diff --git a/src/ImageSharp/IO/BufferedReadStream.cs b/src/ImageSharp/IO/BufferedReadStream.cs
index 7e233c965..39d294ae5 100644
--- a/src/ImageSharp/IO/BufferedReadStream.cs
+++ b/src/ImageSharp/IO/BufferedReadStream.cs
@@ -49,7 +49,6 @@ internal sealed class BufferedReadStream : Stream
 
         this.BaseStream = stream;
         this.Length = stream.Length;
-        this.Position = (int)stream.Position;
         this.BufferSize = configuration.StreamProcessingBufferSize;
         this.maxBufferIndex = this.BufferSize - 1;
         this.readBuffer = ArrayPool<byte>.Shared.Rent(this.BufferSize);
@@ -59,8 +58,8 @@ internal sealed class BufferedReadStream : Stream
             this.pinnedReadBuffer = (byte*)this.readBufferHandle.Pointer;
         }
 
-        // This triggers a full read on first attempt.
-        this.readBufferIndex = this.BufferSize;
+        this.Position = (int)stream.Position;
+        this.FillReadBuffer();
     }
 
     /// <summary>
@@ -99,6 +98,7 @@ internal sealed class BufferedReadStream : Stream
                 this.BaseStream.Seek(value, SeekOrigin.Begin);
                 this.readerPosition = value;
                 this.readBufferIndex = this.BufferSize;
+                this.FillReadBuffer();
             }
         }
     }