diff --git a/src/ImageSharp/Formats/Png/Filters/AverageFilter.cs b/src/ImageSharp/Formats/Png/Filters/AverageFilter.cs
index 4c2b56e515..aabf24b2fc 100644
--- a/src/ImageSharp/Formats/Png/Filters/AverageFilter.cs
+++ b/src/ImageSharp/Formats/Png/Filters/AverageFilter.cs
@@ -22,22 +22,25 @@ namespace ImageSharp.Formats
         /// <param name="bytesPerScanline">The number of bytes per scanline</param>
         /// <param name="bytesPerPixel">The bytes per pixel.</param>
         [MethodImpl(MethodImplOptions.AggressiveInlining)]
-        public static void Decode(ref byte scanline, ref byte previousScanline, int bytesPerScanline, int bytesPerPixel)
+        public static void Decode(BufferSpan<byte> scanline, BufferSpan<byte> previousScanline, int bytesPerScanline, int bytesPerPixel)
         {
+            ref byte scanPointer = ref scanline.DangerousGetPinnableReference();
+            ref byte prevPointer = ref previousScanline.DangerousGetPinnableReference();
+
             // Average(x) + floor((Raw(x-bpp)+Prior(x))/2)
             for (int x = 1; x < bytesPerScanline; x++)
             {
                 if (x - bytesPerPixel < 1)
                 {
-                    ref byte scan = ref Unsafe.Add(ref scanline, x);
-                    ref byte above = ref Unsafe.Add(ref previousScanline, x);
+                    ref byte scan = ref Unsafe.Add(ref scanPointer, x);
+                    byte above = Unsafe.Add(ref prevPointer, x);
                     scan = (byte)((scan + (above >> 1)) % 256);
                 }
                 else
                 {
-                    ref byte scan = ref Unsafe.Add(ref scanline, x);
-                    ref byte left = ref Unsafe.Add(ref scanline, x - bytesPerPixel);
-                    ref byte above = ref Unsafe.Add(ref previousScanline, x);
+                    ref byte scan = ref Unsafe.Add(ref scanPointer, x);
+                    byte left = Unsafe.Add(ref scanPointer, x - bytesPerPixel);
+                    byte above = Unsafe.Add(ref prevPointer, x);
                     scan = (byte)((scan + Average(left, above)) % 256);
                 }
             }
@@ -52,26 +55,30 @@ namespace ImageSharp.Formats
         /// <param name="bytesPerScanline">The number of bytes per scanline</param>
         /// <param name="bytesPerPixel">The bytes per pixel.</param>
         [MethodImpl(MethodImplOptions.AggressiveInlining)]
-        public static void Encode(ref byte scanline, ref byte previousScanline, ref byte result, int bytesPerScanline, int bytesPerPixel)
+        public static void Encode(BufferSpan<byte> scanline, BufferSpan<byte> previousScanline, BufferSpan<byte> result, int bytesPerScanline, int bytesPerPixel)
         {
+            ref byte scanPointer = ref scanline.DangerousGetPinnableReference();
+            ref byte prevPointer = ref previousScanline.DangerousGetPinnableReference();
+            ref byte resultPointer = ref result.DangerousGetPinnableReference();
+
             // Average(x) = Raw(x) - floor((Raw(x-bpp)+Prior(x))/2)
-            result = 3;
+            resultPointer = 3;
 
             for (int x = 0; x < bytesPerScanline; x++)
             {
                 if (x - bytesPerPixel < 0)
                 {
-                    ref byte scan = ref Unsafe.Add(ref scanline, x);
-                    ref byte above = ref Unsafe.Add(ref previousScanline, x);
-                    ref byte res = ref Unsafe.Add(ref result, x + 1);
+                    byte scan = Unsafe.Add(ref scanPointer, x);
+                    byte above = Unsafe.Add(ref prevPointer, x);
+                    ref byte res = ref Unsafe.Add(ref resultPointer, x + 1);
                     res = (byte)((scan - (above >> 1)) % 256);
                 }
                 else
                 {
-                    ref byte scan = ref Unsafe.Add(ref scanline, x);
-                    ref byte left = ref Unsafe.Add(ref scanline, x - bytesPerPixel);
-                    ref byte above = ref Unsafe.Add(ref previousScanline, x);
-                    ref byte res = ref Unsafe.Add(ref result, x + 1);
+                    byte scan = Unsafe.Add(ref scanPointer, x);
+                    byte left = Unsafe.Add(ref scanPointer, x - bytesPerPixel);
+                    byte above = Unsafe.Add(ref prevPointer, x);
+                    ref byte res = ref Unsafe.Add(ref resultPointer, x + 1);
                     res = (byte)((scan - Average(left, above)) % 256);
                 }
             }
diff --git a/src/ImageSharp/Formats/Png/Filters/PaethFilter.cs b/src/ImageSharp/Formats/Png/Filters/PaethFilter.cs
index 79de59cb95..a8e397a300 100644
--- a/src/ImageSharp/Formats/Png/Filters/PaethFilter.cs
+++ b/src/ImageSharp/Formats/Png/Filters/PaethFilter.cs
@@ -23,23 +23,26 @@ namespace ImageSharp.Formats
         /// <param name="bytesPerScanline">The number of bytes per scanline</param>
         /// <param name="bytesPerPixel">The bytes per pixel.</param>
         [MethodImpl(MethodImplOptions.AggressiveInlining)]
-        public static void Decode(ref byte scanline, ref byte previousScanline, int bytesPerScanline, int bytesPerPixel)
+        public static void Decode(BufferSpan<byte> scanline, BufferSpan<byte> previousScanline, int bytesPerScanline, int bytesPerPixel)
         {
+            ref byte scanPointer = ref scanline.DangerousGetPinnableReference();
+            ref byte prevPointer = ref previousScanline.DangerousGetPinnableReference();
+
             // Paeth(x) + PaethPredictor(Raw(x-bpp), Prior(x), Prior(x-bpp))
             for (int x = 1; x < bytesPerScanline; x++)
             {
                 if (x - bytesPerPixel < 1)
                 {
-                    ref byte scan = ref Unsafe.Add(ref scanline, x);
-                    ref byte above = ref Unsafe.Add(ref previousScanline, x);
+                    ref byte scan = ref Unsafe.Add(ref scanPointer, x);
+                    byte above = Unsafe.Add(ref prevPointer, x);
                     scan = (byte)((scan + PaethPredicator(0, above, 0)) % 256);
                 }
                 else
                 {
-                    ref byte scan = ref Unsafe.Add(ref scanline, x);
-                    ref byte left = ref Unsafe.Add(ref scanline, x - bytesPerPixel);
-                    ref byte above = ref Unsafe.Add(ref previousScanline, x);
-                    ref byte upperLeft = ref Unsafe.Add(ref previousScanline, x - bytesPerPixel);
+                    ref byte scan = ref Unsafe.Add(ref scanPointer, x);
+                    byte left = Unsafe.Add(ref scanPointer, x - bytesPerPixel);
+                    byte above = Unsafe.Add(ref prevPointer, x);
+                    byte upperLeft = Unsafe.Add(ref prevPointer, x - bytesPerPixel);
                     scan = (byte)((scan + PaethPredicator(left, above, upperLeft)) % 256);
                 }
             }
@@ -54,27 +57,31 @@ namespace ImageSharp.Formats
         /// <param name="bytesPerScanline">The number of bytes per scanline</param>
         /// <param name="bytesPerPixel">The bytes per pixel.</param>
         [MethodImpl(MethodImplOptions.AggressiveInlining)]
-        public static void Encode(ref byte scanline, ref byte previousScanline, ref byte result, int bytesPerScanline, int bytesPerPixel)
+        public static void Encode(BufferSpan<byte> scanline, BufferSpan<byte> previousScanline, BufferSpan<byte> result, int bytesPerScanline, int bytesPerPixel)
         {
+            ref byte scanPointer = ref scanline.DangerousGetPinnableReference();
+            ref byte prevPointer = ref previousScanline.DangerousGetPinnableReference();
+            ref byte resultPointer = ref result.DangerousGetPinnableReference();
+
             // Paeth(x) = Raw(x) - PaethPredictor(Raw(x-bpp), Prior(x), Prior(x - bpp))
-            result = 4;
+            resultPointer = 4;
 
             for (int x = 0; x < bytesPerScanline; x++)
             {
                 if (x - bytesPerPixel < 0)
                 {
-                    ref byte scan = ref Unsafe.Add(ref scanline, x);
-                    ref byte above = ref Unsafe.Add(ref previousScanline, x);
-                    ref byte res = ref Unsafe.Add(ref result, x + 1);
+                    byte scan = Unsafe.Add(ref scanPointer, x);
+                    byte above = Unsafe.Add(ref prevPointer, x);
+                    ref byte res = ref Unsafe.Add(ref resultPointer, x + 1);
                     res = (byte)((scan - PaethPredicator(0, above, 0)) % 256);
                 }
                 else
                 {
-                    ref byte scan = ref Unsafe.Add(ref scanline, x);
-                    ref byte left = ref Unsafe.Add(ref scanline, x - bytesPerPixel);
-                    ref byte above = ref Unsafe.Add(ref previousScanline, x);
-                    ref byte upperLeft = ref Unsafe.Add(ref previousScanline, x - bytesPerPixel);
-                    ref byte res = ref Unsafe.Add(ref result, x + 1);
+                    byte scan = Unsafe.Add(ref scanPointer, x);
+                    byte left = Unsafe.Add(ref scanPointer, x - bytesPerPixel);
+                    byte above = Unsafe.Add(ref prevPointer, x);
+                    byte upperLeft = Unsafe.Add(ref prevPointer, x - bytesPerPixel);
+                    ref byte res = ref Unsafe.Add(ref resultPointer, x + 1);
                     res = (byte)((scan - PaethPredicator(left, above, upperLeft)) % 256);
                 }
             }
diff --git a/src/ImageSharp/Formats/Png/Filters/SubFilter.cs b/src/ImageSharp/Formats/Png/Filters/SubFilter.cs
index 90d0fa6925..d5ec2e7dc3 100644
--- a/src/ImageSharp/Formats/Png/Filters/SubFilter.cs
+++ b/src/ImageSharp/Formats/Png/Filters/SubFilter.cs
@@ -21,20 +21,22 @@ namespace ImageSharp.Formats
         /// <param name="bytesPerScanline">The number of bytes per scanline</param>
         /// <param name="bytesPerPixel">The bytes per pixel.</param>
         [MethodImpl(MethodImplOptions.AggressiveInlining)]
-        public static void Decode(ref byte scanline, int bytesPerScanline, int bytesPerPixel)
+        public static void Decode(BufferSpan<byte> scanline, int bytesPerScanline, int bytesPerPixel)
         {
+            ref byte scanPointer = ref scanline.DangerousGetPinnableReference();
+
             // Sub(x) + Raw(x-bpp)
             for (int x = 1; x < bytesPerScanline; x++)
             {
                 if (x - bytesPerPixel < 1)
                 {
-                    ref byte scan = ref Unsafe.Add(ref scanline, x);
+                    ref byte scan = ref Unsafe.Add(ref scanPointer, x);
                     scan = (byte)(scan % 256);
                 }
                 else
                 {
-                    ref byte scan = ref Unsafe.Add(ref scanline, x);
-                    ref byte prev = ref Unsafe.Add(ref scanline, x - bytesPerPixel);
+                    ref byte scan = ref Unsafe.Add(ref scanPointer, x);
+                    byte prev = Unsafe.Add(ref scanPointer, x - bytesPerPixel);
                     scan = (byte)((scan + prev) % 256);
                 }
             }
@@ -48,24 +50,27 @@ namespace ImageSharp.Formats
         /// <param name="bytesPerScanline">The number of bytes per scanline</param>
         /// <param name="bytesPerPixel">The bytes per pixel.</param>
         [MethodImpl(MethodImplOptions.AggressiveInlining)]
-        public static void Encode(ref byte scanline, ref byte result, int bytesPerScanline, int bytesPerPixel)
+        public static void Encode(BufferSpan<byte> scanline, BufferSpan<byte> result, int bytesPerScanline, int bytesPerPixel)
         {
+            ref byte scanPointer = ref scanline.DangerousGetPinnableReference();
+            ref byte resultPointer = ref result.DangerousGetPinnableReference();
+
             // Sub(x) = Raw(x) - Raw(x-bpp)
-            result = 1;
+            resultPointer = 1;
 
             for (int x = 0; x < bytesPerScanline; x++)
             {
                 if (x - bytesPerPixel < 0)
                 {
-                    ref byte scan = ref Unsafe.Add(ref scanline, x);
-                    ref byte res = ref Unsafe.Add(ref result, x + 1);
+                    byte scan = Unsafe.Add(ref scanPointer, x);
+                    ref byte res = ref Unsafe.Add(ref resultPointer, x + 1);
                     res = (byte)(scan % 256);
                 }
                 else
                 {
-                    ref byte scan = ref Unsafe.Add(ref scanline, x);
-                    ref byte prev = ref Unsafe.Add(ref scanline, x - bytesPerPixel);
-                    ref byte res = ref Unsafe.Add(ref result, x + 1);
+                    byte scan = Unsafe.Add(ref scanPointer, x);
+                    byte prev = Unsafe.Add(ref scanPointer, x - bytesPerPixel);
+                    ref byte res = ref Unsafe.Add(ref resultPointer, x + 1);
                     res = (byte)((scan - prev) % 256);
                 }
             }
diff --git a/src/ImageSharp/Formats/Png/Filters/UpFilter.cs b/src/ImageSharp/Formats/Png/Filters/UpFilter.cs
index fa67330602..30cdf65a38 100644
--- a/src/ImageSharp/Formats/Png/Filters/UpFilter.cs
+++ b/src/ImageSharp/Formats/Png/Filters/UpFilter.cs
@@ -21,13 +21,16 @@ namespace ImageSharp.Formats
         /// <param name="previousScanline">The previous scanline.</param>
         /// <param name="bytesPerScanline">The number of bytes per scanline</param>
         [MethodImpl(MethodImplOptions.AggressiveInlining)]
-        public static void Decode(ref byte scanline, ref byte previousScanline, int bytesPerScanline)
+        public static void Decode(BufferSpan<byte> scanline, BufferSpan<byte> previousScanline, int bytesPerScanline)
         {
+            ref byte scanPointer = ref scanline.DangerousGetPinnableReference();
+            ref byte prevPointer = ref previousScanline.DangerousGetPinnableReference();
+
             // Up(x) + Prior(x)
             for (int x = 1; x < bytesPerScanline; x++)
             {
-                ref byte scan = ref Unsafe.Add(ref scanline, x);
-                ref byte above = ref Unsafe.Add(ref previousScanline, x);
+                ref byte scan = ref Unsafe.Add(ref scanPointer, x);
+                byte above = Unsafe.Add(ref prevPointer, x);
                 scan = (byte)((scan + above) % 256);
             }
         }
@@ -40,16 +43,20 @@ namespace ImageSharp.Formats
         /// <param name="result">The filtered scanline result.</param>
         /// <param name="bytesPerScanline">The number of bytes per scanline</param>
         [MethodImpl(MethodImplOptions.AggressiveInlining)]
-        public static void Encode(ref byte scanline, ref byte previousScanline, ref byte result, int bytesPerScanline)
+        public static void Encode(BufferSpan<byte> scanline, BufferSpan<byte> previousScanline, BufferSpan<byte> result, int bytesPerScanline)
         {
+            ref byte scanPointer = ref scanline.DangerousGetPinnableReference();
+            ref byte prevPointer = ref previousScanline.DangerousGetPinnableReference();
+            ref byte resultPointer = ref result.DangerousGetPinnableReference();
+
             // Up(x) = Raw(x) - Prior(x)
-            result = 2;
+            resultPointer = 2;
 
             for (int x = 0; x < bytesPerScanline; x++)
             {
-                ref byte scan = ref Unsafe.Add(ref scanline, x);
-                ref byte above = ref Unsafe.Add(ref previousScanline, x);
-                ref byte res = ref Unsafe.Add(ref result, x + 1);
+                byte scan = Unsafe.Add(ref scanPointer, x);
+                byte above = Unsafe.Add(ref prevPointer, x);
+                ref byte res = ref Unsafe.Add(ref resultPointer, x + 1);
                 res = (byte)((scan - above) % 256);
             }
         }
diff --git a/src/ImageSharp/Formats/Png/PngDecoderCore.cs b/src/ImageSharp/Formats/Png/PngDecoderCore.cs
index f4e1d8261a..b165d0935c 100644
--- a/src/ImageSharp/Formats/Png/PngDecoderCore.cs
+++ b/src/ImageSharp/Formats/Png/PngDecoderCore.cs
@@ -440,9 +440,7 @@ namespace ImageSharp.Formats
 
                 var scanSpan = new BufferSpan<byte>(this.scanline);
                 var prevSpan = new BufferSpan<byte>(this.previousScanline);
-                ref byte scanPointer = ref scanSpan.DangerousGetPinnableReference();
-                ref byte prevPointer = ref prevSpan.DangerousGetPinnableReference();
-                var filterType = (FilterType)scanPointer;
+                var filterType = (FilterType)scanSpan[0];
 
                 switch (filterType)
                 {
@@ -451,22 +449,22 @@ namespace ImageSharp.Formats
 
                     case FilterType.Sub:
 
-                        SubFilter.Decode(ref scanPointer, this.bytesPerScanline, this.bytesPerPixel);
+                        SubFilter.Decode(scanSpan, this.bytesPerScanline, this.bytesPerPixel);
                         break;
 
                     case FilterType.Up:
 
-                        UpFilter.Decode(ref scanPointer, ref prevPointer, this.bytesPerScanline);
+                        UpFilter.Decode(scanSpan, prevSpan, this.bytesPerScanline);
                         break;
 
                     case FilterType.Average:
 
-                        AverageFilter.Decode(ref scanPointer, ref prevPointer, this.bytesPerScanline, this.bytesPerPixel);
+                        AverageFilter.Decode(scanSpan, prevSpan, this.bytesPerScanline, this.bytesPerPixel);
                         break;
 
                     case FilterType.Paeth:
 
-                        PaethFilter.Decode(ref scanPointer, ref prevPointer, this.bytesPerScanline, this.bytesPerPixel);
+                        PaethFilter.Decode(scanSpan, prevSpan, this.bytesPerScanline, this.bytesPerPixel);
                         break;
 
                     default:
@@ -517,9 +515,7 @@ namespace ImageSharp.Formats
 
                     var scanSpan = new BufferSpan<byte>(this.scanline);
                     var prevSpan = new BufferSpan<byte>(this.previousScanline);
-                    ref byte scanPointer = ref scanSpan.DangerousGetPinnableReference();
-                    ref byte prevPointer = ref prevSpan.DangerousGetPinnableReference();
-                    var filterType = (FilterType)scanPointer;
+                    var filterType = (FilterType)scanSpan[0];
 
                     switch (filterType)
                     {
@@ -528,22 +524,22 @@ namespace ImageSharp.Formats
 
                         case FilterType.Sub:
 
-                            SubFilter.Decode(ref scanPointer, bytesPerInterlaceScanline, this.bytesPerPixel);
+                            SubFilter.Decode(scanSpan, bytesPerInterlaceScanline, this.bytesPerPixel);
                             break;
 
                         case FilterType.Up:
 
-                            UpFilter.Decode(ref scanPointer, ref prevPointer, bytesPerInterlaceScanline);
+                            UpFilter.Decode(scanSpan, prevSpan, bytesPerInterlaceScanline);
                             break;
 
                         case FilterType.Average:
 
-                            AverageFilter.Decode(ref scanPointer, ref prevPointer, bytesPerInterlaceScanline, this.bytesPerPixel);
+                            AverageFilter.Decode(scanSpan, prevSpan, bytesPerInterlaceScanline, this.bytesPerPixel);
                             break;
 
                         case FilterType.Paeth:
 
-                            PaethFilter.Decode(ref scanPointer, ref prevPointer, bytesPerInterlaceScanline, this.bytesPerPixel);
+                            PaethFilter.Decode(scanSpan, prevSpan, bytesPerInterlaceScanline, this.bytesPerPixel);
                             break;
 
                         default:
diff --git a/src/ImageSharp/Formats/Png/PngEncoderCore.cs b/src/ImageSharp/Formats/Png/PngEncoderCore.cs
index 3420673073..2d63539c2b 100644
--- a/src/ImageSharp/Formats/Png/PngEncoderCore.cs
+++ b/src/ImageSharp/Formats/Png/PngEncoderCore.cs
@@ -361,8 +361,6 @@ namespace ImageSharp.Formats
         {
             var scanSpan = new BufferSpan<byte>(rawScanline);
             var prevSpan = new BufferSpan<byte>(previousScanline);
-            ref byte scanPointer = ref scanSpan.DangerousGetPinnableReference();
-            ref byte prevPointer = ref prevSpan.DangerousGetPinnableReference();
 
             // Palette images don't compress well with adaptive filtering.
             if (this.pngColorType == PngColorType.Palette || this.bitDepth < 8)
@@ -374,17 +372,15 @@ namespace ImageSharp.Formats
             // This order, while different to the enumerated order is more likely to produce a smaller sum
             // early on which shaves a couple of milliseconds off the processing time.
             var upSpan = new BufferSpan<byte>(this.up);
-            ref byte upPointer = ref upSpan.DangerousGetPinnableReference();
-            UpFilter.Encode(ref scanPointer, ref prevPointer, ref upPointer, this.bytesPerScanline);
+            UpFilter.Encode(scanSpan, prevSpan, upSpan, this.bytesPerScanline);
 
-            int currentSum = this.CalculateTotalVariation(ref upPointer, int.MaxValue);
+            int currentSum = this.CalculateTotalVariation(upSpan, int.MaxValue);
             int lowestSum = currentSum;
             result = this.up;
 
             var paethSpan = new BufferSpan<byte>(this.paeth);
-            ref byte paethPointer = ref paethSpan.DangerousGetPinnableReference();
-            PaethFilter.Encode(ref scanPointer, ref prevPointer, ref paethPointer, this.bytesPerScanline, this.bytesPerPixel);
-            currentSum = this.CalculateTotalVariation(ref paethPointer, currentSum);
+            PaethFilter.Encode(scanSpan, prevSpan, paethSpan, this.bytesPerScanline, this.bytesPerPixel);
+            currentSum = this.CalculateTotalVariation(paethSpan, currentSum);
 
             if (currentSum < lowestSum)
             {
@@ -393,9 +389,8 @@ namespace ImageSharp.Formats
             }
 
             var subSpan = new BufferSpan<byte>(this.sub);
-            ref byte subPointer = ref subSpan.DangerousGetPinnableReference();
-            SubFilter.Encode(ref scanPointer, ref subPointer, this.bytesPerScanline, this.bytesPerPixel);
-            currentSum = this.CalculateTotalVariation(ref subPointer, int.MaxValue);
+            SubFilter.Encode(scanSpan, subSpan, this.bytesPerScanline, this.bytesPerPixel);
+            currentSum = this.CalculateTotalVariation(subSpan, int.MaxValue);
 
             if (currentSum < lowestSum)
             {
@@ -404,9 +399,8 @@ namespace ImageSharp.Formats
             }
 
             var averageSpan = new BufferSpan<byte>(this.average);
-            ref byte averagePointer = ref averageSpan.DangerousGetPinnableReference();
-            AverageFilter.Encode(ref scanPointer, ref prevPointer, ref averagePointer, this.bytesPerScanline, this.bytesPerPixel);
-            currentSum = this.CalculateTotalVariation(ref averagePointer, currentSum);
+            AverageFilter.Encode(scanSpan, prevSpan, averageSpan, this.bytesPerScanline, this.bytesPerPixel);
+            currentSum = this.CalculateTotalVariation(averageSpan, currentSum);
 
             if (currentSum < lowestSum)
             {
@@ -424,17 +418,18 @@ namespace ImageSharp.Formats
         /// <param name="lastSum">The last variation sum</param>
         /// <returns>The <see cref="int"/></returns>
         [MethodImpl(MethodImplOptions.AggressiveInlining)]
-        private int CalculateTotalVariation(ref byte scanline, int lastSum)
+        private int CalculateTotalVariation(BufferSpan<byte> scanline, int lastSum)
         {
+            ref byte scanPointer = ref scanline.DangerousGetPinnableReference();
             int sum = 0;
 
             for (int i = 1; i < this.bytesPerScanline; i++)
             {
-                ref byte v = ref Unsafe.Add(ref scanline, i);
+                byte v = Unsafe.Add(ref scanPointer, i);
                 sum += v < 128 ? v : 256 - v;
 
                 // No point continuing if we are larger.
-                if (sum > lastSum)
+                if (sum >= lastSum)
                 {
                     break;
                 }