diff --git a/src/ImageSharp/Formats/Png/Filters/AverageFilter.cs b/src/ImageSharp/Formats/Png/Filters/AverageFilter.cs
index d66aa06dd..0d3a65dbd 100644
--- a/src/ImageSharp/Formats/Png/Filters/AverageFilter.cs
+++ b/src/ImageSharp/Formats/Png/Filters/AverageFilter.cs
@@ -53,8 +53,9 @@ namespace SixLabors.ImageSharp.Formats.Png.Filters
         /// <param name="previousScanline">The previous scanline.</param>
         /// <param name="result">The filtered scanline result.</param>
         /// <param name="bytesPerPixel">The bytes per pixel.</param>
+        /// <param name="sum">The sum of the total variance of the filtered row</param>
         [MethodImpl(MethodImplOptions.AggressiveInlining)]
-        public static void Encode(Span<byte> scanline, Span<byte> previousScanline, Span<byte> result, int bytesPerPixel)
+        public static void Encode(Span<byte> scanline, Span<byte> previousScanline, Span<byte> result, int bytesPerPixel, out int sum)
         {
             DebugGuard.MustBeSameSized(scanline, previousScanline, nameof(scanline));
             DebugGuard.MustBeSizedAtLeast(result, scanline, nameof(result));
@@ -62,6 +63,7 @@ namespace SixLabors.ImageSharp.Formats.Png.Filters
             ref byte scanBaseRef = ref scanline.DangerousGetPinnableReference();
             ref byte prevBaseRef = ref previousScanline.DangerousGetPinnableReference();
             ref byte resultBaseRef = ref result.DangerousGetPinnableReference();
+            sum = 0;
 
             // Average(x) = Raw(x) - floor((Raw(x-bpp)+Prior(x))/2)
             resultBaseRef = 3;
@@ -74,6 +76,7 @@ namespace SixLabors.ImageSharp.Formats.Png.Filters
                     byte above = Unsafe.Add(ref prevBaseRef, x);
                     ref byte res = ref Unsafe.Add(ref resultBaseRef, x + 1);
                     res = (byte)((scan - (above >> 1)) % 256);
+                    sum += res < 128 ? res : 256 - res;
                 }
                 else
                 {
@@ -82,8 +85,11 @@ namespace SixLabors.ImageSharp.Formats.Png.Filters
                     byte above = Unsafe.Add(ref prevBaseRef, x);
                     ref byte res = ref Unsafe.Add(ref resultBaseRef, x + 1);
                     res = (byte)((scan - Average(left, above)) % 256);
+                    sum += res < 128 ? res : 256 - res;
                 }
             }
+
+            sum -= 3;
         }
 
         /// <summary>
diff --git a/src/ImageSharp/Formats/Png/Filters/PaethFilter.cs b/src/ImageSharp/Formats/Png/Filters/PaethFilter.cs
index 630aa56ee..08e493880 100644
--- a/src/ImageSharp/Formats/Png/Filters/PaethFilter.cs
+++ b/src/ImageSharp/Formats/Png/Filters/PaethFilter.cs
@@ -12,7 +12,7 @@ namespace SixLabors.ImageSharp.Formats.Png.Filters
     /// This technique is due to Alan W. Paeth.
     /// <see href="https://www.w3.org/TR/PNG-Filters.html"/>
     /// </summary>
-    internal static unsafe class PaethFilter
+    internal static class PaethFilter
     {
         /// <summary>
         /// Decodes the scanline
@@ -54,8 +54,9 @@ namespace SixLabors.ImageSharp.Formats.Png.Filters
         /// <param name="previousScanline">The previous scanline.</param>
         /// <param name="result">The filtered scanline result.</param>
         /// <param name="bytesPerPixel">The bytes per pixel.</param>
+        /// <param name="sum">The sum of the total variance of the filtered row</param>
         [MethodImpl(MethodImplOptions.AggressiveInlining)]
-        public static void Encode(Span<byte> scanline, Span<byte> previousScanline, Span<byte> result, int bytesPerPixel)
+        public static void Encode(Span<byte> scanline, Span<byte> previousScanline, Span<byte> result, int bytesPerPixel, out int sum)
         {
             DebugGuard.MustBeSameSized(scanline, previousScanline, nameof(scanline));
             DebugGuard.MustBeSizedAtLeast(result, scanline, nameof(result));
@@ -63,6 +64,7 @@ namespace SixLabors.ImageSharp.Formats.Png.Filters
             ref byte scanBaseRef = ref scanline.DangerousGetPinnableReference();
             ref byte prevBaseRef = ref previousScanline.DangerousGetPinnableReference();
             ref byte resultBaseRef = ref result.DangerousGetPinnableReference();
+            sum = 0;
 
             // Paeth(x) = Raw(x) - PaethPredictor(Raw(x-bpp), Prior(x), Prior(x - bpp))
             resultBaseRef = 4;
@@ -75,6 +77,7 @@ namespace SixLabors.ImageSharp.Formats.Png.Filters
                     byte above = Unsafe.Add(ref prevBaseRef, x);
                     ref byte res = ref Unsafe.Add(ref resultBaseRef, x + 1);
                     res = (byte)((scan - PaethPredicator(0, above, 0)) % 256);
+                    sum += res < 128 ? res : 256 - res;
                 }
                 else
                 {
@@ -84,8 +87,11 @@ namespace SixLabors.ImageSharp.Formats.Png.Filters
                     byte upperLeft = Unsafe.Add(ref prevBaseRef, x - bytesPerPixel);
                     ref byte res = ref Unsafe.Add(ref resultBaseRef, x + 1);
                     res = (byte)((scan - PaethPredicator(left, above, upperLeft)) % 256);
+                    sum += res < 128 ? res : 256 - res;
                 }
             }
+
+            sum -= 4;
         }
 
         /// <summary>
diff --git a/src/ImageSharp/Formats/Png/Filters/SubFilter.cs b/src/ImageSharp/Formats/Png/Filters/SubFilter.cs
index 43607dcdd..5ee866440 100644
--- a/src/ImageSharp/Formats/Png/Filters/SubFilter.cs
+++ b/src/ImageSharp/Formats/Png/Filters/SubFilter.cs
@@ -11,7 +11,7 @@ namespace SixLabors.ImageSharp.Formats.Png.Filters
     /// of the prior pixel.
     /// <see href="https://www.w3.org/TR/PNG-Filters.html"/>
     /// </summary>
-    internal static unsafe class SubFilter
+    internal static class SubFilter
     {
         /// <summary>
         /// Decodes the scanline
@@ -46,13 +46,15 @@ namespace SixLabors.ImageSharp.Formats.Png.Filters
         /// <param name="scanline">The scanline to encode</param>
         /// <param name="result">The filtered scanline result.</param>
         /// <param name="bytesPerPixel">The bytes per pixel.</param>
+        /// <param name="sum">The sum of the total variance of the filtered row</param>
         [MethodImpl(MethodImplOptions.AggressiveInlining)]
-        public static void Encode(Span<byte> scanline, Span<byte> result, int bytesPerPixel)
+        public static void Encode(Span<byte> scanline, Span<byte> result, int bytesPerPixel, out int sum)
         {
             DebugGuard.MustBeSizedAtLeast(result, scanline, nameof(result));
 
             ref byte scanBaseRef = ref scanline.DangerousGetPinnableReference();
             ref byte resultBaseRef = ref result.DangerousGetPinnableReference();
+            sum = 0;
 
             // Sub(x) = Raw(x) - Raw(x-bpp)
             resultBaseRef = 1;
@@ -64,6 +66,7 @@ namespace SixLabors.ImageSharp.Formats.Png.Filters
                     byte scan = Unsafe.Add(ref scanBaseRef, x);
                     ref byte res = ref Unsafe.Add(ref resultBaseRef, x + 1);
                     res = (byte)(scan % 256);
+                    sum += res < 128 ? res : 256 - res;
                 }
                 else
                 {
@@ -71,8 +74,11 @@ namespace SixLabors.ImageSharp.Formats.Png.Filters
                     byte prev = Unsafe.Add(ref scanBaseRef, x - bytesPerPixel);
                     ref byte res = ref Unsafe.Add(ref resultBaseRef, x + 1);
                     res = (byte)((scan - prev) % 256);
+                    sum += res < 128 ? res : 256 - res;
                 }
             }
+
+            sum -= 1;
         }
     }
 }
diff --git a/src/ImageSharp/Formats/Png/Filters/UpFilter.cs b/src/ImageSharp/Formats/Png/Filters/UpFilter.cs
index 12a566e32..6e8f780e5 100644
--- a/src/ImageSharp/Formats/Png/Filters/UpFilter.cs
+++ b/src/ImageSharp/Formats/Png/Filters/UpFilter.cs
@@ -11,7 +11,7 @@ namespace SixLabors.ImageSharp.Formats.Png.Filters
     /// rather than just to its left, is used as the predictor.
     /// <see href="https://www.w3.org/TR/PNG-Filters.html"/>
     /// </summary>
-    internal static unsafe class UpFilter
+    internal static class UpFilter
     {
         /// <summary>
         /// Decodes the scanline
@@ -41,8 +41,9 @@ namespace SixLabors.ImageSharp.Formats.Png.Filters
         /// <param name="scanline">The scanline to encode</param>
         /// <param name="previousScanline">The previous scanline.</param>
         /// <param name="result">The filtered scanline result.</param>
+        /// <param name="sum">The sum of the total variance of the filtered row</param>
         [MethodImpl(MethodImplOptions.AggressiveInlining)]
-        public static void Encode(Span<byte> scanline, Span<byte> previousScanline, Span<byte> result)
+        public static void Encode(Span<byte> scanline, Span<byte> previousScanline, Span<byte> result, out int sum)
         {
             DebugGuard.MustBeSameSized(scanline, previousScanline, nameof(scanline));
             DebugGuard.MustBeSizedAtLeast(result, scanline, nameof(result));
@@ -50,6 +51,7 @@ namespace SixLabors.ImageSharp.Formats.Png.Filters
             ref byte scanBaseRef = ref scanline.DangerousGetPinnableReference();
             ref byte prevBaseRef = ref previousScanline.DangerousGetPinnableReference();
             ref byte resultBaseRef = ref result.DangerousGetPinnableReference();
+            sum = 0;
 
             // Up(x) = Raw(x) - Prior(x)
             resultBaseRef = 2;
@@ -60,7 +62,10 @@ namespace SixLabors.ImageSharp.Formats.Png.Filters
                 byte above = Unsafe.Add(ref prevBaseRef, x);
                 ref byte res = ref Unsafe.Add(ref resultBaseRef, x + 1);
                 res = (byte)((scan - above) % 256);
+                sum += res < 128 ? res : 256 - res;
             }
+
+            sum -= 2;
         }
     }
 }
diff --git a/src/ImageSharp/Formats/Png/PngEncoder.cs b/src/ImageSharp/Formats/Png/PngEncoder.cs
index 6b2410e83..2fc6911f0 100644
--- a/src/ImageSharp/Formats/Png/PngEncoder.cs
+++ b/src/ImageSharp/Formats/Png/PngEncoder.cs
@@ -1,7 +1,6 @@
 // Copyright (c) Six Labors and contributors.
 // Licensed under the Apache License, Version 2.0.
 
-using System.Collections.Generic;
 using System.IO;
 using SixLabors.ImageSharp.PixelFormats;
 using SixLabors.ImageSharp.Quantizers;
diff --git a/src/ImageSharp/Formats/Png/PngEncoderCore.cs b/src/ImageSharp/Formats/Png/PngEncoderCore.cs
index 660c37187..86a63f5b4 100644
--- a/src/ImageSharp/Formats/Png/PngEncoderCore.cs
+++ b/src/ImageSharp/Formats/Png/PngEncoderCore.cs
@@ -5,7 +5,6 @@ using System;
 using System.Buffers;
 using System.IO;
 using System.Linq;
-using System.Runtime.CompilerServices;
 using SixLabors.ImageSharp.Advanced;
 using SixLabors.ImageSharp.Formats.Png.Filters;
 using SixLabors.ImageSharp.Formats.Png.Zlib;
@@ -233,7 +232,7 @@ namespace SixLabors.ImageSharp.Formats.Png
             // Collect the indexed pixel data
             if (this.pngColorType == PngColorType.Palette)
             {
-                this.CollectIndexedBytes<TPixel>(image.Frames.RootFrame, stream, header);
+                this.CollectIndexedBytes(image.Frames.RootFrame, stream, header);
             }
 
             this.WritePhysicalChunk(stream, image);
@@ -243,9 +242,7 @@ namespace SixLabors.ImageSharp.Formats.Png
             stream.Flush();
         }
 
-        /// <summary>
-        /// Disposes PngEncoderCore instance, disposing it's internal buffers.
-        /// </summary>
+        /// <inheritdoc />
         public void Dispose()
         {
             this.previousScanline?.Dispose();
@@ -411,14 +408,12 @@ namespace SixLabors.ImageSharp.Formats.Png
 
             // This order, while different to the enumerated order is more likely to produce a smaller sum
             // early on which shaves a couple of milliseconds off the processing time.
-            UpFilter.Encode(scanSpan, prevSpan, this.up);
+            UpFilter.Encode(scanSpan, prevSpan, this.up, out int currentSum);
 
-            int currentSum = this.CalculateTotalVariation(this.up, int.MaxValue);
             int lowestSum = currentSum;
             Buffer<byte> actualResult = this.up;
 
-            PaethFilter.Encode(scanSpan, prevSpan, this.paeth, this.bytesPerPixel);
-            currentSum = this.CalculateTotalVariation(this.paeth, currentSum);
+            PaethFilter.Encode(scanSpan, prevSpan, this.paeth, this.bytesPerPixel, out currentSum);
 
             if (currentSum < lowestSum)
             {
@@ -426,8 +421,7 @@ namespace SixLabors.ImageSharp.Formats.Png
                 actualResult = this.paeth;
             }
 
-            SubFilter.Encode(scanSpan, this.sub, this.bytesPerPixel);
-            currentSum = this.CalculateTotalVariation(this.sub, int.MaxValue);
+            SubFilter.Encode(scanSpan, this.sub, this.bytesPerPixel, out currentSum);
 
             if (currentSum < lowestSum)
             {
@@ -435,8 +429,7 @@ namespace SixLabors.ImageSharp.Formats.Png
                 actualResult = this.sub;
             }
 
-            AverageFilter.Encode(scanSpan, prevSpan, this.average, this.bytesPerPixel);
-            currentSum = this.CalculateTotalVariation(this.average, currentSum);
+            AverageFilter.Encode(scanSpan, prevSpan, this.average, this.bytesPerPixel, out currentSum);
 
             if (currentSum < lowestSum)
             {
@@ -446,34 +439,6 @@ namespace SixLabors.ImageSharp.Formats.Png
             return actualResult;
         }
 
-        /// <summary>
-        /// Calculates the total variation of given byte array. Total variation is the sum of the absolute values of
-        /// neighbor differences.
-        /// </summary>
-        /// <param name="scanline">The scanline bytes</param>
-        /// <param name="lastSum">The last variation sum</param>
-        /// <returns>The <see cref="int"/></returns>
-        [MethodImpl(MethodImplOptions.AggressiveInlining)]
-        private int CalculateTotalVariation(Span<byte> scanline, int lastSum)
-        {
-            ref byte scanBaseRef = ref scanline.DangerousGetPinnableReference();
-            int sum = 0;
-
-            for (int i = 1; i < this.bytesPerScanline; i++)
-            {
-                byte v = Unsafe.Add(ref scanBaseRef, i);
-                sum += v < 128 ? v : 256 - v;
-
-                // No point continuing if we are larger.
-                if (sum >= lastSum)
-                {
-                    break;
-                }
-            }
-
-            return sum;
-        }
-
         /// <summary>
         /// Calculates the correct number of bytes per pixel for the given color type.
         /// </summary>
diff --git a/tests/ImageSharp.Benchmarks/Image/EncodePng.cs b/tests/ImageSharp.Benchmarks/Image/EncodePng.cs
index 8c9fcbbb3..463743b9a 100644
--- a/tests/ImageSharp.Benchmarks/Image/EncodePng.cs
+++ b/tests/ImageSharp.Benchmarks/Image/EncodePng.cs
@@ -10,11 +10,11 @@ namespace SixLabors.ImageSharp.Benchmarks.Image
     using System.IO;
 
     using BenchmarkDotNet.Attributes;
-
-    using SixLabors.ImageSharp.Formats;
     using SixLabors.ImageSharp.Formats.Png;
     using SixLabors.ImageSharp.Quantizers;
     using SixLabors.ImageSharp.Quantizers.Base;
+    using SixLabors.ImageSharp.Tests;
+
     using CoreImage = ImageSharp.Image;
 
     public class EncodePng : BenchmarkBase
@@ -35,9 +35,11 @@ namespace SixLabors.ImageSharp.Benchmarks.Image
         {
             if (this.bmpStream == null)
             {
-                string path = this.LargeImage
-                                  ? "../ImageSharp.Tests/TestImages/Formats/Jpg/baseline/jpeg420exif.jpg"
-                                  : "../ImageSharp.Tests/TestImages/Formats/Bmp/Car.bmp";
+                string path = Path.Combine(
+                    TestEnvironment.InputImagesDirectoryFullPath,
+                    this.LargeImage ? TestImages.Jpeg.Baseline.Jpeg420Exif : TestImages.Bmp.Car);
+
+
                 this.bmpStream = File.OpenRead(path);
                 this.bmpCore = CoreImage.Load<Rgba32>(this.bmpStream);
                 this.bmpStream.Position = 0;
@@ -53,26 +55,26 @@ namespace SixLabors.ImageSharp.Benchmarks.Image
             this.bmpDrawing.Dispose();
         }
 
-        [Benchmark(Baseline = true, Description = "System.Drawing Png")]
-        public void PngSystemDrawing()
-        {
-            using (MemoryStream memoryStream = new MemoryStream())
-            {
-                this.bmpDrawing.Save(memoryStream, ImageFormat.Png);
-            }
-        }
+        //[Benchmark(Baseline = true, Description = "System.Drawing Png")]
+        //public void PngSystemDrawing()
+        //{
+        //    using (var memoryStream = new MemoryStream())
+        //    {
+        //        this.bmpDrawing.Save(memoryStream, ImageFormat.Png);
+        //    }
+        //}
 
         [Benchmark(Description = "ImageSharp Png")]
         public void PngCore()
         {
-            using (MemoryStream memoryStream = new MemoryStream())
+            using (var memoryStream = new MemoryStream())
             {
                 QuantizerBase<Rgba32> quantizer = this.UseOctreeQuantizer
                 ? (QuantizerBase<Rgba32>)
                 new OctreeQuantizer<Rgba32>()
                 : new PaletteQuantizer<Rgba32>();
 
-                PngEncoder options = new PngEncoder() { Quantizer = quantizer };
+                var options = new PngEncoder { Quantizer = quantizer };
                 this.bmpCore.SaveAsPng(memoryStream, options);
             }
         }