diff --git a/src/ImageSharp.Formats.Png/Filters/PaethFilter.cs b/src/ImageSharp.Formats.Png/Filters/PaethFilter.cs
index 323611fbc..ff208f3d7 100644
--- a/src/ImageSharp.Formats.Png/Filters/PaethFilter.cs
+++ b/src/ImageSharp.Formats.Png/Filters/PaethFilter.cs
@@ -79,9 +79,9 @@ namespace ImageSharp.Formats
         private static byte PaethPredicator(byte left, byte above, byte upperLeft)
         {
             int p = left + above - upperLeft;
-            int pa = Math.Abs(p - left);
-            int pb = Math.Abs(p - above);
-            int pc = Math.Abs(p - upperLeft);
+            int pa = ImageMaths.FastAbs(p - left);
+            int pb = ImageMaths.FastAbs(p - above);
+            int pc = ImageMaths.FastAbs(p - upperLeft);
 
             if (pa <= pb && pa <= pc)
             {
diff --git a/src/ImageSharp.Formats.Png/PngEncoderCore.cs b/src/ImageSharp.Formats.Png/PngEncoderCore.cs
index 78df6c72e..609556b45 100644
--- a/src/ImageSharp.Formats.Png/PngEncoderCore.cs
+++ b/src/ImageSharp.Formats.Png/PngEncoderCore.cs
@@ -369,33 +369,33 @@ namespace ImageSharp.Formats
             }
 
             SubFilter.Encode(rawScanline, this.sub, this.bytesPerPixel);
-            int currentTotalVariation = this.CalculateTotalVariation(this.sub);
-            int lowestTotalVariation = currentTotalVariation;
+            int currentSum = this.CalculateTotalVariation(this.sub, int.MaxValue);
+            int lowestSum = currentSum;
 
             result = this.sub;
 
             UpFilter.Encode(rawScanline, previousScanline, this.up);
-            currentTotalVariation = this.CalculateTotalVariation(this.up);
+            currentSum = this.CalculateTotalVariation(this.up, currentSum);
 
-            if (currentTotalVariation < lowestTotalVariation)
+            if (currentSum < lowestSum)
             {
-                lowestTotalVariation = currentTotalVariation;
+                lowestSum = currentSum;
                 result = this.up;
             }
 
             AverageFilter.Encode(rawScanline, previousScanline, this.average, this.bytesPerPixel);
-            currentTotalVariation = this.CalculateTotalVariation(this.average);
+            currentSum = this.CalculateTotalVariation(this.average, currentSum);
 
-            if (currentTotalVariation < lowestTotalVariation)
+            if (currentSum < lowestSum)
             {
-                lowestTotalVariation = currentTotalVariation;
+                lowestSum = currentSum;
                 result = this.average;
             }
 
             PaethFilter.Encode(rawScanline, previousScanline, this.paeth, this.bytesPerPixel);
-            currentTotalVariation = this.CalculateTotalVariation(this.paeth);
+            currentSum = this.CalculateTotalVariation(this.paeth, currentSum);
 
-            if (currentTotalVariation < lowestTotalVariation)
+            if (currentSum < lowestSum)
             {
                 result = this.paeth;
             }
@@ -408,17 +408,25 @@ namespace ImageSharp.Formats
         /// neighbor differences.
         /// </summary>
         /// <param name="scanline">The scanline bytes</param>
+        /// <param name="lastSum">The last variation sum</param>
         /// <returns>The <see cref="int"/></returns>
-        private int CalculateTotalVariation(byte[] scanline)
+        private int CalculateTotalVariation(byte[] scanline, int lastSum)
         {
-            int totalVariation = 0;
+            int sum = 0;
 
             for (int i = 1; i < scanline.Length; i++)
             {
-                totalVariation += Math.Abs(scanline[i] - scanline[i - 1]);
+                byte v = scanline[i];
+                sum += v < 128 ? v : 256 - v;
+
+                // No point continuing if we are larger.
+                if (sum > lastSum)
+                {
+                    break;
+                }
             }
 
-            return totalVariation;
+            return sum;
         }
 
         /// <summary>
diff --git a/src/ImageSharp/Common/Extensions/ByteExtensions.cs b/src/ImageSharp/Common/Extensions/ByteExtensions.cs
index fc9c29e62..a6dc19ded 100644
--- a/src/ImageSharp/Common/Extensions/ByteExtensions.cs
+++ b/src/ImageSharp/Common/Extensions/ByteExtensions.cs
@@ -5,6 +5,8 @@
 
 namespace ImageSharp
 {
+    using System.Runtime.CompilerServices;
+
     /// <summary>
     /// Extension methods for the <see cref="byte"/> struct.
     /// </summary>
@@ -26,6 +28,7 @@ namespace ImageSharp
         /// <param name="index">The index.</param>
         /// <param name="length">The length.</param>
         /// <exception cref="System.ArgumentNullException"><paramref name="source"/> is null.</exception>
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
         public static void ReverseBytes(this byte[] source, int index, int length)
         {
             Guard.NotNull(source, nameof(source));
diff --git a/src/ImageSharp/Common/Helpers/ImageMaths.cs b/src/ImageSharp/Common/Helpers/ImageMaths.cs
index 7455b542d..147a5914b 100644
--- a/src/ImageSharp/Common/Helpers/ImageMaths.cs
+++ b/src/ImageSharp/Common/Helpers/ImageMaths.cs
@@ -8,12 +8,26 @@ namespace ImageSharp
     using System;
     using System.Linq;
     using System.Numerics;
+    using System.Runtime.CompilerServices;
 
     /// <summary>
     /// Provides common mathematical methods.
     /// </summary>
     internal static class ImageMaths
     {
+        /// <summary>
+        /// Returns the absolute value of a 32-bit signed integer. Uses bit shifting to speed up the operation.
+        /// </summary>
+        /// <param name="x">
+        /// A number that is greater than <see cref="int.MinValue"/>, but less than or equal to <see cref="int.MaxValue"/>
+        /// </param>
+        /// <returns>The <see cref="int"/></returns>
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+        public static int FastAbs(int x)
+        {
+            return (x ^ (x >> 31)) - (x >> 31);
+        }
+
         /// <summary>
         /// Returns how many bits are required to store the specified number of colors.
         /// Performs a Log2() on the value.
@@ -273,6 +287,7 @@ namespace ImageSharp
         /// <returns>
         /// The <see cref="float"/>
         /// </returns>.
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
         private static float Clean(float x)
         {
             if (Math.Abs(x) < Constants.Epsilon)
diff --git a/tests/ImageSharp.Benchmarks/General/Abs.cs b/tests/ImageSharp.Benchmarks/General/Abs.cs
new file mode 100644
index 000000000..b0f033b81
--- /dev/null
+++ b/tests/ImageSharp.Benchmarks/General/Abs.cs
@@ -0,0 +1,33 @@
+namespace ImageSharp.Benchmarks.General
+{
+    using System;
+
+    using BenchmarkDotNet.Attributes;
+
+    public class Abs
+    {
+        [Params(-1, 1)]
+        public int X { get; set; }
+
+        [Benchmark(Baseline = true, Description = "Maths Abs")]
+        public int MathAbs()
+        {
+            int x = this.X;
+            return Math.Abs(x);
+        }
+
+        [Benchmark(Description = "Conditional Abs")]
+        public int ConditionalAbs()
+        {
+            int x = this.X;
+            return x < 0 ? -x : x;
+        }
+
+        [Benchmark(Description = "Bitwise Abs")]
+        public int AbsBitwise()
+        {
+            int x = this.X;
+            return (x ^ (x >> 31)) - (x >> 31);
+        }
+    }
+}