diff --git a/src/ImageSharp/Dithering/ErrorDiffusion/ErrorDiffuser.cs b/src/ImageSharp/Dithering/ErrorDiffusion/ErrorDiffuser.cs
index cde146f1e..d6ab8eb64 100644
--- a/src/ImageSharp/Dithering/ErrorDiffusion/ErrorDiffuser.cs
+++ b/src/ImageSharp/Dithering/ErrorDiffusion/ErrorDiffuser.cs
@@ -71,8 +71,19 @@ namespace ImageSharp.Dithering
         public void Dither<TColor>(PixelAccessor<TColor> pixels, TColor source, TColor transformed, int x, int y, int width, int height)
             where TColor : struct, IPixel<TColor>
         {
-            // Assign the transformed pixel to the array.
-            pixels[x, y] = transformed;
+            this.Dither(pixels, source, transformed, x, y, width, height, true);
+        }
+
+        /// <inheritdoc />
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+        public void Dither<TColor>(PixelAccessor<TColor> pixels, TColor source, TColor transformed, int x, int y, int width, int height, bool replacePixel)
+            where TColor : struct, IPixel<TColor>
+        {
+            if (replacePixel)
+            {
+                // Assign the transformed pixel to the array.
+                pixels[x, y] = transformed;
+            }
 
             // Calculate the error
             Vector4 error = source.ToVector4() - transformed.ToVector4();
diff --git a/src/ImageSharp/Dithering/ErrorDiffusion/IErrorDiffuser.cs b/src/ImageSharp/Dithering/ErrorDiffusion/IErrorDiffuser.cs
index 18079b1fb..66ec3d515 100644
--- a/src/ImageSharp/Dithering/ErrorDiffusion/IErrorDiffuser.cs
+++ b/src/ImageSharp/Dithering/ErrorDiffusion/IErrorDiffuser.cs
@@ -25,5 +25,23 @@ namespace ImageSharp.Dithering
         /// <typeparam name="TColor">The pixel format.</typeparam>
         void Dither<TColor>(PixelAccessor<TColor> pixels, TColor source, TColor transformed, int x, int y, int width, int height)
             where TColor : struct, IPixel<TColor>;
+
+        /// <summary>
+        /// Transforms the image applying the dither matrix. This method alters the input pixels array
+        /// </summary>
+        /// <param name="pixels">The pixel accessor </param>
+        /// <param name="source">The source pixel</param>
+        /// <param name="transformed">The transformed pixel</param>
+        /// <param name="x">The column index.</param>
+        /// <param name="y">The row index.</param>
+        /// <param name="width">The image width.</param>
+        /// <param name="height">The image height.</param>
+        /// <param name="replacePixel">
+        /// Whether to replace the pixel at the given coordinates with the transformed value.
+        /// Generally this would be true for standard two-color dithering but when used in conjunction with color quantization this should be false.
+        /// </param>
+        /// <typeparam name="TColor">The pixel format.</typeparam>
+        void Dither<TColor>(PixelAccessor<TColor> pixels, TColor source, TColor transformed, int x, int y, int width, int height, bool replacePixel)
+            where TColor : struct, IPixel<TColor>;
     }
 }
diff --git a/src/ImageSharp/Quantizers/OctreeQuantizer.cs b/src/ImageSharp/Quantizers/OctreeQuantizer.cs
index f95ae5fce..df52ee7f9 100644
--- a/src/ImageSharp/Quantizers/OctreeQuantizer.cs
+++ b/src/ImageSharp/Quantizers/OctreeQuantizer.cs
@@ -101,7 +101,7 @@ namespace ImageSharp.Quantizers
                     if (this.Dither)
                     {
                         // Apply the dithering matrix. We have to reapply the value now as the original has changed.
-                        this.DitherType.Dither(source, sourcePixel, transformedPixel, x, y, width, height);
+                        this.DitherType.Dither(source, sourcePixel, transformedPixel, x, y, width, height, false);
                     }
 
                     output[(y * source.Width) + x] = pixelValue;
diff --git a/src/ImageSharp/Quantizers/PaletteQuantizer.cs b/src/ImageSharp/Quantizers/PaletteQuantizer.cs
index b9e4573e6..f039fe0c5 100644
--- a/src/ImageSharp/Quantizers/PaletteQuantizer.cs
+++ b/src/ImageSharp/Quantizers/PaletteQuantizer.cs
@@ -108,7 +108,7 @@ namespace ImageSharp.Quantizers
                     if (this.Dither)
                     {
                         // Apply the dithering matrix. We have to reapply the value now as the original has changed.
-                        this.DitherType.Dither(source, sourcePixel, transformedPixel, x, y, width, height);
+                        this.DitherType.Dither(source, sourcePixel, transformedPixel, x, y, width, height, false);
                     }
 
                     output[(y * source.Width) + x] = pixelValue;
diff --git a/src/ImageSharp/Quantizers/Quantizer.cs b/src/ImageSharp/Quantizers/Quantizer.cs
index ccf8da3df..bb856ccc5 100644
--- a/src/ImageSharp/Quantizers/Quantizer.cs
+++ b/src/ImageSharp/Quantizers/Quantizer.cs
@@ -66,22 +66,9 @@ namespace ImageSharp.Quantizers
                     this.FirstPass(pixels, width, height);
                 }
 
-                // Collect the palette. Octree requires this to be done before the second pass runs.
+                // Collect the palette. Required before the second pass runs.
                 colorPalette = this.GetPalette();
-
-                if (this.Dither)
-                {
-                    // We clone the image as we don't want to alter the original.
-                    using (Image<TColor> clone = new Image<TColor>(image))
-                    using (PixelAccessor<TColor> clonedPixels = clone.Lock())
-                    {
-                        this.SecondPass(clonedPixels, quantizedPixels, width, height);
-                    }
-                }
-                else
-                {
-                    this.SecondPass(pixels, quantizedPixels, width, height);
-                }
+                this.SecondPass(pixels, quantizedPixels, width, height);
             }
 
             return new QuantizedImage<TColor>(width, height, colorPalette, quantizedPixels);
diff --git a/src/ImageSharp/Quantizers/WuQuantizer.cs b/src/ImageSharp/Quantizers/WuQuantizer.cs
index fdf8e4136..c1c81d0ac 100644
--- a/src/ImageSharp/Quantizers/WuQuantizer.cs
+++ b/src/ImageSharp/Quantizers/WuQuantizer.cs
@@ -275,7 +275,7 @@ namespace ImageSharp.Quantizers
                     if (this.Dither)
                     {
                         // Apply the dithering matrix. We have to reapply the value now as the original has changed.
-                        this.DitherType.Dither(source, sourcePixel, transformedPixel, x, y, width, height);
+                        this.DitherType.Dither(source, sourcePixel, transformedPixel, x, y, width, height, false);
                     }
 
                     output[(y * source.Width) + x] = pixelValue;