diff --git a/src/ImageSharp/Formats/Png/PngEncoderCore.cs b/src/ImageSharp/Formats/Png/PngEncoderCore.cs
index 8a26fb51c..25ccf7bd1 100644
--- a/src/ImageSharp/Formats/Png/PngEncoderCore.cs
+++ b/src/ImageSharp/Formats/Png/PngEncoderCore.cs
@@ -555,49 +555,44 @@ namespace SixLabors.ImageSharp.Formats.Png
 
             // Grab the palette and write it to the stream.
             ReadOnlySpan<TPixel> palette = quantized.Palette.Span;
-            int paletteLength = Math.Min(palette.Length, QuantizerConstants.MaxColors);
-            int colorTableLength = paletteLength * 3;
-            bool anyAlpha = false;
+            int paletteLength = palette.Length;
+            int colorTableLength = paletteLength * Unsafe.SizeOf<Rgb24>();
+            bool hasAlpha = false;
 
-            using (IManagedByteBuffer colorTable = this.memoryAllocator.AllocateManagedByteBuffer(colorTableLength))
-            using (IManagedByteBuffer alphaTable = this.memoryAllocator.AllocateManagedByteBuffer(paletteLength))
-            {
-                ref byte colorTableRef = ref MemoryMarshal.GetReference(colorTable.GetSpan());
-                ref byte alphaTableRef = ref MemoryMarshal.GetReference(alphaTable.GetSpan());
-                ReadOnlySpan<byte> quantizedSpan = quantized.GetPixelBufferSpan();
-
-                Rgba32 rgba = default;
-
-                for (int i = 0; i < paletteLength; i++)
-                {
-                    if (quantizedSpan.IndexOf((byte)i) > -1)
-                    {
-                        int offset = i * 3;
-                        palette[i].ToRgba32(ref rgba);
+            using IManagedByteBuffer colorTable = this.memoryAllocator.AllocateManagedByteBuffer(colorTableLength);
+            using IManagedByteBuffer alphaTable = this.memoryAllocator.AllocateManagedByteBuffer(paletteLength);
 
-                        byte alpha = rgba.A;
+            ref Rgb24 colorTableRef = ref MemoryMarshal.GetReference(MemoryMarshal.Cast<byte, Rgb24>(colorTable.GetSpan()));
+            ref byte alphaTableRef = ref MemoryMarshal.GetReference(alphaTable.GetSpan());
 
-                        Unsafe.Add(ref colorTableRef, offset) = rgba.R;
-                        Unsafe.Add(ref colorTableRef, offset + 1) = rgba.G;
-                        Unsafe.Add(ref colorTableRef, offset + 2) = rgba.B;
+            // Bulk convert our palette to RGBA to allow assignment to tables.
+            // Palette length maxes out at 256 so safe to stackalloc.
+            Span<Rgba32> rgbaPaletteSpan = stackalloc Rgba32[palette.Length];
+            PixelOperations<TPixel>.Instance.ToRgba32(quantized.Configuration, quantized.Palette.Span, rgbaPaletteSpan);
+            ref Rgba32 rgbaPaletteRef = ref MemoryMarshal.GetReference(rgbaPaletteSpan);
 
-                        if (alpha > this.options.Threshold)
-                        {
-                            alpha = byte.MaxValue;
-                        }
+            // Loop, assign, and extract alpha values from the palette.
+            for (int i = 0; i < paletteLength; i++)
+            {
+                Rgba32 rgba = Unsafe.Add(ref rgbaPaletteRef, i);
+                byte alpha = rgba.A;
 
-                        anyAlpha = anyAlpha || alpha < byte.MaxValue;
-                        Unsafe.Add(ref alphaTableRef, i) = alpha;
-                    }
+                Unsafe.Add(ref colorTableRef, i) = rgba.Rgb;
+                if (alpha > this.options.Threshold)
+                {
+                    alpha = byte.MaxValue;
                 }
 
-                this.WriteChunk(stream, PngChunkType.Palette, colorTable.Array, 0, colorTableLength);
+                hasAlpha = hasAlpha || alpha < byte.MaxValue;
+                Unsafe.Add(ref alphaTableRef, i) = alpha;
+            }
 
-                // Write the transparency data
-                if (anyAlpha)
-                {
-                    this.WriteChunk(stream, PngChunkType.Transparency, alphaTable.Array, 0, paletteLength);
-                }
+            this.WriteChunk(stream, PngChunkType.Palette, colorTable.Array, 0, colorTableLength);
+
+            // Write the transparency data
+            if (hasAlpha)
+            {
+                this.WriteChunk(stream, PngChunkType.Transparency, alphaTable.Array, 0, paletteLength);
             }
         }
 
diff --git a/src/ImageSharp/Processing/Processors/Quantization/IndexedImageFrame{TPixel}.cs b/src/ImageSharp/Processing/Processors/Quantization/IndexedImageFrame{TPixel}.cs
index 42aadb5fd..ac737f452 100644
--- a/src/ImageSharp/Processing/Processors/Quantization/IndexedImageFrame{TPixel}.cs
+++ b/src/ImageSharp/Processing/Processors/Quantization/IndexedImageFrame{TPixel}.cs
@@ -32,6 +32,7 @@ namespace SixLabors.ImageSharp.Processing.Processors.Quantization
         internal IndexedImageFrame(Configuration configuration, int width, int height, ReadOnlyMemory<TPixel> palette)
         {
             Guard.NotNull(configuration, nameof(configuration));
+            Guard.MustBeLessThanOrEqualTo(palette.Length, QuantizerConstants.MaxColors, nameof(palette));
             Guard.MustBeGreaterThan(width, 0, nameof(width));
             Guard.MustBeGreaterThan(height, 0, nameof(height));