diff --git a/src/ImageSharp/Formats/Jpeg/GolangPort/Components/Decoder/Bytes.cs b/src/ImageSharp/Formats/Jpeg/GolangPort/Components/Decoder/Bytes.cs
index c10771b46..3fc46093e 100644
--- a/src/ImageSharp/Formats/Jpeg/GolangPort/Components/Decoder/Bytes.cs
+++ b/src/ImageSharp/Formats/Jpeg/GolangPort/Components/Decoder/Bytes.cs
@@ -28,12 +28,12 @@ namespace SixLabors.ImageSharp.Formats.Jpeg.GolangPort.Components.Decoder
         /// buffer[i:j] are the buffered bytes read from the underlying
         /// stream that haven't yet been passed further on.
         /// </summary>
-        public IManagedByteBuffer Buffer;
+        public byte[] Buffer;
 
         /// <summary>
         /// Values of <see cref="Buffer"/> converted to <see cref="int"/>-s
         /// </summary>
-        public IBuffer<int> BufferAsInt;
+        public int[] BufferAsInt;
 
         /// <summary>
         /// Start of bytes read
@@ -58,10 +58,12 @@ namespace SixLabors.ImageSharp.Formats.Jpeg.GolangPort.Components.Decoder
         /// <returns>The bytes created</returns>
         public static Bytes Create(MemoryManager memoryManager)
         {
+            // DO NOT bother with buffers and array pooling here!
+            // It only makes things worse!
             return new Bytes
             {
-                Buffer = memoryManager.AllocateManagedByteBuffer(BufferSize),
-                BufferAsInt = memoryManager.Allocate<int>(BufferSize)
+                Buffer = new byte[BufferSize],
+                BufferAsInt = new int[BufferSize]
             };
         }
 
@@ -70,9 +72,6 @@ namespace SixLabors.ImageSharp.Formats.Jpeg.GolangPort.Components.Decoder
         /// </summary>
         public void Dispose()
         {
-            this.Buffer?.Dispose();
-            this.BufferAsInt?.Dispose();
-
             this.Buffer = null;
             this.BufferAsInt = null;
         }
@@ -88,8 +87,7 @@ namespace SixLabors.ImageSharp.Formats.Jpeg.GolangPort.Components.Decoder
             // Take the fast path if bytes.buf contains at least two bytes.
             if (this.I + 2 <= this.J)
             {
-                Span<int> bufferSpan = this.BufferAsInt.Span;
-                x = bufferSpan[this.I];
+                x = this.BufferAsInt[this.I];
                 this.I++;
                 this.UnreadableBytes = 1;
                 if (x != OrigJpegConstants.Markers.XFFInt)
@@ -97,7 +95,7 @@ namespace SixLabors.ImageSharp.Formats.Jpeg.GolangPort.Components.Decoder
                     return OrigDecoderErrorCode.NoError;
                 }
 
-                if (bufferSpan[this.I] != 0x00)
+                if (this.BufferAsInt[this.I] != 0x00)
                 {
                     return OrigDecoderErrorCode.MissingFF00;
                 }
@@ -171,7 +169,7 @@ namespace SixLabors.ImageSharp.Formats.Jpeg.GolangPort.Components.Decoder
                 }
             }
 
-            result = this.Buffer.Span[this.I];
+            result = this.Buffer[this.I];
             this.I++;
             this.UnreadableBytes = 0;
             return errorCode;
@@ -197,7 +195,7 @@ namespace SixLabors.ImageSharp.Formats.Jpeg.GolangPort.Components.Decoder
                 }
             }
 
-            result = this.BufferAsInt.Span[this.I];
+            result = this.BufferAsInt[this.I];
             this.I++;
             this.UnreadableBytes = 0;
             return errorCode;
@@ -231,20 +229,18 @@ namespace SixLabors.ImageSharp.Formats.Jpeg.GolangPort.Components.Decoder
                 DecoderThrowHelper.ThrowImageFormatException.FillCalledWhenUnreadBytesExist();
             }
 
-            Span<byte> byteSpan = this.Buffer.Span;
-
             // Move the last 2 bytes to the start of the buffer, in case we need
             // to call UnreadByteStuffedByte.
             if (this.J > 2)
             {
-                byteSpan[0] = byteSpan[this.J - 2];
-                byteSpan[1] = byteSpan[this.J - 1];
+                this.Buffer[0] = this.Buffer[this.J - 2];
+                this.Buffer[1] = this.Buffer[this.J - 1];
                 this.I = 2;
                 this.J = 2;
             }
 
             // Fill in the rest of the buffer.
-            int n = inputStream.Read(this.Buffer.Array, this.J, byteSpan.Length - this.J);
+            int n = inputStream.Read(this.Buffer, this.J, this.Buffer.Length - this.J);
             if (n == 0)
             {
                 return OrigDecoderErrorCode.UnexpectedEndOfStream;
@@ -252,10 +248,9 @@ namespace SixLabors.ImageSharp.Formats.Jpeg.GolangPort.Components.Decoder
 
             this.J += n;
 
-            Span<int> intSpan = this.BufferAsInt.Span;
-            for (int i = 0; i < byteSpan.Length; i++)
+            for (int i = 0; i < this.Buffer.Length; i++)
             {
-                intSpan[i] = byteSpan[i];
+                this.BufferAsInt[i] = this.Buffer[i];
             }
 
             return OrigDecoderErrorCode.NoError;
diff --git a/src/ImageSharp/Formats/Jpeg/GolangPort/Components/Decoder/InputProcessor.cs b/src/ImageSharp/Formats/Jpeg/GolangPort/Components/Decoder/InputProcessor.cs
index e9f468a85..e7c58f234 100644
--- a/src/ImageSharp/Formats/Jpeg/GolangPort/Components/Decoder/InputProcessor.cs
+++ b/src/ImageSharp/Formats/Jpeg/GolangPort/Components/Decoder/InputProcessor.cs
@@ -158,13 +158,13 @@ namespace SixLabors.ImageSharp.Formats.Jpeg.GolangPort.Components.Decoder
             {
                 if (this.Bytes.J - this.Bytes.I >= length)
                 {
-                    Array.Copy(this.Bytes.Buffer.Array, this.Bytes.I, data, offset, length);
+                    Array.Copy(this.Bytes.Buffer, this.Bytes.I, data, offset, length);
                     this.Bytes.I += length;
                     length -= length;
                 }
                 else
                 {
-                    Array.Copy(this.Bytes.Buffer.Array, this.Bytes.I, data, offset, this.Bytes.J - this.Bytes.I);
+                    Array.Copy(this.Bytes.Buffer, this.Bytes.I, data, offset, this.Bytes.J - this.Bytes.I);
                     offset += this.Bytes.J - this.Bytes.I;
                     length -= this.Bytes.J - this.Bytes.I;
                     this.Bytes.I += this.Bytes.J - this.Bytes.I;