diff --git a/src/ImageSharp/Formats/Jpeg/Components/Decoder/SpectralConverter{TPixel}.cs b/src/ImageSharp/Formats/Jpeg/Components/Decoder/SpectralConverter{TPixel}.cs
index 532892e060..2c3dccdee6 100644
--- a/src/ImageSharp/Formats/Jpeg/Components/Decoder/SpectralConverter{TPixel}.cs
+++ b/src/ImageSharp/Formats/Jpeg/Components/Decoder/SpectralConverter{TPixel}.cs
@@ -95,7 +95,7 @@ namespace SixLabors.ImageSharp.Formats.Jpeg.Components.Decoder
                 }
             }
 
-            var buffer = this.pixelBuffer;
+            Buffer2D<TPixel> buffer = this.pixelBuffer;
             this.pixelBuffer = null;
             return buffer;
         }
diff --git a/src/ImageSharp/Formats/Jpeg/JpegDecoderCore.cs b/src/ImageSharp/Formats/Jpeg/JpegDecoderCore.cs
index 58c85bd34e..94439f35a9 100644
--- a/src/ImageSharp/Formats/Jpeg/JpegDecoderCore.cs
+++ b/src/ImageSharp/Formats/Jpeg/JpegDecoderCore.cs
@@ -90,11 +90,6 @@ namespace SixLabors.ImageSharp.Formats.Jpeg
         /// </summary>
         private JFifMarker jFif;
 
-        /// <summary>
-        /// Whether the image has a JFIF marker. This is needed to determine, if the colorspace is YCbCr.
-        /// </summary>
-        private bool hasJFif;
-
         /// <summary>
         /// Contains information about the Adobe marker.
         /// </summary>
@@ -516,8 +511,6 @@ namespace SixLabors.ImageSharp.Formats.Jpeg
 
             if (componentCount == 3)
             {
-                // We prioritize adobe marker over jfif marker, if somebody really encoded this image with redundant adobe marker,
-                // then it's most likely an adobe jfif image.
                 if (!this.adobe.Equals(default))
                 {
                     if (this.adobe.ColorTransform == JpegConstants.Adobe.ColorTransformYCbCr)
@@ -529,51 +522,14 @@ namespace SixLabors.ImageSharp.Formats.Jpeg
                     {
                         return JpegColorSpace.RGB;
                     }
-
-                    // Fallback to the id color deduction: If these values are 1-3 for a 3-channel image, then the image is assumed to be YCbCr.
-                    if (this.Components[2].Id == 3 && this.Components[1].Id == 2 && this.Components[0].Id == 1)
-                    {
-                        return JpegColorSpace.YCbCr;
-                    }
-
-                    JpegThrowHelper.ThrowNotSupportedColorSpace();
-                }
-
-                if (this.hasJFif)
-                {
-                    // JFIF implies YCbCr.
-                    return JpegColorSpace.YCbCr;
                 }
 
-                // Fallback to the id color deduction.
-                // If the component Id's are R, G, B in ASCII the colorspace is RGB and not YCbCr.
-                // See: https://docs.oracle.com/javase/7/docs/api/javax/imageio/metadata/doc-files/jpeg_metadata.html#color
-                if (this.Components[2].Id == 66 && this.Components[1].Id == 71 && this.Components[0].Id == 82)
-                {
-                    return JpegColorSpace.RGB;
-                }
-
-                // If these values are 1-3 for a 3-channel image, then the image is assumed to be YCbCr.
-                if (this.Components[2].Id == 3 && this.Components[1].Id == 2 && this.Components[0].Id == 1)
-                {
-                    return JpegColorSpace.YCbCr;
-                }
-
-                // 3-channel non-subsampled images are assumed to be RGB.
-                if (this.Components[2].VerticalSamplingFactor == 1 && this.Components[1].VerticalSamplingFactor == 1 && this.Components[0].VerticalSamplingFactor == 1 &&
-                    this.Components[2].HorizontalSamplingFactor == 1 && this.Components[1].HorizontalSamplingFactor == 1 && this.Components[0].HorizontalSamplingFactor == 1)
-                {
-                    return JpegColorSpace.RGB;
-                }
-
-                // Some images are poorly encoded and contain incorrect colorspace transform metadata.
-                // We ignore that and always fall back to the default colorspace.
+                // Fallback to YCbCr
                 return JpegColorSpace.YCbCr;
             }
 
             if (componentCount == 4)
             {
-                // jfif images doesn't not support 4 component images, so we only check adobe.
                 if (!this.adobe.Equals(default))
                 {
                     if (this.adobe.ColorTransform == JpegConstants.Adobe.ColorTransformYcck)
@@ -585,11 +541,9 @@ namespace SixLabors.ImageSharp.Formats.Jpeg
                     {
                         return JpegColorSpace.Cmyk;
                     }
-
-                    JpegThrowHelper.ThrowNotSupportedColorSpace();
                 }
 
-                // Fallback to cmyk as neither of cmyk nor ycck have 'special' component ids.
+                // Fallback to CMYK
                 return JpegColorSpace.Cmyk;
             }
 
@@ -757,8 +711,6 @@ namespace SixLabors.ImageSharp.Formats.Jpeg
         /// <param name="remaining">The remaining bytes in the segment block.</param>
         private void ProcessApplicationHeaderMarker(BufferedReadStream stream, int remaining)
         {
-            this.hasJFif = true;
-
             // We can only decode JFif identifiers.
             // Some images contain multiple JFIF markers (Issue 1932) so we check to see
             // if it's already been read.