diff --git a/src/ImageSharp/Formats/Jpeg/GolangPort/OrigJpegDecoderCore.cs b/src/ImageSharp/Formats/Jpeg/GolangPort/OrigJpegDecoderCore.cs
index 6b4da1ba1..c7e26e04a 100644
--- a/src/ImageSharp/Formats/Jpeg/GolangPort/OrigJpegDecoderCore.cs
+++ b/src/ImageSharp/Formats/Jpeg/GolangPort/OrigJpegDecoderCore.cs
@@ -361,15 +361,18 @@ namespace SixLabors.ImageSharp.Formats.Jpeg.GolangPort
 
                         break;
                     case OrigJpegConstants.Markers.SOS:
-                        if (!metadataOnly)
+                        if (metadataOnly)
                         {
-                            this.ProcessStartOfScanMarker(remaining);
+                            this.InputProcessor.Skip(remaining);
                         }
-
-                        if (this.InputProcessor.ReachedEOF)
+                        else
                         {
-                            // If unexpected EOF reached. We can stop processing bytes as we now have the image data.
-                            processBytes = false;
+                            this.ProcessStartOfScanMarker(remaining);
+                            if (this.InputProcessor.ReachedEOF)
+                            {
+                                // If unexpected EOF reached. We can stop processing bytes as we now have the image data.
+                                processBytes = false;
+                            }
                         }
 
                         break;
diff --git a/src/ImageSharp/Formats/Jpeg/PdfJsPort/PdfJsJpegDecoderCore.cs b/src/ImageSharp/Formats/Jpeg/PdfJsPort/PdfJsJpegDecoderCore.cs
index c20f283d7..03ca170cf 100644
--- a/src/ImageSharp/Formats/Jpeg/PdfJsPort/PdfJsJpegDecoderCore.cs
+++ b/src/ImageSharp/Formats/Jpeg/PdfJsPort/PdfJsJpegDecoderCore.cs
@@ -246,7 +246,11 @@ namespace SixLabors.ImageSharp.Formats.Jpeg.PdfJsPort
                             break;
 
                         case PdfJsJpegConstants.Markers.SOS:
-                            if (!metadataOnly)
+                            if (metadataOnly)
+                            {
+                                this.InputStream.Skip(remaining);
+                            }
+                            else
                             {
                                 this.ProcessStartOfScanMarker();
                             }