diff --git a/src/ImageSharp/Formats/Jpeg/JpegDecoderCore.cs b/src/ImageSharp/Formats/Jpeg/JpegDecoderCore.cs
index d4517e9f19..d503323d2f 100644
--- a/src/ImageSharp/Formats/Jpeg/JpegDecoderCore.cs
+++ b/src/ImageSharp/Formats/Jpeg/JpegDecoderCore.cs
@@ -385,7 +385,11 @@ internal sealed class JpegDecoderCore : ImageDecoderCore, IRawJpegData
                     case JpegConstants.Markers.SOF1:
                     case JpegConstants.Markers.SOF2:
 
-                        this.ProcessStartOfFrameMarker(stream, markerContentByteSize, fileMarker, ComponentType.Huffman, metadataOnly);
+                        if (!this.ProcessStartOfFrameMarker(stream, markerContentByteSize, fileMarker, ComponentType.Huffman, metadataOnly))
+                        {
+                            return;
+                        }
+
                         break;
 
                     case JpegConstants.Markers.SOF9:
@@ -398,7 +402,11 @@ internal sealed class JpegDecoderCore : ImageDecoderCore, IRawJpegData
                             this.scanDecoder.ResetInterval = this.resetInterval.Value;
                         }
 
-                        this.ProcessStartOfFrameMarker(stream, markerContentByteSize, fileMarker, ComponentType.Arithmetic, metadataOnly);
+                        if (!this.ProcessStartOfFrameMarker(stream, markerContentByteSize, fileMarker, ComponentType.Arithmetic, metadataOnly))
+                        {
+                            return;
+                        }
+
                         break;
 
                     case JpegConstants.Markers.SOF5:
@@ -429,7 +437,9 @@ internal sealed class JpegDecoderCore : ImageDecoderCore, IRawJpegData
                         }
 
                         // It's highly unlikely that APPn related data will be found after the SOS marker
-                        // We should have gathered everything we need by now.
+                        // So we can stop parsing here and return the metadata we have parsed so far, instead
+                        // of trying to parse any APPn markers after the SOS marker and risking running out of
+                        // memory or other exceptions.
                         return;
 
                     case JpegConstants.Markers.DHT:
@@ -1212,13 +1222,19 @@ internal sealed class JpegDecoderCore : ImageDecoderCore, IRawJpegData
     /// <param name="frameMarker">The current frame marker.</param>
     /// <param name="decodingComponentType">The jpeg decoding component type.</param>
     /// <param name="metadataOnly">Whether to parse metadata only.</param>
-    private void ProcessStartOfFrameMarker(BufferedReadStream stream, int remaining, in JpegFileMarker frameMarker, ComponentType decodingComponentType, bool metadataOnly)
+    private bool ProcessStartOfFrameMarker(BufferedReadStream stream, int remaining, in JpegFileMarker frameMarker, ComponentType decodingComponentType, bool metadataOnly)
     {
         if (this.Frame != null)
         {
-            if (metadataOnly)
+            // If we have found the SOS marker, we can stop parsing as we have all
+            // the information we need to decode the image.
+            // It's possible that there are APPn related markers after the SOS marker,
+            // but it's highly unlikely and we would be better off stopping parsing
+            // and decoding the image instead of trying to parse those APPn markers
+            // and risking running out of memory or other exceptions.
+            if (this.hasSOSMarker)
             {
-                return;
+                return false;
             }
 
             JpegThrowHelper.ThrowInvalidImageContentException("Multiple SOF markers. Only single frame jpegs supported.");
@@ -1351,6 +1367,8 @@ internal sealed class JpegDecoderCore : ImageDecoderCore, IRawJpegData
             this.Frame.Init(maxH, maxV);
             this.scanDecoder.InjectFrameData(this.Frame, this);
         }
+
+        return true;
     }
 
     /// <summary>
diff --git a/tests/ImageSharp.Tests/Formats/Jpg/JpegDecoderTests.cs b/tests/ImageSharp.Tests/Formats/Jpg/JpegDecoderTests.cs
index 36847536b3..d7d81bdfde 100644
--- a/tests/ImageSharp.Tests/Formats/Jpg/JpegDecoderTests.cs
+++ b/tests/ImageSharp.Tests/Formats/Jpg/JpegDecoderTests.cs
@@ -457,4 +457,14 @@ public partial class JpegDecoderTests
             byte[] data = [0xFF, 0xD8, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30];
             using Image<Rgba32> image = Image.Load<Rgba32>(data);
         });
+
+    // https://github.com/SixLabors/ImageSharp/issues/3118
+    [Theory]
+    [WithFile(TestImages.Jpeg.Issues.Issue3118, PixelTypes.Rgb24)]
+    public void Issue3118_Multiple_SOF_WithSOS_DoesNotThrow<TPixel>(TestImageProvider<TPixel> provider)
+        where TPixel : unmanaged, IPixel<TPixel>
+    {
+        using Image<TPixel> image = provider.GetImage(JpegDecoder.Instance);
+        image.DebugSave(provider);
+    }
 }
diff --git a/tests/ImageSharp.Tests/TestImages.cs b/tests/ImageSharp.Tests/TestImages.cs
index f7e130d66a..1a8cf948e8 100644
--- a/tests/ImageSharp.Tests/TestImages.cs
+++ b/tests/ImageSharp.Tests/TestImages.cs
@@ -361,6 +361,7 @@ public static class TestImages
             public const string Issue2758 = "Jpg/issues/issue-2758.jpg";
             public const string Issue2857 = "Jpg/issues/issue-2857-subsub-ifds.jpg";
             public const string Issue2948 = "Jpg/issues/issue-2948-sos.jpg";
+            public const string Issue3118 = "Jpg/issues/issue3118-multiple-sof.jpg";
 
             public static class Fuzz
             {
diff --git a/tests/Images/Input/Jpg/issues/issue3118-multiple-sof.jpg b/tests/Images/Input/Jpg/issues/issue3118-multiple-sof.jpg
new file mode 100644
index 0000000000..e8897a511b
--- /dev/null
+++ b/tests/Images/Input/Jpg/issues/issue3118-multiple-sof.jpg
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:29d9073bc60cb8f5965993654ec5a949cdbacebe6365db9831ece860095ca85f
+size 11541050