to keep the ExifReader free from jpeg specific stuff, the Exif Id Code will be skipped when setting the ExifProfile

src/ImageSharp/Formats/Jpeg/GolangPort/GolangJpegDecoderCore.cs

@@ -3,7 +3,7 @@
 
 using System.Collections.Generic;
 using System.IO;
-
+using System.Linq;
 using SixLabors.ImageSharp.Formats.Jpeg.Components;
 using SixLabors.ImageSharp.Formats.Jpeg.Components.Decoder;
 using SixLabors.ImageSharp.Formats.Jpeg.GolangPort.Components.Decoder;
@@ -500,7 +500,7 @@ namespace SixLabors.ImageSharp.Formats.Jpeg.GolangPort
             if (ProfileResolver.IsProfile(profile, ProfileResolver.ExifMarker))
             {
                 this.isExif = true;
-                this.MetaData.ExifProfile = new ExifProfile(profile);
+                this.MetaData.ExifProfile = new ExifProfile(profile.Skip(6).ToArray());
             }
         }
 

src/ImageSharp/Formats/Jpeg/PdfJsPort/PdfJsJpegDecoderCore.cs

@@ -5,6 +5,7 @@ using System;
 using System.Buffers.Binary;
 using System.Collections.Generic;
 using System.IO;
+using System.Linq;
 using System.Runtime.CompilerServices;
 using System.Runtime.InteropServices;
 
@@ -482,7 +483,8 @@ namespace SixLabors.ImageSharp.Formats.Jpeg.PdfJsPort
                 this.isExif = true;
                 if (this.MetaData.ExifProfile == null)
                 {
-                    this.MetaData.ExifProfile = new ExifProfile(profile);
+                    // the first 6 bytes (Exif00) will be skipped, because this is Jpeg specific
+                    this.MetaData.ExifProfile = new ExifProfile(profile.Skip(6).ToArray());
                 }
                 else
                 {

src/ImageSharp/MetaData/Profiles/Exif/ExifReader.cs

@@ -25,7 +25,6 @@ namespace SixLabors.ImageSharp.MetaData.Profiles.Exif
         private Endianness endianness = Endianness.BigEndian;
         private uint exifOffset;
         private uint gpsOffset;
-        private int startIndex;
 
         public ExifReader(byte[] exifData)
         {
@@ -77,20 +76,6 @@ namespace SixLabors.ImageSharp.MetaData.Profiles.Exif
         {
             var values = new List<ExifValue>();
 
-            if (this.ReadString(4) == "Exif")
-            {
-                if (this.ReadUInt16() != 0)
-                {
-                    return values;
-                }
-
-                this.startIndex = 6;
-            }
-            else
-            {
-                this.position = 0;
-            }
-
             if (this.ReadString(2) == "II")
             {
                 this.endianness = Endianness.LittleEndian;
@@ -169,7 +154,7 @@ namespace SixLabors.ImageSharp.MetaData.Profiles.Exif
         /// <param name="index">The index.</param>
         private void AddValues(List<ExifValue> values, int index)
         {
-            this.position = this.startIndex + index;
+            this.position = index;
             int count = this.ReadUInt16();
 
             for (int i = 0; i < count; i++)
@@ -353,7 +338,7 @@ namespace SixLabors.ImageSharp.MetaData.Profiles.Exif
             {
                 int oldIndex = this.position;
 
-                uint newIndex = this.ConvertToUInt32(offsetBuffer) + (uint)this.startIndex;
+                uint newIndex = this.ConvertToUInt32(offsetBuffer);
 
                 // Ensure that the new index does not overrun the data
                 if (newIndex > int.MaxValue)
@@ -454,7 +439,7 @@ namespace SixLabors.ImageSharp.MetaData.Profiles.Exif
             {
                 if (value.Tag == ExifTag.JPEGInterchangeFormat && (value.DataType == ExifDataType.Long))
                 {
-                    this.ThumbnailOffset = (uint)value.Value + (uint)this.startIndex;
+                    this.ThumbnailOffset = (uint)value.Value;
                 }
                 else if (value.Tag == ExifTag.JPEGInterchangeFormatLength && value.DataType == ExifDataType.Long)
                 {