diff --git a/src/ImageSharp/Formats/Tga/TgaDecoderCore.cs b/src/ImageSharp/Formats/Tga/TgaDecoderCore.cs
index eef4dc71b3..7b7f803ca1 100644
--- a/src/ImageSharp/Formats/Tga/TgaDecoderCore.cs
+++ b/src/ImageSharp/Formats/Tga/TgaDecoderCore.cs
@@ -111,6 +111,13 @@ namespace SixLabors.ImageSharp.Formats.Tga
                     return image;
                 }
 
+                // Even if the image type indicates it is not a paletted image, it can still contain a palette. Skip those bytes.
+                if (this.fileHeader.CMapLength > 0)
+                {
+                    int colorMapPixelSizeInBytes = this.fileHeader.CMapDepth / 8;
+                    this.currentStream.Skip(this.fileHeader.CMapLength * colorMapPixelSizeInBytes);
+                }
+
                 switch (this.fileHeader.PixelDepth)
                 {
                     case 8: