Merge pull request #469 from SixLabors/antonfirsov/cover-all-codecs

Improved test coverage for Encoders/Decoders

src/ImageSharp/Image/IImage.cs

@@ -1,12 +1,14 @@
 // Copyright (c) Six Labors and contributors.
 // Licensed under the Apache License, Version 2.0.
 
+using System;
+
 namespace SixLabors.ImageSharp
 {
     /// <summary>
     /// Encapsulates the properties and methods that describe an image.
     /// </summary>
-    public interface IImage : IImageInfo
+    public interface IImage : IImageInfo, IDisposable
     {
     }
 }

src/ImageSharp/Image/Image{TPixel}.cs

@@ -17,7 +17,7 @@ namespace SixLabors.ImageSharp
     /// Encapsulates an image, which consists of the pixel data for a graphics image and its attributes.
     /// </summary>
     /// <typeparam name="TPixel">The pixel format.</typeparam>
-    public sealed partial class Image<TPixel> : IImage, IDisposable, IConfigurable
+    public sealed partial class Image<TPixel> : IImage, IConfigurable
         where TPixel : struct, IPixel<TPixel>
     {
         private Configuration configuration;

tests/ImageSharp.Tests/ComplexIntegrationTests.cs

@@ -0,0 +1,35 @@
+namespace SixLabors.ImageSharp.Tests
+{
+    using SixLabors.ImageSharp.Formats.Jpeg;
+    using SixLabors.ImageSharp.PixelFormats;
+    using SixLabors.ImageSharp.Processing;
+    using SixLabors.Primitives;
+
+    using Xunit;
+
+    /// <summary>
+    /// Might be useful to catch complex bugs
+    /// </summary>
+    public class ComplexIntegrationTests
+    {
+        [Theory]
+        [WithFile(TestImages.Jpeg.Baseline.Snake, PixelTypes.Rgba32, 75, JpegSubsample.Ratio420)]
+        [WithFile(TestImages.Jpeg.Baseline.Lake, PixelTypes.Rgba32, 75, JpegSubsample.Ratio420)]
+        [WithFile(TestImages.Jpeg.Baseline.Snake, PixelTypes.Rgba32, 75, JpegSubsample.Ratio444)]
+        [WithFile(TestImages.Jpeg.Baseline.Lake, PixelTypes.Rgba32, 75, JpegSubsample.Ratio444)]
+        public void LoadResizeSave<TPixel>(TestImageProvider<TPixel> provider, int quality, JpegSubsample subsample)
+            where TPixel : struct, IPixel<TPixel>
+        {
+            using (Image<TPixel> image = provider.GetImage(x => x.Resize(new ResizeOptions { Size = new Size(150, 100), Mode = ResizeMode.Max })))
+            {
+
+                image.MetaData.ExifProfile = null; // Reduce the size of the file
+                JpegEncoder options = new JpegEncoder { Subsample = subsample, Quality = quality };
+
+                provider.Utility.TestName += $"{subsample}_Q{quality}";
+                provider.Utility.SaveTestOutputFile(image, "png");
+                provider.Utility.SaveTestOutputFile(image, "jpg", options);
+            }
+        }
+    }
+}

tests/ImageSharp.Tests/Formats/Bmp/BmpDecoderTests.cs

@@ -1,19 +1,27 @@
 // Copyright (c) Six Labors and contributors.
 // Licensed under the Apache License, Version 2.0.
 
-using SixLabors.ImageSharp.Formats;
+using System.IO;
+using SixLabors.ImageSharp.Formats.Bmp;
 using SixLabors.ImageSharp.PixelFormats;
 using Xunit;
+
 // ReSharper disable InconsistentNaming
 
 namespace SixLabors.ImageSharp.Tests
 {
-    using System.IO;
+    using static TestImages.Bmp;
-
-    using SixLabors.ImageSharp.Formats.Bmp;
 
-    public class BmpDecoderTests : FileTestBase
+    public class BmpDecoderTests
     {
+        public const PixelTypes CommonNonDefaultPixelTypes =
+            PixelTypes.Rgba32 | PixelTypes.Bgra32 | PixelTypes.RgbaVector;
+
+        public static readonly string[] AllBmpFiles =
+            {
+                Car, F, NegHeight, CoreHeader, V5Header, RLE, RLEInverted, Bit8, Bit8Inverted, Bit16, Bit16Inverted
+            };
+
         [Theory]
         [WithFileCollection(nameof(AllBmpFiles), PixelTypes.Rgba32)]
         public void DecodeBmp<TPixel>(TestImageProvider<TPixel> provider)
@@ -27,7 +35,7 @@ namespace SixLabors.ImageSharp.Tests
         }
 
         [Theory]
-        [WithFile(TestImages.Bmp.F, CommonNonDefaultPixelTypes)]
+        [WithFile(F, CommonNonDefaultPixelTypes)]
         public void BmpDecoder_IsNotBoundToSinglePixelType<TPixel>(TestImageProvider<TPixel> provider)
             where TPixel : struct, IPixel<TPixel>
         {
@@ -39,18 +47,18 @@ namespace SixLabors.ImageSharp.Tests
         }
 
         [Theory]
-        [InlineData(TestImages.Bmp.Car, 24)]
+        [InlineData(Car, 24)]
-        [InlineData(TestImages.Bmp.F, 24)]
+        [InlineData(F, 24)]
-        [InlineData(TestImages.Bmp.NegHeight, 24)]
+        [InlineData(NegHeight, 24)]
-        [InlineData(TestImages.Bmp.Bit8, 8)]
+        [InlineData(Bit8, 8)]
-        [InlineData(TestImages.Bmp.Bit8Inverted, 8)]
+        [InlineData(Bit8Inverted, 8)]
         public void DetectPixelSize(string imagePath, int expectedPixelSize)
         {
-            TestFile testFile = TestFile.Create(imagePath);
+            var testFile = TestFile.Create(imagePath);
             using (var stream = new MemoryStream(testFile.Bytes, false))
             {
                 Assert.Equal(expectedPixelSize, Image.Identify(stream)?.PixelType?.BitsPerPixel);
             }
         }
     }
-}
+}

tests/ImageSharp.Tests/Formats/Bmp/BmpEncoderTests.cs

@@ -5,31 +5,64 @@ using SixLabors.ImageSharp.Formats;
 using SixLabors.ImageSharp.Formats.Bmp;
 using SixLabors.ImageSharp.PixelFormats;
 using Xunit;
+// ReSharper disable InconsistentNaming
 
 namespace SixLabors.ImageSharp.Tests
 {
+    using SixLabors.ImageSharp.Tests.TestUtilities.ImageComparison;
+
+    using Xunit.Abstractions;
+
     public class BmpEncoderTests : FileTestBase
     {
-        public static readonly TheoryData<BmpBitsPerPixel> BitsPerPixel
+        private const PixelTypes PixelTypesToTest = PixelTypes.Rgba32 | PixelTypes.Bgra32 | PixelTypes.Rgb24;
-        = new TheoryData<BmpBitsPerPixel>
+
+        public static readonly TheoryData<BmpBitsPerPixel> BitsPerPixel =
+            new TheoryData<BmpBitsPerPixel>
+                {
+                    BmpBitsPerPixel.Pixel24,
+                    BmpBitsPerPixel.Pixel32
+                };
+
+        public BmpEncoderTests(ITestOutputHelper output)
+        {
+            this.Output = output;
+        }
+
+        private ITestOutputHelper Output { get; }
+
+        [Theory]
+        [WithTestPatternImages(nameof(BitsPerPixel), 24, 24, PixelTypes.Rgba32 | PixelTypes.Bgra32 | PixelTypes.Rgb24)]
+        public void Encode_IsNotBoundToSinglePixelType<TPixel>(TestImageProvider<TPixel> provider, BmpBitsPerPixel bitsPerPixel)
+            where TPixel : struct, IPixel<TPixel>
         {
-            BmpBitsPerPixel.Pixel24,
+            TestBmpEncoderCore(provider, bitsPerPixel);
-            BmpBitsPerPixel.Pixel32
+        }
-        };
 
         [Theory]
-        [MemberData(nameof(BitsPerPixel))]
+        [WithTestPatternImages(nameof(BitsPerPixel), 48, 24, PixelTypes.Rgba32)]
-        public void BitmapCanEncodeDifferentBitRates(BmpBitsPerPixel bitsPerPixel)
+        [WithTestPatternImages(nameof(BitsPerPixel), 47, 8, PixelTypes.Rgba32)]
+        [WithTestPatternImages(nameof(BitsPerPixel), 49, 7, PixelTypes.Rgba32)]
+        [WithSolidFilledImages(nameof(BitsPerPixel), 1, 1, 255, 100, 50, 255, PixelTypes.Rgba32)]
+        [WithTestPatternImages(nameof(BitsPerPixel), 7, 5, PixelTypes.Rgba32)]
+        public void Encode_WorksWithDifferentSizes<TPixel>(TestImageProvider<TPixel> provider, BmpBitsPerPixel bitsPerPixel)
+            where TPixel : struct, IPixel<TPixel>
         {
-            string path = TestEnvironment.CreateOutputDirectory("Bmp");
+            TestBmpEncoderCore(provider, bitsPerPixel);
+        }
 
-            foreach (TestFile file in Files)
+        private static void TestBmpEncoderCore<TPixel>(TestImageProvider<TPixel> provider, BmpBitsPerPixel bitsPerPixel)
+            where TPixel : struct, IPixel<TPixel>
+        {
+            using (Image<TPixel> image = provider.GetImage())
             {
-                string filename = file.GetFileNameWithoutExtension(bitsPerPixel);
+                // there is no alpha in bmp!
-                using (Image<Rgba32> image = file.CreateImage())
+                image.Mutate(c => c.Opacity(1));
-                {
+
-                    image.Save($"{path}/{filename}.bmp", new BmpEncoder { BitsPerPixel = bitsPerPixel });
+                var encoder = new BmpEncoder { BitsPerPixel = bitsPerPixel };
-                }
+
+                // Does DebugSave & load reference CompareToReferenceInput():
+                image.VerifyEncoder(provider, "bmp", bitsPerPixel, encoder);
             }
         }
     }

tests/ImageSharp.Tests/Formats/Gif/GifDecoderTests.cs

@@ -6,47 +6,89 @@ using SixLabors.ImageSharp.Formats.Gif;
 using SixLabors.ImageSharp.PixelFormats;
 using SixLabors.Primitives;
 using Xunit;
+using System.IO;
+using SixLabors.ImageSharp.Advanced;
 
 // ReSharper disable InconsistentNaming
 namespace SixLabors.ImageSharp.Tests
 {
-    using System.IO;
+    using System.Collections.Generic;
-    using SixLabors.ImageSharp.Advanced;
+
+    using SixLabors.ImageSharp.Tests.TestUtilities.ImageComparison;
 
     public class GifDecoderTests
     {
-        private const PixelTypes PixelTypes = Tests.PixelTypes.Rgba32 | Tests.PixelTypes.RgbaVector | Tests.PixelTypes.Argb32;
+        private const PixelTypes TestPixelTypes = PixelTypes.Rgba32 | PixelTypes.RgbaVector | PixelTypes.Argb32;
+
+        public static readonly string[] MultiFrameTestFiles =
+            {
+                TestImages.Gif.Giphy, TestImages.Gif.Kumin
+            };
+
+        public static readonly string[] BasicVerificationFiles =
+            {
+                TestImages.Gif.Cheers,
+                TestImages.Gif.Rings,
+
+                // previously DecodeBadApplicationExtensionLength:
+                TestImages.Gif.Issues.BadAppExtLength,
+                TestImages.Gif.Issues.BadAppExtLength_2,
 
-        public static readonly string[] TestFiles = { TestImages.Gif.Giphy, TestImages.Gif.Rings, TestImages.Gif.Trans };
+                // previously DecodeBadDescriptorDimensionsLength:
+                TestImages.Gif.Issues.BadDescriptorWidth
+            };
+
+        private static readonly Dictionary<string, int> BasicVerificationFrameCount =
+            new Dictionary<string, int>
+                {
+                    [TestImages.Gif.Cheers] = 93,
+                    [TestImages.Gif.Issues.BadDescriptorWidth] = 36,
+            };
 
         public static readonly string[] BadAppExtFiles = { TestImages.Gif.Issues.BadAppExtLength, TestImages.Gif.Issues.BadAppExtLength_2 };
 
         [Theory]
-        [WithFileCollection(nameof(TestFiles), PixelTypes)]
+        [WithFileCollection(nameof(MultiFrameTestFiles), PixelTypes.Rgba32)]
-        public void DecodeAndReSave<TPixel>(TestImageProvider<TPixel> imageProvider)
+        public void Decode_VerifyAllFrames<TPixel>(TestImageProvider<TPixel> provider)
             where TPixel : struct, IPixel<TPixel>
         {
-            using (Image<TPixel> image = imageProvider.GetImage())
+            using (Image<TPixel> image = provider.GetImage())
             {
-                imageProvider.Utility.SaveTestOutputFile(image, "bmp");
+                image.DebugSaveMultiFrame(provider);
-                imageProvider.Utility.SaveTestOutputFile(image, "gif");
+                image.CompareToReferenceOutputMultiFrame(provider, ImageComparer.Exact);
             }
         }
 
         [Theory]
-        [WithFileCollection(nameof(TestFiles), PixelTypes)]
+        [WithFile(TestImages.Gif.Trans, TestPixelTypes)]
-        public void DecodeResizeAndSave<TPixel>(TestImageProvider<TPixel> imageProvider)
+        public void GifDecoder_IsNotBoundToSinglePixelType<TPixel>(TestImageProvider<TPixel> provider)
             where TPixel : struct, IPixel<TPixel>
         {
-            using (Image<TPixel> image = imageProvider.GetImage())
+            using (Image<TPixel> image = provider.GetImage())
             {
-                image.Mutate(x => x.Resize(new Size(image.Width / 2, image.Height / 2)));
+                image.DebugSave(provider);
-
+                image.CompareFirstFrameToReferenceOutput(provider, ImageComparer.Exact);
-                imageProvider.Utility.SaveTestOutputFile(image, "bmp");
-                imageProvider.Utility.SaveTestOutputFile(image, "gif");
             }
         }
 
+        [Theory]
+        [WithFileCollection(nameof(BasicVerificationFiles), PixelTypes.Rgba32)]
+        public void Decode_VerifyRootFrameAndFrameCount<TPixel>(TestImageProvider<TPixel> provider)
+            where TPixel : struct, IPixel<TPixel>
+        {
+            if (!BasicVerificationFrameCount.TryGetValue(provider.SourceFileOrDescription, out int expectedFrameCount))
+            {
+                expectedFrameCount = 1;
+            }
+
+            using (Image<TPixel> image = provider.GetImage())
+            {
+                Assert.Equal(expectedFrameCount, image.Frames.Count);
+                image.DebugSave(provider);
+                image.CompareFirstFrameToReferenceOutput(provider, ImageComparer.Exact);
+            }
+        }
+        
         [Fact]
         public void Decode_IgnoreMetadataIsFalse_CommentsAreRead()
         {
@@ -149,27 +191,5 @@ namespace SixLabors.ImageSharp.Tests
                 }
             }
         }
-
-        [Theory]
-        [WithFileCollection(nameof(BadAppExtFiles), PixelTypes.Rgba32)]
-        public void DecodeBadApplicationExtensionLength<TPixel>(TestImageProvider<TPixel> imageProvider)
-            where TPixel : struct, IPixel<TPixel>
-        {
-            using (Image<TPixel> image = imageProvider.GetImage())
-            {
-                imageProvider.Utility.SaveTestOutputFile(image, "bmp");
-            }
-        }
-
-        [Theory]
-        [WithFile(TestImages.Gif.Issues.BadDescriptorWidth, PixelTypes.Rgba32)]
-        public void DecodeBadDescriptorDimensionsLength<TPixel>(TestImageProvider<TPixel> provider)
-            where TPixel : struct, IPixel<TPixel>
-        {
-            using (Image<TPixel> image = provider.GetImage())
-            {
-                provider.Utility.SaveTestOutputFile(image, "bmp");
-            }
-        }
     }
 }

tests/ImageSharp.Tests/Formats/Gif/GifEncoderTests.cs

@@ -7,15 +7,16 @@ using SixLabors.ImageSharp.Formats.Gif;
 using SixLabors.ImageSharp.MetaData;
 using SixLabors.ImageSharp.PixelFormats;
 using Xunit;
+// ReSharper disable InconsistentNaming
 
 namespace SixLabors.ImageSharp.Tests
 {
     public class GifEncoderTests
     {
-        private const PixelTypes PixelTypes = Tests.PixelTypes.Rgba32 | Tests.PixelTypes.RgbaVector | Tests.PixelTypes.Argb32;
+        private const PixelTypes TestPixelTypes = PixelTypes.Rgba32 | PixelTypes.RgbaVector | PixelTypes.Argb32;
 
         [Theory]
-        [WithTestPatternImages(100, 100, PixelTypes)]
+        [WithTestPatternImages(100, 100, TestPixelTypes)]
         public void EncodeGeneratedPatterns<TPixel>(TestImageProvider<TPixel> provider)
             where TPixel : struct, IPixel<TPixel>
         {
@@ -78,7 +79,7 @@ namespace SixLabors.ImageSharp.Tests
         }
 
         [Fact]
-        public void Encode_CommentIsToLong_CommentIsTrimmed()
+        public void Encode_WhenCommentIsTooLong_CommentIsTrimmed()
         {
             using (Image<Rgba32> input = new Image<Rgba32>(1, 1))
             {

tests/ImageSharp.Tests/Formats/Jpg/JpegEncoderTests.cs

@@ -10,117 +10,126 @@ namespace SixLabors.ImageSharp.Tests.Formats.Jpg
     using System.Collections.Generic;
     using System.IO;
 
+    using SixLabors.ImageSharp.Formats.Bmp;
     using SixLabors.ImageSharp.Formats.Jpeg;
     using SixLabors.ImageSharp.PixelFormats;
-    using SixLabors.ImageSharp.Processing;
+    using SixLabors.ImageSharp.Tests.TestUtilities.ImageComparison;
-    using SixLabors.Primitives;
+    using SixLabors.ImageSharp.Tests.TestUtilities.ReferenceCodecs;
 
     using Xunit;
     using Xunit.Abstractions;
 
-    public class JpegEncoderTests : MeasureFixture
+    public class JpegEncoderTests
     {
-        public static IEnumerable<string> AllBmpFiles => TestImages.Bmp.All;
+        public static readonly TheoryData<JpegSubsample, int> BitsPerPixel_Quality =
+            new TheoryData<JpegSubsample, int>
+                {
+                    { JpegSubsample.Ratio420, 40 },
+                    { JpegSubsample.Ratio420, 60 },
+                    { JpegSubsample.Ratio420, 100 },
+
+                    { JpegSubsample.Ratio444, 40 },
+                    { JpegSubsample.Ratio444, 60 },
+                    { JpegSubsample.Ratio444, 100 },
+                };
 
-        public JpegEncoderTests(ITestOutputHelper output)
+        [Theory]
-            : base(output)
+        [WithFile(TestImages.Png.CalliphoraPartial, nameof(BitsPerPixel_Quality), PixelTypes.Rgba32)]
+        [WithTestPatternImages(nameof(BitsPerPixel_Quality), 73, 71, PixelTypes.Rgba32)]
+        [WithTestPatternImages(nameof(BitsPerPixel_Quality), 48, 24, PixelTypes.Rgba32)]
+        [WithTestPatternImages(nameof(BitsPerPixel_Quality), 46, 8, PixelTypes.Rgba32)]
+        [WithTestPatternImages(nameof(BitsPerPixel_Quality), 51, 7, PixelTypes.Rgba32)]
+        [WithSolidFilledImages(nameof(BitsPerPixel_Quality), 1, 1, 255, 100, 50, 255, PixelTypes.Rgba32)]
+        [WithTestPatternImages(nameof(BitsPerPixel_Quality), 7, 5, PixelTypes.Rgba32)]
+        public void EncodeBaseline_WorksWithDifferentSizes<TPixel>(TestImageProvider<TPixel> provider, JpegSubsample subsample, int quality)
+            where TPixel : struct, IPixel<TPixel>
         {
+            TestJpegEncoderCore(provider, subsample, quality);
         }
 
         [Theory]
-        [WithFile(TestImages.Jpeg.Baseline.Snake, PixelTypes.Rgba32, 75, JpegSubsample.Ratio420)]
+        [WithTestPatternImages(nameof(BitsPerPixel_Quality), 48, 48, PixelTypes.Rgba32 | PixelTypes.Bgra32)]
-        [WithFile(TestImages.Jpeg.Baseline.Lake, PixelTypes.Rgba32, 75, JpegSubsample.Ratio420)]
+        public void EncodeBaseline_IsNotBoundToSinglePixelType<TPixel>(TestImageProvider<TPixel> provider, JpegSubsample subsample, int quality)
-        [WithFile(TestImages.Jpeg.Baseline.Snake, PixelTypes.Rgba32, 75, JpegSubsample.Ratio444)]
-        [WithFile(TestImages.Jpeg.Baseline.Lake, PixelTypes.Rgba32, 75, JpegSubsample.Ratio444)]
-        public void LoadResizeSave<TPixel>(TestImageProvider<TPixel> provider, int quality, JpegSubsample subsample)
             where TPixel : struct, IPixel<TPixel>
         {
-            using (Image<TPixel> image = provider.GetImage(x => x.Resize(new ResizeOptions { Size = new Size(150, 100), Mode = ResizeMode.Max })))
+            TestJpegEncoderCore(provider, subsample, quality);
-            {
-
-                image.MetaData.ExifProfile = null; // Reduce the size of the file
-                JpegEncoder options = new JpegEncoder { Subsample = subsample, Quality = quality };
-
-                provider.Utility.TestName += $"{subsample}_Q{quality}";
-                provider.Utility.SaveTestOutputFile(image, "png");
-                provider.Utility.SaveTestOutputFile(image, "jpg", options);
-            }
         }
 
-        [Theory]
+        private static ImageComparer GetComparer(int quality)
-        [WithFileCollection(nameof(AllBmpFiles), PixelTypes.Rgba32 | PixelTypes.Rgba32 | PixelTypes.Argb32, JpegSubsample.Ratio420, 75)]
-        [WithFileCollection(nameof(AllBmpFiles), PixelTypes.Rgba32 | PixelTypes.Rgba32 | PixelTypes.Argb32, JpegSubsample.Ratio444, 75)]
-        public void OpenBmp_SaveJpeg<TPixel>(TestImageProvider<TPixel> provider, JpegSubsample subSample, int quality)
-           where TPixel : struct, IPixel<TPixel>
         {
-            using (Image<TPixel> image = provider.GetImage())
+            if (quality > 90)
             {
-                ImagingTestCaseUtility utility = provider.Utility;
+                return ImageComparer.Tolerant(0.0005f / 100);
-                utility.TestName += "_" + subSample + "_Q" + quality;
+            }
-
+            else if (quality > 50)
-                using (FileStream outputStream = File.OpenWrite(utility.GetTestOutputFileName("jpg")))
+            {
-                {
+                return ImageComparer.Tolerant(0.005f / 100);
-                    image.Save(outputStream, new JpegEncoder()
+            }
-                    {
+            else
-                        Subsample = subSample,
+            {
-                        Quality = quality
+                return ImageComparer.Tolerant(0.01f / 100);
-                    });
-                }
             }
         }
 
-        [Fact]
+        private static void TestJpegEncoderCore<TPixel>(
-        public void Encode_IgnoreMetadataIsFalse_ExifProfileIsWritten()
+            TestImageProvider<TPixel> provider,
+            JpegSubsample subsample,
+            int quality = 100)
+            where TPixel : struct, IPixel<TPixel>
         {
-            JpegEncoder options = new JpegEncoder()
+            using (Image<TPixel> image = provider.GetImage())
-            {
-                IgnoreMetadata = false
-            };
-
-            TestFile testFile = TestFile.Create(TestImages.Jpeg.Baseline.Floorplan);
-
-            using (Image<Rgba32> input = testFile.CreateImage())
             {
-                using (MemoryStream memStream = new MemoryStream())
+                // There is no alpha in Jpeg!
-                {
+                image.Mutate(c => c.Opacity(1));
-                    input.Save(memStream, options);
+
-
+                var encoder = new JpegEncoder()
-                    memStream.Position = 0;
+                                  {
-                    using (Image<Rgba32> output = Image.Load<Rgba32>(memStream))
+                                      Subsample = subsample,
-                    {
+                                      Quality = quality
-                        Assert.NotNull(output.MetaData.ExifProfile);
+                                  };
-                    }
+                string info = $"{subsample}-Q{quality}";
-                }
+                ImageComparer comparer = GetComparer(quality);
+
+                // Does DebugSave & load reference CompareToReferenceInput():
+                image.VerifyEncoder(provider, "jpeg", info, encoder, comparer, referenceImageExtension: "png");
             }
         }
+        
 
-        [Fact]
+        [Theory]
-        public void Encode_IgnoreMetadataIsTrue_ExifProfileIgnored()
+        [InlineData(false)]
+        [InlineData(true)]
+        public void IgnoreMetadata_ControlsIfExifProfileIsWritten(bool ignoreMetaData)
         {
-            JpegEncoder options = new JpegEncoder()
+            var encoder = new JpegEncoder()
             {
-                IgnoreMetadata = true
+                IgnoreMetadata = ignoreMetaData
             };
-
+            
-            TestFile testFile = TestFile.Create(TestImages.Jpeg.Baseline.Floorplan);
+            using (Image<Rgba32> input = TestFile.Create(TestImages.Jpeg.Baseline.Floorplan).CreateImage())
-
-            using (Image<Rgba32> input = testFile.CreateImage())
             {
-                using (MemoryStream memStream = new MemoryStream())
+                using (var memStream = new MemoryStream())
                 {
-                    input.SaveAsJpeg(memStream, options);
+                    input.Save(memStream, encoder);
 
                     memStream.Position = 0;
-                    using (Image<Rgba32> output = Image.Load<Rgba32>(memStream))
+                    using (var output = Image.Load<Rgba32>(memStream))
                     {
-                        Assert.Null(output.MetaData.ExifProfile);
+                        if (ignoreMetaData)
+                        {
+                            Assert.Null(output.MetaData.ExifProfile);
+                        }
+                        else
+                        {
+                            Assert.NotNull(output.MetaData.ExifProfile);
+                        }
                     }
                 }
             }
         }
-
+        
         [Fact]
-        public void Encode_Quality_0_And_1_Are_Identical()
+        public void Quality_0_And_1_Are_Identical()
         {
             var options = new JpegEncoder
             {
@@ -143,7 +152,7 @@ namespace SixLabors.ImageSharp.Tests.Formats.Jpg
         }
 
         [Fact]
-        public void Encode_Quality_0_And_100_Are_Not_Identical()
+        public void Quality_0_And_100_Are_Not_Identical()
         {
             var options = new JpegEncoder
             {

tests/ImageSharp.Tests/Formats/Png/PngEncoderTests.cs

@@ -10,36 +10,145 @@ using SixLabors.ImageSharp.Formats.Png;
 using SixLabors.ImageSharp.IO;
 using SixLabors.ImageSharp.PixelFormats;
 using Xunit;
+// ReSharper disable InconsistentNaming
 
 namespace SixLabors.ImageSharp.Tests
 {
-    public class PngEncoderTests : FileTestBase
+    using SixLabors.ImageSharp.Quantizers;
+    using SixLabors.ImageSharp.Tests.TestUtilities.ImageComparison;
+
+    public class PngEncoderTests
     {
-        private const PixelTypes PixelTypes = Tests.PixelTypes.Rgba32 | Tests.PixelTypes.RgbaVector | Tests.PixelTypes.Argb32;
+        private const float ToleranceThresholdForPaletteEncoder = 0.01f / 100;
+
+        /// <summary>
+        /// All types except Palette
+        /// </summary>
+        public static readonly TheoryData<PngColorType> PngColorTypes = new TheoryData<PngColorType>()
+                                                                           {
+                                                                               PngColorType.RgbWithAlpha,
+                                                                               PngColorType.Rgb,
+                                                                               PngColorType.Grayscale,
+                                                                               PngColorType.GrayscaleWithAlpha,
+                                                                           };
+
+        /// <summary>
+        /// All types except Palette
+        /// </summary>
+        public static readonly TheoryData<int> CompressionLevels = new TheoryData<int>()
+                                                                            {
+                                                                                1, 2, 3, 4, 5, 6, 7, 8, 9
+                                                                            };
+
+        public static readonly TheoryData<int> PaletteSizes = new TheoryData<int>()
+                                                                  {
+                                                                      30, 55, 100, 201, 255
+                                                                  };
+
+        public static readonly TheoryData<int> PaletteLargeOnly = new TheoryData<int>()
+                                                                  {
+                                                                      80, 100, 120, 230
+                                                                  };
+
+        [Theory]
+        [WithFile(TestImages.Png.Palette8Bpp, nameof(PngColorTypes), PixelTypes.Rgba32)]
+        [WithTestPatternImages(nameof(PngColorTypes), 48, 24, PixelTypes.Rgba32)]
+        [WithTestPatternImages(nameof(PngColorTypes), 47, 8, PixelTypes.Rgba32)]
+        [WithTestPatternImages(nameof(PngColorTypes), 49, 7, PixelTypes.Rgba32)]
+        [WithSolidFilledImages(nameof(PngColorTypes), 1, 1, 255, 100, 50, 255, PixelTypes.Rgba32)]
+        [WithTestPatternImages(nameof(PngColorTypes), 7, 5, PixelTypes.Rgba32)]
+        public void WorksWithDifferentSizes<TPixel>(TestImageProvider<TPixel> provider, PngColorType pngColorType)
+            where TPixel : struct, IPixel<TPixel>
+        {
+            TestPngEncoderCore(provider, pngColorType, appendPngColorType: true);
+        }
+        
+        [Theory]
+        [WithTestPatternImages(nameof(PngColorTypes), 24, 24, PixelTypes.Rgba32 | PixelTypes.Bgra32 | PixelTypes.Rgb24)]
+        public void IsNotBoundToSinglePixelType<TPixel>(TestImageProvider<TPixel> provider, PngColorType pngColorType)
+            where TPixel : struct, IPixel<TPixel>
+        {
+            TestPngEncoderCore(provider, pngColorType, appendPixelType: true);
+        }
+
+        [Theory]
+        [WithTestPatternImages(nameof(CompressionLevels), 24, 24, PixelTypes.Rgba32)]
+        public void WorksWithAllCompressionLevels<TPixel>(TestImageProvider<TPixel> provider, int compressionLevel)
+            where TPixel : struct, IPixel<TPixel>
+        {
+            TestPngEncoderCore(provider, PngColorType.RgbWithAlpha, compressionLevel, appendCompressionLevel: true);
+        }
 
         [Theory]
-        [WithTestPatternImages(100, 100, PixelTypes, PngColorType.RgbWithAlpha)]
+        [WithFile(TestImages.Png.Palette8Bpp, nameof(PaletteLargeOnly), PixelTypes.Rgba32)]
-        [WithTestPatternImages(100, 100, PixelTypes, PngColorType.Rgb)]
+        public void PaletteColorType_WuQuantizer_File<TPixel>(
-        [WithTestPatternImages(100, 100, PixelTypes, PngColorType.Palette)]
+            TestImageProvider<TPixel> provider,
-        [WithTestPatternImages(100, 100, PixelTypes, PngColorType.Grayscale)]
+            int paletteSize)
-        [WithTestPatternImages(100, 100, PixelTypes, PngColorType.GrayscaleWithAlpha)]
+            where TPixel : struct, IPixel<TPixel>
-        public void EncodeGeneratedPatterns<TPixel>(TestImageProvider<TPixel> provider, PngColorType pngColorType)
+        {
+            this.PaletteColorType_WuQuantizer(provider, paletteSize);
+        }
+
+        [Theory]
+        [WithTestPatternImages(nameof(PaletteSizes), 72, 72, PixelTypes.Rgba32)]
+        public void PaletteColorType_WuQuantizer<TPixel>(TestImageProvider<TPixel> provider, int paletteSize)
             where TPixel : struct, IPixel<TPixel>
         {
             using (Image<TPixel> image = provider.GetImage())
             {
-                var options = new PngEncoder()
+                var encoder = new PngEncoder
-                {
+                                  {
-                    PngColorType = pngColorType
+                                      PngColorType = PngColorType.Palette,
-                };
+                                      PaletteSize = paletteSize,
-                provider.Utility.TestName += "_" + pngColorType;
+                                      Quantizer = new WuQuantizer<TPixel>()
+                                  };
 
-                provider.Utility.SaveTestOutputFile(image, "png", options);
+                image.VerifyEncoder(provider, "png", $"PaletteSize-{paletteSize}", encoder, appendPixelTypeToFileName: false);
             }
         }
 
+        private static bool HasAlpha(PngColorType pngColorType) =>
+            pngColorType == PngColorType.GrayscaleWithAlpha || pngColorType == PngColorType.RgbWithAlpha;
+
+        private static void TestPngEncoderCore<TPixel>(
+            TestImageProvider<TPixel> provider,
+            PngColorType pngColorType,
+            int compressionLevel = 6,
+            bool appendPngColorType = false,
+            bool appendPixelType = false,
+            bool appendCompressionLevel = false)
+            where TPixel : struct, IPixel<TPixel>
+        {
+            using (Image<TPixel> image = provider.GetImage())
+            {
+                if (!HasAlpha(pngColorType))
+                {
+                    image.Mutate(c => c.Opacity(1));
+                }
+
+                var encoder = new PngEncoder { PngColorType = pngColorType, CompressionLevel = compressionLevel};
+
+                string pngColorTypeInfo = appendPixelType ? pngColorType.ToString() : "";
+                string compressionLevelInfo = appendCompressionLevel ? $"_C{compressionLevel}" : "";
+                string debugInfo = $"{pngColorTypeInfo}{compressionLevelInfo}";
+                string referenceInfo = $"{pngColorTypeInfo}";
+
+                // Does DebugSave & load reference CompareToReferenceInput():
+                string path = ((ITestImageProvider)provider).Utility.SaveTestOutputFile(image, "png", encoder, debugInfo, appendPixelType);
+            
+                IImageDecoder referenceDecoder = TestEnvironment.GetReferenceDecoder(path);
+                string referenceOutputFile = ((ITestImageProvider)provider).Utility.GetReferenceOutputFileName("png", referenceInfo, appendPixelType);
+            
+                using (var encodedImage = Image.Load<TPixel>(referenceOutputFile, referenceDecoder))
+                {
+                    ImageComparer comparer = pngColorType== PngColorType.Palette ? ImageComparer.Tolerant(ToleranceThresholdForPaletteEncoder) : ImageComparer.Exact;
+                    comparer.CompareImagesOrFrames(image, encodedImage);
+                }
+            }
+        }
+        
         [Theory]
-        [WithBlankImages(1, 1, PixelTypes.All)]
+        [WithBlankImages(1, 1, PixelTypes.Rgba32)]
         public void WritesFileMarker<TPixel>(TestImageProvider<TPixel> provider)
             where TPixel : struct, IPixel<TPixel>
         {

tests/ImageSharp.Tests/TestUtilities/ImageProviders/TestPatternProvider.cs

@@ -86,9 +86,10 @@ namespace SixLabors.ImageSharp.Tests
                                      NamedColors<TPixel>.HotPink,
                                      NamedColors<TPixel>.Blue
                                  };
-                int p = 0;
+                
                 for (int y = top; y < bottom; y++)
                 {
+                    int p = 0;
                     for (int x = left; x < right; x++)
                     {
                         if (x % stride == 0)

tests/ImageSharp.Tests/TestUtilities/ImagingTestCaseUtility.cs

@@ -157,16 +157,77 @@ namespace SixLabors.ImageSharp.Tests
             return path;
         }
 
+        public IEnumerable<string> GetTestOutputFileNamesMultiFrame(
+            int frameCount,
+            string extension = null,
+            object testOutputDetails = null,
+            bool appendPixelTypeToFileName = true)
+        {
+            string baseDir = this.GetTestOutputFileName("", testOutputDetails, appendPixelTypeToFileName);
+
+            if (!Directory.Exists(baseDir))
+            {
+                Directory.CreateDirectory(baseDir);
+            }
+            
+            for (int i = 0; i < frameCount; i++)
+            {
+                string filePath = $"{baseDir}/{i:D2}.{extension}";
+                yield return filePath;
+            }
+        }
+
+        public string[] SaveTestOutputFileMultiFrame<TPixel>(
+            Image<TPixel> image,
+            string extension = "png",
+            IImageEncoder encoder = null,
+            object testOutputDetails = null,
+            bool appendPixelTypeToFileName = true)
+            where TPixel : struct, IPixel<TPixel>
+        {
+            encoder = encoder ?? TestEnvironment.GetReferenceEncoder($"foo.{extension}");
+
+            string[] files = this.GetTestOutputFileNamesMultiFrame(
+                image.Frames.Count,
+                extension,
+                testOutputDetails,
+                appendPixelTypeToFileName).ToArray();
+
+            for (int i = 0; i < image.Frames.Count; i++)
+            {
+                using (Image<TPixel> frameImage = image.Frames.CloneFrame(i))
+                {
+                    string filePath = files[i];
+                    using (FileStream stream = File.OpenWrite(filePath))
+                    {
+                        frameImage.Save(stream, encoder);
+                    }
+                }
+            }
+
+            return files;
+        }
+
         internal string GetReferenceOutputFileName(
             string extension,
-            object settings,
+            object testOutputDetails,
             bool appendPixelTypeToFileName)
         {
             return TestEnvironment.GetReferenceOutputFileName(
-                this.GetTestOutputFileName(extension, settings, appendPixelTypeToFileName)
+                this.GetTestOutputFileName(extension, testOutputDetails, appendPixelTypeToFileName)
                 );
         }
 
+        public string[] GetReferenceOutputFileNamesMultiFrame(
+            int frameCount,
+            string extension,
+            object testOutputDetails,
+            bool appendPixelTypeToFileName = true)
+        {
+            return this.GetTestOutputFileNamesMultiFrame(frameCount, extension, testOutputDetails)
+                .Select(TestEnvironment.GetReferenceOutputFileName).ToArray();
+        }
+
         internal void Init(string typeName, string methodName, string outputSubfolderName)
         {
             this.TestGroupName = typeName;

tests/ImageSharp.Tests/TestUtilities/TestImageExtensions.cs

@@ -16,6 +16,7 @@ namespace SixLabors.ImageSharp.Tests
     using System.Numerics;
     using SixLabors.ImageSharp.Advanced;
     using SixLabors.ImageSharp.Memory;
+    using SixLabors.ImageSharp.MetaData;
 
     using Xunit;
 
@@ -52,6 +53,28 @@ namespace SixLabors.ImageSharp.Tests
             return image;
         }
 
+        public static Image<TPixel> DebugSaveMultiFrame<TPixel>(
+            this Image<TPixel> image,
+            ITestImageProvider provider,
+            object testOutputDetails = null,
+            string extension = "png",
+            bool appendPixelTypeToFileName = true)
+            where TPixel : struct, IPixel<TPixel>
+        {
+            if (TestEnvironment.RunsOnCI)
+            {
+                return image;
+            }
+
+            // We are running locally then we want to save it out
+            provider.Utility.SaveTestOutputFileMultiFrame(
+                image,
+                extension,
+                testOutputDetails: testOutputDetails,
+                appendPixelTypeToFileName: appendPixelTypeToFileName);
+            return image;
+        }
+
         /// <summary>
         /// Compares the image against the expected Reference output, throws an exception if the images are not similar enough.
         /// The output file should be named identically to the output produced by <see cref="DebugSave{TPixel}(Image{TPixel}, ITestImageProvider, object, string, bool)"/>.
@@ -118,6 +141,55 @@ namespace SixLabors.ImageSharp.Tests
             return image;
         }
 
+        public static Image<TPixel> CompareFirstFrameToReferenceOutput<TPixel>(
+            this Image<TPixel> image,
+            ITestImageProvider provider,
+            ImageComparer comparer,
+            object testOutputDetails = null,
+            string extension = "png",
+            bool grayscale = false,
+            bool appendPixelTypeToFileName = true)
+            where TPixel : struct, IPixel<TPixel>
+        {
+            using (Image<TPixel> firstFrameOnlyImage = new Image<TPixel>(image.Width, image.Height))
+            using (Image<TPixel> referenceImage = GetReferenceOutputImage<TPixel>(
+                provider,
+                testOutputDetails,
+                extension,
+                appendPixelTypeToFileName))
+            {
+                firstFrameOnlyImage.Frames.AddFrame(image.Frames.RootFrame);
+                firstFrameOnlyImage.Frames.RemoveFrame(0);
+
+                comparer.VerifySimilarity(referenceImage, firstFrameOnlyImage);
+            }
+
+            return image;
+        }
+
+        public static Image<TPixel> CompareToReferenceOutputMultiFrame<TPixel>(
+            this Image<TPixel> image,
+            ITestImageProvider provider,
+            ImageComparer comparer,
+            object testOutputDetails = null,
+            string extension = "png",
+            bool grayscale = false,
+            bool appendPixelTypeToFileName = true)
+            where TPixel : struct, IPixel<TPixel>
+        {
+            using (Image<TPixel> referenceImage = GetReferenceOutputImageMultiFrame<TPixel>(
+                provider,
+                image.Frames.Count,
+                testOutputDetails,
+                extension,
+                appendPixelTypeToFileName))
+            {
+                comparer.VerifySimilarity(referenceImage, image);
+            }
+
+            return image;
+        }
+
         public static Image<TPixel> GetReferenceOutputImage<TPixel>(this ITestImageProvider provider,
                                                                     object testOutputDetails = null,
                                                                     string extension = "png",
@@ -136,6 +208,49 @@ namespace SixLabors.ImageSharp.Tests
             return Image.Load<TPixel>(referenceOutputFile, decoder);
         }
 
+        public static Image<TPixel> GetReferenceOutputImageMultiFrame<TPixel>(this ITestImageProvider provider,
+                                                                             int frameCount,
+                                                                    object testOutputDetails = null,
+                                                                    string extension = "png",
+                                                                    bool appendPixelTypeToFileName = true)
+            where TPixel : struct, IPixel<TPixel>
+        {
+            string[] frameFiles = provider.Utility.GetReferenceOutputFileNamesMultiFrame(
+                frameCount,
+                extension,
+                testOutputDetails,
+                appendPixelTypeToFileName);
+
+            var temporalFrameImages = new List<Image<TPixel>>();
+
+            IImageDecoder decoder = TestEnvironment.GetReferenceDecoder(frameFiles[0]);
+
+            foreach (string path in frameFiles)
+            {
+                if (!File.Exists(path))
+                {
+                    throw new Exception("Reference output file missing: " + path);
+                }
+
+                var tempImage = Image.Load<TPixel>(path, decoder);
+                temporalFrameImages.Add(tempImage);
+            }
+
+            Image<TPixel> firstTemp = temporalFrameImages[0];
+            
+            var result = new Image<TPixel>(firstTemp.Width, firstTemp.Height);
+
+            foreach (Image<TPixel> fi in temporalFrameImages)
+            {
+                result.Frames.AddFrame(fi.Frames.RootFrame);
+                fi.Dispose();
+            }
+
+            // remove the initial empty frame:
+            result.Frames.RemoveFrame(0);
+            return result;
+        }
+
         public static IEnumerable<ImageSimilarityReport> GetReferenceOutputSimilarityReports<TPixel>(
             this Image<TPixel> image,
             ITestImageProvider provider,
@@ -227,6 +342,36 @@ namespace SixLabors.ImageSharp.Tests
             return image;
         }
 
+        /// <summary>
+        /// Loads the expected image with a reference decoder + compares it to <paramref name="image"/>.
+        /// Also performs a debug save using <see cref="ImagingTestCaseUtility.SaveTestOutputFile{TPixel}"/>.
+        /// </summary>
+        internal static void VerifyEncoder<TPixel>(this Image<TPixel> image,
+                                                   ITestImageProvider provider,
+                                                   string extension,
+                                                   object testOutputDetails,
+                                                   IImageEncoder encoder,
+                                                   ImageComparer customComparer = null,
+                                                   bool appendPixelTypeToFileName = true,
+                                                   string referenceImageExtension = null
+                                                   )
+            where TPixel : struct, IPixel<TPixel>
+        {
+
+            provider.Utility.SaveTestOutputFile(image, extension, encoder, testOutputDetails, appendPixelTypeToFileName);
+            
+            referenceImageExtension = referenceImageExtension ?? extension;
+            string referenceOutputFile = provider.Utility.GetReferenceOutputFileName(referenceImageExtension, testOutputDetails, appendPixelTypeToFileName);
+
+            IImageDecoder referenceDecoder = TestEnvironment.GetReferenceDecoder(referenceOutputFile);
+
+            using (var encodedImage = Image.Load<TPixel>(referenceOutputFile, referenceDecoder))
+            {
+                ImageComparer comparer = customComparer ?? ImageComparer.Exact;
+                comparer.CompareImagesOrFrames(image, encodedImage);
+            }
+        }
+
         internal static Image<Rgba32> ToGrayscaleImage(this Buffer2D<float> buffer, float scale)
         {
             var image = new Image<Rgba32>(buffer.Width, buffer.Height);
@@ -242,5 +387,6 @@ namespace SixLabors.ImageSharp.Tests
 
             return image;
         }
+
     }
 }

tests/ImageSharp.Tests/TestUtilities/Tests/TestImageProviderTests.cs

@@ -239,8 +239,28 @@ namespace SixLabors.ImageSharp.Tests
             where TPixel : struct, IPixel<TPixel>
         {
             Assert.NotNull(provider.Utility.SourceFileOrDescription);
-            Image<TPixel> image = provider.GetImage();
+            using (Image<TPixel> image = provider.GetImage())
-            provider.Utility.SaveTestOutputFile(image, "png");
+            {
+                provider.Utility.SaveTestOutputFile(image, "png");
+            }
+        }
+
+        [Theory]
+        [WithFile(TestImages.Gif.Giphy, PixelTypes.Rgba32)]
+        public void SaveTestOutputFileMultiFrame<TPixel>(TestImageProvider<TPixel> provider)
+            where TPixel : struct, IPixel<TPixel>
+        {
+            using (Image<TPixel> image = provider.GetImage())
+            {
+                string[] files = provider.Utility.SaveTestOutputFileMultiFrame(image);
+
+                Assert.True(files.Length > 2);
+                foreach (string path in files)
+                {
+                    this.Output.WriteLine(path);
+                    Assert.True(File.Exists(path));
+                }
+            }
         }
 
         [Theory]

tests/Images/External

@@ -1 +1 @@
-Subproject commit 376605e05bb704d425b2d17bf5b310f5376da22e
+Subproject commit b3be1178d4e970efc624181480094e50b0d57a90