JpegDecoderTests using reference images

tests/ImageSharp.Tests/Formats/Jpg/JpegDecoderTests.cs

@@ -11,6 +11,7 @@ namespace ImageSharp.Tests
 
     using ImageSharp.Formats;
     using ImageSharp.PixelFormats;
+    using ImageSharp.Tests.TestUtilities.ImageComparison;
 
     using Xunit;
 
@@ -25,6 +26,10 @@ namespace ImageSharp.Tests
 
         public static string[] ProgressiveTestJpegs = TestImages.Jpeg.Progressive.All;
 
+        // TODO: We should make this comparer less tolerant ...
+        private static readonly ImageComparer VeryTolerantJpegComparer =
+            ImageComparer.Tolerant(0.005f, pixelThresholdInPixelByteSum: 4);
+
         [Theory]
         [WithFileCollection(nameof(BaselineTestJpegs), PixelTypes.Rgba32 | PixelTypes.Rgba32 | PixelTypes.Argb32)]
         public void DecodeBaselineJpeg<TPixel>(TestImageProvider<TPixel> provider)
@@ -32,7 +37,7 @@ namespace ImageSharp.Tests
         {
             using (Image<TPixel> image = provider.GetImage())
             {
-                image.DebugSave(provider);
+                image.CompareToReferenceOutput(provider, VeryTolerantJpegComparer);
             }
         }
 
@@ -43,7 +48,7 @@ namespace ImageSharp.Tests
         {
             using (Image<TPixel> image = provider.GetImage())
             {
-                image.DebugSave(provider);
+                image.DebugSave(provider, VeryTolerantJpegComparer);
             }
         }
 
@@ -70,11 +75,9 @@ namespace ImageSharp.Tests
                     image.Save(ms, encoder);
                 }
             }
-
+            
-            // TODO: Automatic image comparers could help here a lot :P
             Image<TPixel> mirror = provider.Factory.CreateImage(data);
-            provider.Utility.TestName += $"_{subsample}_Q{quality}";
+            mirror.DebugSave(provider, $"_{subsample}_Q{quality}");
-            mirror.DebugSave(provider);
         }
 
         [Theory]

tests/ImageSharp.Tests/TestUtilities/ImageComparison/ImageSimilarityReport.cs

@@ -7,15 +7,22 @@
 
     public class ImageSimilarityReport
     {
-        public ImageSimilarityReport(IImageBase expectedImage, IImageBase actualImage, IEnumerable<PixelDifference> differences)
+        public ImageSimilarityReport(
+            IImageBase expectedImage,
+            IImageBase actualImage,
+            IEnumerable<PixelDifference> differences,
+            float? totalNormalizedDifference = null)
         {
             this.ExpectedImage = expectedImage;
             this.ActualImage = actualImage;
+            this.TotalNormalizedDifference = totalNormalizedDifference;
             this.Differences = differences.ToArray();
         }
 
         public static ImageSimilarityReport Empty =>
-            new ImageSimilarityReport(null, null, Enumerable.Empty<PixelDifference>());
+            new ImageSimilarityReport(null, null, Enumerable.Empty<PixelDifference>(), null);
+
+        public float? TotalNormalizedDifference { get; }
 
         public IImageBase ExpectedImage { get; }
 
@@ -27,23 +34,27 @@
 
         public override string ToString()
         {
-            return this.IsEmpty ? "[SimilarImages]" : StringifyDifferences(this.Differences);
+            return this.IsEmpty ? "[SimilarImages]" : this.PrintDifference();
         }
         
-        private static string StringifyDifferences(PixelDifference[] differences)
+        private string PrintDifference()
         {
             var sb = new StringBuilder();
-            int max = Math.Min(5, differences.Length);
+            if (this.TotalNormalizedDifference.HasValue)
+            {
+                sb.AppendLine($"Total difference: {this.TotalNormalizedDifference.Value * 100:0.00}%");
+            }
+            int max = Math.Min(5, this.Differences.Length);
 
             for (int i = 0; i < max; i++)
             {
-                sb.Append(differences[i]);
+                sb.Append(this.Differences[i]);
                 if (i < max - 1)
                 {
                     sb.Append("; ");
                 }
             }
-            if (differences.Length >= 5)
+            if (this.Differences.Length >= 5)
             {
                 sb.Append("...");
             }

tests/ImageSharp.Tests/TestUtilities/ImageComparison/TolerantImageComparer.cs

@@ -37,14 +37,14 @@
         /// For an individual <see cref="Rgba32"/> pixel the value it's calculated as: pixel.R + pixel.G + pixel.B + pixel.A
         /// </summary>
         public int PixelThresholdInPixelByteSum { get; }
-
+        
         public override ImageSimilarityReport CompareImagesOrFrames<TPixelA, TPixelB>(ImageBase<TPixelA> expected, ImageBase<TPixelB> actual)
         {
             if (expected.Size() != actual.Size())
             {
                 throw new InvalidOperationException("Calling ImageComparer is invalid when dimensions mismatch!");
             }
-
+            
             int width = actual.Width;
 
             // TODO: Comparing through Rgba32 is not robust enough because of the existance of super high precision pixel types.
@@ -83,7 +83,7 @@
             
             if (normalizedDifference > this.ImageThreshold)
             {
-                return new ImageSimilarityReport(expected, actual, differences);
+                return new ImageSimilarityReport(expected, actual, differences, normalizedDifference);
             }
             else
             {

tests/ImageSharp.Tests/TestUtilities/ImageProviders/FileProvider.cs

@@ -28,29 +28,29 @@ namespace ImageSharp.Tests
             }
 
             private static readonly ConcurrentDictionary<Key, Image<TPixel>> cache = new ConcurrentDictionary<Key, Image<TPixel>>();
-
+            
-            private string filePath;
-
             public FileProvider(string filePath)
             {
-                this.filePath = filePath;
+                this.FilePath = filePath;
             }
 
             public FileProvider()
             {
             }
 
-            public override string SourceFileOrDescription => this.filePath;
+            public string FilePath { get; private set; }
+
+            public override string SourceFileOrDescription => this.FilePath;
 
             public override Image<TPixel> GetImage()
             {
-                Key key = new Key(this.PixelType, this.filePath);
+                Key key = new Key(this.PixelType, this.FilePath);
 
                 Image<TPixel> cachedImage = cache.GetOrAdd(
                     key,
                     fn =>
                         {
-                            TestFile testFile = TestFile.Create(this.filePath);
+                            TestFile testFile = TestFile.Create(this.FilePath);
                             return this.Factory.CreateImage(testFile.Bytes);
                         });
 
@@ -59,7 +59,7 @@ namespace ImageSharp.Tests
 
             public override void Deserialize(IXunitSerializationInfo info)
             {
-                this.filePath = info.GetValue<string>("path");
+                this.FilePath = info.GetValue<string>("path");
 
                 base.Deserialize(info); // must be called last
             }
@@ -67,8 +67,14 @@ namespace ImageSharp.Tests
             public override void Serialize(IXunitSerializationInfo info)
             {
                 base.Serialize(info);
-                info.AddValue("path", this.filePath);
+                info.AddValue("path", this.FilePath);
             }
         }
+
+        public static string GetFilePathOrNull(ITestImageProvider provider)
+        {
+            var fileProvider = provider as FileProvider;
+            return fileProvider?.FilePath;
+        }
     }
 }

tests/ImageSharp.Tests/TestUtilities/ImagingTestCaseUtility.cs

@@ -23,42 +23,42 @@ namespace ImageSharp.Tests
         /// <summary>
         /// Name of the TPixel in the owner <see cref="TestImageProvider{TPixel}"/>
         /// </summary>
-        public string PixelTypeName { get; set; } = string.Empty;
+        public string PixelTypeName { get; set; } = String.Empty;
 
         /// <summary>
         /// The name of the file which is provided by <see cref="TestImageProvider{TPixel}"/>
         /// Or a short string describing the image in the case of a non-file based image provider.
         /// </summary>
-        public string SourceFileOrDescription { get; set; } = string.Empty;
+        public string SourceFileOrDescription { get; set; } = String.Empty;
 
         /// <summary>
         /// By default this is the name of the test class, but it's possible to change it
         /// </summary>
-        public string TestGroupName { get; set; } = string.Empty;
+        public string TestGroupName { get; set; } = String.Empty;
 
         /// <summary>
         /// The name of the test case (by default)
         /// </summary>
-        public string TestName { get; set; } = string.Empty;
+        public string TestName { get; set; } = String.Empty;
 
         
         private string GetTestOutputFileNameImpl(string extension, string tag)
         {
-            string fn = string.Empty;
+            string fn = String.Empty;
 
-            if (string.IsNullOrWhiteSpace(extension))
+            if (String.IsNullOrWhiteSpace(extension))
             {
                 extension = null;
             }
 
             fn = Path.GetFileNameWithoutExtension(this.SourceFileOrDescription);
 
-            if (string.IsNullOrWhiteSpace(extension))
+            if (String.IsNullOrWhiteSpace(extension))
             {
                 extension = Path.GetExtension(this.SourceFileOrDescription);
             }
 
-            if (string.IsNullOrWhiteSpace(extension))
+            if (String.IsNullOrWhiteSpace(extension))
             {
                 extension = ".bmp";
             }
@@ -68,16 +68,16 @@ namespace ImageSharp.Tests
                 extension = '.' + extension;
             }
 
-            if (fn != string.Empty) fn = '_' + fn;
+            if (fn != String.Empty) fn = '_' + fn;
 
             string pixName = this.PixelTypeName;
-            if (pixName != string.Empty)
+            if (pixName != String.Empty)
             {
                 pixName = '_' + pixName;
             }
 
-            tag = tag ?? string.Empty;
+            tag = tag ?? String.Empty;
-            if (tag != string.Empty)
+            if (tag != String.Empty)
             {
                 tag = '_' + tag;
             }
@@ -113,7 +113,7 @@ namespace ImageSharp.Tests
                 {
                     IEnumerable<PropertyInfo> properties = settings.GetType().GetRuntimeProperties();
 
-                    tag = string.Join("_", properties.ToDictionary(x => x.Name, x => x.GetValue(settings)).Select(x => $"{x.Key}-{x.Value}"));
+                    tag = String.Join("_", properties.ToDictionary(x => x.Name, x => x.GetValue(settings)).Select(x => $"{x.Key}-{x.Value}"));
                 }
             }
             return this.GetTestOutputFileNameImpl(extension, tag);
@@ -185,5 +185,52 @@ namespace ImageSharp.Tests
             string testGroupName = Path.GetFileNameWithoutExtension(this.TestGroupName);
             return this.CreateOutputDirectory(testGroupName);
         }
+
+        public static void ModifyPixel<TPixel>(ImageBase<TPixel> img, int x, int y, byte perChannelChange)
+            where TPixel : struct, IPixel<TPixel>
+        {
+            TPixel pixel = img[x, y];
+            var rgbaPixel = default(Rgba32);
+            pixel.ToRgba32(ref rgbaPixel);
+
+            if (rgbaPixel.R + perChannelChange <= 255)
+            {
+                rgbaPixel.R += perChannelChange;
+            }
+            else
+            {
+                rgbaPixel.R -= perChannelChange;
+            }
+
+            if (rgbaPixel.G + perChannelChange <= 255)
+            {
+                rgbaPixel.G += perChannelChange;
+            }
+            else
+            {
+                rgbaPixel.G -= perChannelChange;
+            }
+
+            if (rgbaPixel.B + perChannelChange <= 255)
+            {
+                rgbaPixel.B += perChannelChange;
+            }
+            else
+            {
+                rgbaPixel.B -= perChannelChange;
+            }
+
+            if (rgbaPixel.A + perChannelChange <= 255)
+            {
+                rgbaPixel.A += perChannelChange;
+            }
+            else
+            {
+                rgbaPixel.A -= perChannelChange;
+            }
+
+            pixel.PackFromRgba32(rgbaPixel);
+            img[x, y] = pixel;
+        }
     }
 }

tests/ImageSharp.Tests/TestUtilities/ReferenceCodecs/ReferenceDecoder.cs

@@ -2,6 +2,7 @@ namespace ImageSharp.Tests.TestUtilities.ReferenceCodecs
 {
     using System;
     using System.Drawing;
+    using System.Drawing.Drawing2D;
     using System.IO;
 
     using ImageSharp.Formats;
@@ -28,7 +29,12 @@ namespace ImageSharp.Tests.TestUtilities.ReferenceCodecs
                 {
                     using (var g = Graphics.FromImage(convertedBitmap))
                     {
-                        g.DrawImage(sourceBitmap, new PointF(0, 0));
+                        g.CompositingQuality = System.Drawing.Drawing2D.CompositingQuality.HighQuality;
+                        g.InterpolationMode = System.Drawing.Drawing2D.InterpolationMode.HighQualityBicubic;
+                        g.SmoothingMode = System.Drawing.Drawing2D.SmoothingMode.HighQuality;
+                        g.PixelOffsetMode = PixelOffsetMode.HighQuality;
+
+                        g.DrawImage(sourceBitmap, 0, 0, sourceBitmap.Width, sourceBitmap.Height);
                     }
                     return SystemDrawingBridge.FromSystemDrawingBitmap<TPixel>(convertedBitmap);
                 }

tests/ImageSharp.Tests/TestUtilities/TestImageExtensions.cs

@@ -96,6 +96,7 @@ namespace ImageSharp.Tests
             where TPixel : struct, IPixel<TPixel>
         {
             string referenceOutputFile = provider.Utility.GetReferenceOutputFileName(extension, testOutputDetails);
+            extension = extension.ToLower();
 
             if (!TestEnvironment.RunsOnCI)
             {
@@ -110,13 +111,44 @@ namespace ImageSharp.Tests
             {
                 throw new Exception("Reference output file missing: " + referenceOutputFile);
             }
-
+            
-            using (Image<Rgba32> referenceImage = Image.Load<Rgba32>(referenceOutputFile, ReferenceDecoder.Instance))
+            using (var referenceImage = Image.Load<Rgba32>(referenceOutputFile/*, ReferenceDecoder.Instance*/))
             {
                 comparer.VerifySimilarity(referenceImage, image);
             }
 
             return image;
         }
+
+        public static Image<TPixel> CompareToOriginal<TPixel>(
+            this Image<TPixel> image,
+            ITestImageProvider provider)
+            where TPixel : struct, IPixel<TPixel>
+        {
+            return CompareToOriginal(image, provider, ImageComparer.Tolerant());
+        }
+
+        public static Image<TPixel> CompareToOriginal<TPixel>(
+            this Image<TPixel> image,
+            ITestImageProvider provider,
+            ImageComparer comparer)
+            where TPixel : struct, IPixel<TPixel>
+        {
+            string path = TestImageProvider<TPixel>.GetFilePathOrNull(provider);
+            if (path == null)
+            {
+                throw new InvalidOperationException("CompareToOriginal() works only with file providers!");
+            }
+
+            var testFile = TestFile.Create(path);
+
+            using (var original = Image.Load<TPixel>(testFile.Bytes, ReferenceDecoder.Instance))
+            {
+                //original.DebugSave(provider, "__SYSTEMDRAWING__");
+                comparer.VerifySimilarity(original, image);
+            }
+
+            return image;
+        }
     }
 }

tests/ImageSharp.Tests/TestUtilities/Tests/ImageComparerTests.cs

@@ -42,53 +42,6 @@ namespace ImageSharp.Tests
             }
         }
 
-        private static void ModifyPixel<TPixel>(ImageBase<TPixel> img, int x, int y, byte perChannelChange)
-            where TPixel : struct, IPixel<TPixel>
-        {
-            TPixel pixel = img[x, y];
-            var rgbaPixel = default(Rgba32);
-            pixel.ToRgba32(ref rgbaPixel);
-
-            if (rgbaPixel.R + perChannelChange <= 255)
-            {
-                rgbaPixel.R += perChannelChange;
-            }
-            else
-            {
-                rgbaPixel.R -= perChannelChange;
-            }
-
-            if (rgbaPixel.G + perChannelChange <= 255)
-            {
-                rgbaPixel.G += perChannelChange;
-            }
-            else
-            {
-                rgbaPixel.G -= perChannelChange;
-            }
-
-            if (rgbaPixel.B + perChannelChange <= 255)
-            {
-                rgbaPixel.B += perChannelChange;
-            }
-            else
-            {
-                rgbaPixel.B -= perChannelChange;
-            }
-
-            if (rgbaPixel.A + perChannelChange <= 255)
-            {
-                rgbaPixel.A += perChannelChange;
-            }
-            else
-            {
-                rgbaPixel.A -= perChannelChange;
-            }
-
-            pixel.PackFromRgba32(rgbaPixel);
-            img[x, y] = pixel;
-        }
-
         [Theory]
         [WithTestPatternImages(110, 110, PixelTypes.Rgba32)]
         public void TolerantImageComparer_ApprovesSimilarityBelowTolerance<TPixel>(TestImageProvider<TPixel> provider)
@@ -98,7 +51,7 @@ namespace ImageSharp.Tests
             {
                 using (Image<TPixel> clone = image.Clone())
                 {
-                    ModifyPixel(clone, 0, 0, 1);
+                    ImagingTestCaseUtility.ModifyPixel(clone, 0, 0, 1);
 
                     var comparer = ImageComparer.Tolerant();
                     comparer.VerifySimilarity(image, clone);
@@ -115,7 +68,7 @@ namespace ImageSharp.Tests
             {
                 using (Image<TPixel> clone = image.Clone())
                 {
-                    ModifyPixel(clone, 3, 1, 2);
+                    ImagingTestCaseUtility.ModifyPixel(clone, 3, 1, 2);
 
                     var comparer = ImageComparer.Tolerant();
 
@@ -139,9 +92,9 @@ namespace ImageSharp.Tests
             {
                 using (Image<TPixel> clone = image.Clone())
                 {
-                    ModifyPixel(clone, 0, 0, 10);
+                    ImagingTestCaseUtility.ModifyPixel(clone, 0, 0, 10);
-                    ModifyPixel(clone, 1, 0, 10);
+                    ImagingTestCaseUtility.ModifyPixel(clone, 1, 0, 10);
-                    ModifyPixel(clone, 2, 0, 10);
+                    ImagingTestCaseUtility.ModifyPixel(clone, 2, 0, 10);
 
                     var comparer = ImageComparer.Tolerant(pixelThresholdInPixelByteSum: 42);
                     comparer.VerifySimilarity(image, clone);
@@ -180,7 +133,7 @@ namespace ImageSharp.Tests
             {
                 using (Image<TPixel> clone = image.Clone())
                 {
-                    ModifyPixel(clone.Frames[0], 42, 43, 1);
+                    ImagingTestCaseUtility.ModifyPixel(clone.Frames[0], 42, 43, 1);
 
                     IEnumerable<ImageSimilarityReport> reports = ImageComparer.Exact.CompareImages(image, clone);
 
@@ -214,8 +167,8 @@ namespace ImageSharp.Tests
             {
                 using (Image<TPixel> clone = image.Clone())
                 {
-                    ModifyPixel(clone, 42, 24, 1);
+                    ImagingTestCaseUtility.ModifyPixel(clone, 42, 24, 1);
-                    ModifyPixel(clone, 7, 93, 1);
+                    ImagingTestCaseUtility.ModifyPixel(clone, 7, 93, 1);
 
                     IEnumerable<ImageSimilarityReport> reports = ExactImageComparer.Instance.CompareImages(image, clone);
                     

tests/ImageSharp.Tests/TestUtilities/Tests/TestImageExtensionsTests.cs

@@ -4,6 +4,9 @@ namespace ImageSharp.Tests
     using System;
 
     using ImageSharp.PixelFormats;
+    using ImageSharp.Tests.TestUtilities.ImageComparison;
+
+    using Moq;
 
     using Xunit;
 
@@ -43,5 +46,51 @@ namespace ImageSharp.Tests
                 Assert.ThrowsAny<Exception>(() => image.CompareToReferenceOutput(provider));
             }
         }
+
+        [Theory]
+        [WithFile(TestImages.Png.CalliphoraPartial, PixelTypes.Rgba32)]
+        public void CompareToOriginal_WhenSimilar<TPixel>(TestImageProvider<TPixel> provider)
+            where TPixel : struct, IPixel<TPixel>
+        {
+            using (Image<TPixel> image = provider.GetImage())
+            {
+                using (Image<TPixel> clone = image.Clone())
+                {
+                    clone.CompareToOriginal(provider, ImageComparer.Exact);
+                }
+            }
+        }
+
+        [Theory]
+        [WithFile(TestImages.Png.CalliphoraPartial, PixelTypes.Rgba32)]
+        public void CompareToOriginal_WhenDifferent_Throws<TPixel>(TestImageProvider<TPixel> provider)
+            where TPixel : struct, IPixel<TPixel>
+        {
+            using (Image<TPixel> image = provider.GetImage())
+            {
+                ImagingTestCaseUtility.ModifyPixel(image, 3, 1, 1);
+
+                Assert.ThrowsAny<ImagePixelsAreDifferentException>(
+                    () =>
+                        {
+                            image.CompareToOriginal(provider, ImageComparer.Exact);
+                        });
+            }
+        }
+
+        [Theory]
+        [WithBlankImages(10, 10, PixelTypes.Rgba32)]
+        public void CompareToOriginal_WhenInputIsNotFromFile_Throws<TPixel>(TestImageProvider<TPixel> provider)
+            where TPixel : struct, IPixel<TPixel>
+        {
+            using (Image<TPixel> image = provider.GetImage())
+            {
+                Assert.ThrowsAny<Exception>(
+                    () =>
+                        {
+                            image.CompareToOriginal(provider, Mock.Of<ImageComparer>());
+                        });
+            }
+        }
     }
 }

tests/Images/External

@@ -1 +1 @@
-Subproject commit fcd68139fcc6f0ce9af29b716eb1f3874128315e
+Subproject commit f01bd8ace62e69aa9be9fcdf8bc00789d8d75a53