diff --git a/tests/ImageSharp.Tests/Formats/Jpg/JpegDecoderTests.cs b/tests/ImageSharp.Tests/Formats/Jpg/JpegDecoderTests.cs index d625b8612..ff78f12e6 100644 --- a/tests/ImageSharp.Tests/Formats/Jpg/JpegDecoderTests.cs +++ b/tests/ImageSharp.Tests/Formats/Jpg/JpegDecoderTests.cs @@ -11,6 +11,7 @@ namespace ImageSharp.Tests using ImageSharp.Formats; using ImageSharp.PixelFormats; + using ImageSharp.Tests.TestUtilities.ImageComparison; using Xunit; @@ -25,6 +26,10 @@ namespace ImageSharp.Tests public static string[] ProgressiveTestJpegs = TestImages.Jpeg.Progressive.All; + // TODO: We should make this comparer less tolerant ... + private static readonly ImageComparer VeryTolerantJpegComparer = + ImageComparer.Tolerant(0.005f, pixelThresholdInPixelByteSum: 4); + [Theory] [WithFileCollection(nameof(BaselineTestJpegs), PixelTypes.Rgba32 | PixelTypes.Rgba32 | PixelTypes.Argb32)] public void DecodeBaselineJpeg(TestImageProvider provider) @@ -32,7 +37,7 @@ namespace ImageSharp.Tests { using (Image image = provider.GetImage()) { - image.DebugSave(provider); + image.CompareToReferenceOutput(provider, VeryTolerantJpegComparer); } } @@ -43,7 +48,7 @@ namespace ImageSharp.Tests { using (Image image = provider.GetImage()) { - image.DebugSave(provider); + image.DebugSave(provider, VeryTolerantJpegComparer); } } @@ -70,11 +75,9 @@ namespace ImageSharp.Tests image.Save(ms, encoder); } } - - // TODO: Automatic image comparers could help here a lot :P + Image mirror = provider.Factory.CreateImage(data); - provider.Utility.TestName += $"_{subsample}_Q{quality}"; - mirror.DebugSave(provider); + mirror.DebugSave(provider, $"_{subsample}_Q{quality}"); } [Theory] diff --git a/tests/ImageSharp.Tests/TestUtilities/ImageComparison/ImageSimilarityReport.cs b/tests/ImageSharp.Tests/TestUtilities/ImageComparison/ImageSimilarityReport.cs index 0740a7e82..74995537e 100644 --- a/tests/ImageSharp.Tests/TestUtilities/ImageComparison/ImageSimilarityReport.cs +++ b/tests/ImageSharp.Tests/TestUtilities/ImageComparison/ImageSimilarityReport.cs @@ -7,15 +7,22 @@ public class ImageSimilarityReport { - public ImageSimilarityReport(IImageBase expectedImage, IImageBase actualImage, IEnumerable differences) + public ImageSimilarityReport( + IImageBase expectedImage, + IImageBase actualImage, + IEnumerable differences, + float? totalNormalizedDifference = null) { this.ExpectedImage = expectedImage; this.ActualImage = actualImage; + this.TotalNormalizedDifference = totalNormalizedDifference; this.Differences = differences.ToArray(); } public static ImageSimilarityReport Empty => - new ImageSimilarityReport(null, null, Enumerable.Empty()); + new ImageSimilarityReport(null, null, Enumerable.Empty(), null); + + public float? TotalNormalizedDifference { get; } public IImageBase ExpectedImage { get; } @@ -27,23 +34,27 @@ public override string ToString() { - return this.IsEmpty ? "[SimilarImages]" : StringifyDifferences(this.Differences); + return this.IsEmpty ? "[SimilarImages]" : this.PrintDifference(); } - private static string StringifyDifferences(PixelDifference[] differences) + private string PrintDifference() { var sb = new StringBuilder(); - int max = Math.Min(5, differences.Length); + if (this.TotalNormalizedDifference.HasValue) + { + sb.AppendLine($"Total difference: {this.TotalNormalizedDifference.Value * 100:0.00}%"); + } + int max = Math.Min(5, this.Differences.Length); for (int i = 0; i < max; i++) { - sb.Append(differences[i]); + sb.Append(this.Differences[i]); if (i < max - 1) { sb.Append("; "); } } - if (differences.Length >= 5) + if (this.Differences.Length >= 5) { sb.Append("..."); } diff --git a/tests/ImageSharp.Tests/TestUtilities/ImageComparison/TolerantImageComparer.cs b/tests/ImageSharp.Tests/TestUtilities/ImageComparison/TolerantImageComparer.cs index bc938ca0c..23e576180 100644 --- a/tests/ImageSharp.Tests/TestUtilities/ImageComparison/TolerantImageComparer.cs +++ b/tests/ImageSharp.Tests/TestUtilities/ImageComparison/TolerantImageComparer.cs @@ -37,14 +37,14 @@ /// For an individual pixel the value it's calculated as: pixel.R + pixel.G + pixel.B + pixel.A /// public int PixelThresholdInPixelByteSum { get; } - + public override ImageSimilarityReport CompareImagesOrFrames(ImageBase expected, ImageBase actual) { if (expected.Size() != actual.Size()) { throw new InvalidOperationException("Calling ImageComparer is invalid when dimensions mismatch!"); } - + int width = actual.Width; // TODO: Comparing through Rgba32 is not robust enough because of the existance of super high precision pixel types. @@ -83,7 +83,7 @@ if (normalizedDifference > this.ImageThreshold) { - return new ImageSimilarityReport(expected, actual, differences); + return new ImageSimilarityReport(expected, actual, differences, normalizedDifference); } else { diff --git a/tests/ImageSharp.Tests/TestUtilities/ImageProviders/FileProvider.cs b/tests/ImageSharp.Tests/TestUtilities/ImageProviders/FileProvider.cs index 16eecd2fc..362924de5 100644 --- a/tests/ImageSharp.Tests/TestUtilities/ImageProviders/FileProvider.cs +++ b/tests/ImageSharp.Tests/TestUtilities/ImageProviders/FileProvider.cs @@ -28,29 +28,29 @@ namespace ImageSharp.Tests } private static readonly ConcurrentDictionary> cache = new ConcurrentDictionary>(); - - private string filePath; - + public FileProvider(string filePath) { - this.filePath = filePath; + this.FilePath = filePath; } public FileProvider() { } - public override string SourceFileOrDescription => this.filePath; + public string FilePath { get; private set; } + + public override string SourceFileOrDescription => this.FilePath; public override Image GetImage() { - Key key = new Key(this.PixelType, this.filePath); + Key key = new Key(this.PixelType, this.FilePath); Image cachedImage = cache.GetOrAdd( key, fn => { - TestFile testFile = TestFile.Create(this.filePath); + TestFile testFile = TestFile.Create(this.FilePath); return this.Factory.CreateImage(testFile.Bytes); }); @@ -59,7 +59,7 @@ namespace ImageSharp.Tests public override void Deserialize(IXunitSerializationInfo info) { - this.filePath = info.GetValue("path"); + this.FilePath = info.GetValue("path"); base.Deserialize(info); // must be called last } @@ -67,8 +67,14 @@ namespace ImageSharp.Tests public override void Serialize(IXunitSerializationInfo info) { base.Serialize(info); - info.AddValue("path", this.filePath); + info.AddValue("path", this.FilePath); } } + + public static string GetFilePathOrNull(ITestImageProvider provider) + { + var fileProvider = provider as FileProvider; + return fileProvider?.FilePath; + } } } \ No newline at end of file diff --git a/tests/ImageSharp.Tests/TestUtilities/ImagingTestCaseUtility.cs b/tests/ImageSharp.Tests/TestUtilities/ImagingTestCaseUtility.cs index a009b4f4c..30cb1bac8 100644 --- a/tests/ImageSharp.Tests/TestUtilities/ImagingTestCaseUtility.cs +++ b/tests/ImageSharp.Tests/TestUtilities/ImagingTestCaseUtility.cs @@ -23,42 +23,42 @@ namespace ImageSharp.Tests /// /// Name of the TPixel in the owner /// - public string PixelTypeName { get; set; } = string.Empty; + public string PixelTypeName { get; set; } = String.Empty; /// /// The name of the file which is provided by /// Or a short string describing the image in the case of a non-file based image provider. /// - public string SourceFileOrDescription { get; set; } = string.Empty; + public string SourceFileOrDescription { get; set; } = String.Empty; /// /// By default this is the name of the test class, but it's possible to change it /// - public string TestGroupName { get; set; } = string.Empty; + public string TestGroupName { get; set; } = String.Empty; /// /// The name of the test case (by default) /// - public string TestName { get; set; } = string.Empty; + public string TestName { get; set; } = String.Empty; private string GetTestOutputFileNameImpl(string extension, string tag) { - string fn = string.Empty; + string fn = String.Empty; - if (string.IsNullOrWhiteSpace(extension)) + if (String.IsNullOrWhiteSpace(extension)) { extension = null; } fn = Path.GetFileNameWithoutExtension(this.SourceFileOrDescription); - if (string.IsNullOrWhiteSpace(extension)) + if (String.IsNullOrWhiteSpace(extension)) { extension = Path.GetExtension(this.SourceFileOrDescription); } - if (string.IsNullOrWhiteSpace(extension)) + if (String.IsNullOrWhiteSpace(extension)) { extension = ".bmp"; } @@ -68,16 +68,16 @@ namespace ImageSharp.Tests extension = '.' + extension; } - if (fn != string.Empty) fn = '_' + fn; + if (fn != String.Empty) fn = '_' + fn; string pixName = this.PixelTypeName; - if (pixName != string.Empty) + if (pixName != String.Empty) { pixName = '_' + pixName; } - tag = tag ?? string.Empty; - if (tag != string.Empty) + tag = tag ?? String.Empty; + if (tag != String.Empty) { tag = '_' + tag; } @@ -113,7 +113,7 @@ namespace ImageSharp.Tests { IEnumerable properties = settings.GetType().GetRuntimeProperties(); - tag = string.Join("_", properties.ToDictionary(x => x.Name, x => x.GetValue(settings)).Select(x => $"{x.Key}-{x.Value}")); + tag = String.Join("_", properties.ToDictionary(x => x.Name, x => x.GetValue(settings)).Select(x => $"{x.Key}-{x.Value}")); } } return this.GetTestOutputFileNameImpl(extension, tag); @@ -185,5 +185,52 @@ namespace ImageSharp.Tests string testGroupName = Path.GetFileNameWithoutExtension(this.TestGroupName); return this.CreateOutputDirectory(testGroupName); } + + public static void ModifyPixel(ImageBase img, int x, int y, byte perChannelChange) + where TPixel : struct, IPixel + { + TPixel pixel = img[x, y]; + var rgbaPixel = default(Rgba32); + pixel.ToRgba32(ref rgbaPixel); + + if (rgbaPixel.R + perChannelChange <= 255) + { + rgbaPixel.R += perChannelChange; + } + else + { + rgbaPixel.R -= perChannelChange; + } + + if (rgbaPixel.G + perChannelChange <= 255) + { + rgbaPixel.G += perChannelChange; + } + else + { + rgbaPixel.G -= perChannelChange; + } + + if (rgbaPixel.B + perChannelChange <= 255) + { + rgbaPixel.B += perChannelChange; + } + else + { + rgbaPixel.B -= perChannelChange; + } + + if (rgbaPixel.A + perChannelChange <= 255) + { + rgbaPixel.A += perChannelChange; + } + else + { + rgbaPixel.A -= perChannelChange; + } + + pixel.PackFromRgba32(rgbaPixel); + img[x, y] = pixel; + } } } \ No newline at end of file diff --git a/tests/ImageSharp.Tests/TestUtilities/ReferenceCodecs/ReferenceDecoder.cs b/tests/ImageSharp.Tests/TestUtilities/ReferenceCodecs/ReferenceDecoder.cs index afb588b9f..3ed3248b2 100644 --- a/tests/ImageSharp.Tests/TestUtilities/ReferenceCodecs/ReferenceDecoder.cs +++ b/tests/ImageSharp.Tests/TestUtilities/ReferenceCodecs/ReferenceDecoder.cs @@ -2,6 +2,7 @@ namespace ImageSharp.Tests.TestUtilities.ReferenceCodecs { using System; using System.Drawing; + using System.Drawing.Drawing2D; using System.IO; using ImageSharp.Formats; @@ -28,7 +29,12 @@ namespace ImageSharp.Tests.TestUtilities.ReferenceCodecs { using (var g = Graphics.FromImage(convertedBitmap)) { - g.DrawImage(sourceBitmap, new PointF(0, 0)); + g.CompositingQuality = System.Drawing.Drawing2D.CompositingQuality.HighQuality; + g.InterpolationMode = System.Drawing.Drawing2D.InterpolationMode.HighQualityBicubic; + g.SmoothingMode = System.Drawing.Drawing2D.SmoothingMode.HighQuality; + g.PixelOffsetMode = PixelOffsetMode.HighQuality; + + g.DrawImage(sourceBitmap, 0, 0, sourceBitmap.Width, sourceBitmap.Height); } return SystemDrawingBridge.FromSystemDrawingBitmap(convertedBitmap); } diff --git a/tests/ImageSharp.Tests/TestUtilities/TestImageExtensions.cs b/tests/ImageSharp.Tests/TestUtilities/TestImageExtensions.cs index 41619a3d2..74fa16ad5 100644 --- a/tests/ImageSharp.Tests/TestUtilities/TestImageExtensions.cs +++ b/tests/ImageSharp.Tests/TestUtilities/TestImageExtensions.cs @@ -96,6 +96,7 @@ namespace ImageSharp.Tests where TPixel : struct, IPixel { string referenceOutputFile = provider.Utility.GetReferenceOutputFileName(extension, testOutputDetails); + extension = extension.ToLower(); if (!TestEnvironment.RunsOnCI) { @@ -110,13 +111,44 @@ namespace ImageSharp.Tests { throw new Exception("Reference output file missing: " + referenceOutputFile); } - - using (Image referenceImage = Image.Load(referenceOutputFile, ReferenceDecoder.Instance)) + + using (var referenceImage = Image.Load(referenceOutputFile/*, ReferenceDecoder.Instance*/)) { comparer.VerifySimilarity(referenceImage, image); } return image; } + + public static Image CompareToOriginal( + this Image image, + ITestImageProvider provider) + where TPixel : struct, IPixel + { + return CompareToOriginal(image, provider, ImageComparer.Tolerant()); + } + + public static Image CompareToOriginal( + this Image image, + ITestImageProvider provider, + ImageComparer comparer) + where TPixel : struct, IPixel + { + string path = TestImageProvider.GetFilePathOrNull(provider); + if (path == null) + { + throw new InvalidOperationException("CompareToOriginal() works only with file providers!"); + } + + var testFile = TestFile.Create(path); + + using (var original = Image.Load(testFile.Bytes, ReferenceDecoder.Instance)) + { + //original.DebugSave(provider, "__SYSTEMDRAWING__"); + comparer.VerifySimilarity(original, image); + } + + return image; + } } } \ No newline at end of file diff --git a/tests/ImageSharp.Tests/TestUtilities/Tests/ImageComparerTests.cs b/tests/ImageSharp.Tests/TestUtilities/Tests/ImageComparerTests.cs index 799bd822a..dbcdc78fe 100644 --- a/tests/ImageSharp.Tests/TestUtilities/Tests/ImageComparerTests.cs +++ b/tests/ImageSharp.Tests/TestUtilities/Tests/ImageComparerTests.cs @@ -42,53 +42,6 @@ namespace ImageSharp.Tests } } - private static void ModifyPixel(ImageBase img, int x, int y, byte perChannelChange) - where TPixel : struct, IPixel - { - TPixel pixel = img[x, y]; - var rgbaPixel = default(Rgba32); - pixel.ToRgba32(ref rgbaPixel); - - if (rgbaPixel.R + perChannelChange <= 255) - { - rgbaPixel.R += perChannelChange; - } - else - { - rgbaPixel.R -= perChannelChange; - } - - if (rgbaPixel.G + perChannelChange <= 255) - { - rgbaPixel.G += perChannelChange; - } - else - { - rgbaPixel.G -= perChannelChange; - } - - if (rgbaPixel.B + perChannelChange <= 255) - { - rgbaPixel.B += perChannelChange; - } - else - { - rgbaPixel.B -= perChannelChange; - } - - if (rgbaPixel.A + perChannelChange <= 255) - { - rgbaPixel.A += perChannelChange; - } - else - { - rgbaPixel.A -= perChannelChange; - } - - pixel.PackFromRgba32(rgbaPixel); - img[x, y] = pixel; - } - [Theory] [WithTestPatternImages(110, 110, PixelTypes.Rgba32)] public void TolerantImageComparer_ApprovesSimilarityBelowTolerance(TestImageProvider provider) @@ -98,7 +51,7 @@ namespace ImageSharp.Tests { using (Image clone = image.Clone()) { - ModifyPixel(clone, 0, 0, 1); + ImagingTestCaseUtility.ModifyPixel(clone, 0, 0, 1); var comparer = ImageComparer.Tolerant(); comparer.VerifySimilarity(image, clone); @@ -115,7 +68,7 @@ namespace ImageSharp.Tests { using (Image clone = image.Clone()) { - ModifyPixel(clone, 3, 1, 2); + ImagingTestCaseUtility.ModifyPixel(clone, 3, 1, 2); var comparer = ImageComparer.Tolerant(); @@ -139,9 +92,9 @@ namespace ImageSharp.Tests { using (Image clone = image.Clone()) { - ModifyPixel(clone, 0, 0, 10); - ModifyPixel(clone, 1, 0, 10); - ModifyPixel(clone, 2, 0, 10); + ImagingTestCaseUtility.ModifyPixel(clone, 0, 0, 10); + ImagingTestCaseUtility.ModifyPixel(clone, 1, 0, 10); + ImagingTestCaseUtility.ModifyPixel(clone, 2, 0, 10); var comparer = ImageComparer.Tolerant(pixelThresholdInPixelByteSum: 42); comparer.VerifySimilarity(image, clone); @@ -180,7 +133,7 @@ namespace ImageSharp.Tests { using (Image clone = image.Clone()) { - ModifyPixel(clone.Frames[0], 42, 43, 1); + ImagingTestCaseUtility.ModifyPixel(clone.Frames[0], 42, 43, 1); IEnumerable reports = ImageComparer.Exact.CompareImages(image, clone); @@ -214,8 +167,8 @@ namespace ImageSharp.Tests { using (Image clone = image.Clone()) { - ModifyPixel(clone, 42, 24, 1); - ModifyPixel(clone, 7, 93, 1); + ImagingTestCaseUtility.ModifyPixel(clone, 42, 24, 1); + ImagingTestCaseUtility.ModifyPixel(clone, 7, 93, 1); IEnumerable reports = ExactImageComparer.Instance.CompareImages(image, clone); diff --git a/tests/ImageSharp.Tests/TestUtilities/Tests/TestImageExtensionsTests.cs b/tests/ImageSharp.Tests/TestUtilities/Tests/TestImageExtensionsTests.cs index 1a5e85646..46cd86b5e 100644 --- a/tests/ImageSharp.Tests/TestUtilities/Tests/TestImageExtensionsTests.cs +++ b/tests/ImageSharp.Tests/TestUtilities/Tests/TestImageExtensionsTests.cs @@ -4,6 +4,9 @@ namespace ImageSharp.Tests using System; using ImageSharp.PixelFormats; + using ImageSharp.Tests.TestUtilities.ImageComparison; + + using Moq; using Xunit; @@ -43,5 +46,51 @@ namespace ImageSharp.Tests Assert.ThrowsAny(() => image.CompareToReferenceOutput(provider)); } } + + [Theory] + [WithFile(TestImages.Png.CalliphoraPartial, PixelTypes.Rgba32)] + public void CompareToOriginal_WhenSimilar(TestImageProvider provider) + where TPixel : struct, IPixel + { + using (Image image = provider.GetImage()) + { + using (Image clone = image.Clone()) + { + clone.CompareToOriginal(provider, ImageComparer.Exact); + } + } + } + + [Theory] + [WithFile(TestImages.Png.CalliphoraPartial, PixelTypes.Rgba32)] + public void CompareToOriginal_WhenDifferent_Throws(TestImageProvider provider) + where TPixel : struct, IPixel + { + using (Image image = provider.GetImage()) + { + ImagingTestCaseUtility.ModifyPixel(image, 3, 1, 1); + + Assert.ThrowsAny( + () => + { + image.CompareToOriginal(provider, ImageComparer.Exact); + }); + } + } + + [Theory] + [WithBlankImages(10, 10, PixelTypes.Rgba32)] + public void CompareToOriginal_WhenInputIsNotFromFile_Throws(TestImageProvider provider) + where TPixel : struct, IPixel + { + using (Image image = provider.GetImage()) + { + Assert.ThrowsAny( + () => + { + image.CompareToOriginal(provider, Mock.Of()); + }); + } + } } } \ No newline at end of file diff --git a/tests/Images/External b/tests/Images/External index fcd68139f..f01bd8ace 160000 --- a/tests/Images/External +++ b/tests/Images/External @@ -1 +1 @@ -Subproject commit fcd68139fcc6f0ce9af29b716eb1f3874128315e +Subproject commit f01bd8ace62e69aa9be9fcdf8bc00789d8d75a53