From 9626f46655bb807d633ae4d03e1d373ea208cc5f Mon Sep 17 00:00:00 2001 From: Anton Firszov Date: Tue, 3 Jan 2017 03:56:37 +0100 Subject: [PATCH] BetterBenchmarks(TM): cleanup MultiImageBenchmarkBase, ImageBenchmarkTests: "unit tests" for MultiImageBenchmarkBase, Config.Short for long running benchmarks, EncodeBmpMultiple, EncodeGifMultiple --- tests/ImageSharp.Benchmarks/Config.cs | 14 ++++ .../Image/DecodeJpegMultiple.cs | 11 ++- .../Image/EncodeBmpMultiple.cs | 42 ++++++++++ .../Image/EncodeGifMultiple.cs | 53 ++++++++++++ .../Image/EncodeJpegMultiple.cs | 33 ++++---- .../Image/ImageBenchmarkTests.cs | 81 +++++++++++++++++++ .../Image/MultiImageBenchmarkBase.cs | 73 ++++++++++++++--- 7 files changed, 271 insertions(+), 36 deletions(-) create mode 100644 tests/ImageSharp.Benchmarks/Image/EncodeBmpMultiple.cs create mode 100644 tests/ImageSharp.Benchmarks/Image/EncodeGifMultiple.cs create mode 100644 tests/ImageSharp.Benchmarks/Image/ImageBenchmarkTests.cs diff --git a/tests/ImageSharp.Benchmarks/Config.cs b/tests/ImageSharp.Benchmarks/Config.cs index ecede3596..8e278330b 100644 --- a/tests/ImageSharp.Benchmarks/Config.cs +++ b/tests/ImageSharp.Benchmarks/Config.cs @@ -7,6 +7,8 @@ using BenchmarkDotNet.Configs; namespace ImageSharp.Benchmarks { + using BenchmarkDotNet.Jobs; + public class Config : ManualConfig { public Config() @@ -14,5 +16,17 @@ namespace ImageSharp.Benchmarks // Uncomment if you want to use any of the diagnoser this.Add(new BenchmarkDotNet.Diagnosers.MemoryDiagnoser()); } + + public class Short : Config + { + public Short() + { + this.Add( + Job.Default.WithLaunchCount(1) + .WithWarmupCount(3) + .WithTargetCount(3) + ); + } + } } } \ No newline at end of file diff --git a/tests/ImageSharp.Benchmarks/Image/DecodeJpegMultiple.cs b/tests/ImageSharp.Benchmarks/Image/DecodeJpegMultiple.cs index a99554558..c79d61538 100644 --- a/tests/ImageSharp.Benchmarks/Image/DecodeJpegMultiple.cs +++ b/tests/ImageSharp.Benchmarks/Image/DecodeJpegMultiple.cs @@ -1,14 +1,17 @@ -using System.Collections.Generic; +// +// Copyright (c) James Jackson-South and contributors. +// Licensed under the Apache License, Version 2.0. +// namespace ImageSharp.Benchmarks.Image { - using System.Drawing; - using System.IO; + using System.Collections.Generic; using BenchmarkDotNet.Attributes; using Image = ImageSharp.Image; using ImageSharpSize = ImageSharp.Size; + [Config(typeof(Config.Short))] public class DecodeJpegMultiple : MultiImageBenchmarkBase { protected override IEnumerable InputImageSubfoldersOrFiles => new[] @@ -16,7 +19,7 @@ namespace ImageSharp.Benchmarks.Image "Jpg/" }; - protected override IEnumerable FileFilters => new[] { "*.jpg" }; + protected override IEnumerable SearchPatterns => new[] { "*.jpg" }; [Benchmark(Description = "DecodeJpegMultiple - ImageSharp")] public void DecodeJpegImageSharp() diff --git a/tests/ImageSharp.Benchmarks/Image/EncodeBmpMultiple.cs b/tests/ImageSharp.Benchmarks/Image/EncodeBmpMultiple.cs new file mode 100644 index 000000000..1775f144d --- /dev/null +++ b/tests/ImageSharp.Benchmarks/Image/EncodeBmpMultiple.cs @@ -0,0 +1,42 @@ +// +// Copyright (c) James Jackson-South and contributors. +// Licensed under the Apache License, Version 2.0. +// + +namespace ImageSharp.Benchmarks.Image +{ + using System.Collections.Generic; + using System.Drawing.Imaging; + + using BenchmarkDotNet.Attributes; + + using ImageSharp.Formats; + + [Config(typeof(Config.Short))] + public class EncodeBmpMultiple : MultiImageBenchmarkBase.WithImagesPreloaded + { + protected override IEnumerable InputImageSubfoldersOrFiles => new[] { "Bmp/", "Jpg/baseline" }; + + [Benchmark(Description = "EncodeBmpMultiple - ImageSharp")] + public void EncodeGifImageSharp() + { + this.ForEachImageSharpImage( + (img, ms) => + { + img.Save(ms, new BmpEncoder()); + return null; + }); + } + + [Benchmark(Baseline = true, Description = "EncodeBmpMultiple - System.Drawing")] + public void EncodeGifSystemDrawing() + { + this.ForEachSystemDrawingImage( + (img, ms) => + { + img.Save(ms, ImageFormat.Bmp); + return null; + }); + } + } +} \ No newline at end of file diff --git a/tests/ImageSharp.Benchmarks/Image/EncodeGifMultiple.cs b/tests/ImageSharp.Benchmarks/Image/EncodeGifMultiple.cs new file mode 100644 index 000000000..8d1d14746 --- /dev/null +++ b/tests/ImageSharp.Benchmarks/Image/EncodeGifMultiple.cs @@ -0,0 +1,53 @@ +namespace ImageSharp.Benchmarks.Image +{ + using System.Collections.Generic; + using System.Drawing.Imaging; + + using BenchmarkDotNet.Attributes; + using BenchmarkDotNet.Jobs; + + using ImageSharp.Formats; + + [Config(typeof(SingleRunConfig))] + public class EncodeGifMultiple : MultiImageBenchmarkBase.WithImagesPreloaded + { + public class SingleRunConfig : Config + { + public SingleRunConfig() + { + this.Add( + Job.Default.WithLaunchCount(1) + .WithWarmupCount(1) + .WithTargetCount(1) + ); + } + } + + [Params(InputImageCategory.AllImages)] + public override InputImageCategory InputCategory { get; set; } + + protected override IEnumerable InputImageSubfoldersOrFiles => new[] { "Gif/" }; + + [Benchmark(Description = "EncodeGifMultiple - ImageSharp")] + public void EncodeGifImageSharp() + { + this.ForEachImageSharpImage( + (img, ms) => + { + img.Save(ms, new GifEncoder()); + return null; + }); + } + + [Benchmark(Baseline = true, Description = "EncodeGifMultiple - System.Drawing")] + public void EncodeGifSystemDrawing() + { + this.ForEachSystemDrawingImage( + (img, ms) => + { + img.Save(ms, ImageFormat.Gif); + return null; + }); + } + } +} \ No newline at end of file diff --git a/tests/ImageSharp.Benchmarks/Image/EncodeJpegMultiple.cs b/tests/ImageSharp.Benchmarks/Image/EncodeJpegMultiple.cs index e631de57f..bbf838a9e 100644 --- a/tests/ImageSharp.Benchmarks/Image/EncodeJpegMultiple.cs +++ b/tests/ImageSharp.Benchmarks/Image/EncodeJpegMultiple.cs @@ -1,36 +1,33 @@ -namespace ImageSharp.Benchmarks.Image +// +// Copyright (c) James Jackson-South and contributors. +// Licensed under the Apache License, Version 2.0. +// + +namespace ImageSharp.Benchmarks.Image { - using System; using System.Collections.Generic; using System.Drawing.Imaging; - using System.IO; - using System.Linq; using BenchmarkDotNet.Attributes; - using BenchmarkDotNet.Attributes.Jobs; - using BenchmarkDotNet.Engines; using ImageSharp.Formats; + + [Config(typeof(Config.Short))] // It's long enough to iterate through multiple files public class EncodeJpegMultiple : MultiImageBenchmarkBase.WithImagesPreloaded { - protected override IEnumerable InputImageSubfoldersOrFiles => new[] - { - "Bmp/", - "Jpg/baseline" - }; + protected override IEnumerable InputImageSubfoldersOrFiles => new[] { "Bmp/", "Jpg/baseline" }; - protected override IEnumerable FileFilters => new[] { "*.bmp", "*.jpg" }; + protected override IEnumerable SearchPatterns => new[] { "*.bmp", "*.jpg" }; [Benchmark(Description = "EncodeJpegMultiple - ImageSharp")] public void EncodeJpegImageSharp() { this.ForEachImageSharpImage( - img => + (img, ms) => { - MemoryStream ms = new MemoryStream(); img.Save(ms, new JpegEncoder()); - return ms; + return null; }); } @@ -38,13 +35,11 @@ public void EncodeJpegSystemDrawing() { this.ForEachSystemDrawingImage( - img => + (img, ms) => { - MemoryStream ms = new MemoryStream(); img.Save(ms, ImageFormat.Jpeg); - return ms; + return null; }); } - } } \ No newline at end of file diff --git a/tests/ImageSharp.Benchmarks/Image/ImageBenchmarkTests.cs b/tests/ImageSharp.Benchmarks/Image/ImageBenchmarkTests.cs new file mode 100644 index 000000000..f68f4a418 --- /dev/null +++ b/tests/ImageSharp.Benchmarks/Image/ImageBenchmarkTests.cs @@ -0,0 +1,81 @@ +// +// Copyright (c) James Jackson-South and contributors. +// Licensed under the Apache License, Version 2.0. +// + +// This file contains small, cheap and "unit test" benchmarks to test MultiImageBenchmarkBase. +// Need this because there are no real test cases for the common benchmark utility stuff. + +// Uncomment this to enable benchmark testing +// #define TEST + +#if TEST + +// ReSharper disable InconsistentNaming +namespace ImageSharp.Benchmarks.Image +{ + using System; + using System.Collections.Generic; + using System.Linq; + using System.Runtime.CompilerServices; + + using BenchmarkDotNet.Attributes; + using BenchmarkDotNet.Configs; + using BenchmarkDotNet.Jobs; + + internal class Assert + { + public static void True(bool condition, string meassage) + { + if (!condition) throw new Exception(meassage); + } + + public static void Equal(T a, T b, string message) => True(a.Equals(b), message); + + public static void Equal(object a, object b, string message) => True(a.Equals(b), message); + } + + public class MultiImageBenchmarkBase_ShouldReadSingleFiles : MultiImageBenchmarkBase + { + protected override IEnumerable InputImageSubfoldersOrFiles => new[] { "Bmp/F.bmp", "Jpg/Exif.jpg" }; + + [Benchmark] + public void Run() + { + //Console.WriteLine("FileNames2Bytes.Count(): " + this.FileNames2Bytes.Count()); + Assert.Equal(this.FileNames2Bytes.Count(), 2, "MultiImageBenchmarkBase_ShouldReadFiles failed"); + } + } + + public class MultiImageBenchmarkBase_ShouldReadMixed : MultiImageBenchmarkBase + { + protected override IEnumerable InputImageSubfoldersOrFiles => new[] { "Bmp/F.bmp", "Gif/" }; + + [Benchmark] + public void Run() + { + //Console.WriteLine("FileNames2Bytes.Count(): " + this.FileNames2Bytes.Count()); + Assert.True(this.FileNames2Bytes.Count() > 2, "MultiImageBenchmarkBase_ShouldReadMixed failed"); + Assert.True( + this.FileNames2Bytes.Any(kv => kv.Key.Contains("F.bmp")), + "MultiImageBenchmarkBase_ShouldReadMixed failed" + ); + } + } + + [Config(typeof(Config.Short))] + public class UseCustomConfigTest : MultiImageBenchmarkBase + { + protected override IEnumerable InputImageSubfoldersOrFiles => new[] { "Bmp/" }; + + [Benchmark] + public void Run() + { + this.ForEachStream( + ms => new ImageSharp.Image(ms) + ); + } + } +} + +#endif \ No newline at end of file diff --git a/tests/ImageSharp.Benchmarks/Image/MultiImageBenchmarkBase.cs b/tests/ImageSharp.Benchmarks/Image/MultiImageBenchmarkBase.cs index b2c63fa27..b6d0341c4 100644 --- a/tests/ImageSharp.Benchmarks/Image/MultiImageBenchmarkBase.cs +++ b/tests/ImageSharp.Benchmarks/Image/MultiImageBenchmarkBase.cs @@ -1,4 +1,10 @@ -namespace ImageSharp.Benchmarks.Image +// +// Copyright (c) James Jackson-South and contributors. +// Licensed under the Apache License, Version 2.0. +// + + +namespace ImageSharp.Benchmarks.Image { using System; using System.Collections.Generic; @@ -31,11 +37,11 @@ } [Params(InputImageCategory.AllImages, InputImageCategory.SmallImagesOnly, InputImageCategory.LargeImagesOnly)] - public InputImageCategory InputCategory { get; set; } + public virtual InputImageCategory InputCategory { get; set; } protected virtual string BaseFolder => "../ImageSharp.Tests/TestImages/Formats/"; - protected virtual IEnumerable FileFilters => new[] { "*.*" }; + protected virtual IEnumerable SearchPatterns => new[] { "*.*" }; /// /// Gets the file names containing these strings are substrings are not processed by the benchmark. @@ -81,19 +87,25 @@ public void ReadImages() { // Console.WriteLine("Vector.IsHardwareAccelerated: " + Vector.IsHardwareAccelerated); - this.ReadImagesImpl(); + this.ReadFilesImpl(); } - protected virtual void ReadImagesImpl() + protected virtual void ReadFilesImpl() { - foreach (string folder in this.AllFoldersOrFiles) + foreach (string path in this.AllFoldersOrFiles) { + if (File.Exists(path)) + { + this.FileNamesToBytes[path] = File.ReadAllBytes(path); + continue; + } var allFiles = - this.FileFilters.SelectMany( + this.SearchPatterns.SelectMany( f => - Directory.EnumerateFiles(folder, f, SearchOption.AllDirectories) + Directory.EnumerateFiles(path, f, SearchOption.AllDirectories) .Where(fn => !this.ExcludeSubstringsInFileNames.Any(w => fn.ToLower().Contains(w)))).ToArray(); + foreach (var fn in allFiles) { this.FileNamesToBytes[fn] = File.ReadAllBytes(fn); @@ -101,6 +113,10 @@ } } + /// + /// Execute code for each image stream. If the returned object of the opearation is it will be disposed. + /// + /// The operation to execute. If the returned object is <see cref="IDisposable"/> it will be disposed protected void ForEachStream(Func operation) { foreach (var kv in this.FileNames2Bytes) @@ -123,9 +139,9 @@ public abstract class WithImagesPreloaded : MultiImageBenchmarkBase { - protected override void ReadImagesImpl() + protected override void ReadFilesImpl() { - base.ReadImagesImpl(); + base.ReadFilesImpl(); foreach (var kv in this.FileNamesToBytes) { @@ -174,6 +190,23 @@ } } + protected void ForEachImageSharpImage(Func operation) + { + using (MemoryStream workStream = new MemoryStream()) + { + + this.ForEachImageSharpImage( + img => + { + // ReSharper disable AccessToDisposedClosure + object result = operation(img, workStream); + workStream.Seek(0, SeekOrigin.Begin); + // ReSharper restore AccessToDisposedClosure + return result; + }); + } + } + protected void ForEachSystemDrawingImage(Func operation) { foreach (var kv in this.FileNames2SystemDrawingImages) @@ -189,10 +222,24 @@ } } } - } + protected void ForEachSystemDrawingImage(Func operation) + { + using (MemoryStream workStream = new MemoryStream()) + { - } - + this.ForEachSystemDrawingImage( + img => + { + // ReSharper disable AccessToDisposedClosure + object result = operation(img, workStream); + workStream.Seek(0, SeekOrigin.Begin); + // ReSharper restore AccessToDisposedClosure + return result; + }); + } + } + } + } } \ No newline at end of file