BetterBenchmarks(TM):

cleanup MultiImageBenchmarkBase,
ImageBenchmarkTests: "unit tests" for MultiImageBenchmarkBase,
Config.Short for long running benchmarks,
EncodeBmpMultiple,
EncodeGifMultiple

tests/ImageSharp.Benchmarks/Config.cs

@@ -7,6 +7,8 @@ using BenchmarkDotNet.Configs;
 
 namespace ImageSharp.Benchmarks
 {
+    using BenchmarkDotNet.Jobs;
+
     public class Config : ManualConfig
     {
         public Config()
@@ -14,5 +16,17 @@ namespace ImageSharp.Benchmarks
             // Uncomment if you want to use any of the diagnoser
             this.Add(new BenchmarkDotNet.Diagnosers.MemoryDiagnoser());
         }
+
+        public class Short : Config
+        {
+            public Short()
+            {
+                this.Add(
+                    Job.Default.WithLaunchCount(1)
+                        .WithWarmupCount(3)
+                        .WithTargetCount(3)
+                        );
+            }
+        }
     }
 }

tests/ImageSharp.Benchmarks/Image/DecodeJpegMultiple.cs

@@ -1,14 +1,17 @@
-using System.Collections.Generic;
+// <copyright file="DecodeJpegMultiple.cs" company="James Jackson-South">
+// Copyright (c) James Jackson-South and contributors.
+// Licensed under the Apache License, Version 2.0.
+// </copyright>
 
 namespace ImageSharp.Benchmarks.Image
 {
-    using System.Drawing;
+    using System.Collections.Generic;
-    using System.IO;
     using BenchmarkDotNet.Attributes;
 
     using Image = ImageSharp.Image;
     using ImageSharpSize = ImageSharp.Size;
 
+    [Config(typeof(Config.Short))]
     public class DecodeJpegMultiple : MultiImageBenchmarkBase
     {
         protected override IEnumerable<string> InputImageSubfoldersOrFiles => new[]
@@ -16,7 +19,7 @@ namespace ImageSharp.Benchmarks.Image
             "Jpg/"
         };
 
-        protected override IEnumerable<string> FileFilters => new[] { "*.jpg" };
+        protected override IEnumerable<string> SearchPatterns => new[] { "*.jpg" };
 
         [Benchmark(Description = "DecodeJpegMultiple - ImageSharp")]
         public void DecodeJpegImageSharp()

tests/ImageSharp.Benchmarks/Image/EncodeBmpMultiple.cs

@@ -0,0 +1,42 @@
+// <copyright file="EncodeBmpMultiple.cs" company="James Jackson-South">
+// Copyright (c) James Jackson-South and contributors.
+// Licensed under the Apache License, Version 2.0.
+// </copyright>
+
+namespace ImageSharp.Benchmarks.Image
+{
+    using System.Collections.Generic;
+    using System.Drawing.Imaging;
+
+    using BenchmarkDotNet.Attributes;
+
+    using ImageSharp.Formats;
+
+    [Config(typeof(Config.Short))]
+    public class EncodeBmpMultiple : MultiImageBenchmarkBase.WithImagesPreloaded
+    {
+        protected override IEnumerable<string> InputImageSubfoldersOrFiles => new[] { "Bmp/", "Jpg/baseline" };
+
+        [Benchmark(Description = "EncodeBmpMultiple - ImageSharp")]
+        public void EncodeGifImageSharp()
+        {
+            this.ForEachImageSharpImage(
+                (img, ms) =>
+                    {
+                        img.Save(ms, new BmpEncoder());
+                        return null;
+                    });
+        }
+
+        [Benchmark(Baseline = true, Description = "EncodeBmpMultiple - System.Drawing")]
+        public void EncodeGifSystemDrawing()
+        {
+            this.ForEachSystemDrawingImage(
+                (img, ms) =>
+                    {
+                        img.Save(ms, ImageFormat.Bmp);
+                        return null;
+                    });
+        }
+    }
+}

tests/ImageSharp.Benchmarks/Image/EncodeGifMultiple.cs

@@ -0,0 +1,53 @@
+namespace ImageSharp.Benchmarks.Image
+{
+    using System.Collections.Generic;
+    using System.Drawing.Imaging;
+
+    using BenchmarkDotNet.Attributes;
+    using BenchmarkDotNet.Jobs;
+
+    using ImageSharp.Formats;
+
+    [Config(typeof(SingleRunConfig))]
+    public class EncodeGifMultiple : MultiImageBenchmarkBase.WithImagesPreloaded
+    {
+        public class SingleRunConfig : Config
+        {
+            public SingleRunConfig()
+            {
+                this.Add(
+                    Job.Default.WithLaunchCount(1)
+                        .WithWarmupCount(1)
+                        .WithTargetCount(1)
+                        );
+            }
+        }
+
+        [Params(InputImageCategory.AllImages)]
+        public override InputImageCategory InputCategory { get; set; }
+
+        protected override IEnumerable<string> InputImageSubfoldersOrFiles => new[] { "Gif/" };
+
+        [Benchmark(Description = "EncodeGifMultiple - ImageSharp")]
+        public void EncodeGifImageSharp()
+        {
+            this.ForEachImageSharpImage(
+                (img, ms) =>
+                    {
+                        img.Save(ms, new GifEncoder());
+                        return null;
+                    });
+        }
+
+        [Benchmark(Baseline = true, Description = "EncodeGifMultiple - System.Drawing")]
+        public void EncodeGifSystemDrawing()
+        {
+            this.ForEachSystemDrawingImage(
+                (img, ms) =>
+                    {
+                        img.Save(ms, ImageFormat.Gif);
+                        return null;
+                    });
+        }
+    }
+}

tests/ImageSharp.Benchmarks/Image/EncodeJpegMultiple.cs

@@ -1,36 +1,33 @@
-namespace ImageSharp.Benchmarks.Image
+// <copyright file="EncodeJpegMultiple.cs" company="James Jackson-South">
+// Copyright (c) James Jackson-South and contributors.
+// Licensed under the Apache License, Version 2.0.
+// </copyright>
+
+namespace ImageSharp.Benchmarks.Image
 {
-    using System;
     using System.Collections.Generic;
     using System.Drawing.Imaging;
-    using System.IO;
-    using System.Linq;
 
     using BenchmarkDotNet.Attributes;
-    using BenchmarkDotNet.Attributes.Jobs;
-    using BenchmarkDotNet.Engines;
 
     using ImageSharp.Formats;
 
+    
+    [Config(typeof(Config.Short))] // It's long enough to iterate through multiple files
     public class EncodeJpegMultiple : MultiImageBenchmarkBase.WithImagesPreloaded
     {
-        protected override IEnumerable<string> InputImageSubfoldersOrFiles => new[]
+        protected override IEnumerable<string> InputImageSubfoldersOrFiles => new[] { "Bmp/", "Jpg/baseline" };
-        {
-            "Bmp/",
-            "Jpg/baseline"
-        };
 
-        protected override IEnumerable<string> FileFilters => new[] { "*.bmp", "*.jpg" };
+        protected override IEnumerable<string> SearchPatterns => new[] { "*.bmp", "*.jpg" };
 
         [Benchmark(Description = "EncodeJpegMultiple - ImageSharp")]
         public void EncodeJpegImageSharp()
         {
             this.ForEachImageSharpImage(
-                img =>
+                (img, ms) =>
                     {
-                        MemoryStream ms = new MemoryStream();
                         img.Save(ms, new JpegEncoder());
-                        return ms;
+                        return null;
                     });
         }
 
@@ -38,13 +35,11 @@
         public void EncodeJpegSystemDrawing()
         {
             this.ForEachSystemDrawingImage(
-                img =>
+                (img, ms) =>
                     {
-                        MemoryStream ms = new MemoryStream();
                         img.Save(ms, ImageFormat.Jpeg);
-                        return ms;
+                        return null;
                     });
         }
-
     }
 }

tests/ImageSharp.Benchmarks/Image/ImageBenchmarkTests.cs

@@ -0,0 +1,81 @@
+// <copyright file="ImageBenchmarkTests.cs" company="James Jackson-South">
+// Copyright (c) James Jackson-South and contributors.
+// Licensed under the Apache License, Version 2.0.
+// </copyright>
+
+// This file contains small, cheap and "unit test" benchmarks to test MultiImageBenchmarkBase.
+// Need this because there are no real test cases for the common benchmark utility stuff.
+
+// Uncomment this to enable benchmark testing
+// #define TEST
+
+#if TEST
+
+// ReSharper disable InconsistentNaming
+namespace ImageSharp.Benchmarks.Image
+{
+    using System;
+    using System.Collections.Generic;
+    using System.Linq;
+    using System.Runtime.CompilerServices;
+
+    using BenchmarkDotNet.Attributes;
+    using BenchmarkDotNet.Configs;
+    using BenchmarkDotNet.Jobs;
+
+    internal class Assert
+    {
+        public static void True(bool condition, string meassage)
+        {
+            if (!condition) throw new Exception(meassage);
+        }
+
+        public static void Equal<T>(T a, T b, string message) => True(a.Equals(b), message);
+
+        public static void Equal(object a, object b, string message) => True(a.Equals(b), message);
+    }
+
+    public class MultiImageBenchmarkBase_ShouldReadSingleFiles : MultiImageBenchmarkBase
+    {
+        protected override IEnumerable<string> InputImageSubfoldersOrFiles => new[] { "Bmp/F.bmp", "Jpg/Exif.jpg" };
+
+        [Benchmark]
+        public void Run()
+        {
+            //Console.WriteLine("FileNames2Bytes.Count(): " + this.FileNames2Bytes.Count());
+            Assert.Equal(this.FileNames2Bytes.Count(), 2, "MultiImageBenchmarkBase_ShouldReadFiles failed");
+        }
+    }
+
+    public class MultiImageBenchmarkBase_ShouldReadMixed : MultiImageBenchmarkBase
+    {
+        protected override IEnumerable<string> InputImageSubfoldersOrFiles => new[] { "Bmp/F.bmp", "Gif/" };
+
+        [Benchmark]
+        public void Run()
+        {
+            //Console.WriteLine("FileNames2Bytes.Count(): " + this.FileNames2Bytes.Count());
+            Assert.True(this.FileNames2Bytes.Count() > 2, "MultiImageBenchmarkBase_ShouldReadMixed failed");
+            Assert.True(
+                this.FileNames2Bytes.Any(kv => kv.Key.Contains("F.bmp")),
+                "MultiImageBenchmarkBase_ShouldReadMixed failed"
+                );
+        }
+    }
+
+    [Config(typeof(Config.Short))]
+    public class UseCustomConfigTest : MultiImageBenchmarkBase
+    {
+        protected override IEnumerable<string> InputImageSubfoldersOrFiles => new[] { "Bmp/" };
+
+        [Benchmark]
+        public void Run()
+        {
+            this.ForEachStream(
+                ms => new ImageSharp.Image(ms)
+                );
+        }
+    }
+}
+
+#endif

tests/ImageSharp.Benchmarks/Image/MultiImageBenchmarkBase.cs

@@ -1,4 +1,10 @@
-namespace ImageSharp.Benchmarks.Image
+// <copyright file="MultiImageBenchmarkBase.cs" company="James Jackson-South">
+// Copyright (c) James Jackson-South and contributors.
+// Licensed under the Apache License, Version 2.0.
+// </copyright>
+
+
+namespace ImageSharp.Benchmarks.Image
 {
     using System;
     using System.Collections.Generic;
@@ -31,11 +37,11 @@
         }
 
         [Params(InputImageCategory.AllImages, InputImageCategory.SmallImagesOnly, InputImageCategory.LargeImagesOnly)]
-        public InputImageCategory InputCategory { get; set; }
+        public virtual InputImageCategory InputCategory { get; set; }
         
         protected virtual string BaseFolder => "../ImageSharp.Tests/TestImages/Formats/";
 
-        protected virtual IEnumerable<string> FileFilters => new[] { "*.*" };
+        protected virtual IEnumerable<string> SearchPatterns => new[] { "*.*" };
         
         /// <summary>
         /// Gets the file names containing these strings are substrings are not processed by the benchmark.
@@ -81,19 +87,25 @@
         public void ReadImages()
         {
             // Console.WriteLine("Vector.IsHardwareAccelerated: " + Vector.IsHardwareAccelerated);
-            this.ReadImagesImpl();
+            this.ReadFilesImpl();
         }
 
-        protected virtual void ReadImagesImpl()
+        protected virtual void ReadFilesImpl()
         {
-            foreach (string folder in this.AllFoldersOrFiles)
+            foreach (string path in this.AllFoldersOrFiles)
             {
+                if (File.Exists(path))
+                {
+                    this.FileNamesToBytes[path] = File.ReadAllBytes(path);
+                    continue;
+                }
 
                 var allFiles =
-                    this.FileFilters.SelectMany(
+                    this.SearchPatterns.SelectMany(
                         f =>
-                            Directory.EnumerateFiles(folder, f, SearchOption.AllDirectories)
+                            Directory.EnumerateFiles(path, f, SearchOption.AllDirectories)
                                 .Where(fn => !this.ExcludeSubstringsInFileNames.Any(w => fn.ToLower().Contains(w)))).ToArray();
+
                 foreach (var fn in allFiles)
                 {
                     this.FileNamesToBytes[fn] = File.ReadAllBytes(fn);
@@ -101,6 +113,10 @@
             }
         }
 
+        /// <summary>
+        /// Execute code for each image stream. If the returned object of the opearation <see cref="Func{T, TResult}"/> is <see cref="IDisposable"/> it will be disposed.
+        /// </summary>
+        /// <param name="operation">The operation to execute. If the returned object is &lt;see cref="IDisposable"/&gt; it will be disposed </param>
         protected void ForEachStream(Func<MemoryStream, object> operation)
         {
             foreach (var kv in this.FileNames2Bytes)
@@ -123,9 +139,9 @@
 
         public abstract class WithImagesPreloaded : MultiImageBenchmarkBase
         {
-            protected override void ReadImagesImpl()
+            protected override void ReadFilesImpl()
             {
-                base.ReadImagesImpl();
+                base.ReadFilesImpl();
 
                 foreach (var kv in this.FileNamesToBytes)
                 {
@@ -174,6 +190,23 @@
                 }
             }
 
+            protected void ForEachImageSharpImage(Func<Image, MemoryStream, object> operation)
+            {
+                using (MemoryStream workStream = new MemoryStream())
+                {
+
+                    this.ForEachImageSharpImage(
+                        img =>
+                        {
+                            // ReSharper disable AccessToDisposedClosure
+                            object result = operation(img, workStream);
+                            workStream.Seek(0, SeekOrigin.Begin);
+                            // ReSharper restore AccessToDisposedClosure
+                            return result;
+                        });
+                }
+            }
+
             protected void ForEachSystemDrawingImage(Func<System.Drawing.Bitmap, object> operation)
             {
                 foreach (var kv in this.FileNames2SystemDrawingImages)
@@ -189,10 +222,24 @@
                     }
                 }
             }
-        }
 
+            protected void ForEachSystemDrawingImage(Func<System.Drawing.Bitmap, MemoryStream, object> operation)
+            {
+                using (MemoryStream workStream = new MemoryStream())
+                {
 
-    }
+                    this.ForEachSystemDrawingImage(
-
+                        img =>
+                        {
+                            // ReSharper disable AccessToDisposedClosure
+                            object result = operation(img, workStream);
+                            workStream.Seek(0, SeekOrigin.Begin);
+                            // ReSharper restore AccessToDisposedClosure
+                            return result;
+                        });
+                }
 
+            }
+        }
+    }
 }