bringing back life into Benchmarks project

src/Shared/AssemblyInfo.Common.cs

@@ -33,7 +33,7 @@ using System.Runtime.CompilerServices;
 
 // Ensure the internals can be built and tested.
 [assembly: InternalsVisibleTo("SixLabors.ImageSharp.Drawing")]
-[assembly: InternalsVisibleTo("SixLabors.ImageSharp.Benchmarks")]
+[assembly: InternalsVisibleTo("ImageSharp.Benchmarks")]
 [assembly: InternalsVisibleTo("SixLabors.ImageSharp.Tests")]
 [assembly: InternalsVisibleTo("SixLabors.ImageSharp.Sandbox46")]
 

tests/ImageSharp.Benchmarks/Config.cs

@@ -22,7 +22,7 @@ namespace SixLabors.ImageSharp.Benchmarks
             public Short()
             {
                 this.Add(
-                    Job.Default.WithLaunchCount(1)
+                    Job.Clr.WithLaunchCount(1)
                         .WithWarmupCount(3)
                         .WithTargetCount(3)
                         );

tests/ImageSharp.Benchmarks/Image/DecodeJpeg.cs

@@ -9,38 +9,45 @@ namespace SixLabors.ImageSharp.Benchmarks.Image
     using System.IO;
 
     using BenchmarkDotNet.Attributes;
+    using BenchmarkDotNet.Attributes.Jobs;
+
+    using SixLabors.ImageSharp.Tests;
 
     using CoreImage = ImageSharp.Image;
 
     using CoreSize = SixLabors.Primitives.Size;
-
+    
-    [Config(typeof(Config))]
+    [Config(typeof(Config.Short))]
     public class DecodeJpeg : BenchmarkBase
     {
         private byte[] jpegBytes;
 
+        private static readonly string TestImage = Path.Combine(
+            TestEnvironment.InputImagesDirectoryFullPath,
+            TestImages.Jpeg.Baseline.Calliphora);
+
         [GlobalSetup]
         public void ReadImages()
         {
             if (this.jpegBytes == null)
             {
-                this.jpegBytes = File.ReadAllBytes("../../../../../../../../ImageSharp.Tests/TestImages/Formats/Jpg/Baseline/Calliphora.jpg");
+                this.jpegBytes = File.ReadAllBytes(TestImage);
+            }
+        }
+
+        [Benchmark(Baseline = true, Description = "Decode Jpeg - System.Drawing")]
+        public Size JpegSystemDrawing()
+        {
+            using (MemoryStream memoryStream = new MemoryStream(this.jpegBytes))
+            {
+                using (Image image = Image.FromStream(memoryStream))
+                {
+                    return image.Size;
+                }
             }
         }
 
-        //[Benchmark(Baseline = true, Description = "System.Drawing Jpeg")]
+        [Benchmark(Description = "Decode Jpeg - ImageSharp")]
-        //public Size JpegSystemDrawing()
-        //{
-        //    using (MemoryStream memoryStream = new MemoryStream(this.jpegBytes))
-        //    {
-        //        using (Image image = Image.FromStream(memoryStream))
-        //        {
-        //            return image.Size;
-        //        }
-        //    }
-        //}
-
-        [Benchmark(Description = "ImageSharp Jpeg")]
         public CoreSize JpegCore()
         {
             using (MemoryStream memoryStream = new MemoryStream(this.jpegBytes))

tests/ImageSharp.Benchmarks/Image/DecodeJpegMultiple.cs

@@ -21,28 +21,20 @@ namespace SixLabors.ImageSharp.Benchmarks.Image
     {
         protected override IEnumerable<string> InputImageSubfoldersOrFiles => new[]
         {
-            "Jpg/"
+            "Jpg/baseline",
+            "Jpg/progressive",
         };
 
         protected override IEnumerable<string> SearchPatterns => new[] { "*.jpg" };
 
-        [Benchmark(Description = "DecodeJpegMultiple - ImageSharp NEW")]
+        [Benchmark(Description = "DecodeJpegMultiple - ImageSharp")]
         public void DecodeJpegImageSharpNwq()
         {
             this.ForEachStream(
                 ms => CoreImage.Load<Rgba32>(ms)
                 );
         }
-
+        
-
-        [Benchmark(Description = "DecodeJpegMultiple - ImageSharp Original")]
-        public void DecodeJpegImageSharpOriginal()
-        {
-            this.ForEachStream(
-                ms => CoreImage.Load<Rgba32>(ms, new OriginalJpegDecoder())
-            );
-        }
-
         [Benchmark(Baseline = true, Description = "DecodeJpegMultiple - System.Drawing")]
         public void DecodeJpegSystemDrawing()
         {
@@ -50,26 +42,5 @@ namespace SixLabors.ImageSharp.Benchmarks.Image
                 System.Drawing.Image.FromStream
                 );
         }
-
-
-        public sealed class OriginalJpegDecoder : IImageDecoder, IJpegDecoderOptions
-        {
-            /// <summary>
-            /// Gets or sets a value indicating whether the metadata should be ignored when the image is being decoded.
-            /// </summary>
-            public bool IgnoreMetadata { get; set; }
-
-            /// <inheritdoc/>
-            public Image<TPixel> Decode<TPixel>(Configuration configuration, Stream stream)
-                where TPixel : struct, IPixel<TPixel>
-            {
-                Guard.NotNull(stream, "stream");
-
-                using (var decoder = new OrigJpegDecoderCore(configuration, this))
-                {
-                    return decoder.Decode<TPixel>(stream);
-                }
-            }
-        }
     }
 }

tests/ImageSharp.Benchmarks/Image/MultiImageBenchmarkBase.cs

@@ -15,6 +15,8 @@ namespace SixLabors.ImageSharp.Benchmarks.Image
 
     using BenchmarkDotNet.Attributes;
 
+    using SixLabors.ImageSharp.Tests;
+
     using CoreImage = ImageSharp.Image;
 
     public abstract class MultiImageBenchmarkBase : BenchmarkBase
@@ -39,14 +41,14 @@ namespace SixLabors.ImageSharp.Benchmarks.Image
         [Params(InputImageCategory.AllImages, InputImageCategory.SmallImagesOnly, InputImageCategory.LargeImagesOnly)]
         public virtual InputImageCategory InputCategory { get; set; }
 
-        protected virtual string BaseFolder => "../../../../../../../../ImageSharp.Tests/TestImages/Formats/";
+        protected virtual string BaseFolder => TestEnvironment.InputImagesDirectoryFullPath;
 
         protected virtual IEnumerable<string> SearchPatterns => new[] { "*.*" };
 
         /// <summary>
         /// Gets the file names containing these strings are substrings are not processed by the benchmark.
         /// </summary>
-        protected IEnumerable<string> ExcludeSubstringsInFileNames => new[] { "badeof", "BadEof" };
+        protected IEnumerable<string> ExcludeSubstringsInFileNames => new[] { "badeof", "BadEof", "CriticalEOF" };
 
         /// <summary>
         /// Enumerates folders containing files OR files to be processed by the benchmark.

tests/ImageSharp.Benchmarks/ImageSharp.Benchmarks.csproj

@@ -4,12 +4,16 @@
     <OutputType>Exe</OutputType>
     <AllowUnsafeBlocks>True</AllowUnsafeBlocks>
     <RootNamespace>SixLabors.ImageSharp.Benchmarks</RootNamespace>
-    <AssemblyName>SixLabors.ImageSharp.Benchmarks</AssemblyName>
+    <AssemblyName>ImageSharp.Benchmarks</AssemblyName>
   </PropertyGroup>
   <PropertyGroup Condition="'$(TargetFramework)'=='net461'">
     <RuntimeIdentifier>win7-x64</RuntimeIdentifier>
     <Prefer32Bit>false</Prefer32Bit>
   </PropertyGroup>
+  <ItemGroup>
+    <Compile Include="..\ImageSharp.Tests\TestImages.cs" Link="Tests\TestImages.cs" />
+    <Compile Include="..\ImageSharp.Tests\TestUtilities\TestEnvironment.cs" Link="Tests\TestEnvironment.cs" />
+  </ItemGroup>
   <ItemGroup>
     <PackageReference Include="Colourful" Version="1.1.2" />
     <PackageReference Include="System.Numerics.Vectors" Version="4.4.0" />

tests/ImageSharp.Tests/Formats/Jpg/Block8x8FTests.CopyToBufferArea.cs

@@ -41,8 +41,9 @@ namespace SixLabors.ImageSharp.Tests.Formats.Jpg
                 }
             }
 
-            // TODO: This test occasionally fails. Don't get the reason, BufferArea.CopyTo() is totally OK.
+            // TODO: This test occasionally fails from the same reason certain ICC tests are failing. Should be false negative.
-            [Fact(Skip = "This test occasionally fails. Don't get the reason, BufferArea.CopyTo() is totally OK.")]
+            //[Fact(Skip = "This test occasionally fails from the same reason certain ICC tests are failing. Should be false negative.")]
+            [Fact]
             public void Unscaled()
             {
                 Block8x8F block = CreateRandomFloatBlock(0, 100);
@@ -62,8 +63,9 @@ namespace SixLabors.ImageSharp.Tests.Formats.Jpg
                 }
             }
 
-            // TODO: This test occasionally fails. Don't get the reason, BufferArea.CopyTo() is totally OK.
+            // TODO: This test occasionally fails from the same reason certain ICC tests are failing. Should be false negative.
-            [Theory(Skip = "This test occasionally fails. Don't get the reason, BufferArea.CopyTo() is totally OK.")]
+            //[Theory(Skip = "This test occasionally fails from the same reason certain ICC tests are failing. Should be false negative.")]
+            [Theory]
             [InlineData(1, 1)]
             [InlineData(1, 2)]
             [InlineData(2, 1)]

tests/ImageSharp.Tests/Formats/Jpg/JpegProfilingBenchmarks.cs

@@ -30,8 +30,8 @@ namespace SixLabors.ImageSharp.Tests.Formats.Jpg
             TestImages.Jpeg.Baseline.Jpeg444,
         };
 
-        // [Theory] // Benchmark, enable manually
+        //[Theory] // Benchmark, enable manually
-        // [MemberData(nameof(DecodeJpegData))]
+        //[MemberData(nameof(DecodeJpegData))]
         public void DecodeJpeg(string fileName)
         {
             const int ExecutionCount = 30;

tests/ImageSharp.Tests/TestUtilities/TestEnvironment.Formats.cs

@@ -0,0 +1,84 @@
+// Copyright (c) Six Labors and contributors.
+// Licensed under the Apache License, Version 2.0.
+
+using System;
+using System.IO;
+using SixLabors.ImageSharp.Formats;
+using SixLabors.ImageSharp.Formats.Bmp;
+using SixLabors.ImageSharp.Formats.Gif;
+using SixLabors.ImageSharp.Formats.Jpeg;
+using SixLabors.ImageSharp.Formats.Png;
+using SixLabors.ImageSharp.Tests.TestUtilities.ReferenceCodecs;
+
+namespace SixLabors.ImageSharp.Tests
+{
+    public static partial class TestEnvironment
+    {
+        private static Lazy<Configuration> configuration = new Lazy<Configuration>(CreateDefaultConfiguration);
+
+        internal static Configuration Configuration => configuration.Value;
+        
+        internal static IImageDecoder GetReferenceDecoder(string filePath)
+        {
+            IImageFormat format = GetImageFormat(filePath);
+            return Configuration.FindDecoder(format);
+        }
+
+        internal static IImageEncoder GetReferenceEncoder(string filePath)
+        {
+            IImageFormat format = GetImageFormat(filePath);
+            return Configuration.FindEncoder(format);
+        }
+
+        internal static IImageFormat GetImageFormat(string filePath)
+        {
+            string extension = Path.GetExtension(filePath).ToLower();
+            if (extension[0] == '.') extension = extension.Substring(1);
+            IImageFormat format = Configuration.FindFormatByFileExtension(extension);
+            return format;
+        }
+
+        private static void ConfigureCodecs(
+            this Configuration cfg,
+            IImageFormat imageFormat,
+            IImageDecoder decoder,
+            IImageEncoder encoder,
+            IImageFormatDetector detector)
+        {
+            cfg.SetDecoder(imageFormat, decoder);
+            cfg.SetEncoder(imageFormat, encoder);
+            cfg.AddImageFormatDetector(detector);
+        }
+
+        private static Configuration CreateDefaultConfiguration()
+        {
+            var configuration = new Configuration(
+                new PngConfigurationModule(),
+                new JpegConfigurationModule(),
+                new GifConfigurationModule()
+            );
+
+            if (!IsLinux)
+            {
+                configuration.ConfigureCodecs(
+                    ImageFormats.Png,
+                    SystemDrawingReferenceDecoder.Instance,
+                    SystemDrawingReferenceEncoder.Png,
+                    new PngImageFormatDetector());
+
+                configuration.ConfigureCodecs(
+                    ImageFormats.Bmp,
+                    SystemDrawingReferenceDecoder.Instance,
+                    SystemDrawingReferenceEncoder.Png,
+                    new PngImageFormatDetector());
+            }
+            else
+            {
+                configuration.Configure(new PngConfigurationModule());
+                configuration.Configure(new BmpConfigurationModule());
+            }
+
+            return configuration;
+        }
+    }
+}

tests/ImageSharp.Tests/TestUtilities/TestEnvironment.cs

@@ -5,19 +5,11 @@ using System;
 using System.IO;
 using System.Linq;
 using System.Reflection;
-
+using System.Runtime.InteropServices;
-using SixLabors.ImageSharp.Formats;
-using SixLabors.ImageSharp.Tests.TestUtilities.ReferenceCodecs;
 
 namespace SixLabors.ImageSharp.Tests
 {
-    using System.Runtime.InteropServices;
+    public static partial class TestEnvironment
-    using SixLabors.ImageSharp.Formats.Bmp;
-    using SixLabors.ImageSharp.Formats.Gif;
-    using SixLabors.ImageSharp.Formats.Jpeg;
-    using SixLabors.ImageSharp.Formats.Png;
-
-    public static class TestEnvironment
     {
         private const string ImageSharpSolutionFileName = "ImageSharp.sln";
 
@@ -38,8 +30,6 @@ namespace SixLabors.ImageSharp.Tests
                     return Boolean.TryParse(Environment.GetEnvironmentVariable("CI"), out isCi) && isCi;
                 });
 
-        private static Lazy<Configuration> configuration = new Lazy<Configuration>(CreateDefaultConfiguration);
-        
         // ReSharper disable once InconsistentNaming
         /// <summary>
         /// Gets a value indicating whether test execution runs on CI.
@@ -48,54 +38,9 @@ namespace SixLabors.ImageSharp.Tests
 
         internal static string SolutionDirectoryFullPath => solutionDirectoryFullPath.Value;
 
-        internal static Configuration Configuration => configuration.Value;
-
-        private static void ConfigureCodecs(
-            this Configuration cfg,
-            IImageFormat imageFormat,
-            IImageDecoder decoder,
-            IImageEncoder encoder,
-            IImageFormatDetector detector)
-        {
-            cfg.SetDecoder(imageFormat, decoder);
-            cfg.SetEncoder(imageFormat, encoder);
-            cfg.AddImageFormatDetector(detector);
-        }
-
-        private static Configuration CreateDefaultConfiguration()
-        {
-            var configuration = new Configuration(
-                new PngConfigurationModule(),
-                new JpegConfigurationModule(),
-                new GifConfigurationModule()
-                );
-
-            if (!IsLinux)
-            {
-                configuration.ConfigureCodecs(
-                    ImageFormats.Png,
-                    SystemDrawingReferenceDecoder.Instance,
-                    SystemDrawingReferenceEncoder.Png,
-                    new PngImageFormatDetector());
-
-                configuration.ConfigureCodecs(
-                    ImageFormats.Bmp,
-                    SystemDrawingReferenceDecoder.Instance,
-                    SystemDrawingReferenceEncoder.Png,
-                    new PngImageFormatDetector());
-            }
-            else
-            {
-                configuration.Configure(new PngConfigurationModule());
-                configuration.Configure(new BmpConfigurationModule());
-            }
-            
-            return configuration;
-        }
-
         private static string GetSolutionDirectoryFullPathImpl()
         {
-            string assemblyLocation = typeof(TestFile).GetTypeInfo().Assembly.Location;
+            string assemblyLocation = typeof(TestEnvironment).GetTypeInfo().Assembly.Location;
 
             var assemblyFile = new FileInfo(assemblyLocation);
 
@@ -146,26 +91,6 @@ namespace SixLabors.ImageSharp.Tests
         internal static string GetReferenceOutputFileName(string actualOutputFileName) =>
             actualOutputFileName.Replace("ActualOutput", @"External\ReferenceOutput").Replace('\\', Path.DirectorySeparatorChar);
 
-        internal static IImageDecoder GetReferenceDecoder(string filePath)
-        {
-            IImageFormat format = GetImageFormat(filePath);
-            return Configuration.FindDecoder(format);
-        }
-
-        internal static IImageEncoder GetReferenceEncoder(string filePath)
-        {
-            IImageFormat format = GetImageFormat(filePath);
-            return Configuration.FindEncoder(format);
-        }
-
-        internal static IImageFormat GetImageFormat(string filePath)
-        {
-            string extension = Path.GetExtension(filePath).ToLower();
-            if (extension[0] == '.') extension = extension.Substring(1);
-            IImageFormat format = Configuration.FindFormatByFileExtension(extension);
-            return format;
-        }
-
         internal static bool IsLinux => RuntimeInformation.IsOSPlatform(OSPlatform.Linux);
 
         internal static bool IsWindows => RuntimeInformation.IsOSPlatform(OSPlatform.Windows);