IDCT resizing modes

src/ImageSharp/Formats/Jpeg/Components/Decoder/SpectralConverter{TPixel}.cs

@@ -28,19 +28,19 @@ namespace SixLabors.ImageSharp.Formats.Jpeg.Components.Decoder
         /// <summary>
         /// Supported scaling factors for DCT jpeg scaling.
         /// </summary>
-        private static readonly int[] ScalingFactors = new int[]
+        private static readonly int[] ScaledBlockSizes = new int[]
         {
-            // 8 => 8, no scaling
+            // 8 => 1, 1/8 of the original size
-            8,
+            1,
-
-            // 8 => 4, 1/2 of the original size
-            4,
 
             // 8 => 2, 1/4 of the original size
             2,
 
-            // 8 => 1, 1/8 of the original size
+            // 8 => 4, 1/2 of the original size
-            1,
+            4,
+
+            // 8 => 8, no scaling
+            8,
         };
 
         /// <summary>
@@ -132,45 +132,73 @@ namespace SixLabors.ImageSharp.Formats.Jpeg.Components.Decoder
         }
 
         /// <summary>
-        /// Calculates resulting image size and jpeg block scaling.
+        /// Calculates resulting image size.
         /// </summary>
-        /// <param name="nativeSize">Native size of the image.</param>
+        /// <remarks>
-        /// <param name="blockPixelSize">Resulting jpeg block pixel size.</param>
+        /// If <paramref name="targetSize"/> is null, unchanged <paramref name="size"/> is returned.
+        /// </remarks>
+        /// <param name="size">Size of the image.</param>
+        /// <param name="targetSize">Target image size, can be null.</param>
+        /// <param name="outputBlockSize">Scaled spectral block size if IDCT scaling should be applied</param>
         /// <returns>Scaled jpeg image size.</returns>
-        private Size GetResultingImageSize(Size nativeSize, out int blockPixelSize)
+        // TODO: describe ALL outcomes of the built-in IDCT downscaling
+        public static Size GetResultingImageSize(Size size, Size? targetSize, out int outputBlockSize)
         {
-            if (this.targetSize == null)
+            const int jpegBlockPixelSize = 8;
-            {
+
-                blockPixelSize = 8;
+            // must be at least 5% smaller than current IDCT scaled size
-                return nativeSize;
+            // to perform second pass resizing
-            }
+            // this is a highly experimental value
-            else
+            const float secondPassThresholdRatio = 0.95f;
+
+            outputBlockSize = jpegBlockPixelSize;
+            if (targetSize != null)
             {
-                const uint jpegBlockPixelSize = 8;
+                Size tSize = targetSize.Value;
 
-                Size targetSize = this.targetSize.Value;
+                int widthInBlocks = (int)Numerics.DivideCeil((uint)size.Width, jpegBlockPixelSize);
-                int outputWidth = nativeSize.Width;
+                int heightInBlocks = (int)Numerics.DivideCeil((uint)size.Height, jpegBlockPixelSize);
-                int outputHeight = nativeSize.Height;
-                blockPixelSize = 1;
 
-                for (int i = 1; i < ScalingFactors.Length; i++)
+                for (int i = 0; i < ScaledBlockSizes.Length; i++)
                 {
-                    int scale = ScalingFactors[i];
+                    int blockSize = ScaledBlockSizes[i];
-                    int scaledw = (int)Numerics.DivideCeil((uint)(nativeSize.Width * scale), jpegBlockPixelSize);
+                    int scaledWidth = widthInBlocks * blockSize;
-                    int scaledh = (int)Numerics.DivideCeil((uint)(nativeSize.Height * scale), jpegBlockPixelSize);
+                    int scaledHeight = heightInBlocks * blockSize;
 
-                    if (scaledw < targetSize.Width || scaledh < targetSize.Height)
+                    // skip to next IDCT scaling
+                    if (scaledWidth < tSize.Width || scaledHeight < tSize.Height)
                     {
-                        blockPixelSize = ScalingFactors[i - 1];
+                        // this if segment can be safely removed
-                        break;
+                        continue;
                     }
 
-                    outputWidth = scaledw;
+                    // exact match
-                    outputHeight = scaledh;
+                    if (scaledWidth == tSize.Width && scaledHeight == tSize.Height)
-                }
+                    {
+                        outputBlockSize = blockSize;
+                        return new Size(scaledWidth, scaledHeight);
+                    }
 
-                return new Size(outputWidth, outputHeight);
+                    // center cropping
+                    int widthDiff = Math.Abs(tSize.Width - scaledWidth);
+                    int heightDiff = Math.Abs(tSize.Height - scaledHeight);
+                    if (widthDiff < blockSize && heightDiff < blockSize)
+                    {
+                        throw new NotSupportedException($"Central cropping is not supported yet");
+                    }
+
+                    // small enough for second pass
+                    float secondPassWidthRatio = (float)tSize.Width / scaledWidth;
+                    float secondPassHeightRatio = (float)tSize.Height / scaledHeight;
+                    if (secondPassWidthRatio <= secondPassThresholdRatio && secondPassHeightRatio <= secondPassThresholdRatio)
+                    {
+                        outputBlockSize = blockSize;
+                        return new Size(scaledWidth, scaledHeight);
+                    }
+                }
             }
+
+            return size;
         }
 
         /// <summary>
@@ -233,7 +261,7 @@ namespace SixLabors.ImageSharp.Formats.Jpeg.Components.Decoder
             this.colorConverter = converter;
 
             // Resulting image size
-            Size pixelSize = this.GetResultingImageSize(frame.PixelSize, out int blockPixelSize);
+            Size pixelSize = GetResultingImageSize(frame.PixelSize, this.targetSize, out int blockPixelSize);
 
             // iteration data
             int majorBlockWidth = frame.Components.Max((component) => component.SizeInBlocks.Width);

tests/ImageSharp.Tests.ProfilingSandbox/Program.cs

@@ -6,9 +6,12 @@ using System.Diagnostics;
 using System.IO;
 using System.Reflection;
 using System.Threading;
+using PhotoSauce.MagicScaler;
+using PhotoSauce.MagicScaler.Interpolators;
 using SixLabors.ImageSharp.Formats.Jpeg;
 using SixLabors.ImageSharp.PixelFormats;
 using SixLabors.ImageSharp.Processing;
+using SixLabors.ImageSharp.Processing.Processors.Transforms;
 using SixLabors.ImageSharp.Tests.PixelFormats.PixelOperations;
 using SixLabors.ImageSharp.Tests.ProfilingBenchmarks;
 using Xunit.Abstractions;
@@ -28,29 +31,60 @@ namespace SixLabors.ImageSharp.Tests.ProfilingSandbox
             public void WriteLine(string format, params object[] args) => Console.WriteLine(format, args);
         }
 
-        public static void Main(string[] args)
+        const string pathTemplate = ..."";
-        {
-            //ReEncodeImage("Calliphora");
 
-            // DecodeImageResize__explicit("Calliphora", new Size(101, 150));
+        // Second pass - must be 5% smaller than appropriate IDCT scaled size
-            // DecodeImageResize__experimental("Calliphora_aligned_size", new Size(101, 150));
+        const float scale = 0.75f;
-            //DecodeImageResize__experimental("winter420_noninterleaved", new Size(80, 120));
+        readonly IResampler resampler = KnownResamplers.Box;
 
-            // Decode-Resize-Encode w/ Mutate()
+        public static void Main(string[] args)
-            // Elapsed: 2504ms across 250 iterations
+        {
-            // Average: 10,016ms
+            //ReEncodeImage("jpeg444");
-            BenchmarkResizingLoop__explicit("Calliphora", new Size(80, 120), 250);
+
-
+            //Size targetSize = new Size(808, 1200);
-            // Decode-Resize-Encode w/ downscaling decoder
+
-            // Elapsed: 1157ms across 250 iterations
+            // 808 x 1200
-            // Average: 4,628ms
+            // 404 x 600
-            BenchmarkResizingLoop__experimental("Calliphora", new Size(80, 120), 250);
+            // 202 x 300
+            // 101 x 150
+            string imageName = "Calliphora_aligned_size";
+
+            // Exact matches for 8/4/2/1 scaling
+            //Size exactSizeX8 = new Size(808, 1200);
+            //Size exactSizeX4 = new Size(404, 600);
+            //Size exactSizeX2 = new Size(202, 300);
+            //Size exactSizeX1 = new Size(101, 150);
+            //ReencodeImageResize__experimental(imageName, exactSizeX8);
+            //ReencodeImageResize__experimental(imageName, exactSizeX4);
+            //ReencodeImageResize__experimental(imageName, exactSizeX2);
+            //ReencodeImageResize__experimental(imageName, exactSizeX1);
+
+            Size secondPassSizeX8 = new Size((int)(808 * scale), (int)(1200 * scale));
+            Size secondPassSizeX4 = new Size((int)(404 * scale), (int)(600 * scale));
+            Size secondPassSizeX2 = new Size((int)(202 * scale), (int)(300 * scale));
+            Size secondPassSizeX1 = new Size((int)(101 * scale), (int)(150 * scale));
+            ReencodeImageResize__experimental(imageName, secondPassSizeX8);
+            ReencodeImageResize__experimental(imageName, secondPassSizeX4);
+            ReencodeImageResize__experimental(imageName, secondPassSizeX2);
+            ReencodeImageResize__experimental(imageName, secondPassSizeX1);
+            ReencodeImageResize__explicit(imageName, secondPassSizeX8, resampler);
+            ReencodeImageResize__explicit(imageName, secondPassSizeX4, resampler);
+            ReencodeImageResize__explicit(imageName, secondPassSizeX2, resampler);
+            ReencodeImageResize__explicit(imageName, secondPassSizeX1, resampler);
+
+            // 'native' resizing - only jpeg dct downscaling
+            //ReencodeImageResize_Comparison("Calliphora_ratio1", targetSize, 100);
+
+            // 'native' + software resizing - jpeg dct downscaling + postprocessing
+            //ReencodeImageResize_Comparison("Calliphora_aligned_size", new Size(269, 400), 99);
+
+            //var benchmarkSize = new Size(404, 600);
+            //BenchmarkResizingLoop__explicit("jpeg_quality_100", benchmarkSize, 300);
+            //BenchmarkResizingLoop__experimental("jpeg_quality_100", benchmarkSize, 300);
 
             Console.WriteLine("Done.");
         }
 
-        const string pathTemplate = "C:\\Users\\pl4nu\\Downloads\\{0}.jpg";
-
         private static void BenchmarkEncoder(string fileName, int iterations, int quality, JpegColorType color)
         {
             string loadPath = String.Format(pathTemplate, fileName);
@@ -174,10 +208,10 @@ namespace SixLabors.ImageSharp.Tests.ProfilingSandbox
             img.SaveAsJpeg(savePath, encoder);
         }
 
-        private static void DecodeImageResize__explicit(string fileName, Size targetSize, int? quality = null)
+        private static void ReencodeImageResize__explicit(string fileName, Size targetSize, IResampler sampler, int? quality = null)
         {
             string loadPath = String.Format(pathTemplate, fileName);
-            string savePath = String.Format(pathTemplate, $"q{quality}_test_{fileName}");
+            string savePath = String.Format(pathTemplate, $"is_res_{sampler.GetType().Name}[{targetSize.Width}x{targetSize.Height}]_{fileName}");
 
             var decoder = new JpegDecoder();
             var encoder = new JpegEncoder()
@@ -187,19 +221,18 @@ namespace SixLabors.ImageSharp.Tests.ProfilingSandbox
             };
 
             using Image img = decoder.Decode<Rgb24>(Configuration.Default, File.OpenRead(loadPath), CancellationToken.None);
-            img.Mutate(ctx => ctx.Resize(targetSize, KnownResamplers.Box, false));
+            img.Mutate(ctx => ctx.Resize(targetSize, sampler, compand: false));
             img.SaveAsJpeg(savePath, encoder);
-
         }
 
-        private static void DecodeImageResize__experimental(string fileName, Size targetSize, int? quality = null)
+        private static void ReencodeImageResize__experimental(string fileName, Size targetSize, int? quality = null)
         {
             string loadPath = String.Format(pathTemplate, fileName);
+            string savePath = String.Format(pathTemplate, $"is_res_jpeg[{targetSize.Width}x{targetSize.Height}]_{fileName}");
 
-            var decoder = new JpegDecoder();
+            var decoder = new JpegDecoder { IgnoreMetadata = true };
             using Image img = decoder.Experimental__DecodeInto<Rgb24>(Configuration.Default, File.OpenRead(loadPath), targetSize, CancellationToken.None);
 
-            string savePath = String.Format(pathTemplate, $"q{quality}_test_{fileName}");
             var encoder = new JpegEncoder()
             {
                 Quality = quality,
@@ -208,6 +241,45 @@ namespace SixLabors.ImageSharp.Tests.ProfilingSandbox
             img.SaveAsJpeg(savePath, encoder);
         }
 
+        private static void ReencodeImageResize__Netvips(string fileName, Size targetSize, int? quality)
+        {
+            string loadPath = String.Format(pathTemplate, fileName);
+            string savePath = String.Format(pathTemplate, $"netvips_resize_{fileName}");
+
+            using var thumb = NetVips.Image.Thumbnail(loadPath, targetSize.Width, targetSize.Height);
+
+            // Save the results
+            thumb.Jpegsave(savePath, q: quality, strip: true, subsampleMode: NetVips.Enums.ForeignSubsample.Off);
+        }
+
+        private static void ReencodeImageResize__MagicScaler(string fileName, Size targetSize, int quality)
+        {
+            string loadPath = String.Format(pathTemplate, fileName);
+            string savePath = String.Format(pathTemplate, $"magicscaler_resize_{fileName}");
+
+            var settings = new ProcessImageSettings()
+            {
+                Width = targetSize.Width,
+                Height = targetSize.Height,
+                SaveFormat = FileFormat.Jpeg,
+                JpegQuality = quality,
+                JpegSubsampleMode = ChromaSubsampleMode.Subsample444,
+                Sharpen = false,
+                ColorProfileMode = ColorProfileMode.Ignore,
+                HybridMode = HybridScaleMode.Turbo,
+            };
+
+            using var output = new FileStream(savePath, FileMode.Create);
+            MagicImageProcessor.ProcessImage(loadPath, output, settings);
+        }
+
+        private static void ReencodeImageResize_Comparison(string fileName, Size targetSize, int quality)
+        {
+            ReencodeImageResize__experimental(fileName, targetSize, quality);
+            ReencodeImageResize__Netvips(fileName, targetSize, quality);
+            ReencodeImageResize__MagicScaler(fileName, targetSize, quality);
+        }
+
         private static Version GetNetCoreVersion()
         {
             Assembly assembly = typeof(System.Runtime.GCSettings).GetTypeInfo().Assembly;