tsai
/
ImageSharp
mirror of https://github.com/SixLabors/ImageSharp
1
0
Fork 0
Code Issues Projects Releases Wiki Activity
Browse Source

stress testing improvements

pull/1730/head
Anton Firszov 5 years ago
parent
77e7700857
commit
b43e963b60
3 changed files with 61 additions and 47 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Unified View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 31
      tests/ImageSharp.Benchmarks/LoadResizeSave/LoadResizeSaveStressRunner.cs
  2. 70
      tests/ImageSharp.Tests.ProfilingSandbox/LoadResizeSaveParallelMemoryStress.cs
  3. 7
      tests/ImageSharp.Tests/RunTestsInLoop.ps1

31
tests/ImageSharp.Benchmarks/LoadResizeSave/LoadResizeSaveStressRunner.cs
View File

@ -44,6 +44,8 @@ namespace SixLabors.ImageSharp.Benchmarks.LoadResizeSave
public double TotalProcessedMegapixels { get; private set; } public double TotalProcessedMegapixels { get; private set; }
public Size LastProcessedImageSize { get; private set; }
private string outputDirectory; private string outputDirectory;
public int ImageCount { get; set; } = int.MaxValue; public int ImageCount { get; set; } = int.MaxValue;
@ -54,9 +56,6 @@ namespace SixLabors.ImageSharp.Benchmarks.LoadResizeSave
public int ThumbnailSize { get; set; } = 150; public int ThumbnailSize { get; set; } = 150;
// Inject leaking memory allocation requests to ImageSharp processing code to stress-test finalizer behavior.
public bool EmulateLeakedAllocations { get; set; }
private static readonly string[] ProgressiveFiles = private static readonly string[] ProgressiveFiles =
{ {
"ancyloscelis-apiformis-m-paraguay-face_2014-08-08-095255-zs-pmax_15046500892_o.jpg", "ancyloscelis-apiformis-m-paraguay-face_2014-08-08-095255-zs-pmax_15046500892_o.jpg",
@ -125,8 +124,9 @@ namespace SixLabors.ImageSharp.Benchmarks.LoadResizeSave
new ParallelOptions { MaxDegreeOfParallelism = this.MaxDegreeOfParallelism }, new ParallelOptions { MaxDegreeOfParallelism = this.MaxDegreeOfParallelism },
action); action);
private void IncreaseTotalMegapixels(int width, int height) private void LogImageProcessed(int width, int height)
{ {
this.LastProcessedImageSize = new Size(width, height);
double pixels = width * (double)height; double pixels = width * (double)height;
this.TotalProcessedMegapixels += pixels / 1_000_000.0; this.TotalProcessedMegapixels += pixels / 1_000_000.0;
} }
@ -156,7 +156,7 @@ namespace SixLabors.ImageSharp.Benchmarks.LoadResizeSave
public void SystemDrawingResize(string input) public void SystemDrawingResize(string input)
{ {
using var image = SystemDrawingImage.FromFile(input, true); using var image = SystemDrawingImage.FromFile(input, true);
this.IncreaseTotalMegapixels(image.Width, image.Height); this.LogImageProcessed(image.Width, image.Height);
(int Width, int Height) scaled = this.ScaledSize(image.Width, image.Height, this.ThumbnailSize); (int Width, int Height) scaled = this.ScaledSize(image.Width, image.Height, this.ThumbnailSize);
var resized = new Bitmap(scaled.Width, scaled.Height); var resized = new Bitmap(scaled.Width, scaled.Height);
@ -182,13 +182,7 @@ namespace SixLabors.ImageSharp.Benchmarks.LoadResizeSave
// Resize it to fit a 150x150 square // Resize it to fit a 150x150 square
using var image = ImageSharpImage.Load(input); using var image = ImageSharpImage.Load(input);
this.IncreaseTotalMegapixels(image.Width, image.Height); this.LogImageProcessed(image.Width, image.Height);
if (this.EmulateLeakedAllocations)
{
_ = Configuration.Default.MemoryAllocator.Allocate<long>(image.Width * image.Height);
_ = Configuration.Default.MemoryAllocator.Allocate<byte>(1 << 21);
}
image.Mutate(i => i.Resize(new ResizeOptions image.Mutate(i => i.Resize(new ResizeOptions
{ {
@ -201,17 +195,12 @@ namespace SixLabors.ImageSharp.Benchmarks.LoadResizeSave
// Save the results // Save the results
image.Save(output, this.imageSharpJpegEncoder); image.Save(output, this.imageSharpJpegEncoder);
if (this.EmulateLeakedAllocations)
{
_ = Configuration.Default.MemoryAllocator.Allocate2D<long>(image.Width, image.Height);
}
} }
public void MagickResize(string input) public void MagickResize(string input)
{ {
using var image = new MagickImage(input); using var image = new MagickImage(input);
this.IncreaseTotalMegapixels(image.Width, image.Height); this.LogImageProcessed(image.Width, image.Height);
// Resize it to fit a 150x150 square // Resize it to fit a 150x150 square
image.Resize(this.ThumbnailSize, this.ThumbnailSize); image.Resize(this.ThumbnailSize, this.ThumbnailSize);
@ -246,7 +235,7 @@ namespace SixLabors.ImageSharp.Benchmarks.LoadResizeSave
public void SkiaCanvasResize(string input) public void SkiaCanvasResize(string input)
{ {
using var original = SKBitmap.Decode(input); using var original = SKBitmap.Decode(input);
this.IncreaseTotalMegapixels(original.Width, original.Height); this.LogImageProcessed(original.Width, original.Height);
(int Width, int Height) scaled = this.ScaledSize(original.Width, original.Height, this.ThumbnailSize); (int Width, int Height) scaled = this.ScaledSize(original.Width, original.Height, this.ThumbnailSize);
using var surface = SKSurface.Create(new SKImageInfo(scaled.Width, scaled.Height, original.ColorType, original.AlphaType)); using var surface = SKSurface.Create(new SKImageInfo(scaled.Width, scaled.Height, original.ColorType, original.AlphaType));
using var paint = new SKPaint() { FilterQuality = SKFilterQuality.High }; using var paint = new SKPaint() { FilterQuality = SKFilterQuality.High };
@ -264,7 +253,7 @@ namespace SixLabors.ImageSharp.Benchmarks.LoadResizeSave
public void SkiaBitmapResize(string input) public void SkiaBitmapResize(string input)
{ {
using var original = SKBitmap.Decode(input); using var original = SKBitmap.Decode(input);
this.IncreaseTotalMegapixels(original.Width, original.Height); this.LogImageProcessed(original.Width, original.Height);
(int Width, int Height) scaled = this.ScaledSize(original.Width, original.Height, this.ThumbnailSize); (int Width, int Height) scaled = this.ScaledSize(original.Width, original.Height, this.ThumbnailSize);
using var resized = original.Resize(new SKImageInfo(scaled.Width, scaled.Height), SKFilterQuality.High); using var resized = original.Resize(new SKImageInfo(scaled.Width, scaled.Height), SKFilterQuality.High);
if (resized == null) if (resized == null)
@ -283,7 +272,7 @@ namespace SixLabors.ImageSharp.Benchmarks.LoadResizeSave
using var codec = SKCodec.Create(input); using var codec = SKCodec.Create(input);
SKImageInfo info = codec.Info; SKImageInfo info = codec.Info;
this.IncreaseTotalMegapixels(info.Width, info.Height); this.LogImageProcessed(info.Width, info.Height);
(int Width, int Height) scaled = this.ScaledSize(info.Width, info.Height, this.ThumbnailSize); (int Width, int Height) scaled = this.ScaledSize(info.Width, info.Height, this.ThumbnailSize);
SKSizeI supportedScale = codec.GetScaledDimensions((float)scaled.Width / info.Width); SKSizeI supportedScale = codec.GetScaledDimensions((float)scaled.Width / info.Width);

70
tests/ImageSharp.Tests.ProfilingSandbox/LoadResizeSaveParallelMemoryStress.cs
View File

@ -5,6 +5,7 @@ using System;
using System.Diagnostics; using System.Diagnostics;
using System.Globalization; using System.Globalization;
using System.IO; using System.IO;
using System.Runtime.CompilerServices;
using System.Text; using System.Text;
using System.Threading; using System.Threading;
using CommandLine; using CommandLine;
@ -27,13 +28,19 @@ namespace SixLabors.ImageSharp.Tests.ProfilingSandbox
this.Benchmarks.Init(); this.Benchmarks.Init();
} }
private int gcFrequency;
private int leakFrequency;
private int imageCounter;
public LoadResizeSaveStressRunner Benchmarks { get; } public LoadResizeSaveStressRunner Benchmarks { get; }
public static void Run(string[] args) public static void Run(string[] args)
{ {
Console.WriteLine($"Running: {typeof(LoadResizeSaveParallelMemoryStress).Assembly.Location}"); Console.WriteLine($"Running: {typeof(LoadResizeSaveParallelMemoryStress).Assembly.Location}");
Console.WriteLine($"64 bit: {Environment.Is64BitProcess}"); Console.WriteLine($"64 bit: {Environment.Is64BitProcess}");
var options = args.Length > 0 ? CommandLineOptions.Parse(args) : null; CommandLineOptions options = args.Length > 0 ? CommandLineOptions.Parse(args) : null;
var lrs = new LoadResizeSaveParallelMemoryStress(); var lrs = new LoadResizeSaveParallelMemoryStress();
if (options != null) if (options != null)
@ -55,10 +62,12 @@ namespace SixLabors.ImageSharp.Tests.ProfilingSandbox
if (!options.KeepDefaultAllocator) if (!options.KeepDefaultAllocator)
{ {
MemoryAllocator.Default = Configuration.Default.MemoryAllocator = options.CreateMemoryAllocator(); Configuration.Default.MemoryAllocator = options.CreateMemoryAllocator();
} }
lrs.Benchmarks.EmulateLeakedAllocations = options.LeakAllocations; lrs.leakFrequency = options.LeakFrequency;
lrs.gcFrequency = options.GcFrequency;
timer = Stopwatch.StartNew(); timer = Stopwatch.StartNew();
try try
@ -80,16 +89,23 @@ namespace SixLabors.ImageSharp.Tests.ProfilingSandbox
Configuration.Default.MemoryAllocator.ReleaseRetainedResources(); Configuration.Default.MemoryAllocator.ReleaseRetainedResources();
} }
for (int i = 0; i < options.FinalGcCount; i++) int finalGcCount = -Math.Min(0, options.GcFrequency);
if (finalGcCount > 0)
{ {
GC.Collect(); Console.WriteLine($"TotalOutstandingHandles: {UnmanagedMemoryHandle.TotalOutstandingHandles}");
GC.WaitForPendingFinalizers(); Console.WriteLine($"GC x {finalGcCount}, with 3 seconds wait.");
Thread.Sleep(1000); for (int i = 0; i < finalGcCount; i++)
{
Thread.Sleep(3000);
GC.Collect();
GC.WaitForPendingFinalizers();
}
} }
} }
var stats = new Stats(timer, lrs.Benchmarks.TotalProcessedMegapixels); var stats = new Stats(timer, lrs.Benchmarks.TotalProcessedMegapixels);
Console.WriteLine($"Total Megapixels: {stats.TotalMegapixels}, TotalOomRetries: {UnmanagedMemoryHandle.TotalOomRetries}"); Console.WriteLine($"Total Megapixels: {stats.TotalMegapixels}, TotalOomRetries: {UnmanagedMemoryHandle.TotalOomRetries}, TotalOutstandingHandles: {UnmanagedMemoryHandle.TotalOutstandingHandles}, Total Gen2 GC count: {GC.CollectionCount(2)}");
Console.WriteLine(stats.GetMarkdown()); Console.WriteLine(stats.GetMarkdown());
if (options?.FileOutput != null) if (options?.FileOutput != null)
{ {
@ -121,8 +137,9 @@ namespace SixLabors.ImageSharp.Tests.ProfilingSandbox
2. ImageSharp 2. ImageSharp
3. MagicScaler 3. MagicScaler
4. SkiaSharp 4. SkiaSharp
5. NetVips 5. SkiaSharp - Decode to target size
6. ImageMagick 6. NetVips
7. ImageMagick
"); ");
ConsoleKey key = Console.ReadKey().Key; ConsoleKey key = Console.ReadKey().Key;
@ -149,9 +166,12 @@ namespace SixLabors.ImageSharp.Tests.ProfilingSandbox
lrs.SkiaBitmapBenchmarkParallel(); lrs.SkiaBitmapBenchmarkParallel();
break; break;
case ConsoleKey.D5: case ConsoleKey.D5:
lrs.NetVipsBenchmarkParallel(); lrs.SkiaBitmapDecodeToTargetSizeBenchmarkParallel();
break; break;
case ConsoleKey.D6: case ConsoleKey.D6:
lrs.NetVipsBenchmarkParallel();
break;
case ConsoleKey.D7:
lrs.MagickBenchmarkParallel(); lrs.MagickBenchmarkParallel();
break; break;
} }
@ -222,8 +242,8 @@ namespace SixLabors.ImageSharp.Tests.ProfilingSandbox
[Option('r', "repeat-count", Required = false, Default = 1, HelpText = "Times to run the whole benchmark")] [Option('r', "repeat-count", Required = false, Default = 1, HelpText = "Times to run the whole benchmark")]
public int RepeatCount { get; set; } = 1; public int RepeatCount { get; set; } = 1;
[Option('g', "final-gc-count", Required = false, Default = 0, HelpText = "How many times to GC.Collect after execution")] [Option('g', "gc-frequency", Required = false, Default = 0, HelpText = "Positive number: call GC every 'g'-th resize. Negative number: call GC '-g' times in the end.")]
public int FinalGcCount { get; set; } public int GcFrequency { get; set; }
[Option('e', "release-at-end", Required = false, Default = false, HelpText = "Specify to run ReleaseRetainedResources() after execution")] [Option('e', "release-at-end", Required = false, Default = false, HelpText = "Specify to run ReleaseRetainedResources() after execution")]
public bool ReleaseRetainedResourcesAtEnd { get; set; } public bool ReleaseRetainedResourcesAtEnd { get; set; }
@ -237,8 +257,8 @@ namespace SixLabors.ImageSharp.Tests.ProfilingSandbox
[Option('t', "trim-period", Required = false, Default = null, HelpText = "Trim period for the pool in seconds")] [Option('t', "trim-period", Required = false, Default = null, HelpText = "Trim period for the pool in seconds")]
public int? TrimTimeSeconds { get; set; } public int? TrimTimeSeconds { get; set; }
[Option('l', "leak-allocations", Required = false, Default = false, HelpText = "Inject leaking memory allocation requests to stress-test finalizer behavior.")] [Option('l', "leak-frequency", Required = false, Default = 0, HelpText = "Inject leaking memory allocations after every 'l'-th resize to stress test finalizer behavior.")]
public bool LeakAllocations { get; set; } public int LeakFrequency { get; set; }
public static CommandLineOptions Parse(string[] args) public static CommandLineOptions Parse(string[] args)
{ {
@ -257,7 +277,7 @@ namespace SixLabors.ImageSharp.Tests.ProfilingSandbox
} }
public override string ToString() => public override string ToString() =>
$"p({this.MaxDegreeOfParallelism})_i({this.ImageSharp})_d({this.KeepDefaultAllocator})_m({this.MaxContiguousPoolBufferMegaBytes})_s({this.MaxPoolSizeMegaBytes})_u({this.MaxCapacityOfNonPoolBuffersMegaBytes})_r({this.RepeatCount})_g({this.FinalGcCount})_e({this.ReleaseRetainedResourcesAtEnd}_l({this.LeakAllocations}))"; $"p({this.MaxDegreeOfParallelism})_i({this.ImageSharp})_d({this.KeepDefaultAllocator})_m({this.MaxContiguousPoolBufferMegaBytes})_s({this.MaxPoolSizeMegaBytes})_u({this.MaxCapacityOfNonPoolBuffersMegaBytes})_r({this.RepeatCount})_g({this.GcFrequency})_e({this.ReleaseRetainedResourcesAtEnd})_l({this.LeakFrequency})";
public MemoryAllocator CreateMemoryAllocator() public MemoryAllocator CreateMemoryAllocator()
{ {
@ -290,28 +310,32 @@ namespace SixLabors.ImageSharp.Tests.ProfilingSandbox
private void SystemDrawingBenchmarkParallel() => this.ForEachImage(this.Benchmarks.SystemDrawingResize); private void SystemDrawingBenchmarkParallel() => this.ForEachImage(this.Benchmarks.SystemDrawingResize);
private void ImageSharpBenchmarkParallel() private void ImageSharpBenchmarkParallel() =>
{
int cnt = 0;
this.ForEachImage(f => this.ForEachImage(f =>
{ {
int cnt = Interlocked.Increment(ref this.imageCounter);
this.Benchmarks.ImageSharpResize(f); this.Benchmarks.ImageSharpResize(f);
if (cnt % 4 == 0 && this.Benchmarks.EmulateLeakedAllocations) if (this.leakFrequency > 0 && cnt % this.leakFrequency == 0)
{
_ = Configuration.Default.MemoryAllocator.Allocate<byte>(1 << 16);
Size size = this.Benchmarks.LastProcessedImageSize;
_ = new Image<ImageSharp.PixelFormats.Rgba64>(size.Width, size.Height);
}
if (this.gcFrequency > 0 && cnt % this.gcFrequency == 0)
{ {
GC.Collect(); GC.Collect();
GC.WaitForPendingFinalizers(); GC.WaitForPendingFinalizers();
GC.Collect(); GC.Collect();
} }
cnt++;
}); });
}
private void MagickBenchmarkParallel() => this.ForEachImage(this.Benchmarks.MagickResize); private void MagickBenchmarkParallel() => this.ForEachImage(this.Benchmarks.MagickResize);
private void MagicScalerBenchmarkParallel() => this.ForEachImage(this.Benchmarks.MagicScalerResize); private void MagicScalerBenchmarkParallel() => this.ForEachImage(this.Benchmarks.MagicScalerResize);
private void SkiaBitmapBenchmarkParallel() => this.ForEachImage(this.Benchmarks.SkiaBitmapResize); private void SkiaBitmapBenchmarkParallel() => this.ForEachImage(this.Benchmarks.SkiaBitmapResize);
private void SkiaBitmapDecodeToTargetSizeBenchmarkParallel() => this.ForEachImage(this.Benchmarks.SkiaBitmapDecodeToTargetSize);
private void NetVipsBenchmarkParallel() => this.ForEachImage(this.Benchmarks.NetVipsResize); private void NetVipsBenchmarkParallel() => this.ForEachImage(this.Benchmarks.NetVipsResize);
} }

7
tests/ImageSharp.Tests/RunTestsInLoop.ps1
View File

@ -1,16 +1,17 @@
# This script can be used to collect logs from sporadic bugs # This script can be used to collect logs from sporadic bugs
Param( Param(
[int]$TestRunCount=10, [int]$TestRunCount=10,
[string]$TargetFramework="netcoreapp3.1" [string]$TargetFramework="netcoreapp3.1",
[string]$Configuration="Release"
) )
$runId = Get-Random -Minimum 0 -Maximum 9999 $runId = Get-Random -Minimum 0 -Maximum 9999
dotnet build -c Release -f $TargetFramework dotnet build -c $Configuration -f $TargetFramework
for ($i = 0; $i -lt $TestRunCount; $i++) { for ($i = 0; $i -lt $TestRunCount; $i++) {
$logFile = ".\_testlog-" + $runId.ToString("d4") + "-run-" + $i.ToString("d3") + ".log" $logFile = ".\_testlog-" + $runId.ToString("d4") + "-run-" + $i.ToString("d3") + ".log"
Write-Host "Test run $i ..." Write-Host "Test run $i ..."
& dotnet test --no-build -c Release -f $TargetFramework 3>&1 2>&1 > $logFile & dotnet test --no-build -c $Configuration -f $TargetFramework 3>&1 2>&1 > $logFile
if ($LastExitCode -eq 0) { if ($LastExitCode -eq 0) {
Write-Host "Success!" Write-Host "Success!"
Remove-Item $logFile Remove-Item $logFile

Write Preview
Loading…
Cancel
Save