From d5e91adba6e96adc14f662582671dc7cddff79b6 Mon Sep 17 00:00:00 2001 From: Ynse Hoornenborg Date: Fri, 28 Feb 2025 21:47:52 +0100 Subject: [PATCH] Reference RGB-YUV conversions --- .../Heif/Av1/Av1ReferenceYuvConverter.cs | 156 +++++++++++++++++ .../Formats/Heif/Av1/Av1YuvConverterTests.cs | 163 +++++++++++++++++- 2 files changed, 314 insertions(+), 5 deletions(-) create mode 100644 tests/ImageSharp.Tests/Formats/Heif/Av1/Av1ReferenceYuvConverter.cs diff --git a/tests/ImageSharp.Tests/Formats/Heif/Av1/Av1ReferenceYuvConverter.cs b/tests/ImageSharp.Tests/Formats/Heif/Av1/Av1ReferenceYuvConverter.cs new file mode 100644 index 0000000000..ee728b3453 --- /dev/null +++ b/tests/ImageSharp.Tests/Formats/Heif/Av1/Av1ReferenceYuvConverter.cs @@ -0,0 +1,156 @@ +// Copyright (c) Six Labors. +// Licensed under the Six Labors Split License. + +using System.Numerics; +using SixLabors.ImageSharp.Formats.Heif.Av1; +using SixLabors.ImageSharp.PixelFormats; + +namespace SixLabors.ImageSharp.Tests.Formats.Heif.Av1; + +/// +/// This simulates converting 24-bit RGB values to YUV, then back to 24-bit RGB. +/// Using BT.709 transfer functions: https://en.wikipedia.org/wiki/Rec._709 +/// +/// It demonstrates that converting to 30-bit YUV then back to 24-bit RGB is lossy. +/// Using 10 bits per YUV value appears to be lossless. +/// +/// Converting RGB (24-bit) -> YUV(64-bit floats per channel, normalized[0 - 1]) -> RGB(24-bit) +/// Found 0 inaccurate conversions out of 16581375 RGB values +/// +/// Converting RGB(24-bit) -> YUV(30-bit) -> RGB(24-bit) +/// Found 0 inaccurate conversions out of 16581375 RGB values +/// +/// Converting RGB(24-bit) -> YUV(24-bit) -> RGB(24-bit) +/// Found 4058422 accurate conversions out of 16581375 RGB values +/// Found 12522953 inaccurate conversions out of 16581375 RGB values +/// Off by: {1: 8786792, 2: 3727753, 3: 8408} +/// +/// Ported from Python to C# from: https://gist.github.com/linrock/5be4f365c9c9e61eee9e8984ba13cb25. +internal class Av1ReferenceYuvConverter +{ + // The range of UV values in BT.709 is [-Umax, Umax] and [-Vmax, Vmax] + private const double Umax = 0.436; + private const double Vmax = 0.615; + + // Constants used in BT.709 + private const double Wr = 0.2126; + private const double Wb = 0.0722; + + // Constants used in BT.601 + // private const double Wr = 0.299; + // private const double Wb = 0.114; + + private const double Wg = 1 - Wr - Wb; + + public static Span RgbToYuv(Span row) + { + Rgb24[] result = new Rgb24[row.Length]; + for (int i = 0; i < row.Length; i++) + { + double[] current = RgbToYuv(row[i], false, true, false); + byte y = (byte)current[0]; + byte u = (byte)current[1]; + byte v = (byte)current[2]; + result[i] = new Rgb24(y, u, v); + } + + return result; + } + + public static double[] RgbToYuv(Rgb24 rgb, bool normalize = false, bool is_8bit = false, bool is_10bit = false) + { + double r = rgb.R / 255.0; + double g = rgb.G / 255.0; + double b = rgb.B / 255.0; + double y = (Wr * r) + (Wg * g) + (Wb * b); + double u = Umax * (b - y) / (1 - Wb); + double v = Vmax * (r - y) / (1 - Wr); + + // y[0, 1] u[-Umax, Umax] v[-Vmax, Vmax] + if (normalize) + { + u = (u + Umax) / (2 * Umax); + v = (v + Vmax) / (2 * Vmax); + + // y[0, 1] u[0, 1] v[0, 1] + } + + if (is_8bit) + { + y = Math.Round(y * 255); + u = Math.Round(u * 255); + v = Math.Round(v * 255); + + // y[0, 255] u[0, 255] v[0, 255] + } + + if (is_10bit) + { + y = Math.Round(y * 1023); + u = Math.Round(u * 1023); + v = Math.Round(v * 1023); + + // y[0, 1023] u[0, 1023] v[0, 1023] + } + + return [y, u, v]; + } + + public static Span YuvToRgb(Av1FrameBuffer frameBuffer) + { + Span yRow = frameBuffer.BufferY!.DangerousGetSingleSpan(); + Span uRow = frameBuffer.BufferCb!.DangerousGetSingleSpan(); + Span vRow = frameBuffer.BufferCr!.DangerousGetSingleSpan(); + Rgb24[] result = new Rgb24[yRow.Length]; + double[] yuv = new double[3]; + for (int i = 0; i < yRow.Length; i++) + { + yuv[0] = yRow[i]; + yuv[1] = uRow[i]; + yuv[2] = vRow[i]; + result[i] = YuvToRgb(yuv, false, true, false); + } + + return result; + } + + public static Rgb24 YuvToRgb(double[] yuv, bool normalized = false, bool is_8bit = false, bool is_10bit = false) + { + double y = yuv[0]; + double u = yuv[1]; + double v = yuv[2]; + if (is_8bit) + { + // y[0, 255] u[0, 255] v[0, 255] + y /= 255.0; + u /= 255.0; + v /= 255.0; + } + + if (is_10bit) + { + // y[0, 1023] u[0, 1023] v[0, 1023] + y /= 1023.0; + u /= 1023.0; + v /= 1023.0; + } + + if (normalized) + { + // y [0, 1], u [0, 1], v[0, 1] + u = (u - 0.5) * 2 * Umax; + v = (v - 0.5) * 2 * Vmax; + + // y [0, 1], u [-Umax, Umax], v[-Vmax, Vmax] + } + + // r = y + 1.28033 * v + // g = y - 0.21482 * u - 0.38059 * v + // b = y + 2.12798 * u + double r = y + (v * (1 - Wr) / Vmax); + double g = y - (u * Wb * (1 - Wb) / (Umax * Wg)) - (v * Wr * (1 - Wr) / (Vmax * Wg)); + double b = y + (u * (1 - Wb) / Umax); + + return new Rgb24((byte)Math.Round(r * 255), (byte)Math.Round(g * 255), (byte)Math.Round(b * 255)); + } +} diff --git a/tests/ImageSharp.Tests/Formats/Heif/Av1/Av1YuvConverterTests.cs b/tests/ImageSharp.Tests/Formats/Heif/Av1/Av1YuvConverterTests.cs index 0f6e2f70cf..3b04f84eff 100644 --- a/tests/ImageSharp.Tests/Formats/Heif/Av1/Av1YuvConverterTests.cs +++ b/tests/ImageSharp.Tests/Formats/Heif/Av1/Av1YuvConverterTests.cs @@ -1,12 +1,8 @@ // Copyright (c) Six Labors. // Licensed under the Six Labors Split License. -using Iced.Intel; using SixLabors.ImageSharp.Formats.Heif.Av1; using SixLabors.ImageSharp.Formats.Heif.Av1.OpenBitstreamUnit; -using SixLabors.ImageSharp.Formats.Heif.Av1.Pipeline; -using SixLabors.ImageSharp.Formats.Heif.Av1.Tiling; -using SixLabors.ImageSharp.Memory; using SixLabors.ImageSharp.PixelFormats; using SixLabors.ImageSharp.Tests.TestUtilities.ImageComparison; @@ -15,6 +11,163 @@ namespace SixLabors.ImageSharp.Tests.Formats.Heif.Av1; [Trait("Format", "Avif")] public class Av1YuvConverterTests { + [Theory] + [InlineData(255, 255, 255, 255, 127, 127)] + [InlineData(0, 0, 0, 0, 127, 127)] + [InlineData(42, 42, 42, 42, 127, 127)] + [InlineData(150, 100, 50, 107, 97, 154)] + public void RgbToYuvSinglePixel(byte r, byte g, byte b, int y, int u, int v) + { + // Assign + using Image image = new(1, 1); + ImageFrame frame = image.Frames.RootFrame; + frame.DangerousTryGetSinglePixelMemory(out Memory memory); + memory.Span[0] = new Rgb24(r, g, b); + ObuSequenceHeader sequenceHeader = new(); + sequenceHeader.MaxFrameWidth = 1; + sequenceHeader.MaxFrameHeight = 1; + Av1FrameBuffer frameBuffer = new(Configuration.Default, sequenceHeader, Av1ColorFormat.Yuv444, false); + + // Act + Av1YuvConverter.ConvertFromRgb(Configuration.Default, frame, frameBuffer); + + // Assert + byte actualY = frameBuffer.BufferY.DangerousGetRowSpan(0)[0]; + byte actualU = frameBuffer.BufferCb.DangerousGetRowSpan(0)[0]; + byte actualV = frameBuffer.BufferCr.DangerousGetRowSpan(0)[0]; + Assert.Equal(y, actualY); + Assert.Equal(u, actualU); + Assert.Equal(v, actualV); + } + + [Theory] + [InlineData(255, 255, 255, 255, 127, 127)] + [InlineData(0, 0, 0, 0, 127, 127)] + [InlineData(42, 42, 42, 42, 127, 127)] + [InlineData(150, 100, 50, 107, 97, 154)] + public void YuvToRgbSinglePixel(byte r, byte g, byte b, int y, int u, int v) + { + // Assign + using Image image = new(1, 1); + ImageFrame frame = image.Frames.RootFrame; + ObuSequenceHeader sequenceHeader = new(); + sequenceHeader.MaxFrameWidth = 1; + sequenceHeader.MaxFrameHeight = 1; + Av1FrameBuffer frameBuffer = new(Configuration.Default, sequenceHeader, Av1ColorFormat.Yuv444, false); + frameBuffer.BufferY.DangerousGetRowSpan(0)[0] = (byte)y; + frameBuffer.BufferCb.DangerousGetRowSpan(0)[0] = (byte)u; + frameBuffer.BufferCr.DangerousGetRowSpan(0)[0] = (byte)v; + + // Act + Av1YuvConverter.ConvertToRgb(Configuration.Default, frameBuffer, frame); + + // Assert + frame.DangerousTryGetSinglePixelMemory(out Memory memory); + Rgb24 actual = memory.Span[0]; + Assert.Equal(r, actual.R, 1d); + Assert.Equal(g, actual.G, 1d); + Assert.Equal(b, actual.B, 1d); + } + + [Fact] + public void RgbToYuvCompareToReferenceRandomPixels() + { + const int sampleCount = 1000; + + // Assign + using Image image = new(sampleCount, 1); + ImageFrame frame = image.Frames.RootFrame; + frame.DangerousTryGetSinglePixelMemory(out Memory memory); + Random rnd = new(42); + Span input = new byte[sampleCount * 3]; + CreateTestData(rnd, input); + PixelOperations.Instance.FromBgr24Bytes(Configuration.Default, input, memory.Span, image.Width); + ObuSequenceHeader sequenceHeader = new(); + sequenceHeader.MaxFrameWidth = image.Width; + sequenceHeader.MaxFrameHeight = image.Height; + Av1FrameBuffer frameBuffer = new(Configuration.Default, sequenceHeader, Av1ColorFormat.Yuv444, false); + + // Act + Av1YuvConverter.ConvertFromRgb(Configuration.Default, frame, frameBuffer); + Span referenceOutput = Av1ReferenceYuvConverter.RgbToYuv(memory.Span); + + // Assert + Span actual = new Rgb24[frameBuffer.Width]; + Span yRow = frameBuffer.BufferY!.DangerousGetSingleSpan(); + Span uRow = frameBuffer.BufferCb!.DangerousGetSingleSpan(); + Span vRow = frameBuffer.BufferCr!.DangerousGetSingleSpan(); + for (int i = 0; i < frameBuffer.Width; i++) + { + Rgb24 pixel = new(); + pixel.R = yRow[i]; + pixel.G = uRow[i]; + pixel.B = vRow[i]; + actual[i] = pixel; + } + + Compare(referenceOutput, actual, 1); + } + + [Fact] + public void YuvToRgbCompareToReferenceRandomPixels() + { + const int sampleCount = 1000; + + // Assign + using Image image = new(sampleCount, 1); + ImageFrame frame = image.Frames.RootFrame; + ObuSequenceHeader sequenceHeader = new(); + sequenceHeader.MaxFrameWidth = image.Width; + sequenceHeader.MaxFrameHeight = image.Height; + Av1FrameBuffer frameBuffer = new(Configuration.Default, sequenceHeader, Av1ColorFormat.Yuv444, false); + Random rnd = new(42); + CreateTestData(rnd, frameBuffer.BufferY.DangerousGetRowSpan(0)); + CreateTestData(rnd, frameBuffer.BufferCb.DangerousGetRowSpan(0)); + CreateTestData(rnd, frameBuffer.BufferCr.DangerousGetRowSpan(0)); + + // Act + Av1YuvConverter.ConvertToRgb(Configuration.Default, frameBuffer, frame); + Span referenceOutput = Av1ReferenceYuvConverter.YuvToRgb(frameBuffer); + + // Assert + frame.DangerousTryGetSinglePixelMemory(out Memory memory); + Span actual = memory.Span; + Compare(referenceOutput, actual, 1); + } + + private static void Compare(Span referenceOutput, Span actual, int allowedDifference) + { + for (int i = 0; i < actual.Length; i++) + { + if (Math.Abs(referenceOutput[i].R - actual[i].R) > allowedDifference || + Math.Abs(referenceOutput[i].G - actual[i].G) > allowedDifference || + Math.Abs(referenceOutput[i].B - actual[i].B) > allowedDifference) + { + Assert.Fail($"Difference at index {i}, expected: {referenceOutput[i]} but was {actual[i]}"); + } + } + } + + private static void CreateTestData(Random rnd, Span span, int bitCount = 8) + { + int max = (1 << bitCount) - 1; + for (int i = 0; i < span.Length; i++) + { + byte current = (byte)rnd.Next(max); + span[i] = current; + } + } + + private static void CreateTestData(Random rnd, Span span, int bitCount) + { + int max = (1 << bitCount) - 1; + for (int i = 0; i < span.Length; i++) + { + ushort current = (ushort)rnd.Next(max); + span[i] = current; + } + } + [Theory] [InlineData(255, 255, 255)] [InlineData(0, 0, 0)] @@ -24,7 +177,7 @@ public class Av1YuvConverterTests [InlineData(42, 0, 42)] [InlineData(0, 42, 42)] [InlineData(0, 0, 42)] - [InlineData(50, 100, 150)] + [InlineData(150, 100, 50)] public void RoundTripSinglePixel(byte r, byte g, byte b) { // Assign