From ea1183acdc9dff52f19d34e4e820b4098ffebb02 Mon Sep 17 00:00:00 2001 From: Ynse Hoornenborg Date: Sat, 22 Feb 2025 18:23:08 +0100 Subject: [PATCH] First implementation of YUV conversion --- src/ImageSharp/Formats/Heif/Av1/Av1Decoder.cs | 7 +- .../Formats/Heif/Av1/Av1YuvConverter.cs | 163 ++++++++++++++++++ .../Formats/Heif/Av1/Av1YuvConverterTests.cs | 73 ++++++++ 3 files changed, 241 insertions(+), 2 deletions(-) create mode 100644 src/ImageSharp/Formats/Heif/Av1/Av1YuvConverter.cs create mode 100644 tests/ImageSharp.Tests/Formats/Heif/Av1/Av1YuvConverterTests.cs diff --git a/src/ImageSharp/Formats/Heif/Av1/Av1Decoder.cs b/src/ImageSharp/Formats/Heif/Av1/Av1Decoder.cs index 767743430a..00e3a96ec5 100644 --- a/src/ImageSharp/Formats/Heif/Av1/Av1Decoder.cs +++ b/src/ImageSharp/Formats/Heif/Av1/Av1Decoder.cs @@ -5,6 +5,7 @@ using SixLabors.ImageSharp.Formats.Heif.Av1.OpenBitstreamUnit; using SixLabors.ImageSharp.Formats.Heif.Av1.Pipeline; using SixLabors.ImageSharp.Formats.Heif.Av1.Tiling; using SixLabors.ImageSharp.PixelFormats; +using SixLabors.ImageSharp.PixelFormats.Utils; namespace SixLabors.ImageSharp.Formats.Heif.Av1; @@ -43,8 +44,10 @@ internal class Av1Decoder : IAv1TileReader this.frameDecoder = new(this.SequenceHeader, this.FrameHeader, this.FrameInfo, this.FrameBuffer); this.frameDecoder.DecodeFrame(); - // TODO: Implement returning proper pixels. - return new Image(1, 1); + Image resultImage = new(this.FrameHeader.FrameSize.FrameWidth, this.FrameHeader.FrameSize.FrameHeight); + ImageFrame resultFrame = resultImage.Frames.RootFrame; + Av1YuvConverter.ConvertToRgb(this.configuration, this.FrameBuffer, resultFrame); + return resultImage; } public void ReadTile(Span tileData, int tileNum) diff --git a/src/ImageSharp/Formats/Heif/Av1/Av1YuvConverter.cs b/src/ImageSharp/Formats/Heif/Av1/Av1YuvConverter.cs new file mode 100644 index 0000000000..0d7df3cd84 --- /dev/null +++ b/src/ImageSharp/Formats/Heif/Av1/Av1YuvConverter.cs @@ -0,0 +1,163 @@ +// Copyright (c) Six Labors. +// Licensed under the Six Labors Split License. + +using System.Runtime.CompilerServices; +using SixLabors.ImageSharp.Memory; +using SixLabors.ImageSharp.PixelFormats; + +namespace SixLabors.ImageSharp.Formats.Heif.Av1; + +internal class Av1YuvConverter +{ + public static void ConvertToRgb(Configuration configuration, Av1FrameBuffer frameBuffer, ImageFrame image) + where TPixel : unmanaged, IPixel + { + using Image rgbImage = new(image.Width, image.Height); + ImageFrame rgbFrame = rgbImage.Frames.RootFrame; + + // TODO: Support YUV420 and YUV420 also. + if (frameBuffer.ColorFormat != Av1ColorFormat.Yuv444) + { + throw new NotSupportedException("Only able to convert YUV444 to RGB."); + } + + ConvertYuvToRgb(frameBuffer, rgbFrame, false); + image.ProcessPixelRows(rgbFrame, (resultAcc, rgbAcc) => + { + for (int y = 0; y < rgbImage.Height; y++) + { + Span rgbRow = rgbAcc.GetRowSpan(y); + Span resultRow = resultAcc.GetRowSpan(y); + PixelOperations.Instance.FromRgb24(configuration, rgbRow, resultRow); + } + }); + } + + public static void ConvertFromRgb(Configuration configuration, ImageFrame image, Av1FrameBuffer frameBuffer) + where TPixel : unmanaged, IPixel + { + using Image rgbImage = new(image.Width, image.Height); + ImageFrame rgbFrame = rgbImage.Frames.RootFrame; + + image.ProcessPixelRows(rgbFrame, (sourceAcc, rgbAcc) => + { + for (int y = 0; y < rgbImage.Height; y++) + { + Span rgbRow = rgbAcc.GetRowSpan(y); + Span sourceRow = sourceAcc.GetRowSpan(y); + PixelOperations.Instance.ToRgb24(configuration, sourceRow, rgbRow); + } + }); + + // TODO: Support YUV422 and YUV420 also. + ConvertRgbToYuv444(rgbFrame, frameBuffer); + } + + private static void ConvertYuvToRgb(Av1FrameBuffer buffer, ImageFrame image, bool isSubsampled) + { + // Weight multiplied by 256 to exploit full byte resolution, rounded to the nearest integer. + // Using BT.709 specification + const int rvWeight = (int)(1.28033 * 256); + const int guWeight = (int)(-0.21482 * 256); + const int gvWeight = (int)(-0.38059 * 256); + const int buWeight = (int)(2.12798 * 256); + Guard.NotNull(buffer.BufferY); + Guard.NotNull(buffer.BufferCb); + Guard.NotNull(buffer.BufferCr); + Guard.MustBeGreaterThanOrEqualTo(buffer.BufferY.Width, image.Width, nameof(buffer)); + Guard.MustBeGreaterThanOrEqualTo(buffer.BufferY.Height, image.Height, nameof(buffer)); + if (isSubsampled) + { + Guard.MustBeGreaterThanOrEqualTo(buffer.BufferCb.Width, image.Width >> 1, nameof(buffer)); + Guard.MustBeGreaterThanOrEqualTo(buffer.BufferCb.Height, image.Height >> 1, nameof(buffer)); + Guard.MustBeGreaterThanOrEqualTo(buffer.BufferCr.Width, image.Width >> 1, nameof(buffer)); + Guard.MustBeGreaterThanOrEqualTo(buffer.BufferCr.Height, image.Height >> 1, nameof(buffer)); + } + else + { + Guard.MustBeGreaterThanOrEqualTo(buffer.BufferCb.Width, image.Width, nameof(buffer)); + Guard.MustBeGreaterThanOrEqualTo(buffer.BufferCb.Height, image.Height, nameof(buffer)); + Guard.MustBeGreaterThanOrEqualTo(buffer.BufferCr.Width, image.Width, nameof(buffer)); + Guard.MustBeGreaterThanOrEqualTo(buffer.BufferCr.Height, image.Height, nameof(buffer)); + } + + image.ProcessPixelRows(accessor => + { + Buffer2D yBuffer = buffer.BufferY; + Buffer2D uBuffer = buffer.BufferCb; + Buffer2D vBuffer = buffer.BufferCr; + for (int y = 0; y < image.Height; y++) + { + Span rgbRow = accessor.GetRowSpan(y); + ref Rgb24 pixel = ref rgbRow[0]; + Span ySpan = yBuffer.DangerousGetRowSpan(y); + ref byte yRef = ref ySpan[0]; + Span uSpan = uBuffer.DangerousGetRowSpan(y); + ref byte uRef = ref uSpan[0]; + Span vSpan = vBuffer.DangerousGetRowSpan(y); + ref byte vRef = ref vSpan[0]; + for (int x = 0; x < image.Width; x++) + { + pixel.R = (byte)Av1Math.Clip3(0, 255, ((yRef << 8) + (rvWeight * vRef)) >> 8); + pixel.G = (byte)Av1Math.Clip3(0, 255, ((yRef << 8) + (guWeight * uRef) + (gvWeight * vRef)) >> 8); + pixel.B = (byte)Av1Math.Clip3(0, 255, ((yRef << 8) + (buWeight * uRef)) >> 8); + pixel = ref Unsafe.Add(ref pixel, 1); + yRef = ref Unsafe.Add(ref yRef, 1); + uRef = ref Unsafe.Add(ref uRef, 1); + vRef = ref Unsafe.Add(ref vRef, 1); + } + } + }); + } + + private static void ConvertRgbToYuv444(ImageFrame image, Av1FrameBuffer buffer) + { + // Weight multiplied by 256 to exploit full byte resolution, rounded to the nearest integer. + const int yrWeight = (int)(0.2126 * 256); + const int ygWeight = (int)(0.7152 * 256); + const int ybWeight = (int)(0.0722 * 256); + const int urWeight = (int)(-0.09991 * 256); + const int ugWeight = (int)(-0.33609 * 256); + const int ubWeight = (int)(0.436 * 256); + const int vrWeight = (int)(0.615 * 256); + const int vgWeight = (int)(-0.55861 * 256); + const int vbWeight = (int)(-0.05639 * 256); + Guard.NotNull(buffer.BufferY); + Guard.NotNull(buffer.BufferCb); + Guard.NotNull(buffer.BufferCr); + Guard.MustBeGreaterThanOrEqualTo(buffer.BufferY.Width, image.Width, nameof(buffer)); + Guard.MustBeGreaterThanOrEqualTo(buffer.BufferY.Height, image.Height, nameof(buffer)); + Guard.MustBeGreaterThanOrEqualTo(buffer.BufferCb.Width, image.Width, nameof(buffer)); + Guard.MustBeGreaterThanOrEqualTo(buffer.BufferCb.Height, image.Height, nameof(buffer)); + Guard.MustBeGreaterThanOrEqualTo(buffer.BufferCr.Width, image.Width, nameof(buffer)); + Guard.MustBeGreaterThanOrEqualTo(buffer.BufferCr.Height, image.Height, nameof(buffer)); + + image.ProcessPixelRows(accessor => + { + Buffer2D yBuffer = buffer.BufferY; + Buffer2D uBuffer = buffer.BufferCb; + Buffer2D vBuffer = buffer.BufferCr; + for (int y = 0; y < image.Height; y++) + { + Span rgbRow = accessor.GetRowSpan(y); + ref Rgb24 pixel = ref rgbRow[0]; + Span ySpan = yBuffer.DangerousGetRowSpan(y); + ref byte yRef = ref ySpan[0]; + Span uSpan = uBuffer.DangerousGetRowSpan(y); + ref byte uRef = ref uSpan[0]; + Span vSpan = vBuffer.DangerousGetRowSpan(y); + ref byte vRef = ref vSpan[0]; + for (int x = 0; x < image.Width; x++) + { + yRef = (byte)Av1Math.Clip3(0, 255, ((yrWeight * pixel.R) + (ygWeight * pixel.G) + (ybWeight * pixel.B)) >> 8); + uRef = (byte)Av1Math.Clip3(0, 255, ((urWeight * pixel.R) + (ugWeight * pixel.G) + (ubWeight * pixel.B)) >> 8); + vRef = (byte)Av1Math.Clip3(0, 255, ((vrWeight * pixel.R) + (vgWeight * pixel.G) + (vbWeight * pixel.B)) >> 8); + pixel = ref Unsafe.Add(ref pixel, 1); + yRef = ref Unsafe.Add(ref yRef, 1); + uRef = ref Unsafe.Add(ref uRef, 1); + vRef = ref Unsafe.Add(ref vRef, 1); + } + } + }); + } +} diff --git a/tests/ImageSharp.Tests/Formats/Heif/Av1/Av1YuvConverterTests.cs b/tests/ImageSharp.Tests/Formats/Heif/Av1/Av1YuvConverterTests.cs new file mode 100644 index 0000000000..0f6e2f70cf --- /dev/null +++ b/tests/ImageSharp.Tests/Formats/Heif/Av1/Av1YuvConverterTests.cs @@ -0,0 +1,73 @@ +// Copyright (c) Six Labors. +// Licensed under the Six Labors Split License. + +using Iced.Intel; +using SixLabors.ImageSharp.Formats.Heif.Av1; +using SixLabors.ImageSharp.Formats.Heif.Av1.OpenBitstreamUnit; +using SixLabors.ImageSharp.Formats.Heif.Av1.Pipeline; +using SixLabors.ImageSharp.Formats.Heif.Av1.Tiling; +using SixLabors.ImageSharp.Memory; +using SixLabors.ImageSharp.PixelFormats; +using SixLabors.ImageSharp.Tests.TestUtilities.ImageComparison; + +namespace SixLabors.ImageSharp.Tests.Formats.Heif.Av1; + +[Trait("Format", "Avif")] +public class Av1YuvConverterTests +{ + [Theory] + [InlineData(255, 255, 255)] + [InlineData(0, 0, 0)] + [InlineData(42, 42, 42)] + [InlineData(42, 0, 0)] + [InlineData(42, 42, 0)] + [InlineData(42, 0, 42)] + [InlineData(0, 42, 42)] + [InlineData(0, 0, 42)] + [InlineData(50, 100, 150)] + public void RoundTripSinglePixel(byte r, byte g, byte b) + { + // Assign + using Image image = new(1, 1); + ImageFrame frame = image.Frames.RootFrame; + frame.DangerousTryGetSinglePixelMemory(out Memory memory); + memory.Span[0] = new Rgb24(r, g, b); + ObuSequenceHeader sequenceHeader = new(); + sequenceHeader.MaxFrameWidth = 1; + sequenceHeader.MaxFrameHeight = 1; + Av1FrameBuffer frameBuffer = new(Configuration.Default, sequenceHeader, Av1ColorFormat.Yuv444, false); + using Image actual = new(image.Width, image.Height); + + // Act + Av1YuvConverter.ConvertFromRgb(Configuration.Default, frame, frameBuffer); + Av1YuvConverter.ConvertToRgb(Configuration.Default, frameBuffer, actual.Frames.RootFrame); + + // Assert + actual.Frames.RootFrame.DangerousTryGetSinglePixelMemory(out Memory actualMemory); + Rgb24 actualPixel = actualMemory.Span[0]; + Assert.Equal(r, actualPixel.R, 2d); + Assert.Equal(g, actualPixel.G, 2d); + Assert.Equal(b, actualPixel.B, 2d); + } + + [Theory] + [WithFile(TestImages.Jpeg.Baseline.Winter444_Interleaved, PixelTypes.Rgb24)] + public void RoundTrip(TestImageProvider provider) + { + // Assign + using Image image = provider.GetImage(); + ImageFrame frame = image.Frames.RootFrame; + ObuSequenceHeader sequenceHeader = new(); + sequenceHeader.MaxFrameWidth = image.Width; + sequenceHeader.MaxFrameHeight = image.Height; + Av1FrameBuffer frameBuffer = new(Configuration.Default, sequenceHeader, Av1ColorFormat.Yuv444, false); + using Image actual = new(image.Width, image.Height); + + // Act + Av1YuvConverter.ConvertFromRgb(Configuration.Default, frame, frameBuffer); + Av1YuvConverter.ConvertToRgb(Configuration.Default, frameBuffer, actual.Frames.RootFrame); + + // Assert + ImageComparer.Tolerant(0.002F).VerifySimilarity(image, actual); + } +}