From ab2ae29dc9c66ca8adb4ef15fc0316cda2b05434 Mon Sep 17 00:00:00 2001 From: Ynse Hoornenborg Date: Sun, 4 Aug 2024 11:47:18 +0200 Subject: [PATCH] ObuWriter improvements --- .../Formats/Heif/Av1/Av1BitStreamWriter.cs | 21 ++ .../Formats/Heif/Av1/IAv1TileWriter.cs | 19 ++ .../Heif/Av1/OpenBitstreamUnit/ObuWriter.cs | 308 +++++++++++------- .../Heif/Av1/Transform/Av1FrameDecoder.cs | 8 +- .../Formats/Heif/Av1/Av1TileDecoderStub.cs | 11 +- .../Formats/Heif/Av1/ObuFrameHeaderTests.cs | 92 +++++- 6 files changed, 311 insertions(+), 148 deletions(-) create mode 100644 src/ImageSharp/Formats/Heif/Av1/IAv1TileWriter.cs diff --git a/src/ImageSharp/Formats/Heif/Av1/Av1BitStreamWriter.cs b/src/ImageSharp/Formats/Heif/Av1/Av1BitStreamWriter.cs index 67ae41415..0b447f832 100644 --- a/src/ImageSharp/Formats/Heif/Av1/Av1BitStreamWriter.cs +++ b/src/ImageSharp/Formats/Heif/Av1/Av1BitStreamWriter.cs @@ -113,4 +113,25 @@ internal ref struct Av1BitStreamWriter(Stream stream) this.bitOffset = 0; } } + + public void WriteLittleEndian(uint value, int n) + { + // See section 4.10.4 of the AV1-Specification + DebugGuard.IsTrue(Av1Math.Modulus8(this.BitPosition) == 0, "Writing of Little Endian value only allowed on byte alignment"); + + uint t = value; + for (int i = 0; i < n; i++) + { + this.WriteLiteral(t & 0xff, 8); + t >>= 8; + } + } + + internal void WriteBlob(Span tileData) + { + DebugGuard.IsTrue(Av1Math.Modulus8(this.BitPosition) == 0, "Writing of Tile Data only allowed on byte alignment"); + + this.stream.Write(tileData); + this.bitOffset += tileData.Length << 3; + } } diff --git a/src/ImageSharp/Formats/Heif/Av1/IAv1TileWriter.cs b/src/ImageSharp/Formats/Heif/Av1/IAv1TileWriter.cs new file mode 100644 index 000000000..1e2552b8b --- /dev/null +++ b/src/ImageSharp/Formats/Heif/Av1/IAv1TileWriter.cs @@ -0,0 +1,19 @@ +// Copyright (c) Six Labors. +// Licensed under the Six Labors Split License. + +namespace SixLabors.ImageSharp.Formats.Heif.Av1; + +/// +/// Interface for writing of image tiles. +/// +internal interface IAv1TileWriter +{ + /// + /// Write the information for a single tile. + /// + /// The index of the tile that is to be read. + /// + /// The bytes of encoded data in the bitstream dedicated to this tile. + /// + Span WriteTile(int tileNum); +} diff --git a/src/ImageSharp/Formats/Heif/Av1/OpenBitstreamUnit/ObuWriter.cs b/src/ImageSharp/Formats/Heif/Av1/OpenBitstreamUnit/ObuWriter.cs index ea38b2897..e7025b5cf 100644 --- a/src/ImageSharp/Formats/Heif/Av1/OpenBitstreamUnit/ObuWriter.cs +++ b/src/ImageSharp/Formats/Heif/Av1/OpenBitstreamUnit/ObuWriter.cs @@ -1,16 +1,20 @@ // Copyright (c) Six Labors. // Licensed under the Six Labors Split License. +using System; using SixLabors.ImageSharp.Formats.Heif.Av1.Transform; namespace SixLabors.ImageSharp.Formats.Heif.Av1.OpenBitstreamUnit; internal class ObuWriter { + private int[] previousQIndex = []; + private int[] previousDeltaLoopFilter = []; + /// /// Encode a single frame into OBU's. /// - public static void WriteAll(Stream stream, ObuSequenceHeader sequenceHeader, ObuFrameHeader frameInfo) + public void WriteAll(Stream stream, ObuSequenceHeader sequenceHeader, ObuFrameHeader frameInfo, IAv1TileWriter tileWriter) { MemoryStream bufferStream = new(100); Av1BitStreamWriter writer = new(bufferStream); @@ -27,19 +31,15 @@ internal class ObuWriter if (frameInfo != null && sequenceHeader != null) { bufferStream.Position = 0; - WriteFrameHeader(ref writer, sequenceHeader, frameInfo, true); - int bytesWritten = (writer.BitPosition + 7) >> 3; - writer.Flush(); - WriteObuHeaderAndSize(stream, ObuType.FrameHeader, bufferStream.GetBuffer(), bytesWritten); - } + this.WriteFrameHeader(ref writer, sequenceHeader, frameInfo, true); + if (frameInfo.TilesInfo != null) + { + WriteTileGroup(ref writer, frameInfo.TilesInfo, tileWriter); + } - if (frameInfo?.TilesInfo != null) - { - bufferStream.Position = 0; - WriteTileGroup(ref writer, frameInfo.TilesInfo); - int bytesWritten = (writer.BitPosition + 7) >> 3; + int bytesWritten = 5; // (writer.BitPosition + 7) >> 3; writer.Flush(); - WriteObuHeaderAndSize(stream, ObuType.TileGroup, bufferStream.GetBuffer(), bytesWritten); + WriteObuHeaderAndSize(stream, ObuType.Frame, bufferStream.GetBuffer(), bytesWritten); } } @@ -234,53 +234,53 @@ internal class ObuWriter WriteSuperResolutionParameters(ref writer, sequenceHeader, frameInfo); } - private static void WriteTileInfo(ref Av1BitStreamWriter writer, ObuSequenceHeader sequenceHeader, ObuFrameHeader frameInfo, ObuTileGroupHeader tileInfo) + private static void WriteTileInfo(ref Av1BitStreamWriter writer, ObuSequenceHeader sequenceHeader, ObuFrameHeader frameInfo) { - int superBlockColumnCount; - int superBlockRowCount; - int superBlockShift; - if (sequenceHeader.Use128x128Superblock) - { - superBlockColumnCount = (frameInfo.ModeInfoColumnCount + 31) >> 5; - superBlockRowCount = (frameInfo.ModeInfoRowCount + 31) >> 5; - superBlockShift = 5; - } - else - { - superBlockColumnCount = (frameInfo.ModeInfoColumnCount + 15) >> 4; - superBlockRowCount = (frameInfo.ModeInfoRowCount + 15) >> 4; - superBlockShift = 4; - } - - int superBlockSize = superBlockShift + 2; + ObuTileGroupHeader tileInfo = frameInfo.TilesInfo; + int superblockColumnCount; + int superblockRowCount; + int superblockSizeLog2 = sequenceHeader.SuperblockSizeLog2; + int superblockShift = superblockSizeLog2 - Av1Constants.ModeInfoSizeLog2; + superblockColumnCount = (frameInfo.ModeInfoColumnCount + sequenceHeader.SuperblockModeInfoSize - 1) >> superblockShift; + superblockRowCount = (frameInfo.ModeInfoRowCount + sequenceHeader.SuperblockModeInfoSize - 1) >> superblockShift; + int superBlockSize = superblockShift + 2; int maxTileAreaOfSuperBlock = Av1Constants.MaxTileArea >> (2 * superBlockSize); tileInfo.MaxTileWidthSuperblock = Av1Constants.MaxTileWidth >> superBlockSize; tileInfo.MaxTileHeightSuperblock = (Av1Constants.MaxTileArea / Av1Constants.MaxTileWidth) >> superBlockSize; - tileInfo.MinLog2TileColumnCount = ObuReader.TileLog2(tileInfo.MaxTileWidthSuperblock, superBlockColumnCount); - tileInfo.MaxLog2TileColumnCount = ObuReader.TileLog2(1, Math.Min(superBlockColumnCount, Av1Constants.MaxTileColumnCount)); - tileInfo.MaxLog2TileRowCount = ObuReader.TileLog2(1, Math.Min(superBlockRowCount, Av1Constants.MaxTileRowCount)); - tileInfo.MinLog2TileCount = Math.Max(tileInfo.MinLog2TileColumnCount, ObuReader.TileLog2(maxTileAreaOfSuperBlock, superBlockColumnCount * superBlockRowCount)); + tileInfo.MinLog2TileColumnCount = ObuReader.TileLog2(tileInfo.MaxTileWidthSuperblock, superblockColumnCount); + tileInfo.MaxLog2TileColumnCount = ObuReader.TileLog2(1, Math.Min(superblockColumnCount, Av1Constants.MaxTileColumnCount)); + tileInfo.MaxLog2TileRowCount = ObuReader.TileLog2(1, Math.Min(superblockRowCount, Av1Constants.MaxTileRowCount)); + tileInfo.MinLog2TileCount = Math.Max(tileInfo.MinLog2TileColumnCount, ObuReader.TileLog2(maxTileAreaOfSuperBlock, superblockColumnCount * superblockRowCount)); + + int log2TileColumnCount = Av1Math.Log2(tileInfo.TileColumnCount); + int log2TileRowCount = Av1Math.Log2(tileInfo.TileRowCount); writer.WriteBoolean(tileInfo.HasUniformTileSpacing); if (tileInfo.HasUniformTileSpacing) { - for (int i = 0; i < tileInfo.TileColumnCountLog2; i++) + // Uniform spaced tiles with power-of-two number of rows and columns + // tile columns + int ones = log2TileColumnCount - tileInfo.MinLog2TileColumnCount; + while (ones-- > 0) { writer.WriteBoolean(true); } - if (tileInfo.TileColumnCountLog2 < tileInfo.MaxLog2TileColumnCount) + if (log2TileColumnCount < tileInfo.MaxLog2TileColumnCount) { writer.WriteBoolean(false); } - for (int i = 0; i < tileInfo.TileRowCountLog2; i++) + // rows + tileInfo.MinLog2TileRowCount = Math.Min(tileInfo.MinLog2TileCount - log2TileColumnCount, 0); + ones = log2TileRowCount - tileInfo.MinLog2TileRowCount; + while (ones-- > 0) { writer.WriteBoolean(true); } - if (tileInfo.TileRowCountLog2 < tileInfo.MaxLog2TileRowCount) + if (log2TileRowCount < tileInfo.MaxLog2TileRowCount) { writer.WriteBoolean(false); } @@ -289,29 +289,29 @@ internal class ObuWriter { int startSuperBlock = 0; int i = 0; - for (; startSuperBlock < superBlockColumnCount; i++) + for (; startSuperBlock < superblockColumnCount; i++) { - uint widthInSuperBlocks = (uint)((tileInfo.TileColumnStartModeInfo[i] >> superBlockShift) - startSuperBlock); - uint maxWidth = (uint)Math.Min(superBlockColumnCount - startSuperBlock, tileInfo.MaxTileWidthSuperblock); + uint widthInSuperBlocks = (uint)((tileInfo.TileColumnStartModeInfo[i] >> superblockShift) - startSuperBlock); + uint maxWidth = (uint)Math.Min(superblockColumnCount - startSuperBlock, tileInfo.MaxTileWidthSuperblock); writer.WriteNonSymmetric(widthInSuperBlocks - 1, maxWidth); startSuperBlock += (int)widthInSuperBlocks; } - if (startSuperBlock != superBlockColumnCount) + if (startSuperBlock != superblockColumnCount) { throw new ImageFormatException("Super block tiles width does not add up to total width."); } startSuperBlock = 0; - for (i = 0; startSuperBlock < superBlockRowCount; i++) + for (i = 0; startSuperBlock < superblockRowCount; i++) { - uint heightInSuperBlocks = (uint)((tileInfo.TileRowStartModeInfo[i] >> superBlockShift) - startSuperBlock); - uint maxHeight = (uint)Math.Min(superBlockRowCount - startSuperBlock, tileInfo.MaxTileHeightSuperblock); + uint heightInSuperBlocks = (uint)((tileInfo.TileRowStartModeInfo[i] >> superblockShift) - startSuperBlock); + uint maxHeight = (uint)Math.Min(superblockRowCount - startSuperBlock, tileInfo.MaxTileHeightSuperblock); writer.WriteNonSymmetric(heightInSuperBlocks - 1, maxHeight); startSuperBlock += (int)heightInSuperBlocks; } - if (startSuperBlock != superBlockRowCount) + if (startSuperBlock != superblockRowCount) { throw new ImageFormatException("Super block tiles height does not add up to total height."); } @@ -322,122 +322,163 @@ internal class ObuWriter writer.WriteLiteral(tileInfo.ContextUpdateTileId, tileInfo.TileRowCountLog2 + tileInfo.TileColumnCountLog2); writer.WriteLiteral((uint)tileInfo.TileSizeBytes - 1, 2); } + + frameInfo.TilesInfo = tileInfo; } - private static void WriteUncompressedFrameHeader(ref Av1BitStreamWriter writer, ObuSequenceHeader sequenceHeader, ObuFrameHeader frameInfo, int planesCount) + private void WriteUncompressedFrameHeader(ref Av1BitStreamWriter writer, ObuSequenceHeader sequenceHeader, ObuFrameHeader frameHeader) { - uint previousFrameId = 0; - bool isIntraFrame = true; - int idLength = sequenceHeader.FrameIdLength - 1 + sequenceHeader.DeltaFrameIdLength - 2 + 3; - writer.WriteBoolean(frameInfo.DisableCdfUpdate); - if (frameInfo.AllowScreenContentTools) + // TODO: Make tile count configurable. + int tileCount = 1; + int planesCount = sequenceHeader.ColorConfig.PlaneCount; + writer.WriteBoolean(frameHeader.DisableCdfUpdate); + if (sequenceHeader.ForceScreenContentTools == 2) { - writer.WriteBoolean(frameInfo.AllowScreenContentTools); + writer.WriteBoolean(frameHeader.AllowScreenContentTools); + } + else + { + // Guard.IsTrue(frameInfo.AllowScreenContentTools == sequenceHeader.ForceScreenContentTools); } - if (frameInfo.AllowScreenContentTools) + if (frameHeader.AllowScreenContentTools) { - if (sequenceHeader.ForceIntegerMotionVector == 1) + if (sequenceHeader.ForceIntegerMotionVector == 2) { - writer.WriteBoolean(frameInfo.ForceIntegerMotionVector); + writer.WriteBoolean(frameHeader.ForceIntegerMotionVector); + } + else + { + // Guard.IsTrue(frameInfo.ForceIntegerMotionVector == sequenceHeader.ForceIntegerMotionVector, nameof(frameInfo.ForceIntegerMotionVector), "Frame and sequence must be in sync"); } } - bool havePreviousFrameId = !(frameInfo.FrameType == ObuFrameType.KeyFrame && frameInfo.ShowFrame); - if (havePreviousFrameId) + if (frameHeader.FrameType == ObuFrameType.KeyFrame) + { + if (!frameHeader.ShowFrame) + { + throw new NotImplementedException("No support for hidden frames."); + } + } + else if (frameHeader.FrameType == ObuFrameType.IntraOnlyFrame) { - previousFrameId = frameInfo.CurrentFrameId; + throw new NotImplementedException("No IntraOnly frames supported."); } - if (sequenceHeader.IsFrameIdNumbersPresent) + if (frameHeader.FrameType == ObuFrameType.KeyFrame) { - writer.WriteLiteral(frameInfo.CurrentFrameId, idLength); - if (havePreviousFrameId) + WriteFrameSize(ref writer, sequenceHeader, frameHeader, false); + if (frameHeader.AllowScreenContentTools) { - uint diffFrameId = (frameInfo.CurrentFrameId > previousFrameId) ? - frameInfo.CurrentFrameId - previousFrameId : - (uint)((1 << idLength) + (int)frameInfo.CurrentFrameId - previousFrameId); - if (frameInfo.CurrentFrameId == previousFrameId || diffFrameId >= 1 << (idLength - 1)) - { - throw new ImageFormatException("Current frame ID cannot be same as previous Frame ID"); - } + writer.WriteBoolean(frameHeader.AllowIntraBlockCopy); + } + } + else if (frameHeader.FrameType == ObuFrameType.IntraOnlyFrame) + { + WriteFrameSize(ref writer, sequenceHeader, frameHeader, false); + if (frameHeader.AllowScreenContentTools) + { + writer.WriteBoolean(frameHeader.AllowIntraBlockCopy); } + } + else + { + throw new NotImplementedException("Inter frames not applicable for AVIF."); + } - int diffLength = sequenceHeader.DeltaFrameIdLength; - for (int i = 0; i < Av1Constants.ReferenceFrameCount; i++) + WriteTileInfo(ref writer, sequenceHeader, frameHeader); + WriteQuantizationParameters(ref writer, frameHeader.QuantizationParameters, sequenceHeader.ColorConfig, planesCount); + WriteSegmentationParameters(ref writer, sequenceHeader, frameHeader, planesCount); + + if (frameHeader.QuantizationParameters.BaseQIndex > 0) + { + writer.WriteBoolean(frameHeader.DeltaQParameters.IsPresent); + if (frameHeader.DeltaQParameters.IsPresent) { - if (frameInfo.CurrentFrameId > (1U << diffLength)) + writer.WriteLiteral((uint)frameHeader.DeltaQParameters.Resolution - 1, 2); + for (int tileIndex = 0; tileIndex < tileCount; tileIndex++) { - if ((frameInfo.ReferenceFrameIndex[i] > frameInfo.CurrentFrameId) || - frameInfo.ReferenceFrameIndex[i] > (frameInfo.CurrentFrameId - (1 - diffLength))) - { - frameInfo.ReferenceValid[i] = false; - } + this.previousQIndex[tileIndex] = frameHeader.QuantizationParameters.BaseQIndex; + } + + if (frameHeader.AllowIntraBlockCopy) + { + Guard.IsFalse( + frameHeader.DeltaLoopFilterParameters.IsPresent, + nameof(frameHeader.DeltaLoopFilterParameters.IsPresent), + "Allow INTRA block copy required Loop Filter."); } - else if (frameInfo.ReferenceFrameIndex[i] > frameInfo.CurrentFrameId && - frameInfo.ReferenceFrameIndex[i] < ((1 << idLength) + (frameInfo.CurrentFrameId - (1 << diffLength)))) + else { - frameInfo.ReferenceValid[i] = false; + writer.WriteBoolean(frameHeader.DeltaLoopFilterParameters.IsPresent); } - } - } - - writer.WriteLiteral(frameInfo.OrderHint, sequenceHeader.OrderHintInfo.OrderHintBits); - if (!isIntraFrame && !frameInfo.ErrorResilientMode) - { - writer.WriteLiteral(frameInfo.PrimaryReferenceFrame, Av1Constants.PimaryReferenceBits); + if (frameHeader.DeltaLoopFilterParameters.IsPresent) + { + writer.WriteLiteral((uint)(1 + Av1Math.MostSignificantBit((uint)frameHeader.DeltaLoopFilterParameters.Resolution) - 1), 2); + writer.WriteBoolean(frameHeader.DeltaLoopFilterParameters.IsMulti); + int frameLoopFilterCount = sequenceHeader.ColorConfig.IsMonochrome ? Av1Constants.FrameLoopFilterCount - 2 : Av1Constants.FrameLoopFilterCount; + for (int loopFilterId = 0; loopFilterId < frameLoopFilterCount; loopFilterId++) + { + this.previousDeltaLoopFilter[loopFilterId] = 0; + } + } + } } - // Skipping, as no decoder info model present - frameInfo.AllowHighPrecisionMotionVector = false; - frameInfo.UseReferenceFrameMotionVectors = false; - frameInfo.AllowIntraBlockCopy = false; - if (frameInfo.FrameType != ObuFrameType.SwitchFrame && !(frameInfo.FrameType == ObuFrameType.KeyFrame && frameInfo.ShowFrame)) + if (frameHeader.AllLossless) { - writer.WriteLiteral(frameInfo.RefreshFrameFlags, 8); + throw new NotImplementedException("No entire lossless supported."); } - - if (isIntraFrame) + else { - WriteFrameSize(ref writer, sequenceHeader, frameInfo, false); - WriteRenderSize(ref writer, frameInfo); - if (frameInfo.AllowScreenContentTools && frameInfo.FrameSize.RenderWidth != 0) + if (!frameHeader.CodedLossless) { - if (frameInfo.FrameSize.FrameWidth == frameInfo.FrameSize.SuperResolutionUpscaledWidth) + WriteLoopFilterParameters(ref writer, sequenceHeader, frameHeader, planesCount); + if (sequenceHeader.CdefLevel > 0) { - writer.WriteBoolean(frameInfo.AllowIntraBlockCopy); + WriteCdefParameters(ref writer, sequenceHeader, frameHeader, planesCount); } } + + if (sequenceHeader.EnableRestoration) + { + WriteLoopRestorationParameters(ref writer, sequenceHeader, frameHeader, planesCount); + } } - if (frameInfo.PrimaryReferenceFrame == Av1Constants.PrimaryReferenceFrameNone) + writer.WriteBoolean(frameHeader.TransformMode == Av1TransformMode.Select); + + // No compound INTER-INTER for AVIF. + if (frameHeader.SkipModeParameters.SkipModeAllowed) { - SetupPastIndependence(frameInfo); + writer.WriteBoolean(frameHeader.SkipModeParameters.SkipModeFlag); } - // GenerateNextReferenceFrameMap(sequenceHeader, frameInfo); - WriteTileInfo(ref writer, sequenceHeader, frameInfo, frameInfo.TilesInfo); - WriteQuantizationParameters(ref writer, frameInfo.QuantizationParameters, sequenceHeader.ColorConfig, planesCount); - WriteSegmentationParameters(ref writer, sequenceHeader, frameInfo, planesCount); - WriteFrameDeltaQParameters(ref writer, frameInfo); - WriteFrameDeltaLoopFilterParameters(ref writer, frameInfo); - - WriteLoopFilterParameters(ref writer, sequenceHeader, frameInfo, planesCount); - WriteCdefParameters(ref writer, sequenceHeader, frameInfo, planesCount); - WriteLoopRestorationParameters(ref writer, sequenceHeader, frameInfo, planesCount); - WriteTransformMode(ref writer, frameInfo); + if (FrameMightAllowWarpedMotion(sequenceHeader, frameHeader)) + { + writer.WriteBoolean(frameHeader.AllowWarpedMotion); + } + else + { + Guard.IsFalse(frameHeader.AllowWarpedMotion, nameof(frameHeader.AllowWarpedMotion), "No warped motion allowed."); + } - // Not applicable for INTRA frames. - // WriteFrameReferenceMode(ref writer, frameInfo.ReferenceMode, isIntraFrame); - // WriteSkipModeParameters(ref writer, sequenceHeader, frameInfo, isIntraFrame, frameInfo.ReferenceMode); - writer.WriteBoolean(frameInfo.UseReducedTransformSet); + writer.WriteBoolean(frameHeader.UseReducedTransformSet); - // Not applicable for INTRA frames. - // WriteGlobalMotionParameters(ref writer, sequenceHeader, frameInfo, isIntraFrame); - WriteFilmGrainFilterParameters(ref writer, frameInfo.FilmGrainParameters); + // No global motion for AVIF. + if (sequenceHeader.AreFilmGrainingParametersPresent && (frameHeader.ShowFrame || frameHeader.ShowableFrame)) + { + WriteFilmGrainFilterParameters(ref writer, frameHeader.FilmGrainParameters); + } } + private static bool IsSuperResolutionUnscaled(ObuFrameSize frameSize) + => frameSize.FrameWidth == frameSize.SuperResolutionUpscaledWidth; + + private static bool FrameMightAllowWarpedMotion(ObuSequenceHeader sequenceHeader, ObuFrameHeader frameHeader) + => false; // !frameHeader.ErrorResilientMode && !FrameIsIntraOnly(sequenceHeader) && scs->enable_warped_motion; + private static void SetupPastIndependence(ObuFrameHeader frameInfo) { // TODO: Initialize the loop filter parameters. @@ -460,24 +501,21 @@ internal class ObuWriter } } - private static int WriteFrameHeader(ref Av1BitStreamWriter writer, ObuSequenceHeader sequenceHeader, ObuFrameHeader frameInfo, bool writeTrailingBits) + private int WriteFrameHeader(ref Av1BitStreamWriter writer, ObuSequenceHeader sequenceHeader, ObuFrameHeader frameInfo, bool writeTrailingBits) { - int planeCount = sequenceHeader.ColorConfig.IsMonochrome ? 1 : 3; int startBitPosition = writer.BitPosition; - WriteUncompressedFrameHeader(ref writer, sequenceHeader, frameInfo, planeCount); + this.WriteUncompressedFrameHeader(ref writer, sequenceHeader, frameInfo); if (writeTrailingBits) { WriteTrailingBits(ref writer); } - AlignToByteBoundary(ref writer); - int endPosition = writer.BitPosition; int headerBytes = (endPosition - startBitPosition) / 8; return headerBytes; } - private static int WriteTileGroup(ref Av1BitStreamWriter writer, ObuTileGroupHeader tileInfo) + private static int WriteTileGroup(ref Av1BitStreamWriter writer, ObuTileGroupHeader tileInfo, IAv1TileWriter tileWriter) { int tileCount = tileInfo.TileColumnCount * tileInfo.TileRowCount; int startBitPosition = writer.BitPosition; @@ -494,11 +532,29 @@ internal class ObuWriter } AlignToByteBoundary(ref writer); + + WriteTileData(ref writer, tileInfo, tileWriter); + int endBitPosition = writer.BitPosition; int headerBytes = (endBitPosition - startBitPosition) / 8; return headerBytes; } + private static void WriteTileData(ref Av1BitStreamWriter writer, ObuTileGroupHeader tileInfo, IAv1TileWriter tileWriter) + { + int tileCount = tileInfo.TileColumnCount * tileInfo.TileRowCount; + for (int tileNum = 0; tileNum < tileCount; tileNum++) + { + Span tileData = tileWriter.WriteTile(tileNum); + if (tileNum != tileCount - 1 && tileCount > 1) + { + writer.WriteLittleEndian((uint)tileData.Length - 1U, tileInfo.TileSizeBytes); + } + + writer.WriteBlob(tileData); + } + } + private static int WriteDeltaQ(ref Av1BitStreamWriter writer, int deltaQ) { bool isCoded = deltaQ == 0; diff --git a/src/ImageSharp/Formats/Heif/Av1/Transform/Av1FrameDecoder.cs b/src/ImageSharp/Formats/Heif/Av1/Transform/Av1FrameDecoder.cs index 85744a4f6..040ce87b9 100644 --- a/src/ImageSharp/Formats/Heif/Av1/Transform/Av1FrameDecoder.cs +++ b/src/ImageSharp/Formats/Heif/Av1/Transform/Av1FrameDecoder.cs @@ -8,9 +8,9 @@ namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform; internal class Av1FrameDecoder { - private ObuSequenceHeader sequenceHeader; - private ObuFrameHeader frameHeader; - private Av1FrameBuffer frameBuffer; + private readonly ObuSequenceHeader sequenceHeader; + private readonly ObuFrameHeader frameHeader; + private readonly Av1FrameBuffer frameBuffer; public Av1FrameDecoder(ObuSequenceHeader sequenceHeader, ObuFrameHeader frameHeader, Av1FrameBuffer frameBuffer) { @@ -22,6 +22,8 @@ internal class Av1FrameDecoder public void DecodeFrame() { Guard.NotNull(this.sequenceHeader); + Guard.NotNull(this.frameHeader); + Guard.NotNull(this.frameBuffer); // TODO: Implement. } diff --git a/tests/ImageSharp.Tests/Formats/Heif/Av1/Av1TileDecoderStub.cs b/tests/ImageSharp.Tests/Formats/Heif/Av1/Av1TileDecoderStub.cs index 67f9b0b3e..a2afda2ac 100644 --- a/tests/ImageSharp.Tests/Formats/Heif/Av1/Av1TileDecoderStub.cs +++ b/tests/ImageSharp.Tests/Formats/Heif/Av1/Av1TileDecoderStub.cs @@ -5,10 +5,13 @@ using SixLabors.ImageSharp.Formats.Heif.Av1; namespace SixLabors.ImageSharp.Tests.Formats.Heif.Av1; -internal class Av1TileDecoderStub : IAv1TileReader +internal class Av1TileDecoderStub : IAv1TileReader, IAv1TileWriter { + private readonly Dictionary tileDatas = []; + public void ReadTile(Span tileData, int tileNum) - { - // Intentionally left blank. - } + => this.tileDatas.Add(tileNum, tileData.ToArray()); + + public Span WriteTile(int tileNum) + => this.tileDatas[tileNum]; } diff --git a/tests/ImageSharp.Tests/Formats/Heif/Av1/ObuFrameHeaderTests.cs b/tests/ImageSharp.Tests/Formats/Heif/Av1/ObuFrameHeaderTests.cs index 5b5d82836..64f752e5a 100644 --- a/tests/ImageSharp.Tests/Formats/Heif/Av1/ObuFrameHeaderTests.cs +++ b/tests/ImageSharp.Tests/Formats/Heif/Av1/ObuFrameHeaderTests.cs @@ -12,6 +12,11 @@ public class ObuFrameHeaderTests private static readonly byte[] DefaultSequenceHeaderBitStream = [0x0a, 0x06, 0b001_1_1_000, 0b00_1000_01, 0b11_110101, 0b001_11101, 0b111_1_1_1_0_1, 0b1_0_0_1_1_1_10]; + // TODO: Check with libgav1 test code. + private static readonly byte[] KeyFrameHeaderBitStream = + // libgav1 expects this: [0x32, 0x05, 0x10, 0x00]; + [0x32, 0x05, 0x20, 0x04]; + // Bits Syntax element Value // 1 obu_forbidden_bit 0 // 4 obu_type 2 (OBU_TEMPORAL_DELIMITER) @@ -22,10 +27,10 @@ public class ObuFrameHeaderTests private static readonly byte[] DefaultTemporalDelimiterBitStream = [0x12, 0x00]; [Theory] - // [InlineData(TestImages.Heif.IrvineAvif, 0x0102, 0x000D)] - // [InlineData(TestImages.Heif.IrvineAvif, 0x0198, 0x6BD1)] - [InlineData(TestImages.Heif.XnConvert, 0x010E, 0x03CC)] - [InlineData(TestImages.Heif.Orange4x4, 0x010E, 0x001d)] + // [InlineData(TestImages.Heif.IrvineAvif, 0x0102, 0x000d)] + // [InlineData(TestImages.Heif.IrvineAvif, 0x0198, 0x6bd1)] + [InlineData(TestImages.Heif.XnConvert, 0x010e, 0x03cc)] + [InlineData(TestImages.Heif.Orange4x4, 0x010e, 0x001d)] public void ReadFrameHeader(string filename, int fileOffset, int blockSize) { // Assign @@ -49,26 +54,27 @@ public class ObuFrameHeaderTests /* [Theory] - [InlineData(TestImages.Heif.XnConvert, 0x010E, 0x03CC)] - public void BinaryIdenticalRoundTripFrameHeader(string filename, int fileOffset, int blockSize) + [InlineData(TestImages.Heif.Orange4x4, 0x010e, 0x001d, 0x0128)] + [InlineData(TestImages.Heif.XnConvert, 0x010e, 0x03cc, 0x0114)] + public void BinaryIdenticalRoundTripFrameHeader(string filename, int fileOffset, int blockSize, int tileOffset) { // Assign string filePath = Path.Combine(TestEnvironment.InputImagesDirectoryFullPath, filename); byte[] content = File.ReadAllBytes(filePath); Span span = content.AsSpan(fileOffset, blockSize); - Av1TileDecoderStub tileDecoder = new(); + Av1TileDecoderStub tileStub = new(); Av1BitStreamReader reader = new(span); ObuReader obuReader = new(); // Act 1 - obuReader.ReadAll(ref reader, blockSize, tileDecoder); + obuReader.ReadAll(ref reader, blockSize, tileStub); // Assign 2 MemoryStream encoded = new(); // Act 2 ObuWriter obuWriter = new(); - ObuWriter.Write(encoded, obuReader.SequenceHeader, obuReader.FrameHeader); + obuWriter.WriteAll(encoded, obuReader.SequenceHeader, obuReader.FrameHeader, tileStub); // Assert Assert.Equal(span, encoded.ToArray()); @@ -76,25 +82,27 @@ public class ObuFrameHeaderTests */ [Theory] - [InlineData(TestImages.Heif.XnConvert, 0x010E, 0x03CC)] + [InlineData(TestImages.Heif.Orange4x4, 0x010e, 0x001d)] + [InlineData(TestImages.Heif.XnConvert, 0x010e, 0x03cc)] public void ThreeTimeRoundTripFrameHeader(string filename, int fileOffset, int blockSize) { // Assign string filePath = Path.Combine(TestEnvironment.InputImagesDirectoryFullPath, filename); byte[] content = File.ReadAllBytes(filePath); Span span = content.AsSpan(fileOffset, blockSize); - IAv1TileReader tileDecoder = new Av1TileDecoderStub(); + Av1TileDecoderStub tileStub = new(); Av1BitStreamReader reader = new(span); ObuReader obuReader1 = new(); // Act 1 - obuReader1.ReadAll(ref reader, blockSize, tileDecoder); + obuReader1.ReadAll(ref reader, blockSize, tileStub); // Assign 2 MemoryStream encoded = new(); // Act 2 - ObuWriter.WriteAll(encoded, obuReader1.SequenceHeader, obuReader1.FrameHeader); + ObuWriter obuWriter = new(); + obuWriter.WriteAll(encoded, obuReader1.SequenceHeader, obuReader1.FrameHeader, tileStub); // Assign 2 Span encodedBuffer = encoded.ToArray(); @@ -169,9 +177,10 @@ public class ObuFrameHeaderTests { // Arrange using MemoryStream stream = new(2); + ObuWriter obuWriter = new(); // Act - ObuWriter.WriteAll(stream, null, null); + obuWriter.WriteAll(stream, null, null, null); byte[] actual = stream.GetBuffer(); // Assert @@ -184,9 +193,10 @@ public class ObuFrameHeaderTests // Arrange using MemoryStream stream = new(10); ObuSequenceHeader input = GetDefaultSequenceHeader(); + ObuWriter obuWriter = new(); // Act - ObuWriter.WriteAll(stream, input, null); + obuWriter.WriteAll(stream, input, null, null); byte[] buffer = stream.GetBuffer(); // Assert @@ -195,6 +205,28 @@ public class ObuFrameHeaderTests Assert.Equal(DefaultSequenceHeaderBitStream, actual); } + [Fact] + public void WriteFrameHeader() + { + // Arrange + using MemoryStream stream = new(10); + ObuSequenceHeader sequenceInput = GetDefaultSequenceHeader(); + ObuFrameHeader frameInput = GetKeyFrameHeader(); + Av1TileDecoderStub tileStub = new(); + byte[] empty = []; + tileStub.ReadTile(empty, 0); + ObuWriter obuWriter = new(); + + // Act + obuWriter.WriteAll(stream, sequenceInput, frameInput, tileStub); + byte[] buffer = stream.GetBuffer(); + + // Assert + // Skip over Temporal Delimiter and Sequence header. + byte[] actual = buffer.AsSpan().Slice(DefaultTemporalDelimiterBitStream.Length + DefaultSequenceHeaderBitStream.Length, KeyFrameHeaderBitStream.Length).ToArray(); + Assert.Equal(KeyFrameHeaderBitStream, actual); + } + private static ObuSequenceHeader GetDefaultSequenceHeader() // Offset Bits Syntax element Value @@ -261,4 +293,34 @@ public class ObuFrameHeaderTests }, AreFilmGrainingParametersPresent = true, }; + + private static ObuFrameHeader GetKeyFrameHeader() + => new() + { + FrameType = ObuFrameType.KeyFrame, + ShowFrame = true, + ShowableFrame = false, + DisableFrameEndUpdateCdf = false, + FrameSize = new() + { + FrameWidth = 426, + FrameHeight = 240, + RenderWidth = 426, + RenderHeight = 240, + SuperResolutionUpscaledWidth = 426, + }, + PrimaryReferenceFrame = 7, + ModeInfoRowCount = 60, + ModeInfoColumnCount = 108, + RefreshFrameFlags = 0xff, + ErrorResilientMode = true, + ForceIntegerMotionVector = true, + TilesInfo = new ObuTileGroupHeader() + { + HasUniformTileSpacing = true, + TileColumnCount = 1, + TileRowCount = 1, + } + }; + }