From 41a5b09eb638fc3b1663553a599a3c458909937a Mon Sep 17 00:00:00 2001 From: James Jackson-South Date: Tue, 12 Nov 2019 13:36:31 +1100 Subject: [PATCH 01/16] Initial working commit --- src/ImageSharp/Formats/Png/Zlib/Adler32.cs | 8 +- src/ImageSharp/Formats/Png/Zlib/Deflater.cs | 610 +++++++++++ .../Formats/Png/Zlib/DeflaterConstants.cs | 151 +++ .../Formats/Png/Zlib/DeflaterEngine.cs | 951 +++++++++++++++++ .../Formats/Png/Zlib/DeflaterHuffman.cs | 965 ++++++++++++++++++ .../Formats/Png/Zlib/DeflaterOutputStream.cs | 499 +++++++++ .../Formats/Png/Zlib/DeflaterPending.cs | 21 + .../Formats/Png/Zlib/PendingBuffer.cs | 276 +++++ .../Formats/Png/Zlib/ZlibDeflateStream.cs | 30 +- .../ImageSharp.Benchmarks/Codecs/EncodePng.cs | 8 +- 10 files changed, 3498 insertions(+), 21 deletions(-) create mode 100644 src/ImageSharp/Formats/Png/Zlib/Deflater.cs create mode 100644 src/ImageSharp/Formats/Png/Zlib/DeflaterConstants.cs create mode 100644 src/ImageSharp/Formats/Png/Zlib/DeflaterEngine.cs create mode 100644 src/ImageSharp/Formats/Png/Zlib/DeflaterHuffman.cs create mode 100644 src/ImageSharp/Formats/Png/Zlib/DeflaterOutputStream.cs create mode 100644 src/ImageSharp/Formats/Png/Zlib/DeflaterPending.cs create mode 100644 src/ImageSharp/Formats/Png/Zlib/PendingBuffer.cs diff --git a/src/ImageSharp/Formats/Png/Zlib/Adler32.cs b/src/ImageSharp/Formats/Png/Zlib/Adler32.cs index a06983b9e..f6f6edd12 100644 --- a/src/ImageSharp/Formats/Png/Zlib/Adler32.cs +++ b/src/ImageSharp/Formats/Png/Zlib/Adler32.cs @@ -1,4 +1,4 @@ -// Copyright (c) Six Labors and contributors. +// Copyright (c) Six Labors and contributors. // Licensed under the Apache License, Version 2.0. using System; @@ -133,8 +133,8 @@ namespace SixLabors.ImageSharp.Formats.Png.Zlib count -= n; while (--n >= 0) { - s1 = s1 + (uint)(data[offset++] & 0xff); - s2 = s2 + s1; + s1 += (uint)(data[offset++] & 0xff); + s2 += s1; } s1 %= Base; @@ -144,4 +144,4 @@ namespace SixLabors.ImageSharp.Formats.Png.Zlib this.checksum = (s2 << 16) | s1; } } -} \ No newline at end of file +} diff --git a/src/ImageSharp/Formats/Png/Zlib/Deflater.cs b/src/ImageSharp/Formats/Png/Zlib/Deflater.cs new file mode 100644 index 000000000..358112549 --- /dev/null +++ b/src/ImageSharp/Formats/Png/Zlib/Deflater.cs @@ -0,0 +1,610 @@ +// Copyright (c) Six Labors and contributors. +// Licensed under the Apache License, Version 2.0. + +// +using System; +using System.Collections.Generic; +using System.Text; + +namespace SixLabors.ImageSharp.Formats.Png.Zlib +{ + /// + /// This is the Deflater class. The deflater class compresses input + /// with the deflate algorithm described in RFC 1951. It has several + /// compression levels and three different strategies described below. + /// + /// This class is not thread safe. This is inherent in the API, due + /// to the split of deflate and setInput. + /// + /// author of the original java version : Jochen Hoenicke + /// + public class Deflater + { + #region Deflater Documentation + + /* + * The Deflater can do the following state transitions: + * + * (1) -> INIT_STATE ----> INIT_FINISHING_STATE ---. + * / | (2) (5) | + * / v (5) | + * (3)| SETDICT_STATE ---> SETDICT_FINISHING_STATE |(3) + * \ | (3) | ,--------' + * | | | (3) / + * v v (5) v v + * (1) -> BUSY_STATE ----> FINISHING_STATE + * | (6) + * v + * FINISHED_STATE + * \_____________________________________/ + * | (7) + * v + * CLOSED_STATE + * + * (1) If we should produce a header we start in INIT_STATE, otherwise + * we start in BUSY_STATE. + * (2) A dictionary may be set only when we are in INIT_STATE, then + * we change the state as indicated. + * (3) Whether a dictionary is set or not, on the first call of deflate + * we change to BUSY_STATE. + * (4) -- intentionally left blank -- :) + * (5) FINISHING_STATE is entered, when flush() is called to indicate that + * there is no more INPUT. There are also states indicating, that + * the header wasn't written yet. + * (6) FINISHED_STATE is entered, when everything has been flushed to the + * internal pending output buffer. + * (7) At any time (7) + * + */ + + #endregion Deflater Documentation + + #region Public Constants + + /// + /// The best and slowest compression level. This tries to find very + /// long and distant string repetitions. + /// + public const int BEST_COMPRESSION = 9; + + /// + /// The worst but fastest compression level. + /// + public const int BEST_SPEED = 1; + + /// + /// The default compression level. + /// + public const int DEFAULT_COMPRESSION = -1; + + /// + /// This level won't compress at all but output uncompressed blocks. + /// + public const int NO_COMPRESSION = 0; + + /// + /// The compression method. This is the only method supported so far. + /// There is no need to use this constant at all. + /// + public const int DEFLATED = 8; + + #endregion Public Constants + + #region Public Enum + + /// + /// Compression Level as an enum for safer use + /// + public enum CompressionLevel + { + /// + /// The best and slowest compression level. This tries to find very + /// long and distant string repetitions. + /// + BEST_COMPRESSION = Deflater.BEST_COMPRESSION, + + /// + /// The worst but fastest compression level. + /// + BEST_SPEED = Deflater.BEST_SPEED, + + /// + /// The default compression level. + /// + DEFAULT_COMPRESSION = Deflater.DEFAULT_COMPRESSION, + + /// + /// This level won't compress at all but output uncompressed blocks. + /// + NO_COMPRESSION = Deflater.NO_COMPRESSION, + + /// + /// The compression method. This is the only method supported so far. + /// There is no need to use this constant at all. + /// + DEFLATED = Deflater.DEFLATED + } + + #endregion Public Enum + + #region Local Constants + + private const int IS_SETDICT = 0x01; + private const int IS_FLUSHING = 0x04; + private const int IS_FINISHING = 0x08; + + private const int INIT_STATE = 0x00; + private const int SETDICT_STATE = 0x01; + + // private static int INIT_FINISHING_STATE = 0x08; + // private static int SETDICT_FINISHING_STATE = 0x09; + private const int BUSY_STATE = 0x10; + + private const int FLUSHING_STATE = 0x14; + private const int FINISHING_STATE = 0x1c; + private const int FINISHED_STATE = 0x1e; + private const int CLOSED_STATE = 0x7f; + + #endregion Local Constants + + #region Constructors + + /// + /// Creates a new deflater with default compression level. + /// + public Deflater() : this(DEFAULT_COMPRESSION, false) + { + } + + /// + /// Creates a new deflater with given compression level. + /// + /// + /// the compression level, a value between NO_COMPRESSION + /// and BEST_COMPRESSION, or DEFAULT_COMPRESSION. + /// + /// if lvl is out of range. + public Deflater(int level) : this(level, false) + { + } + + /// + /// Creates a new deflater with given compression level. + /// + /// + /// the compression level, a value between NO_COMPRESSION + /// and BEST_COMPRESSION. + /// + /// + /// true, if we should suppress the Zlib/RFC1950 header at the + /// beginning and the adler checksum at the end of the output. This is + /// useful for the GZIP/PKZIP formats. + /// + /// if lvl is out of range. + public Deflater(int level, bool noZlibHeaderOrFooter) + { + if (level == DEFAULT_COMPRESSION) + { + level = 6; + } + else if (level < NO_COMPRESSION || level > BEST_COMPRESSION) + { + throw new ArgumentOutOfRangeException(nameof(level)); + } + + pending = new DeflaterPending(); + engine = new DeflaterEngine(pending, noZlibHeaderOrFooter); + this.noZlibHeaderOrFooter = noZlibHeaderOrFooter; + SetStrategy(DeflateStrategy.Default); + SetLevel(level); + Reset(); + } + + #endregion Constructors + + /// + /// Resets the deflater. The deflater acts afterwards as if it was + /// just created with the same compression level and strategy as it + /// had before. + /// + public void Reset() + { + state = (noZlibHeaderOrFooter ? BUSY_STATE : INIT_STATE); + totalOut = 0; + pending.Reset(); + engine.Reset(); + } + + /// + /// Gets the current adler checksum of the data that was processed so far. + /// + public int Adler + { + get + { + return engine.Adler; + } + } + + /// + /// Gets the number of input bytes processed so far. + /// + public long TotalIn + { + get + { + return engine.TotalIn; + } + } + + /// + /// Gets the number of output bytes so far. + /// + public long TotalOut + { + get + { + return totalOut; + } + } + + /// + /// Flushes the current input block. Further calls to deflate() will + /// produce enough output to inflate everything in the current input + /// block. This is not part of Sun's JDK so I have made it package + /// private. It is used by DeflaterOutputStream to implement + /// flush(). + /// + public void Flush() + { + state |= IS_FLUSHING; + } + + /// + /// Finishes the deflater with the current input block. It is an error + /// to give more input after this method was called. This method must + /// be called to force all bytes to be flushed. + /// + public void Finish() + { + state |= (IS_FLUSHING | IS_FINISHING); + } + + /// + /// Returns true if the stream was finished and no more output bytes + /// are available. + /// + public bool IsFinished + { + get + { + return (state == FINISHED_STATE) && pending.IsFlushed; + } + } + + /// + /// Returns true, if the input buffer is empty. + /// You should then call setInput(). + /// NOTE: This method can also return true when the stream + /// was finished. + /// + public bool IsNeedingInput + { + get + { + return engine.NeedsInput(); + } + } + + /// + /// Sets the data which should be compressed next. This should be only + /// called when needsInput indicates that more input is needed. + /// If you call setInput when needsInput() returns false, the + /// previous input that is still pending will be thrown away. + /// The given byte array should not be changed, before needsInput() returns + /// true again. + /// This call is equivalent to setInput(input, 0, input.length). + /// + /// + /// the buffer containing the input data. + /// + /// + /// if the buffer was finished() or ended(). + /// + public void SetInput(byte[] input) + { + SetInput(input, 0, input.Length); + } + + /// + /// Sets the data which should be compressed next. This should be + /// only called when needsInput indicates that more input is needed. + /// The given byte array should not be changed, before needsInput() returns + /// true again. + /// + /// + /// the buffer containing the input data. + /// + /// + /// the start of the data. + /// + /// + /// the number of data bytes of input. + /// + /// + /// if the buffer was Finish()ed or if previous input is still pending. + /// + public void SetInput(byte[] input, int offset, int count) + { + if ((state & IS_FINISHING) != 0) + { + throw new InvalidOperationException("Finish() already called"); + } + engine.SetInput(input, offset, count); + } + + /// + /// Sets the compression level. There is no guarantee of the exact + /// position of the change, but if you call this when needsInput is + /// true the change of compression level will occur somewhere near + /// before the end of the so far given input. + /// + /// + /// the new compression level. + /// + public void SetLevel(int level) + { + if (level == DEFAULT_COMPRESSION) + { + level = 6; + } + else if (level < NO_COMPRESSION || level > BEST_COMPRESSION) + { + throw new ArgumentOutOfRangeException(nameof(level)); + } + + if (this.level != level) + { + this.level = level; + engine.SetLevel(level); + } + } + + /// + /// Get current compression level + /// + /// Returns the current compression level + public int GetLevel() + { + return level; + } + + /// + /// Sets the compression strategy. Strategy is one of + /// DEFAULT_STRATEGY, HUFFMAN_ONLY and FILTERED. For the exact + /// position where the strategy is changed, the same as for + /// SetLevel() applies. + /// + /// + /// The new compression strategy. + /// + public void SetStrategy(DeflateStrategy strategy) + { + engine.Strategy = strategy; + } + + /// + /// Deflates the current input block with to the given array. + /// + /// + /// The buffer where compressed data is stored + /// + /// + /// The number of compressed bytes added to the output, or 0 if either + /// IsNeedingInput() or IsFinished returns true or length is zero. + /// + public int Deflate(byte[] output) + { + return Deflate(output, 0, output.Length); + } + + /// + /// Deflates the current input block to the given array. + /// + /// + /// Buffer to store the compressed data. + /// + /// + /// Offset into the output array. + /// + /// + /// The maximum number of bytes that may be stored. + /// + /// + /// The number of compressed bytes added to the output, or 0 if either + /// needsInput() or finished() returns true or length is zero. + /// + /// + /// If Finish() was previously called. + /// + /// + /// If offset or length don't match the array length. + /// + public int Deflate(byte[] output, int offset, int length) + { + int origLength = length; + + if (state == CLOSED_STATE) + { + throw new InvalidOperationException("Deflater closed"); + } + + if (state < BUSY_STATE) + { + // output header + int header = (DEFLATED + + ((DeflaterConstants.MAX_WBITS - 8) << 4)) << 8; + int level_flags = (level - 1) >> 1; + if (level_flags < 0 || level_flags > 3) + { + level_flags = 3; + } + header |= level_flags << 6; + if ((state & IS_SETDICT) != 0) + { + // Dictionary was set + header |= DeflaterConstants.PRESET_DICT; + } + header += 31 - (header % 31); + + pending.WriteShortMSB(header); + if ((state & IS_SETDICT) != 0) + { + int chksum = engine.Adler; + engine.ResetAdler(); + pending.WriteShortMSB(chksum >> 16); + pending.WriteShortMSB(chksum & 0xffff); + } + + state = BUSY_STATE | (state & (IS_FLUSHING | IS_FINISHING)); + } + + for (; ; ) + { + int count = pending.Flush(output, offset, length); + offset += count; + totalOut += count; + length -= count; + + if (length == 0 || state == FINISHED_STATE) + { + break; + } + + if (!engine.Deflate((state & IS_FLUSHING) != 0, (state & IS_FINISHING) != 0)) + { + switch (state) + { + case BUSY_STATE: + // We need more input now + return origLength - length; + + case FLUSHING_STATE: + if (level != NO_COMPRESSION) + { + /* We have to supply some lookahead. 8 bit lookahead + * is needed by the zlib inflater, and we must fill + * the next byte, so that all bits are flushed. + */ + int neededbits = 8 + ((-pending.BitCount) & 7); + while (neededbits > 0) + { + /* write a static tree block consisting solely of + * an EOF: + */ + pending.WriteBits(2, 10); + neededbits -= 10; + } + } + state = BUSY_STATE; + break; + + case FINISHING_STATE: + pending.AlignToByte(); + + // Compressed data is complete. Write footer information if required. + if (!noZlibHeaderOrFooter) + { + int adler = engine.Adler; + pending.WriteShortMSB(adler >> 16); + pending.WriteShortMSB(adler & 0xffff); + } + state = FINISHED_STATE; + break; + } + } + } + return origLength - length; + } + + /// + /// Sets the dictionary which should be used in the deflate process. + /// This call is equivalent to setDictionary(dict, 0, dict.Length). + /// + /// + /// the dictionary. + /// + /// + /// if SetInput () or Deflate () were already called or another dictionary was already set. + /// + public void SetDictionary(byte[] dictionary) + { + SetDictionary(dictionary, 0, dictionary.Length); + } + + /// + /// Sets the dictionary which should be used in the deflate process. + /// The dictionary is a byte array containing strings that are + /// likely to occur in the data which should be compressed. The + /// dictionary is not stored in the compressed output, only a + /// checksum. To decompress the output you need to supply the same + /// dictionary again. + /// + /// + /// The dictionary data + /// + /// + /// The index where dictionary information commences. + /// + /// + /// The number of bytes in the dictionary. + /// + /// + /// If SetInput () or Deflate() were already called or another dictionary was already set. + /// + public void SetDictionary(byte[] dictionary, int index, int count) + { + if (state != INIT_STATE) + { + throw new InvalidOperationException(); + } + + state = SETDICT_STATE; + engine.SetDictionary(dictionary, index, count); + } + + #region Instance Fields + + /// + /// Compression level. + /// + private int level; + + /// + /// If true no Zlib/RFC1950 headers or footers are generated + /// + private bool noZlibHeaderOrFooter; + + /// + /// The current state. + /// + private int state; + + /// + /// The total bytes of output written. + /// + private long totalOut; + + /// + /// The pending output. + /// + private DeflaterPending pending; + + /// + /// The deflater engine. + /// + private DeflaterEngine engine; + + #endregion Instance Fields + } +} diff --git a/src/ImageSharp/Formats/Png/Zlib/DeflaterConstants.cs b/src/ImageSharp/Formats/Png/Zlib/DeflaterConstants.cs new file mode 100644 index 000000000..67e8c6900 --- /dev/null +++ b/src/ImageSharp/Formats/Png/Zlib/DeflaterConstants.cs @@ -0,0 +1,151 @@ +// Copyright (c) Six Labors and contributors. +// Licensed under the Apache License, Version 2.0. + +// +using System; +using System.Collections.Generic; +using System.Text; + +namespace SixLabors.ImageSharp.Formats.Png.Zlib +{ + /// + /// This class contains constants used for deflation. + /// + public static class DeflaterConstants + { + /// + /// Set to true to enable debugging + /// + public const bool DEBUGGING = false; + + /// + /// Written to Zip file to identify a stored block + /// + public const int STORED_BLOCK = 0; + + /// + /// Identifies static tree in Zip file + /// + public const int STATIC_TREES = 1; + + /// + /// Identifies dynamic tree in Zip file + /// + public const int DYN_TREES = 2; + + /// + /// Header flag indicating a preset dictionary for deflation + /// + public const int PRESET_DICT = 0x20; + + /// + /// Sets internal buffer sizes for Huffman encoding + /// + public const int DEFAULT_MEM_LEVEL = 8; + + /// + /// Internal compression engine constant + /// + public const int MAX_MATCH = 258; + + /// + /// Internal compression engine constant + /// + public const int MIN_MATCH = 3; + + /// + /// Internal compression engine constant + /// + public const int MAX_WBITS = 15; + + /// + /// Internal compression engine constant + /// + public const int WSIZE = 1 << MAX_WBITS; + + /// + /// Internal compression engine constant + /// + public const int WMASK = WSIZE - 1; + + /// + /// Internal compression engine constant + /// + public const int HASH_BITS = DEFAULT_MEM_LEVEL + 7; + + /// + /// Internal compression engine constant + /// + public const int HASH_SIZE = 1 << HASH_BITS; + + /// + /// Internal compression engine constant + /// + public const int HASH_MASK = HASH_SIZE - 1; + + /// + /// Internal compression engine constant + /// + public const int HASH_SHIFT = (HASH_BITS + MIN_MATCH - 1) / MIN_MATCH; + + /// + /// Internal compression engine constant + /// + public const int MIN_LOOKAHEAD = MAX_MATCH + MIN_MATCH + 1; + + /// + /// Internal compression engine constant + /// + public const int MAX_DIST = WSIZE - MIN_LOOKAHEAD; + + /// + /// Internal compression engine constant + /// + public const int PENDING_BUF_SIZE = 1 << (DEFAULT_MEM_LEVEL + 8); + + /// + /// Internal compression engine constant + /// + public static int MAX_BLOCK_SIZE = Math.Min(65535, PENDING_BUF_SIZE - 5); + + /// + /// Internal compression engine constant + /// + public const int DEFLATE_STORED = 0; + + /// + /// Internal compression engine constant + /// + public const int DEFLATE_FAST = 1; + + /// + /// Internal compression engine constant + /// + public const int DEFLATE_SLOW = 2; + + /// + /// Internal compression engine constant + /// + public static int[] GOOD_LENGTH = { 0, 4, 4, 4, 4, 8, 8, 8, 32, 32 }; + + /// + /// Internal compression engine constant + /// + public static int[] MAX_LAZY = { 0, 4, 5, 6, 4, 16, 16, 32, 128, 258 }; + + /// + /// Internal compression engine constant + /// + public static int[] NICE_LENGTH = { 0, 8, 16, 32, 16, 32, 128, 128, 258, 258 }; + + /// + /// Internal compression engine constant + /// + public static int[] MAX_CHAIN = { 0, 4, 8, 32, 16, 32, 128, 256, 1024, 4096 }; + + /// + /// Internal compression engine constant + /// + public static int[] COMPR_FUNC = { 0, 1, 1, 1, 1, 2, 2, 2, 2, 2 }; + } +} diff --git a/src/ImageSharp/Formats/Png/Zlib/DeflaterEngine.cs b/src/ImageSharp/Formats/Png/Zlib/DeflaterEngine.cs new file mode 100644 index 000000000..7ac5b6c69 --- /dev/null +++ b/src/ImageSharp/Formats/Png/Zlib/DeflaterEngine.cs @@ -0,0 +1,951 @@ +// Copyright (c) Six Labors and contributors. +// Licensed under the Apache License, Version 2.0. + +// +using System; +using System.Collections.Generic; +using System.Text; + +namespace SixLabors.ImageSharp.Formats.Png.Zlib +{ + /// + /// Strategies for deflater + /// + public enum DeflateStrategy + { + /// + /// The default strategy + /// + Default = 0, + + /// + /// This strategy will only allow longer string repetitions. It is + /// useful for random data with a small character set. + /// + Filtered = 1, + + /// + /// This strategy will not look for string repetitions at all. It + /// only encodes with Huffman trees (which means, that more common + /// characters get a smaller encoding. + /// + HuffmanOnly = 2 + } + + // DEFLATE ALGORITHM: + // + // The uncompressed stream is inserted into the window array. When + // the window array is full the first half is thrown away and the + // second half is copied to the beginning. + // + // The head array is a hash table. Three characters build a hash value + // and they the value points to the corresponding index in window of + // the last string with this hash. The prev array implements a + // linked list of matches with the same hash: prev[index & WMASK] points + // to the previous index with the same hash. + // + + /// + /// Low level compression engine for deflate algorithm which uses a 32K sliding window + /// with secondary compression from Huffman/Shannon-Fano codes. + /// + public class DeflaterEngine + { + #region Constants + + private const int TooFar = 4096; + + #endregion Constants + + #region Constructors + + /// + /// Construct instance with pending buffer + /// Adler calculation will be peformed + /// + /// + /// Pending buffer to use + /// + public DeflaterEngine(DeflaterPending pending) + : this(pending, false) + { + } + + + + /// + /// Construct instance with pending buffer + /// + /// + /// Pending buffer to use + /// + /// + /// If no adler calculation should be performed + /// + public DeflaterEngine(DeflaterPending pending, bool noAdlerCalculation) + { + this.pending = pending; + huffman = new DeflaterHuffman(pending); + if (!noAdlerCalculation) + adler = new Adler32(); + + window = new byte[2 * DeflaterConstants.WSIZE]; + head = new short[DeflaterConstants.HASH_SIZE]; + prev = new short[DeflaterConstants.WSIZE]; + + // We start at index 1, to avoid an implementation deficiency, that + // we cannot build a repeat pattern at index 0. + blockStart = strstart = 1; + } + + #endregion Constructors + + /// + /// Deflate drives actual compression of data + /// + /// True to flush input buffers + /// Finish deflation with the current input. + /// Returns true if progress has been made. + public bool Deflate(bool flush, bool finish) + { + bool progress; + do + { + FillWindow(); + bool canFlush = flush && (inputOff == inputEnd); + +#if DebugDeflation + if (DeflaterConstants.DEBUGGING) { + Console.WriteLine("window: [" + blockStart + "," + strstart + "," + + lookahead + "], " + compressionFunction + "," + canFlush); + } +#endif + switch (compressionFunction) + { + case DeflaterConstants.DEFLATE_STORED: + progress = DeflateStored(canFlush, finish); + break; + + case DeflaterConstants.DEFLATE_FAST: + progress = DeflateFast(canFlush, finish); + break; + + case DeflaterConstants.DEFLATE_SLOW: + progress = DeflateSlow(canFlush, finish); + break; + + default: + throw new InvalidOperationException("unknown compressionFunction"); + } + } while (pending.IsFlushed && progress); // repeat while we have no pending output and progress was made + return progress; + } + + /// + /// Sets input data to be deflated. Should only be called when NeedsInput() + /// returns true + /// + /// The buffer containing input data. + /// The offset of the first byte of data. + /// The number of bytes of data to use as input. + public void SetInput(byte[] buffer, int offset, int count) + { + if (buffer == null) + { + throw new ArgumentNullException(nameof(buffer)); + } + + if (offset < 0) + { + throw new ArgumentOutOfRangeException(nameof(offset)); + } + + if (count < 0) + { + throw new ArgumentOutOfRangeException(nameof(count)); + } + + if (inputOff < inputEnd) + { + throw new InvalidOperationException("Old input was not completely processed"); + } + + int end = offset + count; + + /* We want to throw an ArrayIndexOutOfBoundsException early. The + * check is very tricky: it also handles integer wrap around. + */ + if ((offset > end) || (end > buffer.Length)) + { + throw new ArgumentOutOfRangeException(nameof(count)); + } + + inputBuf = buffer; + inputOff = offset; + inputEnd = end; + } + + /// + /// Determines if more input is needed. + /// + /// Return true if input is needed via SetInput + public bool NeedsInput() + { + return (inputEnd == inputOff); + } + + /// + /// Set compression dictionary + /// + /// The buffer containing the dictionary data + /// The offset in the buffer for the first byte of data + /// The length of the dictionary data. + public void SetDictionary(byte[] buffer, int offset, int length) + { +#if DebugDeflation + if (DeflaterConstants.DEBUGGING && (strstart != 1) ) + { + throw new InvalidOperationException("strstart not 1"); + } +#endif + adler?.Update(new ArraySegment(buffer, offset, length)); + if (length < DeflaterConstants.MIN_MATCH) + { + return; + } + + if (length > DeflaterConstants.MAX_DIST) + { + offset += length - DeflaterConstants.MAX_DIST; + length = DeflaterConstants.MAX_DIST; + } + + System.Array.Copy(buffer, offset, window, strstart, length); + + UpdateHash(); + --length; + while (--length > 0) + { + InsertString(); + strstart++; + } + strstart += 2; + blockStart = strstart; + } + + /// + /// Reset internal state + /// + public void Reset() + { + huffman.Reset(); + adler?.Reset(); + blockStart = strstart = 1; + lookahead = 0; + totalIn = 0; + prevAvailable = false; + matchLen = DeflaterConstants.MIN_MATCH - 1; + + for (int i = 0; i < DeflaterConstants.HASH_SIZE; i++) + { + head[i] = 0; + } + + for (int i = 0; i < DeflaterConstants.WSIZE; i++) + { + prev[i] = 0; + } + } + + /// + /// Reset Adler checksum + /// + public void ResetAdler() + { + adler?.Reset(); + } + + /// + /// Get current value of Adler checksum + /// + public int Adler + { + get + { + return (adler != null) ? unchecked((int)adler.Value) : 0; + } + } + + /// + /// Total data processed + /// + public long TotalIn + { + get + { + return totalIn; + } + } + + /// + /// Get/set the deflate strategy + /// + public DeflateStrategy Strategy + { + get + { + return strategy; + } + set + { + strategy = value; + } + } + + /// + /// Set the deflate level (0-9) + /// + /// The value to set the level to. + public void SetLevel(int level) + { + if ((level < 0) || (level > 9)) + { + throw new ArgumentOutOfRangeException(nameof(level)); + } + + goodLength = DeflaterConstants.GOOD_LENGTH[level]; + max_lazy = DeflaterConstants.MAX_LAZY[level]; + niceLength = DeflaterConstants.NICE_LENGTH[level]; + max_chain = DeflaterConstants.MAX_CHAIN[level]; + + if (DeflaterConstants.COMPR_FUNC[level] != compressionFunction) + { +#if DebugDeflation + if (DeflaterConstants.DEBUGGING) { + Console.WriteLine("Change from " + compressionFunction + " to " + + DeflaterConstants.COMPR_FUNC[level]); + } +#endif + switch (compressionFunction) + { + case DeflaterConstants.DEFLATE_STORED: + if (strstart > blockStart) + { + huffman.FlushStoredBlock(window, blockStart, + strstart - blockStart, false); + blockStart = strstart; + } + UpdateHash(); + break; + + case DeflaterConstants.DEFLATE_FAST: + if (strstart > blockStart) + { + huffman.FlushBlock(window, blockStart, strstart - blockStart, + false); + blockStart = strstart; + } + break; + + case DeflaterConstants.DEFLATE_SLOW: + if (prevAvailable) + { + huffman.TallyLit(window[strstart - 1] & 0xff); + } + if (strstart > blockStart) + { + huffman.FlushBlock(window, blockStart, strstart - blockStart, false); + blockStart = strstart; + } + prevAvailable = false; + matchLen = DeflaterConstants.MIN_MATCH - 1; + break; + } + compressionFunction = DeflaterConstants.COMPR_FUNC[level]; + } + } + + /// + /// Fill the window + /// + public void FillWindow() + { + /* If the window is almost full and there is insufficient lookahead, + * move the upper half to the lower one to make room in the upper half. + */ + if (strstart >= DeflaterConstants.WSIZE + DeflaterConstants.MAX_DIST) + { + SlideWindow(); + } + + /* If there is not enough lookahead, but still some input left, + * read in the input + */ + if (lookahead < DeflaterConstants.MIN_LOOKAHEAD && inputOff < inputEnd) + { + int more = 2 * DeflaterConstants.WSIZE - lookahead - strstart; + + if (more > inputEnd - inputOff) + { + more = inputEnd - inputOff; + } + + System.Array.Copy(inputBuf, inputOff, window, strstart + lookahead, more); + adler?.Update(new ArraySegment(inputBuf, inputOff, more)); + + inputOff += more; + totalIn += more; + lookahead += more; + } + + if (lookahead >= DeflaterConstants.MIN_MATCH) + { + UpdateHash(); + } + } + + private void UpdateHash() + { + /* + if (DEBUGGING) { + Console.WriteLine("updateHash: "+strstart); + } + */ + ins_h = (window[strstart] << DeflaterConstants.HASH_SHIFT) ^ window[strstart + 1]; + } + + /// + /// Inserts the current string in the head hash and returns the previous + /// value for this hash. + /// + /// The previous hash value + private int InsertString() + { + short match; + int hash = ((ins_h << DeflaterConstants.HASH_SHIFT) ^ window[strstart + (DeflaterConstants.MIN_MATCH - 1)]) & DeflaterConstants.HASH_MASK; + +#if DebugDeflation + if (DeflaterConstants.DEBUGGING) + { + if (hash != (((window[strstart] << (2*HASH_SHIFT)) ^ + (window[strstart + 1] << HASH_SHIFT) ^ + (window[strstart + 2])) & HASH_MASK)) { + throw new ImageFormatException("hash inconsistent: " + hash + "/" + +window[strstart] + "," + +window[strstart + 1] + "," + +window[strstart + 2] + "," + HASH_SHIFT); + } + } +#endif + prev[strstart & DeflaterConstants.WMASK] = match = head[hash]; + head[hash] = unchecked((short)strstart); + ins_h = hash; + return match & 0xffff; + } + + private void SlideWindow() + { + Array.Copy(window, DeflaterConstants.WSIZE, window, 0, DeflaterConstants.WSIZE); + matchStart -= DeflaterConstants.WSIZE; + strstart -= DeflaterConstants.WSIZE; + blockStart -= DeflaterConstants.WSIZE; + + // Slide the hash table (could be avoided with 32 bit values + // at the expense of memory usage). + for (int i = 0; i < DeflaterConstants.HASH_SIZE; ++i) + { + int m = head[i] & 0xffff; + head[i] = (short)(m >= DeflaterConstants.WSIZE ? (m - DeflaterConstants.WSIZE) : 0); + } + + // Slide the prev table. + for (int i = 0; i < DeflaterConstants.WSIZE; i++) + { + int m = prev[i] & 0xffff; + prev[i] = (short)(m >= DeflaterConstants.WSIZE ? (m - DeflaterConstants.WSIZE) : 0); + } + } + + /// + /// Find the best (longest) string in the window matching the + /// string starting at strstart. + /// + /// Preconditions: + /// + /// strstart + DeflaterConstants.MAX_MATCH <= window.length. + /// + /// + /// True if a match greater than the minimum length is found + private bool FindLongestMatch(int curMatch) + { + int match; + int scan = strstart; + // scanMax is the highest position that we can look at + int scanMax = scan + Math.Min(DeflaterConstants.MAX_MATCH, lookahead) - 1; + int limit = Math.Max(scan - DeflaterConstants.MAX_DIST, 0); + + byte[] window = this.window; + short[] prev = this.prev; + int chainLength = this.max_chain; + int niceLength = Math.Min(this.niceLength, lookahead); + + matchLen = Math.Max(matchLen, DeflaterConstants.MIN_MATCH - 1); + + if (scan + matchLen > scanMax) return false; + + byte scan_end1 = window[scan + matchLen - 1]; + byte scan_end = window[scan + matchLen]; + + // Do not waste too much time if we already have a good match: + if (matchLen >= this.goodLength) chainLength >>= 2; + + do + { + match = curMatch; + scan = strstart; + + if (window[match + matchLen] != scan_end + || window[match + matchLen - 1] != scan_end1 + || window[match] != window[scan] + || window[++match] != window[++scan]) + { + continue; + } + + // scan is set to strstart+1 and the comparison passed, so + // scanMax - scan is the maximum number of bytes we can compare. + // below we compare 8 bytes at a time, so first we compare + // (scanMax - scan) % 8 bytes, so the remainder is a multiple of 8 + + switch ((scanMax - scan) % 8) + { + case 1: + if (window[++scan] == window[++match]) break; + break; + + case 2: + if (window[++scan] == window[++match] + && window[++scan] == window[++match]) break; + break; + + case 3: + if (window[++scan] == window[++match] + && window[++scan] == window[++match] + && window[++scan] == window[++match]) break; + break; + + case 4: + if (window[++scan] == window[++match] + && window[++scan] == window[++match] + && window[++scan] == window[++match] + && window[++scan] == window[++match]) break; + break; + + case 5: + if (window[++scan] == window[++match] + && window[++scan] == window[++match] + && window[++scan] == window[++match] + && window[++scan] == window[++match] + && window[++scan] == window[++match]) break; + break; + + case 6: + if (window[++scan] == window[++match] + && window[++scan] == window[++match] + && window[++scan] == window[++match] + && window[++scan] == window[++match] + && window[++scan] == window[++match] + && window[++scan] == window[++match]) break; + break; + + case 7: + if (window[++scan] == window[++match] + && window[++scan] == window[++match] + && window[++scan] == window[++match] + && window[++scan] == window[++match] + && window[++scan] == window[++match] + && window[++scan] == window[++match] + && window[++scan] == window[++match]) break; + break; + } + + if (window[scan] == window[match]) + { + /* We check for insufficient lookahead only every 8th comparison; + * the 256th check will be made at strstart + 258 unless lookahead is + * exhausted first. + */ + do + { + if (scan == scanMax) + { + ++scan; // advance to first position not matched + ++match; + + break; + } + } + while (window[++scan] == window[++match] + && window[++scan] == window[++match] + && window[++scan] == window[++match] + && window[++scan] == window[++match] + && window[++scan] == window[++match] + && window[++scan] == window[++match] + && window[++scan] == window[++match] + && window[++scan] == window[++match]); + } + + if (scan - strstart > matchLen) + { +#if DebugDeflation + if (DeflaterConstants.DEBUGGING && (ins_h == 0) ) + Console.Error.WriteLine("Found match: " + curMatch + "-" + (scan - strstart)); +#endif + + matchStart = curMatch; + matchLen = scan - strstart; + + if (matchLen >= niceLength) + break; + + scan_end1 = window[scan - 1]; + scan_end = window[scan]; + } + } while ((curMatch = (prev[curMatch & DeflaterConstants.WMASK] & 0xffff)) > limit && 0 != --chainLength); + + return matchLen >= DeflaterConstants.MIN_MATCH; + } + + private bool DeflateStored(bool flush, bool finish) + { + if (!flush && (lookahead == 0)) + { + return false; + } + + strstart += lookahead; + lookahead = 0; + + int storedLength = strstart - blockStart; + + if ((storedLength >= DeflaterConstants.MAX_BLOCK_SIZE) || // Block is full + (blockStart < DeflaterConstants.WSIZE && storedLength >= DeflaterConstants.MAX_DIST) || // Block may move out of window + flush) + { + bool lastBlock = finish; + if (storedLength > DeflaterConstants.MAX_BLOCK_SIZE) + { + storedLength = DeflaterConstants.MAX_BLOCK_SIZE; + lastBlock = false; + } + +#if DebugDeflation + if (DeflaterConstants.DEBUGGING) + { + Console.WriteLine("storedBlock[" + storedLength + "," + lastBlock + "]"); + } +#endif + + huffman.FlushStoredBlock(window, blockStart, storedLength, lastBlock); + blockStart += storedLength; + return !(lastBlock || storedLength == 0); + } + return true; + } + + private bool DeflateFast(bool flush, bool finish) + { + if (lookahead < DeflaterConstants.MIN_LOOKAHEAD && !flush) + { + return false; + } + + while (lookahead >= DeflaterConstants.MIN_LOOKAHEAD || flush) + { + if (lookahead == 0) + { + // We are flushing everything + huffman.FlushBlock(window, blockStart, strstart - blockStart, finish); + blockStart = strstart; + return false; + } + + if (strstart > 2 * DeflaterConstants.WSIZE - DeflaterConstants.MIN_LOOKAHEAD) + { + /* slide window, as FindLongestMatch needs this. + * This should only happen when flushing and the window + * is almost full. + */ + SlideWindow(); + } + + int hashHead; + if (lookahead >= DeflaterConstants.MIN_MATCH && + (hashHead = InsertString()) != 0 && + strategy != DeflateStrategy.HuffmanOnly && + strstart - hashHead <= DeflaterConstants.MAX_DIST && + FindLongestMatch(hashHead)) + { + // longestMatch sets matchStart and matchLen +#if DebugDeflation + if (DeflaterConstants.DEBUGGING) + { + for (int i = 0 ; i < matchLen; i++) { + if (window[strstart + i] != window[matchStart + i]) { + throw new ImageFormatException("Match failure"); + } + } + } +#endif + + bool full = huffman.TallyDist(strstart - matchStart, matchLen); + + lookahead -= matchLen; + if (matchLen <= max_lazy && lookahead >= DeflaterConstants.MIN_MATCH) + { + while (--matchLen > 0) + { + ++strstart; + InsertString(); + } + ++strstart; + } + else + { + strstart += matchLen; + if (lookahead >= DeflaterConstants.MIN_MATCH - 1) + { + UpdateHash(); + } + } + matchLen = DeflaterConstants.MIN_MATCH - 1; + if (!full) + { + continue; + } + } + else + { + // No match found + huffman.TallyLit(window[strstart] & 0xff); + ++strstart; + --lookahead; + } + + if (huffman.IsFull()) + { + bool lastBlock = finish && (lookahead == 0); + huffman.FlushBlock(window, blockStart, strstart - blockStart, lastBlock); + blockStart = strstart; + return !lastBlock; + } + } + return true; + } + + private bool DeflateSlow(bool flush, bool finish) + { + if (lookahead < DeflaterConstants.MIN_LOOKAHEAD && !flush) + { + return false; + } + + while (lookahead >= DeflaterConstants.MIN_LOOKAHEAD || flush) + { + if (lookahead == 0) + { + if (prevAvailable) + { + huffman.TallyLit(window[strstart - 1] & 0xff); + } + prevAvailable = false; + + // We are flushing everything +#if DebugDeflation + if (DeflaterConstants.DEBUGGING && !flush) + { + throw new ImageFormatException("Not flushing, but no lookahead"); + } +#endif + huffman.FlushBlock(window, blockStart, strstart - blockStart, + finish); + blockStart = strstart; + return false; + } + + if (strstart >= 2 * DeflaterConstants.WSIZE - DeflaterConstants.MIN_LOOKAHEAD) + { + /* slide window, as FindLongestMatch needs this. + * This should only happen when flushing and the window + * is almost full. + */ + SlideWindow(); + } + + int prevMatch = matchStart; + int prevLen = matchLen; + if (lookahead >= DeflaterConstants.MIN_MATCH) + { + int hashHead = InsertString(); + + if (strategy != DeflateStrategy.HuffmanOnly && + hashHead != 0 && + strstart - hashHead <= DeflaterConstants.MAX_DIST && + FindLongestMatch(hashHead)) + { + // longestMatch sets matchStart and matchLen + + // Discard match if too small and too far away + if (matchLen <= 5 && (strategy == DeflateStrategy.Filtered || (matchLen == DeflaterConstants.MIN_MATCH && strstart - matchStart > TooFar))) + { + matchLen = DeflaterConstants.MIN_MATCH - 1; + } + } + } + + // previous match was better + if ((prevLen >= DeflaterConstants.MIN_MATCH) && (matchLen <= prevLen)) + { +#if DebugDeflation + if (DeflaterConstants.DEBUGGING) + { + for (int i = 0 ; i < matchLen; i++) { + if (window[strstart-1+i] != window[prevMatch + i]) + throw new ImageFormatException(); + } + } +#endif + huffman.TallyDist(strstart - 1 - prevMatch, prevLen); + prevLen -= 2; + do + { + strstart++; + lookahead--; + if (lookahead >= DeflaterConstants.MIN_MATCH) + { + InsertString(); + } + } while (--prevLen > 0); + + strstart++; + lookahead--; + prevAvailable = false; + matchLen = DeflaterConstants.MIN_MATCH - 1; + } + else + { + if (prevAvailable) + { + huffman.TallyLit(window[strstart - 1] & 0xff); + } + prevAvailable = true; + strstart++; + lookahead--; + } + + if (huffman.IsFull()) + { + int len = strstart - blockStart; + if (prevAvailable) + { + len--; + } + bool lastBlock = (finish && (lookahead == 0) && !prevAvailable); + huffman.FlushBlock(window, blockStart, len, lastBlock); + blockStart += len; + return !lastBlock; + } + } + return true; + } + + #region Instance Fields + + // Hash index of string to be inserted + private int ins_h; + + /// + /// Hashtable, hashing three characters to an index for window, so + /// that window[index]..window[index+2] have this hash code. + /// Note that the array should really be unsigned short, so you need + /// to and the values with 0xffff. + /// + private short[] head; + + /// + /// prev[index & WMASK] points to the previous index that has the + /// same hash code as the string starting at index. This way + /// entries with the same hash code are in a linked list. + /// Note that the array should really be unsigned short, so you need + /// to and the values with 0xffff. + /// + private short[] prev; + + private int matchStart; + + // Length of best match + private int matchLen; + + // Set if previous match exists + private bool prevAvailable; + + private int blockStart; + + /// + /// Points to the current character in the window. + /// + private int strstart; + + /// + /// lookahead is the number of characters starting at strstart in + /// window that are valid. + /// So window[strstart] until window[strstart+lookahead-1] are valid + /// characters. + /// + private int lookahead; + + /// + /// This array contains the part of the uncompressed stream that + /// is of relevance. The current character is indexed by strstart. + /// + private byte[] window; + + private DeflateStrategy strategy; + private int max_chain, max_lazy, niceLength, goodLength; + + /// + /// The current compression function. + /// + private int compressionFunction; + + /// + /// The input data for compression. + /// + private byte[] inputBuf; + + /// + /// The total bytes of input read. + /// + private long totalIn; + + /// + /// The offset into inputBuf, where input data starts. + /// + private int inputOff; + + /// + /// The end offset of the input data. + /// + private int inputEnd; + + private DeflaterPending pending; + private DeflaterHuffman huffman; + + /// + /// The adler checksum + /// + private Adler32 adler; + + #endregion Instance Fields + } +} diff --git a/src/ImageSharp/Formats/Png/Zlib/DeflaterHuffman.cs b/src/ImageSharp/Formats/Png/Zlib/DeflaterHuffman.cs new file mode 100644 index 000000000..bf506d14c --- /dev/null +++ b/src/ImageSharp/Formats/Png/Zlib/DeflaterHuffman.cs @@ -0,0 +1,965 @@ +// Copyright (c) Six Labors and contributors. +// Licensed under the Apache License, Version 2.0. + +// +using System; +using System.Collections.Generic; +using System.Text; + +namespace SixLabors.ImageSharp.Formats.Png.Zlib +{ + /// + /// This is the DeflaterHuffman class. + /// + /// This class is not thread safe. This is inherent in the API, due + /// to the split of Deflate and SetInput. + /// + /// author of the original java version : Jochen Hoenicke + /// + public class DeflaterHuffman + { + private const int BUFSIZE = 1 << (DeflaterConstants.DEFAULT_MEM_LEVEL + 6); + private const int LITERAL_NUM = 286; + + // Number of distance codes + private const int DIST_NUM = 30; + + // Number of codes used to transfer bit lengths + private const int BITLEN_NUM = 19; + + // repeat previous bit length 3-6 times (2 bits of repeat count) + private const int REP_3_6 = 16; + + // repeat a zero length 3-10 times (3 bits of repeat count) + private const int REP_3_10 = 17; + + // repeat a zero length 11-138 times (7 bits of repeat count) + private const int REP_11_138 = 18; + + private const int EOF_SYMBOL = 256; + + // The lengths of the bit length codes are sent in order of decreasing + // probability, to avoid transmitting the lengths for unused bit length codes. + private static readonly int[] BL_ORDER = { 16, 17, 18, 0, 8, 7, 9, 6, 10, 5, 11, 4, 12, 3, 13, 2, 14, 1, 15 }; + + private static readonly byte[] bit4Reverse = { + 0, + 8, + 4, + 12, + 2, + 10, + 6, + 14, + 1, + 9, + 5, + 13, + 3, + 11, + 7, + 15 + }; + + private static short[] staticLCodes; + private static byte[] staticLLength; + private static short[] staticDCodes; + private static byte[] staticDLength; + + private class Tree + { + #region Instance Fields + + public short[] freqs; + + public byte[] length; + + public int minNumCodes; + + public int numCodes; + + private short[] codes; + private readonly int[] bl_counts; + private readonly int maxLength; + private DeflaterHuffman dh; + + #endregion Instance Fields + + #region Constructors + + public Tree(DeflaterHuffman dh, int elems, int minCodes, int maxLength) + { + this.dh = dh; + this.minNumCodes = minCodes; + this.maxLength = maxLength; + freqs = new short[elems]; + bl_counts = new int[maxLength]; + } + + #endregion Constructors + + /// + /// Resets the internal state of the tree + /// + public void Reset() + { + for (int i = 0; i < freqs.Length; i++) + { + freqs[i] = 0; + } + codes = null; + length = null; + } + + public void WriteSymbol(int code) + { + // if (DeflaterConstants.DEBUGGING) { + // freqs[code]--; + // // Console.Write("writeSymbol("+freqs.length+","+code+"): "); + // } + dh.pending.WriteBits(codes[code] & 0xffff, length[code]); + } + + /// + /// Check that all frequencies are zero + /// + /// + /// At least one frequency is non-zero + /// + public void CheckEmpty() + { + bool empty = true; + for (int i = 0; i < freqs.Length; i++) + { + empty &= freqs[i] == 0; + } + + if (!empty) + { + throw new ImageFormatException("!Empty"); + } + } + + /// + /// Set static codes and length + /// + /// new codes + /// length for new codes + public void SetStaticCodes(short[] staticCodes, byte[] staticLengths) + { + codes = staticCodes; + length = staticLengths; + } + + /// + /// Build dynamic codes and lengths + /// + public void BuildCodes() + { + int numSymbols = freqs.Length; + int[] nextCode = new int[maxLength]; + int code = 0; + + codes = new short[freqs.Length]; + + // if (DeflaterConstants.DEBUGGING) { + // //Console.WriteLine("buildCodes: "+freqs.Length); + // } + + for (int bits = 0; bits < maxLength; bits++) + { + nextCode[bits] = code; + code += bl_counts[bits] << (15 - bits); + + // if (DeflaterConstants.DEBUGGING) { + // //Console.WriteLine("bits: " + ( bits + 1) + " count: " + bl_counts[bits] + // +" nextCode: "+code); + // } + } + +#if DebugDeflation + if ( DeflaterConstants.DEBUGGING && (code != 65536) ) + { + throw new ImageFormatException("Inconsistent bl_counts!"); + } +#endif + for (int i = 0; i < numCodes; i++) + { + int bits = length[i]; + if (bits > 0) + { + // if (DeflaterConstants.DEBUGGING) { + // //Console.WriteLine("codes["+i+"] = rev(" + nextCode[bits-1]+"), + // +bits); + // } + + codes[i] = BitReverse(nextCode[bits - 1]); + nextCode[bits - 1] += 1 << (16 - bits); + } + } + } + + public void BuildTree() + { + int numSymbols = freqs.Length; + + /* heap is a priority queue, sorted by frequency, least frequent + * nodes first. The heap is a binary tree, with the property, that + * the parent node is smaller than both child nodes. This assures + * that the smallest node is the first parent. + * + * The binary tree is encoded in an array: 0 is root node and + * the nodes 2*n+1, 2*n+2 are the child nodes of node n. + */ + int[] heap = new int[numSymbols]; + int heapLen = 0; + int maxCode = 0; + for (int n = 0; n < numSymbols; n++) + { + int freq = freqs[n]; + if (freq != 0) + { + // Insert n into heap + int pos = heapLen++; + int ppos; + while (pos > 0 && freqs[heap[ppos = (pos - 1) / 2]] > freq) + { + heap[pos] = heap[ppos]; + pos = ppos; + } + heap[pos] = n; + + maxCode = n; + } + } + + /* We could encode a single literal with 0 bits but then we + * don't see the literals. Therefore we force at least two + * literals to avoid this case. We don't care about order in + * this case, both literals get a 1 bit code. + */ + while (heapLen < 2) + { + int node = maxCode < 2 ? ++maxCode : 0; + heap[heapLen++] = node; + } + + numCodes = Math.Max(maxCode + 1, minNumCodes); + + int numLeafs = heapLen; + int[] childs = new int[4 * heapLen - 2]; + int[] values = new int[2 * heapLen - 1]; + int numNodes = numLeafs; + for (int i = 0; i < heapLen; i++) + { + int node = heap[i]; + childs[2 * i] = node; + childs[2 * i + 1] = -1; + values[i] = freqs[node] << 8; + heap[i] = i; + } + + /* Construct the Huffman tree by repeatedly combining the least two + * frequent nodes. + */ + do + { + int first = heap[0]; + int last = heap[--heapLen]; + + // Propagate the hole to the leafs of the heap + int ppos = 0; + int path = 1; + + while (path < heapLen) + { + if (path + 1 < heapLen && values[heap[path]] > values[heap[path + 1]]) + { + path++; + } + + heap[ppos] = heap[path]; + ppos = path; + path = path * 2 + 1; + } + + /* Now propagate the last element down along path. Normally + * it shouldn't go too deep. + */ + int lastVal = values[last]; + while ((path = ppos) > 0 && values[heap[ppos = (path - 1) / 2]] > lastVal) + { + heap[path] = heap[ppos]; + } + heap[path] = last; + + int second = heap[0]; + + // Create a new node father of first and second + last = numNodes++; + childs[2 * last] = first; + childs[2 * last + 1] = second; + int mindepth = Math.Min(values[first] & 0xff, values[second] & 0xff); + values[last] = lastVal = values[first] + values[second] - mindepth + 1; + + // Again, propagate the hole to the leafs + ppos = 0; + path = 1; + + while (path < heapLen) + { + if (path + 1 < heapLen && values[heap[path]] > values[heap[path + 1]]) + { + path++; + } + + heap[ppos] = heap[path]; + ppos = path; + path = ppos * 2 + 1; + } + + // Now propagate the new element down along path + while ((path = ppos) > 0 && values[heap[ppos = (path - 1) / 2]] > lastVal) + { + heap[path] = heap[ppos]; + } + heap[path] = last; + } while (heapLen > 1); + + if (heap[0] != childs.Length / 2 - 1) + { + throw new ImageFormatException("Heap invariant violated"); + } + + BuildLength(childs); + } + + /// + /// Get encoded length + /// + /// Encoded length, the sum of frequencies * lengths + public int GetEncodedLength() + { + int len = 0; + for (int i = 0; i < freqs.Length; i++) + { + len += freqs[i] * length[i]; + } + return len; + } + + /// + /// Scan a literal or distance tree to determine the frequencies of the codes + /// in the bit length tree. + /// + public void CalcBLFreq(Tree blTree) + { + int max_count; /* max repeat count */ + int min_count; /* min repeat count */ + int count; /* repeat count of the current code */ + int curlen = -1; /* length of current code */ + + int i = 0; + while (i < numCodes) + { + count = 1; + int nextlen = length[i]; + if (nextlen == 0) + { + max_count = 138; + min_count = 3; + } + else + { + max_count = 6; + min_count = 3; + if (curlen != nextlen) + { + blTree.freqs[nextlen]++; + count = 0; + } + } + curlen = nextlen; + i++; + + while (i < numCodes && curlen == length[i]) + { + i++; + if (++count >= max_count) + { + break; + } + } + + if (count < min_count) + { + blTree.freqs[curlen] += (short)count; + } + else if (curlen != 0) + { + blTree.freqs[REP_3_6]++; + } + else if (count <= 10) + { + blTree.freqs[REP_3_10]++; + } + else + { + blTree.freqs[REP_11_138]++; + } + } + } + + /// + /// Write tree values + /// + /// Tree to write + public void WriteTree(Tree blTree) + { + int max_count; // max repeat count + int min_count; // min repeat count + int count; // repeat count of the current code + int curlen = -1; // length of current code + + int i = 0; + while (i < numCodes) + { + count = 1; + int nextlen = length[i]; + if (nextlen == 0) + { + max_count = 138; + min_count = 3; + } + else + { + max_count = 6; + min_count = 3; + if (curlen != nextlen) + { + blTree.WriteSymbol(nextlen); + count = 0; + } + } + curlen = nextlen; + i++; + + while (i < numCodes && curlen == length[i]) + { + i++; + if (++count >= max_count) + { + break; + } + } + + if (count < min_count) + { + while (count-- > 0) + { + blTree.WriteSymbol(curlen); + } + } + else if (curlen != 0) + { + blTree.WriteSymbol(REP_3_6); + dh.pending.WriteBits(count - 3, 2); + } + else if (count <= 10) + { + blTree.WriteSymbol(REP_3_10); + dh.pending.WriteBits(count - 3, 3); + } + else + { + blTree.WriteSymbol(REP_11_138); + dh.pending.WriteBits(count - 11, 7); + } + } + } + + private void BuildLength(int[] childs) + { + this.length = new byte[freqs.Length]; + int numNodes = childs.Length / 2; + int numLeafs = (numNodes + 1) / 2; + int overflow = 0; + + for (int i = 0; i < maxLength; i++) + { + bl_counts[i] = 0; + } + + // First calculate optimal bit lengths + int[] lengths = new int[numNodes]; + lengths[numNodes - 1] = 0; + + for (int i = numNodes - 1; i >= 0; i--) + { + if (childs[2 * i + 1] != -1) + { + int bitLength = lengths[i] + 1; + if (bitLength > maxLength) + { + bitLength = maxLength; + overflow++; + } + lengths[childs[2 * i]] = lengths[childs[2 * i + 1]] = bitLength; + } + else + { + // A leaf node + int bitLength = lengths[i]; + bl_counts[bitLength - 1]++; + this.length[childs[2 * i]] = (byte)lengths[i]; + } + } + + // if (DeflaterConstants.DEBUGGING) { + // //Console.WriteLine("Tree "+freqs.Length+" lengths:"); + // for (int i=0; i < numLeafs; i++) { + // //Console.WriteLine("Node "+childs[2*i]+" freq: "+freqs[childs[2*i]] + // + " len: "+length[childs[2*i]]); + // } + // } + + if (overflow == 0) + { + return; + } + + int incrBitLen = maxLength - 1; + do + { + // Find the first bit length which could increase: + while (bl_counts[--incrBitLen] == 0) + { + } + + // Move this node one down and remove a corresponding + // number of overflow nodes. + do + { + bl_counts[incrBitLen]--; + bl_counts[++incrBitLen]++; + overflow -= 1 << (maxLength - 1 - incrBitLen); + } while (overflow > 0 && incrBitLen < maxLength - 1); + } while (overflow > 0); + + /* We may have overshot above. Move some nodes from maxLength to + * maxLength-1 in that case. + */ + bl_counts[maxLength - 1] += overflow; + bl_counts[maxLength - 2] -= overflow; + + /* Now recompute all bit lengths, scanning in increasing + * frequency. It is simpler to reconstruct all lengths instead of + * fixing only the wrong ones. This idea is taken from 'ar' + * written by Haruhiko Okumura. + * + * The nodes were inserted with decreasing frequency into the childs + * array. + */ + int nodePtr = 2 * numLeafs; + for (int bits = maxLength; bits != 0; bits--) + { + int n = bl_counts[bits - 1]; + while (n > 0) + { + int childPtr = 2 * childs[nodePtr++]; + if (childs[childPtr + 1] == -1) + { + // We found another leaf + length[childs[childPtr]] = (byte)bits; + n--; + } + } + } + // if (DeflaterConstants.DEBUGGING) { + // //Console.WriteLine("*** After overflow elimination. ***"); + // for (int i=0; i < numLeafs; i++) { + // //Console.WriteLine("Node "+childs[2*i]+" freq: "+freqs[childs[2*i]] + // + " len: "+length[childs[2*i]]); + // } + // } + } + } + + #region Instance Fields + + /// + /// Pending buffer to use + /// + public DeflaterPending pending; + + private Tree literalTree; + private Tree distTree; + private Tree blTree; + + // Buffer for distances + private short[] d_buf; + + private byte[] l_buf; + private int last_lit; + private int extra_bits; + + #endregion Instance Fields + + static DeflaterHuffman() + { + // See RFC 1951 3.2.6 + // Literal codes + staticLCodes = new short[LITERAL_NUM]; + staticLLength = new byte[LITERAL_NUM]; + + int i = 0; + while (i < 144) + { + staticLCodes[i] = BitReverse((0x030 + i) << 8); + staticLLength[i++] = 8; + } + + while (i < 256) + { + staticLCodes[i] = BitReverse((0x190 - 144 + i) << 7); + staticLLength[i++] = 9; + } + + while (i < 280) + { + staticLCodes[i] = BitReverse((0x000 - 256 + i) << 9); + staticLLength[i++] = 7; + } + + while (i < LITERAL_NUM) + { + staticLCodes[i] = BitReverse((0x0c0 - 280 + i) << 8); + staticLLength[i++] = 8; + } + + // Distance codes + staticDCodes = new short[DIST_NUM]; + staticDLength = new byte[DIST_NUM]; + for (i = 0; i < DIST_NUM; i++) + { + staticDCodes[i] = BitReverse(i << 11); + staticDLength[i] = 5; + } + } + + /// + /// Construct instance with pending buffer + /// + /// Pending buffer to use + public DeflaterHuffman(DeflaterPending pending) + { + this.pending = pending; + + literalTree = new Tree(this, LITERAL_NUM, 257, 15); + distTree = new Tree(this, DIST_NUM, 1, 15); + blTree = new Tree(this, BITLEN_NUM, 4, 7); + + d_buf = new short[BUFSIZE]; + l_buf = new byte[BUFSIZE]; + } + + /// + /// Reset internal state + /// + public void Reset() + { + last_lit = 0; + extra_bits = 0; + literalTree.Reset(); + distTree.Reset(); + blTree.Reset(); + } + + /// + /// Write all trees to pending buffer + /// + /// The number/rank of treecodes to send. + public void SendAllTrees(int blTreeCodes) + { + blTree.BuildCodes(); + literalTree.BuildCodes(); + distTree.BuildCodes(); + pending.WriteBits(literalTree.numCodes - 257, 5); + pending.WriteBits(distTree.numCodes - 1, 5); + pending.WriteBits(blTreeCodes - 4, 4); + for (int rank = 0; rank < blTreeCodes; rank++) + { + pending.WriteBits(blTree.length[BL_ORDER[rank]], 3); + } + literalTree.WriteTree(blTree); + distTree.WriteTree(blTree); + +#if DebugDeflation + if (DeflaterConstants.DEBUGGING) { + blTree.CheckEmpty(); + } +#endif + } + + /// + /// Compress current buffer writing data to pending buffer + /// + public void CompressBlock() + { + for (int i = 0; i < last_lit; i++) + { + int litlen = l_buf[i] & 0xff; + int dist = d_buf[i]; + if (dist-- != 0) + { + // if (DeflaterConstants.DEBUGGING) { + // Console.Write("["+(dist+1)+","+(litlen+3)+"]: "); + // } + + int lc = Lcode(litlen); + literalTree.WriteSymbol(lc); + + int bits = (lc - 261) / 4; + if (bits > 0 && bits <= 5) + { + pending.WriteBits(litlen & ((1 << bits) - 1), bits); + } + + int dc = Dcode(dist); + distTree.WriteSymbol(dc); + + bits = dc / 2 - 1; + if (bits > 0) + { + pending.WriteBits(dist & ((1 << bits) - 1), bits); + } + } + else + { + // if (DeflaterConstants.DEBUGGING) { + // if (litlen > 32 && litlen < 127) { + // Console.Write("("+(char)litlen+"): "); + // } else { + // Console.Write("{"+litlen+"}: "); + // } + // } + literalTree.WriteSymbol(litlen); + } + } + +#if DebugDeflation + if (DeflaterConstants.DEBUGGING) { + Console.Write("EOF: "); + } +#endif + literalTree.WriteSymbol(EOF_SYMBOL); + +#if DebugDeflation + if (DeflaterConstants.DEBUGGING) { + literalTree.CheckEmpty(); + distTree.CheckEmpty(); + } +#endif + } + + /// + /// Flush block to output with no compression + /// + /// Data to write + /// Index of first byte to write + /// Count of bytes to write + /// True if this is the last block + public void FlushStoredBlock(byte[] stored, int storedOffset, int storedLength, bool lastBlock) + { +#if DebugDeflation + // if (DeflaterConstants.DEBUGGING) { + // //Console.WriteLine("Flushing stored block "+ storedLength); + // } +#endif + pending.WriteBits((DeflaterConstants.STORED_BLOCK << 1) + (lastBlock ? 1 : 0), 3); + pending.AlignToByte(); + pending.WriteShort(storedLength); + pending.WriteShort(~storedLength); + pending.WriteBlock(stored, storedOffset, storedLength); + Reset(); + } + + /// + /// Flush block to output with compression + /// + /// Data to flush + /// Index of first byte to flush + /// Count of bytes to flush + /// True if this is the last block + public void FlushBlock(byte[] stored, int storedOffset, int storedLength, bool lastBlock) + { + literalTree.freqs[EOF_SYMBOL]++; + + // Build trees + literalTree.BuildTree(); + distTree.BuildTree(); + + // Calculate bitlen frequency + literalTree.CalcBLFreq(blTree); + distTree.CalcBLFreq(blTree); + + // Build bitlen tree + blTree.BuildTree(); + + int blTreeCodes = 4; + for (int i = 18; i > blTreeCodes; i--) + { + if (blTree.length[BL_ORDER[i]] > 0) + { + blTreeCodes = i + 1; + } + } + int opt_len = 14 + blTreeCodes * 3 + blTree.GetEncodedLength() + + literalTree.GetEncodedLength() + distTree.GetEncodedLength() + + extra_bits; + + int static_len = extra_bits; + for (int i = 0; i < LITERAL_NUM; i++) + { + static_len += literalTree.freqs[i] * staticLLength[i]; + } + for (int i = 0; i < DIST_NUM; i++) + { + static_len += distTree.freqs[i] * staticDLength[i]; + } + if (opt_len >= static_len) + { + // Force static trees + opt_len = static_len; + } + + if (storedOffset >= 0 && storedLength + 4 < opt_len >> 3) + { + // Store Block + + // if (DeflaterConstants.DEBUGGING) { + // //Console.WriteLine("Storing, since " + storedLength + " < " + opt_len + // + " <= " + static_len); + // } + FlushStoredBlock(stored, storedOffset, storedLength, lastBlock); + } + else if (opt_len == static_len) + { + // Encode with static tree + pending.WriteBits((DeflaterConstants.STATIC_TREES << 1) + (lastBlock ? 1 : 0), 3); + literalTree.SetStaticCodes(staticLCodes, staticLLength); + distTree.SetStaticCodes(staticDCodes, staticDLength); + CompressBlock(); + Reset(); + } + else + { + // Encode with dynamic tree + pending.WriteBits((DeflaterConstants.DYN_TREES << 1) + (lastBlock ? 1 : 0), 3); + SendAllTrees(blTreeCodes); + CompressBlock(); + Reset(); + } + } + + /// + /// Get value indicating if internal buffer is full + /// + /// true if buffer is full + public bool IsFull() + { + return last_lit >= BUFSIZE; + } + + /// + /// Add literal to buffer + /// + /// Literal value to add to buffer. + /// Value indicating internal buffer is full + public bool TallyLit(int literal) + { + // if (DeflaterConstants.DEBUGGING) { + // if (lit > 32 && lit < 127) { + // //Console.WriteLine("("+(char)lit+")"); + // } else { + // //Console.WriteLine("{"+lit+"}"); + // } + // } + d_buf[last_lit] = 0; + l_buf[last_lit++] = (byte)literal; + literalTree.freqs[literal]++; + return IsFull(); + } + + /// + /// Add distance code and length to literal and distance trees + /// + /// Distance code + /// Length + /// Value indicating if internal buffer is full + public bool TallyDist(int distance, int length) + { + // if (DeflaterConstants.DEBUGGING) { + // //Console.WriteLine("[" + distance + "," + length + "]"); + // } + + d_buf[last_lit] = (short)distance; + l_buf[last_lit++] = (byte)(length - 3); + + int lc = Lcode(length - 3); + literalTree.freqs[lc]++; + if (lc >= 265 && lc < 285) + { + extra_bits += (lc - 261) / 4; + } + + int dc = Dcode(distance - 1); + distTree.freqs[dc]++; + if (dc >= 4) + { + extra_bits += dc / 2 - 1; + } + return IsFull(); + } + + /// + /// Reverse the bits of a 16 bit value. + /// + /// Value to reverse bits + /// Value with bits reversed + public static short BitReverse(int toReverse) + { + return (short)(bit4Reverse[toReverse & 0xF] << 12 | + bit4Reverse[(toReverse >> 4) & 0xF] << 8 | + bit4Reverse[(toReverse >> 8) & 0xF] << 4 | + bit4Reverse[toReverse >> 12]); + } + + private static int Lcode(int length) + { + if (length == 255) + { + return 285; + } + + int code = 257; + while (length >= 8) + { + code += 4; + length >>= 1; + } + return code + length; + } + + private static int Dcode(int distance) + { + int code = 0; + while (distance >= 4) + { + code += 2; + distance >>= 1; + } + return code + distance; + } + } +} diff --git a/src/ImageSharp/Formats/Png/Zlib/DeflaterOutputStream.cs b/src/ImageSharp/Formats/Png/Zlib/DeflaterOutputStream.cs new file mode 100644 index 000000000..31db032ae --- /dev/null +++ b/src/ImageSharp/Formats/Png/Zlib/DeflaterOutputStream.cs @@ -0,0 +1,499 @@ +// Copyright (c) Six Labors and contributors. +// Licensed under the Apache License, Version 2.0. + +// +using System; +using System.Collections.Generic; +using System.IO; +using System.Text; + +namespace SixLabors.ImageSharp.Formats.Png.Zlib +{ + /// + /// A special stream deflating or compressing the bytes that are + /// written to it. It uses a Deflater to perform actual deflating.
+ /// Authors of the original java version : Tom Tromey, Jochen Hoenicke + /// + public class DeflaterOutputStream : Stream + { + #region Constructors + + /// + /// Creates a new DeflaterOutputStream with a default Deflater and default buffer size. + /// + /// + /// the output stream where deflated output should be written. + /// + public DeflaterOutputStream(Stream baseOutputStream) + : this(baseOutputStream, new Deflater(), 512) + { + } + + /// + /// Creates a new DeflaterOutputStream with the given Deflater and + /// default buffer size. + /// + /// + /// the output stream where deflated output should be written. + /// + /// + /// the underlying deflater. + /// + public DeflaterOutputStream(Stream baseOutputStream, Deflater deflater) + : this(baseOutputStream, deflater, 512) + { + } + + /// + /// Creates a new DeflaterOutputStream with the given Deflater and + /// buffer size. + /// + /// + /// The output stream where deflated output is written. + /// + /// + /// The underlying deflater to use + /// + /// + /// The buffer size in bytes to use when deflating (minimum value 512) + /// + /// + /// bufsize is less than or equal to zero. + /// + /// + /// baseOutputStream does not support writing + /// + /// + /// deflater instance is null + /// + public DeflaterOutputStream(Stream baseOutputStream, Deflater deflater, int bufferSize) + { + if (baseOutputStream == null) + { + throw new ArgumentNullException(nameof(baseOutputStream)); + } + + if (baseOutputStream.CanWrite == false) + { + throw new ArgumentException("Must support writing", nameof(baseOutputStream)); + } + + if (bufferSize < 512) + { + throw new ArgumentOutOfRangeException(nameof(bufferSize)); + } + + baseOutputStream_ = baseOutputStream; + buffer_ = new byte[bufferSize]; + deflater_ = deflater ?? throw new ArgumentNullException(nameof(deflater)); + } + + #endregion Constructors + + #region Public API + + /// + /// Finishes the stream by calling finish() on the deflater. + /// + /// + /// Not all input is deflated + /// + public virtual void Finish() + { + deflater_.Finish(); + while (!deflater_.IsFinished) + { + int len = deflater_.Deflate(buffer_, 0, buffer_.Length); + if (len <= 0) + { + break; + } + + //if (cryptoTransform_ != null) + //{ + // EncryptBlock(buffer_, 0, len); + //} + + baseOutputStream_.Write(buffer_, 0, len); + } + + if (!deflater_.IsFinished) + { + throw new ImageFormatException("Can't deflate all input?"); + } + + baseOutputStream_.Flush(); + + //if (cryptoTransform_ != null) + //{ + // if (cryptoTransform_ is ZipAESTransform) + // { + // AESAuthCode = ((ZipAESTransform)cryptoTransform_).GetAuthCode(); + // } + // cryptoTransform_.Dispose(); + // cryptoTransform_ = null; + //} + } + + /// + /// Gets or sets a flag indicating ownership of underlying stream. + /// When the flag is true will close the underlying stream also. + /// + /// The default value is true. + public bool IsStreamOwner { get; set; } = true; + + /// + /// Allows client to determine if an entry can be patched after its added + /// + public bool CanPatchEntries + { + get + { + return baseOutputStream_.CanSeek; + } + } + + #endregion Public API + + //#region Encryption + + //private string password; + + //private ICryptoTransform cryptoTransform_; + + ///// + ///// Returns the 10 byte AUTH CODE to be appended immediately following the AES data stream. + ///// + //protected byte[] AESAuthCode; + + ///// + ///// Get/set the password used for encryption. + ///// + ///// When set to null or if the password is empty no encryption is performed + //public string Password + //{ + // get + // { + // return password; + // } + // set + // { + // if ((value != null) && (value.Length == 0)) + // { + // password = null; + // } + // else + // { + // password = value; + // } + // } + //} + + ///// + ///// Encrypt a block of data + ///// + ///// + ///// Data to encrypt. NOTE the original contents of the buffer are lost + ///// + ///// + ///// Offset of first byte in buffer to encrypt + ///// + ///// + ///// Number of bytes in buffer to encrypt + ///// + //protected void EncryptBlock(byte[] buffer, int offset, int length) + //{ + // cryptoTransform_.TransformBlock(buffer, 0, length, buffer, 0); + //} + + ///// + ///// Initializes encryption keys based on given . + ///// + ///// The password. + //protected void InitializePassword(string password) + //{ + // var pkManaged = new PkzipClassicManaged(); + // byte[] key = PkzipClassic.GenerateKeys(ZipStrings.ConvertToArray(password)); + // cryptoTransform_ = pkManaged.CreateEncryptor(key, null); + //} + + ///// + ///// Initializes encryption keys based on given password. + ///// + //protected void InitializeAESPassword(ZipEntry entry, string rawPassword, + // out byte[] salt, out byte[] pwdVerifier) + //{ + // salt = new byte[entry.AESSaltLen]; + // // Salt needs to be cryptographically random, and unique per file + // if (_aesRnd == null) + // _aesRnd = RandomNumberGenerator.Create(); + // _aesRnd.GetBytes(salt); + // int blockSize = entry.AESKeySize / 8; // bits to bytes + + // cryptoTransform_ = new ZipAESTransform(rawPassword, salt, blockSize, true); + // pwdVerifier = ((ZipAESTransform)cryptoTransform_).PwdVerifier; + //} + + //#endregion Encryption + + #region Deflation Support + + /// + /// Deflates everything in the input buffers. This will call + /// def.deflate() until all bytes from the input buffers + /// are processed. + /// + protected void Deflate() + { + Deflate(false); + } + + private void Deflate(bool flushing) + { + while (flushing || !deflater_.IsNeedingInput) + { + int deflateCount = deflater_.Deflate(buffer_, 0, buffer_.Length); + + if (deflateCount <= 0) + { + break; + } + //if (cryptoTransform_ != null) + //{ + // EncryptBlock(buffer_, 0, deflateCount); + //} + + baseOutputStream_.Write(buffer_, 0, deflateCount); + } + + if (!deflater_.IsNeedingInput) + { + throw new ImageFormatException("DeflaterOutputStream can't deflate all input?"); + } + } + + #endregion Deflation Support + + #region Stream Overrides + + /// + /// Gets value indicating stream can be read from + /// + public override bool CanRead + { + get + { + return false; + } + } + + /// + /// Gets a value indicating if seeking is supported for this stream + /// This property always returns false + /// + public override bool CanSeek + { + get + { + return false; + } + } + + /// + /// Get value indicating if this stream supports writing + /// + public override bool CanWrite + { + get + { + return baseOutputStream_.CanWrite; + } + } + + /// + /// Get current length of stream + /// + public override long Length + { + get + { + return baseOutputStream_.Length; + } + } + + /// + /// Gets the current position within the stream. + /// + /// Any attempt to set position + public override long Position + { + get + { + return baseOutputStream_.Position; + } + set + { + throw new NotSupportedException("Position property not supported"); + } + } + + /// + /// Sets the current position of this stream to the given value. Not supported by this class! + /// + /// The offset relative to the to seek. + /// The to seek from. + /// The new position in the stream. + /// Any access + public override long Seek(long offset, SeekOrigin origin) + { + throw new NotSupportedException("DeflaterOutputStream Seek not supported"); + } + + /// + /// Sets the length of this stream to the given value. Not supported by this class! + /// + /// The new stream length. + /// Any access + public override void SetLength(long value) + { + throw new NotSupportedException("DeflaterOutputStream SetLength not supported"); + } + + /// + /// Read a byte from stream advancing position by one + /// + /// The byte read cast to an int. THe value is -1 if at the end of the stream. + /// Any access + public override int ReadByte() + { + throw new NotSupportedException("DeflaterOutputStream ReadByte not supported"); + } + + /// + /// Read a block of bytes from stream + /// + /// The buffer to store read data in. + /// The offset to start storing at. + /// The maximum number of bytes to read. + /// The actual number of bytes read. Zero if end of stream is detected. + /// Any access + public override int Read(byte[] buffer, int offset, int count) + { + throw new NotSupportedException("DeflaterOutputStream Read not supported"); + } + + /// + /// Flushes the stream by calling Flush on the deflater and then + /// on the underlying stream. This ensures that all bytes are flushed. + /// + public override void Flush() + { + deflater_.Flush(); + Deflate(true); + baseOutputStream_.Flush(); + } + + /// + /// Calls and closes the underlying + /// stream when is true. + /// + protected override void Dispose(bool disposing) + { + if (!isClosed_) + { + isClosed_ = true; + + try + { + Finish(); + //if (cryptoTransform_ != null) + //{ + // GetAuthCodeIfAES(); + // cryptoTransform_.Dispose(); + // cryptoTransform_ = null; + //} + } + finally + { + if (IsStreamOwner) + { + baseOutputStream_.Dispose(); + } + } + } + } + + ///// + ///// Get the Auth code for AES encrypted entries + ///// + //protected void GetAuthCodeIfAES() + //{ + // if (cryptoTransform_ is ZipAESTransform) + // { + // AESAuthCode = ((ZipAESTransform)cryptoTransform_).GetAuthCode(); + // } + //} + + /// + /// Writes a single byte to the compressed output stream. + /// + /// + /// The byte value. + /// + public override void WriteByte(byte value) + { + byte[] b = new byte[1]; + b[0] = value; + Write(b, 0, 1); + } + + /// + /// Writes bytes from an array to the compressed stream. + /// + /// + /// The byte array + /// + /// + /// The offset into the byte array where to start. + /// + /// + /// The number of bytes to write. + /// + public override void Write(byte[] buffer, int offset, int count) + { + deflater_.SetInput(buffer, offset, count); + Deflate(); + } + + #endregion Stream Overrides + + #region Instance Fields + + /// + /// This buffer is used temporarily to retrieve the bytes from the + /// deflater and write them to the underlying output stream. + /// + private byte[] buffer_; + + /// + /// The deflater which is used to deflate the stream. + /// + protected Deflater deflater_; + + /// + /// Base stream the deflater depends on. + /// + protected Stream baseOutputStream_; + + private bool isClosed_; + + #endregion Instance Fields + + #region Static Fields + + // Static to help ensure that multiple files within a zip will get different random salt + //private static RandomNumberGenerator _aesRnd = RandomNumberGenerator.Create(); + + #endregion Static Fields + } +} diff --git a/src/ImageSharp/Formats/Png/Zlib/DeflaterPending.cs b/src/ImageSharp/Formats/Png/Zlib/DeflaterPending.cs new file mode 100644 index 000000000..cc421c5b7 --- /dev/null +++ b/src/ImageSharp/Formats/Png/Zlib/DeflaterPending.cs @@ -0,0 +1,21 @@ +// Copyright (c) Six Labors and contributors. +// Licensed under the Apache License, Version 2.0. + +// +namespace SixLabors.ImageSharp.Formats.Png.Zlib +{ + /// + /// This class stores the pending output of the Deflater. + /// + /// author of the original java version : Jochen Hoenicke + /// + public class DeflaterPending : PendingBuffer + { + /// + /// Construct instance with default buffer size + /// + public DeflaterPending() : base(DeflaterConstants.PENDING_BUF_SIZE) + { + } + } +} diff --git a/src/ImageSharp/Formats/Png/Zlib/PendingBuffer.cs b/src/ImageSharp/Formats/Png/Zlib/PendingBuffer.cs new file mode 100644 index 000000000..ae02c5113 --- /dev/null +++ b/src/ImageSharp/Formats/Png/Zlib/PendingBuffer.cs @@ -0,0 +1,276 @@ +// Copyright (c) Six Labors and contributors. +// Licensed under the Apache License, Version 2.0. + +// +using System; +using System.Collections.Generic; +using System.Text; + +namespace SixLabors.ImageSharp.Formats.Png.Zlib +{ + /// + /// This class is general purpose class for writing data to a buffer. + /// + /// It allows you to write bits as well as bytes + /// Based on DeflaterPending.java + /// + /// author of the original java version : Jochen Hoenicke + /// + public class PendingBuffer + { + #region Instance Fields + + /// + /// Internal work buffer + /// + private readonly byte[] buffer; + + private int start; + private int end; + + private uint bits; + private int bitCount; + + #endregion Instance Fields + + #region Constructors + + /// + /// construct instance using default buffer size of 4096 + /// + public PendingBuffer() : this(4096) + { + } + + /// + /// construct instance using specified buffer size + /// + /// + /// size to use for internal buffer + /// + public PendingBuffer(int bufferSize) + { + buffer = new byte[bufferSize]; + } + + #endregion Constructors + + /// + /// Clear internal state/buffers + /// + public void Reset() + { + start = end = bitCount = 0; + } + + /// + /// Write a byte to buffer + /// + /// + /// The value to write + /// + public void WriteByte(int value) + { +#if DebugDeflation + if (DeflaterConstants.DEBUGGING && (start != 0) ) + { + throw new ImageFormatException("Debug check: start != 0"); + } +#endif + buffer[end++] = unchecked((byte)value); + } + + /// + /// Write a short value to buffer LSB first + /// + /// + /// The value to write. + /// + public void WriteShort(int value) + { +#if DebugDeflation + if (DeflaterConstants.DEBUGGING && (start != 0) ) + { + throw new ImageFormatException("Debug check: start != 0"); + } +#endif + buffer[end++] = unchecked((byte)value); + buffer[end++] = unchecked((byte)(value >> 8)); + } + + /// + /// write an integer LSB first + /// + /// The value to write. + public void WriteInt(int value) + { +#if DebugDeflation + if (DeflaterConstants.DEBUGGING && (start != 0) ) + { + throw new ImageFormatException("Debug check: start != 0"); + } +#endif + buffer[end++] = unchecked((byte)value); + buffer[end++] = unchecked((byte)(value >> 8)); + buffer[end++] = unchecked((byte)(value >> 16)); + buffer[end++] = unchecked((byte)(value >> 24)); + } + + /// + /// Write a block of data to buffer + /// + /// data to write + /// offset of first byte to write + /// number of bytes to write + public void WriteBlock(byte[] block, int offset, int length) + { +#if DebugDeflation + if (DeflaterConstants.DEBUGGING && (start != 0) ) + { + throw new ImageFormatException("Debug check: start != 0"); + } +#endif + System.Array.Copy(block, offset, buffer, end, length); + end += length; + } + + /// + /// The number of bits written to the buffer + /// + public int BitCount + { + get + { + return bitCount; + } + } + + /// + /// Align internal buffer on a byte boundary + /// + public void AlignToByte() + { +#if DebugDeflation + if (DeflaterConstants.DEBUGGING && (start != 0) ) + { + throw new ImageFormatException("Debug check: start != 0"); + } +#endif + if (bitCount > 0) + { + buffer[end++] = unchecked((byte)bits); + if (bitCount > 8) + { + buffer[end++] = unchecked((byte)(bits >> 8)); + } + } + bits = 0; + bitCount = 0; + } + + /// + /// Write bits to internal buffer + /// + /// source of bits + /// number of bits to write + public void WriteBits(int b, int count) + { +#if DebugDeflation + if (DeflaterConstants.DEBUGGING && (start != 0) ) + { + throw new ImageFormatException("Debug check: start != 0"); + } + + // if (DeflaterConstants.DEBUGGING) { + // //Console.WriteLine("writeBits("+b+","+count+")"); + // } +#endif + bits |= (uint)(b << bitCount); + bitCount += count; + if (bitCount >= 16) + { + buffer[end++] = unchecked((byte)bits); + buffer[end++] = unchecked((byte)(bits >> 8)); + bits >>= 16; + bitCount -= 16; + } + } + + /// + /// Write a short value to internal buffer most significant byte first + /// + /// value to write + public void WriteShortMSB(int s) + { +#if DebugDeflation + if (DeflaterConstants.DEBUGGING && (start != 0) ) + { + throw new ImageFormatException("Debug check: start != 0"); + } +#endif + buffer[end++] = unchecked((byte)(s >> 8)); + buffer[end++] = unchecked((byte)s); + } + + /// + /// Indicates if buffer has been flushed + /// + public bool IsFlushed + { + get + { + return end == 0; + } + } + + /// + /// Flushes the pending buffer into the given output array. If the + /// output array is to small, only a partial flush is done. + /// + /// The output array. + /// The offset into output array. + /// The maximum number of bytes to store. + /// The number of bytes flushed. + public int Flush(byte[] output, int offset, int length) + { + if (bitCount >= 8) + { + buffer[end++] = unchecked((byte)bits); + bits >>= 8; + bitCount -= 8; + } + + if (length > end - start) + { + length = end - start; + System.Array.Copy(buffer, start, output, offset, length); + start = 0; + end = 0; + } + else + { + System.Array.Copy(buffer, start, output, offset, length); + start += length; + } + return length; + } + + /// + /// Convert internal buffer to byte array. + /// Buffer is empty on completion + /// + /// + /// The internal buffer contents converted to a byte array. + /// + public byte[] ToByteArray() + { + AlignToByte(); + + byte[] result = new byte[end - start]; + System.Array.Copy(buffer, start, result, 0, result.Length); + start = 0; + end = 0; + return result; + } + } +} diff --git a/src/ImageSharp/Formats/Png/Zlib/ZlibDeflateStream.cs b/src/ImageSharp/Formats/Png/Zlib/ZlibDeflateStream.cs index 8e0bac938..ffe176368 100644 --- a/src/ImageSharp/Formats/Png/Zlib/ZlibDeflateStream.cs +++ b/src/ImageSharp/Formats/Png/Zlib/ZlibDeflateStream.cs @@ -1,4 +1,4 @@ -// Copyright (c) Six Labors and contributors. +// Copyright (c) Six Labors and contributors. // Licensed under the Apache License, Version 2.0. using System; @@ -38,7 +38,8 @@ namespace SixLabors.ImageSharp.Formats.Png.Zlib /// /// The stream responsible for compressing the input stream. /// - private System.IO.Compression.DeflateStream deflateStream; + // private DeflateStream deflateStream; + private DeflaterOutputStream deflateStream; /// /// Initializes a new instance of the class. @@ -89,18 +90,19 @@ namespace SixLabors.ImageSharp.Formats.Png.Zlib this.rawStream.WriteByte((byte)flg); // Initialize the deflate Stream. - CompressionLevel level = CompressionLevel.Optimal; - - if (compressionLevel >= 1 && compressionLevel <= 5) - { - level = CompressionLevel.Fastest; - } - else if (compressionLevel == 0) - { - level = CompressionLevel.NoCompression; - } - - this.deflateStream = new System.IO.Compression.DeflateStream(this.rawStream, level, true); + // CompressionLevel level = CompressionLevel.Optimal; + // + // if (compressionLevel >= 1 && compressionLevel <= 5) + // { + // level = CompressionLevel.Fastest; + // } + // else if (compressionLevel == 0) + // { + // level = CompressionLevel.NoCompression; + // } + this.deflateStream = new DeflaterOutputStream(this.rawStream, new Deflater(compressionLevel, true)) { IsStreamOwner = false }; + + // this.deflateStream = new DeflateStream(this.rawStream, level, true); } /// diff --git a/tests/ImageSharp.Benchmarks/Codecs/EncodePng.cs b/tests/ImageSharp.Benchmarks/Codecs/EncodePng.cs index 157dadd2c..7bd1b8044 100644 --- a/tests/ImageSharp.Benchmarks/Codecs/EncodePng.cs +++ b/tests/ImageSharp.Benchmarks/Codecs/EncodePng.cs @@ -1,9 +1,10 @@ -// Copyright (c) Six Labors and contributors. +// Copyright (c) Six Labors and contributors. // Licensed under the Apache License, Version 2.0. using System.Drawing.Imaging; using System.IO; using BenchmarkDotNet.Attributes; +using SixLabors.ImageSharp.Formats.Png; using SixLabors.ImageSharp.PixelFormats; using SixLabors.ImageSharp.Tests; using SDImage = System.Drawing.Image; @@ -56,8 +57,9 @@ namespace SixLabors.ImageSharp.Benchmarks.Codecs { using (var memoryStream = new MemoryStream()) { - this.bmpCore.SaveAsPng(memoryStream); + var encoder = new PngEncoder { FilterMethod = PngFilterMethod.None }; + this.bmpCore.SaveAsPng(memoryStream, encoder); } } } -} \ No newline at end of file +} From 575f034b2c62fdbc794a78671692472c7e900d39 Mon Sep 17 00:00:00 2001 From: James Jackson-South Date: Tue, 12 Nov 2019 15:44:34 +1100 Subject: [PATCH 02/16] Optimize DeflaterPendingBuffer --- src/ImageSharp/Formats/Png/PngEncoderCore.cs | 4 +- src/ImageSharp/Formats/Png/Zlib/Deflater.cs | 57 ++-- .../Formats/Png/Zlib/DeflaterEngine.cs | 6 +- .../Formats/Png/Zlib/DeflaterHuffman.cs | 4 +- .../Formats/Png/Zlib/DeflaterOutputStream.cs | 12 - .../Formats/Png/Zlib/DeflaterPending.cs | 21 -- .../Formats/Png/Zlib/DeflaterPendingBuffer.cs | 187 ++++++++++++ .../Formats/Png/Zlib/PendingBuffer.cs | 276 ------------------ .../Formats/Png/Zlib/ZlibDeflateStream.cs | 13 +- 9 files changed, 239 insertions(+), 341 deletions(-) delete mode 100644 src/ImageSharp/Formats/Png/Zlib/DeflaterPending.cs create mode 100644 src/ImageSharp/Formats/Png/Zlib/DeflaterPendingBuffer.cs delete mode 100644 src/ImageSharp/Formats/Png/Zlib/PendingBuffer.cs diff --git a/src/ImageSharp/Formats/Png/PngEncoderCore.cs b/src/ImageSharp/Formats/Png/PngEncoderCore.cs index 09575bb28..19c6af27f 100644 --- a/src/ImageSharp/Formats/Png/PngEncoderCore.cs +++ b/src/ImageSharp/Formats/Png/PngEncoderCore.cs @@ -699,7 +699,7 @@ namespace SixLabors.ImageSharp.Formats.Png { using (var memoryStream = new MemoryStream()) { - using (var deflateStream = new ZlibDeflateStream(memoryStream, this.options.CompressionLevel)) + using (var deflateStream = new ZlibDeflateStream(this.memoryAllocator, memoryStream, this.options.CompressionLevel)) { deflateStream.Write(textBytes); } @@ -790,7 +790,7 @@ namespace SixLabors.ImageSharp.Formats.Png using (var memoryStream = new MemoryStream()) { - using (var deflateStream = new ZlibDeflateStream(memoryStream, this.options.CompressionLevel)) + using (var deflateStream = new ZlibDeflateStream(this.memoryAllocator, memoryStream, this.options.CompressionLevel)) { if (this.options.InterlaceMethod == PngInterlaceMode.Adam7) { diff --git a/src/ImageSharp/Formats/Png/Zlib/Deflater.cs b/src/ImageSharp/Formats/Png/Zlib/Deflater.cs index 358112549..90e25fb58 100644 --- a/src/ImageSharp/Formats/Png/Zlib/Deflater.cs +++ b/src/ImageSharp/Formats/Png/Zlib/Deflater.cs @@ -5,6 +5,7 @@ using System; using System.Collections.Generic; using System.Text; +using SixLabors.Memory; namespace SixLabors.ImageSharp.Formats.Png.Zlib { @@ -18,7 +19,7 @@ namespace SixLabors.ImageSharp.Formats.Png.Zlib /// /// author of the original java version : Jochen Hoenicke /// - public class Deflater + public sealed class Deflater : IDisposable { #region Deflater Documentation @@ -149,28 +150,10 @@ namespace SixLabors.ImageSharp.Formats.Png.Zlib #region Constructors - /// - /// Creates a new deflater with default compression level. - /// - public Deflater() : this(DEFAULT_COMPRESSION, false) - { - } - - /// - /// Creates a new deflater with given compression level. - /// - /// - /// the compression level, a value between NO_COMPRESSION - /// and BEST_COMPRESSION, or DEFAULT_COMPRESSION. - /// - /// if lvl is out of range. - public Deflater(int level) : this(level, false) - { - } - /// /// Creates a new deflater with given compression level. /// + /// The memory allocator to use for buffer allocations. /// /// the compression level, a value between NO_COMPRESSION /// and BEST_COMPRESSION. @@ -181,7 +164,7 @@ namespace SixLabors.ImageSharp.Formats.Png.Zlib /// useful for the GZIP/PKZIP formats. /// /// if lvl is out of range. - public Deflater(int level, bool noZlibHeaderOrFooter) + public Deflater(MemoryAllocator memoryAllocator, int level, bool noZlibHeaderOrFooter) { if (level == DEFAULT_COMPRESSION) { @@ -192,7 +175,7 @@ namespace SixLabors.ImageSharp.Formats.Png.Zlib throw new ArgumentOutOfRangeException(nameof(level)); } - pending = new DeflaterPending(); + pending = new DeflaterPendingBuffer(memoryAllocator); engine = new DeflaterEngine(pending, noZlibHeaderOrFooter); this.noZlibHeaderOrFooter = noZlibHeaderOrFooter; SetStrategy(DeflateStrategy.Default); @@ -598,13 +581,41 @@ namespace SixLabors.ImageSharp.Formats.Png.Zlib /// /// The pending output. /// - private DeflaterPending pending; + private DeflaterPendingBuffer pending; /// /// The deflater engine. /// private DeflaterEngine engine; + #region IDisposable Support + private bool disposedValue = false; // To detect redundant calls + + void Dispose(bool disposing) + { + if (!disposedValue) + { + if (disposing) + { + this.pending.Dispose(); + // TODO: dispose managed state (managed objects). + } + + // TODO: free unmanaged resources (unmanaged objects) and override a finalizer below. + // TODO: set large fields to null. + this.pending = null; + disposedValue = true; + } + } + + /// + public void Dispose() + { + Dispose(true); + GC.SuppressFinalize(this); + } + #endregion + #endregion Instance Fields } } diff --git a/src/ImageSharp/Formats/Png/Zlib/DeflaterEngine.cs b/src/ImageSharp/Formats/Png/Zlib/DeflaterEngine.cs index 7ac5b6c69..c9967056e 100644 --- a/src/ImageSharp/Formats/Png/Zlib/DeflaterEngine.cs +++ b/src/ImageSharp/Formats/Png/Zlib/DeflaterEngine.cs @@ -66,7 +66,7 @@ namespace SixLabors.ImageSharp.Formats.Png.Zlib /// /// Pending buffer to use /// - public DeflaterEngine(DeflaterPending pending) + public DeflaterEngine(DeflaterPendingBuffer pending) : this(pending, false) { } @@ -82,7 +82,7 @@ namespace SixLabors.ImageSharp.Formats.Png.Zlib /// /// If no adler calculation should be performed /// - public DeflaterEngine(DeflaterPending pending, bool noAdlerCalculation) + public DeflaterEngine(DeflaterPendingBuffer pending, bool noAdlerCalculation) { this.pending = pending; huffman = new DeflaterHuffman(pending); @@ -938,7 +938,7 @@ namespace SixLabors.ImageSharp.Formats.Png.Zlib /// private int inputEnd; - private DeflaterPending pending; + private DeflaterPendingBuffer pending; private DeflaterHuffman huffman; /// diff --git a/src/ImageSharp/Formats/Png/Zlib/DeflaterHuffman.cs b/src/ImageSharp/Formats/Png/Zlib/DeflaterHuffman.cs index bf506d14c..3d2856b0e 100644 --- a/src/ImageSharp/Formats/Png/Zlib/DeflaterHuffman.cs +++ b/src/ImageSharp/Formats/Png/Zlib/DeflaterHuffman.cs @@ -590,7 +590,7 @@ namespace SixLabors.ImageSharp.Formats.Png.Zlib /// /// Pending buffer to use /// - public DeflaterPending pending; + public DeflaterPendingBuffer pending; private Tree literalTree; private Tree distTree; @@ -651,7 +651,7 @@ namespace SixLabors.ImageSharp.Formats.Png.Zlib /// Construct instance with pending buffer /// /// Pending buffer to use - public DeflaterHuffman(DeflaterPending pending) + public DeflaterHuffman(DeflaterPendingBuffer pending) { this.pending = pending; diff --git a/src/ImageSharp/Formats/Png/Zlib/DeflaterOutputStream.cs b/src/ImageSharp/Formats/Png/Zlib/DeflaterOutputStream.cs index 31db032ae..ac5f229aa 100644 --- a/src/ImageSharp/Formats/Png/Zlib/DeflaterOutputStream.cs +++ b/src/ImageSharp/Formats/Png/Zlib/DeflaterOutputStream.cs @@ -17,18 +17,6 @@ namespace SixLabors.ImageSharp.Formats.Png.Zlib public class DeflaterOutputStream : Stream { #region Constructors - - /// - /// Creates a new DeflaterOutputStream with a default Deflater and default buffer size. - /// - /// - /// the output stream where deflated output should be written. - /// - public DeflaterOutputStream(Stream baseOutputStream) - : this(baseOutputStream, new Deflater(), 512) - { - } - /// /// Creates a new DeflaterOutputStream with the given Deflater and /// default buffer size. diff --git a/src/ImageSharp/Formats/Png/Zlib/DeflaterPending.cs b/src/ImageSharp/Formats/Png/Zlib/DeflaterPending.cs deleted file mode 100644 index cc421c5b7..000000000 --- a/src/ImageSharp/Formats/Png/Zlib/DeflaterPending.cs +++ /dev/null @@ -1,21 +0,0 @@ -// Copyright (c) Six Labors and contributors. -// Licensed under the Apache License, Version 2.0. - -// -namespace SixLabors.ImageSharp.Formats.Png.Zlib -{ - /// - /// This class stores the pending output of the Deflater. - /// - /// author of the original java version : Jochen Hoenicke - /// - public class DeflaterPending : PendingBuffer - { - /// - /// Construct instance with default buffer size - /// - public DeflaterPending() : base(DeflaterConstants.PENDING_BUF_SIZE) - { - } - } -} diff --git a/src/ImageSharp/Formats/Png/Zlib/DeflaterPendingBuffer.cs b/src/ImageSharp/Formats/Png/Zlib/DeflaterPendingBuffer.cs new file mode 100644 index 000000000..64214b47e --- /dev/null +++ b/src/ImageSharp/Formats/Png/Zlib/DeflaterPendingBuffer.cs @@ -0,0 +1,187 @@ +// Copyright (c) Six Labors and contributors. +// Licensed under the Apache License, Version 2.0. + +using System; +using System.Buffers; +using System.Runtime.CompilerServices; +using SixLabors.Memory; + +namespace SixLabors.ImageSharp.Formats.Png.Zlib +{ + /// + /// Stores pending data for writing data to the Deflater. + /// + public sealed unsafe class DeflaterPendingBuffer : IDisposable + { + private readonly byte[] buffer; + private readonly byte* pinnedBuffer; + private readonly IManagedByteBuffer managedBuffer; + private MemoryHandle handle; + + private int start; + private int end; + private uint bits; + private bool isDisposed; + + /// + /// Initializes a new instance of the class. + /// + /// The memory allocator to use for buffer allocations. + public DeflaterPendingBuffer(MemoryAllocator memoryAllocator) + { + this.buffer = new byte[DeflaterConstants.PENDING_BUF_SIZE]; + this.managedBuffer = memoryAllocator.AllocateManagedByteBuffer(DeflaterConstants.PENDING_BUF_SIZE); + this.buffer = this.managedBuffer.Array; + this.handle = this.managedBuffer.Memory.Pin(); + this.pinnedBuffer = (byte*)this.handle.Pointer; + } + + /// + /// Gets the number of bits written to the buffer. + /// + public int BitCount { get; private set; } + + /// + /// Gets a value indicating whether indicates the buffer has been flushed. + /// + public bool IsFlushed => this.end == 0; + + /// + /// Clear internal state/buffers. + /// + [MethodImpl(InliningOptions.ShortMethod)] + public void Reset() => this.start = this.end = this.BitCount = 0; + + /// + /// Write a short value to buffer LSB first. + /// + /// The value to write. + [MethodImpl(InliningOptions.ShortMethod)] + public void WriteShort(int value) + { + byte* pinned = this.pinnedBuffer; + pinned[this.end++] = unchecked((byte)value); + pinned[this.end++] = unchecked((byte)(value >> 8)); + } + + /// + /// Write a block of data to the internal buffer. + /// + /// The data to write. + /// The offset of first byte to write. + /// The number of bytes to write. + [MethodImpl(InliningOptions.ShortMethod)] + public void WriteBlock(byte[] block, int offset, int length) + { + Unsafe.CopyBlockUnaligned(ref this.buffer[this.end], ref block[offset], unchecked((uint)length)); + this.end += length; + } + + /// + /// Aligns internal buffer on a byte boundary. + /// + [MethodImpl(InliningOptions.ShortMethod)] + public void AlignToByte() + { + if (this.BitCount > 0) + { + byte* pinned = this.pinnedBuffer; + pinned[this.end++] = unchecked((byte)this.bits); + if (this.BitCount > 8) + { + pinned[this.end++] = unchecked((byte)(this.bits >> 8)); + } + } + + this.bits = 0; + this.BitCount = 0; + } + + /// + /// Write bits to internal buffer + /// + /// source of bits + /// number of bits to write + [MethodImpl(InliningOptions.ShortMethod)] + public void WriteBits(int b, int count) + { + this.bits |= (uint)(b << this.BitCount); + this.BitCount += count; + if (this.BitCount >= 16) + { + byte* pinned = this.pinnedBuffer; + pinned[this.end++] = unchecked((byte)this.bits); + pinned[this.end++] = unchecked((byte)(this.bits >> 8)); + this.bits >>= 16; + this.BitCount -= 16; + } + } + + /// + /// Write a short value to internal buffer most significant byte first + /// + /// The value to write + [MethodImpl(InliningOptions.ShortMethod)] + public void WriteShortMSB(int value) + { + byte* pinned = this.pinnedBuffer; + pinned[this.end++] = unchecked((byte)(value >> 8)); + pinned[this.end++] = unchecked((byte)value); + } + + /// + /// Flushes the pending buffer into the given output array. + /// If the output array is to small, only a partial flush is done. + /// + /// The output array. + /// The offset into output array. + /// The maximum number of bytes to store. + /// The number of bytes flushed. + public int Flush(byte[] output, int offset, int length) + { + if (this.BitCount >= 8) + { + this.pinnedBuffer[this.end++] = unchecked((byte)this.bits); + this.bits >>= 8; + this.BitCount -= 8; + } + + if (length > this.end - this.start) + { + length = this.end - this.start; + + Unsafe.CopyBlockUnaligned(ref output[offset], ref this.buffer[this.start], unchecked((uint)length)); + this.start = 0; + this.end = 0; + } + else + { + Unsafe.CopyBlockUnaligned(ref output[offset], ref this.buffer[this.start], unchecked((uint)length)); + this.start += length; + } + + return length; + } + + /// + public void Dispose() + { + this.Dispose(true); + GC.SuppressFinalize(this); + } + + private void Dispose(bool disposing) + { + if (!this.isDisposed) + { + if (disposing) + { + this.handle.Dispose(); + this.managedBuffer.Dispose(); + } + + this.isDisposed = true; + } + } + } +} diff --git a/src/ImageSharp/Formats/Png/Zlib/PendingBuffer.cs b/src/ImageSharp/Formats/Png/Zlib/PendingBuffer.cs deleted file mode 100644 index ae02c5113..000000000 --- a/src/ImageSharp/Formats/Png/Zlib/PendingBuffer.cs +++ /dev/null @@ -1,276 +0,0 @@ -// Copyright (c) Six Labors and contributors. -// Licensed under the Apache License, Version 2.0. - -// -using System; -using System.Collections.Generic; -using System.Text; - -namespace SixLabors.ImageSharp.Formats.Png.Zlib -{ - /// - /// This class is general purpose class for writing data to a buffer. - /// - /// It allows you to write bits as well as bytes - /// Based on DeflaterPending.java - /// - /// author of the original java version : Jochen Hoenicke - /// - public class PendingBuffer - { - #region Instance Fields - - /// - /// Internal work buffer - /// - private readonly byte[] buffer; - - private int start; - private int end; - - private uint bits; - private int bitCount; - - #endregion Instance Fields - - #region Constructors - - /// - /// construct instance using default buffer size of 4096 - /// - public PendingBuffer() : this(4096) - { - } - - /// - /// construct instance using specified buffer size - /// - /// - /// size to use for internal buffer - /// - public PendingBuffer(int bufferSize) - { - buffer = new byte[bufferSize]; - } - - #endregion Constructors - - /// - /// Clear internal state/buffers - /// - public void Reset() - { - start = end = bitCount = 0; - } - - /// - /// Write a byte to buffer - /// - /// - /// The value to write - /// - public void WriteByte(int value) - { -#if DebugDeflation - if (DeflaterConstants.DEBUGGING && (start != 0) ) - { - throw new ImageFormatException("Debug check: start != 0"); - } -#endif - buffer[end++] = unchecked((byte)value); - } - - /// - /// Write a short value to buffer LSB first - /// - /// - /// The value to write. - /// - public void WriteShort(int value) - { -#if DebugDeflation - if (DeflaterConstants.DEBUGGING && (start != 0) ) - { - throw new ImageFormatException("Debug check: start != 0"); - } -#endif - buffer[end++] = unchecked((byte)value); - buffer[end++] = unchecked((byte)(value >> 8)); - } - - /// - /// write an integer LSB first - /// - /// The value to write. - public void WriteInt(int value) - { -#if DebugDeflation - if (DeflaterConstants.DEBUGGING && (start != 0) ) - { - throw new ImageFormatException("Debug check: start != 0"); - } -#endif - buffer[end++] = unchecked((byte)value); - buffer[end++] = unchecked((byte)(value >> 8)); - buffer[end++] = unchecked((byte)(value >> 16)); - buffer[end++] = unchecked((byte)(value >> 24)); - } - - /// - /// Write a block of data to buffer - /// - /// data to write - /// offset of first byte to write - /// number of bytes to write - public void WriteBlock(byte[] block, int offset, int length) - { -#if DebugDeflation - if (DeflaterConstants.DEBUGGING && (start != 0) ) - { - throw new ImageFormatException("Debug check: start != 0"); - } -#endif - System.Array.Copy(block, offset, buffer, end, length); - end += length; - } - - /// - /// The number of bits written to the buffer - /// - public int BitCount - { - get - { - return bitCount; - } - } - - /// - /// Align internal buffer on a byte boundary - /// - public void AlignToByte() - { -#if DebugDeflation - if (DeflaterConstants.DEBUGGING && (start != 0) ) - { - throw new ImageFormatException("Debug check: start != 0"); - } -#endif - if (bitCount > 0) - { - buffer[end++] = unchecked((byte)bits); - if (bitCount > 8) - { - buffer[end++] = unchecked((byte)(bits >> 8)); - } - } - bits = 0; - bitCount = 0; - } - - /// - /// Write bits to internal buffer - /// - /// source of bits - /// number of bits to write - public void WriteBits(int b, int count) - { -#if DebugDeflation - if (DeflaterConstants.DEBUGGING && (start != 0) ) - { - throw new ImageFormatException("Debug check: start != 0"); - } - - // if (DeflaterConstants.DEBUGGING) { - // //Console.WriteLine("writeBits("+b+","+count+")"); - // } -#endif - bits |= (uint)(b << bitCount); - bitCount += count; - if (bitCount >= 16) - { - buffer[end++] = unchecked((byte)bits); - buffer[end++] = unchecked((byte)(bits >> 8)); - bits >>= 16; - bitCount -= 16; - } - } - - /// - /// Write a short value to internal buffer most significant byte first - /// - /// value to write - public void WriteShortMSB(int s) - { -#if DebugDeflation - if (DeflaterConstants.DEBUGGING && (start != 0) ) - { - throw new ImageFormatException("Debug check: start != 0"); - } -#endif - buffer[end++] = unchecked((byte)(s >> 8)); - buffer[end++] = unchecked((byte)s); - } - - /// - /// Indicates if buffer has been flushed - /// - public bool IsFlushed - { - get - { - return end == 0; - } - } - - /// - /// Flushes the pending buffer into the given output array. If the - /// output array is to small, only a partial flush is done. - /// - /// The output array. - /// The offset into output array. - /// The maximum number of bytes to store. - /// The number of bytes flushed. - public int Flush(byte[] output, int offset, int length) - { - if (bitCount >= 8) - { - buffer[end++] = unchecked((byte)bits); - bits >>= 8; - bitCount -= 8; - } - - if (length > end - start) - { - length = end - start; - System.Array.Copy(buffer, start, output, offset, length); - start = 0; - end = 0; - } - else - { - System.Array.Copy(buffer, start, output, offset, length); - start += length; - } - return length; - } - - /// - /// Convert internal buffer to byte array. - /// Buffer is empty on completion - /// - /// - /// The internal buffer contents converted to a byte array. - /// - public byte[] ToByteArray() - { - AlignToByte(); - - byte[] result = new byte[end - start]; - System.Array.Copy(buffer, start, result, 0, result.Length); - start = 0; - end = 0; - return result; - } - } -} diff --git a/src/ImageSharp/Formats/Png/Zlib/ZlibDeflateStream.cs b/src/ImageSharp/Formats/Png/Zlib/ZlibDeflateStream.cs index ffe176368..69b3c0602 100644 --- a/src/ImageSharp/Formats/Png/Zlib/ZlibDeflateStream.cs +++ b/src/ImageSharp/Formats/Png/Zlib/ZlibDeflateStream.cs @@ -4,6 +4,7 @@ using System; using System.IO; using System.IO.Compression; +using SixLabors.Memory; namespace SixLabors.ImageSharp.Formats.Png.Zlib { @@ -41,12 +42,15 @@ namespace SixLabors.ImageSharp.Formats.Png.Zlib // private DeflateStream deflateStream; private DeflaterOutputStream deflateStream; + private Deflater deflater; + /// /// Initializes a new instance of the class. /// + /// The memory allocator to use for buffer allocations. /// The stream to compress. /// The compression level. - public ZlibDeflateStream(Stream stream, int compressionLevel) + public ZlibDeflateStream(MemoryAllocator memoryAllocator, Stream stream, int compressionLevel) { this.rawStream = stream; @@ -100,7 +104,8 @@ namespace SixLabors.ImageSharp.Formats.Png.Zlib // { // level = CompressionLevel.NoCompression; // } - this.deflateStream = new DeflaterOutputStream(this.rawStream, new Deflater(compressionLevel, true)) { IsStreamOwner = false }; + this.deflater = new Deflater(memoryAllocator, compressionLevel, true); + this.deflateStream = new DeflaterOutputStream(this.rawStream, this.deflater) { IsStreamOwner = false }; // this.deflateStream = new DeflateStream(this.rawStream, level, true); } @@ -170,6 +175,10 @@ namespace SixLabors.ImageSharp.Formats.Png.Zlib { this.deflateStream.Dispose(); this.deflateStream = null; + + // TODO: Remove temporal coupling here. + this.deflater.Dispose(); + this.deflater = null; } else { From 884ff6db9fd9455bda7d25e63104141fadc9dcd2 Mon Sep 17 00:00:00 2001 From: James Jackson-South Date: Tue, 12 Nov 2019 19:59:30 +1100 Subject: [PATCH 03/16] !st pass cleanup of Deflater --- src/ImageSharp/Formats/Png/Zlib/Deflater.cs | 573 +++++------------- .../Formats/Png/Zlib/DeflaterThrowHelper.cs | 17 + .../Formats/Png/Zlib/ZlibDeflateStream.cs | 9 +- 3 files changed, 165 insertions(+), 434 deletions(-) create mode 100644 src/ImageSharp/Formats/Png/Zlib/DeflaterThrowHelper.cs diff --git a/src/ImageSharp/Formats/Png/Zlib/Deflater.cs b/src/ImageSharp/Formats/Png/Zlib/Deflater.cs index 90e25fb58..26d1f9a45 100644 --- a/src/ImageSharp/Formats/Png/Zlib/Deflater.cs +++ b/src/ImageSharp/Formats/Png/Zlib/Deflater.cs @@ -1,97 +1,92 @@ // Copyright (c) Six Labors and contributors. // Licensed under the Apache License, Version 2.0. -// using System; -using System.Collections.Generic; -using System.Text; +using System.Runtime.CompilerServices; using SixLabors.Memory; namespace SixLabors.ImageSharp.Formats.Png.Zlib { /// - /// This is the Deflater class. The deflater class compresses input - /// with the deflate algorithm described in RFC 1951. It has several - /// compression levels and three different strategies described below. - /// - /// This class is not thread safe. This is inherent in the API, due - /// to the split of deflate and setInput. - /// - /// author of the original java version : Jochen Hoenicke + /// This class compresses input with the deflate algorithm described in RFC 1951. + /// It has several compression levels and three different strategies described below. /// public sealed class Deflater : IDisposable { - #region Deflater Documentation - - /* - * The Deflater can do the following state transitions: - * - * (1) -> INIT_STATE ----> INIT_FINISHING_STATE ---. - * / | (2) (5) | - * / v (5) | - * (3)| SETDICT_STATE ---> SETDICT_FINISHING_STATE |(3) - * \ | (3) | ,--------' - * | | | (3) / - * v v (5) v v - * (1) -> BUSY_STATE ----> FINISHING_STATE - * | (6) - * v - * FINISHED_STATE - * \_____________________________________/ - * | (7) - * v - * CLOSED_STATE - * - * (1) If we should produce a header we start in INIT_STATE, otherwise - * we start in BUSY_STATE. - * (2) A dictionary may be set only when we are in INIT_STATE, then - * we change the state as indicated. - * (3) Whether a dictionary is set or not, on the first call of deflate - * we change to BUSY_STATE. - * (4) -- intentionally left blank -- :) - * (5) FINISHING_STATE is entered, when flush() is called to indicate that - * there is no more INPUT. There are also states indicating, that - * the header wasn't written yet. - * (6) FINISHED_STATE is entered, when everything has been flushed to the - * internal pending output buffer. - * (7) At any time (7) - * - */ - - #endregion Deflater Documentation - - #region Public Constants - /// /// The best and slowest compression level. This tries to find very /// long and distant string repetitions. /// - public const int BEST_COMPRESSION = 9; + public const int BestCompression = 9; /// /// The worst but fastest compression level. /// - public const int BEST_SPEED = 1; + public const int BestSpeed = 1; /// /// The default compression level. /// - public const int DEFAULT_COMPRESSION = -1; + public const int DefaultCompression = -1; /// /// This level won't compress at all but output uncompressed blocks. /// - public const int NO_COMPRESSION = 0; + public const int NoCompression = 0; /// /// The compression method. This is the only method supported so far. /// There is no need to use this constant at all. /// - public const int DEFLATED = 8; + public const int Deflated = 8; - #endregion Public Constants + /// + /// Compression level. + /// + private int level; + + /// + /// The current state. + /// + private int state; + + private DeflaterPendingBuffer pending; + private DeflaterEngine engine; + private bool isDisposed; - #region Public Enum + private const int IsSetDict = 0x01; + private const int IsFlushing = 0x04; + private const int IsFinishing = 0x08; + private const int BusyState = 0x10; + private const int FlushingState = 0x14; + private const int FinishingState = 0x1c; + private const int FinishedState = 0x1e; + private const int ClosedState = 0x7f; + + /// + /// Initializes a new instance of the class. + /// + /// The memory allocator to use for buffer allocations. + /// The compression level, a value between NoCompression and BestCompression. + /// + /// if level is out of range. + public Deflater(MemoryAllocator memoryAllocator, int level) + { + if (level == DefaultCompression) + { + level = 6; + } + else if (level < NoCompression || level > BestCompression) + { + throw new ArgumentOutOfRangeException(nameof(level)); + } + + this.pending = new DeflaterPendingBuffer(memoryAllocator); + this.engine = new DeflaterEngine(this.pending, true); + this.engine.Strategy = DeflateStrategy.Default; + this.SetLevel(level); + this.Reset(); + } /// /// Compression Level as an enum for safer use @@ -102,202 +97,72 @@ namespace SixLabors.ImageSharp.Formats.Png.Zlib /// The best and slowest compression level. This tries to find very /// long and distant string repetitions. /// - BEST_COMPRESSION = Deflater.BEST_COMPRESSION, + BestCompression = Deflater.BestCompression, /// /// The worst but fastest compression level. /// - BEST_SPEED = Deflater.BEST_SPEED, + BestSpeed = Deflater.BestSpeed, /// /// The default compression level. /// - DEFAULT_COMPRESSION = Deflater.DEFAULT_COMPRESSION, + DefaultCompression = Deflater.DefaultCompression, /// /// This level won't compress at all but output uncompressed blocks. /// - NO_COMPRESSION = Deflater.NO_COMPRESSION, + NoCompression = Deflater.NoCompression, /// /// The compression method. This is the only method supported so far. /// There is no need to use this constant at all. /// - DEFLATED = Deflater.DEFLATED + Deflated = Deflater.Deflated } - #endregion Public Enum - - #region Local Constants - - private const int IS_SETDICT = 0x01; - private const int IS_FLUSHING = 0x04; - private const int IS_FINISHING = 0x08; - - private const int INIT_STATE = 0x00; - private const int SETDICT_STATE = 0x01; - - // private static int INIT_FINISHING_STATE = 0x08; - // private static int SETDICT_FINISHING_STATE = 0x09; - private const int BUSY_STATE = 0x10; - - private const int FLUSHING_STATE = 0x14; - private const int FINISHING_STATE = 0x1c; - private const int FINISHED_STATE = 0x1e; - private const int CLOSED_STATE = 0x7f; - - #endregion Local Constants - - #region Constructors - /// - /// Creates a new deflater with given compression level. + /// Gets a value indicating whetherthe stream was finished and no more output bytes + /// are available. /// - /// The memory allocator to use for buffer allocations. - /// - /// the compression level, a value between NO_COMPRESSION - /// and BEST_COMPRESSION. - /// - /// - /// true, if we should suppress the Zlib/RFC1950 header at the - /// beginning and the adler checksum at the end of the output. This is - /// useful for the GZIP/PKZIP formats. - /// - /// if lvl is out of range. - public Deflater(MemoryAllocator memoryAllocator, int level, bool noZlibHeaderOrFooter) - { - if (level == DEFAULT_COMPRESSION) - { - level = 6; - } - else if (level < NO_COMPRESSION || level > BEST_COMPRESSION) - { - throw new ArgumentOutOfRangeException(nameof(level)); - } - - pending = new DeflaterPendingBuffer(memoryAllocator); - engine = new DeflaterEngine(pending, noZlibHeaderOrFooter); - this.noZlibHeaderOrFooter = noZlibHeaderOrFooter; - SetStrategy(DeflateStrategy.Default); - SetLevel(level); - Reset(); - } + public bool IsFinished => (this.state == FinishedState) && this.pending.IsFlushed; - #endregion Constructors + /// + /// Gets a value indicating whether the input buffer is empty. + /// You should then call setInput(). + /// NOTE: This method can also return true when the stream + /// was finished. + /// + public bool IsNeedingInput => this.engine.NeedsInput(); /// /// Resets the deflater. The deflater acts afterwards as if it was /// just created with the same compression level and strategy as it /// had before. /// + [MethodImpl(InliningOptions.ShortMethod)] public void Reset() { - state = (noZlibHeaderOrFooter ? BUSY_STATE : INIT_STATE); - totalOut = 0; - pending.Reset(); - engine.Reset(); - } - - /// - /// Gets the current adler checksum of the data that was processed so far. - /// - public int Adler - { - get - { - return engine.Adler; - } - } - - /// - /// Gets the number of input bytes processed so far. - /// - public long TotalIn - { - get - { - return engine.TotalIn; - } - } - - /// - /// Gets the number of output bytes so far. - /// - public long TotalOut - { - get - { - return totalOut; - } + this.state = BusyState; + this.pending.Reset(); + this.engine.Reset(); } /// - /// Flushes the current input block. Further calls to deflate() will + /// Flushes the current input block. Further calls to Deflate() will /// produce enough output to inflate everything in the current input - /// block. This is not part of Sun's JDK so I have made it package - /// private. It is used by DeflaterOutputStream to implement - /// flush(). + /// block. It is used by DeflaterOutputStream to implement Flush(). /// - public void Flush() - { - state |= IS_FLUSHING; - } + [MethodImpl(InliningOptions.ShortMethod)] + public void Flush() => this.state |= IsFlushing; /// - /// Finishes the deflater with the current input block. It is an error - /// to give more input after this method was called. This method must + /// Finishes the deflater with the current input block. It is an error + /// to give more input after this method was called. This method must /// be called to force all bytes to be flushed. /// - public void Finish() - { - state |= (IS_FLUSHING | IS_FINISHING); - } - - /// - /// Returns true if the stream was finished and no more output bytes - /// are available. - /// - public bool IsFinished - { - get - { - return (state == FINISHED_STATE) && pending.IsFlushed; - } - } - - /// - /// Returns true, if the input buffer is empty. - /// You should then call setInput(). - /// NOTE: This method can also return true when the stream - /// was finished. - /// - public bool IsNeedingInput - { - get - { - return engine.NeedsInput(); - } - } - - /// - /// Sets the data which should be compressed next. This should be only - /// called when needsInput indicates that more input is needed. - /// If you call setInput when needsInput() returns false, the - /// previous input that is still pending will be thrown away. - /// The given byte array should not be changed, before needsInput() returns - /// true again. - /// This call is equivalent to setInput(input, 0, input.length). - /// - /// - /// the buffer containing the input data. - /// - /// - /// if the buffer was finished() or ended(). - /// - public void SetInput(byte[] input) - { - SetInput(input, 0, input.Length); - } + [MethodImpl(InliningOptions.ShortMethod)] + public void Finish() => this.state |= IsFlushing | IsFinishing; /// /// Sets the data which should be compressed next. This should be @@ -305,25 +170,21 @@ namespace SixLabors.ImageSharp.Formats.Png.Zlib /// The given byte array should not be changed, before needsInput() returns /// true again. /// - /// - /// the buffer containing the input data. - /// - /// - /// the start of the data. - /// - /// - /// the number of data bytes of input. - /// - /// - /// if the buffer was Finish()ed or if previous input is still pending. + /// The buffer containing the input data. + /// The start of the data. + /// The number of data bytes of input. + /// + /// if the buffer was finished or if previous input is still pending. /// + [MethodImpl(InliningOptions.ShortMethod)] public void SetInput(byte[] input, int offset, int count) { - if ((state & IS_FINISHING) != 0) + if ((this.state & IsFinishing) != 0) { - throw new InvalidOperationException("Finish() already called"); + DeflaterThrowHelper.ThrowAlreadyFinished(); } - engine.SetInput(input, offset, count); + + this.engine.SetInput(input, offset, count); } /// @@ -337,11 +198,11 @@ namespace SixLabors.ImageSharp.Formats.Png.Zlib /// public void SetLevel(int level) { - if (level == DEFAULT_COMPRESSION) + if (level == DefaultCompression) { level = 6; } - else if (level < NO_COMPRESSION || level > BEST_COMPRESSION) + else if (level < NoCompression || level > BestCompression) { throw new ArgumentOutOfRangeException(nameof(level)); } @@ -349,273 +210,127 @@ namespace SixLabors.ImageSharp.Formats.Png.Zlib if (this.level != level) { this.level = level; - engine.SetLevel(level); + this.engine.SetLevel(level); } } - /// - /// Get current compression level - /// - /// Returns the current compression level - public int GetLevel() - { - return level; - } - - /// - /// Sets the compression strategy. Strategy is one of - /// DEFAULT_STRATEGY, HUFFMAN_ONLY and FILTERED. For the exact - /// position where the strategy is changed, the same as for - /// SetLevel() applies. - /// - /// - /// The new compression strategy. - /// - public void SetStrategy(DeflateStrategy strategy) - { - engine.Strategy = strategy; - } - - /// - /// Deflates the current input block with to the given array. - /// - /// - /// The buffer where compressed data is stored - /// - /// - /// The number of compressed bytes added to the output, or 0 if either - /// IsNeedingInput() or IsFinished returns true or length is zero. - /// - public int Deflate(byte[] output) - { - return Deflate(output, 0, output.Length); - } - /// /// Deflates the current input block to the given array. /// - /// - /// Buffer to store the compressed data. - /// - /// - /// Offset into the output array. - /// - /// - /// The maximum number of bytes that may be stored. - /// + /// Buffer to store the compressed data. + /// Offset into the output array. + /// The maximum number of bytes that may be stored. /// /// The number of compressed bytes added to the output, or 0 if either - /// needsInput() or finished() returns true or length is zero. + /// or returns true or length is zero. /// - /// - /// If Finish() was previously called. - /// - /// - /// If offset or length don't match the array length. - /// public int Deflate(byte[] output, int offset, int length) { int origLength = length; - if (state == CLOSED_STATE) + if (this.state == ClosedState) { - throw new InvalidOperationException("Deflater closed"); + DeflaterThrowHelper.ThrowAlreadyClosed(); } - if (state < BUSY_STATE) + if (this.state < BusyState) { - // output header - int header = (DEFLATED + - ((DeflaterConstants.MAX_WBITS - 8) << 4)) << 8; - int level_flags = (level - 1) >> 1; - if (level_flags < 0 || level_flags > 3) + // Output header + int header = (Deflated + ((DeflaterConstants.MAX_WBITS - 8) << 4)) << 8; + int levelFlags = (this.level - 1) >> 1; + if (levelFlags < 0 || levelFlags > 3) { - level_flags = 3; + levelFlags = 3; } - header |= level_flags << 6; - if ((state & IS_SETDICT) != 0) + + header |= levelFlags << 6; + if ((this.state & IsSetDict) != 0) { // Dictionary was set header |= DeflaterConstants.PRESET_DICT; } + header += 31 - (header % 31); - pending.WriteShortMSB(header); - if ((state & IS_SETDICT) != 0) + this.pending.WriteShortMSB(header); + if ((this.state & IsSetDict) != 0) { - int chksum = engine.Adler; - engine.ResetAdler(); - pending.WriteShortMSB(chksum >> 16); - pending.WriteShortMSB(chksum & 0xffff); + int chksum = this.engine.Adler; + this.engine.ResetAdler(); + this.pending.WriteShortMSB(chksum >> 16); + this.pending.WriteShortMSB(chksum & 0xffff); } - state = BUSY_STATE | (state & (IS_FLUSHING | IS_FINISHING)); + this.state = BusyState | (this.state & (IsFlushing | IsFinishing)); } - for (; ; ) + while (true) { - int count = pending.Flush(output, offset, length); + int count = this.pending.Flush(output, offset, length); offset += count; - totalOut += count; length -= count; - if (length == 0 || state == FINISHED_STATE) + if (length == 0 || this.state == FinishedState) { break; } - if (!engine.Deflate((state & IS_FLUSHING) != 0, (state & IS_FINISHING) != 0)) + if (!this.engine.Deflate((this.state & IsFlushing) != 0, (this.state & IsFinishing) != 0)) { - switch (state) + switch (this.state) { - case BUSY_STATE: + case BusyState: // We need more input now return origLength - length; - case FLUSHING_STATE: - if (level != NO_COMPRESSION) + case FlushingState: + if (this.level != NoCompression) { - /* We have to supply some lookahead. 8 bit lookahead - * is needed by the zlib inflater, and we must fill - * the next byte, so that all bits are flushed. - */ - int neededbits = 8 + ((-pending.BitCount) & 7); + // We have to supply some lookahead. 8 bit lookahead + // is needed by the zlib inflater, and we must fill + // the next byte, so that all bits are flushed. + int neededbits = 8 + ((-this.pending.BitCount) & 7); while (neededbits > 0) { - /* write a static tree block consisting solely of - * an EOF: - */ - pending.WriteBits(2, 10); + // Write a static tree block consisting solely of an EOF: + this.pending.WriteBits(2, 10); neededbits -= 10; } } - state = BUSY_STATE; - break; - case FINISHING_STATE: - pending.AlignToByte(); + this.state = BusyState; + break; - // Compressed data is complete. Write footer information if required. - if (!noZlibHeaderOrFooter) - { - int adler = engine.Adler; - pending.WriteShortMSB(adler >> 16); - pending.WriteShortMSB(adler & 0xffff); - } - state = FINISHED_STATE; + case FinishingState: + this.pending.AlignToByte(); + this.state = FinishedState; break; } } } - return origLength - length; - } - /// - /// Sets the dictionary which should be used in the deflate process. - /// This call is equivalent to setDictionary(dict, 0, dict.Length). - /// - /// - /// the dictionary. - /// - /// - /// if SetInput () or Deflate () were already called or another dictionary was already set. - /// - public void SetDictionary(byte[] dictionary) - { - SetDictionary(dictionary, 0, dictionary.Length); + return origLength - length; } - /// - /// Sets the dictionary which should be used in the deflate process. - /// The dictionary is a byte array containing strings that are - /// likely to occur in the data which should be compressed. The - /// dictionary is not stored in the compressed output, only a - /// checksum. To decompress the output you need to supply the same - /// dictionary again. - /// - /// - /// The dictionary data - /// - /// - /// The index where dictionary information commences. - /// - /// - /// The number of bytes in the dictionary. - /// - /// - /// If SetInput () or Deflate() were already called or another dictionary was already set. - /// - public void SetDictionary(byte[] dictionary, int index, int count) + /// + public void Dispose() { - if (state != INIT_STATE) - { - throw new InvalidOperationException(); - } - - state = SETDICT_STATE; - engine.SetDictionary(dictionary, index, count); + this.Dispose(true); + GC.SuppressFinalize(this); } - #region Instance Fields - - /// - /// Compression level. - /// - private int level; - - /// - /// If true no Zlib/RFC1950 headers or footers are generated - /// - private bool noZlibHeaderOrFooter; - - /// - /// The current state. - /// - private int state; - - /// - /// The total bytes of output written. - /// - private long totalOut; - - /// - /// The pending output. - /// - private DeflaterPendingBuffer pending; - - /// - /// The deflater engine. - /// - private DeflaterEngine engine; - - #region IDisposable Support - private bool disposedValue = false; // To detect redundant calls - - void Dispose(bool disposing) + private void Dispose(bool disposing) { - if (!disposedValue) + if (!this.isDisposed) { if (disposing) { this.pending.Dispose(); - // TODO: dispose managed state (managed objects). } - // TODO: free unmanaged resources (unmanaged objects) and override a finalizer below. - // TODO: set large fields to null. this.pending = null; - disposedValue = true; + this.isDisposed = true; } } - - /// - public void Dispose() - { - Dispose(true); - GC.SuppressFinalize(this); - } - #endregion - - #endregion Instance Fields } } diff --git a/src/ImageSharp/Formats/Png/Zlib/DeflaterThrowHelper.cs b/src/ImageSharp/Formats/Png/Zlib/DeflaterThrowHelper.cs new file mode 100644 index 000000000..b59d32c4d --- /dev/null +++ b/src/ImageSharp/Formats/Png/Zlib/DeflaterThrowHelper.cs @@ -0,0 +1,17 @@ +// Copyright (c) Six Labors and contributors. +// Licensed under the Apache License, Version 2.0. + +using System; +using System.Runtime.CompilerServices; + +namespace SixLabors.ImageSharp.Formats.Png.Zlib +{ + internal static class DeflaterThrowHelper + { + [MethodImpl(InliningOptions.ColdPath)] + public static void ThrowAlreadyFinished() => throw new InvalidOperationException("Finish() already called."); + + [MethodImpl(InliningOptions.ColdPath)] + public static void ThrowAlreadyClosed() => throw new InvalidOperationException("Deflator already closed."); + } +} diff --git a/src/ImageSharp/Formats/Png/Zlib/ZlibDeflateStream.cs b/src/ImageSharp/Formats/Png/Zlib/ZlibDeflateStream.cs index 69b3c0602..5724e027d 100644 --- a/src/ImageSharp/Formats/Png/Zlib/ZlibDeflateStream.cs +++ b/src/ImageSharp/Formats/Png/Zlib/ZlibDeflateStream.cs @@ -3,7 +3,6 @@ using System; using System.IO; -using System.IO.Compression; using SixLabors.Memory; namespace SixLabors.ImageSharp.Formats.Png.Zlib @@ -65,7 +64,7 @@ namespace SixLabors.ImageSharp.Formats.Png.Zlib // +---+---+ // |CMF|FLG| // +---+---+ - int cmf = 0x78; + const int Cmf = 0x78; int flg = 218; // http://stackoverflow.com/a/2331025/277304 @@ -83,14 +82,14 @@ namespace SixLabors.ImageSharp.Formats.Png.Zlib } // Just in case - flg -= ((cmf * 256) + flg) % 31; + flg -= ((Cmf * 256) + flg) % 31; if (flg < 0) { flg += 31; } - this.rawStream.WriteByte((byte)cmf); + this.rawStream.WriteByte(Cmf); this.rawStream.WriteByte((byte)flg); // Initialize the deflate Stream. @@ -104,7 +103,7 @@ namespace SixLabors.ImageSharp.Formats.Png.Zlib // { // level = CompressionLevel.NoCompression; // } - this.deflater = new Deflater(memoryAllocator, compressionLevel, true); + this.deflater = new Deflater(memoryAllocator, compressionLevel); this.deflateStream = new DeflaterOutputStream(this.rawStream, this.deflater) { IsStreamOwner = false }; // this.deflateStream = new DeflateStream(this.rawStream, level, true); From c18d1fef7876ad95c163d46d2e85638fc95de696 Mon Sep 17 00:00:00 2001 From: James Jackson-South Date: Tue, 12 Nov 2019 20:35:37 +1100 Subject: [PATCH 04/16] Stylecop cleanup DeflaterEngine --- ...erThrowHelper.cs => DeflateThrowHelper.cs} | 2 +- src/ImageSharp/Formats/Png/Zlib/Deflater.cs | 4 +- .../Formats/Png/Zlib/DeflaterEngine.cs | 840 ++++++++---------- 3 files changed, 395 insertions(+), 451 deletions(-) rename src/ImageSharp/Formats/Png/Zlib/{DeflaterThrowHelper.cs => DeflateThrowHelper.cs} (92%) diff --git a/src/ImageSharp/Formats/Png/Zlib/DeflaterThrowHelper.cs b/src/ImageSharp/Formats/Png/Zlib/DeflateThrowHelper.cs similarity index 92% rename from src/ImageSharp/Formats/Png/Zlib/DeflaterThrowHelper.cs rename to src/ImageSharp/Formats/Png/Zlib/DeflateThrowHelper.cs index b59d32c4d..a360e5e87 100644 --- a/src/ImageSharp/Formats/Png/Zlib/DeflaterThrowHelper.cs +++ b/src/ImageSharp/Formats/Png/Zlib/DeflateThrowHelper.cs @@ -6,7 +6,7 @@ using System.Runtime.CompilerServices; namespace SixLabors.ImageSharp.Formats.Png.Zlib { - internal static class DeflaterThrowHelper + internal static class DeflateThrowHelper { [MethodImpl(InliningOptions.ColdPath)] public static void ThrowAlreadyFinished() => throw new InvalidOperationException("Finish() already called."); diff --git a/src/ImageSharp/Formats/Png/Zlib/Deflater.cs b/src/ImageSharp/Formats/Png/Zlib/Deflater.cs index 26d1f9a45..33b3019b8 100644 --- a/src/ImageSharp/Formats/Png/Zlib/Deflater.cs +++ b/src/ImageSharp/Formats/Png/Zlib/Deflater.cs @@ -181,7 +181,7 @@ namespace SixLabors.ImageSharp.Formats.Png.Zlib { if ((this.state & IsFinishing) != 0) { - DeflaterThrowHelper.ThrowAlreadyFinished(); + DeflateThrowHelper.ThrowAlreadyFinished(); } this.engine.SetInput(input, offset, count); @@ -230,7 +230,7 @@ namespace SixLabors.ImageSharp.Formats.Png.Zlib if (this.state == ClosedState) { - DeflaterThrowHelper.ThrowAlreadyClosed(); + DeflateThrowHelper.ThrowAlreadyClosed(); } if (this.state < BusyState) diff --git a/src/ImageSharp/Formats/Png/Zlib/DeflaterEngine.cs b/src/ImageSharp/Formats/Png/Zlib/DeflaterEngine.cs index c9967056e..ddf571884 100644 --- a/src/ImageSharp/Formats/Png/Zlib/DeflaterEngine.cs +++ b/src/ImageSharp/Formats/Png/Zlib/DeflaterEngine.cs @@ -1,7 +1,6 @@ // Copyright (c) Six Labors and contributors. // Licensed under the Apache License, Version 2.0. -// using System; using System.Collections.Generic; using System.Text; @@ -51,54 +50,145 @@ namespace SixLabors.ImageSharp.Formats.Png.Zlib /// public class DeflaterEngine { - #region Constants - private const int TooFar = 4096; - #endregion Constants + // Hash index of string to be inserted + private int insertHashIndex; - #region Constructors + /// + /// Hashtable, hashing three characters to an index for window, so + /// that window[index]..window[index+2] have this hash code. + /// Note that the array should really be unsigned short, so you need + /// to and the values with 0xFFFF. + /// + private short[] head; /// - /// Construct instance with pending buffer - /// Adler calculation will be peformed + /// prev[index & WMASK] points to the previous index that has the + /// same hash code as the string starting at index. This way + /// entries with the same hash code are in a linked list. + /// Note that the array should really be unsigned short, so you need + /// to and the values with 0xFFFF. /// - /// - /// Pending buffer to use - /// - public DeflaterEngine(DeflaterPendingBuffer pending) - : this(pending, false) - { - } + private short[] prev; + private int matchStart; + // Length of best match + private int matchLen; + + // Set if previous match exists + private bool prevAvailable; + + private int blockStart; /// - /// Construct instance with pending buffer + /// Points to the current character in the window. /// - /// - /// Pending buffer to use - /// + private int strstart; + + /// + /// lookahead is the number of characters starting at strstart in + /// window that are valid. + /// So window[strstart] until window[strstart+lookahead-1] are valid + /// characters. + /// + private int lookahead; + + /// + /// This array contains the part of the uncompressed stream that + /// is of relevance. The current character is indexed by strstart. + /// + private byte[] window; + private int maxChain; + private int maxLazy; + private int niceLength; + private int goodLength; + + /// + /// The current compression function. + /// + private int compressionFunction; + + /// + /// The input data for compression. + /// + private byte[] inputBuf; + + /// + /// The total bytes of input read. + /// + private long totalIn; + + /// + /// The offset into inputBuf, where input data starts. + /// + private int inputOff; + + /// + /// The end offset of the input data. + /// + private int inputEnd; + + private DeflaterPendingBuffer pending; + private DeflaterHuffman huffman; + + /// + /// The adler checksum + /// + private Adler32 adler; + + /// + /// Initializes a new instance of the class. + /// + /// The pending buffer to use. /// /// If no adler calculation should be performed /// public DeflaterEngine(DeflaterPendingBuffer pending, bool noAdlerCalculation) { this.pending = pending; - huffman = new DeflaterHuffman(pending); + this.huffman = new DeflaterHuffman(pending); if (!noAdlerCalculation) - adler = new Adler32(); + { + this.adler = new Adler32(); + } - window = new byte[2 * DeflaterConstants.WSIZE]; - head = new short[DeflaterConstants.HASH_SIZE]; - prev = new short[DeflaterConstants.WSIZE]; + this.window = new byte[2 * DeflaterConstants.WSIZE]; + this.head = new short[DeflaterConstants.HASH_SIZE]; + this.prev = new short[DeflaterConstants.WSIZE]; // We start at index 1, to avoid an implementation deficiency, that // we cannot build a repeat pattern at index 0. - blockStart = strstart = 1; + this.blockStart = this.strstart = 1; } - #endregion Constructors + /// + /// Gets the current value of Adler checksum + /// + public int Adler + { + get + { + return (this.adler != null) ? unchecked((int)this.adler.Value) : 0; + } + } + + /// + /// Gets the total data processed + /// + public long TotalIn + { + get + { + return this.totalIn; + } + } + + /// + /// Gets or sets the deflate strategy + /// + public DeflateStrategy Strategy { get; set; } /// /// Deflate drives actual compression of data @@ -111,33 +201,28 @@ namespace SixLabors.ImageSharp.Formats.Png.Zlib bool progress; do { - FillWindow(); - bool canFlush = flush && (inputOff == inputEnd); + this.FillWindow(); + bool canFlush = flush && (this.inputOff == this.inputEnd); -#if DebugDeflation - if (DeflaterConstants.DEBUGGING) { - Console.WriteLine("window: [" + blockStart + "," + strstart + "," - + lookahead + "], " + compressionFunction + "," + canFlush); - } -#endif - switch (compressionFunction) + switch (this.compressionFunction) { case DeflaterConstants.DEFLATE_STORED: - progress = DeflateStored(canFlush, finish); + progress = this.DeflateStored(canFlush, finish); break; case DeflaterConstants.DEFLATE_FAST: - progress = DeflateFast(canFlush, finish); + progress = this.DeflateFast(canFlush, finish); break; case DeflaterConstants.DEFLATE_SLOW: - progress = DeflateSlow(canFlush, finish); + progress = this.DeflateSlow(canFlush, finish); break; default: throw new InvalidOperationException("unknown compressionFunction"); } - } while (pending.IsFlushed && progress); // repeat while we have no pending output and progress was made + } + while (this.pending.IsFlushed && progress); // repeat while we have no pending output and progress was made return progress; } @@ -165,24 +250,23 @@ namespace SixLabors.ImageSharp.Formats.Png.Zlib throw new ArgumentOutOfRangeException(nameof(count)); } - if (inputOff < inputEnd) + if (this.inputOff < this.inputEnd) { - throw new InvalidOperationException("Old input was not completely processed"); + throw new InvalidOperationException("Old input was not completely processed."); } int end = offset + count; - /* We want to throw an ArrayIndexOutOfBoundsException early. The - * check is very tricky: it also handles integer wrap around. - */ + // We want to throw an ArrayIndexOutOfBoundsException early. + // The check is very tricky: it also handles integer wrap around. if ((offset > end) || (end > buffer.Length)) { throw new ArgumentOutOfRangeException(nameof(count)); } - inputBuf = buffer; - inputOff = offset; - inputEnd = end; + this.inputBuf = buffer; + this.inputOff = offset; + this.inputEnd = end; } /// @@ -191,7 +275,7 @@ namespace SixLabors.ImageSharp.Formats.Png.Zlib /// Return true if input is needed via SetInput public bool NeedsInput() { - return (inputEnd == inputOff); + return this.inputEnd == this.inputOff; } /// @@ -202,13 +286,7 @@ namespace SixLabors.ImageSharp.Formats.Png.Zlib /// The length of the dictionary data. public void SetDictionary(byte[] buffer, int offset, int length) { -#if DebugDeflation - if (DeflaterConstants.DEBUGGING && (strstart != 1) ) - { - throw new InvalidOperationException("strstart not 1"); - } -#endif - adler?.Update(new ArraySegment(buffer, offset, length)); + this.adler?.Update(new ArraySegment(buffer, offset, length)); if (length < DeflaterConstants.MIN_MATCH) { return; @@ -220,17 +298,18 @@ namespace SixLabors.ImageSharp.Formats.Png.Zlib length = DeflaterConstants.MAX_DIST; } - System.Array.Copy(buffer, offset, window, strstart, length); + System.Array.Copy(buffer, offset, this.window, this.strstart, length); - UpdateHash(); + this.UpdateHash(); --length; while (--length > 0) { - InsertString(); - strstart++; + this.InsertString(); + this.strstart++; } - strstart += 2; - blockStart = strstart; + + this.strstart += 2; + this.blockStart = this.strstart; } /// @@ -238,22 +317,22 @@ namespace SixLabors.ImageSharp.Formats.Png.Zlib /// public void Reset() { - huffman.Reset(); - adler?.Reset(); - blockStart = strstart = 1; - lookahead = 0; - totalIn = 0; - prevAvailable = false; - matchLen = DeflaterConstants.MIN_MATCH - 1; + this.huffman.Reset(); + this.adler?.Reset(); + this.blockStart = this.strstart = 1; + this.lookahead = 0; + this.totalIn = 0; + this.prevAvailable = false; + this.matchLen = DeflaterConstants.MIN_MATCH - 1; for (int i = 0; i < DeflaterConstants.HASH_SIZE; i++) { - head[i] = 0; + this.head[i] = 0; } for (int i = 0; i < DeflaterConstants.WSIZE; i++) { - prev[i] = 0; + this.prev[i] = 0; } } @@ -262,44 +341,7 @@ namespace SixLabors.ImageSharp.Formats.Png.Zlib /// public void ResetAdler() { - adler?.Reset(); - } - - /// - /// Get current value of Adler checksum - /// - public int Adler - { - get - { - return (adler != null) ? unchecked((int)adler.Value) : 0; - } - } - - /// - /// Total data processed - /// - public long TotalIn - { - get - { - return totalIn; - } - } - - /// - /// Get/set the deflate strategy - /// - public DeflateStrategy Strategy - { - get - { - return strategy; - } - set - { - strategy = value; - } + this.adler?.Reset(); } /// @@ -313,55 +355,52 @@ namespace SixLabors.ImageSharp.Formats.Png.Zlib throw new ArgumentOutOfRangeException(nameof(level)); } - goodLength = DeflaterConstants.GOOD_LENGTH[level]; - max_lazy = DeflaterConstants.MAX_LAZY[level]; - niceLength = DeflaterConstants.NICE_LENGTH[level]; - max_chain = DeflaterConstants.MAX_CHAIN[level]; + this.goodLength = DeflaterConstants.GOOD_LENGTH[level]; + this.maxLazy = DeflaterConstants.MAX_LAZY[level]; + this.niceLength = DeflaterConstants.NICE_LENGTH[level]; + this.maxChain = DeflaterConstants.MAX_CHAIN[level]; - if (DeflaterConstants.COMPR_FUNC[level] != compressionFunction) + if (DeflaterConstants.COMPR_FUNC[level] != this.compressionFunction) { -#if DebugDeflation - if (DeflaterConstants.DEBUGGING) { - Console.WriteLine("Change from " + compressionFunction + " to " - + DeflaterConstants.COMPR_FUNC[level]); - } -#endif - switch (compressionFunction) + switch (this.compressionFunction) { case DeflaterConstants.DEFLATE_STORED: - if (strstart > blockStart) + if (this.strstart > this.blockStart) { - huffman.FlushStoredBlock(window, blockStart, - strstart - blockStart, false); - blockStart = strstart; + this.huffman.FlushStoredBlock(this.window, this.blockStart, this.strstart - this.blockStart, false); + this.blockStart = this.strstart; } - UpdateHash(); + + this.UpdateHash(); break; case DeflaterConstants.DEFLATE_FAST: - if (strstart > blockStart) + if (this.strstart > this.blockStart) { - huffman.FlushBlock(window, blockStart, strstart - blockStart, - false); - blockStart = strstart; + this.huffman.FlushBlock(this.window, this.blockStart, this.strstart - this.blockStart, false); + this.blockStart = this.strstart; } + break; case DeflaterConstants.DEFLATE_SLOW: - if (prevAvailable) + if (this.prevAvailable) { - huffman.TallyLit(window[strstart - 1] & 0xff); + this.huffman.TallyLit(this.window[this.strstart - 1] & 0xff); } - if (strstart > blockStart) + + if (this.strstart > this.blockStart) { - huffman.FlushBlock(window, blockStart, strstart - blockStart, false); - blockStart = strstart; + this.huffman.FlushBlock(this.window, this.blockStart, this.strstart - this.blockStart, false); + this.blockStart = this.strstart; } - prevAvailable = false; - matchLen = DeflaterConstants.MIN_MATCH - 1; + + this.prevAvailable = false; + this.matchLen = DeflaterConstants.MIN_MATCH - 1; break; } - compressionFunction = DeflaterConstants.COMPR_FUNC[level]; + + this.compressionFunction = DeflaterConstants.COMPR_FUNC[level]; } } @@ -370,48 +409,40 @@ namespace SixLabors.ImageSharp.Formats.Png.Zlib /// public void FillWindow() { - /* If the window is almost full and there is insufficient lookahead, - * move the upper half to the lower one to make room in the upper half. - */ - if (strstart >= DeflaterConstants.WSIZE + DeflaterConstants.MAX_DIST) + // If the window is almost full and there is insufficient lookahead, + // move the upper half to the lower one to make room in the upper half. + if (this.strstart >= DeflaterConstants.WSIZE + DeflaterConstants.MAX_DIST) { - SlideWindow(); + this.SlideWindow(); } - /* If there is not enough lookahead, but still some input left, - * read in the input - */ - if (lookahead < DeflaterConstants.MIN_LOOKAHEAD && inputOff < inputEnd) + // If there is not enough lookahead, but still some input left, read in the input. + if (this.lookahead < DeflaterConstants.MIN_LOOKAHEAD && this.inputOff < this.inputEnd) { - int more = 2 * DeflaterConstants.WSIZE - lookahead - strstart; + int more = (2 * DeflaterConstants.WSIZE) - this.lookahead - this.strstart; - if (more > inputEnd - inputOff) + if (more > this.inputEnd - this.inputOff) { - more = inputEnd - inputOff; + more = this.inputEnd - this.inputOff; } - System.Array.Copy(inputBuf, inputOff, window, strstart + lookahead, more); - adler?.Update(new ArraySegment(inputBuf, inputOff, more)); + Array.Copy(this.inputBuf, this.inputOff, this.window, this.strstart + this.lookahead, more); + this.adler?.Update(new ArraySegment(this.inputBuf, this.inputOff, more)); - inputOff += more; - totalIn += more; - lookahead += more; + this.inputOff += more; + this.totalIn += more; + this.lookahead += more; } - if (lookahead >= DeflaterConstants.MIN_MATCH) + if (this.lookahead >= DeflaterConstants.MIN_MATCH) { - UpdateHash(); + this.UpdateHash(); } } private void UpdateHash() { - /* - if (DEBUGGING) { - Console.WriteLine("updateHash: "+strstart); - } - */ - ins_h = (window[strstart] << DeflaterConstants.HASH_SHIFT) ^ window[strstart + 1]; + this.insertHashIndex = (this.window[this.strstart] << DeflaterConstants.HASH_SHIFT) ^ this.window[this.strstart + 1]; } /// @@ -422,90 +453,87 @@ namespace SixLabors.ImageSharp.Formats.Png.Zlib private int InsertString() { short match; - int hash = ((ins_h << DeflaterConstants.HASH_SHIFT) ^ window[strstart + (DeflaterConstants.MIN_MATCH - 1)]) & DeflaterConstants.HASH_MASK; + int hash = ((this.insertHashIndex << DeflaterConstants.HASH_SHIFT) ^ this.window[this.strstart + (DeflaterConstants.MIN_MATCH - 1)]) & DeflaterConstants.HASH_MASK; -#if DebugDeflation - if (DeflaterConstants.DEBUGGING) - { - if (hash != (((window[strstart] << (2*HASH_SHIFT)) ^ - (window[strstart + 1] << HASH_SHIFT) ^ - (window[strstart + 2])) & HASH_MASK)) { - throw new ImageFormatException("hash inconsistent: " + hash + "/" - +window[strstart] + "," - +window[strstart + 1] + "," - +window[strstart + 2] + "," + HASH_SHIFT); - } - } -#endif - prev[strstart & DeflaterConstants.WMASK] = match = head[hash]; - head[hash] = unchecked((short)strstart); - ins_h = hash; - return match & 0xffff; + this.prev[this.strstart & DeflaterConstants.WMASK] = match = this.head[hash]; + this.head[hash] = unchecked((short)this.strstart); + this.insertHashIndex = hash; + return match & 0xFFFF; } private void SlideWindow() { - Array.Copy(window, DeflaterConstants.WSIZE, window, 0, DeflaterConstants.WSIZE); - matchStart -= DeflaterConstants.WSIZE; - strstart -= DeflaterConstants.WSIZE; - blockStart -= DeflaterConstants.WSIZE; + Array.Copy(this.window, DeflaterConstants.WSIZE, this.window, 0, DeflaterConstants.WSIZE); + this.matchStart -= DeflaterConstants.WSIZE; + this.strstart -= DeflaterConstants.WSIZE; + this.blockStart -= DeflaterConstants.WSIZE; // Slide the hash table (could be avoided with 32 bit values // at the expense of memory usage). for (int i = 0; i < DeflaterConstants.HASH_SIZE; ++i) { - int m = head[i] & 0xffff; - head[i] = (short)(m >= DeflaterConstants.WSIZE ? (m - DeflaterConstants.WSIZE) : 0); + int m = this.head[i] & 0xFFFF; + this.head[i] = (short)(m >= DeflaterConstants.WSIZE ? (m - DeflaterConstants.WSIZE) : 0); } // Slide the prev table. for (int i = 0; i < DeflaterConstants.WSIZE; i++) { - int m = prev[i] & 0xffff; - prev[i] = (short)(m >= DeflaterConstants.WSIZE ? (m - DeflaterConstants.WSIZE) : 0); + int m = this.prev[i] & 0xFFFF; + this.prev[i] = (short)(m >= DeflaterConstants.WSIZE ? (m - DeflaterConstants.WSIZE) : 0); } } /// + /// /// Find the best (longest) string in the window matching the /// string starting at strstart. - /// + /// + /// /// Preconditions: /// /// strstart + DeflaterConstants.MAX_MATCH <= window.length. + /// /// - /// + /// The current match. /// True if a match greater than the minimum length is found private bool FindLongestMatch(int curMatch) { int match; - int scan = strstart; + int scan = this.strstart; + // scanMax is the highest position that we can look at - int scanMax = scan + Math.Min(DeflaterConstants.MAX_MATCH, lookahead) - 1; + int scanMax = scan + Math.Min(DeflaterConstants.MAX_MATCH, this.lookahead) - 1; int limit = Math.Max(scan - DeflaterConstants.MAX_DIST, 0); byte[] window = this.window; short[] prev = this.prev; - int chainLength = this.max_chain; - int niceLength = Math.Min(this.niceLength, lookahead); + int chainLength = this.maxChain; + int niceLength = Math.Min(this.niceLength, this.lookahead); - matchLen = Math.Max(matchLen, DeflaterConstants.MIN_MATCH - 1); + this.matchLen = Math.Max(this.matchLen, DeflaterConstants.MIN_MATCH - 1); - if (scan + matchLen > scanMax) return false; + if (scan + this.matchLen > scanMax) + { + return false; + } - byte scan_end1 = window[scan + matchLen - 1]; - byte scan_end = window[scan + matchLen]; + byte scan_end1 = window[scan + this.matchLen - 1]; + byte scan_end = window[scan + this.matchLen]; // Do not waste too much time if we already have a good match: - if (matchLen >= this.goodLength) chainLength >>= 2; + if (this.matchLen >= this.goodLength) + { + chainLength >>= 2; + } do { match = curMatch; - scan = strstart; + scan = this.strstart; - if (window[match + matchLen] != scan_end - || window[match + matchLen - 1] != scan_end1 + if (window[match + this.matchLen] != scan_end + || window[match + this.matchLen - 1] != scan_end1 || window[match] != window[scan] || window[++match] != window[++scan]) { @@ -516,120 +544,144 @@ namespace SixLabors.ImageSharp.Formats.Png.Zlib // scanMax - scan is the maximum number of bytes we can compare. // below we compare 8 bytes at a time, so first we compare // (scanMax - scan) % 8 bytes, so the remainder is a multiple of 8 - switch ((scanMax - scan) % 8) { case 1: - if (window[++scan] == window[++match]) break; + if (window[++scan] == window[++match]) + { + break; + } + break; case 2: if (window[++scan] == window[++match] - && window[++scan] == window[++match]) break; + && window[++scan] == window[++match]) + { + break; + } + break; case 3: if (window[++scan] == window[++match] - && window[++scan] == window[++match] - && window[++scan] == window[++match]) break; + && window[++scan] == window[++match] + && window[++scan] == window[++match]) + { + break; + } + break; case 4: if (window[++scan] == window[++match] - && window[++scan] == window[++match] - && window[++scan] == window[++match] - && window[++scan] == window[++match]) break; + && window[++scan] == window[++match] + && window[++scan] == window[++match] + && window[++scan] == window[++match]) + { + break; + } + break; case 5: if (window[++scan] == window[++match] - && window[++scan] == window[++match] - && window[++scan] == window[++match] - && window[++scan] == window[++match] - && window[++scan] == window[++match]) break; + && window[++scan] == window[++match] + && window[++scan] == window[++match] + && window[++scan] == window[++match] + && window[++scan] == window[++match]) + { + break; + } + break; case 6: if (window[++scan] == window[++match] - && window[++scan] == window[++match] - && window[++scan] == window[++match] - && window[++scan] == window[++match] - && window[++scan] == window[++match] - && window[++scan] == window[++match]) break; + && window[++scan] == window[++match] + && window[++scan] == window[++match] + && window[++scan] == window[++match] + && window[++scan] == window[++match] + && window[++scan] == window[++match]) + { + break; + } + break; case 7: if (window[++scan] == window[++match] - && window[++scan] == window[++match] - && window[++scan] == window[++match] - && window[++scan] == window[++match] - && window[++scan] == window[++match] - && window[++scan] == window[++match] - && window[++scan] == window[++match]) break; + && window[++scan] == window[++match] + && window[++scan] == window[++match] + && window[++scan] == window[++match] + && window[++scan] == window[++match] + && window[++scan] == window[++match] + && window[++scan] == window[++match]) + { + break; + } + break; } if (window[scan] == window[match]) { - /* We check for insufficient lookahead only every 8th comparison; - * the 256th check will be made at strstart + 258 unless lookahead is - * exhausted first. - */ + // We check for insufficient lookahead only every 8th comparison; + // the 256th check will be made at strstart + 258 unless lookahead is + // exhausted first. do { if (scan == scanMax) { - ++scan; // advance to first position not matched + ++scan; // advance to first position not matched ++match; break; } } while (window[++scan] == window[++match] - && window[++scan] == window[++match] - && window[++scan] == window[++match] - && window[++scan] == window[++match] - && window[++scan] == window[++match] - && window[++scan] == window[++match] - && window[++scan] == window[++match] - && window[++scan] == window[++match]); + && window[++scan] == window[++match] + && window[++scan] == window[++match] + && window[++scan] == window[++match] + && window[++scan] == window[++match] + && window[++scan] == window[++match] + && window[++scan] == window[++match] + && window[++scan] == window[++match]); } - if (scan - strstart > matchLen) + if (scan - this.strstart > this.matchLen) { -#if DebugDeflation - if (DeflaterConstants.DEBUGGING && (ins_h == 0) ) - Console.Error.WriteLine("Found match: " + curMatch + "-" + (scan - strstart)); -#endif + this.matchStart = curMatch; + this.matchLen = scan - this.strstart; - matchStart = curMatch; - matchLen = scan - strstart; - - if (matchLen >= niceLength) + if (this.matchLen >= niceLength) + { break; + } scan_end1 = window[scan - 1]; scan_end = window[scan]; } - } while ((curMatch = (prev[curMatch & DeflaterConstants.WMASK] & 0xffff)) > limit && 0 != --chainLength); + } + while ((curMatch = prev[curMatch & DeflaterConstants.WMASK] & 0xFFFF) > limit && --chainLength != 0); - return matchLen >= DeflaterConstants.MIN_MATCH; + return this.matchLen >= DeflaterConstants.MIN_MATCH; } private bool DeflateStored(bool flush, bool finish) { - if (!flush && (lookahead == 0)) + if (!flush && (this.lookahead == 0)) { return false; } - strstart += lookahead; - lookahead = 0; + this.strstart += this.lookahead; + this.lookahead = 0; - int storedLength = strstart - blockStart; + int storedLength = this.strstart - this.blockStart; if ((storedLength >= DeflaterConstants.MAX_BLOCK_SIZE) || // Block is full - (blockStart < DeflaterConstants.WSIZE && storedLength >= DeflaterConstants.MAX_DIST) || // Block may move out of window + (this.blockStart < DeflaterConstants.WSIZE && storedLength >= DeflaterConstants.MAX_DIST) || // Block may move out of window flush) { bool lastBlock = finish; @@ -639,86 +691,70 @@ namespace SixLabors.ImageSharp.Formats.Png.Zlib lastBlock = false; } -#if DebugDeflation - if (DeflaterConstants.DEBUGGING) - { - Console.WriteLine("storedBlock[" + storedLength + "," + lastBlock + "]"); - } -#endif - - huffman.FlushStoredBlock(window, blockStart, storedLength, lastBlock); - blockStart += storedLength; + this.huffman.FlushStoredBlock(this.window, this.blockStart, storedLength, lastBlock); + this.blockStart += storedLength; return !(lastBlock || storedLength == 0); } + return true; } private bool DeflateFast(bool flush, bool finish) { - if (lookahead < DeflaterConstants.MIN_LOOKAHEAD && !flush) + if (this.lookahead < DeflaterConstants.MIN_LOOKAHEAD && !flush) { return false; } - while (lookahead >= DeflaterConstants.MIN_LOOKAHEAD || flush) + while (this.lookahead >= DeflaterConstants.MIN_LOOKAHEAD || flush) { - if (lookahead == 0) + if (this.lookahead == 0) { // We are flushing everything - huffman.FlushBlock(window, blockStart, strstart - blockStart, finish); - blockStart = strstart; + this.huffman.FlushBlock(this.window, this.blockStart, this.strstart - this.blockStart, finish); + this.blockStart = this.strstart; return false; } - if (strstart > 2 * DeflaterConstants.WSIZE - DeflaterConstants.MIN_LOOKAHEAD) + if (this.strstart > (2 * DeflaterConstants.WSIZE) - DeflaterConstants.MIN_LOOKAHEAD) { - /* slide window, as FindLongestMatch needs this. - * This should only happen when flushing and the window - * is almost full. - */ - SlideWindow(); + // slide window, as FindLongestMatch needs this. + // This should only happen when flushing and the window + // is almost full. + this.SlideWindow(); } int hashHead; - if (lookahead >= DeflaterConstants.MIN_MATCH && - (hashHead = InsertString()) != 0 && - strategy != DeflateStrategy.HuffmanOnly && - strstart - hashHead <= DeflaterConstants.MAX_DIST && - FindLongestMatch(hashHead)) + if (this.lookahead >= DeflaterConstants.MIN_MATCH && + (hashHead = this.InsertString()) != 0 && + this.Strategy != DeflateStrategy.HuffmanOnly && + this.strstart - hashHead <= DeflaterConstants.MAX_DIST && + this.FindLongestMatch(hashHead)) { // longestMatch sets matchStart and matchLen -#if DebugDeflation - if (DeflaterConstants.DEBUGGING) - { - for (int i = 0 ; i < matchLen; i++) { - if (window[strstart + i] != window[matchStart + i]) { - throw new ImageFormatException("Match failure"); - } - } - } -#endif + bool full = this.huffman.TallyDist(this.strstart - this.matchStart, this.matchLen); - bool full = huffman.TallyDist(strstart - matchStart, matchLen); - - lookahead -= matchLen; - if (matchLen <= max_lazy && lookahead >= DeflaterConstants.MIN_MATCH) + this.lookahead -= this.matchLen; + if (this.matchLen <= this.maxLazy && this.lookahead >= DeflaterConstants.MIN_MATCH) { - while (--matchLen > 0) + while (--this.matchLen > 0) { - ++strstart; - InsertString(); + ++this.strstart; + this.InsertString(); } - ++strstart; + + ++this.strstart; } else { - strstart += matchLen; - if (lookahead >= DeflaterConstants.MIN_MATCH - 1) + this.strstart += this.matchLen; + if (this.lookahead >= DeflaterConstants.MIN_MATCH - 1) { - UpdateHash(); + this.UpdateHash(); } } - matchLen = DeflaterConstants.MIN_MATCH - 1; + + this.matchLen = DeflaterConstants.MIN_MATCH - 1; if (!full) { continue; @@ -727,84 +763,77 @@ namespace SixLabors.ImageSharp.Formats.Png.Zlib else { // No match found - huffman.TallyLit(window[strstart] & 0xff); - ++strstart; - --lookahead; + this.huffman.TallyLit(this.window[this.strstart] & 0xff); + ++this.strstart; + --this.lookahead; } - if (huffman.IsFull()) + if (this.huffman.IsFull()) { - bool lastBlock = finish && (lookahead == 0); - huffman.FlushBlock(window, blockStart, strstart - blockStart, lastBlock); - blockStart = strstart; + bool lastBlock = finish && (this.lookahead == 0); + this.huffman.FlushBlock(this.window, this.blockStart, this.strstart - this.blockStart, lastBlock); + this.blockStart = this.strstart; return !lastBlock; } } + return true; } private bool DeflateSlow(bool flush, bool finish) { - if (lookahead < DeflaterConstants.MIN_LOOKAHEAD && !flush) + if (this.lookahead < DeflaterConstants.MIN_LOOKAHEAD && !flush) { return false; } - while (lookahead >= DeflaterConstants.MIN_LOOKAHEAD || flush) + while (this.lookahead >= DeflaterConstants.MIN_LOOKAHEAD || flush) { - if (lookahead == 0) + if (this.lookahead == 0) { - if (prevAvailable) + if (this.prevAvailable) { - huffman.TallyLit(window[strstart - 1] & 0xff); + this.huffman.TallyLit(this.window[this.strstart - 1] & 0xff); } - prevAvailable = false; + + this.prevAvailable = false; // We are flushing everything -#if DebugDeflation - if (DeflaterConstants.DEBUGGING && !flush) - { - throw new ImageFormatException("Not flushing, but no lookahead"); - } -#endif - huffman.FlushBlock(window, blockStart, strstart - blockStart, - finish); - blockStart = strstart; + this.huffman.FlushBlock(this.window, this.blockStart, this.strstart - this.blockStart, finish); + this.blockStart = this.strstart; return false; } - if (strstart >= 2 * DeflaterConstants.WSIZE - DeflaterConstants.MIN_LOOKAHEAD) + if (this.strstart >= (2 * DeflaterConstants.WSIZE) - DeflaterConstants.MIN_LOOKAHEAD) { - /* slide window, as FindLongestMatch needs this. - * This should only happen when flushing and the window - * is almost full. - */ - SlideWindow(); + // slide window, as FindLongestMatch needs this. + // This should only happen when flushing and the window + // is almost full. + this.SlideWindow(); } - int prevMatch = matchStart; - int prevLen = matchLen; - if (lookahead >= DeflaterConstants.MIN_MATCH) + int prevMatch = this.matchStart; + int prevLen = this.matchLen; + if (this.lookahead >= DeflaterConstants.MIN_MATCH) { - int hashHead = InsertString(); + int hashHead = this.InsertString(); - if (strategy != DeflateStrategy.HuffmanOnly && + if (this.Strategy != DeflateStrategy.HuffmanOnly && hashHead != 0 && - strstart - hashHead <= DeflaterConstants.MAX_DIST && - FindLongestMatch(hashHead)) + this.strstart - hashHead <= DeflaterConstants.MAX_DIST && + this.FindLongestMatch(hashHead)) { // longestMatch sets matchStart and matchLen - // Discard match if too small and too far away - if (matchLen <= 5 && (strategy == DeflateStrategy.Filtered || (matchLen == DeflaterConstants.MIN_MATCH && strstart - matchStart > TooFar))) + if (this.matchLen <= 5 && (this.Strategy == DeflateStrategy.Filtered || (this.matchLen == DeflaterConstants.MIN_MATCH && this.strstart - this.matchStart > TooFar))) { - matchLen = DeflaterConstants.MIN_MATCH - 1; + this.matchLen = DeflaterConstants.MIN_MATCH - 1; } } } // previous match was better - if ((prevLen >= DeflaterConstants.MIN_MATCH) && (matchLen <= prevLen)) + if ((prevLen >= DeflaterConstants.MIN_MATCH) && (this.matchLen <= prevLen)) { #if DebugDeflation if (DeflaterConstants.DEBUGGING) @@ -815,137 +844,52 @@ namespace SixLabors.ImageSharp.Formats.Png.Zlib } } #endif - huffman.TallyDist(strstart - 1 - prevMatch, prevLen); + this.huffman.TallyDist(this.strstart - 1 - prevMatch, prevLen); prevLen -= 2; do { - strstart++; - lookahead--; - if (lookahead >= DeflaterConstants.MIN_MATCH) + this.strstart++; + this.lookahead--; + if (this.lookahead >= DeflaterConstants.MIN_MATCH) { - InsertString(); + this.InsertString(); } - } while (--prevLen > 0); + } + while (--prevLen > 0); - strstart++; - lookahead--; - prevAvailable = false; - matchLen = DeflaterConstants.MIN_MATCH - 1; + this.strstart++; + this.lookahead--; + this.prevAvailable = false; + this.matchLen = DeflaterConstants.MIN_MATCH - 1; } else { - if (prevAvailable) + if (this.prevAvailable) { - huffman.TallyLit(window[strstart - 1] & 0xff); + this.huffman.TallyLit(this.window[this.strstart - 1] & 0xff); } - prevAvailable = true; - strstart++; - lookahead--; + + this.prevAvailable = true; + this.strstart++; + this.lookahead--; } - if (huffman.IsFull()) + if (this.huffman.IsFull()) { - int len = strstart - blockStart; - if (prevAvailable) + int len = this.strstart - this.blockStart; + if (this.prevAvailable) { len--; } - bool lastBlock = (finish && (lookahead == 0) && !prevAvailable); - huffman.FlushBlock(window, blockStart, len, lastBlock); - blockStart += len; + + bool lastBlock = finish && (this.lookahead == 0) && !this.prevAvailable; + this.huffman.FlushBlock(this.window, this.blockStart, len, lastBlock); + this.blockStart += len; return !lastBlock; } } + return true; } - - #region Instance Fields - - // Hash index of string to be inserted - private int ins_h; - - /// - /// Hashtable, hashing three characters to an index for window, so - /// that window[index]..window[index+2] have this hash code. - /// Note that the array should really be unsigned short, so you need - /// to and the values with 0xffff. - /// - private short[] head; - - /// - /// prev[index & WMASK] points to the previous index that has the - /// same hash code as the string starting at index. This way - /// entries with the same hash code are in a linked list. - /// Note that the array should really be unsigned short, so you need - /// to and the values with 0xffff. - /// - private short[] prev; - - private int matchStart; - - // Length of best match - private int matchLen; - - // Set if previous match exists - private bool prevAvailable; - - private int blockStart; - - /// - /// Points to the current character in the window. - /// - private int strstart; - - /// - /// lookahead is the number of characters starting at strstart in - /// window that are valid. - /// So window[strstart] until window[strstart+lookahead-1] are valid - /// characters. - /// - private int lookahead; - - /// - /// This array contains the part of the uncompressed stream that - /// is of relevance. The current character is indexed by strstart. - /// - private byte[] window; - - private DeflateStrategy strategy; - private int max_chain, max_lazy, niceLength, goodLength; - - /// - /// The current compression function. - /// - private int compressionFunction; - - /// - /// The input data for compression. - /// - private byte[] inputBuf; - - /// - /// The total bytes of input read. - /// - private long totalIn; - - /// - /// The offset into inputBuf, where input data starts. - /// - private int inputOff; - - /// - /// The end offset of the input data. - /// - private int inputEnd; - - private DeflaterPendingBuffer pending; - private DeflaterHuffman huffman; - - /// - /// The adler checksum - /// - private Adler32 adler; - - #endregion Instance Fields } } From 7ab868f3ba78e5b1e05dfce0a755ff12af285649 Mon Sep 17 00:00:00 2001 From: James Jackson-South Date: Tue, 12 Nov 2019 21:38:03 +1100 Subject: [PATCH 05/16] Make DeflaterEngine disposable --- src/ImageSharp/Formats/Png/Zlib/Deflater.cs | 40 +-- .../Formats/Png/Zlib/DeflaterEngine.cs | 273 ++++++++---------- 2 files changed, 129 insertions(+), 184 deletions(-) diff --git a/src/ImageSharp/Formats/Png/Zlib/Deflater.cs b/src/ImageSharp/Formats/Png/Zlib/Deflater.cs index 33b3019b8..5732f82d2 100644 --- a/src/ImageSharp/Formats/Png/Zlib/Deflater.cs +++ b/src/ImageSharp/Formats/Png/Zlib/Deflater.cs @@ -54,7 +54,6 @@ namespace SixLabors.ImageSharp.Formats.Png.Zlib private DeflaterEngine engine; private bool isDisposed; - private const int IsSetDict = 0x01; private const int IsFlushing = 0x04; private const int IsFinishing = 0x08; private const int BusyState = 0x10; @@ -82,8 +81,10 @@ namespace SixLabors.ImageSharp.Formats.Png.Zlib } this.pending = new DeflaterPendingBuffer(memoryAllocator); - this.engine = new DeflaterEngine(this.pending, true); - this.engine.Strategy = DeflateStrategy.Default; + + // TODO: Possibly provide DeflateStrategy as an option. + this.engine = new DeflaterEngine(memoryAllocator, this.pending, DeflateStrategy.Default); + this.SetLevel(level); this.Reset(); } @@ -233,37 +234,6 @@ namespace SixLabors.ImageSharp.Formats.Png.Zlib DeflateThrowHelper.ThrowAlreadyClosed(); } - if (this.state < BusyState) - { - // Output header - int header = (Deflated + ((DeflaterConstants.MAX_WBITS - 8) << 4)) << 8; - int levelFlags = (this.level - 1) >> 1; - if (levelFlags < 0 || levelFlags > 3) - { - levelFlags = 3; - } - - header |= levelFlags << 6; - if ((this.state & IsSetDict) != 0) - { - // Dictionary was set - header |= DeflaterConstants.PRESET_DICT; - } - - header += 31 - (header % 31); - - this.pending.WriteShortMSB(header); - if ((this.state & IsSetDict) != 0) - { - int chksum = this.engine.Adler; - this.engine.ResetAdler(); - this.pending.WriteShortMSB(chksum >> 16); - this.pending.WriteShortMSB(chksum & 0xffff); - } - - this.state = BusyState | (this.state & (IsFlushing | IsFinishing)); - } - while (true) { int count = this.pending.Flush(output, offset, length); @@ -326,9 +296,11 @@ namespace SixLabors.ImageSharp.Formats.Png.Zlib if (disposing) { this.pending.Dispose(); + this.engine.Dispose(); } this.pending = null; + this.engine = null; this.isDisposed = true; } } diff --git a/src/ImageSharp/Formats/Png/Zlib/DeflaterEngine.cs b/src/ImageSharp/Formats/Png/Zlib/DeflaterEngine.cs index ddf571884..0870f629c 100644 --- a/src/ImageSharp/Formats/Png/Zlib/DeflaterEngine.cs +++ b/src/ImageSharp/Formats/Png/Zlib/DeflaterEngine.cs @@ -2,8 +2,9 @@ // Licensed under the Apache License, Version 2.0. using System; -using System.Collections.Generic; -using System.Text; +using System.Buffers; +using System.Runtime.CompilerServices; +using SixLabors.Memory; namespace SixLabors.ImageSharp.Formats.Png.Zlib { @@ -48,30 +49,13 @@ namespace SixLabors.ImageSharp.Formats.Png.Zlib /// Low level compression engine for deflate algorithm which uses a 32K sliding window /// with secondary compression from Huffman/Shannon-Fano codes. /// - public class DeflaterEngine + public sealed unsafe class DeflaterEngine : IDisposable { private const int TooFar = 4096; // Hash index of string to be inserted private int insertHashIndex; - /// - /// Hashtable, hashing three characters to an index for window, so - /// that window[index]..window[index+2] have this hash code. - /// Note that the array should really be unsigned short, so you need - /// to and the values with 0xFFFF. - /// - private short[] head; - - /// - /// prev[index & WMASK] points to the previous index that has the - /// same hash code as the string starting at index. This way - /// entries with the same hash code are in a linked list. - /// Note that the array should really be unsigned short, so you need - /// to and the values with 0xFFFF. - /// - private short[] prev; - private int matchStart; // Length of best match @@ -95,16 +79,6 @@ namespace SixLabors.ImageSharp.Formats.Png.Zlib /// private int lookahead; - /// - /// This array contains the part of the uncompressed stream that - /// is of relevance. The current character is indexed by strstart. - /// - private byte[] window; - private int maxChain; - private int maxLazy; - private int niceLength; - private int goodLength; - /// /// The current compression function. /// @@ -115,11 +89,6 @@ namespace SixLabors.ImageSharp.Formats.Png.Zlib /// private byte[] inputBuf; - /// - /// The total bytes of input read. - /// - private long totalIn; - /// /// The offset into inputBuf, where input data starts. /// @@ -130,31 +99,59 @@ namespace SixLabors.ImageSharp.Formats.Png.Zlib /// private int inputEnd; + private DeflateStrategy strategy; private DeflaterPendingBuffer pending; private DeflaterHuffman huffman; + private bool isDisposed; /// - /// The adler checksum + /// Hashtable, hashing three characters to an index for window, so + /// that window[index]..window[index+2] have this hash code. + /// Note that the array should really be unsigned short, so you need + /// to and the values with 0xFFFF. /// - private Adler32 adler; + private short[] head; + + /// + /// prev[index & WMASK] points to the previous index that has the + /// same hash code as the string starting at index. This way + /// entries with the same hash code are in a linked list. + /// Note that the array should really be unsigned short, so you need + /// to and the values with 0xFFFF. + /// + private short[] prev; + + /// + /// This array contains the part of the uncompressed stream that + /// is of relevance. The current character is indexed by strstart. + /// + private readonly IManagedByteBuffer windowBuffer; + private MemoryHandle windowBufferHandle; + private byte[] window; + private readonly byte* pinnedWindowPointer; + + private int maxChain; + private int maxLazy; + private int niceLength; + private int goodLength; /// /// Initializes a new instance of the class. /// + /// The memory allocator to use for buffer allocations. /// The pending buffer to use. - /// - /// If no adler calculation should be performed - /// - public DeflaterEngine(DeflaterPendingBuffer pending, bool noAdlerCalculation) + /// The deflate strategy to use. + public DeflaterEngine(MemoryAllocator memoryAllocator, DeflaterPendingBuffer pending, DeflateStrategy strategy) { this.pending = pending; this.huffman = new DeflaterHuffman(pending); - if (!noAdlerCalculation) - { - this.adler = new Adler32(); - } + this.strategy = strategy; + + this.windowBuffer = memoryAllocator.AllocateManagedByteBuffer(2 * DeflaterConstants.WSIZE); + this.window = this.windowBuffer.Array; + this.windowBufferHandle = this.windowBuffer.Memory.Pin(); + this.pinnedWindowPointer = (byte*)this.windowBufferHandle.Pointer; - this.window = new byte[2 * DeflaterConstants.WSIZE]; this.head = new short[DeflaterConstants.HASH_SIZE]; this.prev = new short[DeflaterConstants.WSIZE]; @@ -163,33 +160,6 @@ namespace SixLabors.ImageSharp.Formats.Png.Zlib this.blockStart = this.strstart = 1; } - /// - /// Gets the current value of Adler checksum - /// - public int Adler - { - get - { - return (this.adler != null) ? unchecked((int)this.adler.Value) : 0; - } - } - - /// - /// Gets the total data processed - /// - public long TotalIn - { - get - { - return this.totalIn; - } - } - - /// - /// Gets or sets the deflate strategy - /// - public DeflateStrategy Strategy { get; set; } - /// /// Deflate drives actual compression of data /// @@ -286,7 +256,6 @@ namespace SixLabors.ImageSharp.Formats.Png.Zlib /// The length of the dictionary data. public void SetDictionary(byte[] buffer, int offset, int length) { - this.adler?.Update(new ArraySegment(buffer, offset, length)); if (length < DeflaterConstants.MIN_MATCH) { return; @@ -318,10 +287,8 @@ namespace SixLabors.ImageSharp.Formats.Png.Zlib public void Reset() { this.huffman.Reset(); - this.adler?.Reset(); this.blockStart = this.strstart = 1; this.lookahead = 0; - this.totalIn = 0; this.prevAvailable = false; this.matchLen = DeflaterConstants.MIN_MATCH - 1; @@ -336,14 +303,6 @@ namespace SixLabors.ImageSharp.Formats.Png.Zlib } } - /// - /// Reset Adler checksum - /// - public void ResetAdler() - { - this.adler?.Reset(); - } - /// /// Set the deflate level (0-9) /// @@ -386,7 +345,7 @@ namespace SixLabors.ImageSharp.Formats.Png.Zlib case DeflaterConstants.DEFLATE_SLOW: if (this.prevAvailable) { - this.huffman.TallyLit(this.window[this.strstart - 1] & 0xff); + this.huffman.TallyLit(this.pinnedWindowPointer[this.strstart - 1] & 0xFF); } if (this.strstart > this.blockStart) @@ -427,10 +386,8 @@ namespace SixLabors.ImageSharp.Formats.Png.Zlib } Array.Copy(this.inputBuf, this.inputOff, this.window, this.strstart + this.lookahead, more); - this.adler?.Update(new ArraySegment(this.inputBuf, this.inputOff, more)); this.inputOff += more; - this.totalIn += more; this.lookahead += more; } @@ -440,9 +397,19 @@ namespace SixLabors.ImageSharp.Formats.Png.Zlib } } + /// + public void Dispose() + { + // Do not change this code. Put cleanup code in Dispose(bool disposing) above. + this.Dispose(true); + GC.SuppressFinalize(this); + } + + [MethodImpl(InliningOptions.ShortMethod)] private void UpdateHash() { - this.insertHashIndex = (this.window[this.strstart] << DeflaterConstants.HASH_SHIFT) ^ this.window[this.strstart + 1]; + byte* pinned = this.pinnedWindowPointer; + this.insertHashIndex = (pinned[this.strstart] << DeflaterConstants.HASH_SHIFT) ^ pinned[this.strstart + 1]; } /// @@ -453,7 +420,7 @@ namespace SixLabors.ImageSharp.Formats.Png.Zlib private int InsertString() { short match; - int hash = ((this.insertHashIndex << DeflaterConstants.HASH_SHIFT) ^ this.window[this.strstart + (DeflaterConstants.MIN_MATCH - 1)]) & DeflaterConstants.HASH_MASK; + int hash = ((this.insertHashIndex << DeflaterConstants.HASH_SHIFT) ^ this.pinnedWindowPointer[this.strstart + (DeflaterConstants.MIN_MATCH - 1)]) & DeflaterConstants.HASH_MASK; this.prev[this.strstart & DeflaterConstants.WMASK] = match = this.head[hash]; this.head[hash] = unchecked((short)this.strstart); @@ -463,7 +430,7 @@ namespace SixLabors.ImageSharp.Formats.Png.Zlib private void SlideWindow() { - Array.Copy(this.window, DeflaterConstants.WSIZE, this.window, 0, DeflaterConstants.WSIZE); + Unsafe.CopyBlockUnaligned(ref this.window[0], ref this.window[DeflaterConstants.WSIZE], DeflaterConstants.WSIZE); this.matchStart -= DeflaterConstants.WSIZE; this.strstart -= DeflaterConstants.WSIZE; this.blockStart -= DeflaterConstants.WSIZE; @@ -518,8 +485,9 @@ namespace SixLabors.ImageSharp.Formats.Png.Zlib return false; } - byte scan_end1 = window[scan + this.matchLen - 1]; - byte scan_end = window[scan + this.matchLen]; + byte* pinnedWindow = this.pinnedWindowPointer; + byte scan_end1 = pinnedWindow[scan + this.matchLen - 1]; + byte scan_end = pinnedWindow[scan + this.matchLen]; // Do not waste too much time if we already have a good match: if (this.matchLen >= this.goodLength) @@ -532,10 +500,10 @@ namespace SixLabors.ImageSharp.Formats.Png.Zlib match = curMatch; scan = this.strstart; - if (window[match + this.matchLen] != scan_end - || window[match + this.matchLen - 1] != scan_end1 - || window[match] != window[scan] - || window[++match] != window[++scan]) + if (pinnedWindow[match + this.matchLen] != scan_end + || pinnedWindow[match + this.matchLen - 1] != scan_end1 + || pinnedWindow[match] != pinnedWindow[scan] + || pinnedWindow[++match] != pinnedWindow[++scan]) { continue; } @@ -547,7 +515,7 @@ namespace SixLabors.ImageSharp.Formats.Png.Zlib switch ((scanMax - scan) % 8) { case 1: - if (window[++scan] == window[++match]) + if (pinnedWindow[++scan] == pinnedWindow[++match]) { break; } @@ -555,8 +523,8 @@ namespace SixLabors.ImageSharp.Formats.Png.Zlib break; case 2: - if (window[++scan] == window[++match] - && window[++scan] == window[++match]) + if (pinnedWindow[++scan] == pinnedWindow[++match] + && pinnedWindow[++scan] == pinnedWindow[++match]) { break; } @@ -564,9 +532,9 @@ namespace SixLabors.ImageSharp.Formats.Png.Zlib break; case 3: - if (window[++scan] == window[++match] - && window[++scan] == window[++match] - && window[++scan] == window[++match]) + if (pinnedWindow[++scan] == pinnedWindow[++match] + && pinnedWindow[++scan] == pinnedWindow[++match] + && pinnedWindow[++scan] == pinnedWindow[++match]) { break; } @@ -574,10 +542,10 @@ namespace SixLabors.ImageSharp.Formats.Png.Zlib break; case 4: - if (window[++scan] == window[++match] - && window[++scan] == window[++match] - && window[++scan] == window[++match] - && window[++scan] == window[++match]) + if (pinnedWindow[++scan] == pinnedWindow[++match] + && pinnedWindow[++scan] == pinnedWindow[++match] + && pinnedWindow[++scan] == pinnedWindow[++match] + && pinnedWindow[++scan] == pinnedWindow[++match]) { break; } @@ -585,11 +553,11 @@ namespace SixLabors.ImageSharp.Formats.Png.Zlib break; case 5: - if (window[++scan] == window[++match] - && window[++scan] == window[++match] - && window[++scan] == window[++match] - && window[++scan] == window[++match] - && window[++scan] == window[++match]) + if (pinnedWindow[++scan] == pinnedWindow[++match] + && pinnedWindow[++scan] == pinnedWindow[++match] + && pinnedWindow[++scan] == pinnedWindow[++match] + && pinnedWindow[++scan] == pinnedWindow[++match] + && pinnedWindow[++scan] == pinnedWindow[++match]) { break; } @@ -597,12 +565,12 @@ namespace SixLabors.ImageSharp.Formats.Png.Zlib break; case 6: - if (window[++scan] == window[++match] - && window[++scan] == window[++match] - && window[++scan] == window[++match] - && window[++scan] == window[++match] - && window[++scan] == window[++match] - && window[++scan] == window[++match]) + if (pinnedWindow[++scan] == pinnedWindow[++match] + && pinnedWindow[++scan] == pinnedWindow[++match] + && pinnedWindow[++scan] == pinnedWindow[++match] + && pinnedWindow[++scan] == pinnedWindow[++match] + && pinnedWindow[++scan] == pinnedWindow[++match] + && pinnedWindow[++scan] == pinnedWindow[++match]) { break; } @@ -610,13 +578,13 @@ namespace SixLabors.ImageSharp.Formats.Png.Zlib break; case 7: - if (window[++scan] == window[++match] - && window[++scan] == window[++match] - && window[++scan] == window[++match] - && window[++scan] == window[++match] - && window[++scan] == window[++match] - && window[++scan] == window[++match] - && window[++scan] == window[++match]) + if (pinnedWindow[++scan] == pinnedWindow[++match] + && pinnedWindow[++scan] == pinnedWindow[++match] + && pinnedWindow[++scan] == pinnedWindow[++match] + && pinnedWindow[++scan] == pinnedWindow[++match] + && pinnedWindow[++scan] == pinnedWindow[++match] + && pinnedWindow[++scan] == pinnedWindow[++match] + && pinnedWindow[++scan] == pinnedWindow[++match]) { break; } @@ -624,7 +592,7 @@ namespace SixLabors.ImageSharp.Formats.Png.Zlib break; } - if (window[scan] == window[match]) + if (pinnedWindow[scan] == pinnedWindow[match]) { // We check for insufficient lookahead only every 8th comparison; // the 256th check will be made at strstart + 258 unless lookahead is @@ -639,14 +607,14 @@ namespace SixLabors.ImageSharp.Formats.Png.Zlib break; } } - while (window[++scan] == window[++match] - && window[++scan] == window[++match] - && window[++scan] == window[++match] - && window[++scan] == window[++match] - && window[++scan] == window[++match] - && window[++scan] == window[++match] - && window[++scan] == window[++match] - && window[++scan] == window[++match]); + while (pinnedWindow[++scan] == pinnedWindow[++match] + && pinnedWindow[++scan] == pinnedWindow[++match] + && pinnedWindow[++scan] == pinnedWindow[++match] + && pinnedWindow[++scan] == pinnedWindow[++match] + && pinnedWindow[++scan] == pinnedWindow[++match] + && pinnedWindow[++scan] == pinnedWindow[++match] + && pinnedWindow[++scan] == pinnedWindow[++match] + && pinnedWindow[++scan] == pinnedWindow[++match]); } if (scan - this.strstart > this.matchLen) @@ -659,8 +627,8 @@ namespace SixLabors.ImageSharp.Formats.Png.Zlib break; } - scan_end1 = window[scan - 1]; - scan_end = window[scan]; + scan_end1 = pinnedWindow[scan - 1]; + scan_end = pinnedWindow[scan]; } } while ((curMatch = prev[curMatch & DeflaterConstants.WMASK] & 0xFFFF) > limit && --chainLength != 0); @@ -727,7 +695,7 @@ namespace SixLabors.ImageSharp.Formats.Png.Zlib int hashHead; if (this.lookahead >= DeflaterConstants.MIN_MATCH && (hashHead = this.InsertString()) != 0 && - this.Strategy != DeflateStrategy.HuffmanOnly && + this.strategy != DeflateStrategy.HuffmanOnly && this.strstart - hashHead <= DeflaterConstants.MAX_DIST && this.FindLongestMatch(hashHead)) { @@ -763,7 +731,7 @@ namespace SixLabors.ImageSharp.Formats.Png.Zlib else { // No match found - this.huffman.TallyLit(this.window[this.strstart] & 0xff); + this.huffman.TallyLit(this.pinnedWindowPointer[this.strstart] & 0xff); ++this.strstart; --this.lookahead; } @@ -793,7 +761,7 @@ namespace SixLabors.ImageSharp.Formats.Png.Zlib { if (this.prevAvailable) { - this.huffman.TallyLit(this.window[this.strstart - 1] & 0xff); + this.huffman.TallyLit(this.pinnedWindowPointer[this.strstart - 1] & 0xff); } this.prevAvailable = false; @@ -818,14 +786,14 @@ namespace SixLabors.ImageSharp.Formats.Png.Zlib { int hashHead = this.InsertString(); - if (this.Strategy != DeflateStrategy.HuffmanOnly && + if (this.strategy != DeflateStrategy.HuffmanOnly && hashHead != 0 && this.strstart - hashHead <= DeflaterConstants.MAX_DIST && this.FindLongestMatch(hashHead)) { // longestMatch sets matchStart and matchLen // Discard match if too small and too far away - if (this.matchLen <= 5 && (this.Strategy == DeflateStrategy.Filtered || (this.matchLen == DeflaterConstants.MIN_MATCH && this.strstart - this.matchStart > TooFar))) + if (this.matchLen <= 5 && (this.strategy == DeflateStrategy.Filtered || (this.matchLen == DeflaterConstants.MIN_MATCH && this.strstart - this.matchStart > TooFar))) { this.matchLen = DeflaterConstants.MIN_MATCH - 1; } @@ -835,15 +803,6 @@ namespace SixLabors.ImageSharp.Formats.Png.Zlib // previous match was better if ((prevLen >= DeflaterConstants.MIN_MATCH) && (this.matchLen <= prevLen)) { -#if DebugDeflation - if (DeflaterConstants.DEBUGGING) - { - for (int i = 0 ; i < matchLen; i++) { - if (window[strstart-1+i] != window[prevMatch + i]) - throw new ImageFormatException(); - } - } -#endif this.huffman.TallyDist(this.strstart - 1 - prevMatch, prevLen); prevLen -= 2; do @@ -866,7 +825,7 @@ namespace SixLabors.ImageSharp.Formats.Png.Zlib { if (this.prevAvailable) { - this.huffman.TallyLit(this.window[this.strstart - 1] & 0xff); + this.huffman.TallyLit(this.pinnedWindowPointer[this.strstart - 1] & 0xff); } this.prevAvailable = true; @@ -891,5 +850,19 @@ namespace SixLabors.ImageSharp.Formats.Png.Zlib return true; } + + private void Dispose(bool disposing) + { + if (!this.isDisposed) + { + if (disposing) + { + this.windowBufferHandle.Dispose(); + this.windowBuffer.Dispose(); + } + + this.isDisposed = true; + } + } } } From e4d3474cf04eb68cafac35af92792022111754f3 Mon Sep 17 00:00:00 2001 From: James Jackson-South Date: Tue, 12 Nov 2019 23:28:01 +1100 Subject: [PATCH 06/16] Optimize DeflaterEngine --- .../Formats/Png/Zlib/DeflateThrowHelper.cs | 11 ++ src/ImageSharp/Formats/Png/Zlib/Deflater.cs | 2 +- .../Formats/Png/Zlib/DeflaterEngine.cs | 133 ++++++++---------- .../Formats/Png/Zlib/DeflaterOutputStream.cs | 21 --- 4 files changed, 70 insertions(+), 97 deletions(-) diff --git a/src/ImageSharp/Formats/Png/Zlib/DeflateThrowHelper.cs b/src/ImageSharp/Formats/Png/Zlib/DeflateThrowHelper.cs index a360e5e87..2698c9b99 100644 --- a/src/ImageSharp/Formats/Png/Zlib/DeflateThrowHelper.cs +++ b/src/ImageSharp/Formats/Png/Zlib/DeflateThrowHelper.cs @@ -13,5 +13,16 @@ namespace SixLabors.ImageSharp.Formats.Png.Zlib [MethodImpl(InliningOptions.ColdPath)] public static void ThrowAlreadyClosed() => throw new InvalidOperationException("Deflator already closed."); + + [MethodImpl(InliningOptions.ColdPath)] + public static void ThrowUnknownCompression() => throw new InvalidOperationException("Unknown compression function."); + + public static void ThrowNotProcessed() => throw new InvalidOperationException("Old input was not completely processed."); + + [MethodImpl(InliningOptions.ColdPath)] + public static void ThrowNull(string name) => throw new ArgumentNullException(name); + + [MethodImpl(InliningOptions.ColdPath)] + public static void ThrowOutOfRange(string name) => throw new ArgumentOutOfRangeException(name); } } diff --git a/src/ImageSharp/Formats/Png/Zlib/Deflater.cs b/src/ImageSharp/Formats/Png/Zlib/Deflater.cs index 5732f82d2..ef1d0e116 100644 --- a/src/ImageSharp/Formats/Png/Zlib/Deflater.cs +++ b/src/ImageSharp/Formats/Png/Zlib/Deflater.cs @@ -295,8 +295,8 @@ namespace SixLabors.ImageSharp.Formats.Png.Zlib { if (disposing) { - this.pending.Dispose(); this.engine.Dispose(); + this.pending.Dispose(); } this.pending = null; diff --git a/src/ImageSharp/Formats/Png/Zlib/DeflaterEngine.cs b/src/ImageSharp/Formats/Png/Zlib/DeflaterEngine.cs index 0870f629c..4db87c57e 100644 --- a/src/ImageSharp/Formats/Png/Zlib/DeflaterEngine.cs +++ b/src/ImageSharp/Formats/Png/Zlib/DeflaterEngine.cs @@ -99,8 +99,8 @@ namespace SixLabors.ImageSharp.Formats.Png.Zlib /// private int inputEnd; - private DeflateStrategy strategy; - private DeflaterPendingBuffer pending; + private readonly DeflateStrategy strategy; + private readonly DeflaterPendingBuffer pending; private DeflaterHuffman huffman; private bool isDisposed; @@ -110,7 +110,10 @@ namespace SixLabors.ImageSharp.Formats.Png.Zlib /// Note that the array should really be unsigned short, so you need /// to and the values with 0xFFFF. /// - private short[] head; + private readonly IMemoryOwner headBuffer; + private MemoryHandle headBufferHandle; + private readonly Memory head; + private readonly short* pinnedHeadPointer; /// /// prev[index & WMASK] points to the previous index that has the @@ -119,7 +122,10 @@ namespace SixLabors.ImageSharp.Formats.Png.Zlib /// Note that the array should really be unsigned short, so you need /// to and the values with 0xFFFF. /// - private short[] prev; + private readonly IMemoryOwner prevBuffer; + private MemoryHandle prevBufferHandle; + private readonly Memory prev; + private readonly short* pinnedPrevPointer; /// /// This array contains the part of the uncompressed stream that @@ -127,7 +133,7 @@ namespace SixLabors.ImageSharp.Formats.Png.Zlib /// private readonly IManagedByteBuffer windowBuffer; private MemoryHandle windowBufferHandle; - private byte[] window; + private readonly byte[] window; private readonly byte* pinnedWindowPointer; private int maxChain; @@ -147,13 +153,22 @@ namespace SixLabors.ImageSharp.Formats.Png.Zlib this.huffman = new DeflaterHuffman(pending); this.strategy = strategy; - this.windowBuffer = memoryAllocator.AllocateManagedByteBuffer(2 * DeflaterConstants.WSIZE); + // Create pinned pointers to the various buffers to allow indexing + // without bounds checks. + this.windowBuffer = memoryAllocator.AllocateManagedByteBuffer(2 * DeflaterConstants.WSIZE, AllocationOptions.Clean); this.window = this.windowBuffer.Array; this.windowBufferHandle = this.windowBuffer.Memory.Pin(); this.pinnedWindowPointer = (byte*)this.windowBufferHandle.Pointer; - this.head = new short[DeflaterConstants.HASH_SIZE]; - this.prev = new short[DeflaterConstants.WSIZE]; + this.headBuffer = memoryAllocator.Allocate(DeflaterConstants.HASH_SIZE, AllocationOptions.Clean); + this.head = this.headBuffer.Memory; + this.headBufferHandle = this.headBuffer.Memory.Pin(); + this.pinnedHeadPointer = (short*)this.headBufferHandle.Pointer; + + this.prevBuffer = memoryAllocator.Allocate(DeflaterConstants.WSIZE, AllocationOptions.Clean); + this.prev = this.prevBuffer.Memory; + this.prevBufferHandle = this.prevBuffer.Memory.Pin(); + this.pinnedPrevPointer = (short*)this.prevBufferHandle.Pointer; // We start at index 1, to avoid an implementation deficiency, that // we cannot build a repeat pattern at index 0. @@ -168,7 +183,7 @@ namespace SixLabors.ImageSharp.Formats.Png.Zlib /// Returns true if progress has been made. public bool Deflate(bool flush, bool finish) { - bool progress; + bool progress = false; do { this.FillWindow(); @@ -189,7 +204,8 @@ namespace SixLabors.ImageSharp.Formats.Png.Zlib break; default: - throw new InvalidOperationException("unknown compressionFunction"); + DeflateThrowHelper.ThrowUnknownCompression(); + break; } } while (this.pending.IsFlushed && progress); // repeat while we have no pending output and progress was made @@ -197,7 +213,7 @@ namespace SixLabors.ImageSharp.Formats.Png.Zlib } /// - /// Sets input data to be deflated. Should only be called when NeedsInput() + /// Sets input data to be deflated. Should only be called when /// returns true /// /// The buffer containing input data. @@ -205,33 +221,33 @@ namespace SixLabors.ImageSharp.Formats.Png.Zlib /// The number of bytes of data to use as input. public void SetInput(byte[] buffer, int offset, int count) { - if (buffer == null) + if (buffer is null) { - throw new ArgumentNullException(nameof(buffer)); + DeflateThrowHelper.ThrowNull(nameof(buffer)); } if (offset < 0) { - throw new ArgumentOutOfRangeException(nameof(offset)); + DeflateThrowHelper.ThrowOutOfRange(nameof(offset)); } if (count < 0) { - throw new ArgumentOutOfRangeException(nameof(count)); + DeflateThrowHelper.ThrowOutOfRange(nameof(count)); } if (this.inputOff < this.inputEnd) { - throw new InvalidOperationException("Old input was not completely processed."); + DeflateThrowHelper.ThrowNotProcessed(); } int end = offset + count; - // We want to throw an ArrayIndexOutOfBoundsException early. + // We want to throw an ArgumentOutOfRangeException early. // The check is very tricky: it also handles integer wrap around. if ((offset > end) || (end > buffer.Length)) { - throw new ArgumentOutOfRangeException(nameof(count)); + DeflateThrowHelper.ThrowOutOfRange(nameof(count)); } this.inputBuf = buffer; @@ -243,47 +259,13 @@ namespace SixLabors.ImageSharp.Formats.Png.Zlib /// Determines if more input is needed. /// /// Return true if input is needed via SetInput - public bool NeedsInput() - { - return this.inputEnd == this.inputOff; - } - - /// - /// Set compression dictionary - /// - /// The buffer containing the dictionary data - /// The offset in the buffer for the first byte of data - /// The length of the dictionary data. - public void SetDictionary(byte[] buffer, int offset, int length) - { - if (length < DeflaterConstants.MIN_MATCH) - { - return; - } - - if (length > DeflaterConstants.MAX_DIST) - { - offset += length - DeflaterConstants.MAX_DIST; - length = DeflaterConstants.MAX_DIST; - } - - System.Array.Copy(buffer, offset, this.window, this.strstart, length); - - this.UpdateHash(); - --length; - while (--length > 0) - { - this.InsertString(); - this.strstart++; - } - - this.strstart += 2; - this.blockStart = this.strstart; - } + [MethodImpl(InliningOptions.ShortMethod)] + public bool NeedsInput() => this.inputEnd == this.inputOff; /// /// Reset internal state /// + [MethodImpl(InliningOptions.ShortMethod)] public void Reset() { this.huffman.Reset(); @@ -291,16 +273,8 @@ namespace SixLabors.ImageSharp.Formats.Png.Zlib this.lookahead = 0; this.prevAvailable = false; this.matchLen = DeflaterConstants.MIN_MATCH - 1; - - for (int i = 0; i < DeflaterConstants.HASH_SIZE; i++) - { - this.head[i] = 0; - } - - for (int i = 0; i < DeflaterConstants.WSIZE; i++) - { - this.prev[i] = 0; - } + this.head.Span.Slice(0, DeflaterConstants.HASH_SIZE).Clear(); + this.prev.Span.Slice(0, DeflaterConstants.WSIZE).Clear(); } /// @@ -311,7 +285,7 @@ namespace SixLabors.ImageSharp.Formats.Png.Zlib { if ((level < 0) || (level > 9)) { - throw new ArgumentOutOfRangeException(nameof(level)); + DeflateThrowHelper.ThrowOutOfRange(nameof(level)); } this.goodLength = DeflaterConstants.GOOD_LENGTH[level]; @@ -417,13 +391,15 @@ namespace SixLabors.ImageSharp.Formats.Png.Zlib /// value for this hash. /// /// The previous hash value + [MethodImpl(InliningOptions.ShortMethod)] private int InsertString() { short match; int hash = ((this.insertHashIndex << DeflaterConstants.HASH_SHIFT) ^ this.pinnedWindowPointer[this.strstart + (DeflaterConstants.MIN_MATCH - 1)]) & DeflaterConstants.HASH_MASK; - this.prev[this.strstart & DeflaterConstants.WMASK] = match = this.head[hash]; - this.head[hash] = unchecked((short)this.strstart); + short* pinnedHead = this.pinnedHeadPointer; + this.pinnedPrevPointer[this.strstart & DeflaterConstants.WMASK] = match = pinnedHead[hash]; + pinnedHead[hash] = unchecked((short)this.strstart); this.insertHashIndex = hash; return match & 0xFFFF; } @@ -437,17 +413,19 @@ namespace SixLabors.ImageSharp.Formats.Png.Zlib // Slide the hash table (could be avoided with 32 bit values // at the expense of memory usage). + short* pinnedHead = this.pinnedHeadPointer; for (int i = 0; i < DeflaterConstants.HASH_SIZE; ++i) { - int m = this.head[i] & 0xFFFF; - this.head[i] = (short)(m >= DeflaterConstants.WSIZE ? (m - DeflaterConstants.WSIZE) : 0); + int m = pinnedHead[i] & 0xFFFF; + pinnedHead[i] = (short)(m >= DeflaterConstants.WSIZE ? (m - DeflaterConstants.WSIZE) : 0); } // Slide the prev table. + short* pinnedPrev = this.pinnedPrevPointer; for (int i = 0; i < DeflaterConstants.WSIZE; i++) { - int m = this.prev[i] & 0xFFFF; - this.prev[i] = (short)(m >= DeflaterConstants.WSIZE ? (m - DeflaterConstants.WSIZE) : 0); + int m = pinnedPrev[i] & 0xFFFF; + pinnedPrev[i] = (short)(m >= DeflaterConstants.WSIZE ? (m - DeflaterConstants.WSIZE) : 0); } } @@ -473,8 +451,6 @@ namespace SixLabors.ImageSharp.Formats.Png.Zlib int scanMax = scan + Math.Min(DeflaterConstants.MAX_MATCH, this.lookahead) - 1; int limit = Math.Max(scan - DeflaterConstants.MAX_DIST, 0); - byte[] window = this.window; - short[] prev = this.prev; int chainLength = this.maxChain; int niceLength = Math.Min(this.niceLength, this.lookahead); @@ -495,6 +471,7 @@ namespace SixLabors.ImageSharp.Formats.Png.Zlib chainLength >>= 2; } + short* pinnedPrev = this.pinnedPrevPointer; do { match = curMatch; @@ -631,7 +608,7 @@ namespace SixLabors.ImageSharp.Formats.Png.Zlib scan_end = pinnedWindow[scan]; } } - while ((curMatch = prev[curMatch & DeflaterConstants.WMASK] & 0xFFFF) > limit && --chainLength != 0); + while ((curMatch = pinnedPrev[curMatch & DeflaterConstants.WMASK] & 0xFFFF) > limit && --chainLength != 0); return this.matchLen >= DeflaterConstants.MIN_MATCH; } @@ -859,6 +836,12 @@ namespace SixLabors.ImageSharp.Formats.Png.Zlib { this.windowBufferHandle.Dispose(); this.windowBuffer.Dispose(); + + this.headBufferHandle.Dispose(); + this.headBuffer.Dispose(); + + this.prevBufferHandle.Dispose(); + this.prevBuffer.Dispose(); } this.isDisposed = true; diff --git a/src/ImageSharp/Formats/Png/Zlib/DeflaterOutputStream.cs b/src/ImageSharp/Formats/Png/Zlib/DeflaterOutputStream.cs index ac5f229aa..837e8b795 100644 --- a/src/ImageSharp/Formats/Png/Zlib/DeflaterOutputStream.cs +++ b/src/ImageSharp/Formats/Png/Zlib/DeflaterOutputStream.cs @@ -97,11 +97,6 @@ namespace SixLabors.ImageSharp.Formats.Png.Zlib break; } - //if (cryptoTransform_ != null) - //{ - // EncryptBlock(buffer_, 0, len); - //} - baseOutputStream_.Write(buffer_, 0, len); } @@ -111,16 +106,6 @@ namespace SixLabors.ImageSharp.Formats.Png.Zlib } baseOutputStream_.Flush(); - - //if (cryptoTransform_ != null) - //{ - // if (cryptoTransform_ is ZipAESTransform) - // { - // AESAuthCode = ((ZipAESTransform)cryptoTransform_).GetAuthCode(); - // } - // cryptoTransform_.Dispose(); - // cryptoTransform_ = null; - //} } /// @@ -394,12 +379,6 @@ namespace SixLabors.ImageSharp.Formats.Png.Zlib try { Finish(); - //if (cryptoTransform_ != null) - //{ - // GetAuthCodeIfAES(); - // cryptoTransform_.Dispose(); - // cryptoTransform_ = null; - //} } finally { From 7ce43270800bdf82fc60fbeab533240e7a04b9af Mon Sep 17 00:00:00 2001 From: James Jackson-South Date: Wed, 13 Nov 2019 13:25:59 +1100 Subject: [PATCH 07/16] No need to clean the arrays. We write over everything. --- src/ImageSharp/Formats/Png/Zlib/DeflaterEngine.cs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/ImageSharp/Formats/Png/Zlib/DeflaterEngine.cs b/src/ImageSharp/Formats/Png/Zlib/DeflaterEngine.cs index 4db87c57e..b14682501 100644 --- a/src/ImageSharp/Formats/Png/Zlib/DeflaterEngine.cs +++ b/src/ImageSharp/Formats/Png/Zlib/DeflaterEngine.cs @@ -155,17 +155,17 @@ namespace SixLabors.ImageSharp.Formats.Png.Zlib // Create pinned pointers to the various buffers to allow indexing // without bounds checks. - this.windowBuffer = memoryAllocator.AllocateManagedByteBuffer(2 * DeflaterConstants.WSIZE, AllocationOptions.Clean); + this.windowBuffer = memoryAllocator.AllocateManagedByteBuffer(2 * DeflaterConstants.WSIZE); this.window = this.windowBuffer.Array; this.windowBufferHandle = this.windowBuffer.Memory.Pin(); this.pinnedWindowPointer = (byte*)this.windowBufferHandle.Pointer; - this.headBuffer = memoryAllocator.Allocate(DeflaterConstants.HASH_SIZE, AllocationOptions.Clean); + this.headBuffer = memoryAllocator.Allocate(DeflaterConstants.HASH_SIZE); this.head = this.headBuffer.Memory; this.headBufferHandle = this.headBuffer.Memory.Pin(); this.pinnedHeadPointer = (short*)this.headBufferHandle.Pointer; - this.prevBuffer = memoryAllocator.Allocate(DeflaterConstants.WSIZE, AllocationOptions.Clean); + this.prevBuffer = memoryAllocator.Allocate(DeflaterConstants.WSIZE); this.prev = this.prevBuffer.Memory; this.prevBufferHandle = this.prevBuffer.Memory.Pin(); this.pinnedPrevPointer = (short*)this.prevBufferHandle.Pointer; From d3da2d33143d2a6c033327da57f7075620e1fcb8 Mon Sep 17 00:00:00 2001 From: James Jackson-South Date: Wed, 13 Nov 2019 15:35:38 +1100 Subject: [PATCH 08/16] Appease StyleCop --- .../Formats/Png/Zlib/DeflaterHuffman.cs | 1022 ++++++++--------- 1 file changed, 458 insertions(+), 564 deletions(-) diff --git a/src/ImageSharp/Formats/Png/Zlib/DeflaterHuffman.cs b/src/ImageSharp/Formats/Png/Zlib/DeflaterHuffman.cs index 3d2856b0e..bc7ad7a38 100644 --- a/src/ImageSharp/Formats/Png/Zlib/DeflaterHuffman.cs +++ b/src/ImageSharp/Formats/Png/Zlib/DeflaterHuffman.cs @@ -1,7 +1,6 @@ // Copyright (c) Six Labors and contributors. // Licensed under the Apache License, Version 2.0. -// using System; using System.Collections.Generic; using System.Text; @@ -18,106 +17,409 @@ namespace SixLabors.ImageSharp.Formats.Png.Zlib /// public class DeflaterHuffman { - private const int BUFSIZE = 1 << (DeflaterConstants.DEFAULT_MEM_LEVEL + 6); - private const int LITERAL_NUM = 286; + private const int BufferSize = 1 << (DeflaterConstants.DEFAULT_MEM_LEVEL + 6); + + // The number of literal codes. + private const int LiteralNumber = 286; // Number of distance codes - private const int DIST_NUM = 30; + private const int DistanceNumber = 30; // Number of codes used to transfer bit lengths - private const int BITLEN_NUM = 19; + private const int BitLengthNumber = 19; // repeat previous bit length 3-6 times (2 bits of repeat count) - private const int REP_3_6 = 16; + private const int Repeat3To6 = 16; // repeat a zero length 3-10 times (3 bits of repeat count) - private const int REP_3_10 = 17; + private const int Repeat3To10 = 17; // repeat a zero length 11-138 times (7 bits of repeat count) - private const int REP_11_138 = 18; + private const int Repeat11To138 = 18; - private const int EOF_SYMBOL = 256; + private const int EofSymbol = 256; // The lengths of the bit length codes are sent in order of decreasing // probability, to avoid transmitting the lengths for unused bit length codes. - private static readonly int[] BL_ORDER = { 16, 17, 18, 0, 8, 7, 9, 6, 10, 5, 11, 4, 12, 3, 13, 2, 14, 1, 15 }; - - private static readonly byte[] bit4Reverse = { - 0, - 8, - 4, - 12, - 2, - 10, - 6, - 14, - 1, - 9, - 5, - 13, - 3, - 11, - 7, - 15 - }; + private static readonly int[] BitLengthOrder = { 16, 17, 18, 0, 8, 7, 9, 6, 10, 5, 11, 4, 12, 3, 13, 2, 14, 1, 15 }; + + private static readonly byte[] Bit4Reverse = { 0, 8, 4, 12, 2, 10, 6, 14, 1, 9, 5, 13, 3, 11, 7, 15 }; private static short[] staticLCodes; private static byte[] staticLLength; private static short[] staticDCodes; private static byte[] staticDLength; - private class Tree + private Tree literalTree; + private Tree distTree; + private Tree blTree; + + // Buffer for distances + private short[] distanceBuffer; + + private byte[] literalBuffer; + private int lastLiteral; + private int extraBits; + + static DeflaterHuffman() + { + // See RFC 1951 3.2.6 + // Literal codes + staticLCodes = new short[LiteralNumber]; + staticLLength = new byte[LiteralNumber]; + + int i = 0; + while (i < 144) + { + staticLCodes[i] = BitReverse((0x030 + i) << 8); + staticLLength[i++] = 8; + } + + while (i < 256) + { + staticLCodes[i] = BitReverse((0x190 - 144 + i) << 7); + staticLLength[i++] = 9; + } + + while (i < 280) + { + staticLCodes[i] = BitReverse((0x000 - 256 + i) << 9); + staticLLength[i++] = 7; + } + + while (i < LiteralNumber) + { + staticLCodes[i] = BitReverse((0x0c0 - 280 + i) << 8); + staticLLength[i++] = 8; + } + + // Distance codes + staticDCodes = new short[DistanceNumber]; + staticDLength = new byte[DistanceNumber]; + for (i = 0; i < DistanceNumber; i++) + { + staticDCodes[i] = BitReverse(i << 11); + staticDLength[i] = 5; + } + } + + /// + /// Initializes a new instance of the class. + /// + /// Pending buffer to use + public DeflaterHuffman(DeflaterPendingBuffer pending) { - #region Instance Fields + this.Pending = pending; - public short[] freqs; + this.literalTree = new Tree(this, LiteralNumber, 257, 15); + this.distTree = new Tree(this, DistanceNumber, 1, 15); + this.blTree = new Tree(this, BitLengthNumber, 4, 7); - public byte[] length; + this.distanceBuffer = new short[BufferSize]; + this.literalBuffer = new byte[BufferSize]; + } - public int minNumCodes; + /// + /// Gets the pending buffer to use. + /// + public DeflaterPendingBuffer Pending { get; } - public int numCodes; + /// + /// Reset internal state + /// + public void Reset() + { + this.lastLiteral = 0; + this.extraBits = 0; + this.literalTree.Reset(); + this.distTree.Reset(); + this.blTree.Reset(); + } - private short[] codes; - private readonly int[] bl_counts; - private readonly int maxLength; - private DeflaterHuffman dh; + /// + /// Write all trees to pending buffer + /// + /// The number/rank of treecodes to send. + public void SendAllTrees(int blTreeCodes) + { + this.blTree.BuildCodes(); + this.literalTree.BuildCodes(); + this.distTree.BuildCodes(); + this.Pending.WriteBits(this.literalTree.NumCodes - 257, 5); + this.Pending.WriteBits(this.distTree.NumCodes - 1, 5); + this.Pending.WriteBits(blTreeCodes - 4, 4); + for (int rank = 0; rank < blTreeCodes; rank++) + { + this.Pending.WriteBits(this.blTree.Length[BitLengthOrder[rank]], 3); + } + + this.literalTree.WriteTree(this.blTree); + this.distTree.WriteTree(this.blTree); + } + + /// + /// Compress current buffer writing data to pending buffer + /// + public void CompressBlock() + { + for (int i = 0; i < this.lastLiteral; i++) + { + int litlen = this.literalBuffer[i] & 0xff; + int dist = this.distanceBuffer[i]; + if (dist-- != 0) + { + int lc = Lcode(litlen); + this.literalTree.WriteSymbol(lc); + + int bits = (lc - 261) / 4; + if (bits > 0 && bits <= 5) + { + this.Pending.WriteBits(litlen & ((1 << bits) - 1), bits); + } + + int dc = Dcode(dist); + this.distTree.WriteSymbol(dc); + + bits = (dc / 2) - 1; + if (bits > 0) + { + this.Pending.WriteBits(dist & ((1 << bits) - 1), bits); + } + } + else + { + this.literalTree.WriteSymbol(litlen); + } + } + + this.literalTree.WriteSymbol(EofSymbol); + } + + /// + /// Flush block to output with no compression + /// + /// Data to write + /// Index of first byte to write + /// Count of bytes to write + /// True if this is the last block + public void FlushStoredBlock(byte[] stored, int storedOffset, int storedLength, bool lastBlock) + { + this.Pending.WriteBits((DeflaterConstants.STORED_BLOCK << 1) + (lastBlock ? 1 : 0), 3); + this.Pending.AlignToByte(); + this.Pending.WriteShort(storedLength); + this.Pending.WriteShort(~storedLength); + this.Pending.WriteBlock(stored, storedOffset, storedLength); + this.Reset(); + } + + /// + /// Flush block to output with compression + /// + /// Data to flush + /// Index of first byte to flush + /// Count of bytes to flush + /// True if this is the last block + public void FlushBlock(byte[] stored, int storedOffset, int storedLength, bool lastBlock) + { + this.literalTree.Freqs[EofSymbol]++; + + // Build trees + this.literalTree.BuildTree(); + this.distTree.BuildTree(); + + // Calculate bitlen frequency + this.literalTree.CalcBLFreq(this.blTree); + this.distTree.CalcBLFreq(this.blTree); + + // Build bitlen tree + this.blTree.BuildTree(); + + int blTreeCodes = 4; + for (int i = 18; i > blTreeCodes; i--) + { + if (this.blTree.Length[BitLengthOrder[i]] > 0) + { + blTreeCodes = i + 1; + } + } + + int opt_len = 14 + (blTreeCodes * 3) + this.blTree.GetEncodedLength() + + this.literalTree.GetEncodedLength() + this.distTree.GetEncodedLength() + + this.extraBits; + + int static_len = this.extraBits; + for (int i = 0; i < LiteralNumber; i++) + { + static_len += this.literalTree.Freqs[i] * staticLLength[i]; + } + + for (int i = 0; i < DistanceNumber; i++) + { + static_len += this.distTree.Freqs[i] * staticDLength[i]; + } + + if (opt_len >= static_len) + { + // Force static trees + opt_len = static_len; + } + + if (storedOffset >= 0 && storedLength + 4 < opt_len >> 3) + { + // Store Block + this.FlushStoredBlock(stored, storedOffset, storedLength, lastBlock); + } + else if (opt_len == static_len) + { + // Encode with static tree + this.Pending.WriteBits((DeflaterConstants.STATIC_TREES << 1) + (lastBlock ? 1 : 0), 3); + this.literalTree.SetStaticCodes(staticLCodes, staticLLength); + this.distTree.SetStaticCodes(staticDCodes, staticDLength); + this.CompressBlock(); + this.Reset(); + } + else + { + // Encode with dynamic tree + this.Pending.WriteBits((DeflaterConstants.DYN_TREES << 1) + (lastBlock ? 1 : 0), 3); + this.SendAllTrees(blTreeCodes); + this.CompressBlock(); + this.Reset(); + } + } + + /// + /// Get value indicating if internal buffer is full + /// + /// true if buffer is full + public bool IsFull() + { + return this.lastLiteral >= BufferSize; + } + + /// + /// Add literal to buffer + /// + /// Literal value to add to buffer. + /// Value indicating internal buffer is full + public bool TallyLit(int literal) + { + this.distanceBuffer[this.lastLiteral] = 0; + this.literalBuffer[this.lastLiteral++] = (byte)literal; + this.literalTree.Freqs[literal]++; + return this.IsFull(); + } + + /// + /// Add distance code and length to literal and distance trees + /// + /// Distance code + /// Length + /// Value indicating if internal buffer is full + public bool TallyDist(int distance, int length) + { + this.distanceBuffer[this.lastLiteral] = (short)distance; + this.literalBuffer[this.lastLiteral++] = (byte)(length - 3); + + int lc = Lcode(length - 3); + this.literalTree.Freqs[lc]++; + if (lc >= 265 && lc < 285) + { + this.extraBits += (lc - 261) / 4; + } + + int dc = Dcode(distance - 1); + this.distTree.Freqs[dc]++; + if (dc >= 4) + { + this.extraBits += (dc / 2) - 1; + } + + return this.IsFull(); + } + + /// + /// Reverse the bits of a 16 bit value. + /// + /// Value to reverse bits + /// Value with bits reversed + public static short BitReverse(int toReverse) + { + return (short)(Bit4Reverse[toReverse & 0xF] << 12 | + Bit4Reverse[(toReverse >> 4) & 0xF] << 8 | + Bit4Reverse[(toReverse >> 8) & 0xF] << 4 | + Bit4Reverse[toReverse >> 12]); + } + + private static int Lcode(int length) + { + if (length == 255) + { + return 285; + } + + int code = 257; + while (length >= 8) + { + code += 4; + length >>= 1; + } - #endregion Instance Fields + return code + length; + } - #region Constructors + private static int Dcode(int distance) + { + int code = 0; + while (distance >= 4) + { + code += 2; + distance >>= 1; + } + + return code + distance; + } + + private class Tree + { + private readonly int minNumCodes; + private short[] codes; + private readonly int[] bitLengthCounts; + private readonly int maxLength; + private readonly DeflaterHuffman dh; public Tree(DeflaterHuffman dh, int elems, int minCodes, int maxLength) { this.dh = dh; this.minNumCodes = minCodes; this.maxLength = maxLength; - freqs = new short[elems]; - bl_counts = new int[maxLength]; + this.Freqs = new short[elems]; + this.bitLengthCounts = new int[maxLength]; } - #endregion Constructors + public int NumCodes { get; private set; } + + public short[] Freqs { get; } + + public byte[] Length { get; set; } /// /// Resets the internal state of the tree /// public void Reset() { - for (int i = 0; i < freqs.Length; i++) + for (int i = 0; i < this.Freqs.Length; i++) { - freqs[i] = 0; + this.Freqs[i] = 0; } - codes = null; - length = null; + + this.codes = null; + this.Length = null; } public void WriteSymbol(int code) { - // if (DeflaterConstants.DEBUGGING) { - // freqs[code]--; - // // Console.Write("writeSymbol("+freqs.length+","+code+"): "); - // } - dh.pending.WriteBits(codes[code] & 0xffff, length[code]); + this.dh.Pending.WriteBits(this.codes[code] & 0xffff, this.Length[code]); } /// @@ -129,9 +431,9 @@ namespace SixLabors.ImageSharp.Formats.Png.Zlib public void CheckEmpty() { bool empty = true; - for (int i = 0; i < freqs.Length; i++) + for (int i = 0; i < this.Freqs.Length; i++) { - empty &= freqs[i] == 0; + empty &= this.Freqs[i] == 0; } if (!empty) @@ -147,8 +449,8 @@ namespace SixLabors.ImageSharp.Formats.Png.Zlib /// length for new codes public void SetStaticCodes(short[] staticCodes, byte[] staticLengths) { - codes = staticCodes; - length = staticLengths; + this.codes = staticCodes; + this.Length = staticLengths; } /// @@ -156,44 +458,24 @@ namespace SixLabors.ImageSharp.Formats.Png.Zlib /// public void BuildCodes() { - int numSymbols = freqs.Length; - int[] nextCode = new int[maxLength]; + int numSymbols = this.Freqs.Length; + int[] nextCode = new int[this.maxLength]; int code = 0; - codes = new short[freqs.Length]; - - // if (DeflaterConstants.DEBUGGING) { - // //Console.WriteLine("buildCodes: "+freqs.Length); - // } + this.codes = new short[this.Freqs.Length]; - for (int bits = 0; bits < maxLength; bits++) + for (int bits = 0; bits < this.maxLength; bits++) { nextCode[bits] = code; - code += bl_counts[bits] << (15 - bits); - - // if (DeflaterConstants.DEBUGGING) { - // //Console.WriteLine("bits: " + ( bits + 1) + " count: " + bl_counts[bits] - // +" nextCode: "+code); - // } + code += this.bitLengthCounts[bits] << (15 - bits); } -#if DebugDeflation - if ( DeflaterConstants.DEBUGGING && (code != 65536) ) - { - throw new ImageFormatException("Inconsistent bl_counts!"); - } -#endif - for (int i = 0; i < numCodes; i++) + for (int i = 0; i < this.NumCodes; i++) { - int bits = length[i]; + int bits = this.Length[i]; if (bits > 0) { - // if (DeflaterConstants.DEBUGGING) { - // //Console.WriteLine("codes["+i+"] = rev(" + nextCode[bits-1]+"), - // +bits); - // } - - codes[i] = BitReverse(nextCode[bits - 1]); + this.codes[i] = BitReverse(nextCode[bits - 1]); nextCode[bits - 1] += 1 << (16 - bits); } } @@ -201,67 +483,65 @@ namespace SixLabors.ImageSharp.Formats.Png.Zlib public void BuildTree() { - int numSymbols = freqs.Length; - - /* heap is a priority queue, sorted by frequency, least frequent - * nodes first. The heap is a binary tree, with the property, that - * the parent node is smaller than both child nodes. This assures - * that the smallest node is the first parent. - * - * The binary tree is encoded in an array: 0 is root node and - * the nodes 2*n+1, 2*n+2 are the child nodes of node n. - */ + int numSymbols = this.Freqs.Length; + + // heap is a priority queue, sorted by frequency, least frequent + // nodes first. The heap is a binary tree, with the property, that + // the parent node is smaller than both child nodes. This assures + // that the smallest node is the first parent. + // + // The binary tree is encoded in an array: 0 is root node and + // the nodes 2*n+1, 2*n+2 are the child nodes of node n. int[] heap = new int[numSymbols]; int heapLen = 0; int maxCode = 0; for (int n = 0; n < numSymbols; n++) { - int freq = freqs[n]; + int freq = this.Freqs[n]; if (freq != 0) { // Insert n into heap int pos = heapLen++; int ppos; - while (pos > 0 && freqs[heap[ppos = (pos - 1) / 2]] > freq) + while (pos > 0 && this.Freqs[heap[ppos = (pos - 1) / 2]] > freq) { heap[pos] = heap[ppos]; pos = ppos; } + heap[pos] = n; maxCode = n; } } - /* We could encode a single literal with 0 bits but then we - * don't see the literals. Therefore we force at least two - * literals to avoid this case. We don't care about order in - * this case, both literals get a 1 bit code. - */ + // We could encode a single literal with 0 bits but then we + // don't see the literals. Therefore we force at least two + // literals to avoid this case. We don't care about order in + // this case, both literals get a 1 bit code. while (heapLen < 2) { int node = maxCode < 2 ? ++maxCode : 0; heap[heapLen++] = node; } - numCodes = Math.Max(maxCode + 1, minNumCodes); + this.NumCodes = Math.Max(maxCode + 1, this.minNumCodes); int numLeafs = heapLen; - int[] childs = new int[4 * heapLen - 2]; - int[] values = new int[2 * heapLen - 1]; + int[] childs = new int[(4 * heapLen) - 2]; + int[] values = new int[(2 * heapLen) - 1]; int numNodes = numLeafs; for (int i = 0; i < heapLen; i++) { int node = heap[i]; childs[2 * i] = node; - childs[2 * i + 1] = -1; - values[i] = freqs[node] << 8; + childs[(2 * i) + 1] = -1; + values[i] = this.Freqs[node] << 8; heap[i] = i; } - /* Construct the Huffman tree by repeatedly combining the least two - * frequent nodes. - */ + // Construct the Huffman tree by repeatedly combining the least two + // frequent nodes. do { int first = heap[0]; @@ -280,17 +560,17 @@ namespace SixLabors.ImageSharp.Formats.Png.Zlib heap[ppos] = heap[path]; ppos = path; - path = path * 2 + 1; + path = (path * 2) + 1; } - /* Now propagate the last element down along path. Normally - * it shouldn't go too deep. - */ + // Now propagate the last element down along path. Normally + // it shouldn't go too deep. int lastVal = values[last]; while ((path = ppos) > 0 && values[heap[ppos = (path - 1) / 2]] > lastVal) { heap[path] = heap[ppos]; } + heap[path] = last; int second = heap[0]; @@ -298,7 +578,7 @@ namespace SixLabors.ImageSharp.Formats.Png.Zlib // Create a new node father of first and second last = numNodes++; childs[2 * last] = first; - childs[2 * last + 1] = second; + childs[(2 * last) + 1] = second; int mindepth = Math.Min(values[first] & 0xff, values[second] & 0xff); values[last] = lastVal = values[first] + values[second] - mindepth + 1; @@ -315,7 +595,7 @@ namespace SixLabors.ImageSharp.Formats.Png.Zlib heap[ppos] = heap[path]; ppos = path; - path = ppos * 2 + 1; + path = (ppos * 2) + 1; } // Now propagate the new element down along path @@ -323,15 +603,17 @@ namespace SixLabors.ImageSharp.Formats.Png.Zlib { heap[path] = heap[ppos]; } + heap[path] = last; - } while (heapLen > 1); + } + while (heapLen > 1); - if (heap[0] != childs.Length / 2 - 1) + if (heap[0] != (childs.Length / 2) - 1) { throw new ImageFormatException("Heap invariant violated"); } - BuildLength(childs); + this.BuildLength(childs); } /// @@ -341,10 +623,11 @@ namespace SixLabors.ImageSharp.Formats.Png.Zlib public int GetEncodedLength() { int len = 0; - for (int i = 0; i < freqs.Length; i++) + for (int i = 0; i < this.Freqs.Length; i++) { - len += freqs[i] * length[i]; + len += this.Freqs[i] * this.Length[i]; } + return len; } @@ -360,10 +643,10 @@ namespace SixLabors.ImageSharp.Formats.Png.Zlib int curlen = -1; /* length of current code */ int i = 0; - while (i < numCodes) + while (i < this.NumCodes) { count = 1; - int nextlen = length[i]; + int nextlen = this.Length[i]; if (nextlen == 0) { max_count = 138; @@ -375,14 +658,15 @@ namespace SixLabors.ImageSharp.Formats.Png.Zlib min_count = 3; if (curlen != nextlen) { - blTree.freqs[nextlen]++; + blTree.Freqs[nextlen]++; count = 0; } } + curlen = nextlen; i++; - while (i < numCodes && curlen == length[i]) + while (i < this.NumCodes && curlen == this.Length[i]) { i++; if (++count >= max_count) @@ -393,19 +677,19 @@ namespace SixLabors.ImageSharp.Formats.Png.Zlib if (count < min_count) { - blTree.freqs[curlen] += (short)count; + blTree.Freqs[curlen] += (short)count; } else if (curlen != 0) { - blTree.freqs[REP_3_6]++; + blTree.Freqs[Repeat3To6]++; } else if (count <= 10) { - blTree.freqs[REP_3_10]++; + blTree.Freqs[Repeat3To10]++; } else { - blTree.freqs[REP_11_138]++; + blTree.Freqs[Repeat11To138]++; } } } @@ -422,10 +706,10 @@ namespace SixLabors.ImageSharp.Formats.Png.Zlib int curlen = -1; // length of current code int i = 0; - while (i < numCodes) + while (i < this.NumCodes) { count = 1; - int nextlen = length[i]; + int nextlen = this.Length[i]; if (nextlen == 0) { max_count = 138; @@ -441,10 +725,11 @@ namespace SixLabors.ImageSharp.Formats.Png.Zlib count = 0; } } + curlen = nextlen; i++; - while (i < numCodes && curlen == length[i]) + while (i < this.NumCodes && curlen == this.Length[i]) { i++; if (++count >= max_count) @@ -462,32 +747,32 @@ namespace SixLabors.ImageSharp.Formats.Png.Zlib } else if (curlen != 0) { - blTree.WriteSymbol(REP_3_6); - dh.pending.WriteBits(count - 3, 2); + blTree.WriteSymbol(Repeat3To6); + this.dh.Pending.WriteBits(count - 3, 2); } else if (count <= 10) { - blTree.WriteSymbol(REP_3_10); - dh.pending.WriteBits(count - 3, 3); + blTree.WriteSymbol(Repeat3To10); + this.dh.Pending.WriteBits(count - 3, 3); } else { - blTree.WriteSymbol(REP_11_138); - dh.pending.WriteBits(count - 11, 7); + blTree.WriteSymbol(Repeat11To138); + this.dh.Pending.WriteBits(count - 11, 7); } } } private void BuildLength(int[] childs) { - this.length = new byte[freqs.Length]; + this.Length = new byte[this.Freqs.Length]; int numNodes = childs.Length / 2; int numLeafs = (numNodes + 1) / 2; int overflow = 0; - for (int i = 0; i < maxLength; i++) + for (int i = 0; i < this.maxLength; i++) { - bl_counts[i] = 0; + this.bitLengthCounts[i] = 0; } // First calculate optimal bit lengths @@ -496,43 +781,36 @@ namespace SixLabors.ImageSharp.Formats.Png.Zlib for (int i = numNodes - 1; i >= 0; i--) { - if (childs[2 * i + 1] != -1) + if (childs[(2 * i) + 1] != -1) { int bitLength = lengths[i] + 1; - if (bitLength > maxLength) + if (bitLength > this.maxLength) { - bitLength = maxLength; + bitLength = this.maxLength; overflow++; } - lengths[childs[2 * i]] = lengths[childs[2 * i + 1]] = bitLength; + + lengths[childs[2 * i]] = lengths[childs[(2 * i) + 1]] = bitLength; } else { // A leaf node int bitLength = lengths[i]; - bl_counts[bitLength - 1]++; - this.length[childs[2 * i]] = (byte)lengths[i]; + this.bitLengthCounts[bitLength - 1]++; + this.Length[childs[2 * i]] = (byte)lengths[i]; } } - // if (DeflaterConstants.DEBUGGING) { - // //Console.WriteLine("Tree "+freqs.Length+" lengths:"); - // for (int i=0; i < numLeafs; i++) { - // //Console.WriteLine("Node "+childs[2*i]+" freq: "+freqs[childs[2*i]] - // + " len: "+length[childs[2*i]]); - // } - // } - if (overflow == 0) { return; } - int incrBitLen = maxLength - 1; + int incrBitLen = this.maxLength - 1; do { // Find the first bit length which could increase: - while (bl_counts[--incrBitLen] == 0) + while (this.bitLengthCounts[--incrBitLen] == 0) { } @@ -540,426 +818,42 @@ namespace SixLabors.ImageSharp.Formats.Png.Zlib // number of overflow nodes. do { - bl_counts[incrBitLen]--; - bl_counts[++incrBitLen]++; - overflow -= 1 << (maxLength - 1 - incrBitLen); - } while (overflow > 0 && incrBitLen < maxLength - 1); - } while (overflow > 0); - - /* We may have overshot above. Move some nodes from maxLength to - * maxLength-1 in that case. - */ - bl_counts[maxLength - 1] += overflow; - bl_counts[maxLength - 2] -= overflow; - - /* Now recompute all bit lengths, scanning in increasing - * frequency. It is simpler to reconstruct all lengths instead of - * fixing only the wrong ones. This idea is taken from 'ar' - * written by Haruhiko Okumura. - * - * The nodes were inserted with decreasing frequency into the childs - * array. - */ + this.bitLengthCounts[incrBitLen]--; + this.bitLengthCounts[++incrBitLen]++; + overflow -= 1 << (this.maxLength - 1 - incrBitLen); + } + while (overflow > 0 && incrBitLen < this.maxLength - 1); + } + while (overflow > 0); + + // We may have overshot above. Move some nodes from maxLength to + // maxLength-1 in that case. + this.bitLengthCounts[this.maxLength - 1] += overflow; + this.bitLengthCounts[this.maxLength - 2] -= overflow; + + // Now recompute all bit lengths, scanning in increasing + // frequency. It is simpler to reconstruct all lengths instead of + // fixing only the wrong ones. This idea is taken from 'ar' + // written by Haruhiko Okumura. + // + // The nodes were inserted with decreasing frequency into the childs + // array. int nodePtr = 2 * numLeafs; - for (int bits = maxLength; bits != 0; bits--) + for (int bits = this.maxLength; bits != 0; bits--) { - int n = bl_counts[bits - 1]; + int n = this.bitLengthCounts[bits - 1]; while (n > 0) { int childPtr = 2 * childs[nodePtr++]; if (childs[childPtr + 1] == -1) { // We found another leaf - length[childs[childPtr]] = (byte)bits; + this.Length[childs[childPtr]] = (byte)bits; n--; } } } - // if (DeflaterConstants.DEBUGGING) { - // //Console.WriteLine("*** After overflow elimination. ***"); - // for (int i=0; i < numLeafs; i++) { - // //Console.WriteLine("Node "+childs[2*i]+" freq: "+freqs[childs[2*i]] - // + " len: "+length[childs[2*i]]); - // } - // } - } - } - - #region Instance Fields - - /// - /// Pending buffer to use - /// - public DeflaterPendingBuffer pending; - - private Tree literalTree; - private Tree distTree; - private Tree blTree; - - // Buffer for distances - private short[] d_buf; - - private byte[] l_buf; - private int last_lit; - private int extra_bits; - - #endregion Instance Fields - - static DeflaterHuffman() - { - // See RFC 1951 3.2.6 - // Literal codes - staticLCodes = new short[LITERAL_NUM]; - staticLLength = new byte[LITERAL_NUM]; - - int i = 0; - while (i < 144) - { - staticLCodes[i] = BitReverse((0x030 + i) << 8); - staticLLength[i++] = 8; - } - - while (i < 256) - { - staticLCodes[i] = BitReverse((0x190 - 144 + i) << 7); - staticLLength[i++] = 9; - } - - while (i < 280) - { - staticLCodes[i] = BitReverse((0x000 - 256 + i) << 9); - staticLLength[i++] = 7; - } - - while (i < LITERAL_NUM) - { - staticLCodes[i] = BitReverse((0x0c0 - 280 + i) << 8); - staticLLength[i++] = 8; - } - - // Distance codes - staticDCodes = new short[DIST_NUM]; - staticDLength = new byte[DIST_NUM]; - for (i = 0; i < DIST_NUM; i++) - { - staticDCodes[i] = BitReverse(i << 11); - staticDLength[i] = 5; - } - } - - /// - /// Construct instance with pending buffer - /// - /// Pending buffer to use - public DeflaterHuffman(DeflaterPendingBuffer pending) - { - this.pending = pending; - - literalTree = new Tree(this, LITERAL_NUM, 257, 15); - distTree = new Tree(this, DIST_NUM, 1, 15); - blTree = new Tree(this, BITLEN_NUM, 4, 7); - - d_buf = new short[BUFSIZE]; - l_buf = new byte[BUFSIZE]; - } - - /// - /// Reset internal state - /// - public void Reset() - { - last_lit = 0; - extra_bits = 0; - literalTree.Reset(); - distTree.Reset(); - blTree.Reset(); - } - - /// - /// Write all trees to pending buffer - /// - /// The number/rank of treecodes to send. - public void SendAllTrees(int blTreeCodes) - { - blTree.BuildCodes(); - literalTree.BuildCodes(); - distTree.BuildCodes(); - pending.WriteBits(literalTree.numCodes - 257, 5); - pending.WriteBits(distTree.numCodes - 1, 5); - pending.WriteBits(blTreeCodes - 4, 4); - for (int rank = 0; rank < blTreeCodes; rank++) - { - pending.WriteBits(blTree.length[BL_ORDER[rank]], 3); - } - literalTree.WriteTree(blTree); - distTree.WriteTree(blTree); - -#if DebugDeflation - if (DeflaterConstants.DEBUGGING) { - blTree.CheckEmpty(); - } -#endif - } - - /// - /// Compress current buffer writing data to pending buffer - /// - public void CompressBlock() - { - for (int i = 0; i < last_lit; i++) - { - int litlen = l_buf[i] & 0xff; - int dist = d_buf[i]; - if (dist-- != 0) - { - // if (DeflaterConstants.DEBUGGING) { - // Console.Write("["+(dist+1)+","+(litlen+3)+"]: "); - // } - - int lc = Lcode(litlen); - literalTree.WriteSymbol(lc); - - int bits = (lc - 261) / 4; - if (bits > 0 && bits <= 5) - { - pending.WriteBits(litlen & ((1 << bits) - 1), bits); - } - - int dc = Dcode(dist); - distTree.WriteSymbol(dc); - - bits = dc / 2 - 1; - if (bits > 0) - { - pending.WriteBits(dist & ((1 << bits) - 1), bits); - } - } - else - { - // if (DeflaterConstants.DEBUGGING) { - // if (litlen > 32 && litlen < 127) { - // Console.Write("("+(char)litlen+"): "); - // } else { - // Console.Write("{"+litlen+"}: "); - // } - // } - literalTree.WriteSymbol(litlen); - } - } - -#if DebugDeflation - if (DeflaterConstants.DEBUGGING) { - Console.Write("EOF: "); - } -#endif - literalTree.WriteSymbol(EOF_SYMBOL); - -#if DebugDeflation - if (DeflaterConstants.DEBUGGING) { - literalTree.CheckEmpty(); - distTree.CheckEmpty(); - } -#endif - } - - /// - /// Flush block to output with no compression - /// - /// Data to write - /// Index of first byte to write - /// Count of bytes to write - /// True if this is the last block - public void FlushStoredBlock(byte[] stored, int storedOffset, int storedLength, bool lastBlock) - { -#if DebugDeflation - // if (DeflaterConstants.DEBUGGING) { - // //Console.WriteLine("Flushing stored block "+ storedLength); - // } -#endif - pending.WriteBits((DeflaterConstants.STORED_BLOCK << 1) + (lastBlock ? 1 : 0), 3); - pending.AlignToByte(); - pending.WriteShort(storedLength); - pending.WriteShort(~storedLength); - pending.WriteBlock(stored, storedOffset, storedLength); - Reset(); - } - - /// - /// Flush block to output with compression - /// - /// Data to flush - /// Index of first byte to flush - /// Count of bytes to flush - /// True if this is the last block - public void FlushBlock(byte[] stored, int storedOffset, int storedLength, bool lastBlock) - { - literalTree.freqs[EOF_SYMBOL]++; - - // Build trees - literalTree.BuildTree(); - distTree.BuildTree(); - - // Calculate bitlen frequency - literalTree.CalcBLFreq(blTree); - distTree.CalcBLFreq(blTree); - - // Build bitlen tree - blTree.BuildTree(); - - int blTreeCodes = 4; - for (int i = 18; i > blTreeCodes; i--) - { - if (blTree.length[BL_ORDER[i]] > 0) - { - blTreeCodes = i + 1; - } - } - int opt_len = 14 + blTreeCodes * 3 + blTree.GetEncodedLength() + - literalTree.GetEncodedLength() + distTree.GetEncodedLength() + - extra_bits; - - int static_len = extra_bits; - for (int i = 0; i < LITERAL_NUM; i++) - { - static_len += literalTree.freqs[i] * staticLLength[i]; - } - for (int i = 0; i < DIST_NUM; i++) - { - static_len += distTree.freqs[i] * staticDLength[i]; - } - if (opt_len >= static_len) - { - // Force static trees - opt_len = static_len; - } - - if (storedOffset >= 0 && storedLength + 4 < opt_len >> 3) - { - // Store Block - - // if (DeflaterConstants.DEBUGGING) { - // //Console.WriteLine("Storing, since " + storedLength + " < " + opt_len - // + " <= " + static_len); - // } - FlushStoredBlock(stored, storedOffset, storedLength, lastBlock); - } - else if (opt_len == static_len) - { - // Encode with static tree - pending.WriteBits((DeflaterConstants.STATIC_TREES << 1) + (lastBlock ? 1 : 0), 3); - literalTree.SetStaticCodes(staticLCodes, staticLLength); - distTree.SetStaticCodes(staticDCodes, staticDLength); - CompressBlock(); - Reset(); - } - else - { - // Encode with dynamic tree - pending.WriteBits((DeflaterConstants.DYN_TREES << 1) + (lastBlock ? 1 : 0), 3); - SendAllTrees(blTreeCodes); - CompressBlock(); - Reset(); - } - } - - /// - /// Get value indicating if internal buffer is full - /// - /// true if buffer is full - public bool IsFull() - { - return last_lit >= BUFSIZE; - } - - /// - /// Add literal to buffer - /// - /// Literal value to add to buffer. - /// Value indicating internal buffer is full - public bool TallyLit(int literal) - { - // if (DeflaterConstants.DEBUGGING) { - // if (lit > 32 && lit < 127) { - // //Console.WriteLine("("+(char)lit+")"); - // } else { - // //Console.WriteLine("{"+lit+"}"); - // } - // } - d_buf[last_lit] = 0; - l_buf[last_lit++] = (byte)literal; - literalTree.freqs[literal]++; - return IsFull(); - } - - /// - /// Add distance code and length to literal and distance trees - /// - /// Distance code - /// Length - /// Value indicating if internal buffer is full - public bool TallyDist(int distance, int length) - { - // if (DeflaterConstants.DEBUGGING) { - // //Console.WriteLine("[" + distance + "," + length + "]"); - // } - - d_buf[last_lit] = (short)distance; - l_buf[last_lit++] = (byte)(length - 3); - - int lc = Lcode(length - 3); - literalTree.freqs[lc]++; - if (lc >= 265 && lc < 285) - { - extra_bits += (lc - 261) / 4; - } - - int dc = Dcode(distance - 1); - distTree.freqs[dc]++; - if (dc >= 4) - { - extra_bits += dc / 2 - 1; - } - return IsFull(); - } - - /// - /// Reverse the bits of a 16 bit value. - /// - /// Value to reverse bits - /// Value with bits reversed - public static short BitReverse(int toReverse) - { - return (short)(bit4Reverse[toReverse & 0xF] << 12 | - bit4Reverse[(toReverse >> 4) & 0xF] << 8 | - bit4Reverse[(toReverse >> 8) & 0xF] << 4 | - bit4Reverse[toReverse >> 12]); - } - - private static int Lcode(int length) - { - if (length == 255) - { - return 285; } - - int code = 257; - while (length >= 8) - { - code += 4; - length >>= 1; - } - return code + length; - } - - private static int Dcode(int distance) - { - int code = 0; - while (distance >= 4) - { - code += 2; - distance >>= 1; - } - return code + distance; } } } From 209895180c71969c1eb43025e365370c6b43b3b4 Mon Sep 17 00:00:00 2001 From: James Jackson-South Date: Wed, 13 Nov 2019 21:44:21 +1100 Subject: [PATCH 09/16] Move pending buffer --- src/ImageSharp/Formats/Png/Zlib/Deflater.cs | 19 ++++------- .../Formats/Png/Zlib/DeflaterEngine.cs | 19 +++++++---- .../Formats/Png/Zlib/DeflaterHuffman.cs | 33 ++++++++++++++++--- 3 files changed, 48 insertions(+), 23 deletions(-) diff --git a/src/ImageSharp/Formats/Png/Zlib/Deflater.cs b/src/ImageSharp/Formats/Png/Zlib/Deflater.cs index ef1d0e116..d1560eb4b 100644 --- a/src/ImageSharp/Formats/Png/Zlib/Deflater.cs +++ b/src/ImageSharp/Formats/Png/Zlib/Deflater.cs @@ -50,7 +50,6 @@ namespace SixLabors.ImageSharp.Formats.Png.Zlib /// private int state; - private DeflaterPendingBuffer pending; private DeflaterEngine engine; private bool isDisposed; @@ -80,10 +79,8 @@ namespace SixLabors.ImageSharp.Formats.Png.Zlib throw new ArgumentOutOfRangeException(nameof(level)); } - this.pending = new DeflaterPendingBuffer(memoryAllocator); - // TODO: Possibly provide DeflateStrategy as an option. - this.engine = new DeflaterEngine(memoryAllocator, this.pending, DeflateStrategy.Default); + this.engine = new DeflaterEngine(memoryAllocator, DeflateStrategy.Default); this.SetLevel(level); this.Reset(); @@ -126,7 +123,7 @@ namespace SixLabors.ImageSharp.Formats.Png.Zlib /// Gets a value indicating whetherthe stream was finished and no more output bytes /// are available. /// - public bool IsFinished => (this.state == FinishedState) && this.pending.IsFlushed; + public bool IsFinished => (this.state == FinishedState) && this.engine.Pending.IsFlushed; /// /// Gets a value indicating whether the input buffer is empty. @@ -145,7 +142,7 @@ namespace SixLabors.ImageSharp.Formats.Png.Zlib public void Reset() { this.state = BusyState; - this.pending.Reset(); + this.engine.Pending.Reset(); this.engine.Reset(); } @@ -236,7 +233,7 @@ namespace SixLabors.ImageSharp.Formats.Png.Zlib while (true) { - int count = this.pending.Flush(output, offset, length); + int count = this.engine.Pending.Flush(output, offset, length); offset += count; length -= count; @@ -259,11 +256,11 @@ namespace SixLabors.ImageSharp.Formats.Png.Zlib // We have to supply some lookahead. 8 bit lookahead // is needed by the zlib inflater, and we must fill // the next byte, so that all bits are flushed. - int neededbits = 8 + ((-this.pending.BitCount) & 7); + int neededbits = 8 + ((-this.engine.Pending.BitCount) & 7); while (neededbits > 0) { // Write a static tree block consisting solely of an EOF: - this.pending.WriteBits(2, 10); + this.engine.Pending.WriteBits(2, 10); neededbits -= 10; } } @@ -272,7 +269,7 @@ namespace SixLabors.ImageSharp.Formats.Png.Zlib break; case FinishingState: - this.pending.AlignToByte(); + this.engine.Pending.AlignToByte(); this.state = FinishedState; break; } @@ -296,10 +293,8 @@ namespace SixLabors.ImageSharp.Formats.Png.Zlib if (disposing) { this.engine.Dispose(); - this.pending.Dispose(); } - this.pending = null; this.engine = null; this.isDisposed = true; } diff --git a/src/ImageSharp/Formats/Png/Zlib/DeflaterEngine.cs b/src/ImageSharp/Formats/Png/Zlib/DeflaterEngine.cs index b14682501..8f57f51f9 100644 --- a/src/ImageSharp/Formats/Png/Zlib/DeflaterEngine.cs +++ b/src/ImageSharp/Formats/Png/Zlib/DeflaterEngine.cs @@ -100,7 +100,6 @@ namespace SixLabors.ImageSharp.Formats.Png.Zlib private int inputEnd; private readonly DeflateStrategy strategy; - private readonly DeflaterPendingBuffer pending; private DeflaterHuffman huffman; private bool isDisposed; @@ -145,12 +144,11 @@ namespace SixLabors.ImageSharp.Formats.Png.Zlib /// Initializes a new instance of the class. /// /// The memory allocator to use for buffer allocations. - /// The pending buffer to use. /// The deflate strategy to use. - public DeflaterEngine(MemoryAllocator memoryAllocator, DeflaterPendingBuffer pending, DeflateStrategy strategy) + public DeflaterEngine(MemoryAllocator memoryAllocator, DeflateStrategy strategy) { - this.pending = pending; - this.huffman = new DeflaterHuffman(pending); + this.huffman = new DeflaterHuffman(memoryAllocator); + this.Pending = this.huffman.Pending; this.strategy = strategy; // Create pinned pointers to the various buffers to allow indexing @@ -175,6 +173,11 @@ namespace SixLabors.ImageSharp.Formats.Png.Zlib this.blockStart = this.strstart = 1; } + /// + /// Gets the pending buffer to use. + /// + public DeflaterPendingBuffer Pending { get; } + /// /// Deflate drives actual compression of data /// @@ -208,7 +211,7 @@ namespace SixLabors.ImageSharp.Formats.Png.Zlib break; } } - while (this.pending.IsFlushed && progress); // repeat while we have no pending output and progress was made + while (this.Pending.IsFlushed && progress); // repeat while we have no pending output and progress was made return progress; } @@ -834,6 +837,8 @@ namespace SixLabors.ImageSharp.Formats.Png.Zlib { if (disposing) { + this.huffman.Dispose(); + this.windowBufferHandle.Dispose(); this.windowBuffer.Dispose(); @@ -844,6 +849,8 @@ namespace SixLabors.ImageSharp.Formats.Png.Zlib this.prevBuffer.Dispose(); } + this.huffman = null; + this.isDisposed = true; } } diff --git a/src/ImageSharp/Formats/Png/Zlib/DeflaterHuffman.cs b/src/ImageSharp/Formats/Png/Zlib/DeflaterHuffman.cs index bc7ad7a38..1a2d661c1 100644 --- a/src/ImageSharp/Formats/Png/Zlib/DeflaterHuffman.cs +++ b/src/ImageSharp/Formats/Png/Zlib/DeflaterHuffman.cs @@ -4,6 +4,7 @@ using System; using System.Collections.Generic; using System.Text; +using SixLabors.Memory; namespace SixLabors.ImageSharp.Formats.Png.Zlib { @@ -15,7 +16,7 @@ namespace SixLabors.ImageSharp.Formats.Png.Zlib /// /// author of the original java version : Jochen Hoenicke /// - public class DeflaterHuffman + public sealed class DeflaterHuffman : IDisposable { private const int BufferSize = 1 << (DeflaterConstants.DEFAULT_MEM_LEVEL + 6); @@ -60,6 +61,7 @@ namespace SixLabors.ImageSharp.Formats.Png.Zlib private byte[] literalBuffer; private int lastLiteral; private int extraBits; + private bool isDisposed; static DeflaterHuffman() { @@ -106,10 +108,10 @@ namespace SixLabors.ImageSharp.Formats.Png.Zlib /// /// Initializes a new instance of the class. /// - /// Pending buffer to use - public DeflaterHuffman(DeflaterPendingBuffer pending) + /// The memory allocator to use for buffer allocations. + public DeflaterHuffman(MemoryAllocator memoryAllocator) { - this.Pending = pending; + this.Pending = new DeflaterPendingBuffer(memoryAllocator); this.literalTree = new Tree(this, LiteralNumber, 257, 15); this.distTree = new Tree(this, DistanceNumber, 1, 15); @@ -122,7 +124,7 @@ namespace SixLabors.ImageSharp.Formats.Png.Zlib /// /// Gets the pending buffer to use. /// - public DeflaterPendingBuffer Pending { get; } + public DeflaterPendingBuffer Pending { get; private set; } /// /// Reset internal state @@ -351,6 +353,13 @@ namespace SixLabors.ImageSharp.Formats.Png.Zlib Bit4Reverse[toReverse >> 12]); } + /// + public void Dispose() + { + this.Dispose(true); + GC.SuppressFinalize(this); + } + private static int Lcode(int length) { if (length == 255) @@ -380,6 +389,20 @@ namespace SixLabors.ImageSharp.Formats.Png.Zlib return code + distance; } + private void Dispose(bool disposing) + { + if (!this.isDisposed) + { + if (disposing) + { + this.Pending.Dispose(); + } + + this.Pending = null; + this.isDisposed = true; + } + } + private class Tree { private readonly int minNumCodes; From d4155c7365d4344c1987ba7190a0671e62831bc0 Mon Sep 17 00:00:00 2001 From: James Jackson-South Date: Thu, 14 Nov 2019 00:01:03 +1100 Subject: [PATCH 10/16] Use pinned buffers for distances --- .../Formats/Png/Zlib/DeflateThrowHelper.cs | 7 + .../Formats/Png/Zlib/DeflaterHuffman.cs | 205 ++++++++++-------- .../Formats/Png/Zlib/DeflaterPendingBuffer.cs | 1 - 3 files changed, 119 insertions(+), 94 deletions(-) diff --git a/src/ImageSharp/Formats/Png/Zlib/DeflateThrowHelper.cs b/src/ImageSharp/Formats/Png/Zlib/DeflateThrowHelper.cs index 2698c9b99..d7198c4ee 100644 --- a/src/ImageSharp/Formats/Png/Zlib/DeflateThrowHelper.cs +++ b/src/ImageSharp/Formats/Png/Zlib/DeflateThrowHelper.cs @@ -17,6 +17,7 @@ namespace SixLabors.ImageSharp.Formats.Png.Zlib [MethodImpl(InliningOptions.ColdPath)] public static void ThrowUnknownCompression() => throw new InvalidOperationException("Unknown compression function."); + [MethodImpl(InliningOptions.ColdPath)] public static void ThrowNotProcessed() => throw new InvalidOperationException("Old input was not completely processed."); [MethodImpl(InliningOptions.ColdPath)] @@ -24,5 +25,11 @@ namespace SixLabors.ImageSharp.Formats.Png.Zlib [MethodImpl(InliningOptions.ColdPath)] public static void ThrowOutOfRange(string name) => throw new ArgumentOutOfRangeException(name); + + [MethodImpl(InliningOptions.ColdPath)] + public static void ThrowFrequencyNotEmpty() => throw new InvalidOperationException("Huffman frequency entry non empty."); + + [MethodImpl(InliningOptions.ColdPath)] + public static void ThrowHeapViolated() => throw new InvalidOperationException("Huffman heap invariant violated."); } } diff --git a/src/ImageSharp/Formats/Png/Zlib/DeflaterHuffman.cs b/src/ImageSharp/Formats/Png/Zlib/DeflaterHuffman.cs index 1a2d661c1..36cf0d559 100644 --- a/src/ImageSharp/Formats/Png/Zlib/DeflaterHuffman.cs +++ b/src/ImageSharp/Formats/Png/Zlib/DeflaterHuffman.cs @@ -2,8 +2,8 @@ // Licensed under the Apache License, Version 2.0. using System; -using System.Collections.Generic; -using System.Text; +using System.Buffers; +using System.Runtime.CompilerServices; using SixLabors.Memory; namespace SixLabors.ImageSharp.Formats.Png.Zlib @@ -16,7 +16,7 @@ namespace SixLabors.ImageSharp.Formats.Png.Zlib /// /// author of the original java version : Jochen Hoenicke /// - public sealed class DeflaterHuffman : IDisposable + public sealed unsafe class DeflaterHuffman : IDisposable { private const int BufferSize = 1 << (DeflaterConstants.DEFAULT_MEM_LEVEL + 6); @@ -29,13 +29,13 @@ namespace SixLabors.ImageSharp.Formats.Png.Zlib // Number of codes used to transfer bit lengths private const int BitLengthNumber = 19; - // repeat previous bit length 3-6 times (2 bits of repeat count) + // Repeat previous bit length 3-6 times (2 bits of repeat count) private const int Repeat3To6 = 16; - // repeat a zero length 3-10 times (3 bits of repeat count) + // Repeat a zero length 3-10 times (3 bits of repeat count) private const int Repeat3To10 = 17; - // repeat a zero length 11-138 times (7 bits of repeat count) + // Repeat a zero length 11-138 times (7 bits of repeat count) private const int Repeat11To138 = 18; private const int EofSymbol = 256; @@ -46,19 +46,24 @@ namespace SixLabors.ImageSharp.Formats.Png.Zlib private static readonly byte[] Bit4Reverse = { 0, 8, 4, 12, 2, 10, 6, 14, 1, 9, 5, 13, 3, 11, 7, 15 }; - private static short[] staticLCodes; - private static byte[] staticLLength; - private static short[] staticDCodes; - private static byte[] staticDLength; + private static readonly short[] StaticLCodes; + private static readonly byte[] StaticLLength; + private static readonly short[] StaticDCodes; + private static readonly byte[] StaticDLength; private Tree literalTree; private Tree distTree; private Tree blTree; // Buffer for distances - private short[] distanceBuffer; + private readonly IMemoryOwner distanceManagedBuffer; + private readonly short* pinnedDistanceBuffer; + private MemoryHandle distanceBufferHandle; + + private readonly IMemoryOwner literalManagedBuffer; + private readonly short* pinnedLiteralBuffer; + private MemoryHandle literalBufferHandle; - private byte[] literalBuffer; private int lastLiteral; private int extraBits; private bool isDisposed; @@ -67,41 +72,41 @@ namespace SixLabors.ImageSharp.Formats.Png.Zlib { // See RFC 1951 3.2.6 // Literal codes - staticLCodes = new short[LiteralNumber]; - staticLLength = new byte[LiteralNumber]; + StaticLCodes = new short[LiteralNumber]; + StaticLLength = new byte[LiteralNumber]; int i = 0; while (i < 144) { - staticLCodes[i] = BitReverse((0x030 + i) << 8); - staticLLength[i++] = 8; + StaticLCodes[i] = BitReverse((0x030 + i) << 8); + StaticLLength[i++] = 8; } while (i < 256) { - staticLCodes[i] = BitReverse((0x190 - 144 + i) << 7); - staticLLength[i++] = 9; + StaticLCodes[i] = BitReverse((0x190 - 144 + i) << 7); + StaticLLength[i++] = 9; } while (i < 280) { - staticLCodes[i] = BitReverse((0x000 - 256 + i) << 9); - staticLLength[i++] = 7; + StaticLCodes[i] = BitReverse((0x000 - 256 + i) << 9); + StaticLLength[i++] = 7; } while (i < LiteralNumber) { - staticLCodes[i] = BitReverse((0x0c0 - 280 + i) << 8); - staticLLength[i++] = 8; + StaticLCodes[i] = BitReverse((0x0c0 - 280 + i) << 8); + StaticLLength[i++] = 8; } // Distance codes - staticDCodes = new short[DistanceNumber]; - staticDLength = new byte[DistanceNumber]; + StaticDCodes = new short[DistanceNumber]; + StaticDLength = new byte[DistanceNumber]; for (i = 0; i < DistanceNumber; i++) { - staticDCodes[i] = BitReverse(i << 11); - staticDLength[i] = 5; + StaticDCodes[i] = BitReverse(i << 11); + StaticDLength[i] = 5; } } @@ -113,12 +118,17 @@ namespace SixLabors.ImageSharp.Formats.Png.Zlib { this.Pending = new DeflaterPendingBuffer(memoryAllocator); - this.literalTree = new Tree(this, LiteralNumber, 257, 15); - this.distTree = new Tree(this, DistanceNumber, 1, 15); - this.blTree = new Tree(this, BitLengthNumber, 4, 7); + this.literalTree = new Tree(LiteralNumber, 257, 15); + this.distTree = new Tree(DistanceNumber, 1, 15); + this.blTree = new Tree(BitLengthNumber, 4, 7); + + this.distanceManagedBuffer = memoryAllocator.Allocate(BufferSize); + this.distanceBufferHandle = this.distanceManagedBuffer.Memory.Pin(); + this.pinnedDistanceBuffer = (short*)this.distanceBufferHandle.Pointer; - this.distanceBuffer = new short[BufferSize]; - this.literalBuffer = new byte[BufferSize]; + this.literalManagedBuffer = memoryAllocator.Allocate(BufferSize); + this.literalBufferHandle = this.literalManagedBuffer.Memory.Pin(); + this.pinnedLiteralBuffer = (short*)this.literalBufferHandle.Pointer; } /// @@ -155,8 +165,8 @@ namespace SixLabors.ImageSharp.Formats.Png.Zlib this.Pending.WriteBits(this.blTree.Length[BitLengthOrder[rank]], 3); } - this.literalTree.WriteTree(this.blTree); - this.distTree.WriteTree(this.blTree); + this.literalTree.WriteTree(this.Pending, this.blTree); + this.distTree.WriteTree(this.Pending, this.blTree); } /// @@ -164,14 +174,18 @@ namespace SixLabors.ImageSharp.Formats.Png.Zlib /// public void CompressBlock() { + DeflaterPendingBuffer pendingBuffer = this.Pending; + short* pinnedDistance = this.pinnedDistanceBuffer; + short* pinnedLiteral = this.pinnedLiteralBuffer; + for (int i = 0; i < this.lastLiteral; i++) { - int litlen = this.literalBuffer[i] & 0xff; - int dist = this.distanceBuffer[i]; + int litlen = pinnedLiteral[i] & 0xFF; + int dist = pinnedDistance[i]; if (dist-- != 0) { int lc = Lcode(litlen); - this.literalTree.WriteSymbol(lc); + this.literalTree.WriteSymbol(pendingBuffer, lc); int bits = (lc - 261) / 4; if (bits > 0 && bits <= 5) @@ -180,7 +194,7 @@ namespace SixLabors.ImageSharp.Formats.Png.Zlib } int dc = Dcode(dist); - this.distTree.WriteSymbol(dc); + this.distTree.WriteSymbol(pendingBuffer, dc); bits = (dc / 2) - 1; if (bits > 0) @@ -190,11 +204,11 @@ namespace SixLabors.ImageSharp.Formats.Png.Zlib } else { - this.literalTree.WriteSymbol(litlen); + this.literalTree.WriteSymbol(pendingBuffer, litlen); } } - this.literalTree.WriteSymbol(EofSymbol); + this.literalTree.WriteSymbol(pendingBuffer, EofSymbol); } /// @@ -245,19 +259,19 @@ namespace SixLabors.ImageSharp.Formats.Png.Zlib } } - int opt_len = 14 + (blTreeCodes * 3) + this.blTree.GetEncodedLength() + - this.literalTree.GetEncodedLength() + this.distTree.GetEncodedLength() + - this.extraBits; + int opt_len = 14 + (blTreeCodes * 3) + this.blTree.GetEncodedLength() + + this.literalTree.GetEncodedLength() + this.distTree.GetEncodedLength() + + this.extraBits; int static_len = this.extraBits; for (int i = 0; i < LiteralNumber; i++) { - static_len += this.literalTree.Freqs[i] * staticLLength[i]; + static_len += this.literalTree.Freqs[i] * StaticLLength[i]; } for (int i = 0; i < DistanceNumber; i++) { - static_len += this.distTree.Freqs[i] * staticDLength[i]; + static_len += this.distTree.Freqs[i] * StaticDLength[i]; } if (opt_len >= static_len) @@ -275,8 +289,8 @@ namespace SixLabors.ImageSharp.Formats.Png.Zlib { // Encode with static tree this.Pending.WriteBits((DeflaterConstants.STATIC_TREES << 1) + (lastBlock ? 1 : 0), 3); - this.literalTree.SetStaticCodes(staticLCodes, staticLLength); - this.distTree.SetStaticCodes(staticDCodes, staticDLength); + this.literalTree.SetStaticCodes(StaticLCodes, StaticLLength); + this.distTree.SetStaticCodes(StaticDCodes, StaticDLength); this.CompressBlock(); this.Reset(); } @@ -294,20 +308,19 @@ namespace SixLabors.ImageSharp.Formats.Png.Zlib /// Get value indicating if internal buffer is full /// /// true if buffer is full - public bool IsFull() - { - return this.lastLiteral >= BufferSize; - } + [MethodImpl(InliningOptions.ShortMethod)] + public bool IsFull() => this.lastLiteral >= BufferSize; /// /// Add literal to buffer /// /// Literal value to add to buffer. /// Value indicating internal buffer is full + [MethodImpl(InliningOptions.ShortMethod)] public bool TallyLit(int literal) { - this.distanceBuffer[this.lastLiteral] = 0; - this.literalBuffer[this.lastLiteral++] = (byte)literal; + this.pinnedDistanceBuffer[this.lastLiteral] = 0; + this.pinnedLiteralBuffer[this.lastLiteral++] = (byte)literal; this.literalTree.Freqs[literal]++; return this.IsFull(); } @@ -318,10 +331,11 @@ namespace SixLabors.ImageSharp.Formats.Png.Zlib /// Distance code /// Length /// Value indicating if internal buffer is full + [MethodImpl(InliningOptions.ShortMethod)] public bool TallyDist(int distance, int length) { - this.distanceBuffer[this.lastLiteral] = (short)distance; - this.literalBuffer[this.lastLiteral++] = (byte)(length - 3); + this.pinnedDistanceBuffer[this.lastLiteral] = (short)distance; + this.pinnedLiteralBuffer[this.lastLiteral++] = (byte)(length - 3); int lc = Lcode(length - 3); this.literalTree.Freqs[lc]++; @@ -345,12 +359,13 @@ namespace SixLabors.ImageSharp.Formats.Png.Zlib /// /// Value to reverse bits /// Value with bits reversed + [MethodImpl(InliningOptions.ShortMethod)] public static short BitReverse(int toReverse) { - return (short)(Bit4Reverse[toReverse & 0xF] << 12 | - Bit4Reverse[(toReverse >> 4) & 0xF] << 8 | - Bit4Reverse[(toReverse >> 8) & 0xF] << 4 | - Bit4Reverse[toReverse >> 12]); + return (short)(Bit4Reverse[toReverse & 0xF] << 12 + | Bit4Reverse[(toReverse >> 4) & 0xF] << 8 + | Bit4Reverse[(toReverse >> 8) & 0xF] << 4 + | Bit4Reverse[toReverse >> 12]); } /// @@ -360,6 +375,7 @@ namespace SixLabors.ImageSharp.Formats.Png.Zlib GC.SuppressFinalize(this); } + [MethodImpl(InliningOptions.ShortMethod)] private static int Lcode(int length) { if (length == 255) @@ -377,6 +393,7 @@ namespace SixLabors.ImageSharp.Formats.Png.Zlib return code + length; } + [MethodImpl(InliningOptions.ShortMethod)] private static int Dcode(int distance) { int code = 0; @@ -396,6 +413,10 @@ namespace SixLabors.ImageSharp.Formats.Png.Zlib if (disposing) { this.Pending.Dispose(); + this.distanceBufferHandle.Dispose(); + this.distanceManagedBuffer.Dispose(); + this.literalBufferHandle.Dispose(); + this.literalManagedBuffer.Dispose(); } this.Pending = null; @@ -403,20 +424,18 @@ namespace SixLabors.ImageSharp.Formats.Png.Zlib } } - private class Tree + private sealed class Tree { private readonly int minNumCodes; private short[] codes; private readonly int[] bitLengthCounts; private readonly int maxLength; - private readonly DeflaterHuffman dh; - public Tree(DeflaterHuffman dh, int elems, int minCodes, int maxLength) + public Tree(int elements, int minCodes, int maxLength) { - this.dh = dh; this.minNumCodes = minCodes; this.maxLength = maxLength; - this.Freqs = new short[elems]; + this.Freqs = new short[elements]; this.bitLengthCounts = new int[maxLength]; } @@ -429,6 +448,7 @@ namespace SixLabors.ImageSharp.Formats.Png.Zlib /// /// Resets the internal state of the tree /// + [MethodImpl(InliningOptions.ShortMethod)] public void Reset() { for (int i = 0; i < this.Freqs.Length; i++) @@ -440,17 +460,17 @@ namespace SixLabors.ImageSharp.Formats.Png.Zlib this.Length = null; } - public void WriteSymbol(int code) - { - this.dh.Pending.WriteBits(this.codes[code] & 0xffff, this.Length[code]); - } + [MethodImpl(InliningOptions.ShortMethod)] + public void WriteSymbol(DeflaterPendingBuffer pendingBuffer, int code) + => pendingBuffer.WriteBits(this.codes[code] & 0xFFFF, this.Length[code]); /// /// Check that all frequencies are zero /// - /// + /// /// At least one frequency is non-zero /// + [MethodImpl(InliningOptions.ShortMethod)] public void CheckEmpty() { bool empty = true; @@ -461,7 +481,7 @@ namespace SixLabors.ImageSharp.Formats.Png.Zlib if (!empty) { - throw new ImageFormatException("!Empty"); + DeflateThrowHelper.ThrowFrequencyNotEmpty(); } } @@ -481,7 +501,6 @@ namespace SixLabors.ImageSharp.Formats.Png.Zlib /// public void BuildCodes() { - int numSymbols = this.Freqs.Length; int[] nextCode = new int[this.maxLength]; int code = 0; @@ -544,8 +563,7 @@ namespace SixLabors.ImageSharp.Formats.Png.Zlib // this case, both literals get a 1 bit code. while (heapLen < 2) { - int node = maxCode < 2 ? ++maxCode : 0; - heap[heapLen++] = node; + heap[heapLen++] = maxCode < 2 ? ++maxCode : 0; } this.NumCodes = Math.Max(maxCode + 1, this.minNumCodes); @@ -602,7 +620,7 @@ namespace SixLabors.ImageSharp.Formats.Png.Zlib last = numNodes++; childs[2 * last] = first; childs[(2 * last) + 1] = second; - int mindepth = Math.Min(values[first] & 0xff, values[second] & 0xff); + int mindepth = Math.Min(values[first] & 0xFF, values[second] & 0xFF); values[last] = lastVal = values[first] + values[second] - mindepth + 1; // Again, propagate the hole to the leafs @@ -633,7 +651,7 @@ namespace SixLabors.ImageSharp.Formats.Png.Zlib if (heap[0] != (childs.Length / 2) - 1) { - throw new ImageFormatException("Heap invariant violated"); + DeflateThrowHelper.ThrowHeapViolated(); } this.BuildLength(childs); @@ -718,10 +736,11 @@ namespace SixLabors.ImageSharp.Formats.Png.Zlib } /// - /// Write tree values + /// Write the tree values. /// - /// Tree to write - public void WriteTree(Tree blTree) + /// The pending buffer. + /// The tree to write. + public void WriteTree(DeflaterPendingBuffer pendingBuffer, Tree bitLengthTree) { int max_count; // max repeat count int min_count; // min repeat count @@ -744,7 +763,7 @@ namespace SixLabors.ImageSharp.Formats.Png.Zlib min_count = 3; if (curlen != nextlen) { - blTree.WriteSymbol(nextlen); + bitLengthTree.WriteSymbol(pendingBuffer, nextlen); count = 0; } } @@ -765,31 +784,31 @@ namespace SixLabors.ImageSharp.Formats.Png.Zlib { while (count-- > 0) { - blTree.WriteSymbol(curlen); + bitLengthTree.WriteSymbol(pendingBuffer, curlen); } } else if (curlen != 0) { - blTree.WriteSymbol(Repeat3To6); - this.dh.Pending.WriteBits(count - 3, 2); + bitLengthTree.WriteSymbol(pendingBuffer, Repeat3To6); + pendingBuffer.WriteBits(count - 3, 2); } else if (count <= 10) { - blTree.WriteSymbol(Repeat3To10); - this.dh.Pending.WriteBits(count - 3, 3); + bitLengthTree.WriteSymbol(pendingBuffer, Repeat3To10); + pendingBuffer.WriteBits(count - 3, 3); } else { - blTree.WriteSymbol(Repeat11To138); - this.dh.Pending.WriteBits(count - 11, 7); + bitLengthTree.WriteSymbol(pendingBuffer, Repeat11To138); + pendingBuffer.WriteBits(count - 11, 7); } } } - private void BuildLength(int[] childs) + private void BuildLength(int[] children) { this.Length = new byte[this.Freqs.Length]; - int numNodes = childs.Length / 2; + int numNodes = children.Length / 2; int numLeafs = (numNodes + 1) / 2; int overflow = 0; @@ -804,7 +823,7 @@ namespace SixLabors.ImageSharp.Formats.Png.Zlib for (int i = numNodes - 1; i >= 0; i--) { - if (childs[(2 * i) + 1] != -1) + if (children[(2 * i) + 1] != -1) { int bitLength = lengths[i] + 1; if (bitLength > this.maxLength) @@ -813,14 +832,14 @@ namespace SixLabors.ImageSharp.Formats.Png.Zlib overflow++; } - lengths[childs[2 * i]] = lengths[childs[(2 * i) + 1]] = bitLength; + lengths[children[2 * i]] = lengths[children[(2 * i) + 1]] = bitLength; } else { // A leaf node int bitLength = lengths[i]; this.bitLengthCounts[bitLength - 1]++; - this.Length[childs[2 * i]] = (byte)lengths[i]; + this.Length[children[2 * i]] = (byte)lengths[i]; } } @@ -867,11 +886,11 @@ namespace SixLabors.ImageSharp.Formats.Png.Zlib int n = this.bitLengthCounts[bits - 1]; while (n > 0) { - int childPtr = 2 * childs[nodePtr++]; - if (childs[childPtr + 1] == -1) + int childPtr = 2 * children[nodePtr++]; + if (children[childPtr + 1] == -1) { // We found another leaf - this.Length[childs[childPtr]] = (byte)bits; + this.Length[children[childPtr]] = (byte)bits; n--; } } diff --git a/src/ImageSharp/Formats/Png/Zlib/DeflaterPendingBuffer.cs b/src/ImageSharp/Formats/Png/Zlib/DeflaterPendingBuffer.cs index 64214b47e..da4d89040 100644 --- a/src/ImageSharp/Formats/Png/Zlib/DeflaterPendingBuffer.cs +++ b/src/ImageSharp/Formats/Png/Zlib/DeflaterPendingBuffer.cs @@ -29,7 +29,6 @@ namespace SixLabors.ImageSharp.Formats.Png.Zlib /// The memory allocator to use for buffer allocations. public DeflaterPendingBuffer(MemoryAllocator memoryAllocator) { - this.buffer = new byte[DeflaterConstants.PENDING_BUF_SIZE]; this.managedBuffer = memoryAllocator.AllocateManagedByteBuffer(DeflaterConstants.PENDING_BUF_SIZE); this.buffer = this.managedBuffer.Array; this.handle = this.managedBuffer.Memory.Pin(); From 89cc2e9fcd50dab2f82ff9452aa9eac4bc57da06 Mon Sep 17 00:00:00 2001 From: James Jackson-South Date: Thu, 14 Nov 2019 00:59:16 +1100 Subject: [PATCH 11/16] Remove a few more bounds checks. --- src/ImageSharp/Formats/Png/Zlib/DeflaterHuffman.cs | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/src/ImageSharp/Formats/Png/Zlib/DeflaterHuffman.cs b/src/ImageSharp/Formats/Png/Zlib/DeflaterHuffman.cs index 36cf0d559..b2eddc37b 100644 --- a/src/ImageSharp/Formats/Png/Zlib/DeflaterHuffman.cs +++ b/src/ImageSharp/Formats/Png/Zlib/DeflaterHuffman.cs @@ -4,17 +4,13 @@ using System; using System.Buffers; using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; using SixLabors.Memory; namespace SixLabors.ImageSharp.Formats.Png.Zlib { /// - /// This is the DeflaterHuffman class. - /// - /// This class is not thread safe. This is inherent in the API, due - /// to the split of Deflate and SetInput. - /// - /// author of the original java version : Jochen Hoenicke + /// Performs Deflate Huffman encoding. /// public sealed unsafe class DeflaterHuffman : IDisposable { @@ -264,14 +260,16 @@ namespace SixLabors.ImageSharp.Formats.Png.Zlib + this.extraBits; int static_len = this.extraBits; + ref byte staticLLengthRef = ref MemoryMarshal.GetReference(StaticLLength); for (int i = 0; i < LiteralNumber; i++) { - static_len += this.literalTree.Freqs[i] * StaticLLength[i]; + static_len += this.literalTree.Freqs[i] * Unsafe.Add(ref staticLLengthRef, i); } + ref byte staticDLengthRef = ref MemoryMarshal.GetReference(StaticDLength); for (int i = 0; i < DistanceNumber; i++) { - static_len += this.distTree.Freqs[i] * StaticDLength[i]; + static_len += this.distTree.Freqs[i] * Unsafe.Add(ref staticDLengthRef, i); } if (opt_len >= static_len) From 645421b803a1685633eef403c1be309fda7c4803 Mon Sep 17 00:00:00 2001 From: James Jackson-South Date: Thu, 14 Nov 2019 15:49:39 +1100 Subject: [PATCH 12/16] More speedup and bounds checks removal. --- .../Formats/Png/Zlib/DeflaterHuffman.cs | 198 ++++++++++++------ .../Formats/Png/Zlib/DeflaterPendingBuffer.cs | 18 +- 2 files changed, 144 insertions(+), 72 deletions(-) diff --git a/src/ImageSharp/Formats/Png/Zlib/DeflaterHuffman.cs b/src/ImageSharp/Formats/Png/Zlib/DeflaterHuffman.cs index b2eddc37b..50aa1c095 100644 --- a/src/ImageSharp/Formats/Png/Zlib/DeflaterHuffman.cs +++ b/src/ImageSharp/Formats/Png/Zlib/DeflaterHuffman.cs @@ -64,6 +64,7 @@ namespace SixLabors.ImageSharp.Formats.Png.Zlib private int extraBits; private bool isDisposed; + // TODO: These should be pre-generated array/readonlyspans. static DeflaterHuffman() { // See RFC 1951 3.2.6 @@ -114,9 +115,9 @@ namespace SixLabors.ImageSharp.Formats.Png.Zlib { this.Pending = new DeflaterPendingBuffer(memoryAllocator); - this.literalTree = new Tree(LiteralNumber, 257, 15); - this.distTree = new Tree(DistanceNumber, 1, 15); - this.blTree = new Tree(BitLengthNumber, 4, 7); + this.literalTree = new Tree(memoryAllocator, LiteralNumber, 257, 15); + this.distTree = new Tree(memoryAllocator, DistanceNumber, 1, 15); + this.blTree = new Tree(memoryAllocator, BitLengthNumber, 4, 7); this.distanceManagedBuffer = memoryAllocator.Allocate(BufferSize); this.distanceBufferHandle = this.distanceManagedBuffer.Memory.Pin(); @@ -135,6 +136,7 @@ namespace SixLabors.ImageSharp.Formats.Png.Zlib /// /// Reset internal state /// + [MethodImpl(InliningOptions.ShortMethod)] public void Reset() { this.lastLiteral = 0; @@ -214,6 +216,7 @@ namespace SixLabors.ImageSharp.Formats.Png.Zlib /// Index of first byte to write /// Count of bytes to write /// True if this is the last block + [MethodImpl(InliningOptions.ShortMethod)] public void FlushStoredBlock(byte[] stored, int storedOffset, int storedLength, bool lastBlock) { this.Pending.WriteBits((DeflaterConstants.STORED_BLOCK << 1) + (lastBlock ? 1 : 0), 3); @@ -233,7 +236,7 @@ namespace SixLabors.ImageSharp.Formats.Png.Zlib /// True if this is the last block public void FlushBlock(byte[] stored, int storedOffset, int storedLength, bool lastBlock) { - this.literalTree.Freqs[EofSymbol]++; + this.literalTree.Frequencies[EofSymbol]++; // Build trees this.literalTree.BuildTree(); @@ -263,13 +266,13 @@ namespace SixLabors.ImageSharp.Formats.Png.Zlib ref byte staticLLengthRef = ref MemoryMarshal.GetReference(StaticLLength); for (int i = 0; i < LiteralNumber; i++) { - static_len += this.literalTree.Freqs[i] * Unsafe.Add(ref staticLLengthRef, i); + static_len += this.literalTree.Frequencies[i] * Unsafe.Add(ref staticLLengthRef, i); } ref byte staticDLengthRef = ref MemoryMarshal.GetReference(StaticDLength); for (int i = 0; i < DistanceNumber; i++) { - static_len += this.distTree.Freqs[i] * Unsafe.Add(ref staticDLengthRef, i); + static_len += this.distTree.Frequencies[i] * Unsafe.Add(ref staticDLengthRef, i); } if (opt_len >= static_len) @@ -319,7 +322,7 @@ namespace SixLabors.ImageSharp.Formats.Png.Zlib { this.pinnedDistanceBuffer[this.lastLiteral] = 0; this.pinnedLiteralBuffer[this.lastLiteral++] = (byte)literal; - this.literalTree.Freqs[literal]++; + this.literalTree.Frequencies[literal]++; return this.IsFull(); } @@ -336,14 +339,14 @@ namespace SixLabors.ImageSharp.Formats.Png.Zlib this.pinnedLiteralBuffer[this.lastLiteral++] = (byte)(length - 3); int lc = Lcode(length - 3); - this.literalTree.Freqs[lc]++; + this.literalTree.Frequencies[lc]++; if (lc >= 265 && lc < 285) { this.extraBits += (lc - 261) / 4; } int dc = Dcode(distance - 1); - this.distTree.Freqs[dc]++; + this.distTree.Frequencies[dc]++; if (dc >= 4) { this.extraBits += (dc / 2) - 1; @@ -415,33 +418,69 @@ namespace SixLabors.ImageSharp.Formats.Png.Zlib this.distanceManagedBuffer.Dispose(); this.literalBufferHandle.Dispose(); this.literalManagedBuffer.Dispose(); + + this.literalTree.Dispose(); + this.blTree.Dispose(); + this.distTree.Dispose(); } this.Pending = null; + + this.literalTree = null; + this.blTree = null; + this.distTree = null; + this.isDisposed = true; } } - private sealed class Tree + private sealed class Tree : IDisposable { private readonly int minNumCodes; - private short[] codes; private readonly int[] bitLengthCounts; private readonly int maxLength; + private bool isDisposed; - public Tree(int elements, int minCodes, int maxLength) + private readonly int elementCount; + + private IMemoryOwner codesMemoryOwner; + private MemoryHandle codesMemoryHandle; + private short* codes; + + private IMemoryOwner frequenciesMemoryOwner; + private MemoryHandle frequenciesMemoryHandle; + + private IManagedByteBuffer lengthsMemoryOwner; + private MemoryHandle lengthsMemoryHandle; + + public Tree(MemoryAllocator memoryAllocator, int elements, int minCodes, int maxLength) { + this.elementCount = elements; + this.minNumCodes = minCodes; this.maxLength = maxLength; - this.Freqs = new short[elements]; + + this.frequenciesMemoryOwner = memoryAllocator.Allocate(elements); + this.frequenciesMemoryHandle = this.frequenciesMemoryOwner.Memory.Pin(); + this.Frequencies = (short*)this.frequenciesMemoryHandle.Pointer; + + this.lengthsMemoryOwner = memoryAllocator.AllocateManagedByteBuffer(elements); + this.lengthsMemoryHandle = this.lengthsMemoryOwner.Memory.Pin(); + this.Length = (byte*)this.lengthsMemoryHandle.Pointer; + + this.codesMemoryOwner = memoryAllocator.Allocate(elements); + this.codesMemoryHandle = this.codesMemoryOwner.Memory.Pin(); + this.codes = (short*)this.codesMemoryHandle.Pointer; + + // Maxes out at 15. this.bitLengthCounts = new int[maxLength]; } public int NumCodes { get; private set; } - public short[] Freqs { get; } + public short* Frequencies { get; } - public byte[] Length { get; set; } + public byte* Length { get; } /// /// Resets the internal state of the tree @@ -449,13 +488,9 @@ namespace SixLabors.ImageSharp.Formats.Png.Zlib [MethodImpl(InliningOptions.ShortMethod)] public void Reset() { - for (int i = 0; i < this.Freqs.Length; i++) - { - this.Freqs[i] = 0; - } - - this.codes = null; - this.Length = null; + this.frequenciesMemoryOwner.Memory.Span.Clear(); + this.lengthsMemoryOwner.Memory.Span.Clear(); + this.codesMemoryOwner.Memory.Span.Clear(); } [MethodImpl(InliningOptions.ShortMethod)] @@ -472,9 +507,9 @@ namespace SixLabors.ImageSharp.Formats.Png.Zlib public void CheckEmpty() { bool empty = true; - for (int i = 0; i < this.Freqs.Length; i++) + for (int i = 0; i < this.elementCount; i++) { - empty &= this.Freqs[i] == 0; + empty &= this.Frequencies[i] == 0; } if (!empty) @@ -488,10 +523,11 @@ namespace SixLabors.ImageSharp.Formats.Png.Zlib /// /// new codes /// length for new codes - public void SetStaticCodes(short[] staticCodes, byte[] staticLengths) + [MethodImpl(InliningOptions.ShortMethod)] + public void SetStaticCodes(ReadOnlySpan staticCodes, ReadOnlySpan staticLengths) { - this.codes = staticCodes; - this.Length = staticLengths; + staticCodes.CopyTo(this.codesMemoryOwner.Memory.Span); + staticLengths.CopyTo(this.lengthsMemoryOwner.Memory.Span); } /// @@ -499,15 +535,16 @@ namespace SixLabors.ImageSharp.Formats.Png.Zlib /// public void BuildCodes() { - int[] nextCode = new int[this.maxLength]; - int code = 0; - - this.codes = new short[this.Freqs.Length]; + // Maxes out at 15 * 4 + Span nextCode = stackalloc int[this.maxLength]; + ref int nextCodeRef = ref MemoryMarshal.GetReference(nextCode); + ref int bitLengthCountsRef = ref MemoryMarshal.GetReference(this.bitLengthCounts); + int code = 0; for (int bits = 0; bits < this.maxLength; bits++) { - nextCode[bits] = code; - code += this.bitLengthCounts[bits] << (15 - bits); + Unsafe.Add(ref nextCodeRef, bits) = code; + code += Unsafe.Add(ref bitLengthCountsRef, bits) << (15 - bits); } for (int i = 0; i < this.NumCodes; i++) @@ -515,15 +552,15 @@ namespace SixLabors.ImageSharp.Formats.Png.Zlib int bits = this.Length[i]; if (bits > 0) { - this.codes[i] = BitReverse(nextCode[bits - 1]); - nextCode[bits - 1] += 1 << (16 - bits); + this.codes[i] = BitReverse(Unsafe.Add(ref nextCodeRef, bits - 1)); + Unsafe.Add(ref nextCodeRef, bits - 1) += 1 << (16 - bits); } } } public void BuildTree() { - int numSymbols = this.Freqs.Length; + int numSymbols = this.elementCount; // heap is a priority queue, sorted by frequency, least frequent // nodes first. The heap is a binary tree, with the property, that @@ -532,20 +569,23 @@ namespace SixLabors.ImageSharp.Formats.Png.Zlib // // The binary tree is encoded in an array: 0 is root node and // the nodes 2*n+1, 2*n+2 are the child nodes of node n. - int[] heap = new int[numSymbols]; + // Maxes out at 286 * 4 + Span heap = stackalloc int[numSymbols]; + ref int heapRef = ref MemoryMarshal.GetReference(heap); + int heapLen = 0; int maxCode = 0; for (int n = 0; n < numSymbols; n++) { - int freq = this.Freqs[n]; + int freq = this.Frequencies[n]; if (freq != 0) { // Insert n into heap int pos = heapLen++; int ppos; - while (pos > 0 && this.Freqs[heap[ppos = (pos - 1) / 2]] > freq) + while (pos > 0 && this.Frequencies[Unsafe.Add(ref heapRef, ppos = (pos - 1) / 2)] > freq) { - heap[pos] = heap[ppos]; + Unsafe.Add(ref heapRef, pos) = Unsafe.Add(ref heapRef, ppos); pos = ppos; } @@ -561,7 +601,7 @@ namespace SixLabors.ImageSharp.Formats.Png.Zlib // this case, both literals get a 1 bit code. while (heapLen < 2) { - heap[heapLen++] = maxCode < 2 ? ++maxCode : 0; + Unsafe.Add(ref heapRef, heapLen++) = maxCode < 2 ? ++maxCode : 0; } this.NumCodes = Math.Max(maxCode + 1, this.minNumCodes); @@ -572,10 +612,10 @@ namespace SixLabors.ImageSharp.Formats.Png.Zlib int numNodes = numLeafs; for (int i = 0; i < heapLen; i++) { - int node = heap[i]; + int node = Unsafe.Add(ref heapRef, i); childs[2 * i] = node; childs[(2 * i) + 1] = -1; - values[i] = this.Freqs[node] << 8; + values[i] = this.Frequencies[node] << 8; heap[i] = i; } @@ -584,7 +624,7 @@ namespace SixLabors.ImageSharp.Formats.Png.Zlib do { int first = heap[0]; - int last = heap[--heapLen]; + int last = Unsafe.Add(ref heapRef, --heapLen); // Propagate the hole to the leafs of the heap int ppos = 0; @@ -592,12 +632,12 @@ namespace SixLabors.ImageSharp.Formats.Png.Zlib while (path < heapLen) { - if (path + 1 < heapLen && values[heap[path]] > values[heap[path + 1]]) + if (path + 1 < heapLen && values[Unsafe.Add(ref heapRef, path)] > values[Unsafe.Add(ref heapRef, path + 1)]) { path++; } - heap[ppos] = heap[path]; + Unsafe.Add(ref heapRef, ppos) = Unsafe.Add(ref heapRef, path); ppos = path; path = (path * 2) + 1; } @@ -605,14 +645,14 @@ namespace SixLabors.ImageSharp.Formats.Png.Zlib // Now propagate the last element down along path. Normally // it shouldn't go too deep. int lastVal = values[last]; - while ((path = ppos) > 0 && values[heap[ppos = (path - 1) / 2]] > lastVal) + while ((path = ppos) > 0 && values[Unsafe.Add(ref heapRef, ppos = (path - 1) / 2)] > lastVal) { - heap[path] = heap[ppos]; + Unsafe.Add(ref heapRef, path) = Unsafe.Add(ref heapRef, ppos); } - heap[path] = last; + Unsafe.Add(ref heapRef, path) = last; - int second = heap[0]; + int second = Unsafe.Add(ref heapRef, 0); // Create a new node father of first and second last = numNodes++; @@ -627,27 +667,27 @@ namespace SixLabors.ImageSharp.Formats.Png.Zlib while (path < heapLen) { - if (path + 1 < heapLen && values[heap[path]] > values[heap[path + 1]]) + if (path + 1 < heapLen && values[Unsafe.Add(ref heapRef, path)] > values[Unsafe.Add(ref heapRef, path + 1)]) { path++; } - heap[ppos] = heap[path]; + Unsafe.Add(ref heapRef, ppos) = Unsafe.Add(ref heapRef, path); ppos = path; path = (ppos * 2) + 1; } // Now propagate the new element down along path - while ((path = ppos) > 0 && values[heap[ppos = (path - 1) / 2]] > lastVal) + while ((path = ppos) > 0 && values[Unsafe.Add(ref heapRef, ppos = (path - 1) / 2)] > lastVal) { - heap[path] = heap[ppos]; + Unsafe.Add(ref heapRef, path) = Unsafe.Add(ref heapRef, ppos); } - heap[path] = last; + Unsafe.Add(ref heapRef, path) = last; } while (heapLen > 1); - if (heap[0] != (childs.Length / 2) - 1) + if (Unsafe.Add(ref heapRef, 0) != (childs.Length / 2) - 1) { DeflateThrowHelper.ThrowHeapViolated(); } @@ -659,12 +699,13 @@ namespace SixLabors.ImageSharp.Formats.Png.Zlib /// Get encoded length /// /// Encoded length, the sum of frequencies * lengths + [MethodImpl(InliningOptions.ShortMethod)] public int GetEncodedLength() { int len = 0; - for (int i = 0; i < this.Freqs.Length; i++) + for (int i = 0; i < this.elementCount; i++) { - len += this.Freqs[i] * this.Length[i]; + len += this.Frequencies[i] * this.Length[i]; } return len; @@ -697,7 +738,7 @@ namespace SixLabors.ImageSharp.Formats.Png.Zlib min_count = 3; if (curlen != nextlen) { - blTree.Freqs[nextlen]++; + blTree.Frequencies[nextlen]++; count = 0; } } @@ -716,19 +757,19 @@ namespace SixLabors.ImageSharp.Formats.Png.Zlib if (count < min_count) { - blTree.Freqs[curlen] += (short)count; + blTree.Frequencies[curlen] += (short)count; } else if (curlen != 0) { - blTree.Freqs[Repeat3To6]++; + blTree.Frequencies[Repeat3To6]++; } else if (count <= 10) { - blTree.Freqs[Repeat3To10]++; + blTree.Frequencies[Repeat3To10]++; } else { - blTree.Freqs[Repeat11To138]++; + blTree.Frequencies[Repeat11To138]++; } } } @@ -805,7 +846,6 @@ namespace SixLabors.ImageSharp.Formats.Png.Zlib private void BuildLength(int[] children) { - this.Length = new byte[this.Freqs.Length]; int numNodes = children.Length / 2; int numLeafs = (numNodes + 1) / 2; int overflow = 0; @@ -894,6 +934,36 @@ namespace SixLabors.ImageSharp.Formats.Png.Zlib } } } + + public void Dispose() + { + this.Dispose(true); + GC.SuppressFinalize(this); + } + + private void Dispose(bool disposing) + { + if (!this.isDisposed) + { + if (disposing) + { + this.frequenciesMemoryHandle.Dispose(); + this.frequenciesMemoryOwner.Dispose(); + + this.lengthsMemoryHandle.Dispose(); + this.lengthsMemoryOwner.Dispose(); + + this.codesMemoryHandle.Dispose(); + this.codesMemoryOwner.Dispose(); + } + + this.frequenciesMemoryOwner = null; + this.lengthsMemoryOwner = null; + this.codesMemoryOwner = null; + + this.isDisposed = true; + } + } } } } diff --git a/src/ImageSharp/Formats/Png/Zlib/DeflaterPendingBuffer.cs b/src/ImageSharp/Formats/Png/Zlib/DeflaterPendingBuffer.cs index da4d89040..d4af8cb5a 100644 --- a/src/ImageSharp/Formats/Png/Zlib/DeflaterPendingBuffer.cs +++ b/src/ImageSharp/Formats/Png/Zlib/DeflaterPendingBuffer.cs @@ -15,8 +15,8 @@ namespace SixLabors.ImageSharp.Formats.Png.Zlib { private readonly byte[] buffer; private readonly byte* pinnedBuffer; - private readonly IManagedByteBuffer managedBuffer; - private MemoryHandle handle; + private IManagedByteBuffer bufferMemoryOwner; + private MemoryHandle bufferMemoryHandle; private int start; private int end; @@ -29,10 +29,10 @@ namespace SixLabors.ImageSharp.Formats.Png.Zlib /// The memory allocator to use for buffer allocations. public DeflaterPendingBuffer(MemoryAllocator memoryAllocator) { - this.managedBuffer = memoryAllocator.AllocateManagedByteBuffer(DeflaterConstants.PENDING_BUF_SIZE); - this.buffer = this.managedBuffer.Array; - this.handle = this.managedBuffer.Memory.Pin(); - this.pinnedBuffer = (byte*)this.handle.Pointer; + this.bufferMemoryOwner = memoryAllocator.AllocateManagedByteBuffer(DeflaterConstants.PENDING_BUF_SIZE); + this.buffer = this.bufferMemoryOwner.Array; + this.bufferMemoryHandle = this.bufferMemoryOwner.Memory.Pin(); + this.pinnedBuffer = (byte*)this.bufferMemoryHandle.Pointer; } /// @@ -175,10 +175,12 @@ namespace SixLabors.ImageSharp.Formats.Png.Zlib { if (disposing) { - this.handle.Dispose(); - this.managedBuffer.Dispose(); + this.bufferMemoryHandle.Dispose(); + this.bufferMemoryOwner.Dispose(); } + this.bufferMemoryOwner = null; + this.isDisposed = true; } } From d69f66457016f5d33d5353698d91bf26c77b5d56 Mon Sep 17 00:00:00 2001 From: James Jackson-South Date: Thu, 14 Nov 2019 23:53:19 +1100 Subject: [PATCH 13/16] Optimize huffman and make all internal --- src/ImageSharp/Formats/Png/Zlib/Crc32.cs | 6 +- .../Formats/Png/Zlib/DeflateThrowHelper.cs | 4 +- src/ImageSharp/Formats/Png/Zlib/Deflater.cs | 2 +- .../Formats/Png/Zlib/DeflaterEngine.cs | 87 ++-- .../Formats/Png/Zlib/DeflaterHuffman.cs | 343 ++++++------- .../Formats/Png/Zlib/DeflaterOutputStream.cs | 477 +++--------------- .../Formats/Png/Zlib/DeflaterPendingBuffer.cs | 19 +- src/ImageSharp/Formats/Png/Zlib/README.md | 7 +- .../Formats/Png/Zlib/ZlibDeflateStream.cs | 41 +- 9 files changed, 317 insertions(+), 669 deletions(-) diff --git a/src/ImageSharp/Formats/Png/Zlib/Crc32.cs b/src/ImageSharp/Formats/Png/Zlib/Crc32.cs index d1588c384..77355e908 100644 --- a/src/ImageSharp/Formats/Png/Zlib/Crc32.cs +++ b/src/ImageSharp/Formats/Png/Zlib/Crc32.cs @@ -1,8 +1,9 @@ -// Copyright (c) Six Labors and contributors. +// Copyright (c) Six Labors and contributors. // Licensed under the Apache License, Version 2.0. using System; using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; namespace SixLabors.ImageSharp.Formats.Png.Zlib { @@ -141,9 +142,10 @@ namespace SixLabors.ImageSharp.Formats.Png.Zlib { this.crc ^= CrcSeed; + ref uint crcTableRef = ref MemoryMarshal.GetReference(CrcTable.AsSpan()); for (int i = 0; i < data.Length; i++) { - this.crc = CrcTable[(this.crc ^ data[i]) & 0xFF] ^ (this.crc >> 8); + this.crc = Unsafe.Add(ref crcTableRef, (int)((this.crc ^ data[i]) & 0xFF)) ^ (this.crc >> 8); } this.crc ^= CrcSeed; diff --git a/src/ImageSharp/Formats/Png/Zlib/DeflateThrowHelper.cs b/src/ImageSharp/Formats/Png/Zlib/DeflateThrowHelper.cs index d7198c4ee..5f62b13c7 100644 --- a/src/ImageSharp/Formats/Png/Zlib/DeflateThrowHelper.cs +++ b/src/ImageSharp/Formats/Png/Zlib/DeflateThrowHelper.cs @@ -27,9 +27,9 @@ namespace SixLabors.ImageSharp.Formats.Png.Zlib public static void ThrowOutOfRange(string name) => throw new ArgumentOutOfRangeException(name); [MethodImpl(InliningOptions.ColdPath)] - public static void ThrowFrequencyNotEmpty() => throw new InvalidOperationException("Huffman frequency entry non empty."); + public static void ThrowHeapViolated() => throw new InvalidOperationException("Huffman heap invariant violated."); [MethodImpl(InliningOptions.ColdPath)] - public static void ThrowHeapViolated() => throw new InvalidOperationException("Huffman heap invariant violated."); + public static void ThrowNoDeflate() => throw new ImageFormatException("Cannot deflate all input."); } } diff --git a/src/ImageSharp/Formats/Png/Zlib/Deflater.cs b/src/ImageSharp/Formats/Png/Zlib/Deflater.cs index d1560eb4b..fb2538f8c 100644 --- a/src/ImageSharp/Formats/Png/Zlib/Deflater.cs +++ b/src/ImageSharp/Formats/Png/Zlib/Deflater.cs @@ -11,7 +11,7 @@ namespace SixLabors.ImageSharp.Formats.Png.Zlib /// This class compresses input with the deflate algorithm described in RFC 1951. /// It has several compression levels and three different strategies described below. /// - public sealed class Deflater : IDisposable + internal sealed class Deflater : IDisposable { /// /// The best and slowest compression level. This tries to find very diff --git a/src/ImageSharp/Formats/Png/Zlib/DeflaterEngine.cs b/src/ImageSharp/Formats/Png/Zlib/DeflaterEngine.cs index 8f57f51f9..327279e72 100644 --- a/src/ImageSharp/Formats/Png/Zlib/DeflaterEngine.cs +++ b/src/ImageSharp/Formats/Png/Zlib/DeflaterEngine.cs @@ -49,7 +49,7 @@ namespace SixLabors.ImageSharp.Formats.Png.Zlib /// Low level compression engine for deflate algorithm which uses a 32K sliding window /// with secondary compression from Huffman/Shannon-Fano codes. /// - public sealed unsafe class DeflaterEngine : IDisposable + internal sealed unsafe class DeflaterEngine : IDisposable { private const int TooFar = 4096; @@ -109,8 +109,8 @@ namespace SixLabors.ImageSharp.Formats.Png.Zlib /// Note that the array should really be unsigned short, so you need /// to and the values with 0xFFFF. /// - private readonly IMemoryOwner headBuffer; - private MemoryHandle headBufferHandle; + private IMemoryOwner headMemoryOwner; + private MemoryHandle headMemoryHandle; private readonly Memory head; private readonly short* pinnedHeadPointer; @@ -121,17 +121,17 @@ namespace SixLabors.ImageSharp.Formats.Png.Zlib /// Note that the array should really be unsigned short, so you need /// to and the values with 0xFFFF. /// - private readonly IMemoryOwner prevBuffer; - private MemoryHandle prevBufferHandle; + private IMemoryOwner prevMemoryOwner; + private MemoryHandle prevMemoryHandle; private readonly Memory prev; private readonly short* pinnedPrevPointer; /// /// This array contains the part of the uncompressed stream that - /// is of relevance. The current character is indexed by strstart. + /// is of relevance. The current character is indexed by strstart. /// - private readonly IManagedByteBuffer windowBuffer; - private MemoryHandle windowBufferHandle; + private IManagedByteBuffer windowMemoryOwner; + private MemoryHandle windowMemoryHandle; private readonly byte[] window; private readonly byte* pinnedWindowPointer; @@ -153,20 +153,20 @@ namespace SixLabors.ImageSharp.Formats.Png.Zlib // Create pinned pointers to the various buffers to allow indexing // without bounds checks. - this.windowBuffer = memoryAllocator.AllocateManagedByteBuffer(2 * DeflaterConstants.WSIZE); - this.window = this.windowBuffer.Array; - this.windowBufferHandle = this.windowBuffer.Memory.Pin(); - this.pinnedWindowPointer = (byte*)this.windowBufferHandle.Pointer; + this.windowMemoryOwner = memoryAllocator.AllocateManagedByteBuffer(2 * DeflaterConstants.WSIZE); + this.window = this.windowMemoryOwner.Array; + this.windowMemoryHandle = this.windowMemoryOwner.Memory.Pin(); + this.pinnedWindowPointer = (byte*)this.windowMemoryHandle.Pointer; - this.headBuffer = memoryAllocator.Allocate(DeflaterConstants.HASH_SIZE); - this.head = this.headBuffer.Memory; - this.headBufferHandle = this.headBuffer.Memory.Pin(); - this.pinnedHeadPointer = (short*)this.headBufferHandle.Pointer; + this.headMemoryOwner = memoryAllocator.Allocate(DeflaterConstants.HASH_SIZE); + this.head = this.headMemoryOwner.Memory; + this.headMemoryHandle = this.headMemoryOwner.Memory.Pin(); + this.pinnedHeadPointer = (short*)this.headMemoryHandle.Pointer; - this.prevBuffer = memoryAllocator.Allocate(DeflaterConstants.WSIZE); - this.prev = this.prevBuffer.Memory; - this.prevBufferHandle = this.prevBuffer.Memory.Pin(); - this.pinnedPrevPointer = (short*)this.prevBufferHandle.Pointer; + this.prevMemoryOwner = memoryAllocator.Allocate(DeflaterConstants.WSIZE); + this.prev = this.prevMemoryOwner.Memory; + this.prevMemoryHandle = this.prevMemoryOwner.Memory.Pin(); + this.pinnedPrevPointer = (short*)this.prevMemoryHandle.Pointer; // We start at index 1, to avoid an implementation deficiency, that // we cannot build a repeat pattern at index 0. @@ -377,8 +377,27 @@ namespace SixLabors.ImageSharp.Formats.Png.Zlib /// public void Dispose() { - // Do not change this code. Put cleanup code in Dispose(bool disposing) above. - this.Dispose(true); + if (!this.isDisposed) + { + this.huffman.Dispose(); + + this.windowMemoryHandle.Dispose(); + this.windowMemoryOwner.Dispose(); + + this.headMemoryHandle.Dispose(); + this.headMemoryOwner.Dispose(); + + this.prevMemoryHandle.Dispose(); + this.prevMemoryOwner.Dispose(); + + this.windowMemoryOwner = null; + this.headMemoryOwner = null; + this.prevMemoryOwner = null; + this.huffman = null; + + this.isDisposed = true; + } + GC.SuppressFinalize(this); } @@ -830,29 +849,5 @@ namespace SixLabors.ImageSharp.Formats.Png.Zlib return true; } - - private void Dispose(bool disposing) - { - if (!this.isDisposed) - { - if (disposing) - { - this.huffman.Dispose(); - - this.windowBufferHandle.Dispose(); - this.windowBuffer.Dispose(); - - this.headBufferHandle.Dispose(); - this.headBuffer.Dispose(); - - this.prevBufferHandle.Dispose(); - this.prevBuffer.Dispose(); - } - - this.huffman = null; - - this.isDisposed = true; - } - } } } diff --git a/src/ImageSharp/Formats/Png/Zlib/DeflaterHuffman.cs b/src/ImageSharp/Formats/Png/Zlib/DeflaterHuffman.cs index 50aa1c095..7118703d0 100644 --- a/src/ImageSharp/Formats/Png/Zlib/DeflaterHuffman.cs +++ b/src/ImageSharp/Formats/Png/Zlib/DeflaterHuffman.cs @@ -12,7 +12,7 @@ namespace SixLabors.ImageSharp.Formats.Png.Zlib /// /// Performs Deflate Huffman encoding. /// - public sealed unsafe class DeflaterHuffman : IDisposable + internal sealed unsafe class DeflaterHuffman : IDisposable { private const int BufferSize = 1 << (DeflaterConstants.DEFAULT_MEM_LEVEL + 6); @@ -194,7 +194,7 @@ namespace SixLabors.ImageSharp.Formats.Png.Zlib int dc = Dcode(dist); this.distTree.WriteSymbol(pendingBuffer, dc); - bits = (dc / 2) - 1; + bits = (dc >> 1) - 1; if (bits > 0) { this.Pending.WriteBits(dist & ((1 << bits) - 1), bits); @@ -349,7 +349,7 @@ namespace SixLabors.ImageSharp.Formats.Png.Zlib this.distTree.Frequencies[dc]++; if (dc >= 4) { - this.extraBits += (dc / 2) - 1; + this.extraBits += (dc >> 1) - 1; } return this.IsFull(); @@ -443,9 +443,11 @@ namespace SixLabors.ImageSharp.Formats.Png.Zlib private readonly int elementCount; + private readonly MemoryAllocator memoryAllocator; + private IMemoryOwner codesMemoryOwner; private MemoryHandle codesMemoryHandle; - private short* codes; + private readonly short* codes; private IMemoryOwner frequenciesMemoryOwner; private MemoryHandle frequenciesMemoryHandle; @@ -455,8 +457,8 @@ namespace SixLabors.ImageSharp.Formats.Png.Zlib public Tree(MemoryAllocator memoryAllocator, int elements, int minCodes, int maxLength) { + this.memoryAllocator = memoryAllocator; this.elementCount = elements; - this.minNumCodes = minCodes; this.maxLength = maxLength; @@ -497,27 +499,6 @@ namespace SixLabors.ImageSharp.Formats.Png.Zlib public void WriteSymbol(DeflaterPendingBuffer pendingBuffer, int code) => pendingBuffer.WriteBits(this.codes[code] & 0xFFFF, this.Length[code]); - /// - /// Check that all frequencies are zero - /// - /// - /// At least one frequency is non-zero - /// - [MethodImpl(InliningOptions.ShortMethod)] - public void CheckEmpty() - { - bool empty = true; - for (int i = 0; i < this.elementCount; i++) - { - empty &= this.Frequencies[i] == 0; - } - - if (!empty) - { - DeflateThrowHelper.ThrowFrequencyNotEmpty(); - } - } - /// /// Set static codes and length /// @@ -569,130 +550,141 @@ namespace SixLabors.ImageSharp.Formats.Png.Zlib // // The binary tree is encoded in an array: 0 is root node and // the nodes 2*n+1, 2*n+2 are the child nodes of node n. - // Maxes out at 286 * 4 - Span heap = stackalloc int[numSymbols]; - ref int heapRef = ref MemoryMarshal.GetReference(heap); - - int heapLen = 0; - int maxCode = 0; - for (int n = 0; n < numSymbols; n++) - { - int freq = this.Frequencies[n]; - if (freq != 0) - { - // Insert n into heap - int pos = heapLen++; - int ppos; - while (pos > 0 && this.Frequencies[Unsafe.Add(ref heapRef, ppos = (pos - 1) / 2)] > freq) - { - Unsafe.Add(ref heapRef, pos) = Unsafe.Add(ref heapRef, ppos); - pos = ppos; - } - - heap[pos] = n; - - maxCode = n; - } - } - - // We could encode a single literal with 0 bits but then we - // don't see the literals. Therefore we force at least two - // literals to avoid this case. We don't care about order in - // this case, both literals get a 1 bit code. - while (heapLen < 2) - { - Unsafe.Add(ref heapRef, heapLen++) = maxCode < 2 ? ++maxCode : 0; - } - - this.NumCodes = Math.Max(maxCode + 1, this.minNumCodes); - - int numLeafs = heapLen; - int[] childs = new int[(4 * heapLen) - 2]; - int[] values = new int[(2 * heapLen) - 1]; - int numNodes = numLeafs; - for (int i = 0; i < heapLen; i++) - { - int node = Unsafe.Add(ref heapRef, i); - childs[2 * i] = node; - childs[(2 * i) + 1] = -1; - values[i] = this.Frequencies[node] << 8; - heap[i] = i; - } - - // Construct the Huffman tree by repeatedly combining the least two - // frequent nodes. - do + // Maxes out at 286 * 4 so too large for the stack. + using (IMemoryOwner heapMemoryOwner = this.memoryAllocator.Allocate(numSymbols)) { - int first = heap[0]; - int last = Unsafe.Add(ref heapRef, --heapLen); - - // Propagate the hole to the leafs of the heap - int ppos = 0; - int path = 1; + ref int heapRef = ref MemoryMarshal.GetReference(heapMemoryOwner.Memory.Span); - while (path < heapLen) + int heapLen = 0; + int maxCode = 0; + for (int n = 0; n < numSymbols; n++) { - if (path + 1 < heapLen && values[Unsafe.Add(ref heapRef, path)] > values[Unsafe.Add(ref heapRef, path + 1)]) + int freq = this.Frequencies[n]; + if (freq != 0) { - path++; + // Insert n into heap + int pos = heapLen++; + int ppos; + while (pos > 0 && this.Frequencies[Unsafe.Add(ref heapRef, ppos = (pos - 1) >> 1)] > freq) + { + Unsafe.Add(ref heapRef, pos) = Unsafe.Add(ref heapRef, ppos); + pos = ppos; + } + + Unsafe.Add(ref heapRef, pos) = n; + + maxCode = n; } - - Unsafe.Add(ref heapRef, ppos) = Unsafe.Add(ref heapRef, path); - ppos = path; - path = (path * 2) + 1; } - // Now propagate the last element down along path. Normally - // it shouldn't go too deep. - int lastVal = values[last]; - while ((path = ppos) > 0 && values[Unsafe.Add(ref heapRef, ppos = (path - 1) / 2)] > lastVal) + // We could encode a single literal with 0 bits but then we + // don't see the literals. Therefore we force at least two + // literals to avoid this case. We don't care about order in + // this case, both literals get a 1 bit code. + while (heapLen < 2) { - Unsafe.Add(ref heapRef, path) = Unsafe.Add(ref heapRef, ppos); + Unsafe.Add(ref heapRef, heapLen++) = maxCode < 2 ? ++maxCode : 0; } - Unsafe.Add(ref heapRef, path) = last; - - int second = Unsafe.Add(ref heapRef, 0); + this.NumCodes = Math.Max(maxCode + 1, this.minNumCodes); - // Create a new node father of first and second - last = numNodes++; - childs[2 * last] = first; - childs[(2 * last) + 1] = second; - int mindepth = Math.Min(values[first] & 0xFF, values[second] & 0xFF); - values[last] = lastVal = values[first] + values[second] - mindepth + 1; + int numLeafs = heapLen; + int childrenLength = (4 * heapLen) - 2; + using (IMemoryOwner childrenMemoryOwner = this.memoryAllocator.Allocate(childrenLength)) + using (IMemoryOwner valuesMemoryOwner = this.memoryAllocator.Allocate((2 * heapLen) - 1)) + { + ref int childrenRef = ref MemoryMarshal.GetReference(childrenMemoryOwner.Memory.Span); + ref int valuesRef = ref MemoryMarshal.GetReference(valuesMemoryOwner.Memory.Span); + int numNodes = numLeafs; - // Again, propagate the hole to the leafs - ppos = 0; - path = 1; + for (int i = 0; i < heapLen; i++) + { + int node = Unsafe.Add(ref heapRef, i); + int i2 = 2 * i; + Unsafe.Add(ref childrenRef, i2) = node; + Unsafe.Add(ref childrenRef, i2 + 1) = -1; + Unsafe.Add(ref valuesRef, i) = this.Frequencies[node] << 8; + Unsafe.Add(ref heapRef, i) = i; + } - while (path < heapLen) - { - if (path + 1 < heapLen && values[Unsafe.Add(ref heapRef, path)] > values[Unsafe.Add(ref heapRef, path + 1)]) + // Construct the Huffman tree by repeatedly combining the least two + // frequent nodes. + do { - path++; + int first = Unsafe.Add(ref heapRef, 0); + int last = Unsafe.Add(ref heapRef, --heapLen); + + // Propagate the hole to the leafs of the heap + int ppos = 0; + int path = 1; + + while (path < heapLen) + { + if (path + 1 < heapLen && Unsafe.Add(ref valuesRef, Unsafe.Add(ref heapRef, path)) > Unsafe.Add(ref valuesRef, Unsafe.Add(ref heapRef, path + 1))) + { + path++; + } + + Unsafe.Add(ref heapRef, ppos) = Unsafe.Add(ref heapRef, path); + ppos = path; + path = (path * 2) + 1; + } + + // Now propagate the last element down along path. Normally + // it shouldn't go too deep. + int lastVal = Unsafe.Add(ref valuesRef, last); + while ((path = ppos) > 0 + && Unsafe.Add(ref valuesRef, Unsafe.Add(ref heapRef, ppos = (path - 1) >> 1)) > lastVal) + { + Unsafe.Add(ref heapRef, path) = Unsafe.Add(ref heapRef, ppos); + } + + Unsafe.Add(ref heapRef, path) = last; + + int second = Unsafe.Add(ref heapRef, 0); + + // Create a new node father of first and second + last = numNodes++; + Unsafe.Add(ref childrenRef, 2 * last) = first; + Unsafe.Add(ref childrenRef, (2 * last) + 1) = second; + int mindepth = Math.Min(Unsafe.Add(ref valuesRef, first) & 0xFF, Unsafe.Add(ref valuesRef, second) & 0xFF); + Unsafe.Add(ref valuesRef, last) = lastVal = Unsafe.Add(ref valuesRef, first) + Unsafe.Add(ref valuesRef, second) - mindepth + 1; + + // Again, propagate the hole to the leafs + ppos = 0; + path = 1; + + while (path < heapLen) + { + if (path + 1 < heapLen + && Unsafe.Add(ref valuesRef, Unsafe.Add(ref heapRef, path)) > Unsafe.Add(ref valuesRef, Unsafe.Add(ref heapRef, path + 1))) + { + path++; + } + + Unsafe.Add(ref heapRef, ppos) = Unsafe.Add(ref heapRef, path); + ppos = path; + path = (ppos * 2) + 1; + } + + // Now propagate the new element down along path + while ((path = ppos) > 0 && Unsafe.Add(ref valuesRef, Unsafe.Add(ref heapRef, ppos = (path - 1) >> 1)) > lastVal) + { + Unsafe.Add(ref heapRef, path) = Unsafe.Add(ref heapRef, ppos); + } + + Unsafe.Add(ref heapRef, path) = last; } + while (heapLen > 1); - Unsafe.Add(ref heapRef, ppos) = Unsafe.Add(ref heapRef, path); - ppos = path; - path = (ppos * 2) + 1; - } + if (Unsafe.Add(ref heapRef, 0) != (childrenLength >> 1) - 1) + { + DeflateThrowHelper.ThrowHeapViolated(); + } - // Now propagate the new element down along path - while ((path = ppos) > 0 && values[Unsafe.Add(ref heapRef, ppos = (path - 1) / 2)] > lastVal) - { - Unsafe.Add(ref heapRef, path) = Unsafe.Add(ref heapRef, ppos); + this.BuildLength(childrenMemoryOwner.Memory.Span); } - - Unsafe.Add(ref heapRef, path) = last; } - while (heapLen > 1); - - if (Unsafe.Add(ref heapRef, 0) != (childs.Length / 2) - 1) - { - DeflateThrowHelper.ThrowHeapViolated(); - } - - this.BuildLength(childs); } /// @@ -717,10 +709,10 @@ namespace SixLabors.ImageSharp.Formats.Png.Zlib /// public void CalcBLFreq(Tree blTree) { - int max_count; /* max repeat count */ - int min_count; /* min repeat count */ - int count; /* repeat count of the current code */ - int curlen = -1; /* length of current code */ + int maxCount; // max repeat count + int minCount; // min repeat count + int count; // repeat count of the current code + int curLen = -1; // length of current code int i = 0; while (i < this.NumCodes) @@ -729,37 +721,37 @@ namespace SixLabors.ImageSharp.Formats.Png.Zlib int nextlen = this.Length[i]; if (nextlen == 0) { - max_count = 138; - min_count = 3; + maxCount = 138; + minCount = 3; } else { - max_count = 6; - min_count = 3; - if (curlen != nextlen) + maxCount = 6; + minCount = 3; + if (curLen != nextlen) { blTree.Frequencies[nextlen]++; count = 0; } } - curlen = nextlen; + curLen = nextlen; i++; - while (i < this.NumCodes && curlen == this.Length[i]) + while (i < this.NumCodes && curLen == this.Length[i]) { i++; - if (++count >= max_count) + if (++count >= maxCount) { break; } } - if (count < min_count) + if (count < minCount) { - blTree.Frequencies[curlen] += (short)count; + blTree.Frequencies[curLen] += (short)count; } - else if (curlen != 0) + else if (curLen != 0) { blTree.Frequencies[Repeat3To6]++; } @@ -781,10 +773,10 @@ namespace SixLabors.ImageSharp.Formats.Png.Zlib /// The tree to write. public void WriteTree(DeflaterPendingBuffer pendingBuffer, Tree bitLengthTree) { - int max_count; // max repeat count - int min_count; // min repeat count - int count; // repeat count of the current code - int curlen = -1; // length of current code + int maxCount; // max repeat count + int minCount; // min repeat count + int count; // repeat count of the current code + int curLen = -1; // length of current code int i = 0; while (i < this.NumCodes) @@ -793,40 +785,40 @@ namespace SixLabors.ImageSharp.Formats.Png.Zlib int nextlen = this.Length[i]; if (nextlen == 0) { - max_count = 138; - min_count = 3; + maxCount = 138; + minCount = 3; } else { - max_count = 6; - min_count = 3; - if (curlen != nextlen) + maxCount = 6; + minCount = 3; + if (curLen != nextlen) { bitLengthTree.WriteSymbol(pendingBuffer, nextlen); count = 0; } } - curlen = nextlen; + curLen = nextlen; i++; - while (i < this.NumCodes && curlen == this.Length[i]) + while (i < this.NumCodes && curLen == this.Length[i]) { i++; - if (++count >= max_count) + if (++count >= maxCount) { break; } } - if (count < min_count) + if (count < minCount) { while (count-- > 0) { - bitLengthTree.WriteSymbol(pendingBuffer, curlen); + bitLengthTree.WriteSymbol(pendingBuffer, curLen); } } - else if (curlen != 0) + else if (curLen != 0) { bitLengthTree.WriteSymbol(pendingBuffer, Repeat3To6); pendingBuffer.WriteBits(count - 3, 2); @@ -844,10 +836,10 @@ namespace SixLabors.ImageSharp.Formats.Png.Zlib } } - private void BuildLength(int[] children) + private void BuildLength(ReadOnlySpan children) { - int numNodes = children.Length / 2; - int numLeafs = (numNodes + 1) / 2; + int numNodes = children.Length >> 1; + int numLeafs = (numNodes + 1) >> 1; int overflow = 0; for (int i = 0; i < this.maxLength; i++) @@ -936,26 +928,17 @@ namespace SixLabors.ImageSharp.Formats.Png.Zlib } public void Dispose() - { - this.Dispose(true); - GC.SuppressFinalize(this); - } - - private void Dispose(bool disposing) { if (!this.isDisposed) { - if (disposing) - { - this.frequenciesMemoryHandle.Dispose(); - this.frequenciesMemoryOwner.Dispose(); + this.frequenciesMemoryHandle.Dispose(); + this.frequenciesMemoryOwner.Dispose(); - this.lengthsMemoryHandle.Dispose(); - this.lengthsMemoryOwner.Dispose(); + this.lengthsMemoryHandle.Dispose(); + this.lengthsMemoryOwner.Dispose(); - this.codesMemoryHandle.Dispose(); - this.codesMemoryOwner.Dispose(); - } + this.codesMemoryHandle.Dispose(); + this.codesMemoryOwner.Dispose(); this.frequenciesMemoryOwner = null; this.lengthsMemoryOwner = null; @@ -963,6 +946,8 @@ namespace SixLabors.ImageSharp.Formats.Png.Zlib this.isDisposed = true; } + + GC.SuppressFinalize(this); } } } diff --git a/src/ImageSharp/Formats/Png/Zlib/DeflaterOutputStream.cs b/src/ImageSharp/Formats/Png/Zlib/DeflaterOutputStream.cs index 837e8b795..eb214aae2 100644 --- a/src/ImageSharp/Formats/Png/Zlib/DeflaterOutputStream.cs +++ b/src/ImageSharp/Formats/Png/Zlib/DeflaterOutputStream.cs @@ -1,466 +1,155 @@ // Copyright (c) Six Labors and contributors. // Licensed under the Apache License, Version 2.0. -// using System; -using System.Collections.Generic; using System.IO; -using System.Text; +using SixLabors.Memory; namespace SixLabors.ImageSharp.Formats.Png.Zlib { /// /// A special stream deflating or compressing the bytes that are - /// written to it. It uses a Deflater to perform actual deflating.
- /// Authors of the original java version : Tom Tromey, Jochen Hoenicke + /// written to it. It uses a Deflater to perform actual deflating. /// - public class DeflaterOutputStream : Stream + internal sealed class DeflaterOutputStream : Stream { - #region Constructors - /// - /// Creates a new DeflaterOutputStream with the given Deflater and - /// default buffer size. - /// - /// - /// the output stream where deflated output should be written. - /// - /// - /// the underlying deflater. - /// - public DeflaterOutputStream(Stream baseOutputStream, Deflater deflater) - : this(baseOutputStream, deflater, 512) - { - } + private const int BufferLength = 512; + private IManagedByteBuffer memoryOwner; + private readonly byte[] buffer; + private Deflater deflater; + private readonly Stream rawStream; + private bool isDisposed; /// - /// Creates a new DeflaterOutputStream with the given Deflater and - /// buffer size. + /// Initializes a new instance of the class. /// - /// - /// The output stream where deflated output is written. - /// - /// - /// The underlying deflater to use - /// - /// - /// The buffer size in bytes to use when deflating (minimum value 512) - /// - /// - /// bufsize is less than or equal to zero. - /// - /// - /// baseOutputStream does not support writing - /// - /// - /// deflater instance is null - /// - public DeflaterOutputStream(Stream baseOutputStream, Deflater deflater, int bufferSize) + /// The memory allocator to use for buffer allocations. + /// The output stream where deflated output is written. + /// The compression level. + public DeflaterOutputStream(MemoryAllocator memoryAllocator, Stream rawStream, int compressionLevel) { - if (baseOutputStream == null) - { - throw new ArgumentNullException(nameof(baseOutputStream)); - } - - if (baseOutputStream.CanWrite == false) - { - throw new ArgumentException("Must support writing", nameof(baseOutputStream)); - } - - if (bufferSize < 512) - { - throw new ArgumentOutOfRangeException(nameof(bufferSize)); - } - - baseOutputStream_ = baseOutputStream; - buffer_ = new byte[bufferSize]; - deflater_ = deflater ?? throw new ArgumentNullException(nameof(deflater)); + this.rawStream = rawStream; + this.memoryOwner = memoryAllocator.AllocateManagedByteBuffer(BufferLength); + this.buffer = this.memoryOwner.Array; + this.deflater = new Deflater(memoryAllocator, compressionLevel); } - #endregion Constructors + /// + public override bool CanRead => false; - #region Public API + /// + public override bool CanSeek => false; - /// - /// Finishes the stream by calling finish() on the deflater. - /// - /// - /// Not all input is deflated - /// - public virtual void Finish() - { - deflater_.Finish(); - while (!deflater_.IsFinished) - { - int len = deflater_.Deflate(buffer_, 0, buffer_.Length); - if (len <= 0) - { - break; - } + /// + public override bool CanWrite => this.rawStream.CanWrite; - baseOutputStream_.Write(buffer_, 0, len); - } + /// + public override long Length => this.rawStream.Length; - if (!deflater_.IsFinished) + /// + public override long Position + { + get { - throw new ImageFormatException("Can't deflate all input?"); + return this.rawStream.Position; } - baseOutputStream_.Flush(); - } - - /// - /// Gets or sets a flag indicating ownership of underlying stream. - /// When the flag is true will close the underlying stream also. - /// - /// The default value is true. - public bool IsStreamOwner { get; set; } = true; - - /// - /// Allows client to determine if an entry can be patched after its added - /// - public bool CanPatchEntries - { - get + set { - return baseOutputStream_.CanSeek; + throw new NotSupportedException(); } } - #endregion Public API - - //#region Encryption - - //private string password; - - //private ICryptoTransform cryptoTransform_; + /// + public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); - ///// - ///// Returns the 10 byte AUTH CODE to be appended immediately following the AES data stream. - ///// - //protected byte[] AESAuthCode; + /// + public override void SetLength(long value) => throw new NotSupportedException(); - ///// - ///// Get/set the password used for encryption. - ///// - ///// When set to null or if the password is empty no encryption is performed - //public string Password - //{ - // get - // { - // return password; - // } - // set - // { - // if ((value != null) && (value.Length == 0)) - // { - // password = null; - // } - // else - // { - // password = value; - // } - // } - //} + /// + public override int ReadByte() => throw new NotSupportedException(); - ///// - ///// Encrypt a block of data - ///// - ///// - ///// Data to encrypt. NOTE the original contents of the buffer are lost - ///// - ///// - ///// Offset of first byte in buffer to encrypt - ///// - ///// - ///// Number of bytes in buffer to encrypt - ///// - //protected void EncryptBlock(byte[] buffer, int offset, int length) - //{ - // cryptoTransform_.TransformBlock(buffer, 0, length, buffer, 0); - //} + /// + public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - ///// - ///// Initializes encryption keys based on given . - ///// - ///// The password. - //protected void InitializePassword(string password) - //{ - // var pkManaged = new PkzipClassicManaged(); - // byte[] key = PkzipClassic.GenerateKeys(ZipStrings.ConvertToArray(password)); - // cryptoTransform_ = pkManaged.CreateEncryptor(key, null); - //} - - ///// - ///// Initializes encryption keys based on given password. - ///// - //protected void InitializeAESPassword(ZipEntry entry, string rawPassword, - // out byte[] salt, out byte[] pwdVerifier) - //{ - // salt = new byte[entry.AESSaltLen]; - // // Salt needs to be cryptographically random, and unique per file - // if (_aesRnd == null) - // _aesRnd = RandomNumberGenerator.Create(); - // _aesRnd.GetBytes(salt); - // int blockSize = entry.AESKeySize / 8; // bits to bytes - - // cryptoTransform_ = new ZipAESTransform(rawPassword, salt, blockSize, true); - // pwdVerifier = ((ZipAESTransform)cryptoTransform_).PwdVerifier; - //} - - //#endregion Encryption - - #region Deflation Support + /// + public override void Flush() + { + this.deflater.Flush(); + this.Deflate(true); + this.rawStream.Flush(); + } - /// - /// Deflates everything in the input buffers. This will call - /// def.deflate() until all bytes from the input buffers - /// are processed. - /// - protected void Deflate() + /// + public override void Write(byte[] buffer, int offset, int count) { - Deflate(false); + this.deflater.SetInput(buffer, offset, count); + this.Deflate(); } + private void Deflate() => this.Deflate(false); + private void Deflate(bool flushing) { - while (flushing || !deflater_.IsNeedingInput) + while (flushing || !this.deflater.IsNeedingInput) { - int deflateCount = deflater_.Deflate(buffer_, 0, buffer_.Length); + int deflateCount = this.deflater.Deflate(this.buffer, 0, BufferLength); if (deflateCount <= 0) { break; } - //if (cryptoTransform_ != null) - //{ - // EncryptBlock(buffer_, 0, deflateCount); - //} - baseOutputStream_.Write(buffer_, 0, deflateCount); + this.rawStream.Write(this.buffer, 0, deflateCount); } - if (!deflater_.IsNeedingInput) + if (!this.deflater.IsNeedingInput) { - throw new ImageFormatException("DeflaterOutputStream can't deflate all input?"); + DeflateThrowHelper.ThrowNoDeflate(); } } - #endregion Deflation Support - - #region Stream Overrides - - /// - /// Gets value indicating stream can be read from - /// - public override bool CanRead + private void Finish() { - get + this.deflater.Finish(); + while (!this.deflater.IsFinished) { - return false; - } - } + int len = this.deflater.Deflate(this.buffer, 0, BufferLength); + if (len <= 0) + { + break; + } - /// - /// Gets a value indicating if seeking is supported for this stream - /// This property always returns false - /// - public override bool CanSeek - { - get - { - return false; + this.rawStream.Write(this.buffer, 0, len); } - } - /// - /// Get value indicating if this stream supports writing - /// - public override bool CanWrite - { - get + if (!this.deflater.IsFinished) { - return baseOutputStream_.CanWrite; + DeflateThrowHelper.ThrowNoDeflate(); } - } - /// - /// Get current length of stream - /// - public override long Length - { - get - { - return baseOutputStream_.Length; - } + this.rawStream.Flush(); } - /// - /// Gets the current position within the stream. - /// - /// Any attempt to set position - public override long Position + /// + protected override void Dispose(bool disposing) { - get - { - return baseOutputStream_.Position; - } - set + if (this.isDisposed) { - throw new NotSupportedException("Position property not supported"); + return; } - } - - /// - /// Sets the current position of this stream to the given value. Not supported by this class! - /// - /// The offset relative to the to seek. - /// The to seek from. - /// The new position in the stream. - /// Any access - public override long Seek(long offset, SeekOrigin origin) - { - throw new NotSupportedException("DeflaterOutputStream Seek not supported"); - } - - /// - /// Sets the length of this stream to the given value. Not supported by this class! - /// - /// The new stream length. - /// Any access - public override void SetLength(long value) - { - throw new NotSupportedException("DeflaterOutputStream SetLength not supported"); - } - - /// - /// Read a byte from stream advancing position by one - /// - /// The byte read cast to an int. THe value is -1 if at the end of the stream. - /// Any access - public override int ReadByte() - { - throw new NotSupportedException("DeflaterOutputStream ReadByte not supported"); - } - - /// - /// Read a block of bytes from stream - /// - /// The buffer to store read data in. - /// The offset to start storing at. - /// The maximum number of bytes to read. - /// The actual number of bytes read. Zero if end of stream is detected. - /// Any access - public override int Read(byte[] buffer, int offset, int count) - { - throw new NotSupportedException("DeflaterOutputStream Read not supported"); - } - - /// - /// Flushes the stream by calling Flush on the deflater and then - /// on the underlying stream. This ensures that all bytes are flushed. - /// - public override void Flush() - { - deflater_.Flush(); - Deflate(true); - baseOutputStream_.Flush(); - } - /// - /// Calls and closes the underlying - /// stream when is true. - /// - protected override void Dispose(bool disposing) - { - if (!isClosed_) + if (disposing) { - isClosed_ = true; - - try - { - Finish(); - } - finally - { - if (IsStreamOwner) - { - baseOutputStream_.Dispose(); - } - } + this.Finish(); + this.deflater.Dispose(); + this.memoryOwner.Dispose(); } - } - - ///// - ///// Get the Auth code for AES encrypted entries - ///// - //protected void GetAuthCodeIfAES() - //{ - // if (cryptoTransform_ is ZipAESTransform) - // { - // AESAuthCode = ((ZipAESTransform)cryptoTransform_).GetAuthCode(); - // } - //} - /// - /// Writes a single byte to the compressed output stream. - /// - /// - /// The byte value. - /// - public override void WriteByte(byte value) - { - byte[] b = new byte[1]; - b[0] = value; - Write(b, 0, 1); - } - - /// - /// Writes bytes from an array to the compressed stream. - /// - /// - /// The byte array - /// - /// - /// The offset into the byte array where to start. - /// - /// - /// The number of bytes to write. - /// - public override void Write(byte[] buffer, int offset, int count) - { - deflater_.SetInput(buffer, offset, count); - Deflate(); + this.deflater = null; + this.memoryOwner = null; + this.isDisposed = true; + base.Dispose(disposing); } - - #endregion Stream Overrides - - #region Instance Fields - - /// - /// This buffer is used temporarily to retrieve the bytes from the - /// deflater and write them to the underlying output stream. - /// - private byte[] buffer_; - - /// - /// The deflater which is used to deflate the stream. - /// - protected Deflater deflater_; - - /// - /// Base stream the deflater depends on. - /// - protected Stream baseOutputStream_; - - private bool isClosed_; - - #endregion Instance Fields - - #region Static Fields - - // Static to help ensure that multiple files within a zip will get different random salt - //private static RandomNumberGenerator _aesRnd = RandomNumberGenerator.Create(); - - #endregion Static Fields } } diff --git a/src/ImageSharp/Formats/Png/Zlib/DeflaterPendingBuffer.cs b/src/ImageSharp/Formats/Png/Zlib/DeflaterPendingBuffer.cs index d4af8cb5a..a5f00f03c 100644 --- a/src/ImageSharp/Formats/Png/Zlib/DeflaterPendingBuffer.cs +++ b/src/ImageSharp/Formats/Png/Zlib/DeflaterPendingBuffer.cs @@ -11,7 +11,7 @@ namespace SixLabors.ImageSharp.Formats.Png.Zlib /// /// Stores pending data for writing data to the Deflater. /// - public sealed unsafe class DeflaterPendingBuffer : IDisposable + internal sealed unsafe class DeflaterPendingBuffer : IDisposable { private readonly byte[] buffer; private readonly byte* pinnedBuffer; @@ -164,25 +164,16 @@ namespace SixLabors.ImageSharp.Formats.Png.Zlib /// public void Dispose() - { - this.Dispose(true); - GC.SuppressFinalize(this); - } - - private void Dispose(bool disposing) { if (!this.isDisposed) { - if (disposing) - { - this.bufferMemoryHandle.Dispose(); - this.bufferMemoryOwner.Dispose(); - } - + this.bufferMemoryHandle.Dispose(); + this.bufferMemoryOwner.Dispose(); this.bufferMemoryOwner = null; - this.isDisposed = true; } + + GC.SuppressFinalize(this); } } } diff --git a/src/ImageSharp/Formats/Png/Zlib/README.md b/src/ImageSharp/Formats/Png/Zlib/README.md index c297a91d5..59f75d05f 100644 --- a/src/ImageSharp/Formats/Png/Zlib/README.md +++ b/src/ImageSharp/Formats/Png/Zlib/README.md @@ -1,2 +1,5 @@ -Adler32.cs and Crc32.cs have been copied from -https://github.com/ygrenier/SharpZipLib.Portable +Deflatestream implementation adapted from + +https://github.com/icsharpcode/SharpZipLib + +LIcensed under MIT diff --git a/src/ImageSharp/Formats/Png/Zlib/ZlibDeflateStream.cs b/src/ImageSharp/Formats/Png/Zlib/ZlibDeflateStream.cs index 5724e027d..36bacc5ec 100644 --- a/src/ImageSharp/Formats/Png/Zlib/ZlibDeflateStream.cs +++ b/src/ImageSharp/Formats/Png/Zlib/ZlibDeflateStream.cs @@ -41,8 +41,6 @@ namespace SixLabors.ImageSharp.Formats.Png.Zlib // private DeflateStream deflateStream; private DeflaterOutputStream deflateStream; - private Deflater deflater; - /// /// Initializes a new instance of the class. /// @@ -92,21 +90,7 @@ namespace SixLabors.ImageSharp.Formats.Png.Zlib this.rawStream.WriteByte(Cmf); this.rawStream.WriteByte((byte)flg); - // Initialize the deflate Stream. - // CompressionLevel level = CompressionLevel.Optimal; - // - // if (compressionLevel >= 1 && compressionLevel <= 5) - // { - // level = CompressionLevel.Fastest; - // } - // else if (compressionLevel == 0) - // { - // level = CompressionLevel.NoCompression; - // } - this.deflater = new Deflater(memoryAllocator, compressionLevel); - this.deflateStream = new DeflaterOutputStream(this.rawStream, this.deflater) { IsStreamOwner = false }; - - // this.deflateStream = new DeflateStream(this.rawStream, level, true); + this.deflateStream = new DeflaterOutputStream(memoryAllocator, this.rawStream, compressionLevel); } /// @@ -116,16 +100,23 @@ namespace SixLabors.ImageSharp.Formats.Png.Zlib public override bool CanSeek => false; /// - public override bool CanWrite => true; + public override bool CanWrite => this.rawStream.CanWrite; /// - public override long Length => throw new NotSupportedException(); + public override long Length => this.rawStream.Length; /// public override long Position { - get => throw new NotSupportedException(); - set => throw new NotSupportedException(); + get + { + return this.rawStream.Position; + } + + set + { + throw new NotSupportedException(); + } } /// @@ -174,10 +165,6 @@ namespace SixLabors.ImageSharp.Formats.Png.Zlib { this.deflateStream.Dispose(); this.deflateStream = null; - - // TODO: Remove temporal coupling here. - this.deflater.Dispose(); - this.deflater = null; } else { @@ -195,10 +182,6 @@ namespace SixLabors.ImageSharp.Formats.Png.Zlib } base.Dispose(disposing); - - // Call the appropriate methods to clean up - // unmanaged resources here. - // Note disposing is done. this.isDisposed = true; } } From e63030c5b29ccfce638c45e41da44360c08fed72 Mon Sep 17 00:00:00 2001 From: James Jackson-South Date: Tue, 19 Nov 2019 13:53:35 +1100 Subject: [PATCH 14/16] Minor perf tweaks. --- src/ImageSharp/Formats/Png/Zlib/Adler32.cs | 5 +-- .../Formats/Png/Zlib/DeflaterEngine.cs | 36 +++++++++++-------- 2 files changed, 24 insertions(+), 17 deletions(-) diff --git a/src/ImageSharp/Formats/Png/Zlib/Adler32.cs b/src/ImageSharp/Formats/Png/Zlib/Adler32.cs index f6f6edd12..c4dc82a4d 100644 --- a/src/ImageSharp/Formats/Png/Zlib/Adler32.cs +++ b/src/ImageSharp/Formats/Png/Zlib/Adler32.cs @@ -3,6 +3,7 @@ using System; using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; namespace SixLabors.ImageSharp.Formats.Png.Zlib { @@ -112,7 +113,7 @@ namespace SixLabors.ImageSharp.Formats.Png.Zlib [MethodImpl(MethodImplOptions.AggressiveInlining)] public void Update(ReadOnlySpan data) { - // (By Per Bothner) + ref byte dataRef = ref MemoryMarshal.GetReference(data); uint s1 = this.checksum & 0xFFFF; uint s2 = this.checksum >> 16; @@ -133,7 +134,7 @@ namespace SixLabors.ImageSharp.Formats.Png.Zlib count -= n; while (--n >= 0) { - s1 += (uint)(data[offset++] & 0xff); + s1 += Unsafe.Add(ref dataRef, offset++); s2 += s1; } diff --git a/src/ImageSharp/Formats/Png/Zlib/DeflaterEngine.cs b/src/ImageSharp/Formats/Png/Zlib/DeflaterEngine.cs index 327279e72..0163eec0b 100644 --- a/src/ImageSharp/Formats/Png/Zlib/DeflaterEngine.cs +++ b/src/ImageSharp/Formats/Png/Zlib/DeflaterEngine.cs @@ -476,19 +476,22 @@ namespace SixLabors.ImageSharp.Formats.Png.Zlib int chainLength = this.maxChain; int niceLength = Math.Min(this.niceLength, this.lookahead); + int matchStrt = this.matchStart; this.matchLen = Math.Max(this.matchLen, DeflaterConstants.MIN_MATCH - 1); + int matchLength = this.matchLen; - if (scan + this.matchLen > scanMax) + if (scan + matchLength > scanMax) { return false; } byte* pinnedWindow = this.pinnedWindowPointer; - byte scan_end1 = pinnedWindow[scan + this.matchLen - 1]; - byte scan_end = pinnedWindow[scan + this.matchLen]; + int scanStart = this.strstart; + byte scanEnd1 = pinnedWindow[scan + matchLength - 1]; + byte scanEnd = pinnedWindow[scan + matchLength]; // Do not waste too much time if we already have a good match: - if (this.matchLen >= this.goodLength) + if (matchLength >= this.goodLength) { chainLength >>= 2; } @@ -497,10 +500,10 @@ namespace SixLabors.ImageSharp.Formats.Png.Zlib do { match = curMatch; - scan = this.strstart; + scan = scanStart; - if (pinnedWindow[match + this.matchLen] != scan_end - || pinnedWindow[match + this.matchLen - 1] != scan_end1 + if (pinnedWindow[match + matchLength] != scanEnd + || pinnedWindow[match + matchLength - 1] != scanEnd1 || pinnedWindow[match] != pinnedWindow[scan] || pinnedWindow[++match] != pinnedWindow[++scan]) { @@ -511,7 +514,8 @@ namespace SixLabors.ImageSharp.Formats.Png.Zlib // scanMax - scan is the maximum number of bytes we can compare. // below we compare 8 bytes at a time, so first we compare // (scanMax - scan) % 8 bytes, so the remainder is a multiple of 8 - switch ((scanMax - scan) % 8) + // n & (8 - 1) == n % 8. + switch ((scanMax - scan) & 7) { case 1: if (pinnedWindow[++scan] == pinnedWindow[++match]) @@ -616,23 +620,25 @@ namespace SixLabors.ImageSharp.Formats.Png.Zlib && pinnedWindow[++scan] == pinnedWindow[++match]); } - if (scan - this.strstart > this.matchLen) + if (scan - scanStart > matchLength) { - this.matchStart = curMatch; - this.matchLen = scan - this.strstart; + matchStrt = curMatch; + matchLength = scan - scanStart; - if (this.matchLen >= niceLength) + if (matchLength >= niceLength) { break; } - scan_end1 = pinnedWindow[scan - 1]; - scan_end = pinnedWindow[scan]; + scanEnd1 = pinnedWindow[scan - 1]; + scanEnd = pinnedWindow[scan]; } } while ((curMatch = pinnedPrev[curMatch & DeflaterConstants.WMASK] & 0xFFFF) > limit && --chainLength != 0); - return this.matchLen >= DeflaterConstants.MIN_MATCH; + this.matchStart = matchStrt; + this.matchLen = matchLength; + return matchLength >= DeflaterConstants.MIN_MATCH; } private bool DeflateStored(bool flush, bool finish) From ab93b0a497fdfe5c3e4daa9551194f261e363d1a Mon Sep 17 00:00:00 2001 From: James Jackson-South Date: Tue, 19 Nov 2019 14:29:28 +1100 Subject: [PATCH 15/16] Improve code coverage. --- src/ImageSharp/Formats/Png/Zlib/Deflater.cs | 14 ++---- .../Formats/Png/Zlib/DeflaterHuffman.cs | 47 ++++++++----------- .../Formats/Png/Zlib/DeflaterOutputStream.cs | 24 +++++----- .../Formats/Png/Zlib/ZlibDeflateStream.cs | 34 +++----------- .../Formats/Png/PngEncoderTests.cs | 2 +- 5 files changed, 41 insertions(+), 80 deletions(-) diff --git a/src/ImageSharp/Formats/Png/Zlib/Deflater.cs b/src/ImageSharp/Formats/Png/Zlib/Deflater.cs index fb2538f8c..6c4ea44d1 100644 --- a/src/ImageSharp/Formats/Png/Zlib/Deflater.cs +++ b/src/ImageSharp/Formats/Png/Zlib/Deflater.cs @@ -281,23 +281,15 @@ namespace SixLabors.ImageSharp.Formats.Png.Zlib /// public void Dispose() - { - this.Dispose(true); - GC.SuppressFinalize(this); - } - - private void Dispose(bool disposing) { if (!this.isDisposed) { - if (disposing) - { - this.engine.Dispose(); - } - + this.engine.Dispose(); this.engine = null; this.isDisposed = true; } + + GC.SuppressFinalize(this); } } } diff --git a/src/ImageSharp/Formats/Png/Zlib/DeflaterHuffman.cs b/src/ImageSharp/Formats/Png/Zlib/DeflaterHuffman.cs index 7118703d0..003e4fbb7 100644 --- a/src/ImageSharp/Formats/Png/Zlib/DeflaterHuffman.cs +++ b/src/ImageSharp/Formats/Png/Zlib/DeflaterHuffman.cs @@ -372,7 +372,25 @@ namespace SixLabors.ImageSharp.Formats.Png.Zlib /// public void Dispose() { - this.Dispose(true); + if (!this.isDisposed) + { + this.Pending.Dispose(); + this.distanceBufferHandle.Dispose(); + this.distanceManagedBuffer.Dispose(); + this.literalBufferHandle.Dispose(); + this.literalManagedBuffer.Dispose(); + + this.literalTree.Dispose(); + this.blTree.Dispose(); + this.distTree.Dispose(); + + this.Pending = null; + this.literalTree = null; + this.blTree = null; + this.distTree = null; + this.isDisposed = true; + } + GC.SuppressFinalize(this); } @@ -407,33 +425,6 @@ namespace SixLabors.ImageSharp.Formats.Png.Zlib return code + distance; } - private void Dispose(bool disposing) - { - if (!this.isDisposed) - { - if (disposing) - { - this.Pending.Dispose(); - this.distanceBufferHandle.Dispose(); - this.distanceManagedBuffer.Dispose(); - this.literalBufferHandle.Dispose(); - this.literalManagedBuffer.Dispose(); - - this.literalTree.Dispose(); - this.blTree.Dispose(); - this.distTree.Dispose(); - } - - this.Pending = null; - - this.literalTree = null; - this.blTree = null; - this.distTree = null; - - this.isDisposed = true; - } - } - private sealed class Tree : IDisposable { private readonly int minNumCodes; diff --git a/src/ImageSharp/Formats/Png/Zlib/DeflaterOutputStream.cs b/src/ImageSharp/Formats/Png/Zlib/DeflaterOutputStream.cs index eb214aae2..9eeb12cb0 100644 --- a/src/ImageSharp/Formats/Png/Zlib/DeflaterOutputStream.cs +++ b/src/ImageSharp/Formats/Png/Zlib/DeflaterOutputStream.cs @@ -134,22 +134,20 @@ namespace SixLabors.ImageSharp.Formats.Png.Zlib /// protected override void Dispose(bool disposing) { - if (this.isDisposed) + if (!this.isDisposed) { - return; - } + if (disposing) + { + this.Finish(); + this.deflater.Dispose(); + this.memoryOwner.Dispose(); + } - if (disposing) - { - this.Finish(); - this.deflater.Dispose(); - this.memoryOwner.Dispose(); + this.deflater = null; + this.memoryOwner = null; + this.isDisposed = true; + base.Dispose(disposing); } - - this.deflater = null; - this.memoryOwner = null; - this.isDisposed = true; - base.Dispose(disposing); } } } diff --git a/src/ImageSharp/Formats/Png/Zlib/ZlibDeflateStream.cs b/src/ImageSharp/Formats/Png/Zlib/ZlibDeflateStream.cs index 36bacc5ec..3c52d306f 100644 --- a/src/ImageSharp/Formats/Png/Zlib/ZlibDeflateStream.cs +++ b/src/ImageSharp/Formats/Png/Zlib/ZlibDeflateStream.cs @@ -120,28 +120,16 @@ namespace SixLabors.ImageSharp.Formats.Png.Zlib } /// - public override void Flush() - { - this.deflateStream?.Flush(); - } + public override void Flush() => this.deflateStream.Flush(); /// - public override int Read(byte[] buffer, int offset, int count) - { - throw new NotSupportedException(); - } + public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); /// - public override long Seek(long offset, SeekOrigin origin) - { - throw new NotSupportedException(); - } + public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); /// - public override void SetLength(long value) - { - throw new NotSupportedException(); - } + public override void SetLength(long value) => throw new NotSupportedException(); /// public override void Write(byte[] buffer, int offset, int count) @@ -161,17 +149,7 @@ namespace SixLabors.ImageSharp.Formats.Png.Zlib if (disposing) { // dispose managed resources - if (this.deflateStream != null) - { - this.deflateStream.Dispose(); - this.deflateStream = null; - } - else - { - // Hack: empty input? - this.rawStream.WriteByte(3); - this.rawStream.WriteByte(0); - } + this.deflateStream.Dispose(); // Add the crc uint crc = (uint)this.adler32.Value; @@ -181,6 +159,8 @@ namespace SixLabors.ImageSharp.Formats.Png.Zlib this.rawStream.WriteByte((byte)(crc & 0xFF)); } + this.deflateStream = null; + base.Dispose(disposing); this.isDisposed = true; } diff --git a/tests/ImageSharp.Tests/Formats/Png/PngEncoderTests.cs b/tests/ImageSharp.Tests/Formats/Png/PngEncoderTests.cs index 2584391bb..8a0cdbfba 100644 --- a/tests/ImageSharp.Tests/Formats/Png/PngEncoderTests.cs +++ b/tests/ImageSharp.Tests/Formats/Png/PngEncoderTests.cs @@ -63,7 +63,7 @@ namespace SixLabors.ImageSharp.Tests.Formats.Png /// public static readonly TheoryData CompressionLevels = new TheoryData { - 1, 2, 3, 4, 5, 6, 7, 8, 9 + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 }; public static readonly TheoryData PaletteSizes = new TheoryData From a92e7e70bd151e6b960ad4769273ad8aee27b2b9 Mon Sep 17 00:00:00 2001 From: James Jackson-South Date: Mon, 25 Nov 2019 15:20:16 +1100 Subject: [PATCH 16/16] Remove allocation and bounds checks --- .../Formats/Png/Zlib/DeflaterHuffman.cs | 76 ++++++++++--------- 1 file changed, 40 insertions(+), 36 deletions(-) diff --git a/src/ImageSharp/Formats/Png/Zlib/DeflaterHuffman.cs b/src/ImageSharp/Formats/Png/Zlib/DeflaterHuffman.cs index 003e4fbb7..96ff6b657 100644 --- a/src/ImageSharp/Formats/Png/Zlib/DeflaterHuffman.cs +++ b/src/ImageSharp/Formats/Png/Zlib/DeflaterHuffman.cs @@ -829,38 +829,42 @@ namespace SixLabors.ImageSharp.Formats.Png.Zlib private void BuildLength(ReadOnlySpan children) { + byte* lengthPtr = this.Length; + ref int childrenRef = ref MemoryMarshal.GetReference(children); + ref int bitLengthCountsRef = ref MemoryMarshal.GetReference(this.bitLengthCounts); + + int maxLen = this.maxLength; int numNodes = children.Length >> 1; int numLeafs = (numNodes + 1) >> 1; int overflow = 0; - for (int i = 0; i < this.maxLength; i++) - { - this.bitLengthCounts[i] = 0; - } + Array.Clear(this.bitLengthCounts, 0, maxLen); // First calculate optimal bit lengths - int[] lengths = new int[numNodes]; - lengths[numNodes - 1] = 0; - - for (int i = numNodes - 1; i >= 0; i--) + using (IMemoryOwner lengthsMemoryOwner = this.memoryAllocator.Allocate(numNodes, AllocationOptions.Clean)) { - if (children[(2 * i) + 1] != -1) + ref int lengthsRef = ref MemoryMarshal.GetReference(lengthsMemoryOwner.Memory.Span); + + for (int i = numNodes - 1; i >= 0; i--) { - int bitLength = lengths[i] + 1; - if (bitLength > this.maxLength) + if (children[(2 * i) + 1] != -1) { - bitLength = this.maxLength; - overflow++; - } + int bitLength = Unsafe.Add(ref lengthsRef, i) + 1; + if (bitLength > maxLen) + { + bitLength = maxLen; + overflow++; + } - lengths[children[2 * i]] = lengths[children[(2 * i) + 1]] = bitLength; - } - else - { - // A leaf node - int bitLength = lengths[i]; - this.bitLengthCounts[bitLength - 1]++; - this.Length[children[2 * i]] = (byte)lengths[i]; + Unsafe.Add(ref lengthsRef, Unsafe.Add(ref childrenRef, 2 * i)) = Unsafe.Add(ref lengthsRef, Unsafe.Add(ref childrenRef, (2 * i) + 1)) = bitLength; + } + else + { + // A leaf node + int bitLength = Unsafe.Add(ref lengthsRef, i); + Unsafe.Add(ref bitLengthCountsRef, bitLength - 1)++; + lengthPtr[Unsafe.Add(ref childrenRef, 2 * i)] = (byte)Unsafe.Add(ref lengthsRef, i); + } } } @@ -869,11 +873,11 @@ namespace SixLabors.ImageSharp.Formats.Png.Zlib return; } - int incrBitLen = this.maxLength - 1; + int incrBitLen = maxLen - 1; do { // Find the first bit length which could increase: - while (this.bitLengthCounts[--incrBitLen] == 0) + while (Unsafe.Add(ref bitLengthCountsRef, --incrBitLen) == 0) { } @@ -881,18 +885,18 @@ namespace SixLabors.ImageSharp.Formats.Png.Zlib // number of overflow nodes. do { - this.bitLengthCounts[incrBitLen]--; - this.bitLengthCounts[++incrBitLen]++; - overflow -= 1 << (this.maxLength - 1 - incrBitLen); + Unsafe.Add(ref bitLengthCountsRef, incrBitLen)--; + Unsafe.Add(ref bitLengthCountsRef, ++incrBitLen)++; + overflow -= 1 << (maxLen - 1 - incrBitLen); } - while (overflow > 0 && incrBitLen < this.maxLength - 1); + while (overflow > 0 && incrBitLen < maxLen - 1); } while (overflow > 0); // We may have overshot above. Move some nodes from maxLength to // maxLength-1 in that case. - this.bitLengthCounts[this.maxLength - 1] += overflow; - this.bitLengthCounts[this.maxLength - 2] -= overflow; + Unsafe.Add(ref bitLengthCountsRef, maxLen - 1) += overflow; + Unsafe.Add(ref bitLengthCountsRef, maxLen - 2) -= overflow; // Now recompute all bit lengths, scanning in increasing // frequency. It is simpler to reconstruct all lengths instead of @@ -901,17 +905,17 @@ namespace SixLabors.ImageSharp.Formats.Png.Zlib // // The nodes were inserted with decreasing frequency into the childs // array. - int nodePtr = 2 * numLeafs; - for (int bits = this.maxLength; bits != 0; bits--) + int nodeIndex = 2 * numLeafs; + for (int bits = maxLen; bits != 0; bits--) { - int n = this.bitLengthCounts[bits - 1]; + int n = Unsafe.Add(ref bitLengthCountsRef, bits - 1); while (n > 0) { - int childPtr = 2 * children[nodePtr++]; - if (children[childPtr + 1] == -1) + int childIndex = 2 * Unsafe.Add(ref childrenRef, nodeIndex++); + if (Unsafe.Add(ref childrenRef, childIndex + 1) == -1) { // We found another leaf - this.Length[children[childPtr]] = (byte)bits; + lengthPtr[Unsafe.Add(ref childrenRef, childIndex)] = (byte)bits; n--; } }