From 278291313938fe30a4abde1148c3ce2e5f502d7e Mon Sep 17 00:00:00 2001 From: Marcus Cuda Date: Sun, 27 Jan 2013 17:49:21 +0200 Subject: [PATCH 1/4] replaced zlib.net with DeflateStream --- .../LinearAlgebra/IO/Matlab/Adler32.cs | 93 +++++++++++++++++++ .../LinearAlgebra/IO/Matlab/MatlabParser.cs | 10 +- .../LinearAlgebra/IO/MatlabWriter.cs | 33 +++++-- src/Numerics.IO/Numerics.IO.csproj | 5 +- src/UnitTests/UnitTests.csproj | 4 - 5 files changed, 125 insertions(+), 20 deletions(-) create mode 100644 src/Numerics.IO/LinearAlgebra/IO/Matlab/Adler32.cs diff --git a/src/Numerics.IO/LinearAlgebra/IO/Matlab/Adler32.cs b/src/Numerics.IO/LinearAlgebra/IO/Matlab/Adler32.cs new file mode 100644 index 00000000..eac7721c --- /dev/null +++ b/src/Numerics.IO/LinearAlgebra/IO/Matlab/Adler32.cs @@ -0,0 +1,93 @@ +// +// Math.NET Numerics, part of the Math.NET Project +// http://numerics.mathdotnet.com +// http://github.com/mathnet/mathnet-numerics +// http://mathnetnumerics.codeplex.com +// +// Copyright (c) 2013 Math.NET +// +// Permission is hereby granted, free of charge, to any person +// obtaining a copy of this software and associated documentation +// files (the "Software"), to deal in the Software without +// restriction, including without limitation the rights to use, +// copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following +// conditions: +// The above copyright notice and this permission notice shall be +// included in all copies or substantial portions of the Software. +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +// OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +// HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +// WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +// OTHER DEALINGS IN THE SOFTWARE. +// + +/* This code is a port and simplification of + adler32.c -- compute the Adler-32 checksum of a data stream + Copyright (C) 1995-2011 Mark Adler + zlib license: + Copyright (C) 1995-2012 Jean-loup Gailly and Mark Adler + + This software is provided 'as-is', without any express or implied + warranty. In no event will the authors be held liable for any damages + arising from the use of this software. + + Permission is granted to anyone to use this software for any purpose, + including commercial applications, and to alter it and redistribute it + freely, subject to the following restrictions: + + 1. The origin of this software must not be misrepresented; you must not + claim that you wrote the original software. If you use this software + in a product, an acknowledgment in the product documentation would be + appreciated but is not required. + 2. Altered source versions must be plainly marked as such, and must not be + misrepresented as being the original software. + 3. This notice may not be removed or altered from any source distribution. + + Jean-loup Gailly Mark Adler + jloup@gzip.org madler@alumni.caltech.edu + +*/ + +namespace MathNet.Numerics.LinearAlgebra.IO.Matlab +{ + internal static class Adler32 + { + /* largest prime smaller than 65536 */ + const int Base = 65521; + /* NMAX is the largest n such that 255n(n+1)/2 + (n+1)(BASE-1) <= 2^32-1 */ + const int Nmax = 5552; + + /// + /// Computes the Adler-32 checksum of the given data. + /// + /// The data to create the checksum. + /// The checksum + public static uint Compute(byte[] data) + { + uint adler = 1; + uint sum2 = 0; + var len = data.Length; + var offset = 0; + while (len > 0) + { + var tlen = len < Nmax ? len : Nmax; + len -= tlen; + + do + { + adler += data[offset++]; + sum2 += adler; + } while (--tlen > 0); + + adler %= Base; + sum2 %= Base; + } + return adler | (sum2 << 16); + } + } +} diff --git a/src/Numerics.IO/LinearAlgebra/IO/Matlab/MatlabParser.cs b/src/Numerics.IO/LinearAlgebra/IO/Matlab/MatlabParser.cs index 1b3f71ad..69c31f5e 100644 --- a/src/Numerics.IO/LinearAlgebra/IO/Matlab/MatlabParser.cs +++ b/src/Numerics.IO/LinearAlgebra/IO/Matlab/MatlabParser.cs @@ -24,6 +24,8 @@ // OTHER DEALINGS IN THE SOFTWARE. // +using System.IO.Compression; + namespace MathNet.Numerics.LinearAlgebra.IO.Matlab { using System; @@ -34,7 +36,6 @@ namespace MathNet.Numerics.LinearAlgebra.IO.Matlab using System.Text; using Generic; using Properties; - using zlib; using Complex32 = Numerics.Complex32; /// @@ -221,11 +222,12 @@ namespace MathNet.Numerics.LinearAlgebra.IO.Matlab private static byte[] DecompressBlock(byte[] compressed, ref DataType type) { byte[] data; - using (var decompressed = new MemoryStream()) + using (var compressedStream = new MemoryStream(compressed, 2, compressed.Length-6)) { - using (var decompressor = new ZOutputStream(decompressed)) + using (var decompressor = new DeflateStream(compressedStream, CompressionMode.Decompress)) + using(var decompressed = new MemoryStream()) { - decompressor.Write(compressed, 0, compressed.Length); + decompressor.CopyTo(decompressed); decompressed.Position = 0; var buf = new byte[4]; decompressed.Read(buf, 0, 4); diff --git a/src/Numerics.IO/LinearAlgebra/IO/MatlabWriter.cs b/src/Numerics.IO/LinearAlgebra/IO/MatlabWriter.cs index 8120a79e..6031f00b 100644 --- a/src/Numerics.IO/LinearAlgebra/IO/MatlabWriter.cs +++ b/src/Numerics.IO/LinearAlgebra/IO/MatlabWriter.cs @@ -4,7 +4,7 @@ // http://github.com/mathnet/mathnet-numerics // http://mathnetnumerics.codeplex.com // -// Copyright (c) 2009-2010 Math.NET +// Copyright (c) 2009-2013 Math.NET // // Permission is hereby granted, free of charge, to any person // obtaining a copy of this software and associated documentation @@ -29,8 +29,8 @@ using System; using System.Collections.Generic; using System.IO; +using System.IO.Compression; using System.Text; -using zlib; namespace MathNet.Numerics.LinearAlgebra.IO { @@ -285,14 +285,31 @@ namespace MathNet.Numerics.LinearAlgebra.IO /// The compressed data. private static byte[] CompressData(byte[] data) { - using (var compressedStream = new MemoryStream()) + var adler = BitConverter.GetBytes(Adler32.Compute(data)); + using (var stream = new MemoryStream()) { - using (var outputStream = new ZOutputStream(compressedStream, zlibConst.Z_DEFAULT_COMPRESSION)) + stream.WriteByte(0x58); + stream.WriteByte(0x85); + using (var compressedStream = new MemoryStream()) { - outputStream.Write(data, 0, data.Length); - } - - return compressedStream.ToArray(); + using (var outputStream = new DeflateStream(compressedStream, CompressionMode.Compress)) + { + outputStream.Write(data, 0, data.Length); + } + //something odd - we need to write the CRC to the memory stream before returning + //however, the CRC is being added before the data is compressed. It seems that + //both the memorystream and deflate stream need to be closed first, but then + //we cannot append the CRC. So I'm adding a second intermediate memory stream. There should + //be a better way --marcus + var compressedData = compressedStream.ToArray(); + stream.Write(compressedData, 0, compressedData.Length); + } + + stream.WriteByte(adler[3]); + stream.WriteByte(adler[2]); + stream.WriteByte(adler[1]); + stream.WriteByte(adler[0]); + return stream.ToArray(); } } diff --git a/src/Numerics.IO/Numerics.IO.csproj b/src/Numerics.IO/Numerics.IO.csproj index 26b146a7..1c8cdb39 100644 --- a/src/Numerics.IO/Numerics.IO.csproj +++ b/src/Numerics.IO/Numerics.IO.csproj @@ -60,10 +60,6 @@ - - False - ..\..\packages\zlib.net.1.0.4.0\lib\zlib.net.dll - @@ -76,6 +72,7 @@ + diff --git a/src/UnitTests/UnitTests.csproj b/src/UnitTests/UnitTests.csproj index ea502cfd..9c263dae 100644 --- a/src/UnitTests/UnitTests.csproj +++ b/src/UnitTests/UnitTests.csproj @@ -62,10 +62,6 @@ - - False - ..\..\packages\zlib.net.1.0.4.0\lib\zlib.net.dll - From 9851d536e1115295c3c0df814cba5ac79b349185 Mon Sep 17 00:00:00 2001 From: Marcus Cuda Date: Mon, 28 Jan 2013 08:59:43 +0200 Subject: [PATCH 2/4] removed extra memory stream --- .../LinearAlgebra/IO/MatlabWriter.cs | 33 +++++++------------ 1 file changed, 11 insertions(+), 22 deletions(-) diff --git a/src/Numerics.IO/LinearAlgebra/IO/MatlabWriter.cs b/src/Numerics.IO/LinearAlgebra/IO/MatlabWriter.cs index 6031f00b..73eb9944 100644 --- a/src/Numerics.IO/LinearAlgebra/IO/MatlabWriter.cs +++ b/src/Numerics.IO/LinearAlgebra/IO/MatlabWriter.cs @@ -286,30 +286,19 @@ namespace MathNet.Numerics.LinearAlgebra.IO private static byte[] CompressData(byte[] data) { var adler = BitConverter.GetBytes(Adler32.Compute(data)); - using (var stream = new MemoryStream()) + using (var compressedStream = new MemoryStream()) { - stream.WriteByte(0x58); - stream.WriteByte(0x85); - using (var compressedStream = new MemoryStream()) + compressedStream.WriteByte(0x58); + compressedStream.WriteByte(0x85); + using (var outputStream = new DeflateStream(compressedStream, CompressionMode.Compress, true)) { - using (var outputStream = new DeflateStream(compressedStream, CompressionMode.Compress)) - { - outputStream.Write(data, 0, data.Length); - } - //something odd - we need to write the CRC to the memory stream before returning - //however, the CRC is being added before the data is compressed. It seems that - //both the memorystream and deflate stream need to be closed first, but then - //we cannot append the CRC. So I'm adding a second intermediate memory stream. There should - //be a better way --marcus - var compressedData = compressedStream.ToArray(); - stream.Write(compressedData, 0, compressedData.Length); - } - - stream.WriteByte(adler[3]); - stream.WriteByte(adler[2]); - stream.WriteByte(adler[1]); - stream.WriteByte(adler[0]); - return stream.ToArray(); + outputStream.Write(data, 0, data.Length); + } + compressedStream.WriteByte(adler[3]); + compressedStream.WriteByte(adler[2]); + compressedStream.WriteByte(adler[1]); + compressedStream.WriteByte(adler[0]); + return compressedStream.ToArray(); } } From 6eb9bca773a8cd18e6128f11f2da1919cb2f097a Mon Sep 17 00:00:00 2001 From: Marcus Cuda Date: Tue, 29 Jan 2013 15:00:51 +0200 Subject: [PATCH 3/4] fixed bug in sparse matrix parsing --- src/Numerics.IO/LinearAlgebra/IO/Matlab/MatlabParser.cs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/Numerics.IO/LinearAlgebra/IO/Matlab/MatlabParser.cs b/src/Numerics.IO/LinearAlgebra/IO/Matlab/MatlabParser.cs index 69c31f5e..4485d710 100644 --- a/src/Numerics.IO/LinearAlgebra/IO/Matlab/MatlabParser.cs +++ b/src/Numerics.IO/LinearAlgebra/IO/Matlab/MatlabParser.cs @@ -412,7 +412,8 @@ namespace MathNet.Numerics.LinearAlgebra.IO.Matlab for (var i = 0; i < ir.Count; i++) { var row = ir[i]; - if (jc[col + 1] == i) + // if (jc[col + 1] == i) + while(jc[col + 1] == i) { col++; } From 3d203b46b7ad55ea72e1b5ea497d45379b3badbb Mon Sep 17 00:00:00 2001 From: Marcus Cuda Date: Tue, 29 Jan 2013 15:23:42 +0200 Subject: [PATCH 4/4] added fix to other data types and updated tests to check that the bug was fixed --- .../LinearAlgebra/IO/Matlab/MatlabParser.cs | 11 +++++------ .../Complex/IO/MatlabWriterTests.cs | 14 ++++++-------- .../Complex32/IO/MatlabWriterTests.cs | 14 ++++++-------- .../Double/IO/MatlabWriterTests.cs | 14 ++++++-------- .../Single/IO/MatlabWriterTests.cs | 14 ++++++-------- 5 files changed, 29 insertions(+), 38 deletions(-) diff --git a/src/Numerics.IO/LinearAlgebra/IO/Matlab/MatlabParser.cs b/src/Numerics.IO/LinearAlgebra/IO/Matlab/MatlabParser.cs index 4485d710..7c8cbdf6 100644 --- a/src/Numerics.IO/LinearAlgebra/IO/Matlab/MatlabParser.cs +++ b/src/Numerics.IO/LinearAlgebra/IO/Matlab/MatlabParser.cs @@ -412,7 +412,6 @@ namespace MathNet.Numerics.LinearAlgebra.IO.Matlab for (var i = 0; i < ir.Count; i++) { var row = ir[i]; - // if (jc[col + 1] == i) while(jc[col + 1] == i) { col++; @@ -470,7 +469,7 @@ namespace MathNet.Numerics.LinearAlgebra.IO.Matlab for (var i = 0; i < ir.Count; i++) { var row = ir[i]; - if (jc[col + 1] == i) + while (jc[col + 1] == i) { col++; } @@ -529,7 +528,7 @@ namespace MathNet.Numerics.LinearAlgebra.IO.Matlab for (var i = 0; i < ir.Count; i++) { var row = ir[i]; - if (jc[col + 1] == i) + while (jc[col + 1] == i) { col++; } @@ -585,7 +584,7 @@ namespace MathNet.Numerics.LinearAlgebra.IO.Matlab for (var i = 0; i < ir.Count; i++) { var row = ir[i]; - if (jc[col + 1] == i) + while (jc[col + 1] == i) { col++; } @@ -646,7 +645,7 @@ namespace MathNet.Numerics.LinearAlgebra.IO.Matlab for (var i = 0; i < ir.Count; i++) { var row = ir[i]; - if (jc[col + 1] == i) + while (jc[col + 1] == i) { col++; } @@ -703,7 +702,7 @@ namespace MathNet.Numerics.LinearAlgebra.IO.Matlab for (var i = 0; i < ir.Count; i++) { var row = ir[i]; - if (jc[col + 1] == i) + while (jc[col + 1] == i) { col++; } diff --git a/src/UnitTests/LinearAlgebraTests/Complex/IO/MatlabWriterTests.cs b/src/UnitTests/LinearAlgebraTests/Complex/IO/MatlabWriterTests.cs index 486251df..ad69ea87 100644 --- a/src/UnitTests/LinearAlgebraTests/Complex/IO/MatlabWriterTests.cs +++ b/src/UnitTests/LinearAlgebraTests/Complex/IO/MatlabWriterTests.cs @@ -97,16 +97,14 @@ namespace MathNet.Numerics.UnitTests.LinearAlgebraTests.Complex.IO } Matrix mat3 = new SparseMatrix(5, 4); - for (var i = 0; i < mat3.ColumnCount; i++) - { - mat3[i, i] = new Complex(i + .1, i + .1); - } + mat3[0, 0] = new Complex(1.1, 1.1); + mat3[0, 2] = new Complex(2.2, 2.2); + mat3[4, 3] = new Complex(3.3, 3.3); Matrix mat4 = new SparseMatrix(3, 5); - for (var i = 0; i < mat4.RowCount; i++) - { - mat4[i, i] = new Complex(i + .1, i + .1); - } + mat4[0, 0] = new Complex(1.1, 1.1); + mat4[0, 2] = new Complex(2.2, 2.2); + mat4[2, 4] = new Complex(3.3, 3.3); var write = new[] { mat1, mat2, mat3, mat4 }; diff --git a/src/UnitTests/LinearAlgebraTests/Complex32/IO/MatlabWriterTests.cs b/src/UnitTests/LinearAlgebraTests/Complex32/IO/MatlabWriterTests.cs index c21d51d9..71aa7338 100644 --- a/src/UnitTests/LinearAlgebraTests/Complex32/IO/MatlabWriterTests.cs +++ b/src/UnitTests/LinearAlgebraTests/Complex32/IO/MatlabWriterTests.cs @@ -97,16 +97,14 @@ namespace MathNet.Numerics.UnitTests.LinearAlgebraTests.Complex32.IO } Matrix mat3 = new SparseMatrix(5, 4); - for (var i = 0; i < mat3.ColumnCount; i++) - { - mat3[i, i] = new Complex32(i + .1f, i + .1f); - } + mat3[0, 0] = new Complex32(1.1f, 1.1f); + mat3[0, 2] = new Complex32(2.2f, 2.2f); + mat3[4, 3] = new Complex32(3.3f, 3.3f); Matrix mat4 = new SparseMatrix(3, 5); - for (var i = 0; i < mat4.RowCount; i++) - { - mat4[i, i] = new Complex32(i + .1f, i + .1f); - } + mat4[0, 0] = new Complex32(1.1f, 1.1f); + mat4[0, 2] = new Complex32(2.2f, 2.2f); + mat4[2, 4] = new Complex32(3.3f, 3.3f); var write = new[] { mat1, mat2, mat3, mat4 }; diff --git a/src/UnitTests/LinearAlgebraTests/Double/IO/MatlabWriterTests.cs b/src/UnitTests/LinearAlgebraTests/Double/IO/MatlabWriterTests.cs index 608d6ef1..f4c3c5db 100644 --- a/src/UnitTests/LinearAlgebraTests/Double/IO/MatlabWriterTests.cs +++ b/src/UnitTests/LinearAlgebraTests/Double/IO/MatlabWriterTests.cs @@ -96,16 +96,14 @@ namespace MathNet.Numerics.UnitTests.LinearAlgebraTests.Double.IO } Matrix mat3 = new SparseMatrix(5, 4); - for (var i = 0; i < mat3.ColumnCount; i++) - { - mat3[i, i] = i + .1; - } + mat3[0, 0] = 1.1; + mat3[0, 2] = 2.2; + mat3[4, 3] = 3.3; Matrix mat4 = new SparseMatrix(3, 5); - for (var i = 0; i < mat4.RowCount; i++) - { - mat4[i, i] = i + .1; - } + mat4[0, 0] = 1.1; + mat4[0, 2] = 2.2; + mat4[2, 4] = 3.3; var write = new[] { mat1, mat2, mat3, mat4 }; diff --git a/src/UnitTests/LinearAlgebraTests/Single/IO/MatlabWriterTests.cs b/src/UnitTests/LinearAlgebraTests/Single/IO/MatlabWriterTests.cs index 3ded9446..694d002b 100644 --- a/src/UnitTests/LinearAlgebraTests/Single/IO/MatlabWriterTests.cs +++ b/src/UnitTests/LinearAlgebraTests/Single/IO/MatlabWriterTests.cs @@ -95,16 +95,14 @@ namespace MathNet.Numerics.UnitTests.LinearAlgebraTests.Single.IO } Matrix mat3 = new SparseMatrix(5, 4); - for (var i = 0; i < mat3.ColumnCount; i++) - { - mat3[i, i] = i + .1f; - } + mat3[0, 0] = 1.1f; + mat3[0, 2] = 2.2f; + mat3[4, 3] = 3.3f; Matrix mat4 = new SparseMatrix(3, 5); - for (var i = 0; i < mat4.RowCount; i++) - { - mat4[i, i] = i + .1f; - } + mat4[0, 0] = 1.1f; + mat4[0, 2] = 2.2f; + mat4[2, 4] = 3.3f; var write = new[] { mat1, mat2, mat3, mat4 };