Browse Source

Data: leverage MATLAB sparse data format for faster reading into sparse matrix (via Christian Woltering)

provider
Christoph Ruegg 12 years ago
parent
commit
d21e076a58
  1. 2
      src/Data/Matlab/ArrayClass.cs
  2. 2
      src/Data/Matlab/ArrayFlags.cs
  3. 2
      src/Data/Matlab/DataType.cs
  4. 2
      src/Data/Matlab/Matlab.csproj
  5. 12
      src/Data/Matlab/MatlabFile.cs
  6. 1396
      src/Data/Matlab/MatlabParser.cs
  7. 2
      src/Data/Matlab/MatlabReader.cs
  8. 14
      src/Data/Matlab/MatlabWriter.cs
  9. 257
      src/Data/Matlab/NumericArrayReader.cs
  10. 251
      src/Data/Matlab/SparseArrayReader.cs

2
src/Data/Matlab/ArrayClass.cs

@ -31,7 +31,7 @@
namespace MathNet.Numerics.Data.Matlab
{
/// <summary>
/// Enumeration for the Matlab array types
/// Enumeration for the MATLAB array types
/// </summary>
internal enum ArrayClass : byte
{

2
src/Data/Matlab/ArrayFlags.cs

@ -33,7 +33,7 @@ using System;
namespace MathNet.Numerics.Data.Matlab
{
/// <summary>
/// Matlab Array Flags
/// MATLAB Array Flags
/// </summary>
[Flags]
internal enum ArrayFlags

2
src/Data/Matlab/DataType.cs

@ -31,7 +31,7 @@
namespace MathNet.Numerics.Data.Matlab
{
/// <summary>
/// Matlab data types
/// MATLAB data types
/// </summary>
internal enum DataType
{

2
src/Data/Matlab/Matlab.csproj

@ -51,6 +51,8 @@
<Compile Include="Adler32.cs" />
<Compile Include="ArrayClass.cs" />
<Compile Include="ArrayFlags.cs" />
<Compile Include="SparseArrayReader.cs" />
<Compile Include="NumericArrayReader.cs" />
<Compile Include="DataType.cs" />
<Compile Include="MatlabFile.cs" />
<Compile Include="MatlabParser.cs" />

12
src/Data/Matlab/MatlabFile.cs

@ -35,13 +35,13 @@ using MathNet.Numerics.LinearAlgebra;
namespace MathNet.Numerics.Data.Matlab
{
/// <summary>
/// Represents a Matlab file
/// Represents a MATLAB file
/// </summary>
/// <typeparam name="TDataType">The data type of the matrix to return.</typeparam>
internal class MatlabFile<TDataType> where TDataType : struct, IEquatable<TDataType>, IFormattable
{
/// <summary>
/// Matrices in a matlab file stored as 1-D arrays
/// Matrices in a MATLAB file stored as 1-D arrays
/// </summary>
readonly IDictionary<string, Matrix<TDataType>> _matrices = new SortedList<string, Matrix<TDataType>>();
@ -49,19 +49,19 @@ namespace MathNet.Numerics.Data.Matlab
/// Gets or sets the header text.
/// </summary>
/// <value>The header text.</value>
public string HeaderText { get; set; }
internal string HeaderText { get; set; }
/// <summary>
/// Gets or sets the first name of the matrix.
/// </summary>
/// <value>The first name of the matrix.</value>
public string FirstMatrixName { get; set; }
internal string FirstMatrixName { get; set; }
/// <summary>
/// Gets the first matrix.
/// </summary>
/// <value>The first matrix.</value>
public Matrix<TDataType> FirstMatrix
internal Matrix<TDataType> FirstMatrix
{
get
{
@ -78,7 +78,7 @@ namespace MathNet.Numerics.Data.Matlab
/// Gets the matrices.
/// </summary>
/// <value>The matrices.</value>
public IDictionary<string, Matrix<TDataType>> Matrices
internal IDictionary<string, Matrix<TDataType>> Matrices
{
get { return _matrices; }
}

1396
src/Data/Matlab/MatlabParser.cs

File diff suppressed because it is too large

2
src/Data/Matlab/MatlabReader.cs

@ -37,7 +37,7 @@ using MathNet.Numerics.LinearAlgebra;
namespace MathNet.Numerics.Data.Matlab
{
/// <summary>
/// Creates matrices from Matlab files.
/// Creates matrices from MATLAB files.
/// </summary>
public static class MatlabMatrixReader
{

14
src/Data/Matlab/MatlabWriter.cs

@ -4,7 +4,7 @@
// http://github.com/mathnet/mathnet-numerics
// http://mathnetnumerics.codeplex.com
//
// Copyright (c) 2009-2013 Math.NET
// Copyright (c) 2009-2014 Math.NET
//
// Permission is hereby granted, free of charge, to any person
// obtaining a copy of this software and associated documentation
@ -41,7 +41,7 @@ using MathNet.Numerics.Properties;
namespace MathNet.Numerics.Data.Matlab
{
/// <summary>
/// Writes matrices to a Matlab file.
/// Writes matrices to a MATLAB file.
/// </summary>
public class MatlabMatrixWriter : IDisposable
{
@ -68,7 +68,7 @@ namespace MathNet.Numerics.Data.Matlab
/// <summary>
/// Initializes a new instance of the <see cref="MatlabMatrixWriter"/> class.
/// </summary>
/// <param name="filename">The name of the Matlab file to save the matrices to.</param>
/// <param name="filename">The name of the MATLAB file to save the matrices to.</param>
public MatlabMatrixWriter(string filename)
{
if (string.IsNullOrEmpty(filename))
@ -95,7 +95,7 @@ namespace MathNet.Numerics.Data.Matlab
}
/// <summary>
/// Writes the given <see cref="Matrix{T}"/> to the file.
/// Writes the given <see cref="Matrix{T}"/> to the file.
/// </summary>
/// <param name="matrix">The matrix to write.</param>
/// <param name="name">The name of the matrix to store in the file.</param>
@ -125,7 +125,7 @@ namespace MathNet.Numerics.Data.Matlab
_headerWritten = true;
}
// write datatype
// write data type
_writer.Write((int)DataType.Compressed);
byte[] data;
@ -212,7 +212,7 @@ namespace MathNet.Numerics.Data.Matlab
/// <param name="writer">The writer we are using.</param>
/// <param name="arrayClass">The array class we are writing.</param>
/// <param name="isComplex">if set to <c>true</c> if this a complex matrix.</param>
/// <param name="name">The name name of the matrix.</param>
/// <param name="name">The name of the matrix.</param>
/// <param name="rows">The number of rows.</param>
/// <param name="columns">The columns of columns.</param>
/// <param name="nzmax">The maximum number of non-zero elements.</param>
@ -228,7 +228,7 @@ namespace MathNet.Numerics.Data.Matlab
writer.Write((int)DataType.UInt32);
writer.Write(8);
// write array class and flags
// write array class and flags
writer.Write((byte)arrayClass);
if (isComplex)
{

257
src/Data/Matlab/NumericArrayReader.cs

@ -0,0 +1,257 @@
// <copyright file="NumericArrayReader.cs" company="Math.NET">
// Math.NET Numerics, part of the Math.NET Project
// http://numerics.mathdotnet.com
// http://github.com/mathnet/mathnet-numerics
// http://mathnetnumerics.codeplex.com
//
// Copyright (c) 2009-2014 Math.NET
//
// Permission is hereby granted, free of charge, to any person
// obtaining a copy of this software and associated documentation
// files (the "Software"), to deal in the Software without
// restriction, including without limitation the rights to use,
// copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the
// Software is furnished to do so, subject to the following
// conditions:
//
// The above copyright notice and this permission notice shall be
// included in all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
// OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
// HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
// WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
// OTHER DEALINGS IN THE SOFTWARE.
// </copyright>
using System;
using System.IO;
using System.Numerics;
using MathNet.Numerics.LinearAlgebra;
namespace MathNet.Numerics.Data.Matlab
{
internal static class NumericArrayReader<TDataType>
where TDataType : struct, IEquatable<TDataType>, IFormattable
{
/// <summary>
/// Populates a dense matrix.
/// </summary>
/// <param name="type">The MATLAB data type.</param>
/// <param name="reader">The reader to read from.</param>
/// <param name="isComplex">if set to <c>true</c> if the MATLAB complex flag is set.</param>
/// <param name="rows">The number of rows.</param>
/// <param name="columns">The number of columns.</param>
/// <param name="size">The length of the stored data.</param>
/// <returns>Returns a populated dense matrix.</returns>
public static Matrix<TDataType> PopulateDenseMatrix(DataType type, BinaryReader reader, bool isComplex, int rows, int columns, int size)
{
var dataType = typeof (TDataType);
Matrix<TDataType> matrix;
if (type == DataType.Double && dataType == typeof (double))
{
var count = rows*columns;
var data = new double[count];
Buffer.BlockCopy(reader.ReadBytes(count*Constants.SizeOfDouble), 0, data, 0, count*Constants.SizeOfDouble);
matrix = (Matrix<TDataType>)(object)new LinearAlgebra.Double.DenseMatrix(rows, columns, data);
}
else if (type == DataType.Single && dataType == typeof (float))
{
var count = rows*columns;
var data = new float[count];
Buffer.BlockCopy(reader.ReadBytes(count*Constants.SizeOfFloat), 0, data, 0, count*Constants.SizeOfFloat);
matrix = (Matrix<TDataType>)(object)new LinearAlgebra.Single.DenseMatrix(rows, columns, data);
}
else
{
matrix = Matrix<TDataType>.Build.Dense(rows, columns);
if (dataType == typeof (double))
{
if (isComplex)
{
throw new ArgumentException("Invalid TDataType. Matrix is stored as a complex matrix, but a real data type was given.");
}
PopulateDoubleDenseMatrix((Matrix<double>)(object)matrix, type, reader, rows, columns);
}
else if (dataType == typeof (float))
{
if (isComplex)
{
throw new ArgumentException("Invalid TDataType. Matrix is stored as a complex matrix, but a real data type was given.");
}
PopulateSingleDenseMatrix((Matrix<float>)(object)matrix, type, reader, rows, columns);
}
else if (dataType == typeof (Complex))
{
PopulateComplexDenseMatrix((Matrix<Complex>)(object)matrix, type, isComplex, reader, rows, columns, size);
}
else if (dataType == typeof (Complex32))
{
PopulateComplex32DenseMatrix((Matrix<Complex32>)(object)matrix, type, isComplex, reader, rows, columns, size);
}
else
{
throw new NotSupportedException();
}
}
return matrix;
}
/// <summary>
/// Populates the double dense matrix.
/// </summary>
/// <param name="matrix">The matrix to populate.</param>
/// <param name="type">The MATLAB data type.</param>
/// <param name="reader">The reader to read from.</param>
/// <param name="rows">The number of rows.</param>
/// <param name="columns">The number of columns.</param>
static void PopulateDoubleDenseMatrix(Matrix<double> matrix, DataType type, BinaryReader reader, int rows, int columns)
{
for (var j = 0; j < columns; j++)
{
for (var i = 0; i < rows; i++)
{
matrix.At(i, j, ReadDoubleValue(type, reader));
}
}
}
/// <summary>
/// Populates the complex dense matrix.
/// </summary>
/// <param name="matrix">The matrix to populate.</param>
/// <param name="type">The MATLAB data type.</param>
/// <param name="isComplex">if set to <c>true</c> if the MATLAB complex flag is set.</param>
/// <param name="reader">The reader to read from.</param>
/// <param name="rows">The number of rows.</param>
/// <param name="columns">The number of columns.</param>
/// <param name="dataSize">The length of the stored data.</param>
static void PopulateComplexDenseMatrix(Matrix<Complex> matrix, DataType type, bool isComplex, BinaryReader reader, int rows, int columns, int dataSize)
{
for (var j = 0; j < columns; j++)
{
for (var i = 0; i < rows; i++)
{
matrix.At(i, j, ReadDoubleValue(type, reader));
}
}
if (isComplex)
{
var skip = dataSize%8;
// skip pad
reader.ReadBytes(skip);
// skip header
type = (DataType)reader.ReadInt32();
reader.ReadInt32();
for (var j = 0; j < columns; j++)
{
for (var i = 0; i < rows; i++)
{
matrix.At(i, j, new Complex(matrix.At(i, j).Real, ReadDoubleValue(type, reader)));
}
}
}
}
/// <summary>
/// Populates the complex32 dense matrix.
/// </summary>
/// <param name="matrix">The matrix to populate.</param>
/// <param name="type">The MATLAB data type.</param>
/// <param name="isComplex">if set to <c>true</c> if the MATLAB complex flag is set.</param>
/// <param name="reader">The reader to read from.</param>
/// <param name="rows">The number of rows.</param>
/// <param name="columns">The number of columns.</param>
/// <param name="dataSize">The length of the stored data.</param>
static void PopulateComplex32DenseMatrix(Matrix<Complex32> matrix, DataType type, bool isComplex, BinaryReader reader, int rows, int columns, int dataSize)
{
for (var j = 0; j < columns; j++)
{
for (var i = 0; i < rows; i++)
{
matrix.At(i, j, (float)ReadDoubleValue(type, reader));
}
}
if (isComplex)
{
var skip = dataSize%8;
// skip pad
reader.ReadBytes(skip);
// skip header
type = (DataType)reader.ReadInt32();
reader.ReadInt32();
for (var j = 0; j < columns; j++)
{
for (var i = 0; i < rows; i++)
{
matrix.At(i, j, new Complex32(matrix.At(i, j).Real, (float)ReadDoubleValue(type, reader)));
}
}
}
}
/// <summary>
/// Populates the float dense matrix.
/// </summary>
/// <param name="matrix">The matrix to populate.</param>
/// <param name="type">The MATLAB data type.</param>
/// <param name="reader">The reader to read from.</param>
/// <param name="rows">The number of rows.</param>
/// <param name="columns">The number of columns.</param>
static void PopulateSingleDenseMatrix(Matrix<float> matrix, DataType type, BinaryReader reader, int rows, int columns)
{
for (var j = 0; j < columns; j++)
{
for (var i = 0; i < rows; i++)
{
matrix.At(i, j, (float)ReadDoubleValue(type, reader));
}
}
}
static double ReadDoubleValue(DataType type, BinaryReader reader)
{
switch (type)
{
case DataType.Double:
return reader.ReadDouble();
case DataType.Int8:
return reader.ReadSByte();
case DataType.UInt8:
return reader.ReadByte();
case DataType.Int16:
return reader.ReadInt16();
case DataType.UInt16:
return reader.ReadUInt16();
case DataType.Int32:
return reader.ReadInt32();
case DataType.UInt32:
return reader.ReadUInt32();
case DataType.Single:
return reader.ReadSingle();
case DataType.Int64:
return reader.ReadInt64();
case DataType.UInt64:
return reader.ReadUInt64();
default:
throw new NotSupportedException();
}
}
}
}

251
src/Data/Matlab/SparseArrayReader.cs

@ -0,0 +1,251 @@
// <copyright file="SparseArrayReader.cs" company="Math.NET">
// Math.NET Numerics, part of the Math.NET Project
// http://numerics.mathdotnet.com
// http://github.com/mathnet/mathnet-numerics
// http://mathnetnumerics.codeplex.com
//
// Copyright (c) 2009-2014 Math.NET
//
// Permission is hereby granted, free of charge, to any person
// obtaining a copy of this software and associated documentation
// files (the "Software"), to deal in the Software without
// restriction, including without limitation the rights to use,
// copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the
// Software is furnished to do so, subject to the following
// conditions:
//
// The above copyright notice and this permission notice shall be
// included in all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
// OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
// HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
// WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
// OTHER DEALINGS IN THE SOFTWARE.
// </copyright>
using System;
using System.IO;
using System.Numerics;
using MathNet.Numerics.LinearAlgebra;
using MathNet.Numerics.LinearAlgebra.Storage;
namespace MathNet.Numerics.Data.Matlab
{
internal static class SparseArrayReader<TDataType>
where TDataType : struct, IEquatable<TDataType>, IFormattable
{
/// <summary>
/// Populates a sparse matrix.
/// </summary>
/// <param name="reader">The reader.</param>
/// <param name="isComplex">if set to <c>true</c> if the MATLAB complex flag is set.</param>
/// <param name="rows">The number of rows.</param>
/// <param name="columns">The number of columns.</param>
/// <param name="size">The size of the block.</param>
/// <returns>A populated sparse matrix.</returns>
public static Matrix<TDataType> PopulateSparseMatrix(BinaryReader reader, bool isComplex, int rows, int columns, int size)
{
// Create matrix with CSR storage.
var matrix = Matrix<TDataType>.Build.Sparse(columns, rows);
// MATLAB sparse matrices are actually stored as CSC, so just read the data and then transpose.
var storage = matrix.Storage as SparseCompressedRowMatrixStorage<TDataType>;
// populate the row data array
var ir = storage.ColumnIndices = new int[size/4];
for (var i = 0; i < ir.Length; i++)
{
ir[i] = reader.ReadInt32();
}
MatlabParser<TDataType>.AlignData(reader.BaseStream, size, false);
// skip data type since it will always be int32
reader.BaseStream.Seek(4, SeekOrigin.Current);
// populate the column data array
var jcsize = reader.ReadInt32();
var jc = storage.RowPointers;
if (jc.Length != jcsize/4)
{
throw new Exception("invalid jcsize");
}
for (var j = 0; j < jc.Length; j++)
{
jc[j] = reader.ReadInt32();
}
MatlabParser<TDataType>.AlignData(reader.BaseStream, jcsize, false);
var type = (DataType)reader.ReadInt32();
var dataSize = reader.ReadInt32();
var dataType = typeof (TDataType);
// Allocate memory for matrix values
var data = storage.Values = new TDataType[jc[columns]];
if (dataType == typeof (double))
{
if (isComplex)
{
throw new ArgumentException("Invalid TDataType. Matrix is stored as a complex matrix, but a real data type was given.");
}
PopulateDoubleSparseMatrix(type, (double[])(object)data, reader);
}
else if (dataType == typeof (float))
{
if (isComplex)
{
throw new ArgumentException("Invalid TDataType. Matrix is stored as a complex matrix, but a real data type was given.");
}
PopulateSingleSparseMatrix(type, (float[])(object)data, reader);
}
else if (dataType == typeof (Complex))
{
PopulateComplexSparseMatrix(type, isComplex, (Complex[])(object)data, reader, dataSize);
}
else if (dataType == typeof (Complex32))
{
PopulateComplex32SparseMatrix(type, isComplex, (Complex32[])(object)data, reader, dataSize);
}
else
{
throw new NotSupportedException();
}
return matrix.Transpose();
}
/// <summary>
/// Populates the double sparse matrix.
/// </summary>
/// <param name="type">The MATLAB data type.</param>
/// <param name="data">The matrix values array.</param>
/// <param name="reader">The reader to read from.</param>
static void PopulateDoubleSparseMatrix(DataType type, double[] data, BinaryReader reader)
{
for (var i = 0; i < data.Length; i++)
{
data[i] = ReadDoubleValue(type, reader);
}
}
/// <summary>
/// Populates the float sparse matrix.
/// </summary>
/// <param name="type">The MATLAB data type.</param>
/// <param name="data">The matrix values array.</param>
/// <param name="reader">The reader to read from.</param>
static void PopulateSingleSparseMatrix(DataType type, float[] data, BinaryReader reader)
{
for (var i = 0; i < data.Length; i++)
{
data[i] = (float)ReadDoubleValue(type, reader);
}
}
/// <summary>
/// Populates the complex sparse matrix.
/// </summary>
/// <param name="type">The MATLAB data type.</param>
/// <param name="isComplex">if set to <c>true</c> if the MATLAB complex flag is set.</param>
/// <param name="data">The matrix values array.</param>
/// <param name="reader">The reader to read from.</param>
/// <param name="dataSize">The length of the stored data.</param>
static void PopulateComplexSparseMatrix(DataType type, bool isComplex, Complex[] data, BinaryReader reader, int dataSize)
{
for (var i = 0; i < data.Length; i++)
{
data[i] = ReadDoubleValue(type, reader);
}
if (isComplex)
{
var skip = dataSize%8;
// skip pad
reader.ReadBytes(skip);
// skip header
type = (DataType)reader.ReadInt32();
reader.ReadInt32();
for (var i = 0; i < data.Length; i++)
{
data[i] += new Complex(0.0, ReadDoubleValue(type, reader));
}
}
}
/// <summary>
/// Populates the complex32 sparse matrix.
/// </summary>
/// <param name="type">The MATLAB data type.</param>
/// <param name="isComplex">if set to <c>true</c> if the MATLAB complex flag is set.</param>
/// <param name="data">The matrix values array.</param>
/// <param name="reader">The reader to read from.</param>
/// <param name="dataSize">The length of the stored data.</param>
static void PopulateComplex32SparseMatrix(DataType type, bool isComplex, Complex32[] data, BinaryReader reader, int dataSize)
{
for (var i = 0; i < data.Length; i++)
{
data[i] = (float)ReadDoubleValue(type, reader);
}
if (isComplex)
{
var skip = dataSize%8;
// skip pad
reader.ReadBytes(skip);
// skip header
type = (DataType)reader.ReadInt32();
reader.ReadInt32();
for (var i = 0; i < data.Length; i++)
{
data[i] += new Complex32(0.0f, (float)ReadDoubleValue(type, reader));
}
}
}
static double ReadDoubleValue(DataType type, BinaryReader reader)
{
switch (type)
{
case DataType.Double:
return reader.ReadDouble();
case DataType.Int8:
return reader.ReadSByte();
case DataType.UInt8:
return reader.ReadByte();
case DataType.Int16:
return reader.ReadInt16();
case DataType.UInt16:
return reader.ReadUInt16();
case DataType.Int32:
return reader.ReadInt32();
case DataType.UInt32:
return reader.ReadUInt32();
case DataType.Single:
return reader.ReadSingle();
case DataType.Int64:
return reader.ReadInt64();
case DataType.UInt64:
return reader.ReadUInt64();
default:
throw new NotSupportedException();
}
}
}
}
Loading…
Cancel
Save