From b66e38041406e2c86246cce2b8df41f17fa6ad88 Mon Sep 17 00:00:00 2001 From: Alexander Luzgarev Date: Sat, 9 Mar 2019 18:53:44 +0100 Subject: [PATCH] Support numerical arrays --- MatFileHandler.Tests/MatFileReaderHdfTests.cs | 60 ++++ MatFileHandler.Tests/test-data/hdf/limits.mat | Bin 0 -> 3896 bytes MatFileHandler.Tests/test-data/hdf/matrix.mat | Bin 0 -> 1960 bytes MatFileHandler/HdfFileReader.cs | 289 +++++++++++++++++- MatFileHandler/MatFileHdfReader.cs | 6 +- 5 files changed, 335 insertions(+), 20 deletions(-) create mode 100644 MatFileHandler.Tests/test-data/hdf/limits.mat create mode 100644 MatFileHandler.Tests/test-data/hdf/matrix.mat diff --git a/MatFileHandler.Tests/MatFileReaderHdfTests.cs b/MatFileHandler.Tests/MatFileReaderHdfTests.cs index cdf2085..a5e7995 100644 --- a/MatFileHandler.Tests/MatFileReaderHdfTests.cs +++ b/MatFileHandler.Tests/MatFileReaderHdfTests.cs @@ -50,6 +50,66 @@ namespace MatFileHandler.Tests Assert.That(arrayUnicodeWide.String, Is.EqualTo("🍆")); } + /// + /// Test reading a two-dimensional double array. + /// + [Test] + public void TestMatrix() + { + var matFile = ReadHdfTestFile("matrix"); + var matrix = matFile["matrix"].Value as IArrayOf; + Assert.That(matrix.Dimensions, Is.EqualTo(new[] { 3, 2 })); + Assert.That(matrix.ConvertToDoubleArray(), Is.EqualTo(new[] { 1.0, 3.0, 5.0, 2.0, 4.0, 6.0 })); + Assert.That(matrix[0, 0], Is.EqualTo(1.0)); + Assert.That(matrix[0, 1], Is.EqualTo(2.0)); + Assert.That(matrix[1, 0], Is.EqualTo(3.0)); + Assert.That(matrix[1, 1], Is.EqualTo(4.0)); + Assert.That(matrix[2, 0], Is.EqualTo(5.0)); + Assert.That(matrix[2, 1], Is.EqualTo(6.0)); + } + + /// + /// Test reading lower and upper limits of integer data types. + /// + [Test] + public void TestLimits() + { + var matFile = ReadHdfTestFile("limits"); + IArray array; + array = matFile["int8_"].Value; + CheckLimits(array as IArrayOf, CommonData.Int8Limits); + Assert.That(array.ConvertToDoubleArray(), Is.EqualTo(new[] { -128.0, 127.0 })); + + array = matFile["uint8_"].Value; + CheckLimits(array as IArrayOf, CommonData.UInt8Limits); + + array = matFile["int16_"].Value; + CheckLimits(array as IArrayOf, CommonData.Int16Limits); + + array = matFile["uint16_"].Value; + CheckLimits(array as IArrayOf, CommonData.UInt16Limits); + + array = matFile["int32_"].Value; + CheckLimits(array as IArrayOf, CommonData.Int32Limits); + + array = matFile["uint32_"].Value; + CheckLimits(array as IArrayOf, CommonData.UInt32Limits); + + array = matFile["int64_"].Value; + CheckLimits(array as IArrayOf, CommonData.Int64Limits); + + array = matFile["uint64_"].Value; + CheckLimits(array as IArrayOf, CommonData.UInt64Limits); + } + + private static void CheckLimits(IArrayOf array, T[] limits) + where T : struct + { + Assert.That(array, Is.Not.Null); + Assert.That(array.Dimensions, Is.EqualTo(new[] { 1, 2 })); + Assert.That(array.Data, Is.EqualTo(limits)); + } + private static AbstractTestDataFactory GetTests(string factoryName) => new MatTestDataFactory(Path.Combine(TestDirectory, factoryName)); diff --git a/MatFileHandler.Tests/test-data/hdf/limits.mat b/MatFileHandler.Tests/test-data/hdf/limits.mat new file mode 100644 index 0000000000000000000000000000000000000000..fa519c0599a9f9f5ad84daf4e0b18e0bfb5001ea GIT binary patch literal 3896 zcmeHJy-piJ5T5fvxDZerXrfe8QV{F>!^L)G2OJp*gB2_#MI>?}Mj#oGY*S;22Hqf# zkVnWPq)eGQRo;Tk?957F8;jU$B~n<+_wDY??a#L}-)*hj*ebuqSE(%4%8i!?N3G^6 z)?0D+;Iw^Q#QOTX&Fv_+itFuW+--is(~~0Z#$Bw%ZNvhmqvBejm9uqClsv(*RuI}BXWnJv!0m^ZTr0n9*B7LpLS){=jEyDcq%k6Q5* zFBQM&`yTKQ;JNlc)A9TL8gDBVAWu7=MSTtOGg2$rjtBg&7^qgtb>8rM-E~#*o5q7g;sTMy{22sIJ~gLSzhG3#8mwIy!?UAnEM>>$^}0nG}FnW ziFJndD49~??b5fn|NB0mGlm-$8uoX=j|k0l_;Ha>_thC7M#kae=KaUq=a|?j<SMIe36+wr_T3_Dy$oy0+E7w9c`VKSA5-ACD)Vo5QZ>gyU)8o7k=2v^!@; z4(oyIgzgBZPbT)95N#(wtYD#R8YQz>#-d)RV5@O)inHOP>pNJ;>pJG~HvuVX+m`AE z|EB`)?95Wh!(^0Q(;yCM@gb^!1<5?|b1r5815y;iH^MosSAahNS+Zw{e>tvy-E0Dj zl=&pmG8n{#rb)vF*Qx@orq$&ezNKALm%<12nJk9ygucao5nbo>z&NJLyZ5h`#L zF)6Ar$n-JdU!KQU=kv%6RzdV`4U8Qy7PMfCS-D>&LJi6hQw(%s{HzK^s|h?H1j007 zHo`s1!a4P?*SlTfQ6R-U8n8>A-S_uH&za3w2<)@5&4$ + /// Initializes a new instance of the class. + /// + /// Dimensions of the array. + protected HdfArray( + int[] dimensions) { Dimensions = dimensions; + } + + /// + public int[] Dimensions { get; } + + /// + public int Count => Dimensions.NumberOfElements(); + + /// + /// Returns a new empty array. + /// + /// Empty array. + public static HdfArray Empty() + { + return new HdfArray(Array.Empty()); + } + + public virtual double[] ConvertToDoubleArray() + { + return null; + } + + public virtual Complex[] ConvertToComplexArray() + { + return null; + } + + /// + public bool IsEmpty => Dimensions.Length == 0; + } + + /// + /// A numerical array. + /// + /// Element type. + internal class HdfNumericalArrayOf : HdfArray, IArrayOf + where T : struct + { + /// + /// Initializes a new instance of the class. + /// + /// Dimensions of the array. + /// Array name. + /// Array contents. + public HdfNumericalArrayOf(int[] dimensions, T[] data) + : base(dimensions) + { + Data = data; + } + + /// + public T[] Data { get; } + + /// + public T this[params int[] list] + { + get => Data[Dimensions.DimFlatten(list)]; + set => Data[Dimensions.DimFlatten(list)] = value; + } + + /// + /// Tries to convert the array to an array of Double values. + /// + /// Array of values of the array, converted to Double, or null if the conversion is not possible. + public override double[] ConvertToDoubleArray() + { + return Data as double[] ?? Data.Select(x => Convert.ToDouble(x)).ToArray(); + } + + /// + /// Tries to convert the array to an array of Complex values. + /// + /// Array of values of the array, converted to Complex, or null if the conversion is not possible. + public override Complex[] ConvertToComplexArray() + { + if (Data is Complex[]) + { + return Data as Complex[]; + } + if (Data is ComplexOf[]) + { + return ConvertToComplex(Data as ComplexOf[]); + } + if (Data is ComplexOf[]) + { + return ConvertToComplex(Data as ComplexOf[]); + } + if (Data is ComplexOf[]) + { + return ConvertToComplex(Data as ComplexOf[]); + } + if (Data is ComplexOf[]) + { + return ConvertToComplex(Data as ComplexOf[]); + } + if (Data is ComplexOf[]) + { + return ConvertToComplex(Data as ComplexOf[]); + } + if (Data is ComplexOf[]) + { + return ConvertToComplex(Data as ComplexOf[]); + } + if (Data is ComplexOf[]) + { + return ConvertToComplex(Data as ComplexOf[]); + } + if (Data is ComplexOf[]) + { + return ConvertToComplex(Data as ComplexOf[]); + } + return ConvertToDoubleArray().Select(x => new Complex(x, 0.0)).ToArray(); + } + + private static Complex[] ConvertToComplex(IEnumerable> array) + where TS : struct + { + return array.Select(x => new Complex(Convert.ToDouble(x.Real), Convert.ToDouble(x.Imaginary))).ToArray(); + } + } + + internal class HdfCharArray : HdfArray, ICharArray + { + public HdfCharArray(int[] dimensions, string data) + : base(dimensions) + { StringData = data; } - public bool IsEmpty => Dimensions.Length == 0; - - public int[] Dimensions { get; } - - public int Count => Dimensions.NumberOfElements(); - - public double[] ConvertToDoubleArray() + public override double[] ConvertToDoubleArray() { return Data.Select(Convert.ToDouble).ToArray(); } - public Complex[] ConvertToComplexArray() + public override Complex[] ConvertToComplexArray() { return ConvertToDoubleArray().Select(x => new Complex(x, 0.0)).ToArray(); } @@ -129,26 +254,158 @@ namespace MatFileHandler return dims.Select(x => (int)x).ToArray(); } + private static ArrayType ArrayTypeFromMatlabClassName(string matlabClassName) + { + switch (matlabClassName) + { + case "char": + return ArrayType.MxChar; + case "int8": + return ArrayType.MxInt8; + case "uint8": + return ArrayType.MxUInt8; + case "int16": + return ArrayType.MxInt16; + case "uint16": + return ArrayType.MxUInt16; + case "int32": + return ArrayType.MxInt32; + case "uint32": + return ArrayType.MxUInt32; + case "int64": + return ArrayType.MxInt64; + case "uint64": + return ArrayType.MxUInt64; + case "double": + return ArrayType.MxDouble; + } + throw new NotImplementedException(); + } + private static IArray ReadDataset(long datasetId) { var dims = GetDimensionsOfDataset(datasetId); var matlabClass = GetMatlabClassOfDataset(datasetId); + var arrayType = ArrayTypeFromMatlabClassName(matlabClass); - if (matlabClass == "char") + switch (arrayType) { - return ReadCharArray(datasetId, dims); + case ArrayType.MxChar: + return ReadCharArray(datasetId, dims); + case ArrayType.MxInt8: + return ReadNumericalArray(datasetId, dims, arrayType); + case ArrayType.MxUInt8: + return ReadNumericalArray(datasetId, dims, arrayType); + case ArrayType.MxInt16: + return ReadNumericalArray(datasetId, dims, arrayType); + case ArrayType.MxUInt16: + return ReadNumericalArray(datasetId, dims, arrayType); + case ArrayType.MxInt32: + return ReadNumericalArray(datasetId, dims, arrayType); + case ArrayType.MxUInt32: + return ReadNumericalArray(datasetId, dims, arrayType); + case ArrayType.MxInt64: + return ReadNumericalArray(datasetId, dims, arrayType); + case ArrayType.MxUInt64: + return ReadNumericalArray(datasetId, dims, arrayType); + case ArrayType.MxSingle: + return ReadNumericalArray(datasetId, dims, arrayType); + case ArrayType.MxDouble: + return ReadNumericalArray(datasetId, dims, arrayType); + } + throw new NotImplementedException($"Unknown array type: {arrayType}."); + } + + private static int SizeOfArrayElement(ArrayType arrayType) + { + switch (arrayType) + { + case ArrayType.MxInt8: + case ArrayType.MxUInt8: + return 1; + case ArrayType.MxInt16: + case ArrayType.MxUInt16: + return 2; + case ArrayType.MxInt32: + case ArrayType.MxUInt32: + case ArrayType.MxSingle: + return 4; + case ArrayType.MxInt64: + case ArrayType.MxUInt64: + case ArrayType.MxDouble: + return 8; + } + + throw new NotImplementedException(); + } + + private static long H5tTypeFromArrayType(ArrayType arrayType) + { + switch (arrayType) + { + case ArrayType.MxInt8: + return H5T.NATIVE_INT8; + case ArrayType.MxUInt8: + return H5T.NATIVE_UINT8; + case ArrayType.MxInt16: + return H5T.NATIVE_INT16; + case ArrayType.MxUInt16: + return H5T.NATIVE_UINT16; + case ArrayType.MxInt32: + return H5T.NATIVE_INT32; + case ArrayType.MxUInt32: + return H5T.NATIVE_UINT32; + case ArrayType.MxInt64: + return H5T.NATIVE_INT64; + case ArrayType.MxUInt64: + return H5T.NATIVE_UINT64; + case ArrayType.MxSingle: + return H5T.NATIVE_FLOAT; + case ArrayType.MxDouble: + return H5T.NATIVE_DOUBLE; } throw new NotImplementedException(); } + private static T[] ConvertDataToProperType(byte[] bytes, ArrayType arrayType) + where T : struct + { + var length = bytes.Length; + var arrayElementSize = SizeOfArrayElement(arrayType); + var data = new T[length / arrayElementSize]; + Buffer.BlockCopy(bytes, 0, data, 0, length); + return data; + } + + private static byte[] ReadDataset(long datasetId, long elementType, int dataSize) + { + var dataBuffer = Marshal.AllocHGlobal(dataSize); + H5D.read(datasetId, elementType, H5S.ALL, H5S.ALL, H5P.DEFAULT, dataBuffer); + var data = new byte[dataSize]; + Marshal.Copy(dataBuffer, data, 0, dataSize); + return data; + } + + private static IArray ReadNumericalArray(long datasetId, int[] dims, ArrayType arrayType) + where T : struct + { + var numberOfElements = dims.NumberOfElements(); + var dataSize = numberOfElements * SizeOfArrayElement(arrayType); + var storageSize = (int)H5D.get_storage_size(datasetId); + if (dataSize != storageSize) + { + throw new Exception("Data size mismatch."); + } + var data = ReadDataset(datasetId, H5tTypeFromArrayType(arrayType), dataSize); + var convertedData = ConvertDataToProperType(data, arrayType); + return new HdfNumericalArrayOf(dims, convertedData); + } + private static IArray ReadCharArray(long datasetId, int[] dims) { var storageSize = (int)H5D.get_storage_size(datasetId); - var data = new byte[storageSize]; - var dataBuffer = Marshal.AllocHGlobal(storageSize); - H5D.read(datasetId, H5T.NATIVE_UINT16, H5S.ALL, H5S.ALL, H5P.DEFAULT, dataBuffer); - Marshal.Copy(dataBuffer, data, 0, storageSize); + var data = ReadDataset(datasetId, H5T.NATIVE_UINT16, storageSize); var str = Encoding.Unicode.GetString(data); return new HdfCharArray(dims, str); } diff --git a/MatFileHandler/MatFileHdfReader.cs b/MatFileHandler/MatFileHdfReader.cs index 285b75b..d2d6036 100644 --- a/MatFileHandler/MatFileHdfReader.cs +++ b/MatFileHandler/MatFileHdfReader.cs @@ -1,9 +1,7 @@ -using System; -using System.Collections.Generic; +using HDF.PInvoke; +using System; using System.IO; using System.Runtime.InteropServices; -using System.Text; -using HDF.PInvoke; namespace MatFileHandler {