From 68256334ff2322a04d276e6721cc752aa393a739 Mon Sep 17 00:00:00 2001 From: Robert Hague Date: Wed, 23 Jul 2025 11:36:59 +0100 Subject: [PATCH] Read/Write bytes directly from data array --- .../MatFileHandler.Tests.csproj | 8 +- MatFileHandler/DataElementReader.cs | 17 +- MatFileHandler/Header.cs | 4 +- MatFileHandler/MatFileHandler.csproj | 4 +- MatFileHandler/MatFileWriter.cs | 165 ++++++------------ MatFileHandler/PolyfillExtensions.cs | 52 ++++++ 6 files changed, 126 insertions(+), 124 deletions(-) create mode 100644 MatFileHandler/PolyfillExtensions.cs diff --git a/MatFileHandler.Tests/MatFileHandler.Tests.csproj b/MatFileHandler.Tests/MatFileHandler.Tests.csproj index d0c201d..5709432 100755 --- a/MatFileHandler.Tests/MatFileHandler.Tests.csproj +++ b/MatFileHandler.Tests/MatFileHandler.Tests.csproj @@ -1,6 +1,6 @@  - net461;net472;net8.0 + net462;net472;net8.0 false latest @@ -10,11 +10,7 @@ - - all - runtime; build; native; contentfiles; analyzers; buildtransitive - - + all runtime; build; native; contentfiles; analyzers; buildtransitive diff --git a/MatFileHandler/DataElementReader.cs b/MatFileHandler/DataElementReader.cs index 34f2bed..eb8b13a 100755 --- a/MatFileHandler/DataElementReader.cs +++ b/MatFileHandler/DataElementReader.cs @@ -1,8 +1,10 @@ using System; using System.Collections.Generic; +using System.Diagnostics; using System.IO; using System.IO.Compression; using System.Linq; +using System.Runtime.InteropServices; using System.Text; namespace MatFileHandler @@ -144,17 +146,18 @@ namespace MatFileHandler return Encoding.ASCII.GetString(element.Data.Select(x => (byte)x).ToArray()); } - private static DataElement ReadNum(Tag tag, BinaryReader reader) - where T : struct + private static MiNum ReadNum(Tag tag, BinaryReader reader) + where T : unmanaged { - var bytes = reader.ReadBytes(tag.Length); - if (tag.Type == DataType.MiUInt8) + T[] result; + + unsafe { - return new MiNum(bytes); + Debug.Assert(tag.ElementSize == sizeof(T)); + result = new T[tag.Length / sizeof(T)]; } - var result = new T[bytes.Length / tag.ElementSize]; - Buffer.BlockCopy(bytes, 0, result, 0, bytes.Length); + reader.BaseStream.ReadExactly(MemoryMarshal.AsBytes(result)); return new MiNum(result); } diff --git a/MatFileHandler/Header.cs b/MatFileHandler/Header.cs index dcfe568..028328b 100755 --- a/MatFileHandler/Header.cs +++ b/MatFileHandler/Header.cs @@ -2,7 +2,7 @@ using System; using System.Globalization; using System.IO; using System.Linq; -#if !NET461 +#if !NET462 using System.Runtime.InteropServices; #endif @@ -79,7 +79,7 @@ namespace MatFileHandler private static string GetOperatingSystem() { -#if NET461 +#if NET462 return "Windows"; #else if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) diff --git a/MatFileHandler/MatFileHandler.csproj b/MatFileHandler/MatFileHandler.csproj index dcebe31..9ac5c2d 100755 --- a/MatFileHandler/MatFileHandler.csproj +++ b/MatFileHandler/MatFileHandler.csproj @@ -1,6 +1,6 @@  - netstandard2.0;net461;net472 + netstandard2.0;net462;net8.0 1.4.0-beta6 MatFileHandler A library for reading and writing MATLAB .mat files. @@ -28,7 +28,7 @@ - + diff --git a/MatFileHandler/MatFileWriter.cs b/MatFileHandler/MatFileWriter.cs index 67648fe..f8487e7 100755 --- a/MatFileHandler/MatFileWriter.cs +++ b/MatFileHandler/MatFileWriter.cs @@ -1,9 +1,12 @@ using System; +using System.Buffers; using System.Collections.Generic; +using System.Diagnostics; using System.IO; using System.IO.Compression; using System.Linq; using System.Numerics; +using System.Runtime.InteropServices; using System.Text; namespace MatFileHandler @@ -114,13 +117,21 @@ namespace MatFileHandler writer.Write((short)tag.Length); } - private static void WriteDataElement(BinaryWriter writer, DataType type, byte[] data) + private static void WriteDataElement(BinaryWriter writer, DataType type, T[] data) + where T : unmanaged { - if (data.Length > 4) + unsafe { - WriteTag(writer, new Tag(type, data.Length)); - writer.Write(data); - var rem = data.Length % 8; + Debug.Assert(type.Size() == sizeof(T)); + } + + ReadOnlySpan bytes = MemoryMarshal.AsBytes(data); + + if (bytes.Length > 4) + { + WriteTag(writer, new Tag(type, bytes.Length)); + writer.Write(bytes); + var rem = bytes.Length % 8; if (rem > 0) { var padding = new byte[8 - rem]; @@ -129,11 +140,11 @@ namespace MatFileHandler } else { - WriteShortTag(writer, new Tag(type, data.Length)); - writer.Write(data); - if (data.Length < 4) + WriteShortTag(writer, new Tag(type, bytes.Length)); + writer.Write(bytes); + if (bytes.Length < 4) { - var padding = new byte[4 - data.Length]; + var padding = new byte[4 - bytes.Length]; writer.Write(padding); } } @@ -141,81 +152,22 @@ namespace MatFileHandler private static void WriteDimensions(BinaryWriter writer, int[] dimensions) { - var buffer = ConvertToByteArray(dimensions); - WriteDataElement(writer, DataType.MiInt32, buffer); + WriteDataElement(writer, DataType.MiInt32, dimensions); } - private static byte[] ConvertToByteArray(T[] data) + private static (T[] real, T[] imaginary) ConvertToPairOfArrays(ComplexOf[] data) where T : struct { - int size; - if (typeof(T) == typeof(sbyte)) - { - size = sizeof(sbyte); - } - else if (typeof(T) == typeof(byte)) - { - size = sizeof(byte); - } - else if (typeof(T) == typeof(short)) - { - size = sizeof(short); - } - else if (typeof(T) == typeof(ushort)) - { - size = sizeof(ushort); - } - else if (typeof(T) == typeof(int)) - { - size = sizeof(int); - } - else if (typeof(T) == typeof(uint)) - { - size = sizeof(uint); - } - else if (typeof(T) == typeof(long)) - { - size = sizeof(long); - } - else if (typeof(T) == typeof(ulong)) - { - size = sizeof(ulong); - } - else if (typeof(T) == typeof(float)) - { - size = sizeof(float); - } - else if (typeof(T) == typeof(double)) - { - size = sizeof(double); - } - else if (typeof(T) == typeof(bool)) - { - size = sizeof(bool); - } - else - { - throw new NotSupportedException(); - } - var buffer = new byte[data.Length * size]; - Buffer.BlockCopy(data, 0, buffer, 0, buffer.Length); - return buffer; + return (data.Select(x => x.Real).ToArray(), data.Select(x => x.Imaginary).ToArray()); } - private static (byte[] real, byte[] imaginary) ConvertToPairOfByteArrays(ComplexOf[] data) - where T : struct + private static (double[] real, double[] imaginary) ConvertToPairOfArrays(Complex[] data) { - return (ConvertToByteArray(data.Select(x => x.Real).ToArray()), - ConvertToByteArray(data.Select(x => x.Imaginary).ToArray())); + return (data.Select(x => x.Real).ToArray(), data.Select(x => x.Imaginary).ToArray()); } - private static (byte[] real, byte[] imaginary) ConvertToPairOfByteArrays(Complex[] data) - { - return (ConvertToByteArray(data.Select(x => x.Real).ToArray()), - ConvertToByteArray(data.Select(x => x.Imaginary).ToArray())); - } - - private static void WriteComplexValues(BinaryWriter writer, DataType type, (byte[] real, byte[] complex) data) + private static void WriteComplexValues(BinaryWriter writer, DataType type, (T[] real, T[] complex) data) + where T : unmanaged { WriteDataElement(writer, type, data.real); WriteDataElement(writer, type, data.complex); @@ -251,67 +203,67 @@ namespace MatFileHandler switch (value) { case IArrayOf sbyteArray: - WriteDataElement(writer, DataType.MiInt8, ConvertToByteArray(sbyteArray.Data)); + WriteDataElement(writer, DataType.MiInt8, sbyteArray.Data); break; case IArrayOf byteArray: - WriteDataElement(writer, DataType.MiUInt8, ConvertToByteArray(byteArray.Data)); + WriteDataElement(writer, DataType.MiUInt8, byteArray.Data); break; case IArrayOf shortArray: - WriteDataElement(writer, DataType.MiInt16, ConvertToByteArray(shortArray.Data)); + WriteDataElement(writer, DataType.MiInt16, shortArray.Data); break; case IArrayOf ushortArray: - WriteDataElement(writer, DataType.MiUInt16, ConvertToByteArray(ushortArray.Data)); + WriteDataElement(writer, DataType.MiUInt16, ushortArray.Data); break; case IArrayOf intArray: - WriteDataElement(writer, DataType.MiInt32, ConvertToByteArray(intArray.Data)); + WriteDataElement(writer, DataType.MiInt32, intArray.Data); break; case IArrayOf uintArray: - WriteDataElement(writer, DataType.MiUInt32, ConvertToByteArray(uintArray.Data)); + WriteDataElement(writer, DataType.MiUInt32, uintArray.Data); break; case IArrayOf longArray: - WriteDataElement(writer, DataType.MiInt64, ConvertToByteArray(longArray.Data)); + WriteDataElement(writer, DataType.MiInt64, longArray.Data); break; case IArrayOf ulongArray: - WriteDataElement(writer, DataType.MiUInt64, ConvertToByteArray(ulongArray.Data)); + WriteDataElement(writer, DataType.MiUInt64, ulongArray.Data); break; case IArrayOf floatArray: - WriteDataElement(writer, DataType.MiSingle, ConvertToByteArray(floatArray.Data)); + WriteDataElement(writer, DataType.MiSingle, floatArray.Data); break; case IArrayOf doubleArray: - WriteDataElement(writer, DataType.MiDouble, ConvertToByteArray(doubleArray.Data)); + WriteDataElement(writer, DataType.MiDouble, doubleArray.Data); break; case IArrayOf boolArray: - WriteDataElement(writer, DataType.MiUInt8, ConvertToByteArray(boolArray.Data)); + WriteDataElement(writer, DataType.MiUInt8, boolArray.Data); break; case IArrayOf> complexSbyteArray: - WriteComplexValues(writer, DataType.MiInt8, ConvertToPairOfByteArrays(complexSbyteArray.Data)); + WriteComplexValues(writer, DataType.MiInt8, ConvertToPairOfArrays(complexSbyteArray.Data)); break; case IArrayOf> complexByteArray: - WriteComplexValues(writer, DataType.MiUInt8, ConvertToPairOfByteArrays(complexByteArray.Data)); + WriteComplexValues(writer, DataType.MiUInt8, ConvertToPairOfArrays(complexByteArray.Data)); break; case IArrayOf> complexShortArray: - WriteComplexValues(writer, DataType.MiInt16, ConvertToPairOfByteArrays(complexShortArray.Data)); + WriteComplexValues(writer, DataType.MiInt16, ConvertToPairOfArrays(complexShortArray.Data)); break; case IArrayOf> complexUshortArray: - WriteComplexValues(writer, DataType.MiUInt16, ConvertToPairOfByteArrays(complexUshortArray.Data)); + WriteComplexValues(writer, DataType.MiUInt16, ConvertToPairOfArrays(complexUshortArray.Data)); break; case IArrayOf> complexIntArray: - WriteComplexValues(writer, DataType.MiInt32, ConvertToPairOfByteArrays(complexIntArray.Data)); + WriteComplexValues(writer, DataType.MiInt32, ConvertToPairOfArrays(complexIntArray.Data)); break; case IArrayOf> complexUintArray: - WriteComplexValues(writer, DataType.MiUInt32, ConvertToPairOfByteArrays(complexUintArray.Data)); + WriteComplexValues(writer, DataType.MiUInt32, ConvertToPairOfArrays(complexUintArray.Data)); break; case IArrayOf> complexLongArray: - WriteComplexValues(writer, DataType.MiInt64, ConvertToPairOfByteArrays(complexLongArray.Data)); + WriteComplexValues(writer, DataType.MiInt64, ConvertToPairOfArrays(complexLongArray.Data)); break; case IArrayOf> complexUlongArray: - WriteComplexValues(writer, DataType.MiUInt64, ConvertToPairOfByteArrays(complexUlongArray.Data)); + WriteComplexValues(writer, DataType.MiUInt64, ConvertToPairOfArrays(complexUlongArray.Data)); break; case IArrayOf> complexFloatArray: - WriteComplexValues(writer, DataType.MiSingle, ConvertToPairOfByteArrays(complexFloatArray.Data)); + WriteComplexValues(writer, DataType.MiSingle, ConvertToPairOfArrays(complexFloatArray.Data)); break; case IArrayOf complexDoubleArray: - WriteComplexValues(writer, DataType.MiDouble, ConvertToPairOfByteArrays(complexDoubleArray.Data)); + WriteComplexValues(writer, DataType.MiDouble, ConvertToPairOfArrays(complexDoubleArray.Data)); break; default: throw new NotSupportedException(); @@ -416,8 +368,7 @@ namespace MatFileHandler WriteArrayFlags(writer, GetCharArrayFlags(isGlobal)); WriteDimensions(writer, charArray.Dimensions); WriteName(writer, name); - var array = charArray.String.ToCharArray().Select(c => (ushort)c).ToArray(); - WriteDataElement(writer, DataType.MiUtf16, ConvertToByteArray(array)); + WriteDataElement(writer, DataType.MiUtf16, charArray.String.ToCharArray()); } private static void WriteCharArray(BinaryWriter writer, ICharArray charArray, string name, bool isGlobal) @@ -431,31 +382,31 @@ namespace MatFileHandler private static void WriteSparseArrayValues( BinaryWriter writer, int[] rows, int[] columns, T[] data) - where T : struct + where T : unmanaged { - WriteDataElement(writer, DataType.MiInt32, ConvertToByteArray(rows)); - WriteDataElement(writer, DataType.MiInt32, ConvertToByteArray(columns)); + WriteDataElement(writer, DataType.MiInt32, rows); + WriteDataElement(writer, DataType.MiInt32, columns); if (data is double[]) { - WriteDataElement(writer, DataType.MiDouble, ConvertToByteArray(data)); + WriteDataElement(writer, DataType.MiDouble, data); } else if (data is Complex[] complexData) { WriteDataElement( writer, DataType.MiDouble, - ConvertToByteArray(complexData.Select(c => c.Real).ToArray())); + complexData.Select(c => c.Real).ToArray()); WriteDataElement( writer, DataType.MiDouble, - ConvertToByteArray(complexData.Select(c => c.Imaginary).ToArray())); + complexData.Select(c => c.Imaginary).ToArray()); } else if (data is bool[] boolData) { WriteDataElement( writer, DataType.MiUInt8, - ConvertToByteArray(boolData)); + boolData); } } @@ -487,7 +438,7 @@ namespace MatFileHandler ISparseArrayOf array, string name, bool isGlobal) - where T : struct, IEquatable + where T : unmanaged, IEquatable { (var rows, var columns, var data, var nonZero) = PrepareSparseArrayData(array); WriteSparseArrayFlags(writer, GetSparseArrayFlags(array, isGlobal, nonZero)); @@ -510,7 +461,7 @@ namespace MatFileHandler { var fieldNamesArray = fieldNames.Select(name => Encoding.ASCII.GetBytes(name)).ToArray(); var maxFieldName = fieldNamesArray.Max(name => name.Length) + 1; - WriteDataElement(writer, DataType.MiInt32, ConvertToByteArray([maxFieldName])); + WriteDataElement(writer, DataType.MiInt32, [maxFieldName]); var buffer = new byte[fieldNamesArray.Length * maxFieldName]; var startPosition = 0; foreach (var name in fieldNamesArray) diff --git a/MatFileHandler/PolyfillExtensions.cs b/MatFileHandler/PolyfillExtensions.cs new file mode 100644 index 0000000..4adebb8 --- /dev/null +++ b/MatFileHandler/PolyfillExtensions.cs @@ -0,0 +1,52 @@ +#if !NET +using System; +using System.Buffers; +using System.IO; + +namespace MatFileHandler +{ + /// + /// Polyfills for methods that do not exist on lower targets. + /// + internal static class PolyfillExtensions + { + public static void ReadExactly(this Stream stream, Span buffer) + { + var array = ArrayPool.Shared.Rent(buffer.Length); + + stream.ReadExactly(array, 0, buffer.Length); + + array.AsSpan(0, buffer.Length).CopyTo(buffer); + + ArrayPool.Shared.Return(array); + } + + public static void ReadExactly(this Stream stream, byte[] buffer, int offset, int count) + { + var totalRead = 0; + + while (totalRead < count) + { + var read = stream.Read(buffer, offset + totalRead, count - totalRead); + if (read == 0) + { + throw new EndOfStreamException(); + } + + totalRead += read; + } + } + + public static void Write(this BinaryWriter writer, ReadOnlySpan buffer) + { + var array = ArrayPool.Shared.Rent(buffer.Length); + + buffer.CopyTo(array); + + writer.Write(array, 0, buffer.Length); + + ArrayPool.Shared.Return(array); + } + } +} +#endif \ No newline at end of file -- 2.45.2