mahalex/MatFileHandler
1
1
Fork 1
Code Issues 13 Pull Requests 1 Actions Packages Projects Releases 9 Wiki Activity

Read/Write bytes directly from data array #41

Open
Rob-Hague wants to merge 1 commits from Rob-Hague/MatFileHandler:span into master
pull from: Rob-Hague/MatFileHandler:span
merge into: mahalex:master
Conversation 0 Commits 1 Files Changed 6 +126 -124
6 changed files with 126 additions and 124 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
MatFileHandler.Tests/MatFileHandler.Tests.csproj
View File

@ -1,6 +1,6 @@
<Project Sdk="Microsoft.NET.Sdk"> <Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup> <PropertyGroup>
<TargetFrameworks>net461;net472;net8.0</TargetFrameworks> <TargetFrameworks>net462;net472;net8.0</TargetFrameworks>
<IsPackable>false</IsPackable> <IsPackable>false</IsPackable>
<LangVersion>latest</LangVersion> <LangVersion>latest</LangVersion>
</PropertyGroup> </PropertyGroup>
@ -10,11 +10,7 @@
<ItemGroup> <ItemGroup>
<PackageReference Include="Microsoft.NET.Test.Sdk" Version="16.10.0" /> <PackageReference Include="Microsoft.NET.Test.Sdk" Version="16.10.0" />
<PackageReference Include="xunit" Version="2.9.0" /> <PackageReference Include="xunit" Version="2.9.0" />
<PackageReference Condition=" '$(TargetFramework)' != 'net461' " Include="xunit.runner.visualstudio" Version="2.8.2"> <PackageReference Include="xunit.runner.visualstudio" Version="2.8.2">
<PrivateAssets>all</PrivateAssets>
<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
</PackageReference>
<PackageReference Condition=" '$(TargetFramework)' == 'net461' " Include="xunit.runner.visualstudio" Version="2.4.3">
<PrivateAssets>all</PrivateAssets> <PrivateAssets>all</PrivateAssets>
<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets> <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
</PackageReference> </PackageReference>

17
MatFileHandler/DataElementReader.cs
View File

@ -1,8 +1,10 @@
using System; using System;
using System.Collections.Generic; using System.Collections.Generic;
using System.Diagnostics;
using System.IO; using System.IO;
using System.IO.Compression; using System.IO.Compression;
using System.Linq; using System.Linq;
using System.Runtime.InteropServices;
using System.Text; using System.Text;
namespace MatFileHandler namespace MatFileHandler
@ -144,17 +146,18 @@ namespace MatFileHandler
return Encoding.ASCII.GetString(element.Data.Select(x => (byte)x).ToArray()); return Encoding.ASCII.GetString(element.Data.Select(x => (byte)x).ToArray());
} }
private static DataElement ReadNum<T>(Tag tag, BinaryReader reader) private static MiNum<T> ReadNum<T>(Tag tag, BinaryReader reader)
where T : struct where T : unmanaged
{ {
var bytes = reader.ReadBytes(tag.Length); T[] result;
if (tag.Type == DataType.MiUInt8)
unsafe
{ {
return new MiNum<byte>(bytes); Debug.Assert(tag.ElementSize == sizeof(T));
result = new T[tag.Length / sizeof(T)];
} }
var result = new T[bytes.Length / tag.ElementSize]; reader.BaseStream.ReadExactly(MemoryMarshal.AsBytes<T>(result));
Buffer.BlockCopy(bytes, 0, result, 0, bytes.Length);
return new MiNum<T>(result); return new MiNum<T>(result);
} }

4
MatFileHandler/Header.cs
View File

@ -2,7 +2,7 @@ using System;
using System.Globalization; using System.Globalization;
using System.IO; using System.IO;
using System.Linq; using System.Linq;
#if !NET461 #if !NET462
using System.Runtime.InteropServices; using System.Runtime.InteropServices;
#endif #endif
@ -79,7 +79,7 @@ namespace MatFileHandler
private static string GetOperatingSystem() private static string GetOperatingSystem()
{ {
#if NET461 #if NET462
return "Windows"; return "Windows";
#else #else
if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows))

4
MatFileHandler/MatFileHandler.csproj
View File

@ -1,6 +1,6 @@
<Project Sdk="Microsoft.NET.Sdk"> <Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup> <PropertyGroup>
<TargetFrameworks>netstandard2.0;net461;net472</TargetFrameworks> <TargetFrameworks>netstandard2.0;net462;net8.0</TargetFrameworks>
<PackageVersion>1.4.0-beta6</PackageVersion> <PackageVersion>1.4.0-beta6</PackageVersion>
<PackageId>MatFileHandler</PackageId> <PackageId>MatFileHandler</PackageId>
<Title>A library for reading and writing MATLAB .mat files.</Title> <Title>A library for reading and writing MATLAB .mat files.</Title>
@ -28,7 +28,7 @@
</PropertyGroup> </PropertyGroup>
<ItemGroup> <ItemGroup>
<PackageReference Include="System.ValueTuple" Version="4.4.0" Condition="'$(TargetFramework)' == 'net461'" /> <PackageReference Include="Microsoft.Bcl.Memory" Version="9.0.7" Condition="!$([MSBuild]::IsTargetFrameworkCompatible('$(TargetFramework)', 'net8.0'))" />
</ItemGroup> </ItemGroup>
<ItemGroup> <ItemGroup>

165
MatFileHandler/MatFileWriter.cs
View File

@ -1,9 +1,12 @@
using System; using System;
using System.Buffers;
using System.Collections.Generic; using System.Collections.Generic;
using System.Diagnostics;
using System.IO; using System.IO;
using System.IO.Compression; using System.IO.Compression;
using System.Linq; using System.Linq;
using System.Numerics; using System.Numerics;
using System.Runtime.InteropServices;
using System.Text; using System.Text;
namespace MatFileHandler namespace MatFileHandler
@ -114,13 +117,21 @@ namespace MatFileHandler
writer.Write((short)tag.Length); writer.Write((short)tag.Length);
} }
private static void WriteDataElement(BinaryWriter writer, DataType type, byte[] data) private static void WriteDataElement<T>(BinaryWriter writer, DataType type, T[] data)
where T : unmanaged
{ {
if (data.Length > 4) unsafe
{ {
WriteTag(writer, new Tag(type, data.Length)); Debug.Assert(type.Size() == sizeof(T));
writer.Write(data); }
var rem = data.Length % 8;
ReadOnlySpan<byte> bytes = MemoryMarshal.AsBytes<T>(data);
if (bytes.Length > 4)
{
WriteTag(writer, new Tag(type, bytes.Length));
writer.Write(bytes);
var rem = bytes.Length % 8;
if (rem > 0) if (rem > 0)
{ {
var padding = new byte[8 - rem]; var padding = new byte[8 - rem];
@ -129,11 +140,11 @@ namespace MatFileHandler
} }
else else
{ {
WriteShortTag(writer, new Tag(type, data.Length)); WriteShortTag(writer, new Tag(type, bytes.Length));
writer.Write(data); writer.Write(bytes);
if (data.Length < 4) if (bytes.Length < 4)
{ {
var padding = new byte[4 - data.Length]; var padding = new byte[4 - bytes.Length];
writer.Write(padding); writer.Write(padding);
} }
} }
@ -141,81 +152,22 @@ namespace MatFileHandler
private static void WriteDimensions(BinaryWriter writer, int[] dimensions) private static void WriteDimensions(BinaryWriter writer, int[] dimensions)
{ {
var buffer = ConvertToByteArray(dimensions); WriteDataElement(writer, DataType.MiInt32, dimensions);
WriteDataElement(writer, DataType.MiInt32, buffer);
} }
private static byte[] ConvertToByteArray<T>(T[] data) private static (T[] real, T[] imaginary) ConvertToPairOfArrays<T>(ComplexOf<T>[] data)
where T : struct where T : struct
{ {
int size; return (data.Select(x => x.Real).ToArray(), data.Select(x => x.Imaginary).ToArray());
if (typeof(T) == typeof(sbyte))
{
size = sizeof(sbyte);
}
else if (typeof(T) == typeof(byte))
{
size = sizeof(byte);
}
else if (typeof(T) == typeof(short))
{
size = sizeof(short);
}
else if (typeof(T) == typeof(ushort))
{
size = sizeof(ushort);
}
else if (typeof(T) == typeof(int))
{
size = sizeof(int);
}
else if (typeof(T) == typeof(uint))
{
size = sizeof(uint);
}
else if (typeof(T) == typeof(long))
{
size = sizeof(long);
}
else if (typeof(T) == typeof(ulong))
{
size = sizeof(ulong);
}
else if (typeof(T) == typeof(float))
{
size = sizeof(float);
}
else if (typeof(T) == typeof(double))
{
size = sizeof(double);
}
else if (typeof(T) == typeof(bool))
{
size = sizeof(bool);
}
else
{
throw new NotSupportedException();
}
var buffer = new byte[data.Length * size];
Buffer.BlockCopy(data, 0, buffer, 0, buffer.Length);
return buffer;
} }
private static (byte[] real, byte[] imaginary) ConvertToPairOfByteArrays<T>(ComplexOf<T>[] data) private static (double[] real, double[] imaginary) ConvertToPairOfArrays(Complex[] data)
where T : struct
{ {
return (ConvertToByteArray(data.Select(x => x.Real).ToArray()), return (data.Select(x => x.Real).ToArray(), data.Select(x => x.Imaginary).ToArray());
ConvertToByteArray(data.Select(x => x.Imaginary).ToArray()));
} }
private static (byte[] real, byte[] imaginary) ConvertToPairOfByteArrays(Complex[] data) private static void WriteComplexValues<T>(BinaryWriter writer, DataType type, (T[] real, T[] complex) data)
{ where T : unmanaged
return (ConvertToByteArray(data.Select(x => x.Real).ToArray()),
ConvertToByteArray(data.Select(x => x.Imaginary).ToArray()));
}
private static void WriteComplexValues(BinaryWriter writer, DataType type, (byte[] real, byte[] complex) data)
{ {
WriteDataElement(writer, type, data.real); WriteDataElement(writer, type, data.real);
WriteDataElement(writer, type, data.complex); WriteDataElement(writer, type, data.complex);
@ -251,67 +203,67 @@ namespace MatFileHandler
switch (value) switch (value)
{ {
case IArrayOf<sbyte> sbyteArray: case IArrayOf<sbyte> sbyteArray:
WriteDataElement(writer, DataType.MiInt8, ConvertToByteArray(sbyteArray.Data)); WriteDataElement(writer, DataType.MiInt8, sbyteArray.Data);
break; break;
case IArrayOf<byte> byteArray: case IArrayOf<byte> byteArray:
WriteDataElement(writer, DataType.MiUInt8, ConvertToByteArray(byteArray.Data)); WriteDataElement(writer, DataType.MiUInt8, byteArray.Data);
break; break;
case IArrayOf<short> shortArray: case IArrayOf<short> shortArray:
WriteDataElement(writer, DataType.MiInt16, ConvertToByteArray(shortArray.Data)); WriteDataElement(writer, DataType.MiInt16, shortArray.Data);
break; break;
case IArrayOf<ushort> ushortArray: case IArrayOf<ushort> ushortArray:
WriteDataElement(writer, DataType.MiUInt16, ConvertToByteArray(ushortArray.Data)); WriteDataElement(writer, DataType.MiUInt16, ushortArray.Data);
break; break;
case IArrayOf<int> intArray: case IArrayOf<int> intArray:
WriteDataElement(writer, DataType.MiInt32, ConvertToByteArray(intArray.Data)); WriteDataElement(writer, DataType.MiInt32, intArray.Data);
break; break;
case IArrayOf<uint> uintArray: case IArrayOf<uint> uintArray:
WriteDataElement(writer, DataType.MiUInt32, ConvertToByteArray(uintArray.Data)); WriteDataElement(writer, DataType.MiUInt32, uintArray.Data);
break; break;
case IArrayOf<long> longArray: case IArrayOf<long> longArray:
WriteDataElement(writer, DataType.MiInt64, ConvertToByteArray(longArray.Data)); WriteDataElement(writer, DataType.MiInt64, longArray.Data);
break; break;
case IArrayOf<ulong> ulongArray: case IArrayOf<ulong> ulongArray:
WriteDataElement(writer, DataType.MiUInt64, ConvertToByteArray(ulongArray.Data)); WriteDataElement(writer, DataType.MiUInt64, ulongArray.Data);
break; break;
case IArrayOf<float> floatArray: case IArrayOf<float> floatArray:
WriteDataElement(writer, DataType.MiSingle, ConvertToByteArray(floatArray.Data)); WriteDataElement(writer, DataType.MiSingle, floatArray.Data);
break; break;
case IArrayOf<double> doubleArray: case IArrayOf<double> doubleArray:
WriteDataElement(writer, DataType.MiDouble, ConvertToByteArray(doubleArray.Data)); WriteDataElement(writer, DataType.MiDouble, doubleArray.Data);
break; break;
case IArrayOf<bool> boolArray: case IArrayOf<bool> boolArray:
WriteDataElement(writer, DataType.MiUInt8, ConvertToByteArray(boolArray.Data)); WriteDataElement(writer, DataType.MiUInt8, boolArray.Data);
break; break;
case IArrayOf<ComplexOf<sbyte>> complexSbyteArray: case IArrayOf<ComplexOf<sbyte>> complexSbyteArray:
WriteComplexValues(writer, DataType.MiInt8, ConvertToPairOfByteArrays(complexSbyteArray.Data)); WriteComplexValues(writer, DataType.MiInt8, ConvertToPairOfArrays(complexSbyteArray.Data));
break; break;
case IArrayOf<ComplexOf<byte>> complexByteArray: case IArrayOf<ComplexOf<byte>> complexByteArray:
WriteComplexValues(writer, DataType.MiUInt8, ConvertToPairOfByteArrays(complexByteArray.Data)); WriteComplexValues(writer, DataType.MiUInt8, ConvertToPairOfArrays(complexByteArray.Data));
break; break;
case IArrayOf<ComplexOf<short>> complexShortArray: case IArrayOf<ComplexOf<short>> complexShortArray:
WriteComplexValues(writer, DataType.MiInt16, ConvertToPairOfByteArrays(complexShortArray.Data)); WriteComplexValues(writer, DataType.MiInt16, ConvertToPairOfArrays(complexShortArray.Data));
break; break;
case IArrayOf<ComplexOf<ushort>> complexUshortArray: case IArrayOf<ComplexOf<ushort>> complexUshortArray:
WriteComplexValues(writer, DataType.MiUInt16, ConvertToPairOfByteArrays(complexUshortArray.Data)); WriteComplexValues(writer, DataType.MiUInt16, ConvertToPairOfArrays(complexUshortArray.Data));
break; break;
case IArrayOf<ComplexOf<int>> complexIntArray: case IArrayOf<ComplexOf<int>> complexIntArray:
WriteComplexValues(writer, DataType.MiInt32, ConvertToPairOfByteArrays(complexIntArray.Data)); WriteComplexValues(writer, DataType.MiInt32, ConvertToPairOfArrays(complexIntArray.Data));
break; break;
case IArrayOf<ComplexOf<uint>> complexUintArray: case IArrayOf<ComplexOf<uint>> complexUintArray:
WriteComplexValues(writer, DataType.MiUInt32, ConvertToPairOfByteArrays(complexUintArray.Data)); WriteComplexValues(writer, DataType.MiUInt32, ConvertToPairOfArrays(complexUintArray.Data));
break; break;
case IArrayOf<ComplexOf<long>> complexLongArray: case IArrayOf<ComplexOf<long>> complexLongArray:
WriteComplexValues(writer, DataType.MiInt64, ConvertToPairOfByteArrays(complexLongArray.Data)); WriteComplexValues(writer, DataType.MiInt64, ConvertToPairOfArrays(complexLongArray.Data));
break; break;
case IArrayOf<ComplexOf<ulong>> complexUlongArray: case IArrayOf<ComplexOf<ulong>> complexUlongArray:
WriteComplexValues(writer, DataType.MiUInt64, ConvertToPairOfByteArrays(complexUlongArray.Data)); WriteComplexValues(writer, DataType.MiUInt64, ConvertToPairOfArrays(complexUlongArray.Data));
break; break;
case IArrayOf<ComplexOf<float>> complexFloatArray: case IArrayOf<ComplexOf<float>> complexFloatArray:
WriteComplexValues(writer, DataType.MiSingle, ConvertToPairOfByteArrays(complexFloatArray.Data)); WriteComplexValues(writer, DataType.MiSingle, ConvertToPairOfArrays(complexFloatArray.Data));
break; break;
case IArrayOf<Complex> complexDoubleArray: case IArrayOf<Complex> complexDoubleArray:
WriteComplexValues(writer, DataType.MiDouble, ConvertToPairOfByteArrays(complexDoubleArray.Data)); WriteComplexValues(writer, DataType.MiDouble, ConvertToPairOfArrays(complexDoubleArray.Data));
break; break;
default: default:
throw new NotSupportedException(); throw new NotSupportedException();
@ -416,8 +368,7 @@ namespace MatFileHandler
WriteArrayFlags(writer, GetCharArrayFlags(isGlobal)); WriteArrayFlags(writer, GetCharArrayFlags(isGlobal));
WriteDimensions(writer, charArray.Dimensions); WriteDimensions(writer, charArray.Dimensions);
WriteName(writer, name); WriteName(writer, name);
var array = charArray.String.ToCharArray().Select(c => (ushort)c).ToArray(); WriteDataElement(writer, DataType.MiUtf16, charArray.String.ToCharArray());
WriteDataElement(writer, DataType.MiUtf16, ConvertToByteArray(array));
} }
private static void WriteCharArray(BinaryWriter writer, ICharArray charArray, string name, bool isGlobal) private static void WriteCharArray(BinaryWriter writer, ICharArray charArray, string name, bool isGlobal)
@ -431,31 +382,31 @@ namespace MatFileHandler
private static void WriteSparseArrayValues<T>( private static void WriteSparseArrayValues<T>(
BinaryWriter writer, int[] rows, int[] columns, T[] data) BinaryWriter writer, int[] rows, int[] columns, T[] data)
where T : struct where T : unmanaged
{ {
WriteDataElement(writer, DataType.MiInt32, ConvertToByteArray(rows)); WriteDataElement(writer, DataType.MiInt32, rows);
WriteDataElement(writer, DataType.MiInt32, ConvertToByteArray(columns)); WriteDataElement(writer, DataType.MiInt32, columns);
if (data is double[]) if (data is double[])
{ {
WriteDataElement(writer, DataType.MiDouble, ConvertToByteArray(data)); WriteDataElement(writer, DataType.MiDouble, data);
} }
else if (data is Complex[] complexData) else if (data is Complex[] complexData)
{ {
WriteDataElement( WriteDataElement(
writer, writer,
DataType.MiDouble, DataType.MiDouble,
ConvertToByteArray(complexData.Select(c => c.Real).ToArray())); complexData.Select(c => c.Real).ToArray());
WriteDataElement( WriteDataElement(
writer, writer,
DataType.MiDouble, DataType.MiDouble,
ConvertToByteArray(complexData.Select(c => c.Imaginary).ToArray())); complexData.Select(c => c.Imaginary).ToArray());
} }
else if (data is bool[] boolData) else if (data is bool[] boolData)
{ {
WriteDataElement( WriteDataElement(
writer, writer,
DataType.MiUInt8, DataType.MiUInt8,
ConvertToByteArray(boolData)); boolData);
} }
} }
@ -487,7 +438,7 @@ namespace MatFileHandler
ISparseArrayOf<T> array, ISparseArrayOf<T> array,
string name, string name,
bool isGlobal) bool isGlobal)
where T : struct, IEquatable<T> where T : unmanaged, IEquatable<T>
{ {
(var rows, var columns, var data, var nonZero) = PrepareSparseArrayData(array); (var rows, var columns, var data, var nonZero) = PrepareSparseArrayData(array);
WriteSparseArrayFlags(writer, GetSparseArrayFlags(array, isGlobal, nonZero)); WriteSparseArrayFlags(writer, GetSparseArrayFlags(array, isGlobal, nonZero));
@ -510,7 +461,7 @@ namespace MatFileHandler
{ {
var fieldNamesArray = fieldNames.Select(name => Encoding.ASCII.GetBytes(name)).ToArray(); var fieldNamesArray = fieldNames.Select(name => Encoding.ASCII.GetBytes(name)).ToArray();
var maxFieldName = fieldNamesArray.Max(name => name.Length) + 1; var maxFieldName = fieldNamesArray.Max(name => name.Length) + 1;
WriteDataElement(writer, DataType.MiInt32, ConvertToByteArray([maxFieldName])); WriteDataElement(writer, DataType.MiInt32, [maxFieldName]);
var buffer = new byte[fieldNamesArray.Length * maxFieldName]; var buffer = new byte[fieldNamesArray.Length * maxFieldName];
var startPosition = 0; var startPosition = 0;
foreach (var name in fieldNamesArray) foreach (var name in fieldNamesArray)

52
MatFileHandler/PolyfillExtensions.cs Normal file
View File

@ -0,0 +1,52 @@
#if !NET
using System;
using System.Buffers;
using System.IO;
namespace MatFileHandler
{
/// <summary>
/// Polyfills for methods that do not exist on lower targets.
/// </summary>
internal static class PolyfillExtensions
{
public static void ReadExactly(this Stream stream, Span<byte> buffer)
{
var array = ArrayPool<byte>.Shared.Rent(buffer.Length);
stream.ReadExactly(array, 0, buffer.Length);
array.AsSpan(0, buffer.Length).CopyTo(buffer);
ArrayPool<byte>.Shared.Return(array);
}
public static void ReadExactly(this Stream stream, byte[] buffer, int offset, int count)
{
var totalRead = 0;
while (totalRead < count)
{
var read = stream.Read(buffer, offset + totalRead, count - totalRead);
if (read == 0)
{
throw new EndOfStreamException();
}
totalRead += read;
}
}
public static void Write(this BinaryWriter writer, ReadOnlySpan<byte> buffer)
{
var array = ArrayPool<byte>.Shared.Rent(buffer.Length);
buffer.CopyTo(array);
writer.Write(array, 0, buffer.Length);
ArrayPool<byte>.Shared.Return(array);
}
}
}
#endif