From e9582723c9791bae5387c0537c2720f3c4cb1485 Mon Sep 17 00:00:00 2001 From: Alexander Luzgarev Date: Sun, 6 Apr 2025 12:27:40 +0200 Subject: [PATCH] Reduce memory consumption of MatFileWriter --- .../ChecksumCalculatingStreamTests.cs | 106 +++++ .../MatFileHandler.Tests.csproj | 1 + MatFileHandler.Tests/MatFileWriterTests.cs | 122 +++--- MatFileHandler.Tests/MatFileWritingMethod.cs | 17 + .../MatFileWritingToMemoryStream.cs | 37 ++ .../MatFileWritingToUnalignedMemoryStream.cs | 43 ++ .../MatFileWritingToUnseekableStream.cs | 38 ++ .../PartialUnseekableReadStream.cs | 3 +- MatFileHandler.Tests/UnseekableWriteStream.cs | 70 +++ MatFileHandler/ChecksumCalculatingStream.cs | 92 ++++ MatFileHandler/FakeWriter.cs | 398 ++++++++++++++++++ MatFileHandler/MatFileHandler.csproj | 4 + MatFileHandler/MatFileWriter.cs | 74 +++- 13 files changed, 928 insertions(+), 77 deletions(-) create mode 100644 MatFileHandler.Tests/ChecksumCalculatingStreamTests.cs create mode 100644 MatFileHandler.Tests/MatFileWritingMethod.cs create mode 100644 MatFileHandler.Tests/MatFileWritingToMemoryStream.cs create mode 100644 MatFileHandler.Tests/MatFileWritingToUnalignedMemoryStream.cs create mode 100644 MatFileHandler.Tests/MatFileWritingToUnseekableStream.cs create mode 100644 MatFileHandler.Tests/UnseekableWriteStream.cs create mode 100644 MatFileHandler/ChecksumCalculatingStream.cs create mode 100644 MatFileHandler/FakeWriter.cs diff --git a/MatFileHandler.Tests/ChecksumCalculatingStreamTests.cs b/MatFileHandler.Tests/ChecksumCalculatingStreamTests.cs new file mode 100644 index 0000000..cdf86ab --- /dev/null +++ b/MatFileHandler.Tests/ChecksumCalculatingStreamTests.cs @@ -0,0 +1,106 @@ +// Copyright 2017-2018 Alexander Luzgarev + +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Text; +using Xunit; + +namespace MatFileHandler.Tests +{ + /// + /// Tests for the class. + /// + public class ChecksumCalculatingStreamTests + { + /// + /// Test writing various things. + /// + /// + [Theory] + [MemberData(nameof(TestData))] + public void Test(Action action) + { + using var stream = new MemoryStream(); + var sut = new ChecksumCalculatingStream(stream); + action(sut); + var actual = sut.GetCrc(); + var expected = ReferenceCalculation(action); + } + + /// + /// Test data for . + /// + /// Test data. + public static IEnumerable TestData() + { + foreach (var data in TestData_Typed()) + { + yield return new object[] { data }; + } + } + + private static IEnumerable> TestData_Typed() + { + yield return BinaryWriterAction(w => w.Write(true)); + yield return BinaryWriterAction(w => w.Write(false)); + yield return BinaryWriterAction(w => w.Write(byte.MinValue)); + yield return BinaryWriterAction(w => w.Write(byte.MaxValue)); + yield return BinaryWriterAction(w => w.Write(short.MinValue)); + yield return BinaryWriterAction(w => w.Write(short.MaxValue)); + yield return BinaryWriterAction(w => w.Write(int.MinValue)); + yield return BinaryWriterAction(w => w.Write(int.MaxValue)); + yield return BinaryWriterAction(w => w.Write(long.MinValue)); + yield return BinaryWriterAction(w => w.Write(long.MaxValue)); + yield return BinaryWriterAction(w => w.Write(decimal.MinValue)); + yield return BinaryWriterAction(w => w.Write(decimal.MaxValue)); + yield return BinaryWriterAction(w => w.Write(double.MinValue)); + yield return BinaryWriterAction(w => w.Write(double.MaxValue)); + yield return BinaryWriterAction(w => w.Write(double.PositiveInfinity)); + yield return BinaryWriterAction(w => w.Write(double.NaN)); + yield return BinaryWriterAction(w => w.Write(new byte[] { 1, 2, 3, 4, 5, 6, 7 })); + yield return BinaryWriterAction(w => w.Write(Enumerable.Range(0, 255).SelectMany(x => Enumerable.Range(0, 255)).Select(x => (byte)x).ToArray())); + } + + private static Action BinaryWriterAction(Action action) + { + return stream => + { + using var writer = new BinaryWriter(stream, Encoding.UTF8, leaveOpen: true); + action(writer); + }; + } + + private uint ReferenceCalculation(Action action) + { + using var stream = new MemoryStream(); + action(stream); + stream.Position = 0; + return CalculateAdler32Checksum(stream); + } + + private static uint CalculateAdler32Checksum(Stream stream) + { + uint s1 = 1; + uint s2 = 0; + const uint bigPrime = 0xFFF1; + const int bufferSize = 2048; + var buffer = new byte[bufferSize]; + while (true) + { + var bytesRead = stream.Read(buffer, 0, bufferSize); + for (var i = 0; i < bytesRead; i++) + { + s1 = (s1 + buffer[i]) % bigPrime; + s2 = (s2 + s1) % bigPrime; + } + if (bytesRead < bufferSize) + { + break; + } + } + return (s2 << 16) | s1; + } + } +} diff --git a/MatFileHandler.Tests/MatFileHandler.Tests.csproj b/MatFileHandler.Tests/MatFileHandler.Tests.csproj index 3f44415..2bab9c1 100755 --- a/MatFileHandler.Tests/MatFileHandler.Tests.csproj +++ b/MatFileHandler.Tests/MatFileHandler.Tests.csproj @@ -2,6 +2,7 @@ net8.0;net472 false + 10.0 ..\MatFileHandler.ruleset diff --git a/MatFileHandler.Tests/MatFileWriterTests.cs b/MatFileHandler.Tests/MatFileWriterTests.cs index 926ce4c..a2da405 100755 --- a/MatFileHandler.Tests/MatFileWriterTests.cs +++ b/MatFileHandler.Tests/MatFileWriterTests.cs @@ -17,8 +17,8 @@ namespace MatFileHandler.Tests /// /// Test writing a simple Double array. /// - [Fact] - public void TestWrite() + [Theory, MemberData(nameof(MatFileWritingMethods))] + public void TestWrite(MatFileWritingMethod method) { var builder = new DataBuilder(); var array = builder.NewArray(1, 2); @@ -26,7 +26,7 @@ namespace MatFileHandler.Tests array[1] = 17.0; var variable = builder.NewVariable("test", array); var actual = builder.NewFile(new[] { variable }); - MatCompareWithTestData("good", "double-array", actual); + MatCompareWithTestData("good", "double-array", actual, method); } /// @@ -51,8 +51,8 @@ namespace MatFileHandler.Tests /// /// Test writing lower and upper limits of integer data types. /// - [Fact] - public void TestLimits() + [Theory, MemberData(nameof(MatFileWritingMethods))] + public void TestLimits(MatFileWritingMethod method) { var builder = new DataBuilder(); var int8 = builder.NewVariable("int8_", builder.NewArray(CommonData.Int8Limits, 1, 2)); @@ -64,14 +64,14 @@ namespace MatFileHandler.Tests var int64 = builder.NewVariable("int64_", builder.NewArray(CommonData.Int64Limits, 1, 2)); var uint64 = builder.NewVariable("uint64_", builder.NewArray(CommonData.UInt64Limits, 1, 2)); var actual = builder.NewFile(new[] { int16, int32, int64, int8, uint16, uint32, uint64, uint8 }); - MatCompareWithTestData("good", "limits", actual); + MatCompareWithTestData("good", "limits", actual, method); } /// /// Test writing lower and upper limits of integer-based complex data types. /// - [Fact] - public void TestLimitsComplex() + [Theory, MemberData(nameof(MatFileWritingMethods))] + public void TestLimitsComplex(MatFileWritingMethod method) { var builder = new DataBuilder(); var int8Complex = builder.NewVariable( @@ -103,26 +103,26 @@ namespace MatFileHandler.Tests int16Complex, int32Complex, int64Complex, int8Complex, uint16Complex, uint32Complex, uint64Complex, uint8Complex, }); - MatCompareWithTestData("good", "limits_complex", actual); + MatCompareWithTestData("good", "limits_complex", actual, method); } /// /// Test writing a wide-Unicode symbol. /// - [Fact] - public void TestUnicodeWide() + [Theory, MemberData(nameof(MatFileWritingMethods))] + public void TestUnicodeWide(MatFileWritingMethod method) { var builder = new DataBuilder(); var s = builder.NewVariable("s", builder.NewCharArray("🍆")); var actual = builder.NewFile(new[] { s }); - MatCompareWithTestData("good", "unicode-wide", actual); + MatCompareWithTestData("good", "unicode-wide", actual, method); } /// /// Test writing a sparse array. /// - [Fact] - public void TestSparseArray() + [Theory, MemberData(nameof(MatFileWritingMethods))] + public void TestSparseArray(MatFileWritingMethod method) { var builder = new DataBuilder(); var sparseArray = builder.NewSparseArray(4, 5); @@ -132,14 +132,14 @@ namespace MatFileHandler.Tests sparseArray[2, 3] = 4; var sparse = builder.NewVariable("sparse_", sparseArray); var actual = builder.NewFile(new[] { sparse }); - MatCompareWithTestData("good", "sparse", actual); + MatCompareWithTestData("good", "sparse", actual, method); } /// /// Test writing a structure array. /// - [Fact] - public void TestStructure() + [Theory, MemberData(nameof(MatFileWritingMethods))] + public void TestStructure(MatFileWritingMethod method) { var builder = new DataBuilder(); var structure = builder.NewStructureArray(new[] { "x", "y" }, 2, 3); @@ -160,27 +160,27 @@ namespace MatFileHandler.Tests structure["y", 1, 2] = builder.NewEmpty(); var struct_ = builder.NewVariable("struct_", structure); var actual = builder.NewFile(new[] { struct_ }); - MatCompareWithTestData("good", "struct", actual); + MatCompareWithTestData("good", "struct", actual, method); } /// /// Test writing a logical array. /// - [Fact] - public void TestLogical() + [Theory, MemberData(nameof(MatFileWritingMethods))] + public void TestLogical(MatFileWritingMethod method) { var builder = new DataBuilder(); var logical = builder.NewArray(new[] { true, false, true, true, false, true }, 2, 3); var logicalVariable = builder.NewVariable("logical_", logical); var actual = builder.NewFile(new[] { logicalVariable }); - MatCompareWithTestData("good", "logical", actual); + MatCompareWithTestData("good", "logical", actual, method); } /// /// Test writing a sparse logical array. /// - [Fact] - public void TestSparseLogical() + [Theory, MemberData(nameof(MatFileWritingMethods))] + public void TestSparseLogical(MatFileWritingMethod method) { var builder = new DataBuilder(); var array = builder.NewSparseArray(2, 3); @@ -190,14 +190,14 @@ namespace MatFileHandler.Tests array[1, 2] = true; var sparseLogical = builder.NewVariable("sparse_logical", array); var actual = builder.NewFile(new[] { sparseLogical }); - MatCompareWithTestData("good", "sparse_logical", actual); + MatCompareWithTestData("good", "sparse_logical", actual, method); } /// /// Test writing a sparse complex array. /// - [Fact] - public void TestSparseComplex() + [Theory, MemberData(nameof(MatFileWritingMethods))] + public void TestSparseComplex(MatFileWritingMethod method) { var builder = new DataBuilder(); var array = builder.NewSparseArray(2, 2); @@ -206,20 +206,42 @@ namespace MatFileHandler.Tests array[1, 1] = 0.5 + Complex.ImaginaryOne; var sparseComplex = builder.NewVariable("sparse_complex", array); var actual = builder.NewFile(new[] { sparseComplex }); - MatCompareWithTestData("good", "sparse_complex", actual); + MatCompareWithTestData("good", "sparse_complex", actual, method); } /// /// Test writing a global variable. /// - [Fact] - public void TestGlobal() + [Theory, MemberData(nameof(MatFileWritingMethods))] + public void TestGlobal(MatFileWritingMethod method) { var builder = new DataBuilder(); var array = builder.NewArray(new double[] { 1, 3, 5 }, 1, 3); var global = builder.NewVariable("global_", array, true); var actual = builder.NewFile(new[] { global }); - MatCompareWithTestData("good", "global", actual); + MatCompareWithTestData("good", "global", actual, method); + } + + /// + /// Various writing methods for testing writing of .mat files. + /// + public static TheoryData MatFileWritingMethods + { + get + { + return new TheoryData + { + new MatFileWritingToMemoryStream(null), + new MatFileWritingToMemoryStream(new MatFileWriterOptions { UseCompression = CompressionUsage.Always }), + new MatFileWritingToMemoryStream(new MatFileWriterOptions { UseCompression = CompressionUsage.Never }), + new MatFileWritingToUnseekableStream(null), + new MatFileWritingToUnseekableStream(new MatFileWriterOptions { UseCompression = CompressionUsage.Always }), + new MatFileWritingToUnseekableStream(new MatFileWriterOptions { UseCompression = CompressionUsage.Never }), + new MatFileWritingToUnalignedMemoryStream(null), + new MatFileWritingToUnalignedMemoryStream(new MatFileWriterOptions { UseCompression = CompressionUsage.Always }), + new MatFileWritingToUnalignedMemoryStream(new MatFileWriterOptions { UseCompression = CompressionUsage.Never }), + }; + } } private static AbstractTestDataFactory GetMatTestData(string factoryName) => @@ -375,40 +397,18 @@ namespace MatFileHandler.Tests } } - private void CompareTestDataWithWritingOptions( - IMatFile expected, + private void MatCompareWithTestData( + string factoryName, + string testName, IMatFile actual, - MatFileWriterOptions? maybeOptions) - { - byte[] buffer; - using (var stream = new MemoryStream()) - { - var writer = maybeOptions is MatFileWriterOptions options - ? new MatFileWriter(stream, options) - : new MatFileWriter(stream); - writer.Write(actual); - buffer = stream.ToArray(); - } - using (var stream = new MemoryStream(buffer)) - { - var reader = new MatFileReader(stream); - var actualRead = reader.Read(); - CompareMatFiles(expected, actualRead); - } - } - - private void MatCompareWithTestData(string factoryName, string testName, IMatFile actual) + MatFileWritingMethod method) { var expected = GetMatTestData(factoryName)[testName]; - CompareTestDataWithWritingOptions(expected, actual, null); - CompareTestDataWithWritingOptions( - expected, - actual, - new MatFileWriterOptions { UseCompression = CompressionUsage.Always }); - CompareTestDataWithWritingOptions( - expected, - actual, - new MatFileWriterOptions { UseCompression = CompressionUsage.Never }); + var buffer = method.WriteMatFile(actual); + using var stream = new MemoryStream(buffer); + var reader = new MatFileReader(stream); + var actualRead = reader.Read(); + CompareMatFiles(expected, actualRead); } private ComplexOf[] CreateComplexLimits(T[] limits) diff --git a/MatFileHandler.Tests/MatFileWritingMethod.cs b/MatFileHandler.Tests/MatFileWritingMethod.cs new file mode 100644 index 0000000..b5ba772 --- /dev/null +++ b/MatFileHandler.Tests/MatFileWritingMethod.cs @@ -0,0 +1,17 @@ +// Copyright 2017-2018 Alexander Luzgarev + +namespace MatFileHandler.Tests +{ + /// + /// A method of writing IMatFile into a byte buffer. + /// + public abstract class MatFileWritingMethod + { + /// + /// Write an IMatFile into a byte buffer. + /// + /// + /// + public abstract byte[] WriteMatFile(IMatFile matFile); + } +} \ No newline at end of file diff --git a/MatFileHandler.Tests/MatFileWritingToMemoryStream.cs b/MatFileHandler.Tests/MatFileWritingToMemoryStream.cs new file mode 100644 index 0000000..5efb26a --- /dev/null +++ b/MatFileHandler.Tests/MatFileWritingToMemoryStream.cs @@ -0,0 +1,37 @@ +// Copyright 2017-2018 Alexander Luzgarev + +using System.IO; + +namespace MatFileHandler.Tests +{ + /// + /// A method of writing an IMatFile into a MemoryStream. + /// + public class MatFileWritingToMemoryStream : MatFileWritingMethod + { + private readonly MatFileWriterOptions? _maybeOptions; + + /// + /// Initializes a new instance of the class. + /// + /// Options for the . + public MatFileWritingToMemoryStream(MatFileWriterOptions? maybeOptions) + { + _maybeOptions = maybeOptions; + } + + /// + public override byte[] WriteMatFile(IMatFile matFile) + { + using var memoryStream = new MemoryStream(); + var matFileWriter = _maybeOptions switch + { + { } options => new MatFileWriter(memoryStream, options), + _ => new MatFileWriter(memoryStream), + }; + + matFileWriter.Write(matFile); + return memoryStream.ToArray(); + } + } +} \ No newline at end of file diff --git a/MatFileHandler.Tests/MatFileWritingToUnalignedMemoryStream.cs b/MatFileHandler.Tests/MatFileWritingToUnalignedMemoryStream.cs new file mode 100644 index 0000000..a419974 --- /dev/null +++ b/MatFileHandler.Tests/MatFileWritingToUnalignedMemoryStream.cs @@ -0,0 +1,43 @@ +// Copyright 2017-2018 Alexander Luzgarev + +using System; +using System.IO; + +namespace MatFileHandler.Tests +{ + /// + /// A method of writing an IMatFile into a stream that is "unaligned". + /// + public class MatFileWritingToUnalignedMemoryStream : MatFileWritingMethod + { + private readonly MatFileWriterOptions? _maybeOptions; + + /// + /// Initializes a new instance of the class. + /// + /// Options for the . + public MatFileWritingToUnalignedMemoryStream(MatFileWriterOptions? maybeOptions) + { + _maybeOptions = maybeOptions; + } + + /// + public override byte[] WriteMatFile(IMatFile matFile) + { + using var memoryStream = new MemoryStream(); + memoryStream.Seek(3, SeekOrigin.Begin); + var matFileWriter = _maybeOptions switch + { + { } options => new MatFileWriter(memoryStream, options), + _ => new MatFileWriter(memoryStream), + }; + + matFileWriter.Write(matFile); + var fullArray = memoryStream.ToArray(); + var length = fullArray.Length - 3; + var result = new byte[length]; + Buffer.BlockCopy(fullArray, 3, result, 0, length); + return result; + } + } +} \ No newline at end of file diff --git a/MatFileHandler.Tests/MatFileWritingToUnseekableStream.cs b/MatFileHandler.Tests/MatFileWritingToUnseekableStream.cs new file mode 100644 index 0000000..1a2c2d8 --- /dev/null +++ b/MatFileHandler.Tests/MatFileWritingToUnseekableStream.cs @@ -0,0 +1,38 @@ +// Copyright 2017-2018 Alexander Luzgarev + +using System.IO; + +namespace MatFileHandler.Tests +{ + /// + /// A method of writing an IMatFile into a stream that is not seekable. + /// + public class MatFileWritingToUnseekableStream : MatFileWritingMethod + { + private readonly MatFileWriterOptions? _maybeOptions; + + /// + /// Initializes a new instance of the class. + /// + /// Options for the . + public MatFileWritingToUnseekableStream(MatFileWriterOptions? maybeOptions) + { + _maybeOptions = maybeOptions; + } + + /// + public override byte[] WriteMatFile(IMatFile matFile) + { + using var memoryStream = new MemoryStream(); + using var unseekableStream = new UnseekableWriteStream(memoryStream); + var matFileWriter = _maybeOptions switch + { + { } options => new MatFileWriter(unseekableStream, options), + _ => new MatFileWriter(unseekableStream), + }; + + matFileWriter.Write(matFile); + return memoryStream.ToArray(); + } + } +} \ No newline at end of file diff --git a/MatFileHandler.Tests/PartialUnseekableReadStream.cs b/MatFileHandler.Tests/PartialUnseekableReadStream.cs index ce828a0..a78104c 100644 --- a/MatFileHandler.Tests/PartialUnseekableReadStream.cs +++ b/MatFileHandler.Tests/PartialUnseekableReadStream.cs @@ -6,7 +6,8 @@ using System.IO; namespace MatFileHandler.Tests { /// - /// A stream which wraps another stream and only reads one byte at a time. + /// A stream which wraps another stream and only reads one byte at a time, + /// while forbidding seeking in it. /// internal class PartialUnseekableReadStream : Stream { diff --git a/MatFileHandler.Tests/UnseekableWriteStream.cs b/MatFileHandler.Tests/UnseekableWriteStream.cs new file mode 100644 index 0000000..2133768 --- /dev/null +++ b/MatFileHandler.Tests/UnseekableWriteStream.cs @@ -0,0 +1,70 @@ +// Copyright 2017-2018 Alexander Luzgarev + +using System; +using System.IO; + +namespace MatFileHandler.Tests +{ + /// + /// A stream which wraps another stream and forbids seeking in it. + /// + internal class UnseekableWriteStream : Stream + { + public UnseekableWriteStream(Stream baseStream) + { + _baseStream = baseStream; + } + + private readonly Stream _baseStream; + + public override bool CanRead => false; + + public override bool CanSeek => false; + + public override bool CanWrite => _baseStream.CanWrite; + + public override long Length => _baseStream.Length; + + public override long Position + { + get => throw new NotSupportedException(); + set => throw new NotSupportedException(); + } + + public override void Flush() + { + _baseStream.Flush(); + } + + public override int Read(byte[] buffer, int offset, int count) + { + throw new NotImplementedException(); + } + + public override long Seek(long offset, SeekOrigin origin) + { + throw new NotSupportedException(); + } + + public override void SetLength(long value) + { + throw new NotSupportedException(); + } + + public override void Write(byte[] buffer, int offset, int count) + { + _baseStream.Write(buffer, offset, count); + } + + /// + protected override void Dispose(bool disposing) + { + if (disposing) + { + _baseStream.Dispose(); + } + + base.Dispose(disposing); + } + } +} diff --git a/MatFileHandler/ChecksumCalculatingStream.cs b/MatFileHandler/ChecksumCalculatingStream.cs new file mode 100644 index 0000000..e0278f0 --- /dev/null +++ b/MatFileHandler/ChecksumCalculatingStream.cs @@ -0,0 +1,92 @@ +// Copyright 2017-2018 Alexander Luzgarev + +using System; +using System.IO; + +namespace MatFileHandler +{ + /// + /// A stream that calculates Adler32 checksum of everything + /// written to it before passing to another stream. + /// + internal class ChecksumCalculatingStream : Stream + { + private const uint BigPrime = 0xFFF1; + private readonly Stream _stream; + private uint s1 = 1; + private uint s2 = 0; + + /// + /// Initializes a new instance of the class. + /// + /// Wrapped stream. + public ChecksumCalculatingStream(Stream stream) + { + _stream = stream; + } + + /// + public override bool CanRead => false; + + /// + public override bool CanSeek => false; + + /// + public override bool CanWrite => true; + + /// + public override long Length => throw new NotImplementedException(); + + /// + public override long Position + { + get => throw new NotImplementedException(); + set => throw new NotImplementedException(); + } + + /// + public override void Flush() + { + _stream.Flush(); + } + + /// + public override int Read(byte[] buffer, int offset, int count) + { + throw new NotImplementedException(); + } + + /// + public override long Seek(long offset, SeekOrigin origin) + { + throw new NotImplementedException(); + } + + /// + public override void SetLength(long value) + { + throw new NotImplementedException(); + } + + /// + public override void Write(byte[] buffer, int offset, int count) + { + for (var i = offset; i < offset + count; i++) + { + s1 = (s1 + buffer[i]) % BigPrime; + s2 = (s2 + s1) % BigPrime; + } + + _stream.Write(buffer, offset, count); + } + + /// + /// Calculate the checksum of everything written to the stream so far. + /// + /// Checksum of everything written to the stream so far. + public uint GetCrc() + { + return (s2 << 16) | s1; + } + } +} \ No newline at end of file diff --git a/MatFileHandler/FakeWriter.cs b/MatFileHandler/FakeWriter.cs new file mode 100644 index 0000000..6e83fdf --- /dev/null +++ b/MatFileHandler/FakeWriter.cs @@ -0,0 +1,398 @@ +// Copyright 2017-2018 Alexander Luzgarev + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Numerics; +using System.Text; + +namespace MatFileHandler +{ + /// + /// A simulated writer of .mat files that just calculate the length of data that would be written. + /// + internal class FakeWriter + { + /// + /// Gets current position of the writer. + /// + public int Position { get; private set; } + + /// + /// Write contents of a numerical array. + /// + /// A numerical array. + /// Name of the array. + public void WriteNumericalArrayContents(IArray array, string name) + { + WriteArrayFlags(); + WriteDimensions(array.Dimensions); + WriteName(name); + WriteNumericalArrayValues(array); + } + + /// + /// Write contents of a char array. + /// + /// A char array. + /// Name of the array. + public void WriteCharArrayContents(ICharArray charArray, string name) + { + WriteArrayFlags(); + WriteDimensions(charArray.Dimensions); + WriteName(name); + WriteDataElement(GetLengthOfByteArray(charArray.String.Length)); + } + + /// + /// Write contents of a sparse array. + /// + /// Array element type. + /// A sparse array. + /// Name of the array. + public void WriteSparseArrayContents( + ISparseArrayOf array, + string name) + where T : unmanaged, IEquatable + { + (var rowsLength, var columnsLength, var dataLength, var nonZero) = PrepareSparseArrayData(array); + WriteSparseArrayFlags(); + WriteDimensions(array.Dimensions); + WriteName(name); + WriteSparseArrayValues(rowsLength, columnsLength, dataLength); + } + + /// + /// Write contents of a structure array. + /// + /// A structure array. + /// Name of the array. + public void WriteStructureArrayContents(IStructureArray array, string name) + { + WriteArrayFlags(); + WriteDimensions(array.Dimensions); + WriteName(name); + WriteFieldNames(array.FieldNames); + WriteStructureArrayValues(array); + } + + /// + /// Write contents of a cell array. + /// + /// A cell array. + /// Name of the array. + public void WriteCellArrayContents(ICellArray array, string name) + { + WriteArrayFlags(); + WriteDimensions(array.Dimensions); + WriteName(name); + WriteCellArrayValues(array); + } + + private void WriteTag() + { + Position += 8; + } + + private void WriteShortTag() + { + Position += 4; + } + + private void WriteWrappingContents(T array, Action writeContents) + where T : IArray + { + if (array.IsEmpty) + { + WriteTag(); + return; + } + + WriteTag(); + writeContents(this); + } + + private void WriteNumericalArrayValues(IArray value) + { + switch (value) + { + case IArrayOf sbyteArray: + WriteDataElement(GetLengthOfByteArray(sbyteArray.Data.Length)); + break; + case IArrayOf byteArray: + WriteDataElement(GetLengthOfByteArray(byteArray.Data.Length)); + break; + case IArrayOf shortArray: + WriteDataElement(GetLengthOfByteArray(shortArray.Data.Length)); + break; + case IArrayOf ushortArray: + WriteDataElement(GetLengthOfByteArray(ushortArray.Data.Length)); + break; + case IArrayOf intArray: + WriteDataElement(GetLengthOfByteArray(intArray.Data.Length)); + break; + case IArrayOf uintArray: + WriteDataElement(GetLengthOfByteArray(uintArray.Data.Length)); + break; + case IArrayOf longArray: + WriteDataElement(GetLengthOfByteArray(longArray.Data.Length)); + break; + case IArrayOf ulongArray: + WriteDataElement(GetLengthOfByteArray(ulongArray.Data.Length)); + break; + case IArrayOf floatArray: + WriteDataElement(GetLengthOfByteArray(floatArray.Data.Length)); + break; + case IArrayOf doubleArray: + WriteDataElement(GetLengthOfByteArray(doubleArray.Data.Length)); + break; + case IArrayOf boolArray: + WriteDataElement(boolArray.Data.Length); + break; + case IArrayOf> complexSbyteArray: + WriteComplexValues(GetLengthOfPairOfByteArrays(complexSbyteArray.Data)); + break; + case IArrayOf> complexByteArray: + WriteComplexValues(GetLengthOfPairOfByteArrays(complexByteArray.Data)); + break; + case IArrayOf> complexShortArray: + WriteComplexValues(GetLengthOfPairOfByteArrays(complexShortArray.Data)); + break; + case IArrayOf> complexUshortArray: + WriteComplexValues(GetLengthOfPairOfByteArrays(complexUshortArray.Data)); + break; + case IArrayOf> complexIntArray: + WriteComplexValues(GetLengthOfPairOfByteArrays(complexIntArray.Data)); + break; + case IArrayOf> complexUintArray: + WriteComplexValues(GetLengthOfPairOfByteArrays(complexUintArray.Data)); + break; + case IArrayOf> complexLongArray: + WriteComplexValues(GetLengthOfPairOfByteArrays(complexLongArray.Data)); + break; + case IArrayOf> complexUlongArray: + WriteComplexValues(GetLengthOfPairOfByteArrays(complexUlongArray.Data)); + break; + case IArrayOf> complexFloatArray: + WriteComplexValues(GetLengthOfPairOfByteArrays(complexFloatArray.Data)); + break; + case IArrayOf complexDoubleArray: + WriteComplexValues(GetLengthOfPairOfByteArrays(complexDoubleArray.Data)); + break; + default: + throw new NotSupportedException(); + } + } + + private void WriteName(string name) + { + var nameBytes = Encoding.ASCII.GetBytes(name); + WriteDataElement(nameBytes.Length); + } + + private void WriteArrayFlags() + { + WriteTag(); + Position += 8; + } + + private void WriteDimensions(int[] dimensions) + { + var buffer = GetLengthOfByteArray(dimensions.Length); + WriteDataElement(buffer); + } + + private unsafe int GetLengthOfByteArray(int dataLength) + where T : unmanaged + { + return dataLength * sizeof(T); + } + + private unsafe int GetLengthOfPairOfByteArrays(ComplexOf[] data) + where T : unmanaged + { + return data.Length * sizeof(T); + } + + private unsafe int GetLengthOfPairOfByteArrays(Complex[] data) + { + return data.Length * sizeof(double); + } + + private int CalculatePadding(int length) + { + var rem = length % 8; + if (rem == 0) + { + return 0; + } + + return 8 - rem; + } + + private void WriteDataElement(int dataLength) + { + var maybePadding = 0; + if (dataLength > 4) + { + WriteTag(); + Position += dataLength; + maybePadding = CalculatePadding(dataLength + 8); + } + else + { + WriteShortTag(); + Position += 4; + } + + Position += maybePadding; + } + + private void WriteComplexValues( + int dataLength) + { + WriteDataElement(dataLength); + WriteDataElement(dataLength); + } + + private void WriteSparseArrayValues(int rowsLength, int columnsLength, int dataLength) + where T : unmanaged + { + WriteDataElement(GetLengthOfByteArray(rowsLength)); + WriteDataElement(GetLengthOfByteArray(columnsLength)); + if (typeof(T) == typeof(double)) + { + WriteDataElement(GetLengthOfByteArray(dataLength)); + } + else if (typeof(T) == typeof(Complex)) + { + WriteDataElement(GetLengthOfByteArray(dataLength)); + WriteDataElement(GetLengthOfByteArray(dataLength)); + } + else if (typeof(T) == typeof(bool)) + { + WriteDataElement(dataLength); + } + } + + private (int rowIndexLength, int columnIndexLength, int dataLength, uint nonZero) PrepareSparseArrayData( + ISparseArrayOf array) + where T : struct, IEquatable + { + var numberOfColumns = array.Dimensions[1]; + var numberOfElements = array.Data.Values.Count(value => !value.Equals(default)); + return (numberOfElements, numberOfColumns + 1, numberOfElements, (uint)numberOfElements); + } + + private void WriteSparseArrayFlags() + { + WriteTag(); + Position += 8; + } + + private void WriteFieldNames(IEnumerable fieldNames) + { + var fieldNamesArray = fieldNames.Select(name => Encoding.ASCII.GetBytes(name)).ToArray(); + var maxFieldName = fieldNamesArray.Select(name => name.Length).Max() + 1; + WriteDataElement(GetLengthOfByteArray(1)); + var buffer = new byte[fieldNamesArray.Length * maxFieldName]; + var startPosition = 0; + foreach (var name in fieldNamesArray) + { + for (var i = 0; i < name.Length; i++) + { + buffer[startPosition + i] = name[i]; + } + startPosition += maxFieldName; + } + WriteDataElement(buffer.Length); + } + + private void WriteStructureArrayValues(IStructureArray array) + { + for (var i = 0; i < array.Count; i++) + { + foreach (var name in array.FieldNames) + { + WriteArray(array[name, i]); + } + } + } + + private void WriteArray(IArray array, string variableName = "", bool isGlobal = false) + { + switch (array) + { + case ICharArray charArray: + WriteCharArray(charArray, variableName); + break; + case ISparseArrayOf doubleSparseArray: + WriteSparseArray(doubleSparseArray, variableName); + break; + case ISparseArrayOf complexSparseArray: + WriteSparseArray(complexSparseArray, variableName); + break; + case ISparseArrayOf boolSparseArray: + WriteSparseArray(boolSparseArray, variableName); + break; + case ICellArray cellArray: + WriteCellArray(cellArray, variableName); + break; + case IStructureArray structureArray: + WriteStructureArray(structureArray, variableName); + break; + default: + WriteNumericalArray(array, variableName); + break; + } + } + + private void WriteCharArray(ICharArray charArray, string name) + { + WriteWrappingContents( + charArray, + fakeWriter => fakeWriter.WriteCharArrayContents(charArray, name)); + } + + private void WriteSparseArray(ISparseArrayOf sparseArray, string name) + where T : unmanaged, IEquatable + { + WriteWrappingContents( + sparseArray, + fakeWriter => fakeWriter.WriteSparseArrayContents(sparseArray, name)); + } + + private void WriteCellArray(ICellArray cellArray, string name) + { + WriteWrappingContents( + cellArray, + fakeWriter => fakeWriter.WriteCellArrayContents(cellArray, name)); + } + + private void WriteCellArrayValues(ICellArray array) + { + for (var i = 0; i < array.Count; i++) + { + WriteArray(array[i]); + } + } + + private void WriteStructureArray( + IStructureArray structureArray, + string name) + { + WriteWrappingContents( + structureArray, + fakeWriter => fakeWriter.WriteStructureArrayContents(structureArray, name)); + } + + private void WriteNumericalArray( + IArray numericalArray, + string name = "") + { + WriteWrappingContents( + numericalArray, + fakeWriter => fakeWriter.WriteNumericalArrayContents(numericalArray, name)); + } + } +} \ No newline at end of file diff --git a/MatFileHandler/MatFileHandler.csproj b/MatFileHandler/MatFileHandler.csproj index c367c12..dc0dfae 100755 --- a/MatFileHandler/MatFileHandler.csproj +++ b/MatFileHandler/MatFileHandler.csproj @@ -22,6 +22,7 @@ true true snupkg + true @@ -46,4 +47,7 @@ + + + diff --git a/MatFileHandler/MatFileWriter.cs b/MatFileHandler/MatFileWriter.cs index 4f9a932..b6077a1 100755 --- a/MatFileHandler/MatFileWriter.cs +++ b/MatFileHandler/MatFileWriter.cs @@ -55,7 +55,15 @@ namespace MatFileHandler switch (_options.UseCompression) { case CompressionUsage.Always: - WriteCompressedVariable(writer, variable); + if (Stream.CanSeek) + { + WriteCompressedVariableToSeekableStream(writer, variable); + } + else + { + WriteCompressedVariableToUnseekableStream(writer, variable); + } + break; case CompressionUsage.Never: WriteVariable(writer, variable); @@ -125,6 +133,12 @@ namespace MatFileHandler { WriteTag(writer, new Tag(type, data.Length)); writer.Write(data); + var rem = data.Length % 8; + if (rem > 0) + { + var padding = new byte[8 - rem]; + writer.Write(padding); + } } else { @@ -136,7 +150,6 @@ namespace MatFileHandler writer.Write(padding); } } - WritePadding(writer); } private void WriteDimensions(BinaryWriter writer, int[] dimensions) @@ -393,7 +406,11 @@ namespace MatFileHandler return new ArrayFlags(ArrayType.MxChar, isGlobal ? Variable.IsGlobal : 0); } - private void WriteWrappingContents(BinaryWriter writer, T array, Action writeContents) + private void WriteWrappingContents( + BinaryWriter writer, + T array, + Action lengthCalculator, + Action writeContents) where T : IArray { if (array.IsEmpty) @@ -401,16 +418,12 @@ namespace MatFileHandler WriteTag(writer, new Tag(DataType.MiMatrix, 0)); return; } - using (var contents = new MemoryStream()) - { - using (var contentsWriter = new BinaryWriter(contents)) - { - writeContents(contentsWriter); - WriteTag(writer, new Tag(DataType.MiMatrix, (int)contents.Length)); - contents.Position = 0; - contents.CopyTo(writer.BaseStream); - } - } + + var fakeWriter = new FakeWriter(); + lengthCalculator(fakeWriter); + var calculatedLength = fakeWriter.Position; + WriteTag(writer, new Tag(DataType.MiMatrix, calculatedLength)); + writeContents(writer); } private void WriteNumericalArrayContents(BinaryWriter writer, IArray array, string name, bool isGlobal) @@ -430,6 +443,7 @@ namespace MatFileHandler WriteWrappingContents( writer, numericalArray, + fakeWriter => fakeWriter.WriteNumericalArrayContents(numericalArray, name), contentsWriter => { WriteNumericalArrayContents(contentsWriter, numericalArray, name, isGlobal); }); } @@ -447,6 +461,7 @@ namespace MatFileHandler WriteWrappingContents( writer, charArray, + fakeWriter => fakeWriter.WriteCharArrayContents(charArray, name), contentsWriter => { WriteCharArrayContents(contentsWriter, charArray, name, isGlobal); }); } @@ -520,11 +535,12 @@ namespace MatFileHandler } private void WriteSparseArray(BinaryWriter writer, ISparseArrayOf sparseArray, string name, bool isGlobal) - where T : struct, IEquatable + where T : unmanaged, IEquatable { WriteWrappingContents( writer, sparseArray, + fakeWriter => fakeWriter.WriteSparseArrayContents(sparseArray, name), contentsWriter => { WriteSparseArrayContents(contentsWriter, sparseArray, name, isGlobal); }); } @@ -575,6 +591,7 @@ namespace MatFileHandler WriteWrappingContents( writer, structureArray, + fakeWriter => fakeWriter.WriteStructureArrayContents(structureArray, name), contentsWriter => { WriteStructureArrayContents(contentsWriter, structureArray, name, isGlobal); }); } @@ -599,6 +616,7 @@ namespace MatFileHandler WriteWrappingContents( writer, cellArray, + fakeWriter => fakeWriter.WriteCellArrayContents(cellArray, name), contentsWriter => { WriteCellArrayContents(contentsWriter, cellArray, name, isGlobal); }); } @@ -635,7 +653,33 @@ namespace MatFileHandler WriteArray(writer, variable.Value, variable.Name, variable.IsGlobal); } - private void WriteCompressedVariable(BinaryWriter writer, IVariable variable) + private void WriteCompressedVariableToSeekableStream(BinaryWriter writer, IVariable variable) + { + var position = writer.BaseStream.Position; + WriteTag(writer, new Tag(DataType.MiCompressed, 0)); + writer.Write((byte)0x78); + writer.Write((byte)0x9c); + int compressedLength; + uint crc; + var before = writer.BaseStream.Position; + using (var compressionStream = new DeflateStream(writer.BaseStream, CompressionMode.Compress, leaveOpen: true)) + { + using var checksumStream = new ChecksumCalculatingStream(compressionStream); + using var internalWriter = new BinaryWriter(checksumStream, Encoding.UTF8, leaveOpen: true); + WriteVariable(internalWriter, variable); + crc = checksumStream.GetCrc(); + } + + var after = writer.BaseStream.Position; + compressedLength = (int)(after - before) + 6; + + writer.Write(BitConverter.GetBytes(crc).Reverse().ToArray()); + writer.BaseStream.Position = position; + WriteTag(writer, new Tag(DataType.MiCompressed, compressedLength)); + writer.BaseStream.Seek(0, SeekOrigin.End); + } + + private void WriteCompressedVariableToUnseekableStream(BinaryWriter writer, IVariable variable) { using (var compressedStream = new MemoryStream()) {