mahalex/MatFileHandler
1
0
Fork 0
Code Issues 12 Pull Requests Actions Packages Projects Releases 9 Wiki Activity

Fix parsing of nested objects.

Browse Source
This commit is contained in:
Alexander Luzgarev 2018-10-17 19:16:06 +02:00
parent fc3af2a3cf
commit 793cc8ddb6
9 changed files with 233 additions and 64 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

41
MatFileHandler.Tests/MatFileReaderTests.cs
View File

@ -347,6 +347,47 @@ namespace MatFileHandler.Tests
Assert.That(variable2.ConvertToDoubleArray(), Is.EqualTo(new[] { 1.0, 3.0, 5.0, 2.0, 4.0, 6.0 }));
}
/// <summary>
/// Test subobjects within objects.
/// </summary>
[Test]
public void TestSubobjects()
{
var matFile = GetTests("good")["pointWithSubpoints"];
var p = matFile["p"].Value as IMatObject;
Assert.That(p.ClassName, Is.EqualTo("Point"));
var x = p["x"] as IMatObject;
Assert.That(x.ClassName, Is.EqualTo("SubPoint"));
Assert.That(x.FieldNames, Is.EquivalentTo(new[] { "a", "b", "c" }));
var y = p["y"] as IMatObject;
Assert.That(y.ClassName, Is.EqualTo("SubPoint"));
Assert.That(y.FieldNames, Is.EquivalentTo(new[] { "a", "b", "c" }));
Assert.That(x["a"].ConvertToDoubleArray(), Is.EquivalentTo(new[] { 1.0 }));
Assert.That(x["b"].ConvertToDoubleArray(), Is.EquivalentTo(new[] { 2.0 }));
Assert.That(x["c"].ConvertToDoubleArray(), Is.EquivalentTo(new[] { 3.0 }));
Assert.That(y["a"].ConvertToDoubleArray(), Is.EquivalentTo(new[] { 14.0 }));
Assert.That(y["b"].ConvertToDoubleArray(), Is.EquivalentTo(new[] { 15.0 }));
Assert.That(y["c"].ConvertToDoubleArray(), Is.EquivalentTo(new[] { 16.0 }));
}
/// <summary>
/// Test nested objects.
/// </summary>
[Test]
public void TestNestedObjects()
{
var matFile = GetTests("good")["subsubPoint"];
var p = matFile["p"].Value as IMatObject;
Assert.That(p.ClassName, Is.EqualTo("Point"));
Assert.That(p["x"].ConvertToDoubleArray(), Is.EquivalentTo(new[] { 1.0 }));
var pp = p["y"] as IMatObject;
Assert.That(pp.ClassName == "Point");
Assert.That(pp["x"].ConvertToDoubleArray(), Is.EquivalentTo(new[] { 10.0 }));
var ppp = pp["y"] as IMatObject;
Assert.That(ppp["x"].ConvertToDoubleArray(), Is.EquivalentTo(new[] { 100.0 }));
Assert.That(ppp["y"].ConvertToDoubleArray(), Is.EquivalentTo(new[] { 200.0 }));
}
private static AbstractTestDataFactory<IMatFile> GetTests(string factoryName) =>
new MatTestDataFactory(Path.Combine(TestDirectory, factoryName));

BIN
MatFileHandler.Tests/test-data/good/pointWithSubpoints.mat Normal file
View File

Binary file not shown.

BIN
MatFileHandler.Tests/test-data/good/subsubPoint.mat Normal file
View File

Binary file not shown.

19
MatFileHandler/DataElementReader.cs
View File

@ -90,14 +90,19 @@ namespace MatFileHandler
return result;
}
private static (int[] dimensions, int[] links, int classIndex) ParseOpaqueData(MatNumericalArrayOf<uint> data)
/// <summary>
/// Parse opaque link data.
/// </summary>
/// <param name="data">Opaque link data.</param>
/// <returns>Dimensions array, links array, class index.</returns>
internal static (int[] dimensions, int[] links, int classIndex) ParseOpaqueData(uint[] data)
{
var nDims = data.Data[1];
var nDims = data[1];
var dimensions = new int[nDims];
var position = 2;
for (var i = 0; i < nDims; i++)
{
dimensions[i] = (int)data.Data[position];
dimensions[i] = (int)data[position];
position++;
}
@ -105,11 +110,11 @@ namespace MatFileHandler
var links = new int[count];
for (var i = 0; i < count; i++)
{
links[i] = (int)data.Data[position];
links[i] = (int)data[position];
position++;
}
var classIndex = (int)data.Data[position];
var classIndex = (int)data[position];
return (dimensions, links, classIndex);
}
@ -239,14 +244,14 @@ namespace MatFileHandler
var data = ReadData(dataElement);
if (data is MatNumericalArrayOf<uint> linkElement)
{
var (dimensions, links, classIndex) = ParseOpaqueData(linkElement);
var (dimensions, indexToObjectId, classIndex) = ParseOpaqueData(linkElement.Data);
return new OpaqueLink(
name,
typeDescription,
className,
dimensions,
data,
links,
indexToObjectId,
classIndex,
subsystemData);
}

35
MatFileHandler/MatFileReader.cs
View File

@ -35,15 +35,19 @@ namespace MatFileHandler
}
/// <summary>
/// Read raw variables from a .mat file.
/// Read a sequence of raw variables from .mat file.
/// </summary>
/// <param name="reader">Binary reader.</param>
/// <param name="subsystemDataOffset">Offset to the subsystem data to use (read from the file header).</param>
/// <returns>Raw variables read.</returns>
internal static List<RawVariable> ReadRawVariables(BinaryReader reader, long subsystemDataOffset)
/// <param name="reader">Reader.</param>
/// <param name="subsystemDataOffset">Offset of subsystem data in the file;
/// we need it because we may encounter it during reading, and
/// the subsystem data should be parsed in a special way.</param>
/// <param name="subsystemData">
/// Link to the current file's subsystem data structure; initially it has dummy value
/// which will be replaced after we parse the whole subsystem data.</param>
/// <returns>List of "raw" variables; the actual variables are constructed from them later.</returns>
internal static List<RawVariable> ReadRawVariables(BinaryReader reader, long subsystemDataOffset, SubsystemData subsystemData)
{
var variables = new List<RawVariable>();
var subsystemData = new SubsystemData();
var dataElementReader = new DataElementReader(subsystemData);
while (true)
{
@ -54,7 +58,8 @@ namespace MatFileHandler
if (position == subsystemDataOffset)
{
var subsystemDataElement = dataElement as IArrayOf<byte>;
subsystemData.Set(ReadSubsystemData(subsystemDataElement.Data));
var newSubsystemData = ReadSubsystemData(subsystemDataElement.Data, subsystemData);
subsystemData.Set(newSubsystemData);
}
else
{
@ -70,6 +75,18 @@ namespace MatFileHandler
return variables;
}
/// <summary>
/// Read raw variables from a .mat file.
/// </summary>
/// <param name="reader">Binary reader.</param>
/// <param name="subsystemDataOffset">Offset to the subsystem data to use (read from the file header).</param>
/// <returns>Raw variables read.</returns>
internal static List<RawVariable> ReadRawVariables(BinaryReader reader, long subsystemDataOffset)
{
var subsystemData = new SubsystemData();
return ReadRawVariables(reader, subsystemDataOffset, subsystemData);
}
private static IMatFile Read(BinaryReader reader)
{
var header = ReadHeader(reader);
@ -97,9 +114,9 @@ namespace MatFileHandler
return Header.Read(reader);
}
private static SubsystemData ReadSubsystemData(byte[] bytes)
private static SubsystemData ReadSubsystemData(byte[] bytes, SubsystemData subsystemData)
{
return SubsystemDataReader.Read(bytes);
return SubsystemDataReader.Read(bytes, subsystemData);
}
}
}

2
MatFileHandler/Opaque.cs
View File

@ -37,7 +37,7 @@ namespace MatFileHandler
/// <summary>
/// Gets class name of the opaque object.
/// </summary>
public string ClassName { get; }
public virtual string ClassName { get; }
/// <summary>
/// Gets raw object's data: either links to subsystem data, or actual data.

25
MatFileHandler/OpaqueLink.cs
View File

@ -22,7 +22,7 @@ namespace MatFileHandler
/// <param name="className">Name of the object's class.</param>
/// <param name="dimensions">Dimensions of the object.</param>
/// <param name="data">Raw data containing links to object's storage.</param>
/// <param name="links">Links to object's storage.</param>
/// <param name="indexToObjectId">Links to object's storage.</param>
/// <param name="classIndex">Index of object's class.</param>
/// <param name="subsystemData">Reference to global subsystem data.</param>
public OpaqueLink(
@ -31,12 +31,12 @@ namespace MatFileHandler
string className,
int[] dimensions,
DataElement data,
int[] links,
int[] indexToObjectId,
int classIndex,
SubsystemData subsystemData)
: base(name, typeDescription, className, dimensions, data)
{
Links = links ?? throw new ArgumentNullException(nameof(links));
IndexToObjectId = indexToObjectId ?? throw new ArgumentNullException(nameof(indexToObjectId));
ClassIndex = classIndex;
this.subsystemData = subsystemData ?? throw new ArgumentNullException(nameof(subsystemData));
}
@ -55,9 +55,14 @@ namespace MatFileHandler
/// <summary>
/// Gets links to the fields stored in subsystem data.
/// </summary>
public int[] Links { get; }
public int[] IndexToObjectId { get; }
private string[] FieldNamesArray => subsystemData.ClassInformation[ClassIndex - 1].FieldNames.ToArray();
/// <summary>
/// Gets name of this object's class.
/// </summary>
public override string ClassName => subsystemData.ClassInformation[ClassIndex].Name;
private string[] FieldNamesArray => subsystemData.ClassInformation[ClassIndex].FieldNames.ToArray();
/// <inheritdoc />
public IArray this[string field, params int[] list]
@ -89,21 +94,23 @@ namespace MatFileHandler
private bool TryGetValue(string field, out IArray output, params int[] list)
{
var index = Dimensions.DimFlatten(list);
var maybeFieldIndex = subsystemData.ClassInformation[ClassIndex - 1].FindField(field);
var maybeFieldIndex = subsystemData.ClassInformation[ClassIndex].FindField(field);
if (!(maybeFieldIndex is int fieldIndex))
{
output = default(IArray);
return false;
}
if (index >= subsystemData.ObjectInformation.Length)
if (index >= IndexToObjectId.Length)
{
output = default(IArray);
return false;
}
var dataIndex = subsystemData.ObjectInformation[index].FieldLinks[fieldIndex + 1];
output = subsystemData.Data[dataIndex];
var objectPosition = IndexToObjectId[index];
var objectInfo = subsystemData.ObjectInformation.First(pair => pair.Value.Position == objectPosition).Value;
var fieldId = objectInfo.FieldLinks[fieldIndex];
output = subsystemData.Data[fieldId];
return true;
}

33
MatFileHandler/SubsystemData.cs
View File

@ -30,7 +30,10 @@ namespace MatFileHandler
/// <param name="classInformation">Information about the classes.</param>
/// <param name="objectInformation">Information about the objects.</param>
/// <param name="data">Field values.</param>
public SubsystemData(ClassInfo[] classInformation, ObjectInfo[] objectInformation, Dictionary<int, IArray> data)
public SubsystemData(
Dictionary<int, ClassInfo> classInformation,
Dictionary<int, ObjectInfo> objectInformation,
Dictionary<int, IArray> data)
{
this.ClassInformation =
classInformation ?? throw new ArgumentNullException(nameof(classInformation));
@ -42,7 +45,7 @@ namespace MatFileHandler
/// <summary>
/// Gets or sets information about all the classes occurring in the file.
/// </summary>
public ClassInfo[] ClassInformation { get; set; }
public Dictionary<int, ClassInfo> ClassInformation { get; set; }
/// <summary>
/// Gets or sets the actual data: mapping of "object field" indices to their values.
@ -52,7 +55,7 @@ namespace MatFileHandler
/// <summary>
/// Gets or sets information about all the objects occurring in the file.
/// </summary>
public ObjectInfo[] ObjectInformation { get; set; }
public Dictionary<int, ObjectInfo> ObjectInformation { get; set; }
/// <summary>
/// Initialize this object from another object.
@ -73,30 +76,23 @@ namespace MatFileHandler
/// </summary>
internal class ClassInfo
{
private readonly string[] fieldNames;
private readonly Dictionary<string, int> fieldToIndex;
/// <summary>
/// Initializes a new instance of the <see cref="ClassInfo"/> class.
/// </summary>
/// <param name="name">Class name.</param>
/// <param name="fieldNames">Names of the fields.</param>
public ClassInfo(string name, string[] fieldNames)
/// <param name="fieldToIndex">A dictionary mapping field names to field ids.</param>
public ClassInfo(string name, Dictionary<string, int> fieldToIndex)
{
Name = name ?? throw new ArgumentNullException(nameof(name));
this.fieldNames = fieldNames ?? throw new ArgumentNullException(nameof(fieldNames));
fieldToIndex = new Dictionary<string, int>();
for (var i = 0; i < fieldNames.Length; i++)
{
fieldToIndex[fieldNames[i]] = i;
}
this.fieldToIndex = fieldToIndex ?? throw new ArgumentNullException(nameof(fieldToIndex));
}
/// <summary>
/// Gets names of the fields in the class.
/// </summary>
public IReadOnlyCollection<string> FieldNames => fieldNames;
public IReadOnlyCollection<string> FieldNames => fieldToIndex.Keys;
/// <summary>
/// Gets name of the class.
@ -129,9 +125,11 @@ namespace MatFileHandler
/// <summary>
/// Initializes a new instance of the <see cref="ObjectInfo"/> class.
/// </summary>
/// <param name="position">Position of object in the object information table.</param>
/// <param name="fieldLinks">A dictionary mapping the field indices to "field values" indices.</param>
public ObjectInfo(Dictionary<int, int> fieldLinks)
public ObjectInfo(int position, Dictionary<int, int> fieldLinks)
{
Position = position;
this.fieldLinks = fieldLinks ?? throw new ArgumentNullException(nameof(fieldLinks));
}
@ -139,6 +137,11 @@ namespace MatFileHandler
/// Gets mapping between the field indices and "field values" indices.
/// </summary>
public IReadOnlyDictionary<int, int> FieldLinks => fieldLinks;
/// <summary>
/// Gets position of object in the object information table.
/// </summary>
public int Position { get; }
}
}
}

142
MatFileHandler/SubsystemDataReader.cs
View File

@ -17,8 +17,11 @@ namespace MatFileHandler
/// Read subsystem data from a given byte array.
/// </summary>
/// <param name="bytes">Byte array with the data.</param>
/// <param name="subsystemData">
/// Link to the existing subsystem data; this will be put in nested OpaqueLink objects
/// and later replaced with the subsystem data that we are currently reading.</param>
/// <returns>Subsystem data read.</returns>
public static SubsystemData Read(byte[] bytes)
public static SubsystemData Read(byte[] bytes, SubsystemData subsystemData)
{
List<RawVariable> rawVariables = null;
using (var stream = new MemoryStream(bytes))
@ -26,7 +29,7 @@ namespace MatFileHandler
using (var reader = new BinaryReader(stream))
{
reader.ReadBytes(8);
rawVariables = MatFileReader.ReadRawVariables(reader, -1);
rawVariables = MatFileReader.ReadRawVariables(reader, -1, subsystemData);
}
}
@ -38,55 +41,152 @@ namespace MatFileHandler
var (offsets, position) = ReadOffsets(info, 0);
var fieldNames = ReadFieldNames(info, position, offsets[1]);
var numberOfClasses = ((offsets[3] - offsets[2]) / 16) - 1;
SubsystemData.ClassInfo[] classInformation = null;
Dictionary<int, string> classIdToName = null;
using (var stream = new MemoryStream(info, offsets[2], offsets[3] - offsets[2]))
{
using (var reader = new BinaryReader(stream))
{
classInformation = ReadClassInformation(reader, fieldNames, numberOfClasses);
classIdToName = ReadClassNames(reader, fieldNames, numberOfClasses);
}
}
var numberOfObjects = ((offsets[5] - offsets[4]) / 24) - 1;
SubsystemData.ObjectInfo[] objectInformation = null;
Dictionary<int, (int, int)> objectClasses = null;
using (var stream = new MemoryStream(info, offsets[4], offsets[5] - offsets[4]))
{
using (var reader = new BinaryReader(stream))
{
objectClasses = ReadObjectClasses(reader, numberOfObjects);
}
}
Dictionary<int, Dictionary<int, int>> objectToFields = null;
using (var stream = new MemoryStream(info, offsets[5], offsets[6] - offsets[5]))
{
using (var reader = new BinaryReader(stream))
{
objectInformation = ReadObjectInformation(reader, numberOfObjects);
objectToFields = ReadObjectToFieldsMapping(reader, numberOfObjects);
}
}
var allFields = objectInformation.SelectMany(obj => obj.FieldLinks.Values);
var (classInformation, objectInformation) =
GatherClassAndObjectInformation(
classIdToName,
fieldNames,
objectClasses,
objectToFields);
var allFields = objectInformation.Values.SelectMany(obj => obj.FieldLinks.Values);
var data = new Dictionary<int, IArray>();
foreach (var i in allFields)
{
data[i] = opaqueData[i + 2];
data[i] = TransformOpaqueData(opaqueData[i + 2], subsystemData);
}
return new SubsystemData(classInformation, objectInformation, data);
}
private static SubsystemData.ObjectInfo ReadObjectInformation(BinaryReader reader)
private static IArray TransformOpaqueData(IArray array, SubsystemData subsystemData)
{
if (array is MatNumericalArrayOf<uint> uintArray)
{
if (uintArray.Data[0] == 3707764736u)
{
var (dimensions, indexToObjectId, classIndex) = DataElementReader.ParseOpaqueData(uintArray.Data);
return new OpaqueLink(
uintArray.Name,
string.Empty,
string.Empty,
dimensions,
array as DataElement,
indexToObjectId,
classIndex,
subsystemData);
}
}
return array;
}
private static Dictionary<int, (int, int)> ReadObjectClasses(BinaryReader reader, int numberOfObjects)
{
var result = new Dictionary<int, (int, int)>();
reader.ReadBytes(24);
for (var i = 0; i < numberOfObjects; i++)
{
var classId = reader.ReadInt32();
reader.ReadBytes(12);
var objectPosition = reader.ReadInt32();
var objectId = reader.ReadInt32();
result[objectPosition] = (objectId, classId);
}
return result;
}
private static (Dictionary<int, SubsystemData.ClassInfo>, Dictionary<int, SubsystemData.ObjectInfo>) GatherClassAndObjectInformation(
Dictionary<int, string> classIdToName,
string[] fieldNames,
Dictionary<int, (int, int)> objectClasses,
Dictionary<int, Dictionary<int, int>> objectToFields)
{
var classInfos = new Dictionary<int, SubsystemData.ClassInfo>();
foreach (var classId in classIdToName.Keys)
{
var className = classIdToName[classId];
var fieldIds = new SortedSet<int>();
foreach (var objectPosition in objectToFields.Keys)
{
var (_, thisObjectClassId) = objectClasses[objectPosition];
if (thisObjectClassId != classId)
{
continue;
}
foreach (var fieldId in objectToFields[objectPosition].Keys)
{
fieldIds.Add(fieldId);
}
}
var fieldToIndex = new Dictionary<string, int>();
foreach (var fieldId in fieldIds)
{
fieldToIndex[fieldNames[fieldId - 1]] = fieldId;
}
classInfos[classId] = new SubsystemData.ClassInfo(className, fieldToIndex);
}
var objectInfos = new Dictionary<int, SubsystemData.ObjectInfo>();
foreach (var objectPosition in objectToFields.Keys)
{
var (objectId, _) = objectClasses[objectPosition];
objectInfos[objectId] = new SubsystemData.ObjectInfo(objectPosition, objectToFields[objectPosition]);
}
return (classInfos, objectInfos);
}
private static Dictionary<int, int> ReadFieldToFieldDataMapping(BinaryReader reader)
{
var length = reader.ReadInt32();
var fieldLinks = new Dictionary<int, int>();
var result = new Dictionary<int, int>();
for (var i = 0; i < length; i++)
{
var x = reader.ReadInt32();
var y = reader.ReadInt32();
var index = x * y;
var link = reader.ReadInt32();
fieldLinks[index] = link;
result[index] = link;
}
return new SubsystemData.ObjectInfo(fieldLinks);
return result;
}
private static SubsystemData.ObjectInfo[] ReadObjectInformation(BinaryReader reader, int numberOfObjects)
private static Dictionary<int, Dictionary<int, int>> ReadObjectToFieldsMapping(BinaryReader reader, int numberOfObjects)
{
var result = new SubsystemData.ObjectInfo[numberOfObjects];
var result = new Dictionary<int, Dictionary<int, int>>();
reader.ReadBytes(8);
for (var objectIndex = 0; objectIndex < numberOfObjects; objectIndex++)
for (var objectPosition = 1; objectPosition <= numberOfObjects; objectPosition++)
{
result[objectIndex] = ReadObjectInformation(reader);
result[objectPosition] = ReadFieldToFieldDataMapping(reader);
var position = reader.BaseStream.Position;
if (position % 8 != 0)
{
@ -96,12 +196,12 @@ namespace MatFileHandler
return result;
}
private static SubsystemData.ClassInfo[] ReadClassInformation(
private static Dictionary<int, string> ReadClassNames(
BinaryReader reader,
string[] fieldNames,
int numberOfClasses)
{
var result = new SubsystemData.ClassInfo[numberOfClasses];
var result = new Dictionary<int, string>();
var indices = new int[numberOfClasses + 1];
for (var i = 0; i <= numberOfClasses; i++)
{
@ -113,11 +213,7 @@ namespace MatFileHandler
for (var i = 0; i < numberOfClasses; i++)
{
var numberOfFields = indices[i + 1] - indices[i] - 1;
var names = new string[numberOfFields];
Array.Copy(fieldNames, indices[i], names, 0, numberOfFields);
var className = fieldNames[indices[i + 1] - 1];
result[i] = new SubsystemData.ClassInfo(className, names);
result[i + 1] = fieldNames[indices[i + 1] - 1];
}
return result;