Fix parsing of nested objects.

MatFileHandler.Tests/MatFileReaderTests.cs

@@ -347,6 +347,47 @@ namespace MatFileHandler.Tests
             Assert.That(variable2.ConvertToDoubleArray(), Is.EqualTo(new[] { 1.0, 3.0, 5.0, 2.0, 4.0, 6.0 }));
         }
 
+        /// <summary>
+        /// Test subobjects within objects.
+        /// </summary>
+        [Test]
+        public void TestSubobjects()
+        {
+            var matFile = GetTests("good")["pointWithSubpoints"];
+            var p = matFile["p"].Value as IMatObject;
+            Assert.That(p.ClassName, Is.EqualTo("Point"));
+            var x = p["x"] as IMatObject;
+            Assert.That(x.ClassName, Is.EqualTo("SubPoint"));
+            Assert.That(x.FieldNames, Is.EquivalentTo(new[] { "a", "b", "c" }));
+            var y = p["y"] as IMatObject;
+            Assert.That(y.ClassName, Is.EqualTo("SubPoint"));
+            Assert.That(y.FieldNames, Is.EquivalentTo(new[] { "a", "b", "c" }));
+            Assert.That(x["a"].ConvertToDoubleArray(), Is.EquivalentTo(new[] { 1.0 }));
+            Assert.That(x["b"].ConvertToDoubleArray(), Is.EquivalentTo(new[] { 2.0 }));
+            Assert.That(x["c"].ConvertToDoubleArray(), Is.EquivalentTo(new[] { 3.0 }));
+            Assert.That(y["a"].ConvertToDoubleArray(), Is.EquivalentTo(new[] { 14.0 }));
+            Assert.That(y["b"].ConvertToDoubleArray(), Is.EquivalentTo(new[] { 15.0 }));
+            Assert.That(y["c"].ConvertToDoubleArray(), Is.EquivalentTo(new[] { 16.0 }));
+        }
+
+        /// <summary>
+        /// Test nested objects.
+        /// </summary>
+        [Test]
+        public void TestNestedObjects()
+        {
+            var matFile = GetTests("good")["subsubPoint"];
+            var p = matFile["p"].Value as IMatObject;
+            Assert.That(p.ClassName, Is.EqualTo("Point"));
+            Assert.That(p["x"].ConvertToDoubleArray(), Is.EquivalentTo(new[] { 1.0 }));
+            var pp = p["y"] as IMatObject;
+            Assert.That(pp.ClassName == "Point");
+            Assert.That(pp["x"].ConvertToDoubleArray(), Is.EquivalentTo(new[] { 10.0 }));
+            var ppp = pp["y"] as IMatObject;
+            Assert.That(ppp["x"].ConvertToDoubleArray(), Is.EquivalentTo(new[] { 100.0 }));
+            Assert.That(ppp["y"].ConvertToDoubleArray(), Is.EquivalentTo(new[] { 200.0 }));
+        }
+
         private static AbstractTestDataFactory<IMatFile> GetTests(string factoryName) =>
             new MatTestDataFactory(Path.Combine(TestDirectory, factoryName));
 

MatFileHandler/DataElementReader.cs

@@ -90,14 +90,19 @@ namespace MatFileHandler
             return result;
         }
 
-        private static (int[] dimensions, int[] links, int classIndex) ParseOpaqueData(MatNumericalArrayOf<uint> data)
+        /// <summary>
+        /// Parse opaque link data.
+        /// </summary>
+        /// <param name="data">Opaque link data.</param>
+        /// <returns>Dimensions array, links array, class index.</returns>
+        internal static (int[] dimensions, int[] links, int classIndex) ParseOpaqueData(uint[] data)
         {
-            var nDims = data.Data[1];
+            var nDims = data[1];
             var dimensions = new int[nDims];
             var position = 2;
             for (var i = 0; i < nDims; i++)
             {
-                dimensions[i] = (int)data.Data[position];
+                dimensions[i] = (int)data[position];
                 position++;
             }
 
@@ -105,11 +110,11 @@ namespace MatFileHandler
             var links = new int[count];
             for (var i = 0; i < count; i++)
             {
-                links[i] = (int)data.Data[position];
+                links[i] = (int)data[position];
                 position++;
             }
 
-            var classIndex = (int)data.Data[position];
+            var classIndex = (int)data[position];
 
             return (dimensions, links, classIndex);
         }
@@ -239,14 +244,14 @@ namespace MatFileHandler
             var data = ReadData(dataElement);
             if (data is MatNumericalArrayOf<uint> linkElement)
             {
-                var (dimensions, links, classIndex) = ParseOpaqueData(linkElement);
+                var (dimensions, indexToObjectId, classIndex) = ParseOpaqueData(linkElement.Data);
                 return new OpaqueLink(
                     name,
                     typeDescription,
                     className,
                     dimensions,
                     data,
-                    links,
+                    indexToObjectId,
                     classIndex,
                     subsystemData);
             }

MatFileHandler/MatFileReader.cs

@@ -35,15 +35,19 @@ namespace MatFileHandler
         }
 
         /// <summary>
-        /// Read raw variables from a .mat file.
+        /// Read a sequence of raw variables from .mat file.
         /// </summary>
-        /// <param name="reader">Binary reader.</param>
+        /// <param name="reader">Reader.</param>
-        /// <param name="subsystemDataOffset">Offset to the subsystem data to use (read from the file header).</param>
+        /// <param name="subsystemDataOffset">Offset of subsystem data in the file;
-        /// <returns>Raw variables read.</returns>
+        /// we need it because we may encounter it during reading, and
-        internal static List<RawVariable> ReadRawVariables(BinaryReader reader, long subsystemDataOffset)
+        /// the subsystem data should be parsed in a special way.</param>
+        /// <param name="subsystemData">
+        /// Link to the current file's subsystem data structure; initially it has dummy value
+        /// which will be replaced after we parse the whole subsystem data.</param>
+        /// <returns>List of "raw" variables; the actual variables are constructed from them later.</returns>
+        internal static List<RawVariable> ReadRawVariables(BinaryReader reader, long subsystemDataOffset, SubsystemData subsystemData)
         {
             var variables = new List<RawVariable>();
-            var subsystemData = new SubsystemData();
             var dataElementReader = new DataElementReader(subsystemData);
             while (true)
             {
@@ -54,7 +58,8 @@ namespace MatFileHandler
                     if (position == subsystemDataOffset)
                     {
                         var subsystemDataElement = dataElement as IArrayOf<byte>;
-                        subsystemData.Set(ReadSubsystemData(subsystemDataElement.Data));
+                        var newSubsystemData = ReadSubsystemData(subsystemDataElement.Data, subsystemData);
+                        subsystemData.Set(newSubsystemData);
                     }
                     else
                     {
@@ -70,6 +75,18 @@ namespace MatFileHandler
             return variables;
         }
 
+        /// <summary>
+        /// Read raw variables from a .mat file.
+        /// </summary>
+        /// <param name="reader">Binary reader.</param>
+        /// <param name="subsystemDataOffset">Offset to the subsystem data to use (read from the file header).</param>
+        /// <returns>Raw variables read.</returns>
+        internal static List<RawVariable> ReadRawVariables(BinaryReader reader, long subsystemDataOffset)
+        {
+            var subsystemData = new SubsystemData();
+            return ReadRawVariables(reader, subsystemDataOffset, subsystemData);
+        }
+
         private static IMatFile Read(BinaryReader reader)
         {
             var header = ReadHeader(reader);
@@ -97,9 +114,9 @@ namespace MatFileHandler
             return Header.Read(reader);
         }
 
-        private static SubsystemData ReadSubsystemData(byte[] bytes)
+        private static SubsystemData ReadSubsystemData(byte[] bytes, SubsystemData subsystemData)
         {
-            return SubsystemDataReader.Read(bytes);
+            return SubsystemDataReader.Read(bytes, subsystemData);
         }
     }
 }

MatFileHandler/Opaque.cs

@@ -37,7 +37,7 @@ namespace MatFileHandler
         /// <summary>
         /// Gets class name of the opaque object.
         /// </summary>
-        public string ClassName { get; }
+        public virtual string ClassName { get; }
 
         /// <summary>
         /// Gets raw object's data: either links to subsystem data, or actual data.

MatFileHandler/OpaqueLink.cs

@@ -22,7 +22,7 @@ namespace MatFileHandler
         /// <param name="className">Name of the object's class.</param>
         /// <param name="dimensions">Dimensions of the object.</param>
         /// <param name="data">Raw data containing links to object's storage.</param>
-        /// <param name="links">Links to object's storage.</param>
+        /// <param name="indexToObjectId">Links to object's storage.</param>
         /// <param name="classIndex">Index of object's class.</param>
         /// <param name="subsystemData">Reference to global subsystem data.</param>
         public OpaqueLink(
@@ -31,12 +31,12 @@ namespace MatFileHandler
             string className,
             int[] dimensions,
             DataElement data,
-            int[] links,
+            int[] indexToObjectId,
             int classIndex,
             SubsystemData subsystemData)
             : base(name, typeDescription, className, dimensions, data)
         {
-            Links = links ?? throw new ArgumentNullException(nameof(links));
+            IndexToObjectId = indexToObjectId ?? throw new ArgumentNullException(nameof(indexToObjectId));
             ClassIndex = classIndex;
             this.subsystemData = subsystemData ?? throw new ArgumentNullException(nameof(subsystemData));
         }
@@ -55,9 +55,14 @@ namespace MatFileHandler
         /// <summary>
         /// Gets links to the fields stored in subsystem data.
         /// </summary>
-        public int[] Links { get; }
+        public int[] IndexToObjectId { get; }
 
-        private string[] FieldNamesArray => subsystemData.ClassInformation[ClassIndex - 1].FieldNames.ToArray();
+        /// <summary>
+        /// Gets name of this object's class.
+        /// </summary>
+        public override string ClassName => subsystemData.ClassInformation[ClassIndex].Name;
+
+        private string[] FieldNamesArray => subsystemData.ClassInformation[ClassIndex].FieldNames.ToArray();
 
         /// <inheritdoc />
         public IArray this[string field, params int[] list]
@@ -89,21 +94,23 @@ namespace MatFileHandler
         private bool TryGetValue(string field, out IArray output, params int[] list)
         {
             var index = Dimensions.DimFlatten(list);
-            var maybeFieldIndex = subsystemData.ClassInformation[ClassIndex - 1].FindField(field);
+            var maybeFieldIndex = subsystemData.ClassInformation[ClassIndex].FindField(field);
             if (!(maybeFieldIndex is int fieldIndex))
             {
                 output = default(IArray);
                 return false;
             }
 
-            if (index >= subsystemData.ObjectInformation.Length)
+            if (index >= IndexToObjectId.Length)
             {
                 output = default(IArray);
                 return false;
             }
 
-            var dataIndex = subsystemData.ObjectInformation[index].FieldLinks[fieldIndex + 1];
+            var objectPosition = IndexToObjectId[index];
-            output = subsystemData.Data[dataIndex];
+            var objectInfo = subsystemData.ObjectInformation.First(pair => pair.Value.Position == objectPosition).Value;
+            var fieldId = objectInfo.FieldLinks[fieldIndex];
+            output = subsystemData.Data[fieldId];
             return true;
         }
 

MatFileHandler/SubsystemData.cs

@@ -30,7 +30,10 @@ namespace MatFileHandler
         /// <param name="classInformation">Information about the classes.</param>
         /// <param name="objectInformation">Information about the objects.</param>
         /// <param name="data">Field values.</param>
-        public SubsystemData(ClassInfo[] classInformation, ObjectInfo[] objectInformation, Dictionary<int, IArray> data)
+        public SubsystemData(
+            Dictionary<int, ClassInfo> classInformation,
+            Dictionary<int, ObjectInfo> objectInformation,
+            Dictionary<int, IArray> data)
         {
             this.ClassInformation =
                 classInformation ?? throw new ArgumentNullException(nameof(classInformation));
@@ -42,7 +45,7 @@ namespace MatFileHandler
         /// <summary>
         /// Gets or sets information about all the classes occurring in the file.
         /// </summary>
-        public ClassInfo[] ClassInformation { get; set; }
+        public Dictionary<int, ClassInfo> ClassInformation { get; set; }
 
         /// <summary>
         /// Gets or sets the actual data: mapping of "object field" indices to their values.
@@ -52,7 +55,7 @@ namespace MatFileHandler
         /// <summary>
         /// Gets or sets information about all the objects occurring in the file.
         /// </summary>
-        public ObjectInfo[] ObjectInformation { get; set; }
+        public Dictionary<int, ObjectInfo> ObjectInformation { get; set; }
 
         /// <summary>
         /// Initialize this object from another object.
@@ -73,30 +76,23 @@ namespace MatFileHandler
         /// </summary>
         internal class ClassInfo
         {
-            private readonly string[] fieldNames;
-
             private readonly Dictionary<string, int> fieldToIndex;
 
             /// <summary>
             /// Initializes a new instance of the <see cref="ClassInfo"/> class.
             /// </summary>
             /// <param name="name">Class name.</param>
-            /// <param name="fieldNames">Names of the fields.</param>
+            /// <param name="fieldToIndex">A dictionary mapping field names to field ids.</param>
-            public ClassInfo(string name, string[] fieldNames)
+            public ClassInfo(string name, Dictionary<string, int> fieldToIndex)
             {
                 Name = name ?? throw new ArgumentNullException(nameof(name));
-                this.fieldNames = fieldNames ?? throw new ArgumentNullException(nameof(fieldNames));
+                this.fieldToIndex = fieldToIndex ?? throw new ArgumentNullException(nameof(fieldToIndex));
-                fieldToIndex = new Dictionary<string, int>();
-                for (var i = 0; i < fieldNames.Length; i++)
-                {
-                    fieldToIndex[fieldNames[i]] = i;
-                }
             }
 
             /// <summary>
             /// Gets names of the fields in the class.
             /// </summary>
-            public IReadOnlyCollection<string> FieldNames => fieldNames;
+            public IReadOnlyCollection<string> FieldNames => fieldToIndex.Keys;
 
             /// <summary>
             /// Gets name of the class.
@@ -129,9 +125,11 @@ namespace MatFileHandler
             /// <summary>
             /// Initializes a new instance of the <see cref="ObjectInfo"/> class.
             /// </summary>
+            /// <param name="position">Position of object in the object information table.</param>
             /// <param name="fieldLinks">A dictionary mapping the field indices to "field values" indices.</param>
-            public ObjectInfo(Dictionary<int, int> fieldLinks)
+            public ObjectInfo(int position, Dictionary<int, int> fieldLinks)
             {
+                Position = position;
                 this.fieldLinks = fieldLinks ?? throw new ArgumentNullException(nameof(fieldLinks));
             }
 
@@ -139,6 +137,11 @@ namespace MatFileHandler
             /// Gets mapping between the field indices and "field values" indices.
             /// </summary>
             public IReadOnlyDictionary<int, int> FieldLinks => fieldLinks;
+
+            /// <summary>
+            /// Gets position of object in the object information table.
+            /// </summary>
+            public int Position { get; }
         }
     }
 }

MatFileHandler/SubsystemDataReader.cs

@@ -17,8 +17,11 @@ namespace MatFileHandler
         /// Read subsystem data from a given byte array.
         /// </summary>
         /// <param name="bytes">Byte array with the data.</param>
+        /// <param name="subsystemData">
+        /// Link to the existing subsystem data; this will be put in nested OpaqueLink objects
+        /// and later replaced with the subsystem data that we are currently reading.</param>
         /// <returns>Subsystem data read.</returns>
-        public static SubsystemData Read(byte[] bytes)
+        public static SubsystemData Read(byte[] bytes, SubsystemData subsystemData)
         {
             List<RawVariable> rawVariables = null;
             using (var stream = new MemoryStream(bytes))
@@ -26,7 +29,7 @@ namespace MatFileHandler
                 using (var reader = new BinaryReader(stream))
                 {
                     reader.ReadBytes(8);
-                    rawVariables = MatFileReader.ReadRawVariables(reader, -1);
+                    rawVariables = MatFileReader.ReadRawVariables(reader, -1, subsystemData);
                 }
             }
 
@@ -38,55 +41,152 @@ namespace MatFileHandler
             var (offsets, position) = ReadOffsets(info, 0);
             var fieldNames = ReadFieldNames(info, position, offsets[1]);
             var numberOfClasses = ((offsets[3] - offsets[2]) / 16) - 1;
-            SubsystemData.ClassInfo[] classInformation = null;
+            Dictionary<int, string> classIdToName = null;
             using (var stream = new MemoryStream(info, offsets[2], offsets[3] - offsets[2]))
             {
                 using (var reader = new BinaryReader(stream))
                 {
-                    classInformation = ReadClassInformation(reader, fieldNames, numberOfClasses);
+                    classIdToName = ReadClassNames(reader, fieldNames, numberOfClasses);
                 }
             }
+
             var numberOfObjects = ((offsets[5] - offsets[4]) / 24) - 1;
-            SubsystemData.ObjectInfo[] objectInformation = null;
+            Dictionary<int, (int, int)> objectClasses = null;
+            using (var stream = new MemoryStream(info, offsets[4], offsets[5] - offsets[4]))
+            {
+                using (var reader = new BinaryReader(stream))
+                {
+                    objectClasses = ReadObjectClasses(reader, numberOfObjects);
+                }
+            }
+
+            Dictionary<int, Dictionary<int, int>> objectToFields = null;
             using (var stream = new MemoryStream(info, offsets[5], offsets[6] - offsets[5]))
             {
                 using (var reader = new BinaryReader(stream))
                 {
-                    objectInformation = ReadObjectInformation(reader, numberOfObjects);
+                    objectToFields = ReadObjectToFieldsMapping(reader, numberOfObjects);
                 }
             }
 
-            var allFields = objectInformation.SelectMany(obj => obj.FieldLinks.Values);
+            var (classInformation, objectInformation) =
+                GatherClassAndObjectInformation(
+                    classIdToName,
+                    fieldNames,
+                    objectClasses,
+                    objectToFields);
+
+            var allFields = objectInformation.Values.SelectMany(obj => obj.FieldLinks.Values);
             var data = new Dictionary<int, IArray>();
             foreach (var i in allFields)
             {
-                data[i] = opaqueData[i + 2];
+                data[i] = TransformOpaqueData(opaqueData[i + 2], subsystemData);
             }
             return new SubsystemData(classInformation, objectInformation, data);
         }
 
-        private static SubsystemData.ObjectInfo ReadObjectInformation(BinaryReader reader)
+        private static IArray TransformOpaqueData(IArray array, SubsystemData subsystemData)
+        {
+            if (array is MatNumericalArrayOf<uint> uintArray)
+            {
+                if (uintArray.Data[0] == 3707764736u)
+                {
+                    var (dimensions, indexToObjectId, classIndex) = DataElementReader.ParseOpaqueData(uintArray.Data);
+                    return new OpaqueLink(
+                        uintArray.Name,
+                        string.Empty,
+                        string.Empty,
+                        dimensions,
+                        array as DataElement,
+                        indexToObjectId,
+                        classIndex,
+                        subsystemData);
+                }
+            }
+
+            return array;
+        }
+
+        private static Dictionary<int, (int, int)> ReadObjectClasses(BinaryReader reader, int numberOfObjects)
+        {
+            var result = new Dictionary<int, (int, int)>();
+            reader.ReadBytes(24);
+            for (var i = 0; i < numberOfObjects; i++)
+            {
+                var classId = reader.ReadInt32();
+                reader.ReadBytes(12);
+                var objectPosition = reader.ReadInt32();
+                var objectId = reader.ReadInt32();
+                result[objectPosition] = (objectId, classId);
+            }
+
+            return result;
+        }
+
+        private static (Dictionary<int, SubsystemData.ClassInfo>, Dictionary<int, SubsystemData.ObjectInfo>) GatherClassAndObjectInformation(
+            Dictionary<int, string> classIdToName,
+            string[] fieldNames,
+            Dictionary<int, (int, int)> objectClasses,
+            Dictionary<int, Dictionary<int, int>> objectToFields)
+        {
+            var classInfos = new Dictionary<int, SubsystemData.ClassInfo>();
+            foreach (var classId in classIdToName.Keys)
+            {
+                var className = classIdToName[classId];
+                var fieldIds = new SortedSet<int>();
+                foreach (var objectPosition in objectToFields.Keys)
+                {
+                    var (_, thisObjectClassId) = objectClasses[objectPosition];
+                    if (thisObjectClassId != classId)
+                    {
+                        continue;
+                    }
+
+                    foreach (var fieldId in objectToFields[objectPosition].Keys)
+                    {
+                        fieldIds.Add(fieldId);
+                    }
+                }
+                var fieldToIndex = new Dictionary<string, int>();
+                foreach (var fieldId in fieldIds)
+                {
+                    fieldToIndex[fieldNames[fieldId - 1]] = fieldId;
+                }
+                classInfos[classId] = new SubsystemData.ClassInfo(className, fieldToIndex);
+            }
+
+            var objectInfos = new Dictionary<int, SubsystemData.ObjectInfo>();
+            foreach (var objectPosition in objectToFields.Keys)
+            {
+                var (objectId, _) = objectClasses[objectPosition];
+                objectInfos[objectId] = new SubsystemData.ObjectInfo(objectPosition, objectToFields[objectPosition]);
+            }
+
+            return (classInfos, objectInfos);
+        }
+
+        private static Dictionary<int, int> ReadFieldToFieldDataMapping(BinaryReader reader)
         {
             var length = reader.ReadInt32();
-            var fieldLinks = new Dictionary<int, int>();
+            var result = new Dictionary<int, int>();
             for (var i = 0; i < length; i++)
             {
                 var x = reader.ReadInt32();
                 var y = reader.ReadInt32();
                 var index = x * y;
                 var link = reader.ReadInt32();
-                fieldLinks[index] = link;
+                result[index] = link;
             }
-            return new SubsystemData.ObjectInfo(fieldLinks);
+            return result;
         }
 
-        private static SubsystemData.ObjectInfo[] ReadObjectInformation(BinaryReader reader, int numberOfObjects)
+        private static Dictionary<int, Dictionary<int, int>> ReadObjectToFieldsMapping(BinaryReader reader, int numberOfObjects)
         {
-            var result = new SubsystemData.ObjectInfo[numberOfObjects];
+            var result = new Dictionary<int, Dictionary<int, int>>();
             reader.ReadBytes(8);
-            for (var objectIndex = 0; objectIndex < numberOfObjects; objectIndex++)
+            for (var objectPosition = 1; objectPosition <= numberOfObjects; objectPosition++)
             {
-                result[objectIndex] = ReadObjectInformation(reader);
+                result[objectPosition] = ReadFieldToFieldDataMapping(reader);
                 var position = reader.BaseStream.Position;
                 if (position % 8 != 0)
                 {
@@ -96,12 +196,12 @@ namespace MatFileHandler
             return result;
         }
 
-        private static SubsystemData.ClassInfo[] ReadClassInformation(
+        private static Dictionary<int, string> ReadClassNames(
             BinaryReader reader,
             string[] fieldNames,
             int numberOfClasses)
         {
-            var result = new SubsystemData.ClassInfo[numberOfClasses];
+            var result = new Dictionary<int, string>();
             var indices = new int[numberOfClasses + 1];
             for (var i = 0; i <= numberOfClasses; i++)
             {
@@ -113,11 +213,7 @@ namespace MatFileHandler
 
             for (var i = 0; i < numberOfClasses; i++)
             {
-                var numberOfFields = indices[i + 1] - indices[i] - 1;
+                result[i + 1] = fieldNames[indices[i + 1] - 1];
-                var names = new string[numberOfFields];
-                Array.Copy(fieldNames, indices[i], names, 0, numberOfFields);
-                var className = fieldNames[indices[i + 1] - 1];
-                result[i] = new SubsystemData.ClassInfo(className, names);
             }
 
             return result;