unittests/test_cfood.py

@@ -112,6 +112,36 @@ class CFoodReTest(unittest.TestCase):
         self.assertTrue(SimpleCFood.match_item("hallo"))
         self.assertFalse(SimpleCFood.match_item("allo"))
 
+    def test_extensions(self):
+        """Test the RE generation."""
+        empty_extensions = []
+        extensions = ["foo", "bar"]
+
+        self.assertIsNone(AbstractFileCFood.re_from_extensions(empty_extensions))
+        self.assertIsNotNone(SimpleCFood.re_from_extensions(extensions))
+
+        class ExtCFood(AbstractFileCFood):
+
+            @staticmethod
+            def get_re():
+                return AbstractFileCFood.re_from_extensions(extensions)
+            create_identifiables = None
+            update_identifiables = None
+
+        # test which paths are matched
+        print(ExtCFood.re_from_extensions(extensions))
+        self.assertTrue(ExtCFood.match_item("hello/world.foo"))
+        self.assertTrue(ExtCFood.match_item("hello/world.bar"))
+        self.assertFalse(ExtCFood.match_item("hello/world.baz"))
+        self.assertFalse(ExtCFood.match_item("hello/world.foo "))  # Mind the space.
+        self.assertFalse(ExtCFood.match_item("hello/world.foobar"))
+        self.assertFalse(ExtCFood.match_item("hello/world.foo|bar"))
+        self.assertFalse(ExtCFood.match_item("hello/world.fobar"))
+        self.assertFalse(ExtCFood.match_item("hello/world.fooar"))
+
+        # Test stored extension
+        self.assertEqual(ExtCFood("hello/world.foo").match["ext"], "foo")
+
 
 class InsertionTest(unittest.TestCase):
     def test_contained_in_list(self):

unittests/test_h5.py

+import unittest
+from tempfile import NamedTemporaryFile
+
+import caosdb as db
+import caosdb.apiutils
+import h5py
+import numpy as np
+from caosadvancedtools.cfoods import h5
+from caosadvancedtools.cfoods.h5 import h5_attr_to_property
+
+from create_dummy_hdf5file import create_hdf5_file
+
+ENTS = {
+    101: db.Record(id=101),
+    102: db.Record(id=102),
+    103: db.Record(id=103).add_property("test", value=101,
+                                        datatype=db.REFERENCE),
+}
+
+
+def dummy_get(eid):
+    return ENTS[eid]
+
+
+class H5CFoodTest(unittest.TestCase):
+    def setUp(self):
+        self.h5file = NamedTemporaryFile(delete=False, suffix=".h5")
+        self.h5file.close()
+        create_hdf5_file(self.h5file.name)
+        self.h5obj = h5py.File(self.h5file.name, mode="a")
+
+    def test_create_record_records(self):
+        result = h5.H5CFood.create_structure(self.h5obj)
+
+        record_list = []
+        parents = ['group_level1_a', 'group_level1_b', 'group_level1_c', 'root_integers']
+
+        for i in parents:
+            record_list.append(db.Record().add_parent(name=i))
+
+        found_parents = []
+
+        for ent in [p.value for p in result.properties]:
+            if ent.parents[0].name == 'group_level1_a':
+                found_parents.append('group_level1_a')
+                self.assertTrue(ent.get_property("group_level2_aa") is not None)
+                self.assertTrue(ent.get_property("group_level1_a") is None)
+            elif ent.parents[0].name == 'group_level1_b':
+                found_parents.append('group_level1_b')
+                pass
+            elif ent.parents[0].name == 'group_level1_c':
+                found_parents.append('group_level1_c')
+                pass
+            elif ent.parents[0].name == 'root_integers':
+                found_parents.append('root_integers')
+                pass
+
+        for p in parents:
+            self.assertTrue(p in found_parents)
+
+        for i in range(len(result.properties)):
+            for j in result.properties[i].value.get_parents():
+                for k in record_list[i].get_parents():
+                    self.assertEqual(j.name, k.name)
+
+        result1 = h5.H5CFood.create_structure(self.h5obj["group_level1_a"])
+
+        for i in result1.get_parents():
+            self.assertEqual(i.name, "group_level1_a")
+
+        result2 = h5.H5CFood.create_structure(self.h5obj["group_level1_a/group_level2_aa"])
+
+        for i in result2.get_parents():
+            self.assertEqual(i.name, "group_level2_aa")
+
+    def test_collect_existing_structure(self):
+        real_retrieve = caosdb.apiutils.retrieve_entity_with_id
+        caosdb.apiutils.retrieve_entity_with_id = dummy_get
+
+        # should run without problem
+        h5.collect_existing_structure(db.Record(), db.Record(id=234), h5.EntityMapping())
+
+        # test with retrieval: both Records have one test Property with one
+        # value -> The referenced Entities are matched
+        r_exist = db.Record(id=234)
+        r_exist.add_property("test", value=101, datatype=db.REFERENCE)
+        r_target = db.Record()
+        r_child = db.Record()
+        r_target.add_property("test", value=r_child, datatype=db.REFERENCE)
+        em = h5.EntityMapping()
+        h5.collect_existing_structure(r_target, r_exist, em)
+        self.assertTrue(em.to_existing[r_child._cuid] is ENTS[101])
+        self.assertTrue(em.to_target[101] is r_child)
+
+        # test with retrieval: the existing Record has another Property
+        # -> The referenced Entities are matched
+        r_exist = db.Record(id=234)
+        r_exist.add_property("test_other", value=101, datatype=db.REFERENCE)
+        r_target = db.Record()
+        r_child = db.Record()
+        r_target.add_property("test", value=r_child, datatype=db.REFERENCE)
+        em = h5.EntityMapping()
+        h5.collect_existing_structure(r_target, r_exist, em)
+        self.assertEqual(em.to_existing, {})
+        self.assertEqual(em.to_target, {})
+
+        # test with retrieval: both Records have one test Property; the
+        # existing is missing the value -> The referenced Entities are matched
+        r_exist = db.Record(id=234)
+        r_exist.add_property("test", value=None, datatype=db.REFERENCE)
+        r_target = db.Record()
+        r_child = db.Record()
+        r_target.add_property("test", value=r_child, datatype=db.REFERENCE)
+        em = h5.EntityMapping()
+        h5.collect_existing_structure(r_target, r_exist, em)
+        self.assertEqual(em.to_existing, {})
+        self.assertEqual(em.to_target, {})
+
+        # test with retrieval: both Records have one test Property with
+        # multiple values -> The referenced Entities are matched
+        r_exist = db.Record(id=234)
+        r_exist.add_property("test", value=[101, 102], datatype=db.LIST(db.REFERENCE))
+        r_target = db.Record()
+        r_child = db.Record()
+        r_child2 = db.Record()
+        r_target.add_property("test", value=[r_child, r_child2],
+                              datatype=db.LIST(db.REFERENCE))
+        em = h5.EntityMapping()
+        h5.collect_existing_structure(r_target, r_exist, em)
+        self.assertEqual(em.to_existing[r_child._cuid], ENTS[101])
+        self.assertEqual(em.to_existing[r_child2._cuid], ENTS[102])
+        self.assertEqual(em.to_target[101], r_child)
+        self.assertEqual(em.to_target[102], r_child2)
+
+        # test with retrieval: both Records have one test Property with one
+        # value; Add another recursion level -> The referenced Entities are matched
+        r_exist = db.Record(id=234)
+        r_exist.add_property("test", value=103, datatype=db.REFERENCE)
+        r_target = db.Record()
+        r_child = db.Record()
+        r_child2 = db.Record()
+        r_target.add_property("test", value=r_child, datatype=db.REFERENCE)
+        r_child.add_property("test", value=r_child2, datatype=db.REFERENCE)
+        em = h5.EntityMapping()
+        h5.collect_existing_structure(r_target, r_exist, em)
+        self.assertEqual(em.to_existing[r_child._cuid], ENTS[103])
+        self.assertEqual(em.to_target[103], r_child)
+        self.assertEqual(em.to_existing[r_child2._cuid], ENTS[101])
+        self.assertEqual(em.to_target[101], r_child2)
+
+        caosdb.apiutils.retrieve_entity_with_id = real_retrieve
+
+    def test_h5_attr_to_property(self):
+
+        test_int: int = 1
+        test_integer = np.int_(1)
+        test_float = np.float_(1.0)
+        test_str = "Test"
+        test_complex: complex = 2+3j
+        self.assertRaises(NotImplementedError, h5_attr_to_property, test_int)  # only numpy-integers processed?
+        self.assertTupleEqual((1, db.INTEGER), h5_attr_to_property(test_integer))
+        self.assertTupleEqual((1.0, db.DOUBLE), h5_attr_to_property(test_float))
+        self.assertTupleEqual(("Test", db.TEXT), h5_attr_to_property(test_str))
+        self.assertTupleEqual((2+3j, db.TEXT), h5_attr_to_property(test_complex))
+        # strings are often represented using a binary format
+        self.assertTupleEqual(("yeti", db.TEXT), h5_attr_to_property(
+            np.array(["yeti"], dtype=h5py.string_dtype(r'utf-8', 8))[0]))
+
+        test_integer_1d = np.arange(10)
+        test_float_1d = np.arange(0, 1, 0.1)
+        test_str_1d = np.array(["a", "b", "c"])
+        self.assertTrue((np.arange(10) == h5_attr_to_property(test_integer_1d)[0]).all())
+        self.assertTrue(db.LIST(db.INTEGER) == h5_attr_to_property(test_integer_1d)[1])
+        self.assertTrue((np.arange(0, 1, 0.1) == h5_attr_to_property(test_float_1d)[0]).all())
+        self.assertTrue(db.LIST(db.DOUBLE) == h5_attr_to_property(test_float_1d)[1])
+        self.assertTrue((np.array(["a", "b", "c"]) == h5_attr_to_property(test_str_1d)[0]).all())
+        self.assertTrue(db.LIST(db.TEXT) == h5_attr_to_property(test_str_1d)[1])
+
+        test_integers_2d = np.diag(np.arange(100))
+        test_floats_2d = np.eye(100)
+        self.assertTupleEqual((None, None), h5_attr_to_property(test_integers_2d))
+        self.assertTupleEqual((None, None), h5_attr_to_property(test_floats_2d))
+
+        self.assertRaises(NotImplementedError, h5_attr_to_property, np.array(1))

unittests/test_structure_mapping.py

+#!/usr/bin/env python3
+
+# This file is a part of the CaosDB Project.
+#
+# Copyright (C) 2021 IndiScale GmbH <www.indiscale.com>
+# Copyright (C) 2021 Henrik tom Wörden <h.tomwoerden@indiscale.com>
+# Copyright (C) 2021 Alexander Kreft <akreft@trineo.org>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+import unittest
+from os import name
+
+import caosdb as db
+from caosadvancedtools.structure_mapping import (EntityMapping,
+                                                 collect_existing_structure)
+from caosdb.common import datatype
+
+
+class structureMappingTest(unittest.TestCase):
+    def test_Entitymapping(self):
+        ex = db.Record(id=100)  # existing Record
+        tar = db.Record()  # target Record
+        em = EntityMapping()
+        em.add(tar, ex)
+
+        for key, val in em.to_existing.items():
+            self.assertEqual(key, tar._cuid)
+            self.assertEqual(val, ex)
+
+        for key, val in em.to_target.items():
+            self.assertEqual(key, ex.id)
+            self.assertEqual(val, tar)
+
+    def test_collect_existing_structure(self):
+        emap = EntityMapping()
+        reca1 = db.Record(name="Animals", id=100)
+        reca2 = db.Record(name="Dogs", id=200)
+        reca3 = db.Record(name="Husky", id=300)
+        reca1.add_property(id=101, name="Cute Animals", datatype=db.REFERENCE, value=reca2)
+        reca2.add_property(id=201, name="Cute Dogs", datatype=db.REFERENCE, value=reca3)
+
+        recb1 = db.Record(name="Animals")
+        recb2 = db.Record(name="Dogs")
+        recb3 = db.Record(name="Husky")
+        recb1.add_property(name="Cute Animals", datatype=db.REFERENCE, value=recb2)
+        recb2.add_property(name="Cute Dogs", datatype=db.REFERENCE, value=recb3)
+
+        collect_existing_structure(recb1, reca1, emap)
+
+        # Test if the two dicts of the entity mapping correctly depend on each other
+
+        for i in emap.to_existing.keys():
+            self.assertEqual(i, emap.to_target[emap.to_existing[i].id]._cuid)
+
+        for j in emap.to_target.keys():
+            self.assertEqual(j, emap.to_existing[emap.to_target[j]._cuid].id)
+
+        # Test if only the right Properties are in the dicts
+        self.assertTrue((reca2 in emap.to_existing.values()) and
+                        (reca3 in emap.to_existing.values()) and
+                        (reca1 not in emap.to_existing.values()))
+        self.assertTrue((recb2 in emap.to_target.values()) and
+                        (recb3 in emap.to_target.values()) and
+                        (recb1 not in emap.to_target.values()))
+
+        # Test the correct assignment of the properties
+        self.assertTrue(reca2 is emap.to_existing[recb2._cuid])
+        self.assertTrue(reca3 is emap.to_existing[recb3._cuid])
+
+        self.assertTrue(recb2 is emap.to_target[reca2.id])
+        self.assertTrue(recb3 is emap.to_target[reca3.id])
+
+        """Test with one additional Property and Properties, which are not Records"""
+        emap2 = EntityMapping()
+        recc1 = db.Record(name="Transportation", id=100)
+        recc2 = db.Record(name="Cars", id=200)
+        recc3 = db.Record(name="Volvo", id=300)
+        recc1.add_property(id=101, name="Type", datatype=db.REFERENCE, value=recc2)
+        recc2.add_property(id=201, name="Brand", datatype=db.REFERENCE, value=recc3)
+        # other datatypes
+        recc3.add_property(id=301, name="max_speed", value=200.2, datatype=db.DOUBLE)
+        recc3.add_property(id=302, name="doors", value=3, datatype=db.INTEGER)
+
+        recd1 = db.Record(name="Transportation")
+        recd2 = db.Record(name="Cars")
+        recd3 = db.Record(name="Volvo")
+        recd4 = db.Record(name="VW")
+        recd1.add_property(name="Type", datatype=db.REFERENCE, value=recd2)
+        recd2.add_property(name="Brand", datatype=db.REFERENCE, value=recd3)
+        # additional Property
+        recd2.add_property(name="Another Brand", datatype=db.REFERENCE, value=recd4)
+        # other datatypes
+        recd3.add_property(name="max_speed", value=200.2, datatype=db.DOUBLE)
+        recd3.add_property(name="doors", value=3, datatype=db.INTEGER)
+        recd4.add_property(name="max_speed", value=210.4, datatype=db.DOUBLE)
+        recd4.add_property(name="doors", value=5, datatype=db.INTEGER)
+        recd4.add_property(name="Warp engine", value=None)
+
+        collect_existing_structure(recd1, recc1, emap2)
+
+        # Test the correct assignment of the properties
+        self.assertTrue(recc2 is emap2.to_existing[recd2._cuid])
+        self.assertTrue(recc3 is emap2.to_existing[recd3._cuid])
+
+        self.assertTrue(recd2 is emap2.to_target[recc2.id])
+        self.assertTrue(recd3 is emap2.to_target[recc3.id])
+
+        """ Test, if the Record `Cars` in `target_structure` have one additional Property """
+        # Test existing structure
+        self.assertEqual(len(recc2.get_properties()), 1)  # number of properties stay unchanged
+        self.assertEqual(len(recd2.get_properties()), 2)  # number of properties stay unchanged
+
+        for prop_record, prop_em in zip(recc2.get_properties(), recd2.get_properties()):
+            self.assertTrue(prop_record.value is emap2.to_existing[prop_em.value._cuid])
+
+        # Test target structure
+        self.assertEqual(len(recc3.get_properties()), 2)  # number of properties stay unchanged
+        self.assertEqual(len(recd3.get_properties()), 2)  # number of properties stay unchanged
+
+        """ Test if the Properties that are not References show up in the entity map """
+        for rec_existing, rec_target in zip(emap2.to_existing.values(), emap2.to_target.values()):
+            self.assertTrue(isinstance(rec_existing, db.Record))
+            self.assertTrue(isinstance(rec_target, db.Record))