wip

integrationtests/basic_example/test_basic.py

@@ -139,6 +139,7 @@ def test_single_insertion(clear_database, usemodel, crawler, ident):
     #            xml.remove(xml.find(tag))
     #    f.write(db.common.utils.xml2str(xml))
 
+    breakpoint()
     assert len(ins) == 18
     assert len(ups) == 0
 

src/caoscrawler/crawl.py

@@ -387,10 +387,13 @@ class Crawler(object):
                 # 3. check on the remote server
                 else:
                     st.check_remote_server(se)
+                    print("checked", se.id)
                     if se.id is None:
                         st.set_missing(se)
+                        print("missing")
                     else:
                         st.set_existing(se)
+                        print("exisitng")
                     entity_was_treated = True
 
             # TODO
@@ -737,6 +740,8 @@ class Crawler(object):
         for record in to_be_updated:
             if record.id is not None:
                 # TODO: use cache here?
+                print(record.id)
+                print(record)
                 identified_records.append(cached_get_entity_by(eid=record.id))
             else:
                 raise Exception("Please report a bug: At this stage all records to be updated"

src/caoscrawler/identifiable.py

@@ -96,7 +96,8 @@ class Identifiable():
             if value.id is not None:
                 return str(value.id)
             else:
-                return "PyID=" + str(id(value))
+                print(value)
+                raise RuntimeError("Python Entity without id not allowed")
         elif isinstance(value, list):
             return "[" + ", ".join([Identifiable._value_representation(el) for el in value]) + "]"
         elif (isinstance(value, str) or isinstance(value, int) or isinstance(value, float)
@@ -105,7 +106,7 @@ class Identifiable():
         else:
             raise ValueError(f"Unknown datatype of the value: {value}")
 
-    @staticmethod
+    @ staticmethod
     def _create_hashable_string(identifiable: Identifiable) -> str:
         """
         creates a string from the attributes of an identifiable that can be hashed

src/caoscrawler/identifiable_adapters.py

@@ -284,18 +284,24 @@ startswith: bool, optional
                         )
                     continue
 
-                options = [f.get_property(prop.name) for f in se.fragments
+                options = [f.get_property(prop.name).value for f in se.fragments
                            if f.get_property(prop.name) is not None]
                 if len(options) == 0:
                     raise NotImplementedError(
                         f"The following record is missing an identifying property:\n"
                         f"RECORD\n{se.fragments[0]}\nIdentifying PROPERTY\n{prop.name}"
                     )
-                if not all([f.value == options[0].value for f in options]):
+                for ii, el in enumerate(options):
+                    if isinstance(el, db.Entity):
+                        options[ii] = el.id
+                        if el.id is None:
+                            raise RuntimeError("reference to unchecked in identifiable")
+                    else:
+                        options[ii] = el
+                if not all([f == options[0] for f in options]):
                     raise RuntimeError("differing prop values in fragments")
-                record_prop = options[0]
 
-                identifiable_props[record_prop.name] = record_prop.value
+                identifiable_props[prop.name] = options[0]
                 property_name_list_A.append(prop.name)
 
             # check for multi properties in the record:

src/caoscrawler/semantic_target.py

@@ -87,6 +87,15 @@ class SemanticEntity():
 class SemanticTarget():
     """ models the target structure of Entities as it shall be created by the Crawler
 
+    The target entities are composed using the information of the entity fragments (db.Entity
+    objects) of SemanticEntities. This is information like name, parents and properties and,
+    importantly, references. Those references are typically given by a Python reference to some
+    other db.Entity object. These references are scanned initially and used to create multiple
+    reference maps that track how SemanticEntities reference each other. These maps are kept up to
+    date when SemanticEntities are merged because they are identified with each other. When
+    creating the final list of db.Entity objects (``create_record_lists``) the Python references
+    are updated according to the reference map.
+
     This model should only be manipulated via three functions:
     - make_identifiable: adds an identifiable to a SemanticEntity what possibly allows to merge it
                          with another SemanticEntity
@@ -105,7 +114,8 @@ class SemanticTarget():
         self._identifiable_look_up: Dict[str, SemanticEntity] = {}
         self._missing: Dict[int, SemanticEntity] = {}
         self._existing: Dict[int, SemanticEntity] = {}
-        self._remote_missing_counter = -1  # TODO: I guess we can now get rid of this...
+        # entities that are missing get negative IDs to allow identifiable creation
+        self._remote_missing_counter = -1
 
         # create initial set of SemanticEntities from provided Entity list
         self.se: List[SemanticEntity] = []  # list of all SemanticEntities
@@ -113,7 +123,9 @@ class SemanticTarget():
         self.se_lookup: Dict[str, SemanticEntity] = {}  # lookup: UUID -> SemanticEntity
         entities = self._create_flat_list(entities)
         self._sanity_check(entities)
+        print("ids")
         for el in entities:
+            print(el.id)
             self.se.append(SemanticEntity(
                 el,
                 self.identifiableAdapter.get_registered_identifiable(el)))
@@ -166,6 +178,8 @@ class SemanticTarget():
         if se.path is None and se.identifiable is None:
             raise RuntimeError("no identifying information")
         se.id = self._remote_missing_counter
+        for f in se.fragments:
+            f.id = self._remote_missing_counter
         self._remote_missing_counter -= 1
         self._add_any(se, self._missing)
         self.unchecked.remove(se)
@@ -177,6 +191,7 @@ class SemanticTarget():
         on the remote server.
         """
         assert se.id is not None
+        assert se.id > 0
         self._add_any(se, self._existing)
         self.unchecked.remove(se)
 
@@ -191,6 +206,10 @@ class SemanticTarget():
             return self._id_look_up[entity.id]
         if entity.path is not None and entity.path in self._path_look_up:
             return self._path_look_up[entity.path]
+        for e in self.se:
+            if e.identifiable is not None:
+                print(e.identifiable._create_hashable_string(e.identifiable))
+                print(self._identifiable_look_up)
         if (entity.identifiable is not None and entity.identifiable.get_representation() in
                 self._identifiable_look_up):
             return self._identifiable_look_up[entity.identifiable.get_representation()]
@@ -217,12 +236,21 @@ class SemanticTarget():
         else:
             return None
 
+    def combine_fragments(self):
+        for se in self.se:
+            if len(se.fragments) < 2:
+                continue
+            for ent in se.fragments[1:]:
+                merge_entities(se.fragments[0], ent, merge_id_with_resolved_entity=True)
+
     def create_record_lists(self):
         for se in self.se:
             for f in se.fragments:
                 f.id = se.id
                 f.path = se.path
+        self._update_reference_values()
         self.combine_fragments()
+        # TODO assure that there is only one fragment each
 
         missing = [el.fragments[0] for el in self._missing.values()]
         # remove negative IDs
@@ -235,6 +263,18 @@ class SemanticTarget():
 
         return (missing, [el.fragments[0] for el in self._existing.values()])
 
+    def _update_reference_values(self):
+        for se in self.se:
+            for f in se.fragments:
+                for p in f.properties:
+                    if isinstance(p.value, list):
+                        for index, val in enumerate(p.value):
+                            if id(val) in self.se_lookup:
+                                p.value[index] = self.se_lookup[id(val)].fragments[0]
+                    else:
+                        if id(p.value) in self.se_lookup:
+                            p.value = self.se_lookup[id(p.value)].fragments[0]
+
     def identity_relies_on_unchecked_entity(self, se: SemanticEntity):
         """
         If a record for which it could not yet be verified whether it exists in LA or not is part
@@ -262,14 +302,6 @@ class SemanticTarget():
             if ent.role == "Record" and len(ent.parents) == 0:
                 raise RuntimeError(f"Records must have a parent.\n{ent}")
 
-    def combine_fragments(self):
-        for se in self.se:
-            if len(se.fragments) < 2:
-                continue
-            for ent in se.fragments[1:]:
-                merge_entities(se.fragments[0], ent, merge_id_with_resolved_entity=True)
-            se.fragments = [se.fragments[0]]
-
     @ staticmethod
     def _create_flat_list(ent_list: List[db.Entity], flat: Optional[List[db.Entity]] = None):
         """
@@ -394,8 +426,9 @@ class SemanticTarget():
                                     el.name == p.name]) > 0:
                                 forward_id_references[se.uuid].add(vse)
                                 backward_id_references[vse.uuid].add(se)
-                            if IdentifiableAdapter.referencing_entity_has_appropriate_type(
-                                    ent.parents, vse.registered_identifiable):
+                            if (vse.registered_identifiable is not None and
+                                    IdentifiableAdapter.referencing_entity_has_appropriate_type(
+                                    ent.parents, vse.registered_identifiable)):
                                 forward_id_referenced_by[se.uuid].add(vse)
                                 backward_id_referenced_by[vse.uuid].add(se)
 
@@ -407,6 +440,7 @@ class SemanticTarget():
         """ A path or an ID is sufficiently identifying. Thus, those entities can be marked as
         checked """
         for semantic_entity in list(self.se[::-1]):
+            print(semantic_entity.uuid)
             assert len(semantic_entity.fragments) == 1
             entity = semantic_entity.fragments[0]
             if entity.id is None and entity.path is None:
@@ -419,9 +453,10 @@ class SemanticTarget():
                 if existing is not None:
                     semantic_entity.identify_with(existing)
 
+            # at this point, semantic_entity has an ID if it is existing
             treated_before = self.get_checked_equivalent(semantic_entity)
             if treated_before is None:
-                if semantic_entity.id is None:
+                if semantic_entity.id is None or semantic_entity.id < 0:
                     self.set_missing(semantic_entity)
                 else:
                     self.set_existing(semantic_entity)
@@ -451,23 +486,6 @@ class SemanticTarget():
                 return None
         return circle
 
-    @ staticmethod
-    def _bend_references_to_new_object(old, new, entities):
-        # TODO still needed???
-        """ Bend references to the other object
-        Iterate over all entities in `entities` and check the values of all properties of
-        occurances of old Entity and replace them with new Entity
-        """
-        for el in entities:
-            for p in el.properties:
-                if isinstance(p.value, list):
-                    for index, val in enumerate(p.value):
-                        if val is old:
-                            p.value[index] = new
-                else:
-                    if p.value is old:
-                        p.value = new
-
     def _add_any(self, entity: SemanticEntity, lookup):
         if entity.id is not None:
             self._id_look_up[entity.id] = entity

unittests/test_crawler.py

@@ -413,6 +413,8 @@ def test_split_into_inserts_and_updates_with_copy_attr(crawler_mocked_identifiab
     crawler.identifiableAdapter.retrieve_identified_record_for_identifiable.assert_called()
 
 
+@ patch("caoscrawler.crawl.cached_get_entity_by",
+        new=Mock(side_effect=mock_get_entity_by))
 @patch("caoscrawler.identifiable_adapters.cached_query",
        new=Mock(side_effect=mock_cached_only_rt))
 def test_split_iiau_with_unmergeable_list_items():
@@ -480,7 +482,7 @@ a: ([b1, b2])
 
     crawler = Crawler(identifiableAdapter=ident_adapter)
 
-    st = SemanticTarget([rec_a, *rec_b, *rec_c], crawler.identifiableAdapter)
+    st = SemanticTarget(deepcopy([rec_a, *rec_b, *rec_c]), crawler.identifiableAdapter)
     assert st.identity_relies_on_unchecked_entity(st.se[0]) is False
     assert st.identity_relies_on_unchecked_entity(st.se[1])
     assert st.identity_relies_on_unchecked_entity(st.se[2])

unittests/test_identifiable.py

@@ -54,11 +54,6 @@ def test_create_hashable_string():
             Identifiable(name="A", record_type="B", properties={
                          'a': [db.Record(id=12), 11]})
         ) == "P<B>N<A>R<[]>a:[12, 11]")
-    assert (
-        Identifiable._create_hashable_string(
-            Identifiable(record_type="B", properties={'a': [db.Record()]})
-        ) != Identifiable._create_hashable_string(
-            Identifiable(record_type="B", properties={'a': [db.Record()]})))
     assert Identifiable._create_hashable_string(
         Identifiable(name="A", record_type="B", backrefs=[123, db.Entity(id=124)],
                      properties={'a': 5})) == "P<B>N<A>R<['123', '124']>a:5"

unittests/test_semantic_target.py

@@ -314,11 +314,13 @@ def test_merging():
         db.File(name='101').add_parent("A").add_property('a', value=1),
         db.File(name='101').add_parent("A").add_property('a', value=1)]
     st = SemanticTarget(entlist, ident_adapter)
+    assert len(st.unchecked) == 2
     st.make_identifiable(st.se[0])
     st.check_remote_server(st.se[0])
     st.set_missing(st.se[0])
     assert len(st.unchecked) == 1
     st.make_identifiable(st.se[1])
+    assert st.se[1].id is None
     assert st.merge_with_equivalent(st.se[1])
     assert len(st.se) == 1
     assert len(st.unchecked) == 0