F retrieve

src/newcrawler/crawl.py

@@ -247,6 +247,8 @@ class Crawler(object):
         that the same identifiable exists twice)
         """
         identifiable = self.identifiableAdapter.get_identifiable(record)
+        if identifiable is None:
+            return None
         if identifiable in self.identified_cache:
             return self.identified_cache[identifiable]
         else:
@@ -261,6 +263,8 @@ class Crawler(object):
         that the same identifiable exists twice)
         """
         identifiable = self.identifiableAdapter.get_identifiable(record)
+        if identifiable is None:
+            raise RuntimeError()
         self.identified_cache.add(identifiable=identifiable, record=record)
 
     def copy_attributes(self, fro: db.Entity, to: db.Entity):
@@ -270,9 +274,14 @@ class Crawler(object):
         to_be_inserted = []
         to_be_updated = []
         flat = list(ent_list)
-        # assure all entities are direct members
+        # assure all entities are direct members TODO Can this be removed at some point?Check only?
         self.create_flat_list(ent_list, flat)
 
+        # TODO: can the following be removed at some point
+        for ent in flat:
+            if len(ent.parents) == 0:
+                raise RuntimeError("Records must have a parent.")
+
         resolved_references = True
         # flat contains Entities which could not yet be checked against the remote server
         while resolved_references and len(flat) > 0:
@@ -310,6 +319,8 @@ class Crawler(object):
                         # side effect
                         record.id = identified_record.id
                         to_be_updated.append(record)
+                        # TODO think this through
+                        # self.add_identified_record_to_local_cache(record)
                         del flat[i]
                     resolved_references = True
 
@@ -324,13 +335,22 @@ class Crawler(object):
 
         return to_be_inserted, to_be_updated
 
+    def replace_entities_by_ids(self, rec: db.Record):
+        for el in rec.properties:
+            if isinstance(el.value, db.Entity):
+                el.value = el.value.id
+            elif isinstance(el.value, list):
+                for index, val in enumerate(el.value):
+                    if isinstance(val, db.Entity):
+                        el.value[index] = val.id
+
     def remove_unnecessary_updates(self, updateList: list[db.Record]):
         """
         checks whether all relevant attributes (especially Property values) are equal
         """
         for i in reversed(range(len(updateList))):
             record = updateList[i]
-            identifiable = self.get_identifiable(record)
+            identifiable = self.identifiableAdapter.retrieve_identifiable(record)
 
             comp = compare_entities(record, identifiable)
             identical = True
@@ -382,13 +402,16 @@ class Crawler(object):
         to_be_inserted, to_be_updated = self.split_into_inserts_and_updates(updateList)
 
         # remove unnecessary updates from list
+        for el in to_be_updated:
+            self.replace_entities_by_ids(el)
+
         self.remove_unnecessary_updates(to_be_updated)
 
         # TODO
-        self.execute_inserts_in_list(insertList)
-        self.execute_updates_in_list(updateList)
+        # self.execute_inserts_in_list(to_be_inserted)
+        # self.execute_updates_in_list(to_be_updated)
 
-        return (insertList, updateList)
+        return (to_be_inserted, to_be_updated)
 
     @staticmethod
     def debug_build_usage_tree(converter: Converter):