WIP: tests pass

src/caoscrawler/crawl.py

@@ -45,6 +45,7 @@ from caosadvancedtools.crawler import Crawler as OldCrawler
 from caosdb.common.datatype import is_reference
 from .stores import GeneralStore, RecordStore
 from .identified_cache import IdentifiedCache
+from .identifiable import Identifiable
 from .structure_elements import StructureElement, Directory
 from .converters import Converter, DirectoryConverter
 from .identifiable_adapters import (IdentifiableAdapter,
@@ -484,16 +485,18 @@ class Crawler(object):
 
         return self._synchronize(self.crawled_data, commit_changes, unique_names=unique_names)
 
-    def has_reference_value_without_id(self, record: Identifiable):
+    def has_reference_value_without_id(self, ident: Identifiable):
         """
-        Returns True if there is at least one property in `record` which:
+        Returns True if there is at least one property in `ident` which:
         a) is a reference property AND
         b) where the value is set to a db.Entity (instead of an ID) AND
         c) where the ID of the value (the db.Entity object in b)) is not set (to an integer)
 
         Returns False otherwise.
         """
-        for pname, pvalue in record.properties.items():
+        if ident is None:
+            return True
+        for pname, pvalue in ident.properties.items():
             if isinstance(pvalue, list):
                 for el in pvalue:
                     if isinstance(el, db.Entity) and el.id is None:
@@ -525,14 +528,16 @@ class Crawler(object):
                         flat.append(p.value)
                         Crawler.create_flat_list([p.value], flat)
 
-    def has_missing_object_in_references(self, record: Identifiable):
+    def has_missing_object_in_references(self, ident: Identifiable):
         """
         returns False if any property value is a db.Entity object that
-        is contained in the `remote_missing_cache`. If the record has such an object in the
+        is contained in the `remote_missing_cache`. If ident has such an object in the
         reference properties, it means that it references another Entity, where we checked
         whether it exists remotely and it was not found.
         """
-        for pname, pvalue in record.properties.items():
+        if ident is None:
+            return True
+        for pname, pvalue in ident.properties.items():
             # if (is_reference(p)
             # Entity instead of ID and not cached locally
             if (isinstance(pvalue, list)):
@@ -540,7 +545,7 @@ class Crawler(object):
                     if (isinstance(el, db.Entity)
                             and self.get_from_remote_missing_cache(el) is not None):
                         return True
-            if (isinstance(p.value, db.Entity)
+            if (isinstance(pvalue, db.Entity)
                     and self.get_from_remote_missing_cache(pvalue) is not None):
                 # might be checked when reference is resolved
                 return True
@@ -611,9 +616,7 @@ class Crawler(object):
             raise RuntimeError("Should not happen.")
         identifiable = self.identifiableAdapter.get_identifiable(record)
         if identifiable is None:
-            # TODO: check whether the same idea as below works here
-            identifiable = record
-            # return None
+            return None
 
         if identifiable in self.remote_existing_cache:
             return self.remote_existing_cache[identifiable]
@@ -670,7 +673,7 @@ class Crawler(object):
             # TODO: check whether that holds:
             #       if there is no identifiable, for the cache that is the same
             #       as if the complete entity is the identifiable:
-            identifiable = record
+            return
         cache.add(identifiable=identifiable, record=record)
 
     @staticmethod
@@ -728,6 +731,7 @@ class Crawler(object):
 
                 # can we check whether the record(identifiable) exists on the remote server?
                 elif not self.has_reference_value_without_id(
+                    # TODO move get_identifiable above if else?
                         self.identifiableAdapter.get_identifiable(record)):
                     # TODO: remove deepcopy?
                     identified_record = (

src/caoscrawler/identifiable.py

@@ -20,6 +20,7 @@
 #
 
 from __future__ import annotations
+from typing import Union
 
 
 class Identifiable():
@@ -42,7 +43,7 @@ class Identifiable():
     """
 
     def __init__(self, record_type: str = None, name: str = None, properties: dict = None,
-                 path: str = None, backrefs: list[int, str] = None):
+                 path: str = None, backrefs: list[Union[int, str]] = None):
         self.record_type = record_type
         self.name = name
         self.properties: dict = {}

src/caoscrawler/identifiable_adapters.py

@@ -95,7 +95,9 @@ class IdentifiableAdapter(metaclass=ABCMeta):
         uses the properties of ident to create a query that can determine
         whether the required record already exists.
         """
-        query_string = "FIND Record " + ident.record_type
+        query_string = "FIND Record "
+        if ident.record_type is not None:
+            query_string += ident.record_type
 
         query_string += " WITH "
 
@@ -166,6 +168,8 @@ class IdentifiableAdapter(metaclass=ABCMeta):
         registered_identifiable = self.get_registered_identifiable(record)
 
         if registered_identifiable is None:
+            if path is not None:
+                return Identifiable(path=path)
             return None
         if len(registered_identifiable.parents) != 1:
             raise RuntimeError("Multiple parents for identifiables"
@@ -326,7 +330,8 @@ class LocalStorageIdentifiableAdapter(IdentifiableAdapter):
         record is the record from the local database to check against.
         identifiable is the record that was created during the crawler run.
         """
-        if not has_parent(record, identifiable.record_type):
+        if (identifiable.record_type is not None
+                and not has_parent(record, identifiable.record_type)):
             return False
         for propname, propvalue in identifiable.properties.items():
             prop_record = record.get_property(propname)
@@ -419,6 +424,8 @@ class CaosDBIdentifiableAdapter(IdentifiableAdapter):
         It is assumed, that there is exactly one identifiable for each RecordType. Only the first
         parent of the given Record is considered; others are ignored
         """
+        if len(record.parents) == 0:
+            return None
         rt_name = record.parents[0].name
         for name, definition in self._registered_identifiables.items():
             if definition.parents[0].name.lower() == rt_name.lower():