Merge branch 'f-update-parents-mitigation' into 'dev'

F update parents mitigation

See merge request !19

CHANGELOG.md

@@ -30,6 +30,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 ### Fixed
 
 * FIX: #12
+* FIX: #14
 * FIX: Variables are now also replaced when the value is given as a list.
 * FIX: #35 Parent cannot be set from value
 * [#6](https://gitlab.com/caosdb/caosdb-crawler/-/issues/6): Fixed many type

integrationtests/test_data/extroot/use_case_simple_presentation/cfood.yml

@@ -22,7 +22,8 @@ extroot:
 
           records:
             mdfile:
-              parents: []
+              parents:
+                - mdfile
               role: File
               path: $DataFile
               file: $DataFile
@@ -64,7 +65,8 @@ extroot:
 
           records:
             mdfile:
-              parents: []
+              parents:
+                - mdfile
               role: File
               path: $DataFile
               file: $DataFile

integrationtests/test_data/extroot/use_case_simple_presentation/model.yml

@@ -17,7 +17,6 @@ ScientificActivity:
       datatype: TEXT
     mdfile:
       description: The file storing information about this record.
-      datatype: FILE
 
 Experiment:
   description: |

integrationtests/test_use_case_simple_presentation.py

@@ -34,15 +34,14 @@ from caosadvancedtools.loadFiles import loadpath
 from caosadvancedtools.models import parser as parser
 from caoscrawler.crawl import crawler_main
 
-
-# TODO(fspreck) Re-eneable once this is part of dev in advancedusertools.
-# from caosadvancedtools.testutils import clear_database, set_test_key
+# TODO: wait for release of this feature in pylib
+# from caosdb.utils.register_tests import clear_database, set_test_key
 # set_test_key("10b128cf8a1372f30aa3697466bb55e76974e0c16a599bb44ace88f19c8f61e2")
 
 DATADIR = os.path.join(os.path.dirname(__file__), "test_data",
                        "extroot", "use_case_simple_presentation")
 
-
+# TODO: remove this
 @pytest.fixture
 def clear_database():
     # TODO(fspreck): Remove once the corresponding advancedtools function can be

src/caoscrawler/crawl.py

@@ -793,10 +793,56 @@ class Crawler(object):
             else:
                 pass
 
+    @staticmethod
+    def execute_parent_updates_in_list(to_be_updated, securityMode, run_id, unique_names):
+        """
+        Execute the updates of changed parents.
+
+        This method is used before the standard inserts and needed
+        because some changes in parents (e.g. of Files) might fail
+        if they are not updated first.
+        """
+        Crawler.set_ids_and_datatype_of_parents_and_properties(to_be_updated)
+        parent_updates = db.Container()
+
+        for record in to_be_updated:
+            old_entity = Crawler._get_entity_by_id(record.id)
+
+            # Check whether the parents have been changed and add them if missing
+            # in the old entity:
+            changes_made = False
+            for parent in record.parents:
+                found = False
+                for old_parent in old_entity.parents:
+                    if old_parent.id == parent.id:
+                        found = True
+                        break
+                if not found:
+                    old_entity.add_parent(id=parent.id)
+                    changes_made = True
+            if changes_made:
+                parent_updates.append(old_entity)
+        logger.debug("RecordTypes need to be added to the following entities:")
+        logger.debug(parent_updates)
+        if len(parent_updates) > 0:
+            if securityMode.value > SecurityMode.INSERT.value:
+                parent_updates.update(unique=False)
+            elif run_id is not None:
+                update_cache = UpdateCache()
+                update_cache.insert(parent_updates, run_id)
+                logger.info("Some entities need to be updated because they are missing a parent "
+                            "RecordType. The update was NOT executed due to the chosen security "
+                            "mode. This might lead to a failure of inserts that follow.")
+                logger.info(parent_updates)
+
     @staticmethod
     def _get_entity_by_name(name):
         return db.Entity(name=name).retrieve()
 
+    @staticmethod
+    def _get_entity_by_id(id):
+        return db.Entity(id=id).retrieve()
+
     @staticmethod
     def execute_inserts_in_list(to_be_inserted, securityMode, run_id: int = None,
                                 unique_names=True):
@@ -874,6 +920,8 @@ class Crawler(object):
         self.remove_unnecessary_updates(to_be_updated, identified_records)
 
         if commit_changes:
+            self.execute_parent_updates_in_list(to_be_updated, securityMode=self.securityMode,
+                                                run_id=self.run_id, unique_names=unique_names)
             self.execute_inserts_in_list(
                 to_be_inserted, self.securityMode, self.run_id, unique_names=unique_names)
             self.execute_updates_in_list(

unittests/simulated_server_data.py

 
 import caosdb as db
-data_model = {"person": (db.RecordType(id=10001, name="Person")
+data_model = {"person": (db.RecordType(id=259, name="Person")
                          .add_property(name="first_name")
                          .add_property(name="last_name")),
-              "measurement": (db.RecordType(id=10002, name="Measurement")
+              "measurement": (db.RecordType(id=278, name="Measurement")
                               .add_property(name="identifier")
                               .add_property(name="date")
                               .add_property(name="project")),
-              "project": (db.RecordType(id=10003, name="Project")
+              "project": (db.RecordType(id=250, name="Project")
                           .add_property(name="date")
                           .add_property(name="identifier")),
-              "first_name": db.Property(name="first_name", datatype=db.TEXT, id=10004),
-              "responsible": db.Property(name="responsible", datatype="Person", id=10005),
-              "last_name": db.Property(name="last_name", datatype=db.TEXT, id=10006),
-              "identifier": db.Property(name="identifier", datatype=db.TEXT, id=10007),
-              "date": db.Property(name="date", datatype=db.DATETIME, id=10008),
+              "first_name": db.Property(name="first_name", datatype=db.TEXT, id=261),
+              "responsible": db.Property(name="responsible", datatype="Person", id=249),
+              "last_name": db.Property(name="last_name", datatype=db.TEXT, id=262),
+              "identifier": db.Property(name="identifier", datatype=db.TEXT, id=248),
+              "date": db.Property(name="date", datatype=db.DATETIME, id=247),
               }
 existing_data = {
 }

unittests/test_tool.py

@@ -29,6 +29,12 @@ def rfp(*pathcomponents):
     return join(dirname(__file__), *pathcomponents)
 
 
+ident = LocalStorageIdentifiableAdapter()
+ident.restore_state(rfp("records.xml"))
+full_data.update({el.name: el for el in ident._records if el.name is not None})
+full_data.update({el.id: el for el in ident._records if el.name is None})
+
+
 def dircheckstr(*pathcomponents):
     """
     Return the debug tree identifier for a given path.
@@ -544,12 +550,21 @@ def test_replace_entities_with_ids(crawler):
     assert a.get_property("C").value == [12345, 233324]
 
 
+def mock_get_entity_by_id(id):
+    candidates = [el for el in list(full_data.values()) if el.id == id]
+    if len(candidates) > 0:
+        return candidates[0]
+    else:
+        raise ValueError()
+
+
 def mock_get_entity_by_name(name):
-    candidates = [el for el in full_data.values() if el.name.lower() == name.lower()]
+    candidates = [el for el in full_data.values()
+                  if (el.name is not None and el.name.lower() == name.lower())]
     if len(candidates) > 0:
         return candidates[0]
     else:
-        return None
+        raise ValueError()
 
 
 def prepare_crawler_with_sec_mode(mode, ident):
@@ -587,6 +602,8 @@ def change_non_identifiable_prop(ident):
     del resps.value[-1]
 
 
+@patch("caoscrawler.crawl.Crawler._get_entity_by_id",
+       new=Mock(side_effect=mock_get_entity_by_id))
 @patch("caoscrawler.crawl.Crawler._get_entity_by_name",
        new=Mock(side_effect=mock_get_entity_by_name))
 @patch("caoscrawler.crawl.db.Container.insert")