MAINT: allow the apply_list_of_updates function to be used elsewhere

Summary

This extracts the old push_identifiables_to_caosdb function from the crawler and allows other programms to use the logic.

Test Environment

Unit and integration tests

Check List for the Author

Please, prepare your MR for a review. Be sure to write a summary and a focus and create gitlab comments for the reviewer. They should guide the reviewer through the changes, explain your changes and also point out open questions. For further good practices have a look at our review guidelines

• All automated tests pass
• Reference related Issues
• Up-to-date CHANGELOG.md
• Annotations in code (Gitlab comments)
  • Intent of new code
  • Problems with old code
  • Why this implementation?

Check List for the Reviewer

• I understand the intent of this MR
• All automated tests pass
• Up-to-date CHANGELOG.md
• The test environment setup works and the intended behavior is reproducible in the test environment
• In-code documentation and comments are up-to-date.
• Check: Are there spezifications? Are they satisfied?

For further good practices have a look at our review guidelines.

src/caosadvancedtools/crawler.py

@@ -66,6 +66,66 @@ def separated(text):
     return "-"*60 + "\n" + text
 
 
+def apply_list_of_updates(to_be_updated, update_flags,
+                          update_cache=None, run_id=None):
+    """
+    Updates the to_be_updated Container, i.e. pushes the changes to CaosDB
+    """
+
+    if len(to_be_updated) == 0:
+        return
+
+    get_ids_for_entities_with_names(to_be_updated)
+
+    # remove duplicates
+    tmp = db.Container()
+
+    for el in to_be_updated:
+        if el not in tmp:
+            tmp.append(el)
+
+    to_be_updated = tmp
+
+    info = "UPDATE: updating the following entities\n"
+
+    baseurl = db.configuration.get_config()["Connection"]["url"]
+
+    for el in to_be_updated:
+        def make_clickable(txt, id):
+            return "<a href='{}/Entity/{}'>{}</a>".format(baseurl, id, txt)
+        info += str("\t" + make_clickable(el.name, el.id)
+                    if el.name is not None
+                    else "\t" + make_clickable(str(el.id), el.id))
+        info += "\n"
+    logger.info(info)
+
+    logger.debug(to_be_updated)
+    try:
+        if len(to_be_updated) > 0:
+            logger.info(
+                "Updating {} Records...".format(
+                    len(to_be_updated)))
+        guard.safe_update(to_be_updated, unique=False,
+                          flags=update_flags)
+    except FileNotFoundError as e:
+        logger.info("Cannot access {}. However, it might be needed for"
+                    " the correct execution".format(e.filename))
+    except ProhibitedException:
+        try:
+            update_cache.insert(to_be_updated, run_id)
+        except IntegrityError as e:
+            logger.warning(
+                "There were problems with the update of {}.".format(
+                    to_be_updated),
+                extra={"identifier": str(to_be_updated),
+                       "category": "update-cache"}
+            )
+            logger.debug(traceback.format_exc())
+            logger.debug(e)
+    except Exception as e:
+        DataModelProblems.evaluate_exception(e)
+
+
 class Crawler(object):
     def __init__(self, cfood_types, use_cache=False,
                  abort_on_exception=True, interactive=True, hideKnown=False,
@@ -318,7 +378,11 @@ class Crawler(object):
                 self._cached_find_or_insert_identifiables(cfood.identifiables)
 
                 cfood.update_identifiables()
-                self.push_identifiables_to_CaosDB(cfood)
+                apply_list_of_updates(
+                    cfood.to_be_updated,
+                    cfood.update_flags,
+                    update_cache=self.update_cache,
+                    run_id=self.run_id)
             except FileNotFoundError as e:
                 logger.info("Cannot access {}. However, it might be needed for"
                             " the correct execution".format(e.filename))
@@ -516,64 +580,8 @@ carefully and if the changes are ok, click on the following link:
             subject="Crawler Update",
             body=text)
 
-    def push_identifiables_to_CaosDB(self, cfood):
-        """
-        Updates the to_be_updated Container, i.e. pushes the changes to CaosDB
-        """
-
-        if len(cfood.to_be_updated) == 0:
-            return
-
-        get_ids_for_entities_with_names(cfood.to_be_updated)
-
-        # remove duplicates
-        tmp = db.Container()
-
-        for el in cfood.to_be_updated:
-            if el not in tmp:
-                tmp.append(el)
-
-        cfood.to_be_updated = tmp
-
-        info = "UPDATE: updating the following entities\n"
-
-        baseurl = db.configuration.get_config()["Connection"]["url"]
-        for el in cfood.to_be_updated:
-            def make_clickable(txt, id):
-                return "<a href='{}/Entity/{}'>{}</a>".format(baseurl, id, txt)
-            info += str("\t" + make_clickable(el.name, el.id)
-                        if el.name is not None
-                        else "\t" + make_clickable(str(el.id), el.id))
-            info += "\n"
-        logger.info(info)
-
-        logger.debug(cfood.to_be_updated)
-        try:
-            if len(cfood.to_be_updated) > 0:
-                logger.info(
-                    "Updating {} Records...".format(
-                        len(cfood.to_be_updated)))
-            guard.safe_update(cfood.to_be_updated, unique=False,
-                              flags=cfood.update_flags)
-        except FileNotFoundError as e:
-            logger.info("Cannot access {}. However, it might be needed for"
-                        " the correct execution".format(e.filename))
-        except ProhibitedException:
-            try:
-                self.update_cache.insert(cfood.to_be_updated, self.run_id)
-            except IntegrityError as e:
-                logger.warning(
-                    "There were problems with the update of {}.".format(
-                        cfood.to_be_updated),
-                    extra={"identifier": str(cfood.to_be_updated),
-                           "category": "update-cache"}
-                )
-                logger.debug(traceback.format_exc())
-                logger.debug(e)
-        except Exception as e:
-            DataModelProblems.evaluate_exception(e)
-
     # TODO remove static?
+
     @staticmethod
     def find_or_insert_identifiables(identifiables):
         """ Sets the ids of identifiables (that do not have already an id from the