From 3d2e2fe5f775cf4e219b3955a2228268945e3f3f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Henrik=20tom=20W=C3=B6rden?= <henrik@trineo.org>
Date: Sun, 28 Nov 2021 12:05:10 +0100
Subject: [PATCH] MAINT: allow the apply_list_of_updates function to be used
elsewhere
---
src/caosadvancedtools/crawler.py | 124 ++++++++++++++++---------------
1 file changed, 66 insertions(+), 58 deletions(-)
diff --git a/src/caosadvancedtools/crawler.py b/src/caosadvancedtools/crawler.py
index 82ada2fa..5c7ff4f9 100644
--- a/src/caosadvancedtools/crawler.py
+++ b/src/caosadvancedtools/crawler.py
@@ -66,6 +66,66 @@ def separated(text):
return "-"*60 + "\n" + text
+def apply_list_of_updates(to_be_updated, update_flags,
+ update_cache=None, run_id=None):
+ """
+ Updates the to_be_updated Container, i.e. pushes the changes to CaosDB
+ """
+
+ if len(to_be_updated) == 0:
+ return
+
+ get_ids_for_entities_with_names(to_be_updated)
+
+ # remove duplicates
+ tmp = db.Container()
+
+ for el in to_be_updated:
+ if el not in tmp:
+ tmp.append(el)
+
+ to_be_updated = tmp
+
+ info = "UPDATE: updating the following entities\n"
+
+ baseurl = db.configuration.get_config()["Connection"]["url"]
+
+ for el in to_be_updated:
+ def make_clickable(txt, id):
+ return "<a href='{}/Entity/{}'>{}</a>".format(baseurl, id, txt)
+ info += str("\t" + make_clickable(el.name, el.id)
+ if el.name is not None
+ else "\t" + make_clickable(str(el.id), el.id))
+ info += "\n"
+ logger.info(info)
+
+ logger.debug(to_be_updated)
+ try:
+ if len(to_be_updated) > 0:
+ logger.info(
+ "Updating {} Records...".format(
+ len(to_be_updated)))
+ guard.safe_update(to_be_updated, unique=False,
+ flags=update_flags)
+ except FileNotFoundError as e:
+ logger.info("Cannot access {}. However, it might be needed for"
+ " the correct execution".format(e.filename))
+ except ProhibitedException:
+ try:
+ update_cache.insert(to_be_updated, run_id)
+ except IntegrityError as e:
+ logger.warning(
+ "There were problems with the update of {}.".format(
+ to_be_updated),
+ extra={"identifier": str(to_be_updated),
+ "category": "update-cache"}
+ )
+ logger.debug(traceback.format_exc())
+ logger.debug(e)
+ except Exception as e:
+ DataModelProblems.evaluate_exception(e)
+
+
class Crawler(object):
def __init__(self, cfood_types, use_cache=False,
abort_on_exception=True, interactive=True, hideKnown=False,
@@ -318,7 +378,11 @@ class Crawler(object):
self._cached_find_or_insert_identifiables(cfood.identifiables)
cfood.update_identifiables()
- self.push_identifiables_to_CaosDB(cfood)
+ self.apply_list_of_updates(
+ cfood.to_be_updated,
+ cfood.update_flags,
+ update_cache=self.update_cache,
+ run_id=self.run_id)
except FileNotFoundError as e:
logger.info("Cannot access {}. However, it might be needed for"
" the correct execution".format(e.filename))
@@ -516,64 +580,8 @@ carefully and if the changes are ok, click on the following link:
subject="Crawler Update",
body=text)
- def push_identifiables_to_CaosDB(self, cfood):
- """
- Updates the to_be_updated Container, i.e. pushes the changes to CaosDB
- """
-
- if len(cfood.to_be_updated) == 0:
- return
-
- get_ids_for_entities_with_names(cfood.to_be_updated)
-
- # remove duplicates
- tmp = db.Container()
-
- for el in cfood.to_be_updated:
- if el not in tmp:
- tmp.append(el)
-
- cfood.to_be_updated = tmp
-
- info = "UPDATE: updating the following entities\n"
-
- baseurl = db.configuration.get_config()["Connection"]["url"]
- for el in cfood.to_be_updated:
- def make_clickable(txt, id):
- return "<a href='{}/Entity/{}'>{}</a>".format(baseurl, id, txt)
- info += str("\t" + make_clickable(el.name, el.id)
- if el.name is not None
- else "\t" + make_clickable(str(el.id), el.id))
- info += "\n"
- logger.info(info)
-
- logger.debug(cfood.to_be_updated)
- try:
- if len(cfood.to_be_updated) > 0:
- logger.info(
- "Updating {} Records...".format(
- len(cfood.to_be_updated)))
- guard.safe_update(cfood.to_be_updated, unique=False,
- flags=cfood.update_flags)
- except FileNotFoundError as e:
- logger.info("Cannot access {}. However, it might be needed for"
- " the correct execution".format(e.filename))
- except ProhibitedException:
- try:
- self.update_cache.insert(cfood.to_be_updated, self.run_id)
- except IntegrityError as e:
- logger.warning(
- "There were problems with the update of {}.".format(
- cfood.to_be_updated),
- extra={"identifier": str(cfood.to_be_updated),
- "category": "update-cache"}
- )
- logger.debug(traceback.format_exc())
- logger.debug(e)
- except Exception as e:
- DataModelProblems.evaluate_exception(e)
-
# TODO remove static?
+
@staticmethod
def find_or_insert_identifiables(identifiables):
""" Sets the ids of identifiables (that do not have already an id from the
--
GitLab