diff --git a/src/caosadvancedtools/crawler.py b/src/caosadvancedtools/crawler.py index 82ada2fa02cd3b87415493e1c1d5499fc3dc8a82..5c7ff4f954fa939afd359b9d4c3fe5db58396206 100644 --- a/src/caosadvancedtools/crawler.py +++ b/src/caosadvancedtools/crawler.py @@ -66,6 +66,66 @@ def separated(text): return "-"*60 + "\n" + text +def apply_list_of_updates(to_be_updated, update_flags, + update_cache=None, run_id=None): + """ + Updates the to_be_updated Container, i.e. pushes the changes to CaosDB + """ + + if len(to_be_updated) == 0: + return + + get_ids_for_entities_with_names(to_be_updated) + + # remove duplicates + tmp = db.Container() + + for el in to_be_updated: + if el not in tmp: + tmp.append(el) + + to_be_updated = tmp + + info = "UPDATE: updating the following entities\n" + + baseurl = db.configuration.get_config()["Connection"]["url"] + + for el in to_be_updated: + def make_clickable(txt, id): + return "<a href='{}/Entity/{}'>{}</a>".format(baseurl, id, txt) + info += str("\t" + make_clickable(el.name, el.id) + if el.name is not None + else "\t" + make_clickable(str(el.id), el.id)) + info += "\n" + logger.info(info) + + logger.debug(to_be_updated) + try: + if len(to_be_updated) > 0: + logger.info( + "Updating {} Records...".format( + len(to_be_updated))) + guard.safe_update(to_be_updated, unique=False, + flags=update_flags) + except FileNotFoundError as e: + logger.info("Cannot access {}. However, it might be needed for" + " the correct execution".format(e.filename)) + except ProhibitedException: + try: + update_cache.insert(to_be_updated, run_id) + except IntegrityError as e: + logger.warning( + "There were problems with the update of {}.".format( + to_be_updated), + extra={"identifier": str(to_be_updated), + "category": "update-cache"} + ) + logger.debug(traceback.format_exc()) + logger.debug(e) + except Exception as e: + DataModelProblems.evaluate_exception(e) + + class Crawler(object): def __init__(self, cfood_types, use_cache=False, abort_on_exception=True, interactive=True, hideKnown=False, @@ -318,7 +378,11 @@ class Crawler(object): self._cached_find_or_insert_identifiables(cfood.identifiables) cfood.update_identifiables() - self.push_identifiables_to_CaosDB(cfood) + self.apply_list_of_updates( + cfood.to_be_updated, + cfood.update_flags, + update_cache=self.update_cache, + run_id=self.run_id) except FileNotFoundError as e: logger.info("Cannot access {}. However, it might be needed for" " the correct execution".format(e.filename)) @@ -516,64 +580,8 @@ carefully and if the changes are ok, click on the following link: subject="Crawler Update", body=text) - def push_identifiables_to_CaosDB(self, cfood): - """ - Updates the to_be_updated Container, i.e. pushes the changes to CaosDB - """ - - if len(cfood.to_be_updated) == 0: - return - - get_ids_for_entities_with_names(cfood.to_be_updated) - - # remove duplicates - tmp = db.Container() - - for el in cfood.to_be_updated: - if el not in tmp: - tmp.append(el) - - cfood.to_be_updated = tmp - - info = "UPDATE: updating the following entities\n" - - baseurl = db.configuration.get_config()["Connection"]["url"] - for el in cfood.to_be_updated: - def make_clickable(txt, id): - return "<a href='{}/Entity/{}'>{}</a>".format(baseurl, id, txt) - info += str("\t" + make_clickable(el.name, el.id) - if el.name is not None - else "\t" + make_clickable(str(el.id), el.id)) - info += "\n" - logger.info(info) - - logger.debug(cfood.to_be_updated) - try: - if len(cfood.to_be_updated) > 0: - logger.info( - "Updating {} Records...".format( - len(cfood.to_be_updated))) - guard.safe_update(cfood.to_be_updated, unique=False, - flags=cfood.update_flags) - except FileNotFoundError as e: - logger.info("Cannot access {}. However, it might be needed for" - " the correct execution".format(e.filename)) - except ProhibitedException: - try: - self.update_cache.insert(cfood.to_be_updated, self.run_id) - except IntegrityError as e: - logger.warning( - "There were problems with the update of {}.".format( - cfood.to_be_updated), - extra={"identifier": str(cfood.to_be_updated), - "category": "update-cache"} - ) - logger.debug(traceback.format_exc()) - logger.debug(e) - except Exception as e: - DataModelProblems.evaluate_exception(e) - # TODO remove static? + @staticmethod def find_or_insert_identifiables(identifiables): """ Sets the ids of identifiables (that do not have already an id from the