From b58b599fe37a79a13ae64e69c0a981e157452710 Mon Sep 17 00:00:00 2001
From: florian <f.spreckelsen@inidscale.com>
Date: Thu, 22 Sep 2022 09:24:05 +0200
Subject: [PATCH] STY: autopep'd
---
src/caoscrawler/crawl.py | 34 +++++++++++++++++++++-------------
1 file changed, 21 insertions(+), 13 deletions(-)
diff --git a/src/caoscrawler/crawl.py b/src/caoscrawler/crawl.py
index 9875acc0..1e09d3d8 100644
--- a/src/caoscrawler/crawl.py
+++ b/src/caoscrawler/crawl.py
@@ -823,7 +823,8 @@ class Crawler(object):
if self.identifiableAdapter is None:
raise RuntimeError("Should not happen.")
- to_be_inserted, to_be_updated = self.split_into_inserts_and_updates(target_data)
+ to_be_inserted, to_be_updated = self.split_into_inserts_and_updates(
+ target_data)
# TODO: refactoring of typo
for el in to_be_updated:
@@ -831,14 +832,17 @@ class Crawler(object):
self.replace_entities_with_ids(el)
identified_records = [
- self.identifiableAdapter.retrieve_identified_record_for_record(record)
+ self.identifiableAdapter.retrieve_identified_record_for_record(
+ record)
for record in to_be_updated]
# remove unnecessary updates from list by comparing the target records to the existing ones
self.remove_unnecessary_updates(to_be_updated, identified_records)
if commit_changes:
- self.execute_inserts_in_list(to_be_inserted, self.securityMode, self.run_id)
- self.execute_updates_in_list(to_be_updated, self.securityMode, self.run_id)
+ self.execute_inserts_in_list(
+ to_be_inserted, self.securityMode, self.run_id)
+ self.execute_updates_in_list(
+ to_be_updated, self.securityMode, self.run_id)
update_cache = UpdateCache()
pending_inserts = update_cache.get_inserts(self.run_id)
@@ -859,7 +863,8 @@ class Crawler(object):
# only done in SSS mode
if "SHARED_DIR" in os.environ:
- filename = OldCrawler.save_form([el[3] for el in pending_changes], path, run_id)
+ filename = OldCrawler.save_form(
+ [el[3] for el in pending_changes], path, run_id)
OldCrawler.send_mail([el[3] for el in pending_changes], filename)
for i, el in enumerate(pending_changes):
@@ -870,7 +875,8 @@ UNAUTHORIZED UPDATE ({} of {}):
____________________\n""".format(i + 1, len(pending_changes)) + str(el[3]))
logger.info("There were unauthorized changes (see above). An "
"email was sent to the curator.\n"
- "You can authorize the " + ("inserts" if inserts else "updates")
+ "You can authorize the " +
+ ("inserts" if inserts else "updates")
+ " by invoking the crawler"
" with the run id: {rid}\n".format(rid=run_id))
@@ -1002,13 +1008,13 @@ ____________________\n""".format(i + 1, len(pending_changes)) + str(el[3]))
def crawler_main(crawled_directory_path: str,
- cfood_file_name: str,
- identifiables_definition_file: str = None,
- debug: bool = False,
- provenance_file: str = None,
- dry_run: bool = False,
- prefix: str = "",
- securityMode: int = SecurityMode.UPDATE):
+ cfood_file_name: str,
+ identifiables_definition_file: str = None,
+ debug: bool = False,
+ provenance_file: str = None,
+ dry_run: bool = False,
+ prefix: str = "",
+ securityMode: int = SecurityMode.UPDATE):
"""
Parameters
@@ -1121,6 +1127,7 @@ def parse_args():
return parser.parse_args()
+
def main():
args = parse_args()
@@ -1147,5 +1154,6 @@ def main():
"update": SecurityMode.UPDATE}[args.security_mode]
))
+
if __name__ == "__main__":
main()
--
GitLab