diff --git a/src/caosadvancedtools/crawler.py b/src/caosadvancedtools/crawler.py index 48ff406f8549d1bfca15ca5f1854d927ff196aed..27ff120d105966299b6d47cd96d455071cc275e5 100644 --- a/src/caosadvancedtools/crawler.py +++ b/src/caosadvancedtools/crawler.py @@ -49,7 +49,7 @@ from .verbosity import DEBUG, INFO, VERBOSE class Crawler(object): def __init__(self, food, access=lambda x: x, use_cache=False, - verbosity=INFO): + abort_on_exception=True, verbosity=INFO): """ Parameters ---------- @@ -68,6 +68,7 @@ class Crawler(object): self.report = db.Container() self.use_cache = use_cache self.verbosity = verbosity + self.abort_on_exception = abort_on_exception if self.use_cache: self.cache = Cache() @@ -103,10 +104,13 @@ class Crawler(object): try: cfoods.append(Cfood(crawled_file, access=self.access, verbosity=self.verbosity, - guard=self.guard)) + guard=self.guard)) except Exception as e: traceback.print_exc() print(e) + + if self.abort_on_exception: + raise e errors_occured = True tbs.append(e) @@ -165,6 +169,9 @@ class Crawler(object): except Exception as e: traceback.print_exc() print(e) + + if self.abort_on_exception: + raise e errors_occured = True tbs.append(e)