From 524b47c9fccacd160608de41dc7771a9039c63b6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Henrik=20tom=20W=C3=B6rden?= <henrik@trineo.org>
Date: Fri, 20 Sep 2019 17:19:09 +0200
Subject: [PATCH] MAINT: allow defining whether to abort
---
src/caosadvancedtools/crawler.py | 11 +++++++++--
1 file changed, 9 insertions(+), 2 deletions(-)
diff --git a/src/caosadvancedtools/crawler.py b/src/caosadvancedtools/crawler.py
index 48ff406f..27ff120d 100644
--- a/src/caosadvancedtools/crawler.py
+++ b/src/caosadvancedtools/crawler.py
@@ -49,7 +49,7 @@ from .verbosity import DEBUG, INFO, VERBOSE
class Crawler(object):
def __init__(self, food, access=lambda x: x, use_cache=False,
- verbosity=INFO):
+ abort_on_exception=True, verbosity=INFO):
"""
Parameters
----------
@@ -68,6 +68,7 @@ class Crawler(object):
self.report = db.Container()
self.use_cache = use_cache
self.verbosity = verbosity
+ self.abort_on_exception = abort_on_exception
if self.use_cache:
self.cache = Cache()
@@ -103,10 +104,13 @@ class Crawler(object):
try:
cfoods.append(Cfood(crawled_file, access=self.access,
verbosity=self.verbosity,
- guard=self.guard))
+ guard=self.guard))
except Exception as e:
traceback.print_exc()
print(e)
+
+ if self.abort_on_exception:
+ raise e
errors_occured = True
tbs.append(e)
@@ -165,6 +169,9 @@ class Crawler(object):
except Exception as e:
traceback.print_exc()
print(e)
+
+ if self.abort_on_exception:
+ raise e
errors_occured = True
tbs.append(e)
--
GitLab