MAINT: allow defining whether to abort

src/caosadvancedtools/crawler.py

@@ -49,7 +49,7 @@ from .verbosity import DEBUG, INFO, VERBOSE
 
 class Crawler(object):
     def __init__(self, food, access=lambda x: x, use_cache=False,
-                 verbosity=INFO):
+                 abort_on_exception=True, verbosity=INFO):
         """
         Parameters
         ----------
@@ -68,6 +68,7 @@ class Crawler(object):
         self.report = db.Container()
         self.use_cache = use_cache
         self.verbosity = verbosity
+        self.abort_on_exception = abort_on_exception
 
         if self.use_cache:
             self.cache = Cache()
@@ -103,10 +104,13 @@ class Crawler(object):
                     try:
                         cfoods.append(Cfood(crawled_file, access=self.access,
                                             verbosity=self.verbosity,
-                                             guard=self.guard))
+                                            guard=self.guard))
                     except Exception as e:
                         traceback.print_exc()
                         print(e)
+
+                        if self.abort_on_exception:
+                            raise e
                         errors_occured = True
                         tbs.append(e)
 
@@ -165,6 +169,9 @@ class Crawler(object):
             except Exception as e:
                 traceback.print_exc()
                 print(e)
+
+                if self.abort_on_exception:
+                    raise e
                 errors_occured = True
                 tbs.append(e)