diff --git a/src/caosadvancedtools/crawler.py b/src/caosadvancedtools/crawler.py
index 4e88de1309d145d94f2a96eaa40648baecb987c6..91b279fc54619952b7597c4e8a8ba5906468ebd8 100644
--- a/src/caosadvancedtools/crawler.py
+++ b/src/caosadvancedtools/crawler.py
@@ -55,7 +55,7 @@ from .datainconsistency import DataInconsistencyError
from .datamodel_problems import DataModelProblems
from .guard import RETRIEVE, ProhibitedException
from .guard import global_guard as guard
-from .suppressable import Suppressable
+from .suppressKnown import SuppressKnown
logger = logging.getLogger(__name__)
@@ -96,12 +96,12 @@ class Crawler(object):
self.debug_file = debug_file
self.abort_on_exception = abort_on_exception
self.update_cache = UpdateCache()
-
- self.sup = Suppressable(logger=logger)
+ self.filterKnown = SuppressKnown()
+ logger.addFilter(self.filterKnown)
if hideKnown is False:
for cat in ["matches", "inconsistency"]:
- self.sup.reset(cat)
+ self.filterKnown.reset(cat)
if self.use_cache:
self.cache = Cache(db_file=cache_file)
@@ -224,7 +224,8 @@ class Crawler(object):
msg = ("The crawler has no matching rules for and is thus "
"ignoring:\n{}".format(item))
- self.sup.warning(msg, identifier=item, category="matches")
+ logging.warning(msg, extra={"identifier": str(item),
+ 'category': "matches"})
if len(matches[idx]) > 1:
msg = ("Attention: More than one matching cfood!\n"
@@ -232,7 +233,8 @@ class Crawler(object):
+ "\tRecordTypes:\t" + ", ".join(
matches[idx])+"\n")
- self.sup.warning(msg, identifier=item, category="matches")
+ logging.warning(msg, extra={"identifier": str(item),
+ 'category': "matches"})
def cached_find_identifiables(self, identifiables):
if self.use_cache:
diff --git a/src/caosadvancedtools/suppressable.py b/src/caosadvancedtools/suppressKnown.py
similarity index 57%
rename from src/caosadvancedtools/suppressable.py
rename to src/caosadvancedtools/suppressKnown.py
index f0ef9f63f051e91f73720de6bef003e9a891709a..127ed04e9a7a0a53545627e90e766dc0a57ad5a7 100644
--- a/src/caosadvancedtools/suppressable.py
+++ b/src/caosadvancedtools/suppressKnown.py
@@ -6,18 +6,28 @@ import sqlite3
from hashlib import sha256
-class Suppressable(object):
+class SuppressKnown(logging.Filter):
+ """
+ This filter allows to suppress log messages that were shown before.
+
+ The python logging module can be used as normal. This Filter needs to be
+ added to the appropriate Logger and logging calls (e.g. to warning, info
+ etc.) need to have an additional `extra` argument.
+ This argument should be a dict that contains an identifier and a category.
+ Example: `extra={"identifier":"<Record>something</Record>",
+ category="entities"}
+ The identifier is used to check whether a message was shown before and
+ should be a string. The category can be used to remove a specific group of
+ messages from memory and the logger would show those messages again even
+ when they are known.
+ """
+
def __init__(self, db_file=None, logger=None):
if db_file:
self.db_file = db_file
else:
self.db_file = "/tmp/caosadvanced_suppressed_cache.db"
- if logger:
- self.logger = logger
- else:
- self.logger = logging.getLogger()
-
if not os.path.exists(self.db_file):
self.create_cache()
@@ -57,32 +67,19 @@ class Suppressable(object):
else:
return True
- def warning(self, txt, identifier, category):
- self.msg(txt, identifier, "warning", category)
-
- def debug(self, txt, identifier, category):
- self.msg(txt, identifier, "debug", category)
-
- def info(self, txt, identifier, category):
- self.msg(txt, identifier, "info", category)
-
- def error(self, txt, identifier, category):
- self.msg(txt, identifier, "error", category)
-
def hash(self, txt, identifier):
return sha256((txt+str(identifier)).encode("utf-8")).hexdigest()
- def msg(self, txt, identifier, kind, category):
- if self.was_tagged(self.hash(txt, identifier)):
- return
+ def filter(self, record):
+ if not hasattr(record, "identifier"):
+ return 1
+
+ if not hasattr(record, "category"):
+ return 1
+
+ if self.was_tagged(self.hash(record.getMessage(), record.identifier)):
+ return 0
- if kind == "debug":
- self.logger.debug(txt)
- elif kind == "info":
- self.logger.info(txt)
- elif kind == "warning":
- self.logger.warning(txt)
- elif kind == "error":
- self.logger.error(txt)
+ self.tag_msg(record.getMessage(), record.identifier, record.category)
- self.tag_msg(txt, identifier, category)
+ return 1
diff --git a/unittests/test_suppressable.py b/unittests/test_suppressKnown.py
similarity index 77%
rename from unittests/test_suppressable.py
rename to unittests/test_suppressKnown.py
index 8c27b11cfc1bf4ddbc550be9618a44e7187f388a..07c2e18ead500f26556e2400611a0357f26f169f 100644
--- a/unittests/test_suppressable.py
+++ b/unittests/test_suppressKnown.py
@@ -27,20 +27,29 @@ import os
import unittest
from tempfile import NamedTemporaryFile
-from caosadvancedtools.suppressable import Suppressable
+from caosadvancedtools.suppressKnown import SuppressKnown
+
+
+class Record(object):
+ identifier = "5"
+ category = "test"
+
+ def getMessage(self):
+ return "hi"
class SupTestBasic(unittest.TestCase):
def setUp(self):
self.db_file = "/tmp/test_suppress_msg_db_file.db"
- self.basic = Suppressable(db_file=self.db_file)
+ self.basic = SuppressKnown(db_file=self.db_file)
def test_msg(self):
+ r = Record()
# testing basic setup
- self.basic.info("hi", 5, "test")
+ self.basic.filter(r)
digest = self.basic.hash("hi", 5)
assert self.basic.was_tagged(digest)
- self.basic.info("hi", 5, "test")
+ self.basic.filter(r)
def tearDown(self):
os.remove(self.db_file)
@@ -49,7 +58,7 @@ class SupTestBasic(unittest.TestCase):
class SupTestAdvanced(SupTestBasic):
def setUp(self):
self.db_file = "/tmp/test_suppress_msg_db_file.db"
- self.basic = Suppressable(db_file=self.db_file)
+ self.basic = SuppressKnown(db_file=self.db_file)
def test_logger(self):
"""
@@ -60,10 +69,10 @@ class SupTestAdvanced(SupTestBasic):
logger = logging.getLogger()
logger.addHandler(logging.FileHandler(logfile.name))
logger.setLevel(logging.DEBUG)
+ sup = SuppressKnown(db_file=self.db_file)
+ logger.addFilter(sup)
- suppressable = Suppressable(db_file=self.db_file, logger=logger)
-
- suppressable.info("hi", 5, "test")
+ logger.info("hi", extra={"identifier": "5", 'category': "test"})
with open(logfile.name) as lf:
log = lf.read()
# assert that the log was written
@@ -72,7 +81,7 @@ class SupTestAdvanced(SupTestBasic):
assert log.count("\n") == 1
# the following is unchanged and should be suppressed
- suppressable.info("hi", 5, "test")
+ logger.info("hi", extra={"identifier": "5", 'category': "test"})
with open(logfile.name) as lf:
log = lf.read()
# one line with one hi
@@ -80,21 +89,21 @@ class SupTestAdvanced(SupTestBasic):
assert log.count("\n") == 1
# the following is a new message and should thus not be suppressed
- suppressable.info("ha", 5, "new")
+ logger.info("ha", extra={"identifier": "5", 'category': "new"})
with open(logfile.name) as lf:
log = lf.read()
assert log.count("ha") == 1
assert log.count("\n") == 2
# the following has a identifier and should thus not be suppressed
- suppressable.info("hi", 6, "test")
+ logger.info("hi", extra={"identifier": "6", 'category': "test"})
with open(logfile.name) as lf:
log = lf.read()
assert log.count("hi") == 2
assert log.count("\n") == 3
# the following should be suppressed again
- suppressable.info("ha", 5, "new")
+ logger.info("ha", extra={"identifier": "5", 'category': "new"})
with open(logfile.name) as lf:
log = lf.read()
assert log.count("ha") == 1
@@ -102,17 +111,17 @@ class SupTestAdvanced(SupTestBasic):
assert log.count("\n") == 3
# resetting test category; hi should be removed
- suppressable.reset("test")
+ sup.reset("test")
# hi should not be suppressed
- suppressable.info("hi", 5, "test")
+ logger.info("hi", extra={"identifier": "5", 'category': "test"})
with open(logfile.name) as lf:
log = lf.read()
assert log.count("hi") == 3
assert log.count("\n") == 4
# the following should be suppressed still
- suppressable.info("ha", 5, "new")
+ logger.info("ha", extra={"identifier": "5", 'category': "new"})
with open(logfile.name) as lf:
log = lf.read()
assert log.count("ha") == 1