MAINT: make handler to logging filter

src/caosadvancedtools/crawler.py

@@ -55,7 +55,7 @@ from .datainconsistency import DataInconsistencyError
 from .datamodel_problems import DataModelProblems
 from .guard import RETRIEVE, ProhibitedException
 from .guard import global_guard as guard
-from .suppressable import Suppressable
+from .suppressKnown import SuppressKnown
 
 logger = logging.getLogger(__name__)
 
@@ -96,12 +96,12 @@ class Crawler(object):
         self.debug_file = debug_file
         self.abort_on_exception = abort_on_exception
         self.update_cache = UpdateCache()
-
-        self.sup = Suppressable(logger=logger)
+        self.filterKnown = SuppressKnown()
+        logger.addFilter(self.filterKnown)
 
         if hideKnown is False:
             for cat in ["matches", "inconsistency"]:
-                self.sup.reset(cat)
+                self.filterKnown.reset(cat)
 
         if self.use_cache:
             self.cache = Cache(db_file=cache_file)
@@ -224,7 +224,8 @@ class Crawler(object):
                 msg = ("The crawler has no matching rules for and is thus "
                        "ignoring:\n{}".format(item))
 
-                self.sup.warning(msg, identifier=item, category="matches")
+                logging.warning(msg, extra={"identifier": str(item),
+                                            'category': "matches"})
 
             if len(matches[idx]) > 1:
                 msg = ("Attention: More than one matching cfood!\n"
@@ -232,7 +233,8 @@ class Crawler(object):
                        + "\tRecordTypes:\t" + ", ".join(
                             matches[idx])+"\n")
 
-                self.sup.warning(msg, identifier=item, category="matches")
+                logging.warning(msg, extra={"identifier": str(item),
+                                            'category': "matches"})
 
     def cached_find_identifiables(self, identifiables):
         if self.use_cache:

src/caosadvancedtools/suppressable.py → src/caosadvancedtools/suppressKnown.py

@@ -6,18 +6,28 @@ import sqlite3
 from hashlib import sha256
 
 
-class Suppressable(object):
+class SuppressKnown(logging.Filter):
+    """
+    This filter allows to suppress log messages that were shown before.
+
+    The python logging module can be used as normal. This Filter needs to be
+    added to the appropriate Logger and logging calls (e.g. to warning, info
+    etc.) need to have an additional `extra` argument.
+    This argument should be a dict that contains an identifier and a category.
+    Example: `extra={"identifier":"<Record>something</Record>",
+                     category="entities"}
+    The identifier is used to check whether a message was shown before and
+    should be a string. The category can be used to remove a specific group of
+    messages from memory and the logger would show those messages again even
+    when they are known.
+    """
+
     def __init__(self, db_file=None, logger=None):
         if db_file:
             self.db_file = db_file
         else:
             self.db_file = "/tmp/caosadvanced_suppressed_cache.db"
 
-        if logger:
-            self.logger = logger
-        else:
-            self.logger = logging.getLogger()
-
         if not os.path.exists(self.db_file):
             self.create_cache()
 
@@ -57,32 +67,19 @@ class Suppressable(object):
         else:
             return True
 
-    def warning(self, txt, identifier, category):
-        self.msg(txt, identifier, "warning", category)
-
-    def debug(self, txt, identifier, category):
-        self.msg(txt, identifier, "debug", category)
-
-    def info(self, txt, identifier, category):
-        self.msg(txt, identifier, "info", category)
-
-    def error(self, txt, identifier, category):
-        self.msg(txt, identifier, "error", category)
-
     def hash(self, txt, identifier):
         return sha256((txt+str(identifier)).encode("utf-8")).hexdigest()
 
-    def msg(self, txt, identifier, kind, category):
-        if self.was_tagged(self.hash(txt, identifier)):
-            return
+    def filter(self, record):
+        if not hasattr(record, "identifier"):
+            return 1
+
+        if not hasattr(record, "category"):
+            return 1
+
+        if self.was_tagged(self.hash(record.getMessage(), record.identifier)):
+            return 0
 
-        if kind == "debug":
-            self.logger.debug(txt)
-        elif kind == "info":
-            self.logger.info(txt)
-        elif kind == "warning":
-            self.logger.warning(txt)
-        elif kind == "error":
-            self.logger.error(txt)
+        self.tag_msg(record.getMessage(), record.identifier, record.category)
 
-        self.tag_msg(txt, identifier, category)
+        return 1

unittests/test_suppressable.py → unittests/test_suppressKnown.py

@@ -27,20 +27,29 @@ import os
 import unittest
 from tempfile import NamedTemporaryFile
 
-from caosadvancedtools.suppressable import Suppressable
+from caosadvancedtools.suppressKnown import SuppressKnown
+
+
+class Record(object):
+    identifier = "5"
+    category = "test"
+
+    def getMessage(self):
+        return "hi"
 
 
 class SupTestBasic(unittest.TestCase):
     def setUp(self):
         self.db_file = "/tmp/test_suppress_msg_db_file.db"
-        self.basic = Suppressable(db_file=self.db_file)
+        self.basic = SuppressKnown(db_file=self.db_file)
 
     def test_msg(self):
+        r = Record()
         # testing basic setup
-        self.basic.info("hi", 5, "test")
+        self.basic.filter(r)
         digest = self.basic.hash("hi", 5)
         assert self.basic.was_tagged(digest)
-        self.basic.info("hi", 5, "test")
+        self.basic.filter(r)
 
     def tearDown(self):
         os.remove(self.db_file)
@@ -49,7 +58,7 @@ class SupTestBasic(unittest.TestCase):
 class SupTestAdvanced(SupTestBasic):
     def setUp(self):
         self.db_file = "/tmp/test_suppress_msg_db_file.db"
-        self.basic = Suppressable(db_file=self.db_file)
+        self.basic = SuppressKnown(db_file=self.db_file)
 
     def test_logger(self):
         """
@@ -60,10 +69,10 @@ class SupTestAdvanced(SupTestBasic):
         logger = logging.getLogger()
         logger.addHandler(logging.FileHandler(logfile.name))
         logger.setLevel(logging.DEBUG)
+        sup = SuppressKnown(db_file=self.db_file)
+        logger.addFilter(sup)
 
-        suppressable = Suppressable(db_file=self.db_file, logger=logger)
-
-        suppressable.info("hi", 5, "test")
+        logger.info("hi", extra={"identifier": "5", 'category': "test"})
         with open(logfile.name) as lf:
             log = lf.read()
             # assert that the log was written
@@ -72,7 +81,7 @@ class SupTestAdvanced(SupTestBasic):
             assert log.count("\n") == 1
 
         # the following is unchanged and should be suppressed
-        suppressable.info("hi", 5, "test")
+        logger.info("hi", extra={"identifier": "5", 'category': "test"})
         with open(logfile.name) as lf:
             log = lf.read()
             # one line with one hi
@@ -80,21 +89,21 @@ class SupTestAdvanced(SupTestBasic):
             assert log.count("\n") == 1
 
         # the following is a new message and should thus not be suppressed
-        suppressable.info("ha", 5, "new")
+        logger.info("ha", extra={"identifier": "5", 'category': "new"})
         with open(logfile.name) as lf:
             log = lf.read()
             assert log.count("ha") == 1
             assert log.count("\n") == 2
 
         # the following has a identifier and should thus not be suppressed
-        suppressable.info("hi", 6, "test")
+        logger.info("hi", extra={"identifier": "6", 'category': "test"})
         with open(logfile.name) as lf:
             log = lf.read()
             assert log.count("hi") == 2
             assert log.count("\n") == 3
 
         # the following should be suppressed again
-        suppressable.info("ha", 5, "new")
+        logger.info("ha", extra={"identifier": "5", 'category': "new"})
         with open(logfile.name) as lf:
             log = lf.read()
             assert log.count("ha") == 1
@@ -102,17 +111,17 @@ class SupTestAdvanced(SupTestBasic):
             assert log.count("\n") == 3
 
         # resetting test category; hi should be removed
-        suppressable.reset("test")
+        sup.reset("test")
 
         # hi should not be suppressed
-        suppressable.info("hi", 5, "test")
+        logger.info("hi", extra={"identifier": "5", 'category': "test"})
         with open(logfile.name) as lf:
             log = lf.read()
             assert log.count("hi") == 3
             assert log.count("\n") == 4
 
         # the following should be suppressed still
-        suppressable.info("ha", 5, "new")
+        logger.info("ha", extra={"identifier": "5", 'category': "new"})
         with open(logfile.name) as lf:
             log = lf.read()
             assert log.count("ha") == 1