Skip to content
Snippets Groups Projects
Commit fdf09f0d authored by Henrik tom Wörden's avatar Henrik tom Wörden
Browse files

MAINT: make handler to logging filter

parent 283f8a3d
No related branches found
No related tags found
1 merge request!22Release 0.3
Hide whitespace changes
Inline Side-by-side
src/caosadvancedtools/crawler.py
+ 8
− 6
View file @ fdf09f0d
......@@ -55,7 +55,7 @@ from .datainconsistency import DataInconsistencyError
from .datamodel_problems import DataModelProblems
from .guard import RETRIEVE, ProhibitedException
from .guard import global_guard as guard
from .suppressable import Suppressable
from .suppressKnown import SuppressKnown
logger = logging.getLogger(__name__)
......@@ -96,12 +96,12 @@ class Crawler(object):
self.debug_file = debug_file
self.abort_on_exception = abort_on_exception
self.update_cache = UpdateCache()
self.sup = Suppressable(logger=logger)
self.filterKnown = SuppressKnown()
logger.addFilter(self.filterKnown)
if hideKnown is False:
for cat in ["matches", "inconsistency"]:
self.sup.reset(cat)
self.filterKnown.reset(cat)
if self.use_cache:
self.cache = Cache(db_file=cache_file)
......@@ -224,7 +224,8 @@ class Crawler(object):
msg = ("The crawler has no matching rules for and is thus "
"ignoring:\n{}".format(item))
self.sup.warning(msg, identifier=item, category="matches")
logging.warning(msg, extra={"identifier": str(item),
'category': "matches"})
if len(matches[idx]) > 1:
msg = ("Attention: More than one matching cfood!\n"
......@@ -232,7 +233,8 @@ class Crawler(object):
+ "\tRecordTypes:\t" + ", ".join(
matches[idx])+"\n")
self.sup.warning(msg, identifier=item, category="matches")
logging.warning(msg, extra={"identifier": str(item),
'category': "matches"})
def cached_find_identifiables(self, identifiables):
if self.use_cache:
......
src/caosadvancedtools/suppressable.py → src/caosadvancedtools/suppressKnown.py
+ 27
− 30
View file @ fdf09f0d
......@@ -6,18 +6,28 @@ import sqlite3
from hashlib import sha256
class Suppressable(object):
class SuppressKnown(logging.Filter):
"""
This filter allows to suppress log messages that were shown before.
The python logging module can be used as normal. This Filter needs to be
added to the appropriate Logger and logging calls (e.g. to warning, info
etc.) need to have an additional `extra` argument.
This argument should be a dict that contains an identifier and a category.
Example: `extra={"identifier":"<Record>something</Record>",
category="entities"}
The identifier is used to check whether a message was shown before and
should be a string. The category can be used to remove a specific group of
messages from memory and the logger would show those messages again even
when they are known.
"""
def __init__(self, db_file=None, logger=None):
if db_file:
self.db_file = db_file
else:
self.db_file = "/tmp/caosadvanced_suppressed_cache.db"
if logger:
self.logger = logger
else:
self.logger = logging.getLogger()
if not os.path.exists(self.db_file):
self.create_cache()
......@@ -57,32 +67,19 @@ class Suppressable(object):
else:
return True
def warning(self, txt, identifier, category):
self.msg(txt, identifier, "warning", category)
def debug(self, txt, identifier, category):
self.msg(txt, identifier, "debug", category)
def info(self, txt, identifier, category):
self.msg(txt, identifier, "info", category)
def error(self, txt, identifier, category):
self.msg(txt, identifier, "error", category)
def hash(self, txt, identifier):
return sha256((txt+str(identifier)).encode("utf-8")).hexdigest()
def msg(self, txt, identifier, kind, category):
if self.was_tagged(self.hash(txt, identifier)):
return
def filter(self, record):
if not hasattr(record, "identifier"):
return 1
if not hasattr(record, "category"):
return 1
if self.was_tagged(self.hash(record.getMessage(), record.identifier)):
return 0
if kind == "debug":
self.logger.debug(txt)
elif kind == "info":
self.logger.info(txt)
elif kind == "warning":
self.logger.warning(txt)
elif kind == "error":
self.logger.error(txt)
self.tag_msg(record.getMessage(), record.identifier, record.category)
self.tag_msg(txt, identifier, category)
return 1
unittests/test_suppressable.py → unittests/test_suppressKnown.py
+ 24
− 15
View file @ fdf09f0d
......@@ -27,20 +27,29 @@ import os
import unittest
from tempfile import NamedTemporaryFile
from caosadvancedtools.suppressable import Suppressable
from caosadvancedtools.suppressKnown import SuppressKnown
class Record(object):
identifier = "5"
category = "test"
def getMessage(self):
return "hi"
class SupTestBasic(unittest.TestCase):
def setUp(self):
self.db_file = "/tmp/test_suppress_msg_db_file.db"
self.basic = Suppressable(db_file=self.db_file)
self.basic = SuppressKnown(db_file=self.db_file)
def test_msg(self):
r = Record()
# testing basic setup
self.basic.info("hi", 5, "test")
self.basic.filter(r)
digest = self.basic.hash("hi", 5)
assert self.basic.was_tagged(digest)
self.basic.info("hi", 5, "test")
self.basic.filter(r)
def tearDown(self):
os.remove(self.db_file)
......@@ -49,7 +58,7 @@ class SupTestBasic(unittest.TestCase):
class SupTestAdvanced(SupTestBasic):
def setUp(self):
self.db_file = "/tmp/test_suppress_msg_db_file.db"
self.basic = Suppressable(db_file=self.db_file)
self.basic = SuppressKnown(db_file=self.db_file)
def test_logger(self):
"""
......@@ -60,10 +69,10 @@ class SupTestAdvanced(SupTestBasic):
logger = logging.getLogger()
logger.addHandler(logging.FileHandler(logfile.name))
logger.setLevel(logging.DEBUG)
sup = SuppressKnown(db_file=self.db_file)
logger.addFilter(sup)
suppressable = Suppressable(db_file=self.db_file, logger=logger)
suppressable.info("hi", 5, "test")
logger.info("hi", extra={"identifier": "5", 'category': "test"})
with open(logfile.name) as lf:
log = lf.read()
# assert that the log was written
......@@ -72,7 +81,7 @@ class SupTestAdvanced(SupTestBasic):
assert log.count("\n") == 1
# the following is unchanged and should be suppressed
suppressable.info("hi", 5, "test")
logger.info("hi", extra={"identifier": "5", 'category': "test"})
with open(logfile.name) as lf:
log = lf.read()
# one line with one hi
......@@ -80,21 +89,21 @@ class SupTestAdvanced(SupTestBasic):
assert log.count("\n") == 1
# the following is a new message and should thus not be suppressed
suppressable.info("ha", 5, "new")
logger.info("ha", extra={"identifier": "5", 'category': "new"})
with open(logfile.name) as lf:
log = lf.read()
assert log.count("ha") == 1
assert log.count("\n") == 2
# the following has a identifier and should thus not be suppressed
suppressable.info("hi", 6, "test")
logger.info("hi", extra={"identifier": "6", 'category': "test"})
with open(logfile.name) as lf:
log = lf.read()
assert log.count("hi") == 2
assert log.count("\n") == 3
# the following should be suppressed again
suppressable.info("ha", 5, "new")
logger.info("ha", extra={"identifier": "5", 'category': "new"})
with open(logfile.name) as lf:
log = lf.read()
assert log.count("ha") == 1
......@@ -102,17 +111,17 @@ class SupTestAdvanced(SupTestBasic):
assert log.count("\n") == 3
# resetting test category; hi should be removed
suppressable.reset("test")
sup.reset("test")
# hi should not be suppressed
suppressable.info("hi", 5, "test")
logger.info("hi", extra={"identifier": "5", 'category': "test"})
with open(logfile.name) as lf:
log = lf.read()
assert log.count("hi") == 3
assert log.count("\n") == 4
# the following should be suppressed still
suppressable.info("ha", 5, "new")
logger.info("ha", extra={"identifier": "5", 'category': "new"})
with open(logfile.name) as lf:
log = lf.read()
assert log.count("ha") == 1
......
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment