Skip to content
Snippets Groups Projects
Commit f8a8397d authored by Florian Spreckelsen's avatar Florian Spreckelsen
Browse files

Merge branch 'f-suppress' into 'dev'

Allow suppressing logging

See merge request caosdb/caosdb-advanced-user-tools!30
parents d435eda2 a548a7fd
Branches
Tags
1 merge request!22Release 0.3
...@@ -55,6 +55,7 @@ from .datainconsistency import DataInconsistencyError ...@@ -55,6 +55,7 @@ from .datainconsistency import DataInconsistencyError
from .datamodel_problems import DataModelProblems from .datamodel_problems import DataModelProblems
from .guard import RETRIEVE, ProhibitedException from .guard import RETRIEVE, ProhibitedException
from .guard import global_guard as guard from .guard import global_guard as guard
from .suppressable import Suppressable
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
...@@ -63,30 +64,6 @@ def separated(text): ...@@ -63,30 +64,6 @@ def separated(text):
return "-"*60 + "\n" + text return "-"*60 + "\n" + text
class UnknownCache(object):
def __init__(self, interactive=False, load=False):
if interactive and "y" == input(
"\nDo you want to load filenames that previously were not "
"matched by any CFood?\nIn that case, they will not show up "
"again. (y)"):
load = True
if load and os.path.exists("known_cache.db"):
with open("known_cache.db") as fi:
self.filenames = [el.strip("\n") for el in fi.readlines()]
else:
self.filenames = []
def save(self):
with open("known_cache.db", "w") as fi:
for name in self.filenames:
fi.write(name + "\n")
def add(self, el):
self.filenames.append(el)
class Crawler(object): class Crawler(object):
def __init__(self, cfood_types, use_cache=False, def __init__(self, cfood_types, use_cache=False,
abort_on_exception=True, interactive=True, hideKnown=False, abort_on_exception=True, interactive=True, hideKnown=False,
...@@ -120,6 +97,12 @@ class Crawler(object): ...@@ -120,6 +97,12 @@ class Crawler(object):
self.abort_on_exception = abort_on_exception self.abort_on_exception = abort_on_exception
self.update_cache = UpdateCache() self.update_cache = UpdateCache()
self.sup = Suppressable(logger=logger)
if hideKnown is False:
for cat in ["matches", "inconsistency"]:
self.sup.reset(cat)
if self.use_cache: if self.use_cache:
self.cache = Cache(db_file=cache_file) self.cache = Cache(db_file=cache_file)
...@@ -236,20 +219,12 @@ class Crawler(object): ...@@ -236,20 +219,12 @@ class Crawler(object):
return cfoods, tbs, errors_occured return cfoods, tbs, errors_occured
def check_matches(self, matches): def check_matches(self, matches):
# possibly load previously encountered "Missing matches" and
# "Multiple matches"
ucache = UnknownCache(interactive=self.interactive, load=self.hideKnown)
for idx, item in self.iteritems(): for idx, item in self.iteritems():
if len(matches[idx]) == 0: if len(matches[idx]) == 0:
msg = ("The crawler has no matching rules for and is thus " msg = ("The crawler has no matching rules for and is thus "
"ignoring:\n{}".format(item)) "ignoring:\n{}".format(item))
if item in ucache.filenames: self.sup.warning(msg, identifier=item, category="matches")
logger.debug(msg)
else:
logger.warning(msg)
ucache.add(item)
if len(matches[idx]) > 1: if len(matches[idx]) > 1:
msg = ("Attention: More than one matching cfood!\n" msg = ("Attention: More than one matching cfood!\n"
...@@ -257,14 +232,7 @@ class Crawler(object): ...@@ -257,14 +232,7 @@ class Crawler(object):
+ "\tRecordTypes:\t" + ", ".join( + "\tRecordTypes:\t" + ", ".join(
matches[idx])+"\n") matches[idx])+"\n")
if item in ucache.filenames: self.sup.warning(msg, identifier=item, category="matches")
logger.debug(msg)
else:
logger.warning(msg)
ucache.add(item)
# Save the encountered prblem matches
ucache.save()
def cached_find_identifiables(self, identifiables): def cached_find_identifiables(self, identifiables):
if self.use_cache: if self.use_cache:
......
import logging
import os
import sqlite3
from hashlib import sha256
class Suppressable(object):
def __init__(self, db_file=None, logger=None):
if db_file:
self.db_file = db_file
else:
self.db_file = "/tmp/caosadvanced_suppressed_cache.db"
if logger:
self.logger = logger
else:
self.logger = logging.getLogger()
if not os.path.exists(self.db_file):
self.create_cache()
def create_cache(self):
conn = sqlite3.connect(self.db_file)
c = conn.cursor()
c.execute('''CREATE TABLE messages (digest text primary key, category text)''')
conn.commit()
conn.close()
def tag_msg(self, txt, identifier, category):
digest = self.hash(txt, identifier)
conn = sqlite3.connect(self.db_file)
c = conn.cursor()
c.execute('''INSERT INTO messages VALUES (?,?)''', (digest, category))
conn.commit()
conn.close()
def reset(self, category):
conn = sqlite3.connect(self.db_file)
c = conn.cursor()
c.execute('''DELETE FROM messages WHERE category=?''',
(category,))
conn.commit()
def was_tagged(self, digest):
conn = sqlite3.connect(self.db_file)
c = conn.cursor()
c.execute('''Select * FROM messages WHERE digest=?''',
(digest,))
res = c.fetchone()
conn.commit()
conn.close()
if res is None:
return False
else:
return True
def warning(self, txt, identifier, category):
self.msg(txt, identifier, "warning", category)
def debug(self, txt, identifier, category):
self.msg(txt, identifier, "debug", category)
def info(self, txt, identifier, category):
self.msg(txt, identifier, "info", category)
def error(self, txt, identifier, category):
self.msg(txt, identifier, "error", category)
def hash(self, txt, identifier):
return sha256((txt+str(identifier)).encode("utf-8")).hexdigest()
def msg(self, txt, identifier, kind, category):
if self.was_tagged(self.hash(txt, identifier)):
return
if kind == "debug":
self.logger.debug(txt)
elif kind == "info":
self.logger.info(txt)
elif kind == "warning":
self.logger.warning(txt)
elif kind == "error":
self.logger.error(txt)
self.tag_msg(txt, identifier, category)
#!/usr/bin/env python
# encoding: utf-8
#
# ** header v3.0
# This file is a part of the CaosDB Project.
#
# Copyright (C) 2020 IndiScale GmbH
# Copyright (C) 2020 Henrik tom Wörden
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as
# published by the Free Software Foundation, either version 3 of the
# License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <https://www.gnu.org/licenses/>.
#
# ** end header
import logging
import os
import unittest
from tempfile import NamedTemporaryFile
from caosadvancedtools.suppressable import Suppressable
class SupTestBasic(unittest.TestCase):
def setUp(self):
self.db_file = "/tmp/test_suppress_msg_db_file.db"
self.basic = Suppressable(db_file=self.db_file)
def test_msg(self):
# testing basic setup
self.basic.info("hi", 5, "test")
digest = self.basic.hash("hi", 5)
assert self.basic.was_tagged(digest)
self.basic.info("hi", 5, "test")
def tearDown(self):
os.remove(self.db_file)
class SupTestAdvanced(SupTestBasic):
def setUp(self):
self.db_file = "/tmp/test_suppress_msg_db_file.db"
self.basic = Suppressable(db_file=self.db_file)
def test_logger(self):
"""
The logging output is directed to a file which is then checked whether
the output is as expected.
"""
logfile = NamedTemporaryFile()
logger = logging.getLogger()
logger.addHandler(logging.FileHandler(logfile.name))
logger.setLevel(logging.DEBUG)
suppressable = Suppressable(db_file=self.db_file, logger=logger)
suppressable.info("hi", 5, "test")
with open(logfile.name) as lf:
log = lf.read()
# assert that the log was written
assert "hi" in log
# there should be one line so far
assert log.count("\n") == 1
# the following is unchanged and should be suppressed
suppressable.info("hi", 5, "test")
with open(logfile.name) as lf:
log = lf.read()
# one line with one hi
assert log.count("hi") == 1
assert log.count("\n") == 1
# the following is a new message and should thus not be suppressed
suppressable.info("ha", 5, "new")
with open(logfile.name) as lf:
log = lf.read()
assert log.count("ha") == 1
assert log.count("\n") == 2
# the following has a identifier and should thus not be suppressed
suppressable.info("hi", 6, "test")
with open(logfile.name) as lf:
log = lf.read()
assert log.count("hi") == 2
assert log.count("\n") == 3
# the following should be suppressed again
suppressable.info("ha", 5, "new")
with open(logfile.name) as lf:
log = lf.read()
assert log.count("ha") == 1
assert log.count("hi") == 2
assert log.count("\n") == 3
# resetting test category; hi should be removed
suppressable.reset("test")
# hi should not be suppressed
suppressable.info("hi", 5, "test")
with open(logfile.name) as lf:
log = lf.read()
assert log.count("hi") == 3
assert log.count("\n") == 4
# the following should be suppressed still
suppressable.info("ha", 5, "new")
with open(logfile.name) as lf:
log = lf.read()
assert log.count("ha") == 1
assert log.count("hi") == 3
assert log.count("\n") == 4
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment