Merge branch 'f-logging' into 'dev'

MAINT: replace prints by logging calls

See merge request !97

src/caoscrawler/converters.py

@@ -43,6 +43,8 @@ from string import Template
 import yaml_header_tools
 
 import pandas as pd
+import logging
+
 
 import yaml
 
@@ -51,6 +53,8 @@ import yaml
 SPECIAL_PROPERTIES = ("description", "name", "id", "path",
                       "file", "checksum", "size")
 
+logger = logging.getLogger(__name__)
+
 
 def _only_max(children_with_keys):
 
@@ -364,7 +368,8 @@ class Converter(object, metaclass=ABCMeta):
 
         if rule not in FILTER_FUNCTIONS:
             raise RuntimeError(
-                f"{rule} is not a known filter rule. Only {list(FILTER_FUNCTIONS.keys())} are implemented."
+                f"{rule} is not a known filter rule. Only "
+                f"{list(FILTER_FUNCTIONS.keys())} are implemented."
             )
 
         to_be_filtered = []
@@ -391,19 +396,21 @@ class Converter(object, metaclass=ABCMeta):
         pass
 
     @staticmethod
-    def _debug_matching_template(name: str, regexp: list[str], matched: list[str], result: Optional[dict]):
+    def _debug_matching_template(name: str, regexp: list[str], matched: list[str],
+                                 result: Optional[dict]):
         """ Template for the debugging output for the match function """
-        print("\n--------", name, "-----------")
+        msg = "\n--------" + name + "-----------"
         for re, ma in zip(regexp, matched):
-            print("matching against:\n" + re)
-            print("matching:\n" + ma)
-            print("---------")
+            msg += "matching against:\n" + re
+            msg += "matching:\n" + ma
+            msg += "---------"
         if result is None:
-            print("No match")
+            msg += "No match"
         else:
-            print("Matched groups:")
-            print(result)
-        print("----------------------------------------")
+            msg += "Matched groups:"
+            msg += str(result)
+        msg += "----------------------------------------"
+        logger.debug(msg)
 
     @staticmethod
     def debug_matching(kind=None):

src/caoscrawler/crawl.py

@@ -1266,7 +1266,7 @@ def crawler_main(crawled_directory_path: str,
     try:
         crawler.crawl_directory(crawled_directory_path, cfood_file_name, restricted_path)
     except ConverterValidationError as err:
-        print(err)
+        logger.error(err)
         return 1
     if provenance_file is not None and debug:
         crawler.save_debug_data(provenance_file)

src/caoscrawler/identifiable.py

@@ -25,6 +25,9 @@ from datetime import datetime
 import json
 from hashlib import sha256
 from typing import Union
+import logging
+
+logger = logging.getLogger(__name__)
 
 
 class Identifiable():

src/caoscrawler/identifiable_adapters.py

@@ -33,6 +33,7 @@ import caosdb as db
 import logging
 from abc import abstractmethod, ABCMeta
 from .utils import has_parent
+
 logger = logging.getLogger(__name__)
 
 

unittests/test_converters.py

@@ -25,6 +25,8 @@ test the converters module
 """
 import json
 import yaml
+import logging
+import sys
 import importlib
 import os
 from itertools import product
@@ -371,7 +373,6 @@ def test_filter_children_of_directory(converter_registry, capsys):
     dc = DirectoryConverter(
         definition={
             "match": "(.*)",
-            "debug_match": True,
             "filter": {
                 "expr": "test_(?P<date>[0-9]{4,4}-[0-9]{2,2}-[0-9]{2,2}).json",
                 "group": "date",
@@ -384,14 +385,6 @@ def test_filter_children_of_directory(converter_registry, capsys):
 
     m = dc.match(test_dir)
     assert m is not None
-    # checking debug output
-    captured = capsys.readouterr()
-    # the name
-    assert "examples_filter_children" in captured.out
-    # the regexp
-    assert "(.*)" in captured.out
-    # the empty result set
-    assert "{}" in captured.out
 
     # This should only contain the youngest json and the csv that doesn't match
     # the above filter expression.
@@ -541,7 +534,8 @@ def test_converter_value_match(converter_registry):
     assert m is not None
 
 
-def test_match_debug(converter_registry, capsys):
+def test_match_debug(converter_registry, caplog):
+    caplog.set_level(logging.DEBUG, logger="caoscrawler.converters")
     for m, mn, mv in product([".*", None], [".*", None], [".*", None]):
         defi = {"debug_match": True}
         if m:
@@ -563,14 +557,13 @@ def test_match_debug(converter_registry, capsys):
             mtch = dc.match(IntegerElement(name="a", value=4))
         if not (m is None and mn is None and mv is None):
             assert mtch is not None
-            # checking debug output
-            captured = capsys.readouterr()
             # the name
-            assert "a" in captured.out
+            assert "a" in caplog.text
             # the regexp
-            assert ".*" in captured.out
+            assert ".*" in caplog.text
             # the empty result set
-            assert "{}" in captured.out
+            assert "{}" in caplog.text
+            caplog.clear()
 
 
 def test_date_converter():

unittests/test_tool.py

@@ -25,6 +25,7 @@
 Tests for the tool using pytest
 Adapted from check-sfs
 """
+import logging
 
 from caoscrawler.stores import GeneralStore, RecordStore
 import os
@@ -780,7 +781,8 @@ def crawler_mocked_for_backref_test(crawler):
     return crawler
 
 
-def test_validation_error_print(capsys):
+def test_validation_error_print(caplog):
+    caplog.set_level(logging.DEBUG, logger="caoscrawler.converters")
     # there should be no server interaction since we only test the behavior if a validation error
     # occurs during the data collection stage
     DATADIR = os.path.join(os.path.dirname(__file__), "test_data", "failing_validation")
@@ -792,8 +794,8 @@ def test_validation_error_print(capsys):
                            None,
                            False,
                            "/use_case_simple_presentation")
-        captured = capsys.readouterr()
-        assert "Couldn't validate" in captured.out
+        assert "Couldn't validate" in caplog.text
+        caplog.clear()
 
 
 def test_split_into_inserts_and_updates_backref(crawler_mocked_for_backref_test):