ENH: add new transform function: replace

src/caoscrawler/default_transformers.yml

@@ -6,3 +6,6 @@ submatch:
 split:
   package: caoscrawler.transformer_functions
   function: split
+replace:
+  package: caoscrawler.transformer_functions
+  function: split

src/caoscrawler/transformer_functions.py

@@ -50,3 +50,14 @@ def split(in_value: Any, in_parameters: dict):
     if not isinstance(in_value, str):
         raise RuntimeError("must be string")
     return in_value.split(in_parameters['marker'])
+
+
+def replace(in_value: Any, in_parameters: dict):
+    """calls the string 'replace' function on the first argument and uses the value of the keys
+    'remove' and 'insert' stored in the second argument
+    """
+    if "remove" not in in_parameters or "insert" not in in_parameters:
+        raise RuntimeError("Mandatory parameter missing.")
+    if not isinstance(in_value, str):
+        raise RuntimeError("must be string")
+    return in_value.replace(in_parameters['remove'], in_parameters['insert'])

unittests/test_converters.py

@@ -51,7 +51,7 @@ from caoscrawler.structure_elements import (BooleanElement, DictElement,
                                             Directory, File, FloatElement,
                                             IntegerElement, ListElement,
                                             TextElement)
-from caoscrawler.transformer_functions import split
+from caoscrawler.transformer_functions import replace, split
 
 UNITTESTDIR = Path(__file__).parent
 
@@ -400,6 +400,23 @@ def test_apply_transformers(converter_registry):
     assert values['a'] == ["a", "b", "c"]
 
 
+def test_apply_replace(converter_registry):
+    cfood_def = {"type": 'ListElement', "match_name": ".*",
+                 'transform': {'test': {'in': '$a', 'out': '$b', 'functions': [{
+                     'replace': {'insert': ':', "remove": "_"}}]}}}
+    values = GeneralStore()
+    values["a"] = "16_45"
+
+    # transformer_functions  = create_transformer_registry(crawler_definition)
+    transformer_functions = {"replace": replace}
+
+    conv = ListElementConverter(definition=cfood_def, name='test',
+                                converter_registry=converter_registry)
+
+    conv.apply_transformers(values, transformer_functions)
+    assert values['b'] == "16:45"
+
+
 def test_filter_children_of_directory(converter_registry, capsys):
     """Verify that children (i.e., files) in a directory are filtered or sorted correctly. """
     test_dir = Directory("examples_filter_children", UNITTESTDIR /