Merge branch 'f-transformer-replace-variables' into 'dev'

Allow variable replacement in parameters of transformer functions

See merge request !204

CHANGELOG.md

@@ -17,6 +17,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - ZipFileConverter that opens zip files and exposes their contents as
   File and Directory structure elements.
 - `linkahead-crawler` script as alias for `caosdb-crawler`.
+- Transformer function definietion in the cfood support variable
+  substitutions now.
 
 ### Changed ###
 

src/caoscrawler/converters/converters.py

@@ -575,10 +575,19 @@ class Converter(object, metaclass=ABCMeta):
                                        " one element with they key being the name"
                                        " of the function!")
                 tr_func_key = list(tr_func_el.keys())[0]
-                tr_func_params = tr_func_el[tr_func_key]
+
                 if tr_func_key not in transformer_functions:
                     raise RuntimeError("Unknown transformer function: {}".format(tr_func_key))
 
+                # Do variable replacment on function parameters:
+                if tr_func_el[tr_func_key] is not None:
+                    # Create a copy of the function parameters:
+                    tr_func_params = dict(tr_func_el[tr_func_key])
+                    for key in tr_func_params:
+                        tr_func_params[key] = replace_variables(tr_func_params[key], values)
+                else:
+                    tr_func_params = None
+
                 # Retrieve the function from the dictionary:
                 tr_func = transformer_functions[tr_func_key]
                 # Call the function:

src/doc/converters/transform_functions.rst

@@ -38,8 +38,33 @@ An example that splits the variable ``a`` and puts the generated list in ``b`` i
 	  Report:
 	    tags: $b
 
-This splits the string in '$a' and stores the resulting list in '$b'. This is here used to add a
-list valued property to the Report Record.
+This splits the string in '$a' and stores the resulting list in
+'$b'. This is here used to add a list valued property to the Report
+Record. Note that from LinkAhead Crawler 0.11.0 onwards, the value of
+``marker`` in the above example can also be read in from a variable in
+the usual ``$`` notation:
+
+.. code-block:: yaml
+
+    # ... variable ``separator`` is defined somewhere above this part, e.g.,
+    # by reading a config file.
+    Experiment:
+	type: Dict
+	match: ".*"
+	transform:
+	  param_split:
+	    in: $a
+	    out: $b
+	    functions:
+	    - split:
+		marker: $separator  # Now the separator is read in from a
+				    # variable, so we can, e.g., change from
+				    # '|' to ';' without changing the cfood
+				    # definition.
+	records:
+	  Report:
+	    tags: $b
+
 
 
 There are a number of transform functions that are defined by default (see

unittests/test_transformers.py

@@ -30,17 +30,14 @@ See: https://gitlab.indiscale.com/caosdb/src/caosdb-crawler/-/issues/107
 
 import importlib
 from pathlib import Path
-from unittest.mock import MagicMock, Mock, patch
+from unittest.mock import Mock
 
-import linkahead as db
 import pytest
-import yaml
-from pytest import raises
-
 from caoscrawler.converters import Converter, ListElementConverter
 from caoscrawler.scanner import create_transformer_registry, scan_directory
 from caoscrawler.stores import GeneralStore
-from caoscrawler.transformer_functions import replace, split
+from caoscrawler.transformer_functions import replace
+from pytest import raises
 
 UNITTESTDIR = Path(__file__).parent
 
@@ -163,3 +160,23 @@ def test_empty_functions_list(converter_registry):
 
     conv.apply_transformers(values, transformer_functions)
     assert values['b'] == "16_45"
+
+
+def test_replace_variables():
+    vals = GeneralStore()
+    vals["test"] = "with"
+    vals["a"] = "str_without_replacement"
+    conv = Mock()
+    conv.definition = {}
+    conv.definition["transform"] = {
+        "test": {
+            "in": "$a",
+            "out": "$a",
+            "functions": [
+                {"replace": {
+                    "remove": "without",
+                    "insert": "$test"
+                }}
+            ]}}
+    Converter.apply_transformers(conv, vals, {"replace": replace})
+    assert vals["a"] == "str_with_replacement"