diff --git a/CHANGELOG.md b/CHANGELOG.md index 88ea70fbfcc9061579d4123b600634d86016944f..4fe401389b5201894f22ad7a829e820f01c8958f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,8 +19,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - ZipFileConverter that opens zip files and exposes their contents as File and Directory structure elements. - `linkahead-crawler` script as alias for `caosdb-crawler`. -- New transformers of the form `cast_to_*` which allow casting variables to `int`, `float`, - `str` and `bool`. +- New transformers of the form `cast_to_*` which allow casting + variables to `int`, `float`, `str` and `bool`. +- Transformer function definition in the cfood support variable + substitutions now. ### Changed ### diff --git a/src/caoscrawler/converters/converters.py b/src/caoscrawler/converters/converters.py index 3a3c7e292a2967cab68228e820fea6880302be89..d06415f78df2949dfee5a7a352b631e4a0b0264f 100644 --- a/src/caoscrawler/converters/converters.py +++ b/src/caoscrawler/converters/converters.py @@ -575,10 +575,19 @@ class Converter(object, metaclass=ABCMeta): " one element with they key being the name" " of the function!") tr_func_key = list(tr_func_el.keys())[0] - tr_func_params = tr_func_el[tr_func_key] + if tr_func_key not in transformer_functions: raise RuntimeError("Unknown transformer function: {}".format(tr_func_key)) + # Do variable replacment on function parameters: + if tr_func_el[tr_func_key] is not None: + # Create a copy of the function parameters: + tr_func_params = dict(tr_func_el[tr_func_key]) + for key in tr_func_params: + tr_func_params[key] = replace_variables(tr_func_params[key], values) + else: + tr_func_params = None + # Retrieve the function from the dictionary: tr_func = transformer_functions[tr_func_key] # Call the function: diff --git a/src/doc/converters/transform_functions.rst b/src/doc/converters/transform_functions.rst index 22df35c8521ea0d70b2ebf7b7c8bc7c52e176bd3..ecd47d2dc004c6f1382279901dfec2d96e0e4a2d 100644 --- a/src/doc/converters/transform_functions.rst +++ b/src/doc/converters/transform_functions.rst @@ -38,8 +38,33 @@ An example that splits the variable ``a`` and puts the generated list in ``b`` i Report: tags: $b -This splits the string in '$a' and stores the resulting list in '$b'. This is here used to add a -list valued property to the Report Record. +This splits the string in '$a' and stores the resulting list in +'$b'. This is here used to add a list valued property to the Report +Record. Note that from LinkAhead Crawler 0.11.0 onwards, the value of +``marker`` in the above example can also be read in from a variable in +the usual ``$`` notation: + +.. code-block:: yaml + + # ... variable ``separator`` is defined somewhere above this part, e.g., + # by reading a config file. + Experiment: + type: Dict + match: ".*" + transform: + param_split: + in: $a + out: $b + functions: + - split: + marker: $separator # Now the separator is read in from a + # variable, so we can, e.g., change from + # '|' to ';' without changing the cfood + # definition. + records: + Report: + tags: $b + There are a number of transform functions that are defined by default (see diff --git a/unittests/test_transformers.py b/unittests/test_transformers.py index 5a1a9cfacca1450158e7fb1ba5330f16eca75169..6b1c4655d7e28a4949d29ce6e344d104c9f74ced 100644 --- a/unittests/test_transformers.py +++ b/unittests/test_transformers.py @@ -30,11 +30,9 @@ See: https://gitlab.indiscale.com/caosdb/src/caosdb-crawler/-/issues/107 import importlib from pathlib import Path -from unittest.mock import MagicMock, Mock, patch +from unittest.mock import Mock -import linkahead as db import pytest -import yaml from caoscrawler.converters import Converter, ListElementConverter from caoscrawler.scanner import create_transformer_registry, scan_directory from caoscrawler.stores import GeneralStore @@ -194,3 +192,23 @@ def test_cast_transformer_functions(): cast_to_float("24dsf", {}) assert cast_to_str(24, {}) == "24" + + +def test_replace_variables(): + vals = GeneralStore() + vals["test"] = "with" + vals["a"] = "str_without_replacement" + conv = Mock() + conv.definition = {} + conv.definition["transform"] = { + "test": { + "in": "$a", + "out": "$a", + "functions": [ + {"replace": { + "remove": "without", + "insert": "$test" + }} + ]}} + Converter.apply_transformers(conv, vals, {"replace": replace}) + assert vals["a"] == "str_with_replacement"