diff --git a/CHANGELOG.md b/CHANGELOG.md index 77ab1a2bbd479a18a883ec210236e48170513e28..4fe401389b5201894f22ad7a829e820f01c8958f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,7 +19,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - ZipFileConverter that opens zip files and exposes their contents as File and Directory structure elements. - `linkahead-crawler` script as alias for `caosdb-crawler`. -- Transformer function definietion in the cfood support variable +- New transformers of the form `cast_to_*` which allow casting + variables to `int`, `float`, `str` and `bool`. +- Transformer function definition in the cfood support variable substitutions now. ### Changed ### diff --git a/src/caoscrawler/default_transformers.yml b/src/caoscrawler/default_transformers.yml index ffcb1b15bd2bad71083cc8f0ba84172ee3daf2b0..0de9a6e0585c5246fa5a21ffcbdfc37cfdc2b88d 100644 --- a/src/caoscrawler/default_transformers.yml +++ b/src/caoscrawler/default_transformers.yml @@ -15,3 +15,15 @@ date_parse: datetime_parse: package: caoscrawler.transformer_functions function: datetime_parse +cast_to_int: + package: caoscrawler.transformer_functions + function: cast_to_int +cast_to_float: + package: caoscrawler.transformer_functions + function: cast_to_float +cast_to_bool: + package: caoscrawler.transformer_functions + function: cast_to_bool +cast_to_str: + package: caoscrawler.transformer_functions + function: cast_to_str diff --git a/src/caoscrawler/transformer_functions.py b/src/caoscrawler/transformer_functions.py index ce08bc6bc05caa84f342cdc25f3243c5bab0b79c..117d0b021d4ec0b0efc79c5db0d7ed397207933f 100644 --- a/src/caoscrawler/transformer_functions.py +++ b/src/caoscrawler/transformer_functions.py @@ -99,3 +99,56 @@ Parameters fmt = params.get("datetime_format", fmt_default) dt_str = datetime.datetime.strptime(in_value, fmt).strftime(fmt_default) return dt_str + + +def cast_to_int(in_value: Any, params: dict) -> int: + """ + Cast the `in_value` to int. + + Parameters + ========== + No parameters. + """ + return int(in_value) + + +def cast_to_float(in_value: Any, params: dict) -> float: + """ + Cast the `in_value` to float. + + Parameters + ========== + No parameters. + """ + return float(in_value) + + +def cast_to_bool(in_value: Any, params: dict) -> bool: + """ + Cast the `in_value` to bool. + + This is done by comparing `in_value` to "True". + Only "true", "True", "False" and "false" are accepted as possible values. + All other input values raise an error. + + Parameters + ========== + No parameters. + """ + val = str(in_value).lower() + if val == "true": + return True + if val == "false": + return False + raise ValueError("Invalid value for type cast to bool: {}".format(in_value)) + + +def cast_to_str(in_value: Any, params: dict) -> str: + """ + Cast the `in_value` to str. + + Parameters + ========== + No parameters. + """ + return str(in_value) diff --git a/unittests/test_transformers.py b/unittests/test_transformers.py index 301aab50b5d4d1ebb86cd2d4bf86dcfdb6eea6a9..a2d227adc5b0c6a8f2f96cb054e1c7670e980e10 100644 --- a/unittests/test_transformers.py +++ b/unittests/test_transformers.py @@ -36,7 +36,9 @@ import pytest from caoscrawler.converters import Converter, ListElementConverter from caoscrawler.scanner import create_transformer_registry, scan_directory from caoscrawler.stores import GeneralStore -from caoscrawler.transformer_functions import replace +from caoscrawler.transformer_functions import (cast_to_bool, cast_to_float, + cast_to_int, cast_to_str, + replace, split) from pytest import raises UNITTESTDIR = Path(__file__).parent @@ -162,6 +164,38 @@ def test_empty_functions_list(converter_registry): assert values['b'] == "16_45" +def test_cast_transformer_functions(): + for val in ("True", "true", "False", "false"): + assert type(cast_to_bool(val, {})) == bool + if val[1] == "r": + assert cast_to_bool(val, {}) is True + else: + assert cast_to_bool(val, {}) is False + for val_err in ("jaksdlfj", "0", 1): + with pytest.raises(ValueError): + cast_to_bool(val_err, {}) + assert cast_to_bool(False, {}) is False + assert cast_to_bool(True, {}) is True + + assert cast_to_int("24", {}) == 24 + assert cast_to_int(24.0, {}) == 24 + assert cast_to_int(24, {}) == 24 + assert cast_to_int("-24", {}) == -24 + with pytest.raises(ValueError): + cast_to_int("24dsf", {}) + with pytest.raises(ValueError): + cast_to_int("24.0", {}) == 24 + + assert cast_to_float("24", {}) == 24.0 + assert cast_to_float("24.0", {}) == 24.0 + assert cast_to_float(24.0, {}) == 24.0 + assert cast_to_float(24, {}) == 24.0 + with pytest.raises(ValueError): + cast_to_float("24dsf", {}) + + assert cast_to_str(24, {}) == "24" + + def test_replace_variables(): vals = GeneralStore() vals["test"] = "with"