diff --git a/src/caoscrawler/converters.py b/src/caoscrawler/converters.py index c690da172ef36f7e1fdbe1fef22b25d18fabc677..6c63dc622b6d26ff1eb1ac69f5143d7b2b61b338 100644 --- a/src/caoscrawler/converters.py +++ b/src/caoscrawler/converters.py @@ -432,6 +432,7 @@ class Converter(object, metaclass=ABCMeta): return for transformer_key, transformer in self.definition["transform"].items(): in_value = replace_variables(transformer["in"], values) + out_value = in_value for tr_func_el in transformer["functions"]: if not isinstance(tr_func_el, dict): diff --git a/unittests/test_transformers.py b/unittests/test_transformers.py index e7fae484c1a0010d952809fc502e25a02bfe6ec5..4ed12751d9052c839aa4db4abd586c419bed1018 100644 --- a/unittests/test_transformers.py +++ b/unittests/test_transformers.py @@ -46,6 +46,38 @@ from pytest import raises UNITTESTDIR = Path(__file__).parent +@pytest.fixture +def converter_registry(): + converter_registry: dict[str, dict[str, str]] = { + "Directory": { + "converter": "DirectoryConverter", + "package": "caoscrawler.converters"}, + "MarkdownFile": { + "converter": "MarkdownFileConverter", + "package": "caoscrawler.converters"}, + "Date": { + "converter": "DateElementConverter", + "package": "caoscrawler.converters"}, + "DictElement": { + "converter": "DictElementConverter", + "package": "caoscrawler.converters"}, + "TextElement": { + "converter": "TextElementConverter", + "package": "caoscrawler.converters"}, + "ListElement": { + "converter": "ListElementConverter", + "package": "caoscrawler.converters"}, + "JSONFile": { + "converter": "JSONFileConverter", + "package": "caoscrawler.converters"}, + } + + for key, value in converter_registry.items(): + module = importlib.import_module(value["package"]) + value["class"] = getattr(module, value["converter"]) + return converter_registry + + def test_simple_transformer(): """ Test the correct list of returned records by the scanner using the @@ -82,38 +114,6 @@ def test_simple_transformer(): assert False -@pytest.fixture -def converter_registry(): - converter_registry: dict[str, dict[str, str]] = { - "Directory": { - "converter": "DirectoryConverter", - "package": "caoscrawler.converters"}, - "MarkdownFile": { - "converter": "MarkdownFileConverter", - "package": "caoscrawler.converters"}, - "Date": { - "converter": "DateElementConverter", - "package": "caoscrawler.converters"}, - "DictElement": { - "converter": "DictElementConverter", - "package": "caoscrawler.converters"}, - "TextElement": { - "converter": "TextElementConverter", - "package": "caoscrawler.converters"}, - "ListElement": { - "converter": "ListElementConverter", - "package": "caoscrawler.converters"}, - "JSONFile": { - "converter": "JSONFileConverter", - "package": "caoscrawler.converters"}, - } - - for key, value in converter_registry.items(): - module = importlib.import_module(value["package"]) - value["class"] = getattr(module, value["converter"]) - return converter_registry - - def test_apply_replace(converter_registry): cfood_def = {"type": 'ListElement', "match_name": ".*", 'transform': {'test': {'in': '$a', 'out': '$b', 'functions': [{ @@ -146,3 +146,21 @@ def test_apply_replace_from_def(converter_registry): conv.apply_transformers(values, transformer_functions) assert values['b'] == "16:45" + + +def test_empty_functions_list(converter_registry): + cfood_def = {"type": 'ListElement', + "match_name": ".*", + 'transform': {'test': {'in': '$a', 'out': '$b', + 'functions': []}}} + values = GeneralStore() + values["a"] = "16_45" + + # transformer_functions = create_transformer_registry(crawler_definition) + transformer_functions = {"replace": replace} + + conv = ListElementConverter(definition=cfood_def, name='test', + converter_registry=converter_registry) + + conv.apply_transformers(values, transformer_functions) + assert values['b'] == "16_45"