diff --git a/src/caoscrawler/converters.py b/src/caoscrawler/converters.py index 5a3f4a090785f762cad49054570d15a62649bfbe..5dba0726bba5351cb75ab363985daa639a6e56ac 100644 --- a/src/caoscrawler/converters.py +++ b/src/caoscrawler/converters.py @@ -365,10 +365,10 @@ class Converter(object, metaclass=ABCMeta): Parameters: ------------ - + values: GeneralStore The GeneralStore to store values in. - + element: StructureElement The StructureElement to extract values from. """ @@ -382,11 +382,11 @@ class Converter(object, metaclass=ABCMeta): transformer_functions: dict): """ Check if transformers are defined using the "transform" keyword. - The apply the transformers to the variables defined in GeneralStore "values". + Then apply the transformers to the variables defined in GeneralStore "values". Parameters: ------------ - + values: GeneralStore The GeneralStore to store values in. @@ -395,6 +395,9 @@ class Converter(object, metaclass=ABCMeta): """ if "transform" in self.definition: + if not isinstance(self.definition["transform"], dict): + raise RuntimeError("The value corresponding to the 'transform' key in the " + "converter definition must be a dict") for transformer_key, transformer in self.definition["transform"].items(): if "in" not in transformer: raise RuntimeError("In-variable not defined!") @@ -402,7 +405,10 @@ class Converter(object, metaclass=ABCMeta): raise RuntimeError("Out-variable not defined!") if "functions" not in transformer: raise RuntimeError("No functions given for transformer!") - + if not isinstance(self.definition["functions"], list): + raise RuntimeError("The value corresponding to the 'functions' key in the " + "transform section must be a dict") + # Currently overwriting the input variable would be possible. # Shall we prevent that? # Forbid overwriting existing variables at all? @@ -414,7 +420,8 @@ class Converter(object, metaclass=ABCMeta): for tr_func_el in transformer["functions"]: if not isinstance(tr_func_el, dict): - raise RuntimeError("List elements must be dictonaries!") + raise RuntimeError("Elements of the list of the functions key " + "must be dictonaries!") if len(tr_func_el) != 1: raise RuntimeError("List element dictionaries must have exactly" " one element with they key being the name" diff --git a/src/caoscrawler/scanner.py b/src/caoscrawler/scanner.py index 6169de213e4a2d33e0329d4c5ed9299392410d2d..fd82a896d88d6470cfe0883fcf85523892e01ae3 100644 --- a/src/caoscrawler/scanner.py +++ b/src/caoscrawler/scanner.py @@ -36,21 +36,19 @@ import importlib import logging import os import warnings -import yaml +from typing import Any, Optional, Type, Union +import caosdb as db +import yaml from importlib_resources import files from jsonschema import validate -from typing import Any, Optional, Type, Union -import caosdb as db from .converters import Converter - +from .debug_tree import DebugTree from .stores import GeneralStore, RecordStore -from .structure_elements import StructureElement, Directory +from .structure_elements import Directory, StructureElement from .version import check_cfood_version -from .debug_tree import DebugTree - logger = logging.getLogger(__name__) @@ -255,8 +253,6 @@ def scanner(items: list[StructureElement], restricted_path: Optional[list[str]] = None, crawled_data: Optional[list[db.Record]] = None, debug_tree: Optional[DebugTree] = None, - # This would fit better after converters parameter (non-optional), - # but is added here for backwards compatibility. registered_transformer_functions: Optional[dict] = None): """Crawl a list of StructureElements and apply any matching converters. @@ -284,14 +280,14 @@ def scanner(items: list[StructureElement], A dictionary of transformer functions that can be used in the "transform" block of a converter and that allows to apply simple transformations to variables extracted either by the current converter or to other variables found in the current variable store. - + Each function is a dictionary: - The key is the name of the function to be looked up in the dictionary of registered transformer functions. - The value is a dictionary with key, value-assignments which will be passed to the transformer function. The transformer function needs to be of the form: - + def func(in_value: Any, in_parameters: dict) -> Any: pass