diff --git a/src/caoscrawler/converters.py b/src/caoscrawler/converters.py
index e0ca0f9bff77ba1ecc63f4d102d6d9869fb11cb0..d28d1d53819ba8fe400bcfe036703bb5686bc47f 100644
--- a/src/caoscrawler/converters.py
+++ b/src/caoscrawler/converters.py
@@ -34,7 +34,7 @@ import warnings
from abc import ABCMeta, abstractmethod
from inspect import signature
from string import Template
-from typing import Any, List, Optional, Tuple, Union
+from typing import Any, Optional, Union
import linkahead as db
import pandas as pd
@@ -134,8 +134,8 @@ def replace_variables(propvalue: Any, values: GeneralStore):
This function replaces variables in property values (and possibly other locations,
where the crawler can replace cfood-internal variables).
- If `propvalue` is a single variable name preceeded with a '$' (e.g. '$var' or '${var}'), then
- the corresponding value stored in `values` is returned.
+ If ``propvalue`` is a single variable name preceeded by a ``$`` (e.g. ``$var`` or ``${var}``),
+ then the corresponding value stored in ``values`` is returned.
In any other case the variable substitution is carried out as defined by string templates
and a new string with the replaced variables is returned.
"""
@@ -160,16 +160,16 @@ def handle_value(value: Union[dict, str, list], values: GeneralStore):
add as an additional property (multiproperty).
Variable names (starting with a "$") are replaced by the corresponding value stored in the
- `values` GeneralStore.
+ ``values`` GeneralStore.
Parameters
----------
-value:
- - if str, the value to be interpreted. E.g. "4", "hallo" or "$a" etc.
- - if dict, must have keys "value" and "collection_mode". The returned tuple is directly
+value: Union[dict, str, list]
+ - If *str*, the value to be interpreted. E.g. "4", "hello" or "$a" etc.
+ - If *dict*, must have keys ``value`` and ``collection_mode``. The returned tuple is directly
created from the corresponding values.
- - if list, each element is checked for replacement and the resulting list will be used
+ - If *list*, each element is checked for replacement and the resulting list will be used
as (list) value for the property
Returns
@@ -181,7 +181,7 @@ out: tuple
"""
# @review Florian Spreckelsen 2022-05-13
- if type(value) == dict:
+ if isinstance(value, dict):
if "value" not in value:
# TODO: how do we handle this case? Just ignore?
# or disallow?
@@ -189,7 +189,7 @@ out: tuple
propvalue = value["value"]
# can be "single", "list" or "multiproperty"
collection_mode = value["collection_mode"]
- elif type(value) == str:
+ elif isinstance(value, str):
propvalue = value
collection_mode = "single"
if propvalue.startswith("+"):
@@ -198,7 +198,7 @@ out: tuple
elif propvalue.startswith("*"):
collection_mode = "multiproperty"
propvalue = propvalue[1:]
- elif type(value) == list:
+ elif isinstance(value, list):
# TODO: (for review)
# This is a bit dirty right now and needed for
# being able to directly set list values. Semantics is, however, a bit
@@ -209,7 +209,7 @@ out: tuple
propvalue = list()
for element in value:
# Do the element-wise replacement only, when its type is string:
- if type(element) == str:
+ if isinstance(element, str):
propvalue.append(replace_variables(element, values))
else:
propvalue.append(element)
@@ -322,10 +322,12 @@ class Converter(object, metaclass=ABCMeta):
Parameters
----------
- definition: dict, Please refer to ``src/doc/converters.rst`` to learn about the structure
- that the definition dict must have.
- converter_registry: dict, A dictionary that contains converter names as keys and dicts as
- values. Those value dicts have the keys 'converter' and 'package'.
+ definition: dict
+ Please refer to ``src/doc/converters.rst`` to learn about the structure that the
+ definition dict must have.
+ converter_registry: dict
+ A dictionary that contains converter names as keys and dicts as values. Those value dicts
+ have the keys 'converter' and 'package'.
"""
self.definition = definition
@@ -424,7 +426,7 @@ class Converter(object, metaclass=ABCMeta):
pass
"""
- if not "transform" in self.definition:
+ if "transform" not in self.definition:
return
for transformer_key, transformer in self.definition["transform"].items():
in_value = replace_variables(transformer["in"], values)
@@ -460,8 +462,7 @@ class Converter(object, metaclass=ABCMeta):
values[match.group('varname')] = out_value
@abstractmethod
- def create_children(self, values: GeneralStore,
- element: StructureElement):
+ def create_children(self, values: GeneralStore, element: StructureElement):
pass
def create_records(self, values: GeneralStore, records: RecordStore,
@@ -477,7 +478,7 @@ class Converter(object, metaclass=ABCMeta):
self.definition["records"])
def filter_children(self, children_with_strings:
- List[Tuple[StructureElement, str]], expr: str,
+ list[tuple[StructureElement, str]], expr: str,
group: str, rule: str):
"""Filter children according to regexp `expr` and `rule`."""
@@ -620,7 +621,7 @@ class DirectoryConverter(Converter):
element: A directory (of type Directory) which will be traversed.
"""
- children: List[StructureElement] = []
+ children: list[StructureElement] = []
for name in sorted(os.listdir(element.path)):
path = os.path.join(element.path, name)
@@ -660,7 +661,7 @@ class SimpleFileConverter(Converter):
class FileConverter(SimpleFileConverter):
def __init__(self, *args, **kwargs):
warnings.warn(DeprecationWarning(
- "This class is depricated. Please use SimpleFileConverter."))
+ "This class is deprecated. Please use SimpleFileConverter."))
super().__init__(*args, **kwargs)
@@ -693,12 +694,12 @@ class MarkdownFileConverter(SimpleFileConverter):
"Error during the validation (yaml header cannot be read) of the markdown file "
"located at the following node in the data structure:\n"
"{}\nError:\n{}".format(path, err))
- children: List[StructureElement] = []
+ children: list[StructureElement] = []
for name, entry in header.items():
- if type(entry) == list:
+ if isinstance(entry, list):
children.append(ListElement(name, entry))
- elif type(entry) == str:
+ elif isinstance(entry, str):
children.append(TextElement(name, entry))
else:
if generalStore is not None and self.name in generalStore:
@@ -814,14 +815,14 @@ class DictElementConverter(Converter):
class DictConverter(DictElementConverter):
def __init__(self, *args, **kwargs):
warnings.warn(DeprecationWarning(
- "This class is depricated. Please use DictConverter."))
+ "This class is deprecated. Please use DictConverter."))
super().__init__(*args, **kwargs)
class DictDictElementConverter(DictElementConverter):
def __init__(self, *args, **kwargs):
warnings.warn(DeprecationWarning(
- "This class is depricated. Please use DictElementConverter."))
+ "This class is deprecated. Please use DictElementConverter."))
super().__init__(*args, **kwargs)
@@ -1009,7 +1010,7 @@ class BooleanElementConverter(_AbstractScalarValueElementConverter):
class DictBooleanElementConverter(BooleanElementConverter):
def __init__(self, *args, **kwargs):
warnings.warn(DeprecationWarning(
- "This class is depricated. Please use BooleanElementConverter."))
+ "This class is deprecated. Please use BooleanElementConverter."))
super().__init__(*args, **kwargs)
@@ -1025,7 +1026,7 @@ class FloatElementConverter(_AbstractScalarValueElementConverter):
class DictFloatElementConverter(FloatElementConverter):
def __init__(self, *args, **kwargs):
warnings.warn(DeprecationWarning(
- "This class is depricated. Please use FloatElementConverter."))
+ "This class is deprecated. Please use FloatElementConverter."))
super().__init__(*args, **kwargs)
@@ -1050,7 +1051,7 @@ the 'match_value' key to match the value of the TextElement and 'match_name' for
class DictTextElementConverter(TextElementConverter):
def __init__(self, *args, **kwargs):
warnings.warn(DeprecationWarning(
- "This class is depricated. Please use TextElementConverter."))
+ "This class is deprecated. Please use TextElementConverter."))
super().__init__(*args, **kwargs)
@@ -1066,7 +1067,7 @@ class IntegerElementConverter(_AbstractScalarValueElementConverter):
class DictIntegerElementConverter(IntegerElementConverter):
def __init__(self, *args, **kwargs):
warnings.warn(DeprecationWarning(
- "This class is depricated. Please use IntegerElementConverter."))
+ "This class is deprecated. Please use IntegerElementConverter."))
super().__init__(*args, **kwargs)
@@ -1108,7 +1109,7 @@ class ListElementConverter(Converter):
class DictListElementConverter(ListElementConverter):
def __init__(self, *args, **kwargs):
warnings.warn(DeprecationWarning(
- "This class is depricated. Please use ListElementConverter."))
+ "This class is deprecated. Please use ListElementConverter."))
super().__init__(*args, **kwargs)