Select Git revision
test_issues.py
-
Daniel Hornung authoredDaniel Hornung authored
Code owners
Assign users and groups as approvers for specific file changes. Learn more.
test_transformers.py 6.73 KiB
#!/usr/bin/env python3
# encoding: utf-8
#
# This file is a part of the CaosDB Project.
#
# Copyright (C) 2023 Research Group Biomedical Physics,
# Max-Planck-Institute for Dynamics and Self-Organization Göttingen
# Alexander Schlemmer <alexander.schlemmer@ds.mpg.de>
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as
# published by the Free Software Foundation, either version 3 of the
# License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <https://www.gnu.org/licenses/>.
#
"""
Unit test functions for the transformer feature of the scanner.
Currently, this is under development.
See: https://gitlab.indiscale.com/caosdb/src/caosdb-crawler/-/issues/107
"""
import importlib
from pathlib import Path
from unittest.mock import Mock
import pytest
from caoscrawler.converters import Converter, ListElementConverter
from caoscrawler.scanner import create_transformer_registry, scan_directory
from caoscrawler.stores import GeneralStore
from caoscrawler.transformer_functions import replace
from pytest import raises
UNITTESTDIR = Path(__file__).parent
@pytest.fixture
def converter_registry():
converter_registry: dict[str, dict[str, str]] = {
"Directory": {
"converter": "DirectoryConverter",
"package": "caoscrawler.converters"},
"MarkdownFile": {
"converter": "MarkdownFileConverter",
"package": "caoscrawler.converters"},
"Date": {
"converter": "DateElementConverter",
"package": "caoscrawler.converters"},
"DictElement": {
"converter": "DictElementConverter",
"package": "caoscrawler.converters"},
"TextElement": {
"converter": "TextElementConverter",
"package": "caoscrawler.converters"},
"ListElement": {
"converter": "ListElementConverter",
"package": "caoscrawler.converters"},
"JSONFile": {
"converter": "JSONFileConverter",
"package": "caoscrawler.converters"},
}
for key, value in converter_registry.items():
module = importlib.import_module(value["package"])
value["class"] = getattr(module, value["converter"])
return converter_registry
def test_simple_transformer():
"""
Test the correct list of returned records by the scanner using the
scifolder example from the article.
"""
records = scan_directory(UNITTESTDIR / "test_directories" / "test_transformers",
UNITTESTDIR / "test_directories" / "test_transformers" /
"cfood.yml")
for r in records:
if r.parents[0].name == "DayFolder":
assert r.get_property("Day") is not None
assert r.get_property("DayShort") is not None
assert r.get_property("DayShort").value != "$day_short"
if r.get_property("DayShort").value == "Unk":
# This unkown folder should not lead to a replacement
assert r.get_property("Day").value == "Unk"
assert r.get_property("DaySplit").value == ["Unk"]
elif r.get_property("DayShort").value == "Mon":
assert r.get_property("Day").value == "Monday"
assert r.get_property("DaySplit").value == ["M", "n"]
elif r.get_property("DayShort").value == "Tue":
assert r.get_property("Day").value == "Tuesday"
assert r.get_property("DaySplit").value == ["Tue"]
else:
# unexpected occurence of a short form, something wrong with test directories
assert False
elif r.parents[0].name == "Number":
assert r.get_property("num") is not None
assert r.get_property("num").value == "'12345 5 '"
else:
# unkown error, something wrong with test directories
assert False
def test_apply_replace(converter_registry):
cfood_def = {"type": 'ListElement', "match_name": ".*",
'transform': {'test': {'in': '$a', 'out': '$b', 'functions': [{
'replace': {'insert': ':', "remove": "_"}}]}}}
values = GeneralStore()
values["a"] = "16_45"
# transformer_functions = create_transformer_registry(crawler_definition)
transformer_functions = {"replace": replace}
conv = ListElementConverter(definition=cfood_def, name='test',
converter_registry=converter_registry)
conv.apply_transformers(values, transformer_functions)
assert values['b'] == "16:45"
def test_apply_replace_from_def(converter_registry):
cfood_def = {"type": 'ListElement', "match_name": ".*",
'transform': {'test': {'in': '$a', 'out': '$b', 'functions': [{
'replace': {'insert': ':', "remove": "_"}}]}}}
values = GeneralStore()
values["a"] = "16_45"
transformer_functions = create_transformer_registry({})
# transformer_functions = {"replace": replace}
conv = ListElementConverter(definition=cfood_def, name='test',
converter_registry=converter_registry)
conv.apply_transformers(values, transformer_functions)
assert values['b'] == "16:45"
def test_empty_functions_list(converter_registry):
cfood_def = {"type": 'ListElement',
"match_name": ".*",
'transform': {'test': {'in': '$a', 'out': '$b',
'functions': []}}}
values = GeneralStore()
values["a"] = "16_45"
# transformer_functions = create_transformer_registry(crawler_definition)
transformer_functions = {"replace": replace}
conv = ListElementConverter(definition=cfood_def, name='test',
converter_registry=converter_registry)
conv.apply_transformers(values, transformer_functions)
assert values['b'] == "16_45"
def test_replace_variables():
vals = GeneralStore()
vals["test"] = "with"
vals["a"] = "str_without_replacement"
conv = Mock()
conv.definition = {}
conv.definition["transform"] = {
"test": {
"in": "$a",
"out": "$a",
"functions": [
{"replace": {
"remove": "without",
"insert": "$test"
}}
]}}
Converter.apply_transformers(conv, vals, {"replace": replace})
assert vals["a"] == "str_with_replacement"