Skip to content
Snippets Groups Projects
Select Git revision
  • 3c44ccfd4382397154d35f5f7dd77592980558e0
  • main default protected
  • dev
  • f-spss-value-label-name
  • f-unmod
  • f-checkidentical
  • f-simple-breakpoint
  • f-new-debug-tree
  • f-existing-file-id
  • f-no-ident
  • f-collect-problems
  • f-refactor-debug-tree
  • v0.13.0
  • v0.12.0
  • v0.11.0
  • v0.10.1
  • v0.10.0
  • v0.9.1
  • v0.9.0
  • v0.8.0
  • v0.7.1
  • v0.7.0
  • v0.6.0
  • v0.5.0
  • v0.4.0
  • v0.3.0
  • v0.2.0
  • v0.1.0
28 results

test_issues.py

Blame
  • Code owners
    Assign users and groups as approvers for specific file changes. Learn more.
    test_transformers.py 6.73 KiB
    #!/usr/bin/env python3
    # encoding: utf-8
    #
    # This file is a part of the CaosDB Project.
    #
    # Copyright (C)  2023 Research Group Biomedical Physics,
    # Max-Planck-Institute for Dynamics and Self-Organization Göttingen
    # Alexander Schlemmer <alexander.schlemmer@ds.mpg.de>
    #
    # This program is free software: you can redistribute it and/or modify
    # it under the terms of the GNU Affero General Public License as
    # published by the Free Software Foundation, either version 3 of the
    # License, or (at your option) any later version.
    #
    # This program is distributed in the hope that it will be useful,
    # but WITHOUT ANY WARRANTY; without even the implied warranty of
    # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    # GNU Affero General Public License for more details.
    #
    # You should have received a copy of the GNU Affero General Public License
    # along with this program. If not, see <https://www.gnu.org/licenses/>.
    #
    
    """
    Unit test functions for the transformer feature of the scanner.
    
    Currently, this is under development.
    See: https://gitlab.indiscale.com/caosdb/src/caosdb-crawler/-/issues/107
    """
    
    import importlib
    from pathlib import Path
    from unittest.mock import Mock
    
    import pytest
    from caoscrawler.converters import Converter, ListElementConverter
    from caoscrawler.scanner import create_transformer_registry, scan_directory
    from caoscrawler.stores import GeneralStore
    from caoscrawler.transformer_functions import replace
    from pytest import raises
    
    UNITTESTDIR = Path(__file__).parent
    
    
    @pytest.fixture
    def converter_registry():
        converter_registry: dict[str, dict[str, str]] = {
            "Directory": {
                "converter": "DirectoryConverter",
                "package": "caoscrawler.converters"},
            "MarkdownFile": {
                "converter": "MarkdownFileConverter",
                "package": "caoscrawler.converters"},
            "Date": {
                "converter": "DateElementConverter",
                "package": "caoscrawler.converters"},
            "DictElement": {
                "converter": "DictElementConverter",
                "package": "caoscrawler.converters"},
            "TextElement": {
                "converter": "TextElementConverter",
                "package": "caoscrawler.converters"},
            "ListElement": {
                "converter": "ListElementConverter",
                "package": "caoscrawler.converters"},
            "JSONFile": {
                "converter": "JSONFileConverter",
                "package": "caoscrawler.converters"},
        }
    
        for key, value in converter_registry.items():
            module = importlib.import_module(value["package"])
            value["class"] = getattr(module, value["converter"])
        return converter_registry
    
    
    def test_simple_transformer():
        """
        Test the correct list of returned records by the scanner using the
        scifolder example from the article.
        """
    
        records = scan_directory(UNITTESTDIR / "test_directories" / "test_transformers",
                                 UNITTESTDIR / "test_directories" / "test_transformers" /
                                 "cfood.yml")
    
        for r in records:
            if r.parents[0].name == "DayFolder":
                assert r.get_property("Day") is not None
                assert r.get_property("DayShort") is not None
                assert r.get_property("DayShort").value != "$day_short"
                if r.get_property("DayShort").value == "Unk":
                    # This unkown folder should not lead to a replacement
                    assert r.get_property("Day").value == "Unk"
                    assert r.get_property("DaySplit").value == ["Unk"]
                elif r.get_property("DayShort").value == "Mon":
                    assert r.get_property("Day").value == "Monday"
                    assert r.get_property("DaySplit").value == ["M", "n"]
                elif r.get_property("DayShort").value == "Tue":
                    assert r.get_property("Day").value == "Tuesday"
                    assert r.get_property("DaySplit").value == ["Tue"]
                else:
                    # unexpected occurence of a short form, something wrong with test directories
                    assert False
            elif r.parents[0].name == "Number":
                assert r.get_property("num") is not None
                assert r.get_property("num").value == "'12345 5 '"
            else:
                # unkown error, something wrong with test directories
                assert False
    
    
    def test_apply_replace(converter_registry):
        cfood_def = {"type": 'ListElement', "match_name": ".*",
                     'transform': {'test': {'in': '$a', 'out': '$b', 'functions': [{
                         'replace': {'insert': ':', "remove": "_"}}]}}}
        values = GeneralStore()
        values["a"] = "16_45"
    
        # transformer_functions  = create_transformer_registry(crawler_definition)
        transformer_functions = {"replace": replace}
    
        conv = ListElementConverter(definition=cfood_def, name='test',
                                    converter_registry=converter_registry)
    
        conv.apply_transformers(values, transformer_functions)
        assert values['b'] == "16:45"
    
    
    def test_apply_replace_from_def(converter_registry):
        cfood_def = {"type": 'ListElement', "match_name": ".*",
                     'transform': {'test': {'in': '$a', 'out': '$b', 'functions': [{
                         'replace': {'insert': ':', "remove": "_"}}]}}}
        values = GeneralStore()
        values["a"] = "16_45"
    
        transformer_functions = create_transformer_registry({})
        # transformer_functions = {"replace": replace}
    
        conv = ListElementConverter(definition=cfood_def, name='test',
                                    converter_registry=converter_registry)
    
        conv.apply_transformers(values, transformer_functions)
        assert values['b'] == "16:45"
    
    
    def test_empty_functions_list(converter_registry):
        cfood_def = {"type": 'ListElement',
                     "match_name": ".*",
                     'transform': {'test': {'in': '$a', 'out': '$b',
                                            'functions': []}}}
        values = GeneralStore()
        values["a"] = "16_45"
    
        # transformer_functions  = create_transformer_registry(crawler_definition)
        transformer_functions = {"replace": replace}
    
        conv = ListElementConverter(definition=cfood_def, name='test',
                                    converter_registry=converter_registry)
    
        conv.apply_transformers(values, transformer_functions)
        assert values['b'] == "16_45"
    
    
    def test_replace_variables():
        vals = GeneralStore()
        vals["test"] = "with"
        vals["a"] = "str_without_replacement"
        conv = Mock()
        conv.definition = {}
        conv.definition["transform"] = {
            "test": {
                "in": "$a",
                "out": "$a",
                "functions": [
                    {"replace": {
                        "remove": "without",
                        "insert": "$test"
                    }}
                ]}}
        Converter.apply_transformers(conv, vals, {"replace": replace})
        assert vals["a"] == "str_with_replacement"