diff --git a/integrationtests/basic_example/test_basic.py b/integrationtests/basic_example/test_basic.py index c906a81d86af56669f7c522169bceb3b5fcb3e01..6fd322e5f6425e9bce25b970d6de7d99892762a5 100755 --- a/integrationtests/basic_example/test_basic.py +++ b/integrationtests/basic_example/test_basic.py @@ -32,7 +32,7 @@ import sys from argparse import RawTextHelpFormatter from pathlib import Path -import caosdb as db +import linkahead as db import pytest import yaml from caosadvancedtools.crawler import Crawler as OldCrawler @@ -42,8 +42,8 @@ from caoscrawler.debug_tree import DebugTree from caoscrawler.identifiable import Identifiable from caoscrawler.identifiable_adapters import CaosDBIdentifiableAdapter from caoscrawler.scanner import scan_directory -from caosdb import EmptyUniqueQueryError -from caosdb.utils.register_tests import clear_database, set_test_key +from linkahead import EmptyUniqueQueryError +from linkahead.utils.register_tests import clear_database, set_test_key set_test_key("10b128cf8a1372f30aa3697466bb55e76974e0c16a599bb44ace88f19c8f61e2") diff --git a/integrationtests/test_use_case_simple_presentation.py b/integrationtests/test_use_case_simple_presentation.py index cf38e951b78534806c0ea76ef58051436aa22704..05b0a543deb03eb524d40d6a386876812e6b54e2 100644 --- a/integrationtests/test_use_case_simple_presentation.py +++ b/integrationtests/test_use_case_simple_presentation.py @@ -27,12 +27,12 @@ import os import pytest from subprocess import run -import caosdb as db +import linkahead as db from caosadvancedtools.loadFiles import loadpath -from caosdb.cached import cache_clear +from linkahead.cached import cache_clear from caosadvancedtools.models import parser as parser from caoscrawler.crawl import crawler_main -from caosdb.utils.register_tests import clear_database, set_test_key +from linkahead.utils.register_tests import clear_database, set_test_key set_test_key("10b128cf8a1372f30aa3697466bb55e76974e0c16a599bb44ace88f19c8f61e2") diff --git a/src/caoscrawler/macros/macro_yaml_object.py b/src/caoscrawler/macros/macro_yaml_object.py index c6b5de27d7f498d9b1db6b6a90d986487340a880..d85883011db3cf651da0dda6c110015128fbe439 100644 --- a/src/caoscrawler/macros/macro_yaml_object.py +++ b/src/caoscrawler/macros/macro_yaml_object.py @@ -25,12 +25,17 @@ # Function to expand a macro in yaml # A. Schlemmer, 05/2022 +import re from dataclasses import dataclass from typing import Any, Dict from copy import deepcopy from string import Template +_SAFE_SUBST_PAT = re.compile(r"^\$(?P<key>\w+)$") +_SAFE_SUBST_PAT_BRACES = re.compile(r"^\$\{(?P<key>\w+)}$") + + @dataclass class MacroDefinition: """ @@ -53,6 +58,12 @@ def substitute(propvalue, values: dict): Substitution of variables in strings using the variable substitution library from python's standard library. """ + # Simple matches are simply replaced by the raw dict entry. + if match := (_SAFE_SUBST_PAT.fullmatch(propvalue) + or _SAFE_SUBST_PAT_BRACES.fullmatch(propvalue)): + key = match.group("key") + if key in values: + return values[key] propvalue_template = Template(propvalue) return propvalue_template.safe_substitute(**values) diff --git a/src/doc/getting_started/helloworld.md b/src/doc/getting_started/helloworld.md index 723fb88d08047350d9f4bc3d3d2bd84ec9b27efb..67fdf88974391ac6209f1010bfb4f2d883e51021 100644 --- a/src/doc/getting_started/helloworld.md +++ b/src/doc/getting_started/helloworld.md @@ -33,7 +33,7 @@ Then you can do the following interactively in (I)Python. But we recommend that copy the code into a script and execute it to spare yourself typing. ```python -import caosdb as db +import linkahead as db from datetime import datetime from caoscrawler import Crawler, SecurityMode from caoscrawler.identifiable_adapters import CaosDBIdentifiableAdapter diff --git a/unittests/example_cfood.yml b/unittests/example_cfood.yml index 713bd4be0f3c816e1e8c8b7a057b30a4b400f13c..798e540fa25e49bf610ea21653db41a0bddc4d5f 100644 --- a/unittests/example_cfood.yml +++ b/unittests/example_cfood.yml @@ -1,6 +1,6 @@ --- metadata: - crawler-version: 0.3.1 + crawler-version: 0.7.2 --- Definitions: type: Definitions diff --git a/unittests/h5_cfood.yml b/unittests/h5_cfood.yml index f688de6a2171da6533626449b030bcd95a43b37b..4b95a0a31bc43a902eb63dc3aa09b805fc28c2aa 100644 --- a/unittests/h5_cfood.yml +++ b/unittests/h5_cfood.yml @@ -1,6 +1,6 @@ --- metadata: - crawler-version: 0.6.1 + crawler-version: 0.7.2 --- Converters: H5Dataset: diff --git a/unittests/scifolder_cfood.yml b/unittests/scifolder_cfood.yml index 9d6e8cf3ea325ad14641530f2e6cafd43f0dc1bb..ca5fa589b5903e0c0d8ef3dcb2528ea79e0f8cee 100644 --- a/unittests/scifolder_cfood.yml +++ b/unittests/scifolder_cfood.yml @@ -4,7 +4,7 @@ --- metadata: - crawler-version: 0.3.1 + crawler-version: 0.7.2 --- Definitions: type: Definitions diff --git a/unittests/test_converters.py b/unittests/test_converters.py index 2f62ef9216974bc4939667c0cb28971044c1f80c..1d2492a7c2eb59b0533d707bad7e1cb3e51529bd 100644 --- a/unittests/test_converters.py +++ b/unittests/test_converters.py @@ -497,7 +497,7 @@ MyElement: two_doc_yaml = """ --- metadata: - crawler-version: 0.3.1 + crawler-version: 0.7.2 Converters: MyNewType: converter: MyNewTypeConverter diff --git a/unittests/test_entity_comparison.py b/unittests/test_entity_comparison.py index 549bc4f42a59765d25446d44fbb845e49ca4d9b9..0f62475b6c61d82feb3e550cf5ab53e91183f80a 100644 --- a/unittests/test_entity_comparison.py +++ b/unittests/test_entity_comparison.py @@ -2,7 +2,7 @@ # Tests for entity comparison # A. Schlemmer, 06/2021 -import caosdb as db +import linkahead as db import pytest from pytest import raises diff --git a/unittests/test_h5_converter.py b/unittests/test_h5_converter.py index 2f7fae5d8d32bb7e5c90a535b63158c33df55daa..7f244e2cbdccb0d4eee6a62f59e9cea5684295a6 100644 --- a/unittests/test_h5_converter.py +++ b/unittests/test_h5_converter.py @@ -23,7 +23,7 @@ from functools import partial from pathlib import Path from pytest import fixture, importorskip -import caosdb as db +import linkahead as db from caoscrawler.debug_tree import DebugTree from caoscrawler.hdf5_converter import (convert_basic_element_with_nd_array, diff --git a/unittests/test_identifiable.py b/unittests/test_identifiable.py index 074c3843e351b20d17813a661974fdc59ca0442a..d94d852583523a3b3f29f002eaacb9ae0b616c4f 100644 --- a/unittests/test_identifiable.py +++ b/unittests/test_identifiable.py @@ -24,7 +24,7 @@ test identifiable module """ -import caosdb as db +import linkahead as db import pytest from caoscrawler.identifiable import Identifiable from caoscrawler.sync_node import SyncNode diff --git a/unittests/test_identifiable_adapters.py b/unittests/test_identifiable_adapters.py index 94685c27523f2f2cebd9d0723078f2da9ff56f78..53490bc0413a95d960d94186c639dac2c6223b80 100644 --- a/unittests/test_identifiable_adapters.py +++ b/unittests/test_identifiable_adapters.py @@ -32,7 +32,7 @@ from datetime import datetime from unittest.mock import MagicMock, Mock, patch from pathlib import Path -import caosdb as db +import linkahead as db import pytest from caoscrawler.exceptions import (InvalidIdentifiableYAML, ) diff --git a/unittests/test_json.py b/unittests/test_json.py index fdb332df60d73dce3356a563e09ae0d02cf845b7..be65a26ea01e11e11968bd927c80513708e73850 100644 --- a/unittests/test_json.py +++ b/unittests/test_json.py @@ -31,7 +31,7 @@ import os from pytest import raises -import caosdb as db +import linkahead as db from caoscrawler.converters import JSONFileConverter from pathlib import Path diff --git a/unittests/test_macros.py b/unittests/test_macros.py index 53837e920e93f2cc318d62549145a0e8ac757372..85fe56cd2d49581bcf07b1c7af8456ad219b0111 100644 --- a/unittests/test_macros.py +++ b/unittests/test_macros.py @@ -142,7 +142,7 @@ def test_multi_macros_toplevel(register_macros, macro_store_reset): dat_loader = list(yaml.safe_load_all(""" --- metadata: - crawler-version: 0.5.1 + crawler-version: 0.7.2 macros: - !defmacro name: test_one @@ -171,7 +171,7 @@ def test_load_definition(register_macros, macro_store_reset): txt = """ --- metadata: - crawler-version: 0.5.1 + crawler-version: 0.7.2 --- extroot: type: Directory @@ -188,7 +188,7 @@ extroot: cfood = _temp_file_load(""" --- metadata: - crawler-version: 0.5.1 + crawler-version: 0.7.2 macros: - !defmacro name: test_one @@ -223,7 +223,6 @@ extroot3: assert cfood["extroot3"]["subtree"]["SimulationData"]["match"] == "SimulationData" -@pytest.mark.xfail def test_replace_arbitrary_objects(register_macros, macro_store_reset): """ See: https://gitlab.indiscale.com/caosdb/src/caosdb-crawler/-/issues/24 @@ -234,27 +233,34 @@ defs: name: test params: b: 25 + testvar_list_empty: [] testvar_list: - a - $b + testvar_dict_empty: {} testvar_dict: t1: a t2: $b definition: replaced1: $b: ok - c: $testvar_dict - d: $testvar_list + dict_empty: $testvar_dict_empty + dict: $testvar_dict + list_empty: $testvar_list_empty + list: ${testvar_list} testnode: obl: !macro test: """, Loader=yaml.SafeLoader) print(yaml.dump(dat)) - assert dat["testnode"]["obl"]["replaced1"]["c"]["t1"] == "a" - assert dat["testnode"]["obl"]["replaced1"]["c"]["t2"] == "25" - assert dat["testnode"]["obl"]["replaced1"]["d"][0] == "a" - assert dat["testnode"]["obl"]["replaced1"]["d"][1] == "25" + replaced = dat["testnode"]["obl"]["replaced1"] + assert replaced["dict_empty"] == {} + assert replaced["dict"]["t1"] == "a" + assert replaced["dict"]["t2"] == 25 + assert replaced["list_empty"] == [] + assert replaced["list"][0] == "a" + assert replaced["list"][1] == 25 def test_macros_in_macros(register_macros, macro_store_reset): @@ -264,7 +270,7 @@ def test_macros_in_macros(register_macros, macro_store_reset): cfood = _temp_file_load(""" --- metadata: - crawler-version: 0.5.1 + crawler-version: 0.7.2 macros: - !defmacro name: one_macro @@ -293,11 +299,11 @@ extroot: !macro assert "test_macro" not in cfood["extroot"] assert cfood["extroot"]["macro_top"]["not_macro"]["a"] == 26 d = cfood["extroot"]["macro_top"] - assert d["macro_sub_17"]["b"] == "17" + assert d["macro_sub_17"]["b"] == 17 assert d["macro_sub_17"]["another_param"] == 3 - assert d["macro_sub_25"]["b"] == "25" + assert d["macro_sub_25"]["b"] == 25 assert d["macro_sub_25"]["another_param"] == 3 - assert d["macro_sub_98"]["b"] == "98" + assert d["macro_sub_98"]["b"] == 98 assert d["macro_sub_98"]["another_param"] == 3 @@ -309,7 +315,7 @@ def test_silent_overwrite(register_macros, macro_store_reset): cfood = _temp_file_load(""" --- metadata: - crawler-version: 0.5.1 + crawler-version: 0.7.2 macros: - !defmacro name: one_macro @@ -340,7 +346,7 @@ def test_circular_macro_definition(register_macros, macro_store_reset): cfood = _temp_file_load(""" --- metadata: - crawler-version: 0.5.1 + crawler-version: 0.7.2 macros: - !defmacro name: test_one @@ -389,7 +395,7 @@ def test_use_macro_twice(): cfood = _temp_file_load(""" --- metadata: - crawler-version: 0.5.1 + crawler-version: 0.7.2 macros: - !defmacro name: test_twice @@ -410,9 +416,9 @@ extroot: !macro """) for name in ["once", "twice", "default_name"]: assert name in cfood["extroot"] - assert cfood["extroot"]["once"]["something"]["a"] == "4" - assert cfood["extroot"]["twice"]["something"]["a"] == "5" - assert cfood["extroot"]["default_name"]["something"]["a"] == "4" + assert cfood["extroot"]["once"]["something"]["a"] == 4 + assert cfood["extroot"]["twice"]["something"]["a"] == 5 + assert cfood["extroot"]["default_name"]["something"]["a"] == 4 # Code sample to generate the expanded macro: # with open("expanded_test_macro.yaml", "w") as f: # f.write(yaml.dump(cfood)) @@ -423,7 +429,7 @@ def test_documentation_example_2(): cfood = _temp_file_load(""" --- metadata: - crawler-version: 0.5.1 + crawler-version: 0.7.2 macros: - !defmacro name: MarkdownFile @@ -461,7 +467,7 @@ def test_documentation_example_1(): cfood = _temp_file_load(""" --- metadata: - crawler-version: 0.5.1 + crawler-version: 0.7.2 macros: - !defmacro name: SimulationDatasetFile @@ -510,7 +516,7 @@ def test_def_replacements(): cfood = _temp_file_load(""" --- metadata: - crawler-version: 0.5.1 + crawler-version: 0.7.2 macros: - !defmacro name: test_def_replacements @@ -573,9 +579,9 @@ testnode: test2: a: 4 """, Loader=yaml.SafeLoader) - assert dat["testnode"]["obl"]["expanded_4"]["param"] == "4" - assert dat["testnode"]["obl"]["expanded_2"]["param"] == "2" - assert dat["testnode"]["obl"]["expanded_4_test2"]["param"] == "4" + assert dat["testnode"]["obl"]["expanded_4"]["param"] == 4 + assert dat["testnode"]["obl"]["expanded_2"]["param"] == 2 + assert dat["testnode"]["obl"]["expanded_4_test2"]["param"] == 4 def test_variable_in_macro_definition(register_macros, macro_store_reset): @@ -598,7 +604,7 @@ testnode: - a: 2 b: 4 """, Loader=yaml.SafeLoader) - assert dat["testnode"]["obl"]["expanded_4"]["param"] == "4" - assert dat["testnode"]["obl"]["expanded_4"]["param_b"] == "4" - assert dat["testnode"]["obl"]["expanded_2"]["param"] == "2" - assert dat["testnode"]["obl"]["expanded_2"]["param_b"] == "4" + assert dat["testnode"]["obl"]["expanded_4"]["param"] == 4 + assert dat["testnode"]["obl"]["expanded_4"]["param_b"] == 4 + assert dat["testnode"]["obl"]["expanded_2"]["param"] == 2 + assert dat["testnode"]["obl"]["expanded_2"]["param_b"] == 4 diff --git a/unittests/test_parent_cfood.yml b/unittests/test_parent_cfood.yml index b8d0eaf597641d311cb70017dc2bc75c7c3434f3..cd63e81b270117841128a34765a9635a036c52ec 100644 --- a/unittests/test_parent_cfood.yml +++ b/unittests/test_parent_cfood.yml @@ -1,6 +1,6 @@ --- metadata: - crawler-version: 0.6.1 + crawler-version: 0.7.2 --- Definitions: type: Definitions diff --git a/unittests/test_scanner.py b/unittests/test_scanner.py index c0ce736fc4bed18f371f1626b6bc451ee103db49..226b5040547f0e003729dba63622edf836552f18 100644 --- a/unittests/test_scanner.py +++ b/unittests/test_scanner.py @@ -31,7 +31,7 @@ from pathlib import Path from tempfile import NamedTemporaryFile from unittest.mock import MagicMock, Mock, patch -import caosdb as db +import linkahead as db import pytest import yaml from caoscrawler.crawl import Crawler diff --git a/unittests/test_schema.py b/unittests/test_schema.py index 0d5bebce98fbc8c789c1080bcf3919f128bdbf54..3b576c9b72e41b799355f927d6e5387f1c187a18 100644 --- a/unittests/test_schema.py +++ b/unittests/test_schema.py @@ -3,7 +3,7 @@ # A. Schlemmer, 06/2021 from importlib_resources import files -import caosdb as db +import linkahead as db from os.path import join, dirname from caoscrawler import Crawler diff --git a/unittests/test_table_converter.py b/unittests/test_table_converter.py index 178393d9345bd8a6846b66e362ce4f7edac382ee..3b563fd3179968fd90b1c92b9bc5bf0db9ed0858 100644 --- a/unittests/test_table_converter.py +++ b/unittests/test_table_converter.py @@ -32,7 +32,7 @@ import os from os.path import basename, dirname, join from pathlib import Path -import caosdb as db +import linkahead as db import pytest from caoscrawler import Crawler from caoscrawler.converters import (Converter, ConverterValidationError, diff --git a/unittests/test_transformers.py b/unittests/test_transformers.py index 02d932d13cc3fad52048b08e2b9fe56f11db2ae7..e7fae484c1a0010d952809fc502e25a02bfe6ec5 100644 --- a/unittests/test_transformers.py +++ b/unittests/test_transformers.py @@ -34,7 +34,7 @@ from pathlib import Path from tempfile import NamedTemporaryFile from unittest.mock import MagicMock, Mock, patch -import caosdb as db +import linkahead as db import pytest import yaml from caoscrawler.converters import Converter, ListElementConverter diff --git a/unittests/test_variable_substitutions.py b/unittests/test_variable_substitutions.py index 09f78df661d82970e7264996102eff8881ee19ec..90d144b04a4e1271f74b769759e3f201007af705 100644 --- a/unittests/test_variable_substitutions.py +++ b/unittests/test_variable_substitutions.py @@ -25,7 +25,7 @@ from os.path import basename, dirname, join from pathlib import Path from unittest.mock import MagicMock, Mock -import caosdb as db +import linkahead as db import pytest import yaml from caoscrawler import Crawler @@ -35,7 +35,7 @@ from caoscrawler.identifiable_adapters import (IdentifiableAdapter, from caoscrawler.scanner import scan_directory from caoscrawler.structure_elements import (DictListElement, DictTextElement, File) -from caosdb.apiutils import compare_entities +from linkahead.apiutils import compare_entities from pytest import raises from utils import dircheckstr as dircheckstr_base