From 9ec83ca9260417cff7ebe8cfd2b91215a0b9f141 Mon Sep 17 00:00:00 2001
From: Daniel <d.hornung@indiscale.com>
Date: Wed, 26 Jun 2024 15:45:36 +0200
Subject: [PATCH] ENH: Allow to replace YAML objects instead of strings .
---
integrationtests/basic_example/test_basic.py | 6 +-
.../test_use_case_simple_presentation.py | 6 +-
src/caoscrawler/macros/macro_yaml_object.py | 11 ++++
src/doc/getting_started/helloworld.md | 2 +-
unittests/example_cfood.yml | 2 +-
unittests/h5_cfood.yml | 2 +-
unittests/scifolder_cfood.yml | 2 +-
unittests/test_converters.py | 2 +-
unittests/test_entity_comparison.py | 2 +-
unittests/test_h5_converter.py | 2 +-
unittests/test_identifiable.py | 2 +-
unittests/test_identifiable_adapters.py | 2 +-
unittests/test_json.py | 2 +-
unittests/test_macros.py | 66 ++++++++++---------
unittests/test_parent_cfood.yml | 2 +-
unittests/test_scanner.py | 2 +-
unittests/test_schema.py | 2 +-
unittests/test_table_converter.py | 2 +-
unittests/test_transformers.py | 2 +-
unittests/test_variable_substitutions.py | 4 +-
20 files changed, 70 insertions(+), 53 deletions(-)
diff --git a/integrationtests/basic_example/test_basic.py b/integrationtests/basic_example/test_basic.py
index c906a81d..6fd322e5 100755
--- a/integrationtests/basic_example/test_basic.py
+++ b/integrationtests/basic_example/test_basic.py
@@ -32,7 +32,7 @@ import sys
from argparse import RawTextHelpFormatter
from pathlib import Path
-import caosdb as db
+import linkahead as db
import pytest
import yaml
from caosadvancedtools.crawler import Crawler as OldCrawler
@@ -42,8 +42,8 @@ from caoscrawler.debug_tree import DebugTree
from caoscrawler.identifiable import Identifiable
from caoscrawler.identifiable_adapters import CaosDBIdentifiableAdapter
from caoscrawler.scanner import scan_directory
-from caosdb import EmptyUniqueQueryError
-from caosdb.utils.register_tests import clear_database, set_test_key
+from linkahead import EmptyUniqueQueryError
+from linkahead.utils.register_tests import clear_database, set_test_key
set_test_key("10b128cf8a1372f30aa3697466bb55e76974e0c16a599bb44ace88f19c8f61e2")
diff --git a/integrationtests/test_use_case_simple_presentation.py b/integrationtests/test_use_case_simple_presentation.py
index cf38e951..05b0a543 100644
--- a/integrationtests/test_use_case_simple_presentation.py
+++ b/integrationtests/test_use_case_simple_presentation.py
@@ -27,12 +27,12 @@ import os
import pytest
from subprocess import run
-import caosdb as db
+import linkahead as db
from caosadvancedtools.loadFiles import loadpath
-from caosdb.cached import cache_clear
+from linkahead.cached import cache_clear
from caosadvancedtools.models import parser as parser
from caoscrawler.crawl import crawler_main
-from caosdb.utils.register_tests import clear_database, set_test_key
+from linkahead.utils.register_tests import clear_database, set_test_key
set_test_key("10b128cf8a1372f30aa3697466bb55e76974e0c16a599bb44ace88f19c8f61e2")
diff --git a/src/caoscrawler/macros/macro_yaml_object.py b/src/caoscrawler/macros/macro_yaml_object.py
index c6b5de27..d8588301 100644
--- a/src/caoscrawler/macros/macro_yaml_object.py
+++ b/src/caoscrawler/macros/macro_yaml_object.py
@@ -25,12 +25,17 @@
# Function to expand a macro in yaml
# A. Schlemmer, 05/2022
+import re
from dataclasses import dataclass
from typing import Any, Dict
from copy import deepcopy
from string import Template
+_SAFE_SUBST_PAT = re.compile(r"^\$(?P<key>\w+)$")
+_SAFE_SUBST_PAT_BRACES = re.compile(r"^\$\{(?P<key>\w+)}$")
+
+
@dataclass
class MacroDefinition:
"""
@@ -53,6 +58,12 @@ def substitute(propvalue, values: dict):
Substitution of variables in strings using the variable substitution
library from python's standard library.
"""
+ # Simple matches are simply replaced by the raw dict entry.
+ if match := (_SAFE_SUBST_PAT.fullmatch(propvalue)
+ or _SAFE_SUBST_PAT_BRACES.fullmatch(propvalue)):
+ key = match.group("key")
+ if key in values:
+ return values[key]
propvalue_template = Template(propvalue)
return propvalue_template.safe_substitute(**values)
diff --git a/src/doc/getting_started/helloworld.md b/src/doc/getting_started/helloworld.md
index 723fb88d..67fdf889 100644
--- a/src/doc/getting_started/helloworld.md
+++ b/src/doc/getting_started/helloworld.md
@@ -33,7 +33,7 @@ Then you can do the following interactively in (I)Python. But we recommend that
copy the code into a script and execute it to spare yourself typing.
```python
-import caosdb as db
+import linkahead as db
from datetime import datetime
from caoscrawler import Crawler, SecurityMode
from caoscrawler.identifiable_adapters import CaosDBIdentifiableAdapter
diff --git a/unittests/example_cfood.yml b/unittests/example_cfood.yml
index 713bd4be..798e540f 100644
--- a/unittests/example_cfood.yml
+++ b/unittests/example_cfood.yml
@@ -1,6 +1,6 @@
---
metadata:
- crawler-version: 0.3.1
+ crawler-version: 0.7.2
---
Definitions:
type: Definitions
diff --git a/unittests/h5_cfood.yml b/unittests/h5_cfood.yml
index f688de6a..4b95a0a3 100644
--- a/unittests/h5_cfood.yml
+++ b/unittests/h5_cfood.yml
@@ -1,6 +1,6 @@
---
metadata:
- crawler-version: 0.6.1
+ crawler-version: 0.7.2
---
Converters:
H5Dataset:
diff --git a/unittests/scifolder_cfood.yml b/unittests/scifolder_cfood.yml
index 9d6e8cf3..ca5fa589 100644
--- a/unittests/scifolder_cfood.yml
+++ b/unittests/scifolder_cfood.yml
@@ -4,7 +4,7 @@
---
metadata:
- crawler-version: 0.3.1
+ crawler-version: 0.7.2
---
Definitions:
type: Definitions
diff --git a/unittests/test_converters.py b/unittests/test_converters.py
index 2f62ef92..1d2492a7 100644
--- a/unittests/test_converters.py
+++ b/unittests/test_converters.py
@@ -497,7 +497,7 @@ MyElement:
two_doc_yaml = """
---
metadata:
- crawler-version: 0.3.1
+ crawler-version: 0.7.2
Converters:
MyNewType:
converter: MyNewTypeConverter
diff --git a/unittests/test_entity_comparison.py b/unittests/test_entity_comparison.py
index 549bc4f4..0f62475b 100644
--- a/unittests/test_entity_comparison.py
+++ b/unittests/test_entity_comparison.py
@@ -2,7 +2,7 @@
# Tests for entity comparison
# A. Schlemmer, 06/2021
-import caosdb as db
+import linkahead as db
import pytest
from pytest import raises
diff --git a/unittests/test_h5_converter.py b/unittests/test_h5_converter.py
index 2f7fae5d..7f244e2c 100644
--- a/unittests/test_h5_converter.py
+++ b/unittests/test_h5_converter.py
@@ -23,7 +23,7 @@ from functools import partial
from pathlib import Path
from pytest import fixture, importorskip
-import caosdb as db
+import linkahead as db
from caoscrawler.debug_tree import DebugTree
from caoscrawler.hdf5_converter import (convert_basic_element_with_nd_array,
diff --git a/unittests/test_identifiable.py b/unittests/test_identifiable.py
index 074c3843..d94d8525 100644
--- a/unittests/test_identifiable.py
+++ b/unittests/test_identifiable.py
@@ -24,7 +24,7 @@
test identifiable module
"""
-import caosdb as db
+import linkahead as db
import pytest
from caoscrawler.identifiable import Identifiable
from caoscrawler.sync_node import SyncNode
diff --git a/unittests/test_identifiable_adapters.py b/unittests/test_identifiable_adapters.py
index 94685c27..53490bc0 100644
--- a/unittests/test_identifiable_adapters.py
+++ b/unittests/test_identifiable_adapters.py
@@ -32,7 +32,7 @@ from datetime import datetime
from unittest.mock import MagicMock, Mock, patch
from pathlib import Path
-import caosdb as db
+import linkahead as db
import pytest
from caoscrawler.exceptions import (InvalidIdentifiableYAML,
)
diff --git a/unittests/test_json.py b/unittests/test_json.py
index fdb332df..be65a26e 100644
--- a/unittests/test_json.py
+++ b/unittests/test_json.py
@@ -31,7 +31,7 @@ import os
from pytest import raises
-import caosdb as db
+import linkahead as db
from caoscrawler.converters import JSONFileConverter
from pathlib import Path
diff --git a/unittests/test_macros.py b/unittests/test_macros.py
index 53837e92..85fe56cd 100644
--- a/unittests/test_macros.py
+++ b/unittests/test_macros.py
@@ -142,7 +142,7 @@ def test_multi_macros_toplevel(register_macros, macro_store_reset):
dat_loader = list(yaml.safe_load_all("""
---
metadata:
- crawler-version: 0.5.1
+ crawler-version: 0.7.2
macros:
- !defmacro
name: test_one
@@ -171,7 +171,7 @@ def test_load_definition(register_macros, macro_store_reset):
txt = """
---
metadata:
- crawler-version: 0.5.1
+ crawler-version: 0.7.2
---
extroot:
type: Directory
@@ -188,7 +188,7 @@ extroot:
cfood = _temp_file_load("""
---
metadata:
- crawler-version: 0.5.1
+ crawler-version: 0.7.2
macros:
- !defmacro
name: test_one
@@ -223,7 +223,6 @@ extroot3:
assert cfood["extroot3"]["subtree"]["SimulationData"]["match"] == "SimulationData"
-@pytest.mark.xfail
def test_replace_arbitrary_objects(register_macros, macro_store_reset):
"""
See: https://gitlab.indiscale.com/caosdb/src/caosdb-crawler/-/issues/24
@@ -234,27 +233,34 @@ defs:
name: test
params:
b: 25
+ testvar_list_empty: []
testvar_list:
- a
- $b
+ testvar_dict_empty: {}
testvar_dict:
t1: a
t2: $b
definition:
replaced1:
$b: ok
- c: $testvar_dict
- d: $testvar_list
+ dict_empty: $testvar_dict_empty
+ dict: $testvar_dict
+ list_empty: $testvar_list_empty
+ list: ${testvar_list}
testnode:
obl: !macro
test:
""", Loader=yaml.SafeLoader)
print(yaml.dump(dat))
- assert dat["testnode"]["obl"]["replaced1"]["c"]["t1"] == "a"
- assert dat["testnode"]["obl"]["replaced1"]["c"]["t2"] == "25"
- assert dat["testnode"]["obl"]["replaced1"]["d"][0] == "a"
- assert dat["testnode"]["obl"]["replaced1"]["d"][1] == "25"
+ replaced = dat["testnode"]["obl"]["replaced1"]
+ assert replaced["dict_empty"] == {}
+ assert replaced["dict"]["t1"] == "a"
+ assert replaced["dict"]["t2"] == 25
+ assert replaced["list_empty"] == []
+ assert replaced["list"][0] == "a"
+ assert replaced["list"][1] == 25
def test_macros_in_macros(register_macros, macro_store_reset):
@@ -264,7 +270,7 @@ def test_macros_in_macros(register_macros, macro_store_reset):
cfood = _temp_file_load("""
---
metadata:
- crawler-version: 0.5.1
+ crawler-version: 0.7.2
macros:
- !defmacro
name: one_macro
@@ -293,11 +299,11 @@ extroot: !macro
assert "test_macro" not in cfood["extroot"]
assert cfood["extroot"]["macro_top"]["not_macro"]["a"] == 26
d = cfood["extroot"]["macro_top"]
- assert d["macro_sub_17"]["b"] == "17"
+ assert d["macro_sub_17"]["b"] == 17
assert d["macro_sub_17"]["another_param"] == 3
- assert d["macro_sub_25"]["b"] == "25"
+ assert d["macro_sub_25"]["b"] == 25
assert d["macro_sub_25"]["another_param"] == 3
- assert d["macro_sub_98"]["b"] == "98"
+ assert d["macro_sub_98"]["b"] == 98
assert d["macro_sub_98"]["another_param"] == 3
@@ -309,7 +315,7 @@ def test_silent_overwrite(register_macros, macro_store_reset):
cfood = _temp_file_load("""
---
metadata:
- crawler-version: 0.5.1
+ crawler-version: 0.7.2
macros:
- !defmacro
name: one_macro
@@ -340,7 +346,7 @@ def test_circular_macro_definition(register_macros, macro_store_reset):
cfood = _temp_file_load("""
---
metadata:
- crawler-version: 0.5.1
+ crawler-version: 0.7.2
macros:
- !defmacro
name: test_one
@@ -389,7 +395,7 @@ def test_use_macro_twice():
cfood = _temp_file_load("""
---
metadata:
- crawler-version: 0.5.1
+ crawler-version: 0.7.2
macros:
- !defmacro
name: test_twice
@@ -410,9 +416,9 @@ extroot: !macro
""")
for name in ["once", "twice", "default_name"]:
assert name in cfood["extroot"]
- assert cfood["extroot"]["once"]["something"]["a"] == "4"
- assert cfood["extroot"]["twice"]["something"]["a"] == "5"
- assert cfood["extroot"]["default_name"]["something"]["a"] == "4"
+ assert cfood["extroot"]["once"]["something"]["a"] == 4
+ assert cfood["extroot"]["twice"]["something"]["a"] == 5
+ assert cfood["extroot"]["default_name"]["something"]["a"] == 4
# Code sample to generate the expanded macro:
# with open("expanded_test_macro.yaml", "w") as f:
# f.write(yaml.dump(cfood))
@@ -423,7 +429,7 @@ def test_documentation_example_2():
cfood = _temp_file_load("""
---
metadata:
- crawler-version: 0.5.1
+ crawler-version: 0.7.2
macros:
- !defmacro
name: MarkdownFile
@@ -461,7 +467,7 @@ def test_documentation_example_1():
cfood = _temp_file_load("""
---
metadata:
- crawler-version: 0.5.1
+ crawler-version: 0.7.2
macros:
- !defmacro
name: SimulationDatasetFile
@@ -510,7 +516,7 @@ def test_def_replacements():
cfood = _temp_file_load("""
---
metadata:
- crawler-version: 0.5.1
+ crawler-version: 0.7.2
macros:
- !defmacro
name: test_def_replacements
@@ -573,9 +579,9 @@ testnode:
test2:
a: 4
""", Loader=yaml.SafeLoader)
- assert dat["testnode"]["obl"]["expanded_4"]["param"] == "4"
- assert dat["testnode"]["obl"]["expanded_2"]["param"] == "2"
- assert dat["testnode"]["obl"]["expanded_4_test2"]["param"] == "4"
+ assert dat["testnode"]["obl"]["expanded_4"]["param"] == 4
+ assert dat["testnode"]["obl"]["expanded_2"]["param"] == 2
+ assert dat["testnode"]["obl"]["expanded_4_test2"]["param"] == 4
def test_variable_in_macro_definition(register_macros, macro_store_reset):
@@ -598,7 +604,7 @@ testnode:
- a: 2
b: 4
""", Loader=yaml.SafeLoader)
- assert dat["testnode"]["obl"]["expanded_4"]["param"] == "4"
- assert dat["testnode"]["obl"]["expanded_4"]["param_b"] == "4"
- assert dat["testnode"]["obl"]["expanded_2"]["param"] == "2"
- assert dat["testnode"]["obl"]["expanded_2"]["param_b"] == "4"
+ assert dat["testnode"]["obl"]["expanded_4"]["param"] == 4
+ assert dat["testnode"]["obl"]["expanded_4"]["param_b"] == 4
+ assert dat["testnode"]["obl"]["expanded_2"]["param"] == 2
+ assert dat["testnode"]["obl"]["expanded_2"]["param_b"] == 4
diff --git a/unittests/test_parent_cfood.yml b/unittests/test_parent_cfood.yml
index b8d0eaf5..cd63e81b 100644
--- a/unittests/test_parent_cfood.yml
+++ b/unittests/test_parent_cfood.yml
@@ -1,6 +1,6 @@
---
metadata:
- crawler-version: 0.6.1
+ crawler-version: 0.7.2
---
Definitions:
type: Definitions
diff --git a/unittests/test_scanner.py b/unittests/test_scanner.py
index c0ce736f..226b5040 100644
--- a/unittests/test_scanner.py
+++ b/unittests/test_scanner.py
@@ -31,7 +31,7 @@ from pathlib import Path
from tempfile import NamedTemporaryFile
from unittest.mock import MagicMock, Mock, patch
-import caosdb as db
+import linkahead as db
import pytest
import yaml
from caoscrawler.crawl import Crawler
diff --git a/unittests/test_schema.py b/unittests/test_schema.py
index 0d5bebce..3b576c9b 100644
--- a/unittests/test_schema.py
+++ b/unittests/test_schema.py
@@ -3,7 +3,7 @@
# A. Schlemmer, 06/2021
from importlib_resources import files
-import caosdb as db
+import linkahead as db
from os.path import join, dirname
from caoscrawler import Crawler
diff --git a/unittests/test_table_converter.py b/unittests/test_table_converter.py
index 178393d9..3b563fd3 100644
--- a/unittests/test_table_converter.py
+++ b/unittests/test_table_converter.py
@@ -32,7 +32,7 @@ import os
from os.path import basename, dirname, join
from pathlib import Path
-import caosdb as db
+import linkahead as db
import pytest
from caoscrawler import Crawler
from caoscrawler.converters import (Converter, ConverterValidationError,
diff --git a/unittests/test_transformers.py b/unittests/test_transformers.py
index 02d932d1..e7fae484 100644
--- a/unittests/test_transformers.py
+++ b/unittests/test_transformers.py
@@ -34,7 +34,7 @@ from pathlib import Path
from tempfile import NamedTemporaryFile
from unittest.mock import MagicMock, Mock, patch
-import caosdb as db
+import linkahead as db
import pytest
import yaml
from caoscrawler.converters import Converter, ListElementConverter
diff --git a/unittests/test_variable_substitutions.py b/unittests/test_variable_substitutions.py
index 09f78df6..90d144b0 100644
--- a/unittests/test_variable_substitutions.py
+++ b/unittests/test_variable_substitutions.py
@@ -25,7 +25,7 @@ from os.path import basename, dirname, join
from pathlib import Path
from unittest.mock import MagicMock, Mock
-import caosdb as db
+import linkahead as db
import pytest
import yaml
from caoscrawler import Crawler
@@ -35,7 +35,7 @@ from caoscrawler.identifiable_adapters import (IdentifiableAdapter,
from caoscrawler.scanner import scan_directory
from caoscrawler.structure_elements import (DictListElement, DictTextElement,
File)
-from caosdb.apiutils import compare_entities
+from linkahead.apiutils import compare_entities
from pytest import raises
from utils import dircheckstr as dircheckstr_base
--
GitLab