unittests/test_macros.py

@@ -59,7 +59,7 @@ def _temp_file_load(txt: str):
 
 def test_macros(register_macros, macro_store_reset):
     dat = yaml.load("""
-defs:
+macros:
 - !defmacro
   name: test
   params:
@@ -85,7 +85,7 @@ testnode:
 
 def test_macro_list_replacment(register_macros, macro_store_reset):
     dat = yaml.load("""
-defs:
+macros:
 - !defmacro
   name: test
   params:
@@ -112,7 +112,7 @@ testnode:
 
 def test_multi_macros(register_macros, macro_store_reset):
     dat = yaml.load("""
-defs:
+macros:
 - !defmacro
   name: test_one
   params: {}
@@ -142,7 +142,7 @@ def test_multi_macros_toplevel(register_macros, macro_store_reset):
     dat_loader = list(yaml.safe_load_all("""
 ---
 metadata:
-  crawler-version: 0.5.1
+  crawler-version: 0.7.2
   macros:
     - !defmacro
       name: test_one
@@ -171,7 +171,7 @@ def test_load_definition(register_macros, macro_store_reset):
     txt = """
 ---
 metadata:
-  crawler-version: 0.5.1
+  crawler-version: 0.7.2
 ---
 extroot:
   type: Directory
@@ -188,12 +188,13 @@ extroot:
     cfood = _temp_file_load("""
 ---
 metadata:
-  crawler-version: 0.5.1
+  crawler-version: 0.7.2
   macros:
     - !defmacro
       name: test_one
       params: {}
       definition:
+        type: TextElement
         replaced1: ok
     - !defmacro
       name: test_two
@@ -213,6 +214,7 @@ extroot:
 extroot2: !macro  # test top level macro
   test_one:
 extroot3:
+  type: Directory
   subtree:
     SimulationData: !macro
       test_two:
@@ -223,38 +225,44 @@ extroot3:
     assert cfood["extroot3"]["subtree"]["SimulationData"]["match"] == "SimulationData"
 
 
-@pytest.mark.xfail
 def test_replace_arbitrary_objects(register_macros, macro_store_reset):
     """
     See: https://gitlab.indiscale.com/caosdb/src/caosdb-crawler/-/issues/24
     """
     dat = yaml.load("""
-defs:
+macros:
 - !defmacro
   name: test
   params:
     b: 25
+    testvar_list_empty: []
     testvar_list:
     - a
     - $b
+    testvar_dict_empty: {}
     testvar_dict:
       t1: a
       t2: $b
   definition:
     replaced1:
       $b: ok
-      c: $testvar_dict
-      d: $testvar_list
+      dict_empty: $testvar_dict_empty
+      dict: $testvar_dict
+      list_empty: $testvar_list_empty
+      list: ${testvar_list}
 
 testnode:
   obl: !macro
     test:
 """, Loader=yaml.SafeLoader)
     print(yaml.dump(dat))
-    assert dat["testnode"]["obl"]["replaced1"]["c"]["t1"] == "a"
-    assert dat["testnode"]["obl"]["replaced1"]["c"]["t2"] == "25"
-    assert dat["testnode"]["obl"]["replaced1"]["d"][0] == "a"
-    assert dat["testnode"]["obl"]["replaced1"]["d"][1] == "25"
+    replaced = dat["testnode"]["obl"]["replaced1"]
+    assert replaced["dict_empty"] == {}
+    assert replaced["dict"]["t1"] == "a"
+    assert replaced["dict"]["t2"] == 25
+    assert replaced["list_empty"] == []
+    assert replaced["list"][0] == "a"
+    assert replaced["list"][1] == 25
 
 
 def test_macros_in_macros(register_macros, macro_store_reset):
@@ -264,13 +272,14 @@ def test_macros_in_macros(register_macros, macro_store_reset):
     cfood = _temp_file_load("""
 ---
 metadata:
-  crawler-version: 0.5.1
+  crawler-version: 0.7.2
   macros:
     - !defmacro
       name: one_macro
       params:
         a: 25
       definition:
+        type: DictElement
         macro_sub_$a:
           b: $a
           another_param: 3
@@ -278,6 +287,7 @@ metadata:
       name: test_macrodef
       params: {}
       definition:
+        type: DictElement
         macro_top: !macro
           one_macro:
           - a: 17
@@ -293,11 +303,11 @@ extroot: !macro
     assert "test_macro" not in cfood["extroot"]
     assert cfood["extroot"]["macro_top"]["not_macro"]["a"] == 26
     d = cfood["extroot"]["macro_top"]
-    assert d["macro_sub_17"]["b"] == "17"
+    assert d["macro_sub_17"]["b"] == 17
     assert d["macro_sub_17"]["another_param"] == 3
-    assert d["macro_sub_25"]["b"] == "25"
+    assert d["macro_sub_25"]["b"] == 25
     assert d["macro_sub_25"]["another_param"] == 3
-    assert d["macro_sub_98"]["b"] == "98"
+    assert d["macro_sub_98"]["b"] == 98
     assert d["macro_sub_98"]["another_param"] == 3
 
 
@@ -309,7 +319,7 @@ def test_silent_overwrite(register_macros, macro_store_reset):
     cfood = _temp_file_load("""
 ---
 metadata:
-  crawler-version: 0.5.1
+  crawler-version: 0.7.2
   macros:
     - !defmacro
       name: one_macro
@@ -340,12 +350,13 @@ def test_circular_macro_definition(register_macros, macro_store_reset):
     cfood = _temp_file_load("""
 ---
 metadata:
-  crawler-version: 0.5.1
+  crawler-version: 0.7.2
   macros:
     - !defmacro
       name: test_one
       params: {}
       definition: !macro
+        type: TextElement
         test_two:
     - !defmacro
       name: test_two
@@ -361,6 +372,7 @@ metadata:
       name: test_four
       params: {}
       definition: !macro
+        type: TextElement
         test_four:
 ---
 extroot: !macro
@@ -389,7 +401,7 @@ def test_use_macro_twice():
     cfood = _temp_file_load("""
 ---
 metadata:
-  crawler-version: 0.5.1
+  crawler-version: 0.7.2
   macros:
     - !defmacro
       name: test_twice
@@ -397,6 +409,7 @@ metadata:
         macro_name: default_name
         a: 4
       definition:
+        type: DictElement
         $macro_name:
           something:
             a: $a
@@ -410,9 +423,9 @@ extroot: !macro
     """)
     for name in ["once", "twice", "default_name"]:
         assert name in cfood["extroot"]
-    assert cfood["extroot"]["once"]["something"]["a"] == "4"
-    assert cfood["extroot"]["twice"]["something"]["a"] == "5"
-    assert cfood["extroot"]["default_name"]["something"]["a"] == "4"
+    assert cfood["extroot"]["once"]["something"]["a"] == 4
+    assert cfood["extroot"]["twice"]["something"]["a"] == 5
+    assert cfood["extroot"]["default_name"]["something"]["a"] == 4
     # Code sample to generate the expanded macro:
     # with open("expanded_test_macro.yaml", "w") as f:
     #     f.write(yaml.dump(cfood))
@@ -423,7 +436,7 @@ def test_documentation_example_2():
     cfood = _temp_file_load("""
 ---
 metadata:
-  crawler-version: 0.5.1
+  crawler-version: 0.7.2
   macros:
   - !defmacro
     name: MarkdownFile
@@ -461,7 +474,7 @@ def test_documentation_example_1():
     cfood = _temp_file_load("""
 ---
 metadata:
-  crawler-version: 0.5.1
+  crawler-version: 0.7.2
   macros:
   - !defmacro
     name: SimulationDatasetFile
@@ -510,7 +523,7 @@ def test_def_replacements():
     cfood = _temp_file_load("""
 ---
 metadata:
-  crawler-version: 0.5.1
+  crawler-version: 0.7.2
   macros:
     - !defmacro
       name: test_def_replacements
@@ -549,7 +562,7 @@ extroot: !macro
 
 def test_list_macro_application(register_macros, macro_store_reset):
     dat = yaml.load("""
-defs:
+macros:
 - !defmacro
   name: test
   params:
@@ -573,14 +586,14 @@ testnode:
     test2:
       a: 4
 """, Loader=yaml.SafeLoader)
-    assert dat["testnode"]["obl"]["expanded_4"]["param"] == "4"
-    assert dat["testnode"]["obl"]["expanded_2"]["param"] == "2"
-    assert dat["testnode"]["obl"]["expanded_4_test2"]["param"] == "4"
+    assert dat["testnode"]["obl"]["expanded_4"]["param"] == 4
+    assert dat["testnode"]["obl"]["expanded_2"]["param"] == 2
+    assert dat["testnode"]["obl"]["expanded_4_test2"]["param"] == 4
 
 
 def test_variable_in_macro_definition(register_macros, macro_store_reset):
     dat = yaml.load("""
-defs:
+macros:
 - !defmacro
   name: test
   params:
@@ -598,7 +611,7 @@ testnode:
     - a: 2
       b: 4
 """, Loader=yaml.SafeLoader)
-    assert dat["testnode"]["obl"]["expanded_4"]["param"] == "4"
-    assert dat["testnode"]["obl"]["expanded_4"]["param_b"] == "4"
-    assert dat["testnode"]["obl"]["expanded_2"]["param"] == "2"
-    assert dat["testnode"]["obl"]["expanded_2"]["param_b"] == "4"
+    assert dat["testnode"]["obl"]["expanded_4"]["param"] == 4
+    assert dat["testnode"]["obl"]["expanded_4"]["param_b"] == 4
+    assert dat["testnode"]["obl"]["expanded_2"]["param"] == 2
+    assert dat["testnode"]["obl"]["expanded_2"]["param_b"] == 4

unittests/test_parent_cfood.yml

 ---
 metadata:
-  crawler-version: 0.6.1
+  crawler-version: 0.7.2
 ---
 Definitions:
   type: Definitions

unittests/test_scanner.py

@@ -31,7 +31,7 @@ from pathlib import Path
 from tempfile import NamedTemporaryFile
 from unittest.mock import MagicMock, Mock, patch
 
-import caosdb as db
+import linkahead as db
 import pytest
 import yaml
 from caoscrawler.crawl import Crawler

unittests/test_schema.py

@@ -3,7 +3,7 @@
 # A. Schlemmer, 06/2021
 
 from importlib_resources import files
-import caosdb as db
+import linkahead as db
 
 from os.path import join, dirname
 from caoscrawler import Crawler
@@ -27,6 +27,13 @@ def rfp(*pathcomponents):
 def test_schema_validation():
     load_definition(rfp("scifolder_cfood.yml"))
     load_definition(rfp("scifolder_extended.yml"))
+    load_definition(rfp("record_from_dict_cfood.yml"))
 
     with raises(ValidationError, match=".*enum.*"):
         load_definition(rfp("broken_cfoods", "broken1.yml"))
+
+    with raises(ValidationError, match=".*required.*"):
+        load_definition(rfp("broken_cfoods", "broken_record_from_dict.yml"))
+
+    with raises(ValidationError, match=".*required.*"):
+        load_definition(rfp("broken_cfoods", "broken_record_from_dict_2.yml"))

unittests/test_sync_graph.py

@@ -25,10 +25,15 @@ from unittest.mock import MagicMock, Mock, patch
 
 import linkahead as db
 import pytest
-from test_crawler import basic_retrieve_by_name_mock_up, mock_get_entity_by
+from test_crawler import (basic_retrieve_by_name_mock_up,
+                          mock_cached_only_rt_allow_empty,
+                          mock_get_entity_by,
+                          )
 
 from caoscrawler.exceptions import (ImpossibleMergeError,
-                                    MissingIdentifyingProperty)
+                                    MissingIdentifyingProperty,
+                                    MissingRecordType,
+                                    )
 from caoscrawler.identifiable import Identifiable
 from caoscrawler.identifiable_adapters import CaosDBIdentifiableAdapter
 from caoscrawler.sync_graph import SyncGraph, _set_each_scalar_value
@@ -651,3 +656,30 @@ def test_set_each_scalar_value():
     assert a.properties[0].value == 42
     _set_each_scalar_value(a, lambda x: x == 42, lambda x: None)
     assert a.properties[0].value is None
+
+
+@patch("caoscrawler.identifiable_adapters.cached_query",
+       new=Mock(side_effect=mock_cached_only_rt_allow_empty))
+def test_merge_referenced_by():
+    """Merging two entities that are referenced by a third entity with nonexistent RecordType.
+
+    See also https://gitlab.com/linkahead/linkahead-crawler/-/issues/95
+    """
+    ident = CaosDBIdentifiableAdapter()
+    ident.load_from_yaml_object({
+        "RT_A": ["name"],
+        "RT_B": [{"is_referenced_by": ["RT_A"]}, "my_id"]
+    })
+    crawled_data: list = []
+    references: list = []
+    for ii in [0, 1]:
+        rec = db.Record().add_parent("RT_B").add_property("my_id", value=ii)
+        references.append(rec)
+        crawled_data.append(rec)
+    rec_a = db.Record(name="Rec_A").add_parent("RT_A")
+    rec_a.add_property("my_ref", value=references)
+    crawled_data.append(rec_a)
+
+    with pytest.raises(MissingRecordType) as mrt:
+        SyncGraph(crawled_data, ident)
+    assert str(mrt.value).endswith("Record type could not be found on server: RT_A")

unittests/test_table_converter.py

@@ -32,7 +32,7 @@ import os
 from os.path import basename, dirname, join
 from pathlib import Path
 
-import caosdb as db
+import linkahead as db
 import pytest
 from caoscrawler import Crawler
 from caoscrawler.converters import (Converter, ConverterValidationError,

unittests/test_transformers.py

@@ -34,7 +34,7 @@ from pathlib import Path
 from tempfile import NamedTemporaryFile
 from unittest.mock import MagicMock, Mock, patch
 
-import caosdb as db
+import linkahead as db
 import pytest
 import yaml
 from caoscrawler.converters import Converter, ListElementConverter
@@ -46,6 +46,38 @@ from pytest import raises
 UNITTESTDIR = Path(__file__).parent
 
 
+@pytest.fixture
+def converter_registry():
+    converter_registry: dict[str, dict[str, str]] = {
+        "Directory": {
+            "converter": "DirectoryConverter",
+            "package": "caoscrawler.converters"},
+        "MarkdownFile": {
+            "converter": "MarkdownFileConverter",
+            "package": "caoscrawler.converters"},
+        "Date": {
+            "converter": "DateElementConverter",
+            "package": "caoscrawler.converters"},
+        "DictElement": {
+            "converter": "DictElementConverter",
+            "package": "caoscrawler.converters"},
+        "TextElement": {
+            "converter": "TextElementConverter",
+            "package": "caoscrawler.converters"},
+        "ListElement": {
+            "converter": "ListElementConverter",
+            "package": "caoscrawler.converters"},
+        "JSONFile": {
+            "converter": "JSONFileConverter",
+            "package": "caoscrawler.converters"},
+    }
+
+    for key, value in converter_registry.items():
+        module = importlib.import_module(value["package"])
+        value["class"] = getattr(module, value["converter"])
+    return converter_registry
+
+
 def test_simple_transformer():
     """
     Test the correct list of returned records by the scanner using the
@@ -82,38 +114,6 @@ def test_simple_transformer():
             assert False
 
 
-@pytest.fixture
-def converter_registry():
-    converter_registry: dict[str, dict[str, str]] = {
-        "Directory": {
-            "converter": "DirectoryConverter",
-            "package": "caoscrawler.converters"},
-        "MarkdownFile": {
-            "converter": "MarkdownFileConverter",
-            "package": "caoscrawler.converters"},
-        "Date": {
-            "converter": "DateElementConverter",
-            "package": "caoscrawler.converters"},
-        "DictElement": {
-            "converter": "DictElementConverter",
-            "package": "caoscrawler.converters"},
-        "TextElement": {
-            "converter": "TextElementConverter",
-            "package": "caoscrawler.converters"},
-        "ListElement": {
-            "converter": "ListElementConverter",
-            "package": "caoscrawler.converters"},
-        "JSONFile": {
-            "converter": "JSONFileConverter",
-            "package": "caoscrawler.converters"},
-    }
-
-    for key, value in converter_registry.items():
-        module = importlib.import_module(value["package"])
-        value["class"] = getattr(module, value["converter"])
-    return converter_registry
-
-
 def test_apply_replace(converter_registry):
     cfood_def = {"type": 'ListElement', "match_name": ".*",
                  'transform': {'test': {'in': '$a', 'out': '$b', 'functions': [{
@@ -146,3 +146,21 @@ def test_apply_replace_from_def(converter_registry):
 
     conv.apply_transformers(values, transformer_functions)
     assert values['b'] == "16:45"
+
+
+def test_empty_functions_list(converter_registry):
+    cfood_def = {"type": 'ListElement',
+                 "match_name": ".*",
+                 'transform': {'test': {'in': '$a', 'out': '$b',
+                                        'functions': []}}}
+    values = GeneralStore()
+    values["a"] = "16_45"
+
+    # transformer_functions  = create_transformer_registry(crawler_definition)
+    transformer_functions = {"replace": replace}
+
+    conv = ListElementConverter(definition=cfood_def, name='test',
+                                converter_registry=converter_registry)
+
+    conv.apply_transformers(values, transformer_functions)
+    assert values['b'] == "16_45"

unittests/test_variable_substitutions.py

@@ -25,7 +25,7 @@ from os.path import basename, dirname, join
 from pathlib import Path
 from unittest.mock import MagicMock, Mock
 
-import caosdb as db
+import linkahead as db
 import pytest
 import yaml
 from caoscrawler import Crawler
@@ -35,7 +35,7 @@ from caoscrawler.identifiable_adapters import (IdentifiableAdapter,
 from caoscrawler.scanner import scan_directory
 from caoscrawler.structure_elements import (DictListElement, DictTextElement,
                                             File)
-from caosdb.apiutils import compare_entities
+from linkahead.apiutils import compare_entities
 from pytest import raises
 
 from utils import dircheckstr as dircheckstr_base