Merge branch 'f-macro-for-objects' into f-csv-cfood-generator

CHANGELOG.md

@@ -12,9 +12,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 * Support for Python 3.12 and experimental support for 3.13
 * `spss_to_datamodel` script.
 * `SPSSConverter` class
+* CFood macros now accept complex objects as values, not just strings.
 
 ### Changed ###
 
+* CFood macros do not render everything into strings now.
+
 ### Deprecated ###
 
 ### Removed ###
@@ -170,6 +173,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - ``add_prefix`` and ``remove_prefix`` arguments for the command line interface
   and the ``crawler_main`` function for the adding/removal of path prefixes when
   creating file entities.
+- More strict checking of `identifiables.yaml`.
+- Better error messages when server does not conform to expected data model.
 
 ### Changed ###
 

integrationtests/basic_example/test_basic.py

@@ -32,7 +32,7 @@ import sys
 from argparse import RawTextHelpFormatter
 from pathlib import Path
 
-import caosdb as db
+import linkahead as db
 import pytest
 import yaml
 from caosadvancedtools.crawler import Crawler as OldCrawler
@@ -42,8 +42,8 @@ from caoscrawler.debug_tree import DebugTree
 from caoscrawler.identifiable import Identifiable
 from caoscrawler.identifiable_adapters import CaosDBIdentifiableAdapter
 from caoscrawler.scanner import scan_directory
-from caosdb import EmptyUniqueQueryError
-from caosdb.utils.register_tests import clear_database, set_test_key
+from linkahead import EmptyUniqueQueryError
+from linkahead.utils.register_tests import clear_database, set_test_key
 
 set_test_key("10b128cf8a1372f30aa3697466bb55e76974e0c16a599bb44ace88f19c8f61e2")
 

integrationtests/test_use_case_simple_presentation.py

@@ -27,12 +27,12 @@ import os
 import pytest
 from subprocess import run
 
-import caosdb as db
+import linkahead as db
 from caosadvancedtools.loadFiles import loadpath
-from caosdb.cached import cache_clear
+from linkahead.cached import cache_clear
 from caosadvancedtools.models import parser as parser
 from caoscrawler.crawl import crawler_main
-from caosdb.utils.register_tests import clear_database, set_test_key
+from linkahead.utils.register_tests import clear_database, set_test_key
 
 
 set_test_key("10b128cf8a1372f30aa3697466bb55e76974e0c16a599bb44ace88f19c8f61e2")

src/caoscrawler/exceptions.py

@@ -27,15 +27,6 @@ class ForbiddenTransaction(Exception):
     pass
 
 
-class MissingReferencingEntityError(Exception):
-    """Thrown if the identifiable requires that some entity references the given entity but there
-    is no such reference """
-
-    def __init__(self, *args, rts=None, **kwargs):
-        self.rts = rts
-        super().__init__(self, *args, **kwargs)
-
-
 class ImpossibleMergeError(Exception):
     """Thrown if due to identifying information, two SyncNodes  or two Properties of SyncNodes
     should be merged, but there is conflicting information that prevents this.
@@ -47,8 +38,29 @@ class ImpossibleMergeError(Exception):
         super().__init__(self, *args, **kwargs)
 
 
+class InvalidIdentifiableYAML(Exception):
+    """Thrown if the identifiable definition is invalid."""
+    pass
+
+
 class MissingIdentifyingProperty(Exception):
     """Thrown if a SyncNode does not have the properties required by the corresponding registered
     identifiable
     """
     pass
+
+
+class MissingRecordType(Exception):
+    """Thrown if an record type can not be found although it is expected that it exists on the
+    server.
+    """
+    pass
+
+
+class MissingReferencingEntityError(Exception):
+    """Thrown if the identifiable requires that some entity references the given entity but there
+    is no such reference """
+
+    def __init__(self, *args, rts=None, **kwargs):
+        self.rts = rts
+        super().__init__(self, *args, **kwargs)

src/caoscrawler/identifiable_adapters.py

@@ -36,7 +36,12 @@ import yaml
 from linkahead.cached import cached_get_entity_by, cached_query
 from linkahead.utils.escape import escape_squoted_text
 
-from .exceptions import MissingIdentifyingProperty, MissingReferencingEntityError
+from .exceptions import (
+    InvalidIdentifiableYAML,
+    MissingIdentifyingProperty,
+    MissingRecordType,
+    MissingReferencingEntityError,
+)
 from .identifiable import Identifiable
 from .sync_node import SyncNode
 from .utils import has_parent
@@ -48,7 +53,10 @@ def get_children_of_rt(rtname):
     """Supply the name of a recordtype. This name and the name of all children RTs are returned in
     a list"""
     escaped = escape_squoted_text(rtname)
-    return [p.name for p in cached_query(f"FIND RECORDTYPE '{escaped}'")]
+    recordtypes = [p.name for p in cached_query(f"FIND RECORDTYPE '{escaped}'")]
+    if not recordtypes:
+        raise MissingRecordType(f"Record type could not be found on server: {rtname}")
+    return recordtypes
 
 
 def convert_value(value: Any) -> str:
@@ -576,19 +584,32 @@ class CaosDBIdentifiableAdapter(IdentifiableAdapter):
         """Load identifiables defined in a yaml file"""
         with open(path, "r", encoding="utf-8") as yaml_f:
             identifiable_data = yaml.safe_load(yaml_f)
+        self.load_from_yaml_object(identifiable_data)
 
-        for key, value in identifiable_data.items():
-            rt = db.RecordType().add_parent(key)
-            for prop_name in value:
+    def load_from_yaml_object(self, identifiable_data):
+        """Load identifiables defined in a yaml object.
+        """
+
+        for rt_name, id_list in identifiable_data.items():
+            rt = db.RecordType().add_parent(rt_name)
+            if not isinstance(id_list, list):
+                raise InvalidIdentifiableYAML(
+                    f"Identifiable contents must be lists, but this was not: {rt_name}")
+            for prop_name in id_list:
                 if isinstance(prop_name, str):
                     rt.add_property(name=prop_name)
                 elif isinstance(prop_name, dict):
                     for k, v in prop_name.items():
+                        if k == "is_referenced_by" and not isinstance(v, list):
+                            raise InvalidIdentifiableYAML(
+                                f"'is_referenced_by' must be a list.  Found in: {rt_name}")
                         rt.add_property(name=k, value=v)
                 else:
-                    NotImplementedError("YAML is not structured correctly")
+                    raise InvalidIdentifiableYAML(
+                        "Identifiable properties must be str or dict, but this one was not:\n"
+                        f"    {rt_name}/{prop_name}")
 
-            self.register_identifiable(key, rt)
+            self.register_identifiable(rt_name, rt)
 
     def register_identifiable(self, name: str, definition: db.RecordType):
         self._registered_identifiables[name] = definition

src/caoscrawler/macros/macro_yaml_object.py

@@ -25,12 +25,17 @@
 # Function to expand a macro in yaml
 # A. Schlemmer, 05/2022
 
+import re
 from dataclasses import dataclass
 from typing import Any, Dict
 from copy import deepcopy
 from string import Template
 
 
+_SAFE_SUBST_PAT = re.compile(r"^\$(?P<key>\w+)$")
+_SAFE_SUBST_PAT_BRACES = re.compile(r"^\$\{(?P<key>\w+)}$")
+
+
 @dataclass
 class MacroDefinition:
     """
@@ -53,6 +58,12 @@ def substitute(propvalue, values: dict):
     Substitution of variables in strings using the variable substitution
     library from python's standard library.
     """
+    # Simple matches are simply replaced by the raw dict entry.
+    if match := (_SAFE_SUBST_PAT.fullmatch(propvalue)
+                 or _SAFE_SUBST_PAT_BRACES.fullmatch(propvalue)):
+        key = match.group("key")
+        if key in values:
+            return values[key]
     propvalue_template = Template(propvalue)
     return propvalue_template.safe_substitute(**values)
 

src/doc/getting_started/helloworld.md

@@ -33,7 +33,7 @@ Then you can do the following interactively in (I)Python. But we recommend that
 copy the code into a script and execute it to spare yourself typing.
 
 ```python
-import caosdb as db
+import linkahead as db
 from datetime import datetime
 from caoscrawler import Crawler, SecurityMode
 from caoscrawler.identifiable_adapters import CaosDBIdentifiableAdapter

unittests/example_cfood.yml

 ---
 metadata:
-  crawler-version: 0.3.1
+  crawler-version: 0.7.2
 ---
 Definitions:
   type: Definitions

unittests/h5_cfood.yml

 ---
 metadata:
-  crawler-version: 0.6.1
+  crawler-version: 0.7.2
 ---
 Converters:
   H5Dataset:

unittests/scifolder_cfood.yml

@@ -4,7 +4,7 @@
 
 ---
 metadata:
-  crawler-version: 0.3.1
+  crawler-version: 0.7.2
 ---
 Definitions:
   type: Definitions

unittests/test_converters.py

@@ -497,7 +497,7 @@ MyElement:
     two_doc_yaml = """
 ---
 metadata:
-  crawler-version: 0.3.1
+  crawler-version: 0.7.2
   Converters:
     MyNewType:
       converter: MyNewTypeConverter

unittests/test_crawler.py

@@ -173,7 +173,15 @@ A:
         model.get_deep("A").id = 2
         return result + [model.get_deep("B")]
     print(query_string)
-    raise NotImplementedError("Mock for this case is missing")
+    raise NotImplementedError(f"Mock for this case is missing: {query_string}")
+
+
+def mock_cached_only_rt_allow_empty(query_string: str):
+    try:
+        result = mock_cached_only_rt(query_string)
+    except NotImplementedError:
+        result = db.Container()
+    return result
 
 
 @pytest.fixture(autouse=True)

unittests/test_data/invalid_identifiable/identifiable_content_no_list.yaml

+Experiment:
+  date:
+    - 1
+    - 2

unittests/test_data/invalid_identifiable/identifiable_no_str_or_dict.yaml

+Experiment:
+- date
+- 23

unittests/test_data/invalid_identifiable/identifiable_referenced_no_list.yaml

+Experiment:
+- date
+Event:
+- is_referenced_by: Experiment
+- event_id

unittests/test_entity_comparison.py

@@ -2,7 +2,7 @@
 # Tests for entity comparison
 # A. Schlemmer, 06/2021
 
-import caosdb as db
+import linkahead as db
 
 import pytest
 from pytest import raises

unittests/test_h5_converter.py

@@ -23,7 +23,7 @@ from functools import partial
 from pathlib import Path
 from pytest import fixture, importorskip
 
-import caosdb as db
+import linkahead as db
 
 from caoscrawler.debug_tree import DebugTree
 from caoscrawler.hdf5_converter import (convert_basic_element_with_nd_array,

unittests/test_identifiable.py

@@ -24,7 +24,7 @@
 test identifiable module
 """
 
-import caosdb as db
+import linkahead as db
 import pytest
 from caoscrawler.identifiable import Identifiable
 from caoscrawler.sync_node import SyncNode

unittests/test_identifiable_adapters.py

@@ -32,8 +32,10 @@ from datetime import datetime
 from unittest.mock import MagicMock, Mock, patch
 from pathlib import Path
 
-import caosdb as db
+import linkahead as db
 import pytest
+from caoscrawler.exceptions import (InvalidIdentifiableYAML,
+                                    )
 from caoscrawler.identifiable import Identifiable
 from caoscrawler.identifiable_adapters import (CaosDBIdentifiableAdapter,
                                                IdentifiableAdapter,
@@ -122,6 +124,23 @@ def test_load_from_yaml_file():
     assert project_i.get_property("title") is not None
 
 
+def test_invalid_yaml():
+    ident = CaosDBIdentifiableAdapter()
+    invalid_dir = UNITTESTDIR / "test_data" / "invalid_identifiable"
+    with pytest.raises(InvalidIdentifiableYAML) as exc:
+        ident.load_from_yaml_definition(invalid_dir / "identifiable_content_no_list.yaml")
+    assert str(exc.value) == "Identifiable contents must be lists, but this was not: Experiment"
+
+    with pytest.raises(InvalidIdentifiableYAML) as exc:
+        ident.load_from_yaml_definition(invalid_dir / "identifiable_referenced_no_list.yaml")
+    assert str(exc.value) == "'is_referenced_by' must be a list.  Found in: Event"
+
+    with pytest.raises(InvalidIdentifiableYAML) as exc:
+        ident.load_from_yaml_definition(invalid_dir / "identifiable_no_str_or_dict.yaml")
+    assert str(exc.value) == ("Identifiable properties must be str or dict, but this one was not:\n"
+                              "    Experiment/23")
+
+
 def test_non_default_name():
     ident = CaosDBIdentifiableAdapter()
     identifiable = ident.get_identifiable(SyncNode(db.Record(name="don't touch it")
@@ -141,8 +160,8 @@ def test_wildcard_ref():
     dummy.id = 1
     identifiable = ident.get_identifiable(SyncNode(rec, db.RecordType()
                                                    .add_parent(name="Person")
-                                                   .add_property(name="is_referenced_by", value=["*"])),
-
+                                                   .add_property(name="is_referenced_by",
+                                                                 value=["*"])),
                                           [dummy]
                                           )
     assert identifiable.backrefs[0] == 1

unittests/test_json.py

@@ -31,7 +31,7 @@ import os
 
 from pytest import raises
 
-import caosdb as db
+import linkahead as db
 
 from caoscrawler.converters import JSONFileConverter
 from pathlib import Path