Revert "Revert "Merge branch 'f-extend-yaml-model' into 'dev'""

Summary

For #72 (closed). Repairs the previous MR which introduced the role and value keywords into the yaml model.

Focus

For the actual changes after the reversion of the revert, have a look at https://gitlab.indiscale.com/caosdb/src/caosdb-advanced-user-tools/-/merge_requests/36/diffs?diff_id=5835&start_sha=619c25e6214aaeb5f91f10252a8dda104b0e46b5. Note that role: File is postponed until later (#74), as is the implementation of the valued_properties keyword (#75).

Test Environment

New unit tests and integration tests (use with the debug profile of caosdb-deploy as described in README_SETUP.md). Depending on your level of paranoia, you can also test it with the AWI server profile:

1. Check out server-profile in local-testing-main and update (and init, if necessary) the integration test submodule.
2. Run integration-tests/tests/test.sh

Check List for the Author

Please, prepare your MR for a review. Be sure to write a summary and a focus and create gitlab comments for the reviewer. They should guide the reviewer through the changes, explain your changes and also point out open questions. For further good practices have a look at our review guidelines

• All automated tests pass
• Reference related issues
• Up-to-date CHANGELOG.md (or not necessary)
• Annotations in code (Gitlab comments)
  • Intent of new code
  • Problems with old code
  • Why this implementation?

Check List for the Reviewer

• I understand the intent of this MR
• All automated tests pass
• Up-to-date CHANGELOG.md (or not necessary)
• The test environment setup works and the intended behavior is reproducible in the test environment
• In-code documentation and comments are up-to-date.
• Check: Are there specifications? Are they satisfied?

For further good practices have a look at our review guidelines.

src/caosadvancedtools/models/parser.py

@@ -25,7 +25,8 @@ import yaml
 from .data_model import DataModel
 
 # Keywords which are allowed in data model descriptions.
-KEYWORDS = ["parent",
+KEYWORDS = ["parent",  # deprecated, use inherit_from_* instead:
+                       # https://gitlab.com/caosdb/caosdb-advanced-user-tools/-/issues/36
             "importance",
             "datatype",  # for example TEXT, INTEGER or REFERENCE
             "unit",
@@ -35,8 +36,12 @@ KEYWORDS = ["parent",
             "suggested_properties",
             "inherit_from_recommended",
             "inherit_from_suggested",
-            "inherit_from_obligatory", ]
+            "inherit_from_obligatory",
+            "role",
+            "value",
+            ]
 
+# TODO: check whether it's really ignored
 # These KEYWORDS are not forbidden as properties, but merely ignored.
 KEYWORDS_IGNORED = [
     "unit",
@@ -109,6 +114,10 @@ def parse_model_from_string(string):
 
 class Parser(object):
     def __init__(self):
+        """Initialize an empty parser object and initialize the dictionary of entities and the list of
+        treated elements.
+
+        """
         self.model = {}
         self.treated = []
 
@@ -177,13 +186,11 @@ class Parser(object):
             ymlmodel["extern"] = []
 
         for name in ymlmodel["extern"]:
-            if db.execute_query("COUNT Property {}".format(name)) > 0:
-                self.model[name] = db.execute_query(
-                    "FIND Property WITH name={}".format(name), unique=True)
-
-            elif db.execute_query("COUNT RecordType {}".format(name)) > 0:
-                self.model[name] = db.execute_query(
-                    "FIND RecordType WITH name={}".format(name), unique=True)
+            for role in ("Property", "RecordType", "Record", "File"):
+                if db.execute_query("COUNT {} {}".format(role, name)) > 0:
+                    self.model[name] = db.execute_query(
+                        "FIND {} WITH name={}".format(role, name), unique=True)
+                    break
             else:
                 raise Exception("Did not find {}".format(name))
 
@@ -235,6 +242,8 @@ class Parser(object):
         """ adds names of Properties and RecordTypes to the model dictionary
 
         Properties are also initialized.
+
+        name is the key of the yaml element and definition the value.
         """
 
         if name == "__line__":
@@ -258,9 +267,29 @@ class Parser(object):
             # and create the new property
             self.model[name] = db.Property(name=name,
                                            datatype=definition["datatype"])
+        elif (self.model[name] is None and isinstance(definition, dict)
+              and "role" in definition):
+            if definition["role"] == "RecordType":
+                self.model[name] = db.RecordType(name=name)
+            elif definition["role"] == "Record":
+                self.model[name] = db.Record(name=name)
+            elif definition["role"] == "File":
+                # TODO(fspreck) Implement files at some later point in time
+                raise NotImplementedError(
+                    "The definition of file objects is not yet implemented.")
+
+                # self.model[name] = db.File(name=name)
+            elif definition["role"] == "Property":
+                self.model[name] = db.Property(name=name)
+            else:
+                raise RuntimeError("Unknown role {} in definition of entity.".format(
+                    definition["role"]))
 
-        # add other definitions recursively
+        # for setting values of properties directly:
+        if not isinstance(definition, dict):
+            return
 
+        # add other definitions recursively
         for prop_type in ["recommended_properties",
                           "suggested_properties", "obligatory_properties"]:
 
@@ -284,7 +313,25 @@ class Parser(object):
                     raise
 
     def _add_to_recordtype(self, ent_name, props, importance):
-        """Add properties to a RecordType."""
+        """Add properties to a RecordType.
+
+        Parameters
+        ----------
+        ent_name : str
+          The name of the entity to which the properties shall be added.
+
+        props : dict [str -> dict or :doc:`Entity`]
+          The properties, indexed by their names.  Properties may be given as :doc:`Entity` objects
+          or as dictionaries.
+
+        importance
+          The importance as used in :doc:`Entity.add_property`.
+
+        Returns
+        -------
+        None
+
+        """
 
         for n, e in props.items():
             if n in KEYWORDS:
@@ -297,15 +344,28 @@ class Parser(object):
                 continue
             n = self._stringify(n)
 
-            if (isinstance(e, dict) and "datatype" in e
-                    and (_get_listdatatype(e["datatype"]) is not None)):
-                self.model[ent_name].add_property(
-                    name=n,
-                    importance=importance,
-                    datatype=db.LIST(_get_listdatatype(e["datatype"])))
+            if isinstance(e, dict):
+                if "datatype" in e and _get_listdatatype(e["datatype"]) is not None:
+                    # Reuse the existing datatype for lists.
+                    datatype = db.LIST(_get_listdatatype(e["datatype"]))
+                else:
+                    # Ignore a possible e["datatype"] here if it's not a list
+                    # since it has been treated in the definition of the
+                    # property (entity) already
+                    datatype = None
+                if "value" in e:
+                    value = e["value"]
+                else:
+                    value = None
+
             else:
-                self.model[ent_name].add_property(name=n,
-                                                  importance=importance)
+                value = e
+                datatype = None
+
+            self.model[ent_name].add_property(name=n,
+                                              value=value,
+                                              importance=importance,
+                                              datatype=datatype)
 
     def _inherit(self, name, prop, inheritance):
         if not isinstance(prop, list):
@@ -328,6 +388,10 @@ class Parser(object):
             if definition is None:
                 return
 
+            # for setting values of properties directly:
+            if not isinstance(definition, dict):
+                return
+
             if ("datatype" in definition
                     and definition["datatype"].startswith("LIST")):
 
@@ -344,6 +408,9 @@ class Parser(object):
                 if prop_name == "unit":
                     self.model[name].unit = prop
 
+                elif prop_name == "value":
+                    self.model[name].value = prop
+
                 elif prop_name == "description":
                     self.model[name].description = prop
 
@@ -372,6 +439,10 @@ class Parser(object):
                 elif prop_name == "datatype":
                     continue
 
+                # role has already been used
+                elif prop_name == "role":
+                    continue
+
                 elif prop_name == "inherit_from_obligatory":
                     self._inherit(name, prop, db.OBLIGATORY)
                 elif prop_name == "inherit_from_recommended":
@@ -432,7 +503,8 @@ class Parser(object):
 
                     continue
 
-                raise ValueError("Property {} has an unknown datatype: {}".format(value.name, value.datatype))
+                raise ValueError("Property {} has an unknown datatype: {}".format(
+                    value.name, value.datatype))
 
     def _set_recordtypes(self):
         """ properties are defined in first iteration; set remaining as RTs """