Merge branch 'f-extend-yaml-model' into 'dev'

Revert "Revert "Merge branch 'f-extend-yaml-model' into 'dev'""

See merge request !36

CHANGELOG.md

@@ -11,6 +11,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - CFood that creates a Record for each line in a csv file
 - `generic_analysis.py` allows to easily call scripts to perform analyses in
   server side scripting [EXPERIMENTAL]
+- New keyword "role" in yaml data model that allows creation of Records and Files.
+- It is now possible to set values of properties and default values of properties
+  directly in the yaml model.
 
 ### Changed ###
 

README_SETUP.md

@@ -38,7 +38,7 @@ Optional h5-crawler:
 
 1. Change directory to `integrationtests/`.
 2. Mount `extroot` to the folder that will be used as extroot. E.g. `sudo mount
-   -o bind extroot ../../caosdb-deploy/profiles/empty/paths/extroot` (or
+   -o bind extroot ../../caosdb-deploy/profiles/debug/paths/extroot` (or
    whatever path the extroot of the empty profile to be used is located at).
 3. Start (or restart) an empty (!) CaosDB instance (with the mounted
    extroot). The database will be cleared during testing, so it's important to

src/caosadvancedtools/models/parser.py

@@ -25,7 +25,8 @@ import yaml
 from .data_model import DataModel
 
 # Keywords which are allowed in data model descriptions.
-KEYWORDS = ["parent",
+KEYWORDS = ["parent",  # deprecated, use inherit_from_* instead:
+                       # https://gitlab.com/caosdb/caosdb-advanced-user-tools/-/issues/36
             "importance",
             "datatype",  # for example TEXT, INTEGER or REFERENCE
             "unit",
@@ -35,8 +36,12 @@ KEYWORDS = ["parent",
             "suggested_properties",
             "inherit_from_recommended",
             "inherit_from_suggested",
-            "inherit_from_obligatory", ]
+            "inherit_from_obligatory",
+            "role",
+            "value",
+            ]
 
+# TODO: check whether it's really ignored
 # These KEYWORDS are not forbidden as properties, but merely ignored.
 KEYWORDS_IGNORED = [
     "unit",
@@ -109,6 +114,10 @@ def parse_model_from_string(string):
 
 class Parser(object):
     def __init__(self):
+        """Initialize an empty parser object and initialize the dictionary of entities and the list of
+        treated elements.
+
+        """
         self.model = {}
         self.treated = []
 
@@ -177,13 +186,11 @@ class Parser(object):
             ymlmodel["extern"] = []
 
         for name in ymlmodel["extern"]:
-            if db.execute_query("COUNT Property {}".format(name)) > 0:
-                self.model[name] = db.execute_query(
-                    "FIND Property WITH name={}".format(name), unique=True)
-
-            elif db.execute_query("COUNT RecordType {}".format(name)) > 0:
-                self.model[name] = db.execute_query(
-                    "FIND RecordType WITH name={}".format(name), unique=True)
+            for role in ("Property", "RecordType", "Record", "File"):
+                if db.execute_query("COUNT {} {}".format(role, name)) > 0:
+                    self.model[name] = db.execute_query(
+                        "FIND {} WITH name={}".format(role, name), unique=True)
+                    break
             else:
                 raise Exception("Did not find {}".format(name))
 
@@ -235,6 +242,8 @@ class Parser(object):
         """ adds names of Properties and RecordTypes to the model dictionary
 
         Properties are also initialized.
+
+        name is the key of the yaml element and definition the value.
         """
 
         if name == "__line__":
@@ -258,9 +267,29 @@ class Parser(object):
             # and create the new property
             self.model[name] = db.Property(name=name,
                                            datatype=definition["datatype"])
+        elif (self.model[name] is None and isinstance(definition, dict)
+              and "role" in definition):
+            if definition["role"] == "RecordType":
+                self.model[name] = db.RecordType(name=name)
+            elif definition["role"] == "Record":
+                self.model[name] = db.Record(name=name)
+            elif definition["role"] == "File":
+                # TODO(fspreck) Implement files at some later point in time
+                raise NotImplementedError(
+                    "The definition of file objects is not yet implemented.")
+
+                # self.model[name] = db.File(name=name)
+            elif definition["role"] == "Property":
+                self.model[name] = db.Property(name=name)
+            else:
+                raise RuntimeError("Unknown role {} in definition of entity.".format(
+                    definition["role"]))
 
-        # add other definitions recursively
+        # for setting values of properties directly:
+        if not isinstance(definition, dict):
+            return
 
+        # add other definitions recursively
         for prop_type in ["recommended_properties",
                           "suggested_properties", "obligatory_properties"]:
 
@@ -284,7 +313,25 @@ class Parser(object):
                     raise
 
     def _add_to_recordtype(self, ent_name, props, importance):
-        """Add properties to a RecordType."""
+        """Add properties to a RecordType.
+
+        Parameters
+        ----------
+        ent_name : str
+          The name of the entity to which the properties shall be added.
+
+        props : dict [str -> dict or :doc:`Entity`]
+          The properties, indexed by their names.  Properties may be given as :doc:`Entity` objects
+          or as dictionaries.
+
+        importance
+          The importance as used in :doc:`Entity.add_property`.
+
+        Returns
+        -------
+        None
+
+        """
 
         for n, e in props.items():
             if n in KEYWORDS:
@@ -297,15 +344,28 @@ class Parser(object):
                 continue
             n = self._stringify(n)
 
-            if (isinstance(e, dict) and "datatype" in e
-                    and (_get_listdatatype(e["datatype"]) is not None)):
-                self.model[ent_name].add_property(
-                    name=n,
-                    importance=importance,
-                    datatype=db.LIST(_get_listdatatype(e["datatype"])))
+            if isinstance(e, dict):
+                if "datatype" in e and _get_listdatatype(e["datatype"]) is not None:
+                    # Reuse the existing datatype for lists.
+                    datatype = db.LIST(_get_listdatatype(e["datatype"]))
+                else:
+                    # Ignore a possible e["datatype"] here if it's not a list
+                    # since it has been treated in the definition of the
+                    # property (entity) already
+                    datatype = None
+                if "value" in e:
+                    value = e["value"]
+                else:
+                    value = None
+
             else:
-                self.model[ent_name].add_property(name=n,
-                                                  importance=importance)
+                value = e
+                datatype = None
+
+            self.model[ent_name].add_property(name=n,
+                                              value=value,
+                                              importance=importance,
+                                              datatype=datatype)
 
     def _inherit(self, name, prop, inheritance):
         if not isinstance(prop, list):
@@ -328,6 +388,10 @@ class Parser(object):
             if definition is None:
                 return
 
+            # for setting values of properties directly:
+            if not isinstance(definition, dict):
+                return
+
             if ("datatype" in definition
                     and definition["datatype"].startswith("LIST")):
 
@@ -344,6 +408,9 @@ class Parser(object):
                 if prop_name == "unit":
                     self.model[name].unit = prop
 
+                elif prop_name == "value":
+                    self.model[name].value = prop
+
                 elif prop_name == "description":
                     self.model[name].description = prop
 
@@ -372,6 +439,10 @@ class Parser(object):
                 elif prop_name == "datatype":
                     continue
 
+                # role has already been used
+                elif prop_name == "role":
+                    continue
+
                 elif prop_name == "inherit_from_obligatory":
                     self._inherit(name, prop, db.OBLIGATORY)
                 elif prop_name == "inherit_from_recommended":
@@ -432,7 +503,8 @@ class Parser(object):
 
                     continue
 
-                raise ValueError("Property {} has an unknown datatype: {}".format(value.name, value.datatype))
+                raise ValueError("Property {} has an unknown datatype: {}".format(
+                    value.name, value.datatype))
 
     def _set_recordtypes(self):
         """ properties are defined in first iteration; set remaining as RTs """

tox.ini

@@ -11,3 +11,6 @@ deps=nose
     xlrd == 1.2
     h5py
 commands=py.test --cov=caosadvancedtools -vv {posargs}
+
+[flake8]
+max-line-length=100

unittests/test_parser.py

 import unittest
+from datetime import date
 from tempfile import NamedTemporaryFile
+from pytest import raises
 
 import caosdb as db
 from caosadvancedtools.models.parser import (TwiceDefinedException,
@@ -15,6 +17,8 @@ def to_file(string):
 
     return f.name
 
+# TODO: check purpose of this function... add documentation
+
 
 def parse_str(string):
     parse_model_from_yaml(to_file(string))
@@ -68,7 +72,8 @@ RT2:
         a:
 """
 
-        self.assertRaises(TwiceDefinedException, lambda: parse_model_from_yaml(to_file(string)))
+        self.assertRaises(TwiceDefinedException,
+                          lambda: parse_model_from_yaml(to_file(string)))
 
     def test_typical_case(self):
         string = """
@@ -103,7 +108,8 @@ RT5:
 - RT1:
 - RT2:
 """
-        self.assertRaises(ValueError, lambda: parse_model_from_yaml(to_file(string)))
+        self.assertRaises(
+            ValueError, lambda: parse_model_from_yaml(to_file(string)))
 
     def test_unknown_kwarg(self):
         string = """
@@ -111,7 +117,8 @@ RT1:
   datetime:
     p1:
 """
-        self.assertRaises(ValueError, lambda: parse_model_from_yaml(to_file(string)))
+        self.assertRaises(
+            ValueError, lambda: parse_model_from_yaml(to_file(string)))
 
     def test_definition_in_inheritance(self):
         string = """
@@ -121,7 +128,8 @@ RT2:
   - RT1:
     description: "tach"
 """
-        self.assertRaises(ValueError, lambda: parse_model_from_yaml(to_file(string)))
+        self.assertRaises(
+            ValueError, lambda: parse_model_from_yaml(to_file(string)))
 
     def test_inheritance(self):
         string = """
@@ -301,6 +309,8 @@ class ExternTest(unittest.TestCase):
 class ErrorMessageTest(unittest.TestCase):
     """Tests for understandable error messages."""
 
+    # Note: This was changed with implementation of role keyword
+    @unittest.expectedFailure
     def test_non_dict(self):
         """When a value is given, where a list or mapping is expected."""
         recordtype_value = """
@@ -328,3 +338,139 @@ A:
             with self.assertRaises(YamlDefinitionError) as yde:
                 parse_str(string)
             assert("line {}".format(line) in yde.exception.args[0])
+
+
+def test_define_role():
+    model = """
+A:
+  role: Record
+"""
+    entities = parse_model_from_string(model)
+    assert "A" in entities
+    assert isinstance(entities["A"], db.Record)
+    assert entities["A"].role == "Record"
+
+    model = """
+A:
+  role: Record
+  inherit_from_obligatory:
+  - C
+  obligatory_properties:
+    b:
+b:
+  datatype: INTEGER
+C:
+  obligatory_properties:
+    b:
+D:
+  role: RecordType
+"""
+    entities = parse_model_from_string(model)
+    for l, ent in (("A", "Record"), ("b", "Property"),
+                   ("C", "RecordType"), ("D", "RecordType")):
+        assert l in entities
+        assert isinstance(entities[l], getattr(db, ent))
+        assert entities[l].role == ent
+
+    assert entities["A"].parents[0].name == "C"
+    assert entities["A"].name == "A"
+
+    assert entities["A"].properties[0].name == "b"
+    assert entities["A"].properties[0].value is None
+
+    assert entities["C"].properties[0].name == "b"
+    assert entities["C"].properties[0].value is None
+
+    model = """
+A:
+  role: Record
+  obligatory_properties:
+    b: 42
+b:
+  datatype: INTEGER
+"""
+
+    entities = parse_model_from_string(model)
+    assert entities["A"].get_property("b").value == 42
+    assert entities["b"].value is None
+
+    model = """
+b:
+  datatype: INTEGER
+  value: 18
+"""
+    entities = parse_model_from_string(model)
+    assert entities["b"].value == 18
+
+
+def test_issue_72():
+    """Tests for
+    https://gitlab.indiscale.com/caosdb/src/caosdb-advanced-user-tools/-/issues/72
+
+    In some cases, faulty values would be read in for properties without a
+    specified value.
+
+    """
+    model = """
+Experiment:
+  obligatory_properties:
+    date:
+      datatype: DATETIME
+      description: 'date of the experiment'
+    identifier:
+      datatype: TEXT
+      description: 'identifier of the experiment'
+    temperature:
+      datatype: DOUBLE
+      description: 'temp'
+TestExperiment:
+  role: Record
+  inherit_from_obligatory:
+    - Experiment
+  obligatory_properties:
+    date: 2022-03-02
+    identifier: Test
+    temperature: 23
+  recommended_properties:
+    additional_prop:
+      datatype: INTEGER
+      value: 7
+"""
+    entities = parse_model_from_string(model)
+    assert "Experiment" in entities
+    assert "date" in entities
+    assert "identifier" in entities
+    assert "temperature" in entities
+    assert "TestExperiment" in entities
+    assert "additional_prop" in entities
+    assert isinstance(entities["Experiment"], db.RecordType)
+
+    assert entities["Experiment"].get_property("date") is not None
+    # No value is set, so this has to be None
+    assert entities["Experiment"].get_property("date").value is None
+
+    assert entities["Experiment"].get_property("identifier") is not None
+    assert entities["Experiment"].get_property("identifier").value is None
+
+    assert entities["Experiment"].get_property("temperature") is not None
+    assert entities["Experiment"].get_property("temperature").value is None
+
+    test_rec = entities["TestExperiment"]
+    assert isinstance(test_rec, db.Record)
+    assert test_rec.get_property("date").value == date(2022, 3, 2)
+    assert test_rec.get_property("identifier").value == "Test"
+    assert test_rec.get_property("temperature").value == 23
+    assert test_rec.get_property("additional_prop").value == 7
+
+
+def test_file_role():
+    """Not implemented for now, see
+    https://gitlab.indiscale.com/caosdb/src/caosdb-advanced-user-tools/-/issues/74.
+
+    """
+    model = """
+F:
+  role: File
+"""
+    with raises(NotImplementedError):
+        entities = parse_model_from_string(model)