diff --git a/CHANGELOG.md b/CHANGELOG.md
index a6b2de738c79b3ad38c6bf77a2abb3611a6511eb..c307c15ac9ceb0a039f6406d0cb9148931c2837e 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -11,6 +11,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
- CFood that creates a Record for each line in a csv file
- `generic_analysis.py` allows to easily call scripts to perform analyses in
server side scripting [EXPERIMENTAL]
+- New keyword "role" in yaml data model that allows creation of Records and Files.
+- It is now possible to set values of properties and default values of properties
+ directly in the yaml model.
### Changed ###
diff --git a/README_SETUP.md b/README_SETUP.md
index d33316612c1d2870c3b2c416e842df4220ecf858..43047d554afbe8ffba11aef67b20dde44d29bdcf 100644
--- a/README_SETUP.md
+++ b/README_SETUP.md
@@ -38,7 +38,7 @@ Optional h5-crawler:
1. Change directory to `integrationtests/`.
2. Mount `extroot` to the folder that will be used as extroot. E.g. `sudo mount
- -o bind extroot ../../caosdb-deploy/profiles/empty/paths/extroot` (or
+ -o bind extroot ../../caosdb-deploy/profiles/debug/paths/extroot` (or
whatever path the extroot of the empty profile to be used is located at).
3. Start (or restart) an empty (!) CaosDB instance (with the mounted
extroot). The database will be cleared during testing, so it's important to
diff --git a/src/caosadvancedtools/models/parser.py b/src/caosadvancedtools/models/parser.py
index e56a492fa3e9199a312d374a622770e7836f42cb..41dcc3cdb76e307bace04f63748373d2d7f9741f 100644
--- a/src/caosadvancedtools/models/parser.py
+++ b/src/caosadvancedtools/models/parser.py
@@ -25,7 +25,8 @@ import yaml
from .data_model import DataModel
# Keywords which are allowed in data model descriptions.
-KEYWORDS = ["parent",
+KEYWORDS = ["parent", # deprecated, use inherit_from_* instead:
+ # https://gitlab.com/caosdb/caosdb-advanced-user-tools/-/issues/36
"importance",
"datatype", # for example TEXT, INTEGER or REFERENCE
"unit",
@@ -35,8 +36,12 @@ KEYWORDS = ["parent",
"suggested_properties",
"inherit_from_recommended",
"inherit_from_suggested",
- "inherit_from_obligatory", ]
+ "inherit_from_obligatory",
+ "role",
+ "value",
+ ]
+# TODO: check whether it's really ignored
# These KEYWORDS are not forbidden as properties, but merely ignored.
KEYWORDS_IGNORED = [
"unit",
@@ -109,6 +114,10 @@ def parse_model_from_string(string):
class Parser(object):
def __init__(self):
+ """Initialize an empty parser object and initialize the dictionary of entities and the list of
+ treated elements.
+
+ """
self.model = {}
self.treated = []
@@ -177,13 +186,11 @@ class Parser(object):
ymlmodel["extern"] = []
for name in ymlmodel["extern"]:
- if db.execute_query("COUNT Property {}".format(name)) > 0:
- self.model[name] = db.execute_query(
- "FIND Property WITH name={}".format(name), unique=True)
-
- elif db.execute_query("COUNT RecordType {}".format(name)) > 0:
- self.model[name] = db.execute_query(
- "FIND RecordType WITH name={}".format(name), unique=True)
+ for role in ("Property", "RecordType", "Record", "File"):
+ if db.execute_query("COUNT {} {}".format(role, name)) > 0:
+ self.model[name] = db.execute_query(
+ "FIND {} WITH name={}".format(role, name), unique=True)
+ break
else:
raise Exception("Did not find {}".format(name))
@@ -235,6 +242,8 @@ class Parser(object):
""" adds names of Properties and RecordTypes to the model dictionary
Properties are also initialized.
+
+ name is the key of the yaml element and definition the value.
"""
if name == "__line__":
@@ -258,9 +267,29 @@ class Parser(object):
# and create the new property
self.model[name] = db.Property(name=name,
datatype=definition["datatype"])
+ elif (self.model[name] is None and isinstance(definition, dict)
+ and "role" in definition):
+ if definition["role"] == "RecordType":
+ self.model[name] = db.RecordType(name=name)
+ elif definition["role"] == "Record":
+ self.model[name] = db.Record(name=name)
+ elif definition["role"] == "File":
+ # TODO(fspreck) Implement files at some later point in time
+ raise NotImplementedError(
+ "The definition of file objects is not yet implemented.")
+
+ # self.model[name] = db.File(name=name)
+ elif definition["role"] == "Property":
+ self.model[name] = db.Property(name=name)
+ else:
+ raise RuntimeError("Unknown role {} in definition of entity.".format(
+ definition["role"]))
- # add other definitions recursively
+ # for setting values of properties directly:
+ if not isinstance(definition, dict):
+ return
+ # add other definitions recursively
for prop_type in ["recommended_properties",
"suggested_properties", "obligatory_properties"]:
@@ -284,7 +313,25 @@ class Parser(object):
raise
def _add_to_recordtype(self, ent_name, props, importance):
- """Add properties to a RecordType."""
+ """Add properties to a RecordType.
+
+ Parameters
+ ----------
+ ent_name : str
+ The name of the entity to which the properties shall be added.
+
+ props : dict [str -> dict or :doc:`Entity`]
+ The properties, indexed by their names. Properties may be given as :doc:`Entity` objects
+ or as dictionaries.
+
+ importance
+ The importance as used in :doc:`Entity.add_property`.
+
+ Returns
+ -------
+ None
+
+ """
for n, e in props.items():
if n in KEYWORDS:
@@ -297,15 +344,28 @@ class Parser(object):
continue
n = self._stringify(n)
- if (isinstance(e, dict) and "datatype" in e
- and (_get_listdatatype(e["datatype"]) is not None)):
- self.model[ent_name].add_property(
- name=n,
- importance=importance,
- datatype=db.LIST(_get_listdatatype(e["datatype"])))
+ if isinstance(e, dict):
+ if "datatype" in e and _get_listdatatype(e["datatype"]) is not None:
+ # Reuse the existing datatype for lists.
+ datatype = db.LIST(_get_listdatatype(e["datatype"]))
+ else:
+ # Ignore a possible e["datatype"] here if it's not a list
+ # since it has been treated in the definition of the
+ # property (entity) already
+ datatype = None
+ if "value" in e:
+ value = e["value"]
+ else:
+ value = None
+
else:
- self.model[ent_name].add_property(name=n,
- importance=importance)
+ value = e
+ datatype = None
+
+ self.model[ent_name].add_property(name=n,
+ value=value,
+ importance=importance,
+ datatype=datatype)
def _inherit(self, name, prop, inheritance):
if not isinstance(prop, list):
@@ -328,6 +388,10 @@ class Parser(object):
if definition is None:
return
+ # for setting values of properties directly:
+ if not isinstance(definition, dict):
+ return
+
if ("datatype" in definition
and definition["datatype"].startswith("LIST")):
@@ -344,6 +408,9 @@ class Parser(object):
if prop_name == "unit":
self.model[name].unit = prop
+ elif prop_name == "value":
+ self.model[name].value = prop
+
elif prop_name == "description":
self.model[name].description = prop
@@ -372,6 +439,10 @@ class Parser(object):
elif prop_name == "datatype":
continue
+ # role has already been used
+ elif prop_name == "role":
+ continue
+
elif prop_name == "inherit_from_obligatory":
self._inherit(name, prop, db.OBLIGATORY)
elif prop_name == "inherit_from_recommended":
@@ -432,7 +503,8 @@ class Parser(object):
continue
- raise ValueError("Property {} has an unknown datatype: {}".format(value.name, value.datatype))
+ raise ValueError("Property {} has an unknown datatype: {}".format(
+ value.name, value.datatype))
def _set_recordtypes(self):
""" properties are defined in first iteration; set remaining as RTs """
diff --git a/tox.ini b/tox.ini
index 1b3cd4ef0d39955197448ace9fdf5d26ea6749b4..f7b5aabf37628f57e8e192dfced969541aecfe25 100644
--- a/tox.ini
+++ b/tox.ini
@@ -11,3 +11,6 @@ deps=nose
xlrd == 1.2
h5py
commands=py.test --cov=caosadvancedtools -vv {posargs}
+
+[flake8]
+max-line-length=100
diff --git a/unittests/test_parser.py b/unittests/test_parser.py
index 161e2873a9c01f9ce415818116b9e4cf9aeadb5c..01730cdb1690c7c4a917475d10c9035177fb58b7 100644
--- a/unittests/test_parser.py
+++ b/unittests/test_parser.py
@@ -1,5 +1,7 @@
import unittest
+from datetime import date
from tempfile import NamedTemporaryFile
+from pytest import raises
import caosdb as db
from caosadvancedtools.models.parser import (TwiceDefinedException,
@@ -15,6 +17,8 @@ def to_file(string):
return f.name
+# TODO: check purpose of this function... add documentation
+
def parse_str(string):
parse_model_from_yaml(to_file(string))
@@ -68,7 +72,8 @@ RT2:
a:
"""
- self.assertRaises(TwiceDefinedException, lambda: parse_model_from_yaml(to_file(string)))
+ self.assertRaises(TwiceDefinedException,
+ lambda: parse_model_from_yaml(to_file(string)))
def test_typical_case(self):
string = """
@@ -103,7 +108,8 @@ RT5:
- RT1:
- RT2:
"""
- self.assertRaises(ValueError, lambda: parse_model_from_yaml(to_file(string)))
+ self.assertRaises(
+ ValueError, lambda: parse_model_from_yaml(to_file(string)))
def test_unknown_kwarg(self):
string = """
@@ -111,7 +117,8 @@ RT1:
datetime:
p1:
"""
- self.assertRaises(ValueError, lambda: parse_model_from_yaml(to_file(string)))
+ self.assertRaises(
+ ValueError, lambda: parse_model_from_yaml(to_file(string)))
def test_definition_in_inheritance(self):
string = """
@@ -121,7 +128,8 @@ RT2:
- RT1:
description: "tach"
"""
- self.assertRaises(ValueError, lambda: parse_model_from_yaml(to_file(string)))
+ self.assertRaises(
+ ValueError, lambda: parse_model_from_yaml(to_file(string)))
def test_inheritance(self):
string = """
@@ -301,6 +309,8 @@ class ExternTest(unittest.TestCase):
class ErrorMessageTest(unittest.TestCase):
"""Tests for understandable error messages."""
+ # Note: This was changed with implementation of role keyword
+ @unittest.expectedFailure
def test_non_dict(self):
"""When a value is given, where a list or mapping is expected."""
recordtype_value = """
@@ -328,3 +338,139 @@ A:
with self.assertRaises(YamlDefinitionError) as yde:
parse_str(string)
assert("line {}".format(line) in yde.exception.args[0])
+
+
+def test_define_role():
+ model = """
+A:
+ role: Record
+"""
+ entities = parse_model_from_string(model)
+ assert "A" in entities
+ assert isinstance(entities["A"], db.Record)
+ assert entities["A"].role == "Record"
+
+ model = """
+A:
+ role: Record
+ inherit_from_obligatory:
+ - C
+ obligatory_properties:
+ b:
+b:
+ datatype: INTEGER
+C:
+ obligatory_properties:
+ b:
+D:
+ role: RecordType
+"""
+ entities = parse_model_from_string(model)
+ for l, ent in (("A", "Record"), ("b", "Property"),
+ ("C", "RecordType"), ("D", "RecordType")):
+ assert l in entities
+ assert isinstance(entities[l], getattr(db, ent))
+ assert entities[l].role == ent
+
+ assert entities["A"].parents[0].name == "C"
+ assert entities["A"].name == "A"
+
+ assert entities["A"].properties[0].name == "b"
+ assert entities["A"].properties[0].value is None
+
+ assert entities["C"].properties[0].name == "b"
+ assert entities["C"].properties[0].value is None
+
+ model = """
+A:
+ role: Record
+ obligatory_properties:
+ b: 42
+b:
+ datatype: INTEGER
+"""
+
+ entities = parse_model_from_string(model)
+ assert entities["A"].get_property("b").value == 42
+ assert entities["b"].value is None
+
+ model = """
+b:
+ datatype: INTEGER
+ value: 18
+"""
+ entities = parse_model_from_string(model)
+ assert entities["b"].value == 18
+
+
+def test_issue_72():
+ """Tests for
+ https://gitlab.indiscale.com/caosdb/src/caosdb-advanced-user-tools/-/issues/72
+
+ In some cases, faulty values would be read in for properties without a
+ specified value.
+
+ """
+ model = """
+Experiment:
+ obligatory_properties:
+ date:
+ datatype: DATETIME
+ description: 'date of the experiment'
+ identifier:
+ datatype: TEXT
+ description: 'identifier of the experiment'
+ temperature:
+ datatype: DOUBLE
+ description: 'temp'
+TestExperiment:
+ role: Record
+ inherit_from_obligatory:
+ - Experiment
+ obligatory_properties:
+ date: 2022-03-02
+ identifier: Test
+ temperature: 23
+ recommended_properties:
+ additional_prop:
+ datatype: INTEGER
+ value: 7
+"""
+ entities = parse_model_from_string(model)
+ assert "Experiment" in entities
+ assert "date" in entities
+ assert "identifier" in entities
+ assert "temperature" in entities
+ assert "TestExperiment" in entities
+ assert "additional_prop" in entities
+ assert isinstance(entities["Experiment"], db.RecordType)
+
+ assert entities["Experiment"].get_property("date") is not None
+ # No value is set, so this has to be None
+ assert entities["Experiment"].get_property("date").value is None
+
+ assert entities["Experiment"].get_property("identifier") is not None
+ assert entities["Experiment"].get_property("identifier").value is None
+
+ assert entities["Experiment"].get_property("temperature") is not None
+ assert entities["Experiment"].get_property("temperature").value is None
+
+ test_rec = entities["TestExperiment"]
+ assert isinstance(test_rec, db.Record)
+ assert test_rec.get_property("date").value == date(2022, 3, 2)
+ assert test_rec.get_property("identifier").value == "Test"
+ assert test_rec.get_property("temperature").value == 23
+ assert test_rec.get_property("additional_prop").value == 7
+
+
+def test_file_role():
+ """Not implemented for now, see
+ https://gitlab.indiscale.com/caosdb/src/caosdb-advanced-user-tools/-/issues/74.
+
+ """
+ model = """
+F:
+ role: File
+"""
+ with raises(NotImplementedError):
+ entities = parse_model_from_string(model)