Skip to content
Snippets Groups Projects
Commit 78c738ed authored by Florian Spreckelsen's avatar Florian Spreckelsen
Browse files

Merge branch 'f-extend-yaml-model' into 'dev'

Revert "Revert "Merge branch 'f-extend-yaml-model' into 'dev'""

See merge request !36
parents 0b26273c 217bbaf2
Branches
Tags
2 merge requests!39Release 0.4.0,!36Revert "Revert "Merge branch 'f-extend-yaml-model' into 'dev'""
Pipeline #21031 passed
......@@ -11,6 +11,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
- CFood that creates a Record for each line in a csv file
- `generic_analysis.py` allows to easily call scripts to perform analyses in
server side scripting [EXPERIMENTAL]
- New keyword "role" in yaml data model that allows creation of Records and Files.
- It is now possible to set values of properties and default values of properties
directly in the yaml model.
### Changed ###
......
......@@ -38,7 +38,7 @@ Optional h5-crawler:
1. Change directory to `integrationtests/`.
2. Mount `extroot` to the folder that will be used as extroot. E.g. `sudo mount
-o bind extroot ../../caosdb-deploy/profiles/empty/paths/extroot` (or
-o bind extroot ../../caosdb-deploy/profiles/debug/paths/extroot` (or
whatever path the extroot of the empty profile to be used is located at).
3. Start (or restart) an empty (!) CaosDB instance (with the mounted
extroot). The database will be cleared during testing, so it's important to
......
......@@ -25,7 +25,8 @@ import yaml
from .data_model import DataModel
# Keywords which are allowed in data model descriptions.
KEYWORDS = ["parent",
KEYWORDS = ["parent", # deprecated, use inherit_from_* instead:
# https://gitlab.com/caosdb/caosdb-advanced-user-tools/-/issues/36
"importance",
"datatype", # for example TEXT, INTEGER or REFERENCE
"unit",
......@@ -35,8 +36,12 @@ KEYWORDS = ["parent",
"suggested_properties",
"inherit_from_recommended",
"inherit_from_suggested",
"inherit_from_obligatory", ]
"inherit_from_obligatory",
"role",
"value",
]
# TODO: check whether it's really ignored
# These KEYWORDS are not forbidden as properties, but merely ignored.
KEYWORDS_IGNORED = [
"unit",
......@@ -109,6 +114,10 @@ def parse_model_from_string(string):
class Parser(object):
def __init__(self):
"""Initialize an empty parser object and initialize the dictionary of entities and the list of
treated elements.
"""
self.model = {}
self.treated = []
......@@ -177,13 +186,11 @@ class Parser(object):
ymlmodel["extern"] = []
for name in ymlmodel["extern"]:
if db.execute_query("COUNT Property {}".format(name)) > 0:
self.model[name] = db.execute_query(
"FIND Property WITH name={}".format(name), unique=True)
elif db.execute_query("COUNT RecordType {}".format(name)) > 0:
self.model[name] = db.execute_query(
"FIND RecordType WITH name={}".format(name), unique=True)
for role in ("Property", "RecordType", "Record", "File"):
if db.execute_query("COUNT {} {}".format(role, name)) > 0:
self.model[name] = db.execute_query(
"FIND {} WITH name={}".format(role, name), unique=True)
break
else:
raise Exception("Did not find {}".format(name))
......@@ -235,6 +242,8 @@ class Parser(object):
""" adds names of Properties and RecordTypes to the model dictionary
Properties are also initialized.
name is the key of the yaml element and definition the value.
"""
if name == "__line__":
......@@ -258,9 +267,29 @@ class Parser(object):
# and create the new property
self.model[name] = db.Property(name=name,
datatype=definition["datatype"])
elif (self.model[name] is None and isinstance(definition, dict)
and "role" in definition):
if definition["role"] == "RecordType":
self.model[name] = db.RecordType(name=name)
elif definition["role"] == "Record":
self.model[name] = db.Record(name=name)
elif definition["role"] == "File":
# TODO(fspreck) Implement files at some later point in time
raise NotImplementedError(
"The definition of file objects is not yet implemented.")
# self.model[name] = db.File(name=name)
elif definition["role"] == "Property":
self.model[name] = db.Property(name=name)
else:
raise RuntimeError("Unknown role {} in definition of entity.".format(
definition["role"]))
# add other definitions recursively
# for setting values of properties directly:
if not isinstance(definition, dict):
return
# add other definitions recursively
for prop_type in ["recommended_properties",
"suggested_properties", "obligatory_properties"]:
......@@ -284,7 +313,25 @@ class Parser(object):
raise
def _add_to_recordtype(self, ent_name, props, importance):
"""Add properties to a RecordType."""
"""Add properties to a RecordType.
Parameters
----------
ent_name : str
The name of the entity to which the properties shall be added.
props : dict [str -> dict or :doc:`Entity`]
The properties, indexed by their names. Properties may be given as :doc:`Entity` objects
or as dictionaries.
importance
The importance as used in :doc:`Entity.add_property`.
Returns
-------
None
"""
for n, e in props.items():
if n in KEYWORDS:
......@@ -297,15 +344,28 @@ class Parser(object):
continue
n = self._stringify(n)
if (isinstance(e, dict) and "datatype" in e
and (_get_listdatatype(e["datatype"]) is not None)):
self.model[ent_name].add_property(
name=n,
importance=importance,
datatype=db.LIST(_get_listdatatype(e["datatype"])))
if isinstance(e, dict):
if "datatype" in e and _get_listdatatype(e["datatype"]) is not None:
# Reuse the existing datatype for lists.
datatype = db.LIST(_get_listdatatype(e["datatype"]))
else:
# Ignore a possible e["datatype"] here if it's not a list
# since it has been treated in the definition of the
# property (entity) already
datatype = None
if "value" in e:
value = e["value"]
else:
value = None
else:
self.model[ent_name].add_property(name=n,
importance=importance)
value = e
datatype = None
self.model[ent_name].add_property(name=n,
value=value,
importance=importance,
datatype=datatype)
def _inherit(self, name, prop, inheritance):
if not isinstance(prop, list):
......@@ -328,6 +388,10 @@ class Parser(object):
if definition is None:
return
# for setting values of properties directly:
if not isinstance(definition, dict):
return
if ("datatype" in definition
and definition["datatype"].startswith("LIST")):
......@@ -344,6 +408,9 @@ class Parser(object):
if prop_name == "unit":
self.model[name].unit = prop
elif prop_name == "value":
self.model[name].value = prop
elif prop_name == "description":
self.model[name].description = prop
......@@ -372,6 +439,10 @@ class Parser(object):
elif prop_name == "datatype":
continue
# role has already been used
elif prop_name == "role":
continue
elif prop_name == "inherit_from_obligatory":
self._inherit(name, prop, db.OBLIGATORY)
elif prop_name == "inherit_from_recommended":
......@@ -432,7 +503,8 @@ class Parser(object):
continue
raise ValueError("Property {} has an unknown datatype: {}".format(value.name, value.datatype))
raise ValueError("Property {} has an unknown datatype: {}".format(
value.name, value.datatype))
def _set_recordtypes(self):
""" properties are defined in first iteration; set remaining as RTs """
......
......@@ -11,3 +11,6 @@ deps=nose
xlrd == 1.2
h5py
commands=py.test --cov=caosadvancedtools -vv {posargs}
[flake8]
max-line-length=100
import unittest
from datetime import date
from tempfile import NamedTemporaryFile
from pytest import raises
import caosdb as db
from caosadvancedtools.models.parser import (TwiceDefinedException,
......@@ -15,6 +17,8 @@ def to_file(string):
return f.name
# TODO: check purpose of this function... add documentation
def parse_str(string):
parse_model_from_yaml(to_file(string))
......@@ -68,7 +72,8 @@ RT2:
a:
"""
self.assertRaises(TwiceDefinedException, lambda: parse_model_from_yaml(to_file(string)))
self.assertRaises(TwiceDefinedException,
lambda: parse_model_from_yaml(to_file(string)))
def test_typical_case(self):
string = """
......@@ -103,7 +108,8 @@ RT5:
- RT1:
- RT2:
"""
self.assertRaises(ValueError, lambda: parse_model_from_yaml(to_file(string)))
self.assertRaises(
ValueError, lambda: parse_model_from_yaml(to_file(string)))
def test_unknown_kwarg(self):
string = """
......@@ -111,7 +117,8 @@ RT1:
datetime:
p1:
"""
self.assertRaises(ValueError, lambda: parse_model_from_yaml(to_file(string)))
self.assertRaises(
ValueError, lambda: parse_model_from_yaml(to_file(string)))
def test_definition_in_inheritance(self):
string = """
......@@ -121,7 +128,8 @@ RT2:
- RT1:
description: "tach"
"""
self.assertRaises(ValueError, lambda: parse_model_from_yaml(to_file(string)))
self.assertRaises(
ValueError, lambda: parse_model_from_yaml(to_file(string)))
def test_inheritance(self):
string = """
......@@ -301,6 +309,8 @@ class ExternTest(unittest.TestCase):
class ErrorMessageTest(unittest.TestCase):
"""Tests for understandable error messages."""
# Note: This was changed with implementation of role keyword
@unittest.expectedFailure
def test_non_dict(self):
"""When a value is given, where a list or mapping is expected."""
recordtype_value = """
......@@ -328,3 +338,139 @@ A:
with self.assertRaises(YamlDefinitionError) as yde:
parse_str(string)
assert("line {}".format(line) in yde.exception.args[0])
def test_define_role():
model = """
A:
role: Record
"""
entities = parse_model_from_string(model)
assert "A" in entities
assert isinstance(entities["A"], db.Record)
assert entities["A"].role == "Record"
model = """
A:
role: Record
inherit_from_obligatory:
- C
obligatory_properties:
b:
b:
datatype: INTEGER
C:
obligatory_properties:
b:
D:
role: RecordType
"""
entities = parse_model_from_string(model)
for l, ent in (("A", "Record"), ("b", "Property"),
("C", "RecordType"), ("D", "RecordType")):
assert l in entities
assert isinstance(entities[l], getattr(db, ent))
assert entities[l].role == ent
assert entities["A"].parents[0].name == "C"
assert entities["A"].name == "A"
assert entities["A"].properties[0].name == "b"
assert entities["A"].properties[0].value is None
assert entities["C"].properties[0].name == "b"
assert entities["C"].properties[0].value is None
model = """
A:
role: Record
obligatory_properties:
b: 42
b:
datatype: INTEGER
"""
entities = parse_model_from_string(model)
assert entities["A"].get_property("b").value == 42
assert entities["b"].value is None
model = """
b:
datatype: INTEGER
value: 18
"""
entities = parse_model_from_string(model)
assert entities["b"].value == 18
def test_issue_72():
"""Tests for
https://gitlab.indiscale.com/caosdb/src/caosdb-advanced-user-tools/-/issues/72
In some cases, faulty values would be read in for properties without a
specified value.
"""
model = """
Experiment:
obligatory_properties:
date:
datatype: DATETIME
description: 'date of the experiment'
identifier:
datatype: TEXT
description: 'identifier of the experiment'
temperature:
datatype: DOUBLE
description: 'temp'
TestExperiment:
role: Record
inherit_from_obligatory:
- Experiment
obligatory_properties:
date: 2022-03-02
identifier: Test
temperature: 23
recommended_properties:
additional_prop:
datatype: INTEGER
value: 7
"""
entities = parse_model_from_string(model)
assert "Experiment" in entities
assert "date" in entities
assert "identifier" in entities
assert "temperature" in entities
assert "TestExperiment" in entities
assert "additional_prop" in entities
assert isinstance(entities["Experiment"], db.RecordType)
assert entities["Experiment"].get_property("date") is not None
# No value is set, so this has to be None
assert entities["Experiment"].get_property("date").value is None
assert entities["Experiment"].get_property("identifier") is not None
assert entities["Experiment"].get_property("identifier").value is None
assert entities["Experiment"].get_property("temperature") is not None
assert entities["Experiment"].get_property("temperature").value is None
test_rec = entities["TestExperiment"]
assert isinstance(test_rec, db.Record)
assert test_rec.get_property("date").value == date(2022, 3, 2)
assert test_rec.get_property("identifier").value == "Test"
assert test_rec.get_property("temperature").value == 23
assert test_rec.get_property("additional_prop").value == 7
def test_file_role():
"""Not implemented for now, see
https://gitlab.indiscale.com/caosdb/src/caosdb-advanced-user-tools/-/issues/74.
"""
model = """
F:
role: File
"""
with raises(NotImplementedError):
entities = parse_model_from_string(model)
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment