diff --git a/CHANGELOG.md b/CHANGELOG.md
index 0d5a5ac2ef93edca05c8e977b4ebb99f0dd3008e..4d86271ba398825534bc36465fa9b0fca72fb561 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -10,6 +10,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
### Added ###
- New function in apiutils that copies an Entity.
+- New EXPERIMENTAL module `high_level_api` which is a completely refactored version of
+ the old `high_level_api` from apiutils. Please see the included documentation for details.
### Changed ###
diff --git a/src/caosdb/apiutils.py b/src/caosdb/apiutils.py
index 08f31daad56c0ab471322197cadc1a1378267f35..a376068c372c1b6f460c7927467b8da8df328545 100644
--- a/src/caosdb/apiutils.py
+++ b/src/caosdb/apiutils.py
@@ -33,6 +33,8 @@ import warnings
from collections.abc import Iterable
from subprocess import call
+from typing import Optional, Any
+
from caosdb.common.datatype import (BOOLEAN, DATETIME, DOUBLE, FILE, INTEGER,
REFERENCE, TEXT, is_reference)
from caosdb.common.models import (Container, Entity, File, Property, Query,
@@ -99,22 +101,6 @@ def create_id_query(ids):
["ID={}".format(id) for id in ids])
-def retrieve_entity_with_id(eid):
- return execute_query("FIND ENTITY WITH ID={}".format(eid), unique=True)
-
-
-def retrieve_entities_with_ids(entities):
- collection = Container()
- step = 20
-
- for i in range(len(entities)//step+1):
- collection.extend(
- execute_query(
- create_id_query(entities[i*step:(i+1)*step])))
-
- return collection
-
-
def get_type_of_entity_with(id_):
objs = retrieve_entities_with_ids([id_])
@@ -138,385 +124,20 @@ def get_type_of_entity_with(id_):
return Entity
-class CaosDBPythonEntity(object):
-
- _last_id = 0
-
- def __init__(self):
- warnings.warn("The CaosDBPythonEntity class is deprecated, replacements will be provided by"
- " the high_level_api module.",
- DeprecationWarning, stacklevel=2)
- # Save a copy of the dry state
- # of this object in order to be
- # able to detect conflicts.
- self.do_not_expand = False
- self._parents = []
- self._id = CaosDBPythonEntity._get_id()
- self._path = None
- self._file = None
- self.pickup = None
- # TODO:
- # 3.) resolve references up to a specific depth (including infinity)
- # 4.) resolve parents function -> partially implemented by
- # get_parent_names
- self._references = {}
- self._properties = set()
- self._datatypes = {}
- self._forbidden = dir(self)
-
- @staticmethod
- def _get_id():
- CaosDBPythonEntity._last_id -= 1
-
- return CaosDBPythonEntity._last_id
-
- def _set_property_from_entity(self, ent):
- name = ent.name
- val = ent.value
- pr = ent.datatype
- val, reference = self._type_converted_value(val, pr)
- self.set_property(name, val, reference, datatype=pr)
-
- def set_property(self, name, value, is_reference=False,
- overwrite=False, datatype=None):
- """
- overwrite: Use this if you definitely only want one property with that name (set to True).
- """
- self._datatypes[name] = datatype
-
- if isinstance(name, Entity):
- name = name.name
-
- if name in self._forbidden:
- raise RuntimeError("Entity cannot be converted to a corresponding "
- "Python representation. Name of property " +
- name + " is forbidden!")
- already_exists = (name in dir(self))
-
- if already_exists and not overwrite:
- # each call to _set_property checks first if it already exists
- # if yes: Turn the attribute into a list and
- # place all the elements into that list.
- att = self.__getattribute__(name)
-
- if isinstance(att, list):
- pass
- else:
- old_att = self.__getattribute__(name)
- self.__setattr__(name, [old_att])
-
- if is_reference:
- self._references[name] = [
- self._references[name]]
- att = self.__getattribute__(name)
- att.append(value)
-
- if is_reference:
- self._references[name].append(int(value))
- else:
- if is_reference:
- self._references[name] = value
- self.__setattr__(name, value)
-
- if not (already_exists and overwrite):
- self._properties.add(name)
-
- add_property = set_property
-
- def set_id(self, idx):
- self._id = idx
-
- def _type_converted_list(self, val, pr):
- """Convert a list to a python list of the correct type."""
- prrealpre = pr.replace("<", "<").replace(">", ">")
- prreal = prrealpre[prrealpre.index("<") + 1:prrealpre.rindex(">")]
- lst = [self._type_converted_value(i, prreal) for i in val]
-
- return ([i[0] for i in lst], lst[0][1])
-
- def _type_converted_value(self, val, pr):
- """Convert val to the correct type which is indicated by the database
- type string in pr.
-
- Returns a tuple with two entries:
- - the converted value
- - True if the value has to be interpreted as an id acting as a reference
- """
-
- if val is None:
- return (None, False)
- elif pr == DOUBLE:
- return (float(val), False)
- elif pr == BOOLEAN:
- return (bool(val), False)
- elif pr == INTEGER:
- return (int(val), False)
- elif pr == TEXT:
- return (val, False)
- elif pr == FILE:
- return (int(val), False)
- elif pr == REFERENCE:
- return (int(val), True)
- elif pr == DATETIME:
- return (val, False)
- elif pr[0:4] == "LIST":
- return self._type_converted_list(val, pr)
- elif isinstance(val, Entity):
- return (convert_to_python_object(val), False)
- else:
- return (int(val), True)
-
- def attribute_as_list(self, name):
- """This is a workaround for the problem that lists containing only one
- element are indistinguishable from simple types in this
- representation."""
- att = self.__getattribute__(name)
-
- if isinstance(att, list):
- return att
- else:
- return [att]
-
- def _add_parent(self, parent):
- self._parents.append(parent.id)
-
- def add_parent(self, parent_id=None, parent_name=None):
- if parent_id is not None:
- self._parents.append(parent_id)
- elif parent_name is not None:
- self._parents.append(parent_name)
- else:
- raise ValueError("no parent identifier supplied")
-
- def get_parent_names(self):
- new_plist = []
-
- for p in self._parents:
- obj_type = get_type_of_entity_with(p)
- ent = obj_type(id=p).retrieve()
- new_plist.append(ent.name)
-
- return new_plist
-
- def resolve_references(self, deep=False, visited=dict()):
- for i in self._references:
- if isinstance(self._references[i], list):
- for j in range(len(self._references[i])):
- new_id = self._references[i][j]
- obj_type = get_type_of_entity_with(new_id)
-
- if new_id in visited:
- new_object = visited[new_id]
- else:
- ent = obj_type(id=new_id).retrieve()
- new_object = convert_to_python_object(ent)
- visited[new_id] = new_object
-
- if deep:
- new_object.resolve_references(deep, visited)
- self.__getattribute__(i)[j] = new_object
- else:
- new_id = self._references[i]
- obj_type = get_type_of_entity_with(new_id)
-
- if new_id in visited:
- new_object = visited[new_id]
- else:
- ent = obj_type(id=new_id).retrieve()
- new_object = convert_to_python_object(ent)
- visited[new_id] = new_object
-
- if deep:
- new_object.resolve_references(deep, visited)
- self.__setattr__(i, new_object)
-
- def __str__(self, indent=1, name=None):
- if name is None:
- result = str(self.__class__.__name__) + "\n"
- else:
- result = name + "\n"
-
- for p in self._properties:
- value = self.__getattribute__(p)
-
- if isinstance(value, CaosDBPythonEntity):
- result += indent * "\t" + \
- value.__str__(indent=indent + 1, name=p)
- else:
- result += indent * "\t" + p + "\n"
-
- return result
-
-
-class CaosDBPythonRecord(CaosDBPythonEntity):
- pass
-
-
-class CaosDBPythonRecordType(CaosDBPythonEntity):
- pass
-
-
-class CaosDBPythonProperty(CaosDBPythonEntity):
- pass
-
-
-class CaosDBPythonFile(CaosDBPythonEntity):
- def get_File(self, target=None):
- f = File(id=self._id).retrieve()
- self._file = f.download(target)
-
-
-def _single_convert_to_python_object(robj, entity):
- robj._id = entity.id
-
- for i in entity.properties:
- robj._set_property_from_entity(i)
-
- for i in entity.parents:
- robj._add_parent(i)
-
- if entity.path is not None:
- robj._path = entity.path
-
- if entity.file is not None:
- robj._file = entity.file
- # if entity.pickup is not None:
- # robj.pickup = entity.pickup
-
- return robj
-
-
-def _single_convert_to_entity(entity, robj, **kwargs):
- if robj._id is not None:
- entity.id = robj._id
-
- if robj._path is not None:
- entity.path = robj._path
-
- if robj._file is not None:
- entity.file = robj._file
+def retrieve_entity_with_id(eid):
+ return execute_query("FIND ENTITY WITH ID={}".format(eid), unique=True)
- if robj.pickup is not None:
- entity.pickup = robj.pickup
- children = []
- for parent in robj._parents:
- if sys.version_info[0] < 3:
- if hasattr(parent, "encode"):
- entity.add_parent(name=parent)
- else:
- entity.add_parent(id=parent)
- else:
- if hasattr(parent, "encode"):
- entity.add_parent(name=parent)
- else:
- entity.add_parent(id=parent)
-
- def add_property(entity, prop, name, recursive=False, datatype=None):
- if datatype is None:
- raise ArgumentError("datatype must not be None")
+def retrieve_entities_with_ids(entities):
+ collection = Container()
+ step = 20
- if isinstance(prop, CaosDBPythonEntity):
- entity.add_property(name=name, value=str(
- prop._id), datatype=datatype)
+ for i in range(len(entities)//step+1):
+ collection.extend(
+ execute_query(
+ create_id_query(entities[i*step:(i+1)*step])))
- if recursive and not prop.do_not_expand:
- return convert_to_entity(prop, recursive=recursive)
- else:
- return []
- else:
- if isinstance(prop, float) or isinstance(prop, int):
- prop = str(prop)
- entity.add_property(name=name, value=prop, datatype=datatype)
-
- return []
-
- for prop in robj._properties:
- value = robj.__getattribute__(prop)
-
- if isinstance(value, list):
- if robj._datatypes[prop][0:4] == "LIST":
- lst = []
-
- for v in value:
- if isinstance(v, CaosDBPythonEntity):
- lst.append(v._id)
-
- if recursive and not v.do_not_expand:
- children.append(convert_to_entity(
- v, recursive=recursive))
- else:
- if isinstance(v, float) or isinstance(v, int):
- lst.append(str(v))
- else:
- lst.append(v)
- entity.add_property(name=prop, value=lst,
- datatype=robj._datatypes[prop])
- else:
- for v in value:
- children.extend(
- add_property(
- entity,
- v,
- prop,
- datatype=robj._datatypes[prop],
- **kwargs))
- else:
- children.extend(
- add_property(
- entity,
- value,
- prop,
- datatype=robj._datatypes[prop],
- **kwargs))
-
- return [entity] + children
-
-
-def convert_to_entity(python_object, **kwargs):
- warnings.warn("The convert_to_entity function is deprecated, replacement will be provided by "
- "the high_level_api module.", DeprecationWarning, stacklevel=2)
-
- if isinstance(python_object, Container):
- # Create a list of objects:
-
- return [convert_to_python_object(i, **kwargs) for i in python_object]
- elif isinstance(python_object, CaosDBPythonRecord):
- return _single_convert_to_entity(Record(), python_object, **kwargs)
- elif isinstance(python_object, CaosDBPythonFile):
- return _single_convert_to_entity(File(), python_object, **kwargs)
- elif isinstance(python_object, CaosDBPythonRecordType):
- return _single_convert_to_entity(RecordType(), python_object, **kwargs)
- elif isinstance(python_object, CaosDBPythonProperty):
- return _single_convert_to_entity(Property(), python_object, **kwargs)
- elif isinstance(python_object, CaosDBPythonEntity):
- return _single_convert_to_entity(Entity(), python_object, **kwargs)
- else:
- raise ValueError("Cannot convert an object of this type.")
-
-
-def convert_to_python_object(entity):
- """"""
-
- warnings.warn("The convert_to_python_object function is deprecated, replacement will be "
- "provided by the high_level_api module.", DeprecationWarning, stacklevel=2)
- if isinstance(entity, Container):
- # Create a list of objects:
-
- return [convert_to_python_object(i) for i in entity]
- elif isinstance(entity, Record):
- return _single_convert_to_python_object(CaosDBPythonRecord(), entity)
- elif isinstance(entity, RecordType):
- return _single_convert_to_python_object(
- CaosDBPythonRecordType(), entity)
- elif isinstance(entity, File):
- return _single_convert_to_python_object(CaosDBPythonFile(), entity)
- elif isinstance(entity, Property):
- return _single_convert_to_python_object(CaosDBPythonProperty(), entity)
- elif isinstance(entity, Entity):
- return _single_convert_to_python_object(CaosDBPythonEntity(), entity)
- else:
- raise ValueError("Cannot convert an object of this type.")
+ return collection
def getOriginUrlIn(folder):
@@ -584,8 +205,8 @@ def compare_entities(old_entity: Entity, new_entity: Entity):
In case of changed information the value listed under the respective key shows the
value that is stored in the respective entity.
"""
- olddiff = {"properties": {}, "parents": []}
- newdiff = {"properties": {}, "parents": []}
+ olddiff: dict[str, Any] = {"properties": {}, "parents": []}
+ newdiff: dict[str, Any] = {"properties": {}, "parents": []}
if old_entity is new_entity:
return (olddiff, newdiff)
@@ -844,3 +465,26 @@ def resolve_reference(prop: Property):
else:
if isinstance(prop.value, int):
prop.value = retrieve_entity_with_id(prop.value)
+
+
+def create_flat_list(ent_list: list[Entity], flat: list[Entity]):
+ """
+ Recursively adds all properties contained in entities from ent_list to
+ the output list flat. Each element will only be added once to the list.
+
+ TODO: Currently this function is also contained in newcrawler module crawl.
+ We are planning to permanently move it to here.
+ """
+ for ent in ent_list:
+ for p in ent.properties:
+ # For lists append each element that is of type Entity to flat:
+ if isinstance(p.value, list):
+ for el in p.value:
+ if isinstance(el, Entity):
+ if el not in flat:
+ flat.append(el)
+ create_flat_list([el], flat) # TODO: move inside if block?
+ elif isinstance(p.value, Entity):
+ if p.value not in flat:
+ flat.append(p.value)
+ create_flat_list([p.value], flat) # TODO: move inside if block?
diff --git a/src/caosdb/high_level_api.py b/src/caosdb/high_level_api.py
new file mode 100644
index 0000000000000000000000000000000000000000..0c936112993ccdbb5afdd91f3286880a16bdf431
--- /dev/null
+++ b/src/caosdb/high_level_api.py
@@ -0,0 +1,1025 @@
+# -*- coding: utf-8 -*-
+#
+# This file is a part of the CaosDB Project.
+#
+# Copyright (C) 2018 Research Group Biomedical Physics,
+# Max-Planck-Institute for Dynamics and Self-Organization Göttingen
+# Copyright (C) 2020 Timm Fitschen <t.fitschen@indiscale.com>
+# Copyright (C) 2020-2022 IndiScale GmbH <info@indiscale.com>
+# Copyright (C) 2022 Alexander Schlemmer <alexander.schlemmer@ds.mpg.de>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see <https://www.gnu.org/licenses/>.
+#
+# ** end header
+#
+
+"""
+A high level API for accessing CaosDB entities from within python.
+
+This is refactored from apiutils.
+"""
+
+from caosdb.common.datatype import (BOOLEAN, DATETIME, DOUBLE, FILE, INTEGER,
+ REFERENCE, TEXT,
+ is_list_datatype,
+ get_list_datatype,
+ is_reference)
+import caosdb as db
+
+from .apiutils import get_type_of_entity_with, create_flat_list
+import warnings
+
+from typing import Any, Optional, List, Union, Dict
+
+import yaml
+
+from dataclasses import dataclass, fields
+from datetime import datetime
+from dateutil import parser
+
+warnings.warn("""EXPERIMENTAL! The high_level_api module is experimental and may be changed or
+removed in future. Its purpose is to give an impression on how the Python client user interface
+might be changed.""")
+
+
+def standard_type_for_high_level_type(high_level_record: "CaosDBPythonEntity",
+ return_string: bool = False):
+ """
+ For a given CaosDBPythonEntity either return the corresponding
+ class in the standard CaosDB API or - if return_string is True - return
+ the role as a string.
+ """
+ if type(high_level_record) == CaosDBPythonRecord:
+ if not return_string:
+ return db.Record
+ return "Record"
+ elif type(high_level_record) == CaosDBPythonFile:
+ if not return_string:
+ return db.File
+ return "File"
+ elif type(high_level_record) == CaosDBPythonProperty:
+ if not return_string:
+ return db.Property
+ return "Property"
+ elif type(high_level_record) == CaosDBPythonRecordType:
+ if not return_string:
+ return db.RecordType
+ return "RecordType"
+ elif type(high_level_record) == CaosDBPythonEntity:
+ if not return_string:
+ return db.Entity
+ return "Entity"
+ raise RuntimeError("Incompatible type.")
+
+
+def high_level_type_for_role(role: str):
+ if role == "Record":
+ return CaosDBPythonRecord
+ if role == "File":
+ return CaosDBPythonFile
+ if role == "Property":
+ return CaosDBPythonProperty
+ if role == "RecordType":
+ return CaosDBPythonRecordType
+ if role == "Entity":
+ return CaosDBPythonEntity
+ raise RuntimeError("Unknown role.")
+
+
+def high_level_type_for_standard_type(standard_record: db.Entity):
+ if not isinstance(standard_record, db.Entity):
+ raise ValueError()
+ role = standard_record.role
+ if role == "Record" or type(standard_record) == db.Record:
+ return CaosDBPythonRecord
+ elif role == "File" or type(standard_record) == db.File:
+ return CaosDBPythonFile
+ elif role == "Property" or type(standard_record) == db.Property:
+ return CaosDBPythonProperty
+ elif role == "RecordType" or type(standard_record) == db.RecordType:
+ return CaosDBPythonRecordType
+ elif role == "Entity" or type(standard_record) == db.Entity:
+ return CaosDBPythonEntity
+ raise RuntimeError("Incompatible type.")
+
+
+@dataclass
+class CaosDBPropertyMetaData:
+ # name is already the name of the attribute
+ unit: Optional[str] = None
+ datatype: Optional[str] = None
+ description: Optional[str] = None
+ id: Optional[int] = None
+ importance: Optional[str] = None
+
+
+class CaosDBPythonUnresolved:
+ pass
+
+
+@dataclass
+class CaosDBPythonUnresolvedParent(CaosDBPythonUnresolved):
+ """
+ Parents can be either given by name or by ID.
+
+ When resolved, both fields should be set.
+ """
+
+ id: Optional[int] = None
+ name: Optional[str] = None
+
+
+@dataclass
+class CaosDBPythonUnresolvedReference(CaosDBPythonUnresolved):
+
+ def __init__(self, id=None):
+ self.id = id
+
+
+class CaosDBPythonEntity(object):
+
+ def __init__(self):
+ """
+ Initialize a new CaosDBPythonEntity for the high level python api.
+
+ Parents are either unresolved references or CaosDB RecordTypes.
+
+ Properties are stored directly as attributes for the object.
+ Property metadata is maintained in a dctionary _properties_metadata that should
+ never be accessed directly, but only using the get_property_metadata function.
+ If property values are references to other objects, they will be stored as
+ CaosDBPythonUnresolvedReference objects that can be resolved later into
+ CaosDBPythonRecords.
+ """
+
+ # Parents are either unresolved references or CaosDB RecordTypes
+ self._parents: List[Union[
+ CaosDBPythonUnresolvedParent, CaosDBPythonRecordType]] = []
+ # self._id: int = CaosDBPythonEntity._get_new_id()
+ self._id: Optional[int] = None
+ self._name: Optional[str] = None
+ self._description: Optional[str] = None
+ self._version: Optional[str] = None
+
+ self._file: Optional[str] = None
+ self._path: Optional[str] = None
+
+ # name: name of property, value: property metadata
+ self._properties_metadata: Dict[CaosDBPropertyMetaData] = dict()
+
+ # Store all current attributes as forbidden attributes
+ # which must not be changed by the set_property function.
+ self._forbidden = dir(self) + ["_forbidden"]
+
+ def use_parameter(self, name, value):
+ self.__setattr__(name, value)
+ return value
+
+ @property
+ def id(self):
+ """
+ Getter for the id.
+ """
+ return self._id
+
+ @id.setter
+ def id(self, val: int):
+ self._id = val
+
+ @property
+ def name(self):
+ """
+ Getter for the name.
+ """
+ return self._name
+
+ @name.setter
+ def name(self, val: str):
+ self._name = val
+
+ @property
+ def file(self):
+ """
+ Getter for the file.
+ """
+ if type(self) != CaosDBPythonFile:
+ raise RuntimeError("Please don't use the file attribute for entities"
+ " that are no files.")
+ return self._file
+
+ @file.setter
+ def file(self, val: str):
+ if val is not None and type(self) != CaosDBPythonFile:
+ raise RuntimeError("Please don't use the file attribute for entities"
+ " that are no files.")
+ self._file = val
+
+ @property
+ def path(self):
+ """
+ Getter for the path.
+ """
+ if type(self) != CaosDBPythonFile:
+ raise RuntimeError("Please don't use the path attribute for entities"
+ " that are no files.")
+ return self._path
+
+ @path.setter
+ def path(self, val: str):
+ if val is not None and type(self) != CaosDBPythonFile:
+ raise RuntimeError("Please don't use the path attribute for entities"
+ " that are no files.")
+ self._path = val
+
+ @property
+ def description(self):
+ """
+ Getter for the description.
+ """
+ return self._description
+
+ @description.setter
+ def description(self, val: str):
+ self._description = val
+
+ @property
+ def version(self):
+ """
+ Getter for the version.
+ """
+ return self._version
+
+ @version.setter
+ def version(self, val: str):
+ self._version = val
+
+ def _set_property_from_entity(self, ent: db.Entity, importance: str,
+ references: Optional[db.Container]):
+ """
+ Set a new property using an entity from the normal python API.
+
+ ent : db.Entity
+ The entity to be set.
+ """
+
+ if ent.name is None:
+ raise RuntimeError("Setting properties without name is impossible.")
+
+ if ent.name in self.get_properties():
+ raise RuntimeError("Multiproperty not implemented yet.")
+
+ val = self._type_converted_value(ent.value, ent.datatype,
+ references)
+ self.set_property(
+ ent.name,
+ val,
+ datatype=ent.datatype)
+ metadata = self.get_property_metadata(ent.name)
+
+ for prop_name in fields(metadata):
+ k = prop_name.name
+ if k == "importance":
+ metadata.importance = importance
+ else:
+ metadata.__setattr__(k, ent.__getattribute__(k))
+
+ def get_property_metadata(self, prop_name: str) -> CaosDBPropertyMetaData:
+ """
+ Retrieve the property metadata for the property with name prop_name.
+
+ If the property with the given name does not exist or is forbidden, raise an exception.
+ Else return the metadata associated with this property.
+
+ If no metadata does exist yet for the given property, a new object will be created
+ and returned.
+
+ prop_name: str
+ Name of the property to retrieve metadata for.
+ """
+
+ if not self.property_exists(prop_name):
+ raise RuntimeError("The property with name {} does not exist.".format(prop_name))
+
+ if prop_name not in self._properties_metadata:
+ self._properties_metadata[prop_name] = CaosDBPropertyMetaData()
+
+ return self._properties_metadata[prop_name]
+
+ def property_exists(self, prop_name: str):
+ """
+ Check whether a property exists already.
+ """
+ return prop_name not in self._forbidden and prop_name in self.__dict__
+
+ def set_property(self,
+ name: str,
+ value: Any,
+ overwrite: bool = False,
+ datatype: Optional[str] = None):
+ """
+ Set a property for this entity with a name and a value.
+
+ If this property is already set convert the value into a list and append the value.
+ This behavior can be overwritten using the overwrite flag, which will just overwrite
+ the existing value.
+
+ name: str
+ Name of the property.
+
+ value: Any
+ Value of the property.
+
+ overwrite: bool
+ Use this if you definitely only want one property with
+ that name (set to True).
+ """
+
+ if name in self._forbidden:
+ raise RuntimeError("Entity cannot be converted to a corresponding "
+ "Python representation. Name of property " +
+ name + " is forbidden!")
+
+ already_exists = self.property_exists(name)
+
+ if already_exists and not overwrite:
+ # each call to set_property checks first if it already exists
+ # if yes: Turn the attribute into a list and
+ # place all the elements into that list.
+ att = self.__getattribute__(name)
+
+ if isinstance(att, list):
+ # just append, see below
+ pass
+ else:
+ old_att = self.__getattribute__(name)
+ self.__setattr__(name, [old_att])
+ att = self.__getattribute__(name)
+ att.append(value)
+ else:
+ self.__setattr__(name, value)
+
+ def __setattr__(self, name: str, val: Any):
+ """
+ Allow setting generic properties.
+ """
+
+ # TODO: implement checking the value to correspond to one of the datatypes
+ # known for conversion.
+
+ super().__setattr__(name, val)
+
+ def _type_converted_list(self,
+ val: List,
+ pr: str,
+ references: Optional[db.Container]):
+ """
+ Convert a list to a python list of the correct type.
+
+ val: List
+ The value of a property containing the list.
+
+ pr: str
+ The datatype according to the database entry.
+ """
+ if not is_list_datatype(pr) and not isinstance(val, list):
+ raise RuntimeError("Not a list.")
+
+ return [
+ self._type_converted_value(i, get_list_datatype(pr), references
+ ) for i in val]
+
+ def _type_converted_value(self,
+ val: Any,
+ pr: str,
+ references: Optional[db.Container]):
+ """
+ Convert val to the correct type which is indicated by the database
+ type string in pr.
+
+ References with ids will be turned into CaosDBPythonUnresolvedReference.
+ """
+
+ if val is None:
+ return None
+ elif isinstance(val, db.Entity):
+ # this needs to be checked as second case as it is the ONLY
+ # case which does not depend on pr
+ # TODO: we might need to pass through the reference container
+ return convert_to_python_object(val, references)
+ elif isinstance(val, list):
+ return self._type_converted_list(val, pr, references)
+ elif pr is None:
+ return val
+ elif pr == DOUBLE:
+ return float(val)
+ elif pr == BOOLEAN:
+ return bool(val)
+ elif pr == INTEGER:
+ return int(val)
+ elif pr == TEXT:
+ return str(val)
+ elif pr == FILE:
+ return CaosDBPythonUnresolvedReference(val)
+ elif pr == REFERENCE:
+ return CaosDBPythonUnresolvedReference(val)
+ elif pr == DATETIME:
+ return self._parse_datetime(val)
+ elif is_list_datatype(pr):
+ return self._type_converted_list(val, pr, references)
+ else:
+ # Generic references to entities:
+ return CaosDBPythonUnresolvedReference(val)
+
+ def _parse_datetime(self, val: Union[str, datetime]):
+ """
+ Convert val into a datetime object.
+ """
+ if isinstance(val, datetime):
+ return val
+ return parser.parse(val)
+
+ def get_property(self, name: str):
+ """
+ Return the value of the property with name name.
+
+ Raise an exception if the property does not exist.
+ """
+ if not self.property_exists(name):
+ raise RuntimeError("Property {} does not exist.".format(name))
+ att = self.__getattribute__(name)
+ return att
+
+ def attribute_as_list(self, name: str):
+ """
+ This is a workaround for the problem that lists containing only one
+ element are indistinguishable from simple types in this
+ representation.
+
+ TODO: still relevant? seems to be only a problem if LIST types are not used.
+ """
+ att = self.get_property(name)
+
+ if isinstance(att, list):
+ return att
+ else:
+ return [att]
+
+ def add_parent(self, parent: Union[
+ CaosDBPythonUnresolvedParent, "CaosDBPythonRecordType", str]):
+ """
+ Add a parent to this entity. Either using an unresolved parent or
+ using a real record type.
+
+ Strings as argument for parent will automatically be converted to an
+ unresolved parent. Likewise, integers as argument will be automatically converted
+ to unresolved parents with just an id.
+ """
+
+ if isinstance(parent, str):
+ parent = CaosDBPythonUnresolvedParent(name=parent)
+
+ if isinstance(parent, int):
+ parent = CaosDBPythonUnresolvedParent(id=parent)
+
+ if self.has_parent(parent):
+ raise RuntimeError("Duplicate parent.")
+ self._parents.append(parent)
+
+ def get_parents(self):
+ """
+ Returns all parents of this entity.
+
+ Use has_parent for checking for existence of parents
+ and add_parent for adding parents to this entity.
+ """
+ return self._parents
+
+ def has_parent(self, parent: Union[
+ CaosDBPythonUnresolvedParent, "CaosDBPythonRecordType"]):
+ """
+ Check whether this parent already exists for this entity.
+
+ Strings as argument for parent will automatically be converted to an
+ unresolved parent. Likewise, integers as argument will be automatically converted
+ to unresolved parents with just an id.
+ """
+
+ if isinstance(parent, str):
+ parent = CaosDBPythonUnresolvedParent(name=parent)
+
+ if isinstance(parent, int):
+ parent = CaosDBPythonUnresolvedParent(id=parent)
+
+ for p in self._parents:
+ if p.id is not None and p.id == parent.id:
+ return True
+ elif p.name is not None and p.name == parent.name:
+ return True
+ return False
+
+ def _resolve_caosdb_python_unresolved_reference(self, propval, deep,
+ references, visited):
+ # This does not make sense for unset ids:
+ if propval.id is None:
+ raise RuntimeError("Unresolved property reference without an ID.")
+ # have we encountered this id before:
+ if propval.id in visited:
+ # self.__setattr__(prop, visited[propval.id])
+ # don't do the lookup in the references container
+ return visited[propval.id]
+
+ if references is None:
+ ent = db.Entity(id=propval.id).retrieve()
+ obj = convert_to_python_object(ent, references)
+ visited[propval.id] = obj
+ if deep:
+ obj.resolve_references(deep, references, visited)
+ return obj
+
+ # lookup in container:
+ for ent in references:
+ # Entities in container without an ID will be skipped:
+ if ent.id is not None and ent.id == propval.id:
+ # resolve this entity:
+ obj = convert_to_python_object(ent, references)
+ visited[propval.id] = obj
+ # self.__setattr__(prop, visited[propval.id])
+ if deep:
+ obj.resolve_references(deep, references, visited)
+ return obj
+ return propval
+
+ def resolve_references(self, deep: bool, references: db.Container,
+ visited: dict[Union[str, int],
+ "CaosDBPythonEntity"] = None):
+ """
+ Resolve this entity's references. This affects unresolved properties as well
+ as unresolved parents.
+
+ deep: bool
+ If True recursively resolve references also for all resolved references.
+
+ references: Optional[db.Container]
+ A container with references that might be resolved.
+ If None is passed as the container, this function tries to resolve entities from a running
+ CaosDB instance directly.
+ """
+
+ # This parameter is used in the recursion to keep track of already visited
+ # entites (in order to detect cycles).
+ if visited is None:
+ visited = dict()
+
+ for parent in self.get_parents():
+ # TODO
+ if isinstance(parent, CaosDBPythonUnresolvedParent):
+ pass
+
+ for prop in self.get_properties():
+ propval = self.__getattribute__(prop)
+ # Resolve all previously unresolved attributes that are entities:
+ if deep and isinstance(propval, CaosDBPythonEntity):
+ propval.resolve_references(deep, references)
+ elif isinstance(propval, list):
+ resolvedelements = []
+ for element in propval:
+ if deep and isinstance(element, CaosDBPythonEntity):
+ element.resolve_references(deep, references)
+ resolvedelements.append(element)
+ if isinstance(element, CaosDBPythonUnresolvedReference):
+ resolvedelements.append(
+ self._resolve_caosdb_python_unresolved_reference(element, deep,
+ references, visited))
+ else:
+ resolvedelements.append(element)
+ self.__setattr__(prop, resolvedelements)
+
+ elif isinstance(propval, CaosDBPythonUnresolvedReference):
+ val = self._resolve_caosdb_python_unresolved_reference(propval, deep,
+ references, visited)
+ self.__setattr__(prop, val)
+
+ def get_properties(self):
+ """
+ Return the names of all properties.
+ """
+
+ return [p for p in self.__dict__
+ if p not in self._forbidden]
+
+ @staticmethod
+ def deserialize(serialization: dict):
+ """
+ Deserialize a yaml representation of an entity in high level API form.
+ """
+
+ if "role" in serialization:
+ entity = high_level_type_for_role(serialization["role"])()
+ else:
+ entity = CaosDBPythonRecord()
+
+ for parent in serialization["parents"]:
+ if "unresolved" in parent:
+ id = None
+ name = None
+ if "id" in parent:
+ id = parent["id"]
+ if "name" in parent:
+ name = parent["name"]
+ entity.add_parent(CaosDBPythonUnresolvedParent(
+ id=id, name=name))
+ else:
+ raise NotImplementedError()
+
+ for baseprop in ("name", "id", "description", "version"):
+ if baseprop in serialization:
+ entity.__setattr__(baseprop, serialization[baseprop])
+
+ if type(entity) == CaosDBPythonFile:
+ entity.file = serialization["file"]
+ entity.path = serialization["path"]
+
+ for p in serialization["properties"]:
+ # The property needs to be set first:
+
+ prop = serialization["properties"][p]
+ if isinstance(prop, dict):
+ if "unresolved" in prop:
+ entity.__setattr__(p, CaosDBPythonUnresolvedReference(
+ id=prop["id"]))
+ else:
+ entity.__setattr__(p,
+ entity.deserialize(prop))
+ else:
+ entity.__setattr__(p, prop)
+
+ # if there is no metadata in the yaml file just initialize an empty metadata object
+ if "metadata" in serialization and p in serialization["metadata"]:
+ metadata = serialization["metadata"][p]
+ propmeta = entity.get_property_metadata(p)
+
+ for f in fields(propmeta):
+ if f.name in metadata:
+ propmeta.__setattr__(f.name, metadata[f.name])
+ else:
+ raise NotImplementedError()
+
+ return entity
+
+ def serialize(self, without_metadata: bool = False, visited: dict = None):
+ """
+ Serialize necessary information into a dict.
+
+ without_metadata: bool
+ If True don't set the metadata field in order to increase
+ readability. Not recommended if deserialization is needed.
+ """
+
+ if visited is None:
+ visited = dict()
+
+ if self in visited:
+ return visited[self]
+
+ metadata: dict[str, Any] = dict()
+ properties = dict()
+ parents = list()
+
+ # The full information to be returned:
+ fulldict = dict()
+ visited[self] = fulldict
+
+ # Add CaosDB role:
+ fulldict["role"] = standard_type_for_high_level_type(self, True)
+
+ for parent in self._parents:
+ if isinstance(parent, CaosDBPythonEntity):
+ parents.append(parent.serialize(without_metadata, visited))
+ elif isinstance(parent, CaosDBPythonUnresolvedParent):
+ parents.append({"name": parent.name, "id": parent.id,
+ "unresolved": True})
+ else:
+ raise RuntimeError("Incompatible class used as parent.")
+
+ for baseprop in ("name", "id", "description", "version"):
+ val = self.__getattribute__(baseprop)
+ if val is not None:
+ fulldict[baseprop] = val
+
+ if type(self) == CaosDBPythonFile:
+ fulldict["file"] = self.file
+ fulldict["path"] = self.path
+
+ for p in self.get_properties():
+ m = self.get_property_metadata(p)
+ metadata[p] = dict()
+ for f in fields(m):
+ val = m.__getattribute__(f.name)
+ if val is not None:
+ metadata[p][f.name] = val
+
+ val = self.get_property(p)
+ if isinstance(val, CaosDBPythonUnresolvedReference):
+ properties[p] = {"id": val.id, "unresolved": True}
+ elif isinstance(val, CaosDBPythonEntity):
+ properties[p] = val.serialize(without_metadata, visited)
+ elif isinstance(val, list):
+ serializedelements = []
+ for element in val:
+ if isinstance(element, CaosDBPythonUnresolvedReference):
+ elm = dict()
+ elm["id"] = element.id
+ elm["unresolved"] = True
+ serializedelements.append(elm)
+ elif isinstance(element, CaosDBPythonEntity):
+ serializedelements.append(
+ element.serialize(without_metadata,
+ visited))
+ else:
+ serializedelements.append(element)
+ properties[p] = serializedelements
+ else:
+ properties[p] = val
+
+ fulldict["properties"] = properties
+ fulldict["parents"] = parents
+
+ if not without_metadata:
+ fulldict["metadata"] = metadata
+ return fulldict
+
+ def __str__(self):
+ return yaml.dump(self.serialize(False))
+
+ # This seemed like a good solution, but makes it difficult to
+ # compare python objects directly:
+ #
+ # def __repr__(self):
+ # return yaml.dump(self.serialize(True))
+
+
+class CaosDBPythonRecord(CaosDBPythonEntity):
+ pass
+
+
+class CaosDBPythonRecordType(CaosDBPythonEntity):
+ pass
+
+
+class CaosDBPythonProperty(CaosDBPythonEntity):
+ pass
+
+
+class CaosDBMultiProperty:
+ """
+ This implements a multi property using a python list.
+ """
+
+ def __init__(self):
+ raise NotImplementedError()
+
+
+class CaosDBPythonFile(CaosDBPythonEntity):
+ def download(self, target=None):
+ if self.id is None:
+ raise RuntimeError("Cannot download file when id is missing.")
+ f = db.File(id=self.id).retrieve()
+ return f.download(target)
+
+
+BASE_ATTRIBUTES = (
+ "id", "name", "description", "version", "path", "file")
+
+
+def _single_convert_to_python_object(robj: CaosDBPythonEntity,
+ entity: db.Entity,
+ references: Optional[db.Container] = None):
+ """
+ Convert a db.Entity from the standard API to a (previously created)
+ CaosDBPythonEntity from the high level API.
+
+ This method will not resolve any unresolved references, so reference properties
+ as well as parents will become unresolved references in the first place.
+
+ The optional third parameter can be used
+ to resolve references that occur in the converted entities and resolve them
+ to their correct representations. (Entities that are not found remain as
+ CaosDBPythonUnresolvedReferences.)
+
+ Returns the input object robj.
+ """
+ for base_attribute in BASE_ATTRIBUTES:
+ val = entity.__getattribute__(base_attribute)
+ if val is not None:
+ if isinstance(val, db.common.models.Version):
+ val = val.id
+ robj.__setattr__(base_attribute, val)
+
+ for prop in entity.properties:
+ robj._set_property_from_entity(prop, entity.get_importance(prop), references)
+
+ for parent in entity.parents:
+ robj.add_parent(CaosDBPythonUnresolvedParent(id=parent.id,
+ name=parent.name))
+
+ return robj
+
+
+def _convert_property_value(propval):
+ if isinstance(propval, CaosDBPythonUnresolvedReference):
+ propval = propval.id
+ elif isinstance(propval, CaosDBPythonEntity):
+ propval = _single_convert_to_entity(
+ standard_type_for_high_level_type(propval)(), propval)
+ elif isinstance(propval, list):
+ propval = [_convert_property_value(element) for element in propval]
+
+ # TODO: test case for list missing
+
+ return propval
+
+
+def _single_convert_to_entity(entity: db.Entity,
+ robj: CaosDBPythonEntity):
+ """
+ Convert a CaosDBPythonEntity to an entity in standard pylib format.
+
+ entity: db.Entity
+ An empty entity.
+
+ robj: CaosDBPythonEntity
+ The CaosDBPythonEntity that is supposed to be converted to the entity.
+ """
+
+ for base_attribute in BASE_ATTRIBUTES:
+ if base_attribute in ("file", "path") and not isinstance(robj, CaosDBPythonFile):
+ continue
+
+ # Skip version:
+ if base_attribute == "version":
+ continue
+
+ val = robj.__getattribute__(base_attribute)
+
+ if val is not None:
+ entity.__setattr__(base_attribute, val)
+
+ for parent in robj.get_parents():
+ if isinstance(parent, CaosDBPythonUnresolvedParent):
+ entity.add_parent(name=parent.name, id=parent.id)
+ elif isinstance(parent, CaosDBPythonRecordType):
+ raise NotImplementedError()
+ else:
+ raise RuntimeError("Incompatible class used as parent.")
+
+ for prop in robj.get_properties():
+ propval = robj.__getattribute__(prop)
+ metadata = robj.get_property_metadata(prop)
+
+ propval = _convert_property_value(propval)
+
+ entity.add_property(
+ name=prop,
+ value=propval,
+ unit=metadata.unit,
+ importance=metadata.importance,
+ datatype=metadata.datatype,
+ description=metadata.description,
+ id=metadata.id)
+
+ return entity
+
+
+def convert_to_entity(python_object):
+ if isinstance(python_object, db.Container):
+ # Create a list of objects:
+
+ return [convert_to_entity(i) for i in python_object]
+ elif isinstance(python_object, CaosDBPythonRecord):
+ return _single_convert_to_entity(db.Record(), python_object)
+ elif isinstance(python_object, CaosDBPythonFile):
+ return _single_convert_to_entity(db.File(), python_object)
+ elif isinstance(python_object, CaosDBPythonRecordType):
+ return _single_convert_to_entity(db.RecordType(), python_object)
+ elif isinstance(python_object, CaosDBPythonProperty):
+ return _single_convert_to_entity(db.Property(), python_object)
+ elif isinstance(python_object, CaosDBPythonEntity):
+ return _single_convert_to_entity(db.Entity(), python_object)
+ else:
+ raise ValueError("Cannot convert an object of this type.")
+
+
+def convert_to_python_object(entity: Union[db.Container, db.Entity],
+ references: Optional[db.Container] = None):
+ """
+ Convert either a container of CaosDB entities or a single CaosDB entity
+ into the high level representation.
+
+ The optional second parameter can be used
+ to resolve references that occur in the converted entities and resolve them
+ to their correct representations. (Entities that are not found remain as
+ CaosDBPythonUnresolvedReferences.)
+ """
+ if isinstance(entity, db.Container):
+ # Create a list of objects:
+ return [convert_to_python_object(i, references) for i in entity]
+
+ return _single_convert_to_python_object(
+ high_level_type_for_standard_type(entity)(), entity, references)
+
+
+def new_high_level_entity(entity: db.RecordType,
+ importance_level: str,
+ name: str = None):
+ """
+ Create an new record in high level format based on a record type in standard format.
+
+ entity: db.RecordType
+ The record type to initialize the new record from.
+
+ importance_level: str
+ None, obligatory, recommended or suggested
+ Initialize new properties up to this level.
+ Properties in the record type with no importance will be added
+ regardless of the importance_level.
+
+ name: str
+ Name of the new record.
+ """
+
+ r = db.Record(name=name)
+ r.add_parent(entity)
+
+ impmap = {
+ None: 0, "SUGGESTED": 3, "RECOMMENDED": 2, "OBLIGATORY": 1}
+
+ for prop in entity.properties:
+ imp = entity.get_importance(prop)
+ if imp is not None and impmap[importance_level] < impmap[imp]:
+ continue
+
+ r.add_property(prop)
+
+ return convert_to_python_object(r)
+
+
+def create_record(rtname: str, name: str = None, **kwargs):
+ """
+ Create a new record based on the name of a record type. The new record is returned.
+
+ rtname: str
+ The name of the record type.
+
+ name: str
+ This is optional. A name for the new record.
+
+ kwargs:
+ Additional arguments are used to set attributes of the
+ new record.
+ """
+ obj = new_high_level_entity(
+ db.RecordType(name=rtname).retrieve(), "SUGGESTED", name)
+ for key, value in kwargs.items():
+ obj.__setattr__(key, value)
+ return obj
+
+
+def load_external_record(record_name: str):
+ """
+ Retrieve a record by name and convert it to the high level API format.
+ """
+ return convert_to_python_object(db.Record(name=record_name).retrieve())
+
+
+def create_entity_container(record: CaosDBPythonEntity):
+ """
+ Convert this record into an entity container in standard format that can be used
+ to insert or update entities in a running CaosDB instance.
+ """
+ ent = convert_to_entity(record)
+ lse: List[db.Entity] = [ent]
+ create_flat_list([ent], lse)
+ return db.Container().extend(lse)
+
+
+def query(query: str, resolve_references: bool = True, references: db.Container = None):
+ """
+
+ """
+ res = db.execute_query(query)
+ objects = convert_to_python_object(res)
+ if resolve_references:
+ for obj in objects:
+ obj.resolve_references(True, references)
+ return objects
diff --git a/src/caosdb/utils/plantuml.py b/src/caosdb/utils/plantuml.py
index acf218c8297028acba1cbcceabd9ba4398b0e7aa..aee515be380eea23ae29b6fc6b8d9f14da868e21 100644
--- a/src/caosdb/utils/plantuml.py
+++ b/src/caosdb/utils/plantuml.py
@@ -296,6 +296,14 @@ def retrieve_substructure(start_record_types, depth, result_id_set=None, result_
name=get_referenced_recordtype(prop.datatype)).retrieve()
retrieve_substructure([rt], depth-1, result_id_set,
result_container, False)
+ # TODO: clean up this hack
+ # TODO: make it also work for files
+ if is_reference(prop.datatype) and prop.value is not None:
+ r = db.Record(id=prop.value).retrieve()
+ retrieve_substructure([r], depth-1, result_id_set, result_container, False)
+ if r.id not in result_id_set:
+ result_container.append(r)
+ result_id_set.add(r.id)
if prop.id not in result_id_set:
result_container.append(prop)
diff --git a/src/doc/future_caosdb.md b/src/doc/future_caosdb.md
new file mode 100644
index 0000000000000000000000000000000000000000..a34d97ef97bc44503ad3da128378333649839d8f
--- /dev/null
+++ b/src/doc/future_caosdb.md
@@ -0,0 +1,112 @@
+# The future of the CaosDB Python Client
+
+The current Python client has done us great services but its structure and the
+way it is used sometimes feels outdated and clumsy. In this document we sketch
+how it might look different in future and invite everyone to comment or
+contribute to this development.
+
+At several locations in this document there will be links to discussion issues.
+If you want to discuss something new, you can create a new issue
+[here](https://gitlab.com/caosdb/caosdb-pylib/-/issues/new).
+
+## Overview
+Let's get a general impression before discussing single aspects.
+
+``` python
+import caosdb as db
+experiments = db.query("FIND Experiment")
+# print name and date for each `Experiment`
+for exp in experiments:
+ print(exp.name, exp.date)
+
+# suppose `Experiments` reference `Projects` which have a `Funding` Property
+one_exp = experiments[0]
+print(one_exp.Project.Funding)
+
+new_one = db.create_record("Experiment")
+new_one.date = "2022-01-01"
+new_one.name = "Needle Measurement"
+new_one.insert()
+```
+Related discussions:
+- [recursive retrieve in query](https://gitlab.com/caosdb/caosdb-pylib/-/issues/57)
+- [create_record function](https://gitlab.com/caosdb/caosdb-pylib/-/issues/58)
+- [data model utility](https://gitlab.com/caosdb/caosdb-pylib/-/issues/59)
+
+## Quickstart
+Note that you can try out one possible implementation using the
+`caosdb.high_level_api` module. It is experimental and might be removed in
+future!
+
+A `resolve_references` function allows to retrieve the referenced entities of
+an entity, container or a query result set (which is a container).
+It has the following parameters which can also be supplied to the `query`
+function:
+
+- `deep`: Whether to use recursive retrieval
+- `depth`: Maximum recursion depth
+- `references`: Whether to use the supplied db.Container to resolve
+ references. This allows offline usage. Set it to None if you want to
+ automatically retrieve entities from the current CaosDB connection.
+
+In order to allow a quick look at the object structures an easily readable
+serialization is provided by the `to_dict` function. It has the following
+argument:
+- `without_metadata`: Set this to True if you don\'t want to see
+ property metadata like \"unit\" or \"importance\".
+
+This function creates a simple dictionary containing a representation of
+the entity, which can be stored to disk and completely deserialized
+using the function `from_dict`.
+
+Furthermore, the `__str__` function uses this to display objects in yaml
+format by default statement
+
+## Design Decisions
+
+### Dot Notation
+Analogue, to what Pandas does. Provide bracket notation
+`rec.properties["test"]` for Properties with names that are in conflict with
+default attributes or contain spaces (or other forbidden characters).
+
+- Raise Exception if attribute does not exist but is accessed?
+- How to deal with lists? b has a list as value: `a.b[0].c = 5`
+
+[Discussion](https://gitlab.com/caosdb/caosdb-pylib/-/issues/60)
+
+### Serialization
+What information needs to be contained in (meta)data? How compatible is it with
+GRPC json serialization?
+
+
+### Recursive Retrieval
+I can resolve later and end up with the same result:
+`recs =db.query("FIND Experiment", depth=2)` equals `recs = db.query("FIND Experiment"); recs = resolve_references(recs, depth=2)`
+
+[Discussion](https://gitlab.com/caosdb/caosdb-pylib/-/issues/57)
+
+### In-Place operations
+Default behavior is to return new objects instead of modifying them in-place.
+This can be changed with the argument `inplace=True`.
+Especially the following functions operate by default NOT in-place:
+- update
+- insert
+- retrieve
+- resolve_references
+[Discussion](https://gitlab.com/caosdb/caosdb-pylib/-/issues/61)
+
+## Extend Example
+``` python
+import caosdb as db
+
+dm = db.get_data_model()
+
+new_one = db.create_record(dm.Experiment)
+new_one.date = "2022-01-01"
+new_one.name = "Needle Measurement"
+new_one.dataset = db.create_record(dm.Dataset)
+new_one.dataset.voltage = (5, "V")
+new_one.dataset.pulses = [5, 5.3]
+inserted = new_one.insert()
+print("The new record has the ID:", inserted.id)
+```
diff --git a/src/doc/high_level_api.org b/src/doc/high_level_api.org
new file mode 100644
index 0000000000000000000000000000000000000000..516df1b41d500fab000a72517fd2d12ba61753b7
--- /dev/null
+++ b/src/doc/high_level_api.org
@@ -0,0 +1,171 @@
+* High Level API
+
+In addition to the old standard pylib API, new versions of pylib ship with a high level API
+that facilitates usage of CaosDB entities within data analysis scripts. In a nutshell that
+API exposes all properties of CaosDB Records as standard python attributes making their
+access easier.
+
+Or to spell it out directly in Python:
+#+BEGIN_SRC python
+
+ import caosdb as db
+ # Old API:
+ r = db.Record()
+ r.add_parent("Experiment")
+ r.add_property(name="alpha", value=5)
+ r.get_property("alpha").value = 25 # setting properties (old api)
+ print(r.get_property("alpha").value + 25) # getting properties (old api)
+
+ from caosdb.high_level_api import convert_to_python_entity
+ obj = convert_to_python_object(r) # create a high level entity
+ obj.r = 25 # setting properties (new api)
+ print(obj.r + 25) # getting properties (new api)
+
+#+END_SRC
+
+
+** Quickstart
+
+The module, needed for the high level API is called:
+~caosdb.high_level_api~
+
+There are two functions converting entities between the two representation (old API and new API):
+- ~convert_to_python_object~: Convert entities from **old** into **new** representation.
+- ~convert_to_entity~: Convert entities from **new** into **old** representation.
+
+Furthermore there are a few utility functions which expose very practical shorthands:
+- ~new_high_level_entity~: Retrieve a record type and create a new high level entity which contains properties of a certain importance level preset.
+- ~create_record~: Create a new high level entity using the name of a record type and a list of key value pairs as properties.
+- ~load_external_record~: Retrieve a record with a specific name and return it as high level entity.
+- ~create_entity_container~: Convert a high level entity into a standard entity including all sub entities.
+- ~query~: Do a CaosDB query and return the result as a container of high level objects.
+
+So as a first example, you could retrieve any record from CaosDB and use it using its high level representation:
+#+BEGIN_SRC python
+ from caosdb.high_level_api import query
+
+ res = query("FIND Record Experiment")
+ experiment = res[0]
+ # Use a property:
+ print(experiment.date)
+
+ # Use sub properties:
+ print(experiment.output[0].path)
+#+END_SRC
+
+The latter example demonstrates, that the function query is very powerful. For its default parameter
+values it automatically resolves and retrieves references recursively, so that sub properties,
+like the list of output files "output", become immediately available.
+
+**Note** that for the old API you were supposed to run the following series of commands
+to achieve the same result:
+#+BEGIN_SRC python
+ import caosdb as db
+
+ res = db.execute_query("FIND Record Experiment")
+ output = res.get_property("output")
+ output_file = db.File(id=output.value[0].id).retrieve()
+ print(output_file.path)
+#+END_SRC
+
+Resolving subproperties makes use of the "resolve_reference" function provided by the high level
+entity class (~CaosDBPythonEntity~), with the following parameters:
+- ~deep~: Whether to use recursive retrieval
+- ~references~: Whether to use the supplied db.Container to resolve references. This allows offline usage. Set it to None if you want to automatically retrieve entities from the current CaosDB connection.
+- ~visited~: Needed for recursion, set this to None.
+
+Objects in the high level representation can be serialized to a simple yaml form using the function
+~serialize~ with the following parameters:
+- ~without_metadata~: Set this to True if you don't want to see property metadata like "unit" or "importance".
+- ~visited~: Needed for recursion, set this to None.
+
+This function creates a simple dictionary containing a representation of the entity, which can be
+stored to disk and completely deserialized using the function ~deserialize~.
+
+Furthermore the "__str__" function is overloaded, so that you can use print to directly inspect
+high level objects using the following statement:
+#+BEGIN_SRC python
+print(str(obj))
+#+END_SRC
+
+
+** Concepts
+
+As described in the section [[Quickstart]] the two functions ~convert_to_python_object~ and ~convert_to_entity~ convert
+entities beetween the high level and the standard representation.
+
+The high level entities are represented using the following classes from the module ~caosdb.high_level_api~:
+- ~CaosDBPythonEntity~: Base class of the following entity classes.
+- ~CaosDBPythonRecord~
+- ~CaosDBPythonRecordType~
+- ~CaosDBPythonProperty~
+- ~CaosDBPythonMultiProperty~: **WARNING** Not implemented yet.
+- ~CaosDBPythonFile~: Used for file entities and provides an additional ~download~ function for being able to directly retrieve files from CaosDB.
+
+In addition, there are the following helper structures which are realized as Python data classes:
+- ~CaosDBPropertyMetaData~: For storing meta data about properties.
+- ~CaosDBPythonUnresolved~: The base class of unresolved "things".
+- ~CaosDBPythonUnresolvedParent~: Parents of entities are stored as unresolved parents by default, storing an id or a name of a parent (or both).
+- ~CaosDBPythonUnresolvedReference~: An unresolved reference is a reference property with an id which has not (yet) been resolved to an Entity.
+
+The function "resolve_references" can be used to recursively replace ~CaosDBPythonUnresolvedReferences~ into members of type ~CaosDBPythonRecords~
+or ~CaosDBPythonFile~.
+
+Each property stored in a CaosDB record corresponds to:
+- a member attribute of ~CaosDBPythonRecord~ **and**
+- an entry in a dict called "metadata" storing a CaosDBPropertyMetadata object with the following information about proeprties:
+ - ~unit~
+ - ~datatype~
+ - ~description~
+ - ~id~
+ - ~importance~
+
+
+* Example
+
+The following shows a more complex example taken from a real world use case:
+A numerical experiment is created to simulate cardiac electric dynamics. The physical problem
+is modelled using the monodomain equation with the local current term given by the Mitchell
+Schaeffer Model.
+
+The data model for the numerical experiment consists of multiple record types which stores assosciated paremeters:
+- `MonodomainTissueSimulation`
+- `MitchellSchaefferModel`
+- `SpatialExtent2d`
+- `SpatialDimension`
+- `ConstantTimestep`
+- `ConstantDiffusion`
+
+First, the data model will be filled with the parameter values for this specific simulation run. It will be stored in the python variable `MonodomainRecord`. Passing the `MonodomainRecord` through the python functions, the simulation parameters can be easily accessed everywhere in the code when needed.
+
+Records are created by the `create_record` function. Parameter values can be set at record creation and also after creation as python properties of the corresponding record instance. The following example shows how to create a record, how to set the parameter at creation and how to set them as python properties
+
+#+BEGIN_SRC python
+ from caosdb.high_level_api import create_record
+
+ MonodomainRecord = create_record("MonodomainTissueSimulation")
+ MonodomainRecord.LocalModel = create_record("MitchellSchaefferModel")
+ MonodomainRecord.SpatialExtent = create_record(
+ "SpatialExtent2d", spatial_extent_x=100, spatial_extent_y=100)
+ MonodomainRecord.SpatialExtent.cell_sizes = [0.1, 0.1] # parameters can be set as properties
+ MonodomainRecord.SpatialDimension = create_record("SpatialDimension",
+ num_dim=2)
+
+ MonodomainRecord.TimestepInformation = create_record("ConstantTimestep")
+ MonodomainRecord.TimestepInformation.DeltaT = 0.1
+
+ D = create_record("ConstantDiffusion", diffusion_constant=0.1)
+ MonodomainRecord.DiffusionConstantType = D
+ model = MonodomainRecord.LocalModel
+ model.t_close = 150
+ model.t_open = 120
+ model.t_out = 6
+ model.t_in = 0.3
+ model.v_gate = 0.13
+ model.nvars = 2
+#+END_SRC
+
+At any position in the algorithm you are free to:
+- Convert this model to the standard python API and insert or update the records in a running instance of CaosDB.
+- Serialize this model in the high level API yaml format. This enables the CaosDB crawler to pickup the file and synchronize it with a running instance
+of CaosDB.
diff --git a/src/doc/high_level_api.rst b/src/doc/high_level_api.rst
new file mode 100644
index 0000000000000000000000000000000000000000..603052b135ad2289caea7e3bed59ae9d3301f811
--- /dev/null
+++ b/src/doc/high_level_api.rst
@@ -0,0 +1,163 @@
+High Level API
+==============
+
+In addition to the old standard pylib API, new versions of pylib ship
+with a high level API that facilitates usage of CaosDB entities within
+data analysis scripts. In a nutshell that API exposes all properties of
+CaosDB Records as standard python attributes making their access easier.
+
+Or to speak it out directly in Python:
+
+.. code:: python
+
+
+ import caosdb as db
+ # Old API:
+ r = db.Record()
+ r.add_parent("Experiment")
+ r.add_property(name="alpha", value=5)
+ r.get_property("alpha").value = 25 # setting properties (old api)
+ print(r.get_property("alpha").value + 25) # getting properties (old api)
+
+ from caosdb.high_level_api import convert_to_python_entity
+ obj = convert_to_python_object(r) # create a high level entity
+ obj.r = 25 # setting properties (new api)
+ print(obj.r + 25) # getting properties (new api)
+
+Quickstart
+----------
+
+The module, needed for the high level API is called:
+``caosdb.high_level_api``
+
+There are two functions converting entities between the two
+representation (old API and new API):
+
+- ``convert_to_python_object``: Convert entities from **old** into
+ **new** representation.
+- ``convert_to_entity``: Convert entities from **new** into **old**
+ representation.
+
+Furthermore there are a few utility functions which expose very
+practical shorthands:
+
+- ``new_high_level_entity``: Retrieve a record type and create a new
+ high level entity which contains properties of a certain importance
+ level preset.
+- ``create_record``: Create a new high level entity using the name of a
+ record type and a list of key value pairs as properties.
+- ``load_external_record``: Retrieve a record with a specific name and
+ return it as high level entity.
+- ``create_entity_container``: Convert a high level entity into a
+ standard entity including all sub entities.
+- ``query``: Do a CaosDB query and return the result as a container of
+ high level objects.
+
+So as a first example, you could retrieve any record from CaosDB and use
+it using its high level representation:
+
+.. code:: python
+
+ from caosdb.high_level_api import query
+
+ res = query("FIND Record Experiment")
+ experiment = res[0]
+ # Use a property:
+ print(experiment.date)
+
+ # Use sub properties:
+ print(experiment.output[0].path)
+
+The latter example demonstrates, that the function query is very
+powerful. For its default parameter values it automatically resolves and
+retrieves references recursively, so that sub properties, like the list
+of output files "output", become immediately available.
+
+**Note** that for the old API you were supposed to run the following
+series of commands to achieve the same result:
+
+.. code:: python
+
+ import caosdb as db
+
+ res = db.execute_query("FIND Record Experiment")
+ output = res.get_property("output")
+ output_file = db.File(id=output.value[0].id).retrieve()
+ print(output_file.path)
+
+Resolving subproperties makes use of the "resolve\ :sub:`reference`"
+function provided by the high level entity class
+(``CaosDBPythonEntity``), with the following parameters:
+
+- ``deep``: Whether to use recursive retrieval
+- ``references``: Whether to use the supplied db.Container to resolve
+ references. This allows offline usage. Set it to None if you want to
+ automatically retrieve entities from the current CaosDB connection.
+- ``visited``: Needed for recursion, set this to None.
+
+Objects in the high level representation can be serialized to a simple
+yaml form using the function ``serialize`` with the following
+parameters:
+
+- ``without_metadata``: Set this to True if you don't want to see
+ property metadata like "unit" or "importance".
+- ``visited``: Needed for recursion, set this to None.
+
+This function creates a simple dictionary containing a representation of
+the entity, which can be stored to disk and completely deserialized
+using the function ``deserialize``.
+
+Furthermore the "*str*" function is overloaded, so that you can use
+print to directly inspect high level objects using the following
+statement:
+
+.. code:: python
+
+ print(str(obj))
+
+Concepts
+--------
+
+As described in the section Quickstart the two functions
+``convert_to_python_object`` and ``convert_to_entity`` convert entities
+beetween the high level and the standard representation.
+
+The high level entities are represented using the following classes from
+the module ``caosdb.high_level_api``:
+
+- ``CaosDBPythonEntity``: Base class of the following entity classes.
+- ``CaosDBPythonRecord``
+- ``CaosDBPythonRecordType``
+- ``CaosDBPythonProperty``
+- ``CaosDBPythonMultiProperty``: **WARNING** Not implemented yet.
+- ``CaosDBPythonFile``: Used for file entities and provides an
+ additional ``download`` function for being able to directly retrieve
+ files from CaosDB.
+
+In addition, there are the following helper structures which are
+realized as Python data classes:
+
+- ``CaosDBPropertyMetaData``: For storing meta data about properties.
+- ``CaosDBPythonUnresolved``: The base class of unresolved "things".
+- ``CaosDBPythonUnresolvedParent``: Parents of entities are stored as
+ unresolved parents by default, storing an id or a name of a parent
+ (or both).
+- ``CaosDBPythonUnresolvedReference``: An unresolved reference is a
+ reference property with an id which has not (yet) been resolved to an
+ Entity.
+
+The function "resolve\ :sub:`references`" can be used to recursively
+replace ``CaosDBPythonUnresolvedReferences`` into members of type
+``CaosDBPythonRecords`` or ``CaosDBPythonFile``.
+
+Each property stored in a CaosDB record corresponds to:
+
+- a member attribute of ``CaosDBPythonRecord`` **and**
+- an entry in a dict called "metadata" storing a CaosDBPropertyMetadata
+ object with the following information about proeprties:
+
+ - ``unit``
+ - ``datatype``
+ - ``description``
+ - ``id``
+ - ``importance``
diff --git a/src/doc/index.rst b/src/doc/index.rst
index 004ae3a9926ed7a9a27720db1f3c28e72f1f3f28..7344b6aacdd55fd75f4940d834104faa00c33069 100644
--- a/src/doc/index.rst
+++ b/src/doc/index.rst
@@ -12,6 +12,7 @@ Welcome to PyCaosDB's documentation!
Concepts <concepts>
Configuration <configuration>
Administration <administration>
+ High Level API <high_level_api>
Code gallery <gallery/index>
API documentation<_apidoc/caosdb>
diff --git a/tox.ini b/tox.ini
index 0d245e03ef9c8fe2151e173cd1a10964d47ef82b..62658ae501234a276db8d570328bbe80f1348a4c 100644
--- a/tox.ini
+++ b/tox.ini
@@ -7,6 +7,7 @@ deps = .
nose
pytest
pytest-cov
+ python-dateutil
jsonschema==4.0.1
commands=py.test --cov=caosdb -vv {posargs}
diff --git a/unittests/test_apiutils.py b/unittests/test_apiutils.py
index 956c0a6b371a11ea98de1beb2a82c160b79d357b..43ab8107183f16bf8df1d0ea8e447b378bcf8123 100644
--- a/unittests/test_apiutils.py
+++ b/unittests/test_apiutils.py
@@ -25,9 +25,8 @@
# Test apiutils
# A. Schlemmer, 02/2018
-import pickle
-import tempfile
+import pytest
import caosdb as db
import caosdb.apiutils
from caosdb.apiutils import (apply_to_ids, compare_entities, create_id_query,
@@ -35,24 +34,6 @@ from caosdb.apiutils import (apply_to_ids, compare_entities, create_id_query,
from caosdb.common.models import SPECIAL_ATTRIBUTES
-from .test_property import testrecord
-
-import pytest
-
-
-def test_convert_object():
- r2 = db.apiutils.convert_to_python_object(testrecord)
- assert r2.species == "Rabbit"
-
-
-def test_pickle_object():
- r2 = db.apiutils.convert_to_python_object(testrecord)
- with tempfile.TemporaryFile() as f:
- pickle.dump(r2, f)
- f.seek(0)
- rn2 = pickle.load(f)
- assert r2.date == rn2.date
-
def test_apply_to_ids():
parent = db.RecordType(id=3456)
diff --git a/unittests/test_entity.py b/unittests/test_entity.py
index 1e88702ac016d7dcfdf00919dd0f93b5d3345e00..f2891fda266e1d62139b4cb2667c31b090ca6498 100644
--- a/unittests/test_entity.py
+++ b/unittests/test_entity.py
@@ -26,10 +26,13 @@
import unittest
from lxml import etree
+import os
from caosdb import (INTEGER, Entity, Property, Record, RecordType,
configure_connection)
from caosdb.connection.mockup import MockUpServerConnection
+UNITTESTDIR = os.path.dirname(os.path.abspath(__file__))
+
class TestEntity(unittest.TestCase):
@@ -87,7 +90,7 @@ class TestEntity(unittest.TestCase):
"""
parser = etree.XMLParser(remove_comments=True)
entity = Entity._from_xml(Entity(),
- etree.parse("unittests/test_record.xml",
+ etree.parse(os.path.join(UNITTESTDIR, "test_record.xml"),
parser).getroot())
self.assertEqual(entity.role, "Record")
diff --git a/unittests/test_high_level_api.py b/unittests/test_high_level_api.py
new file mode 100644
index 0000000000000000000000000000000000000000..a9e55c9c2a79f7ead8bbb3fb652c1b81427e69e9
--- /dev/null
+++ b/unittests/test_high_level_api.py
@@ -0,0 +1,643 @@
+# This file is a part of the CaosDB Project.
+#
+# Copyright (C) 2018 Research Group Biomedical Physics,
+# Max-Planck-Institute for Dynamics and Self-Organization Göttingen
+# Copyright (C) 2022 Alexander Schlemmer <alexander.schlemmer@ds.mpg.de>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see <https://www.gnu.org/licenses/>.
+#
+# ** end header
+#
+# Test high level api module
+# A. Schlemmer, 02/2022
+
+
+import caosdb as db
+from caosdb.high_level_api import (convert_to_entity, convert_to_python_object,
+ new_high_level_entity)
+from caosdb.high_level_api import (CaosDBPythonUnresolvedParent,
+ CaosDBPythonUnresolvedReference,
+ CaosDBPythonRecord, CaosDBPythonFile,
+ high_level_type_for_standard_type,
+ standard_type_for_high_level_type,
+ high_level_type_for_role,
+ CaosDBPythonEntity)
+from caosdb.apiutils import compare_entities
+
+from caosdb.common.datatype import (is_list_datatype,
+ get_list_datatype,
+ is_reference)
+
+import pytest
+from lxml import etree
+import os
+import tempfile
+import pickle
+
+import sys
+import traceback
+import pdb
+
+
+@pytest.fixture
+def testrecord():
+ parser = etree.XMLParser(remove_comments=True)
+ testrecord = db.Record._from_xml(
+ db.Record(),
+ etree.parse(os.path.join(os.path.dirname(__file__), "test_record.xml"),
+ parser).getroot())
+ return testrecord
+
+
+def test_convert_object(testrecord):
+ r2 = convert_to_python_object(testrecord)
+ assert r2.species == "Rabbit"
+
+
+def test_pickle_object(testrecord):
+ r2 = convert_to_python_object(testrecord)
+ with tempfile.TemporaryFile() as f:
+ pickle.dump(r2, f)
+ f.seek(0)
+ rn2 = pickle.load(f)
+ assert r2.date == rn2.date
+
+
+def test_convert_record():
+ """
+ Test the high level python API.
+ """
+ r = db.Record()
+ r.add_parent("bla")
+ r.add_property(name="a", value=42)
+ r.add_property(name="b", value="test")
+
+ obj = convert_to_python_object(r)
+ assert obj.a == 42
+ assert obj.b == "test"
+
+ # There is no such property
+ with pytest.raises(AttributeError):
+ assert obj.c == 18
+
+ assert obj.has_parent("bla") is True
+ assert obj.has_parent(CaosDBPythonUnresolvedParent(name="bla")) is True
+
+ # Check the has_parent function:
+ assert obj.has_parent("test") is False
+ assert obj.has_parent(CaosDBPythonUnresolvedParent(name="test")) is False
+
+ # duplicate parent
+ with pytest.raises(RuntimeError):
+ obj.add_parent("bla")
+
+ # add parent with just an id:
+ obj.add_parent(CaosDBPythonUnresolvedParent(id=225))
+ assert obj.has_parent(225) is True
+ assert obj.has_parent(CaosDBPythonUnresolvedParent(id=225)) is True
+ assert obj.has_parent(226) is False
+ assert obj.has_parent(CaosDBPythonUnresolvedParent(id=228)) is False
+
+ # same with just a name:
+ obj.add_parent(CaosDBPythonUnresolvedParent(name="another"))
+ assert obj.has_parent("another") is True
+
+
+def test_convert_with_references():
+ r_ref = db.Record()
+ r_ref.add_property(name="a", value=42)
+
+ r = db.Record()
+ r.add_property(name="ref", value=r_ref)
+
+ # try:
+ obj = convert_to_python_object(r)
+ # except:
+ # extype, value, tb = sys.exc_info()
+ # traceback.print_exc()
+ # pdb.post_mortem(tb)
+ assert obj.ref.a == 42
+
+ # With datatype:
+ r_ref = db.Record()
+ r_ref.add_parent("bla")
+ r_ref.add_property(name="a", value=42)
+
+ r = db.Record()
+ r.add_property(name="ref", value=r_ref)
+
+ obj = convert_to_python_object(r)
+ assert obj.ref.a == 42
+ # Parent does not automatically lead to a datatype:
+ assert obj.get_property_metadata("ref").datatype is None
+ assert obj.ref.has_parent("bla") is True
+
+ # Add datatype explicitely:
+ r_ref = db.Record()
+ r_ref.add_parent("bla")
+ r_ref.add_property(name="a", value=42)
+
+ r = db.Record()
+ r.add_property(name="ref", value=r_ref, datatype="bla")
+
+ obj = convert_to_python_object(r)
+ assert obj.ref.a == 42
+ # Parent does not automatically lead to a datatype:
+ assert obj.get_property_metadata("ref").datatype is "bla"
+ assert obj.ref.has_parent("bla") is True
+
+ # Unresolved Reference:
+ r = db.Record()
+ r.add_property(name="ref", value=27, datatype="bla")
+
+ obj = convert_to_python_object(r)
+ # Parent does not automatically lead to a datatype:
+ assert obj.get_property_metadata("ref").datatype is "bla"
+ assert isinstance(obj.ref, CaosDBPythonUnresolvedReference)
+ assert obj.ref.id == 27
+
+
+def test_resolve_references():
+ r = db.Record()
+ r.add_property(name="ref", value=27, datatype="bla")
+ r.add_property(name="ref_false", value=27) # this should be interpreted as integer property
+ obj = convert_to_python_object(r)
+
+ ref = db.Record(id=27)
+ ref.add_property(name="a", value=57)
+
+ unused_ref1 = db.Record(id=28)
+ unused_ref2 = db.Record(id=29)
+ unused_ref3 = db.Record(name="bla")
+
+ references = db.Container().extend([
+ unused_ref1, ref, unused_ref2, unused_ref3])
+
+ # Nothing is going to be resolved:
+ obj.resolve_references(False, db.Container())
+ assert isinstance(obj.ref, CaosDBPythonUnresolvedReference)
+ assert obj.ref.id == 27
+ assert obj.ref_false == 27
+
+ # deep == True does not help:
+ obj.resolve_references(True, db.Container())
+ assert isinstance(obj.ref, CaosDBPythonUnresolvedReference)
+ assert obj.ref.id == 27
+
+ # But adding the reference container will do:
+ obj.resolve_references(False, references)
+ assert not isinstance(obj.ref, CaosDBPythonUnresolvedReference)
+ assert isinstance(obj.ref, CaosDBPythonRecord)
+ assert obj.ref.id == 27
+ assert obj.ref.a == 57
+ # Datatypes will not automatically be set:
+ assert obj.ref.get_property_metadata("a").datatype is None
+
+ # Test deep resolve:
+ ref2 = db.Record(id=225)
+ ref2.add_property(name="c", value="test")
+ ref.add_property(name="ref", value=225, datatype="bla")
+
+ obj = convert_to_python_object(r)
+ assert isinstance(obj.ref, CaosDBPythonUnresolvedReference)
+ obj.resolve_references(False, references)
+ assert not isinstance(obj.ref, CaosDBPythonUnresolvedReference)
+ assert isinstance(obj.ref.ref, CaosDBPythonUnresolvedReference)
+ assert obj.ref.ref.id == 225
+
+ # Will not help, because ref2 is missing in container:
+ obj.resolve_references(True, references)
+ assert not isinstance(obj.ref, CaosDBPythonUnresolvedReference)
+ assert isinstance(obj.ref.ref, CaosDBPythonUnresolvedReference)
+ assert obj.ref.ref.id == 225
+
+ references.append(ref2)
+ obj.resolve_references(False, references)
+ assert not isinstance(obj.ref, CaosDBPythonUnresolvedReference)
+ assert isinstance(obj.ref.ref, CaosDBPythonUnresolvedReference)
+ assert obj.ref.ref.id == 225
+
+ obj.resolve_references(True, references)
+ assert not isinstance(obj.ref, CaosDBPythonUnresolvedReference)
+ assert not isinstance(obj.ref.ref, CaosDBPythonUnresolvedReference)
+ assert obj.ref.ref.c == "test"
+
+ # Test circular dependencies:
+ ref2.add_property(name="ref", value=27, datatype="bla")
+ obj = convert_to_python_object(r)
+ obj.resolve_references(True, references)
+ assert obj.ref.ref.ref == obj.ref
+
+
+def equal_entities(r1, r2):
+ res = compare_entities(r1, r2)
+ if len(res) != 2:
+ return False
+ for i in range(2):
+ if len(res[i]["parents"]) != 0 or len(res[i]["properties"]) != 0:
+ return False
+ return True
+
+
+def test_conversion_to_entity():
+ r = db.Record()
+ r.add_parent("bla")
+ r.add_property(name="a", value=42)
+ r.add_property(name="b", value="test")
+ obj = convert_to_python_object(r)
+ rconv = convert_to_entity(obj)
+ assert equal_entities(r, rconv)
+
+ # With a reference:
+ r_ref = db.Record()
+ r_ref.add_parent("bla")
+ r_ref.add_property(name="a", value=42)
+
+ r = db.Record()
+ r.add_property(name="ref", value=r_ref)
+ obj = convert_to_python_object(r)
+ rconv = convert_to_entity(obj)
+ assert (rconv.get_property("ref").value.get_property("a").value
+ == r.get_property("ref").value.get_property("a").value)
+ # TODO: add more tests here
+
+
+def test_base_properties():
+ r = db.Record(id=5, name="test", description="ok")
+ r.add_property(name="v", value=15, datatype=db.INTEGER, unit="kpx",
+ importance="RECOMMENDED", description="description")
+ obj = convert_to_python_object(r)
+ assert obj.name == "test"
+ assert obj.id == 5
+ assert obj.description == "ok"
+ metadata = obj.get_property_metadata("v")
+ assert metadata.id is None
+ assert metadata.datatype == db.INTEGER
+ assert metadata.unit == "kpx"
+ assert metadata.importance == "RECOMMENDED"
+ assert metadata.description == "description"
+
+ rconv = convert_to_entity(obj)
+ assert rconv.name == "test"
+ assert rconv.id == 5
+ assert rconv.description == "ok"
+ prop = rconv.get_property("v")
+ assert prop.value == 15
+ assert prop.datatype == db.INTEGER
+ assert prop.unit == "kpx"
+ assert prop.description == "description"
+ assert rconv.get_importance("v") == "RECOMMENDED"
+
+
+def test_empty():
+ r = db.Record()
+ obj = convert_to_python_object(r)
+ assert isinstance(obj, CaosDBPythonRecord)
+ assert len(obj.get_properties()) == 0
+ assert len(obj.get_parents()) == 0
+
+ rconv = convert_to_entity(obj)
+ assert len(rconv.properties) == 0
+
+
+def test_wrong_entity_for_file():
+ r = db.Record()
+ r.path = "test.dat"
+ r.file = "/local/path/test.dat"
+ assert r.path == "test.dat"
+ assert r.file == "/local/path/test.dat"
+ with pytest.raises(RuntimeError):
+ obj = convert_to_python_object(r)
+
+
+def test_serialization():
+ r = db.Record(id=5, name="test", description="ok")
+ r.add_property(name="v", value=15, datatype=db.INTEGER, unit="kpx",
+ importance="RECOMMENDED")
+
+ obj = convert_to_python_object(r)
+ text = str(obj)
+ teststrs = ["description: ok", "id: 5", "datatype: INTEGER",
+ "importance: RECOMMENDED", "unit: kpx", "name: test", "v: 15"]
+ for teststr in teststrs:
+ assert teststr in text
+
+ r = db.Record(description="ok")
+ r.add_property(name="v", value=15, datatype=db.INTEGER, unit="kpx",
+ importance="RECOMMENDED")
+ obj = convert_to_python_object(r)
+ text = str(obj)
+ assert "name" not in text
+ assert "id" not in text
+
+
+def test_files():
+ # empty file:
+ r = db.File()
+ obj = convert_to_python_object(r)
+ print(type(obj))
+ assert isinstance(obj, CaosDBPythonFile)
+ assert len(obj.get_properties()) == 0
+ assert len(obj.get_parents()) == 0
+
+ rconv = convert_to_entity(obj)
+ assert len(rconv.properties) == 0
+
+ r.path = "test.dat"
+ r.file = "/local/path/test.dat"
+ obj = convert_to_python_object(r)
+ assert r.path == "test.dat"
+ assert r.file == "/local/path/test.dat"
+ assert isinstance(obj, CaosDBPythonFile)
+
+ assert obj.path == "test.dat"
+ assert obj.file == "/local/path/test.dat"
+
+ assert "path: test.dat" in str(obj)
+ assert "file: /local/path/test.dat" in str(obj)
+
+ # record with file property:
+ rec = db.Record()
+ rec.add_property(name="testfile", value=r)
+ assert rec.get_property("testfile").value.file == "/local/path/test.dat"
+ assert rec.get_property("testfile").value.path == "test.dat"
+
+ obj = convert_to_python_object(rec)
+ assert obj.testfile.file == "/local/path/test.dat"
+ assert obj.testfile.path == "test.dat"
+
+ rconv = convert_to_entity(obj)
+ assert rconv.get_property("testfile").value.file == "/local/path/test.dat"
+ assert rconv.get_property("testfile").value.path == "test.dat"
+
+ # record with file property as reference:
+ rec = db.Record()
+ rec.add_property(name="testfile", value=2, datatype=db.FILE)
+ obj = convert_to_python_object(rec)
+ assert type(obj.testfile) == CaosDBPythonUnresolvedReference
+ assert obj.testfile.id == 2
+ assert obj.get_property_metadata("testfile").datatype == db.FILE
+
+ # without resolving references:
+ rconv = convert_to_entity(obj)
+ p = rconv.get_property("testfile")
+ assert p.value == 2
+ assert p.datatype == db.FILE
+
+ # with previously resolved reference (should not work here, because id is missing):
+ obj.resolve_references(True, db.Container().extend(r))
+ rconv = convert_to_entity(obj)
+ p = rconv.get_property("testfile")
+ assert p.value == 2
+ assert p.datatype == db.FILE
+
+ # this time it must work:
+ r.id = 2
+ obj.resolve_references(True, db.Container().extend(r))
+ rconv = convert_to_entity(obj)
+ p = rconv.get_property("testfile")
+ assert type(p.value) == db.File
+ assert p.datatype == db.FILE
+ assert p.value.file == "/local/path/test.dat"
+ assert p.value.path == "test.dat"
+
+
+@pytest.mark.xfail
+def test_record_generator():
+ rt = db.RecordType(name="Simulation")
+ rt.add_property(name="a", datatype=db.INTEGER)
+ rt.add_property(name="b", datatype=db.DOUBLE)
+ rt.add_property(name="inputfile", datatype=db.FILE)
+
+ simrt = db.RecordType(name="SimOutput")
+ rt.add_property(name="outputfile", datatype="SimOutput")
+
+ obj = new_high_level_entity(
+ rt, "SUGGESTED", "", True)
+ print(obj)
+ assert False
+
+
+def test_list_types():
+ r = db.Record()
+ r.add_property(name="a", value=[1, 2, 4])
+
+ assert get_list_datatype(r.get_property("a").datatype) is None
+
+ obj = convert_to_python_object(r)
+ assert type(obj.a) == list
+ assert len(obj.a) == 3
+ assert 4 in obj.a
+ assert obj.get_property_metadata("a").datatype is None
+
+ conv = convert_to_entity(obj)
+ prop = r.get_property("a")
+ assert prop.value == [1, 2, 4]
+ assert prop.datatype is None
+
+ r.get_property("a").datatype = db.LIST(db.INTEGER)
+ assert r.get_property("a").datatype == "LIST<INTEGER>"
+ obj = convert_to_python_object(r)
+ assert type(obj.a) == list
+ assert len(obj.a) == 3
+ assert 4 in obj.a
+ assert obj.get_property_metadata("a").datatype == "LIST<INTEGER>"
+
+ conv = convert_to_entity(obj)
+ prop = r.get_property("a")
+ assert prop.value == [1, 2, 4]
+ assert obj.get_property_metadata("a").datatype == "LIST<INTEGER>"
+
+ # List of referenced objects:
+ r = db.Record()
+ r.add_property(name="a", value=[1, 2, 4], datatype="LIST<TestReference>")
+ obj = convert_to_python_object(r)
+ assert type(obj.a) == list
+ assert len(obj.a) == 3
+ assert obj.get_property_metadata("a").datatype == "LIST<TestReference>"
+ for i in range(3):
+ assert type(obj.a[i]) == CaosDBPythonUnresolvedReference
+ assert obj.a == [CaosDBPythonUnresolvedReference(id=i) for i in [1, 2, 4]]
+
+ # Try resolving:
+
+ # Should not work:
+ obj.resolve_references(False, db.Container())
+ assert type(obj.a) == list
+ assert len(obj.a) == 3
+ assert obj.get_property_metadata("a").datatype == "LIST<TestReference>"
+ for i in range(3):
+ assert type(obj.a[i]) == CaosDBPythonUnresolvedReference
+ assert obj.a == [CaosDBPythonUnresolvedReference(id=i) for i in [1, 2, 4]]
+
+ references = db.Container()
+ for i in [1, 2, 4]:
+ ref = db.Record(id=i)
+ ref.add_property(name="val", value=str(i) + " bla")
+ references.append(ref)
+
+ obj.resolve_references(False, references)
+ assert type(obj.a) == list
+ assert len(obj.a) == 3
+ assert obj.get_property_metadata("a").datatype == "LIST<TestReference>"
+ for i in range(3):
+ assert type(obj.a[i]) == CaosDBPythonRecord
+
+ assert obj.a[0].val == "1 bla"
+
+ # Conversion with embedded records:
+ r2 = db.Record()
+ r2.add_property(name="a", value=4)
+ r3 = db.Record()
+ r3.add_property(name="b", value=8)
+
+ r = db.Record()
+ r.add_property(name="a", value=[r2, r3])
+
+ obj = convert_to_python_object(r)
+ assert type(obj.a) == list
+ assert len(obj.a) == 2
+ assert obj.a[0].a == 4
+ assert obj.a[1].b == 8
+
+ # Serialization
+ text = str(obj)
+ text2 = str(convert_to_python_object(r2)).split("\n")
+ print(text)
+ # cut away first two characters in text
+ text = [line[4:] for line in text.split("\n")]
+ for line in text2:
+ assert line in text
+
+
+# Test utility functions:
+def test_type_conversion():
+ assert high_level_type_for_standard_type(db.Record()) == CaosDBPythonRecord
+ assert high_level_type_for_standard_type(db.Entity()) == CaosDBPythonEntity
+ assert standard_type_for_high_level_type(CaosDBPythonRecord()) == db.Record
+ assert standard_type_for_high_level_type(CaosDBPythonEntity()) == db.Entity
+ assert standard_type_for_high_level_type(CaosDBPythonFile(), True) == "File"
+ assert standard_type_for_high_level_type(CaosDBPythonRecord(), True) == "Record"
+ assert high_level_type_for_role("Record") == CaosDBPythonRecord
+ assert high_level_type_for_role("Entity") == CaosDBPythonEntity
+ assert high_level_type_for_role("File") == CaosDBPythonFile
+ with pytest.raises(RuntimeError, match="Unknown role."):
+ high_level_type_for_role("jkaldjfkaldsjf")
+
+ with pytest.raises(RuntimeError, match="Incompatible type."):
+ standard_type_for_high_level_type(42, True)
+
+ with pytest.raises(ValueError):
+ high_level_type_for_standard_type("ajsdkfjasfkj")
+
+ with pytest.raises(RuntimeError, match="Incompatible type."):
+ class IncompatibleType(db.Entity):
+ pass
+ high_level_type_for_standard_type(IncompatibleType())
+
+
+def test_deserialization():
+ r = db.Record(id=17, name="test")
+ r.add_parent("bla")
+ r.add_property(name="a", value=42)
+ r.add_property(name="b", value="test")
+
+ obj = convert_to_python_object(r)
+
+ serial = obj.serialize()
+ obj_des = CaosDBPythonEntity.deserialize(serial)
+
+ assert obj_des.name == "test"
+ assert obj_des.id == 17
+ assert obj_des.has_parent(CaosDBPythonUnresolvedParent(name="bla"))
+ print(obj)
+ print(obj_des)
+
+ # This test is very strict, and might fail if order in dictionary is not preserved:
+ assert obj.serialize() == obj_des.serialize()
+
+ f = db.File()
+ f.file = "bla.test"
+ f.path = "/test/n/bla.test"
+
+ obj = convert_to_python_object(f)
+
+ serial = obj.serialize()
+ obj_des = CaosDBPythonEntity.deserialize(serial)
+ assert obj_des.file == "bla.test"
+ assert obj_des.path == "/test/n/bla.test"
+
+ r = db.Record(id=17, name="test")
+ r.add_parent("bla")
+ r.add_property(name="a", value=42)
+ r.add_property(name="b", value="test")
+
+ ref = db.Record(id=28)
+ ref.add_parent("bla1")
+ ref.add_parent("bla2")
+ ref.add_property(name="c", value=5,
+ unit="s", description="description missing")
+ r.add_property(name="ref", value=ref)
+
+ obj = convert_to_python_object(r)
+
+ serial = obj.serialize()
+ obj_des = CaosDBPythonEntity.deserialize(serial)
+ assert obj.serialize() == obj_des.serialize()
+
+
+@pytest.fixture
+def get_record_container():
+ record_xml = """
+<Entities>
+ <Record id="109">
+ <Version id="da669fce50554b2835c3826cf717d6a4532f02de" head="true">
+ <Predecessor id="68534369c5fd05e5bb1d37801a3dbc1532a8e094"/>
+ </Version>
+ <Parent id="103" name="Experiment" description="General type for all experiments in our lab"/>
+ <Property id="104" name="alpha" description="A fictitious measurement" datatype="DOUBLE" unit="km" importance="FIX" flag="inheritance:FIX">16.0</Property>
+ <Property id="107" name="date" datatype="DATETIME" importance="FIX" flag="inheritance:FIX">2022-03-16</Property>
+ <Property id="108" name="identifier" datatype="TEXT" importance="FIX" flag="inheritance:FIX">Demonstration</Property>
+ <Property id="111" name="sources" description="The elements of this lists are scientific activities that this scientific activity is based on." datatype="LIST<ScientificActivity>" importance="FIX" flag="inheritance:FIX">
+ <Value>109</Value>
+ </Property>
+ </Record>
+</Entities>"""
+
+ c = db.Container.from_xml(record_xml)
+ return c
+
+
+def test_recursion(get_record_container):
+ r = convert_to_python_object(get_record_container[0])
+ r.resolve_references(r, get_record_container)
+ assert r.id == 109
+ assert r.sources[0].id == 109
+ assert r.sources[0].sources[0].id == 109
+ assert "&id001" in str(r)
+ assert "*id001" in str(r)
+
+ d = r.serialize(True)
+ assert r.sources[0] == r.sources[0].sources[0]
+
+
+@pytest.mark.xfail
+def test_recursion_advanced(get_record_container):
+ # TODO:
+ # This currently fails, because resolve is done in a second step
+ # and therefore a new python object is created for the reference.
+ r = convert_to_python_object(get_record_container[0])
+ r.resolve_references(r, get_record_container)
+ d = r.serialize(True)
+ assert r == r.sources[0]