Skip to content
Snippets Groups Projects
Commit d355c12a authored by Henrik tom Wörden's avatar Henrik tom Wörden
Browse files

Merge branch 'f-refactor-high-level-api' into 'dev' 

F refactor high level api

See merge request !52
parents 4916fd45 f116a460
No related branches found
No related tags found
2 merge requests!57RELEASE 0.7.3,!52F refactor high level api
Pipeline #22139 passed
Stage: code_style
Stage: linting
Stage: setup
Stage: test
Stage: deploy
Expand all Hide whitespace changes
Inline Side-by-side
Showing
with 2168 additions and 414 deletions
CHANGELOG.md
+ 2
− 0
View file @ d355c12a
......@@ -10,6 +10,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
### Added ###
- New function in apiutils that copies an Entity.
- New EXPERIMENTAL module `high_level_api` which is a completely refactored version of
the old `high_level_api` from apiutils. Please see the included documentation for details.
### Changed ###
......
src/caosdb/apiutils.py
+ 37
− 393
View file @ d355c12a
......@@ -33,6 +33,8 @@ import warnings
from collections.abc import Iterable
from subprocess import call
from typing import Optional, Any
from caosdb.common.datatype import (BOOLEAN, DATETIME, DOUBLE, FILE, INTEGER,
REFERENCE, TEXT, is_reference)
from caosdb.common.models import (Container, Entity, File, Property, Query,
......@@ -99,22 +101,6 @@ def create_id_query(ids):
["ID={}".format(id) for id in ids])
def retrieve_entity_with_id(eid):
return execute_query("FIND ENTITY WITH ID={}".format(eid), unique=True)
def retrieve_entities_with_ids(entities):
collection = Container()
step = 20
for i in range(len(entities)//step+1):
collection.extend(
execute_query(
create_id_query(entities[i*step:(i+1)*step])))
return collection
def get_type_of_entity_with(id_):
objs = retrieve_entities_with_ids([id_])
......@@ -138,385 +124,20 @@ def get_type_of_entity_with(id_):
return Entity
class CaosDBPythonEntity(object):
_last_id = 0
def __init__(self):
warnings.warn("The CaosDBPythonEntity class is deprecated, replacements will be provided by"
" the high_level_api module.",
DeprecationWarning, stacklevel=2)
# Save a copy of the dry state
# of this object in order to be
# able to detect conflicts.
self.do_not_expand = False
self._parents = []
self._id = CaosDBPythonEntity._get_id()
self._path = None
self._file = None
self.pickup = None
# TODO:
# 3.) resolve references up to a specific depth (including infinity)
# 4.) resolve parents function -> partially implemented by
# get_parent_names
self._references = {}
self._properties = set()
self._datatypes = {}
self._forbidden = dir(self)
@staticmethod
def _get_id():
CaosDBPythonEntity._last_id -= 1
return CaosDBPythonEntity._last_id
def _set_property_from_entity(self, ent):
name = ent.name
val = ent.value
pr = ent.datatype
val, reference = self._type_converted_value(val, pr)
self.set_property(name, val, reference, datatype=pr)
def set_property(self, name, value, is_reference=False,
overwrite=False, datatype=None):
"""
overwrite: Use this if you definitely only want one property with that name (set to True).
"""
self._datatypes[name] = datatype
if isinstance(name, Entity):
name = name.name
if name in self._forbidden:
raise RuntimeError("Entity cannot be converted to a corresponding "
"Python representation. Name of property " +
name + " is forbidden!")
already_exists = (name in dir(self))
if already_exists and not overwrite:
# each call to _set_property checks first if it already exists
# if yes: Turn the attribute into a list and
# place all the elements into that list.
att = self.__getattribute__(name)
if isinstance(att, list):
pass
else:
old_att = self.__getattribute__(name)
self.__setattr__(name, [old_att])
if is_reference:
self._references[name] = [
self._references[name]]
att = self.__getattribute__(name)
att.append(value)
if is_reference:
self._references[name].append(int(value))
else:
if is_reference:
self._references[name] = value
self.__setattr__(name, value)
if not (already_exists and overwrite):
self._properties.add(name)
add_property = set_property
def set_id(self, idx):
self._id = idx
def _type_converted_list(self, val, pr):
"""Convert a list to a python list of the correct type."""
prrealpre = pr.replace("&lt;", "<").replace("&gt;", ">")
prreal = prrealpre[prrealpre.index("<") + 1:prrealpre.rindex(">")]
lst = [self._type_converted_value(i, prreal) for i in val]
return ([i[0] for i in lst], lst[0][1])
def _type_converted_value(self, val, pr):
"""Convert val to the correct type which is indicated by the database
type string in pr.
Returns a tuple with two entries:
- the converted value
- True if the value has to be interpreted as an id acting as a reference
"""
if val is None:
return (None, False)
elif pr == DOUBLE:
return (float(val), False)
elif pr == BOOLEAN:
return (bool(val), False)
elif pr == INTEGER:
return (int(val), False)
elif pr == TEXT:
return (val, False)
elif pr == FILE:
return (int(val), False)
elif pr == REFERENCE:
return (int(val), True)
elif pr == DATETIME:
return (val, False)
elif pr[0:4] == "LIST":
return self._type_converted_list(val, pr)
elif isinstance(val, Entity):
return (convert_to_python_object(val), False)
else:
return (int(val), True)
def attribute_as_list(self, name):
"""This is a workaround for the problem that lists containing only one
element are indistinguishable from simple types in this
representation."""
att = self.__getattribute__(name)
if isinstance(att, list):
return att
else:
return [att]
def _add_parent(self, parent):
self._parents.append(parent.id)
def add_parent(self, parent_id=None, parent_name=None):
if parent_id is not None:
self._parents.append(parent_id)
elif parent_name is not None:
self._parents.append(parent_name)
else:
raise ValueError("no parent identifier supplied")
def get_parent_names(self):
new_plist = []
for p in self._parents:
obj_type = get_type_of_entity_with(p)
ent = obj_type(id=p).retrieve()
new_plist.append(ent.name)
return new_plist
def resolve_references(self, deep=False, visited=dict()):
for i in self._references:
if isinstance(self._references[i], list):
for j in range(len(self._references[i])):
new_id = self._references[i][j]
obj_type = get_type_of_entity_with(new_id)
if new_id in visited:
new_object = visited[new_id]
else:
ent = obj_type(id=new_id).retrieve()
new_object = convert_to_python_object(ent)
visited[new_id] = new_object
if deep:
new_object.resolve_references(deep, visited)
self.__getattribute__(i)[j] = new_object
else:
new_id = self._references[i]
obj_type = get_type_of_entity_with(new_id)
if new_id in visited:
new_object = visited[new_id]
else:
ent = obj_type(id=new_id).retrieve()
new_object = convert_to_python_object(ent)
visited[new_id] = new_object
if deep:
new_object.resolve_references(deep, visited)
self.__setattr__(i, new_object)
def __str__(self, indent=1, name=None):
if name is None:
result = str(self.__class__.__name__) + "\n"
else:
result = name + "\n"
for p in self._properties:
value = self.__getattribute__(p)
if isinstance(value, CaosDBPythonEntity):
result += indent * "\t" + \
value.__str__(indent=indent + 1, name=p)
else:
result += indent * "\t" + p + "\n"
return result
class CaosDBPythonRecord(CaosDBPythonEntity):
pass
class CaosDBPythonRecordType(CaosDBPythonEntity):
pass
class CaosDBPythonProperty(CaosDBPythonEntity):
pass
class CaosDBPythonFile(CaosDBPythonEntity):
def get_File(self, target=None):
f = File(id=self._id).retrieve()
self._file = f.download(target)
def _single_convert_to_python_object(robj, entity):
robj._id = entity.id
for i in entity.properties:
robj._set_property_from_entity(i)
for i in entity.parents:
robj._add_parent(i)
if entity.path is not None:
robj._path = entity.path
if entity.file is not None:
robj._file = entity.file
# if entity.pickup is not None:
# robj.pickup = entity.pickup
return robj
def _single_convert_to_entity(entity, robj, **kwargs):
if robj._id is not None:
entity.id = robj._id
if robj._path is not None:
entity.path = robj._path
if robj._file is not None:
entity.file = robj._file
def retrieve_entity_with_id(eid):
return execute_query("FIND ENTITY WITH ID={}".format(eid), unique=True)
if robj.pickup is not None:
entity.pickup = robj.pickup
children = []
for parent in robj._parents:
if sys.version_info[0] < 3:
if hasattr(parent, "encode"):
entity.add_parent(name=parent)
else:
entity.add_parent(id=parent)
else:
if hasattr(parent, "encode"):
entity.add_parent(name=parent)
else:
entity.add_parent(id=parent)
def add_property(entity, prop, name, recursive=False, datatype=None):
if datatype is None:
raise ArgumentError("datatype must not be None")
def retrieve_entities_with_ids(entities):
collection = Container()
step = 20
if isinstance(prop, CaosDBPythonEntity):
entity.add_property(name=name, value=str(
prop._id), datatype=datatype)
for i in range(len(entities)//step+1):
collection.extend(
execute_query(
create_id_query(entities[i*step:(i+1)*step])))
if recursive and not prop.do_not_expand:
return convert_to_entity(prop, recursive=recursive)
else:
return []
else:
if isinstance(prop, float) or isinstance(prop, int):
prop = str(prop)
entity.add_property(name=name, value=prop, datatype=datatype)
return []
for prop in robj._properties:
value = robj.__getattribute__(prop)
if isinstance(value, list):
if robj._datatypes[prop][0:4] == "LIST":
lst = []
for v in value:
if isinstance(v, CaosDBPythonEntity):
lst.append(v._id)
if recursive and not v.do_not_expand:
children.append(convert_to_entity(
v, recursive=recursive))
else:
if isinstance(v, float) or isinstance(v, int):
lst.append(str(v))
else:
lst.append(v)
entity.add_property(name=prop, value=lst,
datatype=robj._datatypes[prop])
else:
for v in value:
children.extend(
add_property(
entity,
v,
prop,
datatype=robj._datatypes[prop],
**kwargs))
else:
children.extend(
add_property(
entity,
value,
prop,
datatype=robj._datatypes[prop],
**kwargs))
return [entity] + children
def convert_to_entity(python_object, **kwargs):
warnings.warn("The convert_to_entity function is deprecated, replacement will be provided by "
"the high_level_api module.", DeprecationWarning, stacklevel=2)
if isinstance(python_object, Container):
# Create a list of objects:
return [convert_to_python_object(i, **kwargs) for i in python_object]
elif isinstance(python_object, CaosDBPythonRecord):
return _single_convert_to_entity(Record(), python_object, **kwargs)
elif isinstance(python_object, CaosDBPythonFile):
return _single_convert_to_entity(File(), python_object, **kwargs)
elif isinstance(python_object, CaosDBPythonRecordType):
return _single_convert_to_entity(RecordType(), python_object, **kwargs)
elif isinstance(python_object, CaosDBPythonProperty):
return _single_convert_to_entity(Property(), python_object, **kwargs)
elif isinstance(python_object, CaosDBPythonEntity):
return _single_convert_to_entity(Entity(), python_object, **kwargs)
else:
raise ValueError("Cannot convert an object of this type.")
def convert_to_python_object(entity):
""""""
warnings.warn("The convert_to_python_object function is deprecated, replacement will be "
"provided by the high_level_api module.", DeprecationWarning, stacklevel=2)
if isinstance(entity, Container):
# Create a list of objects:
return [convert_to_python_object(i) for i in entity]
elif isinstance(entity, Record):
return _single_convert_to_python_object(CaosDBPythonRecord(), entity)
elif isinstance(entity, RecordType):
return _single_convert_to_python_object(
CaosDBPythonRecordType(), entity)
elif isinstance(entity, File):
return _single_convert_to_python_object(CaosDBPythonFile(), entity)
elif isinstance(entity, Property):
return _single_convert_to_python_object(CaosDBPythonProperty(), entity)
elif isinstance(entity, Entity):
return _single_convert_to_python_object(CaosDBPythonEntity(), entity)
else:
raise ValueError("Cannot convert an object of this type.")
return collection
def getOriginUrlIn(folder):
......@@ -584,8 +205,8 @@ def compare_entities(old_entity: Entity, new_entity: Entity):
In case of changed information the value listed under the respective key shows the
value that is stored in the respective entity.
"""
olddiff = {"properties": {}, "parents": []}
newdiff = {"properties": {}, "parents": []}
olddiff: dict[str, Any] = {"properties": {}, "parents": []}
newdiff: dict[str, Any] = {"properties": {}, "parents": []}
if old_entity is new_entity:
return (olddiff, newdiff)
......@@ -844,3 +465,26 @@ def resolve_reference(prop: Property):
else:
if isinstance(prop.value, int):
prop.value = retrieve_entity_with_id(prop.value)
def create_flat_list(ent_list: list[Entity], flat: list[Entity]):
"""
Recursively adds all properties contained in entities from ent_list to
the output list flat. Each element will only be added once to the list.
TODO: Currently this function is also contained in newcrawler module crawl.
We are planning to permanently move it to here.
"""
for ent in ent_list:
for p in ent.properties:
# For lists append each element that is of type Entity to flat:
if isinstance(p.value, list):
for el in p.value:
if isinstance(el, Entity):
if el not in flat:
flat.append(el)
create_flat_list([el], flat) # TODO: move inside if block?
elif isinstance(p.value, Entity):
if p.value not in flat:
flat.append(p.value)
create_flat_list([p.value], flat) # TODO: move inside if block?
src/caosdb/high_level_api.py 0 → 100644
+ 1025
− 0
View file @ d355c12a
This diff is collapsed.
src/caosdb/utils/plantuml.py
+ 8
− 0
View file @ d355c12a
......@@ -296,6 +296,14 @@ def retrieve_substructure(start_record_types, depth, result_id_set=None, result_
name=get_referenced_recordtype(prop.datatype)).retrieve()
retrieve_substructure([rt], depth-1, result_id_set,
result_container, False)
# TODO: clean up this hack
# TODO: make it also work for files
if is_reference(prop.datatype) and prop.value is not None:
r = db.Record(id=prop.value).retrieve()
retrieve_substructure([r], depth-1, result_id_set, result_container, False)
if r.id not in result_id_set:
result_container.append(r)
result_id_set.add(r.id)
if prop.id not in result_id_set:
result_container.append(prop)
......
src/doc/future_caosdb.md 0 → 100644
+ 112
− 0
View file @ d355c12a
# The future of the CaosDB Python Client
The current Python client has done us great services but its structure and the
way it is used sometimes feels outdated and clumsy. In this document we sketch
how it might look different in future and invite everyone to comment or
contribute to this development.
At several locations in this document there will be links to discussion issues.
If you want to discuss something new, you can create a new issue
[here](https://gitlab.com/caosdb/caosdb-pylib/-/issues/new).
## Overview
Let's get a general impression before discussing single aspects.
``` python
import caosdb as db
experiments = db.query("FIND Experiment")
# print name and date for each `Experiment`
for exp in experiments:
print(exp.name, exp.date)
# suppose `Experiments` reference `Projects` which have a `Funding` Property
one_exp = experiments[0]
print(one_exp.Project.Funding)
new_one = db.create_record("Experiment")
new_one.date = "2022-01-01"
new_one.name = "Needle Measurement"
new_one.insert()
```
Related discussions:
- [recursive retrieve in query](https://gitlab.com/caosdb/caosdb-pylib/-/issues/57)
- [create_record function](https://gitlab.com/caosdb/caosdb-pylib/-/issues/58)
- [data model utility](https://gitlab.com/caosdb/caosdb-pylib/-/issues/59)
## Quickstart
Note that you can try out one possible implementation using the
`caosdb.high_level_api` module. It is experimental and might be removed in
future!
A `resolve_references` function allows to retrieve the referenced entities of
an entity, container or a query result set (which is a container).
It has the following parameters which can also be supplied to the `query`
function:
- `deep`: Whether to use recursive retrieval
- `depth`: Maximum recursion depth
- `references`: Whether to use the supplied db.Container to resolve
references. This allows offline usage. Set it to None if you want to
automatically retrieve entities from the current CaosDB connection.
In order to allow a quick look at the object structures an easily readable
serialization is provided by the `to_dict` function. It has the following
argument:
- `without_metadata`: Set this to True if you don\'t want to see
property metadata like \"unit\" or \"importance\".
This function creates a simple dictionary containing a representation of
the entity, which can be stored to disk and completely deserialized
using the function `from_dict`.
Furthermore, the `__str__` function uses this to display objects in yaml
format by default statement
## Design Decisions
### Dot Notation
Analogue, to what Pandas does. Provide bracket notation
`rec.properties["test"]` for Properties with names that are in conflict with
default attributes or contain spaces (or other forbidden characters).
- Raise Exception if attribute does not exist but is accessed?
- How to deal with lists? b has a list as value: `a.b[0].c = 5`
[Discussion](https://gitlab.com/caosdb/caosdb-pylib/-/issues/60)
### Serialization
What information needs to be contained in (meta)data? How compatible is it with
GRPC json serialization?
### Recursive Retrieval
I can resolve later and end up with the same result:
`recs =db.query("FIND Experiment", depth=2)` equals `recs = db.query("FIND Experiment"); recs = resolve_references(recs, depth=2)`
[Discussion](https://gitlab.com/caosdb/caosdb-pylib/-/issues/57)
### In-Place operations
Default behavior is to return new objects instead of modifying them in-place.
This can be changed with the argument `inplace=True`.
Especially the following functions operate by default NOT in-place:
- update
- insert
- retrieve
- resolve_references
[Discussion](https://gitlab.com/caosdb/caosdb-pylib/-/issues/61)
## Extend Example
``` python
import caosdb as db
dm = db.get_data_model()
new_one = db.create_record(dm.Experiment)
new_one.date = "2022-01-01"
new_one.name = "Needle Measurement"
new_one.dataset = db.create_record(dm.Dataset)
new_one.dataset.voltage = (5, "V")
new_one.dataset.pulses = [5, 5.3]
inserted = new_one.insert()
print("The new record has the ID:", inserted.id)
```
src/doc/high_level_api.org 0 → 100644
+ 171
− 0
View file @ d355c12a
* High Level API
In addition to the old standard pylib API, new versions of pylib ship with a high level API
that facilitates usage of CaosDB entities within data analysis scripts. In a nutshell that
API exposes all properties of CaosDB Records as standard python attributes making their
access easier.
Or to spell it out directly in Python:
#+BEGIN_SRC python
import caosdb as db
# Old API:
r = db.Record()
r.add_parent("Experiment")
r.add_property(name="alpha", value=5)
r.get_property("alpha").value = 25 # setting properties (old api)
print(r.get_property("alpha").value + 25) # getting properties (old api)
from caosdb.high_level_api import convert_to_python_entity
obj = convert_to_python_object(r) # create a high level entity
obj.r = 25 # setting properties (new api)
print(obj.r + 25) # getting properties (new api)
#+END_SRC
** Quickstart
The module, needed for the high level API is called:
~caosdb.high_level_api~
There are two functions converting entities between the two representation (old API and new API):
- ~convert_to_python_object~: Convert entities from **old** into **new** representation.
- ~convert_to_entity~: Convert entities from **new** into **old** representation.
Furthermore there are a few utility functions which expose very practical shorthands:
- ~new_high_level_entity~: Retrieve a record type and create a new high level entity which contains properties of a certain importance level preset.
- ~create_record~: Create a new high level entity using the name of a record type and a list of key value pairs as properties.
- ~load_external_record~: Retrieve a record with a specific name and return it as high level entity.
- ~create_entity_container~: Convert a high level entity into a standard entity including all sub entities.
- ~query~: Do a CaosDB query and return the result as a container of high level objects.
So as a first example, you could retrieve any record from CaosDB and use it using its high level representation:
#+BEGIN_SRC python
from caosdb.high_level_api import query
res = query("FIND Record Experiment")
experiment = res[0]
# Use a property:
print(experiment.date)
# Use sub properties:
print(experiment.output[0].path)
#+END_SRC
The latter example demonstrates, that the function query is very powerful. For its default parameter
values it automatically resolves and retrieves references recursively, so that sub properties,
like the list of output files "output", become immediately available.
**Note** that for the old API you were supposed to run the following series of commands
to achieve the same result:
#+BEGIN_SRC python
import caosdb as db
res = db.execute_query("FIND Record Experiment")
output = res.get_property("output")
output_file = db.File(id=output.value[0].id).retrieve()
print(output_file.path)
#+END_SRC
Resolving subproperties makes use of the "resolve_reference" function provided by the high level
entity class (~CaosDBPythonEntity~), with the following parameters:
- ~deep~: Whether to use recursive retrieval
- ~references~: Whether to use the supplied db.Container to resolve references. This allows offline usage. Set it to None if you want to automatically retrieve entities from the current CaosDB connection.
- ~visited~: Needed for recursion, set this to None.
Objects in the high level representation can be serialized to a simple yaml form using the function
~serialize~ with the following parameters:
- ~without_metadata~: Set this to True if you don't want to see property metadata like "unit" or "importance".
- ~visited~: Needed for recursion, set this to None.
This function creates a simple dictionary containing a representation of the entity, which can be
stored to disk and completely deserialized using the function ~deserialize~.
Furthermore the "__str__" function is overloaded, so that you can use print to directly inspect
high level objects using the following statement:
#+BEGIN_SRC python
print(str(obj))
#+END_SRC
** Concepts
As described in the section [[Quickstart]] the two functions ~convert_to_python_object~ and ~convert_to_entity~ convert
entities beetween the high level and the standard representation.
The high level entities are represented using the following classes from the module ~caosdb.high_level_api~:
- ~CaosDBPythonEntity~: Base class of the following entity classes.
- ~CaosDBPythonRecord~
- ~CaosDBPythonRecordType~
- ~CaosDBPythonProperty~
- ~CaosDBPythonMultiProperty~: **WARNING** Not implemented yet.
- ~CaosDBPythonFile~: Used for file entities and provides an additional ~download~ function for being able to directly retrieve files from CaosDB.
In addition, there are the following helper structures which are realized as Python data classes:
- ~CaosDBPropertyMetaData~: For storing meta data about properties.
- ~CaosDBPythonUnresolved~: The base class of unresolved "things".
- ~CaosDBPythonUnresolvedParent~: Parents of entities are stored as unresolved parents by default, storing an id or a name of a parent (or both).
- ~CaosDBPythonUnresolvedReference~: An unresolved reference is a reference property with an id which has not (yet) been resolved to an Entity.
The function "resolve_references" can be used to recursively replace ~CaosDBPythonUnresolvedReferences~ into members of type ~CaosDBPythonRecords~
or ~CaosDBPythonFile~.
Each property stored in a CaosDB record corresponds to:
- a member attribute of ~CaosDBPythonRecord~ **and**
- an entry in a dict called "metadata" storing a CaosDBPropertyMetadata object with the following information about proeprties:
- ~unit~
- ~datatype~
- ~description~
- ~id~
- ~importance~
* Example
The following shows a more complex example taken from a real world use case:
A numerical experiment is created to simulate cardiac electric dynamics. The physical problem
is modelled using the monodomain equation with the local current term given by the Mitchell
Schaeffer Model.
The data model for the numerical experiment consists of multiple record types which stores assosciated paremeters:
- `MonodomainTissueSimulation`
- `MitchellSchaefferModel`
- `SpatialExtent2d`
- `SpatialDimension`
- `ConstantTimestep`
- `ConstantDiffusion`
First, the data model will be filled with the parameter values for this specific simulation run. It will be stored in the python variable `MonodomainRecord`. Passing the `MonodomainRecord` through the python functions, the simulation parameters can be easily accessed everywhere in the code when needed.
Records are created by the `create_record` function. Parameter values can be set at record creation and also after creation as python properties of the corresponding record instance. The following example shows how to create a record, how to set the parameter at creation and how to set them as python properties
#+BEGIN_SRC python
from caosdb.high_level_api import create_record
MonodomainRecord = create_record("MonodomainTissueSimulation")
MonodomainRecord.LocalModel = create_record("MitchellSchaefferModel")
MonodomainRecord.SpatialExtent = create_record(
"SpatialExtent2d", spatial_extent_x=100, spatial_extent_y=100)
MonodomainRecord.SpatialExtent.cell_sizes = [0.1, 0.1] # parameters can be set as properties
MonodomainRecord.SpatialDimension = create_record("SpatialDimension",
num_dim=2)
MonodomainRecord.TimestepInformation = create_record("ConstantTimestep")
MonodomainRecord.TimestepInformation.DeltaT = 0.1
D = create_record("ConstantDiffusion", diffusion_constant=0.1)
MonodomainRecord.DiffusionConstantType = D
model = MonodomainRecord.LocalModel
model.t_close = 150
model.t_open = 120
model.t_out = 6
model.t_in = 0.3
model.v_gate = 0.13
model.nvars = 2
#+END_SRC
At any position in the algorithm you are free to:
- Convert this model to the standard python API and insert or update the records in a running instance of CaosDB.
- Serialize this model in the high level API yaml format. This enables the CaosDB crawler to pickup the file and synchronize it with a running instance
of CaosDB.
src/doc/high_level_api.rst 0 → 100644
+ 163
− 0
View file @ d355c12a
High Level API
==============
In addition to the old standard pylib API, new versions of pylib ship
with a high level API that facilitates usage of CaosDB entities within
data analysis scripts. In a nutshell that API exposes all properties of
CaosDB Records as standard python attributes making their access easier.
Or to speak it out directly in Python:
.. code:: python
import caosdb as db
# Old API:
r = db.Record()
r.add_parent("Experiment")
r.add_property(name="alpha", value=5)
r.get_property("alpha").value = 25 # setting properties (old api)
print(r.get_property("alpha").value + 25) # getting properties (old api)
from caosdb.high_level_api import convert_to_python_entity
obj = convert_to_python_object(r) # create a high level entity
obj.r = 25 # setting properties (new api)
print(obj.r + 25) # getting properties (new api)
Quickstart
----------
The module, needed for the high level API is called:
``caosdb.high_level_api``
There are two functions converting entities between the two
representation (old API and new API):
- ``convert_to_python_object``: Convert entities from **old** into
**new** representation.
- ``convert_to_entity``: Convert entities from **new** into **old**
representation.
Furthermore there are a few utility functions which expose very
practical shorthands:
- ``new_high_level_entity``: Retrieve a record type and create a new
high level entity which contains properties of a certain importance
level preset.
- ``create_record``: Create a new high level entity using the name of a
record type and a list of key value pairs as properties.
- ``load_external_record``: Retrieve a record with a specific name and
return it as high level entity.
- ``create_entity_container``: Convert a high level entity into a
standard entity including all sub entities.
- ``query``: Do a CaosDB query and return the result as a container of
high level objects.
So as a first example, you could retrieve any record from CaosDB and use
it using its high level representation:
.. code:: python
from caosdb.high_level_api import query
res = query("FIND Record Experiment")
experiment = res[0]
# Use a property:
print(experiment.date)
# Use sub properties:
print(experiment.output[0].path)
The latter example demonstrates, that the function query is very
powerful. For its default parameter values it automatically resolves and
retrieves references recursively, so that sub properties, like the list
of output files "output", become immediately available.
**Note** that for the old API you were supposed to run the following
series of commands to achieve the same result:
.. code:: python
import caosdb as db
res = db.execute_query("FIND Record Experiment")
output = res.get_property("output")
output_file = db.File(id=output.value[0].id).retrieve()
print(output_file.path)
Resolving subproperties makes use of the "resolve\ :sub:`reference`"
function provided by the high level entity class
(``CaosDBPythonEntity``), with the following parameters:
- ``deep``: Whether to use recursive retrieval
- ``references``: Whether to use the supplied db.Container to resolve
references. This allows offline usage. Set it to None if you want to
automatically retrieve entities from the current CaosDB connection.
- ``visited``: Needed for recursion, set this to None.
Objects in the high level representation can be serialized to a simple
yaml form using the function ``serialize`` with the following
parameters:
- ``without_metadata``: Set this to True if you don't want to see
property metadata like "unit" or "importance".
- ``visited``: Needed for recursion, set this to None.
This function creates a simple dictionary containing a representation of
the entity, which can be stored to disk and completely deserialized
using the function ``deserialize``.
Furthermore the "*str*" function is overloaded, so that you can use
print to directly inspect high level objects using the following
statement:
.. code:: python
print(str(obj))
Concepts
--------
As described in the section Quickstart the two functions
``convert_to_python_object`` and ``convert_to_entity`` convert entities
beetween the high level and the standard representation.
The high level entities are represented using the following classes from
the module ``caosdb.high_level_api``:
- ``CaosDBPythonEntity``: Base class of the following entity classes.
- ``CaosDBPythonRecord``
- ``CaosDBPythonRecordType``
- ``CaosDBPythonProperty``
- ``CaosDBPythonMultiProperty``: **WARNING** Not implemented yet.
- ``CaosDBPythonFile``: Used for file entities and provides an
additional ``download`` function for being able to directly retrieve
files from CaosDB.
In addition, there are the following helper structures which are
realized as Python data classes:
- ``CaosDBPropertyMetaData``: For storing meta data about properties.
- ``CaosDBPythonUnresolved``: The base class of unresolved "things".
- ``CaosDBPythonUnresolvedParent``: Parents of entities are stored as
unresolved parents by default, storing an id or a name of a parent
(or both).
- ``CaosDBPythonUnresolvedReference``: An unresolved reference is a
reference property with an id which has not (yet) been resolved to an
Entity.
The function "resolve\ :sub:`references`" can be used to recursively
replace ``CaosDBPythonUnresolvedReferences`` into members of type
``CaosDBPythonRecords`` or ``CaosDBPythonFile``.
Each property stored in a CaosDB record corresponds to:
- a member attribute of ``CaosDBPythonRecord`` **and**
- an entry in a dict called "metadata" storing a CaosDBPropertyMetadata
object with the following information about proeprties:
- ``unit``
- ``datatype``
- ``description``
- ``id``
- ``importance``
src/doc/index.rst
+ 1
− 0
View file @ d355c12a
......@@ -12,6 +12,7 @@ Welcome to PyCaosDB's documentation!
Concepts <concepts>
Configuration <configuration>
Administration <administration>
High Level API <high_level_api>
Code gallery <gallery/index>
API documentation<_apidoc/caosdb>
......
tox.ini
+ 1
− 0
View file @ d355c12a
......@@ -7,6 +7,7 @@ deps = .
nose
pytest
pytest-cov
python-dateutil
jsonschema==4.0.1
commands=py.test --cov=caosdb -vv {posargs}
......
unittests/test_apiutils.py
+ 1
− 20
View file @ d355c12a
......@@ -25,9 +25,8 @@
# Test apiutils
# A. Schlemmer, 02/2018
import pickle
import tempfile
import pytest
import caosdb as db
import caosdb.apiutils
from caosdb.apiutils import (apply_to_ids, compare_entities, create_id_query,
......@@ -35,24 +34,6 @@ from caosdb.apiutils import (apply_to_ids, compare_entities, create_id_query,
from caosdb.common.models import SPECIAL_ATTRIBUTES
from .test_property import testrecord
import pytest
def test_convert_object():
r2 = db.apiutils.convert_to_python_object(testrecord)
assert r2.species == "Rabbit"
def test_pickle_object():
r2 = db.apiutils.convert_to_python_object(testrecord)
with tempfile.TemporaryFile() as f:
pickle.dump(r2, f)
f.seek(0)
rn2 = pickle.load(f)
assert r2.date == rn2.date
def test_apply_to_ids():
parent = db.RecordType(id=3456)
......
unittests/test_entity.py
+ 4
− 1
View file @ d355c12a
......@@ -26,10 +26,13 @@
import unittest
from lxml import etree
import os
from caosdb import (INTEGER, Entity, Property, Record, RecordType,
configure_connection)
from caosdb.connection.mockup import MockUpServerConnection
UNITTESTDIR = os.path.dirname(os.path.abspath(__file__))
class TestEntity(unittest.TestCase):
......@@ -87,7 +90,7 @@ class TestEntity(unittest.TestCase):
"""
parser = etree.XMLParser(remove_comments=True)
entity = Entity._from_xml(Entity(),
etree.parse("unittests/test_record.xml",
etree.parse(os.path.join(UNITTESTDIR, "test_record.xml"),
parser).getroot())
self.assertEqual(entity.role, "Record")
......
unittests/test_high_level_api.py 0 → 100644
+ 643
− 0
View file @ d355c12a
This diff is collapsed.
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment