diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 5741684242954b0e9b23f4d2c40ca862f17a9570..1db4c0394de48cbd82d5abbe4d7c581abcb9b277 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -19,6 +19,7 @@
# along with this program. If not, see <https://www.gnu.org/licenses/>.
variables:
+ DEPLOY_REF: dev
CI_REGISTRY_IMAGE: $CI_REGISTRY/caosdb/caosdb-pylib/testenv:latest
# When using dind, it's wise to use the overlayfs driver for
# improved performance.
@@ -63,10 +64,11 @@ trigger_build:
script:
- /usr/bin/curl -X POST
-F token=$DEPLOY_TRIGGER_TOKEN
+ -F "variables[F_BRANCH]=$CI_COMMIT_REF_NAME"
-F "variables[PYLIB]=$CI_COMMIT_REF_NAME"
-F "variables[TriggerdBy]=PYLIB"
-F "variables[TriggerdByHash]=$CI_COMMIT_SHORT_SHA"
- -F ref=dev https://gitlab.indiscale.com/api/v4/projects/14/trigger/pipeline
+ -F ref=$DEPLOY_REF https://gitlab.indiscale.com/api/v4/projects/14/trigger/pipeline
# Build a docker image in which tests for this repository can run
build-testenv:
diff --git a/CHANGELOG.md b/CHANGELOG.md
index aa52d2b98fef19a8b1daffde3fb76e9107b6fa13..6428cc07799338f8b17fa3ce83332eec5e0fe9d7 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -9,6 +9,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
### Added ###
+* `[Entity|Container].get_property_values` for deeply nested references, e.g.
+ from results of SELECT queries.
* two new `password_method`s for the `pycaosdb.ini` and the
`configure_connection` function: `unauthenticated` for staying
unauthenticated (and using the anonymous user) and `auth_token`. If
diff --git a/src/caosdb/common/models.py b/src/caosdb/common/models.py
index 0134deef6ac071f044e261947a9dfaa0594d1d33..be6dfdcfd8ce5c90929785a3649da9b745868861 100644
--- a/src/caosdb/common/models.py
+++ b/src/caosdb/common/models.py
@@ -5,6 +5,8 @@
#
# Copyright (C) 2018 Research Group Biomedical Physics,
# Max-Planck-Institute for Dynamics and Self-Organization Göttingen
+# Copyright (C) 2020 Timm Fitschen <t.fitschen@indiscale.com>
+# Copyright (C) 2020 IndiScale GmbH <info@indiscale.com>
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as
@@ -46,10 +48,10 @@ from caosdb.connection.connection import get_connection
from caosdb.connection.encode import MultipartParam, multipart_encode
from caosdb.exceptions import (AmbiguityException, AuthorizationException,
CaosDBException, ConnectionException,
- ConsistencyError,
- EntityDoesNotExistError, EntityError,
- EntityHasNoDatatypeError, TransactionError,
- UniqueNamesError, UnqualifiedParentsError,
+ ConsistencyError, EntityDoesNotExistError,
+ EntityError, EntityHasNoDatatypeError,
+ TransactionError, UniqueNamesError,
+ UnqualifiedParentsError,
UnqualifiedPropertiesError, URITooLongException)
from lxml import etree
@@ -368,7 +370,7 @@ class Entity(object):
del copy_kwargs['importance']
del copy_kwargs['inheritance']
- new_property = Property(value=value, **copy_kwargs)
+ new_property = Property(**copy_kwargs)
if abstract_property is not None:
new_property._wrap(property)
@@ -378,6 +380,8 @@ class Entity(object):
if new_property.datatype is None and isinstance(
property, (RecordType, Record, File)):
new_property.datatype = property
+ new_property.value = value
+
self.properties.append(
property=new_property, importance=(
kwargs['importance'] if 'importance' in kwargs else None), inheritance=(
@@ -606,6 +610,103 @@ class Entity(object):
return None
+ def _get_value_for_selector(self, selector):
+ """return the value described by the selector
+
+ A selector is a list or a tuple of strings describing a path in an
+ entity tree with self as root. The last selector may be a special one
+ like unit or name.
+
+ See also get_property_values()
+ """
+ SPECIAL_SELECTORS = ["unit", "value", "description", "id", "name"]
+
+ if not isinstance(selector, (tuple, list)):
+ selector = [selector]
+
+ val = None
+ ref = self
+
+ # there are some special selectors which can be applied to the
+ # final element; if such a special selector exists we split it
+ # from the list
+
+ if selector[-1].lower() in SPECIAL_SELECTORS:
+ special_selector = selector[-1]
+ selector = selector[:-1]
+ else:
+ special_selector = None
+
+ # iterating through the entity tree according to the selector
+ for subselector in selector:
+ # selector does not match the structure, we cannot get a
+ # property of non-entity
+
+ if not isinstance(ref, Entity):
+ return None
+
+ prop = ref.get_property(subselector)
+
+ # selector does not match the structure, we did not get a
+ # property
+ if prop is None:
+ return None
+
+ # if the property is a reference, we are interested in the
+ # corresponding entities attributes
+ if isinstance(prop.value, Entity):
+ ref = prop.value
+
+ # otherwise in the attributes of the property
+ else:
+ ref = prop
+
+ # if we saved a special selector before, apply it
+ if special_selector is None:
+ return prop.value
+ else:
+ return getattr(ref, special_selector.lower())
+
+ def get_property_values(self, *selectors):
+ """ Return a tuple with the values described by the given selectors.
+
+ This represents an entity's properties as if it was a row of a table
+ with the given columns.
+
+ If the elements of the selectors parameter are tuples, they will return
+ the properties of the referenced entity, if present. E.g. ("window",
+ "height") will return the value of the height property of the
+ referenced window entity.
+
+ The tuple's values correspond to the order of selectors parameter.
+
+ The tuple contains None for all values that are not available in the
+ entity. That does not necessarily mean, that the values are not stored
+ in the database (e.g. if a single entity was retrieved without
+ referenced entities).
+
+ Parameters
+ ----------
+ *selectors : str or tuple of str
+ Each selector is a list or tuple of property names, e.g. `"height",
+ "width"`.
+
+ Returns
+ -------
+ row : tuple
+ A row-like representation of the entity's properties.
+ """
+ row = tuple()
+
+ for selector in selectors:
+ val = self._get_value_for_selector(selector)
+
+ if isinstance(val, Entity):
+ val = val.id if val.id is not None else val.name
+ row += (val,)
+
+ return row
+
def get_messages(self):
"""Get all messages of this entity.
@@ -834,17 +935,27 @@ class Entity(object):
entity.add_message(child)
elif child is None or hasattr(child, "encode"):
vals.append(child)
+ elif isinstance(child, Entity):
+ vals.append(child)
else:
raise TypeError(
'Child was neither a Property, nor a Parent, nor a Message.\
Was ' + str(type(child)))
- # parse VALUE
+ # add VALUE
+ value = None
+
if len(vals):
# The value[s] have been inside a <Value> tag.
- entity.value = vals
+ value = vals
elif elem.text is not None and elem.text.strip() != "":
- entity.value = elem.text.strip()
+ value = elem.text.strip()
+
+ try:
+ entity.value = value
+ except ValueError:
+ # circumvent the parsing.
+ entity.__value = value
return entity
@@ -993,12 +1104,16 @@ class Entity(object):
def _parse_value(datatype, value):
if value is None:
return value
+
if datatype is None:
return value
+
if datatype == DOUBLE:
return float(value)
+
if datatype == INTEGER:
return int(str(value))
+
if datatype == BOOLEAN:
if str(value).lower() == "true":
return True
@@ -1006,14 +1121,17 @@ def _parse_value(datatype, value):
return False
else:
raise ValueError("Boolean value was {}.".format(value))
+
if datatype in [DATETIME, TEXT]:
if isinstance(value, str):
return value
# deal with collections
+
if isinstance(datatype, str):
matcher = re.compile(r"^(?P<col>[^<]+)<(?P<dt>[^>]+)>$")
m = matcher.match(datatype)
+
if m:
col = m.group("col")
dt = m.group("dt")
@@ -1034,14 +1152,18 @@ def _parse_value(datatype, value):
# This is for a special case, where the xml parser could not differentiate
# between single values and lists with one element. As
+
if hasattr(value, "__len__") and len(value) == 1:
return _parse_value(datatype, value[0])
# deal with references
+
if isinstance(value, Entity):
return value
+
if isinstance(value, str) and "@" in value:
# probably this is a versioned reference
+
return str(value)
else:
# for unversioned references
@@ -1049,6 +1171,7 @@ def _parse_value(datatype, value):
return int(value)
except ValueError:
# reference via name
+
return str(value)
@@ -3115,6 +3238,43 @@ class Container(list):
return self
+ def get_property_values(self, *selectors):
+ """ Return a list of tuples with values of the given selectors.
+
+ I.e. a tabular representation of the container's content.
+
+ If the elements of the selectors parameter are tuples, they will return
+ the properties of the referenced entity, if present. E.g. ("window",
+ "height") will return the value of the height property of the
+ referenced window entity.
+
+ All tuples of the returned list have the same length as the selectors
+ parameter and the ordering of the tuple's values correspond to the
+ order of the parameter as well.
+
+ The tuple contains None for all values that are not available in the
+ entity. That does not necessarily mean, that the values are not stored
+ in the database (e.g. if a single entity was retrieved without
+ referenced entities).
+
+ Parameters
+ ----------
+ *selectors : str or tuple of str
+ Each selector is a list or tuple of property names, e.g. `"height",
+ "width"`.
+
+ Returns
+ -------
+ table : list of tuples
+ A tabular representation of the container's content.
+ """
+ table = []
+
+ for e in self:
+ table.append(e.get_property_values(*selectors))
+
+ return table
+
def sync_global_acl():
c = get_connection()
@@ -3546,6 +3706,7 @@ class Info():
http_response = c.retrieve(["Info"])
except ConnectionException as conn_e:
print(conn_e)
+
return
xml = etree.fromstring(http_response.read())
@@ -3663,6 +3824,7 @@ def _parse_single_xml_element(elem):
return ""
elif elem.text is None or elem.text.strip() == "":
return None
+
return str(elem.text.strip())
elif elem.tag.lower() == "querytemplate":
return QueryTemplate._from_xml(elem)
diff --git a/unittests/test_container.py b/unittests/test_container.py
new file mode 100644
index 0000000000000000000000000000000000000000..b34055372fc83a5608ffcf54423a601001add12b
--- /dev/null
+++ b/unittests/test_container.py
@@ -0,0 +1,79 @@
+
+# -*- encoding: utf-8 -*-
+#
+# ** header v3.0
+# This file is a part of the CaosDB Project.
+#
+# Copyright (C) 2020 Timm Fitschen <t.fitschen@indiscale.com>
+# Copyright (C) 2020 IndiScale GmbH <info@indiscale.com>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see <https://www.gnu.org/licenses/>.
+#
+# ** end header
+#
+"""Tests for the Container class."""
+from __future__ import absolute_import
+
+import caosdb as c
+
+
+def test_get_property_values():
+ rt_house = c.RecordType("House")
+ rt_window = c.RecordType("Window")
+ rt_owner = c.RecordType("Owner")
+ p_height = c.Property("Height", datatype=c.DOUBLE)
+
+ window = c.Record().add_parent(rt_window)
+ window.id = 1001
+ window.add_property(p_height, 20.5, unit="m")
+
+ owner = c.Record("The Queen").add_parent(rt_owner)
+
+ house = c.Record("Buckingham Palace")
+ house.add_parent(rt_house)
+ house.add_property(rt_owner, owner)
+ house.add_property(rt_window, window)
+ house.add_property(p_height, 40.2, unit="ft")
+
+ container = c.Container()
+ container.extend([
+ house,
+ owner
+ ])
+
+ assert getattr(house.get_property(p_height), "unit") == "ft"
+ assert getattr(window.get_property(p_height), "unit") == "m"
+
+ table = container.get_property_values("naME",
+ "height",
+ ("height", "unit"),
+ "window",
+ ("window", "non-existing"),
+ ("window", "non-existing", "unit"),
+ ("window", "unit"),
+ ("window", "heiGHT"),
+ ("window", "heiGHT", "value"),
+ ("window", "heiGHT", "unit"),
+ "owner",
+ )
+ assert len(table) == 2
+ house_row = table[0]
+ assert house_row == (house.name, 40.2, "ft", window.id, None, None, None, 20.5, 20.5, "m", owner.name)
+
+ owner_row = table[1]
+ assert owner_row == (owner.name, None, None, None, None, None, None, None, None, None, None)
+
+ assert container.get_property_values("non-existing") == [(None,), (None,)]
+ assert container.get_property_values("name") == [(house.name,),
+ (owner.name,)]