diff --git a/.gitlab/merge_request_templates/Default.md b/.gitlab/merge_request_templates/Default.md
index 35c6d01c5904289b77fc7f1de9419ef91a1510e9..3629e0ca3695000863d8c254516f64bf59a7bf60 100644
--- a/.gitlab/merge_request_templates/Default.md
+++ b/.gitlab/merge_request_templates/Default.md
@@ -28,6 +28,7 @@ guidelines](https://gitlab.com/caosdb/caosdb/-/blob/dev/REVIEW_GUIDELINES.md)
- [ ] Up-to-date CHANGELOG.md (or not necessary)
- [ ] Up-to-date JSON schema (or not necessary)
- [ ] Appropriate user and developer documentation (or not necessary)
+ - Update / write published documentation (`make doc`).
- How do I use the software? Assume "stupid" users.
- How do I develop or debug the software? Assume novice developers.
- [ ] Annotations in code (Gitlab comments)
@@ -41,7 +42,8 @@ guidelines](https://gitlab.com/caosdb/caosdb/-/blob/dev/REVIEW_GUIDELINES.md)
- [ ] I understand the intent of this MR
- [ ] All automated tests pass
- [ ] Up-to-date CHANGELOG.md (or not necessary)
-- [ ] Appropriate user and developer documentation (or not necessary)
+- [ ] Appropriate user and developer documentation (or not necessary), also in published
+ documentation.
- [ ] The test environment setup works and the intended behavior is reproducible in the test
environment
- [ ] In-code documentation and comments are up-to-date.
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 3acce8439d9e3ce3d4a35a4806c72d2f9b16b2a2..111e4bc769a35a502a2a3bb33f6548511e9c7b46 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -8,19 +8,46 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
## [Unreleased] ##
### Added ###
+
+* `Entity.remove_value_from_property` function that removes a given value from a
+ property and optionally removes the property if it is empty afterwards.
+
+### Changed ###
+
+* `_Messages` is now `Messages` and inherits from list instead of dict
+* `Message.__init__` signature changed and `type` defaults to "Info" now.
+* `Message.__eq__` changed. Equality is equality of `type`, `code`, and
+ `description` now.
+
+### Deprecated ###
+
+* The API of Messages has been simplified and some ways to interact with
+ messages have been deprecated. Warnings are raised correspondingly.
+* `Message.get_code`. Use the `code` property instead.
+
+### Removed ###
+
+### Fixed ###
+
+- Detection for cyclic references when converting entites using the high level API.
+
+### Security ###
+
+### Documentation ###
+
+## [0.12.0] - 2023-06-02 ##
+
+### Added ###
+
- Added location argument to `src/caosdb/utils/checkFileSystemConsistency.py`
- Entity getters: `get_entity_by_<name/id/path>`
- Cached versions of entity getters and of `execute_query` (`cached_query`)
-### Changed ###
-
### Deprecated ###
- getOriginUrlIn, getDiffIn, getBranchIn, getCommitIn (formerly apiutils) have been
moved to caosdb.utils.git_utils
-### Removed ###
-
### Fixed ###
- Fixed `src/caosdb/utils/checkFileSystemConsistency.py`
@@ -28,7 +55,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
comparing two `Property` entities or in case of entities with multi
properties.
-### Security ###
+- Fixed `src/caosdb/utils/checkFileSystemConsistency.py`
### Documentation ###
@@ -39,7 +66,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
## [0.11.2] - 2023-03-14 ##
### Fixed ###
-- root logger is no longer used to create warnings. Fixes undesired output in
+- root logger is no longer used to create warnings. Fixes undesired output in
stderr
## [0.11.1] - 2023-03-07 ##
diff --git a/CITATION.cff b/CITATION.cff
index 910e40a2193d527fc8e4eb68c4ca6b10a28d3630..d9126aae6483459f8c8f248ed6a4fdf859f24e45 100644
--- a/CITATION.cff
+++ b/CITATION.cff
@@ -20,6 +20,6 @@ authors:
given-names: Stefan
orcid: https://orcid.org/0000-0001-7214-8125
title: CaosDB - Pylib
-version: 0.11.1
+version: 0.12.0
doi: 10.3390/data4020083
-date-released: 2022-11-14
\ No newline at end of file
+date-released: 2023-06-02
\ No newline at end of file
diff --git a/release.sh b/release.sh
index 1af097f014de6cd9eb3d3e8ba5da34aea0fe1671..f6335ae20d0c29e760b508aac831a35460a59ef3 100755
--- a/release.sh
+++ b/release.sh
@@ -1,4 +1,4 @@
#!/bin/bash
rm -rf dist/ build/ .eggs/
python setup.py sdist bdist_wheel
-python -m twine upload -s dist/*
+python -m twine upload dist/*
diff --git a/setup.py b/setup.py
index a8b948c1c9097ea49d391d0fe0747290d21be4a6..8fdf3b1c63322ec48af398d1dcb1c4028355d473 100755
--- a/setup.py
+++ b/setup.py
@@ -47,8 +47,8 @@ from setuptools import find_packages, setup
ISRELEASED = False
MAJOR = 0
-MINOR = 11
-MICRO = 3
+MINOR = 12
+MICRO = 1
# Do not tag as pre-release until this commit
# https://github.com/pypa/packaging/pull/515
# has made it into a release. Probably we should wait for pypa/packaging>=21.4
diff --git a/src/caosdb/cached.py b/src/caosdb/cached.py
index 4f735bb8e85ba7aa364b211e198840f9f6fb97e2..131526674d7df97d598a6d1bfbc2af7805c63a03 100644
--- a/src/caosdb/cached.py
+++ b/src/caosdb/cached.py
@@ -22,6 +22,14 @@
"""
This module provides some cached versions of functions that retrieve Entities from a remote server.
+
+See also
+========
+
+- ``cache_initialize(...)`` : Re-initialize the cache.
+- ``cache_clear()`` : Clear the cache.
+- ``cached_query(query)`` : A cached version of ``execute_query(query)``.
+- ``cached_get_entity_by(...)`` : Get an Entity by name, id, ...
"""
from enum import Enum
diff --git a/src/caosdb/common/models.py b/src/caosdb/common/models.py
index 9ba54c49d2d4cd776dc2263b850cc095c65fea60..df77bb7311a86abc8a78715e082f115c6a3efc2b 100644
--- a/src/caosdb/common/models.py
+++ b/src/caosdb/common/models.py
@@ -1,12 +1,11 @@
# -*- coding: utf-8 -*-
#
-# ** header v3.0
# This file is a part of the CaosDB Project.
#
# Copyright (C) 2018 Research Group Biomedical Physics,
# Max-Planck-Institute for Dynamics and Self-Organization Göttingen
-# Copyright (C) 2020-2022 Indiscale GmbH <info@indiscale.com>
-# Copyright (C) 2020 Florian Spreckelsen <f.spreckelsen@indiscale.com>
+# Copyright (C) 2020-2023 Indiscale GmbH <info@indiscale.com>
+# Copyright (C) 2020-2023 Florian Spreckelsen <f.spreckelsen@indiscale.com>
# Copyright (C) 2020-2022 Timm Fitschen <t.fitschen@indiscale.com>
#
# This program is free software: you can redistribute it and/or modify
@@ -22,7 +21,6 @@
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <https://www.gnu.org/licenses/>.
#
-# ** end header
#
"""
@@ -34,8 +32,8 @@ All additional classes are either important for the entities or the
transactions.
"""
-from __future__ import print_function, unicode_literals
from __future__ import annotations # Can be removed with 3.10.
+from __future__ import print_function, unicode_literals
import re
import sys
@@ -47,13 +45,14 @@ from os import listdir
from os.path import isdir
from random import randint
from tempfile import NamedTemporaryFile
+from typing import Any, Optional
from warnings import warn
from caosdb.common.datatype import (BOOLEAN, DATETIME, DOUBLE, INTEGER, TEXT,
is_list_datatype, is_reference)
from caosdb.common.state import State
-from caosdb.common.utils import uuid, xml2str
from caosdb.common.timezone import TimeZone
+from caosdb.common.utils import uuid, xml2str
from caosdb.common.versioning import Version
from caosdb.configuration import get_config
from caosdb.connection.connection import get_connection
@@ -113,7 +112,7 @@ class Entity:
self.__datatype = None
self.datatype = datatype
self.value = value
- self.messages = _Messages()
+ self.messages = Messages()
self.properties = _Properties()
self.parents = _ParentList()
self.path = None
@@ -420,7 +419,7 @@ class Entity:
self.acl.is_permitted(permission=permission)
def get_all_messages(self):
- ret = _Messages()
+ ret = Messages()
ret.append(self.messages)
for p in self.properties:
@@ -453,6 +452,66 @@ class Entity:
return self
+ def remove_value_from_property(self, property_name: str, value: Any,
+ remove_if_empty_afterwards: Optional[bool] = True):
+ """Remove a value from a property given by name.
+
+ Do nothing if this entity does not have a property of this
+ ``property_name`` or if the property value is different of the given
+ ``value``. By default, the property is removed from this entity if it
+ becomes empty (i.e., value=None) through removal of the value. This
+ behavior can be changed by setting ``remove_if_empty_afterwards`` to
+ ``False`` in which case the property remains.
+
+ Notes
+ -----
+ If the property value is a list and the value to be removed occurs more
+ than once in this list, only its first occurrance is deleted (similar
+ to the behavior of Python's ``list.remove()``.)
+
+ If the property was empty (prop.value == None) before, the property is
+ not removed afterwards even if ``remove_if_empty_afterwards`` is set to
+ ``True``. Rationale: the property being empty is not an effect of
+ calling this function.
+
+ Parameters
+ ----------
+ property_name : str
+ Name of the property from which the ``value`` will be removed.
+
+ value
+ Value that is to be removed.
+
+ remove_if_empty_afterwards : bool, optional
+ Whether the property shall be removed from this entity if it is
+ emptied by removing the ``value``. Default is ``True``.
+
+ Returns
+ -------
+ self
+ This entity.
+
+ """
+
+ if self.get_property(property_name) is None:
+ return self
+ if self.get_property(property_name).value is None:
+ remove_if_empty_afterwards = False
+ empty_afterwards = False
+ if isinstance(self.get_property(property_name).value, list):
+ if value in self.get_property(property_name).value:
+ self.get_property(property_name).value.remove(value)
+ if self.get_property(property_name).value == []:
+ self.get_property(property_name).value = None
+ empty_afterwards = True
+ elif self.get_property(property_name).value == value:
+ self.get_property(property_name).value = None
+ empty_afterwards = True
+ if remove_if_empty_afterwards and empty_afterwards:
+ self.remove_property(property_name)
+
+ return self
+
def remove_parent(self, parent):
self.parents.remove(parent)
@@ -623,7 +682,8 @@ class Entity:
if msg is not None:
pass
else:
- msg = Message(type, code, description, body)
+ msg = Message(description=description, type=type, code=code,
+ body=body)
self.messages.append(msg)
return self
@@ -997,7 +1057,7 @@ out: List[Entity]
def get_messages(self):
"""Get all messages of this entity.
- @return: _Messages(list)
+ @return: Messages(list)
"""
return self.messages
@@ -1005,9 +1065,9 @@ out: List[Entity]
def get_warnings(self):
"""Get all warning messages of this entity.
- @return _Messages(list): Warning messages.
+ @return Messages(list): Warning messages.
"""
- ret = _Messages()
+ ret = Messages()
for m in self.messages:
if m.type.lower() == "warning":
@@ -1018,9 +1078,9 @@ out: List[Entity]
def get_errors(self):
"""Get all error messages of this entity.
- @return _Messages(list): Error messages.
+ @return Messages(list): Error messages.
"""
- ret = _Messages()
+ ret = Messages()
for m in self.messages:
if m.type.lower() == "error":
@@ -1540,7 +1600,7 @@ class QueryTemplate():
self._cuid = None
self.value = None
self.datatype = None
- self.messages = _Messages()
+ self.messages = Messages()
self.properties = None
self.parents = None
self.path = None
@@ -1660,7 +1720,7 @@ class QueryTemplate():
return self.id is not None
def get_errors(self):
- ret = _Messages()
+ ret = Messages()
for m in self.messages:
if m.type.lower() == "error":
@@ -1811,12 +1871,10 @@ class Property(Entity):
class Message(object):
- # @ReservedAssignment
-
- def __init__(self, type, code=None, description=None, body=None): # @ReservedAssignment
- self.type = type
- self.code = code
+ def __init__(self, type=None, code=None, description=None, body=None): # @ReservedAssignment
self.description = description
+ self.type = type if type is not None else "Info"
+ self.code = int(code) if code is not None else None
self.body = body
def to_xml(self, xml=None):
@@ -1839,11 +1897,13 @@ class Message(object):
def __eq__(self, obj):
if isinstance(obj, Message):
- return self.type == obj.type and self.code == obj.code
+ return self.type == obj.type and self.code == obj.code and self.description == obj.description
return False
def get_code(self):
+ warn(("get_code is deprecated and will be removed in future. "
+ "Use self.code instead."), DeprecationWarning)
return int(self.code)
@@ -2340,10 +2400,9 @@ class _ParentList(list):
raise KeyError(str(parent) + " not found.")
-class _Messages(dict):
-
- """This 'kind of dictionary' stores error, warning, info, and other
- messages. The mentioned three messages types are messages of special use.
+class Messages(list):
+ """This specialization of list stores error, warning, info, and other
+ messages. The mentioned three messages types play a special role.
They are generated by the client and the server while processing the entity
to which the message in question belongs. It is RECOMMENDED NOT to specify
such messages manually. The other messages are ignored by the server unless
@@ -2354,25 +2413,18 @@ class _Messages(dict):
<$Type code=$code description=$description>$body</$Type>
- Messages are treated as 'equal' if and only if both they have the same type (case-insensitive),
- and the same code (or no code). Every message
- MUST NOT occur more than once per entity (to which the message in question belongs).
-
- If a message m2 is added while a messages m1 is already in this _Message object m2 will
- OVERRIDE m1.
-
Error, warning, and info messages will be deleted before any transaction.
Examples:
- <<< msgs = _Messages()
+ <<< msgs = Messages()
<<< # create Message
<<< msg = Message(type="HelloWorld", code=1, description="Greeting the world", body="Hello, world!")
- <<< # append it to the _Messages
+ <<< # append it to the Messages
<<< msgs.append(msg)
- <<< # use _Messages as list of Message objects
+ <<< # use Messages as list of Message objects
<<< for m in msgs:
... assert isinstance(m,Message)
@@ -2383,29 +2435,12 @@ class _Messages(dict):
<<< msgs.append(msg)
<<< # get it back via get(...) and the key tuple (type, code)
<<< assert id(msgs.get("HelloWorld",1))==id(msg)
-
- <<< # delete Message via remove and the (type,code) tuple
- <<< msgs.remove("HelloWorld",1)
- <<< assert msgs.get("HelloWorld",1) == None
-
- <<< # short version of adding/setting/resetting a new Message
- <<< msgs["HelloWorld",2] = "Greeting the world in German", "Hallo, Welt!"
- <<< assert msgs["HelloWorld",2] == ("Greeting the world in German","Hallo, Welt!")
- <<< msgs["HelloWorld",2] = "Greeting the world in German", "Huhu, Welt!"
- <<< assert msgs["HelloWorld",2] == ("Greeting the world in German","Huhu, Welt!")
- <<< del msgs["HelloWorld",2]
- <<< assert msgs.get("HelloWorld",2) == None
-
- # this Message has no code and no description (make easy things easy...)
- <<<
- <<< msgs["HelloWorld"] = "Hello!"
- <<< assert msgs["HelloWorld"] == "Hello!"
-
- (to be continued...)
"""
def clear_server_messages(self):
- """Removes all error, warning and info messages."""
+ """Removes all messages of type error, warning and info. All other
+ messages types are custom types which should be handled by custom
+ code."""
rem = []
for m in self:
@@ -2415,9 +2450,18 @@ class _Messages(dict):
for m in rem:
self.remove(m)
- return self
-
+ #######################################################################
+ # can be removed after 01.07.24
+ # default implementation of list is sufficient
def __setitem__(self, key, value): # @ReservedAssignment
+ if not isinstance(value, Message):
+ warn("__setitem__ will in future only accept Message objects as second argument. "
+ "You will no longe be"
+ " able to pass bodys such that Message object is created on the fly",
+ DeprecationWarning)
+ if not isinstance(key, int):
+ warn("__setitem__ will in future only accept int as first argument",
+ DeprecationWarning)
if isinstance(key, tuple):
if len(key) == 2:
type = key[0] # @ReservedAssignment
@@ -2428,7 +2472,7 @@ class _Messages(dict):
else:
raise TypeError(
"('type', 'code'), ('type'), or 'type' expected.")
- elif isinstance(key, _Messages._msg_key):
+ elif isinstance(key, Messages._msg_key):
type = key._type # @ReservedAssignment
code = key._code
else:
@@ -2449,13 +2493,19 @@ class _Messages(dict):
if isinstance(value, Message):
body = value.body
description = value.description
+ m = Message
else:
body = value
description = None
- m = Message(type=type, code=code, description=description, body=body)
- dict.__setitem__(self, _Messages._msg_key(type, code), m)
+ m = Message(type=type, code=code, description=description, body=body)
+ if isinstance(key, int):
+ super().__setitem__(key, m)
+ else:
+ self.append(m)
def __getitem__(self, key):
+ if not isinstance(key, int):
+ warn("__getitem__ only supports integer keys in future.", DeprecationWarning)
if isinstance(key, tuple):
if len(key) == 2:
type = key[0] # @ReservedAssignment
@@ -2466,113 +2516,118 @@ class _Messages(dict):
else:
raise TypeError(
"('type', 'code'), ('type'), or 'type' expected.")
- elif isinstance(key, int) and int(key) >= 0:
- for m in self.values():
- if key == 0:
- return m
- else:
- key -= 1
- type = key # @ReservedAssignment
- code = None
+ elif isinstance(key, int) and key >= 0:
+ return super().__getitem__(key)
else:
type = key # @ReservedAssignment
code = None
- m = dict.__getitem__(self, _Messages._msg_key(type, code))
-
+ m = self.get(type, code)
+ if m is None:
+ raise KeyError()
if m.description:
return (m.description, m.body)
else:
return m.body
- def __init__(self):
- dict.__init__(self)
-
def __delitem__(self, key):
if isinstance(key, tuple):
- if len(key) == 2:
- type = key[0] # @ReservedAssignment
- code = key[1]
- elif len(key) == 1:
- type = key[0] # @ReservedAssignment
- code = None
- else:
- raise TypeError(
- "('type', 'code'), ('type'), or 'type' expected.")
+ warn("__delitem__ only supports integer keys in future.", DeprecationWarning)
+ if self.get(key[0], key[1]) is not None:
+ self.remove(self.get(key[0], key[1]))
else:
- type = key # @ReservedAssignment
- code = None
-
- return dict.__delitem__(self, _Messages._msg_key(type, code))
+ super().__delitem__(key)
def remove(self, obj, obj2=None):
- if isinstance(obj, Message):
- return dict.__delitem__(self, _Messages._msg_key.get(obj))
+ if obj2 is not None:
+ warn("Supplying a second argument to remove is deprecated.",
+ DeprecationWarning)
+ super().remove(self.get(obj, obj2))
+ else:
+ super().remove(obj)
- return self.__delitem__((obj, obj2))
+ def append(self, msg):
+ if isinstance(msg, Messages) or isinstance(msg, list):
+ warn("Supplying a list-like object to append is deprecated. Please use extend"
+ " instead.", DeprecationWarning)
+ for m in msg:
+ self.append(m)
+ return
- def get(self, type, code=None, default=None): # @ReservedAssignment
- try:
- return dict.__getitem__(self, _Messages._msg_key(type, code))
- except KeyError:
- return default
+ super().append(msg)
- def extend(self, messages):
- self.append(messages)
+ @staticmethod
+ def _hash(t, c):
+ return hash(str(t).lower() + (str(",") + str(c) if c is not None else ''))
+ # end remove
+ #######################################################################
- return self
+ def get(self, type, code=None, default=None, exact=False): # @ReservedAssignment
+ """
+ returns a message from the list that kind of matches type and code
- def append(self, msg):
- if hasattr(msg, "__iter__"):
- for m in msg:
- self.append(m)
+ case and types (str/int) are ignored
- return self
+ If no suitable message is found, the default argument is returned
+ If exact=True, the message has to match code and type exactly
+ """
+ if not exact:
+ warn("The fuzzy mode (exact=False) is deprecated. Please use exact in future.",
+ DeprecationWarning)
+
+ for msg in self:
+ if exact:
+ if msg.type == type and msg.code == code:
+ return msg
+ else:
+ if self._hash(msg.type, msg.code) == self._hash(type, code):
+ return msg
- if isinstance(msg, Message):
- dict.__setitem__(self, _Messages._msg_key.get(msg), msg)
+ return default
- return self
- else:
- raise TypeError("Argument was not a Message")
+ def to_xml(self, add_to_element):
+ for m in self:
+ melem = m.to_xml()
+ add_to_element.append(melem)
- return self
+ def __repr__(self):
+ xml = etree.Element("Messages")
+ self.to_xml(xml)
- def __iter__(self):
- return dict.values(self).__iter__()
+ return xml2str(xml)
+ #######################################################################
+ # can be removed after 01.07.24
class _msg_key:
def __init__(self, type, code): # @ReservedAssignment
+ warn("This class is deprecated.", DeprecationWarning)
self._type = type
self._code = code
@staticmethod
def get(msg):
- return _Messages._msg_key(msg.type, msg.code)
+ return Messages._msg_key(msg.type, msg.code)
def __eq__(self, obj):
return self.__hash__() == obj.__hash__()
def __hash__(self):
- return hash(str(self._type).lower() + (str(",") +
- str(self._code) if self._code is not None else ''))
+ return hash(str(self._type).lower() + (str(",") + str(self._code)
+ if self._code is not None else ''))
def __repr__(self):
return str(self._type) + (str(",") + str(self._code)
if self._code is not None else '')
+ # end remove
+ #######################################################################
- def to_xml(self, add_to_element):
- for m in self:
- melem = m.to_xml()
- add_to_element.append(melem)
-
- return self
- def __repr__(self):
- xml = etree.Element("Messages")
- self.to_xml(xml)
-
- return xml2str(xml)
+class _Messages(Messages):
+ def __init__(self, *args, **kwargs):
+ warn("_Messages is deprecated. "
+ "Use class Messages instead and beware of the slightly different API of the new"
+ " Messages class", DeprecationWarning)
+ super().__init__(*args, **kwargs)
def _basic_sync(e_local, e_remote):
@@ -2775,7 +2830,7 @@ class Container(list):
list.__init__(self)
self._timestamp = None
self._srid = None
- self.messages = _Messages()
+ self.messages = Messages()
def extend(self, entities):
"""Extend this Container by appending all single entities in the given
@@ -2868,11 +2923,11 @@ class Container(list):
def get_errors(self):
"""Get all error messages of this container.
- @return _Messages: Error messages.
+ @return Messages: Error messages.
"""
if self.has_errors():
- ret = _Messages()
+ ret = Messages()
for m in self.messages:
if m.type.lower() == "error":
@@ -2885,11 +2940,11 @@ class Container(list):
def get_warnings(self):
"""Get all warning messages of this container.
- @return _Messages: Warning messages.
+ @return Messages: Warning messages.
"""
if self.has_warnings():
- ret = _Messages()
+ ret = Messages()
for m in self.messages:
if m.type.lower() == "warning":
@@ -2900,7 +2955,7 @@ class Container(list):
return None
def get_all_messages(self):
- ret = _Messages()
+ ret = Messages()
for e in self:
ret.extend(e.get_all_messages())
@@ -3083,7 +3138,7 @@ class Container(list):
msg = "Request was not unique. CUID " + \
str(local_entity._cuid) + " was found " + \
str(len(sync_remote_entities)) + " times."
- local_entity.add_message(Message("Error", None, msg))
+ local_entity.add_message(Message(description=msg, type="Error"))
if raise_exception_on_error:
raise MismatchingEntitiesError(msg)
@@ -3108,7 +3163,7 @@ class Container(list):
msg = "Request was not unique. ID " + \
str(local_entity.id) + " was found " + \
str(len(sync_remote_entities)) + " times."
- local_entity.add_message(Message("Error", None, msg))
+ local_entity.add_message(Message(description=msg, type="Error"))
if raise_exception_on_error:
raise MismatchingEntitiesError(msg)
@@ -3138,7 +3193,7 @@ class Container(list):
msg = "Request was not unique. Path " + \
str(local_entity.path) + " was found " + \
str(len(sync_remote_entities)) + " times."
- local_entity.add_message(Message("Error", None, msg))
+ local_entity.add_message(Message(description=msg, type="Error"))
if raise_exception_on_error:
raise MismatchingEntitiesError(msg)
@@ -3168,7 +3223,7 @@ class Container(list):
msg = "Request was not unique. Name " + \
str(local_entity.name) + " was found " + \
str(len(sync_remote_entities)) + " times."
- local_entity.add_message(Message("Error", None, msg))
+ local_entity.add_message(Message(description=msg, type="Error"))
if raise_exception_on_error:
raise MismatchingEntitiesError(msg)
@@ -3187,7 +3242,7 @@ class Container(list):
msg = "Request was not unique. There are " + \
str(len(sync_remote_entities)) + \
" entities which could not be matched to one of the requested ones."
- remote_container.add_message(Message("Error", None, msg))
+ remote_container.add_message(Message(description=msg, type="Error"))
if raise_exception_on_error:
raise MismatchingEntitiesError(msg)
@@ -4250,7 +4305,7 @@ class Query():
The query string.
flags : dict of str
A dictionary of flags to be send with the query request.
- messages : _Messages()
+ messages : Messages()
A container of messages included in the last query response.
cached : bool
indicates whether the server used the query cache for the execution of
@@ -4273,7 +4328,7 @@ class Query():
def __init__(self, q):
self.flags = dict()
- self.messages = _Messages()
+ self.messages = Messages()
self.cached = None
self.etag = None
@@ -4397,6 +4452,10 @@ def execute_query(q, unique=False, raise_exception_on_error=True, cache=True, fl
class DropOffBox(list):
+ def __init__(self, *args, **kwargs):
+ warn(DeprecationWarning(
+ "The DropOffBox is deprecated and will be removed in future."))
+ super().__init__(*args, **kwargs)
path = None
@@ -4440,7 +4499,7 @@ class UserInfo():
class Info():
def __init__(self):
- self.messages = _Messages()
+ self.messages = Messages()
self.sync()
def sync(self):
@@ -4587,7 +4646,7 @@ def _parse_single_xml_element(elem):
elif elem.tag.lower() == 'stats':
counts = elem.find("counts")
- return Message(type="Counts", body=counts.attrib)
+ return Message(type="Counts", description=None, body=counts.attrib)
elif elem.tag == "EntityACL":
return ACL(xml=elem)
elif elem.tag == "Permissions":
diff --git a/src/caosdb/high_level_api.py b/src/caosdb/high_level_api.py
index 427a095a4bafc0c372b0169298f2980dbd902c49..3509a7b6bfe7ec322f2e0d2590334c6fc6f02cf8 100644
--- a/src/caosdb/high_level_api.py
+++ b/src/caosdb/high_level_api.py
@@ -49,7 +49,7 @@ from datetime import datetime
from dateutil import parser
warnings.warn("""EXPERIMENTAL! The high_level_api module is experimental and may be changed or
-removed in future. Its purpose is to give an impression on how the Python client user interface
+removed in the future. Its purpose is to give an impression on how the Python client user interface
might be changed.""")
@@ -265,7 +265,8 @@ class CaosDBPythonEntity(object):
self._version = val
def _set_property_from_entity(self, ent: db.Entity, importance: str,
- references: Optional[db.Container]):
+ references: Optional[db.Container],
+ visited: Dict[int, "CaosDBPythonEntity"]):
"""
Set a new property using an entity from the normal python API.
@@ -280,7 +281,7 @@ class CaosDBPythonEntity(object):
raise RuntimeError("Multiproperty not implemented yet.")
val = self._type_converted_value(ent.value, ent.datatype,
- references)
+ references, visited)
self.set_property(
ent.name,
val,
@@ -382,7 +383,8 @@ class CaosDBPythonEntity(object):
def _type_converted_list(self,
val: List,
pr: str,
- references: Optional[db.Container]):
+ references: Optional[db.Container],
+ visited: Dict[int, "CaosDBPythonEntity"]):
"""
Convert a list to a python list of the correct type.
@@ -396,13 +398,14 @@ class CaosDBPythonEntity(object):
raise RuntimeError("Not a list.")
return [
- self._type_converted_value(i, get_list_datatype(pr), references
- ) for i in val]
+ self._type_converted_value(i, get_list_datatype(pr), references,
+ visited) for i in val]
def _type_converted_value(self,
val: Any,
pr: str,
- references: Optional[db.Container]):
+ references: Optional[db.Container],
+ visited: Dict[int, "CaosDBPythonEntity"]):
"""
Convert val to the correct type which is indicated by the database
type string in pr.
@@ -416,9 +419,9 @@ class CaosDBPythonEntity(object):
# this needs to be checked as second case as it is the ONLY
# case which does not depend on pr
# TODO: we might need to pass through the reference container
- return convert_to_python_object(val, references)
+ return convert_to_python_object(val, references, visited)
elif isinstance(val, list):
- return self._type_converted_list(val, pr, references)
+ return self._type_converted_list(val, pr, references, visited)
elif pr is None:
return val
elif pr == DOUBLE:
@@ -436,7 +439,7 @@ class CaosDBPythonEntity(object):
elif pr == DATETIME:
return self._parse_datetime(val)
elif is_list_datatype(pr):
- return self._type_converted_list(val, pr, references)
+ return self._type_converted_list(val, pr, references, visited)
else:
# Generic references to entities:
return CaosDBPythonUnresolvedReference(val)
@@ -561,8 +564,8 @@ class CaosDBPythonEntity(object):
return propval
def resolve_references(self, deep: bool, references: db.Container,
- visited: Dict[Union[str, int],
- "CaosDBPythonEntity"] = None):
+ visited: Optional[Dict[Union[str, int],
+ "CaosDBPythonEntity"]] = None):
"""
Resolve this entity's references. This affects unresolved properties as well
as unresolved parents.
@@ -629,18 +632,20 @@ class CaosDBPythonEntity(object):
else:
entity = CaosDBPythonRecord()
- for parent in serialization["parents"]:
- if "unresolved" in parent:
- id = None
- name = None
- if "id" in parent:
- id = parent["id"]
- if "name" in parent:
- name = parent["name"]
- entity.add_parent(CaosDBPythonUnresolvedParent(
- id=id, name=name))
- else:
- raise NotImplementedError()
+ if "parents" in serialization:
+ for parent in serialization["parents"]:
+ if "unresolved" in parent:
+ id = None
+ name = None
+ if "id" in parent:
+ id = parent["id"]
+ if "name" in parent:
+ name = parent["name"]
+ entity.add_parent(CaosDBPythonUnresolvedParent(
+ id=id, name=name))
+ else:
+ raise NotImplementedError(
+ "Currently, only unresolved parents can be deserialized.")
for baseprop in ("name", "id", "description", "version"):
if baseprop in serialization:
@@ -673,7 +678,8 @@ class CaosDBPythonEntity(object):
if f.name in metadata:
propmeta.__setattr__(f.name, metadata[f.name])
else:
- raise NotImplementedError()
+ pass
+ # raise NotImplementedError()
return entity
@@ -804,7 +810,9 @@ BASE_ATTRIBUTES = (
def _single_convert_to_python_object(robj: CaosDBPythonEntity,
entity: db.Entity,
- references: Optional[db.Container] = None):
+ references: Optional[db.Container] = None,
+ visited: Optional[Dict[int,
+ "CaosDBPythonEntity"]] = None):
"""
Convert a db.Entity from the standard API to a (previously created)
CaosDBPythonEntity from the high level API.
@@ -819,6 +827,17 @@ def _single_convert_to_python_object(robj: CaosDBPythonEntity,
Returns the input object robj.
"""
+
+ # This parameter is used in the recursion to keep track of already visited
+ # entites (in order to detect cycles).
+ if visited is None:
+ visited = dict()
+
+ if id(entity) in visited:
+ return visited[id(entity)]
+ else:
+ visited[id(entity)] = robj
+
for base_attribute in BASE_ATTRIBUTES:
val = entity.__getattribute__(base_attribute)
if val is not None:
@@ -827,7 +846,8 @@ def _single_convert_to_python_object(robj: CaosDBPythonEntity,
robj.__setattr__(base_attribute, val)
for prop in entity.properties:
- robj._set_property_from_entity(prop, entity.get_importance(prop), references)
+ robj._set_property_from_entity(prop, entity.get_importance(prop), references,
+ visited)
for parent in entity.parents:
robj.add_parent(CaosDBPythonUnresolvedParent(id=parent.id,
@@ -921,7 +941,9 @@ def convert_to_entity(python_object):
def convert_to_python_object(entity: Union[db.Container, db.Entity],
- references: Optional[db.Container] = None):
+ references: Optional[db.Container] = None,
+ visited: Optional[Dict[int,
+ "CaosDBPythonEntity"]] = None):
"""
Convert either a container of CaosDB entities or a single CaosDB entity
into the high level representation.
@@ -933,15 +955,19 @@ def convert_to_python_object(entity: Union[db.Container, db.Entity],
"""
if isinstance(entity, db.Container):
# Create a list of objects:
- return [convert_to_python_object(i, references) for i in entity]
+ return [convert_to_python_object(i, references, visited) for i in entity]
+ # TODO: recursion problems?
return _single_convert_to_python_object(
- high_level_type_for_standard_type(entity)(), entity, references)
+ high_level_type_for_standard_type(entity)(),
+ entity,
+ references,
+ visited)
def new_high_level_entity(entity: db.RecordType,
importance_level: str,
- name: str = None):
+ name: Optional[str] = None):
"""
Create an new record in high level format based on a record type in standard format.
@@ -974,7 +1000,7 @@ def new_high_level_entity(entity: db.RecordType,
return convert_to_python_object(r)
-def create_record(rtname: str, name: str = None, **kwargs):
+def create_record(rtname: str, name: Optional[str] = None, **kwargs):
"""
Create a new record based on the name of a record type. The new record is returned.
@@ -1013,7 +1039,9 @@ def create_entity_container(record: CaosDBPythonEntity):
return db.Container().extend(lse)
-def query(query: str, resolve_references: bool = True, references: db.Container = None):
+def query(query: str,
+ resolve_references: Optional[bool] = True,
+ references: Optional[db.Container] = None):
"""
"""
diff --git a/src/caosdb/utils/checkFileSystemConsistency.py b/src/caosdb/utils/checkFileSystemConsistency.py
index a142c1dd7ffd1a4e6ee6cfc85891e1bf70f98d89..6c053fdca6acb3a6585589c0e6298ba0704ea590 100755
--- a/src/caosdb/utils/checkFileSystemConsistency.py
+++ b/src/caosdb/utils/checkFileSystemConsistency.py
@@ -30,7 +30,6 @@ import caosdb as db
from argparse import ArgumentParser
from argparse import RawDescriptionHelpFormatter
-from _testcapi import raise_exception
__all__ = []
__version__ = 0.1
diff --git a/src/doc/conf.py b/src/doc/conf.py
index 819ef61d7fb02e752b4a73a86644d1602bbf188a..0fa5de575f5424e267cad8ecc193cca8230faa8b 100644
--- a/src/doc/conf.py
+++ b/src/doc/conf.py
@@ -29,10 +29,10 @@ copyright = '2023, IndiScale GmbH'
author = 'Daniel Hornung'
# The short X.Y version
-version = '0.11.3'
+version = '0.12.1'
# The full version, including alpha/beta/rc tags
# release = '0.5.2-rc2'
-release = '0.11.3-dev'
+release = '0.12.1-dev'
# -- General configuration ---------------------------------------------------
diff --git a/unittests/docker/Dockerfile b/unittests/docker/Dockerfile
index 06f9d6c830068a2c1c85caef79c64f899eaefb33..7c84050b0a55ae6e1e8f2e2583f894a69f691193 100644
--- a/unittests/docker/Dockerfile
+++ b/unittests/docker/Dockerfile
@@ -1,4 +1,4 @@
-FROM debian:latest
+FROM debian:bullseye
# Use local package repository
COPY sources.list.local /etc/apt/
RUN mv /etc/apt/sources.list /etc/apt/sources.list.orig
diff --git a/unittests/test_high_level_api.py b/unittests/test_high_level_api.py
index a9e55c9c2a79f7ead8bbb3fb652c1b81427e69e9..ea5e635eadaa849480de5f3ece10b813a538a1b0 100644
--- a/unittests/test_high_level_api.py
+++ b/unittests/test_high_level_api.py
@@ -154,7 +154,7 @@ def test_convert_with_references():
obj = convert_to_python_object(r)
assert obj.ref.a == 42
# Parent does not automatically lead to a datatype:
- assert obj.get_property_metadata("ref").datatype is "bla"
+ assert obj.get_property_metadata("ref").datatype == "bla"
assert obj.ref.has_parent("bla") is True
# Unresolved Reference:
@@ -163,7 +163,7 @@ def test_convert_with_references():
obj = convert_to_python_object(r)
# Parent does not automatically lead to a datatype:
- assert obj.get_property_metadata("ref").datatype is "bla"
+ assert obj.get_property_metadata("ref").datatype == "bla"
assert isinstance(obj.ref, CaosDBPythonUnresolvedReference)
assert obj.ref.id == 27
@@ -641,3 +641,14 @@ def test_recursion_advanced(get_record_container):
r.resolve_references(r, get_record_container)
d = r.serialize(True)
assert r == r.sources[0]
+
+
+def test_cyclic_references():
+ r1 = db.Record()
+ r2 = db.Record()
+ r1.add_property(name="ref_to_two", value=r2)
+ r2.add_property(name="ref_to_one", value=r1)
+
+ # This would have lead to a recursion error before adding the detection for
+ # cyclic references:
+ r = convert_to_python_object(r1)
diff --git a/unittests/test_issues.py b/unittests/test_issues.py
index 2c45a6d77ba61c3f948e403f708994c0fe31481a..3fb48416511ba654d6f998442319c4ff29ac2956 100644
--- a/unittests/test_issues.py
+++ b/unittests/test_issues.py
@@ -37,3 +37,30 @@ def test_issue_100():
with raises(TypeError) as exc_info:
db.common.models._parse_single_xml_element(xml_el)
assert "Invalid datatype: List valued properties" in str(exc_info.value)
+
+
+def test_issue_156():
+ """Does parse_value make a mistake with entities?
+
+ https://gitlab.indiscale.com/caosdb/src/caosdb-pylib/-/issues/156
+ """
+ project = db.Record(name="foo")
+ project.add_parent(name="RTName")
+ # <Record name="foo">
+ # <Parent name="RTName"/>
+ # </Record>
+ experiment = db.Record()
+ experiment.add_property(name="RTName", value=project)
+ # <Record>
+ # <Property name="RTName" importance="FIX" flag="inheritance:FIX">foo</Property>
+ # </Record>
+ value = experiment.get_property("RTName").value
+ # <Record name="foo">
+ # <Parent name="RTName"/>
+ # </Record>
+ parents = value.get_parents()
+ # <ParentList>
+ # <Parent name="RTName"/>
+ # </ParentList>
+ assert value is project
+ assert parents[0].name == "RTName"
diff --git a/unittests/test_message.py b/unittests/test_message.py
index 5e1003056c1b606a004b63bb7618e5e0474952bc..440e7169501afb0a35acb78df95cefae01bd9426 100644
--- a/unittests/test_message.py
+++ b/unittests/test_message.py
@@ -27,11 +27,14 @@ import caosdb as db
from copy import deepcopy
+import pytest
+
+
def test_messages_dict_behavior():
from caosdb.common.models import Message
- from caosdb.common.models import _Messages
+ from caosdb.common.models import Messages
- msgs = _Messages()
+ msgs = Messages()
# create Message
msg = Message(
@@ -40,12 +43,12 @@ def test_messages_dict_behavior():
description="Greeting the world",
body="Hello, world!")
- # append it to the _Messages
+ # append it to the Messages
assert repr(msg) == '<HelloWorld code="1" description="Greeting the world">Hello, world!</HelloWorld>\n'
msgs.append(msg)
assert len(msgs) == 1
- # use _Messages as list of Message objects
+ # use Messages as list of Message objects
for m in msgs:
assert isinstance(m, Message)
@@ -70,10 +73,6 @@ def test_messages_dict_behavior():
assert msgs["HelloWorld", 2] == (
"Greeting the world in German", "Hallo, Welt!")
- msgs["HelloWorld", 2] = "Greeting the world in German", "Huhu, Welt!"
- assert len(msgs) == 1
- assert msgs["HelloWorld", 2] == (
- "Greeting the world in German", "Huhu, Welt!")
del msgs["HelloWorld", 2]
assert msgs.get("HelloWorld", 2) is None
@@ -83,11 +82,11 @@ def test_messages_dict_behavior():
def test_deepcopy():
- """Test whether deepcopy of _Messages objects doesn't mess up
+ """Test whether deepcopy of Messages objects doesn't mess up
contained Messages objects.
"""
- msgs = db.common.models._Messages()
+ msgs = db.common.models.Messages()
msg = db.Message(type="bla", code=1234, description="desc", body="blabla")
msgs.append(msg)
msg_copy = deepcopy(msgs)[0]
@@ -102,7 +101,7 @@ def test_deepcopy():
def test_deepcopy_clear_server():
- msgs = db.common.models._Messages()
+ msgs = db.common.models.Messages()
msg = db.Message(type="bla", code=1234, description="desc", body="blabla")
err_msg = db.Message(type="Error", code=1357, description="error")
msgs.extend([msg, err_msg])
@@ -116,3 +115,18 @@ def test_deepcopy_clear_server():
copied_msgs.clear_server_messages()
assert len(copied_msgs) == 1
assert copied_msgs[0].code == msg.code
+
+
+def test_list_behavior():
+ msgs = db.common.models.Messages()
+ msgs.append(db.Message("test"))
+ assert len(msgs) == 1
+ assert msgs[0] == db.Message("test")
+ assert msgs[0] != db.Message("test2")
+
+ msgs.append(db.Message("test"))
+ assert len(msgs) == 2
+ assert msgs[0] == msgs[1]
+
+ with pytest.raises(IndexError):
+ msgs[3]
diff --git a/unittests/test_property.py b/unittests/test_property.py
index 7c756117765e510587c00d818e39fb3945d44c53..84f89b5a959192d7831e1bb3eab3a441912afe7e 100644
--- a/unittests/test_property.py
+++ b/unittests/test_property.py
@@ -1,11 +1,11 @@
# -*- encoding: utf-8 -*-
#
-# ** header v3.0
# This file is a part of the CaosDB Project.
#
# Copyright (C) 2018 Research Group Biomedical Physics,
# Max-Planck-Institute for Dynamics and Self-Organization Göttingen
-# Copyright (C) 2020 IndiScale GmbH <info@indiscale.com>
+# Copyright (C) 2020 - 2023 IndiScale GmbH <info@indiscale.com>
+# Copyright (C) 2023 Florian Spreckelsen <f.spreckelsen@indiscale.com>
# Copyright (C) 2020 Timm Fitschen <t.fitschen@indiscale.com>
#
# This program is free software: you can redistribute it and/or modify
@@ -21,8 +21,6 @@
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <https://www.gnu.org/licenses/>.
#
-# ** end header
-#
"""Tests for the Property class."""
import os
@@ -138,3 +136,87 @@ def test_is_reference():
# restore retrieve function with original
Entity.retrieve = real_retrieve
+
+
+def test_remove_value_from_property():
+
+ rec = Record()
+ names_values_dtypes = [
+ ("testListProp1", [1, 2, 3], db.LIST(db.INTEGER)),
+ ("testListProp2", ["a", "b", "a"], db.LIST(db.TEXT)),
+ ("testScalarProp1", "bla", db.TEXT),
+ ("testScalarProp2", False, db.BOOLEAN),
+ ("testEmptyProp", None, db.REFERENCE),
+ ("testNoneListProp", [None, None], db.LIST(db.REFERENCE)),
+ ]
+ for name, value, dtype in names_values_dtypes:
+ rec.add_property(name=name, value=value, datatype=dtype)
+
+ # property doesn't exist, so do nothing
+ returned = rec.remove_value_from_property("nonexisting", "some_value")
+ assert returned is rec
+ for name, value, dtype in names_values_dtypes:
+ assert rec.get_property(name).value == value
+ assert rec.get_property(name).datatype == dtype
+
+ # value doesn't exist so nothing changes either
+ rec.remove_value_from_property("testListProp1", 0)
+ assert rec.get_property("testListProp1").value == [1, 2, 3]
+ assert rec.get_property("testListProp1").datatype == db.LIST(db.INTEGER)
+
+ returned = rec.remove_value_from_property("testScalarProp2", True)
+ assert returned is rec
+ assert rec.get_property("testScalarProp2").value is False
+ assert rec.get_property("testScalarProp2").datatype == db.BOOLEAN
+
+ # Simple removals from lists without emptying them
+ rec.remove_value_from_property("testListProp1", 1)
+ assert rec.get_property("testListProp1").value == [2, 3]
+
+ rec.remove_value_from_property("testListProp1", 2)
+ assert rec.get_property("testListProp1").value == [3]
+
+ # similarly to Python's `list.remove()`, only remove first occurrance
+ rec.remove_value_from_property("testListProp2", "a")
+ assert rec.get_property("testListProp2").value == ["b", "a"]
+
+ # default is to remove an empty property:
+ rec.remove_value_from_property("testListProp1", 3)
+ assert rec.get_property("testListProp1") is None
+
+ rec.remove_value_from_property("testScalarProp1", "bla")
+ assert rec.get_property("testScalarProp1") is None
+
+ # don't remove if `remove_if_empty_afterwards=False`
+ rec.remove_value_from_property("testListProp2", "b")
+ rec.remove_value_from_property("testListProp2", "a", remove_if_empty_afterwards=False)
+ assert rec.get_property("testListProp2") is not None
+ assert rec.get_property("testListProp2").value is None
+ assert rec.get_property("testListProp2").datatype == db.LIST(db.TEXT)
+
+ rec.remove_value_from_property("testScalarProp2", False, remove_if_empty_afterwards=False)
+ assert rec.get_property("testScalarProp2") is not None
+ assert rec.get_property("testScalarProp2").value is None
+ assert rec.get_property("testScalarProp2").datatype == db.BOOLEAN
+
+ # Special case of an already empty property: It is not empty because a value
+ # was removed by `remove_value_from_property` but never had a value in the
+ # first place. So even `remove_if_empty_afterwards=True` should not lead to
+ # its removal.
+ rec.remove_value_from_property("testEmptyProp", 1234, remove_if_empty_afterwards=True)
+ assert rec.get_property("testEmptyProp") is not None
+ assert rec.get_property("testEmptyProp").value is None
+ assert rec.get_property("testEmptyProp").datatype == db.REFERENCE
+
+ # Corner case of corner case: remove with `value=None` and
+ # `remove_if_empty_afterwards=True` keeps the empty property.
+ rec.remove_value_from_property("testEmptyProp", None, remove_if_empty_afterwards=True)
+ assert rec.get_property("testEmptyProp") is not None
+ assert rec.get_property("testEmptyProp").value is None
+ assert rec.get_property("testEmptyProp").datatype == db.REFERENCE
+
+ # Remove `None` from list `[None, None]`
+ rec.remove_value_from_property("testNoneListProp", None, remove_if_empty_afterwards=True)
+ assert rec.get_property("testNoneListProp") is not None
+ assert rec.get_property("testNoneListProp").value == [None]
+ assert rec.get_property("testNoneListProp").datatype == db.LIST(db.REFERENCE)