diff --git a/src/caosdb/common/datatype.py b/src/caosdb/common/datatype.py
index b6e1f7ac8c1307d5e4a2957a802f9a35ad77e1b7..5434f5b6556a13f65754eacc66cb32231366e5b3 100644
--- a/src/caosdb/common/datatype.py
+++ b/src/caosdb/common/datatype.py
@@ -45,6 +45,8 @@ def LIST(datatype):
def get_list_datatype(datatype):
""" returns the datatype of the elements in the list """
+ if not isinstance(datatype, str):
+ return None
match = re.match("LIST(<|<)(?P<datatype>.*)(>|>)", datatype)
if match is not None:
diff --git a/src/caosdb/common/models.py b/src/caosdb/common/models.py
index 6ec49df2722170805fb6230753f36503870a8821..00522ee614cccc79c9c56b469d00b635059260b6 100644
--- a/src/caosdb/common/models.py
+++ b/src/caosdb/common/models.py
@@ -40,26 +40,23 @@ from sys import hexversion
from tempfile import NamedTemporaryFile
from warnings import warn
-from caosdb.common.datatype import (BOOLEAN, DATETIME, DOUBLE, INTEGER, TEXT)
-from caosdb.common.versioning import Version
+from caosdb.common.datatype import (BOOLEAN, DATETIME, DOUBLE, INTEGER, TEXT,
+ is_list_datatype, is_reference)
from caosdb.common.state import State
from caosdb.common.utils import uuid, xml2str
+from caosdb.common.versioning import Version
from caosdb.configuration import get_config
from caosdb.connection.connection import get_connection
from caosdb.connection.encode import MultipartParam, multipart_encode
-from caosdb.exceptions import (AmbiguousEntityError,
- AuthorizationError,
- CaosDBException, CaosDBConnectionError,
- ConsistencyError,
- EmptyUniqueQueryError,
+from caosdb.exceptions import (AmbiguousEntityError, AuthorizationError,
+ CaosDBConnectionError, CaosDBException,
+ ConsistencyError, EmptyUniqueQueryError,
EntityDoesNotExistError, EntityError,
- EntityHasNoDatatypeError,
- MismatchingEntitiesError,
- QueryNotUniqueError, TransactionError,
- UniqueNamesError,
+ EntityHasNoDatatypeError, HTTPURITooLongError,
+ MismatchingEntitiesError, QueryNotUniqueError,
+ TransactionError, UniqueNamesError,
UnqualifiedParentsError,
- UnqualifiedPropertiesError,
- HTTPURITooLongError)
+ UnqualifiedPropertiesError)
from lxml import etree
_ENTITY_URI_SEGMENT = "Entity"
@@ -580,6 +577,7 @@ class Entity(object):
entity, or name.
"""
+
if isinstance(key, int):
for p in self.parents:
if p.id is not None and int(p.id) == int(key):
@@ -588,14 +586,17 @@ class Entity(object):
if key.id is not None:
# first try by id
found = self.get_parent(int(key.id))
+
if found is not None:
return found
# otherwise by name
+
return self.get_parent(key.name)
else:
for p in self.parents:
if (p.name is not None
and str(p.name).lower() == str(key).lower()):
+
return p
return None
@@ -680,6 +681,7 @@ class Entity(object):
special_selector = None
# iterating through the entity tree according to the selector
+
for subselector in selector:
# selector does not match the structure, we cannot get a
# property of non-entity
@@ -691,11 +693,13 @@ class Entity(object):
# selector does not match the structure, we did not get a
# property
+
if prop is None:
return None
# if the property is a reference, we are interested in the
# corresponding entities attributes
+
if isinstance(prop.value, Entity):
ref = prop.value
@@ -704,6 +708,7 @@ class Entity(object):
ref = prop
# if we saved a special selector before, apply it
+
if special_selector is None:
return prop.value
else:
@@ -835,6 +840,7 @@ class Entity(object):
assert isinstance(xml, etree._Element)
# unwrap wrapped entity
+
if self._wrapped_entity is not None:
xml = self._wrapped_entity.to_xml(xml, add_properties)
@@ -1074,6 +1080,7 @@ class Entity(object):
if len(c) == 1:
c[0].messages.extend(c.messages)
+
return c[0]
raise QueryNotUniqueError("This retrieval was not unique!!!")
@@ -2145,6 +2152,7 @@ class _Messages(dict):
else:
raise TypeError(
"('description', 'body'), ('body'), or 'body' expected.")
+
if isinstance(value, Message):
body = value.body
description = value.description
@@ -2314,6 +2322,7 @@ def _deletion_sync(e_local, e_remote):
except KeyError:
# deletion info wasn't there
e_local.messages = e_remote.messages
+
return
_basic_sync(e_local, e_remote)
@@ -2506,6 +2515,7 @@ class Container(list):
tmpid = 0
# users might already have specified some tmpids. -> look for smallest.
+
for e in self:
tmpid = min(tmpid, Container._get_smallest_tmpid(e))
tmpid -= 1
@@ -2725,6 +2735,7 @@ class Container(list):
used_remote_entities = []
# match by cuid
+
for local_entity in self:
sync_dict[local_entity] = None
@@ -2753,6 +2764,7 @@ class Container(list):
raise MismatchingEntitiesError(msg)
# match by id
+
for local_entity in self:
if sync_dict[local_entity] is None and local_entity.id is not None:
sync_remote_entities = []
@@ -2777,6 +2789,7 @@ class Container(list):
raise MismatchingEntitiesError(msg)
# match by path
+
for local_entity in self:
if (sync_dict[local_entity] is None
and local_entity.path is not None):
@@ -2806,6 +2819,7 @@ class Container(list):
raise MismatchingEntitiesError(msg)
# match by name
+
for local_entity in self:
if (sync_dict[local_entity] is None
and local_entity.name is not None):
@@ -2855,7 +2869,60 @@ class Container(list):
return sync_dict
- def delete(self, raise_exception_on_error=True, flags=None):
+ def _test_dependencies_in_container(self, container):
+ """This function returns those elements of a given container that are a dependency of another element of the same container.
+
+ Args:
+ container (Container): a caosdb container
+
+ Returns:
+ [set]: a set of unique elements that are a dependency of another element of `container`
+ """
+ item_id = set()
+ is_parent = set()
+ is_property = set()
+ is_being_referenced = set()
+ dependent_parents = set()
+ dependent_properties = set()
+ dependent_references = set()
+ dependencies = set()
+
+ for container_item in container:
+ item_id.add(container_item.id)
+
+ for parents in container_item.get_parents():
+ is_parent.add(parents.id)
+
+ for references in container_item.get_properties():
+ if is_reference(references.datatype):
+ # add only if it is a reference, not a property
+
+ if isinstance(references.value, int):
+ is_being_referenced.add(references.value)
+ elif is_list_datatype(references.datatype):
+ for list_item in references.value:
+ if isinstance(list_item, int):
+ is_being_referenced.add(list_item)
+ else:
+ is_being_referenced.add(list_item.id)
+ else:
+ try:
+ is_being_referenced.add(references.value.id)
+ except:
+ pass
+
+ if hasattr(references, 'id'):
+ is_property.add(references.id)
+
+ dependent_parents = item_id.intersection(is_parent)
+ dependent_properties = item_id.intersection(is_property)
+ dependent_references = item_id.intersection(is_being_referenced)
+ dependencies = dependent_parents.union(dependent_references)
+ dependencies = dependencies.union(dependent_properties)
+
+ return dependencies
+
+ def delete(self, raise_exception_on_error=True, flags=None, chunk_size=100):
"""Delete all entities in this container.
Entities are identified via their id if present and via their
@@ -2866,16 +2933,43 @@ class Container(list):
this happens, none of them will be deleted. It occurs an error
instead.
"""
- chunk_size = 100
item_count = len(self)
# Split Container in 'chunk_size'-sized containers (if necessary) to avoid error 414 Request-URI Too Long
+
if item_count > chunk_size:
+ dependencies = self._test_dependencies_in_container(self)
+ '''
+ If there are as many dependencies as entities in the container and it is larger than chunk_size it cannot be split and deleted.
+ This case cannot be handled at the moment.
+ '''
+
+ if len(dependencies) == item_count:
+ if raise_exception_on_error:
+ te = TransactionError(
+ msg="The container is too large and with too many dependencies within to be deleted.",
+ container=self)
+ raise te
+
+ return self
+
+ dependencies_delete = Container() # items which have to be deleted later because of dependencies.
+
for i in range(0, int(item_count/chunk_size)+1):
chunk = Container()
+
for j in range(i*chunk_size, min(item_count, (i+1)*chunk_size)):
- chunk.append(self[j])
+ if len(dependencies):
+ if self[j].id in dependencies:
+ dependencies_delete.append(self[j])
+ else:
+ chunk.append(self[j])
+ else:
+ chunk.append(self[j])
+
if len(chunk):
chunk.delete()
+ dependencies_delete.delete()
+
return self
if len(self) == 0:
@@ -3513,6 +3607,7 @@ class ACL():
result._priority_grants.update(self._priority_grants)
result._priority_denials.update(other._priority_denials)
result._priority_denials.update(self._priority_denials)
+
return result
def __eq__(self, other):
@@ -3803,6 +3898,7 @@ class Query():
connection = get_connection()
flags = self.flags
+
if cache is False:
flags["cache"] = "false"
query_dict = dict(flags)
@@ -3829,9 +3925,11 @@ class Query():
if len(cresp) > 1 and raise_exception_on_error:
raise QueryNotUniqueError(
"Query '{}' wasn't unique.".format(self.q))
+
if len(cresp) == 0 and raise_exception_on_error:
raise EmptyUniqueQueryError(
"Query '{}' found no results.".format(self.q))
+
if len(cresp) == 1:
r = cresp[0]
r.messages.extend(cresp.messages)
@@ -3938,6 +4036,7 @@ class Info():
for e in xml:
m = _parse_single_xml_element(e)
+
if isinstance(m, UserInfo):
self.user_info = m
else:
@@ -4097,13 +4196,16 @@ def _evaluate_and_add_error(parent_error, ent):
Parent error with new exception(s) attached to it.
"""
+
if isinstance(ent, (Entity, QueryTemplate)):
# Check all error messages
found114 = False
found116 = False
+
for err in ent.get_errors():
# Evaluate specific EntityErrors depending on the error
# code
+
if err.code is not None:
if int(err.code) == 101: # ent doesn't exist
new_exc = EntityDoesNotExistError(entity=ent,
@@ -4120,6 +4222,7 @@ def _evaluate_and_add_error(parent_error, ent):
found114 = True
new_exc = UnqualifiedPropertiesError(entity=ent,
error=err)
+
for prop in ent.get_properties():
new_exc = _evaluate_and_add_error(new_exc,
prop)
@@ -4127,6 +4230,7 @@ def _evaluate_and_add_error(parent_error, ent):
found116 = True
new_exc = UnqualifiedParentsError(entity=ent,
error=err)
+
for par in ent.get_parents():
new_exc = _evaluate_and_add_error(new_exc,
par)
@@ -4137,21 +4241,28 @@ def _evaluate_and_add_error(parent_error, ent):
parent_error.add_error(new_exc)
# Check for possible errors in parents and properties that
# weren't detected up to here
+
if not found114:
dummy_err = EntityError(entity=ent)
+
for prop in ent.get_properties():
dummy_err = _evaluate_and_add_error(dummy_err, prop)
+
if dummy_err.errors:
parent_error.add_error(dummy_err)
+
if not found116:
dummy_err = EntityError(entity=ent)
+
for par in ent.get_parents():
dummy_err = _evaluate_and_add_error(dummy_err, par)
+
if dummy_err.errors:
parent_error.add_error(dummy_err)
elif isinstance(ent, Container):
parent_error.container = ent
+
if ent.get_errors() is not None:
parent_error.code = ent.get_errors()[0].code
# In the highly unusual case of more than one error
@@ -4159,6 +4270,7 @@ def _evaluate_and_add_error(parent_error, ent):
parent_error.msg = '\n'.join(
[x.description for x in ent.get_errors()])
# Go through all container elements and add them:
+
for elt in ent:
parent_error = _evaluate_and_add_error(parent_error, elt)
@@ -4184,10 +4296,12 @@ def raise_errors(arg0):
transaction_error = _evaluate_and_add_error(TransactionError(),
arg0)
# Raise if any error was found
+
if len(transaction_error.all_errors) > 0:
raise transaction_error
# Cover the special case of an empty container with error
# message(s) (e.g. query syntax error)
+
if (transaction_error.container is not None and
transaction_error.container.has_errors()):
raise transaction_error
diff --git a/unittests/test_container.py b/unittests/test_container.py
index b34055372fc83a5608ffcf54423a601001add12b..2e8dfa2a9b4538f854d79ad08f4500c11bf68945 100644
--- a/unittests/test_container.py
+++ b/unittests/test_container.py
@@ -25,28 +25,28 @@
"""Tests for the Container class."""
from __future__ import absolute_import
-import caosdb as c
+import caosdb as db
def test_get_property_values():
- rt_house = c.RecordType("House")
- rt_window = c.RecordType("Window")
- rt_owner = c.RecordType("Owner")
- p_height = c.Property("Height", datatype=c.DOUBLE)
+ rt_house = db.RecordType("House")
+ rt_window = db.RecordType("Window")
+ rt_owner = db.RecordType("Owner")
+ p_height = db.Property("Height", datatype=db.DOUBLE)
- window = c.Record().add_parent(rt_window)
+ window = db.Record().add_parent(rt_window)
window.id = 1001
window.add_property(p_height, 20.5, unit="m")
- owner = c.Record("The Queen").add_parent(rt_owner)
+ owner = db.Record("The Queen").add_parent(rt_owner)
- house = c.Record("Buckingham Palace")
+ house = db.Record("Buckingham Palace")
house.add_parent(rt_house)
house.add_property(rt_owner, owner)
house.add_property(rt_window, window)
house.add_property(p_height, 40.2, unit="ft")
- container = c.Container()
+ container = db.Container()
container.extend([
house,
owner
@@ -77,3 +77,68 @@ def test_get_property_values():
assert container.get_property_values("non-existing") == [(None,), (None,)]
assert container.get_property_values("name") == [(house.name,),
(owner.name,)]
+
+
+def test_container_dependencies_for_deletion():
+ not_included_rt = 1000
+ rt = db.RecordType("Just a RecordType")
+ rt.id = 1001
+ rt_record_with_parent = db.RecordType("Records with parent")
+ rt_record_with_parent.id = 1005
+ property_which_is_not_a_record = db.Property(
+ "Normal Property", datatype=db.DOUBLE, value=1006)
+ property_which_is_not_a_record.id = 1006
+ property_which_shall_be_deleted = db.Property(
+ "Normal Property 2", datatype=db.DOUBLE, value=1006)
+ property_which_shall_be_deleted .id = 1007
+
+ record_without_dependencies = db.Record().add_parent(not_included_rt)
+ record_without_dependencies.id = 2003
+
+ record_referenced = db.Record().add_parent(not_included_rt)
+ record_referenced.id = 2002
+ record_with_dependencies = db.Record().add_parent(not_included_rt)
+ record_with_dependencies.id = 2004
+ record_with_dependencies.add_property(not_included_rt, record_referenced)
+
+ record_with_parent = db.Record().add_parent(rt_record_with_parent)
+ record_with_parent.id = 2005
+
+ record_with_property_which_is_not_a_record = db.Record(
+ ).add_parent(not_included_rt)
+ record_with_property_which_is_not_a_record.id = 2006
+ record_with_property_which_is_not_a_record.add_property(
+ property_which_is_not_a_record)
+ record_with_property_which_is_not_a_record.add_property(
+ property_which_shall_be_deleted)
+
+ container = db.Container()
+ container.extend([
+ rt,
+ rt_record_with_parent, # 1005, dependency
+ record_without_dependencies,
+ property_which_shall_be_deleted, # 1007, dependency
+ record_referenced, # 2002, dependency
+ record_with_dependencies,
+ record_with_parent,
+ record_with_property_which_is_not_a_record
+ ])
+ assert (db.Container()._test_dependencies_in_container(container)
+ == {2002, 1005, 1007})
+
+
+def test_container_dependencies_for_deletion_with_lists():
+ not_included_rt = 1000
+
+ record_referenced = db.Record().add_parent(not_included_rt)
+ record_referenced.id = 2001
+
+ record_with_list = db.Record().add_parent(not_included_rt)
+ record_with_list.id = 2002
+ record_with_list.add_property(not_included_rt, datatype=db.LIST(
+ not_included_rt), value=[record_referenced, 2003, 2004, 2005, 2006])
+
+ container = db.Container()
+ container.extend([record_with_list, record_referenced])
+
+ assert db.Container()._test_dependencies_in_container(container) == {2001}