FIX: Treat dependencies inside containers when split for deletion

src/caosdb/common/datatype.py

@@ -45,6 +45,8 @@ def LIST(datatype):
 
 def get_list_datatype(datatype):
     """ returns the datatype of the elements in the list """
+    if not isinstance(datatype, str):
+        return None
     match = re.match("LIST(<|&lt;)(?P<datatype>.*)(>|&gt;)", datatype)
 
     if match is not None:

src/caosdb/common/models.py

@@ -40,26 +40,23 @@ from sys import hexversion
 from tempfile import NamedTemporaryFile
 from warnings import warn
 
-from caosdb.common.datatype import (BOOLEAN, DATETIME, DOUBLE, INTEGER, TEXT)
-from caosdb.common.versioning import Version
+from caosdb.common.datatype import (BOOLEAN, DATETIME, DOUBLE, INTEGER, TEXT,
+                                    is_list_datatype, is_reference)
 from caosdb.common.state import State
 from caosdb.common.utils import uuid, xml2str
+from caosdb.common.versioning import Version
 from caosdb.configuration import get_config
 from caosdb.connection.connection import get_connection
 from caosdb.connection.encode import MultipartParam, multipart_encode
-from caosdb.exceptions import (AmbiguousEntityError,
-                               AuthorizationError,
-                               CaosDBException, CaosDBConnectionError,
-                               ConsistencyError,
-                               EmptyUniqueQueryError,
+from caosdb.exceptions import (AmbiguousEntityError, AuthorizationError,
+                               CaosDBConnectionError, CaosDBException,
+                               ConsistencyError, EmptyUniqueQueryError,
                                EntityDoesNotExistError, EntityError,
-                               EntityHasNoDatatypeError,
-                               MismatchingEntitiesError,
-                               QueryNotUniqueError, TransactionError,
-                               UniqueNamesError,
+                               EntityHasNoDatatypeError, HTTPURITooLongError,
+                               MismatchingEntitiesError, QueryNotUniqueError,
+                               TransactionError, UniqueNamesError,
                                UnqualifiedParentsError,
-                               UnqualifiedPropertiesError,
-                               HTTPURITooLongError)
+                               UnqualifiedPropertiesError)
 from lxml import etree
 
 _ENTITY_URI_SEGMENT = "Entity"
@@ -580,6 +577,7 @@ class Entity(object):
             entity, or name.
 
         """
+
         if isinstance(key, int):
             for p in self.parents:
                 if p.id is not None and int(p.id) == int(key):
@@ -588,14 +586,17 @@ class Entity(object):
             if key.id is not None:
                 # first try by id
                 found = self.get_parent(int(key.id))
+
                 if found is not None:
                     return found
             # otherwise by name
+
             return self.get_parent(key.name)
         else:
             for p in self.parents:
                 if (p.name is not None
                         and str(p.name).lower() == str(key).lower()):
+
                     return p
 
         return None
@@ -680,6 +681,7 @@ class Entity(object):
             special_selector = None
 
         # iterating through the entity tree according to the selector
+
         for subselector in selector:
             # selector does not match the structure, we cannot get a
             # property of non-entity
@@ -691,11 +693,13 @@ class Entity(object):
 
             # selector does not match the structure, we did not get a
             # property
+
             if prop is None:
                 return None
 
             # if the property is a reference, we are interested in the
             # corresponding entities attributes
+
             if isinstance(prop.value, Entity):
                 ref = prop.value
 
@@ -704,6 +708,7 @@ class Entity(object):
                 ref = prop
 
         # if we saved a special selector before, apply it
+
         if special_selector is None:
             return prop.value
         else:
@@ -835,6 +840,7 @@ class Entity(object):
         assert isinstance(xml, etree._Element)
 
         # unwrap wrapped entity
+
         if self._wrapped_entity is not None:
             xml = self._wrapped_entity.to_xml(xml, add_properties)
 
@@ -1074,6 +1080,7 @@ class Entity(object):
 
             if len(c) == 1:
                 c[0].messages.extend(c.messages)
+
                 return c[0]
 
             raise QueryNotUniqueError("This retrieval was not unique!!!")
@@ -2145,6 +2152,7 @@ class _Messages(dict):
             else:
                 raise TypeError(
                     "('description', 'body'), ('body'), or 'body' expected.")
+
         if isinstance(value, Message):
             body = value.body
             description = value.description
@@ -2314,6 +2322,7 @@ def _deletion_sync(e_local, e_remote):
     except KeyError:
         # deletion info wasn't there
         e_local.messages = e_remote.messages
+
         return
 
     _basic_sync(e_local, e_remote)
@@ -2506,6 +2515,7 @@ class Container(list):
         tmpid = 0
 
         # users might already have specified some tmpids. -> look for smallest.
+
         for e in self:
             tmpid = min(tmpid, Container._get_smallest_tmpid(e))
         tmpid -= 1
@@ -2725,6 +2735,7 @@ class Container(list):
         used_remote_entities = []
 
         # match by cuid
+
         for local_entity in self:
 
             sync_dict[local_entity] = None
@@ -2753,6 +2764,7 @@ class Container(list):
                         raise MismatchingEntitiesError(msg)
 
         # match by id
+
         for local_entity in self:
             if sync_dict[local_entity] is None and local_entity.id is not None:
                 sync_remote_entities = []
@@ -2777,6 +2789,7 @@ class Container(list):
                         raise MismatchingEntitiesError(msg)
 
         # match by path
+
         for local_entity in self:
             if (sync_dict[local_entity] is None
                     and local_entity.path is not None):
@@ -2806,6 +2819,7 @@ class Container(list):
                         raise MismatchingEntitiesError(msg)
 
         # match by name
+
         for local_entity in self:
             if (sync_dict[local_entity] is None
                     and local_entity.name is not None):
@@ -2855,7 +2869,60 @@ class Container(list):
 
         return sync_dict
 
-    def delete(self, raise_exception_on_error=True, flags=None):
+    def _test_dependencies_in_container(self, container):
+        """This function returns those elements of a given container that are a dependency of another element of the same container.
+
+        Args:
+            container (Container): a caosdb container
+
+        Returns:
+            [set]: a set of unique elements that are a dependency of another element of `container`
+        """
+        item_id = set()
+        is_parent = set()
+        is_property = set()
+        is_being_referenced = set()
+        dependent_parents = set()
+        dependent_properties = set()
+        dependent_references = set()
+        dependencies = set()
+
+        for container_item in container:
+            item_id.add(container_item.id)
+
+            for parents in container_item.get_parents():
+                is_parent.add(parents.id)
+
+            for references in container_item.get_properties():
+                if is_reference(references.datatype):
+                    # add only if it is a reference, not a property
+
+                    if isinstance(references.value, int):
+                        is_being_referenced.add(references.value)
+                    elif is_list_datatype(references.datatype):
+                        for list_item in references.value:
+                            if isinstance(list_item, int):
+                                is_being_referenced.add(list_item)
+                            else:
+                                is_being_referenced.add(list_item.id)
+                    else:
+                        try:
+                            is_being_referenced.add(references.value.id)
+                        except:
+                            pass
+
+                if hasattr(references, 'id'):
+                    is_property.add(references.id)
+
+        dependent_parents = item_id.intersection(is_parent)
+        dependent_properties = item_id.intersection(is_property)
+        dependent_references = item_id.intersection(is_being_referenced)
+        dependencies = dependent_parents.union(dependent_references)
+        dependencies = dependencies.union(dependent_properties)
+
+        return dependencies
+
+    def delete(self, raise_exception_on_error=True, flags=None, chunk_size=100):
         """Delete all entities in this container.
 
         Entities are identified via their id if present and via their
@@ -2866,16 +2933,43 @@ class Container(list):
         this happens, none of them will be deleted. It occurs an error
         instead.
         """
-        chunk_size = 100
         item_count = len(self)
         # Split Container in 'chunk_size'-sized containers (if necessary) to avoid error 414 Request-URI Too Long
+
         if item_count > chunk_size:
+            dependencies = self._test_dependencies_in_container(self)
+            '''
+            If there are as many dependencies as entities in the container and it is larger than chunk_size it cannot be split and deleted.
+            This case cannot be handled at the moment.
+            '''
+
+            if len(dependencies) == item_count:
+                if raise_exception_on_error:
+                    te = TransactionError(
+                        msg="The container is too large and with too many dependencies within to be deleted.",
+                        container=self)
+                    raise te
+
+                return self
+
+            dependencies_delete = Container()  # items which have to be deleted later because of dependencies.
+
             for i in range(0, int(item_count/chunk_size)+1):
                 chunk = Container()
+
                 for j in range(i*chunk_size, min(item_count, (i+1)*chunk_size)):
+                    if len(dependencies):
+                        if self[j].id in dependencies:
+                            dependencies_delete.append(self[j])
+                        else:
+                            chunk.append(self[j])
+                    else:
                         chunk.append(self[j])
+
                 if len(chunk):
                     chunk.delete()
+            dependencies_delete.delete()
+
             return self
 
         if len(self) == 0:
@@ -3513,6 +3607,7 @@ class ACL():
         result._priority_grants.update(self._priority_grants)
         result._priority_denials.update(other._priority_denials)
         result._priority_denials.update(self._priority_denials)
+
         return result
 
     def __eq__(self, other):
@@ -3803,6 +3898,7 @@ class Query():
         connection = get_connection()
 
         flags = self.flags
+
         if cache is False:
             flags["cache"] = "false"
         query_dict = dict(flags)
@@ -3829,9 +3925,11 @@ class Query():
             if len(cresp) > 1 and raise_exception_on_error:
                 raise QueryNotUniqueError(
                     "Query '{}' wasn't unique.".format(self.q))
+
             if len(cresp) == 0 and raise_exception_on_error:
                 raise EmptyUniqueQueryError(
                     "Query '{}' found no results.".format(self.q))
+
             if len(cresp) == 1:
                 r = cresp[0]
                 r.messages.extend(cresp.messages)
@@ -3938,6 +4036,7 @@ class Info():
 
         for e in xml:
             m = _parse_single_xml_element(e)
+
             if isinstance(m, UserInfo):
                 self.user_info = m
             else:
@@ -4097,13 +4196,16 @@ def _evaluate_and_add_error(parent_error, ent):
         Parent error with new exception(s) attached to it.
 
     """
+
     if isinstance(ent, (Entity, QueryTemplate)):
         # Check all error messages
         found114 = False
         found116 = False
+
         for err in ent.get_errors():
             # Evaluate specific EntityErrors depending on the error
             # code
+
             if err.code is not None:
                 if int(err.code) == 101:  # ent doesn't exist
                     new_exc = EntityDoesNotExistError(entity=ent,
@@ -4120,6 +4222,7 @@ def _evaluate_and_add_error(parent_error, ent):
                     found114 = True
                     new_exc = UnqualifiedPropertiesError(entity=ent,
                                                          error=err)
+
                     for prop in ent.get_properties():
                         new_exc = _evaluate_and_add_error(new_exc,
                                                           prop)
@@ -4127,6 +4230,7 @@ def _evaluate_and_add_error(parent_error, ent):
                     found116 = True
                     new_exc = UnqualifiedParentsError(entity=ent,
                                                       error=err)
+
                     for par in ent.get_parents():
                         new_exc = _evaluate_and_add_error(new_exc,
                                                           par)
@@ -4137,21 +4241,28 @@ def _evaluate_and_add_error(parent_error, ent):
             parent_error.add_error(new_exc)
         # Check for possible errors in parents and properties that
         # weren't detected up to here
+
         if not found114:
             dummy_err = EntityError(entity=ent)
+
             for prop in ent.get_properties():
                 dummy_err = _evaluate_and_add_error(dummy_err, prop)
+
             if dummy_err.errors:
                 parent_error.add_error(dummy_err)
+
         if not found116:
             dummy_err = EntityError(entity=ent)
+
             for par in ent.get_parents():
                 dummy_err = _evaluate_and_add_error(dummy_err, par)
+
             if dummy_err.errors:
                 parent_error.add_error(dummy_err)
 
     elif isinstance(ent, Container):
         parent_error.container = ent
+
         if ent.get_errors() is not None:
             parent_error.code = ent.get_errors()[0].code
             # In the highly unusual case of more than one error
@@ -4159,6 +4270,7 @@ def _evaluate_and_add_error(parent_error, ent):
             parent_error.msg = '\n'.join(
                 [x.description for x in ent.get_errors()])
         # Go through all container elements and add them:
+
         for elt in ent:
             parent_error = _evaluate_and_add_error(parent_error, elt)
 
@@ -4184,10 +4296,12 @@ def raise_errors(arg0):
     transaction_error = _evaluate_and_add_error(TransactionError(),
                                                 arg0)
     # Raise if any error was found
+
     if len(transaction_error.all_errors) > 0:
         raise transaction_error
     # Cover the special case of an empty container with error
     # message(s) (e.g. query syntax error)
+
     if (transaction_error.container is not None and
             transaction_error.container.has_errors()):
         raise transaction_error

unittests/test_container.py

@@ -25,28 +25,28 @@
 """Tests for the Container class."""
 from __future__ import absolute_import
 
-import caosdb as c
+import caosdb as db
 
 
 def test_get_property_values():
-    rt_house = c.RecordType("House")
-    rt_window = c.RecordType("Window")
-    rt_owner = c.RecordType("Owner")
-    p_height = c.Property("Height", datatype=c.DOUBLE)
+    rt_house = db.RecordType("House")
+    rt_window = db.RecordType("Window")
+    rt_owner = db.RecordType("Owner")
+    p_height = db.Property("Height", datatype=db.DOUBLE)
 
-    window = c.Record().add_parent(rt_window)
+    window = db.Record().add_parent(rt_window)
     window.id = 1001
     window.add_property(p_height, 20.5, unit="m")
 
-    owner = c.Record("The Queen").add_parent(rt_owner)
+    owner = db.Record("The Queen").add_parent(rt_owner)
 
-    house = c.Record("Buckingham Palace")
+    house = db.Record("Buckingham Palace")
     house.add_parent(rt_house)
     house.add_property(rt_owner, owner)
     house.add_property(rt_window, window)
     house.add_property(p_height, 40.2, unit="ft")
 
-    container = c.Container()
+    container = db.Container()
     container.extend([
         house,
         owner
@@ -77,3 +77,68 @@ def test_get_property_values():
     assert container.get_property_values("non-existing") == [(None,), (None,)]
     assert container.get_property_values("name") == [(house.name,),
                                                      (owner.name,)]
+
+
+def test_container_dependencies_for_deletion():
+    not_included_rt = 1000
+    rt = db.RecordType("Just a RecordType")
+    rt.id = 1001
+    rt_record_with_parent = db.RecordType("Records with parent")
+    rt_record_with_parent.id = 1005
+    property_which_is_not_a_record = db.Property(
+        "Normal Property", datatype=db.DOUBLE, value=1006)
+    property_which_is_not_a_record.id = 1006
+    property_which_shall_be_deleted = db.Property(
+        "Normal Property 2", datatype=db.DOUBLE, value=1006)
+    property_which_shall_be_deleted .id = 1007
+
+    record_without_dependencies = db.Record().add_parent(not_included_rt)
+    record_without_dependencies.id = 2003
+
+    record_referenced = db.Record().add_parent(not_included_rt)
+    record_referenced.id = 2002
+    record_with_dependencies = db.Record().add_parent(not_included_rt)
+    record_with_dependencies.id = 2004
+    record_with_dependencies.add_property(not_included_rt, record_referenced)
+
+    record_with_parent = db.Record().add_parent(rt_record_with_parent)
+    record_with_parent.id = 2005
+
+    record_with_property_which_is_not_a_record = db.Record(
+    ).add_parent(not_included_rt)
+    record_with_property_which_is_not_a_record.id = 2006
+    record_with_property_which_is_not_a_record.add_property(
+        property_which_is_not_a_record)
+    record_with_property_which_is_not_a_record.add_property(
+        property_which_shall_be_deleted)
+
+    container = db.Container()
+    container.extend([
+        rt,
+        rt_record_with_parent,  # 1005, dependency
+        record_without_dependencies,
+        property_which_shall_be_deleted,  # 1007, dependency
+        record_referenced,  # 2002, dependency
+        record_with_dependencies,
+        record_with_parent,
+        record_with_property_which_is_not_a_record
+    ])
+    assert (db.Container()._test_dependencies_in_container(container)
+            == {2002, 1005, 1007})
+
+
+def test_container_dependencies_for_deletion_with_lists():
+    not_included_rt = 1000
+
+    record_referenced = db.Record().add_parent(not_included_rt)
+    record_referenced.id = 2001
+
+    record_with_list = db.Record().add_parent(not_included_rt)
+    record_with_list.id = 2002
+    record_with_list.add_property(not_included_rt, datatype=db.LIST(
+        not_included_rt), value=[record_referenced, 2003, 2004, 2005, 2006])
+
+    container = db.Container()
+    container.extend([record_with_list, record_referenced])
+
+    assert db.Container()._test_dependencies_in_container(container) == {2001}