ENH: implemented support of lists of references

src/caosdb/high_level_api.py

@@ -243,6 +243,12 @@ class CaosDBPythonEntity(object):
         ent : db.Entity
               The entity to be set.
         """
+
+        if ent.name is None:
+            raise RuntimeError("Setting properties without name is impossible.")
+
+        if ent.name in self.get_properties():
+            raise RuntimeError("Multiproperty not implemented yet.")
         
         val = self._type_converted_value(ent.value, ent.datatype,
                                          references)
@@ -357,7 +363,7 @@ class CaosDBPythonEntity(object):
         pr: str
             The datatype according to the database entry.
         """
-        if not is_list_datatype(pr):
+        if not is_list_datatype(pr) and not isinstance(val, list):
             raise RuntimeError("Not a list.")
 
         return [
@@ -382,6 +388,8 @@ class CaosDBPythonEntity(object):
             # case which does not depend on pr
             # TODO: we might need to pass through the reference container
             return convert_to_python_object(val, references)
+        elif isinstance(val, list):
+            return self._type_converted_list(val, pr, references)
         elif pr is None:
             return val
         elif pr == DOUBLE:
@@ -491,6 +499,30 @@ class CaosDBPythonEntity(object):
                 return True
         return False
 
+    def _resolve_caosdb_python_unresolved_reference(self, propval, deep,
+                                                    references, visited):
+        # This does not make sense for unset ids:
+        if propval.id is None:
+            raise RuntimeError("Unresolved property reference without an ID.")
+        # have we encountered this id before:
+        if propval.id in visited:
+            # self.__setattr__(prop, visited[propval.id])
+            # don't do the lookup in the references container
+            return visited[propval.id]
+
+        # lookup in container:
+        for ent in references:
+            # Entities in container without an ID will be skipped:
+            if ent.id is not None and ent.id == propval.id:
+                # resolve this entity:
+                obj = convert_to_python_object(ent, references)
+                visited[propval.id] = obj
+                # self.__setattr__(prop, visited[propval.id])
+                if deep:
+                    obj.resolve_references(deep, references, visited)
+                return obj
+        return propval
+
     def resolve_references(self, deep: bool, references: db.Container,
                            visited: dict[Union[str, int],
                                          "CaosDBPythonEntity"] = None):
@@ -520,27 +552,24 @@ class CaosDBPythonEntity(object):
             # Resolve all previously unresolved attributes that are entities:
             if deep and isinstance(propval, CaosDBPythonEntity):
                 propval.resolve_references(deep, references)
+            elif isinstance(propval, list):
+                resolvedelements = []
+                for element in propval:
+                    if deep and isinstance(element, CaosDBPythonEntity):
+                        element.resolve_references(deep, references)
+                        resolvedelements.append(element)
+                    if isinstance(element, CaosDBPythonUnresolvedReference):
+                        resolvedelements.append(
+                            self._resolve_caosdb_python_unresolved_reference(element, deep,
+                                                                             references, visited))
+                    else:
+                        resolvedelements.append(element)
+                self.__setattr__(prop, resolvedelements)
+                        
             elif isinstance(propval, CaosDBPythonUnresolvedReference):
-                # This does not make sense for unset ids:
-                if propval.id is None:
-                    raise RuntimeError("Unresolved property reference without an ID.")
-                # have we encountered this id before:
-                if propval.id in visited:
-                    self.__setattr__(prop, visited[propval.id])
-                    # don't do the lookup in the references container
-                    continue
-                
-                # lookup in container:
-                for ent in references:
-                    # Entities in container without an ID will be skipped:
-                    if ent.id is not None and ent.id == propval.id:
-                        # resolve this entity:
-                        obj = convert_to_python_object(ent, references)
-                        visited[propval.id] = obj
-                        self.__setattr__(prop, visited[propval.id])
-                        if deep:
-                            obj.resolve_references(deep, references, visited)
-                        break
+                val = self._resolve_caosdb_python_unresolved_reference(propval, deep,
+                                                                       references, visited)
+                self.__setattr__(prop, val)
 
     def get_properties(self):
         """
@@ -599,6 +628,19 @@ class CaosDBPythonEntity(object):
                 properties[p] = {"id": val.id, "unresolved": True}
             elif isinstance(val, CaosDBPythonEntity):
                 properties[p] = val.serialize(without_metadata)
+            elif isinstance(val, list):
+                serializedelements = []
+                for element in val:
+                    if isinstance(element, CaosDBPythonUnresolvedReference):
+                        elm = dict()
+                        elm["id"] = element.id
+                        elm["unresolved"] = True
+                        serializedelements.append(elm)
+                    elif isinstance(element, CaosDBPythonEntity):
+                        serializedelements.append(element.serialize(without_metadata))
+                    else:
+                        serializedelements.append(element)
+                properties[p] = serializedelements
             else:
                 properties[p] = val
         
@@ -717,7 +759,9 @@ def _single_convert_to_entity(entity: db.Entity,
             propval = _single_convert_to_entity(
                 standard_type_for_high_level_type(propval)(), propval)
         elif isinstance(propval, list):
-            raise NotImplementedError()
+            # propval = []
+            if not isinstance(propval[0], int):
+                raise NotImplementedError()
         
         entity.add_property(
             name=prop,
@@ -768,3 +812,44 @@ def convert_to_python_object(entity: Union[db.Container, db.Entity],
     return _single_convert_to_python_object(
         high_level_type_for_standard_type(entity)(), entity, references)
     
+
+def new_high_level_entity_for_record_type(entity: db.RecordType,
+                                          importance_level: str,
+                                          name: str = None,
+                                          deep: bool = True,
+                                          references: Optional[db.Container] = None):
+    """
+    Create an new record in high level format based on a record type in standard format.
+
+    entity: db.RecordType
+            The record type to initialize the new record from.
+
+    importance_level: str
+                      None, obligatory, recommended or suggested
+                      Initialize new properties up to this level.
+                      Properties in the record type with no importance will be added
+                      regardless of the importance_level.
+
+    name: str
+          Name of the new record.
+    """
+
+    r = db.Record(name=name)
+    r.add_parent(entity)
+
+    impmap = {
+        None: 0, "SUGGESTED": 3, "RECOMMENDED": 2, "OBLIGATORY": 1}
+
+    for prop in entity.properties:
+        imp = entity.get_importance(prop)
+        if imp is not None and impmap[importance_level] < impmap[imp]:
+            continue
+
+        r.add_property(prop)
+
+    if deep:
+        raise NotImplementedError("Recursive creation is not possible at the moment.")
+
+    return convert_to_python_object(r)
+            
+    

unittests/test_high_level_api.py

@@ -25,12 +25,17 @@
 
 
 import caosdb as db
-from caosdb.high_level_api import (convert_to_entity, convert_to_python_object)
+from caosdb.high_level_api import (convert_to_entity, convert_to_python_object,
+                                   new_high_level_entity_for_record_type)
 from caosdb.high_level_api import (CaosDBPythonUnresolvedParent,
                                    CaosDBPythonUnresolvedReference,
                                    CaosDBPythonRecord, CaosDBPythonFile)
 from caosdb.apiutils import compare_entities
 
+from caosdb.common.datatype import (is_list_datatype,
+                                    get_list_datatype,
+                                    is_reference)
+
 import pytest
 from lxml import etree
 import os
@@ -396,3 +401,111 @@ def test_files():
     assert p.datatype == db.FILE
     assert p.value.file == "/local/path/test.dat"
     assert p.value.path == "test.dat"
+
+
+@pytest.mark.xfail
+def test_record_generator():
+    rt = db.RecordType(name="Simulation")
+    rt.add_property(name="a", datatype=db.INTEGER)
+    rt.add_property(name="b", datatype=db.DOUBLE)
+    rt.add_property(name="inputfile", datatype=db.FILE)
+
+    simrt = db.RecordType(name="SimOutput")
+    rt.add_property(name="outputfile", datatype="SimOutput")
+
+    obj = new_high_level_entity_for_record_type(
+        rt, "SUGGESTED", "", True)
+    print(obj)
+    assert False
+
+    
+def test_list_types():
+    r = db.Record()
+    r.add_property(name="a", value=[1, 2, 4])
+
+    assert get_list_datatype(r.get_property("a").datatype) is None
+
+    obj = convert_to_python_object(r)
+    assert type(obj.a) == list
+    assert len(obj.a) == 3
+    assert 4 in obj.a
+    assert obj.get_property_metadata("a").datatype is None
+
+    conv = convert_to_entity(obj)
+    prop = r.get_property("a")
+    assert prop.value == [1, 2, 4]
+    assert prop.datatype is None
+
+    r.get_property("a").datatype = db.LIST(db.INTEGER)
+    assert r.get_property("a").datatype == "LIST<INTEGER>"
+    obj = convert_to_python_object(r)
+    assert type(obj.a) == list
+    assert len(obj.a) == 3
+    assert 4 in obj.a
+    assert obj.get_property_metadata("a").datatype == "LIST<INTEGER>"
+
+    conv = convert_to_entity(obj)
+    prop = r.get_property("a")
+    assert prop.value == [1, 2, 4]
+    assert obj.get_property_metadata("a").datatype == "LIST<INTEGER>"
+
+    # List of referenced objects:
+    r = db.Record()
+    r.add_property(name="a", value=[1, 2, 4], datatype="LIST<TestReference>")
+    obj = convert_to_python_object(r)
+    assert type(obj.a) == list
+    assert len(obj.a) == 3
+    assert obj.get_property_metadata("a").datatype == "LIST<TestReference>"
+    for i in range(3):
+        assert type(obj.a[i]) == CaosDBPythonUnresolvedReference
+    assert obj.a == [CaosDBPythonUnresolvedReference(id=i) for i in [1, 2, 4]]
+
+    # Try resolving:
+
+    # Should not work:
+    obj.resolve_references(False, db.Container())
+    assert type(obj.a) == list
+    assert len(obj.a) == 3
+    assert obj.get_property_metadata("a").datatype == "LIST<TestReference>"
+    for i in range(3):
+        assert type(obj.a[i]) == CaosDBPythonUnresolvedReference
+    assert obj.a == [CaosDBPythonUnresolvedReference(id=i) for i in [1, 2, 4]]
+
+    references = db.Container()
+    for i in [1, 2, 4]:
+        ref = db.Record(id=i)
+        ref.add_property(name="val", value=str(i) + " bla")
+        references.append(ref)
+
+    obj.resolve_references(False, references)
+    assert type(obj.a) == list
+    assert len(obj.a) == 3
+    assert obj.get_property_metadata("a").datatype == "LIST<TestReference>"
+    for i in range(3):
+        assert type(obj.a[i]) == CaosDBPythonRecord
+
+    assert obj.a[0].val == "1 bla"
+
+    # Conversion with embedded records:
+    r2 = db.Record()
+    r2.add_property(name="a", value=4)
+    r3 = db.Record()
+    r3.add_property(name="b", value=8)
+    
+    r = db.Record()
+    r.add_property(name="a", value=[r2, r3])
+
+    obj = convert_to_python_object(r)
+    assert type(obj.a) == list
+    assert len(obj.a) == 2
+    assert obj.a[0].a == 4
+    assert obj.a[1].b == 8
+
+    # Serialization
+    text = str(obj)
+    text2 = str(convert_to_python_object(r2)).split("\n")
+    print(text)
+    # cut away first two characters in text
+    text = [line[4:] for line in text.split("\n")]
+    for line in text2:
+        assert line in text