Merge branch 'release-0.4.1' into 'main'

REL: Release 0.4.1

See merge request !43

CHANGELOG.md

@@ -18,6 +18,19 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Security ###
 
+## [0.4.1] - 2022-05-03 ##
+(Henrik tom Wörden)
+
+### Changed ###
+
+- `JsonSchemaParser` now identifies `name` properties in the schema with the
+  CaosDB name property.
+
+### Fixed ###
+
+- [#40](https://gitlab.com/caosdb/caosdb-advanced-user-tools/-/issues/40)
+  `assure_object_is_in_list` now handles adding objects to an initially empty list correctly.
+
 ## [0.4.0] - 2022-04-05  ##
 
 ### Added ###

RELEASE_GUIDELINES.md

@@ -9,7 +9,7 @@ guidelines of the CaosDB Project
 * All tests are passing.
 * FEATURES.md is up-to-date and a public API is being declared in that document.
 * CHANGELOG.md is up-to-date.
-* DEPENDENCIES.md is up-to-date.
+* dependencies in `setup.py` are up-to-date.
 
 ## Steps
 

integrationtests/extroot/Software/2010_TestSoftware/2019-02-03_v0.1/README.md

 ---
 responsible: Responsible, Only
-description: 	A description of this example analysis.
+description: 	A description of another example analysis.
 
 sources:
 - file:	"/ExperimentalData/2010_TestProject/2019-02-03/*.dat"

integrationtests/extroot/Software/2020NewProject0X/2020-02-03/README.md

 ---
 responsible:	
 - Only Responsible MPI DS
-description: 	A description of this example analysis.
+description: 	A description of another example analysis.
 
 sources:
 - file:	"/ExperimentalData/2010_TestProject/2019-02-03/*.dat"

integrationtests/extroot/Software/2020NewProject0X/2020-02-04/README.md

@@ -2,7 +2,7 @@
 responsible:	
 - Some Responsible
 - Responsible, No, MPI DS
-description: 	A description of this example analysis.
+description: 	A description of another example analysis.
 
 sources:
 - file:	"/ExperimentalData/2010_TestProject/2019-02-03/*.dat"

integrationtests/test.sh

@@ -35,14 +35,17 @@ echo "Testing the crawler database"
 python3 -m pytest test_crawler_with_cfoods.py
 echo "make a change"
 cd extroot
-egrep -liRZ 'A description of another example' . | xargs -0 -l sed -i -e 's/A description of another example/A description of this example/g'
+egrep -liRZ 'A description of another example' . \
+    | xargs -0 -l sed -i -e 's/A description of another example/A description of this example/g'
 # remove a file to check that this does not lead to a crawler crash
-mv DataAnalysis/2010_TestProject/2019-02-03_something/README.xlsx DataAnalysis/2010_TestProject/2019-02-03_something/README.xlsx_back
+mv DataAnalysis/2010_TestProject/2019-02-03_something/README.xlsx \
+   DataAnalysis/2010_TestProject/2019-02-03_something/README.xlsx_back
 cd ..
 echo "run crawler"
 ./crawl.py  / | tee $OUT
 # rename the moved file
-mv extroot/DataAnalysis/2010_TestProject/2019-02-03_something/README.xlsx_back extroot/DataAnalysis/2010_TestProject/2019-02-03_something/README.xlsx
+mv extroot/DataAnalysis/2010_TestProject/2019-02-03_something/README.xlsx_back \
+   extroot/DataAnalysis/2010_TestProject/2019-02-03_something/README.xlsx
 # check whether there was something UNAUTHORIZED
 grep "There where unauthorized changes" $OUT
 # get the id of the run which is the last field of the output string
@@ -59,7 +62,8 @@ fi
 set -e
 echo "Undoing previous changes to extroot content..."
 cd extroot
-egrep -liRZ 'A description of this example' . | xargs -0 -l sed -i -e 's/A description of this example/A description of another example/g'
+egrep -liRZ 'A description of this example' . \
+    | xargs -0 -l sed -i -e 's/A description of this example/A description of another example/g'
 cd ..
 echo "Done."
 python3 test_table.py

integrationtests/test_assure_functions.py

 #!/usr/bin/env python
 # encoding: utf-8
 #
-# ** header v3.0
 # This file is a part of the CaosDB Project.
 #
+# Copyright (C) 2022 IndiScale GmbH <info@indiscale.com>
 # Copyright (C) 2021 University Medical Center Göttingen, Institute for Medical Informatics
 # Copyright (C) 2021 Florian Spreckelsen <florian.spreckelsen@med.uni-goettingen.de>
+# Copyright (C) 2022 Florian Spreckelsen <f.spreckelsen@indiscale.com>
 #
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Affero General Public License as
-# published by the Free Software Foundation, either version 3 of the
-# License, or (at your option) any later version.
+# This program is free software: you can redistribute it and/or modify it under
+# the terms of the GNU Affero General Public License as published by the Free
+# Software Foundation, either version 3 of the License, or (at your option) any
+# later version.
 #
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU Affero General Public License for more details.
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more
+# details.
 #
-# You should have received a copy of the GNU Affero General Public License
-# along with this program. If not, see <https://www.gnu.org/licenses/>.
-#
-# ** end header
+# You should have received a copy of the GNU Affero General Public License along
+# with this program. If not, see <https://www.gnu.org/licenses/>.
 """Integration tests for the `assure_...` functions from
 `caosadvancedtools.cfood`. They mainly test the in-place updates when
 no `to_be_updated` is specified.
@@ -90,3 +89,29 @@ def test_assure_list_in_place():
     assert len(rec2.get_property(ref_rt.name).value) == 3
     assert ref_rec2.id in rec2.get_property(ref_rt.name).value
     assert ref_rec3.id in rec2.get_property(ref_rt.name).value
+
+
+def test_add_to_empty_list():
+    """See https://gitlab.com/caosdb/caosdb-advanced-user-tools/-/issues/40."""
+    # @author Florian Spreckelsen
+    # @date 2022-04-19
+    referenced_rt = db.RecordType(name="TestReferencedType").insert()
+    list_prop = db.Property(name="TestListProp",
+                            datatype=db.LIST(referenced_rt)).insert()
+    referencing_rt = db.RecordType(
+        name="TestReferencingType").add_property(list_prop).insert()
+
+    db.Record(name="TestReferencedRecord").add_parent(referenced_rt).insert()
+    db.Record(name="TestReferencingRecord").add_parent(
+        referencing_rt).add_property(list_prop, value=[]).insert()
+
+    referenced_rec = db.execute_query("FIND TestReferencedRecord", unique=True)
+    referencing_rec = db.execute_query(
+        "FIND TestReferencingRecord", unique=True)
+
+    assure_object_is_in_list(referenced_rec, referencing_rec, list_prop.name)
+
+    referencing_rec = db.execute_query(
+        "FIND TestReferencingRecord", unique=True)
+    assert referencing_rec.get_property(list_prop.name).value == [
+        referenced_rec.id]

setup.py

@@ -47,9 +47,9 @@ from setuptools import find_packages, setup
 
 MAJOR = 0
 MINOR = 4
-MICRO = 0
+MICRO = 1
 PRE = ""  # e.g. rc0, alpha.1, 0.beta-23
-ISRELEASED = False
+ISRELEASED = True
 
 if PRE:
     VERSION = "{}.{}.{}-{}".format(MAJOR, MINOR, MICRO, PRE)

src/caosadvancedtools/cfood.py

 #!/usr/bin/env python
 # encoding: utf-8
 #
-# ** header v3.0
 # This file is a part of the CaosDB Project.
 #
 # Copyright (C) 2018 Research Group Biomedical Physics,
 # Max-Planck-Institute for Dynamics and Self-Organization Göttingen
-# Copyright (C) 2019,2020 IndiScale GmbH <info@indiscale.com>
+# Copyright (C) 2019-2022 IndiScale GmbH <info@indiscale.com>
 # Copyright (C) 2019,2020 Henrik tom Wörden
-# Copyright (C) 2020 Florian Spreckelsen <f.spreckelsen@indiscale.com>
+# Copyright (C) 2020-2022 Florian Spreckelsen <f.spreckelsen@indiscale.com>
 # Copyright (C) 2021 University Medical Center Göttingen, Institute for Medical Informatics
 # Copyright (C) 2021 Florian Spreckelsen <florian.spreckelsen@med.uni-goettingen.de>
 #
@@ -24,8 +23,6 @@
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program. If not, see <https://www.gnu.org/licenses/>.
-#
-# ** end header
 """ Defines how something that shall be inserted into CaosDB is treated.
 
 CaosDB can automatically be filled with Records based on some structure, a file
@@ -415,11 +412,13 @@ def assure_object_is_in_list(obj, containing_object, property_name,
                                        datatype=datatype)
     # TODO: case where multiple times the same property exists is not treated
 
-    if not isinstance(containing_object.get_property(property_name).value, list):
-        containing_object.get_property(property_name).value = [
-            containing_object.get_property(property_name).value]
-        containing_object.get_property(property_name).datatype = datatype
-    current_list = containing_object.get_property(property_name).value
+    list_prop = containing_object.get_property(property_name)
+    if list_prop.value is None:
+        list_prop.value = []
+    elif not isinstance(list_prop.value, list):
+        list_prop.value = [list_prop.value]
+        list_prop.datatype = datatype
+    current_list = list_prop.value
 
     if not isinstance(obj, list):
         objects = [obj]
@@ -674,7 +673,8 @@ def assure_has_property(entity, name, value, to_be_updated=None,
             tmp_value = el.value.id
 
         if isinstance(tmp_value, list):
-            tmp_value = [i.id if isinstance(i, db.Entity) else i for i in tmp_value]
+            tmp_value = [i.id if isinstance(
+                i, db.Entity) else i for i in tmp_value]
 
         if tmp_value == value:
             contained = True
@@ -810,7 +810,9 @@ class RowCFood(AbstractCFood):
         for key, value in self.item.iteritems():
             if key in self.unique_cols:
                 continue
-            rec.add_property(key, value)
+            assure_property_is(rec, key,
+                               value,
+                               to_be_updated=self.to_be_updated)
 
 
 class CMeal():

src/caosadvancedtools/crawler.py

@@ -392,7 +392,6 @@ class Crawler(object):
         for cfood in cfoods:
             try:
                 cfood.create_identifiables()
-
                 self._cached_find_or_insert_identifiables(cfood.identifiables)
 
                 cfood.update_identifiables()

src/caosadvancedtools/models/parser.py

@@ -566,9 +566,11 @@ class Parser(object):
                              db.BOOLEAN]:
 
                     if is_list:
-                        value.datatype = db.LIST(db.__getattribute__(dtype))  # pylint: disable=no-member
+                        value.datatype = db.LIST(db.__getattribute__(  # pylint: disable=no-member
+                            dtype))
                     else:
-                        value.datatype = db.__getattribute__(dtype)  # pylint: disable=no-member
+                        value.datatype = db.__getattribute__(  # pylint: disable=no-member
+                            dtype)
 
                     continue
 
@@ -632,8 +634,9 @@ class JsonSchemaParser(Parser):
         return self._create_model_from_dict(model_dict)
 
     def _create_model_from_dict(self, model_dict: [dict, List[dict]]):
-        """Parse a dictionary read in from the model definition in a json schema and
-        return the Datamodel created from it.
+        """Parse a dictionary and return the Datamodel created from it.
+
+        The dictionary was typically created from the model definition in a json schema file.
 
         Parameters
         ----------
@@ -692,6 +695,15 @@ class JsonSchemaParser(Parser):
             # Each element must have a specific type
             raise JsonSchemaDefinitionError(
                 f"`type` is missing in element {name}.")
+        if name == "name":
+            # This is identified with the CaosDB name property as long as the
+            # type is correct.
+            if not elt["type"] == "string":
+                raise JsonSchemaDefinitionError(
+                    "The 'name' property must be string-typed, otherwise it cannot "
+                    "be identified with CaosDB's name property."
+                )
+            return None, force_list
         if "enum" in elt:
             ent = self._treat_enum(elt, name)
         elif elt["type"] in JSON_SCHEMA_ATOMIC_TYPES:
@@ -726,6 +738,10 @@ class JsonSchemaParser(Parser):
                 else:
                     name = self._stringify(key)
                 prop_ent, force_list = self._treat_element(prop, name)
+                if prop_ent is None:
+                    # Nothing to be appended since the property has to be
+                    # treated specially.
+                    continue
                 importance = db.OBLIGATORY if key in required else db.RECOMMENDED
                 if not force_list:
                     rt.add_property(prop_ent, importance=importance)

src/doc/conf.py

@@ -27,9 +27,9 @@ copyright = '2021, IndiScale GmbH'
 author = 'Daniel Hornung'
 
 # The short X.Y version
-version = '0.4.0'
+version = '0.4.1'
 # The full version, including alpha/beta/rc tags
-release = '0.4.0'
+release = '0.4.1'
 
 
 # -- General configuration ---------------------------------------------------

unittests/json-schema-models/datamodel_name.schema.json

+{
+    "title": "Dataset",
+    "type": "object",
+    "properties": {
+        "name": { "type": "string", "description": "Name of this dataset" },
+        "date_time": { "type": "string", "format": "date-time" },
+        "date": { "type": "string", "format": "date" },
+        "integer": { "type": "integer", "description": "Some integer property" },
+        "boolean": { "type": "boolean" },
+        "number_prop": { "type": "number", "description": "Some float property" }
+    }
+}

unittests/json-schema-models/datamodel_name_wrong_type.schema.json

+{
+    "title": "Dataset",
+    "type": "object",
+    "properties": {
+        "name": { "type": "boolean", "description": "Name of this dataset" },
+        "date_time": { "type": "string", "format": "date-time" },
+        "date": { "type": "string", "format": "date" },
+        "integer": { "type": "integer", "description": "Some integer property" },
+        "boolean": { "type": "boolean" },
+        "number_prop": { "type": "number", "description": "Some float property" }
+    }
+}

unittests/test_json_schema_model_parser.py

@@ -340,3 +340,19 @@ def test_list():
         assert model[name].name == name
         assert len(model[name].parents) == 1
         assert model[name].has_parent(model["license"])
+
+
+def test_name_property():
+    model = parse_model_from_json_schema(os.path.join(
+        FILEPATH, "datamodel_name.schema.json"))
+
+    dataset_rt = model["Dataset"]
+    assert dataset_rt.get_property("name") is None
+    assert "name" not in model
+
+    with pytest.raises(JsonSchemaDefinitionError) as err:
+        broken = parse_model_from_json_schema(os.path.join(
+            FILEPATH, "datamodel_name_wrong_type.schema.json"))
+    assert str(err.value).startswith(
+        "The 'name' property must be string-typed, otherwise it cannot be identified with CaosDB's "
+        "name property.")