CHANGELOG.md

@@ -8,9 +8,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 ## [Unreleased] ##
 
 ### Added ###
-- Added location argument to  `src/caosdb/utils/checkFileSystemConsistency.py`
-- Entity getters: `get_entity_by_<name/id/path>`
-- Cached versions of entity getters and of `execute_query` (`cached_query`)
+
+* `Entity.remove_value_from_property` function that removes a given value from a
+  property and optionally removes the property if it is empty afterwards.
 
 ### Changed ###
 
@@ -19,12 +19,30 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 ### Removed ###
 
 ### Fixed ###
-- Fixed `src/caosdb/utils/checkFileSystemConsistency.py`
 
 ### Security ###
 
 ### Documentation ###
 
+## [0.12.0] - 2023-06-02 ##
+
+### Added ###
+
+- Added location argument to  `src/caosdb/utils/checkFileSystemConsistency.py`
+- Entity getters: `get_entity_by_<name/id/path>`
+- Cached versions of entity getters and of `execute_query` (`cached_query`)
+
+### Deprecated ###
+
+- getOriginUrlIn, getDiffIn, getBranchIn, getCommitIn (formerly apiutils) have been
+  moved to caosdb.utils.git_utils
+
+### Fixed ###
+
+- Fixed `src/caosdb/utils/checkFileSystemConsistency.py`
+
+### Documentation ###
+
 * [#83](https://gitlab.com/caosdb/caosdb-pylib/-/issues/83) - Improved
   documentation on adding REFERENCE properties, both in the docstring of
   `Entity.add_property` and in the data-insertion tutorial.
@@ -32,7 +50,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 ## [0.11.2] - 2023-03-14 ##
 
 ### Fixed ###
-- root logger is no longer used to create warnings. Fixes undesired output in 
+- root logger is no longer used to create warnings. Fixes undesired output in
   stderr
 
 ## [0.11.1] - 2023-03-07 ##

CITATION.cff

@@ -20,6 +20,6 @@ authors:
     given-names: Stefan
     orcid: https://orcid.org/0000-0001-7214-8125
 title: CaosDB - Pylib
-version: 0.11.1
+version: 0.12.0
 doi: 10.3390/data4020083
-date-released: 2022-11-14
\ No newline at end of file
+date-released: 2023-06-02
\ No newline at end of file

setup.py

@@ -47,8 +47,8 @@ from setuptools import find_packages, setup
 
 ISRELEASED = False
 MAJOR = 0
-MINOR = 11
-MICRO = 3
+MINOR = 12
+MICRO = 1
 # Do not tag as pre-release until this commit
 # https://github.com/pypa/packaging/pull/515
 # has made it into a release. Probably we should wait for pypa/packaging>=21.4

src/caosdb/apiutils.py

@@ -27,21 +27,20 @@
 """
 
 import logging
-import sys
-import tempfile
 import warnings
 
 from collections.abc import Iterable
-from subprocess import call
-from typing import Optional, Any, Dict, List
+from typing import Any, Dict, List
 
-from caosdb.common.datatype import (BOOLEAN, DATETIME, DOUBLE, FILE, INTEGER,
-                                    REFERENCE, TEXT, is_reference)
-from caosdb.common.models import (Container, Entity, File, Property, Query,
+from caosdb.common.datatype import is_reference
+from caosdb.common.models import (Container, Entity, File, Property,
                                   Record, RecordType, execute_query,
-                                  get_config, SPECIAL_ATTRIBUTES)
+                                  SPECIAL_ATTRIBUTES)
 from caosdb.exceptions import CaosDBException
 
+from caosdb.utils.git_utils import (get_origin_url_in, get_diff_in,
+                                    get_branch_in, get_commit_in)
+
 logger = logging.getLogger(__name__)
 
 
@@ -148,51 +147,35 @@ def retrieve_entities_with_ids(entities):
 
 
 def getOriginUrlIn(folder):
-    """return the Fetch URL of the git repository in the given folder."""
-    with tempfile.NamedTemporaryFile(delete=False, mode="w") as t:
-        call(["git", "remote", "show", "origin"], stdout=t, cwd=folder)
-    with open(t.name, "r") as t:
-        urlString = "Fetch URL:"
-
-        for line in t.readlines():
-            if urlString in line:
-                return line[line.find(urlString) + len(urlString):].strip()
-
-    return None
+    warnings.warn("""
+                  This function is deprecated and will be removed with the next release.
+                  Please use the caosdb.utils.git_utils.get_origin_url_in instead.""",
+                  DeprecationWarning)
+    return get_origin_url_in(folder)
 
 
 def getDiffIn(folder, save_dir=None):
-    """returns the name of a file where the out put of "git diff" in the given
-    folder is stored."""
-    with tempfile.NamedTemporaryFile(delete=False, mode="w", dir=save_dir) as t:
-        call(["git", "diff"], stdout=t, cwd=folder)
-
-    return t.name
+    warnings.warn("""
+                  This function is deprecated and will be removed with the next release.
+                  Please use the caosdb.utils.git_utils.get_diff_in instead.""",
+                  DeprecationWarning)
+    return get_diff_in(folder, save_dir)
 
 
 def getBranchIn(folder):
-    """returns the current branch of the git repository in the given folder.
-
-    The command "git branch" is called in the given folder and the
-    output is returned
-    """
-    with tempfile.NamedTemporaryFile(delete=False, mode="w") as t:
-        call(["git", "rev-parse", "--abbrev-ref", "HEAD"], stdout=t, cwd=folder)
-    with open(t.name, "r") as t:
-        return t.readline().strip()
+    warnings.warn("""
+                  This function is deprecated and will be removed with the next release.
+                  Please use the caosdb.utils.git_utils.get_branch_in instead.""",
+                  DeprecationWarning)
+    return get_branch_in(folder)
 
 
 def getCommitIn(folder):
-    """returns the commit hash in of the git repository in the given folder.
-
-    The command "git log -1 --format=%h" is called in the given folder
-    and the output is returned
-    """
-
-    with tempfile.NamedTemporaryFile(delete=False, mode="w") as t:
-        call(["git", "log", "-1", "--format=%h"], stdout=t, cwd=folder)
-    with open(t.name, "r") as t:
-        return t.readline().strip()
+    warnings.warn("""
+                  This function is deprecated and will be removed with the next release.
+                  Please use the caosdb.utils.git_utils.get_commit_in instead.""",
+                  DeprecationWarning)
+    return get_commit_in(folder)
 
 
 def compare_entities(old_entity: Entity, new_entity: Entity, compare_referenced_records: bool = False):

src/caosdb/common/models.py

 # -*- coding: utf-8 -*-
 #
-# ** header v3.0
 # This file is a part of the CaosDB Project.
 #
 # Copyright (C) 2018 Research Group Biomedical Physics,
 # Max-Planck-Institute for Dynamics and Self-Organization Göttingen
-# Copyright (C) 2020-2022 Indiscale GmbH <info@indiscale.com>
-# Copyright (C) 2020 Florian Spreckelsen <f.spreckelsen@indiscale.com>
+# Copyright (C) 2020-2023 Indiscale GmbH <info@indiscale.com>
+# Copyright (C) 2020-2023 Florian Spreckelsen <f.spreckelsen@indiscale.com>
 # Copyright (C) 2020-2022 Timm Fitschen <t.fitschen@indiscale.com>
 #
 # This program is free software: you can redistribute it and/or modify
@@ -22,7 +21,6 @@
 # You should have received a copy of the GNU Affero General Public License
 # along with this program. If not, see <https://www.gnu.org/licenses/>.
 #
-# ** end header
 #
 
 """
@@ -47,6 +45,7 @@ from os import listdir
 from os.path import isdir
 from random import randint
 from tempfile import NamedTemporaryFile
+from typing import Any, Optional
 from warnings import warn
 
 from caosdb.common.datatype import (BOOLEAN, DATETIME, DOUBLE, INTEGER, TEXT,
@@ -453,6 +452,66 @@ class Entity:
 
         return self
 
+    def remove_value_from_property(self, property_name: str, value: Any,
+                                   remove_if_empty_afterwards: Optional[bool] = True):
+        """Remove a value from a property given by name.
+
+        Do nothing if this entity does not have a property of this
+        ``property_name`` or if the property value is different of the given
+        ``value``. By default, the property is removed from this entity if it
+        becomes empty (i.e., value=None) through removal of the value. This
+        behavior can be changed by setting ``remove_if_empty_afterwards`` to
+        ``False`` in which case the property remains.
+
+        Notes
+        -----
+        If the property value is a list and the value to be removed occurs more
+        than once in this list, only its first occurrance is deleted (similar
+        to the behavior of Python's ``list.remove()``.)
+
+        If the property was empty (prop.value == None) before, the property is
+        not removed afterwards even if ``remove_if_empty_afterwards`` is set to
+        ``True``.  Rationale: the property being empty is not an effect of
+        calling this function.
+
+        Parameters
+        ----------
+        property_name : str
+            Name of the property from which the ``value`` will be removed.
+
+        value
+            Value that is to be removed.
+
+        remove_if_empty_afterwards : bool, optional
+            Whether the property shall be removed from this entity if it is
+            emptied by removing the ``value``. Default is ``True``.
+
+        Returns
+        -------
+        self
+            This entity.
+
+        """
+
+        if self.get_property(property_name) is None:
+            return self
+        if self.get_property(property_name).value is None:
+            remove_if_empty_afterwards = False
+        empty_afterwards = False
+        if isinstance(self.get_property(property_name).value, list):
+            if value in self.get_property(property_name).value:
+                self.get_property(property_name).value.remove(value)
+                if self.get_property(property_name).value == []:
+                    self.get_property(property_name).value = None
+                    empty_afterwards = True
+        elif self.get_property(property_name).value == value:
+            self.get_property(property_name).value = None
+            empty_afterwards = True
+        if remove_if_empty_afterwards and empty_afterwards:
+            self.remove_property(property_name)
+
+        return self
+
     def remove_parent(self, parent):
         self.parents.remove(parent)
 

src/caosdb/high_level_api.py

@@ -629,18 +629,20 @@ class CaosDBPythonEntity(object):
         else:
             entity = CaosDBPythonRecord()
 
-        for parent in serialization["parents"]:
-            if "unresolved" in parent:
-                id = None
-                name = None
-                if "id" in parent:
-                    id = parent["id"]
-                if "name" in parent:
-                    name = parent["name"]
-                entity.add_parent(CaosDBPythonUnresolvedParent(
-                    id=id, name=name))
-            else:
-                raise NotImplementedError()
+        if "parents" in serialization:
+            for parent in serialization["parents"]:
+                if "unresolved" in parent:
+                    id = None
+                    name = None
+                    if "id" in parent:
+                        id = parent["id"]
+                    if "name" in parent:
+                        name = parent["name"]
+                    entity.add_parent(CaosDBPythonUnresolvedParent(
+                        id=id, name=name))
+                else:
+                    raise NotImplementedError(
+                        "Currently, only unresolved parents can be deserialized.")
 
         for baseprop in ("name", "id", "description", "version"):
             if baseprop in serialization:
@@ -673,7 +675,8 @@ class CaosDBPythonEntity(object):
                     if f.name in metadata:
                         propmeta.__setattr__(f.name, metadata[f.name])
             else:
-                raise NotImplementedError()
+                pass
+                # raise NotImplementedError()
 
         return entity
 

src/caosdb/utils/git_utils.py

+# -*- coding: utf-8 -*-
+#
+# This file is a part of the CaosDB Project.
+#
+# Copyright (C) 2018 Research Group Biomedical Physics,
+# Max-Planck-Institute for Dynamics and Self-Organization Göttingen
+# Copyright (C) 2020 Timm Fitschen <t.fitschen@indiscale.com>
+# Copyright (C) 2020-2022 IndiScale GmbH <info@indiscale.com>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see <https://www.gnu.org/licenses/>.
+#
+# ** end header
+#
+"""git-utils: Some functions for retrieving information about git repositories.
+
+"""
+
+import logging
+import tempfile
+
+from subprocess import call
+
+logger = logging.getLogger(__name__)
+
+
+def get_origin_url_in(folder: str):
+    """return the Fetch URL of the git repository in the given folder."""
+    with tempfile.NamedTemporaryFile(delete=False, mode="w") as t:
+        call(["git", "remote", "show", "origin"], stdout=t, cwd=folder)
+    with open(t.name, "r") as t:
+        urlString = "Fetch URL:"
+
+        for line in t.readlines():
+            if urlString in line:
+                return line[line.find(urlString) + len(urlString):].strip()
+
+    return None
+
+
+def get_diff_in(folder: str, save_dir=None):
+    """returns the name of a file where the out put of "git diff" in the given
+    folder is stored."""
+    with tempfile.NamedTemporaryFile(delete=False, mode="w", dir=save_dir) as t:
+        call(["git", "diff"], stdout=t, cwd=folder)
+
+    return t.name
+
+
+def get_branch_in(folder: str):
+    """returns the current branch of the git repository in the given folder.
+
+    The command "git branch" is called in the given folder and the
+    output is returned
+    """
+    with tempfile.NamedTemporaryFile(delete=False, mode="w") as t:
+        call(["git", "rev-parse", "--abbrev-ref", "HEAD"], stdout=t, cwd=folder)
+    with open(t.name, "r") as t:
+        return t.readline().strip()
+
+
+def get_commit_in(folder: str):
+    """returns the commit hash in of the git repository in the given folder.
+
+    The command "git log -1 --format=%h" is called in the given folder
+    and the output is returned
+    """
+
+    with tempfile.NamedTemporaryFile(delete=False, mode="w") as t:
+        call(["git", "log", "-1", "--format=%h"], stdout=t, cwd=folder)
+    with open(t.name, "r") as t:
+        return t.readline().strip()

src/doc/conf.py

@@ -29,10 +29,10 @@ copyright = '2023, IndiScale GmbH'
 author = 'Daniel Hornung'
 
 # The short X.Y version
-version = '0.11.3'
+version = '0.12.1'
 # The full version, including alpha/beta/rc tags
 # release = '0.5.2-rc2'
-release = '0.11.3-dev'
+release = '0.12.1-dev'
 
 
 # -- General configuration ---------------------------------------------------

unittests/docker/Dockerfile

-FROM debian:latest
+FROM debian:bullseye
 # Use local package repository
 COPY sources.list.local /etc/apt/
 RUN mv /etc/apt/sources.list /etc/apt/sources.list.orig

unittests/test_property.py

 # -*- encoding: utf-8 -*-
 #
-# ** header v3.0
 # This file is a part of the CaosDB Project.
 #
 # Copyright (C) 2018 Research Group Biomedical Physics,
 # Max-Planck-Institute for Dynamics and Self-Organization Göttingen
-# Copyright (C) 2020 IndiScale GmbH <info@indiscale.com>
+# Copyright (C) 2020 - 2023 IndiScale GmbH <info@indiscale.com>
+# Copyright (C) 2023 Florian Spreckelsen <f.spreckelsen@indiscale.com>
 # Copyright (C) 2020 Timm Fitschen <t.fitschen@indiscale.com>
 #
 # This program is free software: you can redistribute it and/or modify
@@ -21,8 +21,6 @@
 # You should have received a copy of the GNU Affero General Public License
 # along with this program. If not, see <https://www.gnu.org/licenses/>.
 #
-# ** end header
-#
 """Tests for the Property class."""
 import os
 
@@ -138,3 +136,87 @@ def test_is_reference():
 
     # restore retrieve function with original
     Entity.retrieve = real_retrieve
+
+
+def test_remove_value_from_property():
+
+    rec = Record()
+    names_values_dtypes = [
+        ("testListProp1", [1, 2, 3], db.LIST(db.INTEGER)),
+        ("testListProp2", ["a", "b", "a"], db.LIST(db.TEXT)),
+        ("testScalarProp1", "bla", db.TEXT),
+        ("testScalarProp2", False, db.BOOLEAN),
+        ("testEmptyProp", None, db.REFERENCE),
+        ("testNoneListProp", [None, None], db.LIST(db.REFERENCE)),
+    ]
+    for name, value, dtype in names_values_dtypes:
+        rec.add_property(name=name, value=value, datatype=dtype)
+
+    # property doesn't exist, so do nothing
+    returned = rec.remove_value_from_property("nonexisting", "some_value")
+    assert returned is rec
+    for name, value, dtype in names_values_dtypes:
+        assert rec.get_property(name).value == value
+        assert rec.get_property(name).datatype == dtype
+
+    # value doesn't exist so nothing changes either
+    rec.remove_value_from_property("testListProp1", 0)
+    assert rec.get_property("testListProp1").value == [1, 2, 3]
+    assert rec.get_property("testListProp1").datatype == db.LIST(db.INTEGER)
+
+    returned = rec.remove_value_from_property("testScalarProp2", True)
+    assert returned is rec
+    assert rec.get_property("testScalarProp2").value is False
+    assert rec.get_property("testScalarProp2").datatype == db.BOOLEAN
+
+    # Simple removals from lists without emptying them
+    rec.remove_value_from_property("testListProp1", 1)
+    assert rec.get_property("testListProp1").value == [2, 3]
+
+    rec.remove_value_from_property("testListProp1", 2)
+    assert rec.get_property("testListProp1").value == [3]
+
+    # similarly to Python's `list.remove()`, only remove first occurrance
+    rec.remove_value_from_property("testListProp2", "a")
+    assert rec.get_property("testListProp2").value == ["b", "a"]
+
+    # default is to remove an empty property:
+    rec.remove_value_from_property("testListProp1", 3)
+    assert rec.get_property("testListProp1") is None
+
+    rec.remove_value_from_property("testScalarProp1", "bla")
+    assert rec.get_property("testScalarProp1") is None
+
+    # don't remove if `remove_if_empty_afterwards=False`
+    rec.remove_value_from_property("testListProp2", "b")
+    rec.remove_value_from_property("testListProp2", "a", remove_if_empty_afterwards=False)
+    assert rec.get_property("testListProp2") is not None
+    assert rec.get_property("testListProp2").value is None
+    assert rec.get_property("testListProp2").datatype == db.LIST(db.TEXT)
+
+    rec.remove_value_from_property("testScalarProp2", False, remove_if_empty_afterwards=False)
+    assert rec.get_property("testScalarProp2") is not None
+    assert rec.get_property("testScalarProp2").value is None
+    assert rec.get_property("testScalarProp2").datatype == db.BOOLEAN
+
+    # Special case of an already empty property: It is not empty because a value
+    # was removed by `remove_value_from_property` but never had a value in the
+    # first place. So even `remove_if_empty_afterwards=True` should not lead to
+    # its removal.
+    rec.remove_value_from_property("testEmptyProp", 1234, remove_if_empty_afterwards=True)
+    assert rec.get_property("testEmptyProp") is not None
+    assert rec.get_property("testEmptyProp").value is None
+    assert rec.get_property("testEmptyProp").datatype == db.REFERENCE
+
+    # Corner case of corner case: remove with `value=None` and
+    # `remove_if_empty_afterwards=True` keeps the empty property.
+    rec.remove_value_from_property("testEmptyProp", None, remove_if_empty_afterwards=True)
+    assert rec.get_property("testEmptyProp") is not None
+    assert rec.get_property("testEmptyProp").value is None
+    assert rec.get_property("testEmptyProp").datatype == db.REFERENCE
+
+    # Remove `None` from list `[None, None]`
+    rec.remove_value_from_property("testNoneListProp", None, remove_if_empty_afterwards=True)
+    assert rec.get_property("testNoneListProp") is not None
+    assert rec.get_property("testNoneListProp").value == [None]
+    assert rec.get_property("testNoneListProp").datatype == db.LIST(db.REFERENCE)