diff --git a/CHANGELOG.md b/CHANGELOG.md
index d6f7c5f123942a22cc7de63424bd7fb7ea597569..8c9ed3081d083c1a78c14b590cb4b0e716478e71 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -8,9 +8,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
## [Unreleased] ##
### Added ###
-- Added location argument to `src/caosdb/utils/checkFileSystemConsistency.py`
-- Entity getters: `get_entity_by_<name/id/path>`
-- Cached versions of entity getters and of `execute_query` (`cached_query`)
+
+* `Entity.remove_value_from_property` function that removes a given value from a
+ property and optionally removes the property if it is empty afterwards.
### Changed ###
@@ -19,12 +19,30 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
### Removed ###
### Fixed ###
-- Fixed `src/caosdb/utils/checkFileSystemConsistency.py`
### Security ###
### Documentation ###
+## [0.12.0] - 2023-06-02 ##
+
+### Added ###
+
+- Added location argument to `src/caosdb/utils/checkFileSystemConsistency.py`
+- Entity getters: `get_entity_by_<name/id/path>`
+- Cached versions of entity getters and of `execute_query` (`cached_query`)
+
+### Deprecated ###
+
+- getOriginUrlIn, getDiffIn, getBranchIn, getCommitIn (formerly apiutils) have been
+ moved to caosdb.utils.git_utils
+
+### Fixed ###
+
+- Fixed `src/caosdb/utils/checkFileSystemConsistency.py`
+
+### Documentation ###
+
* [#83](https://gitlab.com/caosdb/caosdb-pylib/-/issues/83) - Improved
documentation on adding REFERENCE properties, both in the docstring of
`Entity.add_property` and in the data-insertion tutorial.
@@ -32,7 +50,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
## [0.11.2] - 2023-03-14 ##
### Fixed ###
-- root logger is no longer used to create warnings. Fixes undesired output in
+- root logger is no longer used to create warnings. Fixes undesired output in
stderr
## [0.11.1] - 2023-03-07 ##
diff --git a/CITATION.cff b/CITATION.cff
index 910e40a2193d527fc8e4eb68c4ca6b10a28d3630..d9126aae6483459f8c8f248ed6a4fdf859f24e45 100644
--- a/CITATION.cff
+++ b/CITATION.cff
@@ -20,6 +20,6 @@ authors:
given-names: Stefan
orcid: https://orcid.org/0000-0001-7214-8125
title: CaosDB - Pylib
-version: 0.11.1
+version: 0.12.0
doi: 10.3390/data4020083
-date-released: 2022-11-14
\ No newline at end of file
+date-released: 2023-06-02
\ No newline at end of file
diff --git a/setup.py b/setup.py
index a8b948c1c9097ea49d391d0fe0747290d21be4a6..8fdf3b1c63322ec48af398d1dcb1c4028355d473 100755
--- a/setup.py
+++ b/setup.py
@@ -47,8 +47,8 @@ from setuptools import find_packages, setup
ISRELEASED = False
MAJOR = 0
-MINOR = 11
-MICRO = 3
+MINOR = 12
+MICRO = 1
# Do not tag as pre-release until this commit
# https://github.com/pypa/packaging/pull/515
# has made it into a release. Probably we should wait for pypa/packaging>=21.4
diff --git a/src/caosdb/apiutils.py b/src/caosdb/apiutils.py
index a5a936c556dd065b56b60ff690baf9a1ce19a583..a46e30375b924d358448e73aece61562c36c700b 100644
--- a/src/caosdb/apiutils.py
+++ b/src/caosdb/apiutils.py
@@ -27,21 +27,20 @@
"""
import logging
-import sys
-import tempfile
import warnings
from collections.abc import Iterable
-from subprocess import call
-from typing import Optional, Any, Dict, List
+from typing import Any, Dict, List
-from caosdb.common.datatype import (BOOLEAN, DATETIME, DOUBLE, FILE, INTEGER,
- REFERENCE, TEXT, is_reference)
-from caosdb.common.models import (Container, Entity, File, Property, Query,
+from caosdb.common.datatype import is_reference
+from caosdb.common.models import (Container, Entity, File, Property,
Record, RecordType, execute_query,
- get_config, SPECIAL_ATTRIBUTES)
+ SPECIAL_ATTRIBUTES)
from caosdb.exceptions import CaosDBException
+from caosdb.utils.git_utils import (get_origin_url_in, get_diff_in,
+ get_branch_in, get_commit_in)
+
logger = logging.getLogger(__name__)
@@ -148,51 +147,35 @@ def retrieve_entities_with_ids(entities):
def getOriginUrlIn(folder):
- """return the Fetch URL of the git repository in the given folder."""
- with tempfile.NamedTemporaryFile(delete=False, mode="w") as t:
- call(["git", "remote", "show", "origin"], stdout=t, cwd=folder)
- with open(t.name, "r") as t:
- urlString = "Fetch URL:"
-
- for line in t.readlines():
- if urlString in line:
- return line[line.find(urlString) + len(urlString):].strip()
-
- return None
+ warnings.warn("""
+ This function is deprecated and will be removed with the next release.
+ Please use the caosdb.utils.git_utils.get_origin_url_in instead.""",
+ DeprecationWarning)
+ return get_origin_url_in(folder)
def getDiffIn(folder, save_dir=None):
- """returns the name of a file where the out put of "git diff" in the given
- folder is stored."""
- with tempfile.NamedTemporaryFile(delete=False, mode="w", dir=save_dir) as t:
- call(["git", "diff"], stdout=t, cwd=folder)
-
- return t.name
+ warnings.warn("""
+ This function is deprecated and will be removed with the next release.
+ Please use the caosdb.utils.git_utils.get_diff_in instead.""",
+ DeprecationWarning)
+ return get_diff_in(folder, save_dir)
def getBranchIn(folder):
- """returns the current branch of the git repository in the given folder.
-
- The command "git branch" is called in the given folder and the
- output is returned
- """
- with tempfile.NamedTemporaryFile(delete=False, mode="w") as t:
- call(["git", "rev-parse", "--abbrev-ref", "HEAD"], stdout=t, cwd=folder)
- with open(t.name, "r") as t:
- return t.readline().strip()
+ warnings.warn("""
+ This function is deprecated and will be removed with the next release.
+ Please use the caosdb.utils.git_utils.get_branch_in instead.""",
+ DeprecationWarning)
+ return get_branch_in(folder)
def getCommitIn(folder):
- """returns the commit hash in of the git repository in the given folder.
-
- The command "git log -1 --format=%h" is called in the given folder
- and the output is returned
- """
-
- with tempfile.NamedTemporaryFile(delete=False, mode="w") as t:
- call(["git", "log", "-1", "--format=%h"], stdout=t, cwd=folder)
- with open(t.name, "r") as t:
- return t.readline().strip()
+ warnings.warn("""
+ This function is deprecated and will be removed with the next release.
+ Please use the caosdb.utils.git_utils.get_commit_in instead.""",
+ DeprecationWarning)
+ return get_commit_in(folder)
def compare_entities(old_entity: Entity, new_entity: Entity, compare_referenced_records: bool = False):
diff --git a/src/caosdb/common/models.py b/src/caosdb/common/models.py
index 40434b9dab03e136bfdaab42615d005f9a5bb17a..8f7a73d85bba41fc2c07c8f675ac659180d29194 100644
--- a/src/caosdb/common/models.py
+++ b/src/caosdb/common/models.py
@@ -1,12 +1,11 @@
# -*- coding: utf-8 -*-
#
-# ** header v3.0
# This file is a part of the CaosDB Project.
#
# Copyright (C) 2018 Research Group Biomedical Physics,
# Max-Planck-Institute for Dynamics and Self-Organization Göttingen
-# Copyright (C) 2020-2022 Indiscale GmbH <info@indiscale.com>
-# Copyright (C) 2020 Florian Spreckelsen <f.spreckelsen@indiscale.com>
+# Copyright (C) 2020-2023 Indiscale GmbH <info@indiscale.com>
+# Copyright (C) 2020-2023 Florian Spreckelsen <f.spreckelsen@indiscale.com>
# Copyright (C) 2020-2022 Timm Fitschen <t.fitschen@indiscale.com>
#
# This program is free software: you can redistribute it and/or modify
@@ -22,7 +21,6 @@
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <https://www.gnu.org/licenses/>.
#
-# ** end header
#
"""
@@ -47,6 +45,7 @@ from os import listdir
from os.path import isdir
from random import randint
from tempfile import NamedTemporaryFile
+from typing import Any, Optional
from warnings import warn
from caosdb.common.datatype import (BOOLEAN, DATETIME, DOUBLE, INTEGER, TEXT,
@@ -453,6 +452,66 @@ class Entity:
return self
+ def remove_value_from_property(self, property_name: str, value: Any,
+ remove_if_empty_afterwards: Optional[bool] = True):
+ """Remove a value from a property given by name.
+
+ Do nothing if this entity does not have a property of this
+ ``property_name`` or if the property value is different of the given
+ ``value``. By default, the property is removed from this entity if it
+ becomes empty (i.e., value=None) through removal of the value. This
+ behavior can be changed by setting ``remove_if_empty_afterwards`` to
+ ``False`` in which case the property remains.
+
+ Notes
+ -----
+ If the property value is a list and the value to be removed occurs more
+ than once in this list, only its first occurrance is deleted (similar
+ to the behavior of Python's ``list.remove()``.)
+
+ If the property was empty (prop.value == None) before, the property is
+ not removed afterwards even if ``remove_if_empty_afterwards`` is set to
+ ``True``. Rationale: the property being empty is not an effect of
+ calling this function.
+
+ Parameters
+ ----------
+ property_name : str
+ Name of the property from which the ``value`` will be removed.
+
+ value
+ Value that is to be removed.
+
+ remove_if_empty_afterwards : bool, optional
+ Whether the property shall be removed from this entity if it is
+ emptied by removing the ``value``. Default is ``True``.
+
+ Returns
+ -------
+ self
+ This entity.
+
+ """
+
+ if self.get_property(property_name) is None:
+ return self
+ if self.get_property(property_name).value is None:
+ remove_if_empty_afterwards = False
+ empty_afterwards = False
+ if isinstance(self.get_property(property_name).value, list):
+ if value in self.get_property(property_name).value:
+ self.get_property(property_name).value.remove(value)
+ if self.get_property(property_name).value == []:
+ self.get_property(property_name).value = None
+ empty_afterwards = True
+ elif self.get_property(property_name).value == value:
+ self.get_property(property_name).value = None
+ empty_afterwards = True
+ if remove_if_empty_afterwards and empty_afterwards:
+ self.remove_property(property_name)
+
+ return self
+
def remove_parent(self, parent):
self.parents.remove(parent)
diff --git a/src/caosdb/high_level_api.py b/src/caosdb/high_level_api.py
index 427a095a4bafc0c372b0169298f2980dbd902c49..005a20bbba26fd5bee16eac612bd8ebe81f1294a 100644
--- a/src/caosdb/high_level_api.py
+++ b/src/caosdb/high_level_api.py
@@ -629,18 +629,20 @@ class CaosDBPythonEntity(object):
else:
entity = CaosDBPythonRecord()
- for parent in serialization["parents"]:
- if "unresolved" in parent:
- id = None
- name = None
- if "id" in parent:
- id = parent["id"]
- if "name" in parent:
- name = parent["name"]
- entity.add_parent(CaosDBPythonUnresolvedParent(
- id=id, name=name))
- else:
- raise NotImplementedError()
+ if "parents" in serialization:
+ for parent in serialization["parents"]:
+ if "unresolved" in parent:
+ id = None
+ name = None
+ if "id" in parent:
+ id = parent["id"]
+ if "name" in parent:
+ name = parent["name"]
+ entity.add_parent(CaosDBPythonUnresolvedParent(
+ id=id, name=name))
+ else:
+ raise NotImplementedError(
+ "Currently, only unresolved parents can be deserialized.")
for baseprop in ("name", "id", "description", "version"):
if baseprop in serialization:
@@ -673,7 +675,8 @@ class CaosDBPythonEntity(object):
if f.name in metadata:
propmeta.__setattr__(f.name, metadata[f.name])
else:
- raise NotImplementedError()
+ pass
+ # raise NotImplementedError()
return entity
diff --git a/src/caosdb/utils/git_utils.py b/src/caosdb/utils/git_utils.py
new file mode 100644
index 0000000000000000000000000000000000000000..7a58272a3bef1930f75a1e08364349388e2bb89f
--- /dev/null
+++ b/src/caosdb/utils/git_utils.py
@@ -0,0 +1,82 @@
+# -*- coding: utf-8 -*-
+#
+# This file is a part of the CaosDB Project.
+#
+# Copyright (C) 2018 Research Group Biomedical Physics,
+# Max-Planck-Institute for Dynamics and Self-Organization Göttingen
+# Copyright (C) 2020 Timm Fitschen <t.fitschen@indiscale.com>
+# Copyright (C) 2020-2022 IndiScale GmbH <info@indiscale.com>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see <https://www.gnu.org/licenses/>.
+#
+# ** end header
+#
+"""git-utils: Some functions for retrieving information about git repositories.
+
+"""
+
+import logging
+import tempfile
+
+from subprocess import call
+
+logger = logging.getLogger(__name__)
+
+
+def get_origin_url_in(folder: str):
+ """return the Fetch URL of the git repository in the given folder."""
+ with tempfile.NamedTemporaryFile(delete=False, mode="w") as t:
+ call(["git", "remote", "show", "origin"], stdout=t, cwd=folder)
+ with open(t.name, "r") as t:
+ urlString = "Fetch URL:"
+
+ for line in t.readlines():
+ if urlString in line:
+ return line[line.find(urlString) + len(urlString):].strip()
+
+ return None
+
+
+def get_diff_in(folder: str, save_dir=None):
+ """returns the name of a file where the out put of "git diff" in the given
+ folder is stored."""
+ with tempfile.NamedTemporaryFile(delete=False, mode="w", dir=save_dir) as t:
+ call(["git", "diff"], stdout=t, cwd=folder)
+
+ return t.name
+
+
+def get_branch_in(folder: str):
+ """returns the current branch of the git repository in the given folder.
+
+ The command "git branch" is called in the given folder and the
+ output is returned
+ """
+ with tempfile.NamedTemporaryFile(delete=False, mode="w") as t:
+ call(["git", "rev-parse", "--abbrev-ref", "HEAD"], stdout=t, cwd=folder)
+ with open(t.name, "r") as t:
+ return t.readline().strip()
+
+
+def get_commit_in(folder: str):
+ """returns the commit hash in of the git repository in the given folder.
+
+ The command "git log -1 --format=%h" is called in the given folder
+ and the output is returned
+ """
+
+ with tempfile.NamedTemporaryFile(delete=False, mode="w") as t:
+ call(["git", "log", "-1", "--format=%h"], stdout=t, cwd=folder)
+ with open(t.name, "r") as t:
+ return t.readline().strip()
diff --git a/src/doc/conf.py b/src/doc/conf.py
index 819ef61d7fb02e752b4a73a86644d1602bbf188a..0fa5de575f5424e267cad8ecc193cca8230faa8b 100644
--- a/src/doc/conf.py
+++ b/src/doc/conf.py
@@ -29,10 +29,10 @@ copyright = '2023, IndiScale GmbH'
author = 'Daniel Hornung'
# The short X.Y version
-version = '0.11.3'
+version = '0.12.1'
# The full version, including alpha/beta/rc tags
# release = '0.5.2-rc2'
-release = '0.11.3-dev'
+release = '0.12.1-dev'
# -- General configuration ---------------------------------------------------
diff --git a/unittests/docker/Dockerfile b/unittests/docker/Dockerfile
index 06f9d6c830068a2c1c85caef79c64f899eaefb33..7c84050b0a55ae6e1e8f2e2583f894a69f691193 100644
--- a/unittests/docker/Dockerfile
+++ b/unittests/docker/Dockerfile
@@ -1,4 +1,4 @@
-FROM debian:latest
+FROM debian:bullseye
# Use local package repository
COPY sources.list.local /etc/apt/
RUN mv /etc/apt/sources.list /etc/apt/sources.list.orig
diff --git a/unittests/test_property.py b/unittests/test_property.py
index 7c756117765e510587c00d818e39fb3945d44c53..84f89b5a959192d7831e1bb3eab3a441912afe7e 100644
--- a/unittests/test_property.py
+++ b/unittests/test_property.py
@@ -1,11 +1,11 @@
# -*- encoding: utf-8 -*-
#
-# ** header v3.0
# This file is a part of the CaosDB Project.
#
# Copyright (C) 2018 Research Group Biomedical Physics,
# Max-Planck-Institute for Dynamics and Self-Organization Göttingen
-# Copyright (C) 2020 IndiScale GmbH <info@indiscale.com>
+# Copyright (C) 2020 - 2023 IndiScale GmbH <info@indiscale.com>
+# Copyright (C) 2023 Florian Spreckelsen <f.spreckelsen@indiscale.com>
# Copyright (C) 2020 Timm Fitschen <t.fitschen@indiscale.com>
#
# This program is free software: you can redistribute it and/or modify
@@ -21,8 +21,6 @@
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <https://www.gnu.org/licenses/>.
#
-# ** end header
-#
"""Tests for the Property class."""
import os
@@ -138,3 +136,87 @@ def test_is_reference():
# restore retrieve function with original
Entity.retrieve = real_retrieve
+
+
+def test_remove_value_from_property():
+
+ rec = Record()
+ names_values_dtypes = [
+ ("testListProp1", [1, 2, 3], db.LIST(db.INTEGER)),
+ ("testListProp2", ["a", "b", "a"], db.LIST(db.TEXT)),
+ ("testScalarProp1", "bla", db.TEXT),
+ ("testScalarProp2", False, db.BOOLEAN),
+ ("testEmptyProp", None, db.REFERENCE),
+ ("testNoneListProp", [None, None], db.LIST(db.REFERENCE)),
+ ]
+ for name, value, dtype in names_values_dtypes:
+ rec.add_property(name=name, value=value, datatype=dtype)
+
+ # property doesn't exist, so do nothing
+ returned = rec.remove_value_from_property("nonexisting", "some_value")
+ assert returned is rec
+ for name, value, dtype in names_values_dtypes:
+ assert rec.get_property(name).value == value
+ assert rec.get_property(name).datatype == dtype
+
+ # value doesn't exist so nothing changes either
+ rec.remove_value_from_property("testListProp1", 0)
+ assert rec.get_property("testListProp1").value == [1, 2, 3]
+ assert rec.get_property("testListProp1").datatype == db.LIST(db.INTEGER)
+
+ returned = rec.remove_value_from_property("testScalarProp2", True)
+ assert returned is rec
+ assert rec.get_property("testScalarProp2").value is False
+ assert rec.get_property("testScalarProp2").datatype == db.BOOLEAN
+
+ # Simple removals from lists without emptying them
+ rec.remove_value_from_property("testListProp1", 1)
+ assert rec.get_property("testListProp1").value == [2, 3]
+
+ rec.remove_value_from_property("testListProp1", 2)
+ assert rec.get_property("testListProp1").value == [3]
+
+ # similarly to Python's `list.remove()`, only remove first occurrance
+ rec.remove_value_from_property("testListProp2", "a")
+ assert rec.get_property("testListProp2").value == ["b", "a"]
+
+ # default is to remove an empty property:
+ rec.remove_value_from_property("testListProp1", 3)
+ assert rec.get_property("testListProp1") is None
+
+ rec.remove_value_from_property("testScalarProp1", "bla")
+ assert rec.get_property("testScalarProp1") is None
+
+ # don't remove if `remove_if_empty_afterwards=False`
+ rec.remove_value_from_property("testListProp2", "b")
+ rec.remove_value_from_property("testListProp2", "a", remove_if_empty_afterwards=False)
+ assert rec.get_property("testListProp2") is not None
+ assert rec.get_property("testListProp2").value is None
+ assert rec.get_property("testListProp2").datatype == db.LIST(db.TEXT)
+
+ rec.remove_value_from_property("testScalarProp2", False, remove_if_empty_afterwards=False)
+ assert rec.get_property("testScalarProp2") is not None
+ assert rec.get_property("testScalarProp2").value is None
+ assert rec.get_property("testScalarProp2").datatype == db.BOOLEAN
+
+ # Special case of an already empty property: It is not empty because a value
+ # was removed by `remove_value_from_property` but never had a value in the
+ # first place. So even `remove_if_empty_afterwards=True` should not lead to
+ # its removal.
+ rec.remove_value_from_property("testEmptyProp", 1234, remove_if_empty_afterwards=True)
+ assert rec.get_property("testEmptyProp") is not None
+ assert rec.get_property("testEmptyProp").value is None
+ assert rec.get_property("testEmptyProp").datatype == db.REFERENCE
+
+ # Corner case of corner case: remove with `value=None` and
+ # `remove_if_empty_afterwards=True` keeps the empty property.
+ rec.remove_value_from_property("testEmptyProp", None, remove_if_empty_afterwards=True)
+ assert rec.get_property("testEmptyProp") is not None
+ assert rec.get_property("testEmptyProp").value is None
+ assert rec.get_property("testEmptyProp").datatype == db.REFERENCE
+
+ # Remove `None` from list `[None, None]`
+ rec.remove_value_from_property("testNoneListProp", None, remove_if_empty_afterwards=True)
+ assert rec.get_property("testNoneListProp") is not None
+ assert rec.get_property("testNoneListProp").value == [None]
+ assert rec.get_property("testNoneListProp").datatype == db.LIST(db.REFERENCE)