From ce4810be91684d299e2ab8fea77c18e4a361302a Mon Sep 17 00:00:00 2001
From: Florian Spreckelsen <f.spreckelsen@indiscale.com>
Date: Wed, 13 Nov 2024 10:02:26 +0100
Subject: [PATCH 01/62] REL: Prepare next release cycle
---
CHANGELOG.md | 16 ++++++++++++++++
setup.py | 4 ++--
src/doc/conf.py | 4 ++--
3 files changed, 20 insertions(+), 4 deletions(-)
diff --git a/CHANGELOG.md b/CHANGELOG.md
index de809931..bedb968d 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -5,6 +5,22 @@ All notable changes to this project will be documented in this file.
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
+## [Unreleased] ##
+
+### Added ###
+
+### Changed ###
+
+### Deprecated ###
+
+### Removed ###
+
+### Fixed ###
+
+### Security ###
+
+### Documentation ###
+
## [0.16.0] - 2024-11-13 ##
### Added ###
diff --git a/setup.py b/setup.py
index b8a04ade..8b0f7ef0 100755
--- a/setup.py
+++ b/setup.py
@@ -46,10 +46,10 @@ from setuptools import find_packages, setup
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
########################################################################
-ISRELEASED = True
+ISRELEASED = False
MAJOR = 0
MINOR = 16
-MICRO = 0
+MICRO = 1
# Do not tag as pre-release until this commit
# https://github.com/pypa/packaging/pull/515
# has made it into a release. Probably we should wait for pypa/packaging>=21.4
diff --git a/src/doc/conf.py b/src/doc/conf.py
index f25ed399..80d5e8a2 100644
--- a/src/doc/conf.py
+++ b/src/doc/conf.py
@@ -29,10 +29,10 @@ copyright = '2024, IndiScale GmbH'
author = 'Daniel Hornung'
# The short X.Y version
-version = '0.16.0'
+version = '0.16.1'
# The full version, including alpha/beta/rc tags
# release = '0.5.2-rc2'
-release = '0.16.0'
+release = '0.16.1-dev'
# -- General configuration ---------------------------------------------------
--
GitLab
From c5e483708bd1bf70bf6a46468fa9354790dfd432 Mon Sep 17 00:00:00 2001
From: "i.nueske" <i.nueske@indiscale.com>
Date: Sat, 16 Nov 2024 12:39:40 +0100
Subject: [PATCH 02/62] BUG: Container.to_xml() now removes noscript elements
to ensure compatibility with from_xml
---
src/linkahead/common/models.py | 6 ++++++
1 file changed, 6 insertions(+)
diff --git a/src/linkahead/common/models.py b/src/linkahead/common/models.py
index 1dbeb802..0dee8239 100644
--- a/src/linkahead/common/models.py
+++ b/src/linkahead/common/models.py
@@ -3284,6 +3284,7 @@ class Container(list):
if add_to_element is None:
add_to_element = etree.Element("Entities")
+ noscript_in_supplied_xml = list(add_to_element.iter("noscript"))
for m in self.messages:
add_to_element.append(m.to_xml())
@@ -3300,6 +3301,11 @@ class Container(list):
elem = e.to_xml()
add_to_element.append(elem)
+ # remove noscript elements added by this function
+ for elem in list(add_to_element.iter("noscript")):
+ if elem not in noscript_in_supplied_xml:
+ elem.getparent().remove(elem)
+
return add_to_element
def get_errors(self):
--
GitLab
From dbddca1e6b0913027d1af7acdaefbb822c6163a4 Mon Sep 17 00:00:00 2001
From: "i.nueske" <i.nueske@indiscale.com>
Date: Sat, 16 Nov 2024 16:54:41 +0100
Subject: [PATCH 03/62] BUG: Request responses without the "Set-Cookie" header
no longer overwrite the current auth_token with None
---
src/linkahead/connection/authentication/interface.py | 5 +++--
1 file changed, 3 insertions(+), 2 deletions(-)
diff --git a/src/linkahead/connection/authentication/interface.py b/src/linkahead/connection/authentication/interface.py
index b48e27c0..f5033fad 100644
--- a/src/linkahead/connection/authentication/interface.py
+++ b/src/linkahead/connection/authentication/interface.py
@@ -125,8 +125,9 @@ class AbstractAuthenticator(ABC):
Returns
-------
"""
- self.auth_token = parse_auth_token(
- response.getheader("Set-Cookie"))
+ new_token = parse_auth_token(response.getheader("Set-Cookie"))
+ if new_token is not None:
+ self.auth_token = new_token
def on_request(self, method: str, path: str, headers: QueryDict, **kwargs):
# pylint: disable=unused-argument
--
GitLab
From 6321ed77af9119c5b69881d27a715252b8119e21 Mon Sep 17 00:00:00 2001
From: "i.nueske" <i.nueske@indiscale.com>
Date: Tue, 19 Nov 2024 16:02:00 +0100
Subject: [PATCH 04/62] TST: Added support for logout to MockUpServerConnection
and tests to fix test_auth_token_connection in test_connection.py
---
src/linkahead/connection/mockup.py | 8 +++++++-
unittests/test_authentication_auth_token.py | 2 +-
unittests/test_connection.py | 4 ++--
3 files changed, 10 insertions(+), 4 deletions(-)
diff --git a/src/linkahead/connection/mockup.py b/src/linkahead/connection/mockup.py
index 9b69971c..d3bc13bb 100644
--- a/src/linkahead/connection/mockup.py
+++ b/src/linkahead/connection/mockup.py
@@ -75,7 +75,7 @@ class MockUpServerConnection(CaosDBServerConnection):
just returns predefined responses which mimic the LinkAhead server."""
def __init__(self):
- self.resources = [self._login]
+ self.resources = [self._login, self._logout]
def _login(self, method, path, headers, body):
if method == "POST" and path == "login":
@@ -84,6 +84,12 @@ class MockUpServerConnection(CaosDBServerConnection):
"mockup-auth-token"},
body="")
+ def _logout(self, method, path, headers, body):
+ if method in ["DELETE", "GET"] and path == "logout":
+ return MockUpResponse(200,
+ headers={},
+ body="")
+
def configure(self, **kwargs):
"""This configure method does nothing."""
diff --git a/unittests/test_authentication_auth_token.py b/unittests/test_authentication_auth_token.py
index 3142f1f9..4eb17bcc 100644
--- a/unittests/test_authentication_auth_token.py
+++ b/unittests/test_authentication_auth_token.py
@@ -96,6 +96,6 @@ def test_logout_calls_delete():
auth_token="[request token]",
implementation=MockUpServerConnection)
- c._delegate_connection.resources.append(logout_resource)
+ c._delegate_connection.resources.insert(1, logout_resource)
c._logout()
mock.method.assert_called_once()
diff --git a/unittests/test_connection.py b/unittests/test_connection.py
index a3a1eff7..d00d852e 100644
--- a/unittests/test_connection.py
+++ b/unittests/test_connection.py
@@ -216,9 +216,9 @@ def test_init_connection():
def test_resources_list():
connection = test_init_connection()
assert hasattr(connection, "resources")
- assert len(connection.resources) == 1
- connection.resources.append(lambda **kwargs: test_init_response())
assert len(connection.resources) == 2
+ connection.resources.append(lambda **kwargs: test_init_response())
+ assert len(connection.resources) == 3
return connection
--
GitLab
From 285eeb0dfdb2c79338f5af1f70d92cabd66832c2 Mon Sep 17 00:00:00 2001
From: "i.nueske" <i.nueske@indiscale.com>
Date: Fri, 22 Nov 2024 08:45:31 +0100
Subject: [PATCH 05/62] BUG: to_xml now also filters out TransactionBenchmark
tags, root being noscript or TransactionBenchmark does not cause error
---
src/linkahead/common/models.py | 10 ++++++----
1 file changed, 6 insertions(+), 4 deletions(-)
diff --git a/src/linkahead/common/models.py b/src/linkahead/common/models.py
index 0dee8239..6204e119 100644
--- a/src/linkahead/common/models.py
+++ b/src/linkahead/common/models.py
@@ -3284,7 +3284,7 @@ class Container(list):
if add_to_element is None:
add_to_element = etree.Element("Entities")
- noscript_in_supplied_xml = list(add_to_element.iter("noscript"))
+ noscript_in_supplied_xml = list(add_to_element.iter("noscript", "TransactionBenchmark"))
for m in self.messages:
add_to_element.append(m.to_xml())
@@ -3301,10 +3301,12 @@ class Container(list):
elem = e.to_xml()
add_to_element.append(elem)
- # remove noscript elements added by this function
- for elem in list(add_to_element.iter("noscript")):
+ # remove noscript and benchmark elements added by this function
+ for elem in list(add_to_element.iter("noscript", "TransactionBenchmark")):
if elem not in noscript_in_supplied_xml:
- elem.getparent().remove(elem)
+ parent = elem.getparent()
+ if parent is not None:
+ parent.remove(elem)
return add_to_element
--
GitLab
From 1d81b1d5257d4f6caeee203a133095bfeec2ea6f Mon Sep 17 00:00:00 2001
From: "i.nueske" <i.nueske@indiscale.com>
Date: Fri, 22 Nov 2024 09:16:23 +0100
Subject: [PATCH 06/62] MNT: Update Changelog
---
CHANGELOG.md | 2 ++
1 file changed, 2 insertions(+)
diff --git a/CHANGELOG.md b/CHANGELOG.md
index bedb968d..40da08b0 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -16,6 +16,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
### Removed ###
### Fixed ###
+* [#89](https://gitlab.com/linkahead/linkahead-pylib/-/issues/89)
+ `to_xml` does not add `noscript` or `TransactionBenchmark` tags anymore
### Security ###
--
GitLab
From 152377eb19d733f68f29fcd572b6e9f1da1052d1 Mon Sep 17 00:00:00 2001
From: "i.nueske" <i.nueske@indiscale.com>
Date: Fri, 22 Nov 2024 11:59:28 +0100
Subject: [PATCH 07/62] MNT: Update Changelog, fix link
---
CHANGELOG.md | 5 ++++-
1 file changed, 4 insertions(+), 1 deletion(-)
diff --git a/CHANGELOG.md b/CHANGELOG.md
index bedb968d..17885f75 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -16,6 +16,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
### Removed ###
### Fixed ###
+* [#103](https://gitlab.com/linkahead/linkahead-pylib/-/issues/103)
+ `authentication/interface/on_response()` does not overwrite
+ `auth_token` if new value is `None`
### Security ###
@@ -47,7 +50,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
### Fixed ###
-* [gitlab.indiscale.com#200](https://gitlab.indiscale.com/caosdb/src/caosdb-pylib/-/merge_requests/153)
+* [gitlab.indiscale.com#200](https://gitlab.indiscale.com/caosdb/src/caosdb-pylib/-/issues/200)
``linkahead_admin.py`` prints reasonable error messages when users
or roles don't exist.
--
GitLab
From 96fee41aed6ec97b53250f90fc71fb6952d96d8c Mon Sep 17 00:00:00 2001
From: "i.nueske" <i.nueske@indiscale.com>
Date: Fri, 22 Nov 2024 12:24:36 +0100
Subject: [PATCH 08/62] MNT: Adjust logout request to match documentation
---
src/linkahead/connection/authentication/interface.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/src/linkahead/connection/authentication/interface.py b/src/linkahead/connection/authentication/interface.py
index f5033fad..82888805 100644
--- a/src/linkahead/connection/authentication/interface.py
+++ b/src/linkahead/connection/authentication/interface.py
@@ -191,7 +191,7 @@ class CredentialsAuthenticator(AbstractAuthenticator):
def _logout(self):
self.logger.debug("[LOGOUT]")
if self.auth_token is not None:
- self._connection.request(method="DELETE", path="logout")
+ self._connection.request(method="GET", path="logout")
self.auth_token = None
def _login(self):
--
GitLab
From 5449a72659d573f57599eb980eed975f156bc861 Mon Sep 17 00:00:00 2001
From: Daniel <d.hornung@indiscale.com>
Date: Fri, 22 Nov 2024 16:39:33 +0100
Subject: [PATCH 09/62] WIP: Modernizing setup.
---
.gitlab-ci.yml | 3 +--
setup.py | 14 ++++++++++----
2 files changed, 11 insertions(+), 6 deletions(-)
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 8845e407..57af3f4f 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -70,8 +70,7 @@ unittest_py3.8:
script: &python_test_script
# Python docker has problems with tox and pip so use plain pytest here
- touch ~/.pylinkahead.ini
- - pip install pynose pytest pytest-cov jsonschema>=4.4.0 setuptools
- - pip install .
+ - pip install .[test]
- python -m pytest unittests
# This needs to be changed once Python 3.9 isn't the standard Python in Debian
diff --git a/setup.py b/setup.py
index 8b0f7ef0..7e3dfb93 100755
--- a/setup.py
+++ b/setup.py
@@ -187,11 +187,17 @@ def setup_package():
'PyYAML>=5.4.1',
'future',
],
- extras_require={'keyring': ['keyring>=13.0.0'],
- 'jsonschema': ['jsonschema>=4.4.0']},
+ extras_require={
+ 'jsonschema': ['jsonschema>=4.4.0'],
+ 'keyring': ['keyring>=13.0.0'],
+ "test": [
+ "pytest",
+ "pytest-cov",
+ "coverage>=4.4.2",
+ "jsonschema>=4.4.0",
+ ]
+ },
setup_requires=["pytest-runner>=2.0,<3dev"],
- tests_require=["pytest", "pytest-cov", "coverage>=4.4.2",
- "jsonschema>=4.4.0"],
package_data={
'linkahead': ['py.typed', 'cert/indiscale.ca.crt', 'schema-pycaosdb-ini.yml'],
},
--
GitLab
From 9ae5cd871573e78527b94e31768abbf617203b8c Mon Sep 17 00:00:00 2001
From: Daniel <d.hornung@indiscale.com>
Date: Fri, 22 Nov 2024 17:26:41 +0100
Subject: [PATCH 10/62] WIP: mypy
---
.gitlab-ci.yml | 4 ++--
setup.py | 24 ++++++++++++++++++++----
src/linkahead/common/models.py | 9 ++++-----
3 files changed, 26 insertions(+), 11 deletions(-)
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 8845e407..e11ef9f6 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -57,9 +57,9 @@ mypy:
tags: [ docker ]
stage: linting
script:
- - pip install mypy types-PyYAML types-jsonschema types-requests types-setuptools types-lxml types-python-dateutil pytest
+ - pip install .[mypy,test]
- make mypy
- allow_failure: true
+ # allow_failure: true
# run unit tests
unittest_py3.8:
diff --git a/setup.py b/setup.py
index 8b0f7ef0..ec20e88b 100755
--- a/setup.py
+++ b/setup.py
@@ -187,11 +187,27 @@ def setup_package():
'PyYAML>=5.4.1',
'future',
],
- extras_require={'keyring': ['keyring>=13.0.0'],
- 'jsonschema': ['jsonschema>=4.4.0']},
+ extras_require={
+ 'keyring': ['keyring>=13.0.0'],
+ 'jsonschema': ['jsonschema>=4.4.0'],
+ "mypy": [
+ "mypy",
+ "types-PyYAML",
+ "types-jsonschema",
+ "types-requests",
+ "types-setuptools",
+ "types-lxml",
+ "types-python-dateutil",
+ ],
+ "test": [
+ "pytest",
+ "pytest-cov",
+ "coverage>=4.4.2",
+ "jsonschema>=4.4.0",
+ ]
+
+ },
setup_requires=["pytest-runner>=2.0,<3dev"],
- tests_require=["pytest", "pytest-cov", "coverage>=4.4.2",
- "jsonschema>=4.4.0"],
package_data={
'linkahead': ['py.typed', 'cert/indiscale.ca.crt', 'schema-pycaosdb-ini.yml'],
},
diff --git a/src/linkahead/common/models.py b/src/linkahead/common/models.py
index 6204e119..326e5f6d 100644
--- a/src/linkahead/common/models.py
+++ b/src/linkahead/common/models.py
@@ -41,7 +41,6 @@ import warnings
from builtins import str
from copy import deepcopy
from datetime import date, datetime
-from enum import Enum
from functools import cmp_to_key
from hashlib import sha512
from os import listdir
@@ -1120,7 +1119,7 @@ class Entity:
else:
return getattr(ref, special_selector.lower())
- def get_property_values(self, *selectors):
+ def get_property_values(self, *selectors) -> tuple:
""" Return a tuple with the values described by the given selectors.
This represents an entity's properties as if it was a row of a table
@@ -2409,7 +2408,7 @@ class PropertyList(list):
This class provides addional functionality like get/set_importance or get_by_name.
"""
- def __init__(self):
+ def __init__(self) -> None:
super().__init__()
self._importance: dict[Entity, IMPORTANCE] = dict()
self._inheritance: dict[Entity, INHERITANCE] = dict()
@@ -4544,7 +4543,7 @@ class ACL():
return len(self._grants) + len(self._priority_grants) + \
len(self._priority_denials) + len(self._denials) == 0
- def clear(self):
+ def clear(self) -> None:
self._grants: set[ACI] = set()
self._denials: set[ACI] = set()
self._priority_grants: set[ACI] = set()
@@ -5148,7 +5147,7 @@ class Info():
"""
- def __init__(self):
+ def __init__(self) -> None:
self.messages = Messages()
self.user_info: Optional[UserInfo] = None
self.time_zone: Optional[TimeZone] = None
--
GitLab
From 74f96a33db4658a73f2a2793152bcbdc5bfdedb1 Mon Sep 17 00:00:00 2001
From: Daniel <d.hornung@indiscale.com>
Date: Fri, 22 Nov 2024 17:40:45 +0100
Subject: [PATCH 11/62] WIP: mypy
---
src/linkahead/common/versioning.py | 13 ++++++++-----
src/linkahead/connection/connection.py | 21 ++++++++++++---------
src/linkahead/utils/git_utils.py | 18 +++++++++---------
3 files changed, 29 insertions(+), 23 deletions(-)
diff --git a/src/linkahead/common/versioning.py b/src/linkahead/common/versioning.py
index 11cf5f69..1c2999df 100644
--- a/src/linkahead/common/versioning.py
+++ b/src/linkahead/common/versioning.py
@@ -101,11 +101,14 @@ class Version():
# pylint: disable=redefined-builtin
def __init__(self, id: Optional[str] = None, date: Optional[str] = None,
username: Optional[str] = None, realm: Optional[str] = None,
- predecessors: Optional[List[Version]] = None, successors: Optional[List[Version]] = None,
+ predecessors: Optional[List[Version]] = None,
+ successors: Optional[List[Version]] = None,
is_head: Union[bool, str, None] = False,
is_complete_history: Union[bool, str, None] = False):
- """Typically the `predecessors` or `successors` should not "link back" to an existing Version
- object."""
+ """Typically the `predecessors` or `successors` should not "link back" to an existing
+ Version object.
+
+ """
self.id = id
self.date = date
self.username = username
@@ -205,8 +208,8 @@ class Version():
version : Version
a new version instance
"""
- predecessors = [Version.from_xml(p) for p in xml if p.tag.lower() == "predecessor"]
- successors = [Version.from_xml(s) for s in xml if s.tag.lower() == "successor"]
+ predecessors = [Version.from_xml(p) for p in xml if str(p.tag).lower() == "predecessor"]
+ successors = [Version.from_xml(s) for s in xml if str(s.tag).lower() == "successor"]
return Version(id=xml.get("id"), date=xml.get("date"),
is_head=xml.get("head"),
is_complete_history=xml.get("completeHistory"),
diff --git a/src/linkahead/connection/connection.py b/src/linkahead/connection/connection.py
index c95134fe..f75226d4 100644
--- a/src/linkahead/connection/connection.py
+++ b/src/linkahead/connection/connection.py
@@ -103,9 +103,12 @@ class _WrappedHTTPResponse(CaosDBHTTPResponse):
if len(self._buffer) >= size:
# still enough bytes in the buffer
# FIXME: `chunk`` is used before definition
- result = chunk[:size]
- self._buffer = chunk[size:]
- return result
+ raise NotImplementedError("chunk is undefined")
+
+ # # old code:
+ # result = chunk[:size]
+ # self._buffer = chunk[size:]
+ # return result
if self._generator is None:
# first call to this method
@@ -218,7 +221,7 @@ class _DefaultCaosDBServerConnection(CaosDBServerConnection):
"Connection failed. Network or server down? " + str(conn_err)
)
- def configure(self, **config):
+ def configure(self, **config) -> None:
"""configure.
Configure the http connection.
@@ -551,9 +554,9 @@ class _Connection(object): # pylint: disable=useless-object-inheritance
__instance = None
- def __init__(self):
+ def __init__(self) -> None:
self._delegate_connection: Optional[CaosDBServerConnection] = None
- self._authenticator: Optional[CredentialsAuthenticator] = None
+ self._authenticator: Optional[AbstractAuthenticator] = None
self.is_configured = False
@classmethod
@@ -563,7 +566,7 @@ class _Connection(object): # pylint: disable=useless-object-inheritance
return cls.__instance
- def configure(self, **config):
+ def configure(self, **config) -> _Connection:
self.is_configured = True
if "implementation" not in config:
@@ -571,8 +574,7 @@ class _Connection(object): # pylint: disable=useless-object-inheritance
"Missing CaosDBServerConnection implementation. You did not "
"specify an `implementation` for the connection.")
try:
- self._delegate_connection: CaosDBServerConnection = config["implementation"](
- )
+ self._delegate_connection = config["implementation"]()
if not isinstance(self._delegate_connection,
CaosDBServerConnection):
@@ -762,6 +764,7 @@ class _Connection(object): # pylint: disable=useless-object-inheritance
if self._authenticator is None:
raise ValueError(
"No authenticator set. Please call configure_connection() first.")
+ assert isinstance(self._authenticator, CredentialsAuthenticator)
if self._authenticator._credentials_provider is None:
raise ValueError(
"No credentials provider set. Please call configure_connection() first.")
diff --git a/src/linkahead/utils/git_utils.py b/src/linkahead/utils/git_utils.py
index 7a58272a..4824d619 100644
--- a/src/linkahead/utils/git_utils.py
+++ b/src/linkahead/utils/git_utils.py
@@ -36,9 +36,9 @@ logger = logging.getLogger(__name__)
def get_origin_url_in(folder: str):
"""return the Fetch URL of the git repository in the given folder."""
- with tempfile.NamedTemporaryFile(delete=False, mode="w") as t:
- call(["git", "remote", "show", "origin"], stdout=t, cwd=folder)
- with open(t.name, "r") as t:
+ with tempfile.NamedTemporaryFile(delete=False, mode="w", encoding="utf8") as tempf:
+ call(["git", "remote", "show", "origin"], stdout=tempf, cwd=folder)
+ with open(tempf.name, "r", encoding="utf8") as t:
urlString = "Fetch URL:"
for line in t.readlines():
@@ -63,9 +63,9 @@ def get_branch_in(folder: str):
The command "git branch" is called in the given folder and the
output is returned
"""
- with tempfile.NamedTemporaryFile(delete=False, mode="w") as t:
- call(["git", "rev-parse", "--abbrev-ref", "HEAD"], stdout=t, cwd=folder)
- with open(t.name, "r") as t:
+ with tempfile.NamedTemporaryFile(delete=False, mode="w") as tempf:
+ call(["git", "rev-parse", "--abbrev-ref", "HEAD"], stdout=tempf, cwd=folder)
+ with open(tempf.name, "r") as t:
return t.readline().strip()
@@ -76,7 +76,7 @@ def get_commit_in(folder: str):
and the output is returned
"""
- with tempfile.NamedTemporaryFile(delete=False, mode="w") as t:
- call(["git", "log", "-1", "--format=%h"], stdout=t, cwd=folder)
- with open(t.name, "r") as t:
+ with tempfile.NamedTemporaryFile(delete=False, mode="w") as tempf:
+ call(["git", "log", "-1", "--format=%h"], stdout=tempf, cwd=folder)
+ with open(tempf.name, "r") as t:
return t.readline().strip()
--
GitLab
From 1eeed446b79d173ca6e69557d731e40a20994b55 Mon Sep 17 00:00:00 2001
From: Daniel <d.hornung@indiscale.com>
Date: Fri, 22 Nov 2024 17:49:01 +0100
Subject: [PATCH 12/62] WIP: mypy
---
src/linkahead/common/models.py | 38 +++++++++++++++++-----------------
src/linkahead/common/state.py | 15 +++++++-------
2 files changed, 27 insertions(+), 26 deletions(-)
diff --git a/src/linkahead/common/models.py b/src/linkahead/common/models.py
index 326e5f6d..ed80c600 100644
--- a/src/linkahead/common/models.py
+++ b/src/linkahead/common/models.py
@@ -1860,12 +1860,12 @@ class QueryTemplate():
@staticmethod
def _from_xml(xml: etree._Element):
- if xml.tag.lower() == "querytemplate":
+ if str(xml.tag).lower() == "querytemplate":
q = QueryTemplate(name=xml.get("name"),
description=xml.get("description"), query=None)
for e in xml:
- if e.tag.lower() == "query":
+ if str(e.tag).lower() == "query":
q.query = e.text
else:
child = _parse_single_xml_element(e)
@@ -1902,7 +1902,7 @@ class QueryTemplate():
ret = Messages()
for m in self.messages:
- if m.type.lower() == "error":
+ if str(m.type).lower() == "error":
ret.append(m)
return ret
@@ -2732,7 +2732,7 @@ class ParentList(list):
# by name
for e in self:
- if e.name is not None and e.name.lower() == parent.name.lower():
+ if e.name is not None and str(e.name).lower() == str(parent.name).lower():
list.remove(self, e)
return
@@ -4885,7 +4885,7 @@ class Query():
self.etag = q.get("etag")
for m in q:
- if m.tag.lower() == 'warning' or m.tag.lower() == 'error':
+ if str(m.tag).lower() == 'warning' or str(m.tag).lower() == 'error':
self.messages.append(_parse_single_xml_element(m))
else:
self.q = q
@@ -5092,13 +5092,13 @@ class DropOffBox(list):
xml = etree.fromstring(body)
for child in xml:
- if child.tag.lower() == "stats":
+ if str(child.tag).lower() == "stats":
infoelem = child
break
for child in infoelem:
- if child.tag.lower() == "dropoffbox":
+ if str(child.tag).lower() == "dropoffbox":
dropoffboxelem = child
break
@@ -5273,36 +5273,36 @@ def _parse_single_xml_element(elem: etree._Element):
"entity": Entity,
}
- if elem.tag.lower() in classmap:
- klass = classmap.get(elem.tag.lower())
+ if str(elem.tag).lower() in classmap:
+ klass = classmap.get(str(elem.tag).lower())
if klass is None:
raise LinkAheadException("No class for tag '{}' found.".format(elem.tag))
entity = klass()
Entity._from_xml(entity, elem)
return entity
- elif elem.tag.lower() == "version":
+ elif str(elem.tag).lower() == "version":
return Version.from_xml(elem)
- elif elem.tag.lower() == "state":
+ elif str(elem.tag).lower() == "state":
return State.from_xml(elem)
- elif elem.tag.lower() == "emptystring":
+ elif str(elem.tag).lower() == "emptystring":
return ""
- elif elem.tag.lower() == "value":
- if len(elem) == 1 and elem[0].tag.lower() == "emptystring":
+ elif str(elem.tag).lower() == "value":
+ if len(elem) == 1 and str(elem[0].tag).lower() == "emptystring":
return ""
- elif len(elem) == 1 and elem[0].tag.lower() in classmap:
+ elif len(elem) == 1 and str(elem[0].tag).lower() in classmap:
return _parse_single_xml_element(elem[0])
elif elem.text is None or elem.text.strip() == "":
return None
return str(elem.text.strip())
- elif elem.tag.lower() == "querytemplate":
+ elif str(elem.tag).lower() == "querytemplate":
return QueryTemplate._from_xml(elem)
- elif elem.tag.lower() == 'query':
+ elif str(elem.tag).lower() == 'query':
return Query(elem)
- elif elem.tag.lower() == 'history':
+ elif str(elem.tag).lower() == 'history':
return Message(type='History', description=elem.get("transaction"))
- elif elem.tag.lower() == 'stats':
+ elif str(elem.tag).lower() == 'stats':
counts = elem.find("counts")
if counts is None:
raise LinkAheadException("'stats' element without a 'count' found.")
diff --git a/src/linkahead/common/state.py b/src/linkahead/common/state.py
index e352f82d..b708ca13 100644
--- a/src/linkahead/common/state.py
+++ b/src/linkahead/common/state.py
@@ -20,11 +20,11 @@
# ** end header
from __future__ import annotations # Can be removed with 3.10.
-import copy
-from lxml import etree
+import copy
from typing import TYPE_CHECKING
-import sys
+
+from lxml import etree
if TYPE_CHECKING:
from typing import Optional
@@ -87,7 +87,8 @@ class Transition:
return self._to_state
def __repr__(self):
- return f'Transition(name="{self.name}", from_state="{self.from_state}", to_state="{self.to_state}", description="{self.description}")'
+ return (f'Transition(name="{self.name}", from_state="{self.from_state}", '
+ f'to_state="{self.to_state}", description="{self.description}")')
def __eq__(self, other):
return (
@@ -103,9 +104,9 @@ class Transition:
@staticmethod
def from_xml(xml: etree._Element) -> "Transition":
to_state = [to.get("name")
- for to in xml if to.tag.lower() == "tostate"]
+ for to in xml if str(to.tag).lower() == "tostate"]
from_state = [
- from_.get("name") for from_ in xml if from_.tag.lower() == "fromstate"
+ from_.get("name") for from_ in xml if str(from_.tag).lower() == "fromstate"
]
return Transition(
name=xml.get("name"),
@@ -199,7 +200,7 @@ class State:
result._id = xml.get("id")
result._description = xml.get("description")
transitions = [
- Transition.from_xml(t) for t in xml if t.tag.lower() == "transition"
+ Transition.from_xml(t) for t in xml if str(t.tag).lower() == "transition"
]
if transitions:
result._transitions = set(transitions)
--
GitLab
From e00a4a2ee7279b907ff5bf7f011b71dc989cbd9d Mon Sep 17 00:00:00 2001
From: Daniel <d.hornung@indiscale.com>
Date: Fri, 22 Nov 2024 17:51:41 +0100
Subject: [PATCH 13/62] WIP: mypy
---
src/linkahead/common/models.py | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/src/linkahead/common/models.py b/src/linkahead/common/models.py
index ed80c600..aff85556 100644
--- a/src/linkahead/common/models.py
+++ b/src/linkahead/common/models.py
@@ -5276,7 +5276,7 @@ def _parse_single_xml_element(elem: etree._Element):
if str(elem.tag).lower() in classmap:
klass = classmap.get(str(elem.tag).lower())
if klass is None:
- raise LinkAheadException("No class for tag '{}' found.".format(elem.tag))
+ raise LinkAheadException("No class for tag '{}' found.".format(str(elem.tag)))
entity = klass()
Entity._from_xml(entity, elem)
@@ -5322,7 +5322,7 @@ def _parse_single_xml_element(elem: etree._Element):
else:
code = elem.get("code")
return Message(
- type=elem.tag,
+ type=str(elem.tag),
code=int(code) if code is not None else None,
description=elem.get("description"),
body=elem.text,
--
GitLab
From 3f2099c2e1aeae54cf41a3203f81e9eecf6b45db Mon Sep 17 00:00:00 2001
From: Daniel <d.hornung@indiscale.com>
Date: Fri, 22 Nov 2024 18:02:49 +0100
Subject: [PATCH 14/62] WIP: mypy
---
src/linkahead/apiutils.py | 13 ++++++----
src/linkahead/common/administration.py | 35 ++++++++++++++++++--------
src/linkahead/utils/plantuml.py | 9 ++++---
3 files changed, 38 insertions(+), 19 deletions(-)
diff --git a/src/linkahead/apiutils.py b/src/linkahead/apiutils.py
index 49336aa8..31e49aa9 100644
--- a/src/linkahead/apiutils.py
+++ b/src/linkahead/apiutils.py
@@ -275,9 +275,11 @@ def compare_entities(entity0: Optional[Entity] = None,
if entity1 is not None:
raise ValueError("You cannot use both entity1 and new_entity")
entity1 = new_entity
+ assert entity0 is not None
+ assert entity1 is not None
- diff: tuple = ({"properties": {}, "parents": []},
- {"properties": {}, "parents": []})
+ diff: tuple[dict[str, Any], dict[str, Any]] = ({"properties": {}, "parents": []},
+ {"properties": {}, "parents": []})
if entity0 is entity1:
return diff
@@ -550,9 +552,10 @@ def merge_entities(entity_a: Entity,
"""
# Compare both entities:
- diff_r1, diff_r2 = compare_entities(entity_a, entity_b,
- entity_name_id_equivalency=merge_id_with_resolved_entity,
- compare_referenced_records=merge_references_with_empty_diffs)
+ diff_r1, diff_r2 = compare_entities(
+ entity_a, entity_b,
+ entity_name_id_equivalency=merge_id_with_resolved_entity,
+ compare_referenced_records=merge_references_with_empty_diffs)
# Go through the comparison and try to apply changes to entity_a:
for key in diff_r2["parents"]:
diff --git a/src/linkahead/common/administration.py b/src/linkahead/common/administration.py
index 28ef1075..4f39d556 100644
--- a/src/linkahead/common/administration.py
+++ b/src/linkahead/common/administration.py
@@ -91,7 +91,7 @@ def get_server_properties() -> dict[str, Optional[str]]:
props: dict[str, Optional[str]] = dict()
for elem in xml.getroot():
- props[elem.tag] = elem.text
+ props[elem.tag] = str(elem.text)
return props
@@ -156,7 +156,10 @@ def generate_password(length: int):
def _retrieve_user(name: str, realm: Optional[str] = None, **kwargs):
con = get_connection()
try:
- return con._http_request(method="GET", path="User/" + (realm + "/" + name if realm is not None else name), **kwargs).read()
+ return con._http_request(
+ method="GET",
+ path="User/" + (realm + "/" + name if realm is not None else name),
+ **kwargs).read()
except HTTPForbiddenError as e:
e.msg = "You are not permitted to retrieve this user."
raise
@@ -198,7 +201,9 @@ def _update_user(name: str,
if entity is not None:
params["entity"] = str(entity)
try:
- return con.put_form_data(entity_uri_segment="User/" + (realm + "/" + name if realm is not None else name), params=params, **kwargs).read()
+ return con.put_form_data(entity_uri_segment="User/" + (realm + "/" +
+ name if realm is not None else name),
+ params=params, **kwargs).read()
except HTTPResourceNotFoundError as e:
e.msg = "User does not exist."
raise e
@@ -246,7 +251,9 @@ def _insert_user(name: str,
def _insert_role(name, description, **kwargs):
con = get_connection()
try:
- return con.post_form_data(entity_uri_segment="Role", params={"role_name": name, "role_description": description}, **kwargs).read()
+ return con.post_form_data(entity_uri_segment="Role",
+ params={"role_name": name, "role_description": description},
+ **kwargs).read()
except HTTPForbiddenError as e:
e.msg = "You are not permitted to insert a new role."
raise
@@ -259,7 +266,9 @@ def _insert_role(name, description, **kwargs):
def _update_role(name, description, **kwargs):
con = get_connection()
try:
- return con.put_form_data(entity_uri_segment="Role/" + name, params={"role_description": description}, **kwargs).read()
+ return con.put_form_data(entity_uri_segment="Role/" + name,
+ params={"role_description": description},
+ **kwargs).read()
except HTTPForbiddenError as e:
e.msg = "You are not permitted to update this role."
raise
@@ -301,8 +310,10 @@ def _set_roles(username, roles, realm=None, **kwargs):
body = xml2str(xml)
con = get_connection()
try:
- body = con._http_request(method="PUT", path="UserRoles/" + (realm + "/" +
- username if realm is not None else username), body=body, **kwargs).read()
+ body = con._http_request(method="PUT",
+ path="UserRoles/" + (realm + "/" +
+ username if realm is not None else username),
+ body=body, **kwargs).read()
except HTTPForbiddenError as e:
e.msg = "You are not permitted to set this user's roles."
raise
@@ -369,7 +380,8 @@ def _set_permissions(role, permission_rules, **kwargs):
body = xml2str(xml)
con = get_connection()
try:
- return con._http_request(method="PUT", path="PermissionRules/" + role, body=body, **kwargs).read()
+ return con._http_request(method="PUT", path="PermissionRules/" + role, body=body,
+ **kwargs).read()
except HTTPForbiddenError as e:
e.msg = "You are not permitted to set this role's permissions."
raise
@@ -381,7 +393,9 @@ def _set_permissions(role, permission_rules, **kwargs):
def _get_permissions(role, **kwargs):
con = get_connection()
try:
- return PermissionRule._parse_body(con._http_request(method="GET", path="PermissionRules/" + role, **kwargs).read())
+ return PermissionRule._parse_body(con._http_request(method="GET",
+ path="PermissionRules/" + role,
+ **kwargs).read())
except HTTPForbiddenError as e:
e.msg = "You are not permitted to retrieve this role's permissions."
raise
@@ -429,7 +443,8 @@ class PermissionRule():
if permission is None:
raise ValueError(f"Permission is missing in PermissionRule xml: {elem}")
priority = PermissionRule._parse_boolean(elem.get("priority"))
- return PermissionRule(elem.tag, permission, priority if priority is not None else False)
+ return PermissionRule(str(elem.tag), permission,
+ priority if priority is not None else False)
@staticmethod
def _parse_body(body: str):
diff --git a/src/linkahead/utils/plantuml.py b/src/linkahead/utils/plantuml.py
index 19594d6e..59e3c34d 100644
--- a/src/linkahead/utils/plantuml.py
+++ b/src/linkahead/utils/plantuml.py
@@ -130,9 +130,9 @@ def recordtypes_to_plantuml_string(iterable,
classes = [el for el in iterable
if isinstance(el, db.RecordType)]
- dependencies = {}
- inheritances = {}
- properties = [p for p in iterable if isinstance(p, db.Property)]
+ dependencies: dict = {}
+ inheritances: dict = {}
+ properties: list = [p for p in iterable if isinstance(p, db.Property)]
grouped = [g for g in iterable if isinstance(g, Grouped)]
def _add_properties(c, importance=None):
@@ -272,7 +272,8 @@ package \"The property P references an instance of D\" <<Rectangle>> {
return result
-def retrieve_substructure(start_record_types, depth, result_id_set=None, result_container=None, cleanup=True):
+def retrieve_substructure(start_record_types, depth, result_id_set=None, result_container=None,
+ cleanup=True):
"""Recursively retrieves LinkAhead record types and properties, starting
from given initial types up to a specific depth.
--
GitLab
From 1f6af16ba929dd497d5fd3bdbd442e254add6f2f Mon Sep 17 00:00:00 2001
From: Daniel <d.hornung@indiscale.com>
Date: Fri, 22 Nov 2024 18:05:00 +0100
Subject: [PATCH 15/62] WIP: mypy
---
src/linkahead/common/administration.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/src/linkahead/common/administration.py b/src/linkahead/common/administration.py
index 4f39d556..9d9d4f01 100644
--- a/src/linkahead/common/administration.py
+++ b/src/linkahead/common/administration.py
@@ -91,7 +91,7 @@ def get_server_properties() -> dict[str, Optional[str]]:
props: dict[str, Optional[str]] = dict()
for elem in xml.getroot():
- props[elem.tag] = str(elem.text)
+ props[str(elem.tag)] = str(elem.text)
return props
--
GitLab
From ab023283b0a23c01f9ce5866a6c3a398da725136 Mon Sep 17 00:00:00 2001
From: Daniel <d.hornung@indiscale.com>
Date: Fri, 22 Nov 2024 18:05:45 +0100
Subject: [PATCH 16/62] DOC: Changelog
---
CHANGELOG.md | 3 +++
1 file changed, 3 insertions(+)
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 40da08b0..1991634e 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -9,6 +9,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
### Added ###
+* Added a few type definitions.
+
### Changed ###
### Deprecated ###
@@ -16,6 +18,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
### Removed ###
### Fixed ###
+
* [#89](https://gitlab.com/linkahead/linkahead-pylib/-/issues/89)
`to_xml` does not add `noscript` or `TransactionBenchmark` tags anymore
--
GitLab
From 0fb754fa5cea2d3b8100da0313b8cc8fe56f8869 Mon Sep 17 00:00:00 2001
From: Daniel <d.hornung@indiscale.com>
Date: Fri, 22 Nov 2024 18:19:49 +0100
Subject: [PATCH 17/62] MAINT: Enforcing mypy
---
.gitlab-ci.yml | 1 -
1 file changed, 1 deletion(-)
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 6eccfe07..463b1d56 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -59,7 +59,6 @@ mypy:
script:
- pip install .[mypy,test]
- make mypy
- # allow_failure: true
# run unit tests
unittest_py3.8:
--
GitLab
From 2b649e0f88487af86e56c709970d55af52a5d440 Mon Sep 17 00:00:00 2001
From: Daniel <d.hornung@indiscale.com>
Date: Mon, 25 Nov 2024 13:26:00 +0100
Subject: [PATCH 18/62] DOC: Changelog.
---
CHANGELOG.md | 2 ++
1 file changed, 2 insertions(+)
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 40da08b0..270188c2 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -9,6 +9,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
### Added ###
+* New setup extra `test` which installs the dependencies for testing.
+
### Changed ###
### Deprecated ###
--
GitLab
From 937ceb4829d2f4badc98aec317ddbbfc101dced5 Mon Sep 17 00:00:00 2001
From: Florian Spreckelsen <f.spreckelsen@indiscale.com>
Date: Mon, 25 Nov 2024 14:29:59 +0100
Subject: [PATCH 19/62] DOC: Update changelog
---
CHANGELOG.md | 2 --
1 file changed, 2 deletions(-)
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 1991634e..63a3df1a 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -9,8 +9,6 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
### Added ###
-* Added a few type definitions.
-
### Changed ###
### Deprecated ###
--
GitLab
From cb5c7db7f006c4852f3072751c766e629ecb57c3 Mon Sep 17 00:00:00 2001
From: Daniel <d.hornung@indiscale.com>
Date: Mon, 25 Nov 2024 16:23:03 +0100
Subject: [PATCH 20/62] FIX: Fixed undefined variable.
---
src/linkahead/connection/connection.py | 25 ++++++-------
unittests/test_connection.py | 52 ++++++++++++++++++++++++++
2 files changed, 64 insertions(+), 13 deletions(-)
diff --git a/src/linkahead/connection/connection.py b/src/linkahead/connection/connection.py
index f75226d4..4c40842a 100644
--- a/src/linkahead/connection/connection.py
+++ b/src/linkahead/connection/connection.py
@@ -83,8 +83,10 @@ class _WrappedHTTPResponse(CaosDBHTTPResponse):
return self.response.status_code
def read(self, size: Optional[int] = None):
+ # FIXME This function behaves unexpectedly if `size` is larger than in the first run.
+
if self._stream_consumed is True:
- raise RuntimeError("Stream is consumed")
+ raise BufferError("Stream is consumed")
if self._buffer is None:
# the buffer has been drained in the previous call.
@@ -97,18 +99,15 @@ class _WrappedHTTPResponse(CaosDBHTTPResponse):
return self.response.content
if size is None or size == 0:
- raise RuntimeError(
- "size parameter should not be None if the stream is not consumed yet")
+ raise BufferError(
+ "`size` parameter can not be None or zero once reading has started with a non-zero "
+ "value.")
if len(self._buffer) >= size:
# still enough bytes in the buffer
- # FIXME: `chunk`` is used before definition
- raise NotImplementedError("chunk is undefined")
-
- # # old code:
- # result = chunk[:size]
- # self._buffer = chunk[size:]
- # return result
+ result = self._buffer[:size]
+ self._buffer = self._buffer[size:]
+ return result
if self._generator is None:
# first call to this method
@@ -119,16 +118,16 @@ class _WrappedHTTPResponse(CaosDBHTTPResponse):
try:
# read new data into the buffer
chunk = self._buffer + next(self._generator)
- result = chunk[:size]
+ result = chunk[:size] # FIXME what if `size` is larger than at `iter_content(size)`?
if len(result) == 0:
self._stream_consumed = True
self._buffer = chunk[size:]
return result
except StopIteration:
# drain buffer
- result = self._buffer
+ last_result = self._buffer
self._buffer = None
- return result
+ return last_result
def getheader(self, name: str, default=None):
return self.response.headers[name] if name in self.response.headers else default
diff --git a/unittests/test_connection.py b/unittests/test_connection.py
index a3a1eff7..a0d280c2 100644
--- a/unittests/test_connection.py
+++ b/unittests/test_connection.py
@@ -25,14 +25,18 @@
# pylint: disable=missing-docstring
from __future__ import print_function, unicode_literals
+import io
import re
from builtins import bytes, str # pylint: disable=redefined-builtin
+import requests
+
from linkahead import execute_query
from linkahead.configuration import _reset_config, get_config
from linkahead.connection.authentication.interface import CredentialsAuthenticator
from linkahead.connection.connection import (CaosDBServerConnection,
_DefaultCaosDBServerConnection,
+ _WrappedHTTPResponse,
configure_connection)
from linkahead.connection.mockup import (MockUpResponse, MockUpServerConnection,
_request_log_message)
@@ -324,3 +328,51 @@ def test_auth_token_connection():
"auth_token authenticator cannot log in "
"again. You must provide a new authentication "
"token.")
+
+
+def test_buffer_read():
+ """Test the buffering in _WrappedHTTPResponse.read()"""
+
+ class MockResponse(requests.Response):
+ def __init__(self, content: bytes):
+ """A mock response
+
+ Parameters
+ ----------
+ content : bytes
+ The fake content.
+ """
+ super().__init__()
+ self._content = content
+ bio = io.BytesIO(expected)
+ self.raw = bio
+
+ expected = b"This response."
+ MockResponse(expected)
+
+ #############################
+ # Check for some exceptions #
+ #############################
+ resp = _WrappedHTTPResponse(response=MockResponse(expected))
+ with raises(BufferError) as rte:
+ resp.read(4)
+ resp.read()
+ assert "`size` parameter can not be None" in str(rte.value)
+
+ resp = _WrappedHTTPResponse(response=MockResponse(expected))
+ with raises(BufferError) as rte:
+ resp.read(4)
+ resp.read(0)
+ assert "`size` parameter can not be None" in str(rte.value)
+
+ print("---")
+ resp = _WrappedHTTPResponse(response=MockResponse(expected))
+ result = (
+ resp.read(4)
+ + resp.read(2)
+ + resp.read(2) # This line failed before.
+ + resp.read(4) # Reading the rest in two chunks, because of current limitations in read().
+ + resp.read(2)
+ )
+
+ assert result == expected
--
GitLab
From 7796b956d493d9ae1d732919577d2a4fdfeb303e Mon Sep 17 00:00:00 2001
From: Florian Spreckelsen <f.spreckelsen@indiscale.com>
Date: Mon, 25 Nov 2024 16:38:23 +0100
Subject: [PATCH 21/62] DOC: Update changelog
---
CHANGELOG.md | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
diff --git a/CHANGELOG.md b/CHANGELOG.md
index eac0ed0f..05d780ff 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -16,12 +16,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
### Removed ###
### Fixed ###
-* [#103](https://gitlab.com/linkahead/linkahead-pylib/-/issues/103)
- `authentication/interface/on_response()` does not overwrite
- `auth_token` if new value is `None`
* [#89](https://gitlab.com/linkahead/linkahead-pylib/-/issues/89)
`to_xml` does not add `noscript` or `TransactionBenchmark` tags anymore
+* [#103](https://gitlab.com/linkahead/linkahead-pylib/-/issues/103)
+ `authentication/interface/on_response()` does not overwrite
+ `auth_token` if new value is `None`
### Security ###
--
GitLab
From 7309b0c06e947ce979574be923f982bb5c478254 Mon Sep 17 00:00:00 2001
From: "i.nueske" <i.nueske@indiscale.com>
Date: Thu, 28 Nov 2024 13:23:09 +0100
Subject: [PATCH 22/62] DOC: Document pip extras in README_SETUP.md
---
README_SETUP.md | 35 ++++++++++++++++++-----------------
1 file changed, 18 insertions(+), 17 deletions(-)
diff --git a/README_SETUP.md b/README_SETUP.md
index 8a32fbfa..ddc36c62 100644
--- a/README_SETUP.md
+++ b/README_SETUP.md
@@ -2,19 +2,6 @@
## Installation ##
-### Requirements ###
-
-PyCaosDB needs at least Python 3.8. Additionally, the following packages are required (they will
-typically be installed automatically):
-
-- `lxml`
-- `PyYaml`
-- `PySocks`
-
-Optional packages:
-- `keyring`
-- `jsonschema`
-
### How to install ###
#### Linux ####
@@ -73,12 +60,25 @@ cd caosdb-pylib
pip3 install --user .
```
-For installation of optional packages, install with an additional option, e.g. for
-validating with the caosdb json schema:
+#### Additional dependencies ####
+
+To test using tox, you also need to install tox:
+`pip3 install tox --user`
+
+To install dependencies used by optional functionality, the following pip extras
+keywords are defined:
+- `test` for testing with pytest
+- `mypy` for mypy and types
+- `jsonschema`
+- `keyring`
+These extras can be installed using:
```sh
-pip3 install --user .[jsonschema]
+pip3 install --user .[KEYWORD]
```
+A current list of the dependencies installed with this program as well as those installed with
+the keywords can be found in `setup.py`s `setup_package()` method, in the `metadata` dictionary
+entries `install_requires` and `extras_require`.
## Configuration ##
@@ -87,7 +87,7 @@ is described in detail in the [configuration section of the documentation](https
## Try it out ##
-Start Python and check whether the you can access the database. (You will be asked for the
+Start Python and check whether you can access the database. (You will be asked for the
password):
```python
@@ -107,6 +107,7 @@ Now would be a good time to continue with the [tutorials](tutorials/index).
- Run all tests: `tox` or `make unittest`
- Run a specific test file: e.g. `tox -- unittests/test_schema.py`
- Run a specific test function: e.g. `tox -- unittests/test_schema.py::test_config_files`
+- To run using pytest: `pytest .`
## Documentation ##
We use sphinx to create the documentation. Docstrings in the code should comply
--
GitLab
From 7ebf0caa7321611d61187c29dd9aa145923d9c8a Mon Sep 17 00:00:00 2001
From: "i.nueske" <i.nueske@indiscale.com>
Date: Thu, 28 Nov 2024 13:28:18 +0100
Subject: [PATCH 23/62] DOC: Move python installation into optional section
---
README_SETUP.md | 76 ++++++++++++++++++++++++++-----------------------
1 file changed, 40 insertions(+), 36 deletions(-)
diff --git a/README_SETUP.md b/README_SETUP.md
index ddc36c62..b1fa9b7f 100644
--- a/README_SETUP.md
+++ b/README_SETUP.md
@@ -4,42 +4,8 @@
### How to install ###
-#### Linux ####
-
-Make sure that Python (at least version 3.8) and pip is installed, using your system tools and
-documentation.
-
-Then open a terminal and continue in the [Generic installation](#generic-installation) section.
-
-#### Windows ####
-
-If a Python distribution is not yet installed, we recommend Anaconda Python, which you can download
-for free from [https://www.anaconda.com](https://www.anaconda.com). The "Anaconda Individual Edition" provides most of all
-packages you will ever need out of the box. If you prefer, you may also install the leaner
-"Miniconda" installer, which allows you to install packages as you need them.
-
-After installation, open an Anaconda prompt from the Windows menu and continue in the [Generic
-installation](#generic-installation) section.
-
-#### MacOS ####
-
-If there is no Python 3 installed yet, there are two main ways to
-obtain it: Either get the binary package from
-[python.org](https://www.python.org/downloads/) or, for advanced
-users, install via [Homebrew](https://brew.sh/). After installation
-from python.org, it is recommended to also update the TLS certificates
-for Python (this requires administrator rights for your user):
-
-```sh
-# Replace this with your Python version number:
-cd /Applications/Python\ 3.9/
-
-# This needs administrator rights:
-sudo ./Install\ Certificates.command
-```
-
-After these steps, you may continue with the [Generic
-installation](#generic-installation).
+First ensure that python with at least version 3.8 is installed. Should this not be
+the case, you can use the [Installing python](#installing-python-) guide for your OS.
#### Generic installation ####
@@ -80,6 +46,44 @@ A current list of the dependencies installed with this program as well as those
the keywords can be found in `setup.py`s `setup_package()` method, in the `metadata` dictionary
entries `install_requires` and `extras_require`.
+### Installing python ###
+
+#### Linux ####
+
+Make sure that Python (at least version 3.8) and pip is installed, using your system tools and
+documentation.
+
+Then open a terminal and continue in the [Generic installation](#generic-installation) section.
+
+#### Windows ####
+
+If a Python distribution is not yet installed, we recommend Anaconda Python, which you can download
+for free from [https://www.anaconda.com](https://www.anaconda.com). The "Anaconda Individual Edition" provides most of all
+packages you will ever need out of the box. If you prefer, you may also install the leaner
+"Miniconda" installer, which allows you to install packages as you need them.
+
+After installation, open an Anaconda prompt from the Windows menu and continue in the [Generic
+installation](#generic-installation) section.
+
+#### MacOS ####
+
+If there is no Python 3 installed yet, there are two main ways to
+obtain it: Either get the binary package from
+[python.org](https://www.python.org/downloads/) or, for advanced
+users, install via [Homebrew](https://brew.sh/). After installation
+from python.org, it is recommended to also update the TLS certificates
+for Python (this requires administrator rights for your user):
+
+```sh
+# Replace this with your Python version number:
+cd /Applications/Python\ 3.9/
+
+# This needs administrator rights:
+sudo ./Install\ Certificates.command
+```
+
+After these steps, you may continue with the [Generic installation](#generic-installation) section.
+
## Configuration ##
The configuration is done using `ini` configuration files. The content of these configuration files
--
GitLab
From c1dd2b8baaab3065aa9bb501202e304909474ddf Mon Sep 17 00:00:00 2001
From: "i.nueske" <i.nueske@indiscale.com>
Date: Thu, 28 Nov 2024 19:06:06 +0100
Subject: [PATCH 24/62] DOC: Update pip commands, and remove git link
---
README_SETUP.md | 28 +++++-----------------------
1 file changed, 5 insertions(+), 23 deletions(-)
diff --git a/README_SETUP.md b/README_SETUP.md
index b1fa9b7f..8fc93474 100644
--- a/README_SETUP.md
+++ b/README_SETUP.md
@@ -9,27 +9,16 @@ the case, you can use the [Installing python](#installing-python-) guide for you
#### Generic installation ####
-To install PyCaosDB locally, use `pip3` (also called `pip` on some systems):
+To install this LinkAhead python client locally, use `pip`/`pip3`:
```sh
-pip3 install --user caosdb
-```
-
----
-
-Alternatively, obtain the sources from GitLab and install from there (`git` must be installed for
-this option):
-
-```sh
-git clone https://gitlab.com/caosdb/caosdb-pylib
-cd caosdb-pylib
-pip3 install --user .
+pip install linkahead
```
#### Additional dependencies ####
To test using tox, you also need to install tox:
-`pip3 install tox --user`
+`pip install tox`
To install dependencies used by optional functionality, the following pip extras
keywords are defined:
@@ -40,7 +29,7 @@ keywords are defined:
These extras can be installed using:
```sh
-pip3 install --user .[KEYWORD]
+pip install .[KEYWORD]
```
A current list of the dependencies installed with this program as well as those installed with
the keywords can be found in `setup.py`s `setup_package()` method, in the `metadata` dictionary
@@ -119,13 +108,6 @@ with the Googly style (see link below).
Build documentation in `build/` with `make doc`.
-### Requirements ###
-
-- `sphinx`
-- `sphinx-autoapi`
-- `recommonmark`
-- `sphinx_rtd_theme`
-
### How to contribute ###
- [Google Style Python Docstrings](https://sphinxcontrib-napoleon.readthedocs.io/en/latest/example_google.html)
@@ -133,7 +115,7 @@ Build documentation in `build/` with `make doc`.
- [References to other documentation](https://www.sphinx-doc.org/en/master/usage/extensions/intersphinx.html#role-external)
### Troubleshooting ###
-If the client is to be executed directly from the `/src` folder, an initial `.\setup.py install --user` must be called.
+If the client is to be executed directly from the `/src` folder, an initial `.\setup.py install` must be called.
## Migration ##
TODO
--
GitLab
From d9a8ba5d725624709a9fc76c923b70f9f64f1a9f Mon Sep 17 00:00:00 2001
From: "i.nueske" <i.nueske@indiscale.com>
Date: Sat, 30 Nov 2024 11:58:45 +0100
Subject: [PATCH 25/62] BUG: The diff returned by compare_entities now uses id
instead of name as key if either of the compared properties do not have a
name
---
src/linkahead/apiutils.py | 25 +++++++++++++++----------
1 file changed, 15 insertions(+), 10 deletions(-)
diff --git a/src/linkahead/apiutils.py b/src/linkahead/apiutils.py
index 31e49aa9..1504db8e 100644
--- a/src/linkahead/apiutils.py
+++ b/src/linkahead/apiutils.py
@@ -377,15 +377,18 @@ def compare_entities(entity0: Optional[Entity] = None,
# compare properties
for prop in entity0.properties:
- matching = entity1.properties.filter(name=prop.name, pid=prop.id)
+ key = prop.name if prop.name else prop.id # ToDo: Would making id default break anything?
+ matching = entity1.properties.filter(prop)
if len(matching) == 0:
# entity1 has prop, entity0 does not
- diff[0]["properties"][prop.name] = {}
+ diff[0]["properties"][key] = {}
elif len(matching) == 1:
- diff[0]["properties"][prop.name] = {}
- diff[1]["properties"][prop.name] = {}
- propdiff = (diff[0]["properties"][prop.name],
- diff[1]["properties"][prop.name])
+ # It's possible that prop has name and id, but match only has id
+ key = prop.name if prop.name and matching[0].name else prop.id
+ diff[0]["properties"][key] = {}
+ diff[1]["properties"][key] = {}
+ propdiff = (diff[0]["properties"][key],
+ diff[1]["properties"][key])
# We should compare the wrapped properties instead of the
# wrapping entities if possible:
@@ -417,8 +420,8 @@ def compare_entities(entity0: Optional[Entity] = None,
# in case there is no difference, we remove the dict keys again
if len(propdiff[0]) == 0 and len(propdiff[1]) == 0:
- diff[0]["properties"].pop(prop.name)
- diff[1]["properties"].pop(prop.name)
+ diff[0]["properties"].pop(key)
+ diff[1]["properties"].pop(key)
else:
raise NotImplementedError(
@@ -426,11 +429,12 @@ def compare_entities(entity0: Optional[Entity] = None,
# we have not yet compared properties that do not exist in entity0
for prop in entity1.properties:
+ key = prop.name if prop.name else prop.id # ToDo: Would making id default break anything?
# check how often the property appears in entity0
num_prop_in_ent0 = len(entity0.properties.filter(prop))
if num_prop_in_ent0 == 0:
# property is only present in entity0 - add to diff
- diff[1]["properties"][prop.name] = {}
+ diff[1]["properties"][key] = {}
if num_prop_in_ent0 > 1:
# Check whether the property is present multiple times in entity0
# and raise error - result would be incorrect
@@ -441,9 +445,10 @@ def compare_entities(entity0: Optional[Entity] = None,
for index, parents, other_entity in [(0, entity0.parents, entity1),
(1, entity1.parents, entity0)]:
for parent in parents:
+ key = parent.name if parent.name else parent.id # ToDo: Would making id default break anything?
matching = other_entity.parents.filter(parent)
if len(matching) == 0:
- diff[index]["parents"].append(parent.name)
+ diff[index]["parents"].append(key)
continue
return diff
--
GitLab
From 790fb8b6c78d4add7e641924a22a08ca325978bb Mon Sep 17 00:00:00 2001
From: "i.nueske" <i.nueske@indiscale.com>
Date: Sat, 30 Nov 2024 12:10:26 +0100
Subject: [PATCH 26/62] DOC: Update Changelog
---
CHANGELOG.md | 2 ++
1 file changed, 2 insertions(+)
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 0544d70b..03afb6a4 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -24,6 +24,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
* [#103](https://gitlab.com/linkahead/linkahead-pylib/-/issues/103)
`authentication/interface/on_response()` does not overwrite
`auth_token` if new value is `None`
+* [#119](https://gitlab.com/linkahead/linkahead-pylib/-/issues/119)
+ The diff returned by compare_entities now uses id instead of name as key if either property does not have a name
### Security ###
--
GitLab
From 3bd7191cbaec2d365acabf75478af53b2db73be0 Mon Sep 17 00:00:00 2001
From: "i.nueske" <i.nueske@indiscale.com>
Date: Sat, 30 Nov 2024 12:49:39 +0100
Subject: [PATCH 27/62] DOC: Update compare_entities Docstring to describe
which keys are used in the diff
---
src/linkahead/apiutils.py | 6 ++++--
1 file changed, 4 insertions(+), 2 deletions(-)
diff --git a/src/linkahead/apiutils.py b/src/linkahead/apiutils.py
index 1504db8e..fd2c23d6 100644
--- a/src/linkahead/apiutils.py
+++ b/src/linkahead/apiutils.py
@@ -196,6 +196,8 @@ def compare_entities(entity0: Optional[Entity] = None,
that are missing in the other entity, and the 'properties' dict contains
properties and SPECIAL_ATTRIBUTES if they are missing or different from
their counterparts in the other entity.
+ The key used to represent a parent or property is the entities name if the
+ name is present for both compared entities, the id otherwise.
The value of the properties dict for each listed property is again a dict
detailing the differences between this property and its counterpart.
@@ -224,9 +226,9 @@ def compare_entities(entity0: Optional[Entity] = None,
Params
------
- entity0 : Entity
+ entity0: Entity
First entity to be compared.
- entity1 : Entity
+ entity1: Entity
Second entity to be compared.
compare_referenced_records: bool, default: False
If set to True, values with referenced records
--
GitLab
From c8b0728a21ff9659738c117f558f92d5e15083a1 Mon Sep 17 00:00:00 2001
From: "i.nueske" <i.nueske@indiscale.com>
Date: Sat, 30 Nov 2024 16:06:28 +0100
Subject: [PATCH 28/62] MNT: Accept "" as a valid name in compare_entities
---
src/linkahead/apiutils.py | 9 +++++----
1 file changed, 5 insertions(+), 4 deletions(-)
diff --git a/src/linkahead/apiutils.py b/src/linkahead/apiutils.py
index fd2c23d6..9e4ba622 100644
--- a/src/linkahead/apiutils.py
+++ b/src/linkahead/apiutils.py
@@ -379,14 +379,15 @@ def compare_entities(entity0: Optional[Entity] = None,
# compare properties
for prop in entity0.properties:
- key = prop.name if prop.name else prop.id # ToDo: Would making id default break anything?
+ key = prop.name if prop.name is not None else prop.id # ToDo: Would making id default break anything?
matching = entity1.properties.filter(prop)
if len(matching) == 0:
# entity1 has prop, entity0 does not
diff[0]["properties"][key] = {}
elif len(matching) == 1:
# It's possible that prop has name and id, but match only has id
- key = prop.name if prop.name and matching[0].name else prop.id
+ key = prop.name if (prop.name is not None and
+ matching[0].name is not None) else prop.id
diff[0]["properties"][key] = {}
diff[1]["properties"][key] = {}
propdiff = (diff[0]["properties"][key],
@@ -431,7 +432,7 @@ def compare_entities(entity0: Optional[Entity] = None,
# we have not yet compared properties that do not exist in entity0
for prop in entity1.properties:
- key = prop.name if prop.name else prop.id # ToDo: Would making id default break anything?
+ key = prop.name if prop.name is not None else prop.id # ToDo: Would making id default break anything?
# check how often the property appears in entity0
num_prop_in_ent0 = len(entity0.properties.filter(prop))
if num_prop_in_ent0 == 0:
@@ -447,7 +448,7 @@ def compare_entities(entity0: Optional[Entity] = None,
for index, parents, other_entity in [(0, entity0.parents, entity1),
(1, entity1.parents, entity0)]:
for parent in parents:
- key = parent.name if parent.name else parent.id # ToDo: Would making id default break anything?
+ key = parent.name if parent.name is not None else parent.id # ToDo: Would making id default break anything?
matching = other_entity.parents.filter(parent)
if len(matching) == 0:
diff[index]["parents"].append(key)
--
GitLab
From f202d9400bbed0d7f737258da03d897d7a374f78 Mon Sep 17 00:00:00 2001
From: "i.nueske" <i.nueske@indiscale.com>
Date: Tue, 3 Dec 2024 11:52:40 +0100
Subject: [PATCH 29/62] BUG: Prevent infinite recursion in Entity.to_xml:
- Added a visited_entities parameter to save all Entities already treated in to_xml. New calls to to_xml are checked against this to identify potential infinite recursion
- Added test
- Updated changelog
---
CHANGELOG.md | 2 +
src/linkahead/common/models.py | 77 ++++++++++++++++++++++++++++------
unittests/test_issues.py | 35 ++++++++++++++++
3 files changed, 101 insertions(+), 13 deletions(-)
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 0544d70b..cb648b1d 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -19,6 +19,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
### Fixed ###
+* [#73](https://gitlab.com/linkahead/linkahead-pylib/-/issues/73)
+ `Entity.to_xml` now detects potentially infinite recursion and prevents an error
* [#89](https://gitlab.com/linkahead/linkahead-pylib/-/issues/89)
`to_xml` does not add `noscript` or `TransactionBenchmark` tags anymore
* [#103](https://gitlab.com/linkahead/linkahead-pylib/-/issues/103)
diff --git a/src/linkahead/common/models.py b/src/linkahead/common/models.py
index aff85556..74f7fa10 100644
--- a/src/linkahead/common/models.py
+++ b/src/linkahead/common/models.py
@@ -1232,6 +1232,7 @@ class Entity:
xml: Optional[etree._Element] = None,
add_properties: INHERITANCE = "ALL",
local_serialization: bool = False,
+ visited_entities: Optional[Union[list, None]] = None
) -> etree._Element:
"""Generate an xml representation of this entity. If the parameter xml
is given, all attributes, parents, properties, and messages of this
@@ -1242,9 +1243,10 @@ class Entity:
@param xml: an xml element to which all attributes, parents,
properties, and messages
are to be added.
+ @param visited_entities: recursion check, should never be set manually
FIXME: Add documentation for the add_properties parameter.
- FIXME: Add docuemntation for the local_serialization parameter.
+ FIXME: Add documentation for the local_serialization parameter.
@return: xml representation of this entity.
"""
@@ -1255,9 +1257,17 @@ class Entity:
xml = etree.Element(elem_tag)
assert isinstance(xml, etree._Element)
+ if visited_entities is None:
+ visited_entities = []
+ if self in visited_entities:
+ xml.text = "..."
+ return xml
+ visited_entities.append(self)
+
# unwrap wrapped entity
if self._wrapped_entity is not None:
- xml = self._wrapped_entity.to_xml(xml, add_properties)
+ xml = self._wrapped_entity.to_xml(xml, add_properties,
+ visited_entities=visited_entities.copy())
if self.id is not None:
xml.set("id", str(self.id))
@@ -1272,6 +1282,8 @@ class Entity:
xml.set("description", str(self.description))
if self.version is not None:
+ # Does version.to_xml need visited_entities support?
+ # It does have some recursion with predecessors / successors.
xml.append(self.version.to_xml())
if self.value is not None:
@@ -1281,7 +1293,7 @@ class Entity:
elif self.value.name is not None:
xml.text = str(self.value.name)
else:
- xml.text = str(self.value)
+ self.value.to_xml(xml, visited_entities=visited_entities.copy())
elif isinstance(self.value, list):
for v in self.value:
v_elem = etree.Element("Value")
@@ -1292,7 +1304,7 @@ class Entity:
elif v.name is not None:
v_elem.text = str(v.name)
else:
- v_elem.text = str(v)
+ v.to_xml(v_elem, visited_entities=visited_entities.copy())
elif v == "":
v_elem.append(etree.Element("EmptyString"))
elif v is None:
@@ -1314,7 +1326,10 @@ class Entity:
elif self.datatype.name is not None:
xml.set("datatype", str(self.datatype.name))
else:
- xml.set("datatype", str(self.datatype))
+ dt_str = xml2str(self.datatype.to_xml(visited_entities=visited_entities.copy()))
+ # Todo: Use for pretty-printing with calls from _repr_ only?
+ # dt_str = dt_str.replace('<', 'á¸').replace('>', 'á³').replace(' ', 'â €').replace('"', '\'').replace('\n', '')
+ xml.set("datatype", dt_str)
else:
xml.set("datatype", str(self.datatype))
@@ -1337,10 +1352,11 @@ class Entity:
self.messages.to_xml(xml)
if self.parents is not None:
- self.parents.to_xml(xml)
+ self.parents.to_xml(xml, visited_entities=visited_entities.copy())
if self.properties is not None:
- self.properties.to_xml(xml, add_properties)
+ self.properties.to_xml(xml, add_properties,
+ visited_entities=visited_entities.copy())
if len(self._flags) > 0:
flagattr = ""
@@ -1948,11 +1964,16 @@ class Parent(Entity):
xml: Optional[etree._Element] = None,
add_properties: INHERITANCE = "NONE",
local_serialization: bool = False,
+ visited_entities: Optional[Union[list, None]] = None,
):
if xml is None:
xml = etree.Element("Parent")
- return super().to_xml(xml=xml, add_properties=add_properties)
+ if visited_entities is None:
+ visited_entities = []
+
+ return super().to_xml(xml=xml, add_properties=add_properties,
+ visited_entities=visited_entities)
class _EntityWrapper(object):
@@ -2023,14 +2044,19 @@ class Property(Entity):
xml: Optional[etree._Element] = None,
add_properties: INHERITANCE = "ALL",
local_serialization: bool = False,
+ visited_entities: Optional[Union[list, None]] = None,
):
if xml is None:
xml = etree.Element("Property")
+ if visited_entities is None:
+ visited_entities = []
+
return super(Property, self).to_xml(
xml=xml,
add_properties=add_properties,
local_serialization=local_serialization,
+ visited_entities=visited_entities,
)
def is_reference(self, server_retrieval: bool = False) -> Optional[bool]:
@@ -2188,15 +2214,20 @@ class RecordType(Entity):
xml: Optional[etree._Element] = None,
add_properties: INHERITANCE = "ALL",
local_serialization: bool = False,
+ visited_entities: Optional[Union[list, None]] = None,
) -> etree._Element:
if xml is None:
xml = etree.Element("RecordType")
+ if visited_entities is None:
+ visited_entities = []
+
return Entity.to_xml(
self,
xml=xml,
add_properties=add_properties,
local_serialization=local_serialization,
+ visited_entities=visited_entities,
)
@@ -2227,14 +2258,19 @@ class Record(Entity):
xml: Optional[etree._Element] = None,
add_properties: INHERITANCE = "ALL",
local_serialization: bool = False,
+ visited_entities: Optional[Union[list, None]] = None,
):
if xml is None:
xml = etree.Element("Record")
+ if visited_entities is None:
+ visited_entities = []
+
return super().to_xml(
xml=xml,
add_properties=add_properties,
local_serialization=local_serialization,
+ visited_entities=visited_entities
)
@@ -2304,6 +2340,7 @@ class File(Record):
xml: Optional[etree._Element] = None,
add_properties: INHERITANCE = "ALL",
local_serialization: bool = False,
+ visited_entities: Optional[Union[list, None]] = None,
) -> etree._Element:
"""Convert this file to an xml element.
@@ -2313,8 +2350,12 @@ class File(Record):
if xml is None:
xml = etree.Element("File")
+ if visited_entities is None:
+ visited_entities = []
+
return Entity.to_xml(self, xml=xml, add_properties=add_properties,
- local_serialization=local_serialization)
+ local_serialization=local_serialization,
+ visited_entities=visited_entities)
def download(self, target: Optional[str] = None) -> str:
"""Download this file-entity's actual file from the file server. It
@@ -2476,15 +2517,20 @@ class PropertyList(list):
return self
- def to_xml(self, add_to_element: etree._Element, add_properties: INHERITANCE):
+ def to_xml(self, add_to_element: etree._Element, add_properties: INHERITANCE,
+ visited_entities: Optional[Union[list, None]] = None):
+
+ if visited_entities is None:
+ visited_entities = []
+
p: Property
for p in self:
importance = self._importance.get(p)
if add_properties == FIX and not importance == FIX:
continue
-
- pelem = p.to_xml(xml=etree.Element("Property"), add_properties=FIX)
+ pelem = p.to_xml(xml=etree.Element("Property"), add_properties=FIX,
+ visited_entities=visited_entities.copy())
if p in self._importance:
pelem.set("importance", str(importance))
@@ -2631,7 +2677,12 @@ class ParentList(list):
return self
- def to_xml(self, add_to_element: etree._Element):
+ def to_xml(self, add_to_element: etree._Element,
+ visited_entities: Optional[Union[list, None]] = None):
+
+ if visited_entities is None:
+ visited_entities = []
+
for p in self:
pelem = etree.Element("Parent")
diff --git a/unittests/test_issues.py b/unittests/test_issues.py
index e24afbe8..9d422ff8 100644
--- a/unittests/test_issues.py
+++ b/unittests/test_issues.py
@@ -26,6 +26,7 @@ import linkahead as db
from datetime import date, datetime
from pytest import raises
+from linkahead.common.utils import xml2str
def test_issue_100():
@@ -90,3 +91,37 @@ def test_issue_128():
assert prop_list.value == [today, today]
prop_list.value = [now, now]
assert prop_list.value == [now, now]
+
+
+def test_issue_73():
+ """
+ Test to_xml infinite recursion handling with cross- and self-references.
+ https://gitlab.com/linkahead/linkahead-pylib/-/issues/73
+ """
+ # Cross-reference in the property values
+ rt = db.RecordType(name="RT")
+ recA = db.Record().add_parent(rt)
+ recB = db.Record().add_parent(rt)
+ recA.add_property(name="RT", value=recB)
+ recB.add_property(name="RT", value=recA)
+ xml_str = xml2str(recB.to_xml())
+ assert "<Parent name=\"RT" in xml_str
+ assert "<Property name=\"RT" in xml_str
+ assert len(xml_str) < 500
+
+ # Cross-reference in the properties themselves
+ prop1 = db.Property(name="Prop1")
+ prop2 = db.Property(name="Prop2")
+ prop1.add_property(prop2)
+ prop2.add_property(prop1)
+ xml_str = xml2str(prop2.to_xml())
+ assert "<Property name=\"Prop1" in xml_str
+ assert "<Property name=\"Prop2" in xml_str
+ assert len(xml_str) < 500
+
+ # Self-reference in the datatype
+ prop = db.Property()
+ prop.datatype = prop
+ xml_str = xml2str(prop.to_xml())
+ assert "datatype=" in xml_str
+ assert len(xml_str) < 100
--
GitLab
From 4fbc941c70aad5bc82a42cf21764921e6ca93b6f Mon Sep 17 00:00:00 2001
From: "i.nueske" <i.nueske@indiscale.com>
Date: Tue, 3 Dec 2024 12:26:29 +0100
Subject: [PATCH 30/62] MNT: Revert unwanted behaviour change introduced by
bugfix in to_xml with lists of entities
---
src/linkahead/common/models.py | 6 +++++-
1 file changed, 5 insertions(+), 1 deletion(-)
diff --git a/src/linkahead/common/models.py b/src/linkahead/common/models.py
index 74f7fa10..1112aafc 100644
--- a/src/linkahead/common/models.py
+++ b/src/linkahead/common/models.py
@@ -1304,7 +1304,11 @@ class Entity:
elif v.name is not None:
v_elem.text = str(v.name)
else:
- v.to_xml(v_elem, visited_entities=visited_entities.copy())
+ # We could consider reworking this to use value
+ # instead of text analogously to scalar Entity
+ # values, if this is supported server-side?
+ dt_str = xml2str(v.to_xml(visited_entities=visited_entities.copy()))
+ v_elem.text = dt_str
elif v == "":
v_elem.append(etree.Element("EmptyString"))
elif v is None:
--
GitLab
From a78473630a6a1989fbf1ad89ffda60ba626b2c33 Mon Sep 17 00:00:00 2001
From: "i.nueske" <i.nueske@indiscale.com>
Date: Tue, 3 Dec 2024 12:30:09 +0100
Subject: [PATCH 31/62] MNT: Revert unwanted behaviour change introduced by
bugfix in to_xml with lists of entities
---
src/linkahead/common/models.py | 6 ++----
1 file changed, 2 insertions(+), 4 deletions(-)
diff --git a/src/linkahead/common/models.py b/src/linkahead/common/models.py
index 1112aafc..6f37c217 100644
--- a/src/linkahead/common/models.py
+++ b/src/linkahead/common/models.py
@@ -1293,7 +1293,8 @@ class Entity:
elif self.value.name is not None:
xml.text = str(self.value.name)
else:
- self.value.to_xml(xml, visited_entities=visited_entities.copy())
+ dt_str = xml2str(self.value.to_xml(visited_entities=visited_entities.copy()))
+ xml.text = dt_str
elif isinstance(self.value, list):
for v in self.value:
v_elem = etree.Element("Value")
@@ -1304,9 +1305,6 @@ class Entity:
elif v.name is not None:
v_elem.text = str(v.name)
else:
- # We could consider reworking this to use value
- # instead of text analogously to scalar Entity
- # values, if this is supported server-side?
dt_str = xml2str(v.to_xml(visited_entities=visited_entities.copy()))
v_elem.text = dt_str
elif v == "":
--
GitLab
From 593a3acb24796f728deba463db646d2a1722cef7 Mon Sep 17 00:00:00 2001
From: "i.nueske" <i.nueske@indiscale.com>
Date: Tue, 3 Dec 2024 12:58:23 +0100
Subject: [PATCH 32/62] BUG: Only use name as key in compare_entities diff if
names match
---
src/linkahead/apiutils.py | 9 +++++----
1 file changed, 5 insertions(+), 4 deletions(-)
diff --git a/src/linkahead/apiutils.py b/src/linkahead/apiutils.py
index 9e4ba622..e887ee4b 100644
--- a/src/linkahead/apiutils.py
+++ b/src/linkahead/apiutils.py
@@ -379,7 +379,8 @@ def compare_entities(entity0: Optional[Entity] = None,
# compare properties
for prop in entity0.properties:
- key = prop.name if prop.name is not None else prop.id # ToDo: Would making id default break anything?
+ # ToDo: Would making id default break anything?
+ key = prop.name if prop.name is not None else prop.id
matching = entity1.properties.filter(prop)
if len(matching) == 0:
# entity1 has prop, entity0 does not
@@ -387,7 +388,7 @@ def compare_entities(entity0: Optional[Entity] = None,
elif len(matching) == 1:
# It's possible that prop has name and id, but match only has id
key = prop.name if (prop.name is not None and
- matching[0].name is not None) else prop.id
+ matching[0].name == prop.name) else prop.id
diff[0]["properties"][key] = {}
diff[1]["properties"][key] = {}
propdiff = (diff[0]["properties"][key],
@@ -432,7 +433,7 @@ def compare_entities(entity0: Optional[Entity] = None,
# we have not yet compared properties that do not exist in entity0
for prop in entity1.properties:
- key = prop.name if prop.name is not None else prop.id # ToDo: Would making id default break anything?
+ key = prop.name if prop.name is not None else prop.id
# check how often the property appears in entity0
num_prop_in_ent0 = len(entity0.properties.filter(prop))
if num_prop_in_ent0 == 0:
@@ -448,7 +449,7 @@ def compare_entities(entity0: Optional[Entity] = None,
for index, parents, other_entity in [(0, entity0.parents, entity1),
(1, entity1.parents, entity0)]:
for parent in parents:
- key = parent.name if parent.name is not None else parent.id # ToDo: Would making id default break anything?
+ key = parent.name if parent.name is not None else parent.id
matching = other_entity.parents.filter(parent)
if len(matching) == 0:
diff[index]["parents"].append(key)
--
GitLab
From 3af22cc528744a9f4b0660b40164ec86478fe62c Mon Sep 17 00:00:00 2001
From: Florian Spreckelsen <f.spreckelsen@indiscale.com>
Date: Tue, 3 Dec 2024 15:25:51 +0100
Subject: [PATCH 33/62] TST: Add unittests for
https://gitlab.com/linkahead/linkahead-pylib/-/issues/119
---
unittests/test_apiutils.py | 28 ++++++++++++++++++++++++++++
1 file changed, 28 insertions(+)
diff --git a/unittests/test_apiutils.py b/unittests/test_apiutils.py
index fdd5adda..6667089a 100644
--- a/unittests/test_apiutils.py
+++ b/unittests/test_apiutils.py
@@ -991,3 +991,31 @@ def test_describe_diff():
assert "first" not in diffout
assert "second" not in diffout
+
+
+def test_diff_without_names():
+ """Test compare_entities in case of properties and parents with
+ ids and without names
+ (cf. https://gitlab.com/linkahead/linkahead-pylib/-/issues/119).
+
+ """
+
+ r1 = db.Record(name="Test").add_parent(name="TestType")
+ r2 = db.Record(name="Test").add_parent(name="TestType")
+ r2.add_property(id=123, value="Test")
+
+ diff1, diff2 = compare_entities(r1, r2)
+ assert len(diff1["properties"]) == 0
+ assert len(diff2["properties"]) == 1
+ assert 123 in diff2["properties"]
+ assert None not in diff2["properties"]
+
+ r3 = db.Record().add_parent(id=101)
+ r4 = db.Record().add_parent(id=102)
+ diff3, diff4 = compare_entities(r3, r4)
+ assert len(diff3["parents"]) == 1
+ assert 101 in diff3["parents"]
+ assert None not in diff3["parents"]
+ assert len(diff4["parents"]) == 1
+ assert 102 in diff4["parents"]
+ assert None not in diff3["parents"]
--
GitLab
From c6f14140b02ee98708a543368a87ff13c9fd15be Mon Sep 17 00:00:00 2001
From: Florian Spreckelsen <f.spreckelsen@indiscale.com>
Date: Tue, 3 Dec 2024 15:41:17 +0100
Subject: [PATCH 34/62] DOC: Update compare_entities docstring
---
src/linkahead/apiutils.py | 13 +++++++++----
1 file changed, 9 insertions(+), 4 deletions(-)
diff --git a/src/linkahead/apiutils.py b/src/linkahead/apiutils.py
index e887ee4b..1aa127d3 100644
--- a/src/linkahead/apiutils.py
+++ b/src/linkahead/apiutils.py
@@ -196,16 +196,20 @@ def compare_entities(entity0: Optional[Entity] = None,
that are missing in the other entity, and the 'properties' dict contains
properties and SPECIAL_ATTRIBUTES if they are missing or different from
their counterparts in the other entity.
- The key used to represent a parent or property is the entities name if the
+
+ The key used to represent a parent in the parent list or a
+ property in the property dictionary is the entity's name if the
name is present for both compared entities, the id otherwise.
The value of the properties dict for each listed property is again a dict
detailing the differences between this property and its counterpart.
The characteristics that are checked to determine whether two properties
match are the following:
- - datatype
- - importance
- - value
+
+ - datatype
+ - importance
+ - value
+
If any of these characteristics differ for a property, the respective
string (datatype, importance, value) is added as a key to the dict of the
property with its value being the characteristics value,
@@ -244,6 +248,7 @@ def compare_entities(entity0: Optional[Entity] = None,
entity and an int or str also checks whether
the int/str matches the name or id of the
entity, so Entity(id=100) == 100 == "100".
+
"""
# ToDo: Discuss intended behaviour
# Questions that need clarification:
--
GitLab
From 6f276cc8477f695ead4647ccd96e9985908b4af5 Mon Sep 17 00:00:00 2001
From: "i.nueske" <i.nueske@indiscale.com>
Date: Wed, 4 Dec 2024 14:19:35 +0100
Subject: [PATCH 35/62] WIP: Support separate connect/read timeouts and timeout
None
---
src/linkahead/configuration.py | 25 ++++++++++++++++++++-----
src/linkahead/connection/connection.py | 16 ++++++++++++----
src/linkahead/schema-pycaosdb-ini.yml | 8 +++++++-
3 files changed, 39 insertions(+), 10 deletions(-)
diff --git a/src/linkahead/configuration.py b/src/linkahead/configuration.py
index f57289d7..017743e5 100644
--- a/src/linkahead/configuration.py
+++ b/src/linkahead/configuration.py
@@ -30,6 +30,14 @@ import yaml
try:
optional_jsonschema_validate: Optional[Callable] = None
from jsonschema import validate as optional_jsonschema_validate
+
+ # Adapted from https://github.com/python-jsonschema/jsonschema/issues/148
+ # Defines Validator to allow parsing of all iterables as array in jsonschema
+ from collections.abc import Iterable
+ from jsonschema import validators
+ default = validators._LATEST_VERSION
+ t_c = (default.TYPE_CHECKER.redefine('array', lambda x, y: isinstance(y, Iterable)))
+ CustomValidator = validators.extend(default, type_checker=t_c)
except ImportError:
pass
@@ -72,14 +80,20 @@ def get_config() -> ConfigParser:
return _pycaosdbconf
-def config_to_yaml(config: ConfigParser) -> dict[str, dict[str, Union[int, str, bool]]]:
- valobj: dict[str, dict[str, Union[int, str, bool]]] = {}
+def config_to_yaml(config: ConfigParser) -> dict[str, dict[str, Union[int, str, bool, tuple, None]]]:
+ valobj: dict[str, dict[str, Union[int, str, bool, tuple, None]]] = {}
for s in config.sections():
valobj[s] = {}
for key, value in config[s].items():
# TODO: Can the type be inferred from the config object?
if key in ["timeout", "debug"]:
- valobj[s][key] = int(value)
+ if str(value).lower() in ["none", "null"]:
+ valobj[s][key] = None
+ elif value.startswith('(') and value.endswith(')'):
+ content = [int(s) for s in value[1:-1].split(',')]
+ valobj[s][key] = tuple(content)
+ else:
+ valobj[s][key] = int(value)
elif key in ["ssl_insecure"]:
valobj[s][key] = bool(value)
else:
@@ -88,11 +102,12 @@ def config_to_yaml(config: ConfigParser) -> dict[str, dict[str, Union[int, str,
return valobj
-def validate_yaml_schema(valobj: dict[str, dict[str, Union[int, str, bool]]]):
+def validate_yaml_schema(valobj: dict[str, dict[str, Union[int, str, bool, tuple, None]]]):
if optional_jsonschema_validate:
with open(os.path.join(os.path.dirname(__file__), "schema-pycaosdb-ini.yml")) as f:
schema = yaml.load(f, Loader=yaml.SafeLoader)
- optional_jsonschema_validate(instance=valobj, schema=schema["schema-pycaosdb-ini"])
+ optional_jsonschema_validate(instance=valobj, schema=schema["schema-pycaosdb-ini"],
+ cls=CustomValidator)
else:
warnings.warn("""
Warning: The validation could not be performed because `jsonschema` is not installed.
diff --git a/src/linkahead/connection/connection.py b/src/linkahead/connection/connection.py
index 4c40842a..a9f07567 100644
--- a/src/linkahead/connection/connection.py
+++ b/src/linkahead/connection/connection.py
@@ -39,7 +39,7 @@ from requests.adapters import HTTPAdapter
from requests.exceptions import ConnectionError as HTTPConnectionError
from urllib3.poolmanager import PoolManager
-from ..configuration import get_config
+from ..configuration import get_config, config_to_yaml
from ..exceptions import (ConfigurationError, HTTPClientError,
HTTPForbiddenError, HTTPResourceNotFoundError,
HTTPServerError, HTTPURITooLongError,
@@ -465,21 +465,29 @@ def configure_connection(**kwargs):
global_conf = {}
conf = get_config()
# Convert config to dict, with preserving types
- int_opts = ["timeout"]
+ int_opts = []
bool_opts = ["ssl_insecure"]
+ other_opts = ["timeout"]
if conf.has_section("Connection"):
global_conf = dict(conf.items("Connection"))
- # Integer options
+ # Integer options
for opt in int_opts:
if opt in global_conf:
global_conf[opt] = conf.getint("Connection", opt)
- # Boolean options
+ # Boolean options
for opt in bool_opts:
if opt in global_conf:
global_conf[opt] = conf.getboolean("Connection", opt)
+
+ # Other options, defer parsing to configuration.config_to_yaml:
+ connection_config = config_to_yaml(conf)["Connection"]
+ for opt in other_opts:
+ if opt in global_conf:
+ global_conf[opt] = connection_config[opt]
+
local_conf = _make_conf(_DEFAULT_CONF, global_conf, kwargs)
connection = _Connection.get_instance()
diff --git a/src/linkahead/schema-pycaosdb-ini.yml b/src/linkahead/schema-pycaosdb-ini.yml
index 89ce9857..ae46b905 100644
--- a/src/linkahead/schema-pycaosdb-ini.yml
+++ b/src/linkahead/schema-pycaosdb-ini.yml
@@ -67,7 +67,13 @@ schema-pycaosdb-ini:
description: This option is used internally and for testing. Do not override.
examples: [_DefaultCaosDBServerConnection]
timeout:
- type: integer
+ oneOf:
+ - type: [integer, "null"]
+ - type: array
+ items:
+ type: [integer, "null"]
+ minItems: 2
+ maxItems: 2
allOf:
- if:
properties:
--
GitLab
From 822069ccfd227ca7d31353f858a38c3fe4000f32 Mon Sep 17 00:00:00 2001
From: "i.nueske" <i.nueske@indiscale.com>
Date: Wed, 4 Dec 2024 14:38:12 +0100
Subject: [PATCH 36/62] ENH: Support separate connect/read timeouts and timeout
None in pylinkahead.ini
---
src/linkahead/configuration.py | 7 +++++--
1 file changed, 5 insertions(+), 2 deletions(-)
diff --git a/src/linkahead/configuration.py b/src/linkahead/configuration.py
index 017743e5..c72c46e8 100644
--- a/src/linkahead/configuration.py
+++ b/src/linkahead/configuration.py
@@ -33,9 +33,10 @@ try:
# Adapted from https://github.com/python-jsonschema/jsonschema/issues/148
# Defines Validator to allow parsing of all iterables as array in jsonschema
+ # CustomValidator can be removed if/once jsonschema allows tuples for arrays
from collections.abc import Iterable
from jsonschema import validators
- default = validators._LATEST_VERSION
+ default = validators.validator_for(True) # Returns latest supported draft
t_c = (default.TYPE_CHECKER.redefine('array', lambda x, y: isinstance(y, Iterable)))
CustomValidator = validators.extend(default, type_checker=t_c)
except ImportError:
@@ -86,7 +87,9 @@ def config_to_yaml(config: ConfigParser) -> dict[str, dict[str, Union[int, str,
valobj[s] = {}
for key, value in config[s].items():
# TODO: Can the type be inferred from the config object?
- if key in ["timeout", "debug"]:
+ if key in ["debug"]:
+ valobj[s][key] = int(value)
+ if key in ["timeout"]:
if str(value).lower() in ["none", "null"]:
valobj[s][key] = None
elif value.startswith('(') and value.endswith(')'):
--
GitLab
From 74684a84f6e810612a4ff1fd7c1aa1dcbf4e24de Mon Sep 17 00:00:00 2001
From: "i.nueske" <i.nueske@indiscale.com>
Date: Wed, 4 Dec 2024 14:41:43 +0100
Subject: [PATCH 37/62] MNT: Fix Typo
---
src/linkahead/configuration.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/src/linkahead/configuration.py b/src/linkahead/configuration.py
index c72c46e8..e11749fb 100644
--- a/src/linkahead/configuration.py
+++ b/src/linkahead/configuration.py
@@ -89,7 +89,7 @@ def config_to_yaml(config: ConfigParser) -> dict[str, dict[str, Union[int, str,
# TODO: Can the type be inferred from the config object?
if key in ["debug"]:
valobj[s][key] = int(value)
- if key in ["timeout"]:
+ elif key in ["timeout"]:
if str(value).lower() in ["none", "null"]:
valobj[s][key] = None
elif value.startswith('(') and value.endswith(')'):
--
GitLab
From 99db34bb1df539643993c691594773094bddab0c Mon Sep 17 00:00:00 2001
From: "i.nueske" <i.nueske@indiscale.com>
Date: Wed, 4 Dec 2024 17:29:50 +0100
Subject: [PATCH 38/62] MNT: More robust timeout parsing, update changelog
---
CHANGELOG.md | 2 ++
src/linkahead/configuration.py | 4 +++-
2 files changed, 5 insertions(+), 1 deletion(-)
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 03afb6a4..fc480143 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -26,6 +26,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
`auth_token` if new value is `None`
* [#119](https://gitlab.com/linkahead/linkahead-pylib/-/issues/119)
The diff returned by compare_entities now uses id instead of name as key if either property does not have a name
+* [#127](https://gitlab.com/linkahead/linkahead-pylib/-/issues/127)
+ pylinkahead.ini now supports None and tuples as values for the `timeout` keyword
### Security ###
diff --git a/src/linkahead/configuration.py b/src/linkahead/configuration.py
index e11749fb..a87aca4f 100644
--- a/src/linkahead/configuration.py
+++ b/src/linkahead/configuration.py
@@ -90,10 +90,12 @@ def config_to_yaml(config: ConfigParser) -> dict[str, dict[str, Union[int, str,
if key in ["debug"]:
valobj[s][key] = int(value)
elif key in ["timeout"]:
+ value = "".join(value.split()) # Remove whitespace
if str(value).lower() in ["none", "null"]:
valobj[s][key] = None
elif value.startswith('(') and value.endswith(')'):
- content = [int(s) for s in value[1:-1].split(',')]
+ content = [None if str(s).lower() in ["none", "null"] else int(s)
+ for s in value[1:-1].split(',')]
valobj[s][key] = tuple(content)
else:
valobj[s][key] = int(value)
--
GitLab
From 8778db51c72e8595938389f0a57b5c9900984b22 Mon Sep 17 00:00:00 2001
From: Florian Spreckelsen <f.spreckelsen@indiscale.com>
Date: Thu, 5 Dec 2024 14:48:17 +0100
Subject: [PATCH 39/62] DOC: Fix docstring and inline comment
---
src/linkahead/common/models.py | 34 +++++++++++++++++++++++-----------
1 file changed, 23 insertions(+), 11 deletions(-)
diff --git a/src/linkahead/common/models.py b/src/linkahead/common/models.py
index 6f37c217..6cafad9c 100644
--- a/src/linkahead/common/models.py
+++ b/src/linkahead/common/models.py
@@ -1232,7 +1232,7 @@ class Entity:
xml: Optional[etree._Element] = None,
add_properties: INHERITANCE = "ALL",
local_serialization: bool = False,
- visited_entities: Optional[Union[list, None]] = None
+ visited_entities: Optional[list] = None
) -> etree._Element:
"""Generate an xml representation of this entity. If the parameter xml
is given, all attributes, parents, properties, and messages of this
@@ -1240,15 +1240,25 @@ class Entity:
Raise an error if xml is not a lxml.etree.Element
- @param xml: an xml element to which all attributes, parents,
- properties, and messages
- are to be added.
- @param visited_entities: recursion check, should never be set manually
-
- FIXME: Add documentation for the add_properties parameter.
- FIXME: Add documentation for the local_serialization parameter.
+ Parameters
+ ----------
+ xml : etree._Element, optional
+ an xml element to which all attributes, parents,
+ properties, and messages are to be added. Default is None.
+ visited_entities : list, optional
+ list of enties that are being printed for recursion check,
+ should never be set manually. Default is None.
+ add_properties : INHERITANCE, optional
+ FIXME: Add documentation for the add_properties
+ parameter. Default is "ALL".
+ local_serialization : bool, optional
+ FIXME: Add documentation for the local_serialization
+ parameter. Default is False.
- @return: xml representation of this entity.
+ Returns
+ -------
+ xml : etree._Element
+ xml representation of this entity.
"""
if xml is None:
@@ -1282,8 +1292,10 @@ class Entity:
xml.set("description", str(self.description))
if self.version is not None:
- # Does version.to_xml need visited_entities support?
- # It does have some recursion with predecessors / successors.
+ # If this ever causes problems, we might add
+ # visited_entities support here since it does have some
+ # recursion with predecessors / successors. But should be
+ # fine for now, since it is always set by the server.
xml.append(self.version.to_xml())
if self.value is not None:
--
GitLab
From 7c96253a27f30f5427185e72172d4516007f0f0f Mon Sep 17 00:00:00 2001
From: Florian Spreckelsen <f.spreckelsen@indiscale.com>
Date: Thu, 5 Dec 2024 15:28:01 +0100
Subject: [PATCH 40/62] MAINT: Change ellipses to etree.Comment
---
src/linkahead/common/models.py | 3 +--
1 file changed, 1 insertion(+), 2 deletions(-)
diff --git a/src/linkahead/common/models.py b/src/linkahead/common/models.py
index 6cafad9c..740c81aa 100644
--- a/src/linkahead/common/models.py
+++ b/src/linkahead/common/models.py
@@ -1270,8 +1270,7 @@ class Entity:
if visited_entities is None:
visited_entities = []
if self in visited_entities:
- xml.text = "..."
- return xml
+ return etree.Comment("Recursive reference")
visited_entities.append(self)
# unwrap wrapped entity
--
GitLab
From 063d186feb004c0c10250d6e990338229662ba73 Mon Sep 17 00:00:00 2001
From: Florian Spreckelsen <f.spreckelsen@indiscale.com>
Date: Thu, 5 Dec 2024 16:07:43 +0100
Subject: [PATCH 41/62] TST: Test for XML comment text
---
src/linkahead/common/models.py | 3 ++-
unittests/test_issues.py | 5 ++++-
2 files changed, 6 insertions(+), 2 deletions(-)
diff --git a/src/linkahead/common/models.py b/src/linkahead/common/models.py
index 740c81aa..1caa6a4d 100644
--- a/src/linkahead/common/models.py
+++ b/src/linkahead/common/models.py
@@ -1270,7 +1270,8 @@ class Entity:
if visited_entities is None:
visited_entities = []
if self in visited_entities:
- return etree.Comment("Recursive reference")
+ xml.text = xml2str(etree.Comment("Recursive reference"))
+ return xml
visited_entities.append(self)
# unwrap wrapped entity
diff --git a/unittests/test_issues.py b/unittests/test_issues.py
index 9d422ff8..3b0117b2 100644
--- a/unittests/test_issues.py
+++ b/unittests/test_issues.py
@@ -107,6 +107,7 @@ def test_issue_73():
xml_str = xml2str(recB.to_xml())
assert "<Parent name=\"RT" in xml_str
assert "<Property name=\"RT" in xml_str
+ assert "Recursive reference" in xml_str
assert len(xml_str) < 500
# Cross-reference in the properties themselves
@@ -117,6 +118,7 @@ def test_issue_73():
xml_str = xml2str(prop2.to_xml())
assert "<Property name=\"Prop1" in xml_str
assert "<Property name=\"Prop2" in xml_str
+ assert "Recursive reference" in xml_str
assert len(xml_str) < 500
# Self-reference in the datatype
@@ -124,4 +126,5 @@ def test_issue_73():
prop.datatype = prop
xml_str = xml2str(prop.to_xml())
assert "datatype=" in xml_str
- assert len(xml_str) < 100
+ assert "Recursive reference" in xml_str
+ assert len(xml_str) < 500
--
GitLab
From 89b25118c7514d4dcbd0fb574ca015c70fbc6d28 Mon Sep 17 00:00:00 2001
From: "i.nueske" <i.nueske@indiscale.com>
Date: Sun, 8 Dec 2024 17:53:39 +0100
Subject: [PATCH 42/62] TST: Unittest for config timeout option support of
tuples and None
---
unittests/test_configs/pylinkahead-timeout1.ini | 4 ++++
unittests/test_configs/pylinkahead-timeout2.ini | 4 ++++
unittests/test_configuration.py | 16 ++++++++++++++++
3 files changed, 24 insertions(+)
create mode 100644 unittests/test_configs/pylinkahead-timeout1.ini
create mode 100644 unittests/test_configs/pylinkahead-timeout2.ini
diff --git a/unittests/test_configs/pylinkahead-timeout1.ini b/unittests/test_configs/pylinkahead-timeout1.ini
new file mode 100644
index 00000000..d9f894bf
--- /dev/null
+++ b/unittests/test_configs/pylinkahead-timeout1.ini
@@ -0,0 +1,4 @@
+[Connection]
+url=https://localhost:10443/
+password_method = unauthenticated
+timeout = None
diff --git a/unittests/test_configs/pylinkahead-timeout2.ini b/unittests/test_configs/pylinkahead-timeout2.ini
new file mode 100644
index 00000000..b3d3796f
--- /dev/null
+++ b/unittests/test_configs/pylinkahead-timeout2.ini
@@ -0,0 +1,4 @@
+[Connection]
+url=https://localhost:10443/
+password_method = unauthenticated
+timeout = (1,20)
diff --git a/unittests/test_configuration.py b/unittests/test_configuration.py
index 95bc906c..772e872c 100644
--- a/unittests/test_configuration.py
+++ b/unittests/test_configuration.py
@@ -24,6 +24,7 @@
from os import environ, getcwd, remove
from os.path import expanduser, isfile, join
+from pathlib import Path
import linkahead as db
import pytest
@@ -66,3 +67,18 @@ def test_config_ini_via_envvar(temp_ini_files):
assert expanduser("~/.pylinkahead.ini") in db.configuration._read_config_files()
# test configuration file in cwd
assert join(getcwd(), "pylinkahead.ini") in db.configuration._read_config_files()
+
+
+def test_config_timeout_option():
+ expected_results = [None, (1, 20)]
+ # Iterate through timeout test configs
+ test_configs = Path(__file__).parent/'test_configs'
+ for test_config in test_configs.rglob('pylinkahead-timeout*.ini'):
+ # Test that test configs can be parsed
+ db.configure(str(test_config))
+ dct = db.configuration.config_to_yaml(db.get_config())
+ # Test that resulting dict has correct content for timeout
+ assert 'Connection' in dct
+ assert 'timeout' in dct['Connection']
+ assert dct['Connection']['timeout'] in expected_results
+ expected_results.remove(dct['Connection']['timeout'])
--
GitLab
From a2e5671921b374fe93268f07edead0d2b6e5d9e6 Mon Sep 17 00:00:00 2001
From: Florian Spreckelsen <f.spreckelsen@indiscale.com>
Date: Thu, 12 Dec 2024 11:25:55 +0100
Subject: [PATCH 43/62] TST: Add test for
https://gitlab.com/linkahead/linkahead-pylib/-/issues/87
---
unittests/test_error_handling.py | 8 +++++++-
1 file changed, 7 insertions(+), 1 deletion(-)
diff --git a/unittests/test_error_handling.py b/unittests/test_error_handling.py
index 3f524146..c7325341 100644
--- a/unittests/test_error_handling.py
+++ b/unittests/test_error_handling.py
@@ -30,7 +30,7 @@ import linkahead as db
from linkahead.common.models import raise_errors
from linkahead.exceptions import (AuthorizationError,
EntityDoesNotExistError, EntityError,
- EntityHasNoDatatypeError,
+ EntityHasNoDatatypeError, HTTPServerError,
TransactionError, UniqueNamesError,
UnqualifiedParentsError,
UnqualifiedPropertiesError)
@@ -315,3 +315,9 @@ def test_container_with_faulty_elements():
# record raises both of them
assert (isinstance(err, UnqualifiedParentsError) or
isinstance(err, UnqualifiedPropertiesError))
+
+
+def test_incomplete_server_error_response():
+ """The reason behind https://gitlab.com/linkahead/linkahead-pylib/-/issues/87."""
+ err = HTTPServerError("Bla")
+ assert str(err) == "Bla"
--
GitLab
From 4d6d1c8619d3e72dc3b87e1e05b4b318d849526b Mon Sep 17 00:00:00 2001
From: Florian Spreckelsen <f.spreckelsen@indiscale.com>
Date: Thu, 12 Dec 2024 11:40:58 +0100
Subject: [PATCH 44/62] TST: extend tests for
https://gitlab.com/linkahead/linkahead-pylib/-/issues/87
---
unittests/test_error_handling.py | 17 +++++++++++++++++
1 file changed, 17 insertions(+)
diff --git a/unittests/test_error_handling.py b/unittests/test_error_handling.py
index c7325341..64f743c8 100644
--- a/unittests/test_error_handling.py
+++ b/unittests/test_error_handling.py
@@ -319,5 +319,22 @@ def test_container_with_faulty_elements():
def test_incomplete_server_error_response():
"""The reason behind https://gitlab.com/linkahead/linkahead-pylib/-/issues/87."""
+ # Case 1: Response is no XML at all
err = HTTPServerError("Bla")
assert str(err) == "Bla"
+
+ # Case 2: Response is an incomplete XML, e.g. due to very unlucky timeout
+ err = HTTPServerError("<incomplete>XML</inc")
+ assert str(err) == "<incomplete>XML</inc"
+
+ # Case 3: Response is complete XML but doesn't have response and or error information
+ err = HTTPServerError("<complete>XML</complete>")
+ assert str(err) == "<complete>XML</complete>"
+
+ # Case 4: Response is an XML response but the error is lacking a description
+ err = HTTPServerError("<Response><Error>complete error</Error></Response>")
+ assert str(err) == "complete error"
+
+ # Case 5: Healthy error Response
+ err = HTTPServerError("<Response><Error description='Error'>complete error</Error></Response>")
+ assert str(err) == "Error\n\ncomplete error"
--
GitLab
From 8883815a4a7a16c8ef1ed333bcee1fa8f79629fa Mon Sep 17 00:00:00 2001
From: Florian Spreckelsen <f.spreckelsen@indiscale.com>
Date: Thu, 12 Dec 2024 11:41:19 +0100
Subject: [PATCH 45/62] BUG: Fix
https://gitlab.com/linkahead/linkahead-pylib/-/issues/87
---
src/linkahead/exceptions.py | 26 ++++++++++++++++++++------
1 file changed, 20 insertions(+), 6 deletions(-)
diff --git a/src/linkahead/exceptions.py b/src/linkahead/exceptions.py
index 609d3654..7d4dc085 100644
--- a/src/linkahead/exceptions.py
+++ b/src/linkahead/exceptions.py
@@ -94,12 +94,26 @@ class HTTPServerError(LinkAheadException):
"""HTTPServerError represents 5xx HTTP server errors."""
def __init__(self, body):
- xml = etree.fromstring(body)
- error = xml.xpath('/Response/Error')[0]
- msg = error.get("description")
-
- if error.text is not None:
- msg = msg + "\n\n" + error.text
+ try:
+ # This only works if the server sends a valid XML
+ # response. Then it can be parsed for more information.
+ xml = etree.fromstring(body)
+ if xml.xpath('/Response/Error'):
+ error = xml.xpath('/Response/Error')[0]
+ msg = error.get("description") if error.get("description") is not None else ""
+
+ if error.text is not None:
+ if msg:
+ msg = msg + "\n\n" + error.text
+ else:
+ msg = error.text
+ else:
+ # Valid XML, but no error information
+ msg = body
+ except etree.XMLSyntaxError:
+ # Handling of incomplete responses, e.g., due to timeouts,
+ # c.f. https://gitlab.com/linkahead/linkahead-pylib/-/issues/87.
+ msg = body
LinkAheadException.__init__(self, msg)
--
GitLab
From 84443119172283b71f60a3516784a2595a06e69a Mon Sep 17 00:00:00 2001
From: Florian Spreckelsen <f.spreckelsen@indiscale.com>
Date: Thu, 12 Dec 2024 12:02:38 +0100
Subject: [PATCH 46/62] DOC: Update Changelog
---
CHANGELOG.md | 6 +++++-
1 file changed, 5 insertions(+), 1 deletion(-)
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 928c5230..e1fd615a 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -27,7 +27,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
`authentication/interface/on_response()` does not overwrite
`auth_token` if new value is `None`
* [#119](https://gitlab.com/linkahead/linkahead-pylib/-/issues/119)
- The diff returned by compare_entities now uses id instead of name as key if either property does not have a name
+ The diff returned by compare_entities now uses id instead of name as
+ key if either property does not have a name
+* [#87](https://gitlab.com/linkahead/linkahead-pylib/-/issues/87)
+ `XMLSyntaxError` messages when parsing (incomplete) responses in
+ case of certain connection timeouts.
### Security ###
--
GitLab
From 17ed5c12d78c3f8d74a2b4ec14131a1389d54fd4 Mon Sep 17 00:00:00 2001
From: "i.nueske" <i.nueske@indiscale.com>
Date: Mon, 16 Dec 2024 17:19:38 +0100
Subject: [PATCH 47/62] MNT: Add getters for some protected Entity parameters
---
src/linkahead/common/models.py | 9 +++++++++
1 file changed, 9 insertions(+)
diff --git a/src/linkahead/common/models.py b/src/linkahead/common/models.py
index 1caa6a4d..b5747e05 100644
--- a/src/linkahead/common/models.py
+++ b/src/linkahead/common/models.py
@@ -200,6 +200,15 @@ class Entity:
return self._wrapped_entity.version
+ @property # getter for _cuid
+ def cuid(self):
+ # Set if None?
+ return self._cuid
+
+ @property # getter for _flags
+ def flags(self):
+ return self._flags.copy() # for dict[str, str] shallow copy is enough
+
@version.setter
def version(self, version: Optional[Version]):
self._version = version
--
GitLab
From d307a24e8aff3e6003bc3f10f95f2a6256b2bbf9 Mon Sep 17 00:00:00 2001
From: "i.nueske" <i.nueske@indiscale.com>
Date: Mon, 16 Dec 2024 17:48:59 +0100
Subject: [PATCH 48/62] MNT: Fix incorrect property order introduced by new
getters
---
src/linkahead/common/models.py | 18 +++++++++---------
1 file changed, 9 insertions(+), 9 deletions(-)
diff --git a/src/linkahead/common/models.py b/src/linkahead/common/models.py
index b5747e05..9093ebcc 100644
--- a/src/linkahead/common/models.py
+++ b/src/linkahead/common/models.py
@@ -200,15 +200,6 @@ class Entity:
return self._wrapped_entity.version
- @property # getter for _cuid
- def cuid(self):
- # Set if None?
- return self._cuid
-
- @property # getter for _flags
- def flags(self):
- return self._flags.copy() # for dict[str, str] shallow copy is enough
-
@version.setter
def version(self, version: Optional[Version]):
self._version = version
@@ -358,6 +349,15 @@ class Entity:
def pickup(self, new_pickup):
self.__pickup = new_pickup
+ @property # getter for _cuid
+ def cuid(self):
+ # Set if None?
+ return self._cuid
+
+ @property # getter for _flags
+ def flags(self):
+ return self._flags.copy() # for dict[str, str] shallow copy is enough
+
def grant(
self,
realm: Optional[str] = None,
--
GitLab
From c0c6d54326f335417da8eff75a1eaf244c00b124 Mon Sep 17 00:00:00 2001
From: "i.nueske" <i.nueske@indiscale.com>
Date: Tue, 17 Dec 2024 09:41:57 +0100
Subject: [PATCH 49/62] CI: Remove python 3.8 tests from pipeline
---
.gitlab-ci.yml | 18 ++++++------------
1 file changed, 6 insertions(+), 12 deletions(-)
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 463b1d56..db600343 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -61,16 +61,6 @@ mypy:
- make mypy
# run unit tests
-unittest_py3.8:
- tags: [ docker ]
- stage: test
- needs: [ ]
- image: python:3.8
- script: &python_test_script
- # Python docker has problems with tox and pip so use plain pytest here
- - touch ~/.pylinkahead.ini
- - pip install .[test]
- - python -m pytest unittests
# This needs to be changed once Python 3.9 isn't the standard Python in Debian
# anymore.
@@ -90,7 +80,11 @@ unittest_py3.10:
stage: test
needs: [ ]
image: python:3.10
- script: *python_test_script
+ script: &python_test_script
+ # Python docker has problems with tox and pip so use plain pytest here
+ - touch ~/.pylinkahead.ini
+ - pip install .[test]
+ - python -m pytest unittests
unittest_py3.11:
tags: [ docker ]
@@ -158,7 +152,7 @@ build-testenv:
pages_prepare: &pages_prepare
tags: [ cached-dind ]
stage: deploy
- needs: [ code_style, pylint, unittest_py3.8, unittest_py3.9, unittest_py3.10 ]
+ needs: [ code_style, pylint, unittest_py3.9, unittest_py3.10 ]
only:
refs:
- /^release-.*$/i
--
GitLab
From 7fbbea90293b62851e3779d96f8b432accde2f85 Mon Sep 17 00:00:00 2001
From: "i.nueske" <i.nueske@indiscale.com>
Date: Tue, 17 Dec 2024 10:00:14 +0100
Subject: [PATCH 50/62] MNT: Remove support for python 3.8
---
CHANGELOG.md | 2 ++
DEPENDENCIES.md | 2 +-
README_SETUP.md | 4 ++--
setup.py | 2 +-
4 files changed, 6 insertions(+), 4 deletions(-)
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 928c5230..7a0cf22b 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -17,6 +17,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
### Removed ###
+* Support for Python 3.8
+
### Fixed ###
* [#73](https://gitlab.com/linkahead/linkahead-pylib/-/issues/73)
diff --git a/DEPENDENCIES.md b/DEPENDENCIES.md
index e2326b83..e9bd54a1 100644
--- a/DEPENDENCIES.md
+++ b/DEPENDENCIES.md
@@ -1,5 +1,5 @@
* caosdb-server >= 0.12.0
-* Python >= 3.8
+* Python >= 3.9
* pip >= 20.0.2
Any other dependencies are defined in the setup.py and are being installed via pip
diff --git a/README_SETUP.md b/README_SETUP.md
index 8fc93474..f4c92138 100644
--- a/README_SETUP.md
+++ b/README_SETUP.md
@@ -4,7 +4,7 @@
### How to install ###
-First ensure that python with at least version 3.8 is installed. Should this not be
+First ensure that python with at least version 3.9 is installed. Should this not be
the case, you can use the [Installing python](#installing-python-) guide for your OS.
#### Generic installation ####
@@ -39,7 +39,7 @@ entries `install_requires` and `extras_require`.
#### Linux ####
-Make sure that Python (at least version 3.8) and pip is installed, using your system tools and
+Make sure that Python (at least version 3.9) and pip is installed, using your system tools and
documentation.
Then open a terminal and continue in the [Generic installation](#generic-installation) section.
diff --git a/setup.py b/setup.py
index daf36764..7a7bf9bd 100755
--- a/setup.py
+++ b/setup.py
@@ -179,7 +179,7 @@ def setup_package():
"Topic :: Scientific/Engineering :: Information Analysis",
],
packages=find_packages('src'),
- python_requires='>=3.8',
+ python_requires='>=3.9',
package_dir={'': 'src'},
install_requires=['lxml>=4.6.3',
"requests[socks]>=2.26",
--
GitLab
From fab901d3153af65a65051a27752e9d036734225c Mon Sep 17 00:00:00 2001
From: "i.nueske" <i.nueske@indiscale.com>
Date: Tue, 17 Dec 2024 12:10:46 +0100
Subject: [PATCH 51/62] DOC: Update changelog
---
CHANGELOG.md | 2 ++
1 file changed, 2 insertions(+)
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 928c5230..de37f046 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -10,6 +10,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
### Added ###
* New setup extra `test` which installs the dependencies for testing.
+* The `Entity` properties `_cuid` and `_flags` are now available for read-only access
+ as `cuid` and `flags`, respectively.
### Changed ###
--
GitLab
From 6d8b49cc090d43c5baf3286840b937b264e5181a Mon Sep 17 00:00:00 2001
From: "i.nueske" <i.nueske@indiscale.com>
Date: Tue, 17 Dec 2024 12:37:55 +0100
Subject: [PATCH 52/62] DOC: Update docstrings
---
src/linkahead/configuration.py | 16 ++++++++++++++++
src/linkahead/connection/connection.py | 4 +++-
2 files changed, 19 insertions(+), 1 deletion(-)
diff --git a/src/linkahead/configuration.py b/src/linkahead/configuration.py
index a87aca4f..5081c28a 100644
--- a/src/linkahead/configuration.py
+++ b/src/linkahead/configuration.py
@@ -82,6 +82,22 @@ def get_config() -> ConfigParser:
def config_to_yaml(config: ConfigParser) -> dict[str, dict[str, Union[int, str, bool, tuple, None]]]:
+ """
+ Generates and returns a dict with all config options and their values
+ defined in the config.
+ The values of the options 'debug', 'timeout', and 'ssl_insecure' are
+ parsed, all other values are saved as string.
+
+ Parameters
+ ----------
+ config : ConfigParser
+ The config to be converted to a dict
+
+ Returns
+ -------
+ valobj : dict
+ A dict with config options and their values as key value pairs
+ """
valobj: dict[str, dict[str, Union[int, str, bool, tuple, None]]] = {}
for s in config.sections():
valobj[s] = {}
diff --git a/src/linkahead/connection/connection.py b/src/linkahead/connection/connection.py
index a9f07567..74dd2317 100644
--- a/src/linkahead/connection/connection.py
+++ b/src/linkahead/connection/connection.py
@@ -422,8 +422,10 @@ def configure_connection(**kwargs):
- "keyring" Uses the `keyring` library.
- "auth_token" Uses only a given auth_token.
- timeout : int
+ timeout : int, tuple, or None
A connection timeout in seconds. (Default: 210)
+ If a tuple is given, they are used as connect and read timeouts
+ respectively, timeout None disables the timeout.
ssl_insecure : bool
Whether SSL certificate warnings should be ignored. Only use this for
--
GitLab
From df8f9231f490e62ee55c36c369184cbbfc09ca27 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Henrik=20tom=20W=C3=B6rden?= <h.tomwoerden@indiscale.com>
Date: Fri, 3 Jan 2025 14:58:08 +0100
Subject: [PATCH 53/62] ENH: add new member function filter to Container class
---
CHANGELOG.md | 1 +
src/linkahead/common/models.py | 40 +++++++++++++++++++++++++++++-----
2 files changed, 35 insertions(+), 6 deletions(-)
diff --git a/CHANGELOG.md b/CHANGELOG.md
index e1fd615a..8fe5c3ac 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -10,6 +10,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
### Added ###
* New setup extra `test` which installs the dependencies for testing.
+* The Container class has a new member function `filter` which is based o `_filter_entity_list`.
### Changed ###
diff --git a/src/linkahead/common/models.py b/src/linkahead/common/models.py
index 1caa6a4d..031d8058 100644
--- a/src/linkahead/common/models.py
+++ b/src/linkahead/common/models.py
@@ -2575,8 +2575,6 @@ class PropertyList(list):
Params
------
- listobject : Iterable(Property)
- List to be filtered
prop : Property
Property to match name and ID with. Cannot be set
simultaneously with ID or name.
@@ -3093,12 +3091,12 @@ def _basic_sync(e_local, e_remote):
if e_local.role is None:
e_local.role = e_remote.role
elif e_remote.role is not None and not e_local.role.lower() == e_remote.role.lower():
- raise ValueError("The resulting entity had a different role ({0}) "
- "than the local one ({1}). This probably means, that "
+ raise ValueError(f"The resulting entity had a different role ({e_remote.role}) "
+ f"than the local one ({e_local.role}). This probably means, that "
"the entity was intialized with a wrong class "
"by this client or it has changed in the past and "
- "this client did't know about it yet.".format(
- e_remote.role, e_local.role))
+ "this client did't know about it yet.\nThis is the local version of the"
+ f" Entity:\n{e_local}\nThis is the remote one:\n{e_remote}")
e_local.id = e_remote.id
e_local.name = e_remote.name
@@ -3730,6 +3728,36 @@ class Container(list):
return sync_dict
+ def filter(self, entity: Optional[Entity] = None,
+ pid: Union[None, str, int] = None,
+ name: Optional[str] = None,
+ conjunction: bool = False) -> list:
+ """
+ Return all Entities from this Container that match the selection criteria.
+
+ Please refer to the documentation of _filter_entity_list for a detailed
+ description of behaviour.
+
+ Params
+ ------
+ entity : Entity
+ Entity to match name and ID with
+ pid : str, int
+ Parent ID to match
+ name : str
+ Parent name to match
+ simultaneously with ID or name.
+ conjunction : bool, defaults to False
+ Set to return only entities that match both id and name
+ if both are given.
+
+ Returns
+ -------
+ matches : list
+ List containing all matching Entities
+ """
+ return _filter_entity_list(self, pid=pid, name=name, entity=entity,
+ conjunction=conjunction)
@staticmethod
def _find_dependencies_in_container(container: Container):
"""Find elements in a container that are a dependency of another element of the same.
--
GitLab
From 1f2c002304e7b9e4b053fb6ee8321972faad0532 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Henrik=20tom=20W=C3=B6rden?= <h.tomwoerden@indiscale.com>
Date: Fri, 3 Jan 2025 15:03:53 +0100
Subject: [PATCH 54/62] TST: add test for filter function of Container
---
unittests/test_container.py | 9 +++++++++
1 file changed, 9 insertions(+)
diff --git a/unittests/test_container.py b/unittests/test_container.py
index c3a60140..4ef85910 100644
--- a/unittests/test_container.py
+++ b/unittests/test_container.py
@@ -199,3 +199,12 @@ def test_container_slicing():
with pytest.raises(TypeError):
cont[[0, 2, 3]]
+
+def test_container_filter():
+ # this is a very rudimentary test since filter is based on _filter_entity_list which is tested
+ # separately
+ cont = db.Container()
+ cont.extend([db.Record(name=f"TestRec{ii+1}") for ii in range(5)])
+ recs = cont.filter(name="TestRec2")
+ assert len(recs)==1
+ recs[0].name =="TestRec2"
--
GitLab
From cea96eee220e16258aea5320e20d9dfa7cd04974 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Henrik=20tom=20W=C3=B6rden?= <h.tomwoerden@indiscale.com>
Date: Fri, 3 Jan 2025 15:11:01 +0100
Subject: [PATCH 55/62] DOC: document new filter function
---
src/doc/tutorials/complex_data_models.rst | 18 +++++++++++++++++-
1 file changed, 17 insertions(+), 1 deletion(-)
diff --git a/src/doc/tutorials/complex_data_models.rst b/src/doc/tutorials/complex_data_models.rst
index 168cf3b9..6c9520c9 100644
--- a/src/doc/tutorials/complex_data_models.rst
+++ b/src/doc/tutorials/complex_data_models.rst
@@ -78,7 +78,7 @@ Examples
Finding parents and properties
---------
+------------------------------
To find a specific parent or property of an Entity, its
ParentList or PropertyList can be filtered using names, ids, or
entities. A short example:
@@ -126,3 +126,19 @@ entities. A short example:
# Result: [p2_1]
The filter function of ParentList works analogously.
+
+Finding entities in a Container
+-------------------------------
+In the same way as described above, Container can be filtered.
+A short example:
+
+.. code-block:: python3
+
+ import linkahead as db
+
+ # Setup a record with six properties
+ p1 = db.Property(id=101, name="Property 1")
+ p2 = db.Property(name="Property 2")
+ c = db.Container().extend([p1,p2])
+ c.filter(name="Property 1")
+ # Result: [p1]
--
GitLab
From dbeef18c62e26bd61a087fc3e478731444ccce88 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Henrik=20tom=20W=C3=B6rden?= <h.tomwoerden@indiscale.com>
Date: Fri, 10 Jan 2025 09:24:51 +0100
Subject: [PATCH 56/62] MAINT: rename filter to filter_by_identity
---
CHANGELOG.md | 1 +
src/doc/tutorials/complex_data_models.rst | 14 +--
src/linkahead/apiutils.py | 6 +-
src/linkahead/common/models.py | 22 ++--
unittests/test_entity.py | 126 +++++++++++-----------
5 files changed, 89 insertions(+), 80 deletions(-)
diff --git a/CHANGELOG.md b/CHANGELOG.md
index e1fd615a..90fd8f11 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -12,6 +12,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
* New setup extra `test` which installs the dependencies for testing.
### Changed ###
+* Renamed the `filter` function of Container, ParentList and PropertyList to `filter_by_identity`.
### Deprecated ###
diff --git a/src/doc/tutorials/complex_data_models.rst b/src/doc/tutorials/complex_data_models.rst
index 168cf3b9..873c8bd5 100644
--- a/src/doc/tutorials/complex_data_models.rst
+++ b/src/doc/tutorials/complex_data_models.rst
@@ -100,29 +100,29 @@ entities. A short example:
properties = r.properties
# As r only has one property with id 101, this returns a list containing only p1_1
- properties.filter(pid=101)
+ properties.filter_by_identity(pid=101)
# Result: [p1_1]
# Filtering with name="Property 1" returns both p1_1 and p1_2, as they share their name
- properties.filter(name="Property 1")
+ properties.filter_by_identity(name="Property 1")
# Result: [p1_1, p1_2]
# If both name and pid are given, matching is based only on pid for all entities that have an id
- properties.filter(pid="102", name="Other Property")
+ properties.filter_by_identity(pid="102", name="Other Property")
# Result: [p2_1, p2_2, p2_3]
- # However, filtering with name="Property 1" and id=101 returns both p1_1 and p1_2, because
+ # However, filter_by_identity with name="Property 1" and id=101 returns both p1_1 and p1_2, because
# p1_2 does not have an id and matches the name
- properties.filter(pid="101", name="Property 1")
+ properties.filter_by_identity(pid="101", name="Property 1")
# Result: [p1_1, p1_2]
# We can also filter using an entity, in which case the name and id of the entity are used:
- properties.filter(pid="102", name="Property 2") == properties.filter(p2_1)
+ properties.filter_by_identity(pid="102", name="Property 2") == properties.filter_by_identity(p2_1)
# Result: True
# If we only need properties that match both id and name, we can set the parameter
# conjunction to True:
- properties.filter(pid="102", name="Property 2", conjunction=True)
+ properties.filter_by_identity(pid="102", name="Property 2", conjunction=True)
# Result: [p2_1]
The filter function of ParentList works analogously.
diff --git a/src/linkahead/apiutils.py b/src/linkahead/apiutils.py
index 1aa127d3..b2a612fa 100644
--- a/src/linkahead/apiutils.py
+++ b/src/linkahead/apiutils.py
@@ -386,7 +386,7 @@ def compare_entities(entity0: Optional[Entity] = None,
for prop in entity0.properties:
# ToDo: Would making id default break anything?
key = prop.name if prop.name is not None else prop.id
- matching = entity1.properties.filter(prop)
+ matching = entity1.properties.filter_by_identity(prop)
if len(matching) == 0:
# entity1 has prop, entity0 does not
diff[0]["properties"][key] = {}
@@ -440,7 +440,7 @@ def compare_entities(entity0: Optional[Entity] = None,
for prop in entity1.properties:
key = prop.name if prop.name is not None else prop.id
# check how often the property appears in entity0
- num_prop_in_ent0 = len(entity0.properties.filter(prop))
+ num_prop_in_ent0 = len(entity0.properties.filter_by_identity(prop))
if num_prop_in_ent0 == 0:
# property is only present in entity0 - add to diff
diff[1]["properties"][key] = {}
@@ -455,7 +455,7 @@ def compare_entities(entity0: Optional[Entity] = None,
(1, entity1.parents, entity0)]:
for parent in parents:
key = parent.name if parent.name is not None else parent.id
- matching = other_entity.parents.filter(parent)
+ matching = other_entity.parents.filter_by_identity(parent)
if len(matching) == 0:
diff[index]["parents"].append(key)
continue
diff --git a/src/linkahead/common/models.py b/src/linkahead/common/models.py
index 1caa6a4d..526a6f43 100644
--- a/src/linkahead/common/models.py
+++ b/src/linkahead/common/models.py
@@ -2562,7 +2562,11 @@ class PropertyList(list):
return xml2str(xml)
- def filter(self, prop: Optional[Property] = None,
+ def filter(self, *args, **kwargs):
+ warnings.warn(DeprecationWarning("This function was renamed to filter_by_identity."))
+ return self.filter(*args, **kwargs)
+
+ def filter_by_identity(self, prop: Optional[Property] = None,
pid: Union[None, str, int] = None,
name: Optional[str] = None,
conjunction: bool = False) -> list:
@@ -2570,7 +2574,7 @@ class PropertyList(list):
Return all Properties from the given PropertyList that match the
selection criteria.
- Please refer to the documentation of _filter_entity_list for a detailed
+ Please refer to the documentation of _filter_entity_list_by_identity for a detailed
description of behaviour.
Params
@@ -2593,7 +2597,7 @@ class PropertyList(list):
matches : list
List containing all matching Properties
"""
- return _filter_entity_list(self, pid=pid, name=name, entity=prop,
+ return _filter_entity_list_by_identity(self, pid=pid, name=name, entity=prop,
conjunction=conjunction)
def _get_entity_by_cuid(self, cuid: str):
@@ -2731,7 +2735,11 @@ class ParentList(list):
return xml2str(xml)
- def filter(self, parent: Optional[Parent] = None,
+ def filter(self, *args, **kwargs):
+ warnings.warn(DeprecationWarning("This function was renamed to filter_by_identity."))
+ return self.filter(*args, **kwargs)
+
+ def filter_by_identity(self, parent: Optional[Parent] = None,
pid: Union[None, str, int] = None,
name: Optional[str] = None,
conjunction: bool = False) -> list:
@@ -2739,7 +2747,7 @@ class ParentList(list):
Return all Parents from the given ParentList that match the selection
criteria.
- Please refer to the documentation of _filter_entity_list for a detailed
+ Please refer to the documentation of _filter_entity_list_by_identity for a detailed
description of behaviour.
Params
@@ -2762,7 +2770,7 @@ class ParentList(list):
matches : list
List containing all matching Parents
"""
- return _filter_entity_list(self, pid=pid, name=name, entity=parent,
+ return _filter_entity_list_by_identity(self, pid=pid, name=name, entity=parent,
conjunction=conjunction)
def remove(self, parent: Union[Entity, int, str]):
@@ -5537,7 +5545,7 @@ def delete(ids: Union[list[int], range], raise_exception_on_error: bool = True):
return c.delete(raise_exception_on_error=raise_exception_on_error)
-def _filter_entity_list(listobject: list[Entity],
+def _filter_entity_list_by_identity(listobject: list[Entity],
entity: Optional[Entity] = None,
pid: Union[None, str, int] = None,
name: Optional[str] = None,
diff --git a/unittests/test_entity.py b/unittests/test_entity.py
index 2127ce02..e946fd40 100644
--- a/unittests/test_entity.py
+++ b/unittests/test_entity.py
@@ -161,7 +161,7 @@ def test_property_list():
pl.append(p3)
-def test_filter():
+def test_filter_by_identity():
rt1 = RecordType(id=100)
rt2 = RecordType(id=101, name="RT")
rt3 = RecordType(name="")
@@ -184,7 +184,7 @@ def test_filter():
for coll in [entity.properties, entity.parents]:
for ent in test_ents:
assert ent not in coll
- assert ent not in coll.filter(ent)
+ assert ent not in coll.filter_by_identity(ent)
# Checks with each type
t, t_props, t_pars = entity, entity.properties, entity.parents
@@ -194,23 +194,23 @@ def test_filter():
tp1 = t.properties[-1]
t.add_property(p3)
tp3 = t.properties[-1]
- assert len(t_props.filter(pid=100)) == 1
- assert tp1 in t_props.filter(pid=100)
- assert len(t_props.filter(pid="100")) == 1
- assert tp1 in t_props.filter(pid="100")
- assert len(t_props.filter(pid=101, name="RT")) == 1
- assert tp3 in t_props.filter(pid=101, name="RT")
+ assert len(t_props.filter_by_identity(pid=100)) == 1
+ assert tp1 in t_props.filter_by_identity(pid=100)
+ assert len(t_props.filter_by_identity(pid="100")) == 1
+ assert tp1 in t_props.filter_by_identity(pid="100")
+ assert len(t_props.filter_by_identity(pid=101, name="RT")) == 1
+ assert tp3 in t_props.filter_by_identity(pid=101, name="RT")
for entity in [rt1, p2, r1, r2]:
- assert entity not in t_props.filter(pid=100)
- assert tp1 in t_props.filter(entity)
+ assert entity not in t_props.filter_by_identity(pid=100)
+ assert tp1 in t_props.filter_by_identity(entity)
# Check that direct addition (not wrapped) works
t_props.append(p2)
tp2 = t_props[-1]
- assert tp2 in t_props.filter(pid=100)
- assert tp2 not in t_props.filter(pid=101, name="RT")
+ assert tp2 in t_props.filter_by_identity(pid=100)
+ assert tp2 not in t_props.filter_by_identity(pid=101, name="RT")
for entity in [rt1, r1, r2]:
- assert entity not in t_props.filter(pid=100)
- assert tp2 in t_props.filter(entity)
+ assert entity not in t_props.filter_by_identity(pid=100)
+ assert tp2 in t_props.filter_by_identity(entity)
# Parents
# Filtering with both name and id
@@ -218,67 +218,67 @@ def test_filter():
tr3 = t.parents[-1]
t.add_parent(r5)
tr5 = t.parents[-1]
- assert tr3 in t_pars.filter(pid=101)
- assert tr5 not in t_pars.filter(pid=101)
- assert tr3 not in t_pars.filter(name="R")
- assert tr5 in t_pars.filter(name="R")
- assert tr3 in t_pars.filter(pid=101, name="R")
- assert tr5 not in t_pars.filter(pid=101, name="R")
- assert tr3 not in t_pars.filter(pid=104, name="RT")
- assert tr5 in t_pars.filter(pid=104, name="RT")
- assert tr3 not in t_pars.filter(pid=105, name="T")
- assert tr5 not in t_pars.filter(pid=105, name="T")
+ assert tr3 in t_pars.filter_by_identity(pid=101)
+ assert tr5 not in t_pars.filter_by_identity(pid=101)
+ assert tr3 not in t_pars.filter_by_identity(name="R")
+ assert tr5 in t_pars.filter_by_identity(name="R")
+ assert tr3 in t_pars.filter_by_identity(pid=101, name="R")
+ assert tr5 not in t_pars.filter_by_identity(pid=101, name="R")
+ assert tr3 not in t_pars.filter_by_identity(pid=104, name="RT")
+ assert tr5 in t_pars.filter_by_identity(pid=104, name="RT")
+ assert tr3 not in t_pars.filter_by_identity(pid=105, name="T")
+ assert tr5 not in t_pars.filter_by_identity(pid=105, name="T")
# Works also without id / name and with duplicate parents
for ent in test_ents:
t.add_parent(ent)
for ent in t_pars:
- assert ent in t_pars.filter(ent)
+ assert ent in t_pars.filter_by_identity(ent)
# Grid-Based
r7 = Record()
r7.add_property(Property()).add_property(name="A").add_property(name="B")
r7.add_property(id=27).add_property(id=27, name="A").add_property(id=27, name="B")
r7.add_property(id=43).add_property(id=43, name="A").add_property(id=43, name="B")
- assert len(r7.properties.filter(pid=27)) == 3
- assert len(r7.properties.filter(pid=43)) == 3
- assert len(r7.properties.filter(pid=43, conjunction=True)) == 3
- assert len(r7.properties.filter(name="A")) == 3
- assert len(r7.properties.filter(name="B")) == 3
- assert len(r7.properties.filter(name="B", conjunction=True)) == 3
- assert len(r7.properties.filter(pid=1, name="A")) == 1
- assert len(r7.properties.filter(pid=1, name="A", conjunction=True)) == 0
- assert len(r7.properties.filter(pid=27, name="B")) == 4
- assert len(r7.properties.filter(pid=27, name="B", conjunction=True)) == 1
- assert len(r7.properties.filter(pid=27, name="C")) == 3
- assert len(r7.properties.filter(pid=27, name="C", conjunction=True)) == 0
+ assert len(r7.properties.filter_by_identity(pid=27)) == 3
+ assert len(r7.properties.filter_by_identity(pid=43)) == 3
+ assert len(r7.properties.filter_by_identity(pid=43, conjunction=True)) == 3
+ assert len(r7.properties.filter_by_identity(name="A")) == 3
+ assert len(r7.properties.filter_by_identity(name="B")) == 3
+ assert len(r7.properties.filter_by_identity(name="B", conjunction=True)) == 3
+ assert len(r7.properties.filter_by_identity(pid=1, name="A")) == 1
+ assert len(r7.properties.filter_by_identity(pid=1, name="A", conjunction=True)) == 0
+ assert len(r7.properties.filter_by_identity(pid=27, name="B")) == 4
+ assert len(r7.properties.filter_by_identity(pid=27, name="B", conjunction=True)) == 1
+ assert len(r7.properties.filter_by_identity(pid=27, name="C")) == 3
+ assert len(r7.properties.filter_by_identity(pid=27, name="C", conjunction=True)) == 0
# Entity based filtering behaves the same
- assert (r7.properties.filter(pid=27) ==
- r7.properties.filter(Property(id=27)))
- assert (r7.properties.filter(pid=43, conjunction=True) ==
- r7.properties.filter(Property(id=43), conjunction=True))
- assert (r7.properties.filter(name="A") ==
- r7.properties.filter(Property(name="A")))
- assert (r7.properties.filter(name="B") ==
- r7.properties.filter(Property(name="B")))
- assert (r7.properties.filter(name="B", conjunction=True) ==
- r7.properties.filter(Property(name="B"), conjunction=True))
- assert (r7.properties.filter(pid=1, name="A") ==
- r7.properties.filter(Property(id=1, name="A")))
- assert (r7.properties.filter(pid=1, name="A", conjunction=True) ==
- r7.properties.filter(Property(id=1, name="A"), conjunction=True))
- assert (r7.properties.filter(pid=27, name="B") ==
- r7.properties.filter(Property(id=27, name="B")))
- assert (r7.properties.filter(pid=27, name="B", conjunction=True) ==
- r7.properties.filter(Property(id=27, name="B"), conjunction=True))
- assert (r7.properties.filter(pid=27, name="C") ==
- r7.properties.filter(Property(id=27, name="C")))
- assert (r7.properties.filter(pid=27, name="C", conjunction=True) ==
- r7.properties.filter(Property(id=27, name="C"), conjunction=True))
+ assert (r7.properties.filter_by_identity(pid=27) ==
+ r7.properties.filter_by_identity(Property(id=27)))
+ assert (r7.properties.filter_by_identity(pid=43, conjunction=True) ==
+ r7.properties.filter_by_identity(Property(id=43), conjunction=True))
+ assert (r7.properties.filter_by_identity(name="A") ==
+ r7.properties.filter_by_identity(Property(name="A")))
+ assert (r7.properties.filter_by_identity(name="B") ==
+ r7.properties.filter_by_identity(Property(name="B")))
+ assert (r7.properties.filter_by_identity(name="B", conjunction=True) ==
+ r7.properties.filter_by_identity(Property(name="B"), conjunction=True))
+ assert (r7.properties.filter_by_identity(pid=1, name="A") ==
+ r7.properties.filter_by_identity(Property(id=1, name="A")))
+ assert (r7.properties.filter_by_identity(pid=1, name="A", conjunction=True) ==
+ r7.properties.filter_by_identity(Property(id=1, name="A"), conjunction=True))
+ assert (r7.properties.filter_by_identity(pid=27, name="B") ==
+ r7.properties.filter_by_identity(Property(id=27, name="B")))
+ assert (r7.properties.filter_by_identity(pid=27, name="B", conjunction=True) ==
+ r7.properties.filter_by_identity(Property(id=27, name="B"), conjunction=True))
+ assert (r7.properties.filter_by_identity(pid=27, name="C") ==
+ r7.properties.filter_by_identity(Property(id=27, name="C")))
+ assert (r7.properties.filter_by_identity(pid=27, name="C", conjunction=True) ==
+ r7.properties.filter_by_identity(Property(id=27, name="C"), conjunction=True))
# Name only matching and name overwrite
r8 = Record().add_property(name="A").add_property(name="B").add_property(name="B")
r8.add_property(Property(name="A"), name="B")
r8.add_property(Property(name="A", id=12), name="C")
- assert len(r8.properties.filter(name="A")) == 1
- assert len(r8.properties.filter(name="B")) == 3
- assert len(r8.properties.filter(name="C")) == 1
- assert len(r8.properties.filter(pid=12)) == 1
+ assert len(r8.properties.filter_by_identity(name="A")) == 1
+ assert len(r8.properties.filter_by_identity(name="B")) == 3
+ assert len(r8.properties.filter_by_identity(name="C")) == 1
+ assert len(r8.properties.filter_by_identity(pid=12)) == 1
--
GitLab
From e0554c5e740cdd9c9d29a1ca4e689d1630fe0ec3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Henrik=20tom=20W=C3=B6rden?= <h.tomwoerden@indiscale.com>
Date: Fri, 10 Jan 2025 09:30:19 +0100
Subject: [PATCH 57/62] MAINT: rename the filter function of Container
---
src/doc/tutorials/complex_data_models.rst | 2 +-
src/linkahead/common/models.py | 10 +++++-----
unittests/test_container.py | 5 +++--
3 files changed, 9 insertions(+), 8 deletions(-)
diff --git a/src/doc/tutorials/complex_data_models.rst b/src/doc/tutorials/complex_data_models.rst
index d418298a..52757c32 100644
--- a/src/doc/tutorials/complex_data_models.rst
+++ b/src/doc/tutorials/complex_data_models.rst
@@ -140,5 +140,5 @@ A short example:
p1 = db.Property(id=101, name="Property 1")
p2 = db.Property(name="Property 2")
c = db.Container().extend([p1,p2])
- c.filter(name="Property 1")
+ c.filter_by_identity(name="Property 1")
# Result: [p1]
diff --git a/src/linkahead/common/models.py b/src/linkahead/common/models.py
index 0fc669ce..c83354f8 100644
--- a/src/linkahead/common/models.py
+++ b/src/linkahead/common/models.py
@@ -3736,21 +3736,21 @@ class Container(list):
return sync_dict
- def filter(self, entity: Optional[Entity] = None,
- pid: Union[None, str, int] = None,
+ def filter_by_identity(self, entity: Optional[Entity] = None,
+ entity_id: Union[None, str, int] = None,
name: Optional[str] = None,
conjunction: bool = False) -> list:
"""
Return all Entities from this Container that match the selection criteria.
- Please refer to the documentation of _filter_entity_list for a detailed
+ Please refer to the documentation of _filter_entity_list_by_identity for a detailed
description of behaviour.
Params
------
entity : Entity
Entity to match name and ID with
- pid : str, int
+ entity_id : str, int
Parent ID to match
name : str
Parent name to match
@@ -3764,7 +3764,7 @@ class Container(list):
matches : list
List containing all matching Entities
"""
- return _filter_entity_list(self, pid=pid, name=name, entity=entity,
+ return _filter_entity_list_by_identity(self, pid=entity_id, name=name, entity=entity,
conjunction=conjunction)
@staticmethod
def _find_dependencies_in_container(container: Container):
diff --git a/unittests/test_container.py b/unittests/test_container.py
index 4ef85910..70498fd9 100644
--- a/unittests/test_container.py
+++ b/unittests/test_container.py
@@ -201,10 +201,11 @@ def test_container_slicing():
cont[[0, 2, 3]]
def test_container_filter():
- # this is a very rudimentary test since filter is based on _filter_entity_list which is tested
+ # this is a very rudimentary test since filter_by_identity is based on
+ # _filter_entity_list_by_identity which is tested
# separately
cont = db.Container()
cont.extend([db.Record(name=f"TestRec{ii+1}") for ii in range(5)])
- recs = cont.filter(name="TestRec2")
+ recs = cont.filter_by_identity(name="TestRec2")
assert len(recs)==1
recs[0].name =="TestRec2"
--
GitLab
From 71f51dd30ad8cfabaf78db805ecc49c2978b40af Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Henrik=20tom=20W=C3=B6rden?= <h.tomwoerden@indiscale.com>
Date: Fri, 10 Jan 2025 12:46:21 +0100
Subject: [PATCH 58/62] TST: add unittest for deprecation warning
---
src/linkahead/common/models.py | 4 ++--
unittests/test_entity.py | 14 ++++++++++++++
2 files changed, 16 insertions(+), 2 deletions(-)
diff --git a/src/linkahead/common/models.py b/src/linkahead/common/models.py
index c83354f8..18508d5a 100644
--- a/src/linkahead/common/models.py
+++ b/src/linkahead/common/models.py
@@ -2564,7 +2564,7 @@ class PropertyList(list):
def filter(self, *args, **kwargs):
warnings.warn(DeprecationWarning("This function was renamed to filter_by_identity."))
- return self.filter(*args, **kwargs)
+ return self.filter_by_identity(*args, **kwargs)
def filter_by_identity(self, prop: Optional[Property] = None,
pid: Union[None, str, int] = None,
@@ -2735,7 +2735,7 @@ class ParentList(list):
def filter(self, *args, **kwargs):
warnings.warn(DeprecationWarning("This function was renamed to filter_by_identity."))
- return self.filter(*args, **kwargs)
+ return self.filter_by_identity(*args, **kwargs)
def filter_by_identity(self, parent: Optional[Parent] = None,
pid: Union[None, str, int] = None,
diff --git a/unittests/test_entity.py b/unittests/test_entity.py
index e946fd40..da057783 100644
--- a/unittests/test_entity.py
+++ b/unittests/test_entity.py
@@ -29,6 +29,7 @@ import unittest
import linkahead
from linkahead import (INTEGER, Entity, Parent, Property, Record, RecordType,
configure_connection)
+import warnings
from linkahead.common.models import SPECIAL_ATTRIBUTES
from linkahead.connection.mockup import MockUpServerConnection
from lxml import etree
@@ -282,3 +283,16 @@ def test_filter_by_identity():
assert len(r8.properties.filter_by_identity(name="B")) == 3
assert len(r8.properties.filter_by_identity(name="C")) == 1
assert len(r8.properties.filter_by_identity(pid=12)) == 1
+
+
+ with warnings.catch_warnings(record=True) as w:
+ # Cause all warnings to always be triggered.
+ warnings.simplefilter("always")
+
+ r7.properties.filter(pid=34)
+ assert issubclass(w[-1].category, DeprecationWarning)
+ assert "This function was renamed" in str(w[-1].message)
+
+ t.parents.filter(pid=234)
+ assert issubclass(w[-1].category, DeprecationWarning)
+ assert "This function was renamed" in str(w[-1].message)
--
GitLab
From 47a81833846f2b687a72016e3e97bd75d6056c72 Mon Sep 17 00:00:00 2001
From: Daniel <d.hornung@indiscale.com>
Date: Fri, 10 Jan 2025 14:57:46 +0100
Subject: [PATCH 59/62] STY: Only a few style fixes.
---
src/linkahead/common/models.py | 4 +++-
1 file changed, 3 insertions(+), 1 deletion(-)
diff --git a/src/linkahead/common/models.py b/src/linkahead/common/models.py
index c556629a..8bd8eacd 100644
--- a/src/linkahead/common/models.py
+++ b/src/linkahead/common/models.py
@@ -1351,7 +1351,8 @@ class Entity:
else:
dt_str = xml2str(self.datatype.to_xml(visited_entities=visited_entities.copy()))
# Todo: Use for pretty-printing with calls from _repr_ only?
- # dt_str = dt_str.replace('<', 'á¸').replace('>', 'á³').replace(' ', 'â €').replace('"', '\'').replace('\n', '')
+ # dt_str = dt_str.replace('<', 'á¸').replace('>', 'á³').replace(' ', 'â €').replace(
+ # '"', '\'').replace('\n', '')
xml.set("datatype", dt_str)
else:
xml.set("datatype", str(self.datatype))
@@ -3767,6 +3768,7 @@ class Container(list):
"""
return _filter_entity_list(self, pid=pid, name=name, entity=entity,
conjunction=conjunction)
+
@staticmethod
def _find_dependencies_in_container(container: Container):
"""Find elements in a container that are a dependency of another element of the same.
--
GitLab
From 5b5094b924aaaf0947a782abe359676dfc186414 Mon Sep 17 00:00:00 2001
From: Daniel <d.hornung@indiscale.com>
Date: Fri, 10 Jan 2025 15:00:00 +0100
Subject: [PATCH 60/62] STY: Only a few style fixes.
---
unittests/test_container.py | 8 +++++---
1 file changed, 5 insertions(+), 3 deletions(-)
diff --git a/unittests/test_container.py b/unittests/test_container.py
index 4ef85910..715a6eb8 100644
--- a/unittests/test_container.py
+++ b/unittests/test_container.py
@@ -70,7 +70,8 @@ def test_get_property_values():
)
assert len(table) == 2
house_row = table[0]
- assert house_row == (house.name, 40.2, "ft", window.id, None, None, None, 20.5, 20.5, "m", owner.name)
+ assert house_row == (house.name, 40.2, "ft", window.id, None, None, None, 20.5, 20.5, "m",
+ owner.name)
owner_row = table[1]
assert owner_row == (owner.name, None, None, None, None, None, None, None, None, None, None)
@@ -200,11 +201,12 @@ def test_container_slicing():
with pytest.raises(TypeError):
cont[[0, 2, 3]]
+
def test_container_filter():
# this is a very rudimentary test since filter is based on _filter_entity_list which is tested
# separately
cont = db.Container()
cont.extend([db.Record(name=f"TestRec{ii+1}") for ii in range(5)])
recs = cont.filter(name="TestRec2")
- assert len(recs)==1
- recs[0].name =="TestRec2"
+ assert len(recs) == 1
+ recs[0].name == "TestRec2"
--
GitLab
From c9d6b2700a310c8e01fa7de8054c480feb447055 Mon Sep 17 00:00:00 2001
From: Florian Spreckelsen <f.spreckelsen@indiscale.com>
Date: Mon, 13 Jan 2025 15:28:50 +0100
Subject: [PATCH 61/62] STY: autopep8'd
---
src/linkahead/common/models.py | 32 ++++++++++++++++----------------
unittests/test_container.py | 2 +-
unittests/test_entity.py | 1 -
3 files changed, 17 insertions(+), 18 deletions(-)
diff --git a/src/linkahead/common/models.py b/src/linkahead/common/models.py
index 7bd08a66..75b03b70 100644
--- a/src/linkahead/common/models.py
+++ b/src/linkahead/common/models.py
@@ -2577,9 +2577,9 @@ class PropertyList(list):
return self.filter_by_identity(*args, **kwargs)
def filter_by_identity(self, prop: Optional[Property] = None,
- pid: Union[None, str, int] = None,
- name: Optional[str] = None,
- conjunction: bool = False) -> list:
+ pid: Union[None, str, int] = None,
+ name: Optional[str] = None,
+ conjunction: bool = False) -> list:
"""
Return all Properties from the given PropertyList that match the
selection criteria.
@@ -2606,7 +2606,7 @@ class PropertyList(list):
List containing all matching Properties
"""
return _filter_entity_list_by_identity(self, pid=pid, name=name, entity=prop,
- conjunction=conjunction)
+ conjunction=conjunction)
def _get_entity_by_cuid(self, cuid: str):
'''
@@ -2748,9 +2748,9 @@ class ParentList(list):
return self.filter_by_identity(*args, **kwargs)
def filter_by_identity(self, parent: Optional[Parent] = None,
- pid: Union[None, str, int] = None,
- name: Optional[str] = None,
- conjunction: bool = False) -> list:
+ pid: Union[None, str, int] = None,
+ name: Optional[str] = None,
+ conjunction: bool = False) -> list:
"""
Return all Parents from the given ParentList that match the selection
criteria.
@@ -2779,7 +2779,7 @@ class ParentList(list):
List containing all matching Parents
"""
return _filter_entity_list_by_identity(self, pid=pid, name=name, entity=parent,
- conjunction=conjunction)
+ conjunction=conjunction)
def remove(self, parent: Union[Entity, int, str]):
"""
@@ -3747,9 +3747,9 @@ class Container(list):
return sync_dict
def filter_by_identity(self, entity: Optional[Entity] = None,
- entity_id: Union[None, str, int] = None,
- name: Optional[str] = None,
- conjunction: bool = False) -> list:
+ entity_id: Union[None, str, int] = None,
+ name: Optional[str] = None,
+ conjunction: bool = False) -> list:
"""
Return all Entities from this Container that match the selection criteria.
@@ -3775,7 +3775,7 @@ class Container(list):
List containing all matching Entities
"""
return _filter_entity_list_by_identity(self, pid=entity_id, name=name, entity=entity,
- conjunction=conjunction)
+ conjunction=conjunction)
@staticmethod
def _find_dependencies_in_container(container: Container):
@@ -5585,10 +5585,10 @@ def delete(ids: Union[list[int], range], raise_exception_on_error: bool = True):
def _filter_entity_list_by_identity(listobject: list[Entity],
- entity: Optional[Entity] = None,
- pid: Union[None, str, int] = None,
- name: Optional[str] = None,
- conjunction: bool = False) -> list:
+ entity: Optional[Entity] = None,
+ pid: Union[None, str, int] = None,
+ name: Optional[str] = None,
+ conjunction: bool = False) -> list:
"""
Returns a subset of entities from the list based on whether their id and
name matches the selection criterion.
diff --git a/unittests/test_container.py b/unittests/test_container.py
index 54e87674..9df40ffb 100644
--- a/unittests/test_container.py
+++ b/unittests/test_container.py
@@ -210,5 +210,5 @@ def test_container_filter():
cont.extend([db.Record(name=f"TestRec{ii+1}") for ii in range(5)])
recs = cont.filter_by_identity(name="TestRec2")
- assert len(recs)== 1
+ assert len(recs) == 1
recs[0].name == "TestRec2"
diff --git a/unittests/test_entity.py b/unittests/test_entity.py
index da057783..855e5a39 100644
--- a/unittests/test_entity.py
+++ b/unittests/test_entity.py
@@ -284,7 +284,6 @@ def test_filter_by_identity():
assert len(r8.properties.filter_by_identity(name="C")) == 1
assert len(r8.properties.filter_by_identity(pid=12)) == 1
-
with warnings.catch_warnings(record=True) as w:
# Cause all warnings to always be triggered.
warnings.simplefilter("always")
--
GitLab
From de82b057d6f07c5c87b04bfe7f6456ac65023aa2 Mon Sep 17 00:00:00 2001
From: Florian Spreckelsen <f.spreckelsen@indiscale.com>
Date: Tue, 14 Jan 2025 15:24:58 +0100
Subject: [PATCH 62/62] REL: Prepare release of 0.17.0
---
CHANGELOG.md | 16 +++++++++-------
CITATION.cff | 4 ++--
README.md | 2 +-
setup.py | 6 +++---
src/doc/conf.py | 4 ++--
5 files changed, 17 insertions(+), 15 deletions(-)
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 196d4b71..f56bc3ab 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -5,20 +5,26 @@ All notable changes to this project will be documented in this file.
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
-## [Unreleased] ##
+## [0.17.0] - 2025-01-14 ##
### Added ###
* New setup extra `test` which installs the dependencies for testing.
-* The Container class has a new member function `filter` which is based o `_filter_entity_list`.
+* The Container class has a new member function `filter_by_identity`
+ which is based on `_filter_entity_list`.
* The `Entity` properties `_cuid` and `_flags` are now available for read-only access
as `cuid` and `flags`, respectively.
### Changed ###
-* Renamed the `filter` function of Container, ParentList and PropertyList to `filter_by_identity`.
+
+* Renamed the `filter` function of Container, ParentList and
+ PropertyList to `filter_by_identity`.
### Deprecated ###
+* `ParentList.filter` and `PropertyList.filter` functions, use
+ `filter_by_identity` instead.
+
### Removed ###
* Support for Python 3.8
@@ -42,10 +48,6 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
* [#127](https://gitlab.com/linkahead/linkahead-pylib/-/issues/127)
pylinkahead.ini now supports None and tuples as values for the `timeout` keyword
-### Security ###
-
-### Documentation ###
-
## [0.16.0] - 2024-11-13 ##
### Added ###
diff --git a/CITATION.cff b/CITATION.cff
index 123289ca..bcecc2fd 100644
--- a/CITATION.cff
+++ b/CITATION.cff
@@ -20,6 +20,6 @@ authors:
given-names: Stefan
orcid: https://orcid.org/0000-0001-7214-8125
title: CaosDB - Pylib
-version: 0.16.0
+version: 0.17.0
doi: 10.3390/data4020083
-date-released: 2024-11-13
+date-released: 2025-01-14
diff --git a/README.md b/README.md
index 193cb8f0..d630e879 100644
--- a/README.md
+++ b/README.md
@@ -47,7 +47,7 @@ However, you can also create an issue for it.
* Copyright (C) 2018 Research Group Biomedical Physics, Max Planck Institute
for Dynamics and Self-Organization Göttingen.
-* Copyright (C) 2020-2022 Indiscale GmbH <info@indiscale.com>
+* Copyright (C) 2020-2025 Indiscale GmbH <info@indiscale.com>
All files in this repository are licensed under a [GNU Affero General Public
License](LICENCE.md) (version 3 or later).
diff --git a/setup.py b/setup.py
index 7a7bf9bd..75bcf0c7 100755
--- a/setup.py
+++ b/setup.py
@@ -46,10 +46,10 @@ from setuptools import find_packages, setup
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
########################################################################
-ISRELEASED = False
+ISRELEASED = True
MAJOR = 0
-MINOR = 16
-MICRO = 1
+MINOR = 17
+MICRO = 0
# Do not tag as pre-release until this commit
# https://github.com/pypa/packaging/pull/515
# has made it into a release. Probably we should wait for pypa/packaging>=21.4
diff --git a/src/doc/conf.py b/src/doc/conf.py
index 80d5e8a2..65600678 100644
--- a/src/doc/conf.py
+++ b/src/doc/conf.py
@@ -29,10 +29,10 @@ copyright = '2024, IndiScale GmbH'
author = 'Daniel Hornung'
# The short X.Y version
-version = '0.16.1'
+version = '0.17.0'
# The full version, including alpha/beta/rc tags
# release = '0.5.2-rc2'
-release = '0.16.1-dev'
+release = '0.17.0'
# -- General configuration ---------------------------------------------------
--
GitLab