diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 463b1d56507cb970e5ec15e6fa90ce6adf3e123b..db600343569930a436a593a8ab5d511a35bc7aca 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -61,16 +61,6 @@ mypy:
- make mypy
# run unit tests
-unittest_py3.8:
- tags: [ docker ]
- stage: test
- needs: [ ]
- image: python:3.8
- script: &python_test_script
- # Python docker has problems with tox and pip so use plain pytest here
- - touch ~/.pylinkahead.ini
- - pip install .[test]
- - python -m pytest unittests
# This needs to be changed once Python 3.9 isn't the standard Python in Debian
# anymore.
@@ -90,7 +80,11 @@ unittest_py3.10:
stage: test
needs: [ ]
image: python:3.10
- script: *python_test_script
+ script: &python_test_script
+ # Python docker has problems with tox and pip so use plain pytest here
+ - touch ~/.pylinkahead.ini
+ - pip install .[test]
+ - python -m pytest unittests
unittest_py3.11:
tags: [ docker ]
@@ -158,7 +152,7 @@ build-testenv:
pages_prepare: &pages_prepare
tags: [ cached-dind ]
stage: deploy
- needs: [ code_style, pylint, unittest_py3.8, unittest_py3.9, unittest_py3.10 ]
+ needs: [ code_style, pylint, unittest_py3.9, unittest_py3.10 ]
only:
refs:
- /^release-.*$/i
diff --git a/CHANGELOG.md b/CHANGELOG.md
index fe33166bc9bacfd57147e53243ed9489f26cd2bb..196d4b7189c39ae361eeb1bb7f7782be150c7d88 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -11,6 +11,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
* New setup extra `test` which installs the dependencies for testing.
* The Container class has a new member function `filter` which is based o `_filter_entity_list`.
+* The `Entity` properties `_cuid` and `_flags` are now available for read-only access
+ as `cuid` and `flags`, respectively.
### Changed ###
* Renamed the `filter` function of Container, ParentList and PropertyList to `filter_by_identity`.
@@ -19,6 +21,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
### Removed ###
+* Support for Python 3.8
+
### Fixed ###
* [#73](https://gitlab.com/linkahead/linkahead-pylib/-/issues/73)
@@ -34,6 +38,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
* [#87](https://gitlab.com/linkahead/linkahead-pylib/-/issues/87)
`XMLSyntaxError` messages when parsing (incomplete) responses in
case of certain connection timeouts.
+ The diff returned by compare_entities now uses id instead of name as key if either property does not have a name
+* [#127](https://gitlab.com/linkahead/linkahead-pylib/-/issues/127)
+ pylinkahead.ini now supports None and tuples as values for the `timeout` keyword
### Security ###
diff --git a/DEPENDENCIES.md b/DEPENDENCIES.md
index e2326b831a71751265c6c2d5a333ccc37145bfa5..e9bd54a1459df22afa307e256625d05e74bdc6a8 100644
--- a/DEPENDENCIES.md
+++ b/DEPENDENCIES.md
@@ -1,5 +1,5 @@
* caosdb-server >= 0.12.0
-* Python >= 3.8
+* Python >= 3.9
* pip >= 20.0.2
Any other dependencies are defined in the setup.py and are being installed via pip
diff --git a/README_SETUP.md b/README_SETUP.md
index 8fc93474f31697112fcc237ca890e0c0c4603ffa..f4c921382edb26776391590298faed06a5391396 100644
--- a/README_SETUP.md
+++ b/README_SETUP.md
@@ -4,7 +4,7 @@
### How to install ###
-First ensure that python with at least version 3.8 is installed. Should this not be
+First ensure that python with at least version 3.9 is installed. Should this not be
the case, you can use the [Installing python](#installing-python-) guide for your OS.
#### Generic installation ####
@@ -39,7 +39,7 @@ entries `install_requires` and `extras_require`.
#### Linux ####
-Make sure that Python (at least version 3.8) and pip is installed, using your system tools and
+Make sure that Python (at least version 3.9) and pip is installed, using your system tools and
documentation.
Then open a terminal and continue in the [Generic installation](#generic-installation) section.
diff --git a/setup.py b/setup.py
index daf36764ef043916bea44694bf0620505e6fcd9c..7a7bf9bdf4c01bd87681f232df44a3982381ac60 100755
--- a/setup.py
+++ b/setup.py
@@ -179,7 +179,7 @@ def setup_package():
"Topic :: Scientific/Engineering :: Information Analysis",
],
packages=find_packages('src'),
- python_requires='>=3.8',
+ python_requires='>=3.9',
package_dir={'': 'src'},
install_requires=['lxml>=4.6.3',
"requests[socks]>=2.26",
diff --git a/src/linkahead/common/models.py b/src/linkahead/common/models.py
index 18508d5ae75455795a2a7dceaa5ece56fedf4c60..7bd08a6638e931176497e08509b440133625e6c4 100644
--- a/src/linkahead/common/models.py
+++ b/src/linkahead/common/models.py
@@ -349,6 +349,15 @@ class Entity:
def pickup(self, new_pickup):
self.__pickup = new_pickup
+ @property # getter for _cuid
+ def cuid(self):
+ # Set if None?
+ return self._cuid
+
+ @property # getter for _flags
+ def flags(self):
+ return self._flags.copy() # for dict[str, str] shallow copy is enough
+
def grant(
self,
realm: Optional[str] = None,
@@ -1342,7 +1351,8 @@ class Entity:
else:
dt_str = xml2str(self.datatype.to_xml(visited_entities=visited_entities.copy()))
# Todo: Use for pretty-printing with calls from _repr_ only?
- # dt_str = dt_str.replace('<', 'á¸').replace('>', 'á³').replace(' ', 'â €').replace('"', '\'').replace('\n', '')
+ # dt_str = dt_str.replace('<', 'á¸').replace('>', 'á³').replace(' ', 'â €').replace(
+ # '"', '\'').replace('\n', '')
xml.set("datatype", dt_str)
else:
xml.set("datatype", str(self.datatype))
@@ -3766,6 +3776,7 @@ class Container(list):
"""
return _filter_entity_list_by_identity(self, pid=entity_id, name=name, entity=entity,
conjunction=conjunction)
+
@staticmethod
def _find_dependencies_in_container(container: Container):
"""Find elements in a container that are a dependency of another element of the same.
diff --git a/src/linkahead/configuration.py b/src/linkahead/configuration.py
index f57289d7dcb6d7ab062024dc697dbda557670d7a..5081c28af253d3da31926ab1c9449309cc171c4f 100644
--- a/src/linkahead/configuration.py
+++ b/src/linkahead/configuration.py
@@ -30,6 +30,15 @@ import yaml
try:
optional_jsonschema_validate: Optional[Callable] = None
from jsonschema import validate as optional_jsonschema_validate
+
+ # Adapted from https://github.com/python-jsonschema/jsonschema/issues/148
+ # Defines Validator to allow parsing of all iterables as array in jsonschema
+ # CustomValidator can be removed if/once jsonschema allows tuples for arrays
+ from collections.abc import Iterable
+ from jsonschema import validators
+ default = validators.validator_for(True) # Returns latest supported draft
+ t_c = (default.TYPE_CHECKER.redefine('array', lambda x, y: isinstance(y, Iterable)))
+ CustomValidator = validators.extend(default, type_checker=t_c)
except ImportError:
pass
@@ -72,14 +81,40 @@ def get_config() -> ConfigParser:
return _pycaosdbconf
-def config_to_yaml(config: ConfigParser) -> dict[str, dict[str, Union[int, str, bool]]]:
- valobj: dict[str, dict[str, Union[int, str, bool]]] = {}
+def config_to_yaml(config: ConfigParser) -> dict[str, dict[str, Union[int, str, bool, tuple, None]]]:
+ """
+ Generates and returns a dict with all config options and their values
+ defined in the config.
+ The values of the options 'debug', 'timeout', and 'ssl_insecure' are
+ parsed, all other values are saved as string.
+
+ Parameters
+ ----------
+ config : ConfigParser
+ The config to be converted to a dict
+
+ Returns
+ -------
+ valobj : dict
+ A dict with config options and their values as key value pairs
+ """
+ valobj: dict[str, dict[str, Union[int, str, bool, tuple, None]]] = {}
for s in config.sections():
valobj[s] = {}
for key, value in config[s].items():
# TODO: Can the type be inferred from the config object?
- if key in ["timeout", "debug"]:
+ if key in ["debug"]:
valobj[s][key] = int(value)
+ elif key in ["timeout"]:
+ value = "".join(value.split()) # Remove whitespace
+ if str(value).lower() in ["none", "null"]:
+ valobj[s][key] = None
+ elif value.startswith('(') and value.endswith(')'):
+ content = [None if str(s).lower() in ["none", "null"] else int(s)
+ for s in value[1:-1].split(',')]
+ valobj[s][key] = tuple(content)
+ else:
+ valobj[s][key] = int(value)
elif key in ["ssl_insecure"]:
valobj[s][key] = bool(value)
else:
@@ -88,11 +123,12 @@ def config_to_yaml(config: ConfigParser) -> dict[str, dict[str, Union[int, str,
return valobj
-def validate_yaml_schema(valobj: dict[str, dict[str, Union[int, str, bool]]]):
+def validate_yaml_schema(valobj: dict[str, dict[str, Union[int, str, bool, tuple, None]]]):
if optional_jsonschema_validate:
with open(os.path.join(os.path.dirname(__file__), "schema-pycaosdb-ini.yml")) as f:
schema = yaml.load(f, Loader=yaml.SafeLoader)
- optional_jsonschema_validate(instance=valobj, schema=schema["schema-pycaosdb-ini"])
+ optional_jsonschema_validate(instance=valobj, schema=schema["schema-pycaosdb-ini"],
+ cls=CustomValidator)
else:
warnings.warn("""
Warning: The validation could not be performed because `jsonschema` is not installed.
diff --git a/src/linkahead/connection/connection.py b/src/linkahead/connection/connection.py
index 4c40842a7eaf5bbf0e35c978e658d7b4494a58e7..74dd23177c548dd640c6dd1c03ce4069c366802b 100644
--- a/src/linkahead/connection/connection.py
+++ b/src/linkahead/connection/connection.py
@@ -39,7 +39,7 @@ from requests.adapters import HTTPAdapter
from requests.exceptions import ConnectionError as HTTPConnectionError
from urllib3.poolmanager import PoolManager
-from ..configuration import get_config
+from ..configuration import get_config, config_to_yaml
from ..exceptions import (ConfigurationError, HTTPClientError,
HTTPForbiddenError, HTTPResourceNotFoundError,
HTTPServerError, HTTPURITooLongError,
@@ -422,8 +422,10 @@ def configure_connection(**kwargs):
- "keyring" Uses the `keyring` library.
- "auth_token" Uses only a given auth_token.
- timeout : int
+ timeout : int, tuple, or None
A connection timeout in seconds. (Default: 210)
+ If a tuple is given, they are used as connect and read timeouts
+ respectively, timeout None disables the timeout.
ssl_insecure : bool
Whether SSL certificate warnings should be ignored. Only use this for
@@ -465,21 +467,29 @@ def configure_connection(**kwargs):
global_conf = {}
conf = get_config()
# Convert config to dict, with preserving types
- int_opts = ["timeout"]
+ int_opts = []
bool_opts = ["ssl_insecure"]
+ other_opts = ["timeout"]
if conf.has_section("Connection"):
global_conf = dict(conf.items("Connection"))
- # Integer options
+ # Integer options
for opt in int_opts:
if opt in global_conf:
global_conf[opt] = conf.getint("Connection", opt)
- # Boolean options
+ # Boolean options
for opt in bool_opts:
if opt in global_conf:
global_conf[opt] = conf.getboolean("Connection", opt)
+
+ # Other options, defer parsing to configuration.config_to_yaml:
+ connection_config = config_to_yaml(conf)["Connection"]
+ for opt in other_opts:
+ if opt in global_conf:
+ global_conf[opt] = connection_config[opt]
+
local_conf = _make_conf(_DEFAULT_CONF, global_conf, kwargs)
connection = _Connection.get_instance()
diff --git a/src/linkahead/schema-pycaosdb-ini.yml b/src/linkahead/schema-pycaosdb-ini.yml
index 89ce98570738fdd29dba81de25a2c022c1581467..ae46b905c62d2ab168229d92ff138937279c7aed 100644
--- a/src/linkahead/schema-pycaosdb-ini.yml
+++ b/src/linkahead/schema-pycaosdb-ini.yml
@@ -67,7 +67,13 @@ schema-pycaosdb-ini:
description: This option is used internally and for testing. Do not override.
examples: [_DefaultCaosDBServerConnection]
timeout:
- type: integer
+ oneOf:
+ - type: [integer, "null"]
+ - type: array
+ items:
+ type: [integer, "null"]
+ minItems: 2
+ maxItems: 2
allOf:
- if:
properties:
diff --git a/unittests/test_configs/pylinkahead-timeout1.ini b/unittests/test_configs/pylinkahead-timeout1.ini
new file mode 100644
index 0000000000000000000000000000000000000000..d9f894bfeba4f98ed30d96d8c29e057b5a1e643a
--- /dev/null
+++ b/unittests/test_configs/pylinkahead-timeout1.ini
@@ -0,0 +1,4 @@
+[Connection]
+url=https://localhost:10443/
+password_method = unauthenticated
+timeout = None
diff --git a/unittests/test_configs/pylinkahead-timeout2.ini b/unittests/test_configs/pylinkahead-timeout2.ini
new file mode 100644
index 0000000000000000000000000000000000000000..b3d3796f82148459efb8e19344fe11af9e7934ec
--- /dev/null
+++ b/unittests/test_configs/pylinkahead-timeout2.ini
@@ -0,0 +1,4 @@
+[Connection]
+url=https://localhost:10443/
+password_method = unauthenticated
+timeout = (1,20)
diff --git a/unittests/test_configuration.py b/unittests/test_configuration.py
index 95bc906c6c044c51548aa864326cc93f29a6042a..772e872c08e0a7c4aae3feffdb58244f6ad0c849 100644
--- a/unittests/test_configuration.py
+++ b/unittests/test_configuration.py
@@ -24,6 +24,7 @@
from os import environ, getcwd, remove
from os.path import expanduser, isfile, join
+from pathlib import Path
import linkahead as db
import pytest
@@ -66,3 +67,18 @@ def test_config_ini_via_envvar(temp_ini_files):
assert expanduser("~/.pylinkahead.ini") in db.configuration._read_config_files()
# test configuration file in cwd
assert join(getcwd(), "pylinkahead.ini") in db.configuration._read_config_files()
+
+
+def test_config_timeout_option():
+ expected_results = [None, (1, 20)]
+ # Iterate through timeout test configs
+ test_configs = Path(__file__).parent/'test_configs'
+ for test_config in test_configs.rglob('pylinkahead-timeout*.ini'):
+ # Test that test configs can be parsed
+ db.configure(str(test_config))
+ dct = db.configuration.config_to_yaml(db.get_config())
+ # Test that resulting dict has correct content for timeout
+ assert 'Connection' in dct
+ assert 'timeout' in dct['Connection']
+ assert dct['Connection']['timeout'] in expected_results
+ expected_results.remove(dct['Connection']['timeout'])
diff --git a/unittests/test_container.py b/unittests/test_container.py
index 70498fd9c99488bf2d9ce602a6871d8a8750ea7e..54e876747a0581dba0293ce5a94ee40ac74abfb7 100644
--- a/unittests/test_container.py
+++ b/unittests/test_container.py
@@ -70,7 +70,8 @@ def test_get_property_values():
)
assert len(table) == 2
house_row = table[0]
- assert house_row == (house.name, 40.2, "ft", window.id, None, None, None, 20.5, 20.5, "m", owner.name)
+ assert house_row == (house.name, 40.2, "ft", window.id, None, None, None, 20.5, 20.5, "m",
+ owner.name)
owner_row = table[1]
assert owner_row == (owner.name, None, None, None, None, None, None, None, None, None, None)
@@ -200,12 +201,14 @@ def test_container_slicing():
with pytest.raises(TypeError):
cont[[0, 2, 3]]
+
def test_container_filter():
# this is a very rudimentary test since filter_by_identity is based on
# _filter_entity_list_by_identity which is tested
# separately
cont = db.Container()
cont.extend([db.Record(name=f"TestRec{ii+1}") for ii in range(5)])
+
recs = cont.filter_by_identity(name="TestRec2")
- assert len(recs)==1
- recs[0].name =="TestRec2"
+ assert len(recs)== 1
+ recs[0].name == "TestRec2"