Merge branch 'f-enh-fit-93-pylinkahead-separate-timeouts' into 'dev'

Separate connect/read timeouts in pylinkahead.ini

See merge request !167

CHANGELOG.md

@@ -37,6 +37,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 * [#87](https://gitlab.com/linkahead/linkahead-pylib/-/issues/87)
   `XMLSyntaxError` messages when parsing (incomplete) responses in
   case of certain connection timeouts.
+  The diff returned by compare_entities now uses id instead of name as key if either property does not have a name
+* [#127](https://gitlab.com/linkahead/linkahead-pylib/-/issues/127)
+  pylinkahead.ini now supports None and tuples as values for the `timeout` keyword
 
 ### Security ###
 

src/linkahead/configuration.py

@@ -30,6 +30,15 @@ import yaml
 try:
     optional_jsonschema_validate: Optional[Callable] = None
     from jsonschema import validate as optional_jsonschema_validate
+
+    # Adapted from https://github.com/python-jsonschema/jsonschema/issues/148
+    # Defines Validator to allow parsing of all iterables as array in jsonschema
+    # CustomValidator can be removed if/once jsonschema allows tuples for arrays
+    from collections.abc import Iterable
+    from jsonschema import validators
+    default = validators.validator_for(True)   # Returns latest supported draft
+    t_c = (default.TYPE_CHECKER.redefine('array', lambda x, y: isinstance(y, Iterable)))
+    CustomValidator = validators.extend(default, type_checker=t_c)
 except ImportError:
     pass
 
@@ -72,14 +81,40 @@ def get_config() -> ConfigParser:
     return _pycaosdbconf
 
 
-def config_to_yaml(config: ConfigParser) -> dict[str, dict[str, Union[int, str, bool]]]:
-    valobj: dict[str, dict[str, Union[int, str, bool]]] = {}
+def config_to_yaml(config: ConfigParser) -> dict[str, dict[str, Union[int, str, bool, tuple, None]]]:
+    """
+    Generates and returns a dict with all config options and their values
+    defined in the config.
+    The values of the options 'debug', 'timeout', and 'ssl_insecure' are
+    parsed, all other values are saved as string.
+
+    Parameters
+    ----------
+    config : ConfigParser
+        The config to be converted to a dict
+
+    Returns
+    -------
+    valobj : dict
+        A dict with config options and their values as key value pairs
+    """
+    valobj: dict[str, dict[str, Union[int, str, bool, tuple, None]]] = {}
     for s in config.sections():
         valobj[s] = {}
         for key, value in config[s].items():
             # TODO: Can the type be inferred from the config object?
-            if key in ["timeout", "debug"]:
+            if key in ["debug"]:
                 valobj[s][key] = int(value)
+            elif key in ["timeout"]:
+                value = "".join(value.split())          # Remove whitespace
+                if str(value).lower() in ["none", "null"]:
+                    valobj[s][key] = None
+                elif value.startswith('(') and value.endswith(')'):
+                    content = [None if str(s).lower() in ["none", "null"] else int(s)
+                               for s in value[1:-1].split(',')]
+                    valobj[s][key] = tuple(content)
+                else:
+                    valobj[s][key] = int(value)
             elif key in ["ssl_insecure"]:
                 valobj[s][key] = bool(value)
             else:
@@ -88,11 +123,12 @@ def config_to_yaml(config: ConfigParser) -> dict[str, dict[str, Union[int, str,
     return valobj
 
 
-def validate_yaml_schema(valobj: dict[str, dict[str, Union[int, str, bool]]]):
+def validate_yaml_schema(valobj: dict[str, dict[str, Union[int, str, bool, tuple, None]]]):
     if optional_jsonschema_validate:
         with open(os.path.join(os.path.dirname(__file__), "schema-pycaosdb-ini.yml")) as f:
             schema = yaml.load(f, Loader=yaml.SafeLoader)
-        optional_jsonschema_validate(instance=valobj, schema=schema["schema-pycaosdb-ini"])
+        optional_jsonschema_validate(instance=valobj, schema=schema["schema-pycaosdb-ini"],
+                                     cls=CustomValidator)
     else:
         warnings.warn("""
             Warning: The validation could not be performed because `jsonschema` is not installed.

src/linkahead/connection/connection.py

@@ -39,7 +39,7 @@ from requests.adapters import HTTPAdapter
 from requests.exceptions import ConnectionError as HTTPConnectionError
 from urllib3.poolmanager import PoolManager
 
-from ..configuration import get_config
+from ..configuration import get_config, config_to_yaml
 from ..exceptions import (ConfigurationError, HTTPClientError,
                           HTTPForbiddenError, HTTPResourceNotFoundError,
                           HTTPServerError, HTTPURITooLongError,
@@ -422,8 +422,10 @@ def configure_connection(**kwargs):
         - "keyring"  Uses the `keyring` library.
         - "auth_token" Uses only a given auth_token.
 
-    timeout : int
+    timeout : int, tuple, or None
         A connection timeout in seconds. (Default: 210)
+        If a tuple is given, they are used as connect and read timeouts
+        respectively, timeout None disables the timeout.
 
     ssl_insecure : bool
         Whether SSL certificate warnings should be ignored. Only use this for
@@ -465,21 +467,29 @@ def configure_connection(**kwargs):
     global_conf = {}
     conf = get_config()
     # Convert config to dict, with preserving types
-    int_opts = ["timeout"]
+    int_opts = []
     bool_opts = ["ssl_insecure"]
+    other_opts = ["timeout"]
 
     if conf.has_section("Connection"):
         global_conf = dict(conf.items("Connection"))
-        # Integer options
 
+        # Integer options
         for opt in int_opts:
             if opt in global_conf:
                 global_conf[opt] = conf.getint("Connection", opt)
-        # Boolean options
 
+        # Boolean options
         for opt in bool_opts:
             if opt in global_conf:
                 global_conf[opt] = conf.getboolean("Connection", opt)
+
+        # Other options, defer parsing to configuration.config_to_yaml:
+        connection_config = config_to_yaml(conf)["Connection"]
+        for opt in other_opts:
+            if opt in global_conf:
+                global_conf[opt] = connection_config[opt]
+
     local_conf = _make_conf(_DEFAULT_CONF, global_conf, kwargs)
 
     connection = _Connection.get_instance()

src/linkahead/schema-pycaosdb-ini.yml

@@ -67,7 +67,13 @@ schema-pycaosdb-ini:
           description: This option is used internally and for testing. Do not override.
           examples: [_DefaultCaosDBServerConnection]
         timeout:
-          type: integer
+          oneOf:
+            - type: [integer, "null"]
+            - type: array
+              items:
+                type: [integer, "null"]
+              minItems: 2
+              maxItems: 2
       allOf:
         - if:
             properties:

unittests/test_configs/pylinkahead-timeout1.ini

+[Connection]
+url=https://localhost:10443/
+password_method = unauthenticated
+timeout = None

unittests/test_configs/pylinkahead-timeout2.ini

+[Connection]
+url=https://localhost:10443/
+password_method = unauthenticated
+timeout = (1,20)

unittests/test_configuration.py

@@ -24,6 +24,7 @@
 
 from os import environ, getcwd, remove
 from os.path import expanduser, isfile, join
+from pathlib import Path
 
 import linkahead as db
 import pytest
@@ -66,3 +67,18 @@ def test_config_ini_via_envvar(temp_ini_files):
     assert expanduser("~/.pylinkahead.ini") in db.configuration._read_config_files()
     # test configuration file in cwd
     assert join(getcwd(), "pylinkahead.ini") in db.configuration._read_config_files()
+
+
+def test_config_timeout_option():
+    expected_results = [None, (1, 20)]
+    # Iterate through timeout test configs
+    test_configs = Path(__file__).parent/'test_configs'
+    for test_config in test_configs.rglob('pylinkahead-timeout*.ini'):
+        # Test that test configs can be parsed
+        db.configure(str(test_config))
+        dct = db.configuration.config_to_yaml(db.get_config())
+        # Test that resulting dict has correct content for timeout
+        assert 'Connection' in dct
+        assert 'timeout' in dct['Connection']
+        assert dct['Connection']['timeout'] in expected_results
+        expected_results.remove(dct['Connection']['timeout'])