Merge branch 'release-0.10.0' into 'main'

Release 0.10.0

See merge request !79

.gitlab-ci.yml

@@ -54,14 +54,38 @@ pylint:
   allow_failure: true
 
 # run unit tests
-unittest:
+unittest_py3.8:
+  tags: [ docker ]
+  stage: test
+  needs: [ ]
+  image: python:3.8
+  script: &python_test_script
+    # Python docker has problems with tox and pip so use plain pytest here
+    - touch ~/.pycaosdb.ini
+    - pip install nose pytest pytest-cov python-dateutil jsonschema==4.0.1
+    - pip install .
+    - python -m pytest unittests
+
+# This needs to be changed once Python 3.9 isn't the standard Python in Debian
+# anymore.
+unittest_py3.9:
   tags: [ docker ]
   stage: test
   needs: [ ]
   script:
+    # verify that this actually is Python 3.9
+    - python3 -c "import sys; assert sys.version.startswith('3.9')"
     - touch ~/.pycaosdb.ini
     - make unittest
 
+
+unittest_py3.10:
+  tags: [ docker ]
+  stage: test
+  needs: [ ]
+  image: python:3.10
+  script: *python_test_script
+
 # Trigger building of server image and integration tests
 trigger_build:
   stage: deploy
@@ -88,12 +112,12 @@ build-testenv:
   stage: setup
   only:
       - schedules
-  script: 
+  script:
     - cd unittests/docker
     - docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY
       # use here general latest or specific branch latest...
     - docker pull $CI_REGISTRY_IMAGE|| true
-    - docker build 
+    - docker build
       --pull
       --build-arg COMMIT=$CI_COMMIT_SHORT_SHA
       --cache-from $CI_REGISTRY_IMAGE
@@ -105,7 +129,7 @@ build-testenv:
 pages_prepare: &pages_prepare
   tags: [ cached-dind ]
   stage: deploy
-  needs: [ code_style, pylint, unittest ]
+  needs: [ code_style, pylint, unittest_py3.8, unittest_py3.9, unittest_py3.10 ]
   only:
     refs:
       - /^release-.*$/i

CHANGELOG.md

@@ -5,6 +5,50 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [0.10.0] - 2022-11-14
+
+### Added ###
+
+* HTTP connections are allowed additionally to HTTPS connections.
+* Dependency on the `requests` package.
+* Dependency on the `python-dateutil` package.
+* `Connection.https_proxy` and `Connection.http_proxy` option of the
+  pycaosdb.ini and the `https_proxy` and `http_proxy` parameter of the
+  `configure_connection` function. See the documentation of the
+  latter for more information.
+  Note that the `HTTP_PROXY` and `HTTPS_PROXY` environment variables are
+  respected as well, unless overridden programmatically.
+* `apiutils.empty_diff` function that returns `True` if the diffs of two
+  entities found with the `compare_entitis` function are empty, `False`
+  otherwise.
+
+### Changed ###
+
+* `apiutils.compare_entities` now has an optional `compare_referenced_records`
+  argument to compare referenced Entities recursively (fomerly, only the
+  referenced Python objects would be compared). The default is `False` to
+  recover the original behavior.
+* `apiutils.merge_entities` now has an optional
+  `merge_references_with_empty_diffs` argument that determines whether a merge
+  of two entities will be performed if they reference identical records (w.r.t
+  th above `empty_diff` function). Formerly this would have caused a merge
+  conflict if the referenced record(s) were identical, but stored in different
+  Python objects.
+* `apiutils.merge_entities` now has an optional `force` argument (defaults to
+  `False`, i.e., the old behavior) which determines whether in case of merge
+  conflicts errors will be raised or the properties and attributes of entity A
+  will be overwritten by entity B.
+
+### Deprecated ###
+
+* `Connection.socket_proxy` option of the pycaosdb.ini. Please use
+  `Connection.https_proxy` or `Connection.http_proxy` instead. The deprecated
+  option will be removed with the next minor release.
+
+### Fixed ###
+
+* handling of special attributes (name, id, ...) in `apiutils.empty_diff`
+
 ## [0.9.0] - 2022-10-24
 (Florian Spreckelsen)
 

setup.py

@@ -47,7 +47,7 @@ from setuptools import find_packages, setup
 
 ISRELEASED = True
 MAJOR = 0
-MINOR = 9
+MINOR = 10
 MICRO = 0
 # Do not tag as pre-release until this commit
 # https://github.com/pypa/packaging/pull/515
@@ -97,6 +97,9 @@ def get_version_info():
 
     if os.path.exists('.git'):
         GIT_REVISION = git_version()
+    elif os.path.exists('caosdb_pylib_commit'):
+        with open('caosdb_pylib_commit', 'r') as f:
+            GIT_REVISION = f.read().strip()
     elif os.path.exists('src/caosdb/version.py'):
         # must be a source distribution, use existing version file
         try:
@@ -171,7 +174,11 @@ def setup_package():
         python_requires='>=3.8',
         package_dir={'': 'src'},
         install_requires=['lxml>=4.6.3',
-                          'PyYAML>=5.4.1', 'future', 'PySocks>=1.6.7'],
+                          "requests[socks]>=2.28.1",
+                          "python-dateutil>=2.8.2",
+                          'PyYAML>=5.4.1',
+                          'future',
+                         ],
         extras_require={'keyring': ['keyring>=13.0.0'],
                         'jsonschema': ['jsonschema>=4.4.0']},
         setup_requires=["pytest-runner>=2.0,<3dev"],

src/caosdb/apiutils.py

@@ -188,9 +188,8 @@ def getCommitIn(folder):
         return t.readline().strip()
 
 
-def compare_entities(old_entity: Entity, new_entity: Entity):
-    """
-    Compare two entites.
+def compare_entities(old_entity: Entity, new_entity: Entity, compare_referenced_records: bool = False):
+    """Compare two entites.
 
     Return a tuple of dictionaries, the first index belongs to additional information for old
     entity, the second index belongs to additional information for new entity.
@@ -204,6 +203,23 @@ def compare_entities(old_entity: Entity, new_entity: Entity):
         - ... value (not implemented yet)
         In case of changed information the value listed under the respective key shows the
         value that is stored in the respective entity.
+
+    If `compare_referenced_records` is `True`, also referenced entities will be
+    compared using this function (which is then called with
+    `compare_referenced_records = False` to prevent infinite recursion in case
+    of circular references).
+
+    Parameters
+    ----------
+    old_entity, new_entity : Entity
+        Entities to be compared
+    compare_referenced_records : bool, optional
+        Whether to compare referenced records in case of both, `old_entity` and
+        `new_entity`, have the same reference properties and both have a Record
+        object as value. If set to `False`, only the corresponding Python
+        objects are compared which may lead to unexpected behavior when
+        identical records are stored in different objects. Default is False.
+
     """
     olddiff: Dict[str, Any] = {"properties": {}, "parents": []}
     newdiff: Dict[str, Any] = {"properties": {}, "parents": []}
@@ -270,9 +286,29 @@ def compare_entities(old_entity: Entity, new_entity: Entity):
                     matching[0].unit
 
             if (prop.value != matching[0].value):
-                olddiff["properties"][prop.name]["value"] = prop.value
-                newdiff["properties"][prop.name]["value"] = \
-                    matching[0].value
+                # basic comparison of value objects says they are different
+                same_value = False
+                if compare_referenced_records:
+                    # scalar reference
+                    if isinstance(prop.value, Entity) and isinstance(matching[0].value, Entity):
+                        # explicitely not recursive to prevent infinite recursion
+                        same_value = empty_diff(
+                            prop.value, matching[0].value, compare_referenced_records=False)
+                    # list of references
+                    elif isinstance(prop.value, list) and isinstance(matching[0].value, list):
+                        # all elements in both lists actually are entity objects
+                        # TODO: check, whether mixed cases can be allowed or should lead to an error
+                        if all([isinstance(x, Entity) for x in prop.value]) and all([isinstance(x, Entity) for x in matching[0].value]):
+                            # can't be the same if the lengths are different
+                            if len(prop.value) == len(matching[0].value):
+                                # do a one-by-one comparison; the values are the same, if all diffs are empty
+                                same_value = all(
+                                    [empty_diff(x, y, False) for x, y in zip(prop.value, matching[0].value)])
+
+                if not same_value:
+                    olddiff["properties"][prop.name]["value"] = prop.value
+                    newdiff["properties"][prop.name]["value"] = \
+                        matching[0].value
 
             if (len(newdiff["properties"][prop.name]) == 0
                     and len(olddiff["properties"][prop.name]) == 0):
@@ -300,7 +336,36 @@ def compare_entities(old_entity: Entity, new_entity: Entity):
     return (olddiff, newdiff)
 
 
-def merge_entities(entity_a: Entity, entity_b: Entity):
+def empty_diff(old_entity: Entity, new_entity: Entity, compare_referenced_records: bool = False):
+    """Check whether the `compare_entities` found any differences between
+    old_entity and new_entity.
+
+    Parameters
+    ----------
+    old_entity, new_entity : Entity
+        Entities to be compared
+    compare_referenced_records : bool, optional
+        Whether to compare referenced records in case of both, `old_entity` and
+        `new_entity`, have the same reference properties and both have a Record
+        object as value.
+
+    """
+    olddiff, newdiff = compare_entities(
+        old_entity, new_entity, compare_referenced_records)
+    for diff in [olddiff, newdiff]:
+        for key in ["parents", "properties"]:
+            if len(diff[key]) > 0:
+                # There is a difference somewhere in the diff
+                return False
+        for key in SPECIAL_ATTRIBUTES:
+            if key in diff and diff[key]:
+                # There is a difference in at least one special attribute
+                return False
+    # all elements of the two diffs were empty
+    return True
+
+
+def merge_entities(entity_a: Entity, entity_b: Entity, merge_references_with_empty_diffs=True, force=False):
     """
     Merge entity_b into entity_a such that they have the same parents and properties.
 
@@ -314,13 +379,34 @@ def merge_entities(entity_a: Entity, entity_b: Entity):
     Returns entity_a.
 
     WARNING: This function is currently experimental and insufficiently tested. Use with care.
+
+    Parameters
+    ----------
+    entity_a, entity_b : Entity
+       The entities to be merged. entity_b will be merged into entity_a in place
+    merge_references_with_empty_diffs : bool, optional
+       Whether the merge is performed if entity_a and entity_b both reference
+       record(s) that may be different Python objects but have empty diffs. If
+       set to `False` a merge conflict will be raised in this case
+       instead. Default is True.
+    force : bool, optional
+       If True, in case `entity_a` and `entity_b` have the same properties, the
+       values of `entity_a` are replaced by those of `entity_b` in the merge.
+       If `False`, a RuntimeError is raised instead. Default is False.
+
+    Returns
+    -------
+    entity_a : Entity
+       The initial entity_a after the in-place merge
+
     """
 
     logging.warning(
         "This function is currently experimental and insufficiently tested. Use with care.")
 
     # Compare both entities:
-    diff_r1, diff_r2 = compare_entities(entity_a, entity_b)
+    diff_r1, diff_r2 = compare_entities(
+        entity_a, entity_b, compare_referenced_records=merge_references_with_empty_diffs)
 
     # Go through the comparison and try to apply changes to entity_a:
     for key in diff_r2["parents"]:
@@ -343,6 +429,9 @@ def merge_entities(entity_a: Entity, entity_b: Entity):
                     if (diff_r1["properties"][key][attribute] is None):
                         setattr(entity_a.get_property(key), attribute,
                                 diff_r2["properties"][key][attribute])
+                    elif force:
+                        setattr(entity_a.get_property(key), attribute,
+                                diff_r2["properties"][key][attribute])
                     else:
                         raise RuntimeError(
                             f"Merge conflict:\nEntity a ({entity_a.id}, {entity_a.name}) "
@@ -370,6 +459,9 @@ def merge_entities(entity_a: Entity, entity_b: Entity):
         if sa_a != sa_b:
             if sa_a is None:
                 setattr(entity_a, special_attribute, sa_b)
+            elif force:
+                # force overwrite
+                setattr(entity_a, special_attribute, sa_b)
             else:
                 raise RuntimeError("Merge conflict.")
     return entity_a

src/caosdb/cert/indiscale.ca.crt

------BEGIN CERTIFICATE-----
-MIIJ6TCCBdGgAwIBAgIIFVYzdrEDk6EwDQYJKoZIhvcNAQENBQAwgZMxCzAJBgNV
-BAYTAkRFMRUwEwYDVQQIEwxMb3dlciBTYXhvbnkxEzARBgNVBAcTCkdvZXR0aW5n
-ZW4xFzAVBgNVBAoTDkluZGlTY2FsZSBHbWJIMRwwGgYDVQQDExNJbmRpU2NhbGUg
-Um9vdCBDQSAxMSEwHwYJKoZIhvcNAQkBFhJpbmZvQGluZGlzY2FsZS5jb20wHhcN
-MTkwODA3MDAwMDAwWhcNMzQwODA2MjM1OTU5WjCBkzELMAkGA1UEBhMCREUxFTAT
-BgNVBAgTDExvd2VyIFNheG9ueTETMBEGA1UEBxMKR29ldHRpbmdlbjEXMBUGA1UE
-ChMOSW5kaVNjYWxlIEdtYkgxHDAaBgNVBAMTE0luZGlTY2FsZSBSb290IENBIDEx
-ITAfBgkqhkiG9w0BCQEWEmluZm9AaW5kaXNjYWxlLmNvbTCCBCIwDQYJKoZIhvcN
-AQEBBQADggQPADCCBAoCggQBAKxJO3XOqrUxFU3qdVyk9tmZEHwhwntcLO+kRR5t
-64/1Z/+VIPSgVN5phkSCukj2BPJITWKplWzJDAYWSvA/7cqavCtx8yP+m3AHWrRa
-CeHbtkGZ1nzwyFel3GIr93e65REeWqBE3knzem+qxTlZ2hp8/w3oxUlhy7tGxjBs
-JlekgLRDrnj4Opyb4GVjcVfcELmu3sLrrPX1wdYJrqaMQUR4BKZnbXxKdOYyX+kR
-/W2P4sihCCJh7Wy29VXHwSSCM1qEkU3REjvPEmEElCG7UpqOfg+3jaNZDqnvfskf
-okU4GuFCxSWQituyP9jm/hFVEhz59tUMYCllcjEi2jGmD2DBKpiru4t4/z0Aymf4
-Pep9hNtH1yhZMxpQeCYK9ESEE5d7do0bu/4YFp7jAg5vWZ8KlILZakmypVBFUw8I
-U/QJoJ55j95vIp+kjFdXelIVcr5La/zOR82JldaoPfyoBKObzwpwqaWQwYm8pj4p
-XkUdJTf8rpW21SSGWZm8JoFSYDfGvI61rPEjl/ohKhlG0tV6E2tCc406HNo/7pPe
-pmx/v9ZWLbYDAH7MVMB4tv6zDRE/c4KTbh5/s70VbXbAeOG6DNwegdDLDYZOv6Yw
-YQMz9NWtKGzvoFehP2vY5nGK95JVUcd90jaNaoURLB102VtxAjPIEQA1PjbQxLvC
-7A6kshlpQiN7zS/R9IgiEkYP/9gjy6mMuQVxH7C+9cqmCnXvVmpHmxXGUqk61r/B
-h12htsx5qjbbkToZYhUXBmwRq4LDtyoxNeaF2Jc+gE762obbHsSYMuSuh0kTFUUd
-uqfrI8OyzX4r1w5dYf2FEetZTT2Obyxb3Cy0btJF5+zEerBX44RulkdC+TPTMhJw
-b1jrPCACKywy9b6vJcSQ2V1+uLk7rH2JKD+fQRIKUqZZkhNKFYz5dnYYTgS45M0/
-C+vIvRnhgNSNb4efG6wyFvWEF8poDSPnJ4mM+0jHG/+cLqF/M2CMFvC+yU8Hj9YH
-B+H2L6V1QlCkpw5Ai4ji6OaQmnrsjE8EJj58vwYKsjmLGuf4j5AivkQTxfgCPGrT
-6CxSesoFmYDPSg/2eO+IfYEwnd7Rbs4aAhW8eo+lGpmK0DQxNjlejYt/Cgp7HWCq
-m/VNqWPIDMSTTqyk1GTmp67NjEZKt2ukJxI2CpL8s/9x4f3GTjNyI750pKM/uzMk
-OBKTMuWJQ6xeMR3h9RQlqlmwcErLXoUGInOTHHjRGXDI+ZBeLqT5DikcFiwbHG3+
-6FOuxXO0eqqg2tBW8cQ5kuRI0YFznipDUcfgDZt0JEkEXmRuL0nxYO35WKKdpGcF
-xFRJtO4FRB4nVWekVRuK9m47IPm6vC4eo+pCNPPoQ+FjyQ8CAwEAAaM/MD0wDAYD
-VR0TBAUwAwEB/zAdBgNVHQ4EFgQUFjE2TLaKASKEJ0LKOO+37/Hu7qowDgYDVR0P
-AQH/BAQDAgEGMA0GCSqGSIb3DQEBDQUAA4IEAQB2j1GL1G0ferWp9wmuDdF2oumn
-k+JIctRaGHaSrqHy4zjwj3Oqm2JA1ds+WfWozz+d38ZcLqSHo+g9ku5h/XOogQEU
-O4/y7j44pxIUg0EcIpMHtf7KPocPfvi9lw/4vE/3V/WKh4E09SXWgyY5tMUlEMaB
-6t8n7gg943HY2MJE1QU2wOPMXB1krFbunYxJcrUMs21w9jRWVv/wvaj6rkdvvSbU
-Yre11J+VlhC6fxx+STohQopzE6jnsaHile56b9xAmCCKcPEpWeKKBFS7pVNHEIHF
-uHWpgVjhoheEMMbYgu6l5E5K32TNYCKU49jNRWEKETjmYQSNl9dsSip+XlvaU8wQ
-VRR8UMHZPiJDW/AAHCr+bXEarZ9mSj/y+R512YtVw95zCnGUtzOJViThoIk/IAOR
-AJdnvsFmZSIKtFHpSEFYlTDq2yr1ulzbaDhuPRzita8b0cP27UvqRebZw5CvHN48
-B9a9tTYowKuJqmtjE6D00QA4xS8fRizLnx54uNmDbwf/8WavVk6MzDERwRE3OsSy
-D0dV6gy3t2AqEpVBrICrFqvgAQa4fcFcIwz3Qbt5o5uEi7acRomY57YrxrlfNTwh
-2oDQz+HQ/ZTDwZ3DrIgel7GrQ5fXrXDLL3ebtsbuIeBx8crOWQask832HcLtDVpu
-E/FdJEMMjglzIcy2dHpuODIGFmgEVfHR4DOOSBl0hfNdlrYnhC0h8/6QFswtlYFF
-8aQbGX7inK8L2in5wQ7ypeoMuXkQVYxlU1TEGmgB8aDke47MuX1FH+clsCaZ3s1E
-ka6lV6cjNYcosS718B6b2JgDUzmGBn2Sdm1xFmJM16dXp7TSmC5/fYxXuE/CynDs
-PmaUb9Ms6XUYSwKKhZ5HZdeRoNz8w62WNAeF7o7iX6IVrd/G1bJnSBN01istckyR
-BDuIkaoBQ9yvHN6Bo/J3KR08ixF1dHFPo/oSgkBxkLakb/yeslBTP/oISiFeQ4+q
-Gld1mhAvmG99dVZfoysrMjZSyghNbqwScjbYYN115lExV5ZeRtSwA7JCYE2lBjmB
-vocmz/hh/ifbmmqIvSv0NtiBnM6mNqngZEWD/rAloVOQoq0KVJJ5lUCQrBSFtR4+
-G1JGMX6b7uRp4mfdqqDE62KxxfkWBUwzUTIKGb5K42ji1Gy5li/TIWJtLNGNNQ2A
-0ui2RhwioaGGfYyomSFuAo5IPE/NF0ASjrTDW6GoNxypTSYE4/7oSoxeryafVnqN
-S0fRyrgSLiuT5tAiZ3b5Q3EFYUM2OcU3ezr/ZUabf9qIsqOnCi91SqE88BQbenot
-0HyUMdp/7QX9SyWM/azhcRiReAtkmq9pgeQA2TTZADDNTkKRljG9VeFDSwl7
------END CERTIFICATE-----

src/caosdb/common/administration.py

@@ -76,7 +76,7 @@ def get_server_properties():
     con = get_connection()
     try:
         body = con._http_request(
-            method="GET", path="_server_properties").response
+            method="GET", path="_server_properties")
     except EntityDoesNotExistError:
         raise ServerConfigurationException(
             "Debug mode in server is probably disabled.") from None

src/caosdb/common/timezone.py

@@ -12,6 +12,7 @@ class TimeZone():
     display_name : string
         A human-friendly name of the time zone:
     """
+
     def __init__(self, zone_id, offset, display_name):
         self.zone_id = zone_id
         self.offset = offset

src/caosdb/configuration.py

@@ -31,12 +31,7 @@ try:
 except ImportError:
     pass
 
-try:
-    # python2
-    from ConfigParser import ConfigParser
-except ImportError:
-    # python3
-    from configparser import ConfigParser
+from configparser import ConfigParser
 
 from os import environ, getcwd
 from os.path import expanduser, join, isfile
@@ -59,6 +54,11 @@ def configure(inifile):
         _reset_config()
     read_config = _pycaosdbconf.read(inifile)
     validate_yaml_schema(config_to_yaml(_pycaosdbconf))
+
+    if "HTTPS_PROXY" in environ:
+        _pycaosdbconf["Connection"]["https_proxy"] = environ["HTTPS_PROXY"]
+    if "HTTP_PROXY" in environ:
+        _pycaosdbconf["Connection"]["http_proxy"] = environ["HTTP_PROXY"]
     return read_config
 
 

src/caosdb/connection/connection.py

@@ -28,9 +28,15 @@ from __future__ import absolute_import, print_function, unicode_literals
 import logging
 import ssl
 import sys
+import warnings
 from builtins import str  # pylint: disable=redefined-builtin
 from errno import EPIPE as BrokenPipe
 from socket import error as SocketError
+from urllib.parse import quote, urlparse
+from requests import Session as HTTPSession
+from requests.exceptions import ConnectionError as HTTPConnectionError
+from urllib3.poolmanager import PoolManager
+from requests.adapters import HTTPAdapter
 
 from caosdb.configuration import get_config
 from caosdb.exceptions import (CaosDBException, HTTPClientError,
@@ -49,16 +55,8 @@ except ModuleNotFoundError:
 from pkg_resources import resource_filename
 
 from .interface import CaosDBHTTPResponse, CaosDBServerConnection
-from .streaminghttp import StreamingHTTPSConnection
 from .utils import make_uri_path, parse_url, urlencode
-
-try:
-    from urllib.parse import quote, urlparse
-except ImportError:
-    from urllib import quote
-    from urlparse import urlparse
-
-# pylint: disable=missing-docstring
+from .encode import MultipartYielder, ReadableMultiparts
 
 _LOGGER = logging.getLogger(__name__)
 
@@ -67,6 +65,9 @@ class _WrappedHTTPResponse(CaosDBHTTPResponse):
 
     def __init__(self, response):
         self.response = response
+        self._generator = None
+        self._buffer = b''
+        self._stream_consumed = False
 
     @property
     def reason(self):
@@ -74,21 +75,71 @@ class _WrappedHTTPResponse(CaosDBHTTPResponse):
 
     @property
     def status(self):
-        return self.response.status
+        return self.response.status_code
 
     def read(self, size=None):
-        return self.response.read(size)
+        if self._stream_consumed is True:
+            raise RuntimeError("Stream is consumed")
+
+        if self._buffer is None:
+            # the buffer has been drained in the previous call.
+            self._stream_consumed = True
+            return b''
+
+        if self._generator is None and (size is None or size == 0):
+            # return full content at once
+            self._stream_consumed = True
+            return self.response.content
+
+        if len(self._buffer) >= size:
+            # still enough bytes in the buffer
+            result = chunk[:size]
+            self._buffer = chunk[size:]
+            return result
+
+        if self._generator is None:
+            # first call to this method
+            if size is None or size == 0:
+                size = 512
+            self._generator = self.response.iter_content(size)
+
+        try:
+            # read new data into the buffer
+            chunk = self._buffer + next(self._generator)
+            result = chunk[:size]
+            if len(result) == 0:
+                self._stream_consumed = True
+            self._buffer = chunk[size:]
+            return result
+        except StopIteration:
+            # drain buffer
+            result = self._buffer
+            self._buffer = None
+            return result
 
     def getheader(self, name, default=None):
-        return self.response.getheader(name=name, default=default)
+        return self.response.headers[name] if name in self.response.headers else default
 
     def getheaders(self):
-        return self.response.getheaders()
+        return self.response.headers.items()
 
     def close(self):
         self.response.close()
 
 
+class _SSLAdapter(HTTPAdapter):
+    """Transport adapter that allows us to use different SSL versions."""
+
+    def __init__(self, ssl_version):
+        self.ssl_version = ssl_version
+        super().__init__()
+
+    def init_poolmanager(self, connections, maxsize, block=False):
+        self.poolmanager = PoolManager(
+            num_pools=connections, maxsize=maxsize,
+            block=block, ssl_version=self.ssl_version)
+
+
 class _DefaultCaosDBServerConnection(CaosDBServerConnection):
     """_DefaultCaosDBServerConnection.
 
@@ -101,10 +152,11 @@ class _DefaultCaosDBServerConnection(CaosDBServerConnection):
     def __init__(self):
         self._useragent = ("caosdb-pylib/{version} - {implementation}".format(
             version=version, implementation=type(self).__name__))
-        self._http_con = None
         self._base_path = None
+        self._session = None
+        self._timeout = None
 
-    def request(self, method, path, headers=None, body=None, **kwargs):
+    def request(self, method, path, headers=None, body=None):
         """request.
 
         Send a HTTP request to the server.
@@ -118,38 +170,40 @@ class _DefaultCaosDBServerConnection(CaosDBServerConnection):
             including query and frament segments.
         headers : dict of str -> str, optional
             HTTP request headers. (Defautl: None)
-        body : str or bytes or readable, opional
+        body : str or bytes or readable, optional
             The body of the HTTP request. Bytes should be a utf-8 encoded
             string.
-        **kwargs :
-            Any keyword arguments will be ignored.
-            TODO: Why are they allowed then?
 
         Returns
         -------
-            TODO: What?
+        response : CaosDBHTTPResponse
         """
 
         if headers is None:
             headers = {}
         headers["User-Agent"] = self._useragent
+
+        if path.endswith("/."):
+            path = path[:-1] + "%2E"
+
+        if isinstance(body, MultipartYielder):
+            body = ReadableMultiparts(body)
+
         try:
-            self._http_con = StreamingHTTPSConnection(
-                # TODO looks as if configure needs to be done first.
-                # That is however not assured.
-                host=self.setup_fields["host"],
-                timeout=self.setup_fields["timeout"],
-                context=self.setup_fields["context"],
-                socket_proxy=self.setup_fields["socket_proxy"])
-            self._http_con.request(method=method, url=self._base_path + path,
-                                   headers=headers, body=body)
-        except SocketError as socket_err:
+            response = self._session.request(
+                method=method,
+                url=self._base_path + path,
+                headers=headers,
+                data=body,
+                timeout=self._timeout,
+                stream=True)
+
+            return _WrappedHTTPResponse(response)
+        except HTTPConnectionError as conn_err:
             raise CaosDBConnectionError(
-                "Connection failed. Network or server down? " + str(socket_err)
+                "Connection failed. Network or server down? " + str(conn_err)
             )
 
-        return _WrappedHTTPResponse(self._http_con.getresponse())
-
     def configure(self, **config):
         """configure.
 
@@ -173,55 +227,69 @@ class _DefaultCaosDBServerConnection(CaosDBServerConnection):
             loaded.
         """
 
+        if "url" not in config:
+            raise CaosDBConnectionError(
+                "No connection url specified. Please "
+                "do so via caosdb.configure_connection(...) or in a config "
+                "file.")
+        if (not config["url"].lower().startswith("https://") and not config["url"].lower().startswith("http://")):
+            raise CaosDBConnectionError("The connection url is expected "
+                                        "to be a http or https url and "
+                                        "must include the url scheme "
+                                        "(i.e. start with https:// or "
+                                        "http://).")
+
+        url = urlparse(config["url"])
+        path = url.path.strip("/")
+        if len(path) > 0:
+            path = path + "/"
+        self._base_path = url.scheme + "://" + url.netloc + "/" + path
+
+        self._session = HTTPSession()
+
+        if url.scheme == "https":
+            self._setup_ssl(config)
+
+        # TODO(tf) remove in next release
+        socket_proxy = config["socket_proxy"] if "socket_proxy" in config else None
+        if socket_proxy is not None:
+            self._session.proxies = {
+                "https": "socks5://" + socket_proxy,
+                "http": "socks5://" + socket_proxy,
+            }
+
+        if "https_proxy" in config:
+            if self._session.proxies is None:
+                self._session.proxies = {}
+            self._session.proxies["https"] = config["https_proxy"]
+
+        if "http_proxy" in config:
+            if self._session.proxies is None:
+                self._session.proxies = {}
+            self._session.proxies["http"] = config["http_proxy"]
+
+        if "timeout" in config:
+            self._timeout = config["timeout"]
+
+    def _setup_ssl(self, config):
         if "ssl_version" in config and config["cacert"] is not None:
             ssl_version = getattr(ssl, config["ssl_version"])
         else:
             ssl_version = ssl.PROTOCOL_TLS
-        context = ssl.SSLContext(ssl_version)
-        context.verify_mode = ssl.CERT_REQUIRED
 
-        if config.get("ssl_insecure"):
+        self._session.mount(self._base_path, _SSLAdapter(ssl_version))
+
+        verify = True
+        if "cacert" in config:
+            verify = config["cacert"]
+        if "ssl_insecure" in config and config["ssl_insecure"]:
             _LOGGER.warning("*** Warning! ***\n"
                             "Insecure SSL mode, certificate will not be checked! "
                             "Please consider removing the `ssl_insecure` configuration option.\n"
                             "****************")
-            context.verify_mode = ssl.CERT_NONE
-
-        if (not context.verify_mode == ssl.CERT_NONE and
-                hasattr(context, "check_hostname")):
-            context.check_hostname = True
-
-        if ("cacert" in config and config["cacert"] is not None and
-                config["cacert"]):
-            try:
-                context.load_verify_locations(config["cacert"])
-            except Exception as exc:
-                raise CaosDBConnectionError("Could not load the cacert in"
-                                            "`{}`: {}".format(config["cacert"],
-                                                              exc))
-
-        context.load_default_certs()
-
-        if "url" in config:
-            parsed_url = parse_url(config["url"])
-            host = parsed_url.netloc
-            self._base_path = parsed_url.path
-        else:
-            raise CaosDBConnectionError(
-                "No connection url specified. Please "
-                "do so via caosdb.configure_connection(...) or in a config "
-                "file.")
-
-        socket_proxy = None
-
-        if "socket_proxy" in config:
-            socket_proxy = config["socket_proxy"]
-
-        self.setup_fields = {
-            "host": host,
-            "timeout": int(config.get("timeout")),
-            "context": context,
-            "socket_proxy": socket_proxy}
+            verify = False
+        if verify is not None:
+            self._session.verify = verify
 
 
 def _make_conf(*conf):
@@ -252,7 +320,6 @@ _DEFAULT_CONF = {
     "password_method": "input",
     "implementation": _DefaultCaosDBServerConnection,
     "timeout": 210,
-    "cacert": resource_filename("caosdb", 'cert/indiscale.ca.crt')
 }
 
 
@@ -314,6 +381,10 @@ def configure_connection(**kwargs):
 
     Parameters
     ----------
+    url : str
+        The url of the CaosDB Server. HTTP and HTTPS urls are allowed. However,
+        it is **highly** recommend to avoid HTTP because passwords and
+        authentication token are send over the network in plain text.
 
     username : str
         Username for login; e.g. 'admin'.
@@ -342,6 +413,24 @@ def configure_connection(**kwargs):
         An authentication token which has been issued by the CaosDB Server.
         Implies `password_method="auth_token"` if set.  An example token string would be `["O","OneTimeAuthenticationToken","anonymous",["administration"],[],1592995200000,604800000,"3ZZ4WKRB-5I7DG2Q6-ZZE6T64P-VQ","197d0d081615c52dc18fb323c300d7be077beaad4020773bb58920b55023fa6ee49355e35754a4277b9ac525c882bcd3a22e7227ba36dfcbbdbf8f15f19d1ee9",1,30000]`.
 
+    https_proxy : str, optional
+        Define a proxy for the https connections, e.g. `http://localhost:8888`,
+        `socks5://localhost:8888`, or `socks4://localhost:8888`. These are
+        either (non-TLS) HTTP proxies, SOCKS4 proxies, or SOCKS5 proxies. HTTPS
+        proxies are not supported. However, the connection will be secured
+        using TLS in the tunneled connection nonetheless. Only the connection
+        to the proxy is insecure which is why it is not recommended to use HTTP
+        proxies when authentication against the proxy is necessary. If
+        unspecified, the https_proxy option of the pycaosdb.ini or the HTTPS_PROXY
+        environment variable are being used. Use `None` to override these
+        options with a no-proxy setting.
+
+    http_proxy : str, optional
+        Define a proxy for the http connections, e.g. `http://localhost:8888`.
+        If unspecified, the http_proxy option of the pycaosdb.ini or the
+        HTTP_PROXY environment variable are being used. Use `None` to override
+        these options with a no-proxy setting.
+
     implementation : CaosDBServerConnection
         The class which implements the connection. (Default:
         _DefaultCaosDBServerConnection)
@@ -372,6 +461,11 @@ def configure_connection(**kwargs):
     local_conf = _make_conf(_DEFAULT_CONF, global_conf, kwargs)
 
     connection = _Connection.get_instance()
+
+    if "socket_proxy" in local_conf:
+        warnings.warn("Deprecated configuration option: socket_proxy. Use "
+                      "the new https_proxy option instead",
+                      DeprecationWarning, stacklevel=1)
     connection.configure(**local_conf)
 
     return connection
@@ -599,7 +693,7 @@ class _Connection(object):  # pylint: disable=useless-object-inheritance
             method=method,
             path=path,
             headers=headers,
-            body=body, **kwargs)
+            body=body)
         _LOGGER.debug("response: %s %s", str(http_response.status),
                       str(http_response.getheaders()))
         self._authenticator.on_response(http_response)

src/caosdb/connection/encode.py

@@ -51,7 +51,8 @@ multipart/form-data is the standard way to upload files over HTTP
 
 __all__ = [
     'gen_boundary', 'encode_and_quote', 'MultipartParam', 'encode_string',
-    'encode_file_header', 'get_body_size', 'get_headers', 'multipart_encode'
+    'encode_file_header', 'get_body_size', 'get_headers', 'multipart_encode',
+    'ReadableMultiparts',
 ]
 from urllib.parse import quote_plus
 from io import UnsupportedOperation
@@ -475,3 +476,40 @@ def multipart_encode(params, boundary=None, callback=None):
     params = MultipartParam.from_params(params)
 
     return MultipartYielder(params, boundary, callback), headers
+
+
+class ReadableMultiparts(object):
+    """Wraps instances of the MultipartYielder class as a readable and withable
+    object."""
+
+    def __init__(self, multipart_yielder):
+        self.multipart_yielder = multipart_yielder
+        self.current_block = None
+        self.left_over = b''
+
+    def read(self, size=-1):
+        result = self.left_over
+        while size == -1 or len(result) < size:
+            try:
+                next_chunk = self.multipart_yielder.next()
+                if hasattr(next_chunk, "encode"):
+                    next_chunk = next_chunk.encode("utf8")
+                result += next_chunk
+            except StopIteration:
+                break
+
+        if size == -1:
+            self.left_over = b''
+            return result
+
+        self.left_over = result[size:]
+        return result[:size]
+
+    def __enter__(self):
+        pass
+
+    def __exit__(self, type, value, traceback):
+        self.close()
+
+    def close(self):
+        self.multipart_yielder.reset()

src/caosdb/connection/streaminghttp.py

-# -*- encoding: utf-8 -*-
-#
-# ** header v3.0
-# This file is a part of the CaosDB Project.
-#
-# Copyright (C) 2018 Research Group Biomedical Physics,
-# Max-Planck-Institute for Dynamics and Self-Organization Göttingen
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Affero General Public License as
-# published by the Free Software Foundation, either version 3 of the
-# License, or (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU Affero General Public License for more details.
-#
-# You should have received a copy of the GNU Affero General Public License
-# along with this program. If not, see <https://www.gnu.org/licenses/>.
-#
-# ** end header
-#
-# Original work Copyright (c) 2011 Chris AtLee
-# Modified work Copyright (c) 2017 Biomedical Physics, MPI for Dynamics and Self-Organization
-#
-# Permission is hereby granted, free of charge, to any person obtaining a copy
-# of this software and associated documentation files (the "Software"), to deal
-# in the Software without restriction, including without limitation the rights
-# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-# copies of the Software, and to permit persons to whom the Software is
-# furnished to do so, subject to the following conditions:
-#
-# The above copyright notice and this permission notice shall be included in
-# all copies or substantial portions of the Software.
-#
-# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
-# THE SOFTWARE.
-"""Streaming HTTP uploads module.
-
-This module extends the standard httplib and http.client HTTPConnection so that
-iterable objects can be used in the body of HTTP requests.
-
-**N.B.** You must specify a Content-Length header if using an iterable object
-since there is no way to determine in advance the total size that will be
-yielded, and there is no way to reset an interator.
-"""
-
-from __future__ import unicode_literals, print_function, absolute_import
-import socks
-import socket
-try:
-    # python3
-    from http import client as client
-except ImportError:
-    # python2
-    import httplib as client
-
-
-__all__ = ['StreamingHTTPSConnection']
-
-
-class StreamingHTTPSConnection(client.HTTPSConnection, object):
-    """Subclass of `http.client.HTTSConnection` or `httplib.HTTPSConnection`
-    that overrides the `send()` method to support iterable body objects."""
-    # pylint: disable=unused-argument, arguments-differ
-
-    def __init__(self, socket_proxy=None, **kwargs):
-        if socket_proxy is not None:
-            host, port = socket_proxy.split(":")
-            socks.setdefaultproxy(socks.PROXY_TYPE_SOCKS5, host,
-                                  int(port))
-            socket.socket = socks.socksocket
-        super(StreamingHTTPSConnection, self).__init__(**kwargs)
-
-    def _send_output(self, body, **kwargs):
-        """Send the currently buffered request and clear the buffer.
-
-        Appends an extra \\r\\n to the buffer.
-        A message_body may be specified, to be appended to the request.
-
-        This method is implemented in differently in the various python
-        versions (which is extremely annoying). So we provide a unified but
-        relatively dumb implementaion which only serves our needs.
-        """
-        self._buffer.extend(("".encode("utf-8"), "".encode("utf-8")))
-        headers = "\r\n".encode("utf-8").join(self._buffer)
-        del self._buffer[:]
-
-        self.send(headers)
-        if body is not None:
-            self.send(body)
-
-    # pylint: disable=too-many-branches
-
-    def send(self, value):
-        """Send ``value`` to the server.
-
-        ``value`` can be a string-like object which supports a 'encode' method,
-        a file-like object that supports a .read() method, or an iterable object
-        that supports a .next() method.
-
-        An encode()able ``value`` will be utf-8 encoded before sending.
-        """
-        # Based on python 2.6's httplib.HTTPConnection.send()
-        if self.sock is None:
-            if self.auto_open:
-                self.connect()
-            else:
-                raise client.NotConnected()
-
-        # send the data to the server. if we get a broken pipe, then close
-        # the socket. we want to reconnect when somebody tries to send again.
-        #
-        # NOTE: we DO propagate the error, though, because we cannot simply
-        #       ignore the error... the caller will know if they can retry.
-        if self.debuglevel > 0:
-            print("send: ", repr(value))
-        try:
-            blocksize = 8192
-            if hasattr(value, 'read'):
-                if hasattr(value, 'seek'):
-                    value.seek(0)
-                if self.debuglevel > 0:
-                    print("sendIng a read()able")
-                data = value.read(blocksize)
-                while data:
-                    self.sock.sendall(data)
-                    data = value.read(blocksize)
-            elif hasattr(value, 'next'):
-                if hasattr(value, 'reset'):
-                    value.reset()
-                if self.debuglevel > 0:
-                    print("sendIng an iterable")
-                for data in value:
-                    if hasattr(data, "encode"):
-                        self.sock.sendall(data.encode('utf-8'))
-                    else:
-                        self.sock.sendall(data)
-            else:
-                if self.debuglevel > 0:
-                    print("sendIng a byte-like")
-                self.sock.sendall(value)
-        except socket.error as err:
-            if err.args[0] == 32:      # Broken pipe
-                self.close()
-            raise

src/caosdb/schema-pycaosdb-ini.yml

@@ -14,10 +14,10 @@ schema-pycaosdb-ini:
       additionalProperties: false
       properties:
         url:
-          description: URL of the CaosDB server
+          description: "URL of the CaosDB server. Allowed are HTTP and HTTPS connections. However, since authentication tokens and sometimes even passwords are send in plain text to the server it is **highly** recommended to use HTTPS connections whenever possible. HTTP is ok for testing and debugging."
           type: string
-          pattern: https://[-a-zA-Z0-9\.]+(:[0-9]+)?(/)?
-          examples: ["https://demo.indiscale.com/", "https://localhost:10443/"]
+          pattern: http(s)?://[-a-zA-Z0-9\.]+(:[0-9]+)?(/)?
+          examples: ["https://demo.indiscale.com/", "http://localhost:10080/"]
         username:
           type: string
           description: User name used for authentication with the server
@@ -54,7 +54,15 @@ schema-pycaosdb-ini:
         socket_proxy:
           examples: ["localhost:12345"]
           type: string
-          description: You can define a socket proxy to be used. This is for the case that the server sits behind a firewall which is being tunnelled with a socket proxy (SOCKS4 or SOCKS5) (e.g. via ssh's -D option or a dedicated proxy server).
+          description: Deprecated. Please use https_proxy instead.
+        https_proxy:
+          examples: ["http://localhost:8888", "socks5://localhost:8888", "socks4://localhost:8888"]
+          type: string
+          description: "Define a proxy for the https connections. These are either (non-TLS) HTTP proxies, SOCKS4 proxies, or SOCKS5 proxies. HTTPS proxies are not supported. However, the connection will be secured using TLS in the tunneled connection nonetheless. Only the connection to the proxy is insecure which is why it is not recommended to use HTTP proxies when authentication against the proxy is necessary. Note: this option is overridden by the HTTPS_PROXY environment variable, if present."
+        http_proxy:
+          examples: ["http://localhost:8888", "socks5://localhost:8888", "socks4://localhost:8888"]
+          type: string
+          description: "Define a proxy for the http connections. These are either (non-TLS) HTTP proxies, SOCKS4 proxies, or SOCKS5 proxies. HTTPS proxies are not supported. Note: this option is overridden by the HTTP_PROXY environment variable, if present."
         implementation:
           description: This option is used internally and for testing. Do not override.
           examples: [_DefaultCaosDBServerConnection]

src/caosdb/utils/server_side_scripting.py

@@ -30,7 +30,8 @@ from lxml import etree
 
 from caosdb.connection.connection import get_connection
 from caosdb.connection.utils import urlencode
-from caosdb.connection.encode import MultipartParam, multipart_encode
+from caosdb.connection.encode import (MultipartParam, multipart_encode,
+                                      ReadableMultiparts)
 
 
 def _make_params(pos_args, opts):
@@ -63,6 +64,7 @@ def _make_multipart_request(call, pos_args, opts, files):
                                               filename=filename))
 
     body, headers = multipart_encode(parts)
+    body = ReadableMultiparts(body)
     return body, headers
 
 

src/doc/conf.py

@@ -29,10 +29,10 @@ copyright = '2022, IndiScale GmbH'
 author = 'Daniel Hornung'
 
 # The short X.Y version
-version = '0.9.0'
+version = '0.10.0'
 # The full version, including alpha/beta/rc tags
 # release = '0.5.2-rc2'
-release = '0.9.0'
+release = '0.10.0'
 
 
 # -- General configuration ---------------------------------------------------

tox.ini

@@ -7,7 +7,6 @@ deps = .
     nose
     pytest
     pytest-cov
-    python-dateutil
     jsonschema==4.0.1
 commands=py.test --cov=caosdb -vv {posargs}
 

unittests/test_apiutils.py

-# -*- encoding: utf-8 -*-
 #
 # This file is a part of the CaosDB Project.
 #
-# Copyright (C) 2018 Research Group Biomedical Physics,
-# Max-Planck-Institute for Dynamics and Self-Organization Göttingen
 # Copyright (C) 2020 Timm Fitschen <t.fitschen@indiscale.com>
+# Copyright (C) 2022 Florian Spreckelsen <f.spreckelsen@indiscale.com>
 # Copyright (C) 2020-2022 IndiScale GmbH <info@indiscale.com>
+# Copyright (C) 2018 Research Group Biomedical Physics,
+# Max-Planck-Institute for Dynamics and Self-Organization Göttingen
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as
@@ -20,7 +20,6 @@
 # You should have received a copy of the GNU Affero General Public License
 # along with this program. If not, see <https://www.gnu.org/licenses/>.
 #
-# ** end header
 #
 # Test apiutils
 # A. Schlemmer, 02/2018
@@ -30,7 +29,7 @@ import pytest
 import caosdb as db
 import caosdb.apiutils
 from caosdb.apiutils import (apply_to_ids, compare_entities, create_id_query,
-                             resolve_reference, merge_entities)
+                             empty_diff, resolve_reference, merge_entities)
 
 from caosdb.common.models import SPECIAL_ATTRIBUTES
 
@@ -272,8 +271,10 @@ def test_copy_entities():
     for i in [0, 1]:
         assert c.properties[i] is not r.properties[i]
         for special in SPECIAL_ATTRIBUTES:
-            assert getattr(c.properties[i], special) == getattr(r.properties[i], special)
-        assert c.get_importance(c.properties[i]) == r.get_importance(r.properties[i])
+            assert getattr(c.properties[i], special) == getattr(
+                r.properties[i], special)
+        assert c.get_importance(
+            c.properties[i]) == r.get_importance(r.properties[i])
 
 
 def test_merge_entities():
@@ -326,10 +327,12 @@ def test_merge_bug_109():
     assert r_a.get_property("test_bug_property").value == [18, 19]
 
     assert "<Value>18</Value>\n    <Value>19</Value>" in str(r_b)
-    assert "<Value>18</Value>\n    <Value>19</Value>\n    <Value>18</Value>\n    <Value>19</Value>" not in str(r_b)
+    assert "<Value>18</Value>\n    <Value>19</Value>\n    <Value>18</Value>\n    <Value>19</Value>" not in str(
+        r_b)
 
     assert "<Value>18</Value>\n    <Value>19</Value>" in str(r_a)
-    assert "<Value>18</Value>\n    <Value>19</Value>\n    <Value>18</Value>\n    <Value>19</Value>" not in str(r_a)
+    assert "<Value>18</Value>\n    <Value>19</Value>\n    <Value>18</Value>\n    <Value>19</Value>" not in str(
+        r_a)
 
 
 @pytest.mark.xfail
@@ -349,7 +352,212 @@ def test_bug_109():
     assert r_a.get_property("test_bug_property").value == [18, 19]
 
     assert "<Value>18</Value>\n    <Value>19</Value>" in str(r_b)
-    assert "<Value>18</Value>\n    <Value>19</Value>\n    <Value>18</Value>\n    <Value>19</Value>" not in str(r_b)
+    assert "<Value>18</Value>\n    <Value>19</Value>\n    <Value>18</Value>\n    <Value>19</Value>" not in str(
+        r_b)
 
     assert "<Value>18</Value>\n    <Value>19</Value>" in str(r_a)
-    assert "<Value>18</Value>\n    <Value>19</Value>\n    <Value>18</Value>\n    <Value>19</Value>" not in str(r_a)
+    assert "<Value>18</Value>\n    <Value>19</Value>\n    <Value>18</Value>\n    <Value>19</Value>" not in str(
+        r_a)
+
+
+def test_wrong_merge_conflict_reference():
+    """Test a wrongly detected merge conflict in case of two records referencing
+    two different, but identical objects.
+
+    """
+    # Two identical license records will be referenced from both records to be
+    # merged
+    license_rt = db.RecordType(name="license")
+    license_rec_a = db.Record(name="CC-BY-3.0").add_parent(license_rt)
+    license_rec_b = db.Record(name="CC-BY-3.0").add_parent(license_rt)
+
+    # two referencing records
+    dataset_rt = db.RecordType(name="Dataset")
+    title_prop = db.Property(name="title", datatype=db.TEXT)
+    doi_prop = db.Property(name="DOI", datatype=db.TEXT)
+    rec_a = db.Record().add_parent(dataset_rt)
+    rec_a.add_property(name=license_rt.name,
+                       datatype=license_rt.name, value=license_rec_a)
+    rec_a.add_property(name=title_prop.name, value="Some dataset title")
+
+    rec_b = db.Record().add_parent(dataset_rt)
+    rec_b.add_property(name=license_rt.name,
+                       datatype=license_rt.name, value=license_rec_b)
+    rec_b.add_property(name=doi_prop.name, value="https://doi.org/12345.678")
+
+    merge_entities(rec_a, rec_b)
+    assert rec_a.get_property(license_rt.name) is not None
+    assert rec_a.get_property(license_rt.name).value is not None
+    assert isinstance(rec_a.get_property(license_rt.name).value, db.Record)
+    assert rec_a.get_property(license_rt.name).value.name == license_rec_a.name
+    assert rec_a.get_property(license_rt.name).value.name == license_rec_b.name
+    assert rec_a.get_property("title").value == "Some dataset title"
+    assert rec_a.get_property("doi").value == "https://doi.org/12345.678"
+
+    # Reset rec_a
+    rec_a = db.Record().add_parent(dataset_rt)
+    rec_a.add_property(name=license_rt.name,
+                       datatype=license_rt.name, value=license_rec_a)
+    rec_a.add_property(name=title_prop.name, value="Some dataset title")
+
+    # this does not compare referenced records, so it will fail
+    with pytest.raises(RuntimeError) as re:
+        merge_entities(rec_a, rec_b, merge_references_with_empty_diffs=False)
+    assert "Merge conflict" in str(re.value)
+
+    # ... as should this, of course
+    rec_b.get_property(license_rt.name).value.name = "Another license"
+    with pytest.raises(RuntimeError) as re:
+        merge_entities(rec_a, rec_b)
+    assert "Merge conflict" in str(re.value)
+
+
+def test_empty_diff():
+
+    rec_a = db.Record(name="A")
+    rec_b = db.Record(name="B")
+
+    assert empty_diff(rec_a, rec_a)
+    assert not empty_diff(rec_a, rec_b)
+
+    rec_a.add_parent(name="RT")
+    rec_b.add_parent(name="RT")
+    assert empty_diff(rec_a, rec_a)
+    assert not empty_diff(rec_a, rec_b)
+
+    rec_b.name = "A"
+    assert empty_diff(rec_a, rec_b)
+
+    rec_a.add_property(name="some_prop", value=1)
+    assert not empty_diff(rec_a, rec_b)
+
+    rec_b.add_property(name="some_prop", value=1)
+    assert empty_diff(rec_a, rec_b)
+
+    rec_b.get_property("some_prop").value = 2
+    assert not empty_diff(rec_a, rec_b)
+
+    rec_b.get_property("some_prop").value = 1
+    rec_b.add_property(name="some_other_prop", value="Test")
+    assert not empty_diff(rec_a, rec_b)
+
+    rec_a.add_property(name="some_other_prop", value="Test")
+    assert empty_diff(rec_a, rec_b)
+
+    # reference identical records, but different Python Record objects
+    ref_rec_a = db.Record(name="Ref").add_parent(name="RefType")
+    ref_rec_b = db.Record(name="Ref").add_parent(name="RefType")
+    rec_a.add_property(name="RefType", datatype="RefType", value=ref_rec_a)
+    rec_b.add_property(name="RefType", datatype="RefType", value=ref_rec_b)
+    # the default is `compare_referenced_records=False`, so the diff shouldn't
+    # be empty (different Python objects are referenced.)
+    assert not empty_diff(rec_a, rec_b)
+    # when looking into the referenced record, the diffs should be empty again
+    assert empty_diff(rec_a, rec_b, compare_referenced_records=True)
+
+    # The same for lists of references
+    rec_a.remove_property("RefType")
+    rec_b.remove_property("RefType")
+    assert empty_diff(rec_a, rec_b)
+    rec_a.add_property(name="RefType", datatype=db.LIST(
+        "RefType"), value=[ref_rec_a, ref_rec_a])
+    rec_b.add_property(name="RefType", datatype=db.LIST(
+        "RefType"), value=[ref_rec_b, ref_rec_b])
+    assert not empty_diff(rec_a, rec_b)
+    assert empty_diff(rec_a, rec_b, compare_referenced_records=True)
+
+    # special case of ids
+    rec_a = db.Record(id=12)
+    rec_b = db.Record()
+    assert not empty_diff(rec_a, rec_b)
+    rec_b.id = 13
+    assert not empty_diff(rec_a, rec_b)
+    rec_b.id = 12
+    assert empty_diff(rec_a, rec_b)
+
+
+def test_force_merge():
+    """Test whether a forced merge overwrites existing properties correctly."""
+
+    # name overwrite
+    recA = db.Record(name="A")
+    recB = db.Record(name="B")
+
+    with pytest.raises(RuntimeError) as re:
+        merge_entities(recA, recB)
+    assert "Merge conflict" in str(re.value)
+
+    merge_entities(recA, recB, force=True)
+    assert "B" == recA.name
+    # unchanged
+    assert "B" == recB.name
+
+    # description overwrite
+    recA = db.Record()
+    recA.description = "something"
+    recB = db.Record()
+    recB.description = "something else"
+
+    with pytest.raises(RuntimeError) as re:
+        merge_entities(recA, recB)
+    assert "Merge conflict" in str(re.value)
+
+    merge_entities(recA, recB, force=True)
+    assert recA.description == "something else"
+    # unchanged
+    assert recB.description == "something else"
+
+    # property overwrite
+    recA = db.Record()
+    recA.add_property(name="propA", value="something")
+    recB = db.Record()
+    recB.add_property(name="propA", value="something else")
+
+    with pytest.raises(RuntimeError) as re:
+        merge_entities(recA, recB)
+    assert "Merge conflict" in str(re.value)
+
+    merge_entities(recA, recB, force=True)
+    assert recA.get_property("propA").value == "something else"
+    # unchanged
+    assert recB.get_property("propA").value == "something else"
+
+    # don't remove a property that's not in recB
+    recA = db.Record()
+    recA.add_property(name="propA", value="something")
+    recA.add_property(name="propB", value=5.0)
+    recB = db.Record()
+    recB.add_property(name="propA", value="something else")
+
+    merge_entities(recA, recB, force=True)
+    assert recA.get_property("propA").value == "something else"
+    assert recA.get_property("propB").value == 5.0
+
+    # also overwrite datatypes ...
+    rtA = db.RecordType()
+    rtA.add_property(name="propA", datatype=db.INTEGER)
+    rtB = db.RecordType()
+    rtB.add_property(name="propA", datatype=db.TEXT)
+
+    with pytest.raises(RuntimeError) as re:
+        merge_entities(rtA, rtB)
+    assert "Merge conflict" in str(re.value)
+
+    merge_entities(rtA, rtB, force=True)
+    assert rtA.get_property("propA").datatype == db.TEXT
+    # unchanged
+    assert rtB.get_property("propA").datatype == db.TEXT
+
+    # ... and units
+    recA = db.Record()
+    recA.add_property(name="propA", value=5, unit="m")
+    recB = db.Record()
+    recB.add_property(name="propA", value=5, unit="cm")
+
+    with pytest.raises(RuntimeError) as re:
+        merge_entities(recA, recB)
+    assert "Merge conflict" in str(re.value)
+    merge_entities(recA, recB, force=True)
+    assert recA.get_property("propA").unit == "cm"
+    # unchanged
+    assert recB.get_property("propA").unit == "cm"

unittests/test_connection.py

@@ -37,7 +37,8 @@ from caosdb.connection.connection import (CaosDBServerConnection,
 from caosdb.connection.mockup import (MockUpResponse, MockUpServerConnection,
                                       _request_log_message)
 from caosdb.connection.utils import make_uri_path, quote, urlencode
-from caosdb.exceptions import ConfigurationError, LoginFailedError
+from caosdb.exceptions import (ConfigurationError, LoginFailedError,
+                               CaosDBConnectionError)
 from nose.tools import assert_equal as eq
 from nose.tools import assert_false as falz
 from nose.tools import assert_is_not_none as there
@@ -46,6 +47,13 @@ from nose.tools import assert_true as tru
 from pytest import raises
 
 
+def setup_function(function):
+    configure_connection(url="http://localhost:8888/some/path",
+                         password_method="plain", username="test",
+                         password="blub",
+                         implementation=MockUpServerConnection)
+
+
 def setup_module():
     _reset_config()
 
@@ -103,6 +111,7 @@ def test_configure_connection():
     get_config().set("Connection", "password_method", "plain")
     get_config().set("Connection", "password", "test_password")
     get_config().set("Connection", "timeout", "200")
+    get_config().set("Connection", "ssl_insecure", "True")
 
     there(configure_connection)
     tru(hasattr(configure_connection, "__call__"))
@@ -116,6 +125,18 @@ def test_configure_connection():
     tru(isinstance(c._delegate_connection, MockUpServerConnection))
 
 
+def test_configure_connection_bad_url():
+    configure_connection(url="https://localhost:8888")
+    with raises(CaosDBConnectionError) as exc_info:
+        configure_connection(url="ftp://localhost:8888")
+    assert exc_info.value.args[0].startswith(
+        "The connection url is expected to be a http or https url")
+    with raises(CaosDBConnectionError) as exc_info:
+        configure_connection(url="localhost:8888")
+    assert exc_info.value.args[0].startswith(
+        "The connection url is expected to be a http or https url")
+
+
 def test_connection_interface():
     with raiz(TypeError) as cm:
         CaosDBServerConnection()

unittests/test_server_side_scripting.py

@@ -46,7 +46,7 @@ def setup_module():
         content_type = kwargs["headers"]["Content-Type"]
 
         if content_type.startswith("multipart/form-data; boundary"):
-            parts = kwargs["body"]
+            parts = kwargs["body"].multipart_yielder
             stdout = []
             for part in parts:
                 if hasattr(part, "decode"):