.gitignore

@@ -16,3 +16,4 @@ src/caosdb/version.py
 
 # documentation
 _apidoc
+*~

.gitlab-ci.yml

@@ -54,14 +54,38 @@ pylint:
   allow_failure: true
 
 # run unit tests
-unittest:
+unittest_py3.8:
+  tags: [ docker ]
+  stage: test
+  needs: [ ]
+  image: python:3.8
+  script: &python_test_script
+    # Python docker has problems with tox and pip so use plain pytest here
+    - touch ~/.pycaosdb.ini
+    - pip install nose pytest pytest-cov python-dateutil jsonschema==4.0.1
+    - pip install .
+    - python -m pytest unittests
+
+# This needs to be changed once Python 3.9 isn't the standard Python in Debian
+# anymore.
+unittest_py3.9:
   tags: [ docker ]
   stage: test
   needs: [ ]
   script:
+    # verify that this actually is Python 3.9
+    - python3 -c "import sys; assert sys.version.startswith('3.9')"
     - touch ~/.pycaosdb.ini
     - make unittest
 
+
+unittest_py3.10:
+  tags: [ docker ]
+  stage: test
+  needs: [ ]
+  image: python:3.10
+  script: *python_test_script
+
 # Trigger building of server image and integration tests
 trigger_build:
   stage: deploy
@@ -105,7 +129,7 @@ build-testenv:
 pages_prepare: &pages_prepare
   tags: [ cached-dind ]
   stage: deploy
-  needs: [ code_style, pylint, unittest ]
+  needs: [ code_style, pylint, unittest_py3.8, unittest_py3.9, unittest_py3.10 ]
   only:
     refs:
       - /^release-.*$/i

CHANGELOG.md

@@ -5,12 +5,18 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
-## [Unreleased]
+## [Unreleased] ##
 
 ### Added ###
 
+* `apiutils.EntityMergeConflictError` class for unresesolvable merge conflicts
+  when merging two entities
+
 ### Changed ###
 
+* `apiutils.merge_entities` now raises an `EntityMergeConflictError` in case of
+  unresolvable merge conflicts.
+
 ### Deprecated ###
 
 ### Removed ###
@@ -21,6 +27,71 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Documentation ###
 
+* [Fixed](https://gitlab.com/caosdb/caosdb-pylib/-/issues/79) `{action}_entity_permissions` help line.
+
+## [0.10.0] - 2022-11-14
+(Florian Spreckelsen)
+
+### Added ###
+
+* HTTP connections are allowed additionally to HTTPS connections.
+* Dependency on the `requests` package.
+* Dependency on the `python-dateutil` package.
+* `Connection.https_proxy` and `Connection.http_proxy` option of the
+  pycaosdb.ini and the `https_proxy` and `http_proxy` parameter of the
+  `configure_connection` function. See the documentation of the
+  latter for more information.
+  Note that the `HTTP_PROXY` and `HTTPS_PROXY` environment variables are
+  respected as well, unless overridden programmatically.
+* `apiutils.empty_diff` function that returns `True` if the diffs of two
+  entities found with the `compare_entitis` function are empty, `False`
+  otherwise.
+
+### Changed ###
+
+* `apiutils.compare_entities` now has an optional `compare_referenced_records`
+  argument to compare referenced Entities recursively (fomerly, only the
+  referenced Python objects would be compared). The default is `False` to
+  recover the original behavior.
+* `apiutils.merge_entities` now has an optional
+  `merge_references_with_empty_diffs` argument that determines whether a merge
+  of two entities will be performed if they reference identical records (w.r.t
+  th above `empty_diff` function). Formerly this would have caused a merge
+  conflict if the referenced record(s) were identical, but stored in different
+  Python objects.
+* `apiutils.merge_entities` now has an optional `force` argument (defaults to
+  `False`, i.e., the old behavior) which determines whether in case of merge
+  conflicts errors will be raised or the properties and attributes of entity A
+  will be overwritten by entity B.
+
+### Deprecated ###
+
+* `Connection.socket_proxy` option of the pycaosdb.ini. Please use
+  `Connection.https_proxy` or `Connection.http_proxy` instead. The deprecated
+  option will be removed with the next minor release.
+
+### Fixed ###
+
+* handling of special attributes (name, id, ...) in `apiutils.empty_diff`
+
+## [0.9.0] - 2022-10-24
+(Florian Spreckelsen)
+
+### Added ###
+
+* Add TimeZone class and parse the server's time zone in the Info response.
+
+### Fixed ###
+
+* [#141](https://gitlab.indiscale.com/caosdb/src/caosdb-pylib/-/issues/141)
+  `password_method = unauthenticated` not allowed by schema
+* Set PyYAML dependency back to PyYaml>=5.4.1 (from 6.0) for better
+  compatibility with docker-compose
+
+### Documentation ###
+
+* Added curator role permissions example to code gallery
+
 ## [0.8.0] - 2022-07-12
 (Timm Fitschen)
 

RELEASE_GUIDELINES.md

@@ -40,8 +40,9 @@ guidelines of the CaosDB Project
 
 11. Merge the main branch back into the dev branch.
 
-12. After the merge of main to dev, start a new development version by
-    setting `ISRELEASED` to `False` and by increasing at least the `MICRO`
-    version in [setup.py](./setup.py).
-    Also update CHANGELOG.md (new "Unreleased" section).
-    Also update `src/doc/conf.py`.
+12. After the merge of main to dev, start a new development version by setting
+    `ISRELEASED` to `False` and by increasing at least the `MICRO` version in
+    [setup.py](./setup.py). Please note that due to a bug in pip, the `PRE`
+    version has to remain empty in the setup.py.  
+    Also update CHANGELOG.md (new "Unreleased" section). Also update
+    `src/doc/conf.py`.

setup.py

@@ -47,13 +47,13 @@ from setuptools import find_packages, setup
 
 ISRELEASED = False
 MAJOR = 0
-MINOR = 9
-MICRO = 0
+MINOR = 10
+MICRO = 1
 # Do not tag as pre-release until this commit
 # https://github.com/pypa/packaging/pull/515
 # has made it into a release. Probably we should wait for pypa/packaging>=21.4
 # https://github.com/pypa/packaging/releases
-PRE = "dev" # "dev"  # e.g. rc0, alpha.1, 0.beta-23
+PRE = "" # "dev"  # e.g. rc0, alpha.1, 0.beta-23
 
 if PRE:
     VERSION = "{}.{}.{}-{}".format(MAJOR, MINOR, MICRO, PRE)
@@ -97,6 +97,9 @@ def get_version_info():
 
     if os.path.exists('.git'):
         GIT_REVISION = git_version()
+    elif os.path.exists('caosdb_pylib_commit'):
+        with open('caosdb_pylib_commit', 'r') as f:
+            GIT_REVISION = f.read().strip()
     elif os.path.exists('src/caosdb/version.py'):
         # must be a source distribution, use existing version file
         try:
@@ -171,7 +174,11 @@ def setup_package():
         python_requires='>=3.8',
         package_dir={'': 'src'},
         install_requires=['lxml>=4.6.3',
-                          'PyYAML>=6.0', 'future', 'PySocks>=1.6.7'],
+                          "requests[socks]>=2.28.1",
+                          "python-dateutil>=2.8.2",
+                          'PyYAML>=5.4.1',
+                          'future',
+                         ],
         extras_require={'keyring': ['keyring>=13.0.0'],
                         'jsonschema': ['jsonschema>=4.4.0']},
         setup_requires=["pytest-runner>=2.0,<3dev"],

src/caosdb/apiutils.py

@@ -27,12 +27,13 @@
 Some simplified functions for generation of records etc.
 """
 
+import logging
 import sys
 import tempfile
 import warnings
+
 from collections.abc import Iterable
 from subprocess import call
-
 from typing import Optional, Any, Dict, List
 
 from caosdb.common.datatype import (BOOLEAN, DATETIME, DOUBLE, FILE, INTEGER,
@@ -40,8 +41,13 @@ from caosdb.common.datatype import (BOOLEAN, DATETIME, DOUBLE, FILE, INTEGER,
 from caosdb.common.models import (Container, Entity, File, Property, Query,
                                   Record, RecordType, execute_query,
                                   get_config, SPECIAL_ATTRIBUTES)
+from caosdb.exceptions import CaosDBException
 
-import logging
+
+class EntityMergeConflictError(CaosDBException):
+    """An error that is raised in case of an unresolvable conflict when merging
+    two entities.
+    """
 
 
 def new_record(record_type, name=None, description=None,
@@ -188,9 +194,8 @@ def getCommitIn(folder):
         return t.readline().strip()
 
 
-def compare_entities(old_entity: Entity, new_entity: Entity):
-    """
-    Compare two entites.
+def compare_entities(old_entity: Entity, new_entity: Entity, compare_referenced_records: bool = False):
+    """Compare two entites.
 
     Return a tuple of dictionaries, the first index belongs to additional information for old
     entity, the second index belongs to additional information for new entity.
@@ -204,6 +209,23 @@ def compare_entities(old_entity: Entity, new_entity: Entity):
         - ... value (not implemented yet)
         In case of changed information the value listed under the respective key shows the
         value that is stored in the respective entity.
+
+    If `compare_referenced_records` is `True`, also referenced entities will be
+    compared using this function (which is then called with
+    `compare_referenced_records = False` to prevent infinite recursion in case
+    of circular references).
+
+    Parameters
+    ----------
+    old_entity, new_entity : Entity
+        Entities to be compared
+    compare_referenced_records : bool, optional
+        Whether to compare referenced records in case of both, `old_entity` and
+        `new_entity`, have the same reference properties and both have a Record
+        object as value. If set to `False`, only the corresponding Python
+        objects are compared which may lead to unexpected behavior when
+        identical records are stored in different objects. Default is False.
+
     """
     olddiff: Dict[str, Any] = {"properties": {}, "parents": []}
     newdiff: Dict[str, Any] = {"properties": {}, "parents": []}
@@ -270,6 +292,26 @@ def compare_entities(old_entity: Entity, new_entity: Entity):
                     matching[0].unit
 
             if (prop.value != matching[0].value):
+                # basic comparison of value objects says they are different
+                same_value = False
+                if compare_referenced_records:
+                    # scalar reference
+                    if isinstance(prop.value, Entity) and isinstance(matching[0].value, Entity):
+                        # explicitely not recursive to prevent infinite recursion
+                        same_value = empty_diff(
+                            prop.value, matching[0].value, compare_referenced_records=False)
+                    # list of references
+                    elif isinstance(prop.value, list) and isinstance(matching[0].value, list):
+                        # all elements in both lists actually are entity objects
+                        # TODO: check, whether mixed cases can be allowed or should lead to an error
+                        if all([isinstance(x, Entity) for x in prop.value]) and all([isinstance(x, Entity) for x in matching[0].value]):
+                            # can't be the same if the lengths are different
+                            if len(prop.value) == len(matching[0].value):
+                                # do a one-by-one comparison; the values are the same, if all diffs are empty
+                                same_value = all(
+                                    [empty_diff(x, y, False) for x, y in zip(prop.value, matching[0].value)])
+
+                if not same_value:
                     olddiff["properties"][prop.name]["value"] = prop.value
                     newdiff["properties"][prop.name]["value"] = \
                         matching[0].value
@@ -300,27 +342,83 @@ def compare_entities(old_entity: Entity, new_entity: Entity):
     return (olddiff, newdiff)
 
 
-def merge_entities(entity_a: Entity, entity_b: Entity):
-    """
-    Merge entity_b into entity_a such that they have the same parents and properties.
+def empty_diff(old_entity: Entity, new_entity: Entity, compare_referenced_records: bool = False):
+    """Check whether the `compare_entities` found any differences between
+    old_entity and new_entity.
+
+    Parameters
+    ----------
+    old_entity, new_entity : Entity
+        Entities to be compared
+    compare_referenced_records : bool, optional
+        Whether to compare referenced records in case of both, `old_entity` and
+        `new_entity`, have the same reference properties and both have a Record
+        object as value.
 
-    datatype, unit, value, name and description will only be changed in entity_a if they
-    are None for entity_a and set for entity_b. If there is a corresponding value
-    for entity_a different from None a RuntimeError will be raised informing of an
-    unresolvable merge conflict.
+    """
+    olddiff, newdiff = compare_entities(
+        old_entity, new_entity, compare_referenced_records)
+    for diff in [olddiff, newdiff]:
+        for key in ["parents", "properties"]:
+            if len(diff[key]) > 0:
+                # There is a difference somewhere in the diff
+                return False
+        for key in SPECIAL_ATTRIBUTES:
+            if key in diff and diff[key]:
+                # There is a difference in at least one special attribute
+                return False
+    # all elements of the two diffs were empty
+    return True
+
+
+def merge_entities(entity_a: Entity, entity_b: Entity, merge_references_with_empty_diffs=True,
+                   force=False):
+    """Merge entity_b into entity_a such that they have the same parents and properties.
+
+    datatype, unit, value, name and description will only be changed in entity_a
+    if they are None for entity_a and set for entity_b. If there is a
+    corresponding value for entity_a different from None, an
+    EntityMergeConflictError will be raised to inform about an unresolvable merge
+    conflict.
 
     The merge operation is done in place.
 
     Returns entity_a.
 
     WARNING: This function is currently experimental and insufficiently tested. Use with care.
+
+    Parameters
+    ----------
+    entity_a, entity_b : Entity
+       The entities to be merged. entity_b will be merged into entity_a in place
+    merge_references_with_empty_diffs : bool, optional
+       Whether the merge is performed if entity_a and entity_b both reference
+       record(s) that may be different Python objects but have empty diffs. If
+       set to `False` a merge conflict will be raised in this case
+       instead. Default is True.
+    force : bool, optional
+       If True, in case `entity_a` and `entity_b` have the same properties, the
+       values of `entity_a` are replaced by those of `entity_b` in the merge.
+       If `False`, an EntityMergeConflictError is raised instead. Default is False.
+
+    Returns
+    -------
+    entity_a : Entity
+       The initial entity_a after the in-place merge
+
+    Raises
+    ------
+    EntityMergeConflictError
+        In case of an unresolvable merge conflict.
+
     """
 
     logging.warning(
         "This function is currently experimental and insufficiently tested. Use with care.")
 
     # Compare both entities:
-    diff_r1, diff_r2 = compare_entities(entity_a, entity_b)
+    diff_r1, diff_r2 = compare_entities(
+        entity_a, entity_b, compare_referenced_records=merge_references_with_empty_diffs)
 
     # Go through the comparison and try to apply changes to entity_a:
     for key in diff_r2["parents"]:
@@ -338,11 +436,22 @@ def merge_entities(entity_a: Entity, entity_b: Entity):
                 raise NotImplementedError()
 
             for attribute in ("datatype", "unit", "value"):
-                if diff_r1["properties"][key][attribute] is None:
+                if (attribute in diff_r2["properties"][key] and
+                        diff_r2["properties"][key][attribute] is not None):
+                    if (diff_r1["properties"][key][attribute] is None):
+                        setattr(entity_a.get_property(key), attribute,
+                                diff_r2["properties"][key][attribute])
+                    elif force:
                         setattr(entity_a.get_property(key), attribute,
                                 diff_r2["properties"][key][attribute])
                     else:
-                    raise RuntimeError("Merge conflict.")
+                        raise EntityMergeConflictError(
+                            f"Entity a ({entity_a.id}, {entity_a.name}) "
+                            f"has a Property '{key}' with {attribute}="
+                            f"{diff_r2['properties'][key][attribute]}\n"
+                            f"Entity b ({entity_b.id}, {entity_b.name}) "
+                            f"has a Property '{key}' with {attribute}="
+                            f"{diff_r1['properties'][key][attribute]}")
         else:
             # TODO: This is a temporary FIX for
             #       https://gitlab.indiscale.com/caosdb/src/caosdb-pylib/-/issues/105
@@ -362,8 +471,13 @@ def merge_entities(entity_a: Entity, entity_b: Entity):
         if sa_a != sa_b:
             if sa_a is None:
                 setattr(entity_a, special_attribute, sa_b)
+            elif force:
+                # force overwrite
+                setattr(entity_a, special_attribute, sa_b)
             else:
-                raise RuntimeError("Merge conflict.")
+                raise EntityMergeConflictError(
+                    f"Conflict in special attribute {special_attribute}:\n"
+                    f"A: {sa_a}\nB: {sa_b}")
     return entity_a
 
 

src/caosdb/cert/indiscale.ca.crt

------BEGIN CERTIFICATE-----
-MIIJ6TCCBdGgAwIBAgIIFVYzdrEDk6EwDQYJKoZIhvcNAQENBQAwgZMxCzAJBgNV
-BAYTAkRFMRUwEwYDVQQIEwxMb3dlciBTYXhvbnkxEzARBgNVBAcTCkdvZXR0aW5n
-ZW4xFzAVBgNVBAoTDkluZGlTY2FsZSBHbWJIMRwwGgYDVQQDExNJbmRpU2NhbGUg
-Um9vdCBDQSAxMSEwHwYJKoZIhvcNAQkBFhJpbmZvQGluZGlzY2FsZS5jb20wHhcN
-MTkwODA3MDAwMDAwWhcNMzQwODA2MjM1OTU5WjCBkzELMAkGA1UEBhMCREUxFTAT
-BgNVBAgTDExvd2VyIFNheG9ueTETMBEGA1UEBxMKR29ldHRpbmdlbjEXMBUGA1UE
-ChMOSW5kaVNjYWxlIEdtYkgxHDAaBgNVBAMTE0luZGlTY2FsZSBSb290IENBIDEx
-ITAfBgkqhkiG9w0BCQEWEmluZm9AaW5kaXNjYWxlLmNvbTCCBCIwDQYJKoZIhvcN
-AQEBBQADggQPADCCBAoCggQBAKxJO3XOqrUxFU3qdVyk9tmZEHwhwntcLO+kRR5t
-64/1Z/+VIPSgVN5phkSCukj2BPJITWKplWzJDAYWSvA/7cqavCtx8yP+m3AHWrRa
-CeHbtkGZ1nzwyFel3GIr93e65REeWqBE3knzem+qxTlZ2hp8/w3oxUlhy7tGxjBs
-JlekgLRDrnj4Opyb4GVjcVfcELmu3sLrrPX1wdYJrqaMQUR4BKZnbXxKdOYyX+kR
-/W2P4sihCCJh7Wy29VXHwSSCM1qEkU3REjvPEmEElCG7UpqOfg+3jaNZDqnvfskf
-okU4GuFCxSWQituyP9jm/hFVEhz59tUMYCllcjEi2jGmD2DBKpiru4t4/z0Aymf4
-Pep9hNtH1yhZMxpQeCYK9ESEE5d7do0bu/4YFp7jAg5vWZ8KlILZakmypVBFUw8I
-U/QJoJ55j95vIp+kjFdXelIVcr5La/zOR82JldaoPfyoBKObzwpwqaWQwYm8pj4p
-XkUdJTf8rpW21SSGWZm8JoFSYDfGvI61rPEjl/ohKhlG0tV6E2tCc406HNo/7pPe
-pmx/v9ZWLbYDAH7MVMB4tv6zDRE/c4KTbh5/s70VbXbAeOG6DNwegdDLDYZOv6Yw
-YQMz9NWtKGzvoFehP2vY5nGK95JVUcd90jaNaoURLB102VtxAjPIEQA1PjbQxLvC
-7A6kshlpQiN7zS/R9IgiEkYP/9gjy6mMuQVxH7C+9cqmCnXvVmpHmxXGUqk61r/B
-h12htsx5qjbbkToZYhUXBmwRq4LDtyoxNeaF2Jc+gE762obbHsSYMuSuh0kTFUUd
-uqfrI8OyzX4r1w5dYf2FEetZTT2Obyxb3Cy0btJF5+zEerBX44RulkdC+TPTMhJw
-b1jrPCACKywy9b6vJcSQ2V1+uLk7rH2JKD+fQRIKUqZZkhNKFYz5dnYYTgS45M0/
-C+vIvRnhgNSNb4efG6wyFvWEF8poDSPnJ4mM+0jHG/+cLqF/M2CMFvC+yU8Hj9YH
-B+H2L6V1QlCkpw5Ai4ji6OaQmnrsjE8EJj58vwYKsjmLGuf4j5AivkQTxfgCPGrT
-6CxSesoFmYDPSg/2eO+IfYEwnd7Rbs4aAhW8eo+lGpmK0DQxNjlejYt/Cgp7HWCq
-m/VNqWPIDMSTTqyk1GTmp67NjEZKt2ukJxI2CpL8s/9x4f3GTjNyI750pKM/uzMk
-OBKTMuWJQ6xeMR3h9RQlqlmwcErLXoUGInOTHHjRGXDI+ZBeLqT5DikcFiwbHG3+
-6FOuxXO0eqqg2tBW8cQ5kuRI0YFznipDUcfgDZt0JEkEXmRuL0nxYO35WKKdpGcF
-xFRJtO4FRB4nVWekVRuK9m47IPm6vC4eo+pCNPPoQ+FjyQ8CAwEAAaM/MD0wDAYD
-VR0TBAUwAwEB/zAdBgNVHQ4EFgQUFjE2TLaKASKEJ0LKOO+37/Hu7qowDgYDVR0P
-AQH/BAQDAgEGMA0GCSqGSIb3DQEBDQUAA4IEAQB2j1GL1G0ferWp9wmuDdF2oumn
-k+JIctRaGHaSrqHy4zjwj3Oqm2JA1ds+WfWozz+d38ZcLqSHo+g9ku5h/XOogQEU
-O4/y7j44pxIUg0EcIpMHtf7KPocPfvi9lw/4vE/3V/WKh4E09SXWgyY5tMUlEMaB
-6t8n7gg943HY2MJE1QU2wOPMXB1krFbunYxJcrUMs21w9jRWVv/wvaj6rkdvvSbU
-Yre11J+VlhC6fxx+STohQopzE6jnsaHile56b9xAmCCKcPEpWeKKBFS7pVNHEIHF
-uHWpgVjhoheEMMbYgu6l5E5K32TNYCKU49jNRWEKETjmYQSNl9dsSip+XlvaU8wQ
-VRR8UMHZPiJDW/AAHCr+bXEarZ9mSj/y+R512YtVw95zCnGUtzOJViThoIk/IAOR
-AJdnvsFmZSIKtFHpSEFYlTDq2yr1ulzbaDhuPRzita8b0cP27UvqRebZw5CvHN48
-B9a9tTYowKuJqmtjE6D00QA4xS8fRizLnx54uNmDbwf/8WavVk6MzDERwRE3OsSy
-D0dV6gy3t2AqEpVBrICrFqvgAQa4fcFcIwz3Qbt5o5uEi7acRomY57YrxrlfNTwh
-2oDQz+HQ/ZTDwZ3DrIgel7GrQ5fXrXDLL3ebtsbuIeBx8crOWQask832HcLtDVpu
-E/FdJEMMjglzIcy2dHpuODIGFmgEVfHR4DOOSBl0hfNdlrYnhC0h8/6QFswtlYFF
-8aQbGX7inK8L2in5wQ7ypeoMuXkQVYxlU1TEGmgB8aDke47MuX1FH+clsCaZ3s1E
-ka6lV6cjNYcosS718B6b2JgDUzmGBn2Sdm1xFmJM16dXp7TSmC5/fYxXuE/CynDs
-PmaUb9Ms6XUYSwKKhZ5HZdeRoNz8w62WNAeF7o7iX6IVrd/G1bJnSBN01istckyR
-BDuIkaoBQ9yvHN6Bo/J3KR08ixF1dHFPo/oSgkBxkLakb/yeslBTP/oISiFeQ4+q
-Gld1mhAvmG99dVZfoysrMjZSyghNbqwScjbYYN115lExV5ZeRtSwA7JCYE2lBjmB
-vocmz/hh/ifbmmqIvSv0NtiBnM6mNqngZEWD/rAloVOQoq0KVJJ5lUCQrBSFtR4+
-G1JGMX6b7uRp4mfdqqDE62KxxfkWBUwzUTIKGb5K42ji1Gy5li/TIWJtLNGNNQ2A
-0ui2RhwioaGGfYyomSFuAo5IPE/NF0ASjrTDW6GoNxypTSYE4/7oSoxeryafVnqN
-S0fRyrgSLiuT5tAiZ3b5Q3EFYUM2OcU3ezr/ZUabf9qIsqOnCi91SqE88BQbenot
-0HyUMdp/7QX9SyWM/azhcRiReAtkmq9pgeQA2TTZADDNTkKRljG9VeFDSwl7
------END CERTIFICATE-----

src/caosdb/common/administration.py

@@ -76,7 +76,7 @@ def get_server_properties():
     con = get_connection()
     try:
         body = con._http_request(
-            method="GET", path="_server_properties").response
+            method="GET", path="_server_properties")
     except EntityDoesNotExistError:
         raise ServerConfigurationException(
             "Debug mode in server is probably disabled.") from None

src/caosdb/common/models.py

@@ -51,6 +51,7 @@ from caosdb.common.datatype import (BOOLEAN, DATETIME, DOUBLE, INTEGER, TEXT,
                                     is_list_datatype, is_reference)
 from caosdb.common.state import State
 from caosdb.common.utils import uuid, xml2str
+from caosdb.common.timezone import TimeZone
 from caosdb.common.versioning import Version
 from caosdb.configuration import get_config
 from caosdb.connection.connection import get_connection
@@ -4327,6 +4328,8 @@ class Info():
 
             if isinstance(m, UserInfo):
                 self.user_info = m
+            elif isinstance(m, TimeZone):
+                self.time_zone = m
             else:
                 self.messages.append(m)
 
@@ -4460,6 +4463,9 @@ def _parse_single_xml_element(elem):
         return Permissions(xml=elem)
     elif elem.tag == "UserInfo":
         return UserInfo(xml=elem)
+    elif elem.tag == "TimeZone":
+        return TimeZone(zone_id=elem.get("id"), offset=elem.get("offset"),
+                        display_name=elem.text.strip())
     else:
         return Message(type=elem.tag, code=elem.get(
             "code"), description=elem.get("description"), body=elem.text)

src/caosdb/common/timezone.py

+class TimeZone():
+    """
+    TimeZone, e.g. CEST, Europe/Berlin, UTC+4.
+
+
+    Attributes
+    ----------
+    zone_id : string
+        ID of the time zone.
+    offset : int
+        Offset to UTC in seconds.
+    display_name : string
+        A human-friendly name of the time zone:
+    """
+
+    def __init__(self, zone_id, offset, display_name):
+        self.zone_id = zone_id
+        self.offset = offset
+        self.display_name = display_name

src/caosdb/configuration.py

@@ -31,11 +31,6 @@ try:
 except ImportError:
     pass
 
-try:
-    # python2
-    from ConfigParser import ConfigParser
-except ImportError:
-    # python3
 from configparser import ConfigParser
 
 from os import environ, getcwd
@@ -59,6 +54,11 @@ def configure(inifile):
         _reset_config()
     read_config = _pycaosdbconf.read(inifile)
     validate_yaml_schema(config_to_yaml(_pycaosdbconf))
+
+    if "HTTPS_PROXY" in environ:
+        _pycaosdbconf["Connection"]["https_proxy"] = environ["HTTPS_PROXY"]
+    if "HTTP_PROXY" in environ:
+        _pycaosdbconf["Connection"]["http_proxy"] = environ["HTTP_PROXY"]
     return read_config
 
 

src/caosdb/connection/connection.py

@@ -28,9 +28,15 @@ from __future__ import absolute_import, print_function, unicode_literals
 import logging
 import ssl
 import sys
+import warnings
 from builtins import str  # pylint: disable=redefined-builtin
 from errno import EPIPE as BrokenPipe
 from socket import error as SocketError
+from urllib.parse import quote, urlparse
+from requests import Session as HTTPSession
+from requests.exceptions import ConnectionError as HTTPConnectionError
+from urllib3.poolmanager import PoolManager
+from requests.adapters import HTTPAdapter
 
 from caosdb.configuration import get_config
 from caosdb.exceptions import (CaosDBException, HTTPClientError,
@@ -49,16 +55,8 @@ except ModuleNotFoundError:
 from pkg_resources import resource_filename
 
 from .interface import CaosDBHTTPResponse, CaosDBServerConnection
-from .streaminghttp import StreamingHTTPSConnection
 from .utils import make_uri_path, parse_url, urlencode
-
-try:
-    from urllib.parse import quote, urlparse
-except ImportError:
-    from urllib import quote
-    from urlparse import urlparse
-
-# pylint: disable=missing-docstring
+from .encode import MultipartYielder, ReadableMultiparts
 
 _LOGGER = logging.getLogger(__name__)
 
@@ -67,6 +65,9 @@ class _WrappedHTTPResponse(CaosDBHTTPResponse):
 
     def __init__(self, response):
         self.response = response
+        self._generator = None
+        self._buffer = b''
+        self._stream_consumed = False
 
     @property
     def reason(self):
@@ -74,21 +75,71 @@ class _WrappedHTTPResponse(CaosDBHTTPResponse):
 
     @property
     def status(self):
-        return self.response.status
+        return self.response.status_code
 
     def read(self, size=None):
-        return self.response.read(size)
+        if self._stream_consumed is True:
+            raise RuntimeError("Stream is consumed")
+
+        if self._buffer is None:
+            # the buffer has been drained in the previous call.
+            self._stream_consumed = True
+            return b''
+
+        if self._generator is None and (size is None or size == 0):
+            # return full content at once
+            self._stream_consumed = True
+            return self.response.content
+
+        if len(self._buffer) >= size:
+            # still enough bytes in the buffer
+            result = chunk[:size]
+            self._buffer = chunk[size:]
+            return result
+
+        if self._generator is None:
+            # first call to this method
+            if size is None or size == 0:
+                size = 512
+            self._generator = self.response.iter_content(size)
+
+        try:
+            # read new data into the buffer
+            chunk = self._buffer + next(self._generator)
+            result = chunk[:size]
+            if len(result) == 0:
+                self._stream_consumed = True
+            self._buffer = chunk[size:]
+            return result
+        except StopIteration:
+            # drain buffer
+            result = self._buffer
+            self._buffer = None
+            return result
 
     def getheader(self, name, default=None):
-        return self.response.getheader(name=name, default=default)
+        return self.response.headers[name] if name in self.response.headers else default
 
     def getheaders(self):
-        return self.response.getheaders()
+        return self.response.headers.items()
 
     def close(self):
         self.response.close()
 
 
+class _SSLAdapter(HTTPAdapter):
+    """Transport adapter that allows us to use different SSL versions."""
+
+    def __init__(self, ssl_version):
+        self.ssl_version = ssl_version
+        super().__init__()
+
+    def init_poolmanager(self, connections, maxsize, block=False):
+        self.poolmanager = PoolManager(
+            num_pools=connections, maxsize=maxsize,
+            block=block, ssl_version=self.ssl_version)
+
+
 class _DefaultCaosDBServerConnection(CaosDBServerConnection):
     """_DefaultCaosDBServerConnection.
 
@@ -101,10 +152,11 @@ class _DefaultCaosDBServerConnection(CaosDBServerConnection):
     def __init__(self):
         self._useragent = ("caosdb-pylib/{version} - {implementation}".format(
             version=version, implementation=type(self).__name__))
-        self._http_con = None
         self._base_path = None
+        self._session = None
+        self._timeout = None
 
-    def request(self, method, path, headers=None, body=None, **kwargs):
+    def request(self, method, path, headers=None, body=None):
         """request.
 
         Send a HTTP request to the server.
@@ -118,38 +170,40 @@ class _DefaultCaosDBServerConnection(CaosDBServerConnection):
             including query and frament segments.
         headers : dict of str -> str, optional
             HTTP request headers. (Defautl: None)
-        body : str or bytes or readable, opional
+        body : str or bytes or readable, optional
             The body of the HTTP request. Bytes should be a utf-8 encoded
             string.
-        **kwargs :
-            Any keyword arguments will be ignored.
-            TODO: Why are they allowed then?
 
         Returns
         -------
-            TODO: What?
+        response : CaosDBHTTPResponse
         """
 
         if headers is None:
             headers = {}
         headers["User-Agent"] = self._useragent
+
+        if path.endswith("/."):
+            path = path[:-1] + "%2E"
+
+        if isinstance(body, MultipartYielder):
+            body = ReadableMultiparts(body)
+
         try:
-            self._http_con = StreamingHTTPSConnection(
-                # TODO looks as if configure needs to be done first.
-                # That is however not assured.
-                host=self.setup_fields["host"],
-                timeout=self.setup_fields["timeout"],
-                context=self.setup_fields["context"],
-                socket_proxy=self.setup_fields["socket_proxy"])
-            self._http_con.request(method=method, url=self._base_path + path,
-                                   headers=headers, body=body)
-        except SocketError as socket_err:
+            response = self._session.request(
+                method=method,
+                url=self._base_path + path,
+                headers=headers,
+                data=body,
+                timeout=self._timeout,
+                stream=True)
+
+            return _WrappedHTTPResponse(response)
+        except HTTPConnectionError as conn_err:
             raise CaosDBConnectionError(
-                "Connection failed. Network or server down? " + str(socket_err)
+                "Connection failed. Network or server down? " + str(conn_err)
             )
 
-        return _WrappedHTTPResponse(self._http_con.getresponse())
-
     def configure(self, **config):
         """configure.
 
@@ -173,55 +227,69 @@ class _DefaultCaosDBServerConnection(CaosDBServerConnection):
             loaded.
         """
 
+        if "url" not in config:
+            raise CaosDBConnectionError(
+                "No connection url specified. Please "
+                "do so via caosdb.configure_connection(...) or in a config "
+                "file.")
+        if (not config["url"].lower().startswith("https://") and not config["url"].lower().startswith("http://")):
+            raise CaosDBConnectionError("The connection url is expected "
+                                        "to be a http or https url and "
+                                        "must include the url scheme "
+                                        "(i.e. start with https:// or "
+                                        "http://).")
+
+        url = urlparse(config["url"])
+        path = url.path.strip("/")
+        if len(path) > 0:
+            path = path + "/"
+        self._base_path = url.scheme + "://" + url.netloc + "/" + path
+
+        self._session = HTTPSession()
+
+        if url.scheme == "https":
+            self._setup_ssl(config)
+
+        # TODO(tf) remove in next release
+        socket_proxy = config["socket_proxy"] if "socket_proxy" in config else None
+        if socket_proxy is not None:
+            self._session.proxies = {
+                "https": "socks5://" + socket_proxy,
+                "http": "socks5://" + socket_proxy,
+            }
+
+        if "https_proxy" in config:
+            if self._session.proxies is None:
+                self._session.proxies = {}
+            self._session.proxies["https"] = config["https_proxy"]
+
+        if "http_proxy" in config:
+            if self._session.proxies is None:
+                self._session.proxies = {}
+            self._session.proxies["http"] = config["http_proxy"]
+
+        if "timeout" in config:
+            self._timeout = config["timeout"]
+
+    def _setup_ssl(self, config):
         if "ssl_version" in config and config["cacert"] is not None:
             ssl_version = getattr(ssl, config["ssl_version"])
         else:
             ssl_version = ssl.PROTOCOL_TLS
-        context = ssl.SSLContext(ssl_version)
-        context.verify_mode = ssl.CERT_REQUIRED
 
-        if config.get("ssl_insecure"):
+        self._session.mount(self._base_path, _SSLAdapter(ssl_version))
+
+        verify = True
+        if "cacert" in config:
+            verify = config["cacert"]
+        if "ssl_insecure" in config and config["ssl_insecure"]:
             _LOGGER.warning("*** Warning! ***\n"
                             "Insecure SSL mode, certificate will not be checked! "
                             "Please consider removing the `ssl_insecure` configuration option.\n"
                             "****************")
-            context.verify_mode = ssl.CERT_NONE
-
-        if (not context.verify_mode == ssl.CERT_NONE and
-                hasattr(context, "check_hostname")):
-            context.check_hostname = True
-
-        if ("cacert" in config and config["cacert"] is not None and
-                config["cacert"]):
-            try:
-                context.load_verify_locations(config["cacert"])
-            except Exception as exc:
-                raise CaosDBConnectionError("Could not load the cacert in"
-                                            "`{}`: {}".format(config["cacert"],
-                                                              exc))
-
-        context.load_default_certs()
-
-        if "url" in config:
-            parsed_url = parse_url(config["url"])
-            host = parsed_url.netloc
-            self._base_path = parsed_url.path
-        else:
-            raise CaosDBConnectionError(
-                "No connection url specified. Please "
-                "do so via caosdb.configure_connection(...) or in a config "
-                "file.")
-
-        socket_proxy = None
-
-        if "socket_proxy" in config:
-            socket_proxy = config["socket_proxy"]
-
-        self.setup_fields = {
-            "host": host,
-            "timeout": int(config.get("timeout")),
-            "context": context,
-            "socket_proxy": socket_proxy}
+            verify = False
+        if verify is not None:
+            self._session.verify = verify
 
 
 def _make_conf(*conf):
@@ -252,7 +320,6 @@ _DEFAULT_CONF = {
     "password_method": "input",
     "implementation": _DefaultCaosDBServerConnection,
     "timeout": 210,
-    "cacert": resource_filename("caosdb", 'cert/indiscale.ca.crt')
 }
 
 
@@ -314,6 +381,10 @@ def configure_connection(**kwargs):
 
     Parameters
     ----------
+    url : str
+        The url of the CaosDB Server. HTTP and HTTPS urls are allowed. However,
+        it is **highly** recommend to avoid HTTP because passwords and
+        authentication token are send over the network in plain text.
 
     username : str
         Username for login; e.g. 'admin'.
@@ -342,6 +413,24 @@ def configure_connection(**kwargs):
         An authentication token which has been issued by the CaosDB Server.
         Implies `password_method="auth_token"` if set.  An example token string would be `["O","OneTimeAuthenticationToken","anonymous",["administration"],[],1592995200000,604800000,"3ZZ4WKRB-5I7DG2Q6-ZZE6T64P-VQ","197d0d081615c52dc18fb323c300d7be077beaad4020773bb58920b55023fa6ee49355e35754a4277b9ac525c882bcd3a22e7227ba36dfcbbdbf8f15f19d1ee9",1,30000]`.
 
+    https_proxy : str, optional
+        Define a proxy for the https connections, e.g. `http://localhost:8888`,
+        `socks5://localhost:8888`, or `socks4://localhost:8888`. These are
+        either (non-TLS) HTTP proxies, SOCKS4 proxies, or SOCKS5 proxies. HTTPS
+        proxies are not supported. However, the connection will be secured
+        using TLS in the tunneled connection nonetheless. Only the connection
+        to the proxy is insecure which is why it is not recommended to use HTTP
+        proxies when authentication against the proxy is necessary. If
+        unspecified, the https_proxy option of the pycaosdb.ini or the HTTPS_PROXY
+        environment variable are being used. Use `None` to override these
+        options with a no-proxy setting.
+
+    http_proxy : str, optional
+        Define a proxy for the http connections, e.g. `http://localhost:8888`.
+        If unspecified, the http_proxy option of the pycaosdb.ini or the
+        HTTP_PROXY environment variable are being used. Use `None` to override
+        these options with a no-proxy setting.
+
     implementation : CaosDBServerConnection
         The class which implements the connection. (Default:
         _DefaultCaosDBServerConnection)
@@ -372,6 +461,11 @@ def configure_connection(**kwargs):
     local_conf = _make_conf(_DEFAULT_CONF, global_conf, kwargs)
 
     connection = _Connection.get_instance()
+
+    if "socket_proxy" in local_conf:
+        warnings.warn("Deprecated configuration option: socket_proxy. Use "
+                      "the new https_proxy option instead",
+                      DeprecationWarning, stacklevel=1)
     connection.configure(**local_conf)
 
     return connection
@@ -599,7 +693,7 @@ class _Connection(object):  # pylint: disable=useless-object-inheritance
             method=method,
             path=path,
             headers=headers,
-            body=body, **kwargs)
+            body=body)
         _LOGGER.debug("response: %s %s", str(http_response.status),
                       str(http_response.getheaders()))
         self._authenticator.on_response(http_response)

src/caosdb/connection/encode.py

@@ -51,7 +51,8 @@ multipart/form-data is the standard way to upload files over HTTP
 
 __all__ = [
     'gen_boundary', 'encode_and_quote', 'MultipartParam', 'encode_string',
-    'encode_file_header', 'get_body_size', 'get_headers', 'multipart_encode'
+    'encode_file_header', 'get_body_size', 'get_headers', 'multipart_encode',
+    'ReadableMultiparts',
 ]
 from urllib.parse import quote_plus
 from io import UnsupportedOperation
@@ -475,3 +476,40 @@ def multipart_encode(params, boundary=None, callback=None):
     params = MultipartParam.from_params(params)
 
     return MultipartYielder(params, boundary, callback), headers
+
+
+class ReadableMultiparts(object):
+    """Wraps instances of the MultipartYielder class as a readable and withable
+    object."""
+
+    def __init__(self, multipart_yielder):
+        self.multipart_yielder = multipart_yielder
+        self.current_block = None
+        self.left_over = b''
+
+    def read(self, size=-1):
+        result = self.left_over
+        while size == -1 or len(result) < size:
+            try:
+                next_chunk = self.multipart_yielder.next()
+                if hasattr(next_chunk, "encode"):
+                    next_chunk = next_chunk.encode("utf8")
+                result += next_chunk
+            except StopIteration:
+                break
+
+        if size == -1:
+            self.left_over = b''
+            return result
+
+        self.left_over = result[size:]
+        return result[:size]
+
+    def __enter__(self):
+        pass
+
+    def __exit__(self, type, value, traceback):
+        self.close()
+
+    def close(self):
+        self.multipart_yielder.reset()

src/caosdb/connection/streaminghttp.py

-# -*- encoding: utf-8 -*-
-#
-# ** header v3.0
-# This file is a part of the CaosDB Project.
-#
-# Copyright (C) 2018 Research Group Biomedical Physics,
-# Max-Planck-Institute for Dynamics and Self-Organization Göttingen
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Affero General Public License as
-# published by the Free Software Foundation, either version 3 of the
-# License, or (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU Affero General Public License for more details.
-#
-# You should have received a copy of the GNU Affero General Public License
-# along with this program. If not, see <https://www.gnu.org/licenses/>.
-#
-# ** end header
-#
-# Original work Copyright (c) 2011 Chris AtLee
-# Modified work Copyright (c) 2017 Biomedical Physics, MPI for Dynamics and Self-Organization
-#
-# Permission is hereby granted, free of charge, to any person obtaining a copy
-# of this software and associated documentation files (the "Software"), to deal
-# in the Software without restriction, including without limitation the rights
-# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-# copies of the Software, and to permit persons to whom the Software is
-# furnished to do so, subject to the following conditions:
-#
-# The above copyright notice and this permission notice shall be included in
-# all copies or substantial portions of the Software.
-#
-# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
-# THE SOFTWARE.
-"""Streaming HTTP uploads module.
-
-This module extends the standard httplib and http.client HTTPConnection so that
-iterable objects can be used in the body of HTTP requests.
-
-**N.B.** You must specify a Content-Length header if using an iterable object
-since there is no way to determine in advance the total size that will be
-yielded, and there is no way to reset an interator.
-"""
-
-from __future__ import unicode_literals, print_function, absolute_import
-import socks
-import socket
-try:
-    # python3
-    from http import client as client
-except ImportError:
-    # python2
-    import httplib as client
-
-
-__all__ = ['StreamingHTTPSConnection']
-
-
-class StreamingHTTPSConnection(client.HTTPSConnection, object):
-    """Subclass of `http.client.HTTSConnection` or `httplib.HTTPSConnection`
-    that overrides the `send()` method to support iterable body objects."""
-    # pylint: disable=unused-argument, arguments-differ
-
-    def __init__(self, socket_proxy=None, **kwargs):
-        if socket_proxy is not None:
-            host, port = socket_proxy.split(":")
-            socks.setdefaultproxy(socks.PROXY_TYPE_SOCKS5, host,
-                                  int(port))
-            socket.socket = socks.socksocket
-        super(StreamingHTTPSConnection, self).__init__(**kwargs)
-
-    def _send_output(self, body, **kwargs):
-        """Send the currently buffered request and clear the buffer.
-
-        Appends an extra \\r\\n to the buffer.
-        A message_body may be specified, to be appended to the request.
-
-        This method is implemented in differently in the various python
-        versions (which is extremely annoying). So we provide a unified but
-        relatively dumb implementaion which only serves our needs.
-        """
-        self._buffer.extend(("".encode("utf-8"), "".encode("utf-8")))
-        headers = "\r\n".encode("utf-8").join(self._buffer)
-        del self._buffer[:]
-
-        self.send(headers)
-        if body is not None:
-            self.send(body)
-
-    # pylint: disable=too-many-branches
-
-    def send(self, value):
-        """Send ``value`` to the server.
-
-        ``value`` can be a string-like object which supports a 'encode' method,
-        a file-like object that supports a .read() method, or an iterable object
-        that supports a .next() method.
-
-        An encode()able ``value`` will be utf-8 encoded before sending.
-        """
-        # Based on python 2.6's httplib.HTTPConnection.send()
-        if self.sock is None:
-            if self.auto_open:
-                self.connect()
-            else:
-                raise client.NotConnected()
-
-        # send the data to the server. if we get a broken pipe, then close
-        # the socket. we want to reconnect when somebody tries to send again.
-        #
-        # NOTE: we DO propagate the error, though, because we cannot simply
-        #       ignore the error... the caller will know if they can retry.
-        if self.debuglevel > 0:
-            print("send: ", repr(value))
-        try:
-            blocksize = 8192
-            if hasattr(value, 'read'):
-                if hasattr(value, 'seek'):
-                    value.seek(0)
-                if self.debuglevel > 0:
-                    print("sendIng a read()able")
-                data = value.read(blocksize)
-                while data:
-                    self.sock.sendall(data)
-                    data = value.read(blocksize)
-            elif hasattr(value, 'next'):
-                if hasattr(value, 'reset'):
-                    value.reset()
-                if self.debuglevel > 0:
-                    print("sendIng an iterable")
-                for data in value:
-                    if hasattr(data, "encode"):
-                        self.sock.sendall(data.encode('utf-8'))
-                    else:
-                        self.sock.sendall(data)
-            else:
-                if self.debuglevel > 0:
-                    print("sendIng a byte-like")
-                self.sock.sendall(value)
-        except socket.error as err:
-            if err.args[0] == 32:      # Broken pipe
-                self.close()
-            raise

src/caosdb/schema-pycaosdb-ini.yml

@@ -14,10 +14,10 @@ schema-pycaosdb-ini:
       additionalProperties: false
       properties:
         url:
-          description: URL of the CaosDB server
+          description: "URL of the CaosDB server. Allowed are HTTP and HTTPS connections. However, since authentication tokens and sometimes even passwords are send in plain text to the server it is **highly** recommended to use HTTPS connections whenever possible. HTTP is ok for testing and debugging."
           type: string
-          pattern: https://[-a-zA-Z0-9\.]+(:[0-9]+)?(/)?
-          examples: ["https://demo.indiscale.com/", "https://localhost:10443/"]
+          pattern: http(s)?://[-a-zA-Z0-9\.]+(:[0-9]+)?(/)?
+          examples: ["https://demo.indiscale.com/", "http://localhost:10080/"]
         username:
           type: string
           description: User name used for authentication with the server
@@ -26,7 +26,7 @@ schema-pycaosdb-ini:
           description: The password input method defines how the password is supplied that is used for authentication with the server.
           type: string
           default: input
-          enum: [input, plain, pass, keyring]
+          enum: [input, unauthenticated, plain, pass, keyring]
         password_identifier:
           type: string
         password:
@@ -54,7 +54,15 @@ schema-pycaosdb-ini:
         socket_proxy:
           examples: ["localhost:12345"]
           type: string
-          description: You can define a socket proxy to be used. This is for the case that the server sits behind a firewall which is being tunnelled with a socket proxy (SOCKS4 or SOCKS5) (e.g. via ssh's -D option or a dedicated proxy server).
+          description: Deprecated. Please use https_proxy instead.
+        https_proxy:
+          examples: ["http://localhost:8888", "socks5://localhost:8888", "socks4://localhost:8888"]
+          type: string
+          description: "Define a proxy for the https connections. These are either (non-TLS) HTTP proxies, SOCKS4 proxies, or SOCKS5 proxies. HTTPS proxies are not supported. However, the connection will be secured using TLS in the tunneled connection nonetheless. Only the connection to the proxy is insecure which is why it is not recommended to use HTTP proxies when authentication against the proxy is necessary. Note: this option is overridden by the HTTPS_PROXY environment variable, if present."
+        http_proxy:
+          examples: ["http://localhost:8888", "socks5://localhost:8888", "socks4://localhost:8888"]
+          type: string
+          description: "Define a proxy for the http connections. These are either (non-TLS) HTTP proxies, SOCKS4 proxies, or SOCKS5 proxies. HTTPS proxies are not supported. Note: this option is overridden by the HTTP_PROXY environment variable, if present."
         implementation:
           description: This option is used internally and for testing. Do not override.
           examples: [_DefaultCaosDBServerConnection]

src/caosdb/utils/caosdb_admin.py

@@ -621,8 +621,8 @@ USAGE
 
     for action in ["grant", "deny", "revoke_denial", "revoke_grant"]:
         action_entity_permissions_parser = subparsers.add_parser(
-            "{}_entity_permissions".format(action),
-            help="{} entity permissions to a role.".format(action))
+            f"{action}_entity_permissions",
+            help=f"{action} entity permissions to one or more Entities.")
         action_entity_permissions_parser.set_defaults(
             call=do_action_entity_permissions, action=action)
         action_entity_permissions_parser.add_argument(dest="query", metavar="QUERY",

src/caosdb/utils/register_tests.py

+#!/usr/bin/env python
+# encoding: utf-8
+#
+# This file is a part of the CaosDB Project.
+#
+# Copyright (C) 2022 Alexander Schlemmer <alexander.schlemmer@ds.mpg.de>
+# Copyright (C) 2022 Timm Fitschen <t.fitschen@indiscale.com>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+import caosdb as db
+from caosdb import administration as admin
+
+"""
+This module implements a registration procedure for integration tests which
+need a running CaosDB instance.
+
+It ensures that tests do not accidentally overwrite data in real CaosDB
+instances, as it checks whether the running CaosDB instance is actually the
+correct one, that
+should be used for these tests.
+
+The test files have to define a global variable TEST_KEY which must be unique
+for each test using
+
+set_test_key("ABCDE")
+
+The test procedure (invoked by pytest) checks whether a registration
+information is stored in one of the server properties or otherwise
+- offers to register this test in the currently running database ONLY if this
+  is empty.
+- fails otherwise with a RuntimeError
+
+NOTE: you probably need to use pytest with the -s option to be able to
+      register the test interactively. Otherwise, the server property has to be
+      set before server start-up in the server.conf of the CaosDB server.
+
+This module is intended to be used with pytest.
+
+There is a pytest fixture "clear_database" that performs the above mentioned
+checks and clears the database in case of success.
+"""
+
+TEST_KEY = None
+
+
+def set_test_key(KEY):
+    global TEST_KEY
+    TEST_KEY = KEY
+
+
+def _register_test():
+    res = db.execute_query("COUNT Entity")
+    if not isinstance(res, int):
+        raise RuntimeError("Response from server for Info could not be interpreted.")
+    if res > 0:
+        raise RuntimeError("This instance of CaosDB contains entities already."
+                           "It must be empty in order to register a new test.")
+
+    print("Current host of CaosDB instance is: {}".format(
+        db.connection.connection.get_connection()._delegate_connection.setup_fields["host"]))
+    answer = input("This method will register your current test with key {} with the currently"
+                   " running instance of CaosDB. Do you want to continue (y/N)?".format(
+                       TEST_KEY))
+    if answer != "y":
+        raise RuntimeError("Test registration aborted by user.")
+
+    admin.set_server_property("_CAOSDB_INTEGRATION_TEST_SUITE_KEY",
+                              TEST_KEY)
+
+
+def _get_registered_test_key():
+    try:
+        return admin.get_server_property("_CAOSDB_INTEGRATION_TEST_SUITE_KEY")
+    except KeyError:
+        return None
+
+
+def _is_registered():
+    registered_test_key = _get_registered_test_key()
+    if not registered_test_key:
+        return False
+    elif registered_test_key == TEST_KEY:
+        return True
+    else:
+        raise RuntimeError("The database has been setup for a different test.")
+
+
+def _assure_test_is_registered():
+    global TEST_KEY
+    if TEST_KEY is None:
+        raise RuntimeError("TEST_KEY is not defined.")
+    if not _is_registered():
+        answer = input("Do you want to register this instance of CaosDB"
+                       " with the current test? Do you want to continue (y/N)?")
+        if answer == "y":
+            _register_test()
+            raise RuntimeError("Test has been registered. Please rerun tests.")
+        else:
+            raise RuntimeError("The database has not been setup for this test.")
+
+
+def _clear_database():
+    c = db.execute_query("FIND ENTITY WITH ID>99")
+    c.delete(raise_exception_on_error=False)
+    return None
+
+
+try:
+    import pytest
+
+    @pytest.fixture
+    def clear_database():
+        """Remove Records, RecordTypes, Properties, and Files ONLY IF the CaosDB
+        server the current connection points to was registered with the appropriate key.
+
+        PyTestInfo Records and the corresponding RecordType and Property are preserved.
+        """
+        _assure_test_is_registered()
+        yield _clear_database()  # called before the test function
+        _clear_database()  # called after the test function
+except ImportError:
+    raise Warning("""The register_tests module depends on pytest and is
+                  intended to be used in integration test suites for the
+                  caosdb-pylib library only.""")

src/caosdb/utils/server_side_scripting.py

@@ -30,7 +30,8 @@ from lxml import etree
 
 from caosdb.connection.connection import get_connection
 from caosdb.connection.utils import urlencode
-from caosdb.connection.encode import MultipartParam, multipart_encode
+from caosdb.connection.encode import (MultipartParam, multipart_encode,
+                                      ReadableMultiparts)
 
 
 def _make_params(pos_args, opts):
@@ -63,6 +64,7 @@ def _make_multipart_request(call, pos_args, opts, files):
                                               filename=filename))
 
     body, headers = multipart_encode(parts)
+    body = ReadableMultiparts(body)
     return body, headers
 
 

src/doc/administration.rst

@@ -5,10 +5,12 @@ The Python script ``caosdb_admin.py`` should be used for administrative tasks.
 Call ``caosdb_admin.py --help`` to see how to use it.
 
 The most common task is to create a new user (in the CaosDB realm) and set a 
-password for the user (note that a user typically needs to be activated)::
+password for the user (note that a user typically needs to be activated):
 
-     caosdb_admin.py create_user anna
-     caosdb_admin.py set_user_password anna
-     caosdb_admin.py add_user_roles anna administration
-     caosdb_admin.py activate_user anna
+.. code:: console
+
+   $ caosdb_admin.py create_user anna
+   $ caosdb_admin.py set_user_password anna
+   $ caosdb_admin.py add_user_roles anna administration
+   $ caosdb_admin.py activate_user anna