diff --git a/.gitignore b/.gitignore
index b522b1da9176e59756bffe89cd4eafe0d751a23c..55fb3f0d1bc6c101704557da8f35d6e784b5ea89 100644
--- a/.gitignore
+++ b/.gitignore
@@ -15,4 +15,5 @@ build/
src/caosdb/version.py
# documentation
-_apidoc
\ No newline at end of file
+_apidoc
+*~
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 8b15e764abae0844f7d39fc7b9a5098edbc47c3c..792ab22f1524f8d4dc1db90f2b65c7f8f28f90ed 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -54,14 +54,38 @@ pylint:
allow_failure: true
# run unit tests
-unittest:
+unittest_py3.8:
+ tags: [ docker ]
+ stage: test
+ needs: [ ]
+ image: python:3.8
+ script: &python_test_script
+ # Python docker has problems with tox and pip so use plain pytest here
+ - touch ~/.pycaosdb.ini
+ - pip install nose pytest pytest-cov python-dateutil jsonschema==4.0.1
+ - pip install .
+ - python -m pytest unittests
+
+# This needs to be changed once Python 3.9 isn't the standard Python in Debian
+# anymore.
+unittest_py3.9:
tags: [ docker ]
stage: test
needs: [ ]
script:
+ # verify that this actually is Python 3.9
+ - python3 -c "import sys; assert sys.version.startswith('3.9')"
- touch ~/.pycaosdb.ini
- make unittest
+
+unittest_py3.10:
+ tags: [ docker ]
+ stage: test
+ needs: [ ]
+ image: python:3.10
+ script: *python_test_script
+
# Trigger building of server image and integration tests
trigger_build:
stage: deploy
@@ -88,12 +112,12 @@ build-testenv:
stage: setup
only:
- schedules
- script:
+ script:
- cd unittests/docker
- docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY
# use here general latest or specific branch latest...
- docker pull $CI_REGISTRY_IMAGE|| true
- - docker build
+ - docker build
--pull
--build-arg COMMIT=$CI_COMMIT_SHORT_SHA
--cache-from $CI_REGISTRY_IMAGE
@@ -105,7 +129,7 @@ build-testenv:
pages_prepare: &pages_prepare
tags: [ cached-dind ]
stage: deploy
- needs: [ code_style, pylint, unittest ]
+ needs: [ code_style, pylint, unittest_py3.8, unittest_py3.9, unittest_py3.10 ]
only:
refs:
- /^release-.*$/i
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 1e6cfd27875571bf2acaff5673163c996182baa0..7f19eed9471823ee93356d230a10d7f14f2fb287 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -5,17 +5,21 @@ All notable changes to this project will be documented in this file.
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
-## [Unreleased]
+## [Unreleased] ##
### Added ###
* Directory (EXPERIMENTAL)
+* `apiutils.EntityMergeConflictError` class for unresesolvable merge conflicts
+ when merging two entities
### Changed ###
* It is not possible anymore to upload complete directories in one go by just
using a File entity with `file=/my/dir/`. The functionality of creating
directory entities is handled be the new Directory class.
+* `apiutils.merge_entities` now raises an `EntityMergeConflictError` in case of
+ unresolvable merge conflicts.
### Deprecated ###
@@ -31,6 +35,71 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
### Documentation ###
+* [Fixed](https://gitlab.com/caosdb/caosdb-pylib/-/issues/79) `{action}_entity_permissions` help line.
+
+## [0.10.0] - 2022-11-14
+(Florian Spreckelsen)
+
+### Added ###
+
+* HTTP connections are allowed additionally to HTTPS connections.
+* Dependency on the `requests` package.
+* Dependency on the `python-dateutil` package.
+* `Connection.https_proxy` and `Connection.http_proxy` option of the
+ pycaosdb.ini and the `https_proxy` and `http_proxy` parameter of the
+ `configure_connection` function. See the documentation of the
+ latter for more information.
+ Note that the `HTTP_PROXY` and `HTTPS_PROXY` environment variables are
+ respected as well, unless overridden programmatically.
+* `apiutils.empty_diff` function that returns `True` if the diffs of two
+ entities found with the `compare_entitis` function are empty, `False`
+ otherwise.
+
+### Changed ###
+
+* `apiutils.compare_entities` now has an optional `compare_referenced_records`
+ argument to compare referenced Entities recursively (fomerly, only the
+ referenced Python objects would be compared). The default is `False` to
+ recover the original behavior.
+* `apiutils.merge_entities` now has an optional
+ `merge_references_with_empty_diffs` argument that determines whether a merge
+ of two entities will be performed if they reference identical records (w.r.t
+ th above `empty_diff` function). Formerly this would have caused a merge
+ conflict if the referenced record(s) were identical, but stored in different
+ Python objects.
+* `apiutils.merge_entities` now has an optional `force` argument (defaults to
+ `False`, i.e., the old behavior) which determines whether in case of merge
+ conflicts errors will be raised or the properties and attributes of entity A
+ will be overwritten by entity B.
+
+### Deprecated ###
+
+* `Connection.socket_proxy` option of the pycaosdb.ini. Please use
+ `Connection.https_proxy` or `Connection.http_proxy` instead. The deprecated
+ option will be removed with the next minor release.
+
+### Fixed ###
+
+* handling of special attributes (name, id, ...) in `apiutils.empty_diff`
+
+## [0.9.0] - 2022-10-24
+(Florian Spreckelsen)
+
+### Added ###
+
+* Add TimeZone class and parse the server's time zone in the Info response.
+
+### Fixed ###
+
+* [#141](https://gitlab.indiscale.com/caosdb/src/caosdb-pylib/-/issues/141)
+ `password_method = unauthenticated` not allowed by schema
+* Set PyYAML dependency back to PyYaml>=5.4.1 (from 6.0) for better
+ compatibility with docker-compose
+
+### Documentation ###
+
+* Added curator role permissions example to code gallery
+
## [0.8.0] - 2022-07-12
(Timm Fitschen)
@@ -224,7 +293,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
### Removed ###
-* Dynamic exception type `EntityMultiError`.
+* Dynamic exception type `EntityMultiError`.
* `get_something` functions from all error object in `exceptions.py`
* `AmbiguityException`
diff --git a/RELEASE_GUIDELINES.md b/RELEASE_GUIDELINES.md
index 00d0362e6630267c135e2566b993a91fccf1fd91..863afb8f3ac7d6770c372620523638b900785227 100644
--- a/RELEASE_GUIDELINES.md
+++ b/RELEASE_GUIDELINES.md
@@ -40,8 +40,9 @@ guidelines of the CaosDB Project
11. Merge the main branch back into the dev branch.
-12. After the merge of main to dev, start a new development version by
- setting `ISRELEASED` to `False` and by increasing at least the `MICRO`
- version in [setup.py](./setup.py).
- Also update CHANGELOG.md (new "Unreleased" section).
- Also update `src/doc/conf.py`.
+12. After the merge of main to dev, start a new development version by setting
+ `ISRELEASED` to `False` and by increasing at least the `MICRO` version in
+ [setup.py](./setup.py). Please note that due to a bug in pip, the `PRE`
+ version has to remain empty in the setup.py.
+ Also update CHANGELOG.md (new "Unreleased" section). Also update
+ `src/doc/conf.py`.
diff --git a/setup.py b/setup.py
index 9f392889c314bb54b0557f53793a034f6d5161b5..35d6d69a83338a0dc543b4b6438b56c1372b7237 100755
--- a/setup.py
+++ b/setup.py
@@ -47,13 +47,13 @@ from setuptools import find_packages, setup
ISRELEASED = False
MAJOR = 0
-MINOR = 9
-MICRO = 0
+MINOR = 10
+MICRO = 1
# Do not tag as pre-release until this commit
# https://github.com/pypa/packaging/pull/515
# has made it into a release. Probably we should wait for pypa/packaging>=21.4
# https://github.com/pypa/packaging/releases
-PRE = "dev" # "dev" # e.g. rc0, alpha.1, 0.beta-23
+PRE = "" # "dev" # e.g. rc0, alpha.1, 0.beta-23
if PRE:
VERSION = "{}.{}.{}-{}".format(MAJOR, MINOR, MICRO, PRE)
@@ -97,6 +97,9 @@ def get_version_info():
if os.path.exists('.git'):
GIT_REVISION = git_version()
+ elif os.path.exists('caosdb_pylib_commit'):
+ with open('caosdb_pylib_commit', 'r') as f:
+ GIT_REVISION = f.read().strip()
elif os.path.exists('src/caosdb/version.py'):
# must be a source distribution, use existing version file
try:
@@ -171,7 +174,11 @@ def setup_package():
python_requires='>=3.8',
package_dir={'': 'src'},
install_requires=['lxml>=4.6.3',
- 'PyYAML>=6.0', 'future', 'PySocks>=1.6.7'],
+ "requests[socks]>=2.28.1",
+ "python-dateutil>=2.8.2",
+ 'PyYAML>=5.4.1',
+ 'future',
+ ],
extras_require={'keyring': ['keyring>=13.0.0'],
'jsonschema': ['jsonschema>=4.4.0']},
setup_requires=["pytest-runner>=2.0,<3dev"],
diff --git a/src/caosdb/apiutils.py b/src/caosdb/apiutils.py
index 4c8393111bcbb4f9f91e309b81bebdcac55ba626..f3195b8e152f0cb13e5dab3e3a449b7bb36623b4 100644
--- a/src/caosdb/apiutils.py
+++ b/src/caosdb/apiutils.py
@@ -27,12 +27,13 @@
Some simplified functions for generation of records etc.
"""
+import logging
import sys
import tempfile
import warnings
+
from collections.abc import Iterable
from subprocess import call
-
from typing import Optional, Any, Dict, List
from caosdb.common.datatype import (BOOLEAN, DATETIME, DOUBLE, FILE, INTEGER,
@@ -40,8 +41,13 @@ from caosdb.common.datatype import (BOOLEAN, DATETIME, DOUBLE, FILE, INTEGER,
from caosdb.common.models import (Container, Entity, File, Property, Query,
Record, RecordType, execute_query,
get_config, SPECIAL_ATTRIBUTES)
+from caosdb.exceptions import CaosDBException
-import logging
+
+class EntityMergeConflictError(CaosDBException):
+ """An error that is raised in case of an unresolvable conflict when merging
+ two entities.
+ """
def new_record(record_type, name=None, description=None,
@@ -188,9 +194,8 @@ def getCommitIn(folder):
return t.readline().strip()
-def compare_entities(old_entity: Entity, new_entity: Entity):
- """
- Compare two entites.
+def compare_entities(old_entity: Entity, new_entity: Entity, compare_referenced_records: bool = False):
+ """Compare two entites.
Return a tuple of dictionaries, the first index belongs to additional information for old
entity, the second index belongs to additional information for new entity.
@@ -204,6 +209,23 @@ def compare_entities(old_entity: Entity, new_entity: Entity):
- ... value (not implemented yet)
In case of changed information the value listed under the respective key shows the
value that is stored in the respective entity.
+
+ If `compare_referenced_records` is `True`, also referenced entities will be
+ compared using this function (which is then called with
+ `compare_referenced_records = False` to prevent infinite recursion in case
+ of circular references).
+
+ Parameters
+ ----------
+ old_entity, new_entity : Entity
+ Entities to be compared
+ compare_referenced_records : bool, optional
+ Whether to compare referenced records in case of both, `old_entity` and
+ `new_entity`, have the same reference properties and both have a Record
+ object as value. If set to `False`, only the corresponding Python
+ objects are compared which may lead to unexpected behavior when
+ identical records are stored in different objects. Default is False.
+
"""
olddiff: Dict[str, Any] = {"properties": {}, "parents": []}
newdiff: Dict[str, Any] = {"properties": {}, "parents": []}
@@ -270,9 +292,29 @@ def compare_entities(old_entity: Entity, new_entity: Entity):
matching[0].unit
if (prop.value != matching[0].value):
- olddiff["properties"][prop.name]["value"] = prop.value
- newdiff["properties"][prop.name]["value"] = \
- matching[0].value
+ # basic comparison of value objects says they are different
+ same_value = False
+ if compare_referenced_records:
+ # scalar reference
+ if isinstance(prop.value, Entity) and isinstance(matching[0].value, Entity):
+ # explicitely not recursive to prevent infinite recursion
+ same_value = empty_diff(
+ prop.value, matching[0].value, compare_referenced_records=False)
+ # list of references
+ elif isinstance(prop.value, list) and isinstance(matching[0].value, list):
+ # all elements in both lists actually are entity objects
+ # TODO: check, whether mixed cases can be allowed or should lead to an error
+ if all([isinstance(x, Entity) for x in prop.value]) and all([isinstance(x, Entity) for x in matching[0].value]):
+ # can't be the same if the lengths are different
+ if len(prop.value) == len(matching[0].value):
+ # do a one-by-one comparison; the values are the same, if all diffs are empty
+ same_value = all(
+ [empty_diff(x, y, False) for x, y in zip(prop.value, matching[0].value)])
+
+ if not same_value:
+ olddiff["properties"][prop.name]["value"] = prop.value
+ newdiff["properties"][prop.name]["value"] = \
+ matching[0].value
if (len(newdiff["properties"][prop.name]) == 0
and len(olddiff["properties"][prop.name]) == 0):
@@ -300,27 +342,83 @@ def compare_entities(old_entity: Entity, new_entity: Entity):
return (olddiff, newdiff)
-def merge_entities(entity_a: Entity, entity_b: Entity):
- """
- Merge entity_b into entity_a such that they have the same parents and properties.
+def empty_diff(old_entity: Entity, new_entity: Entity, compare_referenced_records: bool = False):
+ """Check whether the `compare_entities` found any differences between
+ old_entity and new_entity.
+
+ Parameters
+ ----------
+ old_entity, new_entity : Entity
+ Entities to be compared
+ compare_referenced_records : bool, optional
+ Whether to compare referenced records in case of both, `old_entity` and
+ `new_entity`, have the same reference properties and both have a Record
+ object as value.
- datatype, unit, value, name and description will only be changed in entity_a if they
- are None for entity_a and set for entity_b. If there is a corresponding value
- for entity_a different from None a RuntimeError will be raised informing of an
- unresolvable merge conflict.
+ """
+ olddiff, newdiff = compare_entities(
+ old_entity, new_entity, compare_referenced_records)
+ for diff in [olddiff, newdiff]:
+ for key in ["parents", "properties"]:
+ if len(diff[key]) > 0:
+ # There is a difference somewhere in the diff
+ return False
+ for key in SPECIAL_ATTRIBUTES:
+ if key in diff and diff[key]:
+ # There is a difference in at least one special attribute
+ return False
+ # all elements of the two diffs were empty
+ return True
+
+
+def merge_entities(entity_a: Entity, entity_b: Entity, merge_references_with_empty_diffs=True,
+ force=False):
+ """Merge entity_b into entity_a such that they have the same parents and properties.
+
+ datatype, unit, value, name and description will only be changed in entity_a
+ if they are None for entity_a and set for entity_b. If there is a
+ corresponding value for entity_a different from None, an
+ EntityMergeConflictError will be raised to inform about an unresolvable merge
+ conflict.
The merge operation is done in place.
Returns entity_a.
WARNING: This function is currently experimental and insufficiently tested. Use with care.
+
+ Parameters
+ ----------
+ entity_a, entity_b : Entity
+ The entities to be merged. entity_b will be merged into entity_a in place
+ merge_references_with_empty_diffs : bool, optional
+ Whether the merge is performed if entity_a and entity_b both reference
+ record(s) that may be different Python objects but have empty diffs. If
+ set to `False` a merge conflict will be raised in this case
+ instead. Default is True.
+ force : bool, optional
+ If True, in case `entity_a` and `entity_b` have the same properties, the
+ values of `entity_a` are replaced by those of `entity_b` in the merge.
+ If `False`, an EntityMergeConflictError is raised instead. Default is False.
+
+ Returns
+ -------
+ entity_a : Entity
+ The initial entity_a after the in-place merge
+
+ Raises
+ ------
+ EntityMergeConflictError
+ In case of an unresolvable merge conflict.
+
"""
logging.warning(
"This function is currently experimental and insufficiently tested. Use with care.")
# Compare both entities:
- diff_r1, diff_r2 = compare_entities(entity_a, entity_b)
+ diff_r1, diff_r2 = compare_entities(
+ entity_a, entity_b, compare_referenced_records=merge_references_with_empty_diffs)
# Go through the comparison and try to apply changes to entity_a:
for key in diff_r2["parents"]:
@@ -338,11 +436,22 @@ def merge_entities(entity_a: Entity, entity_b: Entity):
raise NotImplementedError()
for attribute in ("datatype", "unit", "value"):
- if diff_r1["properties"][key][attribute] is None:
- setattr(entity_a.get_property(key), attribute,
- diff_r2["properties"][key][attribute])
- else:
- raise RuntimeError("Merge conflict.")
+ if (attribute in diff_r2["properties"][key] and
+ diff_r2["properties"][key][attribute] is not None):
+ if (diff_r1["properties"][key][attribute] is None):
+ setattr(entity_a.get_property(key), attribute,
+ diff_r2["properties"][key][attribute])
+ elif force:
+ setattr(entity_a.get_property(key), attribute,
+ diff_r2["properties"][key][attribute])
+ else:
+ raise EntityMergeConflictError(
+ f"Entity a ({entity_a.id}, {entity_a.name}) "
+ f"has a Property '{key}' with {attribute}="
+ f"{diff_r2['properties'][key][attribute]}\n"
+ f"Entity b ({entity_b.id}, {entity_b.name}) "
+ f"has a Property '{key}' with {attribute}="
+ f"{diff_r1['properties'][key][attribute]}")
else:
# TODO: This is a temporary FIX for
# https://gitlab.indiscale.com/caosdb/src/caosdb-pylib/-/issues/105
@@ -362,8 +471,13 @@ def merge_entities(entity_a: Entity, entity_b: Entity):
if sa_a != sa_b:
if sa_a is None:
setattr(entity_a, special_attribute, sa_b)
+ elif force:
+ # force overwrite
+ setattr(entity_a, special_attribute, sa_b)
else:
- raise RuntimeError("Merge conflict.")
+ raise EntityMergeConflictError(
+ f"Conflict in special attribute {special_attribute}:\n"
+ f"A: {sa_a}\nB: {sa_b}")
return entity_a
diff --git a/src/caosdb/cert/indiscale.ca.crt b/src/caosdb/cert/indiscale.ca.crt
deleted file mode 100644
index 08a79d60c5d34626eb96f1a92e33d0ac22494f3c..0000000000000000000000000000000000000000
--- a/src/caosdb/cert/indiscale.ca.crt
+++ /dev/null
@@ -1,55 +0,0 @@
------BEGIN CERTIFICATE-----
-MIIJ6TCCBdGgAwIBAgIIFVYzdrEDk6EwDQYJKoZIhvcNAQENBQAwgZMxCzAJBgNV
-BAYTAkRFMRUwEwYDVQQIEwxMb3dlciBTYXhvbnkxEzARBgNVBAcTCkdvZXR0aW5n
-ZW4xFzAVBgNVBAoTDkluZGlTY2FsZSBHbWJIMRwwGgYDVQQDExNJbmRpU2NhbGUg
-Um9vdCBDQSAxMSEwHwYJKoZIhvcNAQkBFhJpbmZvQGluZGlzY2FsZS5jb20wHhcN
-MTkwODA3MDAwMDAwWhcNMzQwODA2MjM1OTU5WjCBkzELMAkGA1UEBhMCREUxFTAT
-BgNVBAgTDExvd2VyIFNheG9ueTETMBEGA1UEBxMKR29ldHRpbmdlbjEXMBUGA1UE
-ChMOSW5kaVNjYWxlIEdtYkgxHDAaBgNVBAMTE0luZGlTY2FsZSBSb290IENBIDEx
-ITAfBgkqhkiG9w0BCQEWEmluZm9AaW5kaXNjYWxlLmNvbTCCBCIwDQYJKoZIhvcN
-AQEBBQADggQPADCCBAoCggQBAKxJO3XOqrUxFU3qdVyk9tmZEHwhwntcLO+kRR5t
-64/1Z/+VIPSgVN5phkSCukj2BPJITWKplWzJDAYWSvA/7cqavCtx8yP+m3AHWrRa
-CeHbtkGZ1nzwyFel3GIr93e65REeWqBE3knzem+qxTlZ2hp8/w3oxUlhy7tGxjBs
-JlekgLRDrnj4Opyb4GVjcVfcELmu3sLrrPX1wdYJrqaMQUR4BKZnbXxKdOYyX+kR
-/W2P4sihCCJh7Wy29VXHwSSCM1qEkU3REjvPEmEElCG7UpqOfg+3jaNZDqnvfskf
-okU4GuFCxSWQituyP9jm/hFVEhz59tUMYCllcjEi2jGmD2DBKpiru4t4/z0Aymf4
-Pep9hNtH1yhZMxpQeCYK9ESEE5d7do0bu/4YFp7jAg5vWZ8KlILZakmypVBFUw8I
-U/QJoJ55j95vIp+kjFdXelIVcr5La/zOR82JldaoPfyoBKObzwpwqaWQwYm8pj4p
-XkUdJTf8rpW21SSGWZm8JoFSYDfGvI61rPEjl/ohKhlG0tV6E2tCc406HNo/7pPe
-pmx/v9ZWLbYDAH7MVMB4tv6zDRE/c4KTbh5/s70VbXbAeOG6DNwegdDLDYZOv6Yw
-YQMz9NWtKGzvoFehP2vY5nGK95JVUcd90jaNaoURLB102VtxAjPIEQA1PjbQxLvC
-7A6kshlpQiN7zS/R9IgiEkYP/9gjy6mMuQVxH7C+9cqmCnXvVmpHmxXGUqk61r/B
-h12htsx5qjbbkToZYhUXBmwRq4LDtyoxNeaF2Jc+gE762obbHsSYMuSuh0kTFUUd
-uqfrI8OyzX4r1w5dYf2FEetZTT2Obyxb3Cy0btJF5+zEerBX44RulkdC+TPTMhJw
-b1jrPCACKywy9b6vJcSQ2V1+uLk7rH2JKD+fQRIKUqZZkhNKFYz5dnYYTgS45M0/
-C+vIvRnhgNSNb4efG6wyFvWEF8poDSPnJ4mM+0jHG/+cLqF/M2CMFvC+yU8Hj9YH
-B+H2L6V1QlCkpw5Ai4ji6OaQmnrsjE8EJj58vwYKsjmLGuf4j5AivkQTxfgCPGrT
-6CxSesoFmYDPSg/2eO+IfYEwnd7Rbs4aAhW8eo+lGpmK0DQxNjlejYt/Cgp7HWCq
-m/VNqWPIDMSTTqyk1GTmp67NjEZKt2ukJxI2CpL8s/9x4f3GTjNyI750pKM/uzMk
-OBKTMuWJQ6xeMR3h9RQlqlmwcErLXoUGInOTHHjRGXDI+ZBeLqT5DikcFiwbHG3+
-6FOuxXO0eqqg2tBW8cQ5kuRI0YFznipDUcfgDZt0JEkEXmRuL0nxYO35WKKdpGcF
-xFRJtO4FRB4nVWekVRuK9m47IPm6vC4eo+pCNPPoQ+FjyQ8CAwEAAaM/MD0wDAYD
-VR0TBAUwAwEB/zAdBgNVHQ4EFgQUFjE2TLaKASKEJ0LKOO+37/Hu7qowDgYDVR0P
-AQH/BAQDAgEGMA0GCSqGSIb3DQEBDQUAA4IEAQB2j1GL1G0ferWp9wmuDdF2oumn
-k+JIctRaGHaSrqHy4zjwj3Oqm2JA1ds+WfWozz+d38ZcLqSHo+g9ku5h/XOogQEU
-O4/y7j44pxIUg0EcIpMHtf7KPocPfvi9lw/4vE/3V/WKh4E09SXWgyY5tMUlEMaB
-6t8n7gg943HY2MJE1QU2wOPMXB1krFbunYxJcrUMs21w9jRWVv/wvaj6rkdvvSbU
-Yre11J+VlhC6fxx+STohQopzE6jnsaHile56b9xAmCCKcPEpWeKKBFS7pVNHEIHF
-uHWpgVjhoheEMMbYgu6l5E5K32TNYCKU49jNRWEKETjmYQSNl9dsSip+XlvaU8wQ
-VRR8UMHZPiJDW/AAHCr+bXEarZ9mSj/y+R512YtVw95zCnGUtzOJViThoIk/IAOR
-AJdnvsFmZSIKtFHpSEFYlTDq2yr1ulzbaDhuPRzita8b0cP27UvqRebZw5CvHN48
-B9a9tTYowKuJqmtjE6D00QA4xS8fRizLnx54uNmDbwf/8WavVk6MzDERwRE3OsSy
-D0dV6gy3t2AqEpVBrICrFqvgAQa4fcFcIwz3Qbt5o5uEi7acRomY57YrxrlfNTwh
-2oDQz+HQ/ZTDwZ3DrIgel7GrQ5fXrXDLL3ebtsbuIeBx8crOWQask832HcLtDVpu
-E/FdJEMMjglzIcy2dHpuODIGFmgEVfHR4DOOSBl0hfNdlrYnhC0h8/6QFswtlYFF
-8aQbGX7inK8L2in5wQ7ypeoMuXkQVYxlU1TEGmgB8aDke47MuX1FH+clsCaZ3s1E
-ka6lV6cjNYcosS718B6b2JgDUzmGBn2Sdm1xFmJM16dXp7TSmC5/fYxXuE/CynDs
-PmaUb9Ms6XUYSwKKhZ5HZdeRoNz8w62WNAeF7o7iX6IVrd/G1bJnSBN01istckyR
-BDuIkaoBQ9yvHN6Bo/J3KR08ixF1dHFPo/oSgkBxkLakb/yeslBTP/oISiFeQ4+q
-Gld1mhAvmG99dVZfoysrMjZSyghNbqwScjbYYN115lExV5ZeRtSwA7JCYE2lBjmB
-vocmz/hh/ifbmmqIvSv0NtiBnM6mNqngZEWD/rAloVOQoq0KVJJ5lUCQrBSFtR4+
-G1JGMX6b7uRp4mfdqqDE62KxxfkWBUwzUTIKGb5K42ji1Gy5li/TIWJtLNGNNQ2A
-0ui2RhwioaGGfYyomSFuAo5IPE/NF0ASjrTDW6GoNxypTSYE4/7oSoxeryafVnqN
-S0fRyrgSLiuT5tAiZ3b5Q3EFYUM2OcU3ezr/ZUabf9qIsqOnCi91SqE88BQbenot
-0HyUMdp/7QX9SyWM/azhcRiReAtkmq9pgeQA2TTZADDNTkKRljG9VeFDSwl7
------END CERTIFICATE-----
diff --git a/src/caosdb/common/administration.py b/src/caosdb/common/administration.py
index 98d4d2826da7131ef79b5c3cc9b3d9597abc0248..bd09c626d5a9e48bc9d3610f23cb46660cb10db3 100644
--- a/src/caosdb/common/administration.py
+++ b/src/caosdb/common/administration.py
@@ -76,7 +76,7 @@ def get_server_properties():
con = get_connection()
try:
body = con._http_request(
- method="GET", path="_server_properties").response
+ method="GET", path="_server_properties")
except EntityDoesNotExistError:
raise ServerConfigurationException(
"Debug mode in server is probably disabled.") from None
diff --git a/src/caosdb/common/models.py b/src/caosdb/common/models.py
index a50411253fa2c83438e0b91e51c6e004a143acd3..05f661c9b6ef2c89027e2a56fe22e6a9d1e59dff 100644
--- a/src/caosdb/common/models.py
+++ b/src/caosdb/common/models.py
@@ -52,6 +52,7 @@ from caosdb.common.datatype import (BOOLEAN, DATETIME, DOUBLE, INTEGER, TEXT,
is_list_datatype, is_reference)
from caosdb.common.state import State
from caosdb.common.utils import uuid, xml2str, experimental
+from caosdb.common.timezone import TimeZone
from caosdb.common.versioning import Version
from caosdb.configuration import get_config
from caosdb.connection.connection import get_connection
@@ -4354,6 +4355,8 @@ class Info():
if isinstance(m, UserInfo):
self.user_info = m
+ elif isinstance(m, TimeZone):
+ self.time_zone = m
else:
self.messages.append(m)
@@ -4488,6 +4491,9 @@ def _parse_single_xml_element(elem):
return Permissions(xml=elem)
elif elem.tag == "UserInfo":
return UserInfo(xml=elem)
+ elif elem.tag == "TimeZone":
+ return TimeZone(zone_id=elem.get("id"), offset=elem.get("offset"),
+ display_name=elem.text.strip())
else:
return Message(type=elem.tag, code=elem.get(
"code"), description=elem.get("description"), body=elem.text)
diff --git a/src/caosdb/common/timezone.py b/src/caosdb/common/timezone.py
new file mode 100644
index 0000000000000000000000000000000000000000..8fc5e710d3cbf6f20cf81397573f972db3b22f12
--- /dev/null
+++ b/src/caosdb/common/timezone.py
@@ -0,0 +1,19 @@
+class TimeZone():
+ """
+ TimeZone, e.g. CEST, Europe/Berlin, UTC+4.
+
+
+ Attributes
+ ----------
+ zone_id : string
+ ID of the time zone.
+ offset : int
+ Offset to UTC in seconds.
+ display_name : string
+ A human-friendly name of the time zone:
+ """
+
+ def __init__(self, zone_id, offset, display_name):
+ self.zone_id = zone_id
+ self.offset = offset
+ self.display_name = display_name
diff --git a/src/caosdb/configuration.py b/src/caosdb/configuration.py
index 75827df0d00d6c82251c2c04fa47413ac2801928..1c108ac1d39c135dbc90f477be8b8f2f630391ce 100644
--- a/src/caosdb/configuration.py
+++ b/src/caosdb/configuration.py
@@ -31,12 +31,7 @@ try:
except ImportError:
pass
-try:
- # python2
- from ConfigParser import ConfigParser
-except ImportError:
- # python3
- from configparser import ConfigParser
+from configparser import ConfigParser
from os import environ, getcwd
from os.path import expanduser, join, isfile
@@ -59,6 +54,11 @@ def configure(inifile):
_reset_config()
read_config = _pycaosdbconf.read(inifile)
validate_yaml_schema(config_to_yaml(_pycaosdbconf))
+
+ if "HTTPS_PROXY" in environ:
+ _pycaosdbconf["Connection"]["https_proxy"] = environ["HTTPS_PROXY"]
+ if "HTTP_PROXY" in environ:
+ _pycaosdbconf["Connection"]["http_proxy"] = environ["HTTP_PROXY"]
return read_config
diff --git a/src/caosdb/connection/SocksiPy.zip b/src/caosdb/connection/SocksiPy.zip
deleted file mode 100644
index e81f1f9393c766a3acd41b44245f9e17f090cbe5..0000000000000000000000000000000000000000
Binary files a/src/caosdb/connection/SocksiPy.zip and /dev/null differ
diff --git a/src/caosdb/connection/connection.py b/src/caosdb/connection/connection.py
index 43eb3410d8d5bdc8323a811fd2b6424fb75f3fda..a2802848af7bae8fb65378532156d8469f31a9b8 100644
--- a/src/caosdb/connection/connection.py
+++ b/src/caosdb/connection/connection.py
@@ -28,9 +28,15 @@ from __future__ import absolute_import, print_function, unicode_literals
import logging
import ssl
import sys
+import warnings
from builtins import str # pylint: disable=redefined-builtin
from errno import EPIPE as BrokenPipe
from socket import error as SocketError
+from urllib.parse import quote, urlparse
+from requests import Session as HTTPSession
+from requests.exceptions import ConnectionError as HTTPConnectionError
+from urllib3.poolmanager import PoolManager
+from requests.adapters import HTTPAdapter
from caosdb.configuration import get_config
from caosdb.exceptions import (CaosDBException, HTTPClientError,
@@ -49,16 +55,8 @@ except ModuleNotFoundError:
from pkg_resources import resource_filename
from .interface import CaosDBHTTPResponse, CaosDBServerConnection
-from .streaminghttp import StreamingHTTPSConnection
from .utils import make_uri_path, parse_url, urlencode
-
-try:
- from urllib.parse import quote, urlparse
-except ImportError:
- from urllib import quote
- from urlparse import urlparse
-
-# pylint: disable=missing-docstring
+from .encode import MultipartYielder, ReadableMultiparts
_LOGGER = logging.getLogger(__name__)
@@ -67,6 +65,9 @@ class _WrappedHTTPResponse(CaosDBHTTPResponse):
def __init__(self, response):
self.response = response
+ self._generator = None
+ self._buffer = b''
+ self._stream_consumed = False
@property
def reason(self):
@@ -74,21 +75,71 @@ class _WrappedHTTPResponse(CaosDBHTTPResponse):
@property
def status(self):
- return self.response.status
+ return self.response.status_code
def read(self, size=None):
- return self.response.read(size)
+ if self._stream_consumed is True:
+ raise RuntimeError("Stream is consumed")
+
+ if self._buffer is None:
+ # the buffer has been drained in the previous call.
+ self._stream_consumed = True
+ return b''
+
+ if self._generator is None and (size is None or size == 0):
+ # return full content at once
+ self._stream_consumed = True
+ return self.response.content
+
+ if len(self._buffer) >= size:
+ # still enough bytes in the buffer
+ result = chunk[:size]
+ self._buffer = chunk[size:]
+ return result
+
+ if self._generator is None:
+ # first call to this method
+ if size is None or size == 0:
+ size = 512
+ self._generator = self.response.iter_content(size)
+
+ try:
+ # read new data into the buffer
+ chunk = self._buffer + next(self._generator)
+ result = chunk[:size]
+ if len(result) == 0:
+ self._stream_consumed = True
+ self._buffer = chunk[size:]
+ return result
+ except StopIteration:
+ # drain buffer
+ result = self._buffer
+ self._buffer = None
+ return result
def getheader(self, name, default=None):
- return self.response.getheader(name=name, default=default)
+ return self.response.headers[name] if name in self.response.headers else default
def getheaders(self):
- return self.response.getheaders()
+ return self.response.headers.items()
def close(self):
self.response.close()
+class _SSLAdapter(HTTPAdapter):
+ """Transport adapter that allows us to use different SSL versions."""
+
+ def __init__(self, ssl_version):
+ self.ssl_version = ssl_version
+ super().__init__()
+
+ def init_poolmanager(self, connections, maxsize, block=False):
+ self.poolmanager = PoolManager(
+ num_pools=connections, maxsize=maxsize,
+ block=block, ssl_version=self.ssl_version)
+
+
class _DefaultCaosDBServerConnection(CaosDBServerConnection):
"""_DefaultCaosDBServerConnection.
@@ -101,10 +152,11 @@ class _DefaultCaosDBServerConnection(CaosDBServerConnection):
def __init__(self):
self._useragent = ("caosdb-pylib/{version} - {implementation}".format(
version=version, implementation=type(self).__name__))
- self._http_con = None
self._base_path = None
+ self._session = None
+ self._timeout = None
- def request(self, method, path, headers=None, body=None, **kwargs):
+ def request(self, method, path, headers=None, body=None):
"""request.
Send a HTTP request to the server.
@@ -118,38 +170,40 @@ class _DefaultCaosDBServerConnection(CaosDBServerConnection):
including query and frament segments.
headers : dict of str -> str, optional
HTTP request headers. (Defautl: None)
- body : str or bytes or readable, opional
+ body : str or bytes or readable, optional
The body of the HTTP request. Bytes should be a utf-8 encoded
string.
- **kwargs :
- Any keyword arguments will be ignored.
- TODO: Why are they allowed then?
Returns
-------
- TODO: What?
+ response : CaosDBHTTPResponse
"""
if headers is None:
headers = {}
headers["User-Agent"] = self._useragent
+
+ if path.endswith("/."):
+ path = path[:-1] + "%2E"
+
+ if isinstance(body, MultipartYielder):
+ body = ReadableMultiparts(body)
+
try:
- self._http_con = StreamingHTTPSConnection(
- # TODO looks as if configure needs to be done first.
- # That is however not assured.
- host=self.setup_fields["host"],
- timeout=self.setup_fields["timeout"],
- context=self.setup_fields["context"],
- socket_proxy=self.setup_fields["socket_proxy"])
- self._http_con.request(method=method, url=self._base_path + path,
- headers=headers, body=body)
- except SocketError as socket_err:
+ response = self._session.request(
+ method=method,
+ url=self._base_path + path,
+ headers=headers,
+ data=body,
+ timeout=self._timeout,
+ stream=True)
+
+ return _WrappedHTTPResponse(response)
+ except HTTPConnectionError as conn_err:
raise CaosDBConnectionError(
- "Connection failed. Network or server down? " + str(socket_err)
+ "Connection failed. Network or server down? " + str(conn_err)
)
- return _WrappedHTTPResponse(self._http_con.getresponse())
-
def configure(self, **config):
"""configure.
@@ -173,55 +227,69 @@ class _DefaultCaosDBServerConnection(CaosDBServerConnection):
loaded.
"""
+ if "url" not in config:
+ raise CaosDBConnectionError(
+ "No connection url specified. Please "
+ "do so via caosdb.configure_connection(...) or in a config "
+ "file.")
+ if (not config["url"].lower().startswith("https://") and not config["url"].lower().startswith("http://")):
+ raise CaosDBConnectionError("The connection url is expected "
+ "to be a http or https url and "
+ "must include the url scheme "
+ "(i.e. start with https:// or "
+ "http://).")
+
+ url = urlparse(config["url"])
+ path = url.path.strip("/")
+ if len(path) > 0:
+ path = path + "/"
+ self._base_path = url.scheme + "://" + url.netloc + "/" + path
+
+ self._session = HTTPSession()
+
+ if url.scheme == "https":
+ self._setup_ssl(config)
+
+ # TODO(tf) remove in next release
+ socket_proxy = config["socket_proxy"] if "socket_proxy" in config else None
+ if socket_proxy is not None:
+ self._session.proxies = {
+ "https": "socks5://" + socket_proxy,
+ "http": "socks5://" + socket_proxy,
+ }
+
+ if "https_proxy" in config:
+ if self._session.proxies is None:
+ self._session.proxies = {}
+ self._session.proxies["https"] = config["https_proxy"]
+
+ if "http_proxy" in config:
+ if self._session.proxies is None:
+ self._session.proxies = {}
+ self._session.proxies["http"] = config["http_proxy"]
+
+ if "timeout" in config:
+ self._timeout = config["timeout"]
+
+ def _setup_ssl(self, config):
if "ssl_version" in config and config["cacert"] is not None:
ssl_version = getattr(ssl, config["ssl_version"])
else:
ssl_version = ssl.PROTOCOL_TLS
- context = ssl.SSLContext(ssl_version)
- context.verify_mode = ssl.CERT_REQUIRED
- if config.get("ssl_insecure"):
+ self._session.mount(self._base_path, _SSLAdapter(ssl_version))
+
+ verify = True
+ if "cacert" in config:
+ verify = config["cacert"]
+ if "ssl_insecure" in config and config["ssl_insecure"]:
_LOGGER.warning("*** Warning! ***\n"
"Insecure SSL mode, certificate will not be checked! "
"Please consider removing the `ssl_insecure` configuration option.\n"
"****************")
- context.verify_mode = ssl.CERT_NONE
-
- if (not context.verify_mode == ssl.CERT_NONE and
- hasattr(context, "check_hostname")):
- context.check_hostname = True
-
- if ("cacert" in config and config["cacert"] is not None and
- config["cacert"]):
- try:
- context.load_verify_locations(config["cacert"])
- except Exception as exc:
- raise CaosDBConnectionError("Could not load the cacert in"
- "`{}`: {}".format(config["cacert"],
- exc))
-
- context.load_default_certs()
-
- if "url" in config:
- parsed_url = parse_url(config["url"])
- host = parsed_url.netloc
- self._base_path = parsed_url.path
- else:
- raise CaosDBConnectionError(
- "No connection url specified. Please "
- "do so via caosdb.configure_connection(...) or in a config "
- "file.")
-
- socket_proxy = None
-
- if "socket_proxy" in config:
- socket_proxy = config["socket_proxy"]
-
- self.setup_fields = {
- "host": host,
- "timeout": int(config.get("timeout")),
- "context": context,
- "socket_proxy": socket_proxy}
+ verify = False
+ if verify is not None:
+ self._session.verify = verify
def _make_conf(*conf):
@@ -252,7 +320,6 @@ _DEFAULT_CONF = {
"password_method": "input",
"implementation": _DefaultCaosDBServerConnection,
"timeout": 210,
- "cacert": resource_filename("caosdb", 'cert/indiscale.ca.crt')
}
@@ -314,6 +381,10 @@ def configure_connection(**kwargs):
Parameters
----------
+ url : str
+ The url of the CaosDB Server. HTTP and HTTPS urls are allowed. However,
+ it is **highly** recommend to avoid HTTP because passwords and
+ authentication token are send over the network in plain text.
username : str
Username for login; e.g. 'admin'.
@@ -342,6 +413,24 @@ def configure_connection(**kwargs):
An authentication token which has been issued by the CaosDB Server.
Implies `password_method="auth_token"` if set. An example token string would be `["O","OneTimeAuthenticationToken","anonymous",["administration"],[],1592995200000,604800000,"3ZZ4WKRB-5I7DG2Q6-ZZE6T64P-VQ","197d0d081615c52dc18fb323c300d7be077beaad4020773bb58920b55023fa6ee49355e35754a4277b9ac525c882bcd3a22e7227ba36dfcbbdbf8f15f19d1ee9",1,30000]`.
+ https_proxy : str, optional
+ Define a proxy for the https connections, e.g. `http://localhost:8888`,
+ `socks5://localhost:8888`, or `socks4://localhost:8888`. These are
+ either (non-TLS) HTTP proxies, SOCKS4 proxies, or SOCKS5 proxies. HTTPS
+ proxies are not supported. However, the connection will be secured
+ using TLS in the tunneled connection nonetheless. Only the connection
+ to the proxy is insecure which is why it is not recommended to use HTTP
+ proxies when authentication against the proxy is necessary. If
+ unspecified, the https_proxy option of the pycaosdb.ini or the HTTPS_PROXY
+ environment variable are being used. Use `None` to override these
+ options with a no-proxy setting.
+
+ http_proxy : str, optional
+ Define a proxy for the http connections, e.g. `http://localhost:8888`.
+ If unspecified, the http_proxy option of the pycaosdb.ini or the
+ HTTP_PROXY environment variable are being used. Use `None` to override
+ these options with a no-proxy setting.
+
implementation : CaosDBServerConnection
The class which implements the connection. (Default:
_DefaultCaosDBServerConnection)
@@ -372,6 +461,11 @@ def configure_connection(**kwargs):
local_conf = _make_conf(_DEFAULT_CONF, global_conf, kwargs)
connection = _Connection.get_instance()
+
+ if "socket_proxy" in local_conf:
+ warnings.warn("Deprecated configuration option: socket_proxy. Use "
+ "the new https_proxy option instead",
+ DeprecationWarning, stacklevel=1)
connection.configure(**local_conf)
return connection
@@ -599,7 +693,7 @@ class _Connection(object): # pylint: disable=useless-object-inheritance
method=method,
path=path,
headers=headers,
- body=body, **kwargs)
+ body=body)
_LOGGER.debug("response: %s %s", str(http_response.status),
str(http_response.getheaders()))
self._authenticator.on_response(http_response)
diff --git a/src/caosdb/connection/encode.py b/src/caosdb/connection/encode.py
index 7b092aae784a76abec0104ef7269df7ae0111b3b..0b826cc4400275a2374308ee104cdbdabb619b75 100644
--- a/src/caosdb/connection/encode.py
+++ b/src/caosdb/connection/encode.py
@@ -51,7 +51,8 @@ multipart/form-data is the standard way to upload files over HTTP
__all__ = [
'gen_boundary', 'encode_and_quote', 'MultipartParam', 'encode_string',
- 'encode_file_header', 'get_body_size', 'get_headers', 'multipart_encode'
+ 'encode_file_header', 'get_body_size', 'get_headers', 'multipart_encode',
+ 'ReadableMultiparts',
]
from urllib.parse import quote_plus
from io import UnsupportedOperation
@@ -475,3 +476,40 @@ def multipart_encode(params, boundary=None, callback=None):
params = MultipartParam.from_params(params)
return MultipartYielder(params, boundary, callback), headers
+
+
+class ReadableMultiparts(object):
+ """Wraps instances of the MultipartYielder class as a readable and withable
+ object."""
+
+ def __init__(self, multipart_yielder):
+ self.multipart_yielder = multipart_yielder
+ self.current_block = None
+ self.left_over = b''
+
+ def read(self, size=-1):
+ result = self.left_over
+ while size == -1 or len(result) < size:
+ try:
+ next_chunk = self.multipart_yielder.next()
+ if hasattr(next_chunk, "encode"):
+ next_chunk = next_chunk.encode("utf8")
+ result += next_chunk
+ except StopIteration:
+ break
+
+ if size == -1:
+ self.left_over = b''
+ return result
+
+ self.left_over = result[size:]
+ return result[:size]
+
+ def __enter__(self):
+ pass
+
+ def __exit__(self, type, value, traceback):
+ self.close()
+
+ def close(self):
+ self.multipart_yielder.reset()
diff --git a/src/caosdb/connection/streaminghttp.py b/src/caosdb/connection/streaminghttp.py
deleted file mode 100644
index 01774301b9bdb55bdbf6b56695042aaf354dba97..0000000000000000000000000000000000000000
--- a/src/caosdb/connection/streaminghttp.py
+++ /dev/null
@@ -1,152 +0,0 @@
-# -*- encoding: utf-8 -*-
-#
-# ** header v3.0
-# This file is a part of the CaosDB Project.
-#
-# Copyright (C) 2018 Research Group Biomedical Physics,
-# Max-Planck-Institute for Dynamics and Self-Organization Göttingen
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Affero General Public License as
-# published by the Free Software Foundation, either version 3 of the
-# License, or (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU Affero General Public License for more details.
-#
-# You should have received a copy of the GNU Affero General Public License
-# along with this program. If not, see <https://www.gnu.org/licenses/>.
-#
-# ** end header
-#
-# Original work Copyright (c) 2011 Chris AtLee
-# Modified work Copyright (c) 2017 Biomedical Physics, MPI for Dynamics and Self-Organization
-#
-# Permission is hereby granted, free of charge, to any person obtaining a copy
-# of this software and associated documentation files (the "Software"), to deal
-# in the Software without restriction, including without limitation the rights
-# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-# copies of the Software, and to permit persons to whom the Software is
-# furnished to do so, subject to the following conditions:
-#
-# The above copyright notice and this permission notice shall be included in
-# all copies or substantial portions of the Software.
-#
-# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
-# THE SOFTWARE.
-"""Streaming HTTP uploads module.
-
-This module extends the standard httplib and http.client HTTPConnection so that
-iterable objects can be used in the body of HTTP requests.
-
-**N.B.** You must specify a Content-Length header if using an iterable object
-since there is no way to determine in advance the total size that will be
-yielded, and there is no way to reset an interator.
-"""
-
-from __future__ import unicode_literals, print_function, absolute_import
-import socks
-import socket
-try:
- # python3
- from http import client as client
-except ImportError:
- # python2
- import httplib as client
-
-
-__all__ = ['StreamingHTTPSConnection']
-
-
-class StreamingHTTPSConnection(client.HTTPSConnection, object):
- """Subclass of `http.client.HTTSConnection` or `httplib.HTTPSConnection`
- that overrides the `send()` method to support iterable body objects."""
- # pylint: disable=unused-argument, arguments-differ
-
- def __init__(self, socket_proxy=None, **kwargs):
- if socket_proxy is not None:
- host, port = socket_proxy.split(":")
- socks.setdefaultproxy(socks.PROXY_TYPE_SOCKS5, host,
- int(port))
- socket.socket = socks.socksocket
- super(StreamingHTTPSConnection, self).__init__(**kwargs)
-
- def _send_output(self, body, **kwargs):
- """Send the currently buffered request and clear the buffer.
-
- Appends an extra \\r\\n to the buffer.
- A message_body may be specified, to be appended to the request.
-
- This method is implemented in differently in the various python
- versions (which is extremely annoying). So we provide a unified but
- relatively dumb implementaion which only serves our needs.
- """
- self._buffer.extend(("".encode("utf-8"), "".encode("utf-8")))
- headers = "\r\n".encode("utf-8").join(self._buffer)
- del self._buffer[:]
-
- self.send(headers)
- if body is not None:
- self.send(body)
-
- # pylint: disable=too-many-branches
-
- def send(self, value):
- """Send ``value`` to the server.
-
- ``value`` can be a string-like object which supports a 'encode' method,
- a file-like object that supports a .read() method, or an iterable object
- that supports a .next() method.
-
- An encode()able ``value`` will be utf-8 encoded before sending.
- """
- # Based on python 2.6's httplib.HTTPConnection.send()
- if self.sock is None:
- if self.auto_open:
- self.connect()
- else:
- raise client.NotConnected()
-
- # send the data to the server. if we get a broken pipe, then close
- # the socket. we want to reconnect when somebody tries to send again.
- #
- # NOTE: we DO propagate the error, though, because we cannot simply
- # ignore the error... the caller will know if they can retry.
- if self.debuglevel > 0:
- print("send: ", repr(value))
- try:
- blocksize = 8192
- if hasattr(value, 'read'):
- if hasattr(value, 'seek'):
- value.seek(0)
- if self.debuglevel > 0:
- print("sendIng a read()able")
- data = value.read(blocksize)
- while data:
- self.sock.sendall(data)
- data = value.read(blocksize)
- elif hasattr(value, 'next'):
- if hasattr(value, 'reset'):
- value.reset()
- if self.debuglevel > 0:
- print("sendIng an iterable")
- for data in value:
- if hasattr(data, "encode"):
- self.sock.sendall(data.encode('utf-8'))
- else:
- self.sock.sendall(data)
- else:
- if self.debuglevel > 0:
- print("sendIng a byte-like")
- self.sock.sendall(value)
- except socket.error as err:
- if err.args[0] == 32: # Broken pipe
- self.close()
- raise
diff --git a/src/caosdb/schema-pycaosdb-ini.yml b/src/caosdb/schema-pycaosdb-ini.yml
index 5dabdd89795e19a757209e03cc843776be705777..64451a24da1587abbd9815b8ddefc4c85fa36e5a 100644
--- a/src/caosdb/schema-pycaosdb-ini.yml
+++ b/src/caosdb/schema-pycaosdb-ini.yml
@@ -14,10 +14,10 @@ schema-pycaosdb-ini:
additionalProperties: false
properties:
url:
- description: URL of the CaosDB server
+ description: "URL of the CaosDB server. Allowed are HTTP and HTTPS connections. However, since authentication tokens and sometimes even passwords are send in plain text to the server it is **highly** recommended to use HTTPS connections whenever possible. HTTP is ok for testing and debugging."
type: string
- pattern: https://[-a-zA-Z0-9\.]+(:[0-9]+)?(/)?
- examples: ["https://demo.indiscale.com/", "https://localhost:10443/"]
+ pattern: http(s)?://[-a-zA-Z0-9\.]+(:[0-9]+)?(/)?
+ examples: ["https://demo.indiscale.com/", "http://localhost:10080/"]
username:
type: string
description: User name used for authentication with the server
@@ -26,7 +26,7 @@ schema-pycaosdb-ini:
description: The password input method defines how the password is supplied that is used for authentication with the server.
type: string
default: input
- enum: [input, plain, pass, keyring]
+ enum: [input, unauthenticated, plain, pass, keyring]
password_identifier:
type: string
password:
@@ -54,7 +54,15 @@ schema-pycaosdb-ini:
socket_proxy:
examples: ["localhost:12345"]
type: string
- description: You can define a socket proxy to be used. This is for the case that the server sits behind a firewall which is being tunnelled with a socket proxy (SOCKS4 or SOCKS5) (e.g. via ssh's -D option or a dedicated proxy server).
+ description: Deprecated. Please use https_proxy instead.
+ https_proxy:
+ examples: ["http://localhost:8888", "socks5://localhost:8888", "socks4://localhost:8888"]
+ type: string
+ description: "Define a proxy for the https connections. These are either (non-TLS) HTTP proxies, SOCKS4 proxies, or SOCKS5 proxies. HTTPS proxies are not supported. However, the connection will be secured using TLS in the tunneled connection nonetheless. Only the connection to the proxy is insecure which is why it is not recommended to use HTTP proxies when authentication against the proxy is necessary. Note: this option is overridden by the HTTPS_PROXY environment variable, if present."
+ http_proxy:
+ examples: ["http://localhost:8888", "socks5://localhost:8888", "socks4://localhost:8888"]
+ type: string
+ description: "Define a proxy for the http connections. These are either (non-TLS) HTTP proxies, SOCKS4 proxies, or SOCKS5 proxies. HTTPS proxies are not supported. Note: this option is overridden by the HTTP_PROXY environment variable, if present."
implementation:
description: This option is used internally and for testing. Do not override.
examples: [_DefaultCaosDBServerConnection]
diff --git a/src/caosdb/utils/caosdb_admin.py b/src/caosdb/utils/caosdb_admin.py
index 9fb94f57683036f5432a40198cc4ae98893665fb..09a8f64a3c6b9f0825089949840a8791604d1ded 100755
--- a/src/caosdb/utils/caosdb_admin.py
+++ b/src/caosdb/utils/caosdb_admin.py
@@ -621,8 +621,8 @@ USAGE
for action in ["grant", "deny", "revoke_denial", "revoke_grant"]:
action_entity_permissions_parser = subparsers.add_parser(
- "{}_entity_permissions".format(action),
- help="{} entity permissions to a role.".format(action))
+ f"{action}_entity_permissions",
+ help=f"{action} entity permissions to one or more Entities.")
action_entity_permissions_parser.set_defaults(
call=do_action_entity_permissions, action=action)
action_entity_permissions_parser.add_argument(dest="query", metavar="QUERY",
diff --git a/src/caosdb/utils/register_tests.py b/src/caosdb/utils/register_tests.py
new file mode 100644
index 0000000000000000000000000000000000000000..9d0afcbb0845e1d8d31622e8ab9926f26f7e78f6
--- /dev/null
+++ b/src/caosdb/utils/register_tests.py
@@ -0,0 +1,136 @@
+#!/usr/bin/env python
+# encoding: utf-8
+#
+# This file is a part of the CaosDB Project.
+#
+# Copyright (C) 2022 Alexander Schlemmer <alexander.schlemmer@ds.mpg.de>
+# Copyright (C) 2022 Timm Fitschen <t.fitschen@indiscale.com>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+import caosdb as db
+from caosdb import administration as admin
+
+"""
+This module implements a registration procedure for integration tests which
+need a running CaosDB instance.
+
+It ensures that tests do not accidentally overwrite data in real CaosDB
+instances, as it checks whether the running CaosDB instance is actually the
+correct one, that
+should be used for these tests.
+
+The test files have to define a global variable TEST_KEY which must be unique
+for each test using
+
+set_test_key("ABCDE")
+
+The test procedure (invoked by pytest) checks whether a registration
+information is stored in one of the server properties or otherwise
+- offers to register this test in the currently running database ONLY if this
+ is empty.
+- fails otherwise with a RuntimeError
+
+NOTE: you probably need to use pytest with the -s option to be able to
+ register the test interactively. Otherwise, the server property has to be
+ set before server start-up in the server.conf of the CaosDB server.
+
+This module is intended to be used with pytest.
+
+There is a pytest fixture "clear_database" that performs the above mentioned
+checks and clears the database in case of success.
+"""
+
+TEST_KEY = None
+
+
+def set_test_key(KEY):
+ global TEST_KEY
+ TEST_KEY = KEY
+
+
+def _register_test():
+ res = db.execute_query("COUNT Entity")
+ if not isinstance(res, int):
+ raise RuntimeError("Response from server for Info could not be interpreted.")
+ if res > 0:
+ raise RuntimeError("This instance of CaosDB contains entities already."
+ "It must be empty in order to register a new test.")
+
+ print("Current host of CaosDB instance is: {}".format(
+ db.connection.connection.get_connection()._delegate_connection.setup_fields["host"]))
+ answer = input("This method will register your current test with key {} with the currently"
+ " running instance of CaosDB. Do you want to continue (y/N)?".format(
+ TEST_KEY))
+ if answer != "y":
+ raise RuntimeError("Test registration aborted by user.")
+
+ admin.set_server_property("_CAOSDB_INTEGRATION_TEST_SUITE_KEY",
+ TEST_KEY)
+
+
+def _get_registered_test_key():
+ try:
+ return admin.get_server_property("_CAOSDB_INTEGRATION_TEST_SUITE_KEY")
+ except KeyError:
+ return None
+
+
+def _is_registered():
+ registered_test_key = _get_registered_test_key()
+ if not registered_test_key:
+ return False
+ elif registered_test_key == TEST_KEY:
+ return True
+ else:
+ raise RuntimeError("The database has been setup for a different test.")
+
+
+def _assure_test_is_registered():
+ global TEST_KEY
+ if TEST_KEY is None:
+ raise RuntimeError("TEST_KEY is not defined.")
+ if not _is_registered():
+ answer = input("Do you want to register this instance of CaosDB"
+ " with the current test? Do you want to continue (y/N)?")
+ if answer == "y":
+ _register_test()
+ raise RuntimeError("Test has been registered. Please rerun tests.")
+ else:
+ raise RuntimeError("The database has not been setup for this test.")
+
+
+def _clear_database():
+ c = db.execute_query("FIND ENTITY WITH ID>99")
+ c.delete(raise_exception_on_error=False)
+ return None
+
+
+try:
+ import pytest
+
+ @pytest.fixture
+ def clear_database():
+ """Remove Records, RecordTypes, Properties, and Files ONLY IF the CaosDB
+ server the current connection points to was registered with the appropriate key.
+
+ PyTestInfo Records and the corresponding RecordType and Property are preserved.
+ """
+ _assure_test_is_registered()
+ yield _clear_database() # called before the test function
+ _clear_database() # called after the test function
+except ImportError:
+ raise Warning("""The register_tests module depends on pytest and is
+ intended to be used in integration test suites for the
+ caosdb-pylib library only.""")
diff --git a/src/caosdb/utils/server_side_scripting.py b/src/caosdb/utils/server_side_scripting.py
index 663178dcbda4293cb30dff88efbfb7b7302df70d..7e5ee4390ae3314792d12fd2942980aa3d9c9773 100644
--- a/src/caosdb/utils/server_side_scripting.py
+++ b/src/caosdb/utils/server_side_scripting.py
@@ -30,7 +30,8 @@ from lxml import etree
from caosdb.connection.connection import get_connection
from caosdb.connection.utils import urlencode
-from caosdb.connection.encode import MultipartParam, multipart_encode
+from caosdb.connection.encode import (MultipartParam, multipart_encode,
+ ReadableMultiparts)
def _make_params(pos_args, opts):
@@ -63,6 +64,7 @@ def _make_multipart_request(call, pos_args, opts, files):
filename=filename))
body, headers = multipart_encode(parts)
+ body = ReadableMultiparts(body)
return body, headers
diff --git a/src/doc/administration.rst b/src/doc/administration.rst
index 061acc8364d2ef62f743a20d7b9e6562baac0fc5..eab02e43a833559dc21ea7a9fa5edfaf6431facf 100644
--- a/src/doc/administration.rst
+++ b/src/doc/administration.rst
@@ -5,10 +5,12 @@ The Python script ``caosdb_admin.py`` should be used for administrative tasks.
Call ``caosdb_admin.py --help`` to see how to use it.
The most common task is to create a new user (in the CaosDB realm) and set a
-password for the user (note that a user typically needs to be activated)::
+password for the user (note that a user typically needs to be activated):
- caosdb_admin.py create_user anna
- caosdb_admin.py set_user_password anna
- caosdb_admin.py add_user_roles anna administration
- caosdb_admin.py activate_user anna
+.. code:: console
+
+ $ caosdb_admin.py create_user anna
+ $ caosdb_admin.py set_user_password anna
+ $ caosdb_admin.py add_user_roles anna administration
+ $ caosdb_admin.py activate_user anna
diff --git a/src/doc/conf.py b/src/doc/conf.py
index 9e65bff1eba5d114a77d3ed9405e883df6ad7470..15fa844e546a40efa99f577944c377f0a6a0c19a 100644
--- a/src/doc/conf.py
+++ b/src/doc/conf.py
@@ -29,10 +29,10 @@ copyright = '2022, IndiScale GmbH'
author = 'Daniel Hornung'
# The short X.Y version
-version = '0.9.0'
+version = '0.10.1'
# The full version, including alpha/beta/rc tags
# release = '0.5.2-rc2'
-release = '0.9.0-dev'
+release = '0.10.1-dev'
# -- General configuration ---------------------------------------------------
diff --git a/src/doc/gallery/curator-permissions.rst b/src/doc/gallery/curator-permissions.rst
new file mode 100644
index 0000000000000000000000000000000000000000..fa6b4022b7fbc1d042ed00f265e63a2675794a21
--- /dev/null
+++ b/src/doc/gallery/curator-permissions.rst
@@ -0,0 +1,123 @@
+
+Setting permissions for a curator role
+======================================
+
+The following example shows how to create and set permissions for a ``curator``
+role that is allowed to insert, update, or delete any entity apart from a set of
+RecordTypes and properties that define a "core data model" which can only be
+altered with administration permissions.
+
+In the following, you'll learn how to
+
+1. create the ``curator`` role.
+2. configure the ``global_entity_permissions.xml`` s.th. the ``curator`` role is
+ allowed to insert, update, or delete any entity by default.
+3. use a Python script to override the above configuration for the entities in
+ the externally defined core data model.
+
+Prerequisites
+-------------
+
+This example needs some preparations regarding your CaosDB setup that have to
+(or, for the sake of simplicity, should) be done outside the actual Python
+example script.
+
+The curator role
+~~~~~~~~~~~~~~~~
+
+First, a ``curator`` role is created with a meaningful description. We'll use
+``caosdb_admin.py`` for this which leads to the following command:
+
+.. code:: console
+
+ $ caosdb_admin.py create_role "curator" "A user who is permitted to create new Records, Properties, and RecordTypes but who is not allowed to change the core data model."
+
+To actually see how this role's permissions change, we also need a user with
+this role. Assume you already have created and activated (see
+:doc:`Administration <../administration>`) a ``test_curator`` user, then
+``caosdb_admin.py`` is used again to assign it the correct role:
+
+.. code:: console
+
+ $ caosdb_admin.py add_user_roles test_curator curator
+
+.. note::
+
+ The ``test_curator`` user shouldn't have administration privileges, otherwise
+ the below changes won't have any effect.
+
+The core data model and caosdb-advanced-user-tools
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+In principle, the following script works with any data model defined in a json
+or yaml file (just adapt lines 39-42 accordingly). In this example, we'll use the
+`metadata schema <https://github.com/leibniz-zmt/zmt-metadata-schema>`_ that was
+developed by J. Schmidt at the `Leibniz Centre for Tropical Marine Research
+<https://www.leibniz-zmt.de/en/>`_.
+
+Clone the schemata into the same directory containing the below script via
+
+.. code:: console
+
+ $ git clone https://github.com/leibniz-zmt/zmt-metadata-schema.git
+
+Furthermore, we'll need the `CaosDB Advanced User Tools
+<https://gitlab.com/caosdb/caosdb-advanced-user-tools>`_ for loading the
+metadata schemata from the json files, so install them via
+
+.. code:: console
+
+ $ pip install caosadvancedtools
+
+The global entity permissions file
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Users with the ``curator`` role should be able to have any permission for all
+entities by default. The exceptions for the core data model entities will be set
+with the script below. These default settings are best done via the
+``global_entities_permissions.xml`` config file (see the `server documentation
+<https://docs.indiscale.com/caosdb-server/permissions.html#how-to-set-permissions>`_). Simply
+add the following line to the file
+
+.. code:: xml
+
+ <Grant priority="true" role="curator"><Permission name="*"/></Grant>
+
+This means that, by default, all users with the ``curator`` role are **granted**
+all entity permissions (including insert, update, and delete as specified in the
+beginning) **with priority**. This ensures, that no normal user is allowed to
+overrule these permissions (since it is granted with priority), but it can still
+be denied for the core data model entities by a **deny** rule with priority. See
+the server documentation on `permission
+calculation <https://docs.indiscale.com/caosdb-server/permissions.html#permission-calculation>`_
+for more information on which permission rules can or can't be overruled.
+
+Your complete ``global_entities_permissions.xml`` might then look like
+
+.. code:: xml
+
+ <globalPermissions>
+ <Grant priority="false" role="?OWNER?"><Permission name="*"/></Grant>
+ <Grant priority="false" role="?OTHER?"><Permission name="RETRIEVE:*"/></Grant>
+ <Grant priority="false" role="?OTHER?"><Permission name="USE:*"/></Grant>
+ <Grant priority="false" role="anonymous"><Permission name="RETRIEVE:*"/></Grant>
+ <Grant priority="true" role="curator"><Permission name="*"/></Grant>
+ <Deny priority="false" role="?OTHER?"><Permission name="UPDATE:*"/></Deny>
+ <Deny priority="false" role="?OTHER?"><Permission name="DELETE"/></Deny>
+ <Deny priority="true" role="?OTHER?"><Permission name="EDIT:ACL"/></Deny>
+ </globalPermissions>
+
+.. note::
+
+ Note that you have to restart your CaosDB server after modifying the
+ ``global_entities_permissions.xml``.
+
+The code
+--------
+
+After having applied all of the above prerequisites and restarting your CaosDB
+server, execute the following code.
+
+:download:`Download full code<curator_permissions.py>`
+
+.. literalinclude:: curator_permissions.py
diff --git a/src/doc/gallery/curator_permissions.py b/src/doc/gallery/curator_permissions.py
new file mode 100644
index 0000000000000000000000000000000000000000..16b4b7f6f1bb9abfb7e191c6a1101181984bce9a
--- /dev/null
+++ b/src/doc/gallery/curator_permissions.py
@@ -0,0 +1,100 @@
+#!/usr/bin/env python3
+# encoding: utf-8
+#
+# This file is a part of the CaosDB Project.
+#
+# Copyright (C) 2022 Indiscale GmbH <info@indiscale.com>
+# Copyright (C) 2022 Florian Spreckelsen <f.spreckelsen@indiscale.com>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see <https://www.gnu.org/licenses/>.
+#
+
+import os
+import sys
+
+import caosdb as db
+from caosadvancedtools.models.parser import parse_model_from_json_schema
+from caosdb import administration as admin
+
+CURATOR = "curator"
+
+
+def main():
+ """Set curator role permissions: Is allowed to edit all Records; is allowed
+ to create new RTs and Properties and change them, but is not allowed to
+ change anything defined in the core data model, i.e., in the schemas.
+
+ """
+ dataspace_definitions = parse_model_from_json_schema(
+ "zmt-metadata-schema/schemas/dataspace.schema.json")
+ dataset_definitions = parse_model_from_json_schema(
+ "zmt-metadata-schema/schemas/dataset.schema.json")
+
+ # Set general permissions. The curator users should be allowed to perform
+ # any transaction.
+ perms = admin._get_permissions(CURATOR)
+ general_grant_perms = [
+ "TRANSACTION:*"
+ ]
+
+ for p in general_grant_perms:
+
+ g = admin.PermissionRule(action="Grant", permission=p, priority=True)
+ d = admin.PermissionRule(action="Deny", permission=p, priority=True)
+
+ if g in perms:
+ perms.remove(g)
+ if d in perms:
+ perms.remove(d)
+ perms.add(g)
+
+ admin._set_permissions(CURATOR, permission_rules=perms)
+
+ # Deny all permissions that could change the data model ...
+ core_model_deny_permissions = [
+ "DELETE",
+ "UPDATE:*",
+ "EDIT:ACL"
+ ]
+ # ... but allow read-access and of course using the entities as parents,
+ # properties, ...
+ core_model_grant_permissions = [
+ "RETRIEVE:*",
+ "USE:*",
+ ]
+
+ # Iterate over all entities defined in the schemas and update their access control list (ACL) accordingly.
+ updates = db.Container()
+ for model in [dataspace_definitions, dataset_definitions]:
+
+ for ent in model.values():
+ if ent.name in [u.name for u in updates]:
+ # Skip entities that have been updated already
+ continue
+ # The entity needs to be retrieved with the ACL flag to update the
+ # ACL down the road
+ ent.retrieve(flags={"ACL": None})
+ for d in core_model_deny_permissions:
+ ent.deny(role=CURATOR, priority=True, permission=d)
+ ent.update_acl()
+ ent.retrieve(flags={"ACL": None})
+ for g in core_model_grant_permissions:
+ ent.grant(role=CURATOR, priority=True, permission=g)
+ updates.append(ent)
+ ent.update_acl()
+
+
+if __name__ == "__main__":
+
+ sys.exit(main())
diff --git a/src/doc/gallery/index.rst b/src/doc/gallery/index.rst
index a6ef53e4c7d1272c5dbc8c62b4d90a89591cac0f..bfba4317c3556d0692eb402f42ba3699be586d5a 100644
--- a/src/doc/gallery/index.rst
+++ b/src/doc/gallery/index.rst
@@ -14,3 +14,4 @@ This chapter collects code examples which can be immediately run against an empt
:caption: The code examples:
simulation
+ curator-permissions
diff --git a/src/doc/tutorials/basic_analysis.rst b/src/doc/tutorials/basic_analysis.rst
index cc185e0ee08f9e5ee0f890c0ab55f52972882d17..c40cad28b8c9a3be537c641b9614da2eb4df8dd9 100644
--- a/src/doc/tutorials/basic_analysis.rst
+++ b/src/doc/tutorials/basic_analysis.rst
@@ -34,7 +34,7 @@ Often we are interested in table like data for our processing. And the disentang
>>> from caosadvancedtools.table_converter import to_table
>>> # Let us retrieve the data in a table like form using `SELECT`
->>> data = db.execute_query("SELECT quality_factor FROM RECORD Analysis with quality_factor" )
+>>> data = db.execute_query("SELECT quality_factor FROM Analysis with quality_factor" )
>>> table = to_table(data)
>>> print(table)
quality_factor
diff --git a/src/doc/tutorials/first_steps.rst b/src/doc/tutorials/first_steps.rst
index 34b96bbeca416107fb34feb4707b9ef46fc49fe7..486cd4d437c8b13a253cadc8ed45f49b4a7634e4 100644
--- a/src/doc/tutorials/first_steps.rst
+++ b/src/doc/tutorials/first_steps.rst
@@ -87,7 +87,7 @@ Ids can also come in handy when searching. Suppose you have some complicated con
>>> # This condition is not that complicated and long but let's suppose it was.
->>> record = db.execute_query("FIND Analysis with quality_factor=0.08", unique=True)
+>>> record = db.execute_query("FIND MusicalAnalysis with quality_factor=0.08", unique=True)
>>> # You can use unique=True when you only expect one result Entity. An error will be
>>> # thrown if the number of results is unequal to 1 and the resulting object will be
>>> # an Entity and not a Container
diff --git a/tox.ini b/tox.ini
index e321891883b9425d24543a41f2d1283e0be52109..50c22d5716769ef2ec818f6c8fb94491ea372434 100644
--- a/tox.ini
+++ b/tox.ini
@@ -7,7 +7,6 @@ deps = .
nose
pytest
pytest-cov
- python-dateutil
jsonschema==4.0.1
commands=py.test --cov=caosdb -vv {posargs}
diff --git a/unittests/docker/Dockerfile b/unittests/docker/Dockerfile
index 7fa3f75bd198724628dee48ab328829fa071a639..06f9d6c830068a2c1c85caef79c64f899eaefb33 100644
--- a/unittests/docker/Dockerfile
+++ b/unittests/docker/Dockerfile
@@ -1,4 +1,8 @@
FROM debian:latest
+# Use local package repository
+COPY sources.list.local /etc/apt/
+RUN mv /etc/apt/sources.list /etc/apt/sources.list.orig
+RUN cat /etc/apt/sources.list.local /etc/apt/sources.list.orig > /etc/apt/sources.list
RUN apt-get update && \
apt-get install -y \
pylint3 python3-pip tox git \
diff --git a/unittests/docker/sources.list.local b/unittests/docker/sources.list.local
new file mode 100644
index 0000000000000000000000000000000000000000..c0b4107350ba37e77aa95d5a56c31976979e51e1
--- /dev/null
+++ b/unittests/docker/sources.list.local
@@ -0,0 +1,6 @@
+# Local repositories at Netcup
+deb http://debian.netcup.net/debian/ buster main
+deb http://mirrors.n-ix.net/debian-security buster/updates main
+deb http://debian.netcup.net/debian/ buster-updates main
+
+# The original content follows here:
\ No newline at end of file
diff --git a/unittests/test_apiutils.py b/unittests/test_apiutils.py
index 43ab8107183f16bf8df1d0ea8e447b378bcf8123..9b87a743b201213feb49cfc660e006f2c6217387 100644
--- a/unittests/test_apiutils.py
+++ b/unittests/test_apiutils.py
@@ -1,11 +1,12 @@
-# -*- encoding: utf-8 -*-
#
# This file is a part of the CaosDB Project.
#
-# Copyright (C) 2018 Research Group Biomedical Physics,
-# Max-Planck-Institute for Dynamics and Self-Organization Göttingen
# Copyright (C) 2020 Timm Fitschen <t.fitschen@indiscale.com>
+# Copyright (C) 2022 Florian Spreckelsen <f.spreckelsen@indiscale.com>
+# Copyright (C) 2022 Daniel Hornung <d.hornung@indiscale.com>
# Copyright (C) 2020-2022 IndiScale GmbH <info@indiscale.com>
+# Copyright (C) 2018 Research Group Biomedical Physics,
+# Max-Planck-Institute for Dynamics and Self-Organization Göttingen
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as
@@ -20,7 +21,6 @@
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <https://www.gnu.org/licenses/>.
#
-# ** end header
#
# Test apiutils
# A. Schlemmer, 02/2018
@@ -30,6 +30,7 @@ import pytest
import caosdb as db
import caosdb.apiutils
from caosdb.apiutils import (apply_to_ids, compare_entities, create_id_query,
+ empty_diff, EntityMergeConflictError,
resolve_reference, merge_entities)
from caosdb.common.models import SPECIAL_ATTRIBUTES
@@ -272,8 +273,10 @@ def test_copy_entities():
for i in [0, 1]:
assert c.properties[i] is not r.properties[i]
for special in SPECIAL_ATTRIBUTES:
- assert getattr(c.properties[i], special) == getattr(r.properties[i], special)
- assert c.get_importance(c.properties[i]) == r.get_importance(r.properties[i])
+ assert getattr(c.properties[i], special) == getattr(
+ r.properties[i], special)
+ assert c.get_importance(
+ c.properties[i]) == r.get_importance(r.properties[i])
def test_merge_entities():
@@ -296,6 +299,19 @@ def test_merge_entities():
assert r2.get_property("F").value == "text"
+def test_merge_bug_conflict():
+ r = db.Record()
+ r.add_property(name="C", value=4)
+ r2 = db.Record()
+ r2.add_property(name="C", value=4, datatype="TEXT")
+ merge_entities(r, r2)
+
+ r3 = db.Record()
+ r3.add_property(name="C", value=4, datatype="INTEGER")
+ with pytest.raises(EntityMergeConflictError):
+ merge_entities(r3, r2)
+
+
def test_merge_bug_109():
rt = db.RecordType(name="TestBug")
p = db.Property(name="test_bug_property", datatype=db.LIST(db.INTEGER))
@@ -313,10 +329,12 @@ def test_merge_bug_109():
assert r_a.get_property("test_bug_property").value == [18, 19]
assert "<Value>18</Value>\n <Value>19</Value>" in str(r_b)
- assert "<Value>18</Value>\n <Value>19</Value>\n <Value>18</Value>\n <Value>19</Value>" not in str(r_b)
+ assert "<Value>18</Value>\n <Value>19</Value>\n <Value>18</Value>\n <Value>19</Value>" not in str(
+ r_b)
assert "<Value>18</Value>\n <Value>19</Value>" in str(r_a)
- assert "<Value>18</Value>\n <Value>19</Value>\n <Value>18</Value>\n <Value>19</Value>" not in str(r_a)
+ assert "<Value>18</Value>\n <Value>19</Value>\n <Value>18</Value>\n <Value>19</Value>" not in str(
+ r_a)
@pytest.mark.xfail
@@ -336,7 +354,208 @@ def test_bug_109():
assert r_a.get_property("test_bug_property").value == [18, 19]
assert "<Value>18</Value>\n <Value>19</Value>" in str(r_b)
- assert "<Value>18</Value>\n <Value>19</Value>\n <Value>18</Value>\n <Value>19</Value>" not in str(r_b)
+ assert "<Value>18</Value>\n <Value>19</Value>\n <Value>18</Value>\n <Value>19</Value>" not in str(
+ r_b)
assert "<Value>18</Value>\n <Value>19</Value>" in str(r_a)
- assert "<Value>18</Value>\n <Value>19</Value>\n <Value>18</Value>\n <Value>19</Value>" not in str(r_a)
+ assert "<Value>18</Value>\n <Value>19</Value>\n <Value>18</Value>\n <Value>19</Value>" not in str(
+ r_a)
+
+
+def test_wrong_merge_conflict_reference():
+ """Test a wrongly detected merge conflict in case of two records referencing
+ two different, but identical objects.
+
+ """
+ # Two identical license records will be referenced from both records to be
+ # merged
+ license_rt = db.RecordType(name="license")
+ license_rec_a = db.Record(name="CC-BY-3.0").add_parent(license_rt)
+ license_rec_b = db.Record(name="CC-BY-3.0").add_parent(license_rt)
+
+ # two referencing records
+ dataset_rt = db.RecordType(name="Dataset")
+ title_prop = db.Property(name="title", datatype=db.TEXT)
+ doi_prop = db.Property(name="DOI", datatype=db.TEXT)
+ rec_a = db.Record().add_parent(dataset_rt)
+ rec_a.add_property(name=license_rt.name,
+ datatype=license_rt.name, value=license_rec_a)
+ rec_a.add_property(name=title_prop.name, value="Some dataset title")
+
+ rec_b = db.Record().add_parent(dataset_rt)
+ rec_b.add_property(name=license_rt.name,
+ datatype=license_rt.name, value=license_rec_b)
+ rec_b.add_property(name=doi_prop.name, value="https://doi.org/12345.678")
+
+ merge_entities(rec_a, rec_b)
+ assert rec_a.get_property(license_rt.name) is not None
+ assert rec_a.get_property(license_rt.name).value is not None
+ assert isinstance(rec_a.get_property(license_rt.name).value, db.Record)
+ assert rec_a.get_property(license_rt.name).value.name == license_rec_a.name
+ assert rec_a.get_property(license_rt.name).value.name == license_rec_b.name
+ assert rec_a.get_property("title").value == "Some dataset title"
+ assert rec_a.get_property("doi").value == "https://doi.org/12345.678"
+
+ # Reset rec_a
+ rec_a = db.Record().add_parent(dataset_rt)
+ rec_a.add_property(name=license_rt.name,
+ datatype=license_rt.name, value=license_rec_a)
+ rec_a.add_property(name=title_prop.name, value="Some dataset title")
+
+ # this does not compare referenced records, so it will fail
+ with pytest.raises(EntityMergeConflictError):
+ merge_entities(rec_a, rec_b, merge_references_with_empty_diffs=False)
+
+ # ... as should this, of course
+ rec_b.get_property(license_rt.name).value.name = "Another license"
+ with pytest.raises(EntityMergeConflictError) as re:
+ merge_entities(rec_a, rec_b)
+
+
+def test_empty_diff():
+
+ rec_a = db.Record(name="A")
+ rec_b = db.Record(name="B")
+
+ assert empty_diff(rec_a, rec_a)
+ assert not empty_diff(rec_a, rec_b)
+
+ rec_a.add_parent(name="RT")
+ rec_b.add_parent(name="RT")
+ assert empty_diff(rec_a, rec_a)
+ assert not empty_diff(rec_a, rec_b)
+
+ rec_b.name = "A"
+ assert empty_diff(rec_a, rec_b)
+
+ rec_a.add_property(name="some_prop", value=1)
+ assert not empty_diff(rec_a, rec_b)
+
+ rec_b.add_property(name="some_prop", value=1)
+ assert empty_diff(rec_a, rec_b)
+
+ rec_b.get_property("some_prop").value = 2
+ assert not empty_diff(rec_a, rec_b)
+
+ rec_b.get_property("some_prop").value = 1
+ rec_b.add_property(name="some_other_prop", value="Test")
+ assert not empty_diff(rec_a, rec_b)
+
+ rec_a.add_property(name="some_other_prop", value="Test")
+ assert empty_diff(rec_a, rec_b)
+
+ # reference identical records, but different Python Record objects
+ ref_rec_a = db.Record(name="Ref").add_parent(name="RefType")
+ ref_rec_b = db.Record(name="Ref").add_parent(name="RefType")
+ rec_a.add_property(name="RefType", datatype="RefType", value=ref_rec_a)
+ rec_b.add_property(name="RefType", datatype="RefType", value=ref_rec_b)
+ # the default is `compare_referenced_records=False`, so the diff shouldn't
+ # be empty (different Python objects are referenced.)
+ assert not empty_diff(rec_a, rec_b)
+ # when looking into the referenced record, the diffs should be empty again
+ assert empty_diff(rec_a, rec_b, compare_referenced_records=True)
+
+ # The same for lists of references
+ rec_a.remove_property("RefType")
+ rec_b.remove_property("RefType")
+ assert empty_diff(rec_a, rec_b)
+ rec_a.add_property(name="RefType", datatype=db.LIST(
+ "RefType"), value=[ref_rec_a, ref_rec_a])
+ rec_b.add_property(name="RefType", datatype=db.LIST(
+ "RefType"), value=[ref_rec_b, ref_rec_b])
+ assert not empty_diff(rec_a, rec_b)
+ assert empty_diff(rec_a, rec_b, compare_referenced_records=True)
+
+ # special case of ids
+ rec_a = db.Record(id=12)
+ rec_b = db.Record()
+ assert not empty_diff(rec_a, rec_b)
+ rec_b.id = 13
+ assert not empty_diff(rec_a, rec_b)
+ rec_b.id = 12
+ assert empty_diff(rec_a, rec_b)
+
+
+def test_force_merge():
+ """Test whether a forced merge overwrites existing properties correctly."""
+
+ # name overwrite
+ recA = db.Record(name="A")
+ recB = db.Record(name="B")
+
+ with pytest.raises(EntityMergeConflictError):
+ merge_entities(recA, recB)
+
+ merge_entities(recA, recB, force=True)
+ assert "B" == recA.name
+ # unchanged
+ assert "B" == recB.name
+
+ # description overwrite
+ recA = db.Record()
+ recA.description = "something"
+ recB = db.Record()
+ recB.description = "something else"
+
+ with pytest.raises(EntityMergeConflictError) as emce:
+ merge_entities(recA, recB)
+ assert str(emce.value) == """Conflict in special attribute description:
+A: something
+B: something else"""
+
+ merge_entities(recA, recB, force=True)
+ assert recA.description == "something else"
+ # unchanged
+ assert recB.description == "something else"
+
+ # property overwrite
+ recA = db.Record()
+ recA.add_property(name="propA", value="something")
+ recB = db.Record()
+ recB.add_property(name="propA", value="something else")
+
+ with pytest.raises(EntityMergeConflictError):
+ merge_entities(recA, recB)
+
+ merge_entities(recA, recB, force=True)
+ assert recA.get_property("propA").value == "something else"
+ # unchanged
+ assert recB.get_property("propA").value == "something else"
+
+ # don't remove a property that's not in recB
+ recA = db.Record()
+ recA.add_property(name="propA", value="something")
+ recA.add_property(name="propB", value=5.0)
+ recB = db.Record()
+ recB.add_property(name="propA", value="something else")
+
+ merge_entities(recA, recB, force=True)
+ assert recA.get_property("propA").value == "something else"
+ assert recA.get_property("propB").value == 5.0
+
+ # also overwrite datatypes ...
+ rtA = db.RecordType()
+ rtA.add_property(name="propA", datatype=db.INTEGER)
+ rtB = db.RecordType()
+ rtB.add_property(name="propA", datatype=db.TEXT)
+
+ with pytest.raises(EntityMergeConflictError):
+ merge_entities(rtA, rtB)
+
+ merge_entities(rtA, rtB, force=True)
+ assert rtA.get_property("propA").datatype == db.TEXT
+ # unchanged
+ assert rtB.get_property("propA").datatype == db.TEXT
+
+ # ... and units
+ recA = db.Record()
+ recA.add_property(name="propA", value=5, unit="m")
+ recB = db.Record()
+ recB.add_property(name="propA", value=5, unit="cm")
+
+ with pytest.raises(EntityMergeConflictError):
+ merge_entities(recA, recB)
+ merge_entities(recA, recB, force=True)
+ assert recA.get_property("propA").unit == "cm"
+ # unchanged
+ assert recB.get_property("propA").unit == "cm"
diff --git a/unittests/test_configs/pycaosdb6.ini b/unittests/test_configs/pycaosdb6.ini
new file mode 100644
index 0000000000000000000000000000000000000000..3826564f043c5702385a3d093cb4ebb8d4c24cd2
--- /dev/null
+++ b/unittests/test_configs/pycaosdb6.ini
@@ -0,0 +1,4 @@
+[Connection]
+url=https://localhost:10443/
+# No username, unauthenticated connection
+password_method = unauthenticated
diff --git a/unittests/test_connection.py b/unittests/test_connection.py
index ee564ea033f9afc80522d75a85557f70819ece1e..6cc23d87c5cdcf639709a444849a856a8c70af5f 100644
--- a/unittests/test_connection.py
+++ b/unittests/test_connection.py
@@ -37,7 +37,8 @@ from caosdb.connection.connection import (CaosDBServerConnection,
from caosdb.connection.mockup import (MockUpResponse, MockUpServerConnection,
_request_log_message)
from caosdb.connection.utils import make_uri_path, quote, urlencode
-from caosdb.exceptions import ConfigurationError, LoginFailedError
+from caosdb.exceptions import (ConfigurationError, LoginFailedError,
+ CaosDBConnectionError)
from nose.tools import assert_equal as eq
from nose.tools import assert_false as falz
from nose.tools import assert_is_not_none as there
@@ -46,6 +47,13 @@ from nose.tools import assert_true as tru
from pytest import raises
+def setup_function(function):
+ configure_connection(url="http://localhost:8888/some/path",
+ password_method="plain", username="test",
+ password="blub",
+ implementation=MockUpServerConnection)
+
+
def setup_module():
_reset_config()
@@ -103,6 +111,7 @@ def test_configure_connection():
get_config().set("Connection", "password_method", "plain")
get_config().set("Connection", "password", "test_password")
get_config().set("Connection", "timeout", "200")
+ get_config().set("Connection", "ssl_insecure", "True")
there(configure_connection)
tru(hasattr(configure_connection, "__call__"))
@@ -116,6 +125,18 @@ def test_configure_connection():
tru(isinstance(c._delegate_connection, MockUpServerConnection))
+def test_configure_connection_bad_url():
+ configure_connection(url="https://localhost:8888")
+ with raises(CaosDBConnectionError) as exc_info:
+ configure_connection(url="ftp://localhost:8888")
+ assert exc_info.value.args[0].startswith(
+ "The connection url is expected to be a http or https url")
+ with raises(CaosDBConnectionError) as exc_info:
+ configure_connection(url="localhost:8888")
+ assert exc_info.value.args[0].startswith(
+ "The connection url is expected to be a http or https url")
+
+
def test_connection_interface():
with raiz(TypeError) as cm:
CaosDBServerConnection()
diff --git a/unittests/test_server_side_scripting.py b/unittests/test_server_side_scripting.py
index 1fb24d7e40bb843391a971c5f69680b541e1de0e..b699c4482d02972282167eb9683a956097ebc5e9 100644
--- a/unittests/test_server_side_scripting.py
+++ b/unittests/test_server_side_scripting.py
@@ -46,7 +46,7 @@ def setup_module():
content_type = kwargs["headers"]["Content-Type"]
if content_type.startswith("multipart/form-data; boundary"):
- parts = kwargs["body"]
+ parts = kwargs["body"].multipart_yielder
stdout = []
for part in parts:
if hasattr(part, "decode"):