Merge branch 'dev' into f-mypy

.gitlab-ci.yml

@@ -39,15 +39,18 @@ stages:
 code_style:
   tags: [ docker ]
   stage: code_style
+  needs: [ ]
   script:
-    - pycodestyle --count ./
+    - make style
   allow_failure: true
 
+# pylint tests for pycaosdb
 pylint:
   tags: [ docker ]
   stage: linting
+  needs: [ ]
   script:
-    - pylint3 --unsafe-load-any-extension=y -d all -e E,F src/caosdb/common
+    - make lint
   allow_failure: true
 
 mypy:
@@ -57,26 +60,33 @@ mypy:
     - mypy  src/caosdb
   allow_failure: true
 
-# pylint tests for pycaosdb
-test:
+# run unit tests
+unittest:
   tags: [ docker ]
   stage: test
+  needs: [ ]
   script:
     - touch ~/.pycaosdb.ini
-    - tox -r 
+    - make unittest
 
 # Trigger building of server image and integration tests
 trigger_build:
-  tags: [ docker ]
   stage: deploy
-  script:
-    - /usr/bin/curl -X POST
-      -F token=$CI_JOB_TOKEN
-      -F "variables[F_BRANCH]=$CI_COMMIT_REF_NAME"
-      -F "variables[PYLIB]=$CI_COMMIT_REF_NAME"
-      -F "variables[TriggerdBy]=PYLIB"
-      -F "variables[TriggerdByHash]=$CI_COMMIT_SHORT_SHA"
-      -F ref=$DEPLOY_REF https://gitlab.indiscale.com/api/v4/projects/14/trigger/pipeline
+  inherit:
+    variables:
+      - DEPLOY_REF
+  variables:
+    # Renaming variables.
+    F_BRANCH: $CI_COMMIT_REF_NAME
+    PYLIB: $CI_COMMIT_REF_NAME
+    TRIGGERED_BY_REPO: PYLIB
+    TRIGGERED_BY_REF: $CI_COMMIT_REF_NAME
+    TRIGGERED_BY_HASH: $CI_COMMIT_SHORT_SHA
+
+  trigger:
+    project: caosdb/src/caosdb-deploy
+    branch: $DEPLOY_REF
+    strategy: depend
 
 # Build a docker image in which tests for this repository can run
 build-testenv:
@@ -103,6 +113,7 @@ build-testenv:
 pages_prepare: &pages_prepare
   tags: [ cached-dind ]
   stage: deploy
+  needs: [ code_style, pylint, unittest ]
   only:
     refs:
       - /^release-.*$/i

.gitlab/merge_request_templates/Default.md

-# Summary
-
-    Insert a meaningful description for this merge request here.  What is the
-    new/changed behavior? Which bug has been fixed? Are there related Issues?
-
-# Focus
-
-    Point the reviewer to the core of the code change. Where should they start
-    reading? What should they focus on (e.g. security, performance,
-    maintainability, user-friendliness, compliance with the specs, finding more
-    corner cases, concrete questions)?
-
-# Test Environment
-
-    How to set up a test environment for manual testing?
-
-# Check List for the Author
-
-Please, prepare your MR for a review. Be sure to write a summary and a
-focus and create gitlab comments for the reviewer. They should guide the
-reviewer through the changes, explain your changes and also point out open
-questions. For further good practices have a look at [our review
-guidelines](https://gitlab.com/caosdb/caosdb/-/blob/dev/REVIEW_GUIDELINES.md)
-
-- [ ] All automated tests pass
-- [ ] Reference related Issues
-- [ ] Up-to-date CHANGELOG.md
-- [ ] Annotations in code (Gitlab comments)
-  - Intent of new code
-  - Problems with old code
-  - Why this implementation?
-
-
-# Check List for the Reviewer
-
-
-- [ ] I understand the intent of this MR
-- [ ] All automated tests pass
-- [ ] Up-to-date CHANGELOG.md
-- [ ] The test environment setup works and the intended behavior is
-  reproducible in the test environment
-- [ ] In-code documentation and comments are up-to-date.
-- [ ] Check: Are there spezifications? Are they satisfied?
-
-For further good practices have a look at [our review guidelines](https://gitlab.com/caosdb/caosdb/-/blob/dev/REVIEW_GUIDELINES.md).
-
-
-/assign me
-/target_branch dev

CHANGELOG.md

@@ -5,26 +5,153 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
-## [Unreleased] ##
+## [Unreleased]
 
 ### Added ###
 
+### Changed ###
+
+* Set PyYAML dependency back to PyYaml>=5.4.1 (from 6.0) for better
+  compatibility with docker-compose)
+
+### Deprecated ###
+
+### Removed ###
+
+### Fixed ###
+
+### Security ###
+
+### Documentation ###
+
+## [0.8.0] - 2022-07-12
+(Timm Fitschen)
+
+### Removed ###
+
+* Support for Python 3.6 and Python 3.7
+
+### Fixed ###
+
+* `read()` of MockupResponse returns now an appropriate type on modern systems
+* [caosdb-server#142](https://gitlab.com/caosdb/caosdb-server/-/issues/142)
+  Can't create users with dots in their user names
+
+## [0.7.4] - 2022-05-31
+(Florian Spreckelsen)
+
+### Fixed ###
+
+* [#64](https://gitlab.com/caosdb/caosdb-pylib/-/issues/64) Use `Dict[]` and
+  `List[]` from `typing` for type hinting instead of `dict[]` and `list[]` for
+  compatibility with Python<3.9.
+
+## [0.7.3] - 2022-05-03
+(Henrik tom Wörden)
+
+### Added ###
+
+- New function in apiutils that copies an Entity.
+- New EXPERIMENTAL module `high_level_api` which is a completely refactored version of
+  the old `high_level_api` from apiutils. Please see the included documentation for details.
+- `to_graphics` now has  `no_shadow` option.
+
+### Changed ###
+
+- Added additional customization options to the plantuml module.
+- The to_graphics function in the plantuml module uses a temporary directory now for creating the output files.
+
+### Fixed ###
+
+* [#75](https://gitlab.indiscale.com/caosdb/src/caosdb-pylib/-/issues/75), [#103](https://gitlab.indiscale.com/caosdb/src/caosdb-pylib/-/issues/103) Fixed JSON schema to allow more sections, and correct requirements for
+  password method.
+
+## [0.7.2] - 2022-03-25 ##
+(Timm Fitschen)
+
+### Deprecated ###
+
+* In module `caosdb.apiutils`:
+  * `CaosDBPythonEntity` class
+  * `convert_to_entity` function
+  * `convert_to_python_object` function
+
+### Fixed ###
+
+* [caosdb-pylib#106](https://gitlab.indiscale.com/caosdb/src/caosdb-pylib/-/issues/106)
+  Parsing Error in class caosdb.common.models.ACL. This may lead to the
+  unintentional revocation of permissions for some users or roles during
+  updates. However, no additional permissions are being granted.
+
+### Documentation ###
+
+## [0.7.1] - 2022-03-11 ##
+(Daniel Hornung)
+
+### Documentation ###
+
+- `timeout` option in example pycaosdb.ini
+
+## [0.7.0] - 2022-01-21 ##
+
+### Added ###
+
+- Function in administration that generates passwords that comply with the rules.
+
+### Fixed ###
+
+- #90 compare_entities function in apiutils does not check units
+- #94 some special properties were not checked in compare_entities
+
+### Security ###
+
+## [0.6.1] - 2021-12-03 ##
+
+### Fixed ###
+
+- #50 keyring can be used as password input method again
+* #81 compare_entities from apiutils does not compare entity values
+
+## [0.6.0] - 2021-10-19 ##
+
+### Added ###
+
+- It is possible now to supply a password for caosdb_admin on the command line
+  and also activate the user directly using "-c".
 * Added examples for complex data models to documentation
 * extended apiutils with `resolve_reference(Property)`
 * is_reference function for Properties
+* function `retrieve_substructure` that recursively adds connected entities.
 
 ### Changed ###
 
-- It is possible now to supply a password for caosdb_admin on the command line and also activate the user directly using "-c".
+* Retrievals of entities where the class does not match the entity role raise
+  a ValueError now. See
+  [here](https://gitlab.indiscale.com/caosdb/src/caosdb-pylib/-/issues/66) for more
+  information. Updating a role is now being done by setting the `Entity.role`
+  to the new role (as string).
+* Entity.add_property and Entity.add_parent do not accept `**kwargs`-style
+  keywords anymore. Formerly known keywords have been refactored into named
+  parameters.
+* [#35](https://gitlab.com/caosdb/caosdb-pylib/-/issues/35) Loggers now use the
+  name of the unit where they are called instead of a static name
 
 ### Deprecated ###
 
-* `id_query(ids)` in apiutils
-
-### Removed ###
+* `id_query(ids)` in apiutils (to be removed with >=0.5.4)
+* The whole yamlapi with the following functions (to be removed with >=0.5.4):
+  * `append_sublist`
+  * `kv_to_xml`
+  * `dict_to_xml`
+  * `yaml_to_xml`
+  * `process`
+  * `yaml_file_to_xml`
+  * `insert_yaml_file`
 
 ### Fixed ###
 
+* #60 Unintuitive behavior of `Entity.role` after a `Entity(id).retrieve()`
+  Originally the role was `None`. The correct role is present now.
 * #53 Documentation of inheritance
 * #38 Dependencies in chunk-deletion of containers
 
@@ -45,10 +172,6 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 * Updated error-handling tutorial in documentation to reflect the new
   error classes
 
-### Deprecated ###
-
-### Removed ###
-
 ### Fixed ###
 * #45 - test_config_ini_via_envvar
 
@@ -141,14 +264,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
   assumed that `password_method` is `auth_token` if the the `auth_token` is
   set.
 
-### Removed ###
-
 ### Fixed ###
 
 - Replaced deprecated Logger.warn() method.
 
-### Security ###
-
 ## [0.3.0] - 2020-04-24##
 
 ### Added ###
@@ -156,10 +275,6 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 * `apiutils.apply_to_ids` -- a helper which applies a function to all ids which
   are used by an entity (own entity, parents, properties, references etc.).
 
-### Changed ###
-
-### Deprecated ###
-
 ### Fixed ###
 
 * import bugs in apiutils

DEPENDENCIES.md

-* caosdb-server == 0.3
-* Python >= 3.5
+* caosdb-server >= 0.8.0
+* Python >= 3.8
 * pip >= 20.0.2
 
-
-Any other dependencies are being installed via pip
+Any other dependencies are defined in the setup.py and are being installed via pip

FEATURES.md

+# Experimental Features
+
+- High Level API in the module `high_level_api` is experimental and might be removed in future. It is for playing around with a possible future implementation of the Python client. See `src/doc/future_caosdb.md`
+
+
+# Features
+TODO: This is currently an incomplete list.
+- `to_graphics` defined in `caosdb.utils.plantuml` can be used to create an UML diagram of a data model

Makefile

@@ -31,3 +31,18 @@ doc:
 
 install:
 	@echo "Not implemented yet, use pip for installation."
+
+check: style lint
+.PHONY: check
+
+style:
+	pycodestyle --count examples src unittests
+.PHONY: style
+
+lint:
+	pylint --unsafe-load-any-extension=y -d all -e E,F src/caosdb/common
+.PHONY: lint
+
+unittest:
+	tox -r
+.PHONY: unittest

README.md

@@ -47,7 +47,7 @@ However, you can also create an issue for it.
 
 * Copyright (C) 2018 Research Group Biomedical Physics, Max Planck Institute
   for Dynamics and Self-Organization Göttingen.
-* Copyright (C) 2020-2021 Indiscale GmbH <info@indiscale.com>
+* Copyright (C) 2020-2022 Indiscale GmbH <info@indiscale.com>
 
 All files in this repository are licensed under a [GNU Affero General Public
 License](LICENCE.md) (version 3 or later).

README_SETUP.md

@@ -4,18 +4,22 @@
 
 ### Requirements ###
 
-PyCaosDB needs at least Python 3.6.  Additionally, the following packages are required (they will
+PyCaosDB needs at least Python 3.8.  Additionally, the following packages are required (they will
 typically be installed automatically):
 
 - `lxml`
 - `PyYaml`
 - `PySocks`
 
+Optional packages:
+- `keyring`
+- `jsonschema`
+
 ### How to install ###
 
 #### Linux ####
 
-Make sure that Python (at least version 3.6) and pip is installed, using your system tools and
+Make sure that Python (at least version 3.8) and pip is installed, using your system tools and
 documentation.
 
 Then open a terminal and continue in the [Generic installation](#generic-installation) section.
@@ -30,12 +34,14 @@ packages you will ever need out of the box.  If you prefer, you may also install
 After installation, open an Anaconda prompt from the Windows menu and continue in the [Generic
 installation](#generic-installation) section.
 
-#### iOS ####
+#### MacOS ####
 
-If there is no Python 3 installed yet, there are two main ways to obtain it: Either get the binary
-package from [python.org](https://www.python.org/downloads/) or, for advanced users, install via [Homebrew](https://brew.sh/).  After installation
-from python.org, it is recommended to also update the TLS certificates for Python (this requires
-administrator rights for your user):
+If there is no Python 3 installed yet, there are two main ways to
+obtain it: Either get the binary package from
+[python.org](https://www.python.org/downloads/) or, for advanced
+users, install via [Homebrew](https://brew.sh/). After installation
+from python.org, it is recommended to also update the TLS certificates
+for Python (this requires administrator rights for your user):
 
 ```sh
 # Replace this with your Python version number:
@@ -45,7 +51,8 @@ cd /Applications/Python\ 3.9/
 sudo ./Install\ Certificates.command
 ```
 
-After these steps, you may continue with the [Generic installation](#generic-installation).
+After these steps, you may continue with the [Generic
+installation](#generic-installation).
 
 #### Generic installation ####
 
@@ -66,62 +73,17 @@ cd caosdb-pylib
 pip3 install --user .
 ```
 
-## Configuration ##
-
-The  configuration is done using `ini` configuration files.
-PyCaosDB tries to read from the inifile specified in the environment variable `PYCAOSDBINI` or
-alternatively in `~/.pycaosdb.ini` upon import.  After that, the ini file `pycaosdb.ini` in the
-current working directory will be read additionally, if it exists.
-
-Here, we will look at the most common configuration options. For a full and 
-comprehensive description please check out 
-[pycaosdb.ini file](https://gitlab.com/caosdb/caosdb-pylib/-/blob/main/examples/pycaosdb.ini) 
-You can download this file and use it as a starting point.
-
-
-Typically, you need to change at least the `url` and `username` fields as required. 
-(Ask your CaosDB administrator or IT crowd if
-you do not know what to put there, but for the demo instances https://demo.indiscale.com, `username=admin`
-and `password=caosdb` should work).
-
-### Authentication ###
-
-The default configuration (that your are asked for your password when ever a connection is created
-can be changed by setting `password_method`:
+For installation of optional packages, install with an additional option, e.g. for 
+validating with the caosdb json schema:
 
-* with `password_method=input` password (and possibly user) will be queried on demand (**default**)
-* use the password manager [pass](https://www.passwordstore.org) by using `pass` as value, see also the [ArchWiki
-  entry](https://wiki.archlinux.org/index.php/Pass#Basic_usage). This also requires `password_identifier` which refers to the identifier within pass
-  for the desired password.
-* install the python package [keyring](https://pypi.org/project/keyring), to use the system keyring/wallet (macOS, GNOME, KDE,
-  Windows). The password will be queried on first usage.
-* with `password_method=plain` (**strongly discouraged**)
-
-The following illustrates the recommended options:
-
-```ini
-[Connection]
-# using "pass" password manager
-#password_method=pass
-#password_identifier=...
-
-# using the system keyring/wallet (macOS, GNOME, KDE, Windows)
-#password_method=keyring
+```sh
+pip3 install --user .[jsonschema]
 ```
 
-### SSL Certificate ###
-In some cases (especially if you are testing CaosDB) you might need to supply 
-an SSL certificate to allow SSL encryption.
-
-```ini
-[Connection]
-cacert=/path/to/caosdb.ca.pem
-```
+## Configuration ##
 
-### Further Settings ###
-As mentioned above, a complete list of options can be found in the 
-[pycaosdb.ini file](https://gitlab.com/caosdb/caosdb-pylib/-/blob/main/examples/pycaosdb.ini) in 
-the examples folder of the source code.
+The configuration is done using `ini` configuration files.  The content of these configuration files
+is described in detail in the [configuration section of the documentation](https://docs.indiscale.com/caosdb-pylib/configuration.html).
 
 ## Try it out ##
 
@@ -141,7 +103,10 @@ like this, check out the "Authentication" section in the [configuration document
 Now would be a good time to continue with the [tutorials](tutorials/index).
 
 ## Run Unit Tests
-tox
+
+- Run all tests: `tox` or `make unittest`
+- Run a specific test file: e.g. `tox -- unittests/test_schema.py`
+- Run a specific test function: e.g. `tox -- unittests/test_schema.py::test_config_files`
 
 ## Documentation ##
 

RELEASE_GUIDELINES.md

 # Release Guidelines for the CaosDB Python Client Library
 
-This document specifies release guidelines in addition to the generel release
+This document specifies release guidelines in addition to the general release
 guidelines of the CaosDB Project
 ([RELEASE_GUIDELINES.md](https://gitlab.com/caosdb/caosdb/blob/dev/RELEASE_GUIDELINES.md))
 
@@ -20,9 +20,10 @@ guidelines of the CaosDB Project
 
 3. Check all general prerequisites.
 
-4. Prepare [setup.py](./setup.py): Check the `MAJOR`, `MINOR`, `MICRO`, `PRE`
-   variables and set `ISRELEASED` to `True`. Use the possibility to issue
-   pre-release versions for testing.
+4. Update the version:
+   - `version` variables in `src/doc/conf.py`
+   - Version on [setup.py](./setup.py): Check the `MAJOR`, `MINOR`, `MICRO`, `PRE` variables and set
+     `ISRELEASED` to `True`. Use the possibility to issue pre-release versions for testing.
 
 5. Merge the release branch into the main branch.
 
@@ -35,8 +36,12 @@ guidelines of the CaosDB Project
 9. Publish the release by executing `./release.sh` with uploads the caosdb
    module to the Python Package Index [pypi.org](https://pypi.org).
 
-10. Merge the main branch back into the dev branch.
+10. Create a gitlab release on gitlab.indiscale.com and gitlab.com
 
-11. After the merge of main to dev, start a new development version by
-    setting `ISRELEASED` to `False` and by increasing at least the `MIRCO`
-    version in [setup.py](./setup.py)
+11. Merge the main branch back into the dev branch.
+
+12. After the merge of main to dev, start a new development version by
+    setting `ISRELEASED` to `False` and by increasing at least the `MICRO`
+    version in [setup.py](./setup.py).
+    Also update CHANGELOG.md (new "Unreleased" section).
+    Also update `src/doc/conf.py`.

examples/pycaosdb.ini

@@ -67,3 +67,6 @@
 
 # This option is used internally and for testing. Do not override.
 # implementation=_DefaultCaosDBServerConnection
+
+# The timeout for requests to the server.
+# timeout=1000

setup.py

@@ -45,11 +45,15 @@ from setuptools import find_packages, setup
 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 ########################################################################
 
-MAJOR = 0
-MINOR = 5
-MICRO = 3
-PRE = ""  # e.g. rc0, alpha.1, 0.beta-23
 ISRELEASED = False
+MAJOR = 0
+MINOR = 9
+MICRO = 0
+# Do not tag as pre-release until this commit
+# https://github.com/pypa/packaging/pull/515
+# has made it into a release. Probably we should wait for pypa/packaging>=21.4
+# https://github.com/pypa/packaging/releases
+PRE = "dev" # "dev"  # e.g. rc0, alpha.1, 0.beta-23
 
 if PRE:
     VERSION = "{}.{}.{}-{}".format(MAJOR, MINOR, MICRO, PRE)
@@ -154,16 +158,27 @@ def setup_package():
         long_description_content_type="text/markdown",
         author='Timm Fitschen',
         author_email='t.fitschen@indiscale.com',
+        url='https://www.caosdb.org',
+        license="AGPLv3+",
+        classifiers=[
+            "Programming Language :: Python :: 3",
+            "License :: OSI Approved :: GNU Affero General Public License v3 or later (AGPLv3+)",
+            "Operating System :: OS Independent",
+            "Topic :: Database",
+            "Topic :: Scientific/Engineering :: Information Analysis",
+        ],
         packages=find_packages('src'),
-        python_requires='>=3.6',
+        python_requires='>=3.8',
         package_dir={'': 'src'},
-        install_requires=['lxml>=3.6.4',
-                          'PyYaml>=3.12', 'future', 'PySocks>=1.6.7'],
-        extras_require={'keyring': ['keyring>=13.0.0']},
+        install_requires=['lxml>=4.6.3',
+                          'PyYAML>=5.4.1', 'future', 'PySocks>=1.6.7'],
+        extras_require={'keyring': ['keyring>=13.0.0'],
+                        'jsonschema': ['jsonschema>=4.4.0']},
         setup_requires=["pytest-runner>=2.0,<3dev"],
-        tests_require=["pytest", "pytest-cov", "coverage>=4.4.2"],
+        tests_require=["pytest", "pytest-cov", "coverage>=4.4.2",
+                       "jsonschema>=4.4.0"],
         package_data={
-            'caosdb': ['cert/indiscale.ca.crt'],
+            'caosdb': ['cert/indiscale.ca.crt', 'schema-pycaosdb-ini.yml'],
         },
         scripts=["src/caosdb/utils/caosdb_admin.py"]
     )

src/caosdb/common/administration.py

@@ -26,16 +26,15 @@
 
 """missing docstring."""
 
-from lxml import etree
-
+import re
+import string
+import random
 from caosdb.common.utils import xml2str
 from caosdb.connection.connection import get_connection
-from caosdb.exceptions import (HTTPClientError,
-                               HTTPForbiddenError,
-                               HTTPResourceNotFoundError,
-                               EntityDoesNotExistError,
-                               ServerConfigurationException,
-                               )
+from caosdb.exceptions import (EntityDoesNotExistError, HTTPClientError,
+                               HTTPForbiddenError, HTTPResourceNotFoundError,
+                               ServerConfigurationException)
+from lxml import etree
 
 
 def set_server_property(key, value):
@@ -60,7 +59,8 @@ def set_server_property(key, value):
         con._form_data_request(method="POST", path="_server_properties",
                                params={key: value}).read()
     except EntityDoesNotExistError:
-        raise ServerConfigurationException("Debug mode in server is probably disabled.") from None
+        raise ServerConfigurationException(
+            "Debug mode in server is probably disabled.") from None
 
 
 def get_server_properties():
@@ -75,9 +75,11 @@ def get_server_properties():
     """
     con = get_connection()
     try:
-        body = con._http_request(method="GET", path="_server_properties").response
+        body = con._http_request(
+            method="GET", path="_server_properties").response
     except EntityDoesNotExistError:
-        raise ServerConfigurationException("Debug mode in server is probably disabled.") from None
+        raise ServerConfigurationException(
+            "Debug mode in server is probably disabled.") from None
 
     xml = etree.parse(body)
     props = dict()
@@ -112,6 +114,39 @@ def get_server_property(key):
     return get_server_properties()[key]
 
 
+def generate_password(length: int):
+    """Create a random password that fulfills the security requirements
+
+    Parameters
+    ----------
+    length : int
+        Length of the generated password.  Has to be greater than 7.
+
+    Returns
+    -------
+    password : string
+        Generated random password of the given length
+
+    Raises
+    ------
+    ValueError:
+        If the length is less than 8.
+    """
+    minimum_password_length = 8
+    if length < minimum_password_length:
+        raise ValueError("CaosDB passwords have to be at least {} characters.".format(
+            minimum_password_length))
+    sample_letters = string.ascii_letters + string.digits + "!#$%*+-/:;?_"
+    password = ''.join((random.choice(sample_letters) for i in range(length)))
+
+    while not re.match(r"(?=.*[A-Z])(?=.*[a-z])(?=.*\d)(?=.*[\W_]).{8,}",
+                       password):
+        password = ''.join((random.choice(sample_letters)
+                            for i in range(length)))
+
+    return password
+
+
 def _retrieve_user(name, realm=None, **kwargs):
     con = get_connection()
     try:
@@ -156,13 +191,14 @@ def _update_user(name, realm=None, password=None, status=None,
         return con.put_form_data(entity_uri_segment="User/" + (realm + "/" + name if realm is not None else name), params=params, **kwargs).read()
     except HTTPResourceNotFoundError as e:
         e.msg = "User does not exist."
-        raise
+        raise e
     except HTTPForbiddenError as e:
         e.msg = "You are not permitted to update this user."
-        raise
+        raise e
     except HTTPClientError as e:
-        if e.status == 409:
-            e.msg = "Entity does not exist."
+        for elem in etree.fromstring(e.body):
+            if elem.tag == "Error":
+                e.msg = elem.get("description")
         raise
 
 
@@ -187,11 +223,9 @@ def _insert_user(name, password=None, status=None, email=None, entity=None, **kw
         e.msg = "You are not permitted to insert a new user."
         raise e
     except HTTPClientError as e:
-        if e.status == 409:
-            e.msg = "User name is already in use."
-
-        if e.status == 422:
-            e.msg = "Maybe the password does not match the required standard?"
+        for elem in etree.fromstring(e.body):
+            if elem.tag == "Error":
+                e.msg = elem.get("description")
         raise e
 
 

src/caosdb/common/datatype.py

@@ -62,10 +62,21 @@ def is_list_datatype(datatype):
 
 
 def is_reference(datatype):
-    """ returns whether the value is a reference
+    """Returns whether the value is a reference
 
     FILE and REFERENCE properties are examples, but also datatypes that are
-    RecordTypes
+    RecordTypes.
+
+    Parameters
+    ----------
+    datatype : str
+               The datatype to check.
+
+    Returns
+    -------
+    bool
+        True if the datatype is a not base datatype or a list of a base datatype.
+        Otherwise False is returned.
     """
 
     if datatype is None:
@@ -80,6 +91,41 @@ def is_reference(datatype):
         return True
 
 
+def get_referenced_recordtype(datatype):
+    """Return the record type of the referenced datatype.
+
+    Raises
+    ------
+    ValueError
+              In cases where datatype is not a reference, the list does not have
+              a referenced record type or the datatype is a FILE.
+
+    Parameters
+    ----------
+    datatype : str
+               The datatype to check.
+
+    Returns
+    -------
+    str
+       String containing the name of the referenced datatype.
+    """
+
+    if not is_reference(datatype):
+        raise ValueError("datatype must be a reference")
+
+    if is_list_datatype(datatype):
+        datatype = get_list_datatype(datatype)
+        if datatype is None:
+            raise ValueError("list does not have a list datatype")
+
+    if datatype == FILE:
+        raise ValueError(
+            "FILE references are not considered references with a record type")
+
+    return datatype
+
+
 def get_id_of_datatype(datatype):
     """ returns the id of a Record Type
 

src/caosdb/common/utils.py

@@ -26,7 +26,6 @@
 from lxml import etree
 from multiprocessing import Lock
 from uuid import uuid4
-from sys import hexversion
 _uuid_lock = Lock()
 
 

src/caosdb/configuration.py

@@ -21,6 +21,16 @@
 #
 # ** end header
 #
+
+import os
+import yaml
+import warnings
+try:
+    optional_jsonschema_validate = None
+    from jsonschema import validate as optional_jsonschema_validate
+except ImportError:
+    pass
+
 try:
     # python2
     from ConfigParser import ConfigParser
@@ -47,7 +57,9 @@ def configure(inifile):
         _pycaosdbconf = None
     if _pycaosdbconf is None:
         _reset_config()
-    return _pycaosdbconf.read(inifile)
+    read_config = _pycaosdbconf.read(inifile)
+    validate_yaml_schema(config_to_yaml(_pycaosdbconf))
+    return read_config
 
 
 def get_config():
@@ -55,11 +67,45 @@ def get_config():
     return _pycaosdbconf
 
 
+def config_to_yaml(config):
+    valobj = {}
+    for s in config.sections():
+        valobj[s] = {}
+        for key, value in config[s].items():
+            # TODO: Can the type be inferred from the config object?
+            if key in ["timeout", "debug"]:
+                valobj[s][key] = int(value)
+            elif key in ["ssl_insecure"]:
+                valobj[s][key] = bool(value)
+            else:
+                valobj[s][key] = value
+
+    return valobj
+
+
+def validate_yaml_schema(valobj):
+    if optional_jsonschema_validate:
+        with open(os.path.join(os.path.dirname(__file__), "schema-pycaosdb-ini.yml")) as f:
+            schema = yaml.load(f, Loader=yaml.SafeLoader)
+        optional_jsonschema_validate(instance=valobj, schema=schema["schema-pycaosdb-ini"])
+    else:
+        warnings.warn("""
+            Warning: The validation could not be performed because `jsonschema` is not installed.
+        """)
+
+
 def _read_config_files():
-    """Function to read config files from different paths. Checks for path in $PYCAOSDBINI or home directory (.pycaosdb.ini) and in the current working directory (pycaosdb.ini).
+    """Function to read config files from different paths.
+
+    Checks for path either in ``$PYCAOSDBINI`` or home directory (``.pycaosdb.ini``), and
+    additionally in the current working directory (``pycaosdb.ini``).
+
+    Returns
+    -------
+
+    ini files: list
+      The successfully parsed ini-files. Order: env_var or home directory, cwd. Used for testing the function.
 
-    Returns:
-        [list]: list with successfully parsed ini-files. Order: env_var or home directory, cwd. Used for testing the function.
     """
     return_var = []
     if "PYCAOSDBINI" in environ:

src/caosdb/connection/__init__.py

-# -*- encoding: utf-8 -*-
-#
-# ** header v3.0
-# This file is a part of the CaosDB Project.
-#
-# Copyright (C) 2018 Research Group Biomedical Physics,
-# Max-Planck-Institute for Dynamics and Self-Organization Göttingen
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Affero General Public License as
-# published by the Free Software Foundation, either version 3 of the
-# License, or (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU Affero General Public License for more details.
-#
-# You should have received a copy of the GNU Affero General Public License
-# along with this program. If not, see <https://www.gnu.org/licenses/>.
-#
-# ** end header
-#
-from sys import hexversion
-from .utils import check_python_ssl_version
-
-check_python_ssl_version(hexversion)
-del check_python_ssl_version
-del hexversion

src/caosdb/connection/authentication/interface.py

@@ -36,7 +36,7 @@ from caosdb.exceptions import LoginFailedError
 # meta class compatible with Python 2 *and* 3:
 ABC = ABCMeta('ABC', (object, ), {'__slots__': ()})
 
-_LOGGER = logging.getLogger("authentication")
+_LOGGER = logging.getLogger(__name__)
 
 
 class AbstractAuthenticator(ABC):

src/caosdb/connection/authentication/keyring.py

@@ -28,7 +28,7 @@ retrieve the password.
 """
 
 import sys
-import imp
+import importlib
 from getpass import getpass
 from caosdb.exceptions import ConfigurationError
 from .external_credentials_provider import ExternalCredentialsProvider
@@ -52,17 +52,13 @@ def get_authentication_provider():
 
 def _get_external_keyring():
     try:
-        fil, pathname, desc = imp.find_module("keyring", sys.path[1:])
-        module = imp.load_module("external_keyring", fil, pathname, desc)
-        return module
+        return importlib.import_module("keyring")
     except ImportError:
         raise RuntimeError(
             "The keyring password method requires installation of the"
             "keyring python package. On linux with python < 3.5, "
             "this requires the installation of dbus-python as a "
             "system package.")
-    finally:
-        fil.close()
 
 
 def _call_keyring(**config):
@@ -74,7 +70,6 @@ def _call_keyring(**config):
     url = config.get("url")
     username = config.get("username")
     app = "caosdb — {}".format(url)
-    password = _call_keyring(app=app, username=username)
     external_keyring = _get_external_keyring()
     password = external_keyring.get_password(app, username)
     if password is None: