diff --git a/.gitignore b/.gitignore
index 341bfea4f1381b6eb59e548f395859bad35520d7..b522b1da9176e59756bffe89cd4eafe0d751a23c 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,3 +1,5 @@
+# -*- mode:conf; -*-
+
# dot files
.*
!/.gitignore
@@ -11,3 +13,6 @@ __pycache__/
dist/
build/
src/caosdb/version.py
+
+# documentation
+_apidoc
\ No newline at end of file
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 5741684242954b0e9b23f4d2c40ca862f17a9570..9f8c131968c050fb18001b5dc7c5468d0ed26dae 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -4,6 +4,8 @@
# Copyright (C) 2018 Research Group Biomedical Physics,
# Max-Planck-Institute for Dynamics and Self-Organization Göttingen
# Copyright (C) 2019 Henrik tom Wörden
+# Copyright (C) 2020 Indiscale GmbH <info@indiscale.com>
+# Copyright (C) 2020 Daniel Hornung <d.hornung@indiscale.com>
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as
@@ -19,7 +21,8 @@
# along with this program. If not, see <https://www.gnu.org/licenses/>.
variables:
- CI_REGISTRY_IMAGE: $CI_REGISTRY/caosdb/caosdb-pylib/testenv:latest
+ DEPLOY_REF: dev
+ CI_REGISTRY_IMAGE: $CI_REGISTRY/caosdb/src/caosdb-pylib/testenv:latest
# When using dind, it's wise to use the overlayfs driver for
# improved performance.
@@ -62,11 +65,12 @@ trigger_build:
stage: deploy
script:
- /usr/bin/curl -X POST
- -F token=$DEPLOY_TRIGGER_TOKEN
+ -F token=$CI_JOB_TOKEN
+ -F "variables[F_BRANCH]=$CI_COMMIT_REF_NAME"
-F "variables[PYLIB]=$CI_COMMIT_REF_NAME"
-F "variables[TriggerdBy]=PYLIB"
-F "variables[TriggerdByHash]=$CI_COMMIT_SHORT_SHA"
- -F ref=dev https://gitlab.indiscale.com/api/v4/projects/14/trigger/pipeline
+ -F ref=$DEPLOY_REF https://gitlab.indiscale.com/api/v4/projects/14/trigger/pipeline
# Build a docker image in which tests for this repository can run
build-testenv:
@@ -86,3 +90,24 @@ build-testenv:
--cache-from $CI_REGISTRY_IMAGE
-t $CI_REGISTRY_IMAGE .
- docker push $CI_REGISTRY_IMAGE
+
+# Build the sphinx documentation and make it ready for deployment by Gitlab Pages
+# Special job for serving a static website. See https://docs.gitlab.com/ee/ci/yaml/README.html#pages
+pages_prepare: &pages_prepare
+ tags: [ cached-dind ]
+ stage: deploy
+ only:
+ refs:
+ - /^release-.*$/i
+ script:
+ - echo "Deploying"
+ - make doc
+ - cp -r build/doc/html public
+ artifacts:
+ paths:
+ - public
+pages:
+ <<: *pages_prepare
+ only:
+ refs:
+ - main
diff --git a/.gitlab/merge_request_templates/Default.md b/.gitlab/merge_request_templates/Default.md
new file mode 100644
index 0000000000000000000000000000000000000000..77a95da1cc40c815e4952a1283d345af56e80461
--- /dev/null
+++ b/.gitlab/merge_request_templates/Default.md
@@ -0,0 +1,49 @@
+# Summary
+
+ Insert a meaningful description for this merge request here. What is the
+ new/changed behavior? Which bug has been fixed? Are there related Issues?
+
+# Focus
+
+ Point the reviewer to the core of the code change. Where should they start
+ reading? What should they focus on (e.g. security, performance,
+ maintainability, user-friendliness, compliance with the specs, finding more
+ corner cases, concrete questions)?
+
+# Test Environment
+
+ How to set up a test environment for manual testing?
+
+# Check List for the Author
+
+Please, prepare your MR for a review. Be sure to write a summary and a
+focus and create gitlab comments for the reviewer. They should guide the
+reviewer through the changes, explain your changes and also point out open
+questions. For further good practices have a look at [our review
+guidelines](https://gitlab.com/caosdb/caosdb/-/blob/dev/REVIEW_GUIDELINES.md)
+
+- [ ] All automated tests pass
+- [ ] Reference related Issues
+- [ ] Up-to-date CHANGELOG.md
+- [ ] Annotations in code (Gitlab comments)
+ - Intent of new code
+ - Problems with old code
+ - Why this implementation?
+
+
+# Check List for the Reviewer
+
+
+- [ ] I understand the intent of this MR
+- [ ] All automated tests pass
+- [ ] Up-to-date CHANGELOG.md
+- [ ] The test environment setup works and the intended behavior is
+ reproducible in the test environment
+- [ ] In-code documentation and comments are up-to-date.
+- [ ] Check: Are there spezifications? Are they satisfied?
+
+For further good practices have a look at [our review guidelines](https://gitlab.com/caosdb/caosdb/-/blob/dev/REVIEW_GUIDELINES.md).
+
+
+/assign me
+/target_branch dev
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 73403111ac141553237a3f1691adc22ad2183a16..622d5678c38257fecba88e899aa571a731a5b3a9 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -9,6 +9,108 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
### Added ###
+### Changed ###
+
+### Deprecated ###
+
+### Removed ###
+
+### Fixed ###
+
+### Security ###
+
+## [0.5.2] - 2021-06-03 ##
+
+### Added ###
+
+* Entity State support (experimental, no StateModel support yet). See the
+ `caosdb.State` class for more information.
+* `etag` property for the `caosdb.Query` class. The etag allows to debug the
+ caching and to decide whether the server has changed between queries.
+* function `_read_config_files` to read `pycaosdb.ini` files from different paths.
+
+### Changed ###
+
+* Updated error-handling tutorial in documentation to reflect the new
+ error classes
+
+### Deprecated ###
+
+### Removed ###
+
+### Fixed ###
+* #45 - test_config_ini_via_envvar
+
+### Security ###
+
+## [0.5.1] - 2021-02-12 ##
+
+### Fixed ###
+
+* #43 - Error with `execute_query` when server doesn't support query caching.
+
+## [0.5.0] - 2021-02-11 ##
+
+### Added ###
+
+* New exceptions `HTTPForbiddenException` and
+ `HTTPResourceNotFoundException` for HTTP 403 and 404 errors,
+ respectively.
+* `BadQueryError`, `EmptyUniqueQueryError`, and `QueryNotUniqueError`
+ for bad (unique) queries.
+* Added `cache` paramter to `execute_query` and `Query.execute` which indicates
+ whether server may use the cache for the query execution.
+* Added `cached` property to the `Query` class which indicates whether the
+ server used the cache for the execution of the last query.
+* Documentation moved from wiki to this repository and enhanced.
+
+### Changed ###
+
+* Renaming of `URITooLongException` to `HTTPURITooLongError`.
+* Raising of entity exceptions and transaction errors. Whenever any
+ transaction fails, a `TransactionError` is raised. If one ore more
+ entities caused that failure, corresponding entity errors are
+ attached as direct and indirect children of the
+ `TransactionError`. They can be accessed via the `get_errors`
+ (direct children) and `get_all_errors` (direct and indirect
+ children) methods; the causing entities are accessed by
+ `get_entities` and `get_all_entities`. The `has_error` method can be
+ used to check whether a `TransactionError` was caused by a specific
+ `EntityError`.
+* Unique queries will now result in `EmptyUniqueQueryError` or
+ `QueryNotUniqueError` if no or more than one possible candidate is
+ found, respectively.
+
+### Removed ###
+
+* Dynamic exception type `EntityMultiError`.
+* `get_something` functions from all error object in `exceptions.py`
+* `AmbiguityException`
+
+## [0.4.1] - 2021-02-10 ##
+
+### Added ###
+
+* Versioning support (experimental). The version db.Version class can
+ represents particular entity versions and also the complete history of an
+ entity.
+* Automated documentation builds: `make doc`
+
+### Fixed ###
+
+* deepcopy of `_Messages` objects
+
+## [0.4.0] - 2020-07-17##
+
+### Added ###
+
+* `[Entity|Container].get_property_values` for deeply nested references, e.g.
+ from results of SELECT queries.
+* two new `password_method`s for the `pycaosdb.ini` and the
+ `configure_connection` function: `unauthenticated` for staying
+ unauthenticated (and using the anonymous user) and `auth_token`. If
+ `password_method == "auth_token"` the `auth_token` option is required as
+ well.
* Empty string support (See caosdb-server#33)
### Changed ###
@@ -22,6 +124,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
### Deprecated ###
+* Setting the `auth_token` option without setting the `password_method` to
+ `auth_token`. This affects both the `pycaosdb.ini` and the
+ `configure_connection` function. During the deprecation phase it will be
+ assumed that `password_method` is `auth_token` if the the `auth_token` is
+ set.
+
### Removed ###
### Fixed ###
@@ -45,7 +153,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
* import bugs in apiutils
-## [0.2.4] -- 2020-04-23
+## [0.2.4] - 2020-04-23
### Added
diff --git a/DEPENDENCIES.md b/DEPENDENCIES.md
new file mode 100644
index 0000000000000000000000000000000000000000..04e783a7fa31b1d5c3a600a1009c8f040db1620d
--- /dev/null
+++ b/DEPENDENCIES.md
@@ -0,0 +1,6 @@
+* caosdb-server == 0.3
+* Python >= 3.5
+* pip >= 20.0.2
+
+
+Any other dependencies are being installed via pip
diff --git a/Makefile b/Makefile
new file mode 100644
index 0000000000000000000000000000000000000000..4bc3459d209936c17a445c64e77180d9559e4653
--- /dev/null
+++ b/Makefile
@@ -0,0 +1,33 @@
+# ** header v3.0
+# This file is a part of the CaosDB Project.
+#
+# Copyright (C) 2020 IndiScale GmbH <info@indiscale.com>
+# Copyright (C) 2020 Daniel Hornung <d.hornung@indiscale.com>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see <https://www.gnu.org/licenses/>.
+#
+# ** end header
+
+# This Makefile is a wrapper for several other scripts.
+
+.PHONY: help
+
+help:
+ @echo 'Type `make doc` for documentation, or `make install` for (local) installation.'
+
+doc:
+ $(MAKE) -C src/doc html
+
+install:
+ @echo "Not implemented yet, use pip for installation."
diff --git a/README.md b/README.md
index ef26a604905d5140ae9a775065002af35ffe2121..f81202e0c3cd1a54cefcf4b112a7778bb4ddb8f2 100644
--- a/README.md
+++ b/README.md
@@ -1,25 +1,49 @@
-# Welcome
+
+# README
+
+## Welcome
This is the **CaosDB Python Client Library** repository and a part of the
CaosDB project.
-# Setup
+## Setup
Please read the [README_SETUP.md](README_SETUP.md) for instructions on how to
setup this code.
-# Further Reading
+## Further Reading
+
+Please refer to the [official documentation](https://docs.indiscale.com/caosdb-pylib/) for more information.
+
+## Contributing
+
+Thank you very much to all contributers—[past, present](https://gitlab.com/caosdb/caosdb/-/blob/dev/HUMANS.md), and prospective ones.
-Please refer to the [official gitlab repository of the CaosDB
-project](https://gitlab.com/caosdb/caosdb) for more information.
+### Code of Conduct
-# License
+By participating, you are expected to uphold our [Code of Conduct](https://gitlab.com/caosdb/caosdb/-/blob/dev/CODE_OF_CONDUCT.md).
-Copyright (C) 2018 Research Group Biomedical Physics, Max Planck Institute for
-Dynamics and Self-Organization Göttingen.
-Copyright (C) 2020 Indiscale GmbH <info@indiscale.com>
+### How to Contribute
+
+* You found a bug, have a question, or want to request a feature? Please
+[create an issue](https://gitlab.com/caosdb/caosdb-pylib/-/issues).
+* You want to contribute code? Please fork the repository and create a merge
+request in GitLab and choose this repository as target. Make sure to select
+"Allow commits from members who can merge the target branch" under Contribution
+when creating the merge request. This allows our team to work with you on your request.
+- If you have a suggestion for the [documentation](https://docs.indiscale.com/caosdb-pylib/),
+the preferred way is also a merge request as describe above (the documentation resides in `src/doc`).
+However, you can also create an issue for it.
+- You can also contact us at **info (AT) caosdb.de** and join the
+ CaosDB community on
+ [#caosdb:matrix.org](https://matrix.to/#/!unwwlTfOznjEnMMXxf:matrix.org).
+
+## License
+
+* Copyright (C) 2018 Research Group Biomedical Physics, Max Planck Institute
+ for Dynamics and Self-Organization Göttingen.
+* Copyright (C) 2020-2021 Indiscale GmbH <info@indiscale.com>
All files in this repository are licensed under a [GNU Affero General Public
License](LICENCE.md) (version 3 or later).
-
diff --git a/README_SETUP.md b/README_SETUP.md
index a172a878a7aeed3c0ab5f51587330a6f351c991f..30e53ac1c65763fb52b711a215f682c01d2e570e 100644
--- a/README_SETUP.md
+++ b/README_SETUP.md
@@ -1,10 +1,161 @@
-# Installation
-pip install . --user
-pip install tox --user
+# Getting started with PyCaosDB #
-# Run Unit Tests
+## Installation ##
+
+### Requirements ###
+
+PyCaosDB needs at least Python 3.6. Additionally, the following packages are required (they will
+typically be installed automatically):
+
+- `lxml`
+- `PyYaml`
+- `PySocks`
+
+### How to install ###
+
+#### Linux ####
+
+Make sure that Python (at least version 3.6) and pip is installed, using your system tools and
+documentation.
+
+Then open a terminal and continue in the [Generic installation](#generic-installation) section.
+
+#### Windows ####
+
+If a Python distribution is not yet installed, we recommend Anaconda Python, which you can download
+for free from [https://www.anaconda.com](https://www.anaconda.com). The "Anaconda Individual Edition" provides most of all
+packages you will ever need out of the box. If you prefer, you may also install the leaner
+"Miniconda" installer, which allows you to install packages as you need them.
+
+After installation, open an Anaconda prompt from the Windows menu and continue in the [Generic
+installation](#generic-installation) section.
+
+#### iOS ####
+
+If there is no Python 3 installed yet, there are two main ways to obtain it: Either get the binary
+package from [python.org](https://www.python.org/downloads/) or, for advanced users, install via [Homebrew](https://brew.sh/). After installation
+from python.org, it is recommended to also update the TLS certificates for Python (this requires
+administrator rights for your user):
+
+```sh
+# Replace this with your Python version number:
+cd /Applications/Python\ 3.9/
+
+# This needs administrator rights:
+sudo ./Install\ Certificates.command
+```
+
+After these steps, you may continue with the [Generic installation](#generic-installation).
+
+#### Generic installation ####
+
+To install PyCaosDB locally, use `pip3` (also called `pip` on some systems):
+
+```sh
+pip3 install --user caosdb
+```
+
+---
+
+Alternatively, obtain the sources from GitLab and install from there (`git` must be installed for
+this option):
+
+```sh
+git clone https://gitlab.com/caosdb/caosdb-pylib
+cd caosdb-pylib
+pip3 install --user .
+```
+
+## Configuration ##
+
+The configuration is done using `ini` configuration files.
+PyCaosDB tries to read from the inifile specified in the environment variable `PYCAOSDBINI` or
+alternatively in `~/.pycaosdb.ini` upon import. After that, the ini file `pycaosdb.ini` in the
+current working directory will be read additionally, if it exists.
+
+Here, we will look at the most common configuration options. For a full and
+comprehensive description please check out
+[pycaosdb.ini file](https://gitlab.com/caosdb/caosdb-pylib/-/blob/main/examples/pycaosdb.ini)
+You can download this file and use it as a starting point.
+
+
+Typically, you need to change at least the `url` and `username` fields as required.
+(Ask your CaosDB administrator or IT crowd if
+you do not know what to put there, but for the demo instances https://demo.indiscale.com, `username=admin`
+and `password=caosdb` should work).
+
+### Authentication ###
+
+The default configuration (that your are asked for your password when ever a connection is created
+can be changed by setting `password_method`:
+
+* with `password_method=input` password (and possibly user) will be queried on demand (**default**)
+* use the password manager [pass](https://www.passwordstore.org) by using `pass` as value, see also the [ArchWiki
+ entry](https://wiki.archlinux.org/index.php/Pass#Basic_usage). This also requires `password_identifier` which refers to the identifier within pass
+ for the desired password.
+* install the python package [keyring](https://pypi.org/project/keyring), to use the system keyring/wallet (macOS, GNOME, KDE,
+ Windows). The password will be queried on first usage.
+* with `password_method=plain` (**strongly discouraged**)
+
+The following illustrates the recommended options:
+
+```ini
+[Connection]
+# using "pass" password manager
+#password_method=pass
+#password_identifier=...
+
+# using the system keyring/wallet (macOS, GNOME, KDE, Windows)
+#password_method=keyring
+```
+
+### SSL Certificate ###
+In some cases (especially if you are testing CaosDB) you might need to supply
+an SSL certificate to allow SSL encryption.
+
+```ini
+[Connection]
+cacert=/path/to/caosdb.ca.pem
+```
+
+### Further Settings ###
+As mentioned above, a complete list of options can be found in the
+[pycaosdb.ini file](https://gitlab.com/caosdb/caosdb-pylib/-/blob/main/examples/pycaosdb.ini) in
+the examples folder of the source code.
+
+## Try it out ##
+
+Start Python and check whether the you can access the database. (You will be asked for the
+password):
+
+```python
+In [1]: import caosdb as db
+In [2]: db.Info()
+Please enter the password: # It's `caosdb` for the demo server.
+Out[2]: Connection to CaosDB with 501 Records.
+```
+
+Note: This setup will ask you for your password whenever a new connection is created. If you do not
+like this, check out the "Authentication" section in the [configuration documentation](configuration.md).
+
+Now would be a good time to continue with the [tutorials](tutorials/index).
+
+## Run Unit Tests
tox
-# Code Formatting
+## Code Formatting
autopep8 -i -r ./
+
+## Documentation ##
+
+Build documentation in `build/` with `make doc`.
+
+### Requirements ###
+
+- `sphinx`
+- `sphinx-autoapi`
+- `recommonmark`
+
+### Troubleshooting ###
+If the client is to be executed directly from the `/src` folder, an initial `.\setup.py install --user` must be called.
diff --git a/RELEASE_GUIDELINES.md b/RELEASE_GUIDELINES.md
index 02be5c1ad19f6a3a405fb08d62e23dab350ad445..e015b598117abdcd575cf17e2f095fec459a4c4c 100644
--- a/RELEASE_GUIDELINES.md
+++ b/RELEASE_GUIDELINES.md
@@ -16,21 +16,27 @@ guidelines of the CaosDB Project
1. Create a release branch from the dev branch. This prevents further changes
to the code base and a never ending release process. Naming: `release-<VERSION>`
-2. Check all general prerequisites.
+2. Update CHANGELOG.md
-3. Prepare [setup.py](./setup.py): Update the `MAJOR`, `MINOR`, `MICRO`, `PRE`
+3. Check all general prerequisites.
+
+4. Prepare [setup.py](./setup.py): Check the `MAJOR`, `MINOR`, `MICRO`, `PRE`
variables and set `ISRELEASED` to `True`. Use the possibility to issue
pre-release versions for testing.
-4. Merge the release branch into the master branch.
+5. Merge the release branch into the main branch.
-5. Tag the latest commit of the master branch with `v<VERSION>`.
+6. Tag the latest commit of the main branch with `v<VERSION>`.
-6. Delete the release branch.
+7. Delete the release branch.
-7. Remove possibly existing `./dist` directory with old release.
+8. Remove possibly existing `./dist` directory with old release.
-8. Publish the release by executing `./release.sh` with uploads the caosdb
+9. Publish the release by executing `./release.sh` with uploads the caosdb
module to the Python Package Index [pypi.org](https://pypi.org).
-9. Merge the master branch back into the dev branch.
+10. Merge the main branch back into the dev branch.
+
+11. After the merge of main to dev, start a new development version by
+ setting `ISRELEASED` to `False` and by increasing at least the `MIRCO`
+ version in [setup.py](./setup.py)
diff --git a/examples/pycaosdb.ini b/examples/pycaosdb.ini
index 625a3878341265637e6669291ba9d3c517906534..edc32195fbb364bb355d67b8733e8c7bccbb0d34 100644
--- a/examples/pycaosdb.ini
+++ b/examples/pycaosdb.ini
@@ -4,34 +4,66 @@
# - the location given in the env variable PYCAOSDBINI
[Connection]
-url=https://demo.indiscale.com/
-
-## For local installations, enter your server address and SSL certificate here.
+# URL of the CaosDB server
+# url=https://demo.indiscale.com/
+# For local installations this typically is
# url=https://localhost:10443/
-# cacert=/path/to/caosdb.ca.pem
-
-## If this option is set, the SSL certificate will be ignored. Use with care!
-# ssl_insecure=1
-username=admin
+# User name used for authentication with the server
+# username=admin
-## The password input method can be chosen with the `password_method` setting,
-## which by default is set to `plain`.
-##
-## DEFAULT: the password method is `plain`, with this method the password must
-## be saved as plain text.
+# The password input method defines how the password is supplied that is used
+# for authentication with the server.
+#
+# DEFAULT: `input`
+# The username is optional in this case. The password is entered directly by the user.
+# password_method=input
+#
+# OR: `plain`
+# This implies that the password must # be saved as plain text in a
+# configuration under the 'password' key.
# password_method=plain
# password=caosdb
+#
+# OR `pass`
+# The password is retrieved from the "pass" password manager
+# This password manager uses identifiers to distinguish passwords. Supply this
+# identifier under the key 'password_identifier'.
+# password_method=pass
+# password_identifier=caosdb_password
+#
+# OR: `keyring`
+# Using the system keyring/wallet (macOS, GNOME, KDE, Windows)
+# requires installation of the keyring python package (pip install keyring).
+# password_method=keyring
-## OR: `input`: username is optional, password is entered directly by the user.
-## The default password for the demo server is `caosdb`.
-password_method=input
+# Using an authentication token to connect with the server. This setting is
+# not recommended for users.
+# auth_token=TOKEN
-## OR: `pass`: password is retrieved from the "pass" password manager
-# password_method=pass
-# password_identifier=...
+# If the server's SSL certificate cannot be validated by your installed
+# certificates (default or installed by your admins), you may also need to
+# supply the matching key (pem file):
+# cacert=/path/to/caosdb.ca.pem
-## OR: `keyring`: using the system keyring/wallet (macOS, GNOME, KDE, Windows)
-## requires installation of the keyring python package:
-## pip install keyring
-# password_method=keyring
+# If this option is set, the SSL certificate of the server will not be
+# validated. This has the potential of a man-in-the-middle attack. Use with care!
+# ssl_insecure=True
+
+# You may define the ssl version to be used. It has to be the name of the
+# corresponding attribute in the Python ssl module.
+# ssl_version=PROTOCOL_TLS
+
+# The debug key allows control the verbosity. Set it to 1 or 2 in case you
+# want to see debugging output or if you want to learn more about
+# the internals of the protocol. 0 disables debugging output.
+# debug=0
+
+# You can define a socket proxy to be used.
+# This is for the case that the server sits behind a firewall which is being
+# tunnelled with a socket proxy (SOCKS4 or SOCKS5) (e.g. via ssh's -D option or
+# a dedicated proxy server).
+# socket_proxy=localhost:12345
+
+# This option is used internally and for testing. Do not override.
+# implementation=_DefaultCaosDBServerConnection
diff --git a/examples/set_permissions.py b/examples/set_permissions.py
index 8b2b59f10ac033110af846eef0ed90356a09553d..8162b11bfefb41b1bcdbc74b8e314f99a61d1a4e 100755
--- a/examples/set_permissions.py
+++ b/examples/set_permissions.py
@@ -25,15 +25,13 @@
As a result, only a specific user or group may access it.
-This script assumes that data similar to the demo server of IndiScale (at
-demo.indiscale.com) exists on the server specified in the pycaosdb.ini
-configuration.
+This script assumes that the user specified in the pycaosdb.ini
+configuration can create new entities.
"""
import caosdb as db
from caosdb import administration as admin
-import lxml
def assert_user_and_role():
@@ -50,27 +48,27 @@ out : tuple
"""
try:
human_user = admin._retrieve_user("jane")
- _activate_user("jane")
- except db.EntityDoesNotExistError:
+ admin._update_user(name="jane", status="ACTIVE")
+ except db.HTTPResourceNotFoundError:
human_user = admin._insert_user(
"jane", password="Human_Rememberable_Password_1234", status="ACTIVE")
try:
alien_user = admin._retrieve_user("xaxys")
- _activate_user("xaxys")
- except db.EntityDoesNotExistError:
+ admin._update_user(name="xaxys", status="ACTIVE")
+ except db.HTTPResourceNotFoundError:
alien_user = admin._insert_user("xaxys", password="4321_Syxax",
status="ACTIVE")
# At the moment, the return value is only "ok" for successful insertions.
try:
human_role = admin._retrieve_role("human")
- except db.EntityDoesNotExistError:
+ except db.HTTPResourceNotFoundError:
human_role = admin._insert_role("human", "An Earthling.")
try:
alien_role = admin._retrieve_role("alien")
- except db.EntityDoesNotExistError:
+ except db.HTTPResourceNotFoundError:
alien_role = admin._insert_role("alien", "An Extra-terrestrial.")
admin._set_roles("jane", ["human"])
@@ -80,24 +78,6 @@ out : tuple
("xaxys", list(admin._get_roles("xaxys"))))
-def _activate_user(user):
- """Set the user state to "ACTIVE" if necessary.
-
-Parameters
-----------
-user : str
- The user to activate.
-
-Returns
--------
-None
-
- """
- user_xml = lxml.etree.fromstring(admin._retrieve_user(user))
- if user_xml.xpath("User")[0].attrib["status"] != "ACTIVE":
- admin._update_user(user, status="ACTIVE")
-
-
def get_entities(count=1):
"""Retrieve one or more entities.
@@ -111,9 +91,11 @@ Returns
out : Container
A container of retrieved entities, the length is given by the parameter count.
"""
- cont = db.execute_query("FIND RECORD Guitar", flags={"P": "0L{n}".format(n=count)})
+ cont = db.execute_query("FIND RECORD 'Human Food'", flags={
+ "P": "0L{n}".format(n=count)})
if len(cont) != count:
- raise db.CaosDBException(msg="Incorrect number of entitities returned.")
+ raise db.CaosDBException(
+ msg="Incorrect number of entitities returned.")
return cont
@@ -138,7 +120,8 @@ general : bool, optional
# Set general permissions
if general:
- grant = admin.PermissionRule(action="grant", permission="RETRIEVE:OWNER")
+ grant = admin.PermissionRule(
+ action="grant", permission="RETRIEVE:OWNER")
deny = admin.PermissionRule(action="deny", permission="RETRIEVE:FILE")
admin._set_permissions(role=role_grant, permission_rules=[grant])
@@ -189,9 +172,12 @@ None
for ent in cont:
ent.retrieve()
print("Successfully retrieved all entities.")
- except db.AuthorizationException:
- print(ent)
- print("Could not retrieve this entity although it should have been possible!")
+ except db.TransactionError as te:
+ if te.has_error(db.AuthorizationError):
+ print(ent)
+ print("Could not retrieve this entity although it should have been possible!")
+ else:
+ raise te
# Switch to user without permissions
db.configure_connection(username=denied_user[0], password=denied_user[1],
@@ -206,23 +192,45 @@ None
denied_all = False
print(ent)
print("Could retrieve this entity although it should not have been possible!")
- except db.AuthorizationException:
- pass
+ except db.TransactionError as te:
+ # Only do something if an error wasn't caused by an
+ # AuthorizationError
+ if not te.has_error(db.AuthorizationError):
+ raise te
if denied_all:
print("Retrieval of all entities was successfully denied.")
+def create_test_entities():
+ """Create some test entities.
+ After calling this function, there will be a RecordType "Human Food" with the corresponding Records
+ "Bread", "Tomatoes", and "Twinkies" inserted in the database.
+ """
+ rt = db.RecordType(
+ name="Human Food", description="Food that can be eaten only by humans").insert()
+ food = ("Bread", "Tomatoes", "Twinkies")
+
+ cont = db.Container()
+ for i in range(len(food)):
+ rec = db.Record(food[i])
+ rec.add_parent(name="Human Food")
+ cont.append(rec)
+
+ cont.insert()
+
+
def main():
"""The main function of this script."""
- db.connection.connection.get_connection()._login()
-
+ """Create some test entities"""
+ create_test_entities()
+ """Create new users"""
human, alien = assert_user_and_role()
-
- # public, private, undefined entities
+ """Load the newly created entities."""
entities = get_entities(count=3)
-
+ """Set permission for the entities (only humans are allowed to eat human food)"""
set_permission(human[1][0], alien[1][0], entities)
+ """Test the permissions"""
test_permission((human[0], "Human_Rememberable_Password_1234"),
(alien[0], "4321_Syxax"), entities)
diff --git a/release.sh b/release.sh
index a5f3654291743e5ac22fd3b938719e294b3a1e90..1af097f014de6cd9eb3d3e8ba5da34aea0fe1671 100755
--- a/release.sh
+++ b/release.sh
@@ -1,4 +1,4 @@
#!/bin/bash
-rm -r dist/ build/ .eggs/
+rm -rf dist/ build/ .eggs/
python setup.py sdist bdist_wheel
python -m twine upload -s dist/*
diff --git a/setup.py b/setup.py
index 8893a126a7a02d7646625af7fc82787238344173..e1d39458ea8d1b0b17ea12a82ebd7133b27b045a 100755
--- a/setup.py
+++ b/setup.py
@@ -46,8 +46,8 @@ from setuptools import find_packages, setup
########################################################################
MAJOR = 0
-MINOR = 3
-MICRO = 2
+MINOR = 5
+MICRO = 3
PRE = "" # e.g. rc0, alpha.1, 0.beta-23
ISRELEASED = False
@@ -155,6 +155,7 @@ def setup_package():
author='Timm Fitschen',
author_email='t.fitschen@indiscale.com',
packages=find_packages('src'),
+ python_requires='>=3.6',
package_dir={'': 'src'},
install_requires=['lxml>=3.6.4',
'PyYaml>=3.12', 'future', 'PySocks>=1.6.7'],
@@ -163,7 +164,8 @@ def setup_package():
tests_require=["pytest", "pytest-cov", "coverage>=4.4.2"],
package_data={
'caosdb': ['cert/indiscale.ca.crt'],
- }
+ },
+ scripts=["src/caosdb/utils/caosdb_admin.py"]
)
try:
setup(**metadata)
diff --git a/src/caosdb/__init__.py b/src/caosdb/__init__.py
index 92922f79c6b5105abc5e849175c4271561dd92a9..4043bbed3283c07d5a1ff7e3dbd77593a7f82fb1 100644
--- a/src/caosdb/__init__.py
+++ b/src/caosdb/__init__.py
@@ -22,6 +22,13 @@
# ** end header
#
+"""CaosDB Python bindings.
+
+Tries to read from the inifile specified in the environment variable `PYCAOSDBINI` or alternatively
+in `~/.pycaosdb.ini` upon import. After that, the ini file `pycaosdb.ini` in the current working
+directory will be read additionally, if it exists.
+"""
+
from os import environ, getcwd
# Import of the connection function (which is used to connect to the DB):
from os.path import expanduser, join
@@ -30,7 +37,8 @@ from os.path import expanduser, join
import caosdb.apiutils
from caosdb.common import administration
from caosdb.common.datatype import (BOOLEAN, DATETIME, DOUBLE, FILE, INTEGER,
- REFERENCE, TEXT, LIST)
+ LIST, REFERENCE, TEXT)
+from caosdb.common.state import State, Transition
# Import of the basic API classes:
from caosdb.common.models import (ACL, ALL, FIX, NONE, OBLIGATORY, RECOMMENDED,
SUGGESTED, Container, DropOffBox, Entity,
@@ -38,14 +46,9 @@ from caosdb.common.models import (ACL, ALL, FIX, NONE, OBLIGATORY, RECOMMENDED,
Query, QueryTemplate, Record, RecordType,
delete, execute_query, get_global_acl,
get_known_permissions, raise_errors)
-from caosdb.configuration import configure, get_config
+from caosdb.configuration import _read_config_files, configure, get_config
from caosdb.connection.connection import configure_connection, get_connection
from caosdb.exceptions import *
+from caosdb.version import version as __version__
-# read configuration these files
-
-if "PYCAOSDBINI" in environ:
- configure(expanduser(environ["PYCAOSDBINI"]))
-else:
- configure(expanduser('~/.pycaosdb.ini'))
-configure(join(getcwd(), "pycaosdb.ini"))
+_read_config_files()
diff --git a/src/caosdb/apiutils.py b/src/caosdb/apiutils.py
index bd279fcfe5c394b9b5cff787169cff5b9f2d3031..73074efc3057e0548c5abfd56ef3cf1ac9e9bf47 100644
--- a/src/caosdb/apiutils.py
+++ b/src/caosdb/apiutils.py
@@ -34,10 +34,10 @@ from collections.abc import Iterable
from subprocess import call
from caosdb.common.datatype import (BOOLEAN, DATETIME, DOUBLE, FILE, INTEGER,
- REFERENCE, TEXT)
+ REFERENCE, TEXT, is_reference)
from caosdb.common.models import (Container, Entity, File, Property, Query,
Record, RecordType, get_config,
- execute_query, is_reference)
+ execute_query)
def new_record(record_type, name=None, description=None,
@@ -244,6 +244,8 @@ class CaosDBPythonEntity(object):
return (val, False)
elif pr[0:4] == "LIST":
return self._type_converted_list(val, pr)
+ elif isinstance(val, Entity):
+ return (convert_to_python_object(val), False)
else:
return (int(val), True)
diff --git a/src/caosdb/common/__init__.py b/src/caosdb/common/__init__.py
index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..436281df8077b2cbf357537d36b21567b86ea5a2 100644
--- a/src/caosdb/common/__init__.py
+++ b/src/caosdb/common/__init__.py
@@ -0,0 +1 @@
+"""Commonly used classes for CaosDB."""
diff --git a/src/caosdb/common/administration.py b/src/caosdb/common/administration.py
index 157518f82294257d7267b881e96d225ecce4304f..e7ba94182d7a4d8b60c6400cd1d804f62f7bf03c 100644
--- a/src/caosdb/common/administration.py
+++ b/src/caosdb/common/administration.py
@@ -5,6 +5,8 @@
#
# Copyright (C) 2018 Research Group Biomedical Physics,
# Max-Planck-Institute for Dynamics and Self-Organization Göttingen
+# Copyright (C) 2020 Timm Fitschen <t.fitschen@indiscale.com>
+# Copyright (C) 2020 IndiScale GmbH <info@indiscale.com>
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as
@@ -28,8 +30,12 @@ from lxml import etree
from caosdb.common.utils import xml2str
from caosdb.connection.connection import get_connection
-from caosdb.exceptions import (AuthorizationException, ClientErrorException,
- EntityDoesNotExistError)
+from caosdb.exceptions import (HTTPClientError,
+ HTTPForbiddenError,
+ HTTPResourceNotFoundError,
+ EntityDoesNotExistError,
+ ServerConfigurationException,
+ )
def set_server_property(key, value):
@@ -50,9 +56,11 @@ def set_server_property(key, value):
None
"""
con = get_connection()
-
- con._form_data_request(method="POST", path="_server_properties",
- params={key: value}).read()
+ try:
+ con._form_data_request(method="POST", path="_server_properties",
+ params={key: value}).read()
+ except EntityDoesNotExistError:
+ raise ServerConfigurationException("Debug mode in server is probably disabled.") from None
def get_server_properties():
@@ -66,7 +74,11 @@ def get_server_properties():
The server properties.
"""
con = get_connection()
- body = con._http_request(method="GET", path="_server_properties").response
+ try:
+ body = con._http_request(method="GET", path="_server_properties").response
+ except EntityDoesNotExistError:
+ raise ServerConfigurationException("Debug mode in server is probably disabled.") from None
+
xml = etree.parse(body)
props = dict()
@@ -104,10 +116,10 @@ def _retrieve_user(name, realm=None, **kwargs):
con = get_connection()
try:
return con._http_request(method="GET", path="User/" + (realm + "/" + name if realm is not None else name), **kwargs).read()
- except AuthorizationException as e:
+ except HTTPForbiddenError as e:
e.msg = "You are not permitted to retrieve this user."
raise
- except EntityDoesNotExistError as e:
+ except HTTPResourceNotFoundError as e:
e.msg = "User does not exist."
raise
@@ -116,10 +128,10 @@ def _delete_user(name, **kwargs):
con = get_connection()
try:
return con._http_request(method="DELETE", path="User/" + name, **kwargs).read()
- except AuthorizationException as e:
+ except HTTPForbiddenError as e:
e.msg = "You are not permitted to delete this user."
raise
- except EntityDoesNotExistError as e:
+ except HTTPResourceNotFoundError as e:
e.msg = "User does not exist."
raise
@@ -142,13 +154,13 @@ def _update_user(name, realm=None, password=None, status=None,
params["entity"] = str(entity)
try:
return con.put_form_data(entity_uri_segment="User/" + (realm + "/" + name if realm is not None else name), params=params, **kwargs).read()
- except EntityDoesNotExistError as e:
+ except HTTPResourceNotFoundError as e:
e.msg = "User does not exist."
raise
- except AuthorizationException as e:
+ except HTTPForbiddenError as e:
e.msg = "You are not permitted to update this user."
raise
- except ClientErrorException as e:
+ except HTTPClientError as e:
if e.status == 409:
e.msg = "Entity does not exist."
raise
@@ -171,10 +183,10 @@ def _insert_user(name, password=None, status=None, email=None, entity=None, **kw
params["entity"] = entity
try:
return con.post_form_data(entity_uri_segment="User", params=params, **kwargs).read()
- except AuthorizationException as e:
+ except HTTPForbiddenError as e:
e.msg = "You are not permitted to insert a new user."
raise e
- except ClientErrorException as e:
+ except HTTPClientError as e:
if e.status == 409:
e.msg = "User name is already in use."
@@ -187,10 +199,10 @@ def _insert_role(name, description, **kwargs):
con = get_connection()
try:
return con.post_form_data(entity_uri_segment="Role", params={"role_name": name, "role_description": description}, **kwargs).read()
- except AuthorizationException as e:
+ except HTTPForbiddenError as e:
e.msg = "You are not permitted to insert a new role."
raise
- except ClientErrorException as e:
+ except HTTPClientError as e:
if e.status == 409:
e.msg = "Role name is already in use. Choose a different name."
raise
@@ -200,10 +212,10 @@ def _update_role(name, description, **kwargs):
con = get_connection()
try:
return con.put_form_data(entity_uri_segment="Role/" + name, params={"role_description": description}, **kwargs).read()
- except AuthorizationException as e:
+ except HTTPForbiddenError as e:
e.msg = "You are not permitted to update this role."
raise
- except EntityDoesNotExistError as e:
+ except HTTPResourceNotFoundError as e:
e.msg = "Role does not exist."
raise
@@ -212,10 +224,10 @@ def _retrieve_role(name, **kwargs):
con = get_connection()
try:
return con._http_request(method="GET", path="Role/" + name, **kwargs).read()
- except AuthorizationException as e:
+ except HTTPForbiddenError as e:
e.msg = "You are not permitted to retrieve this role."
raise
- except EntityDoesNotExistError as e:
+ except HTTPResourceNotFoundError as e:
e.msg = "Role does not exist."
raise
@@ -224,10 +236,10 @@ def _delete_role(name, **kwargs):
con = get_connection()
try:
return con._http_request(method="DELETE", path="Role/" + name, **kwargs).read()
- except AuthorizationException as e:
+ except HTTPForbiddenError as e:
e.msg = "You are not permitted to delete this role."
raise
- except EntityDoesNotExistError as e:
+ except HTTPResourceNotFoundError as e:
e.msg = "Role does not exist."
raise
@@ -241,14 +253,15 @@ def _set_roles(username, roles, realm=None, **kwargs):
body = xml2str(xml)
con = get_connection()
try:
- body = con._http_request(method="PUT", path="UserRoles/" + (realm + "/" + username if realm is not None else username), body=body, **kwargs).read()
- except AuthorizationException as e:
+ body = con._http_request(method="PUT", path="UserRoles/" + (realm + "/" +
+ username if realm is not None else username), body=body, **kwargs).read()
+ except HTTPForbiddenError as e:
e.msg = "You are not permitted to set this user's roles."
raise
- except EntityDoesNotExistError as e:
+ except HTTPResourceNotFoundError as e:
e.msg = "User does not exist."
raise
- except ClientErrorException as e:
+ except HTTPClientError as e:
if e.status == 409:
e.msg = "Role does not exist."
raise
@@ -264,11 +277,12 @@ def _set_roles(username, roles, realm=None, **kwargs):
def _get_roles(username, realm=None, **kwargs):
con = get_connection()
try:
- body = con._http_request(method="GET", path="UserRoles/" + (realm + "/" + username if realm is not None else username), **kwargs).read()
- except AuthorizationException as e:
+ body = con._http_request(method="GET", path="UserRoles/" + (
+ realm + "/" + username if realm is not None else username), **kwargs).read()
+ except HTTPForbiddenError as e:
e.msg = "You are not permitted to retrieve this user's roles."
raise
- except EntityDoesNotExistError as e:
+ except HTTPResourceNotFoundError as e:
e.msg = "User does not exist."
raise
ret = set()
@@ -308,10 +322,10 @@ Returns
con = get_connection()
try:
return con._http_request(method="PUT", path="PermissionRules/" + role, body=body, **kwargs).read()
- except AuthorizationException as e:
+ except HTTPForbiddenError as e:
e.msg = "You are not permitted to set this role's permissions."
raise
- except EntityDoesNotExistError as e:
+ except HTTPResourceNotFoundError as e:
e.msg = "Role does not exist."
raise
@@ -320,10 +334,10 @@ def _get_permissions(role, **kwargs):
con = get_connection()
try:
return PermissionRule._parse_body(con._http_request(method="GET", path="PermissionRules/" + role, **kwargs).read())
- except AuthorizationException as e:
+ except HTTPForbiddenError as e:
e.msg = "You are not permitted to retrieve this role's permissions."
raise
- except EntityDoesNotExistError as e:
+ except HTTPResourceNotFoundError as e:
e.msg = "Role does not exist."
raise
diff --git a/src/caosdb/common/datatype.py b/src/caosdb/common/datatype.py
index 246485c3957462fc98ac83f9d904413ad7518302..eb8c1e4e0088f1924940a104ec3916b9d5d40f99 100644
--- a/src/caosdb/common/datatype.py
+++ b/src/caosdb/common/datatype.py
@@ -25,7 +25,7 @@
import re
-from ..exceptions import AmbiguityException, EntityDoesNotExistError
+from ..exceptions import EmptyUniqueQueryError, QueryNotUniqueError
DOUBLE = "DOUBLE"
REFERENCE = "REFERENCE"
@@ -91,9 +91,9 @@ def get_id_of_datatype(datatype):
Raises
------
- AmbiguityException
+ QueryNotUniqueError
If there are more than one entities with the same name as the datatype.
- EntityDoesNotExistError
+ EmptyUniqueQueryError
If there is no entity with the name of the datatype.
"""
from caosdb import execute_query
@@ -107,11 +107,11 @@ def get_id_of_datatype(datatype):
res = [el for el in res if el.name.lower() == datatype.lower()]
if len(res) > 1:
- raise AmbiguityException(
+ raise QueryNotUniqueError(
"Name {} did not lead to unique result; Missing "
"implementation".format(datatype))
elif len(res) != 1:
- raise EntityDoesNotExistError(
+ raise EmptyUniqueQueryError(
"No RecordType named {}".format(datatype))
return res[0].id
diff --git a/src/caosdb/common/models.py b/src/caosdb/common/models.py
index 5ab31f482454a408c8b692e24adea4099a28f6cc..efd4a3e7af55297588bf03f869f18846d991f3f0 100644
--- a/src/caosdb/common/models.py
+++ b/src/caosdb/common/models.py
@@ -5,6 +5,9 @@
#
# Copyright (C) 2018 Research Group Biomedical Physics,
# Max-Planck-Institute for Dynamics and Self-Organization Göttingen
+# Copyright (C) 2020 Indiscale GmbH <info@indiscale.com>
+# Copyright (C) 2020 Florian Spreckelsen <f.spreckelsen@indiscale.com>
+# Copyright (C) 2020 Timm Fitschen <t.fitschen@indiscale.com>
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as
@@ -27,7 +30,6 @@ from __future__ import print_function, unicode_literals
import re
import sys
-import traceback
from builtins import str
from functools import cmp_to_key
from hashlib import sha512
@@ -38,23 +40,28 @@ from sys import hexversion
from tempfile import NamedTemporaryFile
from warnings import warn
-from caosdb.common.datatype import (BOOLEAN, DATETIME, DOUBLE, FILE, INTEGER,
- LIST, REFERENCE, TEXT, is_reference)
+from caosdb.common.datatype import (BOOLEAN, DATETIME, DOUBLE, INTEGER, TEXT)
+from caosdb.common.versioning import Version
+from caosdb.common.state import State
from caosdb.common.utils import uuid, xml2str
from caosdb.configuration import get_config
from caosdb.connection.connection import get_connection
from caosdb.connection.encode import MultipartParam, multipart_encode
-from caosdb.exceptions import (AmbiguityException, AuthorizationException,
- CaosDBException, ConnectionException,
+from caosdb.exceptions import (AmbiguousEntityError,
+ AuthorizationError,
+ CaosDBException, CaosDBConnectionError,
ConsistencyError,
+ EmptyUniqueQueryError,
EntityDoesNotExistError, EntityError,
- EntityHasNoDatatypeError, TransactionError,
- UniqueNamesError, UnqualifiedParentsError,
- UnqualifiedPropertiesError, URITooLongException)
+ EntityHasNoDatatypeError,
+ MismatchingEntitiesError,
+ QueryNotUniqueError, TransactionError,
+ UniqueNamesError,
+ UnqualifiedParentsError,
+ UnqualifiedPropertiesError,
+ HTTPURITooLongError)
from lxml import etree
-from .datatype import is_reference
-
_ENTITY_URI_SEGMENT = "Entity"
# importances/inheritance
@@ -86,6 +93,7 @@ class Entity(object):
self._size = None
self._upload = None
self._wrapped_entity = None
+ self._version = None
self._cuid = None
self._flags = dict()
self.__value = None
@@ -105,6 +113,18 @@ class Entity(object):
self.name = name
self.description = description
self.id = id
+ self.state = None
+
+ @property
+ def version(self):
+ if self._version is not None or self._wrapped_entity is None:
+ return self._version
+
+ return self._wrapped_entity.version
+
+ @version.setter
+ def version(self, version):
+ self._version = version
@property
def role(self):
@@ -368,7 +388,7 @@ class Entity(object):
del copy_kwargs['importance']
del copy_kwargs['inheritance']
- new_property = Property(value=value, **copy_kwargs)
+ new_property = Property(**copy_kwargs)
if abstract_property is not None:
new_property._wrap(property)
@@ -378,6 +398,8 @@ class Entity(object):
if new_property.datatype is None and isinstance(
property, (RecordType, Record, File)):
new_property.datatype = property
+ new_property.value = value
+
self.properties.append(
property=new_property, importance=(
kwargs['importance'] if 'importance' in kwargs else None), inheritance=(
@@ -517,18 +539,38 @@ class Entity(object):
w_parent._get_parent_recursively(all_parents)
def get_parent(self, key):
+ """Return the first parent matching the key or None if no match exists.
+
+ Parameters
+ ---------
+ key : int or Enity or str
+ The id, Entity, or name of the parent that should be
+ returned. If an Entity is given, its id or its name is
+ used to find a matching parent.
+
+ Returns
+ -------
+ parent : Entity
+ The first parent of this entity that matches the given id,
+ entity, or name.
+
+ """
if isinstance(key, int):
for p in self.parents:
if p.id is not None and int(p.id) == int(key):
return p
elif isinstance(key, Entity):
- if self.id is not None:
- return self.get_parent(int(key.id))
- else:
- return self.get_parent(key.name)
+ if key.id is not None:
+ # first try by id
+ found = self.get_parent(int(key.id))
+ if found is not None:
+ return found
+ # otherwise by name
+ return self.get_parent(key.name)
else:
for p in self.parents:
- if p.name is not None and str(p.name) == str(key):
+ if (p.name is not None
+ and str(p.name).lower() == str(key).lower()):
return p
return None
@@ -586,6 +628,102 @@ class Entity(object):
return None
+ def _get_value_for_selector(self, selector):
+ """return the value described by the selector
+
+ A selector is a list or a tuple of strings describing a path in an
+ entity tree with self as root. The last selector may be a special one
+ like unit or name.
+
+ See also get_property_values()
+ """
+ SPECIAL_SELECTORS = ["unit", "value", "description", "id", "name"]
+
+ if not isinstance(selector, (tuple, list)):
+ selector = [selector]
+
+ ref = self
+
+ # there are some special selectors which can be applied to the
+ # final element; if such a special selector exists we split it
+ # from the list
+
+ if selector[-1].lower() in SPECIAL_SELECTORS:
+ special_selector = selector[-1]
+ selector = selector[:-1]
+ else:
+ special_selector = None
+
+ # iterating through the entity tree according to the selector
+ for subselector in selector:
+ # selector does not match the structure, we cannot get a
+ # property of non-entity
+
+ if not isinstance(ref, Entity):
+ return None
+
+ prop = ref.get_property(subselector)
+
+ # selector does not match the structure, we did not get a
+ # property
+ if prop is None:
+ return None
+
+ # if the property is a reference, we are interested in the
+ # corresponding entities attributes
+ if isinstance(prop.value, Entity):
+ ref = prop.value
+
+ # otherwise in the attributes of the property
+ else:
+ ref = prop
+
+ # if we saved a special selector before, apply it
+ if special_selector is None:
+ return prop.value
+ else:
+ return getattr(ref, special_selector.lower())
+
+ def get_property_values(self, *selectors):
+ """ Return a tuple with the values described by the given selectors.
+
+ This represents an entity's properties as if it was a row of a table
+ with the given columns.
+
+ If the elements of the selectors parameter are tuples, they will return
+ the properties of the referenced entity, if present. E.g. ("window",
+ "height") will return the value of the height property of the
+ referenced window entity.
+
+ The tuple's values correspond to the order of selectors parameter.
+
+ The tuple contains None for all values that are not available in the
+ entity. That does not necessarily mean, that the values are not stored
+ in the database (e.g. if a single entity was retrieved without
+ referenced entities).
+
+ Parameters
+ ----------
+ *selectors : str or tuple of str
+ Each selector is a list or tuple of property names, e.g. `"height",
+ "width"`.
+
+ Returns
+ -------
+ row : tuple
+ A row-like representation of the entity's properties.
+ """
+ row = tuple()
+
+ for selector in selectors:
+ val = self._get_value_for_selector(selector)
+
+ if isinstance(val, Entity):
+ val = val.id if val.id is not None else val.name
+ row += (val,)
+
+ return row
+
def get_messages(self):
"""Get all messages of this entity.
@@ -623,15 +761,14 @@ class Entity(object):
return ret
- def get_errors_deep(self, roots=[]):
+ def get_errors_deep(self, roots=None):
"""Get all error messages of this entity and all sub-entities /
- parents.
-
- / properties.
+ parents / properties.
@return A list of tuples. Tuple index 0 contains the error message
and tuple index 1 contains the tree.
"""
+ roots = [] if roots is None else roots
result_list = list()
ret_self = self.get_errors()
result_list.extend([
@@ -672,8 +809,7 @@ class Entity(object):
xml = etree.Element("Entity")
assert isinstance(xml, etree._Element)
- ''' unwrap wrapped entity '''
-
+ # unwrap wrapped entity
if self._wrapped_entity is not None:
xml = self._wrapped_entity.to_xml(xml, add_properties)
@@ -689,6 +825,9 @@ class Entity(object):
if self.description is not None:
xml.set("description", str(self.description))
+ if self.version is not None:
+ xml.append(self.version.to_xml())
+
if self.value is not None:
if isinstance(self.value, Entity):
if self.value.id is not None:
@@ -717,6 +856,8 @@ class Entity(object):
xml.append(v_elem)
elif self.value == "":
xml.append(etree.Element("EmptyString"))
+ elif str(self.value) == "nan":
+ xml.text = "NaN"
else:
xml.text = str(self.value)
@@ -770,6 +911,9 @@ class Entity(object):
if self.acl is not None:
xml.append(self.acl.to_xml())
+ if self.state is not None:
+ xml.append(self.state.to_xml())
+
return xml
@staticmethod
@@ -812,19 +956,33 @@ class Entity(object):
entity.permissions = child
elif isinstance(child, Message):
entity.add_message(child)
+ elif isinstance(child, Version):
+ entity.version = child
+ elif isinstance(child, State):
+ entity.state = child
elif child is None or hasattr(child, "encode"):
vals.append(child)
+ elif isinstance(child, Entity):
+ vals.append(child)
else:
raise TypeError(
'Child was neither a Property, nor a Parent, nor a Message.\
Was ' + str(type(child)))
- # parse VALUE
- if len(vals):
+ # add VALUE
+ value = None
+
+ if vals:
# The value[s] have been inside a <Value> tag.
- entity.value = vals
+ value = vals
elif elem.text is not None and elem.text.strip() != "":
- entity.value = elem.text.strip()
+ value = elem.text.strip()
+
+ try:
+ entity.value = value
+ except ValueError:
+ # circumvent the parsing.
+ entity.__value = value
return entity
@@ -839,11 +997,20 @@ class Entity(object):
if self.id is None:
c = Container().retrieve(query=self.name, sync=False)
- if len(c == 1):
+ if len(c) == 1:
e = c[0]
+ elif len(c) == 0:
+ ee = EntityDoesNotExistError(
+ "The entity to be updated does not exist on the server.",
+ entity=self
+ )
+ raise TransactionError(ee)
else:
- raise AmbiguityException(
- "Could not determine the desired Entity which is to be updated by its name.")
+ ae = AmbiguousEntityError(
+ "Could not determine the desired Entity which is to be updated by its name.",
+ entity=self
+ )
+ raise TransactionError(ae)
else:
e = Container().retrieve(query=self.id, sync=False)[0]
e.acl = ACL(self.acl.to_xml())
@@ -882,13 +1049,12 @@ class Entity(object):
if len(c) == 1:
c[0].messages.extend(c.messages)
-
return c[0]
- else:
- raise AmbiguityException("This retrieval was not unique!!!")
- else:
- return Container().append(self).retrieve(
- unique=unique, raise_exception_on_error=raise_exception_on_error, flags=flags)
+
+ raise QueryNotUniqueError("This retrieval was not unique!!!")
+
+ return Container().append(self).retrieve(
+ unique=unique, raise_exception_on_error=raise_exception_on_error, flags=flags)
def insert(self, raise_exception_on_error=True, unique=True,
sync=True, strict=False, flags=None):
@@ -921,7 +1087,7 @@ class Entity(object):
flags=flags)[0]
def update(self, strict=False, raise_exception_on_error=True,
- unique=True, flags=None):
+ unique=True, flags=None, sync=True):
"""Update this entity.
There are two possible work-flows to perform this update:
@@ -955,6 +1121,7 @@ class Entity(object):
return Container().append(self).update(
strict=strict,
+ sync=sync,
raise_exception_on_error=raise_exception_on_error,
unique=unique,
flags=flags)[0]
@@ -973,12 +1140,16 @@ class Entity(object):
def _parse_value(datatype, value):
if value is None:
return value
+
if datatype is None:
return value
+
if datatype == DOUBLE:
return float(value)
+
if datatype == INTEGER:
return int(str(value))
+
if datatype == BOOLEAN:
if str(value).lower() == "true":
return True
@@ -986,14 +1157,17 @@ def _parse_value(datatype, value):
return False
else:
raise ValueError("Boolean value was {}.".format(value))
+
if datatype in [DATETIME, TEXT]:
if isinstance(value, str):
return value
# deal with collections
+
if isinstance(datatype, str):
matcher = re.compile(r"^(?P<col>[^<]+)<(?P<dt>[^>]+)>$")
m = matcher.match(datatype)
+
if m:
col = m.group("col")
dt = m.group("dt")
@@ -1014,14 +1188,18 @@ def _parse_value(datatype, value):
# This is for a special case, where the xml parser could not differentiate
# between single values and lists with one element. As
+
if hasattr(value, "__len__") and len(value) == 1:
return _parse_value(datatype, value[0])
# deal with references
+
if isinstance(value, Entity):
return value
+
if isinstance(value, str) and "@" in value:
# probably this is a versioned reference
+
return str(value)
else:
# for unversioned references
@@ -1029,6 +1207,7 @@ def _parse_value(datatype, value):
return int(value)
except ValueError:
# reference via name
+
return str(value)
@@ -1077,9 +1256,11 @@ class QueryTemplate():
self.permissions = None
self.is_valid = lambda: False
self.is_deleted = lambda: False
+ self.version = None
+ self.state = None
- def retrieve(self, strict=True, raise_exception_on_error=True,
- unique=True, sync=True, flags=None):
+ def retrieve(self, raise_exception_on_error=True, unique=True, sync=True,
+ flags=None):
return Container().append(self).retrieve(
raise_exception_on_error=raise_exception_on_error,
@@ -1127,6 +1308,9 @@ class QueryTemplate():
if self.description is not None:
xml.set("description", self.description)
+ if self.version is not None:
+ xml.append(self.version.to_xml())
+
if self.query is not None:
queryElem = etree.Element("Query")
queryElem.text = self.query
@@ -1156,6 +1340,8 @@ class QueryTemplate():
q.messages.append(child)
elif isinstance(child, ACL):
q.acl = child
+ elif isinstance(child, Version):
+ q.version = child
elif isinstance(child, Permissions):
q.permissions = child
q.id = int(xml.get("id"))
@@ -1219,8 +1405,7 @@ class Parent(Entity):
if xml is None:
xml = etree.Element("Parent")
- return super(Parent, self).to_xml(
- xml=xml, add_properties=add_properties)
+ return super().to_xml(xml=xml, add_properties=add_properties)
class _EntityWrapper(object):
@@ -1285,7 +1470,7 @@ class Message(object):
if xml is None:
xml = etree.Element(str(self.type))
- if self.code:
+ if self.code is not None:
xml.set("code", str(self.code))
if self.description:
@@ -1301,7 +1486,7 @@ class Message(object):
def __eq__(self, obj):
if isinstance(obj, Message):
- return self.type == obj.type and self.code == obj.codes
+ return self.type == obj.type and self.code == obj.code
return False
@@ -1324,8 +1509,8 @@ class RecordType(Entity):
# set default importance
copy_kwargs['inheritance'] = FIX
- return super(RecordType, self).add_property(
- property=property, value=value, **copy_kwargs)
+ return super().add_property(property=property, value=value,
+ **copy_kwargs)
def add_parent(self, parent=None, **kwargs):
copy_kwargs = kwargs.copy()
@@ -1334,7 +1519,7 @@ class RecordType(Entity):
# set default importance
copy_kwargs['inheritance'] = OBLIGATORY
- return super(RecordType, self).add_parent(parent=parent, **copy_kwargs)
+ return super().add_parent(parent=parent, **copy_kwargs)
def __init__(self, name=None, id=None, description=None, datatype=None): # @ReservedAssignment
Entity.__init__(self, name=name, id=id, description=description,
@@ -1362,7 +1547,7 @@ class Record(Entity):
# set default importance
copy_kwargs['inheritance'] = FIX
- return super(Record, self).add_property(
+ return super().add_property(
property=property, value=value, **copy_kwargs)
def __init__(self, name=None, id=None, description=None): # @ReservedAssignment
@@ -1526,7 +1711,7 @@ class File(Record):
# set default importance
copy_kwargs['inheritance'] = FIX
- return super(File, self).add_property(
+ return super().add_property(
property=property, value=value, **copy_kwargs)
@@ -1851,7 +2036,8 @@ class _Messages(dict):
<<< del msgs["HelloWorld",2]
<<< assert msgs.get("HelloWorld",2) == None
- <<< # this Message has no code and no description (make easy things easy...)
+ # this Message has no code and no description (make easy things easy...)
+ <<<
<<< msgs["HelloWorld"] = "Hello!"
<<< assert msgs["HelloWorld"] == "Hello!"
@@ -1882,6 +2068,9 @@ class _Messages(dict):
else:
raise TypeError(
"('type', 'code'), ('type'), or 'type' expected.")
+ elif isinstance(key, _Messages._msg_key):
+ type = key._type # @ReservedAssignment
+ code = key._code
else:
type = key # @ReservedAssignment
code = None
@@ -1896,11 +2085,14 @@ class _Messages(dict):
else:
raise TypeError(
"('description', 'body'), ('body'), or 'body' expected.")
+ if isinstance(value, Message):
+ body = value.body
+ description = value.description
else:
body = value
description = None
m = Message(type=type, code=code, description=description, body=body)
- self.append(m)
+ dict.__setitem__(self, _Messages._msg_key(type, code), m)
def __getitem__(self, key):
if isinstance(key, tuple):
@@ -2041,6 +2233,8 @@ def _basic_sync(e_local, e_remote):
e_local.permissions = e_remote.permissions
e_local.is_valid = e_remote.is_valid
e_local.is_deleted = e_remote.is_deleted
+ e_local.version = e_remote.version
+ e_local.state = e_remote.state
if hasattr(e_remote, "query"):
e_local.query = e_remote.query
@@ -2053,15 +2247,19 @@ def _basic_sync(e_local, e_remote):
def _deletion_sync(e_local, e_remote):
if e_local is None or e_remote is None:
- return None
+ return
+
try:
- e_remote.get_messages()[('info', 10)]
- _basic_sync(e_local, e_remote)
- e_local.is_valid = lambda: False
- e_local.is_deleted = lambda: True
- e_local.id = None
+ e_remote.get_messages()["info", 10] # try and get the deletion info
except KeyError:
+ # deletion info wasn't there
e_local.messages = e_remote.messages
+ return
+
+ _basic_sync(e_local, e_remote)
+ e_local.is_valid = lambda: False
+ e_local.is_deleted = lambda: True
+ e_local.id = None
class Container(list):
@@ -2102,11 +2300,11 @@ class Container(list):
"""
if entity in self:
- super(Container, self).remove(entity)
+ super().remove(entity)
else:
for ee in self:
if entity == ee.id:
- super(Container, self).remove(ee)
+ super().remove(ee)
return ee
raise ValueError(
@@ -2224,13 +2422,13 @@ class Container(list):
"""
if isinstance(entity, Entity):
- super(Container, self).append(entity)
+ super().append(entity)
elif isinstance(entity, int):
- super(Container, self).append(Entity(id=entity))
+ super().append(Entity(id=entity))
elif hasattr(entity, "encode"):
- super(Container, self).append(Entity(name=entity))
+ super().append(Entity(name=entity))
elif isinstance(entity, QueryTemplate):
- super(Container, self).append(entity)
+ super().append(entity)
else:
raise TypeError(
"Entity was neither an id nor a name nor an entity." +
@@ -2246,8 +2444,8 @@ class Container(list):
@return xml element
"""
tmpid = 0
- ''' users might already have specified some tmpids. -> look for smallest.'''
+ # users might already have specified some tmpids. -> look for smallest.
for e in self:
tmpid = min(tmpid, Container._get_smallest_tmpid(e))
tmpid -= 1
@@ -2466,8 +2664,7 @@ class Container(list):
# list of remote entities which already have a local equivalent
used_remote_entities = []
- ''' match by cuid '''
-
+ # match by cuid
for local_entity in self:
sync_dict[local_entity] = None
@@ -2493,10 +2690,9 @@ class Container(list):
local_entity.add_message(Message("Error", None, msg))
if raise_exception_on_error:
- raise AmbiguityException(msg)
-
- ''' match by id '''
+ raise MismatchingEntitiesError(msg)
+ # match by id
for local_entity in self:
if sync_dict[local_entity] is None and local_entity.id is not None:
sync_remote_entities = []
@@ -2518,10 +2714,9 @@ class Container(list):
local_entity.add_message(Message("Error", None, msg))
if raise_exception_on_error:
- raise AmbiguityException(msg)
-
- ''' match by path '''
+ raise MismatchingEntitiesError(msg)
+ # match by path
for local_entity in self:
if (sync_dict[local_entity] is None
and local_entity.path is not None):
@@ -2548,10 +2743,9 @@ class Container(list):
local_entity.add_message(Message("Error", None, msg))
if raise_exception_on_error:
- raise AmbiguityException(msg)
-
- ''' match by name '''
+ raise MismatchingEntitiesError(msg)
+ # match by name
for local_entity in self:
if (sync_dict[local_entity] is None
and local_entity.name is not None):
@@ -2578,7 +2772,7 @@ class Container(list):
local_entity.add_message(Message("Error", None, msg))
if raise_exception_on_error:
- raise AmbiguityException(msg)
+ raise MismatchingEntitiesError(msg)
# add remaining entities to this remote_container
sync_remote_entities = []
@@ -2597,7 +2791,7 @@ class Container(list):
remote_container.add_message(Message("Error", None, msg))
if raise_exception_on_error:
- raise AmbiguityException(msg)
+ raise MismatchingEntitiesError(msg)
return sync_dict
@@ -2612,11 +2806,24 @@ class Container(list):
this happens, none of them will be deleted. It occurs an error
instead.
"""
+ chunk_size = 100
+ item_count = len(self)
+ # Split Container in 'chunk_size'-sized containers (if necessary) to avoid error 414 Request-URI Too Long
+ if item_count > chunk_size:
+ for i in range(0, int(item_count/chunk_size)+1):
+ chunk = Container()
+ for j in range(i*chunk_size, min(item_count, (i+1)*chunk_size)):
+ chunk.append(self[j])
+ if len(chunk):
+ chunk.delete()
+ return self
if len(self) == 0:
if raise_exception_on_error:
- raise TransactionError(
- self, "There are no entities to be deleted. This container is empty.")
+ te = TransactionError(
+ msg="There are no entities to be deleted. This container is empty.",
+ container=self)
+ raise te
return self
self.clear_server_messages()
@@ -2640,21 +2847,23 @@ class Container(list):
description="This entity has no identifier. It cannot be deleted."))
if raise_exception_on_error:
- raise EntityError(
+ ee = EntityError(
"This entity has no identifier. It cannot be deleted.", entity)
- else:
- entity.is_valid = lambda: False
+ raise TransactionError(ee)
+ entity.is_valid = lambda: False
if len(id_str) == 0:
if raise_exception_on_error:
- raise TransactionError(
- self, "There are no entities to be deleted.")
+ te = TransactionError(
+ msg="There are no entities to be deleted.",
+ container=self)
+ raise te
return self
entity_url_segments = [_ENTITY_URI_SEGMENT, "&".join(id_str)]
_log_request("DELETE: " + str(entity_url_segments) +
- ("?" + flags if flags is not None else ''))
+ ("?" + str(flags) if flags is not None else ''))
http_response = c.delete(entity_url_segments, query_dict=flags)
cresp = Container._response_to_entities(http_response)
@@ -2708,10 +2917,11 @@ class Container(list):
description="This entity has no identifier. It cannot be retrieved."))
if raise_exception_on_error:
- raise EntityError(
- "This entity has no identifier. It cannot be retrieved.", entity)
- else:
- entity.is_valid = lambda: False
+ ee = EntityError(
+ "This entity has no identifier. It cannot be retrieved.",
+ entity)
+ raise TransactionError(ee)
+ entity.is_valid = lambda: False
else:
entities_str.append(str(query))
@@ -2752,12 +2962,12 @@ class Container(list):
"&".join(entities))], query_dict=flags)
return Container._response_to_entities(http_response)
- except URITooLongException as uri_e:
+ except HTTPURITooLongError as uri_e:
try:
# split up
uri1, uri2 = Container._split_uri_string(entities)
- except ValueError:
- raise uri_e
+ except ValueError as val_e:
+ raise uri_e from val_e
c1 = self._retrieve(entities=uri1, flags=flags)
c2 = self._retrieve(entities=uri2, flags=flags)
c1.extend(c2)
@@ -2780,7 +2990,8 @@ class Container(list):
for f in listdir(x):
if isdir(x + '/' + f):
- part = MultipartParam(name=hex(randint(0, sys.maxsize)), value="")
+ part = MultipartParam(
+ name=hex(randint(0, sys.maxsize)), value="")
part.filename = upload + \
('/' + d + '/' if d is not None else '/') + f + '/'
ret.extend(Container._dir_to_http_parts(
@@ -2799,9 +3010,10 @@ class Container(list):
"""Update these entites."""
if len(self) < 1:
- raise TransactionError(
- container=self,
- msg="There are no entities to be updated. This container is empty.")
+ te = TransactionError(
+ msg="There are no entities to be updated. This container is empty.",
+ container=self)
+ raise te
self.clear_server_messages()
insert_xml = etree.Element("Update")
@@ -2818,8 +3030,10 @@ class Container(list):
for entity in self:
if (entity.id is None or entity.id < 0):
- raise TransactionError(
- self, "You tried to update an entity without a valid id.")
+ ee = EntityError(
+ "You tried to update an entity without a valid id.",
+ entity)
+ raise TransactionError(ee)
self._linearize()
@@ -2989,9 +3203,10 @@ class Container(list):
insert_xml.append(entity_xml)
if len(self) > 0 and len(insert_xml) < 1:
- raise TransactionError(
- container=self,
- msg="There are no entities to be inserted. This container contains existent entities only.")
+ te = TransactionError(
+ msg="There are no entities to be inserted. This container contains existent entities only.",
+ container=self)
+ raise te
_log_request("POST: " + _ENTITY_URI_SEGMENT +
('' if flags is None else "?" + str(flags)), insert_xml)
@@ -3094,6 +3309,43 @@ class Container(list):
return self
+ def get_property_values(self, *selectors):
+ """ Return a list of tuples with values of the given selectors.
+
+ I.e. a tabular representation of the container's content.
+
+ If the elements of the selectors parameter are tuples, they will return
+ the properties of the referenced entity, if present. E.g. ("window",
+ "height") will return the value of the height property of the
+ referenced window entity.
+
+ All tuples of the returned list have the same length as the selectors
+ parameter and the ordering of the tuple's values correspond to the
+ order of the parameter as well.
+
+ The tuple contains None for all values that are not available in the
+ entity. That does not necessarily mean, that the values are not stored
+ in the database (e.g. if a single entity was retrieved without
+ referenced entities).
+
+ Parameters
+ ----------
+ *selectors : str or tuple of str
+ Each selector is a list or tuple of property names, e.g. `"height",
+ "width"`.
+
+ Returns
+ -------
+ table : list of tuples
+ A tabular representation of the container's content.
+ """
+ table = []
+
+ for e in self:
+ table.append(e.get_property_values(*selectors))
+
+ return table
+
def sync_global_acl():
c = get_connection()
@@ -3190,6 +3442,22 @@ class ACL():
self.deny(username=username, realm=realm, role=role,
permission=permission, priority=priority)
+ def combine(self, other):
+ """ Combine and return new instance."""
+ result = ACL()
+ result._grants.update(other._grants)
+ result._grants.update(self._grants)
+ result._denials.update(other._denials)
+ result._denials.update(self._denials)
+ result._priority_grants.update(other._priority_grants)
+ result._priority_grants.update(self._priority_grants)
+ result._priority_denials.update(other._priority_denials)
+ result._priority_denials.update(self._priority_denials)
+ return result
+
+ def __eq__(self, other):
+ return isinstance(other, ACL) and other._grants == self._grants and self._denials == other._denials and self._priority_grants == other._priority_grants and self._priority_denials == other._priority_denials
+
def is_empty(self):
return len(self._grants) + len(self._priority_grants) + \
len(self._priority_denials) + len(self._denials) == 0
@@ -3399,6 +3667,23 @@ class ACL():
class Query():
+ """Query
+
+ Attributes
+ ----------
+ q : str
+ The query string.
+ flags : dict of str
+ A dictionary of flags to be send with the query request.
+ messages : _Messages()
+ A container of messages included in the last query response.
+ cached : bool
+ indicates whether the server used the query cache for the execution of
+ this query.
+ results : int or Container
+ The number of results (when this was a count query) or the container
+ with the resulting entities.
+ """
def putFlag(self, key, value=None):
self.flags[key] = value
@@ -3414,28 +3699,63 @@ class Query():
def __init__(self, q):
self.flags = dict()
self.messages = _Messages()
+ self.cached = None
+ self.etag = None
if isinstance(q, etree._Element):
self.q = q.get("string")
self.results = int(q.get("results"))
+ if q.get("cached") is None:
+ self.cached = False
+ else:
+ self.cached = q.get("cached").lower() == "true"
+ self.etag = q.get("etag")
+
for m in q:
if m.tag.lower() == 'warning' or m.tag.lower() == 'error':
self.messages.append(_parse_single_xml_element(m))
else:
self.q = q
- def execute(self, unique=False, raise_exception_on_error=True,
- **kwargs):
+ def execute(self, unique=False, raise_exception_on_error=True, cache=True):
+ """Execute a query (via a server-requests) and return the results.
+
+ Parameters
+ ----------
+
+ unique : bool
+ Whether the query is expected to have only one entity as result.
+ Defaults to False.
+ raise_exception_on_error : bool
+ Whether an exception should be raises when there are errors in the
+ resulting entities. Defaults to True.
+ cache : bool
+ Whether to use the query cache (equivalent to adding a "cache"
+ flag) to the Query object. Defaults to True.
+
+ Returns
+ -------
+ results : Container or integer
+ Returns an integer when it was a `COUNT` query. Otherwise, returns a
+ Container with the resulting entities.
+ """
connection = get_connection()
- query_dict = dict(self.flags)
+
+ flags = self.flags
+ if cache is False:
+ flags["cache"] = "false"
+ query_dict = dict(flags)
query_dict["query"] = str(self.q)
+
_log_request("GET Entity?" + str(query_dict), None)
http_response = connection.retrieve(
entity_uri_segments=["Entity"],
- query_dict=query_dict, **kwargs)
+ query_dict=query_dict)
cresp = Container._response_to_entities(http_response)
self.results = cresp.query.results
+ self.cached = cresp.query.cached
+ self.etag = cresp.query.etag
if self.q.lower().startswith('count') and len(cresp) == 0:
# this was a count query
@@ -3447,10 +3767,12 @@ class Query():
if unique:
if len(cresp) > 1 and raise_exception_on_error:
- raise AmbiguityException("This query wasn't unique")
- elif len(cresp) == 0 and raise_exception_on_error:
- raise EntityDoesNotExistError("No such entity found.")
- elif len(cresp) == 1:
+ raise QueryNotUniqueError(
+ "Query '{}' wasn't unique.".format(self.q))
+ if len(cresp) == 0 and raise_exception_on_error:
+ raise EmptyUniqueQueryError(
+ "Query '{}' found no results.".format(self.q))
+ if len(cresp) == 1:
r = cresp[0]
r.messages.extend(cresp.messages)
@@ -3460,8 +3782,32 @@ class Query():
return cresp
-def execute_query(q, unique=False, raise_exception_on_error=True, flags=None,
- **kwargs):
+def execute_query(q, unique=False, raise_exception_on_error=True, cache=True, flags=None):
+ """Execute a query (via a server-requests) and return the results.
+
+ Parameters
+ ----------
+
+ q : str
+ The query string.
+ unique : bool
+ Whether the query is expected to have only one entity as result.
+ Defaults to False.
+ raise_exception_on_error : bool
+ Whether an exception should be raises when there are errors in the
+ resulting entities. Defaults to True.
+ cache : bool
+ Whether to use the query cache (equivalent to adding a "cache" flag).
+ Defaults to True.
+ flags : dict of str
+ Flags to be added to the request.
+
+ Returns
+ -------
+ results : Container or integer
+ Returns an integer when it was a `COUNT` query. Otherwise, returns a
+ Container with the resulting entities.
+ """
query = Query(q)
if flags is not None:
@@ -3469,7 +3815,7 @@ def execute_query(q, unique=False, raise_exception_on_error=True, flags=None,
return query.execute(unique=unique,
raise_exception_on_error=raise_exception_on_error,
- **kwargs)
+ cache=cache)
class DropOffBox(list):
@@ -3505,6 +3851,14 @@ class DropOffBox(list):
return self
+class UserInfo():
+
+ def __init__(self, xml):
+ self.roles = [role.text for role in xml.findall("Roles/Role")]
+ self.name = xml.get("username")
+ self.realm = xml.get("realm")
+
+
class Info():
def __init__(self):
@@ -3515,15 +3869,19 @@ class Info():
c = get_connection()
try:
http_response = c.retrieve(["Info"])
- except ConnectionException as conn_e:
+ except CaosDBConnectionError as conn_e:
print(conn_e)
+
return
xml = etree.fromstring(http_response.read())
for e in xml:
m = _parse_single_xml_element(e)
- self.messages.append(m)
+ if isinstance(m, UserInfo):
+ self.user_info = m
+ else:
+ self.messages.append(m)
def __str__(self):
if "counts" not in self.messages:
@@ -3624,13 +3982,20 @@ def _parse_single_xml_element(elem):
Entity._from_xml(entity, elem)
return entity
+ elif elem.tag.lower() == "version":
+ return Version.from_xml(elem)
+ elif elem.tag.lower() == "state":
+ return State.from_xml(elem)
elif elem.tag.lower() == "emptystring":
return ""
elif elem.tag.lower() == "value":
if len(elem) == 1 and elem[0].tag.lower() == "emptystring":
return ""
+ elif len(elem) == 1 and elem[0].tag.lower() in classmap:
+ return _parse_single_xml_element(elem[0])
elif elem.text is None or elem.text.strip() == "":
return None
+
return str(elem.text.strip())
elif elem.tag.lower() == "querytemplate":
return QueryTemplate._from_xml(elem)
@@ -3646,129 +4011,126 @@ def _parse_single_xml_element(elem):
return ACL(xml=elem)
elif elem.tag == "Permissions":
return Permissions(xml=elem)
+ elif elem.tag == "UserInfo":
+ return UserInfo(xml=elem)
else:
return Message(type=elem.tag, code=elem.get(
"code"), description=elem.get("description"), body=elem.text)
-def raise_errors(arg0):
- if isinstance(arg0, (Entity, QueryTemplate)):
- entity_error = EntityError(
- entity=arg0, error=Message('Error', 0, 'EntityMultiError'))
- found114 = False
- found116 = False
+def _evaluate_and_add_error(parent_error, ent):
+ """Evaluate the error message(s) attached to entity and add a
+ corresponding exception to parent_error.
- for e in arg0.get_errors():
- try:
- if e.code is not None:
- if int(e.code) == 101: # arg0 does not exist
- raise EntityDoesNotExistError(error=e, entity=arg0)
- elif int(e.code) == 110: # entity has no data type
- raise EntityHasNoDatatypeError(error=e, entity=arg0)
- elif int(e.code) == 403: # Transaction not permitted
- raise AuthorizationException(error=e, entity=arg0)
- elif int(e.code) == 114: # unqualified properties
- found114 = True
- unqualified_properties_error = UnqualifiedPropertiesError(
- error=e, entity=arg0)
-
- for p in arg0.get_properties():
- try:
- raise_errors(p)
- except EntityError as pe:
- unqualified_properties_error.add_error(pe)
- raise unqualified_properties_error
- elif int(e.code) == 116: # unqualified parents
- found116 = True
- unqualified_parents_error = UnqualifiedParentsError(
- error=e, entity=arg0)
-
- for p in arg0.get_parents():
- try:
- raise_errors(p)
- except EntityError as pe:
- unqualified_parents_error.add_error(pe)
- raise unqualified_parents_error
- elif int(e.code) == 152: # name was not unique
- raise UniqueNamesError(error=e, entity=arg0)
- raise EntityError(error=e, entity=arg0)
- except EntityError as ee:
- entity_error.add_error(ee)
+ Parameters:
+ -----------
+ parent_error : TrancactionError
+ Parent error to which the new exception will be attached. This
+ exception will be a direct child.
+ ent : Entity
+ Entity that caused the TransactionError. An exception is
+ created depending on its error message(s).
- if not found114:
- for p in arg0.get_properties():
- try:
- raise_errors(p)
- except EntityError as pe:
- entity_error.add_error(pe)
+ Returns:
+ --------
+ TransactionError :
+ Parent error with new exception(s) attached to it.
+ """
+ if isinstance(ent, (Entity, QueryTemplate)):
+ # Check all error messages
+ found114 = False
+ found116 = False
+ for err in ent.get_errors():
+ # Evaluate specific EntityErrors depending on the error
+ # code
+ if err.code is not None:
+ if int(err.code) == 101: # ent doesn't exist
+ new_exc = EntityDoesNotExistError(entity=ent,
+ error=err)
+ elif int(err.code) == 110: # ent has no data type
+ new_exc = EntityHasNoDatatypeError(entity=ent,
+ error=err)
+ elif int(err.code) == 403: # no permission
+ new_exc = AuthorizationError(entity=ent,
+ error=err)
+ elif int(err.code) == 152: # name wasn't unique
+ new_exc = UniqueNamesError(entity=ent, error=err)
+ elif int(err.code) == 114: # unqualified properties
+ found114 = True
+ new_exc = UnqualifiedPropertiesError(entity=ent,
+ error=err)
+ for prop in ent.get_properties():
+ new_exc = _evaluate_and_add_error(new_exc,
+ prop)
+ elif int(err.code) == 116: # unqualified parents
+ found116 = True
+ new_exc = UnqualifiedParentsError(entity=ent,
+ error=err)
+ for par in ent.get_parents():
+ new_exc = _evaluate_and_add_error(new_exc,
+ par)
+ else: # General EntityError for other codes
+ new_exc = EntityError(entity=ent, error=err)
+ else: # No error code causes a general EntityError, too
+ new_exc = EntityError(entity=ent, error=err)
+ parent_error.add_error(new_exc)
+ # Check for possible errors in parents and properties that
+ # weren't detected up to here
+ if not found114:
+ dummy_err = EntityError(entity=ent)
+ for prop in ent.get_properties():
+ dummy_err = _evaluate_and_add_error(dummy_err, prop)
+ if dummy_err.errors:
+ parent_error.add_error(dummy_err)
if not found116:
- for p in arg0.get_parents():
- try:
- raise_errors(p)
- except EntityError as pe:
- entity_error.add_error(pe)
-
- if len(entity_error.get_errors()) == 1:
- r = entity_error.get_errors().pop()
- raise r
- elif len(entity_error.get_errors()) > 1:
- r = entity_error._convert()
- raise r
- elif isinstance(arg0, Container):
- transaction_error = TransactionError(
- container=arg0, msg="This transaction terminated with Errors.")
- doRaise = False
- found12 = False
-
- if arg0.get_errors() is not None:
- for er in arg0.get_errors():
- if er.code is not None:
- if int(er.code) == 12: # atomicity violation
- found12 = True
- atomic_error = TransactionError(
- container=arg0, error=er, msg=er.description)
-
- for e in arg0:
- try:
- raise_errors(e)
- except EntityError as ee:
- atomic_error.add_error(ee)
-
- if len(atomic_error.get_errors()) > 0:
- transaction_error.add_error(
- atomic_error._convert())
- doRaise = True
- else:
- transaction_error.add_error(atomic_error)
- doRaise = True
- else:
- te = TransactionError(
- container=arg0, error=er, msg=er.description)
- transaction_error.add_error(te)
- doRaise = True
-
- if len(transaction_error.get_errors()) == 1:
- transaction_error = transaction_error.get_errors().pop()
-
- if not found12:
- for e in arg0:
- try:
- raise_errors(e)
- except EntityError as ee:
- transaction_error.add_error(ee)
- doRaise = True
-
- if len(transaction_error.get_errors()) == 1:
- t = transaction_error.get_errors().pop()
- raise t
- elif len(transaction_error.get_errors()) > 1:
- t = transaction_error._convert()
- raise t
- elif doRaise:
- raise transaction_error
+ dummy_err = EntityError(entity=ent)
+ for par in ent.get_parents():
+ dummy_err = _evaluate_and_add_error(dummy_err, par)
+ if dummy_err.errors:
+ parent_error.add_error(dummy_err)
+
+ elif isinstance(ent, Container):
+ parent_error.container = ent
+ if ent.get_errors() is not None:
+ parent_error.code = ent.get_errors()[0].code
+ # In the highly unusual case of more than one error
+ # message, attach all of them.
+ parent_error.msg = '\n'.join(
+ [x.description for x in ent.get_errors()])
+ # Go through all container elements and add them:
+ for elt in ent:
+ parent_error = _evaluate_and_add_error(parent_error, elt)
+
else:
- raise TypeError("Parameter arg0 is to be an Entity or a Container")
+ raise TypeError("Parameter ent is to be an Entity or a Container")
+
+ return parent_error
+
+
+def raise_errors(arg0):
+ """Raise a TransactionError depending on the error code(s) inside
+ Entity, QueryTemplate or Container arg0. More detailed errors may
+ be attached to the TransactionError depending on the contents of
+ arg0.
+
+ Parameters:
+ -----------
+ arg0 : Entity, QueryTemplate, or Container
+ CaosDB object whose messages are evaluated according to their
+ error codes
+
+ """
+ transaction_error = _evaluate_and_add_error(TransactionError(),
+ arg0)
+ # Raise if any error was found
+ if len(transaction_error.all_errors) > 0:
+ raise transaction_error
+ # Cover the special case of an empty container with error
+ # message(s) (e.g. query syntax error)
+ if (transaction_error.container is not None and
+ transaction_error.container.has_errors()):
+ raise transaction_error
def delete(ids, raise_exception_on_error=True):
diff --git a/src/caosdb/common/state.py b/src/caosdb/common/state.py
new file mode 100644
index 0000000000000000000000000000000000000000..cb74022bef57a77c8270b2033c904eecabaadf83
--- /dev/null
+++ b/src/caosdb/common/state.py
@@ -0,0 +1,198 @@
+# ** header v3.0
+# This file is a part of the CaosDB Project.
+#
+# Copyright (C) 2020 IndiScale GmbH <info@indiscale.com>
+# Copyright (C) 2020 Timm Fitschen <t.fitschen@indiscale.com>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see <https://www.gnu.org/licenses/>.
+#
+# ** end header
+
+import copy
+from lxml import etree
+
+
+def _translate_to_state_acis(acis):
+ result = set()
+ for aci in acis:
+ aci = copy.copy(aci)
+ if aci.role:
+ aci.role = "?STATE?" + aci.role + "?"
+ result.add(aci)
+ return result
+
+
+class Transition:
+ """Transition
+
+ Represents allowed transitions from one state to another.
+
+ Properties
+ ----------
+ name : str
+ The name of the transition
+ description: str
+ The description of the transition
+ from_state : str
+ A state name
+ to_state : str
+ A state name
+ """
+
+ def __init__(self, name, from_state, to_state, description=None):
+ self._name = name
+ self._from_state = from_state
+ self._to_state = to_state
+ self._description = description
+
+ @property
+ def name(self):
+ return self._name
+
+ @property
+ def description(self):
+ return self._description
+
+ @property
+ def from_state(self):
+ return self._from_state
+
+ @property
+ def to_state(self):
+ return self._to_state
+
+ def __repr__(self):
+ return f'Transition(name="{self.name}", from_state="{self.from_state}", to_state="{self.to_state}", description="{self.description}")'
+
+ def __eq__(self, other):
+ return (isinstance(other, Transition)
+ and other.name == self.name
+ and other.to_state == self.to_state
+ and other.from_state == self.from_state)
+
+ def __hash__(self):
+ return 23472 + hash(self.name) + hash(self.from_state) + hash(self.to_state)
+
+ @staticmethod
+ def from_xml(xml):
+ to_state = [to.get("name") for to in xml
+ if to.tag.lower() == "tostate"]
+ from_state = [from_.get("name") for from_ in xml
+ if from_.tag.lower() == "fromstate"]
+ result = Transition(name=xml.get("name"),
+ description=xml.get("description"),
+ from_state=from_state[0] if from_state else None,
+ to_state=to_state[0] if to_state else None)
+ return result
+
+
+class State:
+ """State
+
+ Represents the state of an entity and take care of the serialization and
+ deserialization of xml for the entity state.
+
+ An entity state is always a State of a StateModel.
+
+ Properties
+ ----------
+ name : str
+ Name of the State
+ model : str
+ Name of the StateModel
+ description : str
+ Description of the State (read-only)
+ id : str
+ Id of the undelying State record (read-only)
+ transitions : set of Transition
+ All transitions which are available from this state (read-only)
+ """
+
+ def __init__(self, model, name):
+ self.name = name
+ self.model = model
+ self._id = None
+ self._description = None
+ self._transitions = None
+
+ @property
+ def id(self):
+ return self._id
+
+ @property
+ def description(self):
+ return self._description
+
+ @property
+ def transitions(self):
+ return self._transitions
+
+ def __eq__(self, other):
+ return (isinstance(other, State)
+ and self.name == other.name
+ and self.model == other.model)
+
+ def __hash__(self):
+ return hash(self.name) + hash(self.model)
+
+ def __repr__(self):
+ return f"State('{self.model}', '{self.name}')"
+
+ def to_xml(self):
+ """Serialize this State to xml.
+
+ Returns
+ -------
+ xml : etree.Element
+ """
+ xml = etree.Element("State")
+ if self.name is not None:
+ xml.set("name", self.name)
+ if self.model is not None:
+ xml.set("model", self.model)
+ return xml
+
+ @staticmethod
+ def from_xml(xml):
+ """Create a new State instance from an xml Element.
+
+ Parameters
+ ----------
+ xml : etree.Element
+
+ Returns
+ -------
+ state : State
+ """
+ name = xml.get("name")
+ model = xml.get("model")
+ result = State(name=name, model=model)
+ result._id = xml.get("id")
+ result._description = xml.get("description")
+ transitions = [Transition.from_xml(t) for t in xml if t.tag.lower() ==
+ "transition"]
+ if transitions:
+ result._transitions = set(transitions)
+
+ return result
+
+ @staticmethod
+ def create_state_acl(acl):
+ from .models import ACL
+ state_acl = ACL()
+ state_acl._grants = _translate_to_state_acis(acl._grants)
+ state_acl._denials = _translate_to_state_acis(acl._denials)
+ state_acl._priority_grants = _translate_to_state_acis(acl._priority_grants)
+ state_acl._priority_denials = _translate_to_state_acis(acl._priority_denials)
+ return state_acl
diff --git a/src/caosdb/common/versioning.py b/src/caosdb/common/versioning.py
new file mode 100644
index 0000000000000000000000000000000000000000..2875486a13347a2eb834d22580497033699ebd37
--- /dev/null
+++ b/src/caosdb/common/versioning.py
@@ -0,0 +1,250 @@
+# -*- coding: utf-8 -*-
+#
+# ** header v3.0
+# This file is a part of the CaosDB Project.
+#
+# Copyright (C) 2020 Timm Fitschen <t.fitschen@indiscale.com>
+# Copyright (C) 2020 IndiScale GmbH <info@indiscale.com>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see <https://www.gnu.org/licenses/>.
+#
+# ** end header
+#
+""" Versioning module for anything related to entity versions.
+
+Currently this module defines nothing but a single class, `Version`.
+"""
+
+from __future__ import absolute_import
+from caosdb.common.utils import xml2str
+from lxml import etree
+
+
+class Version():
+ """The version of an entity.
+
+ An entity version has a version id (string), a date (UTC timestamp), a
+ list of predecessors and a list of successors.
+
+ Parameters
+ ----------
+ id : str, optional
+ See attribute `id`. Default: None
+ date : str, optional
+ See attribute `date`. Default: None
+ username : str, optional
+ See attribute `username`. Default: None
+ realm : str, optional
+ See attribute `realm`. Default: None
+ predecessors : list of Version, optional
+ See attribute `predecessors`. Default: empty list.
+ successors : list of Version, optional
+ See attribute `successors`. Default: empty list.
+ is_head : bool
+ See attribute `is_head`. Default: False
+ is_complete_history : bool
+ See attribute `is_complete_history`. Default: False
+
+ Attributes
+ ----------
+ id : str
+ Version ID (not the entity's id).
+ date : str
+ UTC Timestamp of the version, i.e. the date and time when the entity of
+ this version has been inserted or modified.
+ username : str
+ The username of the user who inserted or updated this version.
+ realm : str
+ The realm of the user who inserted or updated this version.
+ predecessors : list of Version
+ Predecessors are the older entity versions which have been modified
+ into this version. Usually, there is only one predecessor. However,
+ this API allows for entities to be merged into one entity, which would
+ result in more than one predecessor.
+ successors : list of Version
+ Successors are newer versions of this entity. If there are successors,
+ this version is not the latest version of this entity. Usually, there
+ is only one successor. However, this API allows that a single entity
+ may co-exist in several versions (e.g. several proposals for the next
+ entity status). That would result in more than one successor.
+ is_head : bool or string
+ If true, this indicates that this version is the HEAD if true.
+ Otherwise it is not known whether this is the head or not. Any string
+ matching "true" (case-insensitively) is regarded as True.
+ Nota bene: This property should typically be set if the server response
+ indicated that this is the head version.
+ is_complete_history : bool or string
+ If true, this indicates that this version contains the full version
+ history. That means, that the predecessors and successors have their
+ respective predecessors and successors attached as well and the tree is
+ completely available. Any string matching "true" (case-insensitively)
+ is regarded as True.
+ Nota bene: This property should typically be set if the server response
+ indicated that the full version history is included in its response.
+ """
+
+ # pylint: disable=redefined-builtin
+ def __init__(self, id=None, date=None, username=None, realm=None,
+ predecessors=None, successors=None, is_head=False,
+ is_complete_history=False):
+ """Typically the `predecessors` or `successors` should not "link back" to an existing Version
+object."""
+ self.id = id
+ self.date = date
+ self.username = username
+ self.realm = realm
+ self.predecessors = predecessors if predecessors is not None else []
+ self.successors = successors if successors is not None else []
+ self.is_head = str(is_head).lower() == "true"
+ self.is_complete_history = str(is_complete_history).lower() == "true"
+
+ def get_history(self):
+ """ Returns a flat list of Version instances representing the history
+ of the entity.
+
+ The list items are ordered by the relation between the versions,
+ starting with the oldest version.
+
+ The items in the list have no predecessors or successors attached.
+
+ Note: This method only returns reliable results if
+ `self.is_complete_history is True` and it will not retrieve the full
+ version history if it is not present.
+
+ Returns
+ -------
+ list of Version
+ """
+ versions = []
+ for p in self.predecessors:
+ # assuming that predecessors don't have any successors
+ versions = p.get_history()
+ versions.append(Version(id=self.id, date=self.date,
+ username=self.username, realm=self.realm))
+ for s in self.successors:
+ # assuming that successors don't have any predecessors
+ versions.extend(s.get_history())
+ return versions
+
+ def to_xml(self, tag="Version"):
+ """Serialize this version to xml.
+
+ The tag name is 'Version' per default. But since this method is called
+ recursively for the predecessors and successors as well, the tag name
+ can be configured.
+
+ The resulting xml element contains attributes 'id' and 'date' and
+ 'Predecessor' and 'Successor' child elements.
+
+ Parameters
+ ----------
+ tag : str, optional
+ The name of the returned xml element. Defaults to 'Version'.
+
+ Returns
+ -------
+ xml : etree.Element
+ """
+ xml = etree.Element(tag)
+ if self.id is not None:
+ xml.set("id", self.id)
+ if self.date is not None:
+ xml.set("date", self.date)
+ if self.username is not None:
+ xml.set("username", self.username)
+ if self.realm is not None:
+ xml.set("realm", self.realm)
+ if self.predecessors is not None:
+ for p in self.predecessors:
+ xml.append(p.to_xml(tag="Predecessor"))
+ if self.is_head is True:
+ xml.set("head", "true")
+ if self.successors is not None:
+ for s in self.successors:
+ xml.append(s.to_xml(tag="Successor"))
+ return xml
+
+ def __str__(self):
+ """Return a stringified xml representation."""
+ return self.__repr__()
+
+ def __repr__(self):
+ """Return a stringified xml representation."""
+ return xml2str(self.to_xml())
+
+ @staticmethod
+ def from_xml(xml):
+ """Parse a version object from a 'Version' xml element.
+
+ Parameters
+ ----------
+ xml : etree.Element
+ A 'Version' xml element, with 'id', possibly 'date', `username`,
+ `realm`, and `head` attributes as well as 'Predecessor' and
+ 'Successor' child elements.
+
+ Returns
+ -------
+ version : Version
+ a new version instance
+ """
+ predecessors = [Version.from_xml(p) for p in xml if p.tag.lower() == "predecessor"]
+ successors = [Version.from_xml(s) for s in xml if s.tag.lower() == "successor"]
+ return Version(id=xml.get("id"), date=xml.get("date"),
+ is_head=xml.get("head"),
+ is_complete_history=xml.get("completeHistory"),
+ username=xml.get("username"), realm=xml.get("realm"),
+ predecessors=predecessors, successors=successors)
+
+ def __hash__(self):
+ """Hash of the version instance.
+
+ Also hashes the predecessors and successors.
+ """
+ return (hash(self.id)
+ + hash(self.date)
+ + (Version._hash_list(self.predecessors)
+ if self.predecessors else 26335)
+ + (Version._hash_list(self.successors)
+ if self.successors else -23432))
+
+ @staticmethod
+ def _hash_list(_list):
+ """Hash a list by hashing each element and its index."""
+ result = 12352
+ for idx, val in enumerate(_list):
+ result += hash(val) + idx
+ return result
+
+ @staticmethod
+ def _eq_list(this, that):
+ """List equality.
+
+ List equality is defined as equality of each element, the order
+ and length.
+ """
+ if len(this) != len(that):
+ return False
+ for v1, v2 in zip(this, that):
+ if v1 != v2:
+ return False
+ return True
+
+ def __eq__(self, other):
+ """Equality of versions is defined by equality of id, date, and list
+ equality of the predecessors and successors."""
+ return (self.id == other.id
+ and self.date == other.date
+ and Version._eq_list(self.predecessors, other.predecessors)
+ and Version._eq_list(self.successors, other.successors))
diff --git a/src/caosdb/configuration.py b/src/caosdb/configuration.py
index 4d0797844182b8465ef5f97a869e31ee4fcaf47d..6e8a9c6ff2083b0c30324722003fb3c08a592191 100644
--- a/src/caosdb/configuration.py
+++ b/src/caosdb/configuration.py
@@ -28,6 +28,9 @@ except ImportError:
# python3
from configparser import ConfigParser
+from os import environ, getcwd
+from os.path import expanduser, join, isfile
+
def _reset_config():
global _pycaosdbconf
@@ -49,3 +52,20 @@ def configure(inifile):
def get_config():
return _pycaosdbconf
+
+
+def _read_config_files():
+ """Function to read config files from different paths. Checks for path in $PYCAOSDBINI or home directory (.pycaosdb.ini) and in the current working directory (pycaosdb.ini).
+
+ Returns:
+ [list]: list with successfully parsed ini-files. Order: env_var or home directory, cwd. Used for testing the function.
+ """
+ return_var = []
+ if "PYCAOSDBINI" in environ:
+ return_var.extend(configure(expanduser(environ["PYCAOSDBINI"])))
+ else:
+ return_var.extend(configure(expanduser('~/.pycaosdb.ini')))
+
+ if isfile(join(getcwd(), "pycaosdb.ini")):
+ return_var.extend(configure(join(getcwd(), "pycaosdb.ini")))
+ return return_var
diff --git a/src/caosdb/connection/authentication/auth_token.py b/src/caosdb/connection/authentication/auth_token.py
new file mode 100644
index 0000000000000000000000000000000000000000..688123867f68153d3631bb8559baa235f6f02da5
--- /dev/null
+++ b/src/caosdb/connection/authentication/auth_token.py
@@ -0,0 +1,96 @@
+#! -*- coding: utf-8 -*-
+#
+# ** header v3.0
+# This file is a part of the CaosDB Project.
+#
+# Copyright (C) 2018 Research Group Biomedical Physics,
+# Max-Planck-Institute for Dynamics and Self-Organization Göttingen
+# Copyright (C) 2020 IndiScale GmbH <info@indiscale.com>
+# Copyright (C) 2020 Timm Fitschen <f.fitschen@indiscale.com>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see <https://www.gnu.org/licenses/>.
+#
+# ** end header
+#
+"""auth_token.
+
+An Authentictor which only uses only a pre-supplied authentication token.
+"""
+from __future__ import absolute_import, unicode_literals, print_function
+from .interface import AbstractAuthenticator, CaosDBServerConnection
+from caosdb.connection.utils import auth_token_to_cookie
+from caosdb.exceptions import LoginFailedError
+
+
+def get_authentication_provider():
+ """get_authentication_provider.
+
+ Return an authenticator which only uses a pre-supplied authentication
+ token.
+
+ Returns
+ -------
+ AuthTokenAuthenticator
+ """
+ return AuthTokenAuthenticator()
+
+
+class AuthTokenAuthenticator(AbstractAuthenticator):
+ """AuthTokenAuthenticator.
+
+ Subclass of AbstractAuthenticator which provides authentication only via
+ a given authentication token.
+
+ Methods
+ -------
+ login
+ logout
+ configure
+ """
+
+ def __init__(self):
+ super(AuthTokenAuthenticator, self).__init__()
+ self.auth_token = None
+ self._connection = None
+
+ def login(self):
+ self._login()
+
+ def _login(self):
+ raise LoginFailedError("The authentication token is expired or you "
+ "have been logged out otherwise. The "
+ "auth_token authenticator cannot log in "
+ "again. You must provide a new "
+ "authentication token.")
+
+ def logout(self):
+ self._logout()
+
+ def _logout(self):
+ self.logger.debug("[LOGOUT]")
+ if self.auth_token is not None:
+ headers = {'Cookie': auth_token_to_cookie(self.auth_token)}
+ self._connection.request(method="DELETE", path="logout",
+ headers=headers)
+ self.auth_token = None
+
+ def configure(self, **config):
+ if "auth_token" in config:
+ self.auth_token = config["auth_token"]
+ if "connection" in config:
+ self._connection = config["connection"]
+ if not isinstance(self._connection, CaosDBServerConnection):
+ raise Exception("""Bad configuration of the caosdb connection.
+ The `connection` must be an instance of
+ `CaosDBConnection`.""")
diff --git a/src/caosdb/connection/authentication/interface.py b/src/caosdb/connection/authentication/interface.py
index f156345afcb9d77c118daa7bf53b57b8499d87cc..a364aeb564ee929d995b2f8098bd21e30e9733ab 100644
--- a/src/caosdb/connection/authentication/interface.py
+++ b/src/caosdb/connection/authentication/interface.py
@@ -31,7 +31,7 @@ import logging
from caosdb.connection.utils import urlencode
from caosdb.connection.interface import CaosDBServerConnection
from caosdb.connection.utils import parse_auth_token, auth_token_to_cookie
-from caosdb.exceptions import LoginFailedException
+from caosdb.exceptions import LoginFailedError
# meta class compatible with Python 2 *and* 3:
ABC = ABCMeta('ABC', (object, ), {'__slots__': ()})
@@ -50,6 +50,8 @@ class AbstractAuthenticator(ABC):
logger : Logger
A logger which should be used for all logging which has to do with
authentication.
+ auth_token : str
+ A string representation of a CaosDB Auth Token.
Methods
-------
@@ -59,10 +61,6 @@ class AbstractAuthenticator(ABC):
on_request
on_response
- Attributes
- ----------
- auth_token : str
- A string representation of a CaosDB Auth Token.
"""
def __init__(self):
@@ -187,7 +185,7 @@ class CredentialsAuthenticator(AbstractAuthenticator):
self._logout()
def _logout(self):
- _LOGGER.debug("[LOGOUT]")
+ self.logger.debug("[LOGOUT]")
if self.auth_token is not None:
self._connection.request(method="DELETE", path="logout")
self.auth_token = None
@@ -195,13 +193,13 @@ class CredentialsAuthenticator(AbstractAuthenticator):
def _login(self):
username = self._credentials_provider.username
password = self._credentials_provider.password
- _LOGGER.debug("[LOGIN] %s", username)
+ self.logger.debug("[LOGIN] %s", username)
# we need a username for this:
if username is None:
- raise LoginFailedException("No username was given.")
+ raise LoginFailedError("No username was given.")
if password is None:
- raise LoginFailedException("No password was given")
+ raise LoginFailedError("No password was given")
headers = {}
headers["Content-Type"] = "application/x-www-form-urlencoded"
@@ -212,7 +210,7 @@ class CredentialsAuthenticator(AbstractAuthenticator):
response.read() # clear socket
if response.status != 200:
- raise LoginFailedException("LOGIN WAS NOT SUCCESSFUL")
+ raise LoginFailedError("LOGIN WAS NOT SUCCESSFUL")
self.on_response(response)
return response
diff --git a/src/caosdb/connection/authentication/keyring.py b/src/caosdb/connection/authentication/keyring.py
index 1dc986174acbe23191305632afda91cda0c718d2..d8be7ddf030577545230c9111fdad542b6d6e7e2 100644
--- a/src/caosdb/connection/authentication/keyring.py
+++ b/src/caosdb/connection/authentication/keyring.py
@@ -30,7 +30,7 @@ retrieve the password.
import sys
import imp
from getpass import getpass
-from caosdb.exceptions import ConfigurationException
+from caosdb.exceptions import ConfigurationError
from .external_credentials_provider import ExternalCredentialsProvider
from .interface import CredentialsAuthenticator
@@ -67,10 +67,10 @@ def _get_external_keyring():
def _call_keyring(**config):
if "username" not in config:
- raise ConfigurationException("Your configuration did not provide a "
- "`username` which is needed by the "
- "`KeyringCaller` to retrieve the "
- "password in question.")
+ raise ConfigurationError("Your configuration did not provide a "
+ "`username` which is needed by the "
+ "`KeyringCaller` to retrieve the "
+ "password in question.")
url = config.get("url")
username = config.get("username")
app = "caosdb — {}".format(url)
diff --git a/src/caosdb/connection/authentication/pass.py b/src/caosdb/connection/authentication/pass.py
index 9399fc4f4a76407ad94618785adcfbb945d4c788..853cdf0ed92039e7b5fc9beda8bb76cc0f3cc030 100644
--- a/src/caosdb/connection/authentication/pass.py
+++ b/src/caosdb/connection/authentication/pass.py
@@ -28,7 +28,7 @@ password.
"""
from subprocess import check_output, CalledProcessError
-from caosdb.exceptions import ConfigurationException
+from caosdb.exceptions import ConfigurationError
from .interface import CredentialsAuthenticator
from .external_credentials_provider import ExternalCredentialsProvider
@@ -50,10 +50,10 @@ def get_authentication_provider():
def _call_pass(**config):
if "password_identifier" not in config:
- raise ConfigurationException("Your configuration did not provide a "
- "`password_identifier` which is needed "
- "by the `PassCaller` to retrieve the "
- "password in question.")
+ raise ConfigurationError("Your configuration did not provide a "
+ "`password_identifier` which is needed "
+ "by the `PassCaller` to retrieve the "
+ "password in question.")
try:
return check_output(
diff --git a/src/caosdb/connection/authentication/unauthenticated.py b/src/caosdb/connection/authentication/unauthenticated.py
new file mode 100644
index 0000000000000000000000000000000000000000..65febae8fd8f02f3ee0d339fafb36af512fc7be7
--- /dev/null
+++ b/src/caosdb/connection/authentication/unauthenticated.py
@@ -0,0 +1,119 @@
+#! -*- coding: utf-8 -*-
+#
+# ** header v3.0
+# This file is a part of the CaosDB Project.
+#
+# Copyright (C) 2018 Research Group Biomedical Physics,
+# Max-Planck-Institute for Dynamics and Self-Organization Göttingen
+# Copyright (C) 2020 IndiScale GmbH <info@indiscale.com>
+# Copyright (C) 2020 Timm Fitschen <f.fitschen@indiscale.com>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see <https://www.gnu.org/licenses/>.
+#
+# ** end header
+#
+"""unauthenticated.
+
+An Authenticator which suppresses any authentication and also ignores auth_token
+cookies.
+"""
+from __future__ import absolute_import, unicode_literals, print_function
+from .interface import AbstractAuthenticator, CaosDBServerConnection
+from caosdb.exceptions import LoginFailedError
+
+
+def get_authentication_provider():
+ """get_authentication_provider.
+
+ Return an authenticator which only uses a pre-supplied authentication
+ token.
+
+ Returns
+ -------
+ AuthTokenAuthenticator
+ """
+ return Unauthenticated()
+
+
+class Unauthenticated(AbstractAuthenticator):
+ """Unauthenticated.
+
+ Subclass of AbstractAuthenticator which suppresses any authentication and
+ also ignores auth_token cookies.
+
+ Methods
+ -------
+ login
+ logout
+ configure
+ on_request
+ on_response
+ """
+
+ def __init__(self):
+ super(Unauthenticated, self).__init__()
+ self.auth_token = None
+ self._connection = None
+
+ def login(self):
+ self._login()
+
+ def _login(self):
+ raise LoginFailedError("This caosdb client is configured to stay "
+ "unauthenticated. Change your "
+ "`password_method` and provide an "
+ "`auth_token` or credentials if you want "
+ "to authenticate this client.")
+
+ def logout(self):
+ self._logout()
+
+ def _logout(self):
+ self.auth_token = None
+
+ def configure(self, **config):
+ self.auth_token = None
+
+ def on_request(self, method, path, headers, **kwargs):
+ # pylint: disable=unused-argument
+ """on_request.
+
+ This implementation does not attempt to login or authenticate in any
+ form.
+
+ Parameters
+ ----------
+ method
+ unused
+ path
+ unused
+ headers
+ unused
+ **kwargs
+ unused
+ """
+ pass
+
+ def on_response(self, response):
+ # pylint: disable=unused-argument
+ """on_response.
+
+ This implementation ignores any auth_token cookie sent by the server.
+
+ Parameters
+ ----------
+ response
+ unused
+ """
+ pass
diff --git a/src/caosdb/connection/connection.py b/src/caosdb/connection/connection.py
index 5fbca32ed9a8d8f5028b340821c84a2b65ef03a4..2cb31f425b14bf382fae32f62af79a4bd6ef0301 100644
--- a/src/caosdb/connection/connection.py
+++ b/src/caosdb/connection/connection.py
@@ -33,11 +33,14 @@ from errno import EPIPE as BrokenPipe
from socket import error as SocketError
from caosdb.configuration import get_config
-from caosdb.exceptions import (AuthorizationException, CaosDBException,
- ClientErrorException, ConfigurationException,
- ConnectionException, EntityDoesNotExistError,
- LoginFailedException, ServerErrorException,
- URITooLongException)
+from caosdb.exceptions import (CaosDBException, HTTPClientError,
+ ConfigurationError,
+ CaosDBConnectionError,
+ HTTPForbiddenError,
+ LoginFailedError,
+ HTTPResourceNotFoundError,
+ HTTPServerError,
+ HTTPURITooLongError)
from caosdb.version import version
from pkg_resources import resource_filename
@@ -61,6 +64,10 @@ class _WrappedHTTPResponse(CaosDBHTTPResponse):
def __init__(self, response):
self.response = response
+ @property
+ def reason(self):
+ return self.response.reason
+
@property
def status(self):
return self.response.status
@@ -88,7 +95,8 @@ class _DefaultCaosDBServerConnection(CaosDBServerConnection):
"""
def __init__(self):
- self._useragent = ("caosdb-pylib/{version} - {implementation}".format(version=version, implementation=type(self).__name__))
+ self._useragent = ("caosdb-pylib/{version} - {implementation}".format(
+ version=version, implementation=type(self).__name__))
self._http_con = None
self._base_path = None
@@ -132,7 +140,7 @@ class _DefaultCaosDBServerConnection(CaosDBServerConnection):
self._http_con.request(method=method, url=self._base_path + path,
headers=headers, body=body)
except SocketError as socket_err:
- raise ConnectionException(
+ raise CaosDBConnectionError(
"Connection failed. Network or server down? " + str(socket_err)
)
@@ -156,7 +164,7 @@ class _DefaultCaosDBServerConnection(CaosDBServerConnection):
Raises
------
- ConnectionException
+ CaosDBConnectionError
If no url has been specified, or if the CA certificate cannot be
loaded.
"""
@@ -193,9 +201,9 @@ class _DefaultCaosDBServerConnection(CaosDBServerConnection):
try:
context.load_verify_locations(config["cacert"])
except Exception as exc:
- raise ConnectionException("Could not load the cacert in"
- "`{}`: {}".format(config["cacert"],
- exc))
+ raise CaosDBConnectionError("Could not load the cacert in"
+ "`{}`: {}".format(config["cacert"],
+ exc))
context.load_default_certs()
@@ -204,7 +212,7 @@ class _DefaultCaosDBServerConnection(CaosDBServerConnection):
host = parsed_url.netloc
self._base_path = parsed_url.path
else:
- raise ConnectionException(
+ raise CaosDBConnectionError(
"No connection url specified. Please "
"do so via caosdb.configure_connection(...) or in a config "
"file.")
@@ -246,7 +254,7 @@ def _make_conf(*conf):
_DEFAULT_CONF = {
- "password_method": "plain",
+ "password_method": "input",
"implementation": _DefaultCaosDBServerConnection,
"timeout": 210,
"cacert": resource_filename("caosdb", 'cert/indiscale.ca.crt')
@@ -262,8 +270,8 @@ def _get_authenticator(**config):
----------
password_method : str
The simple name of a submodule of caosdb.connection.authentication.
- Currently, there are three valid values for this parameter: 'plain',
- 'pass', and 'keyring'.
+ Currently, there are four valid values for this parameter: 'plain',
+ 'pass', 'keyring' and 'auth_token'.
**config :
Any other keyword arguments are passed the configre method of the
password_method.
@@ -276,7 +284,7 @@ def _get_authenticator(**config):
Raises
------
- ConnectionException
+ ConfigurationError
If the password_method string cannot be resolved to a CaosAuthenticator
class.
"""
@@ -292,9 +300,10 @@ def _get_authenticator(**config):
return auth_provider
except ImportError:
- raise ConfigurationException("Password method \"{}\" not implemented. "
- "Valid methods: plain, pass, or keyring."
- .format(config["password_method"]))
+ raise ConfigurationError("Password method \"{}\" not implemented. "
+ "Try `plain`, `pass`, `keyring`, or "
+ "`auth_token`."
+ .format(config["password_method"]))
def configure_connection(**kwargs):
@@ -325,6 +334,7 @@ def configure_connection(**kwargs):
- "input" Asks for the password.
- "pass" Uses the `pass` password manager.
- "keyring" Uses the `keyring` library.
+ - "auth_token" Uses only a given auth_token.
timeout : int
A connection timeout in seconds. (Default: 210)
@@ -333,6 +343,10 @@ def configure_connection(**kwargs):
Whether SSL certificate warnings should be ignored. Only use this for
development purposes! (Default: False)
+ auth_token : str (optional)
+ An authentication token which has been issued by the CaosDB Server.
+ Implies `password_method="auth_token"` if set. An example token string would be `["O","OneTimeAuthenticationToken","anonymous",["administration"],[],1592995200000,604800000,"3ZZ4WKRB-5I7DG2Q6-ZZE6T64P-VQ","197d0d081615c52dc18fb323c300d7be077beaad4020773bb58920b55023fa6ee49355e35754a4277b9ac525c882bcd3a22e7227ba36dfcbbdbf8f15f19d1ee9",1,30000]`.
+
implementation : CaosDBServerConnection
The class which implements the connection. (Default:
_DefaultCaosDBServerConnection)
@@ -392,29 +406,24 @@ def _handle_response_status(http_response):
# emtpy response buffer
body = http_response.read()
+ if status == 404:
+ raise HTTPResourceNotFoundError("This resource has not been found.")
+ elif status > 499:
+ raise HTTPServerError(body=body)
+
+ reason = http_response.reason
+ standard_message = ("Request failed. The response returned with status "
+ "{} - {}.".format(status, reason))
if status == 401:
- raise LoginFailedException(
- "Request failed. The response returned with status "
- "{}.".format(status))
+ raise LoginFailedError(standard_message)
elif status == 403:
- raise AuthorizationException(
- "Request failed. The response returned with status "
- "{}.".format(status))
- elif status == 404:
- raise EntityDoesNotExistError("This entity does not exist.")
+ raise HTTPForbiddenError(standard_message)
elif status in (413, 414):
- raise URITooLongException(
- "Request failed. The response returned with status "
- "{}.".format(status))
+ raise HTTPURITooLongError(standard_message)
elif 399 < status < 500:
- raise ClientErrorException(msg=("Request failed. The response returned "
- "with status {}.").format(status), status=status, body=body)
- elif status > 499:
- raise ServerErrorException(body=body)
+ raise HTTPClientError(msg=standard_message, status=status, body=body)
else:
- raise CaosDBException(
- "Request failed. The response returned with status "
- "{}.".format(status))
+ raise CaosDBException(standard_message)
class _Connection(object): # pylint: disable=useless-object-inheritance
@@ -448,7 +457,7 @@ class _Connection(object): # pylint: disable=useless-object-inheritance
self.is_configured = True
if "implementation" not in config:
- raise ConfigurationException(
+ raise ConfigurationError(
"Missing CaosDBServerConnection implementation. You did not "
"specify an `implementation` for the connection.")
try:
@@ -459,23 +468,23 @@ class _Connection(object): # pylint: disable=useless-object-inheritance
raise TypeError("The `implementation` callable did not return "
"an instance of CaosDBServerConnection.")
except TypeError as type_err:
- raise ConfigurationException(
+ raise ConfigurationError(
"Bad CaosDBServerConnection implementation. The "
"implementation must be a callable object which returns an "
"instance of `CaosDBServerConnection` (e.g. a constructor "
- "or a factory).", type_err)
+ "or a factory).\n{}".format(type_err.args[0]))
self._delegate_connection.configure(**config)
+ if "auth_token" in config:
+ # deprecated, needed for older scripts
+ config["password_method"] = "auth_token"
if "password_method" not in config:
- raise ConfigurationException("Missing password_method. You did "
- "not specify a `password_method` for"
- "the connection.")
+ raise ConfigurationError("Missing password_method. You did "
+ "not specify a `password_method` for"
+ "the connection.")
self._authenticator = _get_authenticator(
connection=self._delegate_connection, **config)
- if "auth_token" in config:
- self._authenticator.auth_token = config["auth_token"]
-
return self
def retrieve(self, entity_uri_segments=None, query_dict=None, **kwargs):
@@ -548,8 +557,8 @@ class _Connection(object): # pylint: disable=useless-object-inheritance
uri_segments.extend(path.split("/"))
return self.retrieve(entity_uri_segments=uri_segments)
- except EntityDoesNotExistError:
- raise EntityDoesNotExistError("This file does not exist.")
+ except HTTPResourceNotFoundError:
+ raise HTTPResourceNotFoundError("This file does not exist.")
def _login(self):
self._authenticator.login()
@@ -570,7 +579,7 @@ class _Connection(object): # pylint: disable=useless-object-inheritance
headers=headers, body=body,
reconnect=False,
**kwargs)
- except LoginFailedException:
+ except LoginFailedError:
if kwargs.get("reconnect", True) is True:
self._login()
@@ -598,7 +607,15 @@ class _Connection(object): # pylint: disable=useless-object-inheritance
body=body, **kwargs)
_LOGGER.debug("response: %s %s", str(http_response.status),
str(http_response.getheaders()))
- _handle_response_status(http_response)
self._authenticator.on_response(http_response)
+ _handle_response_status(http_response)
return http_response
+
+ def get_username(self):
+ """
+ Return the username of the current connection.
+
+ Shortcut for: get_connection()._authenticator._credentials_provider.username
+ """
+ return self._authenticator._credentials_provider.username
diff --git a/src/caosdb/exceptions.py b/src/caosdb/exceptions.py
index 5763bef4ac805a7dd1dfde8da2f6f3b7e5fd4bd8..f02a4630356726f99d8439fd821b6dd327ab22c7 100644
--- a/src/caosdb/exceptions.py
+++ b/src/caosdb/exceptions.py
@@ -5,6 +5,8 @@
#
# Copyright (C) 2018 Research Group Biomedical Physics,
# Max-Planck-Institute for Dynamics and Self-Organization Göttingen
+# Copyright (C) 2020 Indiscale GmbH <info@indiscale.com>
+# Copyright (C) 2020 Florian Spreckelsen <f.spreckelsen@indiscale.com>
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as
@@ -21,6 +23,10 @@
#
# ** end header
#
+"""The exceptions module defines exceptions for HTTP Errors (4xx and 5xx and
+HTTP response codes) and for transaction errors (i.e. missing permissions,
+dependencies, non-passing consistency checks etc.).
+"""
from lxml import etree
@@ -28,13 +34,13 @@ from lxml import etree
class CaosDBException(Exception):
"""Base class of all CaosDB exceptions."""
- def __init__(self, msg=None, *args):
- Exception.__init__(self, msg, *args)
+ def __init__(self, msg):
+ Exception.__init__(self, msg)
self.msg = msg
-class ConfigurationException(CaosDBException):
- """ConfigurationException.
+class ConfigurationError(CaosDBException):
+ """ConfigurationError.
Indicates a misconfiguration.
@@ -43,7 +49,6 @@ class ConfigurationException(CaosDBException):
msg : str
A descriptin of the misconfiguration. The constructor adds
a few lines with explainingg where to find the configuration.
- *args
Attributes
----------
@@ -51,23 +56,32 @@ class ConfigurationException(CaosDBException):
A description of the misconfiguration.
"""
- def __init__(self, msg, *args):
- super(ConfigurationException, self).__init__(msg +
- ConfigurationException._INFO,
- *args)
+ def __init__(self, msg):
+ super().__init__(msg + ConfigurationError._INFO)
_INFO = ("\n\nPlease check your ~/.pycaosdb.ini and your $PWD/"
- ".pycaosdb.ini. Do at least one of them exist and are they correct?")
+ ".pycaosdb.ini. Does at least one of them exist and are they correct?")
-class ClientErrorException(CaosDBException):
+class ServerConfigurationException(CaosDBException):
+ """The server is configured in a different way than expected.
+
+ This can be for example unexpected flags or settings or missing extensions.
+ """
+
+
+class HTTPClientError(CaosDBException):
+ """HTTPClientError represents 4xx HTTP client errors."""
+
def __init__(self, msg, status, body):
self.status = status
self.body = body
CaosDBException.__init__(self, msg)
-class ServerErrorException(CaosDBException):
+class HTTPServerError(CaosDBException):
+ """HTTPServerError represents 5xx HTTP server errors."""
+
def __init__(self, body):
xml = etree.fromstring(body)
error = xml.xpath('/Response/Error')[0]
@@ -78,107 +92,126 @@ class ServerErrorException(CaosDBException):
CaosDBException.__init__(self, msg)
-class ConnectionException(CaosDBException):
+class CaosDBConnectionError(CaosDBException):
"""Connection is not configured or the network is down."""
def __init__(self, msg=None):
CaosDBException.__init__(self, msg)
-class URITooLongException(CaosDBException):
+class HTTPURITooLongError(HTTPClientError):
"""The URI of the last request was too long."""
def __init__(self, msg=None):
- CaosDBException.__init__(self, msg)
+ HTTPClientError.__init__(self, msg=msg, status=414, body=None)
+
+class LoginFailedError(CaosDBException):
+ """Login failed.
-class AmbiguityException(CaosDBException):
- """A retrieval of an entity that was supposed to be uniquely identifiable
- returned two or more entities."""
+ Probably, your username/password pair is wrong.
+ """
def __init__(self, msg=None):
- CaosDBException.__init__(self, msg)
+ CaosDBException.__init__(self, msg=msg)
-class LoginFailedException(CaosDBException):
- """Login failed.
+class HTTPForbiddenError(HTTPClientError):
+ """You're lacking the required permissions. Corresponds to HTTP status
+ 403.
- Probably, your username/password pair is wrong.
"""
def __init__(self, msg=None):
- CaosDBException.__init__(self, msg=msg)
+ HTTPClientError.__init__(self, msg=msg, status=403, body=None)
-class TransactionError(CaosDBException):
+class HTTPResourceNotFoundError(HTTPClientError):
+ """The requested resource doesn't exist; corresponds to HTTP status
+ 404.
- def _calc_bases(self):
- types = dict()
- # collect each class once
+ """
- for err in self.errors:
- types[id(type(err))] = type(err)
- # delete redundant super classes
+ def __init__(self, msg=None):
+ HTTPClientError.__init__(self, msg=msg, status=404, body=None)
- if len(types.values()) > 1:
- # remove TransactionError
- try:
- del types[id(TransactionError)]
- except KeyError:
- pass
- if len(types.values()) > 1:
- # remove EntityError
- try:
- del types[id(EntityError)]
- except KeyError:
- pass
+class MismatchingEntitiesError(CaosDBException):
+ """Mismatching entities were found during container sync."""
- ret = ()
- for t in types.values():
- ret += (t,)
+# ######################### Bad query errors ###########################
- if ret == ():
- ret = (type(self),)
- return ret
+class BadQueryError(CaosDBException):
+ """Base class for query errors that are not transaction errors."""
- def __init__(self, container=None, error=None, msg=None):
- self.container = container
- self.errors = []
- self.msg = msg if msg is not None else str(error)
- self.error = error
- def print_errs(self):
- print(self)
+class QueryNotUniqueError(BadQueryError):
+ """A unique query or retrieve found more than one entity."""
- for err in self.errors:
- err.print_errs()
- def _convert(self):
- t = self._calc_bases()
- try:
- newtype = type('TransactionError', t, {})
- except BaseException:
- self.print_errs()
- raise
- newinstance = newtype(container=self.container, error=self.msg)
- newinstance.errors = self.errors
- newinstance.get_entities = self.get_entities
+class EmptyUniqueQueryError(BadQueryError):
+ """A unique query or retrieve dound no result."""
- return newinstance
- def get_container(self):
- '''
- @return: The container that raised this TransactionError during the last
- transaction.
- '''
+# ######################### Transaction errors #########################
- return self.container
+
+class TransactionError(CaosDBException):
+ """An error of this type is raised whenever any transaction fails with
+ one or more entities between client and CaosDB server. More
+ detailed errors are collected as direct and indirect children in
+ the 'errors' list (direct children) and the 'all_errors' set (set
+ of all direct and indirect children).
+
+ """
+
+ def __init__(self, error=None,
+ msg="An error occured during the transaction.",
+ container=None):
+ CaosDBException.__init__(self, msg=msg)
+ self.errors = []
+ self.all_errors = set()
+ self.entities = []
+ self.all_entities = set()
+ self.container = container
+ # special case of faulty container
+ if container is not None and container.get_errors() is not None:
+ self.code = container.get_errors()[0].code
+ else:
+ self.code = None
+ if error is not None:
+ self.add_error(error)
+
+ def has_error(self, error_t, direct_children_only=False):
+ """Check whether this transaction error contains an error of type
+ error_t. If direct_children_only is True, only direct children
+ are checked.
+
+ Parameters:
+ -----------
+ error_t : EntityError
+ error type to be checked
+ direct_children_only: bool, optional
+ If True, only direct children, i.e., all errors in
+ self.errors are checked. Else all direct and indirect
+ children, i.e., all errors in self.all_errors are
+ used. Default is false.
+
+ Returns:
+ --------
+ has_error : bool
+ True if at least one of the children is of type error_t,
+ False otherwise.
+
+ """
+
+ test_set = self.errors if direct_children_only else self.all_errors
+ return any([isinstance(err, error_t) for err in test_set])
def add_error(self, error):
- """Add an error to this TransactionError.
+ """Add an error as a direct child to this TransactionError.
@param error: An EntityError or a list of EntityErrors.
@@ -189,35 +222,31 @@ class TransactionError(CaosDBException):
"""
if hasattr(error, "__iter__"):
- for e in error:
- self.add_error(e)
+ for err in error:
+ self.add_error(err)
return self
- elif isinstance(error, TransactionError):
+ elif isinstance(error, EntityError):
self.errors.append(error)
+ self.entities.append(error.entity)
+
+ self.all_errors.add(error)
+ self.all_errors.update(error.all_errors)
+ self.all_entities.add(error.entity)
+ self.all_entities.update(error.all_entities)
return self
else:
raise TypeError(
- "Argument is to be an TransactionError or a list of TransactionErrors.")
-
- def get_errors(self):
- '''
- @return: A list of all EntityError objects.
- '''
-
- if hasattr(self, 'errors'):
- return self.errors
-
- return None
+ "Argument is to be an EntityError or a list of EntityErrors.")
def _repr_reasons(self, indent):
- if self.get_errors() is not None and len(self.get_errors()) > 0:
+ if self.errors is not None and len(self.errors) > 0:
ret = "\n" + indent + " +--| REASONS |--"
- for c in self.get_errors():
+ for err in self.errors:
ret += '\n' + indent + ' | -> ' + \
- c.__str__(indent=indent + ' |')
+ err.__str__(indent=indent + ' |')
ret += "\n" + indent + " +----------------"
return ret
@@ -225,8 +254,11 @@ class TransactionError(CaosDBException):
return ''
def _repr_head(self, indent):
- return str(type(self).__name__) + ((': ' + self.msg)
- if hasattr(self, 'msg') and self.msg is not None else '')
+ return indent + str(type(self).__name__) + (
+ (': ' + self.msg)
+ if hasattr(self, 'msg') and self.msg is not None
+ else ''
+ )
def __str__(self, indent=''):
ret = self._repr_head(indent=indent)
@@ -237,58 +269,21 @@ class TransactionError(CaosDBException):
def __repr__(self):
return self.__str__()
- def get_entities(self):
- '''
- @return: A list of all Entity objects with errors.
- '''
- ret = []
-
- if hasattr(self, 'get_entity') and self.get_entity() is not None:
- ret.append(self.get_entity())
-
- for error in self.errors:
- if hasattr(error, 'get_entity'):
- if error.get_entity() not in ret:
- ret.append(error.get_entity())
-# if hasattr(error, 'get_entities'):
-# for e in error.get_entities():
-# if e not in ret:
-# ret.append(e)
- return ret
-
- def get_error(self):
- return self.error
-
class EntityError(TransactionError):
+ """This is the most basic entity error. It is constructed using an
+ entity that caused the error and the error message attached by the
+ server.
- @staticmethod
- def _sort_t(t):
- if len(t) > 1:
- ret = ()
- '''remove EntityError'''
-
- for i in range(len(t)):
- if t[i] != EntityError:
- ret += (t[i],)
- t = ret
-
- return t
-
- def _convert(self):
- t = self._calc_bases()
- # TODO is it really a good idea to create dynamically types here?
- newtype = type('EntityMultiError', t+(Exception,), {})
- newinstance = newtype(error=self.error, entity=self.entity)
- setattr(newinstance, 'msg', self.msg)
- setattr(newinstance, 'errors', self.errors)
- setattr(newinstance, 'container', self.container)
-
- return newinstance
+ """
- def __init__(self, error=None, container=None, entity=None):
- TransactionError.__init__(self, container=container)
+ def __init__(self, error=None, entity=None):
+ TransactionError.__init__(self)
self.error = error
+ if hasattr(error, "code"):
+ self.code = error.code
+ else:
+ self.code = None
self.entity = entity
if error is not None and hasattr(error, "encode"):
@@ -300,34 +295,17 @@ class EntityError(TransactionError):
else:
self.msg = str(error)
- def get_entity(self):
- '''
- @return: The entity that caused this error.
- '''
-
- if hasattr(self, 'entity'):
- return self.entity
-
- return None
-
@property
def description(self):
+ """The description of the error."""
return self.error.description if self.error is not None else None
- def get_code(self):
- return self.error.code if self.error is not None else None
-
- def get_error(self):
- '''
- @return: Error Message object of this Error.
- '''
-
- return self.error
-
def _repr_head(self, indent):
if hasattr(self, 'entity') and self.entity is not None:
- return str(type(self.entity).__name__).upper() + " (" + str(self.entity.id) + (("," + "'" + str(self.entity.name) + "'")
- if self.entity.name is not None else '') + ") CAUSED " + TransactionError._repr_head(self, indent)
+ return (str(type(self.entity).__name__).upper() + " (id: " +
+ str(self.entity.id) + ((", name: " + "'" + str(self.entity.name) + "'") if
+ self.entity.name is not None else '') + ") CAUSED " +
+ TransactionError._repr_head(self, indent))
else:
return TransactionError._repr_head(self, indent)
@@ -337,15 +315,19 @@ class UniqueNamesError(EntityError):
class UnqualifiedParentsError(EntityError):
- """This entity has unqualified parents (call 'get_errors()' for a list of
- errors of the parent entities or 'get_entities()' for a list of parent
- entities with errors)."""
+ """This entity has unqualified parents (see 'errors' attribute for a
+ list of errors of the parent entities or 'entities' attribute for
+ a list of parent entities with errors).
+
+ """
class UnqualifiedPropertiesError(EntityError):
- """This entity has unqualified properties (call 'get_errors()' for a list
- of errors of the properties or 'get_entities()' for a list of properties
- with errors)."""
+ """This entity has unqualified properties (see 'errors' attribute for
+ a list of errors of the properties or 'entities' attribute for a
+ list of properties with errors).
+
+ """
class EntityDoesNotExistError(EntityError):
@@ -357,11 +339,17 @@ class EntityHasNoDatatypeError(EntityError):
class ConsistencyError(EntityError):
- pass
+ """The transaction violates database consistency."""
-class AuthorizationException(EntityError):
+class AuthorizationError(EntityError):
"""You are not allowed to do what ever you tried to do.
- Maybe you need more privileges or a user account at all.
+ Maybe you need more privileges or a user account.
+ """
+
+
+class AmbiguousEntityError(EntityError):
+ """A retrieval of the entity was not possible because there is more
+ than one possible candidate.
"""
diff --git a/src/caosdb/utils/caosdb_admin.py b/src/caosdb/utils/caosdb_admin.py
index b97bedf484208bf382d71ec58274ca2066a05240..7fd4a141a86642ff43cac859830c96317356d63d 100755
--- a/src/caosdb/utils/caosdb_admin.py
+++ b/src/caosdb/utils/caosdb_admin.py
@@ -33,7 +33,7 @@ from argparse import ArgumentParser, RawDescriptionHelpFormatter
import caosdb as db
from caosdb import administration as admin
-from caosdb.exceptions import ClientErrorException
+from caosdb.exceptions import HTTPClientError
__all__ = []
__version__ = 0.3
@@ -73,7 +73,7 @@ def do_retrieve(args):
c.append(db.Entity(id=eid))
except ValueError:
c.append(db.Entity(name=i))
- c.retrieve()
+ c.retrieve(flags=eval(args.flags))
print(c)
@@ -123,7 +123,7 @@ def do_create_user(args):
try:
admin._insert_user(name=args.user_name,
email=args.user_email, password=password)
- except ClientErrorException as e:
+ except HTTPClientError as e:
print(e.msg)
@@ -284,6 +284,15 @@ USAGE
formatter_class=RawDescriptionHelpFormatter)
parser.add_argument('-V', '--version', action='version',
version=program_version_message)
+ parser.add_argument("--auth-token", metavar="AUTH_TOKEN",
+ dest="auth_token",
+ help=("A CaosDB authentication token (default: None). "
+ "If the authentication token is passed, the "
+ "`password_method` of the connection is set to "
+ "`auth_token` and the respective configuration "
+ "from the pycaosdb.ini is effectively being "
+ "overridden.\nTODO: Also allow passing the token "
+ "via environmenty variables."))
subparsers = parser.add_subparsers(
title="commands",
metavar="COMMAND",
@@ -603,8 +612,12 @@ USAGE
# Process arguments
args = parser.parse_args()
-
- db.configure_connection()._login()
+ auth_token = args.auth_token
+ if auth_token is not None:
+ db.configure_connection(password_method="auth_token",
+ auth_token=auth_token)
+ else:
+ db.configure_connection()
return args.call(args)
diff --git a/src/caosdb/utils/server_side_scripting.py b/src/caosdb/utils/server_side_scripting.py
new file mode 100644
index 0000000000000000000000000000000000000000..663178dcbda4293cb30dff88efbfb7b7302df70d
--- /dev/null
+++ b/src/caosdb/utils/server_side_scripting.py
@@ -0,0 +1,146 @@
+# -*- coding: utf-8 -*-
+#
+# ** header v3.0
+# This file is a part of the CaosDB Project.
+#
+# Copyright (C) 2020 Timm Fitschen <t.fitschen@indiscale.com>
+# Copyright (C) 2020 IndiScale GmbH <info@indiscale.com>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see <https://www.gnu.org/licenses/>.
+#
+# ** end header
+#
+"""server_side_scripting
+
+Helper functions for calling server-side scripts.
+"""
+from urllib.parse import quote
+from lxml import etree
+
+from caosdb.connection.connection import get_connection
+from caosdb.connection.utils import urlencode
+from caosdb.connection.encode import MultipartParam, multipart_encode
+
+
+def _make_params(pos_args, opts):
+ """Create and return option string components.
+
+The return value is a dict with be something like `-O<key>`:`<value>` from `opts` and
+`-p{0,1,2,3,...}`:`<value>` from `pos_args`.
+
+ """
+ result = {}
+ for key, val in opts.items():
+ result["-O{key}".format(key=key)] = str(val)
+ for i, val in enumerate(pos_args):
+ result["-p{i}".format(i=i)] = str(val)
+ return result
+
+
+def _make_multipart_request(call, pos_args, opts, files):
+ """Return body and header for an HTTP request.
+ """
+ parts = list()
+ params = _make_params(pos_args, opts)
+
+ parts.append(MultipartParam("call", call))
+ for key, val in params.items():
+ parts.append(MultipartParam(key, val))
+
+ for paramname, filename in files.items():
+ parts.append(MultipartParam.from_file(paramname=paramname,
+ filename=filename))
+
+ body, headers = multipart_encode(parts)
+ return body, headers
+
+
+def _make_form_request(call, pos_args, opts):
+ """Return URL from call and argumewnts, and headers for urlencoding."""
+ form = dict()
+ form["call"] = call
+
+ params = _make_params(pos_args, opts)
+ for key, val in params.items():
+ form[key] = val
+
+ headers = {}
+ headers["Content-Type"] = "application/x-www-form-urlencoded"
+ return urlencode(form), headers
+
+
+def _make_request(call, pos_args, opts, files=None):
+ """
+ Multipart if with files, otherwise url-encoded.
+
+ Return
+ ------
+ path_segments, body, headers
+ """
+
+ if files is not None:
+ return _make_multipart_request(call, pos_args, opts, files)
+
+ return _make_form_request(call, pos_args, opts)
+
+
+def run_server_side_script(call, *args, files=None, **kwargs):
+ """
+
+ Return
+ ------
+ response : ScriptingResponse
+ """
+ body, headers = _make_request(call=call, pos_args=args,
+ opts=kwargs, files=files)
+ response = get_connection()._http_request(method="POST",
+ path=quote("scripting"),
+ body=body,
+ headers=headers)
+ xml = etree.parse(response)
+ code = int(xml.xpath("/Response/script/@code")[0])
+ call = xml.xpath("/Response/script/call")[0].text
+ stdout = xml.xpath("/Response/script/stdout")[0].text
+ stderr = xml.xpath("/Response/script/stderr")[0].text
+
+ return ScriptingResponse(call=call,
+ code=code,
+ stdout=stdout,
+ stderr=stderr)
+
+
+class ScriptingResponse():
+ """ScriptingResponse
+
+ A data class for the response of server-side scripting calls.
+
+ Properties
+ ----------
+ code : int
+ The return code of the script process.
+ call : str
+ The complete call of the script minus the absolute path and the
+ auth_token.
+ stdout : str
+ The STDOUT of the script process.
+ stderr : str
+ The STDERR of the script process.
+
+ """
+
+ def __init__(self, call, code, stdout, stderr):
+ self.call = call
+ self.code = code
+ self.stdout = stdout
+ self.stderr = stderr
diff --git a/src/doc/Makefile b/src/doc/Makefile
new file mode 100644
index 0000000000000000000000000000000000000000..8b533626f6665fa2a3491ea5d319b58740c05299
--- /dev/null
+++ b/src/doc/Makefile
@@ -0,0 +1,48 @@
+# ** header v3.0
+# This file is a part of the CaosDB Project.
+#
+# Copyright (C) 2020 IndiScale GmbH <info@indiscale.com>
+# Copyright (C) 2020 Daniel Hornung <d.hornung@indiscale.com>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see <https://www.gnu.org/licenses/>.
+#
+# ** end header
+
+# This Makefile is a wrapper for sphinx scripts.
+#
+# It is based upon the autocreated makefile for Sphinx documentation.
+
+# You can set these variables from the command line, and also
+# from the environment for the first two.
+SPHINXOPTS ?= -a
+SPHINXBUILD ?= sphinx-build
+SPHINXAPIDOC ?= sphinx-apidoc
+PY_BASEDIR = ../caosdb
+SOURCEDIR = .
+BUILDDIR = ../../build/doc
+
+
+.PHONY: doc-help Makefile
+
+# Put it first so that "make" without argument is like "make help".
+doc-help:
+ @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
+
+# Catch-all target: route all unknown targets to Sphinx using the new
+# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS).
+%: Makefile apidoc
+ @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
+
+apidoc:
+ @$(SPHINXAPIDOC) -o _apidoc $(PY_BASEDIR) --separate
diff --git a/src/doc/README_SETUP.md b/src/doc/README_SETUP.md
new file mode 120000
index 0000000000000000000000000000000000000000..88332e357f5e06f3de522768ccdcd9e513c15f62
--- /dev/null
+++ b/src/doc/README_SETUP.md
@@ -0,0 +1 @@
+../../README_SETUP.md
\ No newline at end of file
diff --git a/src/doc/administration.rst b/src/doc/administration.rst
new file mode 100644
index 0000000000000000000000000000000000000000..061acc8364d2ef62f743a20d7b9e6562baac0fc5
--- /dev/null
+++ b/src/doc/administration.rst
@@ -0,0 +1,14 @@
+Administration
+==============
+
+The Python script ``caosdb_admin.py`` should be used for administrative tasks.
+Call ``caosdb_admin.py --help`` to see how to use it.
+
+The most common task is to create a new user (in the CaosDB realm) and set a
+password for the user (note that a user typically needs to be activated)::
+
+ caosdb_admin.py create_user anna
+ caosdb_admin.py set_user_password anna
+ caosdb_admin.py add_user_roles anna administration
+ caosdb_admin.py activate_user anna
+
diff --git a/src/doc/concepts.rst b/src/doc/concepts.rst
new file mode 100644
index 0000000000000000000000000000000000000000..29625a0a105dacdea2183eac743d1904a7743ec7
--- /dev/null
+++ b/src/doc/concepts.rst
@@ -0,0 +1,6 @@
+========================
+The concepts of PyCaosDB
+========================
+
+- `Configuration <configuration>`
+
diff --git a/src/doc/conf.py b/src/doc/conf.py
new file mode 100644
index 0000000000000000000000000000000000000000..1865af123ab1abb2e44e9ae714a229b1383794f0
--- /dev/null
+++ b/src/doc/conf.py
@@ -0,0 +1,202 @@
+# -*- coding: utf-8 -*-
+#
+# Configuration file for the Sphinx documentation builder.
+#
+# This file only contains a selection of the most common options. For a full
+# list see the documentation:
+# https://www.sphinx-doc.org/en/master/usage/configuration.html
+
+# -- Path setup --------------------------------------------------------------
+
+# If extensions (or modules to document with autodoc) are in another directory,
+# add these directories to sys.path here. If the directory is relative to the
+# documentation root, use os.path.abspath to make it absolute, like shown here.
+#
+# import os
+# import sys
+# sys.path.insert(0, os.path.abspath('../caosdb'))
+
+
+import sphinx_rtd_theme
+
+
+# -- Project information -----------------------------------------------------
+
+project = 'pycaosdb'
+copyright = '2020, IndiScale GmbH'
+author = 'Daniel Hornung'
+
+# The short X.Y version
+version = '0.5.2'
+# The full version, including alpha/beta/rc tags
+#release = '0.5.2-rc2'
+release = '0.5.2'
+
+
+# -- General configuration ---------------------------------------------------
+
+# If your documentation needs a minimal Sphinx version, state it here.
+#
+# needs_sphinx = '1.0'
+
+# Add any Sphinx extension module names here, as strings. They can be
+# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
+# ones.
+extensions = [
+ 'sphinx.ext.autodoc',
+ 'sphinx.ext.autosectionlabel',
+ 'sphinx.ext.intersphinx',
+ 'sphinx.ext.napoleon', # For Google style docstrings
+ "recommonmark", # For markdown files.
+ "sphinx_rtd_theme",
+]
+
+# Add any paths that contain templates here, relative to this directory.
+templates_path = ['_templates']
+
+# The suffix(es) of source filenames.
+# You can specify multiple suffix as a list of string:
+source_suffix = ['.rst', '.md']
+
+# The master toctree document.
+master_doc = 'index'
+
+# The language for content autogenerated by Sphinx. Refer to documentation
+# for a list of supported languages.
+#
+# This is also used if you do content translation via gettext catalogs.
+# Usually you set "language" from the command line for these cases.
+language = None
+
+# List of patterns, relative to source directory, that match files and
+# directories to ignore when looking for source files.
+# This pattern also affects html_static_path and html_extra_path.
+exclude_patterns = []
+
+# The name of the Pygments (syntax highlighting) style to use.
+pygments_style = None
+
+
+# -- Options for HTML output -------------------------------------------------
+
+# The theme to use for HTML and HTML Help pages. See the documentation for
+# a list of builtin themes.
+#
+
+html_theme = "sphinx_rtd_theme"
+
+# Theme options are theme-specific and customize the look and feel of a theme
+# further. For a list of options available for each theme, see the
+# documentation.
+#
+# html_theme_options = {}
+
+# Add any paths that contain custom static files (such as style sheets) here,
+# relative to this directory. They are copied after the builtin static files,
+# so a file named "default.css" will overwrite the builtin "default.css".
+html_static_path = ['_static']
+
+# Custom sidebar templates, must be a dictionary that maps document names
+# to template names.
+#
+# The default sidebars (for documents that don't match any pattern) are
+# defined by theme itself. Builtin themes are using these templates by
+# default: ``['localtoc.html', 'relations.html', 'sourcelink.html',
+# 'searchbox.html']``.
+#
+# html_sidebars = {}
+
+
+# -- Options for HTMLHelp output ---------------------------------------------
+
+# Output file base name for HTML help builder.
+htmlhelp_basename = 'caosdb-pylibdoc'
+
+
+# -- Options for LaTeX output ------------------------------------------------
+
+latex_elements = {
+ # The paper size ('letterpaper' or 'a4paper').
+ #
+ # 'papersize': 'letterpaper',
+
+ # The font size ('10pt', '11pt' or '12pt').
+ #
+ # 'pointsize': '10pt',
+
+ # Additional stuff for the LaTeX preamble.
+ #
+ # 'preamble': '',
+
+ # Latex figure (float) alignment
+ #
+ # 'figure_align': 'htbp',
+}
+
+# Grouping the document tree into LaTeX files. List of tuples
+# (source start file, target name, title,
+# author, documentclass [howto, manual, or own class]).
+latex_documents = [
+ (master_doc, 'caosdb-pylib.tex', 'caosdb-pylib Documentation',
+ 'IndiScale GmbH', 'manual'),
+]
+
+
+# -- Options for manual page output ------------------------------------------
+
+# One entry per manual page. List of tuples
+# (source start file, name, description, authors, manual section).
+man_pages = [
+ (master_doc, 'pycaosdb', 'pycaosdb documentation',
+ [author], 1)
+]
+
+
+# -- Options for Texinfo output ----------------------------------------------
+
+# Grouping the document tree into Texinfo files. List of tuples
+# (source start file, target name, title, author,
+# dir menu entry, description, category)
+texinfo_documents = [
+ (master_doc, 'pycaosdb', 'pycaosdb documentation',
+ author, 'pycaosdb', 'One line description of project.',
+ 'Miscellaneous'),
+]
+
+
+# -- Options for Epub output -------------------------------------------------
+
+# Bibliographic Dublin Core info.
+epub_title = project
+
+# The unique identifier of the text. This can be a ISBN number
+# or the project homepage.
+#
+# epub_identifier = ''
+
+# A unique identification for the text.
+#
+# epub_uid = ''
+
+# A list of files that should not be packed into the epub file.
+epub_exclude_files = ['search.html']
+
+
+# -- Extension configuration -------------------------------------------------
+
+# -- Options for intersphinx -------------------------------------------------
+
+# https://www.sphinx-doc.org/en/master/usage/extensions/intersphinx.html#confval-intersphinx_mapping
+intersphinx_mapping = {
+ "python": ("https://docs.python.org/", None),
+ "caosdb-mysqlbackend": ("https://caosdb.gitlab.io/caosdb-mysqlbackend/",
+ None),
+ "caosdb-server": ("https://caosdb.gitlab.io/caosdb-server/", None),
+}
+
+
+# TODO Which options do we want?
+autodoc_default_options = {
+ 'members': None,
+ 'undoc-members': None,
+}
diff --git a/src/doc/configuration.md b/src/doc/configuration.md
new file mode 100644
index 0000000000000000000000000000000000000000..6e53542f661dcae94622fef24a67cecf7491df9c
--- /dev/null
+++ b/src/doc/configuration.md
@@ -0,0 +1,54 @@
+# Configuration of PyCaosDB #
+The behavior of PyCaosDB is defined via a configuration that is provided using configuration files.
+PyCaosDB tries to read from the inifile specified in the environment variable `PYCAOSDBINI` or
+alternatively in `~/.pycaosdb.ini` upon import. After that, the ini file `pycaosdb.ini` in the
+current working directory will be read additionally, if it exists.
+
+## Authentication ##
+
+The default configuration (that your are asked for your password when ever a connection is created
+can be changed by setting `password_method`:
+
+* with `password_method=input` password (and possibly user) will be queried on demand (**default**)
+* use the password manager [pass](https://www.passwordstore.org) by using `pass` as value, see also the [ArchWiki
+ entry](https://wiki.archlinux.org/index.php/Pass#Basic_usage). This also requires `password_identifier` which refers to the identifier within pass
+ for the desired password.
+* install the python package [keyring](https://pypi.org/project/keyring), to use the system keyring/wallet (macOS, GNOME, KDE,
+ Windows). The password will be queried on first usage.
+* with `password_method=plain` (**strongly discouraged**)
+
+```ini
+[Connection]
+username=YOUR_USERNAME
+
+# password using "pass" password manager
+#password_method=pass
+#password_identifier=...
+
+# using the system keyring/wallet (macOS, GNOME, KDE, Windows)
+#password_method=keyring
+
+#discouraged: password in plain text
+#password_method=plain
+#password=YOUR_PASSWORD
+```
+
+## SSL Certificate ##
+
+You can set the pass to the ssl certificate to be used:
+
+```ini
+[Connection]
+cacert=/path/to/caosdb.ca.pem
+```
+
+## Further Settings ##
+
+`debug=0` ensures that debug information is **not** printed to the terminal every time you interact
+with CaosDB which makes the experience much less verbose. Set it to 1 or 2 in case you want to help
+debugging (which I hope will not be necessary for this tutorial) or if you want to learn more about
+the internals of the protocol.
+
+A complete list of options can be found in the
+[pycaosdb.ini file](https://gitlab.com/caosdb/caosdb-pylib/-/blob/main/examples/pycaosdb.ini) in
+the examples folder of the source code.
diff --git a/src/doc/index.rst b/src/doc/index.rst
new file mode 100644
index 0000000000000000000000000000000000000000..bd29c6c56acf5c173e94ae6471a6aeba56ea4b93
--- /dev/null
+++ b/src/doc/index.rst
@@ -0,0 +1,28 @@
+
+Welcome to PyCaosDB's documentation!
+====================================
+
+.. toctree::
+ :maxdepth: 2
+ :caption: Contents:
+ :hidden:
+
+ Getting started <README_SETUP>
+ tutorials/index
+ Concepts <concepts>
+ Configuration <configuration>
+ Administration <administration>
+ API documentation<_apidoc/caosdb>
+
+This is the documentation for the Python client library for CaosDB, ``PyCaosDB``.
+
+This documentation helps you to :doc:`get started<README_SETUP>`, explains the most important
+:doc:`concepts<concepts>` and offers a range of :doc:`tutorials<tutorials/index>`.
+
+
+Indices and tables
+==================
+
+* :ref:`genindex`
+* :ref:`modindex`
+* :ref:`search`
diff --git a/src/doc/tutorials/Data-Insertion.rst b/src/doc/tutorials/Data-Insertion.rst
new file mode 100644
index 0000000000000000000000000000000000000000..2ead1cc259c807ee12f2459f7e452558eb1b63a2
--- /dev/null
+++ b/src/doc/tutorials/Data-Insertion.rst
@@ -0,0 +1,163 @@
+Data Insertion
+==============
+
+Data Models
+~~~~~~~~~~~
+
+Data is stored and structured in CaosDB using a concept of RecordTypes,
+Properties, Records etc. If you do not know what these are, please look
+at the chapter :any:`caosdb-server:Data Model` .
+
+In order to insert some actual data, we need to create a data model
+using RecordTypes and Properties (You may skip this if you use a CaosDB
+instance that already has the required types). So, let’s create a simple
+Property called “a” of datatype double. This is very easy in pylib:
+
+.. code:: python
+
+ a = db.Property(name="a", datatype=db.DOUBLE)
+
+There are a few basic datatypes: db.INTEGER, db.TEXT. See `data
+type <Specification/Datatype>`__ for a full list.
+
+We can create our own small data model for e.g. a simulation by adding
+two more Properties and a RecordType:
+
+.. code:: python
+
+ b = db.Property(name="b", datatype=db.DOUBLE)
+ epsilon = db.Property(name="epsilon", datatype=db.DOUBLE)
+ recordtype = db.RecordType(name="BarkleySimulation")
+ recordtype.add_property(a)
+ recordtype.add_property(b)
+ recordtype.add_property(epsilon)
+ container = db.Container()
+ container.extend([a, b, epsilon, recordtype])
+ container.insert()
+
+Insert Actual Data
+~~~~~~~~~~~~~~~~~~
+
+Suppose the RecordType “Experiment” and the Property “date” exist in the
+database. You can then create single data Records by using the
+corresponding python class:
+
+.. code:: python
+
+ rec = db.Record()
+ rec.add_parent(name="Experiment")
+ rec.add_property(name="date", value="2020-01-07")
+ rec.insert()
+
+Here, the record has a parent: The RecordType “Experiment”. And a
+Property: date.
+
+Note, that if you want to use a property that is not a primitive
+datatype like db.INTEGER and so on, you need to use the ID of the Entity
+that you are referencing.
+
+.. code:: python
+
+ rec = db.Record()
+ rec.add_parent(name="Experiment")
+ rec.add_property(name="report", value=235507)
+ rec.add_property(name="Analysis", value=230007)
+ rec.insert()
+
+Of course, the IDs 235507 and 230007 need to exist in CaosDB. The first
+example shows how to use a db.REFERENCE Property (report) and the second
+shows that you can use any RecordType as Property to reference a Record
+that has such a parent.
+
+Most Records do not have name however it can absolutely make sense. In
+that case use the name argument when creating it. Another useful feature
+is the fact that properties can have units:
+
+.. code:: python
+
+ rec = db.Record("DeviceNo-AB110")
+ rec.add_parent(name="SlicingMachine")
+ rec.add_property(name="weight", value="1749", unit="kg")
+ rec.insert()
+
+If you are in some kind of analysis you can do this in batch mode with a
+container. E.g. if you have a python list ``analysis_results``:
+
+.. code:: python
+
+ cont = db.Container()
+ for date, result in analysis_results:
+ rec = db.Record()
+ rec.add_parent(name="Experiment")
+ rec.add_property(name="date", value=date)
+ rec.add_property(name="result", value=result)
+ cont.append(rec)
+
+ cont.insert()
+
+Useful is also, that you can insert directly tabular data.
+
+.. code:: python
+
+ from caosadvancedtools.table_converter import from_tsv
+
+ recs = from_tsv("test.csv", "Experiment")
+ print(recs)
+ recs.insert()
+
+With this example file
+`test.csv <uploads/4f2c8756a26a3984c0af09d206d583e5/test.csv>`__.
+
+Inheritance of Properties
+-------------------------
+
+Given, you want to insert a new RecordType “Fridge temperatur
+experiment” as a child of the existing RecordType “Experiment”. The
+latter may have an obligatory Property “date” (since every experiment is
+conducted at some time). It is a natural way of thinking, that every sub
+type of “Experiment” also has this obligatory Property—in terms of
+object oriented programing the “Fridge temperatur experiment” *inherits*
+that Property.
+
+::
+
+ rt = h.RecordType(name="Fridge temperatur experiment",
+ description="RecordType which inherits all obligatory properties from Experiment"
+ ).add_parent(name="Experiment", inheritance="obligatory").insert()
+
+ print(rt.get_property(name="date").importance) ### rt now has a "date"-property -> this line prints "obligatory"
+
+The parameter *``inheritance=(obligatory|recommended|fix|all|none)``* of
+``add_parent`` tells the server to assign obligatory:: properties of the
+parent to the child automatically, recommended:: properties of the
+parent to the child automatically, fix:: properties of the parent to the
+child automatically, all:: properties of the parent to the child
+automatically, none:: of the properties of the parent to child
+automatically,
+
+File Update
+-----------
+
+Updating an existing file by uploading a new version.
+
+1. Retrieve the file record of interest, e.g. by ID:
+
+.. code:: python
+
+ import caosdb as db
+
+ file_upd = db.File(id=174).retrieve()
+
+2. Set the new local file path. The remote file path is stored in the
+ file object as ``file_upd.path`` while the local path can be found in
+ ``file_upd.file``.
+
+.. code:: python
+
+ file_upd.file = "./supplements.pdf"
+
+3. Update the file:
+
+.. code:: python
+
+ file_upd.update()
diff --git a/src/doc/tutorials/basic_analysis.rst b/src/doc/tutorials/basic_analysis.rst
new file mode 100644
index 0000000000000000000000000000000000000000..cc185e0ee08f9e5ee0f890c0ab55f52972882d17
--- /dev/null
+++ b/src/doc/tutorials/basic_analysis.rst
@@ -0,0 +1,47 @@
+
+Basic Analysis
+==============
+
+If you have not yet, configure a connection with the demo instance. E.g.:
+
+>>> import caosdb as db
+>>> _ = db.configure_connection(
+... url="https://demo.indiscale.com/",
+... password_method="plain",
+... username="admin",
+... password="caosdb")
+
+A basic Analysis of data in CaosDB could start like:
+
+>>>
+>>> analyses = db.execute_query("FIND RECORD Analysis with quality_factor")
+>>> qfs = [el.get_property("quality_factor").value for el in analyses]
+
+This first retrieves all analysis records that have a ``quality_factor`` and
+then creates a Python list that contains the values. You could create a
+histogram of those for example by (**warning** this is a very boring histogram)::
+
+ import matplotlib
+ import matplotlib.pyplot as plt
+ plt.hist(qfs)
+ plt.xlabel("quality factors")
+ plt.ylabel("count")
+ plt.show()
+
+
+
+Often we are interested in table like data for our processing. And the disentangling of the property values as above is a bit annoying. Thus there is a convenience function for that.
+
+>>> from caosadvancedtools.table_converter import to_table
+>>> # Let us retrieve the data in a table like form using `SELECT`
+>>> data = db.execute_query("SELECT quality_factor FROM RECORD Analysis with quality_factor" )
+>>> table = to_table(data)
+>>> print(table)
+ quality_factor
+0 ...
+
+Summary
+-------
+
+Now you know, how you can collect query results in lists or tables that can then
+be used for further processing.
diff --git a/src/doc/tutorials/data-model-interface.md b/src/doc/tutorials/data-model-interface.md
new file mode 100644
index 0000000000000000000000000000000000000000..f6967c57a0a3de6e7c6fd3d2b64d3f59620526de
--- /dev/null
+++ b/src/doc/tutorials/data-model-interface.md
@@ -0,0 +1,36 @@
+# Data Models
+
+
+
+You also want to change the datamodel? Also call
+```bash
+pip3 install --user --no-deps .
+```
+in
+```bash
+CaosDB/data_models
+```
+
+Change to the appropriate directory
+```bash
+cd CaosDB/data_models
+```
+There are "data models" defined in
+```bash
+caosdb_models
+```
+having an ending like "_model.py"
+A set of data models is also considered to be a model
+You can create an UML representation of a model or a set of models by calling
+```bash
+./model_interface.py -u model_name [model_name2]
+```
+If you have troubles look at
+```bash
+./model_interface.py -h
+```
+You can change existing models (but be careful! I hope you know what you are doing) or add new ones by changing the appropriate files or adding a new XXXX_model.py
+Once you are done, you can sync your changes with the server
+```bash
+./model_interface.py -s model_name [model_name2]
+```
diff --git a/src/doc/tutorials/errors.rst b/src/doc/tutorials/errors.rst
new file mode 100644
index 0000000000000000000000000000000000000000..37c53c9b527a0435f9f24ae6c6e71687e73eb963
--- /dev/null
+++ b/src/doc/tutorials/errors.rst
@@ -0,0 +1,176 @@
+
+Error Handling
+==============
+
+In case of erroneous transactions, connection problems and a lot of
+other cases, PyCaosDB may raise specific errors in order to pinpoint
+the problem as precisely as possible. Some of these errors a
+representations of errors in the CaosDB server, others stem from
+problems that occurred on the client side.
+
+The errors and exceptions are ordered hierarchically form the most
+general exceptions to specific transaction or connection problems. The
+most important error types and the hierarchy will be explained in the
+following. For more information on specific error types, see also the
+:doc:`source code<../_apidoc/caosdb.exceptions>`.
+
+.. note::
+
+ Starting from PyCaosDB 0.5, the error handling has changed
+ significantly. New error classes have been introduced and the
+ behavior of ``TransactionError`` and ``EntityError`` has been
+ re-worked. In the following, only the "new" errors are
+ discussed. Please refer to the documentation of PyCaosDB 0.4.1 and
+ earlier for the old error handling.
+
+CaosDBException
+----------------
+
+``CaosDBException`` is the most generic exception and all other error classes inherit
+from this one. Because of its generality, it doesn't tell you much
+except that some component of PyCaosDB raised an exception. If you
+want to catch all possible CaosDB errors, this is the class to use.
+
+TransactionError
+----------------
+
+Every transaction (calling ``insert``, ``update``, ``retrieve``, or
+``delete`` on a container or an entity) may finish with errors. They
+indicate, for instance, that an entity does not exist or that you need
+to specify a data type for your property and much more. If and only if
+one or more errors occur during a transaction a ``TransactionError``
+will be raised by the transaction method. The ``TransactionError``
+class is a container for all errors which occur during a
+transaction. It usually contains one or more :ref:`entity
+errors<EntityError>` which you can inspect in order to learn why the
+transaction failed. For this inspection, there are some helpful
+attributes and methods provided by the ``TransactionError``:
+
+* ``entities``: a list of all entities that directly caused at least one error
+ in this transaction.
+
+* ``errors``: a list of all ``EntityError`` objects that directly caused the
+ transaction to fail.
+
+* ``all_entities``, ``all_errors``: sets of all entities and errors
+ that, directly or indirectly, caused either this ``TransactionError`` or any of the
+ ``EntityError`` objects it contains.
+
+* ``has_error(error_t)``: Check whether an error of type ``error_t``
+ occurred during the transaction.
+
+Additionally, ``print(transaction_error)`` prints a tree-like
+representation of all errors regarding the transaction in question.
+
+EntityError
+-----------
+
+An ``EntityError`` specifies the entity and the error proper that
+caused a transaction to fail. It is never raised on its own but is
+contained in a ``TransactionError`` (which may or may not contain
+other ``EntityError`` objects) which is then raised. ``EntityError``
+has several :ref:`subclasses<Special Errors>` that further specify the
+error that occurred.
+
+The ``EntityError`` class is in fact a subclass of
+``TransactionError``. Thus, it has the same methods and attributes as
+the ``TransactionError`` explained
+:ref:`above<TransactionError>`. This is important in case of an
+``EntityError`` that was caused by other faulty entities (e.g., broken
+parents or properties). In that case these problematic entities and
+errors can again be inspected by visiting the ``entities`` and
+``errors`` lists as above.
+
+Special Errors
+~~~~~~~~~~~~~~
+
+Subclasses of ``EntityError`` for special purposes:
+
+* ``EntityDoesNotExistError``
+
+* ``EntityHasNoDataTypeError``
+
+* ``UniqueNamesError``
+
+* ``UnqualifiedParentsError``
+
+* ``UnqualifiedPropertiesError``
+
+* ``ConsistencyError``
+
+* ``AuthorizationError``
+
+* ``AmbiguousEntityError``
+
+BadQueryError
+-------------
+
+A ``BadQueryError`` is raised when a query could not be processed by
+the server. In contrast to a ``TransactionError`` it is not
+necessarily caused by problematic entities or
+containers. ``BadQueryError`` has the two important subclasses
+``EmptyUniqueQueryError`` and ``QueryNotUniqueError`` for queries with
+``unique=True`` which found no or ambiguous entities, respectively.
+
+HTTP Errors
+-----------
+
+An ``HTTPClientError`` or an ``HTTPServerError`` is raised in case of
+http(s) connection problems caused by the Python client or the CaosDB
+server, respectively. There are the following subclasses of
+``HTTPClientError`` that are used to specify the connection problem:
+
+* ``HTTPURITooLongError``: The URI of the request was too long to be
+ processed by the server.
+
+* ``HTTPForbiddenError``: You're not allowed to access this resource.
+
+* ``HTTPResourceNotFoundError``: The requested resource doesn't exist.
+
+Other Errors
+------------
+
+There are further subclasses of ``CaosDBException`` that are raised in
+case of faulty configurations or other problems. They should be rather
+self-explanatory from their names; see the :doc:`source code<../_apidoc/caosdb.exceptions>`
+for further information.
+
+* ``ConfigurationError``
+
+* ``LoginFailedError``
+
+* ``MismatchingEntitiesError``
+
+* ``ServerConfigurationException``
+
+Examples
+--------
+
+.. code-block:: python3
+
+ import caosdb as db
+
+ def link_and_insert(entity, linked, link=True):
+ """Link the ENTITY to LINKED and insert it."""
+ if link:
+ entity.add_property(db.Property(name="link", value=linked))
+ try:
+ entity.insert()
+ except db.TransactionError as tre:
+ # Unique names problem may be worked around by using another name
+ if tre.has_error(db.UniqueNamesError):
+ for ent_error in tre.errors:
+ if (isinstance(ent_error, db.UniqueNamesError)
+ and entity in ent_error.entities):
+ entity.name = entity.name + "_new" # Try again with new name.
+ link_and_insert(entity, linked, link=False)
+ break
+ # Unqualified properties will be handled by the caller
+ elif tre.has_error(db.UnqualifiedPropertiesError):
+ for ent_error in tre.errors:
+ if (isinstance(ent_error, db.UnqualifiedPropertiesError_
+ and entity in ent_error.entities):
+ raise RuntimeError("One of the properties was unqualified: " + str(ent_error))
+ # Other problems are not covered by this tutorial
+ else:
+ raise NotImplementedError("Unhandled TransactionError: " + str(tre))
diff --git a/src/doc/tutorials/first_steps.rst b/src/doc/tutorials/first_steps.rst
new file mode 100644
index 0000000000000000000000000000000000000000..34b96bbeca416107fb34feb4707b9ef46fc49fe7
--- /dev/null
+++ b/src/doc/tutorials/first_steps.rst
@@ -0,0 +1,130 @@
+First Steps
+===========
+
+You should have a working connection to a CaosDB instance now. If not, please check out the
+:doc:`Getting Started secton</README_SETUP>`.
+
+If you are not yet familiar with Records, RecordTypes and Properties used in CaosDB,
+please check out the respective part in the `Web Interface tutorial`_.
+You should also know the basics of the CaosDB Query Language (a tutorial is
+`here <https://docs.indiscale.com/caosdb-webui/tutorials/query.html>`_).
+
+We recommend that you connect to the `demo instance`_ (hosted by `Indiscale`_) in order to try out
+the following examples. You can do this with
+
+>>> import caosdb as db
+>>> _ = db.configure_connection(
+... url="https://demo.indiscale.com/",
+... password_method="plain",
+... username="admin",
+... password="caosdb")
+
+or by using corresponding settings in the configuration file
+(see :doc:`Getting Started secton</README_SETUP>`.).
+However, you can also translate the examples to the data model that you have at hand.
+
+Let's start with a simple query.
+
+>>> response = db.execute_query("FIND RECORD Guitar")
+
+Queries work the same way as in the web interface. You simply provide the
+query string to the corresponding function (``db.execute_query``). However, the result is not
+displayed as beautifully as in the web interface (Try ``print(response)``). That is why browsing through
+data is the strength of the web interface while the automated processing of
+data is the strength of the Python client.
+
+>>> type(response)
+<class 'caosdb.common.models.Container'>
+
+As you can see the type of the returned object is Container. Containers are
+simply lists of LinkAhead objects with useful functions to interact with LinkAhead.
+Thus we can easily find out how many Records where returned:
+
+>>> len(response)
+3
+
+Let's look at the first element:
+
+>>> firstguitar = response[0]
+>>> print(type(firstguitar))
+<class 'caosdb.common.models.Record'>
+>>> print(firstguitar)
+<Record ...
+
+.. The above example needs doctest ELLIPSIS
+You see that the object is a Record. It has a Parent and two Properties.
+
+.. note::
+
+ Many useful functions and classes are directly available top level in the module::
+
+ db.Container()
+ db.Record()
+
+Accessing Properties
+--------------------
+
+Often it is necessary to access the value of a property.
+
+>>> # get the property object
+>>> print(firstguitar.get_property("price"))
+<Property id="100" name="price" datatype="DOUBLE" unit="€">48.0</Property>
+<BLANKLINE>
+>>> # the value of it
+>>> print(firstguitar.get_property("price").value)
+48.0
+>>> # What is this?
+>>> print(firstguitar.get_property(100))
+<Property id="100" name="price" datatype="DOUBLE" unit="€">48.0</Property>
+<BLANKLINE>
+
+
+Why did the second version work? In the web interface we do not realize it that easily, but there is only one thing that uniquely identifies Entities in LinkAhead: the id.
+
+In the xml output you see, that the properties have the ids 100 and 106. Often names of entities are also unique, but this is not guaranteed. Thus in many cases it is preferable or even necessary to use the id for identifying LinkAhead Entities.
+
+Ids can also come in handy when searching. Suppose you have some complicated condition for the object that you want
+
+
+>>> # This condition is not that complicated and long but let's suppose it was.
+>>> record = db.execute_query("FIND Analysis with quality_factor=0.08", unique=True)
+>>> # You can use unique=True when you only expect one result Entity. An error will be
+>>> # thrown if the number of results is unequal to 1 and the resulting object will be
+>>> # an Entity and not a Container
+>>> print(type(record))
+<class 'caosdb.common.models.Record'>
+>>> print(record.id)
+123
+
+Now we can continue using the id of the first query. This for example allows to formulate a second query with a condition involving this object without including the potentially long and complicated subquery in this one:
+
+>>> query = "FIND Guitar WHICH IS REFERENCED BY {id}".format(id=record.id)
+>>> guitar = db.execute_query(query, unique=True)
+>>> print(guitar)
+<Record id="115" ...
+
+Files
+-----
+
+You can download files (if the LinkAhead server has access to them)
+
+>>> file = db.execute_query("FIND FILE *2019-023" , unique=True)
+>>> target_path = el = file.download()
+
+The file will be saved under target_path.
+If the files are large data files, it is often a better idea to only retrieve the path of the file and access them via a local mount.
+
+
+
+Summary
+-------
+
+Now you know, how you can use Python to send queries to CaosDB and you can access
+the result Records and their properties.
+
+The next tutorial shows how to make some meaningful use of this.
+
+
+.. _`demo instance`: https://demo.indiscale.com
+.. _`IndiScale`: https://indiscale.com
+.. _`Web Interface tutorial`: https://docs.indiscale.com/caosdb-webui/tutorials/first_steps.html
diff --git a/src/doc/tutorials/index.rst b/src/doc/tutorials/index.rst
new file mode 100644
index 0000000000000000000000000000000000000000..3889edb8f47e973cc7ae25c9134d75cfeab95f65
--- /dev/null
+++ b/src/doc/tutorials/index.rst
@@ -0,0 +1,18 @@
+
+PyCaosDB Tutorials
+==================
+
+This chapter contains tutorials that lead you from the first steps to
+advanced usage of the Python client.
+
+.. toctree::
+ :maxdepth: 2
+ :caption: Contents:
+ :hidden:
+
+ first_steps
+ basic_analysis
+ Data-Insertion
+ errors
+ data-model-interface
+
diff --git a/tox.ini b/tox.ini
index 97f7753720f8b8647476654518e0c65e1ec7906e..94c2dc8affb280d3e7f6cff4536636432c9f7749 100644
--- a/tox.ini
+++ b/tox.ini
@@ -1,5 +1,5 @@
[tox]
-envlist=py34, py35, py36, py37, py38
+envlist=py36, py37, py38, py39
skip_missing_interpreters = true
[testenv]
diff --git a/unittests/docker/Dockerfile b/unittests/docker/Dockerfile
index da45618fa6f60fdcc20042dcaf3df30f4e2d166c..7fa3f75bd198724628dee48ab328829fa071a639 100644
--- a/unittests/docker/Dockerfile
+++ b/unittests/docker/Dockerfile
@@ -1,7 +1,10 @@
FROM debian:latest
RUN apt-get update && \
- apt-get install pylint3 python3-pip tox git \
- curl pycodestyle -y
+ apt-get install -y \
+ pylint3 python3-pip tox git \
+ curl pycodestyle \
+ python3-sphinx
ARG COMMIT="dev"
-RUN git clone -b dev https://gitlab.com/caosdb/caosdb-pylib.git && \
+RUN git clone -b dev https://gitlab.indiscale.com/caosdb/src/caosdb-pylib.git && \
cd caosdb-pylib && git checkout $COMMIT && pip3 install .
+RUN pip3 install recommonmark sphinx-rtd-theme
diff --git a/unittests/test_add_property.py b/unittests/test_add_property.py
index 874314ee62d5471a9e859d8c4dc44c4e4a6532c7..5bae6c219732f0170f5c351eae58148c9d3d065a 100644
--- a/unittests/test_add_property.py
+++ b/unittests/test_add_property.py
@@ -5,6 +5,8 @@
#
# Copyright (C) 2018 Research Group Biomedical Physics,
# Max-Planck-Institute for Dynamics and Self-Organization Göttingen
+# Copyright (C) 2020 Indiscale GmbH <info@indiscale.com>
+# Copyright (C) 2020 Timm Fitschen <f.fitschen@indiscale.com>
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as
@@ -21,69 +23,82 @@
#
# ** end header
#
-import caosdb as db
from pytest import raises
-from nose.tools import assert_is, assert_is_none, assert_equals, assert_is_not_none, assert_raises
+import caosdb as db
def test_no_parameter():
rec = db.Record()
- assert_equals(0, len(rec.get_properties()))
+ assert 0 == len(rec.get_properties())
- with assert_raises(UserWarning) as cm:
+ with raises(UserWarning) as cm:
rec.add_property()
- assert_equals(
- cm.exception.args[0],
- "This method expects you to pass at least an entity, a name or an id.")
- assert_equals(0, len(rec.get_properties()))
+ assert cm.value.args[0] == ("This method expects you to pass at "
+ "least an entity, a name or an id.")
+ assert 0 == len(rec.get_properties())
def test_only_value_parameter():
rec = db.Record()
- assert_equals(0, len(rec.get_properties()))
+ assert 0 == len(rec.get_properties())
- with assert_raises(UserWarning) as cm:
+ with raises(UserWarning) as cm:
rec.add_property(value="bla")
- assert_equals(
- cm.exception.args[0],
- "This method expects you to pass at least an entity, a name or an id.")
- assert_equals(0, len(rec.get_properties()))
+ assert cm.value.args[0] == ("This method expects you to pass at "
+ "least an entity, a name or an id.")
+ assert 0 == len(rec.get_properties())
def test_property_name_ambiguity_1():
rec = db.Record()
- assert_equals(0, len(rec.get_properties()))
+ assert 0 == len(rec.get_properties())
- with assert_raises(UserWarning) as cm:
+ with raises(UserWarning) as cm:
rec.add_property("one_name", name="another_name")
- assert_equals(
- cm.exception.args[0],
- "The first parameter was neither an instance of Entity nor an integer. Therefore the string representation of your first parameter would normally be interpreted name of the property which is to be added. But you have also specified a parameter 'name' in the method call. This is ambiguous and cannot be processed.")
- assert_equals(0, len(rec.get_properties()))
+ assert cm.value.args[0] == ("The first parameter was neither an "
+ "instance of Entity nor an integer. "
+ "Therefore the string representation of "
+ "your first parameter would normally be "
+ "interpreted name of the property which "
+ "is to be added. But you have also "
+ "specified a parameter 'name' in the "
+ "method call. This is ambiguous and "
+ "cannot be processed.")
+ assert 0 == len(rec.get_properties())
def test_property_name_ambiguity_2():
rec = db.Record()
- assert_equals(0, len(rec.get_properties()))
+ assert 0 == len(rec.get_properties())
- with assert_raises(UserWarning) as cm:
+ with raises(UserWarning) as cm:
rec.add_property({}, name="another_name")
- assert_equals(
- cm.exception.args[0],
- "The first parameter was neither an instance of Entity nor an integer. Therefore the string representation of your first parameter would normally be interpreted name of the property which is to be added. But you have also specified a parameter 'name' in the method call. This is ambiguous and cannot be processed.")
- assert_equals(0, len(rec.get_properties()))
+ assert cm.value.args[0] == ("The first parameter was neither an "
+ "instance of Entity nor an integer. "
+ "Therefore the string representation of "
+ "your first parameter would normally be "
+ "interpreted name of the property which "
+ "is to be added. But you have also "
+ "specified a parameter 'name' in the "
+ "method call. This is ambiguous and "
+ "cannot be processed.")
+ assert 0 == len(rec.get_properties())
def test_property_id_ambiguity():
rec = db.Record()
- assert_equals(0, len(rec.get_properties()))
+ assert 0 == len(rec.get_properties())
- with assert_raises(UserWarning) as cm:
+ with raises(UserWarning) as cm:
rec.add_property(25, id=26)
- assert_equals(
- cm.exception.args[0],
- "The first parameter was an integer which would normally be interpreted as the id of the property which is to be added. But you have also specified a parameter 'id' in the method call. This is ambiguous and cannot be processed.")
- assert_equals(0, len(rec.get_properties()))
+
+ assert cm.value.args[0] == ("The first parameter was an integer which "
+ "would normally be interpreted as the id of "
+ "the property which is to be added. But you "
+ "have also specified a parameter 'id' in the "
+ "method call. This is ambiguous and cannot be "
+ "processed.")
+ assert 0 == len(rec.get_properties())
def test_property_parameter_with_entity():
@@ -95,18 +110,17 @@ def test_property_parameter_with_entity():
unit="m",
description="This is the length of something.")
- assert_equals(0, len(rec.get_properties()))
+ assert 0 == len(rec.get_properties())
rec.add_property(abstract_property)
- assert_equals(1, len(rec.get_properties()))
+ assert 1 == len(rec.get_properties())
concrete_property = rec.get_property("length")
- assert_is_not_none(concrete_property)
- assert_equals(concrete_property.name, "length")
- assert_equals(concrete_property.id, 512)
- assert_equals(concrete_property.description,
- "This is the length of something.")
- assert_equals(concrete_property.unit, "m")
- assert_equals(concrete_property.datatype, db.DOUBLE)
- assert_is(concrete_property._wrapped_entity, abstract_property)
+ assert concrete_property is not None
+ assert concrete_property.name == "length"
+ assert concrete_property.id == 512
+ assert concrete_property.description == "This is the length of something."
+ assert concrete_property.unit == "m"
+ assert concrete_property.datatype == db.DOUBLE
+ assert concrete_property._wrapped_entity == abstract_property
def test_property_parameter_with_entity_and_value():
@@ -118,54 +132,53 @@ def test_property_parameter_with_entity_and_value():
unit="m",
description="This is the length of something.")
- assert_equals(0, len(rec.get_properties()))
+ assert 0 == len(rec.get_properties())
rec.add_property(abstract_property, 3.14)
- assert_equals(1, len(rec.get_properties()))
+ assert 1 == len(rec.get_properties())
concrete_property = rec.get_property("length")
- assert_is_not_none(concrete_property)
- assert_equals(concrete_property.name, "length")
- assert_equals(concrete_property.id, 512)
- assert_equals(concrete_property.description,
- "This is the length of something.")
- assert_equals(concrete_property.unit, "m")
- assert_equals(concrete_property.value, 3.14)
- assert_equals(concrete_property.datatype, db.DOUBLE)
- assert_is(concrete_property._wrapped_entity, abstract_property)
+ assert concrete_property is not None
+ assert concrete_property.name == "length"
+ assert concrete_property.id == 512
+ assert concrete_property.description == "This is the length of something."
+ assert concrete_property.unit == "m"
+ assert concrete_property.value == 3.14
+ assert concrete_property.datatype == db.DOUBLE
+ assert concrete_property._wrapped_entity == abstract_property
def test_property_parameter_with_id():
rec = db.Record()
- assert_equals(0, len(rec.get_properties()))
+ assert 0 == len(rec.get_properties())
rec.add_property(512)
- assert_equals(1, len(rec.get_properties()))
+ assert 1 == len(rec.get_properties())
concrete_property = rec.get_property(512)
- assert_is_not_none(concrete_property)
- assert_equals(concrete_property.id, 512)
+ assert concrete_property is not None
+ assert concrete_property.id == 512
def test_property_parameter_with_id_and_value():
rec = db.Record()
- assert_equals(0, len(rec.get_properties()))
+ assert 0 == len(rec.get_properties())
rec.add_property(512, 3.14)
- assert_equals(1, len(rec.get_properties()))
+ assert 1 == len(rec.get_properties())
concrete_property = rec.get_property(512)
- assert_is_not_none(concrete_property)
- assert_equals(concrete_property.id, 512)
- assert_equals(concrete_property.value, 3.14)
+ assert concrete_property is not None
+ assert concrete_property.id == 512
+ assert concrete_property.value == 3.14
def test_datatype():
rec = db.Record()
- assert_equals(0, len(rec.get_properties()))
+ assert 0 == len(rec.get_properties())
rec.add_property(512, 3.14)
- assert_equals(1, len(rec.get_properties()))
+ assert 1 == len(rec.get_properties())
concrete_property = rec.get_property(512)
- assert_is_not_none(concrete_property)
- assert_equals(concrete_property.id, 512)
- assert_equals(concrete_property.value, 3.14)
+ assert concrete_property is not None
+ assert concrete_property.id == 512
+ assert concrete_property.value == 3.14
def test_property_parameter_with_entity_and_datatype():
@@ -214,23 +227,23 @@ def test_property_parameter_with_entity_and_datatype():
def test_kw_name_and_value():
rec = db.Record()
- assert_equals(0, len(rec.get_properties()))
+ assert 0 == len(rec.get_properties())
rec.add_property(name="length", value=3.14)
- assert_equals(1, len(rec.get_properties()))
+ assert 1 == len(rec.get_properties())
concrete_property = rec.get_property("length")
- assert_is_not_none(concrete_property)
- assert_equals(concrete_property.value, 3.14)
+ assert concrete_property is not None
+ assert concrete_property.value == 3.14
def test_kw_id_and_value():
rec = db.Record()
- assert_equals(0, len(rec.get_properties()))
+ assert 0 == len(rec.get_properties())
rec.add_property(id=512, value=3.14)
- assert_equals(1, len(rec.get_properties()))
+ assert 1 == len(rec.get_properties())
concrete_property = rec.get_property(512)
- assert_is_not_none(concrete_property)
- assert_equals(concrete_property.value, 3.14)
+ assert concrete_property is not None
+ assert concrete_property.value == 3.14
def test_add_list_of_entitities():
@@ -240,7 +253,7 @@ def test_add_list_of_entitities():
values.append(db.Record(name=str(i)))
rec.add_property("listOfEntities", values)
for e in rec.get_property("listOfEntities").value:
- assert_is_none(e.id)
+ assert e.id is None
i = 0
for val in values:
@@ -249,5 +262,5 @@ def test_add_list_of_entitities():
i = 0
for e in rec.get_property("listOfEntities").value:
- assert_equals(i, e.id)
+ assert i == e.id
i += 1
diff --git a/unittests/test_apiutils.py b/unittests/test_apiutils.py
index df6fe52d5b6f7ceddea791167a2355d7b3bf2fc8..c560b5e3c7c424b762bc8381c7cc9f42617288d0 100644
--- a/unittests/test_apiutils.py
+++ b/unittests/test_apiutils.py
@@ -27,9 +27,9 @@
# A. Schlemmer, 02/2018
import caosdb as db
-from caosdb.apiutils import apply_to_ids
import pickle
import tempfile
+from caosdb.apiutils import apply_to_ids
from .test_property import testrecord
diff --git a/unittests/test_authentication_auth_token.py b/unittests/test_authentication_auth_token.py
new file mode 100644
index 0000000000000000000000000000000000000000..15e54121fc0d7b5c2be645cdb88bc20804a10980
--- /dev/null
+++ b/unittests/test_authentication_auth_token.py
@@ -0,0 +1,101 @@
+# -*- coding: utf-8 -*-
+#
+# ** header v3.0
+# This file is a part of the CaosDB Project.
+#
+# Copyright (C) 2020 Timm Fitschen <t.fitschen@indiscale.com>
+# Copyright (C) 2020 IndiScale GmbH <info@indiscale.com>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see <https://www.gnu.org/licenses/>.
+#
+# ** end header
+#
+"""test_authentication_auth_token
+
+Unit tests for the module caosdb.connection.authentication.auth_token
+"""
+
+from __future__ import unicode_literals
+from pytest import raises
+from unittest.mock import Mock
+from caosdb.connection.authentication import auth_token as at
+from caosdb.connection.mockup import MockUpServerConnection, MockUpResponse
+from caosdb.connection.utils import parse_auth_token
+from caosdb.exceptions import LoginFailedError
+from caosdb import configure_connection
+
+
+def test_get_authentication_provider():
+ ap = at.get_authentication_provider()
+ assert isinstance(ap, at.AuthTokenAuthenticator)
+
+
+def response_with_auth_token():
+ token = "SessionToken=[response token];"
+ assert parse_auth_token(token) is not None, "cookie not ok"
+
+ return MockUpResponse(200, {"Set-Cookie": token}, "ok")
+
+
+def test_configure_connection():
+ def request_has_auth_token(**kwargs):
+ """test resources"""
+ cookie = kwargs["headers"]["Cookie"]
+ assert cookie is not None
+ assert cookie == "SessionToken=%5Brequest%20token%5D;"
+
+ return response_with_auth_token()
+
+ c = configure_connection(password_method="auth_token",
+ auth_token="[request token]",
+ implementation=MockUpServerConnection)
+ assert isinstance(c._authenticator, at.AuthTokenAuthenticator)
+
+ c._delegate_connection.resources.append(request_has_auth_token)
+ assert c._authenticator.auth_token == "[request token]"
+ response = c._http_request(method="GET", path="test")
+ assert response.read() == "ok"
+ assert c._authenticator.auth_token == "[response token]"
+
+
+def test_login_raises():
+ c = configure_connection(url="https://example.com",
+ password_method="auth_token",
+ auth_token="[auth_token]")
+ with raises(LoginFailedError):
+ c._login()
+
+
+def test_logout_calls_delete():
+ mock = Mock()
+
+ def logout_resource(**kwargs):
+ """logout with auth_token"""
+ mock.method()
+ assert kwargs["path"] == "logout"
+ assert kwargs["method"] == "DELETE"
+
+ cookie = kwargs["headers"]["Cookie"]
+ assert cookie is not None
+ assert cookie == "SessionToken=%5Brequest%20token%5D;"
+
+ return MockUpResponse(200, {}, "ok")
+
+ c = configure_connection(password_method="auth_token",
+ auth_token="[request token]",
+ implementation=MockUpServerConnection)
+
+ c._delegate_connection.resources.append(logout_resource)
+ c._logout()
+ mock.method.assert_called_once()
diff --git a/unittests/test_authentication_plain.py b/unittests/test_authentication_plain.py
index 26c57e953f28a2a0ab80e35026c27ff74f18d370..10cbc418df8bd81c81568a4df0cf1e8a4ac498f8 100644
--- a/unittests/test_authentication_plain.py
+++ b/unittests/test_authentication_plain.py
@@ -27,11 +27,18 @@ Unit tests for the modul caosdb.connection.authentication.plain.
"""
from __future__ import unicode_literals
-from pytest import raises
+
from caosdb.connection.authentication.plain import PlainTextCredentialsProvider
+from pytest import raises
def test_subclass_configure():
+ # TODO I do not see the meaning of this test.
+ # It only tests, that the call of the super version of configure sets the
+ # password property. And that due to the subclassing no longer a password
+ # argument can be provided.
+ # Suggestion: Either remove this test or state in what context this test
+ # is meanigful.
"""Test the correct passing of the password argument."""
class SubClassOf(PlainTextCredentialsProvider):
"""A simple subclass of PlainTextCredentialsProvider."""
@@ -51,8 +58,8 @@ def test_subclass_configure():
with raises(TypeError) as exc_info:
instance.configure(password="OH NO!")
- assert exc_info.value.args[0] == ("configure() got multiple values for "
- "keyword argument 'password'")
+ assert exc_info.value.args[0].endswith("configure() got multiple values for "
+ "keyword argument 'password'")
def test_plain_has_logger():
diff --git a/unittests/test_authentication_unauthenticated.py b/unittests/test_authentication_unauthenticated.py
new file mode 100644
index 0000000000000000000000000000000000000000..52146b08ed4e1026660eebacedf348aeb2ff2721
--- /dev/null
+++ b/unittests/test_authentication_unauthenticated.py
@@ -0,0 +1,71 @@
+# -*- coding: utf-8 -*-
+#
+# ** header v3.0
+# This file is a part of the CaosDB Project.
+#
+# Copyright (C) 2020 Timm Fitschen <t.fitschen@indiscale.com>
+# Copyright (C) 2020 IndiScale GmbH <info@indiscale.com>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see <https://www.gnu.org/licenses/>.
+#
+# ** end header
+#
+"""test_authentication_unauthenticated
+
+Unit tests for the module caosdb.connection.authentication.unauthenticated.
+"""
+
+from __future__ import unicode_literals
+from pytest import raises
+from unittest.mock import Mock
+from caosdb.connection.authentication import unauthenticated
+from caosdb.connection.mockup import MockUpServerConnection, MockUpResponse
+from caosdb.connection.utils import parse_auth_token
+from caosdb.exceptions import LoginFailedError
+from caosdb import configure_connection
+from .test_authentication_auth_token import response_with_auth_token
+
+
+def test_get_authentication_provider():
+ ap = unauthenticated.get_authentication_provider()
+ assert isinstance(ap, unauthenticated.Unauthenticated)
+
+
+def test_configure_connection():
+ mock = Mock()
+
+ def request_has_no_auth_token(**kwargs):
+ """test resource"""
+ assert "Cookie" not in kwargs["headers"]
+ mock.method()
+ return response_with_auth_token()
+
+ c = configure_connection(password_method="unauthenticated",
+ implementation=MockUpServerConnection)
+ assert isinstance(c._authenticator, unauthenticated.Unauthenticated)
+
+ c._delegate_connection.resources.append(request_has_no_auth_token)
+
+ assert c._authenticator.auth_token is None
+ response = c._http_request(method="GET", path="test")
+ assert response.read() == "ok"
+ mock.method.assert_called_once()
+ assert c._authenticator.auth_token is None
+
+
+def test_login_raises():
+ c = configure_connection(url="https://example.com",
+ password_method="unauthenticated")
+ with raises(LoginFailedError):
+ c._login()
diff --git a/unittests/test_concrete_property.py b/unittests/test_concrete_property.py
index b806098732c8a77a4913bbda0f8b2b8ebdc4a68c..0e5c28534c7ac404b829df575225f42e908adb01 100644
--- a/unittests/test_concrete_property.py
+++ b/unittests/test_concrete_property.py
@@ -24,17 +24,19 @@
"""Tests for the _ConcreteProperty class."""
-# pylint: disable=missing-docstring
-from nose.tools import (assert_is_not_none as there, assert_true as tru,
- assert_equal as eq)
-from caosdb.common.models import _ConcreteProperty
from caosdb import configure_connection
+from caosdb.common.models import _ConcreteProperty
from caosdb.connection.mockup import MockUpServerConnection
+# pylint: disable=missing-docstring
+from nose.tools import assert_equal as eq
+from nose.tools import assert_is_not_none as there
+from nose.tools import assert_true as tru
def setup_module():
there(_ConcreteProperty)
configure_connection(url="unittests", username="testuser",
+ password_method="plain",
password="testpassword", timeout=200,
implementation=MockUpServerConnection)
diff --git a/unittests/test_configuration.py b/unittests/test_configuration.py
index 76445b6f262120d6a29c73527a9bf042f85f8a05..b135e7cd65b11be7cb6c4ef2237a41a6639ccbb7 100644
--- a/unittests/test_configuration.py
+++ b/unittests/test_configuration.py
@@ -22,19 +22,45 @@
# ** end header
#
+import pytest
import caosdb as db
+from os import environ, getcwd, remove
+from os.path import expanduser, isfile, join
from pytest import raises
-def test_config_ini_via_envvar():
- from os import environ
- from os.path import expanduser
+@pytest.fixture
+def temp_ini_files():
+ created_temp_ini_cwd = False
+ created_temp_ini_home = False
+ if not isfile(join(getcwd(), "pycaosdb.ini")):
+ open("pycaosdb.ini", 'a').close() # create temporary ini file
+ created_temp_ini_cwd = True
+ if not isfile(expanduser("~/.pycaosdb.ini")):
+ open(expanduser("~/.pycaosdb.ini"), 'a').close() # create temporary ini file in home directory
+ created_temp_ini_home = True
+ yield 0
+ if created_temp_ini_cwd:
+ remove("pycaosdb.ini")
+ if created_temp_ini_home:
+ remove(expanduser("~/.pycaosdb.ini"))
+ environ["PYCAOSDBINI"] = "~/.pycaosdb.ini"
+
+
+def test_config_ini_via_envvar(temp_ini_files):
with raises(KeyError):
environ["PYCAOSDBINI"]
environ["PYCAOSDBINI"] = "bla bla"
assert environ["PYCAOSDBINI"] == "bla bla"
- assert db.configuration.configure(environ["PYCAOSDBINI"]) == []
+ # test wrong configuration file in envvar
+ assert not expanduser(environ["PYCAOSDBINI"]) in db.configuration._read_config_files()
+ # test good configuration file in envvar
environ["PYCAOSDBINI"] = "~/.pycaosdb.ini"
- assert db.configuration.configure(expanduser(environ["PYCAOSDBINI"])) == [expanduser("~/.pycaosdb.ini")]
+ assert expanduser("~/.pycaosdb.ini") in db.configuration._read_config_files()
+ # test without envvar
+ environ.pop("PYCAOSDBINI")
+ assert expanduser("~/.pycaosdb.ini") in db.configuration._read_config_files()
+ # test configuration file in cwd
+ assert join(getcwd(), "pycaosdb.ini") in db.configuration._read_config_files()
diff --git a/unittests/test_connection.py b/unittests/test_connection.py
index c1f62088c425b2ef587a6b0ca0028f38180ba2ee..16370f00b7d5e3389582befaac1762b1d2992fcf 100644
--- a/unittests/test_connection.py
+++ b/unittests/test_connection.py
@@ -23,23 +23,27 @@
#
"""Test caosdb.connection."""
# pylint: disable=missing-docstring
-from __future__ import unicode_literals, print_function
-from builtins import bytes, str # pylint: disable=redefined-builtin
+from __future__ import print_function, unicode_literals
+
import re
-from pytest import raises
-from nose.tools import (assert_equal as eq, assert_raises as raiz, assert_true
- as tru, assert_is_not_none as there, assert_false as
- falz)
-from caosdb.exceptions import ConfigurationException, LoginFailedException
-from caosdb.connection.utils import urlencode, make_uri_path, quote
-from caosdb.connection.connection import (
- configure_connection, CaosDBServerConnection,
- _DefaultCaosDBServerConnection)
-from caosdb.connection.mockup import (MockUpServerConnection, MockUpResponse,
- _request_log_message)
-from caosdb.configuration import get_config, _reset_config
-from caosdb.connection.authentication.interface import CredentialsAuthenticator
+from builtins import bytes, str # pylint: disable=redefined-builtin
+
from caosdb import execute_query
+from caosdb.configuration import _reset_config, get_config
+from caosdb.connection.authentication.interface import CredentialsAuthenticator
+from caosdb.connection.connection import (CaosDBServerConnection,
+ _DefaultCaosDBServerConnection,
+ configure_connection)
+from caosdb.connection.mockup import (MockUpResponse, MockUpServerConnection,
+ _request_log_message)
+from caosdb.connection.utils import make_uri_path, quote, urlencode
+from caosdb.exceptions import ConfigurationError, LoginFailedError
+from nose.tools import assert_equal as eq
+from nose.tools import assert_false as falz
+from nose.tools import assert_is_not_none as there
+from nose.tools import assert_raises as raiz
+from nose.tools import assert_true as tru
+from pytest import raises
def setup_module():
@@ -96,6 +100,7 @@ def test_configure_connection():
get_config().add_section("Connection")
get_config().set("Connection", "url", "https://host.de")
get_config().set("Connection", "username", "test_username")
+ get_config().set("Connection", "password_method", "plain")
get_config().set("Connection", "password", "test_password")
get_config().set("Connection", "timeout", "200")
@@ -152,6 +157,7 @@ def test_init_response():
response = MockUpResponse(
status=200, headers={"sessionToken": "SessionToken"}, body="Body")
there(response)
+
return response
@@ -175,6 +181,7 @@ def test_getter_session_token():
def test_init_connection():
connection = MockUpServerConnection()
there(connection)
+
return connection
@@ -184,6 +191,7 @@ def test_resources_list():
assert len(connection.resources) == 1
connection.resources.append(lambda **kwargs: test_init_response())
assert len(connection.resources) == 2
+
return connection
@@ -214,6 +222,7 @@ def setup_two_resources():
connection = test_init_connection()
connection.resources.extend([r1, r2, r3])
+
return connection
@@ -225,7 +234,7 @@ def test_test_request_with_two_responses():
def test_missing_implementation():
connection = configure_connection()
- with raises(ConfigurationException) as exc_info:
+ with raises(ConfigurationError) as exc_info:
connection.configure()
assert exc_info.value.args[0].startswith(
"Missing CaosDBServerConnection implementation.")
@@ -233,27 +242,26 @@ def test_missing_implementation():
def test_bad_implementation_not_callable():
connection = configure_connection()
- with raises(ConfigurationException) as exc_info:
+ with raises(ConfigurationError) as exc_info:
connection.configure(implementation=None)
assert exc_info.value.args[0].startswith(
"Bad CaosDBServerConnection implementation.")
- assert exc_info.value.args[1].args[0] == "'NoneType' object is not callable"
+ assert "'NoneType' object is not callable" in exc_info.value.args[0]
def test_bad_implementation_wrong_class():
connection = configure_connection()
- with raises(ConfigurationException) as exc_info:
+ with raises(ConfigurationError) as exc_info:
connection.configure(implementation=dict)
assert exc_info.value.args[0].startswith(
"Bad CaosDBServerConnection implementation.")
- assert exc_info.value.args[1].args[0] == (
- "The `implementation` callable did not return an instance of "
- "CaosDBServerConnection.")
+ assert ("The `implementation` callable did not return an instance of "
+ "CaosDBServerConnection.") in exc_info.value.args[0]
def test_missing_auth_method():
connection = configure_connection()
- with raises(ConfigurationException) as exc_info:
+ with raises(ConfigurationError) as exc_info:
connection.configure(implementation=MockUpServerConnection)
assert exc_info.value.args[0].startswith("Missing password_method.")
@@ -261,9 +269,30 @@ def test_missing_auth_method():
def test_missing_password():
connection = configure_connection()
connection.configure(implementation=setup_two_resources,
- password_method="plain", auth_token="[test-auth-token]")
- assert connection.retrieve(["some"]).headers["Cookie"] == "SessionToken=%5Btest-auth-token%5D;"
+ password_method="plain")
+ connection._authenticator.auth_token = "[test-auth-token]"
+ assert connection.retrieve(
+ ["some"]).headers["Cookie"] == "SessionToken=%5Btest-auth-token%5D;"
+
connection.configure(implementation=setup_two_resources,
password_method="plain")
- with raises(LoginFailedException):
+ with raises(LoginFailedError):
connection.delete(["401"])
+
+
+def test_auth_token_connection():
+ connection = configure_connection(auth_token="blablabla",
+ password_method="auth_token",
+ implementation=setup_two_resources)
+ connection.retrieve(
+ ["some"]).headers["Cookie"] == "SessionToken=blablabla;"
+
+ connection._logout()
+ with raises(LoginFailedError) as cm:
+ connection.retrieve(
+ ["some"]).headers["Cookie"] == "SessionToken=blablabla;"
+ assert cm.value.args[0] == ("The authentication token is expired or you "
+ "have been logged out otherwise. The "
+ "auth_token authenticator cannot log in "
+ "again. You must provide a new authentication "
+ "token.")
diff --git a/unittests/test_connection_utils.py b/unittests/test_connection_utils.py
index c21b453e3a7588f30d86be8d2ed39bb4d7a1d31e..3890ae05cfe38b78a5ba0829753420246bdb560d 100644
--- a/unittests/test_connection_utils.py
+++ b/unittests/test_connection_utils.py
@@ -28,7 +28,7 @@ from pytest import raises
from nose.tools import (assert_equal as eq, assert_raises as raiz, assert_true
as tru, assert_is_not_none as there, assert_false as
falz)
-from caosdb.exceptions import ConfigurationException, LoginFailedException
+from caosdb.exceptions import ConfigurationError, LoginFailedError
from caosdb.connection.utils import parse_auth_token, auth_token_to_cookie
from caosdb.connection.connection import (
configure_connection, CaosDBServerConnection,
@@ -45,8 +45,10 @@ def setup_module():
def test_parse_auth_token():
- assert parse_auth_token("SessionToken=%5Bblablabla%5D; expires=bla; ...") == "[blablabla]"
+ assert parse_auth_token(
+ "SessionToken=%5Bblablabla%5D; expires=bla; ...") == "[blablabla]"
def test_auth_token_to_cookie():
- assert auth_token_to_cookie("[blablabla]") == "SessionToken=%5Bblablabla%5D;"
+ assert auth_token_to_cookie(
+ "[blablabla]") == "SessionToken=%5Bblablabla%5D;"
diff --git a/unittests/test_container.py b/unittests/test_container.py
new file mode 100644
index 0000000000000000000000000000000000000000..b34055372fc83a5608ffcf54423a601001add12b
--- /dev/null
+++ b/unittests/test_container.py
@@ -0,0 +1,79 @@
+
+# -*- encoding: utf-8 -*-
+#
+# ** header v3.0
+# This file is a part of the CaosDB Project.
+#
+# Copyright (C) 2020 Timm Fitschen <t.fitschen@indiscale.com>
+# Copyright (C) 2020 IndiScale GmbH <info@indiscale.com>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see <https://www.gnu.org/licenses/>.
+#
+# ** end header
+#
+"""Tests for the Container class."""
+from __future__ import absolute_import
+
+import caosdb as c
+
+
+def test_get_property_values():
+ rt_house = c.RecordType("House")
+ rt_window = c.RecordType("Window")
+ rt_owner = c.RecordType("Owner")
+ p_height = c.Property("Height", datatype=c.DOUBLE)
+
+ window = c.Record().add_parent(rt_window)
+ window.id = 1001
+ window.add_property(p_height, 20.5, unit="m")
+
+ owner = c.Record("The Queen").add_parent(rt_owner)
+
+ house = c.Record("Buckingham Palace")
+ house.add_parent(rt_house)
+ house.add_property(rt_owner, owner)
+ house.add_property(rt_window, window)
+ house.add_property(p_height, 40.2, unit="ft")
+
+ container = c.Container()
+ container.extend([
+ house,
+ owner
+ ])
+
+ assert getattr(house.get_property(p_height), "unit") == "ft"
+ assert getattr(window.get_property(p_height), "unit") == "m"
+
+ table = container.get_property_values("naME",
+ "height",
+ ("height", "unit"),
+ "window",
+ ("window", "non-existing"),
+ ("window", "non-existing", "unit"),
+ ("window", "unit"),
+ ("window", "heiGHT"),
+ ("window", "heiGHT", "value"),
+ ("window", "heiGHT", "unit"),
+ "owner",
+ )
+ assert len(table) == 2
+ house_row = table[0]
+ assert house_row == (house.name, 40.2, "ft", window.id, None, None, None, 20.5, 20.5, "m", owner.name)
+
+ owner_row = table[1]
+ assert owner_row == (owner.name, None, None, None, None, None, None, None, None, None, None)
+
+ assert container.get_property_values("non-existing") == [(None,), (None,)]
+ assert container.get_property_values("name") == [(house.name,),
+ (owner.name,)]
diff --git a/unittests/test_entity.py b/unittests/test_entity.py
index d877e1ab7267977a7e73d65033d6f02c16ea5521..e98dfbef5b6b5e5f691e8aecc2fa7d4a86991452 100644
--- a/unittests/test_entity.py
+++ b/unittests/test_entity.py
@@ -35,6 +35,7 @@ class TestEntity(unittest.TestCase):
def setUp(self):
self.assertIsNotNone(Entity)
configure_connection(url="unittests", username="testuser",
+ password_method="plain",
password="testpassword", timeout=200,
implementation=MockUpServerConnection)
diff --git a/unittests/test_error_handling.py b/unittests/test_error_handling.py
new file mode 100644
index 0000000000000000000000000000000000000000..7f974e7db826d093e335b250953658b08db062cd
--- /dev/null
+++ b/unittests/test_error_handling.py
@@ -0,0 +1,317 @@
+# -*- encoding: utf-8 -*-
+#
+# ** header v3.0
+# This file is a part of the CaosDB Project.
+#
+# Copyright (C) 2020 Indiscale GmbH <info@indiscale.com>
+# Copyright (C) 2020 Florian Spreckelsen <f.spreckelsen@indiscale.com>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see <https://www.gnu.org/licenses/>.
+#
+# ** end header
+#
+"""Test the new (as of June 2020) error handling. All errors should
+be TransactionErrors at first which may have one or more level of
+children.
+
+"""
+import caosdb as db
+from caosdb.common.models import raise_errors
+from caosdb.exceptions import (AuthorizationError,
+ EntityDoesNotExistError, EntityError,
+ EntityHasNoDatatypeError,
+ TransactionError, UniqueNamesError,
+ UnqualifiedParentsError,
+ UnqualifiedPropertiesError)
+
+from pytest import raises
+
+
+# #################### Single Error Tests ####################
+
+
+def _add_error_message_to_entity(entity, code, description="Error message"):
+ """Attach error message with code and description to entity"""
+ message = db.Message(type="Error", code=code,
+ description=description)
+ entity.add_message(message)
+
+ return entity
+
+
+def test_has_no_datatype_error():
+ """Code 110; property without datatype"""
+ code = 110
+ prop = _add_error_message_to_entity(db.Property(name="TestProp"),
+ code)
+ with raises(TransactionError) as e:
+ raise_errors(prop)
+ # There should be exactly one child
+ assert len(e.value.errors) == 1
+ err = e.value.errors[0]
+ # check type and entity of only child
+ assert isinstance(err, EntityHasNoDatatypeError)
+ assert err.entity.name == prop.name
+
+
+def test_entity_does_not_exist_error():
+ """Code 101; entity does not exist"""
+ code = 101
+ ent = _add_error_message_to_entity(db.Entity(name="TestEnt"),
+ code)
+ with raises(TransactionError) as e:
+ raise_errors(ent)
+ # There should be exactly one child
+ assert len(e.value.errors) == 1
+ err = e.value.errors[0]
+ # check type and entity of only child
+ assert isinstance(err, EntityDoesNotExistError)
+ assert err.entity.name == ent.name
+
+
+def test_entity_error():
+ """Code 0; most basic."""
+ code = 0
+ ent = _add_error_message_to_entity(db.Entity(name="TestEnt"),
+ code)
+ with raises(TransactionError) as e:
+ raise_errors(ent)
+ assert len(e.value.errors) == 1
+ err = e.value.errors[0]
+ assert isinstance(err, EntityError)
+ assert err.entity.name == ent.name
+
+
+def test_unique_names_error():
+ """Code 152; name is not unique"""
+ code = 152
+ ent = _add_error_message_to_entity(db.Entity(name="TestEnt"),
+ code)
+ with raises(TransactionError) as e:
+ raise_errors(ent)
+ assert len(e.value.errors) == 1
+ err = e.value.errors[0]
+ assert isinstance(err, UniqueNamesError)
+ assert err.entity.name == ent.name
+
+
+def test_authorization_exception():
+ """Code 403; transaction not allowed"""
+ code = 403
+ ent = _add_error_message_to_entity(db.Entity(name="TestEnt"),
+ code)
+ with raises(TransactionError) as e:
+ raise_errors(ent)
+ assert len(e.value.errors) == 1
+ err = e.value.errors[0]
+ assert isinstance(err, AuthorizationError)
+ assert err.entity.name == ent.name
+
+
+def test_empty_container_with_error():
+ """Has to raise an error, even though container is empty."""
+ code = 0
+ cont = _add_error_message_to_entity(db.Container(), code)
+ with raises(TransactionError) as e:
+ raise_errors(cont)
+ # No entity errors
+ assert len(e.value.errors) == 0
+ assert e.value.container == cont
+ assert int(e.value.code) == code
+
+
+def test_faulty_container_with_healthy_entities():
+ """Raises a TransactionError without any EntityErrors since only the
+ container, but none of its entities has an error.
+
+ """
+ code = 0
+ cont = _add_error_message_to_entity(db.Container(), code)
+ cont.append(db.Entity("TestHealthyEnt1"))
+ cont.append(db.Entity("TestHealthyEnt2"))
+ with raises(TransactionError) as e:
+ raise_errors(cont)
+ # No entity errors
+ assert len(e.value.errors) == 0
+ assert len(e.value.entities) == 0
+ assert e.value.container == cont
+ assert int(e.value.code) == code
+
+
+# #################### Children with children ####################
+
+
+def test_unqualified_parents_error():
+ """Code 116; parent does not exist"""
+ code = 116
+ entity_does_not_exist_code = 101
+ parent = _add_error_message_to_entity(
+ db.RecordType(name="TestParent"),
+ entity_does_not_exist_code)
+ rec = _add_error_message_to_entity(db.Record(name="TestRecord"),
+ code)
+ rec.add_parent(parent)
+ with raises(TransactionError) as e:
+ raise_errors(rec)
+ te = e.value
+ # One direct child, two errors in total
+ assert len(te.errors) == 1
+ assert len(te.all_errors) == 2
+ # UnqualifiedParentsError in Record ...
+ assert isinstance(te.errors[0], UnqualifiedParentsError)
+ assert te.errors[0].entity.name == rec.name
+ # ... caused by non-existing parent
+ assert isinstance(te.errors[0].errors[0], EntityDoesNotExistError)
+ assert te.errors[0].errors[0].entity.name == parent.name
+
+
+def test_unqualified_properties_error():
+ """Code 114; properties do not exist or have wrong data types or
+ values.
+
+ """
+ code = 114
+ entity_code = 0
+ no_entity_code = 101
+ prop1 = _add_error_message_to_entity(db.Property(
+ name="TestProp1"), entity_code)
+ prop2 = _add_error_message_to_entity(db.Property(
+ name="TestProp2"), no_entity_code)
+ rec = _add_error_message_to_entity(db.Record(name="TestRecord"),
+ code)
+ rec.add_property(prop1).add_property(prop2)
+ with raises(TransactionError) as e:
+ raise_errors(rec)
+ te = e.value
+ assert len(te.errors) == 1
+ upe = te.errors[0]
+ assert upe.entity.name == rec.name
+ assert len(upe.errors) == 2
+ for error_t in [UnqualifiedPropertiesError, EntityError,
+ EntityDoesNotExistError]:
+ assert any([isinstance(x, error_t) for x in te.all_errors])
+ assert upe.code == code
+
+
+# #################### Multiple errors ####################
+
+def test_parent_and_properties_errors():
+ """Record with UnqualifiedParentsError and UnqualifiedPropertiesError,
+ and corresponding parent and properties with their errors as
+ above. Test whether all levels are in order.
+
+ """
+ prop_code = 114
+ parent_code = 116
+ entity_code = 0
+ no_entity_code = 101
+ parent = _add_error_message_to_entity(
+ db.RecordType(name="TestParent"), no_entity_code)
+ prop1 = _add_error_message_to_entity(db.Property(
+ name="TestProp1"), entity_code)
+ prop2 = _add_error_message_to_entity(db.Property(
+ name="TestProp2"), no_entity_code)
+ rec = _add_error_message_to_entity(db.Record(name="TestRecord"),
+ prop_code)
+ rec = _add_error_message_to_entity(rec, parent_code)
+ rec.add_parent(parent)
+ rec.add_property(prop1).add_property(prop2)
+ with raises(TransactionError) as e:
+ raise_errors(rec)
+ # Now there should be two direct children; both have to be
+ # displayed correctly.
+ te = e.value
+ # exactly two children:
+ assert len(te.errors) == 2
+ # both have to have the right codes and entities
+ found_parent = False
+ found_prop = False
+ for err in te.errors:
+ if err.code == parent_code:
+ found_parent = True
+ assert err.errors[0].entity.name == parent.name
+ assert prop1.name not in [x.name for x in
+ err.all_entities]
+ assert prop2.name not in [x.name for x in
+ err.all_entities]
+ elif err.code == prop_code:
+ found_prop = True
+ assert parent.name not in [x.name for x in
+ err.all_entities]
+ for sub_err in err.errors:
+ if sub_err.code == entity_code:
+ assert sub_err.entity.name == prop1.name
+ elif sub_err.code == no_entity_code:
+ assert sub_err.entity.name == prop2.name
+ assert found_parent
+ assert found_prop
+
+
+def test_container_with_faulty_elements():
+ """Code 12; container with valid and invalid entities. All faulty
+ entities have to be reflected correctly in the errors list of the
+ TransactionError raised by the container.
+
+ """
+ container_code = 12
+ prop_code = 114
+ parent_code = 116
+ name_code = 152
+ auth_code = 403
+ entity_code = 0
+ no_entity_code = 101
+ # Broken parents and properties
+ parent = _add_error_message_to_entity(
+ db.RecordType(name="TestParent"), no_entity_code)
+ prop1 = _add_error_message_to_entity(db.Property(
+ name="TestProp1"), entity_code)
+ prop2 = _add_error_message_to_entity(db.Property(
+ name="TestProp2"), no_entity_code)
+ cont = _add_error_message_to_entity(db.Container(),
+ container_code)
+ # healthy record and property
+ good_rec = db.Record(name="TestRecord1")
+ good_prop = db.Property(name="TestProp3")
+ cont.extend([good_rec, good_prop])
+ # broken records with single and multiole errors
+ rec_name = _add_error_message_to_entity(db.Record(name="TestRecord2"),
+ code=name_code)
+ rec_auth = _add_error_message_to_entity(db.Record(name="TestRecord3"),
+ code=auth_code)
+ rec_par_prop = _add_error_message_to_entity(
+ db.Record(name="TestRecord"), prop_code)
+ rec_par_prop = _add_error_message_to_entity(rec_par_prop, parent_code)
+ rec_par_prop.add_parent(parent)
+ rec_par_prop.add_property(prop1).add_property(prop2)
+ cont.extend([rec_name, rec_auth, rec_par_prop])
+ with raises(TransactionError) as e:
+ raise_errors(cont)
+ te = e.value
+ assert te.container == cont
+ assert te.code == container_code
+ # no healthy entity caused an error
+ for good in [good_rec, good_prop]:
+ assert good not in te.all_entities
+ # all records that caused problems
+ assert {rec_name, rec_auth, rec_par_prop}.issubset(te.all_entities)
+ # the container error contains the errors caused by the records
+ for err in te.errors:
+ if err.entity.name == rec_name.name:
+ assert isinstance(err, UniqueNamesError)
+ elif err.entity.name == rec_auth.name:
+ assert isinstance(err, AuthorizationError)
+ elif err.entity.name == rec_par_prop.name:
+ # record raises both of them
+ assert (isinstance(err, UnqualifiedParentsError) or
+ isinstance(err, UnqualifiedPropertiesError))
diff --git a/unittests/test_file.py b/unittests/test_file.py
index 957c578062b25c74547b599a25802938b057a2e0..3c80af7f362a7cdabe0a9ebc89cd2986d04fe242 100644
--- a/unittests/test_file.py
+++ b/unittests/test_file.py
@@ -22,16 +22,18 @@
# ** end header
#
"""Tests for the File class."""
-# pylint: disable=missing-docstring
-from nose.tools import (assert_is_not_none as there, assert_true as tru,
- assert_equal as eq)
-from caosdb import Record, File, configure_connection
+from caosdb import File, Record, configure_connection
from caosdb.connection.mockup import MockUpServerConnection
+# pylint: disable=missing-docstring
+from nose.tools import assert_equal as eq
+from nose.tools import assert_is_not_none as there
+from nose.tools import assert_true as tru
def setup_module():
there(File)
configure_connection(url="unittests", username="testuser",
+ password_method="plain",
password="testpassword", timeout=200,
implementation=MockUpServerConnection)
diff --git a/unittests/test_message.py b/unittests/test_message.py
new file mode 100644
index 0000000000000000000000000000000000000000..5e1003056c1b606a004b63bb7618e5e0474952bc
--- /dev/null
+++ b/unittests/test_message.py
@@ -0,0 +1,118 @@
+# encoding: utf-8
+#
+# ** header v3.0
+# This file is a part of the CaosDB Project.
+#
+# Copyright (C) 2018 Research Group Biomedical Physics,
+# Max-Planck-Institute for Dynamics and Self-Organization Göttingen
+# Copyright (C) 2020 IndiScale GmbH <info@indiscale.com>
+# Copyright (C) 2020 Florian Spreckelsen <f.spreckelsen@indiscale.com>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see <https://www.gnu.org/licenses/>.
+#
+# ** end header
+#
+import caosdb as db
+from copy import deepcopy
+
+
+def test_messages_dict_behavior():
+ from caosdb.common.models import Message
+ from caosdb.common.models import _Messages
+
+ msgs = _Messages()
+
+ # create Message
+ msg = Message(
+ type="HelloWorld",
+ code=1,
+ description="Greeting the world",
+ body="Hello, world!")
+
+ # append it to the _Messages
+ assert repr(msg) == '<HelloWorld code="1" description="Greeting the world">Hello, world!</HelloWorld>\n'
+ msgs.append(msg)
+ assert len(msgs) == 1
+
+ # use _Messages as list of Message objects
+ for m in msgs:
+ assert isinstance(m, Message)
+
+ # remove it
+ msgs.remove(msg)
+ assert len(msgs) == 0
+
+ # ok append it again ...
+ msgs.append(msg)
+ assert len(msgs) == 1
+ # get it back via get(...) and the key tuple (type, code)
+ assert id(msgs.get("HelloWorld", 1)) == id(msg)
+
+ # delete Message via remove and the (type,code) tuple
+ msgs.remove("HelloWorld", 1)
+ assert msgs.get("HelloWorld", 1) is None
+ assert len(msgs) == 0
+
+ # short version of adding/setting/resetting a new Message
+ msgs["HelloWorld", 2] = "Greeting the world in German", "Hallo, Welt!"
+ assert len(msgs) == 1
+ assert msgs["HelloWorld", 2] == (
+ "Greeting the world in German", "Hallo, Welt!")
+
+ msgs["HelloWorld", 2] = "Greeting the world in German", "Huhu, Welt!"
+ assert len(msgs) == 1
+ assert msgs["HelloWorld", 2] == (
+ "Greeting the world in German", "Huhu, Welt!")
+ del msgs["HelloWorld", 2]
+ assert msgs.get("HelloWorld", 2) is None
+
+ # this Message has no code and no description (make easy things easy...)
+ msgs["HelloWorld"] = "Hello!"
+ assert msgs["HelloWorld"] == "Hello!"
+
+
+def test_deepcopy():
+ """Test whether deepcopy of _Messages objects doesn't mess up
+ contained Messages objects.
+
+ """
+ msgs = db.common.models._Messages()
+ msg = db.Message(type="bla", code=1234, description="desc", body="blabla")
+ msgs.append(msg)
+ msg_copy = deepcopy(msgs)[0]
+
+ # make sure type is string-like (formerly caused problems)
+ assert hasattr(msg_copy.type, "lower")
+ assert msg_copy.type == msg.type
+ assert msg_copy.code == msg.code
+ assert msg_copy.description == msg.description
+ assert msg_copy.body == msg.body
+
+
+def test_deepcopy_clear_server():
+
+ msgs = db.common.models._Messages()
+ msg = db.Message(type="bla", code=1234, description="desc", body="blabla")
+ err_msg = db.Message(type="Error", code=1357, description="error")
+ msgs.extend([msg, err_msg])
+ copied_msgs = deepcopy(msgs)
+
+ assert len(copied_msgs) == 2
+ assert copied_msgs.get("Error", err_msg.code).code == err_msg.code
+ assert copied_msgs.get("bla", msg.code).code == msg.code
+
+ # Only the error should be removed
+ copied_msgs.clear_server_messages()
+ assert len(copied_msgs) == 1
+ assert copied_msgs[0].code == msg.code
diff --git a/unittests/test_property.py b/unittests/test_property.py
index 8b2deeb4b06cfbf3d42881201307a1e2122124e3..752ee01f0eafef14dbffd1e62c99d1c816c45d05 100644
--- a/unittests/test_property.py
+++ b/unittests/test_property.py
@@ -84,3 +84,8 @@ def test_get_property_with_entity():
p = Property(id=1234)
r.add_property(id=1234, value="bla")
assert r.get_property(p).value == "bla"
+
+
+def test_selected_reference_list():
+ assert len(testrecord.get_property("Conductor").value) == 1
+ assert isinstance(testrecord.get_property("Conductor").value[0], Entity)
diff --git a/unittests/test_query.py b/unittests/test_query.py
new file mode 100644
index 0000000000000000000000000000000000000000..12622ea486dda717ca1fbc1255510575c5e0c8e6
--- /dev/null
+++ b/unittests/test_query.py
@@ -0,0 +1,48 @@
+# -*- encoding: utf-8 -*-
+#
+# ** header v3.0
+# This file is a part of the CaosDB Project.
+#
+# Copyright (C) 2021 Indiscale GmbH <info@indiscale.com>
+# Copyright (C) 2021 Timm Fitschen <f.fitschen@indiscale.com>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see <https://www.gnu.org/licenses/>.
+#
+# ** end header
+#
+from lxml import etree
+import caosdb as db
+
+
+def test_query_parsing():
+ s = '<Query string="FIND bla" results="0" cached="true" etag="asdf"/>'
+ q = db.Query(etree.fromstring(s))
+ assert q.q == "FIND bla"
+ assert q.results == 0
+ assert q.cached is True
+ assert q.etag == "asdf"
+
+ s = '<Query string="COUNT bla" results="1" cached="false" etag="asdf"/>'
+ q = db.Query(etree.fromstring(s))
+ assert q.q == "COUNT bla"
+ assert q.results == 1
+ assert q.cached is False
+ assert q.etag == "asdf"
+
+ s = '<Query string="COUNT blub" results="4"/>'
+ q = db.Query(etree.fromstring(s))
+ assert q.q == "COUNT blub"
+ assert q.results == 4
+ assert q.cached is False
+ assert q.etag is None
diff --git a/unittests/test_record.py b/unittests/test_record.py
index 3b4eb4bbd34110706f7b0b1e88daac6c070dc1c6..c08a3eb1605d25ce4a9f142895e50647fe02cc3a 100644
--- a/unittests/test_record.py
+++ b/unittests/test_record.py
@@ -6,6 +6,8 @@
# Copyright (C) 2018 Research Group Biomedical Physics,
# Max-Planck-Institute for Dynamics and Self-Organization Göttingen
# Copyright (C) 2019 Henrik tom Wörden
+# Copyright (C) 2020 Timm Fitschen <t.fitschen@indiscale.com>
+# Copyright (C) 2020 IndiScale GmbH <info@indiscale.com>
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as
@@ -24,9 +26,7 @@
#
"""Tests for the Record class."""
# pylint: disable=missing-docstring
-import unittest
-
-from caosdb import Entity, Record
+from caosdb import Entity, Record, RecordType
def test_is_entity():
@@ -39,11 +39,42 @@ def test_role():
assert record.role == "Record"
-class TestRecord(unittest.TestCase):
- def test_property_access(self):
- rec = Record()
- rec.add_property("Prop")
- assert rec.get_property("Pop") is None
- assert rec.get_property("Prop") is not None
- assert rec.get_property("prop") is not None
- assert rec.get_property("prOp") is not None
+def test_property_access():
+ rec = Record(id=123)
+ rec.add_property("Prop")
+ assert rec.get_property("Pop") is None
+ assert rec.get_property("Prop") is not None
+ assert rec.get_property("prop") is not None
+ assert rec.get_property("prOp") is not None
+
+
+def test_get_parent_by_name():
+ rec = Record(id="123")
+ rec.add_parent(name="Test")
+ assert rec.get_parent(None) is None
+ assert rec.get_parent("Not existing") is None
+ assert rec.get_parent("Test") is not None
+ assert rec.get_parent("test") is not None
+ assert rec.get_parent("tEsT") is not None
+
+
+def test_get_parent_by_id():
+ rec = Record(id="123")
+ rec.add_parent(234)
+ assert rec.get_parent(None) is None
+ assert rec.get_parent(234) is not None
+
+
+def test_get_parent_by_entity():
+ rec = Record(id="123")
+ rt = RecordType("Test", id=234)
+ rec.add_parent(rt)
+ assert rec.get_parent(rt) is not None
+
+ rec = Record()
+ rec.add_parent(234)
+ assert rec.get_parent(rt) is not None
+
+ rec = Record()
+ rec.add_parent("Test")
+ assert rec.get_parent(rt) is not None
diff --git a/unittests/test_record.xml b/unittests/test_record.xml
index 5567e59050fdbcd07ce9b13cdc640c7bccf6c165..018c747c11027a2c3996c65d1deab1d18514e17b 100644
--- a/unittests/test_record.xml
+++ b/unittests/test_record.xml
@@ -1,4 +1,5 @@
<Record id="171179">
+ <Version id="version-str" date="2019-04-02T12:22:34.837UTC"/>
<TransactionBenchmark></TransactionBenchmark>
<Parent description="Experiment for the LEAP project." id="163454" name="LeapExperiment"/>
<Property datatype="TEXT" description="A unique identifier for experiments" flag="inheritance:FIX" id="163414" importance="FIX" name="experimentId">KCdSYWJiaXQnLCAnZXhfdml2bycsICdzeW5jaHJvbml6YXRpb25fbWFwJywgTm9uZSwgJzIwMTctMDgtMTEnLCBOb25lLCBOb25lKQ==</Property>
@@ -327,4 +328,10 @@
<Value>45531</Value>
<Value>45532</Value>
</Property>
+ <Property datatype="LIST<Person>" description="DESCRIBE ME!" id="1634561234" importance="FIX" name="Conductor">
+ <Value>
+ <Record id="23456543">
+ </Record>
+ </Value>
+ </Property>
</Record>
diff --git a/unittests/test_record_type.py b/unittests/test_record_type.py
index c105220c53579537994648b2f9bae5b4d378be13..f31c56decfc394211940296babc83200a470cc8a 100644
--- a/unittests/test_record_type.py
+++ b/unittests/test_record_type.py
@@ -22,16 +22,18 @@
# ** end header
#
"""Tests for the RecordType class."""
-# pylint: disable=missing-docstring
-from nose.tools import (assert_is_not_none as there, assert_true as tru,
- assert_equal as eq)
from caosdb import Entity, RecordType, configure_connection
from caosdb.connection.mockup import MockUpServerConnection
+# pylint: disable=missing-docstring
+from nose.tools import assert_equal as eq
+from nose.tools import assert_is_not_none as there
+from nose.tools import assert_true as tru
def setup_module():
there(RecordType)
configure_connection(url="unittests", username="testuser",
+ password_method="plain",
password="testpassword", timeout=200,
implementation=MockUpServerConnection)
diff --git a/unittests/test_server_side_scripting.py b/unittests/test_server_side_scripting.py
new file mode 100644
index 0000000000000000000000000000000000000000..1fb24d7e40bb843391a971c5f69680b541e1de0e
--- /dev/null
+++ b/unittests/test_server_side_scripting.py
@@ -0,0 +1,122 @@
+# -*- coding: utf-8 -*-
+#
+# ** header v3.0
+# This file is a part of the CaosDB Project.
+#
+# Copyright (C) 2020 Timm Fitschen <t.fitschen@indiscale.com>
+# Copyright (C) 2020 IndiScale GmbH <info@indiscale.com>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see <https://www.gnu.org/licenses/>.
+#
+# ** end header
+#
+
+import json
+from urllib.parse import parse_qs
+from unittest.mock import Mock
+from caosdb.utils import server_side_scripting as sss
+from caosdb.connection.mockup import MockUpServerConnection, MockUpResponse
+from caosdb import configure_connection
+
+_REMOVE_FILES_AFTERWARDS = []
+
+
+def setup_module():
+ c = configure_connection(password_method="unauthenticated",
+ implementation=MockUpServerConnection)
+ xml = ('<Response><script code="{code}">'
+ ' <call>{call}</call>'
+ ' <stdout>{stdout}</stdout>'
+ ' <stderr>{stderr}</stderr>'
+ '</script></Response>')
+
+ def scripting_resource(**kwargs):
+ assert kwargs["path"] == "scripting"
+ content_type = kwargs["headers"]["Content-Type"]
+
+ if content_type.startswith("multipart/form-data; boundary"):
+ parts = kwargs["body"]
+ stdout = []
+ for part in parts:
+ if hasattr(part, "decode"):
+ stdout.append(part.decode("utf-8"))
+ else:
+ stdout.append(part)
+ stdout = json.dumps(stdout)
+ else:
+ assert content_type == "application/x-www-form-urlencoded"
+ stdout = json.dumps(parse_qs(kwargs["body"].decode("utf-8"),
+ encoding="utf-8"))
+ scripting_response = xml.format(code="123",
+ call="call string",
+ stdout=stdout,
+ stderr="stderr string")
+ return MockUpResponse(200, {}, scripting_response)
+ c._delegate_connection.resources.append(scripting_resource)
+
+
+def teardown_module():
+ from os import remove
+ from os.path import exists, isdir
+ from shutil import rmtree
+ for obsolete in _REMOVE_FILES_AFTERWARDS:
+ if exists(obsolete):
+ if isdir(obsolete):
+ rmtree(obsolete)
+ else:
+ remove(obsolete)
+
+
+def test_run_server_side_script():
+ assert type(sss.run_server_side_script).__name__ == "function"
+ r = sss.run_server_side_script("cat", "/etc/passwd", files=None,
+ option1="val1")
+ assert r.call == "call string"
+ assert r.code == 123
+ assert r.stderr == "stderr string"
+
+ form = json.loads(r.stdout)
+ assert form["call"] == ["cat"]
+ assert form["-p0"] == ["/etc/passwd"]
+ assert form["-Ooption1"] == ["val1"]
+
+
+def test_run_server_side_script_with_file():
+ _REMOVE_FILES_AFTERWARDS.append("test_file.txt")
+ with open("test_file.txt", "w") as f:
+ f.write("this is a test")
+
+ assert type(sss.run_server_side_script).__name__ == "function"
+ r = sss.run_server_side_script("cat", "/etc/passwd",
+ files={"file1": "test_file.txt"},
+ option1="val1")
+ assert r.call == "call string"
+ assert r.code == 123
+ assert r.stderr == "stderr string"
+
+ parts = json.loads(r.stdout)
+ print(parts)
+ assert 'name="call"' in parts[0]
+ assert "\r\n\r\ncat\r\n" in parts[0]
+
+ assert 'name="-Ooption1"' in parts[1]
+ assert "\r\n\r\nval1\r\n" in parts[1]
+
+ assert 'name="-p0"' in parts[2]
+ assert "\r\n\r\n/etc/passwd\r\n" in parts[2]
+
+ assert 'name="file1"' in parts[3]
+ assert 'filename="test_file.txt"' in parts[3]
+
+ assert parts[4] == "this is a test"
diff --git a/unittests/test_state.py b/unittests/test_state.py
new file mode 100644
index 0000000000000000000000000000000000000000..202c7a02af3db28434406626e5164def46febed7
--- /dev/null
+++ b/unittests/test_state.py
@@ -0,0 +1,77 @@
+import pytest
+import caosdb as db
+from caosdb import State, Transition
+from caosdb.common.models import parse_xml, ACL
+from lxml import etree
+
+
+def test_state_xml():
+ state = State(model="model1", name="state1")
+ xml = etree.tostring(state.to_xml())
+
+ assert xml == b'<State name="state1" model="model1"/>'
+ state = State.from_xml(etree.fromstring(xml))
+ assert state.name == "state1"
+ assert state.model == "model1"
+
+ assert xml == etree.tostring(state.to_xml())
+
+
+def test_entity_xml():
+ r = db.Record()
+ assert r.state is None
+ r.state = State(model="model1", name="state1")
+
+ xml = etree.tostring(r.to_xml())
+ assert xml == b'<Record><State name="state1" model="model1"/></Record>'
+
+ r = parse_xml(xml)
+ assert r.state == State(model="model1", name="state1")
+
+
+def test_description():
+ xml = b'<State name="state1" model="model1"/>'
+ state = State.from_xml(etree.fromstring(xml))
+ assert state.description is None
+
+ with pytest.raises(AttributeError):
+ state.description = "test"
+
+ xml = b'<State name="state1" model="model1" description="test2"/>'
+ state = State.from_xml(etree.fromstring(xml))
+ assert state.description == "test2"
+
+
+def test_id():
+ xml = b'<State name="state1" model="model1"/>'
+ state = State.from_xml(etree.fromstring(xml))
+ assert state.id is None
+
+ with pytest.raises(AttributeError):
+ state.id = "2345"
+
+ xml = b'<State name="state1" model="model1" id="1234"/>'
+ state = State.from_xml(etree.fromstring(xml))
+ assert state.id == "1234"
+
+
+def test_create_state_acl():
+ acl = ACL()
+ acl.grant(role="role1", permission="DO:IT")
+ acl.grant(role="?OWNER?", permission="DO:THAT")
+ state_acl = State.create_state_acl(acl)
+ assert state_acl.get_permissions_for_role("?STATE?role1?") == {"DO:IT"}
+ assert state_acl.get_permissions_for_role("?STATE??OWNER??") == {"DO:THAT"}
+
+
+def test_transitions():
+ xml = b'<State name="state1" model="model1"/>'
+ state = State.from_xml(etree.fromstring(xml))
+ assert state.transitions is None
+
+ with pytest.raises(AttributeError):
+ state.transitions = []
+
+ xml = b'<State name="state1" model="model1" id="1234"><Transition name="t1"><FromState name="state1"/><ToState name="state2"/></Transition></State>'
+ state = State.from_xml(etree.fromstring(xml))
+ assert state.transitions == set([Transition(name="t1", from_state="state1", to_state="state2")])
diff --git a/unittests/test_utils.py b/unittests/test_utils.py
index f308445ed1a06331a54ebf67411dc154836557ca..42d18ba06eb7516bb318de54cb537f548cfe9081 100644
--- a/unittests/test_utils.py
+++ b/unittests/test_utils.py
@@ -24,7 +24,6 @@
"""Tests for caosdb.common.utils."""
from __future__ import unicode_literals
from lxml.etree import Element
-from nose.tools import assert_equals as eq
from caosdb.common.utils import xml2str
@@ -32,4 +31,4 @@ def test_xml2str():
name = 'Björn'
element = Element(name)
serialized = xml2str(element)
- eq(serialized, "<Björn/>\n")
+ assert serialized == "<Björn/>\n"
diff --git a/unittests/test_versioning.py b/unittests/test_versioning.py
new file mode 100644
index 0000000000000000000000000000000000000000..5047069ca17b573b8b54dcaab984419083d06859
--- /dev/null
+++ b/unittests/test_versioning.py
@@ -0,0 +1,182 @@
+# -*- coding: utf-8 -*-
+#
+# ** header v3.0
+# This file is a part of the CaosDB Project.
+#
+# Copyright (C) 2020 Timm Fitschen <t.fitschen@indiscale.com>
+# Copyright (C) 2020 IndiScale GmbH <info@indiscale.com>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see <https://www.gnu.org/licenses/>.
+#
+# ** end header
+#
+
+from __future__ import absolute_import
+from caosdb import Record
+from caosdb.common.utils import xml2str
+from caosdb.common.versioning import Version
+from .test_property import testrecord
+from lxml import etree
+
+
+def test_constructor():
+ v = Version(id="1234abcd", date="2020-01-01T20:15:00.000UTC",
+ username="testuser", realm="CaosDB", is_head=True,
+ predecessors=[Version(id="2345abdc",
+ date="2020-01-01T20:00:00.000UTC")],
+ successors=[Version(id="3465abdc",
+ date="2020-01-01T20:30:00.000UTC")])
+ assert v.id == "1234abcd"
+ assert v.date == "2020-01-01T20:15:00.000UTC"
+ assert v.username == "testuser"
+ assert v.realm == "CaosDB"
+ assert v.is_head is True
+ assert isinstance(v.predecessors, list)
+ assert isinstance(v.predecessors[0], Version)
+ assert isinstance(v.successors, list)
+ assert isinstance(v.successors[0], Version)
+
+ return v
+
+
+def test_to_xml():
+ v = test_constructor()
+ xmlstr = xml2str(v.to_xml())
+ assert xmlstr == ('<Version id="{i}" date="{d}" username="{u}" realm="{r}"'
+ ' head="{h}">\n'
+ ' <Predecessor id="{pi}" date="{pd}"/>\n'
+ ' <Successor id="{si}" date="{sd}"/>\n'
+ '</Version>\n').format(i=v.id, d=v.date,
+ u=v.username, r=v.realm,
+ h=str(v.is_head).lower(),
+ pi=v.predecessors[0].id,
+ pd=v.predecessors[0].date,
+ si=v.successors[0].id,
+ sd=v.successors[0].date)
+
+ xmlstr2 = xml2str(v.to_xml(tag="OtherVersionTag"))
+ assert xmlstr2 == ('<OtherVersionTag id="{i}" date="{d}" username="{u}" '
+ 'realm="{r}" head="{h}">\n'
+ ' <Predecessor id="{pi}" date="{pd}"/>\n'
+ ' <Successor id="{si}" date="{sd}"/>\n'
+ '</OtherVersionTag>\n'
+ ).format(i=v.id, d=v.date, u=v.username, r=v.realm,
+ h=str(v.is_head).lower(),
+ pi=v.predecessors[0].id,
+ pd=v.predecessors[0].date,
+ si=v.successors[0].id, sd=v.successors[0].date)
+
+
+def test_equality():
+ v = test_constructor()
+ assert hash(v) == hash(v)
+ v2 = test_constructor()
+ assert hash(v) == hash(v2)
+ assert v == v2
+
+ v = Version()
+ v2 = Version()
+ assert hash(v) == hash(v2)
+ assert v == v2
+
+ v = Version(id="123")
+ v2 = Version(id="123")
+ v3 = Version(id="2345")
+ assert hash(v) == hash(v2)
+ assert v == v2
+ assert hash(v) != hash(v3)
+ assert v != v3
+
+ v = Version(id="123", date="2345", predecessors=None)
+ v2 = Version(id="123", date="2345", predecessors=[])
+ v3 = Version(id="123", date="Another date", predecessors=[])
+ assert hash(v) == hash(v2)
+ assert v == v2
+ assert hash(v) != hash(v3)
+ assert v != v3
+
+ v = Version(id="123", date="2345", predecessors=[Version(id="bla")])
+ v2 = Version(id="123", date="2345", predecessors=[Version(id="bla")])
+ v3 = Version(id="123", date="2345", predecessors=[Version(id="blub")])
+ v4 = Version(id="123", date="2345", predecessors=[Version(id="bla"),
+ Version(id="bla")])
+ assert hash(v) == hash(v2)
+ assert v == v2
+ assert hash(v) != hash(v3)
+ assert v != v3
+ assert hash(v) != hash(v4)
+ assert v != v4
+
+ v = Version(id="123", date="2345", predecessors=[Version(id="bla")],
+ successors=[Version(id="234")])
+ v2 = Version(id="123", date="2345", predecessors=[Version(id="bla")],
+ successors=[Version(id="234")])
+ v3 = Version(id="123", date="2345", predecessors=[Version(id="bla")],
+ successors=[Version(id="bluup")])
+ assert hash(v) == hash(v2)
+ assert v == v2
+ assert hash(v) != hash(v3)
+ assert v != v3
+
+
+def test_from_xml():
+ v = test_constructor()
+ xml = v.to_xml()
+
+ v2 = Version.from_xml(xml)
+
+ assert hash(v) == hash(v2)
+ assert v == v2
+ assert str(v) == str(v2)
+
+
+def test_version_deserialization():
+ assert testrecord.version == Version(id="version-str",
+ date="2019-04-02T12:22:34.837UTC")
+
+
+def test_version_serialization():
+ r = Record()
+ r.version = Version(id="test-version", date="asdfsadf")
+
+ # <Record><Version id="test-version" date="asdfsadf"/></Record>
+ assert "test-version" == r.to_xml().xpath("/Record/Version/@id")[0]
+ assert "asdfsadf" == r.to_xml().xpath("/Record/Version/@date")[0]
+
+
+def test_get_history():
+ xml_str = """
+ <Version id="vid6" username="user1" realm="Realm1" date="date6" completeHistory="true">
+ <Predecessor id="vid5" username="user1" realm="Realm1" date="date5">
+ <Predecessor id="vid4" username="user1" realm="Realm1" date="date4">
+ <Predecessor id="vid3" username="user1" realm="Realm1" date="date3">
+ <Predecessor id="vid2" username="user1" realm="Realm1" date="date2">
+ <Predecessor id="vid1" username="user1" realm="Realm1" date="date1" />
+ </Predecessor>
+ </Predecessor>
+ </Predecessor>
+ </Predecessor>
+ <Successor id="vid7" username="user1" realm="Realm1" date="date7">
+ <Successor id="vid8" username="user1" realm="Realm1" date="date8">
+ <Successor id="vid9" username="user1" realm="Realm1" date="date9">
+ <Successor id="vid10" username="user1" realm="Realm1" date="date10" />
+ </Successor>
+ </Successor>
+ </Successor>
+ </Version>"""
+ version = Version.from_xml(etree.fromstring(xml_str))
+ assert version.is_complete_history is True
+ assert version.get_history() == [Version(id=f"vid{i+1}", date=f"date{i+1}",
+ username="user1", realm="Realm1")
+ for i in range(10)]