diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index b45dde5de1e4cafdbc02a15c4b775b763d5447bd..72b4381aebcaf5edd16c10c479df6ca908128ec6 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -22,7 +22,7 @@
variables:
DEPLOY_REF: dev
- CI_REGISTRY_IMAGE: $CI_REGISTRY/caosdb/caosdb-pylib/testenv:latest
+ CI_REGISTRY_IMAGE: $CI_REGISTRY/caosdb/src/caosdb-pylib/testenv:latest
# When using dind, it's wise to use the overlayfs driver for
# improved performance.
@@ -47,7 +47,7 @@ pylint:
tags: [ docker ]
stage: linting
script:
- - pylint3 --unsafe-load-any-extension=y -d all -e E,F src/caosdb/common
+ - pylint --unsafe-load-any-extension=y -d all -e E,F src/caosdb/common
allow_failure: true
@@ -65,7 +65,7 @@ trigger_build:
stage: deploy
script:
- /usr/bin/curl -X POST
- -F token=$DEPLOY_TRIGGER_TOKEN
+ -F token=$CI_JOB_TOKEN
-F "variables[F_BRANCH]=$CI_COMMIT_REF_NAME"
-F "variables[PYLIB]=$CI_COMMIT_REF_NAME"
-F "variables[TriggerdBy]=PYLIB"
@@ -92,15 +92,13 @@ build-testenv:
- docker push $CI_REGISTRY_IMAGE
# Build the sphinx documentation and make it ready for deployment by Gitlab Pages
-# documentation:
-# stage: deploy
-
# Special job for serving a static website. See https://docs.gitlab.com/ee/ci/yaml/README.html#pages
-pages:
+pages_prepare: &pages_prepare
+ tags: [ cached-dind ]
stage: deploy
only:
- # TODO this should be for master only, once releases are more regularly
- - dev
+ refs:
+ - /^release-.*$/i
script:
- echo "Deploying"
- make doc
@@ -108,3 +106,8 @@ pages:
artifacts:
paths:
- public
+pages:
+ <<: *pages_prepare
+ only:
+ refs:
+ - main
diff --git a/.gitlab/merge_request_templates/Default.md b/.gitlab/merge_request_templates/Default.md
index 77a95da1cc40c815e4952a1283d345af56e80461..e43435cb31f415fc4f9c8447983b411627612ad7 100644
--- a/.gitlab/merge_request_templates/Default.md
+++ b/.gitlab/merge_request_templates/Default.md
@@ -25,6 +25,7 @@ guidelines](https://gitlab.com/caosdb/caosdb/-/blob/dev/REVIEW_GUIDELINES.md)
- [ ] All automated tests pass
- [ ] Reference related Issues
- [ ] Up-to-date CHANGELOG.md
+- [ ] Add type hints in created/changed code
- [ ] Annotations in code (Gitlab comments)
- Intent of new code
- Problems with old code
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 7e190313f02f94d31bb3a0e9f0310eed7b729ea4..9d211b7e2ad48363cc414f2bd02263bc30c209e5 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -5,36 +5,90 @@ All notable changes to this project will be documented in this file.
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
-## [Unreleased] ##
+## [0.6.1] - 2021-12-03 ##
+
+### Fixed ###
+
+- #50 keyring can be used as password input method again
+* #81 compare_entities from apiutils does not compare entity values
+
+## [0.6.0] - 2021-10-19 ##
### Added ###
+- It is possible now to supply a password for caosdb_admin on the command line
+ and also activate the user directly using "-c".
+* Added examples for complex data models to documentation
+* extended apiutils with `resolve_reference(Property)`
+* is_reference function for Properties
+* function `retrieve_substructure` that recursively adds connected entities.
+
### Changed ###
+* Retrievals of entities where the class does not match the entity role raise
+ a ValueError now. See
+ [here](https://gitlab.indiscale.com/caosdb/src/caosdb-pylib/-/issues/66) for more
+ information. Updating a role is now being done by setting the `Entity.role`
+ to the new role (as string).
+* Entity.add_property and Entity.add_parent do not accept `**kwargs`-style
+ keywords anymore. Formerly known keywords have been refactored into named
+ parameters.
+* [#35](https://gitlab.com/caosdb/caosdb-pylib/-/issues/35) Loggers now use the
+ name of the unit where they are called instead of a static name
+
### Deprecated ###
+* `id_query(ids)` in apiutils (to be removed with >=0.5.4)
+* The whole yamlapi with the following functions (to be removed with >=0.5.4):
+ * `append_sublist`
+ * `kv_to_xml`
+ * `dict_to_xml`
+ * `yaml_to_xml`
+ * `process`
+ * `yaml_file_to_xml`
+ * `insert_yaml_file`
+
### Removed ###
### Fixed ###
+* #60 Unintuitive behavior of `Entity.role` after a `Entity(id).retrieve()`
+ Originally the role was `None`. The correct role is present now.
+* #53 Documentation of inheritance
+* #38 Dependencies in chunk-deletion of containers
+
### Security ###
-## [0.5.1] - 2021-02-12 ##
+## [0.5.2] - 2021-06-03 ##
### Added ###
+* Entity State support (experimental, no StateModel support yet). See the
+ `caosdb.State` class for more information.
+* `etag` property for the `caosdb.Query` class. The etag allows to debug the
+ caching and to decide whether the server has changed between queries.
+* function `_read_config_files` to read `pycaosdb.ini` files from different paths.
+
### Changed ###
+* Updated error-handling tutorial in documentation to reflect the new
+ error classes
+
### Deprecated ###
### Removed ###
### Fixed ###
-
-* #43 - Error with `execute_query` when server doesn't support query caching.
+* #45 - test_config_ini_via_envvar
### Security ###
+## [0.5.1] - 2021-02-12 ##
+
+### Fixed ###
+
+* #43 - Error with `execute_query` when server doesn't support query caching.
+
## [0.5.0] - 2021-02-11 ##
### Added ###
@@ -67,19 +121,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
`QueryNotUniqueError` if no or more than one possible candidate is
found, respectively.
-### Deprecated ###
-
### Removed ###
* Dynamic exception type `EntityMultiError`.
* `get_something` functions from all error object in `exceptions.py`
* `AmbiguityException`
-
-### Fixed ###
-
-### Security ###
-
## [0.4.1] - 2021-02-10 ##
### Added ###
@@ -89,18 +136,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
entity.
* Automated documentation builds: `make doc`
-### Changed ###
-
-### Deprecated ###
-
-### Removed ###
-
### Fixed ###
* deepcopy of `_Messages` objects
-### Security ###
-
## [0.4.0] - 2020-07-17##
### Added ###
diff --git a/README.md b/README.md
index e44702a3a22c28af986d641b5d7e454f915326c8..04b34cbc07c98e73740b13200ed83fe067af99d2 100644
--- a/README.md
+++ b/README.md
@@ -1,20 +1,49 @@
-# Welcome
+
+# README
+
+## Welcome
This is the **CaosDB Python Client Library** repository and a part of the
CaosDB project.
-# Setup
+## Setup
Please read the [README_SETUP.md](README_SETUP.md) for instructions on how to
setup this code.
-# Further Reading
+## Further Reading
+
+Please refer to the [official documentation](https://docs.indiscale.com/caosdb-pylib/) for more information.
+
+## Contributing
+
+Thank you very much to all contributers—[past, present](https://gitlab.com/caosdb/caosdb/-/blob/dev/HUMANS.md), and prospective ones.
-Please refer to the [official gitlab repository of the CaosDB
-project](https://gitlab.com/caosdb/caosdb) for more information.
+### Code of Conduct
-# License
+By participating, you are expected to uphold our [Code of Conduct](https://gitlab.com/caosdb/caosdb/-/blob/dev/CODE_OF_CONDUCT.md).
+
+### How to Contribute
+
+* You found a bug, have a question, or want to request a feature? Please
+[create an issue](https://gitlab.com/caosdb/caosdb-pylib/-/issues).
+* You want to contribute code?
+ * **Forking:** Please fork the repository and create a merge request in GitLab and choose this repository as
+ target. Make sure to select "Allow commits from members who can merge the target branch" under
+ Contribution when creating the merge request. This allows our team to work with you on your
+ request.
+ * **Code style:** This project adhers to the PEP8 recommendations, you can test your code style
+ using the `autopep8` tool (`autopep8 -i -r ./`). Please write your doc strings following the
+ [NumpyDoc](https://numpydoc.readthedocs.io/en/latest/format.html) conventions.
+* If you have a suggestion for the [documentation](https://docs.indiscale.com/caosdb-pylib/),
+the preferred way is also a merge request as describe above (the documentation resides in `src/doc`).
+However, you can also create an issue for it.
+* You can also contact us at **info (AT) caosdb.de** and join the
+ CaosDB community on
+ [#caosdb:matrix.org](https://matrix.to/#/!unwwlTfOznjEnMMXxf:matrix.org).
+
+## License
* Copyright (C) 2018 Research Group Biomedical Physics, Max Planck Institute
for Dynamics and Self-Organization Göttingen.
@@ -22,4 +51,3 @@ project](https://gitlab.com/caosdb/caosdb) for more information.
All files in this repository are licensed under a [GNU Affero General Public
License](LICENCE.md) (version 3 or later).
-
diff --git a/README_SETUP.md b/README_SETUP.md
index 2db73cfaec2f6aadfc7fa3742892d970d562c946..e58f934ceba176e4b5ba42239565f8e3bd48171a 100644
--- a/README_SETUP.md
+++ b/README_SETUP.md
@@ -11,6 +11,10 @@ typically be installed automatically):
- `PyYaml`
- `PySocks`
+Optional packages:
+- `keyring`
+- `jsonschema`
+
### How to install ###
#### Linux ####
@@ -30,12 +34,14 @@ packages you will ever need out of the box. If you prefer, you may also install
After installation, open an Anaconda prompt from the Windows menu and continue in the [Generic
installation](#generic-installation) section.
-#### iOS ####
+#### MacOS ####
-If there is no Python 3 installed yet, there are two main ways to obtain it: Either get the binary
-package from [python.org](https://www.python.org/downloads/) or, for advanced users, install via [Homebrew](https://brew.sh/). After installation
-from python.org, it is recommended to also update the TLS certificates for Python (this requires
-administrator rights for your user):
+If there is no Python 3 installed yet, there are two main ways to
+obtain it: Either get the binary package from
+[python.org](https://www.python.org/downloads/) or, for advanced
+users, install via [Homebrew](https://brew.sh/). After installation
+from python.org, it is recommended to also update the TLS certificates
+for Python (this requires administrator rights for your user):
```sh
# Replace this with your Python version number:
@@ -45,7 +51,8 @@ cd /Applications/Python\ 3.9/
sudo ./Install\ Certificates.command
```
-After these steps, you may continue with the [Generic installation](#generic-installation).
+After these steps, you may continue with the [Generic
+installation](#generic-installation).
#### Generic installation ####
@@ -66,6 +73,13 @@ cd caosdb-pylib
pip3 install --user .
```
+For installation of optional packages, install with an additional option, e.g. for
+validating with the caosdb json schema:
+
+```sh
+pip3 install --user .[jsonschema]
+```
+
## Configuration ##
The configuration is done using `ini` configuration files.
@@ -75,7 +89,7 @@ current working directory will be read additionally, if it exists.
Here, we will look at the most common configuration options. For a full and
comprehensive description please check out
-[pycaosdb.ini file](https://gitlab.com/caosdb/caosdb-pylib/-/blob/master/examples/pycaosdb.ini)
+[pycaosdb.ini file](https://gitlab.com/caosdb/caosdb-pylib/-/blob/main/examples/pycaosdb.ini)
You can download this file and use it as a starting point.
@@ -84,7 +98,7 @@ Typically, you need to change at least the `url` and `username` fields as requir
you do not know what to put there, but for the demo instances https://demo.indiscale.com, `username=admin`
and `password=caosdb` should work).
-### Authentication ##
+### Authentication ###
The default configuration (that your are asked for your password when ever a connection is created
can be changed by setting `password_method`:
@@ -109,7 +123,7 @@ The following illustrates the recommended options:
#password_method=keyring
```
-### SSL Certificate ##
+### SSL Certificate ###
In some cases (especially if you are testing CaosDB) you might need to supply
an SSL certificate to allow SSL encryption.
@@ -118,9 +132,9 @@ an SSL certificate to allow SSL encryption.
cacert=/path/to/caosdb.ca.pem
```
-### Further Settings ##
+### Further Settings ###
As mentioned above, a complete list of options can be found in the
-[pycaosdb.ini file](https://gitlab.com/caosdb/caosdb-pylib/-/blob/master/examples/pycaosdb.ini) in
+[pycaosdb.ini file](https://gitlab.com/caosdb/caosdb-pylib/-/blob/main/examples/pycaosdb.ini) in
the examples folder of the source code.
## Try it out ##
@@ -138,21 +152,20 @@ Out[2]: Connection to CaosDB with 501 Records.
Note: This setup will ask you for your password whenever a new connection is created. If you do not
like this, check out the "Authentication" section in the [configuration documentation](configuration.md).
-Now would be a good time to continue with the [tutorials](tutorials.html).
+Now would be a good time to continue with the [tutorials](tutorials/index).
## Run Unit Tests
tox
-## Code Formatting
-
-autopep8 -i -r ./
-
-## Documentation #
+## Documentation ##
Build documentation in `build/` with `make doc`.
-### Requirements ##
+### Requirements ###
- `sphinx`
- `sphinx-autoapi`
- `recommonmark`
+
+### Troubleshooting ###
+If the client is to be executed directly from the `/src` folder, an initial `.\setup.py install --user` must be called.
diff --git a/RELEASE_GUIDELINES.md b/RELEASE_GUIDELINES.md
index dd6d5af4c43e1eea17c70f29da837c905694fed4..e015b598117abdcd575cf17e2f095fec459a4c4c 100644
--- a/RELEASE_GUIDELINES.md
+++ b/RELEASE_GUIDELINES.md
@@ -24,9 +24,9 @@ guidelines of the CaosDB Project
variables and set `ISRELEASED` to `True`. Use the possibility to issue
pre-release versions for testing.
-5. Merge the release branch into the master branch.
+5. Merge the release branch into the main branch.
-6. Tag the latest commit of the master branch with `v<VERSION>`.
+6. Tag the latest commit of the main branch with `v<VERSION>`.
7. Delete the release branch.
@@ -35,8 +35,8 @@ guidelines of the CaosDB Project
9. Publish the release by executing `./release.sh` with uploads the caosdb
module to the Python Package Index [pypi.org](https://pypi.org).
-10. Merge the master branch back into the dev branch.
+10. Merge the main branch back into the dev branch.
-11. After the merge of master to dev, start a new development version by
+11. After the merge of main to dev, start a new development version by
setting `ISRELEASED` to `False` and by increasing at least the `MIRCO`
version in [setup.py](./setup.py)
diff --git a/examples/set_permissions.py b/examples/set_permissions.py
index dfc0a1510823a36d963f5d868052abb17b3fe12d..8162b11bfefb41b1bcdbc74b8e314f99a61d1a4e 100755
--- a/examples/set_permissions.py
+++ b/examples/set_permissions.py
@@ -25,15 +25,13 @@
As a result, only a specific user or group may access it.
-This script assumes that data similar to the demo server of IndiScale (at
-demo.indiscale.com) exists on the server specified in the pycaosdb.ini
-configuration.
+This script assumes that the user specified in the pycaosdb.ini
+configuration can create new entities.
"""
import caosdb as db
from caosdb import administration as admin
-import lxml
def assert_user_and_role():
@@ -50,27 +48,27 @@ out : tuple
"""
try:
human_user = admin._retrieve_user("jane")
- _activate_user("jane")
- except db.ResourceNotFoundError:
+ admin._update_user(name="jane", status="ACTIVE")
+ except db.HTTPResourceNotFoundError:
human_user = admin._insert_user(
"jane", password="Human_Rememberable_Password_1234", status="ACTIVE")
try:
alien_user = admin._retrieve_user("xaxys")
- _activate_user("xaxys")
- except db.ResourceNotFoundError:
+ admin._update_user(name="xaxys", status="ACTIVE")
+ except db.HTTPResourceNotFoundError:
alien_user = admin._insert_user("xaxys", password="4321_Syxax",
status="ACTIVE")
# At the moment, the return value is only "ok" for successful insertions.
try:
human_role = admin._retrieve_role("human")
- except db.ResourceNotFoundError:
+ except db.HTTPResourceNotFoundError:
human_role = admin._insert_role("human", "An Earthling.")
try:
alien_role = admin._retrieve_role("alien")
- except db.ResourceNotFoundError:
+ except db.HTTPResourceNotFoundError:
alien_role = admin._insert_role("alien", "An Extra-terrestrial.")
admin._set_roles("jane", ["human"])
@@ -80,24 +78,6 @@ out : tuple
("xaxys", list(admin._get_roles("xaxys"))))
-def _activate_user(user):
- """Set the user state to "ACTIVE" if necessary.
-
-Parameters
-----------
-user : str
- The user to activate.
-
-Returns
--------
-None
-
- """
- user_xml = lxml.etree.fromstring(admin._retrieve_user(user))
- if user_xml.xpath("User")[0].attrib["status"] != "ACTIVE":
- admin._update_user(user, status="ACTIVE")
-
-
def get_entities(count=1):
"""Retrieve one or more entities.
@@ -111,7 +91,7 @@ Returns
out : Container
A container of retrieved entities, the length is given by the parameter count.
"""
- cont = db.execute_query("FIND RECORD Guitar", flags={
+ cont = db.execute_query("FIND RECORD 'Human Food'", flags={
"P": "0L{n}".format(n=count)})
if len(cont) != count:
raise db.CaosDBException(
@@ -221,17 +201,36 @@ None
print("Retrieval of all entities was successfully denied.")
+def create_test_entities():
+ """Create some test entities.
+ After calling this function, there will be a RecordType "Human Food" with the corresponding Records
+ "Bread", "Tomatoes", and "Twinkies" inserted in the database.
+ """
+ rt = db.RecordType(
+ name="Human Food", description="Food that can be eaten only by humans").insert()
+ food = ("Bread", "Tomatoes", "Twinkies")
+
+ cont = db.Container()
+ for i in range(len(food)):
+ rec = db.Record(food[i])
+ rec.add_parent(name="Human Food")
+ cont.append(rec)
+
+ cont.insert()
+
+
def main():
"""The main function of this script."""
- db.connection.connection.get_connection()._login()
-
+ """Create some test entities"""
+ create_test_entities()
+ """Create new users"""
human, alien = assert_user_and_role()
-
- # public, private, undefined entities
+ """Load the newly created entities."""
entities = get_entities(count=3)
-
+ """Set permission for the entities (only humans are allowed to eat human food)"""
set_permission(human[1][0], alien[1][0], entities)
+ """Test the permissions"""
test_permission((human[0], "Human_Rememberable_Password_1234"),
(alien[0], "4321_Syxax"), entities)
diff --git a/setup.py b/setup.py
index d491a1d340796b9b4701b307003c3e62e75d6001..0eda58e813bb88e3d63b8ad064db362069328dd1 100755
--- a/setup.py
+++ b/setup.py
@@ -46,10 +46,10 @@ from setuptools import find_packages, setup
########################################################################
MAJOR = 0
-MINOR = 5
-MICRO = 2
+MINOR = 6
+MICRO = 1
PRE = "" # e.g. rc0, alpha.1, 0.beta-23
-ISRELEASED = False
+ISRELEASED = True
if PRE:
VERSION = "{}.{}.{}-{}".format(MAJOR, MINOR, MICRO, PRE)
@@ -159,11 +159,12 @@ def setup_package():
package_dir={'': 'src'},
install_requires=['lxml>=3.6.4',
'PyYaml>=3.12', 'future', 'PySocks>=1.6.7'],
- extras_require={'keyring': ['keyring>=13.0.0']},
+ extras_require={'keyring': ['keyring>=13.0.0'],
+ 'jsonschema': ['jsonschema==4.0.1']},
setup_requires=["pytest-runner>=2.0,<3dev"],
- tests_require=["pytest", "pytest-cov", "coverage>=4.4.2"],
+ tests_require=["pytest", "pytest-cov", "coverage>=4.4.2", "jsonschema==4.0.1"],
package_data={
- 'caosdb': ['cert/indiscale.ca.crt'],
+ 'caosdb': ['cert/indiscale.ca.crt', 'schema-pycaosdb-ini.yml'],
},
scripts=["src/caosdb/utils/caosdb_admin.py"]
)
diff --git a/src/caosdb/__init__.py b/src/caosdb/__init__.py
index b86fcd638c7321c2e0464b603fb736e3adfbafe3..7e06885fe495c1e8c4ccc99b7d0c0f8ff8c34b5b 100644
--- a/src/caosdb/__init__.py
+++ b/src/caosdb/__init__.py
@@ -37,7 +37,8 @@ from os.path import expanduser, join
import caosdb.apiutils
from caosdb.common import administration
from caosdb.common.datatype import (BOOLEAN, DATETIME, DOUBLE, FILE, INTEGER,
- REFERENCE, TEXT, LIST)
+ LIST, REFERENCE, TEXT)
+from caosdb.common.state import State, Transition
# Import of the basic API classes:
from caosdb.common.models import (ACL, ALL, FIX, NONE, OBLIGATORY, RECOMMENDED,
SUGGESTED, Container, DropOffBox, Entity,
@@ -45,15 +46,13 @@ from caosdb.common.models import (ACL, ALL, FIX, NONE, OBLIGATORY, RECOMMENDED,
Query, QueryTemplate, Record, RecordType,
delete, execute_query, get_global_acl,
get_known_permissions, raise_errors)
-from caosdb.configuration import configure, get_config
+from caosdb.configuration import _read_config_files, configure, get_config
from caosdb.connection.connection import configure_connection, get_connection
-from caosdb.version import version as __version__
from caosdb.exceptions import *
+try:
+ from caosdb.version import version as __version__
+except ModuleNotFoundError:
+ version = "uninstalled"
+ __version__ = version
-# read configuration these files
-
-if "PYCAOSDBINI" in environ:
- configure(expanduser(environ["PYCAOSDBINI"]))
-else:
- configure(expanduser('~/.pycaosdb.ini'))
-configure(join(getcwd(), "pycaosdb.ini"))
+_read_config_files()
diff --git a/src/caosdb/apiutils.py b/src/caosdb/apiutils.py
index 73074efc3057e0548c5abfd56ef3cf1ac9e9bf47..21a8cc7ca700a8b119786dbef082db286ead5d57 100644
--- a/src/caosdb/apiutils.py
+++ b/src/caosdb/apiutils.py
@@ -30,14 +30,15 @@ Some simplified functions for generation of records etc.
import sys
import tempfile
+import warnings
from collections.abc import Iterable
from subprocess import call
from caosdb.common.datatype import (BOOLEAN, DATETIME, DOUBLE, FILE, INTEGER,
REFERENCE, TEXT, is_reference)
from caosdb.common.models import (Container, Entity, File, Property, Query,
- Record, RecordType, get_config,
- execute_query)
+ Record, RecordType, execute_query,
+ get_config)
def new_record(record_type, name=None, description=None,
@@ -86,9 +87,19 @@ def new_record(record_type, name=None, description=None,
def id_query(ids):
- q = "FIND Entity with " + " OR ".join(["id={}".format(id) for id in ids])
+ warnings.warn("Please use 'create_id_query', which only creates"
+ "the string.", DeprecationWarning)
- return execute_query(q)
+ return execute_query(create_id_query(ids))
+
+
+def create_id_query(ids):
+ return "FIND ENTITY WITH " + " OR ".join(
+ ["ID={}".format(id) for id in ids])
+
+
+def retrieve_entity_with_id(eid):
+ return execute_query("FIND ENTITY WITH ID={}".format(eid), unique=True)
def retrieve_entities_with_ids(entities):
@@ -96,7 +107,9 @@ def retrieve_entities_with_ids(entities):
step = 20
for i in range(len(entities)//step+1):
- collection.extend(id_query(entities[i*step:(i+1)*step]))
+ collection.extend(
+ execute_query(
+ create_id_query(entities[i*step:(i+1)*step])))
return collection
@@ -504,9 +517,9 @@ def getOriginUrlIn(folder):
with open(t.name, "r") as t:
urlString = "Fetch URL:"
- for l in t.readlines():
- if urlString in l:
- return l[l.find(urlString) + len(urlString):].strip()
+ for line in t.readlines():
+ if urlString in line:
+ return line[line.find(urlString) + len(urlString):].strip()
return None
@@ -548,7 +561,23 @@ def getCommitIn(folder):
COMPARED = ["name", "role", "datatype", "description", "importance"]
-def compare_entities(old_entity, new_entity):
+def compare_entities(old_entity: Entity, new_entity: Entity):
+ """
+ Compare two entites.
+
+ Return a tuple of dictionaries, the first index belongs to additional information for old
+ entity, the second index belongs to additional information for new entity.
+
+ Additional information means in detail:
+ - Additional parents (a list under key "parents")
+ - Information about properties:
+ - Each property lists either an additional property or a property with a changed:
+ - ... datatype
+ - ... importance or
+ - ... value (not implemented yet)
+ In case of changed information the value listed under the respective key shows the
+ value that is stored in the respective entity.
+ """
olddiff = {"properties": {}, "parents": []}
newdiff = {"properties": {}, "parents": []}
@@ -603,13 +632,16 @@ def compare_entities(old_entity, new_entity):
newdiff["properties"][prop.name]["importance"] = \
new_entity.get_importance(prop.name)
- if ((prop.datatype is not None and
- matching[0].datatype is not None) and
- (prop.datatype != matching[0].datatype)):
+ if (prop.datatype != matching[0].datatype):
olddiff["properties"][prop.name]["datatype"] = prop.datatype
newdiff["properties"][prop.name]["datatype"] = \
matching[0].datatype
+ if (prop.value != matching[0].value):
+ olddiff["properties"][prop.name]["value"] = prop.value
+ newdiff["properties"][prop.name]["value"] = \
+ matching[0].value
+
if (len(newdiff["properties"][prop.name]) == 0
and len(olddiff["properties"][prop.name]) == 0):
newdiff["properties"].pop(prop.name)
@@ -687,6 +719,7 @@ def apply_to_ids(entities, func):
entities : list of Entity
func : function with one parameter.
"""
+
for entity in entities:
_apply_to_ids_of_entity(entity, func)
@@ -707,3 +740,28 @@ def _apply_to_ids_of_entity(entity, func):
else:
if prop.value is not None:
prop.value = func(prop.value)
+
+
+def resolve_reference(prop: Property):
+ """resolves the value of a reference property
+
+ The integer value is replaced with the entity object.
+ If the property is not a reference, then the function returns without
+ change.
+ """
+
+ if not prop.is_reference(server_retrieval=True):
+ return
+
+ if isinstance(prop.value, list):
+ referenced = []
+
+ for val in prop.value:
+ if isinstance(val, int):
+ referenced.append(retrieve_entity_with_id(val))
+ else:
+ referenced.append(val)
+ prop.value = referenced
+ else:
+ if isinstance(prop.value, int):
+ prop.value = retrieve_entity_with_id(prop.value)
diff --git a/src/caosdb/common/administration.py b/src/caosdb/common/administration.py
index 7f9336c7473ac759aab4461c84b708a3b9049d8b..dff461e7fb0ed5270119907bd4ad859503b3ce21 100644
--- a/src/caosdb/common/administration.py
+++ b/src/caosdb/common/administration.py
@@ -26,13 +26,12 @@
"""missing docstring."""
-from lxml import etree
-
from caosdb.common.utils import xml2str
from caosdb.connection.connection import get_connection
-from caosdb.exceptions import (HTTPClientError,
- HTTPForbiddenError,
- HTTPResourceNotFoundError)
+from caosdb.exceptions import (EntityDoesNotExistError, HTTPClientError,
+ HTTPForbiddenError, HTTPResourceNotFoundError,
+ ServerConfigurationException)
+from lxml import etree
def set_server_property(key, value):
@@ -53,9 +52,11 @@ def set_server_property(key, value):
None
"""
con = get_connection()
-
- con._form_data_request(method="POST", path="_server_properties",
- params={key: value}).read()
+ try:
+ con._form_data_request(method="POST", path="_server_properties",
+ params={key: value}).read()
+ except EntityDoesNotExistError:
+ raise ServerConfigurationException("Debug mode in server is probably disabled.") from None
def get_server_properties():
@@ -69,7 +70,11 @@ def get_server_properties():
The server properties.
"""
con = get_connection()
- body = con._http_request(method="GET", path="_server_properties").response
+ try:
+ body = con._http_request(method="GET", path="_server_properties").response
+ except EntityDoesNotExistError:
+ raise ServerConfigurationException("Debug mode in server is probably disabled.") from None
+
xml = etree.parse(body)
props = dict()
@@ -147,13 +152,22 @@ def _update_user(name, realm=None, password=None, status=None,
return con.put_form_data(entity_uri_segment="User/" + (realm + "/" + name if realm is not None else name), params=params, **kwargs).read()
except HTTPResourceNotFoundError as e:
e.msg = "User does not exist."
- raise
+ raise e
except HTTPForbiddenError as e:
e.msg = "You are not permitted to update this user."
- raise
+ raise e
except HTTPClientError as e:
if e.status == 409:
e.msg = "Entity does not exist."
+
+ if e.status == 422:
+ e.msg = """Maybe the password does not match the required standard?
+ The current requirements are:
+ - at least 8 characters
+ - at least 1 number
+ - at least 1 lower case character
+ - at least 1 upper case character
+ - at least 1 special character"""
raise
@@ -182,7 +196,13 @@ def _insert_user(name, password=None, status=None, email=None, entity=None, **kw
e.msg = "User name is already in use."
if e.status == 422:
- e.msg = "Maybe the password does not match the required standard?"
+ e.msg = """Maybe the password does not match the required standard?
+ The current requirements are:
+ - at least 8 characters
+ - at least 1 number
+ - at least 1 lower case character
+ - at least 1 upper case character
+ - at least 1 special character"""
raise e
diff --git a/src/caosdb/common/datatype.py b/src/caosdb/common/datatype.py
index eb8c1e4e0088f1924940a104ec3916b9d5d40f99..03ff6d023ab0d3005c37d56c65353c1a1072518e 100644
--- a/src/caosdb/common/datatype.py
+++ b/src/caosdb/common/datatype.py
@@ -45,6 +45,8 @@ def LIST(datatype):
def get_list_datatype(datatype):
""" returns the datatype of the elements in the list """
+ if not isinstance(datatype, str):
+ return None
match = re.match("LIST(<|<)(?P<datatype>.*)(>|>)", datatype)
if match is not None:
@@ -60,12 +62,27 @@ def is_list_datatype(datatype):
def is_reference(datatype):
- """ returns whether the value is a reference
+ """Returns whether the value is a reference
FILE and REFERENCE properties are examples, but also datatypes that are
- RecordTypes
+ RecordTypes.
+
+ Parameters
+ ----------
+ datatype : str
+ The datatype to check.
+
+ Returns
+ -------
+ bool
+ True if the datatype is a not base datatype or a list of a base datatype.
+ Otherwise False is returned.
"""
+ if datatype is None:
+ raise ValueError("Cannot decide whether datatype is reference if None"
+ " is supplied")
+
if datatype in [DOUBLE, BOOLEAN, INTEGER, TEXT, DATETIME]:
return False
elif is_list_datatype(datatype):
@@ -74,6 +91,41 @@ def is_reference(datatype):
return True
+def get_referenced_recordtype(datatype):
+ """Return the record type of the referenced datatype.
+
+ Raises
+ ------
+ ValueError
+ In cases where datatype is not a reference, the list does not have
+ a referenced record type or the datatype is a FILE.
+
+ Parameters
+ ----------
+ datatype : str
+ The datatype to check.
+
+ Returns
+ -------
+ str
+ String containing the name of the referenced datatype.
+ """
+
+ if not is_reference(datatype):
+ raise ValueError("datatype must be a reference")
+
+ if is_list_datatype(datatype):
+ datatype = get_list_datatype(datatype)
+ if datatype is None:
+ raise ValueError("list does not have a list datatype")
+
+ if datatype == FILE:
+ raise ValueError(
+ "FILE references are not considered references with a record type")
+
+ return datatype
+
+
def get_id_of_datatype(datatype):
""" returns the id of a Record Type
diff --git a/src/caosdb/common/models.py b/src/caosdb/common/models.py
index 9592e484654d3a610d5516cb7ca52c3df71682ba..ec45dec070d1ea731648fe8d45e44ba89e393f76 100644
--- a/src/caosdb/common/models.py
+++ b/src/caosdb/common/models.py
@@ -31,6 +31,7 @@ from __future__ import print_function, unicode_literals
import re
import sys
from builtins import str
+from copy import deepcopy
from functools import cmp_to_key
from hashlib import sha512
from os import listdir
@@ -40,25 +41,23 @@ from sys import hexversion
from tempfile import NamedTemporaryFile
from warnings import warn
-from caosdb.common.datatype import (BOOLEAN, DATETIME, DOUBLE, INTEGER, TEXT)
-from caosdb.common.versioning import Version
+from caosdb.common.datatype import (BOOLEAN, DATETIME, DOUBLE, INTEGER, TEXT,
+ is_list_datatype, is_reference)
+from caosdb.common.state import State
from caosdb.common.utils import uuid, xml2str
+from caosdb.common.versioning import Version
from caosdb.configuration import get_config
from caosdb.connection.connection import get_connection
from caosdb.connection.encode import MultipartParam, multipart_encode
-from caosdb.exceptions import (AmbiguousEntityError,
- AuthorizationError,
- CaosDBException, CaosDBConnectionError,
- ConsistencyError,
- EmptyUniqueQueryError,
+from caosdb.exceptions import (AmbiguousEntityError, AuthorizationError,
+ CaosDBConnectionError, CaosDBException,
+ ConsistencyError, EmptyUniqueQueryError,
EntityDoesNotExistError, EntityError,
- EntityHasNoDatatypeError,
- MismatchingEntitiesError,
- QueryNotUniqueError, TransactionError,
- UniqueNamesError,
+ EntityHasNoDatatypeError, HTTPURITooLongError,
+ MismatchingEntitiesError, QueryNotUniqueError,
+ TransactionError, UniqueNamesError,
UnqualifiedParentsError,
- UnqualifiedPropertiesError,
- HTTPURITooLongError)
+ UnqualifiedPropertiesError)
from lxml import etree
_ENTITY_URI_SEGMENT = "Entity"
@@ -112,6 +111,7 @@ class Entity(object):
self.name = name
self.description = description
self.id = id
+ self.state = None
@property
def version(self):
@@ -130,7 +130,10 @@ class Entity(object):
@role.setter
def role(self, role):
- self.__role = role
+ if role is not None and role.lower() == "entity":
+ self.__role = None
+ else:
+ self.__role = role
@property
def size(self):
@@ -339,54 +342,80 @@ class Entity(object):
return self
- def add_property(self, property=None, value=None, **kwargs): # @ReservedAssignment
+ def add_property(self, property=None, value=None, id=None, name=None, description=None, datatype=None,
+ unit=None, importance=None, inheritance=None): # @ReservedAssignment
"""Add a property to this entity.
The first parameter is meant to identify the property entity. So the method expects an instance of
Entity, an integer or a string here. The second parameter is the value of the new property. Any
- other named parameter may be passed by means of the **kwargs. Accepted keywords are:
+ other named parameter may be passed by means of the keywwords. Accepted keywords are:
id, name, description, importance, inheritance, datatype, and unit. Any other keyword will be
ignored right now. But that may change in the future.
- @param property: An identifying parameter (name, id or abstract property).
- @param value: The value of the new property.
- @param **kwargs: Any other specification for this property. Accepted keywords: id, name, description, importance, inheritance, datatype, and unit.
- @raise UserWarning:
- If the first parameter is None then kwargs['id'] or kwargs['name'] must be defined and not be None.
- Otherwise a UserWarning is raised.
-
- If the first parameter is an integer then it is interpreted as the id and kwargs['id'] must be
- undefined or None. Otherwise a UserWarning is raised.
+ Parameters
+ ----------
+ property : int, str, Property, optional
+ An identifying parameter, by default None
+ value : int, str, Property, optional
+ The value of the new property, by default None
+ id : int, optional
+ Id of the property, by default None
+ name : str, optional
+ Name of the property, by default None
+ description : str, optional
+ Description of the property, by default None
+ datatype : str, optional
+ Datatype of the property, by default None
+ unit : str, optional
+ Unit of the property, by default None
+ importance :str, optional
+ Importance of the property, by default None
+ inheritance : str, optional
+ Inheritance of the property, by default None
- If the first parameter is not None and neither an instance of Entity nor an integer it is
- interpreted as the name and kwargs['name'] must be undefined or None. Otherwise a UserWarning is
- raised.
+ Returns
+ -------
+ Entity
+
+ Raises
+ ------
+ UserWarning
+ If the first parameter is None then id or name must be defined and not be None.
+ UserWarning
+ If the first parameter is an integer then it is interpreted as the id and id must be
+ undefined or None.
+ UserWarning
+ If the first parameter is not None and neither an instance of Entity nor an integer it is
+ interpreted as the name and name must be undefined or None.
"""
- copy_kwargs = kwargs.copy()
- name = (kwargs['name'] if 'name' in kwargs else None)
- pid = (kwargs['id'] if 'id' in kwargs else None)
+
+ pid = id
abstract_property = None
if isinstance(property, Entity):
+ if property.role is not None and property.role.lower() in ["record", "file"]:
+ raise ValueError("The property parameter is a {0}. This "
+ "is very unusual and probably not what you "
+ "want. Otherwise, construct a property from "
+ "a {0} using the Property class and add "
+ "that to this entity.".format(property.role))
abstract_property = property
elif isinstance(property, int):
if pid is not None:
raise UserWarning("The first parameter was an integer which would normally be interpreted as the id of the property which is to be added. But you have also specified a parameter 'id' in the method call. This is ambiguous and cannot be processed.")
pid = property
- copy_kwargs['id'] = pid
+ id = pid
elif property is not None:
if name is not None:
raise UserWarning("The first parameter was neither an instance of Entity nor an integer. Therefore the string representation of your first parameter would normally be interpreted name of the property which is to be added. But you have also specified a parameter 'name' in the method call. This is ambiguous and cannot be processed.")
name = str(property)
- copy_kwargs['name'] = name
if property is None and name is None and pid is None:
raise UserWarning(
"This method expects you to pass at least an entity, a name or an id.")
- del copy_kwargs['importance']
- del copy_kwargs['inheritance']
- new_property = Property(**copy_kwargs)
+ new_property = Property(name=name, id=id, description=description, datatype=datatype,
+ value=value, unit=unit)
if abstract_property is not None:
new_property._wrap(property)
@@ -399,9 +428,7 @@ class Entity(object):
new_property.value = value
self.properties.append(
- property=new_property, importance=(
- kwargs['importance'] if 'importance' in kwargs else None), inheritance=(
- kwargs['inheritance'] if 'inheritance' in kwargs else None))
+ property=new_property, importance=importance, inheritance=inheritance)
return self
@@ -426,20 +453,41 @@ class Entity(object):
return self
- def add_parent(self, parent=None, **kwargs): # @ReservedAssignment
+ def add_parent(self, parent=None, id=None, name=None, inheritance=None): # @ReservedAssignment
"""Add a parent to this entity.
- The first parameter is meant to identify the parent entity. So the method expects an instance of
- Entity, an integer or a string here. Even though, by means of the **kwargs parameter you may pass
- more parameters to this method. Accepted keywords are: id, name, inheritance. Any other keyword is
- ignored right now but this may change in the future.
+ Parameters
+ ----------
+ parent : Entity or int or str or None
+ The parent entity, either specified by the Entity object
+ itself, or its id or its name. Default is None.
+ id : int
+ Integer id of the parent entity. Ignored if `parent`
+ is not None.
+ name : str
+ Name of the parent entity. Ignored if `parent is not
+ none`.
+ inheritance : str
+ One of ``obligatory``, ``recommended``, ``suggested``, or ``fix``. Specifies the
+ minimum importance which parent properties need to have to be inherited by this
+ entity. If no `inheritance` is given, no properties will be inherited by the child.
+ This parameter is case-insensitive.
+
+ Note that the behaviour is currently not yet specified when assigning parents to
+ Records, it only works for inheritance of RecordTypes (and Properties).
+
+ For more information, it is recommended to look into the
+ :ref:`data insertion tutorial<tutorial-inheritance-properties>`.
+
+ Raises
+ ------
+ UserWarning
+ If neither a `parent` parameter, nor the `id`, nor `name`
+ parameter is passed to this method.
- @param parent: An entity, an id or a name.
- @param **kwargs: Accepted keywords: id, name, inheritance.
- @raise UserWarning: If neither a 'parent' parameter, nor the 'id', nor 'name' parameter is passed to this method.
"""
- name = (kwargs['name'] if 'name' in kwargs else None)
- pid = (kwargs['id'] if 'id' in kwargs else None)
+
+ pid = id
parent_entity = None
if isinstance(parent, Entity):
@@ -453,9 +501,6 @@ class Entity(object):
raise UserWarning(
"This method expects you to pass at least an entity, a name or an id.")
- inheritance = (kwargs['inheritance']
- if 'inheritance' in kwargs else None)
-
addp = Parent(id=pid, name=name, inheritance=inheritance)
if parent_entity is not None:
@@ -553,6 +598,7 @@ class Entity(object):
entity, or name.
"""
+
if isinstance(key, int):
for p in self.parents:
if p.id is not None and int(p.id) == int(key):
@@ -561,14 +607,17 @@ class Entity(object):
if key.id is not None:
# first try by id
found = self.get_parent(int(key.id))
+
if found is not None:
return found
# otherwise by name
+
return self.get_parent(key.name)
else:
for p in self.parents:
if (p.name is not None
and str(p.name).lower() == str(key).lower()):
+
return p
return None
@@ -653,6 +702,7 @@ class Entity(object):
special_selector = None
# iterating through the entity tree according to the selector
+
for subselector in selector:
# selector does not match the structure, we cannot get a
# property of non-entity
@@ -664,11 +714,13 @@ class Entity(object):
# selector does not match the structure, we did not get a
# property
+
if prop is None:
return None
# if the property is a reference, we are interested in the
# corresponding entities attributes
+
if isinstance(prop.value, Entity):
ref = prop.value
@@ -677,6 +729,7 @@ class Entity(object):
ref = prop
# if we saved a special selector before, apply it
+
if special_selector is None:
return prop.value
else:
@@ -804,10 +857,13 @@ class Entity(object):
"""
if xml is None:
- xml = etree.Element("Entity")
+ # use role as xml tag name, fall-back to "Entity"
+ elem_tag = "Entity" if self.role is None else self.role
+ xml = etree.Element(elem_tag)
assert isinstance(xml, etree._Element)
# unwrap wrapped entity
+
if self._wrapped_entity is not None:
xml = self._wrapped_entity.to_xml(xml, add_properties)
@@ -854,6 +910,8 @@ class Entity(object):
xml.append(v_elem)
elif self.value == "":
xml.append(etree.Element("EmptyString"))
+ elif str(self.value) == "nan":
+ xml.text = "NaN"
else:
xml.text = str(self.value)
@@ -907,6 +965,9 @@ class Entity(object):
if self.acl is not None:
xml.append(self.acl.to_xml())
+ if self.state is not None:
+ xml.append(self.state.to_xml())
+
return xml
@staticmethod
@@ -917,6 +978,8 @@ class Entity(object):
@param elem: the xml element
"""
+ if isinstance(entity, Entity):
+ entity.role = elem.tag
entity._cuid = elem.get("cuid")
entity.id = elem.get("id") # @ReservedAssignment
entity.name = elem.get("name")
@@ -951,6 +1014,8 @@ class Entity(object):
entity.add_message(child)
elif isinstance(child, Version):
entity.version = child
+ elif isinstance(child, State):
+ entity.state = child
elif child is None or hasattr(child, "encode"):
vals.append(child)
elif isinstance(child, Entity):
@@ -1040,6 +1105,7 @@ class Entity(object):
if len(c) == 1:
c[0].messages.extend(c.messages)
+
return c[0]
raise QueryNotUniqueError("This retrieval was not unique!!!")
@@ -1078,7 +1144,7 @@ class Entity(object):
flags=flags)[0]
def update(self, strict=False, raise_exception_on_error=True,
- unique=True, flags=None):
+ unique=True, flags=None, sync=True):
"""Update this entity.
There are two possible work-flows to perform this update:
@@ -1112,6 +1178,7 @@ class Entity(object):
return Container().append(self).update(
strict=strict,
+ sync=sync,
raise_exception_on_error=raise_exception_on_error,
unique=unique,
flags=flags)[0]
@@ -1227,6 +1294,7 @@ class QueryTemplate():
def __init__(self, id=None, name=None, query=None, description=None): # @ReservedAssignment
self.id = (int(id) if id is not None else None)
+ self.role = "QueryTemplate"
self.name = name
self.description = description
self.query = query
@@ -1247,6 +1315,7 @@ class QueryTemplate():
self.is_valid = lambda: False
self.is_deleted = lambda: False
self.version = None
+ self.state = None
def retrieve(self, raise_exception_on_error=True, unique=True, sync=True,
flags=None):
@@ -1409,28 +1478,42 @@ class Property(Entity):
"""CaosDB's Property object."""
- def add_property(self, property=None, value=None, **kwargs): # @ReservedAssignment
- copy_kwargs = kwargs.copy()
+ def add_property(self, property=None, value=None, id=None, name=None, description=None, datatype=None,
+ unit=None, importance=FIX, inheritance=FIX): # @ReservedAssignment
- if 'importance' not in copy_kwargs:
- # set default importance
- copy_kwargs['importance'] = FIX
-
- if 'inheritance' not in copy_kwargs:
- # set default importance
- copy_kwargs['inheritance'] = FIX
+ return super().add_property(
+ property=property, id=id, name=name, description=description, datatype=datatype,
+ value=value, unit=unit, importance=importance, inheritance=inheritance)
- return super(Property, self).add_property(
- property=property, value=value, **copy_kwargs)
+ def add_parent(self, parent=None, id=None, name=None, inheritance=FIX):
+ """Add a parent Entity to this Property.
- def add_parent(self, parent=None, **kwargs):
- copy_kwargs = kwargs.copy()
+ Parameters
+ ----------
+ Parameters
+ ----------
+ parent : Entity or int or str or None
+ The parent entity, either specified by the Entity object
+ itself, or its id or its name. Default is None.
+ id : int
+ Integer id of the parent entity. Ignored if `parent`
+ is not None.
+ name : str
+ Name of the parent entity. Ignored if `parent is not
+ none`.
+ inheritance : str, default: FIX
+ One of ``obligatory``, ``recommended``, ``suggested``, or ``fix``. Specifies the
+ minimum importance which parent properties need to have to be inherited by this
+ entity. If no `inheritance` is given, no properties will be inherited by the child.
+ This parameter is case-insensitive.
+
+ See Also
+ --------
+ Entity.add_parent
- if 'inheritance' not in copy_kwargs:
- # set default importance
- copy_kwargs['inheritance'] = FIX
+ """
- return super(Property, self).add_parent(parent=parent, **copy_kwargs)
+ return super(Property, self).add_parent(parent=parent, id=id, name=name, inheritance=inheritance)
def __init__(self, name=None, id=None, description=None, datatype=None,
value=None, unit=None):
@@ -1444,6 +1527,47 @@ class Property(Entity):
return super(Property, self).to_xml(xml, add_properties)
+ def is_reference(self, server_retrieval=False):
+ """Returns whether this Property is a reference
+
+ Parameters
+ ----------
+ server_retrieval : bool, optional
+ If True and the datatype is not set, the Property is retrieved from the server, by default False
+
+ Returns
+ -------
+ bool, NoneType
+ Returns whether this Property is a reference or None if a server call is needed to
+ check correctly, but server_retrieval is set to False.
+
+ """
+
+ if self.datatype is None:
+
+ if not self.is_valid():
+ # this is a workaround to prevent side effects
+ # since retrieve currently changes the object
+
+ if server_retrieval:
+ tmp_prop = deepcopy(self)
+ """
+ remove role to avoid unnessecary ValueError while
+ retrieving the Entity.
+ """
+ tmp_prop.role = None
+ tmp_prop.retrieve()
+
+ return tmp_prop.is_reference()
+ else:
+ return None
+ else:
+ # a valid property without datatype has to be an RT
+
+ return True
+ else:
+ return is_reference(self.datatype)
+
class Message(object):
@@ -1487,28 +1611,45 @@ class RecordType(Entity):
"""This class represents CaosDB's RecordType entities."""
- def add_property(self, property=None, value=None, **kwargs): # @ReservedAssignment
- copy_kwargs = kwargs.copy()
-
- if 'importance' not in copy_kwargs:
- # set default importance
- copy_kwargs['importance'] = RECOMMENDED
+ def add_property(self, property=None, value=None, id=None, name=None, description=None, datatype=None,
+ unit=None, importance=RECOMMENDED, inheritance=FIX): # @ReservedAssignment
- if 'inheritance' not in copy_kwargs:
- # set default importance
- copy_kwargs['inheritance'] = FIX
+ return super().add_property(
+ property=property, id=id, name=name, description=description, datatype=datatype,
+ value=value, unit=unit, importance=importance, inheritance=inheritance)
- return super().add_property(property=property, value=value,
- **copy_kwargs)
+ def add_parent(self, parent=None, id=None, name=None, inheritance=OBLIGATORY):
+ """Add a parent to this RecordType
- def add_parent(self, parent=None, **kwargs):
- copy_kwargs = kwargs.copy()
+ Parameters
+ ----------
+ parent : Entity or int or str or None, optional
+ The parent entity, either specified by the Entity object
+ itself, or its id or its name. Default is None.
+ Parameters
+ ----------
+ parent : Entity or int or str or None
+ The parent entity, either specified by the Entity object
+ itself, or its id or its name. Default is None.
+ id : int
+ Integer id of the parent entity. Ignored if `parent`
+ is not None.
+ name : str
+ Name of the parent entity. Ignored if `parent is not
+ none`.
+ inheritance : str, default OBLIGATORY
+ One of ``obligatory``, ``recommended``, ``suggested``, or ``fix``. Specifies the
+ minimum importance which parent properties need to have to be inherited by this
+ entity. If no `inheritance` is given, no properties will be inherited by the child.
+ This parameter is case-insensitive.
+
+ See Also
+ --------
+ Entity.add_parent
- if 'inheritance' not in copy_kwargs:
- # set default importance
- copy_kwargs['inheritance'] = OBLIGATORY
+ """
- return super().add_parent(parent=parent, **copy_kwargs)
+ return super().add_parent(parent=parent, id=id, name=name, inheritance=inheritance)
def __init__(self, name=None, id=None, description=None, datatype=None): # @ReservedAssignment
Entity.__init__(self, name=name, id=id, description=description,
@@ -1525,19 +1666,12 @@ class Record(Entity):
"""This class represents CaosDB's Record entities."""
- def add_property(self, property=None, value=None, **kwargs): # @ReservedAssignment
- copy_kwargs = kwargs.copy()
-
- if 'importance' not in copy_kwargs:
- # set default importance
- copy_kwargs['importance'] = FIX
-
- if 'inheritance' not in copy_kwargs:
- # set default importance
- copy_kwargs['inheritance'] = FIX
+ def add_property(self, property=None, value=None, id=None, name=None, description=None, datatype=None,
+ unit=None, importance=FIX, inheritance=FIX): # @ReservedAssignment
return super().add_property(
- property=property, value=value, **copy_kwargs)
+ property=property, id=id, name=name, description=description, datatype=datatype,
+ value=value, unit=unit, importance=importance, inheritance=inheritance)
def __init__(self, name=None, id=None, description=None): # @ReservedAssignment
Entity.__init__(self, name=name, id=id, description=description,
@@ -1689,19 +1823,12 @@ class File(Record):
return checksum.hexdigest()
- def add_property(self, property=None, value=None, **kwargs): # @ReservedAssignment
- copy_kwargs = kwargs.copy()
-
- if 'importance' not in copy_kwargs:
- # set default importance
- copy_kwargs['importance'] = FIX
-
- if 'inheritance' not in copy_kwargs:
- # set default importance
- copy_kwargs['inheritance'] = FIX
+ def add_property(self, property=None, id=None, name=None, description=None, datatype=None,
+ value=None, unit=None, importance=FIX, inheritance=FIX): # @ReservedAssignment
return super().add_property(
- property=property, value=value, **copy_kwargs)
+ property=property, id=id, name=name, description=description, datatype=datatype,
+ value=value, unit=unit, importance=importance, inheritance=inheritance)
class _Properties(list):
@@ -2074,6 +2201,7 @@ class _Messages(dict):
else:
raise TypeError(
"('description', 'body'), ('body'), or 'body' expected.")
+
if isinstance(value, Message):
body = value.body
description = value.description
@@ -2204,15 +2332,45 @@ class _Messages(dict):
def _basic_sync(e_local, e_remote):
+ '''Copy all state from a one entity to another.
+
+ This method is used to syncronize an entity with a remote (i.e. a newly
+ retrieved) one.
+
+ Any entity state of the local one will be overriden.
+
+ Parameters
+ ----------
+ e_local : Entity
+ Destination of the copy.
+ e_local : Entity
+ Source of the copy.
+
+
+ Returns
+ -------
+ e_local : Entity
+ The syncronized entity.
+ '''
if e_local is None or e_remote is None:
return None
+ if e_local.role is None:
+ e_local.role = e_remote.role
+ elif e_remote.role is not None and not e_local.role.lower() == e_remote.role.lower():
+ raise ValueError("The resulting entity had a different role ({0}) "
+ "than the local one ({1}). This probably means, that "
+ "the entity was intialized with a wrong class "
+ "by this client or it has changed in the past and "
+ "this client did't know about it yet.".format(
+ e_remote.role, e_local.role))
+
e_local.id = e_remote.id
e_local.name = e_remote.name
e_local.description = e_remote.description
e_local.path = e_remote.path
e_local._checksum = e_remote._checksum
e_local._size = e_remote._size
- e_local.datatype = e_remote.datatype # @ReservedAssignment
+ e_local.datatype = e_remote.datatype
e_local.unit = e_remote.unit
e_local.value = e_remote.value
e_local.properties = e_remote.properties
@@ -2223,6 +2381,7 @@ def _basic_sync(e_local, e_remote):
e_local.is_valid = e_remote.is_valid
e_local.is_deleted = e_remote.is_deleted
e_local.version = e_remote.version
+ e_local.state = e_remote.state
if hasattr(e_remote, "query"):
e_local.query = e_remote.query
@@ -2242,6 +2401,7 @@ def _deletion_sync(e_local, e_remote):
except KeyError:
# deletion info wasn't there
e_local.messages = e_remote.messages
+
return
_basic_sync(e_local, e_remote)
@@ -2434,6 +2594,7 @@ class Container(list):
tmpid = 0
# users might already have specified some tmpids. -> look for smallest.
+
for e in self:
tmpid = min(tmpid, Container._get_smallest_tmpid(e))
tmpid -= 1
@@ -2653,6 +2814,7 @@ class Container(list):
used_remote_entities = []
# match by cuid
+
for local_entity in self:
sync_dict[local_entity] = None
@@ -2681,6 +2843,7 @@ class Container(list):
raise MismatchingEntitiesError(msg)
# match by id
+
for local_entity in self:
if sync_dict[local_entity] is None and local_entity.id is not None:
sync_remote_entities = []
@@ -2705,6 +2868,7 @@ class Container(list):
raise MismatchingEntitiesError(msg)
# match by path
+
for local_entity in self:
if (sync_dict[local_entity] is None
and local_entity.path is not None):
@@ -2734,6 +2898,7 @@ class Container(list):
raise MismatchingEntitiesError(msg)
# match by name
+
for local_entity in self:
if (sync_dict[local_entity] is None
and local_entity.name is not None):
@@ -2783,7 +2948,62 @@ class Container(list):
return sync_dict
- def delete(self, raise_exception_on_error=True, flags=None):
+ def _test_dependencies_in_container(self, container):
+ """This function returns those elements of a given container that are a dependency of another element of the same container.
+
+ Args:
+ container (Container): a caosdb container
+
+ Returns:
+ [set]: a set of unique elements that are a dependency of another element of `container`
+ """
+ item_id = set()
+ is_parent = set()
+ is_property = set()
+ is_being_referenced = set()
+ dependent_parents = set()
+ dependent_properties = set()
+ dependent_references = set()
+ dependencies = set()
+
+ for container_item in container:
+ item_id.add(container_item.id)
+
+ for parents in container_item.get_parents():
+ is_parent.add(parents.id)
+
+ for references in container_item.get_properties():
+ if is_reference(references.datatype):
+ # add only if it is a reference, not a property
+
+ if references.value is None:
+ continue
+ elif isinstance(references.value, int):
+ is_being_referenced.add(references.value)
+ elif is_list_datatype(references.datatype):
+ for list_item in references.value:
+ if isinstance(list_item, int):
+ is_being_referenced.add(list_item)
+ else:
+ is_being_referenced.add(list_item.id)
+ else:
+ try:
+ is_being_referenced.add(references.value.id)
+ except AttributeError:
+ pass
+
+ if hasattr(references, 'id'):
+ is_property.add(references.id)
+
+ dependent_parents = item_id.intersection(is_parent)
+ dependent_properties = item_id.intersection(is_property)
+ dependent_references = item_id.intersection(is_being_referenced)
+ dependencies = dependent_parents.union(dependent_references)
+ dependencies = dependencies.union(dependent_properties)
+
+ return dependencies
+
+ def delete(self, raise_exception_on_error=True, flags=None, chunk_size=100):
"""Delete all entities in this container.
Entities are identified via their id if present and via their
@@ -2794,6 +3014,46 @@ class Container(list):
this happens, none of them will be deleted. It occurs an error
instead.
"""
+ item_count = len(self)
+ # Split Container in 'chunk_size'-sized containers (if necessary) to avoid error 414 Request-URI Too Long
+
+ if item_count > chunk_size:
+ dependencies = self._test_dependencies_in_container(self)
+ '''
+ If there are as many dependencies as entities in the container and it is larger than chunk_size it cannot be split and deleted.
+ This case cannot be handled at the moment.
+ '''
+
+ if len(dependencies) == item_count:
+ if raise_exception_on_error:
+ te = TransactionError(
+ msg="The container is too large and with too many dependencies within to be deleted.",
+ container=self)
+ raise te
+
+ return self
+
+ # items which have to be deleted later because of dependencies.
+ dependencies_delete = Container()
+
+ for i in range(0, int(item_count/chunk_size)+1):
+ chunk = Container()
+
+ for j in range(i*chunk_size, min(item_count, (i+1)*chunk_size)):
+ if len(dependencies):
+ if self[j].id in dependencies:
+ dependencies_delete.append(self[j])
+ else:
+ chunk.append(self[j])
+ else:
+ chunk.append(self[j])
+
+ if len(chunk):
+ chunk.delete()
+ if len(dependencies_delete):
+ dependencies_delete.delete()
+
+ return self
if len(self) == 0:
if raise_exception_on_error:
@@ -2840,7 +3100,7 @@ class Container(list):
entity_url_segments = [_ENTITY_URI_SEGMENT, "&".join(id_str)]
_log_request("DELETE: " + str(entity_url_segments) +
- ("?" + flags if flags is not None else ''))
+ ("?" + str(flags) if flags is not None else ''))
http_response = c.delete(entity_url_segments, query_dict=flags)
cresp = Container._response_to_entities(http_response)
@@ -3419,6 +3679,23 @@ class ACL():
self.deny(username=username, realm=realm, role=role,
permission=permission, priority=priority)
+ def combine(self, other):
+ """ Combine and return new instance."""
+ result = ACL()
+ result._grants.update(other._grants)
+ result._grants.update(self._grants)
+ result._denials.update(other._denials)
+ result._denials.update(self._denials)
+ result._priority_grants.update(other._priority_grants)
+ result._priority_grants.update(self._priority_grants)
+ result._priority_denials.update(other._priority_denials)
+ result._priority_denials.update(self._priority_denials)
+
+ return result
+
+ def __eq__(self, other):
+ return isinstance(other, ACL) and other._grants == self._grants and self._denials == other._denials and self._priority_grants == other._priority_grants and self._priority_denials == other._priority_denials
+
def is_empty(self):
return len(self._grants) + len(self._priority_grants) + \
len(self._priority_denials) + len(self._denials) == 0
@@ -3661,6 +3938,7 @@ class Query():
self.flags = dict()
self.messages = _Messages()
self.cached = None
+ self.etag = None
if isinstance(q, etree._Element):
self.q = q.get("string")
@@ -3670,6 +3948,7 @@ class Query():
self.cached = False
else:
self.cached = q.get("cached").lower() == "true"
+ self.etag = q.get("etag")
for m in q:
if m.tag.lower() == 'warning' or m.tag.lower() == 'error':
@@ -3702,6 +3981,7 @@ class Query():
connection = get_connection()
flags = self.flags
+
if cache is False:
flags["cache"] = "false"
query_dict = dict(flags)
@@ -3714,6 +3994,7 @@ class Query():
cresp = Container._response_to_entities(http_response)
self.results = cresp.query.results
self.cached = cresp.query.cached
+ self.etag = cresp.query.etag
if self.q.lower().startswith('count') and len(cresp) == 0:
# this was a count query
@@ -3727,9 +4008,11 @@ class Query():
if len(cresp) > 1 and raise_exception_on_error:
raise QueryNotUniqueError(
"Query '{}' wasn't unique.".format(self.q))
+
if len(cresp) == 0 and raise_exception_on_error:
raise EmptyUniqueQueryError(
"Query '{}' found no results.".format(self.q))
+
if len(cresp) == 1:
r = cresp[0]
r.messages.extend(cresp.messages)
@@ -3836,6 +4119,7 @@ class Info():
for e in xml:
m = _parse_single_xml_element(e)
+
if isinstance(m, UserInfo):
self.user_info = m
else:
@@ -3942,6 +4226,8 @@ def _parse_single_xml_element(elem):
return entity
elif elem.tag.lower() == "version":
return Version.from_xml(elem)
+ elif elem.tag.lower() == "state":
+ return State.from_xml(elem)
elif elem.tag.lower() == "emptystring":
return ""
elif elem.tag.lower() == "value":
@@ -3980,7 +4266,7 @@ def _evaluate_and_add_error(parent_error, ent):
Parameters:
-----------
- parent_error : TrancactionError
+ parent_error : TransactionError
Parent error to which the new exception will be attached. This
exception will be a direct child.
ent : Entity
@@ -3993,13 +4279,16 @@ def _evaluate_and_add_error(parent_error, ent):
Parent error with new exception(s) attached to it.
"""
+
if isinstance(ent, (Entity, QueryTemplate)):
# Check all error messages
found114 = False
found116 = False
+
for err in ent.get_errors():
# Evaluate specific EntityErrors depending on the error
# code
+
if err.code is not None:
if int(err.code) == 101: # ent doesn't exist
new_exc = EntityDoesNotExistError(entity=ent,
@@ -4016,6 +4305,7 @@ def _evaluate_and_add_error(parent_error, ent):
found114 = True
new_exc = UnqualifiedPropertiesError(entity=ent,
error=err)
+
for prop in ent.get_properties():
new_exc = _evaluate_and_add_error(new_exc,
prop)
@@ -4023,6 +4313,7 @@ def _evaluate_and_add_error(parent_error, ent):
found116 = True
new_exc = UnqualifiedParentsError(entity=ent,
error=err)
+
for par in ent.get_parents():
new_exc = _evaluate_and_add_error(new_exc,
par)
@@ -4033,21 +4324,28 @@ def _evaluate_and_add_error(parent_error, ent):
parent_error.add_error(new_exc)
# Check for possible errors in parents and properties that
# weren't detected up to here
+
if not found114:
dummy_err = EntityError(entity=ent)
+
for prop in ent.get_properties():
dummy_err = _evaluate_and_add_error(dummy_err, prop)
+
if dummy_err.errors:
parent_error.add_error(dummy_err)
+
if not found116:
dummy_err = EntityError(entity=ent)
+
for par in ent.get_parents():
dummy_err = _evaluate_and_add_error(dummy_err, par)
+
if dummy_err.errors:
parent_error.add_error(dummy_err)
elif isinstance(ent, Container):
parent_error.container = ent
+
if ent.get_errors() is not None:
parent_error.code = ent.get_errors()[0].code
# In the highly unusual case of more than one error
@@ -4055,6 +4353,7 @@ def _evaluate_and_add_error(parent_error, ent):
parent_error.msg = '\n'.join(
[x.description for x in ent.get_errors()])
# Go through all container elements and add them:
+
for elt in ent:
parent_error = _evaluate_and_add_error(parent_error, elt)
@@ -4080,10 +4379,12 @@ def raise_errors(arg0):
transaction_error = _evaluate_and_add_error(TransactionError(),
arg0)
# Raise if any error was found
+
if len(transaction_error.all_errors) > 0:
raise transaction_error
# Cover the special case of an empty container with error
# message(s) (e.g. query syntax error)
+
if (transaction_error.container is not None and
transaction_error.container.has_errors()):
raise transaction_error
diff --git a/src/caosdb/common/state.py b/src/caosdb/common/state.py
new file mode 100644
index 0000000000000000000000000000000000000000..cb74022bef57a77c8270b2033c904eecabaadf83
--- /dev/null
+++ b/src/caosdb/common/state.py
@@ -0,0 +1,198 @@
+# ** header v3.0
+# This file is a part of the CaosDB Project.
+#
+# Copyright (C) 2020 IndiScale GmbH <info@indiscale.com>
+# Copyright (C) 2020 Timm Fitschen <t.fitschen@indiscale.com>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see <https://www.gnu.org/licenses/>.
+#
+# ** end header
+
+import copy
+from lxml import etree
+
+
+def _translate_to_state_acis(acis):
+ result = set()
+ for aci in acis:
+ aci = copy.copy(aci)
+ if aci.role:
+ aci.role = "?STATE?" + aci.role + "?"
+ result.add(aci)
+ return result
+
+
+class Transition:
+ """Transition
+
+ Represents allowed transitions from one state to another.
+
+ Properties
+ ----------
+ name : str
+ The name of the transition
+ description: str
+ The description of the transition
+ from_state : str
+ A state name
+ to_state : str
+ A state name
+ """
+
+ def __init__(self, name, from_state, to_state, description=None):
+ self._name = name
+ self._from_state = from_state
+ self._to_state = to_state
+ self._description = description
+
+ @property
+ def name(self):
+ return self._name
+
+ @property
+ def description(self):
+ return self._description
+
+ @property
+ def from_state(self):
+ return self._from_state
+
+ @property
+ def to_state(self):
+ return self._to_state
+
+ def __repr__(self):
+ return f'Transition(name="{self.name}", from_state="{self.from_state}", to_state="{self.to_state}", description="{self.description}")'
+
+ def __eq__(self, other):
+ return (isinstance(other, Transition)
+ and other.name == self.name
+ and other.to_state == self.to_state
+ and other.from_state == self.from_state)
+
+ def __hash__(self):
+ return 23472 + hash(self.name) + hash(self.from_state) + hash(self.to_state)
+
+ @staticmethod
+ def from_xml(xml):
+ to_state = [to.get("name") for to in xml
+ if to.tag.lower() == "tostate"]
+ from_state = [from_.get("name") for from_ in xml
+ if from_.tag.lower() == "fromstate"]
+ result = Transition(name=xml.get("name"),
+ description=xml.get("description"),
+ from_state=from_state[0] if from_state else None,
+ to_state=to_state[0] if to_state else None)
+ return result
+
+
+class State:
+ """State
+
+ Represents the state of an entity and take care of the serialization and
+ deserialization of xml for the entity state.
+
+ An entity state is always a State of a StateModel.
+
+ Properties
+ ----------
+ name : str
+ Name of the State
+ model : str
+ Name of the StateModel
+ description : str
+ Description of the State (read-only)
+ id : str
+ Id of the undelying State record (read-only)
+ transitions : set of Transition
+ All transitions which are available from this state (read-only)
+ """
+
+ def __init__(self, model, name):
+ self.name = name
+ self.model = model
+ self._id = None
+ self._description = None
+ self._transitions = None
+
+ @property
+ def id(self):
+ return self._id
+
+ @property
+ def description(self):
+ return self._description
+
+ @property
+ def transitions(self):
+ return self._transitions
+
+ def __eq__(self, other):
+ return (isinstance(other, State)
+ and self.name == other.name
+ and self.model == other.model)
+
+ def __hash__(self):
+ return hash(self.name) + hash(self.model)
+
+ def __repr__(self):
+ return f"State('{self.model}', '{self.name}')"
+
+ def to_xml(self):
+ """Serialize this State to xml.
+
+ Returns
+ -------
+ xml : etree.Element
+ """
+ xml = etree.Element("State")
+ if self.name is not None:
+ xml.set("name", self.name)
+ if self.model is not None:
+ xml.set("model", self.model)
+ return xml
+
+ @staticmethod
+ def from_xml(xml):
+ """Create a new State instance from an xml Element.
+
+ Parameters
+ ----------
+ xml : etree.Element
+
+ Returns
+ -------
+ state : State
+ """
+ name = xml.get("name")
+ model = xml.get("model")
+ result = State(name=name, model=model)
+ result._id = xml.get("id")
+ result._description = xml.get("description")
+ transitions = [Transition.from_xml(t) for t in xml if t.tag.lower() ==
+ "transition"]
+ if transitions:
+ result._transitions = set(transitions)
+
+ return result
+
+ @staticmethod
+ def create_state_acl(acl):
+ from .models import ACL
+ state_acl = ACL()
+ state_acl._grants = _translate_to_state_acis(acl._grants)
+ state_acl._denials = _translate_to_state_acis(acl._denials)
+ state_acl._priority_grants = _translate_to_state_acis(acl._priority_grants)
+ state_acl._priority_denials = _translate_to_state_acis(acl._priority_denials)
+ return state_acl
diff --git a/src/caosdb/configuration.py b/src/caosdb/configuration.py
index 4d0797844182b8465ef5f97a869e31ee4fcaf47d..51e3749aaca3045afec9334ef987a174d5d19f26 100644
--- a/src/caosdb/configuration.py
+++ b/src/caosdb/configuration.py
@@ -21,6 +21,16 @@
#
# ** end header
#
+
+import os
+import yaml
+import warnings
+try:
+ optional_jsonschema_validate = None
+ from jsonschema import validate as optional_jsonschema_validate
+except ImportError:
+ pass
+
try:
# python2
from ConfigParser import ConfigParser
@@ -28,6 +38,9 @@ except ImportError:
# python3
from configparser import ConfigParser
+from os import environ, getcwd
+from os.path import expanduser, join, isfile
+
def _reset_config():
global _pycaosdbconf
@@ -44,8 +57,57 @@ def configure(inifile):
_pycaosdbconf = None
if _pycaosdbconf is None:
_reset_config()
- return _pycaosdbconf.read(inifile)
+ read_config = _pycaosdbconf.read(inifile)
+ validate_yaml_schema(config_to_yaml(_pycaosdbconf))
+ return read_config
def get_config():
+ global _pycaosdbconf
return _pycaosdbconf
+
+
+def config_to_yaml(config):
+ valobj = {}
+ for s in config.sections():
+ valobj[s] = {}
+ for key, value in config[s].items():
+ # TODO: Can the type be inferred from the config object?
+ if key in ["timeout", "debug"]:
+ valobj[s][key] = int(value)
+ elif key in ["ssl_insecure"]:
+ valobj[s][key] = bool(value)
+ else:
+ valobj[s][key] = value
+
+ return valobj
+
+
+def validate_yaml_schema(valobj):
+ # TODO: Re-enable warning once the schema has been extended to also cover
+ # SSS pycaosdb.inis and integration tests.
+ if optional_jsonschema_validate:
+ with open(os.path.join(os.path.dirname(__file__), "schema-pycaosdb-ini.yml")) as f:
+ schema = yaml.load(f, Loader=yaml.SafeLoader)
+ optional_jsonschema_validate(instance=valobj, schema=schema["schema-pycaosdb-ini"])
+ # else:
+ # warnings.warn("""
+ # Warning: The validation could not be performed because `jsonschema` is not installed.
+ # """)
+
+
+def _read_config_files():
+ """Function to read config files from different paths. Checks for path in $PYCAOSDBINI or home directory (.pycaosdb.ini) and in the current working directory (pycaosdb.ini).
+
+ Returns:
+ [list]: list with successfully parsed ini-files. Order: env_var or home directory, cwd. Used for testing the function.
+ """
+ return_var = []
+ if "PYCAOSDBINI" in environ:
+ return_var.extend(configure(expanduser(environ["PYCAOSDBINI"])))
+ else:
+ return_var.extend(configure(expanduser('~/.pycaosdb.ini')))
+
+ if isfile(join(getcwd(), "pycaosdb.ini")):
+ return_var.extend(configure(join(getcwd(), "pycaosdb.ini")))
+ return return_var
diff --git a/src/caosdb/connection/authentication/interface.py b/src/caosdb/connection/authentication/interface.py
index a364aeb564ee929d995b2f8098bd21e30e9733ab..f2cc5001cf8fa0f6d61ec65346f6a200ba0dfcd8 100644
--- a/src/caosdb/connection/authentication/interface.py
+++ b/src/caosdb/connection/authentication/interface.py
@@ -36,7 +36,7 @@ from caosdb.exceptions import LoginFailedError
# meta class compatible with Python 2 *and* 3:
ABC = ABCMeta('ABC', (object, ), {'__slots__': ()})
-_LOGGER = logging.getLogger("authentication")
+_LOGGER = logging.getLogger(__name__)
class AbstractAuthenticator(ABC):
diff --git a/src/caosdb/connection/authentication/keyring.py b/src/caosdb/connection/authentication/keyring.py
index d8be7ddf030577545230c9111fdad542b6d6e7e2..99d184136c20b23557efea0b54c648095a8d3ab2 100644
--- a/src/caosdb/connection/authentication/keyring.py
+++ b/src/caosdb/connection/authentication/keyring.py
@@ -28,7 +28,7 @@ retrieve the password.
"""
import sys
-import imp
+import importlib
from getpass import getpass
from caosdb.exceptions import ConfigurationError
from .external_credentials_provider import ExternalCredentialsProvider
@@ -52,17 +52,13 @@ def get_authentication_provider():
def _get_external_keyring():
try:
- fil, pathname, desc = imp.find_module("keyring", sys.path[1:])
- module = imp.load_module("external_keyring", fil, pathname, desc)
- return module
+ return importlib.import_module("keyring")
except ImportError:
raise RuntimeError(
"The keyring password method requires installation of the"
"keyring python package. On linux with python < 3.5, "
"this requires the installation of dbus-python as a "
"system package.")
- finally:
- fil.close()
def _call_keyring(**config):
@@ -74,7 +70,6 @@ def _call_keyring(**config):
url = config.get("url")
username = config.get("username")
app = "caosdb — {}".format(url)
- password = _call_keyring(app=app, username=username)
external_keyring = _get_external_keyring()
password = external_keyring.get_password(app, username)
if password is None:
diff --git a/src/caosdb/connection/connection.py b/src/caosdb/connection/connection.py
index fc699ab5db1db36bc1ee63034b6828eec4d16bc1..6c3946ee639872b0edb0b5b3c30a808cf8c028d4 100644
--- a/src/caosdb/connection/connection.py
+++ b/src/caosdb/connection/connection.py
@@ -41,7 +41,11 @@ from caosdb.exceptions import (CaosDBException, HTTPClientError,
HTTPResourceNotFoundError,
HTTPServerError,
HTTPURITooLongError)
-from caosdb.version import version
+try:
+ from caosdb.version import version
+except ModuleNotFoundError:
+ version = "uninstalled"
+
from pkg_resources import resource_filename
from .interface import CaosDBHTTPResponse, CaosDBServerConnection
@@ -56,7 +60,7 @@ except ImportError:
# pylint: disable=missing-docstring
-_LOGGER = logging.getLogger("connection")
+_LOGGER = logging.getLogger(__name__)
class _WrappedHTTPResponse(CaosDBHTTPResponse):
@@ -611,3 +615,11 @@ class _Connection(object): # pylint: disable=useless-object-inheritance
_handle_response_status(http_response)
return http_response
+
+ def get_username(self):
+ """
+ Return the username of the current connection.
+
+ Shortcut for: get_connection()._authenticator._credentials_provider.username
+ """
+ return self._authenticator._credentials_provider.username
diff --git a/src/caosdb/exceptions.py b/src/caosdb/exceptions.py
index c9da22ce59b4eee37ea35e58d24ef436a6e88255..fdd2e11f1dfb8857f86942df2534d732bad9a793 100644
--- a/src/caosdb/exceptions.py
+++ b/src/caosdb/exceptions.py
@@ -63,6 +63,13 @@ class ConfigurationError(CaosDBException):
".pycaosdb.ini. Does at least one of them exist and are they correct?")
+class ServerConfigurationException(CaosDBException):
+ """The server is configured in a different way than expected.
+
+ This can be for example unexpected flags or settings or missing extensions.
+ """
+
+
class HTTPClientError(CaosDBException):
"""HTTPClientError represents 4xx HTTP client errors."""
@@ -182,8 +189,8 @@ class TransactionError(CaosDBException):
error_t. If direct_children_only is True, only direct children
are checked.
- Parameters:
- -----------
+ Parameters
+ ----------
error_t : EntityError
error type to be checked
direct_children_only: bool, optional
@@ -192,8 +199,8 @@ class TransactionError(CaosDBException):
children, i.e., all errors in self.all_errors are
used. Default is false.
- Returns:
- --------
+ Returns
+ -------
has_error : bool
True if at least one of the children is of type error_t,
False otherwise.
diff --git a/src/caosdb/schema-pycaosdb-ini.yml b/src/caosdb/schema-pycaosdb-ini.yml
new file mode 100644
index 0000000000000000000000000000000000000000..2653aeceeee94bfc26f17af19fda1618c580bbf1
--- /dev/null
+++ b/src/caosdb/schema-pycaosdb-ini.yml
@@ -0,0 +1,90 @@
+schema-pycaosdb-ini:
+ type: object
+ additionalProperties: false
+ properties:
+ Container:
+ additionalProperties: false
+ properties:
+ debug:
+ default: 0
+ type: integer
+ enum: [0, 1, 2]
+ Connection:
+ description: Settings for the connection to the CaosDB server
+ additionalProperties: false
+ properties:
+ url:
+ description: URL of the CaosDB server
+ type: string
+ pattern: https://[-a-zA-Z0-9\.]+(:[0-9]+)?(/)?
+ examples: [https://demo.indiscale.com/, https://localhost:10443/]
+ username:
+ type: string
+ description: User name used for authentication with the server
+ examples: [admin]
+ password_method:
+ description: The password input method defines how the password is supplied that is used for authentication with the server.
+ type: string
+ default: input
+ enum: [input, plain, pass, keyring]
+ password_identifier:
+ type: string
+ password:
+ type: string
+ examples: [caosdb]
+ auth_token:
+ type: string
+ description: Using an authentication token to connect with the server. This setting is not recommended for users.
+ cacert:
+ type: string
+ description: If the server's SSL certificate cannot be validated by your installed certificates (default or installed by your admins), you may also need to supply the matching key (pem file)
+ examples: [/path/to/caosdb.ca.pem]
+ ssl_insecure:
+ description: If this option is set, the SSL certificate of the server will not be validated. This has the potential of a man-in-the-middle attack. Use with care!
+ type: boolean
+ default: false
+ ssl_version:
+ description: You may define the ssl version to be used. It has to be the name of the corresponding attribute in the Python ssl module.
+ examples: [PROTOCOL_TLS]
+ debug:
+ default: 0
+ type: integer
+ enum: [0, 1, 2]
+ description: The debug key allows control the verbosity. Set it to 1 or 2 in case you want to see debugging output or if you want to learn more about the internals of the protocol. 0 disables debugging output.
+ socket_proxy:
+ examples: [localhost:12345]
+ type: string
+ description: You can define a socket proxy to be used. This is for the case that the server sits behind a firewall which is being tunnelled with a socket proxy (SOCKS4 or SOCKS5) (e.g. via ssh's -D option or a dedicated proxy server).
+ implementation:
+ description: This option is used internally and for testing. Do not override.
+ examples: [_DefaultCaosDBServerConnection]
+ timeout:
+ type: integer
+ allOf:
+ - if:
+ properties:
+ password_method:
+ const: input
+ then:
+ required: [url]
+ - if:
+ properties:
+ password_method:
+ const: plain
+ then:
+ required: [url, username, password]
+ - if:
+ properties:
+ password_method:
+ const: pass
+ then:
+ required: [url, username, password_identifier]
+ - if:
+ properties:
+ password_method:
+ const: keyring
+ then:
+ required: [url, username]
+ IntegrationTests:
+ description: "Used by the integration test suite from the caosdb-pyinttest repo."
+ additionalProperties: true
diff --git a/src/caosdb/utils/caosdb_admin.py b/src/caosdb/utils/caosdb_admin.py
index ff06624e3edc1379eff6dcbdfb4e8563433886d0..9fb94f57683036f5432a40198cc4ae98893665fb 100755
--- a/src/caosdb/utils/caosdb_admin.py
+++ b/src/caosdb/utils/caosdb_admin.py
@@ -73,7 +73,7 @@ def do_retrieve(args):
c.append(db.Entity(id=eid))
except ValueError:
c.append(db.Entity(name=i))
- c.retrieve()
+ c.retrieve(flags=eval(args.flags))
print(c)
@@ -116,28 +116,43 @@ def _promt_for_pw():
def do_create_user(args):
- password = None
+ password = args.user_password
if args.ask_password is True:
password = _promt_for_pw()
try:
admin._insert_user(name=args.user_name,
email=args.user_email, password=password)
+
+ if args.activate_user:
+ do_activate_user(args)
except HTTPClientError as e:
print(e.msg)
def do_activate_user(args):
- admin._update_user(name=args.user_name, status="ACTIVE")
+ try:
+ admin._update_user(name=args.user_name, status="ACTIVE")
+ except HTTPClientError as e:
+ print(e.msg)
def do_deactivate_user(args):
- admin._update_user(name=args.user_name, status="INACTIVE")
+ try:
+ admin._update_user(name=args.user_name, status="INACTIVE")
+ except HTTPClientError as e:
+ print(e.msg)
def do_set_user_password(args):
- password = _promt_for_pw()
- admin._update_user(name=args.user_name, password=password)
+ if args.user_password is None:
+ password = _promt_for_pw()
+ else:
+ password = args.user_password
+ try:
+ admin._update_user(name=args.user_name, password=password)
+ except HTTPClientError as e:
+ print(e.msg)
def do_add_user_roles(args):
@@ -301,10 +316,21 @@ USAGE
# users (CRUD)
subparser = subparsers.add_parser(
"create_user",
- help="Create a new user in caosdb's internal user database.")
+ help="Create a new user in caosdb's internal user database. You need "
+ " to activate the user before use.")
subparser.set_defaults(call=do_create_user)
- subparser.add_argument("-a", "--ask-password",
- help="Prompt for a password.", action="store_true")
+ mg = subparser.add_mutually_exclusive_group()
+ mg.add_argument("-a", "--ask-password",
+ help="Prompt for a password.", action="store_true")
+ mg.add_argument(
+ "--password",
+ dest="user_password",
+ default=None,
+ help="Alternative way to provide the new user's password. Please "
+ "consider to use the more secure, interactive way (-a option).")
+ subparser.add_argument("-c", "--activate-user",
+ help="Activate the user after creation.",
+ action="store_true")
subparser.add_argument(
metavar='USERNAME',
dest="user_name",
@@ -331,12 +357,20 @@ USAGE
subparser = subparsers.add_parser(
"set_user_password",
- help="Set a new password for a user. The password is not to be given on the command line for security reasons. You will be prompted for the password.")
+ help="Set a new password for a user. "
+ "By default, you will be prompted for the password.")
subparser.set_defaults(call=do_set_user_password)
subparser.add_argument(
metavar='USERNAME',
dest="user_name",
help="The name of the user who's password is to be set.")
+ subparser.add_argument(
+ metavar='PASSWORD',
+ nargs="?",
+ dest="user_password",
+ default=None,
+ help="Alternative way to provide the user's new password. "
+ "The more secure (and default way) is to provide it interactively.")
subparser = subparsers.add_parser(
"set_user_entity",
diff --git a/src/caosdb/utils/plantuml.py b/src/caosdb/utils/plantuml.py
index 75b96ee3aa28ba916adc69e418de398abfe23356..be34b2604f3682bb71b48bbd73e00fe854b3af51 100644
--- a/src/caosdb/utils/plantuml.py
+++ b/src/caosdb/utils/plantuml.py
@@ -36,11 +36,24 @@ plantuml FILENAME.pu -> FILENAME.png
import os
import caosdb as db
+from caosdb.common.datatype import is_reference, get_referenced_recordtype
REFERENCE = "REFERENCE"
def get_description(description_str):
+ """Extract and format a description string from a record type or property.
+
+ Parameters
+ ----------
+ description_str : str
+ The description string that is going to be formatted.
+
+ Returns
+ -------
+ str
+ The reformatted description ending in a line break.
+ """
words = description_str.split()
lines = []
lines.append("")
@@ -211,15 +224,76 @@ package \"The property P references an instance of D\" <<Rectangle>> {
return result
+def retrieve_substructure(start_record_types, depth, result_id_set=None, result_container=None, cleanup=True):
+ """Recursively retrieves CaosDB record types and properties, starting
+ from given initial types up to a specific depth.
+
+ Parameters
+ ----------
+ start_record_types : Iterable[db.Entity]
+ Iterable with the entities to be displayed. Starting from these
+ entities more entities will be retrieved.
+ depth : int
+ The maximum depth up to which to retriev sub entities.
+ result_id_set : set[int]
+ Used by recursion. Filled with already visited ids.
+ result_container : db.Container
+ Used by recursion. Filled with already visited entities.
+ cleanup : bool
+ Used by recursion. If True return the resulting result_container.
+ Don't return anything otherwise.
+
+ Returns
+ -------
+ db.Container
+ A container containing all the retrieved entites or None if cleanup is False.
+ """
+ # Initialize the id set and result container for level zero recursion depth:
+ if result_id_set is None:
+ result_id_set = set()
+ if result_container is None:
+ result_container = db.Container()
+
+ for entity in start_record_types:
+ entity.retrieve()
+ if entity.id not in result_id_set:
+ result_container.append(entity)
+ result_id_set.add(entity.id)
+ for prop in entity.properties:
+ if is_reference(prop.datatype) and prop.datatype != db.FILE and depth > 0:
+ rt = db.RecordType(name=get_referenced_recordtype(prop.datatype)).retrieve()
+ retrieve_substructure([rt], depth-1, result_id_set, result_container, False)
+
+ if prop.id not in result_id_set:
+ result_container.append(prop)
+ result_id_set.add(prop.id)
+
+ for parent in entity.parents:
+ rt = db.RecordType(id=parent.id).retrieve()
+ if parent.id not in result_id_set:
+ result_container.append(rt)
+ result_id_set.add(parent.id)
+ if depth > 0:
+ retrieve_substructure([rt], depth-1, result_id_set, result_container, False)
+
+ if cleanup:
+ return result_container
+ return None
+
+
def to_graphics(recordtypes, filename):
- """ calls recordtypes_to_plantuml_string(), saves result to file and
+ """Calls recordtypes_to_plantuml_string(), saves result to file and
creates an svg image
- plantuml needs to be installed
- @params:
- recordtypes: itrable with the record types to be displayed
- filname: filename of the image (e.g. data_structure; data_structure.pu and
- data_structure.svg will be created.
+ plantuml needs to be installed.
+
+ Parameters
+ ----------
+ recordtypes : Iterable[db.Entity]
+ Iterable with the entities to be displayed.
+ filename : str
+ filename of the image without the extension(e.g. data_structure;
+ data_structure.pu and data_structure.svg will be created.)
"""
pu = recordtypes_to_plantuml_string(recordtypes)
diff --git a/src/caosdb/yamlapi.py b/src/caosdb/yamlapi.py
index b2e527983cbbeca2ee71a63e87f487d24d7c0301..80bb4b13e4d1626c5d29c8950f3a22bbb73e0fdb 100644
--- a/src/caosdb/yamlapi.py
+++ b/src/caosdb/yamlapi.py
@@ -22,7 +22,7 @@
# ** end header
#
-"""YAML interface for the database (caosdb)"""
+"""!!! Deprecated !!! YAML interface for the database (caosdb)"""
import yaml
from lxml import etree
@@ -31,9 +31,14 @@ import re
import caosdb
import caosdb.common.utils as utils
from caosdb.connection.connection import get_connection
+import warnings
def append_sublist(v, newel, def_entity_type):
+ warnings.warn("""
+ This function is deprecated and will be removed with the next release.
+ Please use caosdb-advanced-user-tools/models/data_model.py for a
+ similar functionality.""", DeprecationWarning)
if v is None:
return
for i in v:
@@ -46,6 +51,10 @@ def append_sublist(v, newel, def_entity_type):
def kv_to_xml(k, v):
+ warnings.warn("""
+ This function is deprecated and will be removed with the next release.
+ Please use caosdb-advanced-user-tools/models/data_model.py for a
+ similar functionality.""", DeprecationWarning)
newel = Element(k)
# code.interact(local=locals())
if isinstance(v, list): # Top level loop
@@ -69,15 +78,35 @@ def dict_to_xml(d):
d: The dictionary (possibly loaded from yaml)
to convert to caosdb-xml.
"""
+ warnings.warn("""
+ This function is deprecated and will be removed with the next release.
+ Please use caosdb-advanced-user-tools/models/data_model.py for a
+ similar functionality.""", DeprecationWarning)
return kv_to_xml("Entities", d)
def yaml_to_xml(yamlstr):
- return dict_to_xml(yaml.load(yamlstr))
+ warnings.warn("""
+ This function is deprecated and will be removed with the next release.
+ Please use caosdb-advanced-user-tools/models/data_model.py for a
+ similar functionality.""", DeprecationWarning)
+ """Load a yaml document from yamlstr and converts it to XML.
+
+ Parameters
+ ----------
+ yamlstr : str
+ The string to load the yaml document from.
+
+ """
+ return dict_to_xml(yaml.load(yamlstr, Loader=yaml.SafeLoader))
def process(text):
"""Do some replacements on the original file to obtain valid yaml."""
+ warnings.warn("""
+ This function is deprecated and will be removed with the next release.
+ Please use caosdb-advanced-user-tools/models/data_model.py for a
+ similar functionality.""", DeprecationWarning)
processed = re.sub(
"^(\\s*)-\\s*\\{?(.*)\\}?\\s*$",
"\\1- {\\2}",
@@ -90,6 +119,10 @@ def process(text):
def yaml_file_to_xml(yamlfilename):
+ warnings.warn("""
+ This function is deprecated and will be removed with the next release.
+ Please use caosdb-advanced-user-tools/models/data_model.py for a
+ similar functionality.""", DeprecationWarning)
with open(yamlfilename, "r") as f:
return yaml_to_xml(process(f.read()))
@@ -100,6 +133,10 @@ def insert_yaml_file(yamlfilename, simulate=False):
Set 'simulate' to True if you don't actually want to insert the xml,
but only receive what would be sent.
"""
+ warnings.warn("""
+ This function is deprecated and will be removed with the next release.
+ Please use caosdb-advanced-user-tools/models/data_model.py for a
+ similar functionality.""", DeprecationWarning)
con = get_connection()
prs = etree.XMLParser(remove_blank_text=True)
sent_xml = etree.tostring(
diff --git a/src/doc/Makefile b/src/doc/Makefile
index 5458c5300efc82e55686bc1cd6934182c5c8e39a..64219c5957ee963e84f9305685f2ec4e8ed3d761 100644
--- a/src/doc/Makefile
+++ b/src/doc/Makefile
@@ -32,6 +32,7 @@ PY_BASEDIR = ../caosdb
SOURCEDIR = .
BUILDDIR = ../../build/doc
+
.PHONY: doc-help Makefile
# Put it first so that "make" without argument is like "make help".
@@ -44,4 +45,4 @@ doc-help:
@$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
apidoc:
- @$(SPHINXAPIDOC) -o _apidoc $(PY_BASEDIR)
+ @$(SPHINXAPIDOC) -o _apidoc --separate $(PY_BASEDIR)
diff --git a/src/doc/administration.rst b/src/doc/administration.rst
index 91b85344a018618284349b4e8dd34fe6b365b94d..061acc8364d2ef62f743a20d7b9e6562baac0fc5 100644
--- a/src/doc/administration.rst
+++ b/src/doc/administration.rst
@@ -2,13 +2,13 @@ Administration
==============
The Python script ``caosdb_admin.py`` should be used for administrative tasks.
-Call ``python3 caosdb_admin.py --help`` to see how to use it.
+Call ``caosdb_admin.py --help`` to see how to use it.
The most common task is to create a new user (in the CaosDB realm) and set a
password for the user (note that a user typically needs to be activated)::
- python3 caosdb_admin.py create_user anna
- python3 caosdb_admin.py set_user_password anna
- python3 caosdb_admin.py add_user_roles anna administration
- python3 caosdb_admin.py activate_user anna
+ caosdb_admin.py create_user anna
+ caosdb_admin.py set_user_password anna
+ caosdb_admin.py add_user_roles anna administration
+ caosdb_admin.py activate_user anna
diff --git a/src/doc/conf.py b/src/doc/conf.py
index f276f325273b71d4b697bc57990259e842b2dbc3..b05fa1c71c1dcd0b59916594818449d2ebc574bd 100644
--- a/src/doc/conf.py
+++ b/src/doc/conf.py
@@ -8,16 +8,18 @@
# -- Path setup --------------------------------------------------------------
-# If extensions (or modules to document with autodoc) are in another directory,
-# add these directories to sys.path here. If the directory is relative to the
-# documentation root, use os.path.abspath to make it absolute, like shown here.
+# If extensions (or modules to document with autodoc) are in another directory, add these
+# directories to sys.path here. This is particularly necessary if this package is installed at a
+# different version, for example via `pip install`.
#
-# import os
-# import sys
-# sys.path.insert(0, os.path.abspath('../caosdb'))
-
+# If the directory is relative to the documentation root, use os.path.abspath to make it absolute,
+# like shown here.
+#
+import os
+import sys
+sys.path.insert(0, os.path.abspath('..'))
-import sphinx_rtd_theme
+import sphinx_rtd_theme # noqa: E402
# -- Project information -----------------------------------------------------
@@ -27,9 +29,10 @@ copyright = '2020, IndiScale GmbH'
author = 'Daniel Hornung'
# The short X.Y version
-version = '0.4.0'
+version = '0.5.2'
# The full version, including alpha/beta/rc tags
-release = '0.4.0-rc'
+# release = '0.5.2-rc2'
+release = '0.5.2'
# -- General configuration ---------------------------------------------------
@@ -43,6 +46,7 @@ release = '0.4.0-rc'
# ones.
extensions = [
'sphinx.ext.autodoc',
+ 'sphinx.ext.autosectionlabel',
'sphinx.ext.intersphinx',
'sphinx.ext.napoleon', # For Google style docstrings
"recommonmark", # For markdown files.
@@ -54,7 +58,6 @@ templates_path = ['_templates']
# The suffix(es) of source filenames.
# You can specify multiple suffix as a list of string:
-#
source_suffix = ['.rst', '.md']
# The master toctree document.
@@ -81,6 +84,7 @@ pygments_style = None
# The theme to use for HTML and HTML Help pages. See the documentation for
# a list of builtin themes.
#
+
html_theme = "sphinx_rtd_theme"
# Theme options are theme-specific and customize the look and feel of a theme
@@ -182,14 +186,22 @@ epub_exclude_files = ['search.html']
# -- Extension configuration -------------------------------------------------
+# True to prefix each section label with the name of the document it is in, followed by a colon. For
+# example, index:Introduction for a section called Introduction that appears in document
+# index.rst. Useful for avoiding ambiguity when the same section heading appears in different
+# documents.
+#
+# Note: This stops "normal" links from working, so it should be kept at False.
+# autosectionlabel_prefix_document = True
+
# -- Options for intersphinx -------------------------------------------------
# https://www.sphinx-doc.org/en/master/usage/extensions/intersphinx.html#confval-intersphinx_mapping
intersphinx_mapping = {
"python": ("https://docs.python.org/", None),
- "caosdb-mysqlbackend": ("https://caosdb.gitlab.io/caosdb-mysqlbackend/",
+ "caosdb-mysqlbackend": ("https://docs.indiscale.com/caosdb-mysqlbackend/",
None),
- "caosdb-server": ("https://caosdb.gitlab.io/caosdb-server/", None),
+ "caosdb-server": ("https://docs.indiscale.com/caosdb-server/", None),
}
diff --git a/src/doc/configuration.md b/src/doc/configuration.md
index b2de2781d5adff4d59cb3648cd912e142327f676..6e53542f661dcae94622fef24a67cecf7491df9c 100644
--- a/src/doc/configuration.md
+++ b/src/doc/configuration.md
@@ -50,5 +50,5 @@ debugging (which I hope will not be necessary for this tutorial) or if you want
the internals of the protocol.
A complete list of options can be found in the
-[pycaosdb.ini file](https://gitlab.com/caosdb/caosdb-pylib/-/blob/master/examples/pycaosdb.ini) in
+[pycaosdb.ini file](https://gitlab.com/caosdb/caosdb-pylib/-/blob/main/examples/pycaosdb.ini) in
the examples folder of the source code.
diff --git a/src/doc/index.rst b/src/doc/index.rst
index 76a2f88f6d31dd9b5f17995b6d54ccc63eb33631..bd29c6c56acf5c173e94ae6471a6aeba56ea4b93 100644
--- a/src/doc/index.rst
+++ b/src/doc/index.rst
@@ -12,12 +12,12 @@ Welcome to PyCaosDB's documentation!
Concepts <concepts>
Configuration <configuration>
Administration <administration>
- API documentation<_apidoc/modules>
+ API documentation<_apidoc/caosdb>
This is the documentation for the Python client library for CaosDB, ``PyCaosDB``.
-This documentation helps you to :doc:`get started<getting_started>`, explains the most important
-:doc:`concepts<concepts>` and offers a range of :doc:`tutorials<tutorials>`.
+This documentation helps you to :doc:`get started<README_SETUP>`, explains the most important
+:doc:`concepts<concepts>` and offers a range of :doc:`tutorials<tutorials/index>`.
Indices and tables
diff --git a/src/doc/tutorials/Data-Insertion.rst b/src/doc/tutorials/Data-Insertion.rst
index 2ead1cc259c807ee12f2459f7e452558eb1b63a2..f2c7f830d1403fbdf45354d1f36a4ea339759058 100644
--- a/src/doc/tutorials/Data-Insertion.rst
+++ b/src/doc/tutorials/Data-Insertion.rst
@@ -4,21 +4,24 @@ Data Insertion
Data Models
~~~~~~~~~~~
-Data is stored and structured in CaosDB using a concept of RecordTypes,
-Properties, Records etc. If you do not know what these are, please look
-at the chapter :any:`caosdb-server:Data Model` .
+Data is stored and structured in CaosDB using a concept of RecordTypes, Properties, Records etc. If
+you do not know what these are, please look at the chapter :doc:`Data
+Model<caosdb-server:Data-Model>` in the CaosDB server documentation.
In order to insert some actual data, we need to create a data model
using RecordTypes and Properties (You may skip this if you use a CaosDB
-instance that already has the required types). So, let’s create a simple
+instance that already has the required types). When you create a new Property
+you must supply a datatype. So, let’s create a simple
Property called “a” of datatype double. This is very easy in pylib:
.. code:: python
a = db.Property(name="a", datatype=db.DOUBLE)
-There are a few basic datatypes: db.INTEGER, db.TEXT. See `data
-type <Specification/Datatype>`__ for a full list.
+There are a few basic datatypes like db.INTEGER, db.DOUBLE, or db.TEXT. See the
+`data types
+<https://docs.indiscale.com/caosdb-server/specification/Datatype.html>`_ for a
+full list.
We can create our own small data model for e.g. a simulation by adding
two more Properties and a RecordType:
@@ -35,6 +38,42 @@ two more Properties and a RecordType:
container.extend([a, b, epsilon, recordtype])
container.insert()
+.. _tutorial-inheritance-properties:
+
+Inheritance of Properties
+-------------------------
+
+Suppose you want to create a new RecordType “2D_BarkleySimulation”
+that denotes spatially extended Barkley simulations. This is a subtype
+of the “BarkleySimulation” RecordType above and should have all its
+parameters, i.e., properties. It may be assigned more, e.g., spatial
+resolution, but we'll omit this for the sake of brevity for now.
+
+.. code:: python
+
+ rt = db.RecordType(name="2D_BarkleySimulation",
+ description="Spatially extended Barkley simulation")
+ # inherit all properties from the BarkleySimulation RecordType
+ rt.add_parent(name="BarkleySimulation", inheritance="all")
+ rt.insert()
+
+ print(rt.get_property(name="epsilon").importance) ### rt has a "epsilon" property with the same importance as "BarkleySimulation"
+
+The parameter ``inheritance=(obligatory|recommended|fix|all|none)`` of
+:py:meth:`Entity.add_parent()<caosdb.common.models.Entity.add_parent>` tells the server to assign
+all properties of the parent RecordType with the chosen importance (and properties with a higher
+importance) to the child RecordType
+automatically upon insertion. See the chapter on `importance
+<https://docs.indiscale.com/caosdb-server/specification/RecordType.html#importance>`_ in the
+documentation of the CaosDB server for more information on the importance and inheritance of
+properties.
+
+.. note::
+
+ The inherited properties will only be visible after the insertion since they are set by the
+ CaosDB server, not by the Python client.
+
+
Insert Actual Data
~~~~~~~~~~~~~~~~~~
@@ -99,41 +138,37 @@ Useful is also, that you can insert directly tabular data.
.. code:: python
- from caosadvancedtools.table_converter import from_tsv
-
- recs = from_tsv("test.csv", "Experiment")
- print(recs)
- recs.insert()
+ from caosadvancedtools.table_converter import from_tsv
+
+ recs = from_tsv("test.csv", "Experiment")
+ print(recs)
+ recs.insert()
With this example file
`test.csv <uploads/4f2c8756a26a3984c0af09d206d583e5/test.csv>`__.
-Inheritance of Properties
--------------------------
+List Properties
+---------------
+
+As you may already know, properties can also have list values instead of scalar
+values. They can be accessed, set, and updated as you would expect from any
+list-valued attribute in Python, as the following example illustrates.
+
+.. code:: python
+
+ import caosdb as db
+ db.Property(name="TestList", datatype=db.LIST(db.DOUBLE)).insert()
+ db.RecordType(name="TestType").add_property(name="TestList").insert()
+ db.Record(name="TestRec").add_parent("TestType").add_property(
+ name="TestList", value=[1,2,3]).insert()
+ retrieved = db.Record(name="TestRec").retrieve()
+ retrieved.get_property("TestList").value += [4,5]
+ retrieved.update()
+
+ # Check update
+ retrieved = db.Record(name="TestRec").retrieve()
+ print(retrieved.get_property("TestList").value)
-Given, you want to insert a new RecordType “Fridge temperatur
-experiment” as a child of the existing RecordType “Experiment”. The
-latter may have an obligatory Property “date” (since every experiment is
-conducted at some time). It is a natural way of thinking, that every sub
-type of “Experiment” also has this obligatory Property—in terms of
-object oriented programing the “Fridge temperatur experiment” *inherits*
-that Property.
-
-::
-
- rt = h.RecordType(name="Fridge temperatur experiment",
- description="RecordType which inherits all obligatory properties from Experiment"
- ).add_parent(name="Experiment", inheritance="obligatory").insert()
-
- print(rt.get_property(name="date").importance) ### rt now has a "date"-property -> this line prints "obligatory"
-
-The parameter *``inheritance=(obligatory|recommended|fix|all|none)``* of
-``add_parent`` tells the server to assign obligatory:: properties of the
-parent to the child automatically, recommended:: properties of the
-parent to the child automatically, fix:: properties of the parent to the
-child automatically, all:: properties of the parent to the child
-automatically, none:: of the properties of the parent to child
-automatically,
File Update
-----------
diff --git a/src/doc/tutorials/complex_data_models.rst b/src/doc/tutorials/complex_data_models.rst
new file mode 100644
index 0000000000000000000000000000000000000000..0fa868e78bb45a2905dc99392a3a28a9832d369e
--- /dev/null
+++ b/src/doc/tutorials/complex_data_models.rst
@@ -0,0 +1,76 @@
+Complex Data Models
+-------------------
+
+With CaosDB it is possible to create very complex data models.
+
+E.g. it is possible to add properties to properties to cover complex relations
+in data management workflows.
+
+One example for a use case is meta data that is added to very specific properties of
+datasets, e.g. data privacy information can be added to properties which themselves
+could already be considered meta data of a dataset.
+
+The example below tries to cover some complex cases for data models:
+
+Examples
+--------
+
+.. code-block:: python3
+
+ import caosdb as db
+
+ # Create two record types with descriptions:
+ rt1 = db.RecordType(name="TypeA", description="The first type")
+ rt2 = db.RecordType(name="TypeB", description="The second type")
+
+ # Create a record using the first record type as parent:
+ r1 = db.Record(name="Test_R_1", description="A record")
+ r1.add_parent(rt1)
+
+ # Create two files (the files named test.txt and testfile2.txt should exist in the
+ # current working directory:
+ f1 = db.File(name="Test file", path="/test.txt", file="test.txt")
+ f2 = db.File(name="Test file 2", path="/testfile2.txt", file="testfile2.txt")
+
+ # Create two properties with different data types:
+ p1 = db.Property(name="TestFileProperty", datatype=db.FILE)
+ p2 = db.Property(name="TestDoubleProperty", datatype=db.DOUBLE, unit="m")
+ p3 = db.Property(name="TestIntegerProperty", datatype=db.INTEGER, unit="s")
+
+ # Create a reference property that points to records of record type 2:
+ p4 = db.Property(name="TestReferenceProperty", datatype=rt2)
+
+ # Create a complex record:
+ r2 = db.Record(name="Test_R_2", description="A second record")
+ r2.add_parent(rt2)
+ r2.add_property(rt1, value=r1) # this is a reference to the first record type
+ r2.add_property(p1, value=f1) # this is a reference to the first file
+ r2.add_property(p2, value=24.8) # this is a double property with a value
+ r2.add_property(p3, value=1) # this is an integer property with a value
+
+ # Very complex part of the data model:
+ # Case 1: File added to another file
+ f2.add_property(p1, value=f1) # this adds a file property with value first file
+ # to the second file
+
+ # Case 2: Property added to a property
+ p2.add_property(p3, value=27) # this adds an integer property with value 27 to the
+ # double property
+
+ # Case 3: Reference property added to a property
+ # The property p2 now has two sub properties, one is pointing to
+ # record p2 which itself has the property p2, therefore this can be
+ # considered a loop in the data model.
+ p2.add_property(p4, value=r2) # this adds a reference property pointing to
+ # record 2 to the double property
+
+ # Insert a container containing all the newly created entities:
+ c = db.Container().extend([rt1, rt2, r1, r2, f1, p1, p2, p3, f2, p4])
+ c.insert()
+
+ # Useful for testing: wait until the user presses a key
+ # Meanwhile have a look at the WebUI: You can e.g. query "FIND Test*" to view
+ # all the entities created here and see the relations and links between them.
+ b = input("Press any key to cleanup.")
+ # cleanup everything after the user presses any button.
+ c.delete()
diff --git a/src/doc/tutorials/errors.rst b/src/doc/tutorials/errors.rst
index ba386dc31baad1021f01d53b2b12a2623d8278ad..37c53c9b527a0435f9f24ae6c6e71687e73eb963 100644
--- a/src/doc/tutorials/errors.rst
+++ b/src/doc/tutorials/errors.rst
@@ -1,53 +1,176 @@
Error Handling
---------------
+==============
-HeartDBException
-~~~~~~~~~~~~~~~~
+In case of erroneous transactions, connection problems and a lot of
+other cases, PyCaosDB may raise specific errors in order to pinpoint
+the problem as precisely as possible. Some of these errors a
+representations of errors in the CaosDB server, others stem from
+problems that occurred on the client side.
+
+The errors and exceptions are ordered hierarchically form the most
+general exceptions to specific transaction or connection problems. The
+most important error types and the hierarchy will be explained in the
+following. For more information on specific error types, see also the
+:doc:`source code<../_apidoc/caosdb.exceptions>`.
+
+.. note::
+
+ Starting from PyCaosDB 0.5, the error handling has changed
+ significantly. New error classes have been introduced and the
+ behavior of ``TransactionError`` and ``EntityError`` has been
+ re-worked. In the following, only the "new" errors are
+ discussed. Please refer to the documentation of PyCaosDB 0.4.1 and
+ earlier for the old error handling.
+
+CaosDBException
+----------------
+
+``CaosDBException`` is the most generic exception and all other error classes inherit
+from this one. Because of its generality, it doesn't tell you much
+except that some component of PyCaosDB raised an exception. If you
+want to catch all possible CaosDB errors, this is the class to use.
TransactionError
-~~~~~~~~~~~~~~~~
+----------------
Every transaction (calling ``insert``, ``update``, ``retrieve``, or
``delete`` on a container or an entity) may finish with errors. They
indicate, for instance, that an entity does not exist or that you need
to specify a data type for your property and much more. If and only if
one or more errors occur during a transaction a ``TransactionError``
-will be raised by the transaction method. The ``TransactionError`` class
-is a container for all errors which occur during a transaction. It can
-help you to find the crucial problems with your transaction by two
-important methods: \* ``get_errors()`` which returns a list of instances
-of ``EntityError``. \* ``get_entities()`` which returns a list of
-entities in the transaction container which are erroneous.
-
-Additionally, ``print(transaction_error`` prints a tree-like
+will be raised by the transaction method. The ``TransactionError``
+class is a container for all errors which occur during a
+transaction. It usually contains one or more :ref:`entity
+errors<EntityError>` which you can inspect in order to learn why the
+transaction failed. For this inspection, there are some helpful
+attributes and methods provided by the ``TransactionError``:
+
+* ``entities``: a list of all entities that directly caused at least one error
+ in this transaction.
+
+* ``errors``: a list of all ``EntityError`` objects that directly caused the
+ transaction to fail.
+
+* ``all_entities``, ``all_errors``: sets of all entities and errors
+ that, directly or indirectly, caused either this ``TransactionError`` or any of the
+ ``EntityError`` objects it contains.
+
+* ``has_error(error_t)``: Check whether an error of type ``error_t``
+ occurred during the transaction.
+
+Additionally, ``print(transaction_error)`` prints a tree-like
representation of all errors regarding the transaction in question.
EntityError
-~~~~~~~~~~~
+-----------
-An ``EntityError`` represents a single error that has been returned by
-the server. You might call \* ``get_entity()`` which returns the entity
-which caused the error. \* ``get_description()`` which returns a
-description of the error. \* ``get_code()`` which returns the error code
-(if specified) or 0 (if not).
+An ``EntityError`` specifies the entity and the error proper that
+caused a transaction to fail. It is never raised on its own but is
+contained in a ``TransactionError`` (which may or may not contain
+other ``EntityError`` objects) which is then raised. ``EntityError``
+has several :ref:`subclasses<Special Errors>` that further specify the
+error that occurred.
-In fact, the ``EntityError`` class is a subclass of
-``TransactionError``. So, it inherits the ``get_entities()``. Unless
-overridden by subclasses of ``EntityError``, it return a list with only
-one item—the entity which caused this error. Similarly, unless
-overridden by subclasses, the ``get_errors()`` method returns a list
-with only one item—``[self]``.
+The ``EntityError`` class is in fact a subclass of
+``TransactionError``. Thus, it has the same methods and attributes as
+the ``TransactionError`` explained
+:ref:`above<TransactionError>`. This is important in case of an
+``EntityError`` that was caused by other faulty entities (e.g., broken
+parents or properties). In that case these problematic entities and
+errors can again be inspected by visiting the ``entities`` and
+``errors`` lists as above.
Special Errors
~~~~~~~~~~~~~~
-Subclasses of ``EntityError`` for special purposes: \*
-``EntityDoesNotExistError`` \* ``EntityHasNoDataTypeError`` \*
-``UniqueNamesError`` \* ``UnqualifiedParentsError`` - overrides
-``get_entities()``: returns all parent entities with errors. - overrides
-``get_errors()``: returns a list of EntityErrors which have been caused
-by parent entities. \* ``UnqualifiedPropertiesError`` - overrides
-``get_entities()``: returns all properties with errors. - overrides
-``get_errors()``: returns a list of EntityErrors which have been caused
-by properties.
+Subclasses of ``EntityError`` for special purposes:
+
+* ``EntityDoesNotExistError``
+
+* ``EntityHasNoDataTypeError``
+
+* ``UniqueNamesError``
+
+* ``UnqualifiedParentsError``
+
+* ``UnqualifiedPropertiesError``
+
+* ``ConsistencyError``
+
+* ``AuthorizationError``
+
+* ``AmbiguousEntityError``
+
+BadQueryError
+-------------
+
+A ``BadQueryError`` is raised when a query could not be processed by
+the server. In contrast to a ``TransactionError`` it is not
+necessarily caused by problematic entities or
+containers. ``BadQueryError`` has the two important subclasses
+``EmptyUniqueQueryError`` and ``QueryNotUniqueError`` for queries with
+``unique=True`` which found no or ambiguous entities, respectively.
+
+HTTP Errors
+-----------
+
+An ``HTTPClientError`` or an ``HTTPServerError`` is raised in case of
+http(s) connection problems caused by the Python client or the CaosDB
+server, respectively. There are the following subclasses of
+``HTTPClientError`` that are used to specify the connection problem:
+
+* ``HTTPURITooLongError``: The URI of the request was too long to be
+ processed by the server.
+
+* ``HTTPForbiddenError``: You're not allowed to access this resource.
+
+* ``HTTPResourceNotFoundError``: The requested resource doesn't exist.
+
+Other Errors
+------------
+
+There are further subclasses of ``CaosDBException`` that are raised in
+case of faulty configurations or other problems. They should be rather
+self-explanatory from their names; see the :doc:`source code<../_apidoc/caosdb.exceptions>`
+for further information.
+
+* ``ConfigurationError``
+
+* ``LoginFailedError``
+
+* ``MismatchingEntitiesError``
+
+* ``ServerConfigurationException``
+
+Examples
+--------
+
+.. code-block:: python3
+
+ import caosdb as db
+
+ def link_and_insert(entity, linked, link=True):
+ """Link the ENTITY to LINKED and insert it."""
+ if link:
+ entity.add_property(db.Property(name="link", value=linked))
+ try:
+ entity.insert()
+ except db.TransactionError as tre:
+ # Unique names problem may be worked around by using another name
+ if tre.has_error(db.UniqueNamesError):
+ for ent_error in tre.errors:
+ if (isinstance(ent_error, db.UniqueNamesError)
+ and entity in ent_error.entities):
+ entity.name = entity.name + "_new" # Try again with new name.
+ link_and_insert(entity, linked, link=False)
+ break
+ # Unqualified properties will be handled by the caller
+ elif tre.has_error(db.UnqualifiedPropertiesError):
+ for ent_error in tre.errors:
+ if (isinstance(ent_error, db.UnqualifiedPropertiesError_
+ and entity in ent_error.entities):
+ raise RuntimeError("One of the properties was unqualified: " + str(ent_error))
+ # Other problems are not covered by this tutorial
+ else:
+ raise NotImplementedError("Unhandled TransactionError: " + str(tre))
diff --git a/src/doc/tutorials/first_steps.rst b/src/doc/tutorials/first_steps.rst
index 3eb804776960ba93acb9baf78824f0cf8a06e1f7..34b96bbeca416107fb34feb4707b9ef46fc49fe7 100644
--- a/src/doc/tutorials/first_steps.rst
+++ b/src/doc/tutorials/first_steps.rst
@@ -2,14 +2,15 @@ First Steps
===========
You should have a working connection to a CaosDB instance now. If not, please check out the
-:doc:`Getting Started secton</getting_started>`.
+:doc:`Getting Started secton</README_SETUP>`.
If you are not yet familiar with Records, RecordTypes and Properties used in CaosDB,
-please check out the respective part in the `Web Interface Tutorial`_.
-You should also know the basics of the CaosDB Query Language (a tutorial is here_).
+please check out the respective part in the `Web Interface tutorial`_.
+You should also know the basics of the CaosDB Query Language (a tutorial is
+`here <https://docs.indiscale.com/caosdb-webui/tutorials/query.html>`_).
-We recommend that you connect to the demo instance in order to try out the following
-examples. You can do this with
+We recommend that you connect to the `demo instance`_ (hosted by `Indiscale`_) in order to try out
+the following examples. You can do this with
>>> import caosdb as db
>>> _ = db.configure_connection(
@@ -19,7 +20,7 @@ examples. You can do this with
... password="caosdb")
or by using corresponding settings in the configuration file
-(see :doc:`Getting Started secton</getting_started>`.).
+(see :doc:`Getting Started secton</README_SETUP>`.).
However, you can also translate the examples to the data model that you have at hand.
Let's start with a simple query.
@@ -124,9 +125,6 @@ the result Records and their properties.
The next tutorial shows how to make some meaningful use of this.
-
-.. _here: https://gitlabio.something
.. _`demo instance`: https://demo.indiscale.com
.. _`IndiScale`: https://indiscale.com
-.. _`Web Interface Tutorial`: https://caosdb.gitlab.io/caosdb-webui/tutorials/model.html
-.. _here: https://caosdb.gitlab.io/caosdb-webui/tutorials/cql.html
+.. _`Web Interface tutorial`: https://docs.indiscale.com/caosdb-webui/tutorials/first_steps.html
diff --git a/src/doc/tutorials/index.rst b/src/doc/tutorials/index.rst
index 3889edb8f47e973cc7ae25c9134d75cfeab95f65..79068e9201498c87b2eb61b4ffbea0969845b404 100644
--- a/src/doc/tutorials/index.rst
+++ b/src/doc/tutorials/index.rst
@@ -15,4 +15,5 @@ advanced usage of the Python client.
Data-Insertion
errors
data-model-interface
-
+ complex_data_models
+
diff --git a/tox.ini b/tox.ini
index 94c2dc8affb280d3e7f6cff4536636432c9f7749..22c89f765c612ff78572ee2cab20dfab2e740e84 100644
--- a/tox.ini
+++ b/tox.ini
@@ -7,4 +7,5 @@ deps = .
nose
pytest
pytest-cov
+ jsonschema==4.0.1
commands=py.test --cov=caosdb -vv {posargs}
diff --git a/unittests/broken_configs/pycaosdb1.ini b/unittests/broken_configs/pycaosdb1.ini
new file mode 100644
index 0000000000000000000000000000000000000000..71180286881399c35e251bba89a54a345cd948ac
--- /dev/null
+++ b/unittests/broken_configs/pycaosdb1.ini
@@ -0,0 +1,5 @@
+[Connection]
+cacert=/very/long/path/to/self/signed/pem/file/caosdb.ca.pem
+url=https://hostname:8833/playground
+username=username
+password_method=pass
diff --git a/unittests/broken_configs/pycaosdb2.ini b/unittests/broken_configs/pycaosdb2.ini
new file mode 100644
index 0000000000000000000000000000000000000000..6bdf58ea812b83a622e647b2a18b01bb1a1e3099
--- /dev/null
+++ b/unittests/broken_configs/pycaosdb2.ini
@@ -0,0 +1,9 @@
+[Connection]
+url=https://0.0.0.0/
+username=username
+password_identifier=SECTION/SUBSECTION/identifier
+password_method=pass
+cacert=/etc/ssl/cert.pem
+ssl_insecure=true
+timeout=10000
+debug=9
diff --git a/unittests/broken_configs/pycaosdb3.ini b/unittests/broken_configs/pycaosdb3.ini
new file mode 100644
index 0000000000000000000000000000000000000000..62d1fed9497c0c258c13aa2ed8fecb23a3006849
--- /dev/null
+++ b/unittests/broken_configs/pycaosdb3.ini
@@ -0,0 +1,12 @@
+[connection]
+ssl_insecure=true
+url=https://localhost:10443/
+password=caosdb
+username=admin
+password_method=plain
+
+timeout=10000
+debug=0
+
+[Container]
+debug=0
diff --git a/unittests/broken_configs/pycaosdb4.ini b/unittests/broken_configs/pycaosdb4.ini
new file mode 100644
index 0000000000000000000000000000000000000000..e96604ac453ff2be4e2b419aa9ccbbf3598fa231
--- /dev/null
+++ b/unittests/broken_configs/pycaosdb4.ini
@@ -0,0 +1,10 @@
+[Connection]
+ssl_insecure=true
+url=https://localhost:10443/
+password=caosdb
+username=admin
+password_method=plain
+
+timeout=10000
+debug=0
+key=bla
\ No newline at end of file
diff --git a/unittests/docker/Dockerfile b/unittests/docker/Dockerfile
index e41fb91b3e9ae54cff277519c03f481c21264e9e..7fa3f75bd198724628dee48ab328829fa071a639 100644
--- a/unittests/docker/Dockerfile
+++ b/unittests/docker/Dockerfile
@@ -5,6 +5,6 @@ RUN apt-get update && \
curl pycodestyle \
python3-sphinx
ARG COMMIT="dev"
-RUN git clone -b dev https://gitlab.com/caosdb/caosdb-pylib.git && \
+RUN git clone -b dev https://gitlab.indiscale.com/caosdb/src/caosdb-pylib.git && \
cd caosdb-pylib && git checkout $COMMIT && pip3 install .
RUN pip3 install recommonmark sphinx-rtd-theme
diff --git a/unittests/test_add_property.py b/unittests/test_add_property.py
index 5bae6c219732f0170f5c351eae58148c9d3d065a..0d3183b4c0ca5517ecea68d0e49bbf335bb2a13e 100644
--- a/unittests/test_add_property.py
+++ b/unittests/test_add_property.py
@@ -264,3 +264,33 @@ def test_add_list_of_entitities():
for e in rec.get_property("listOfEntities").value:
assert i == e.id
i += 1
+
+
+def test_add_property_with_wrong_role():
+ entity = db.Entity()
+
+ r = db.Record()
+ rt = db.RecordType()
+ p = db.Property()
+ f = db.File()
+ e = db.Entity()
+
+ entity.add_property(rt)
+ entity.add_property(p)
+ entity.add_property(e)
+
+ with raises(ValueError) as cm:
+ entity.add_property(r)
+ assert cm.value.args[0] == ("The property parameter is a Record. This is "
+ "very unusual and probably not what you want. "
+ "Otherwise, construct a property from a "
+ "Record using the Property class and add that "
+ "to this entity.")
+
+ with raises(ValueError) as cm:
+ entity.add_property(f)
+ assert cm.value.args[0] == ("The property parameter is a File. This is "
+ "very unusual and probably not what you want. "
+ "Otherwise, construct a property from a File "
+ "using the Property class and add that to "
+ "this entity.")
diff --git a/unittests/test_apiutils.py b/unittests/test_apiutils.py
index c560b5e3c7c424b762bc8381c7cc9f42617288d0..717595d049a5a7c27b644dc6079d02efa1cefb71 100644
--- a/unittests/test_apiutils.py
+++ b/unittests/test_apiutils.py
@@ -26,10 +26,14 @@
# Test apiutils
# A. Schlemmer, 02/2018
-import caosdb as db
import pickle
import tempfile
-from caosdb.apiutils import apply_to_ids
+
+import caosdb as db
+import caosdb.apiutils
+from caosdb.apiutils import (apply_to_ids, compare_entities, create_id_query,
+ resolve_reference)
+
from .test_property import testrecord
@@ -62,3 +66,81 @@ def test_apply_to_ids():
assert rec.parents[0].id == -3456
assert rec.properties[0].id == -23345
assert rec.id == -23
+
+
+def test_id_query():
+ ids = [1, 2, 3, 4, 5]
+ assert create_id_query(ids) == 'FIND ENTITY WITH ID=1 OR ID=2 OR ID=3 OR '\
+ 'ID=4 OR ID=5'
+
+
+def test_resolve_reference():
+ original_retrieve_entity_with_id = caosdb.apiutils.retrieve_entity_with_id
+ caosdb.apiutils.retrieve_entity_with_id = lambda eid: db.Record(id=eid)
+
+ prop = db.Property(id=1, datatype=db.REFERENCE, value=100)
+ prop.is_valid = lambda: True
+ items = [200, 300, 400]
+ prop_list = db.Property(datatype=db.LIST(db.REFERENCE),
+ value=items)
+ prop_list2 = db.Property(datatype=db.LIST(db.REFERENCE),
+ value=[db.Record(id=500)])
+ resolve_reference(prop)
+ resolve_reference(prop_list)
+ resolve_reference(prop_list2)
+ assert prop.value.id == 100
+ assert isinstance(prop.value, db.Entity)
+
+ prop_list_ids = []
+
+ for i in prop_list.value:
+ prop_list_ids.append(i.id)
+ assert isinstance(i, db.Entity)
+ assert prop_list_ids == items
+
+ for i in prop_list2.value:
+ assert i.id == 500
+ assert isinstance(i, db.Entity)
+
+ no_reference = db.Property(id=5000, datatype=db.INTEGER, value=2)
+ resolve_reference(no_reference)
+ assert no_reference.value == 2
+ assert no_reference.datatype is db.INTEGER
+
+ # restore retrive_entity_with_id
+ caosdb.apiutils.retrieve_entity_with_id = original_retrieve_entity_with_id
+
+
+def test_compare_entities():
+ r1 = db.Record()
+ r2 = db.Record()
+ r1.add_parent("bla")
+ r2.add_parent("bla")
+ r1.add_parent("lopp")
+ r1.add_property("test", value=2)
+ r2.add_property("test", value=2)
+ r1.add_property("tests", value=3)
+ r2.add_property("tests", value=45)
+ r1.add_property("tester", value=3)
+ r2.add_property("tester", )
+ r1.add_property("tests_234234", value=45)
+ r2.add_property("tests_TT", value=45)
+
+ diff_r1, diff_r2 = compare_entities(r1, r2)
+
+ assert len(diff_r1["parents"]) == 1
+ assert len(diff_r2["parents"]) == 0
+ assert len(diff_r1["properties"]) == 3
+ assert len(diff_r2["properties"]) == 3
+
+ assert "test" not in diff_r1["properties"]
+ assert "test" not in diff_r2["properties"]
+
+ assert "tests" in diff_r1["properties"]
+ assert "tests" in diff_r2["properties"]
+
+ assert "tester" in diff_r1["properties"]
+ assert "tester" in diff_r2["properties"]
+
+ assert "tests_234234" in diff_r1["properties"]
+ assert "tests_TT" in diff_r2["properties"]
diff --git a/unittests/test_authentication_plain.py b/unittests/test_authentication_plain.py
index 10cbc418df8bd81c81568a4df0cf1e8a4ac498f8..146b59889c71c86ea77fb4ae962118cdda1afb06 100644
--- a/unittests/test_authentication_plain.py
+++ b/unittests/test_authentication_plain.py
@@ -65,4 +65,6 @@ def test_subclass_configure():
def test_plain_has_logger():
p = PlainTextCredentialsProvider()
assert hasattr(p, "logger")
- assert p.logger.name == "authentication"
+ assert "authentication" in p.logger.name
+ assert "connection" in p.logger.name
+ assert "caosdb" in p.logger.name
diff --git a/unittests/test_configs/pycaosdb-indiscale-demo.ini b/unittests/test_configs/pycaosdb-indiscale-demo.ini
new file mode 100644
index 0000000000000000000000000000000000000000..9010343467f78c7c9c3e25ea3a57520deac18c8e
--- /dev/null
+++ b/unittests/test_configs/pycaosdb-indiscale-demo.ini
@@ -0,0 +1,12 @@
+[Connection]
+url=https://demo.indiscale.com/
+username=admin
+password=caosdb
+password_method=plain
+cacert=/etc/ssl/cert.pem
+
+timeout=10000
+debug=0
+
+[Container]
+debug=0
\ No newline at end of file
diff --git a/unittests/test_configs/pycaosdb1.ini b/unittests/test_configs/pycaosdb1.ini
new file mode 100644
index 0000000000000000000000000000000000000000..dcfa7c21fac735d81ab92b33f0abd31df25fc1ad
--- /dev/null
+++ b/unittests/test_configs/pycaosdb1.ini
@@ -0,0 +1,6 @@
+[Connection]
+cacert=/very/long/path/to/self/signed/pem/file/caosdb.ca.pem
+url=https://hostname:8833/playground
+password_identifier=SECTION/caosdb
+username=username
+password_method=pass
diff --git a/unittests/test_configs/pycaosdb2.ini b/unittests/test_configs/pycaosdb2.ini
new file mode 100644
index 0000000000000000000000000000000000000000..e5493bde3be0f84f38e427edb4d42fba9c75482d
--- /dev/null
+++ b/unittests/test_configs/pycaosdb2.ini
@@ -0,0 +1,9 @@
+[Connection]
+url=https://0.0.0.0/
+username=username
+password_identifier=SECTION/SUBSECTION/identifier
+password_method=pass
+cacert=/etc/ssl/cert.pem
+ssl_insecure=true
+timeout=10000
+debug=0
diff --git a/unittests/test_configs/pycaosdb3.ini b/unittests/test_configs/pycaosdb3.ini
new file mode 100644
index 0000000000000000000000000000000000000000..6c4934039c99855566f38c69f7511d774f81efbd
--- /dev/null
+++ b/unittests/test_configs/pycaosdb3.ini
@@ -0,0 +1,12 @@
+[Connection]
+ssl_insecure=true
+url=https://localhost:10443/
+password=caosdb
+username=admin
+password_method=plain
+
+timeout=10000
+debug=0
+
+[Container]
+debug=0
diff --git a/unittests/test_configuration.py b/unittests/test_configuration.py
index 76445b6f262120d6a29c73527a9bf042f85f8a05..b135e7cd65b11be7cb6c4ef2237a41a6639ccbb7 100644
--- a/unittests/test_configuration.py
+++ b/unittests/test_configuration.py
@@ -22,19 +22,45 @@
# ** end header
#
+import pytest
import caosdb as db
+from os import environ, getcwd, remove
+from os.path import expanduser, isfile, join
from pytest import raises
-def test_config_ini_via_envvar():
- from os import environ
- from os.path import expanduser
+@pytest.fixture
+def temp_ini_files():
+ created_temp_ini_cwd = False
+ created_temp_ini_home = False
+ if not isfile(join(getcwd(), "pycaosdb.ini")):
+ open("pycaosdb.ini", 'a').close() # create temporary ini file
+ created_temp_ini_cwd = True
+ if not isfile(expanduser("~/.pycaosdb.ini")):
+ open(expanduser("~/.pycaosdb.ini"), 'a').close() # create temporary ini file in home directory
+ created_temp_ini_home = True
+ yield 0
+ if created_temp_ini_cwd:
+ remove("pycaosdb.ini")
+ if created_temp_ini_home:
+ remove(expanduser("~/.pycaosdb.ini"))
+ environ["PYCAOSDBINI"] = "~/.pycaosdb.ini"
+
+
+def test_config_ini_via_envvar(temp_ini_files):
with raises(KeyError):
environ["PYCAOSDBINI"]
environ["PYCAOSDBINI"] = "bla bla"
assert environ["PYCAOSDBINI"] == "bla bla"
- assert db.configuration.configure(environ["PYCAOSDBINI"]) == []
+ # test wrong configuration file in envvar
+ assert not expanduser(environ["PYCAOSDBINI"]) in db.configuration._read_config_files()
+ # test good configuration file in envvar
environ["PYCAOSDBINI"] = "~/.pycaosdb.ini"
- assert db.configuration.configure(expanduser(environ["PYCAOSDBINI"])) == [expanduser("~/.pycaosdb.ini")]
+ assert expanduser("~/.pycaosdb.ini") in db.configuration._read_config_files()
+ # test without envvar
+ environ.pop("PYCAOSDBINI")
+ assert expanduser("~/.pycaosdb.ini") in db.configuration._read_config_files()
+ # test configuration file in cwd
+ assert join(getcwd(), "pycaosdb.ini") in db.configuration._read_config_files()
diff --git a/unittests/test_container.py b/unittests/test_container.py
index b34055372fc83a5608ffcf54423a601001add12b..0ac4be44826825aa3302119c8bca08f335ab68d3 100644
--- a/unittests/test_container.py
+++ b/unittests/test_container.py
@@ -25,28 +25,28 @@
"""Tests for the Container class."""
from __future__ import absolute_import
-import caosdb as c
+import caosdb as db
def test_get_property_values():
- rt_house = c.RecordType("House")
- rt_window = c.RecordType("Window")
- rt_owner = c.RecordType("Owner")
- p_height = c.Property("Height", datatype=c.DOUBLE)
+ rt_house = db.RecordType("House")
+ rt_window = db.RecordType("Window")
+ rt_owner = db.RecordType("Owner")
+ p_height = db.Property("Height", datatype=db.DOUBLE)
- window = c.Record().add_parent(rt_window)
+ window = db.Record().add_parent(rt_window)
window.id = 1001
window.add_property(p_height, 20.5, unit="m")
- owner = c.Record("The Queen").add_parent(rt_owner)
+ owner = db.Record("The Queen").add_parent(rt_owner)
- house = c.Record("Buckingham Palace")
+ house = db.Record("Buckingham Palace")
house.add_parent(rt_house)
house.add_property(rt_owner, owner)
house.add_property(rt_window, window)
house.add_property(p_height, 40.2, unit="ft")
- container = c.Container()
+ container = db.Container()
container.extend([
house,
owner
@@ -77,3 +77,70 @@ def test_get_property_values():
assert container.get_property_values("non-existing") == [(None,), (None,)]
assert container.get_property_values("name") == [(house.name,),
(owner.name,)]
+
+
+def test_container_dependencies_for_deletion():
+ not_included_rt = 1000
+ rt = db.RecordType("Just a RecordType")
+ rt.id = 1001
+ rt_record_with_parent = db.RecordType("Records with parent")
+ rt_record_with_parent.id = 1005
+ property_which_is_not_a_record = db.Property(
+ "Normal Property", datatype=db.DOUBLE, value=1006)
+ property_which_is_not_a_record.id = 1006
+ property_which_shall_be_deleted = db.Property(
+ "Normal Property 2", datatype=db.DOUBLE, value=1006)
+ property_which_shall_be_deleted .id = 1007
+
+ record_without_dependencies = db.Record().add_parent(not_included_rt)
+ record_without_dependencies.id = 2003
+
+ record_referenced = db.Record().add_parent(not_included_rt)
+ record_referenced.id = 2002
+ record_with_dependencies = db.Record().add_parent(not_included_rt)
+ record_with_dependencies.id = 2004
+ record_with_dependencies.add_property(not_included_rt,
+ record_referenced,
+ datatype="not_included_rt")
+
+ record_with_parent = db.Record().add_parent(rt_record_with_parent)
+ record_with_parent.id = 2005
+
+ record_with_property_which_is_not_a_record = db.Record(
+ ).add_parent(not_included_rt)
+ record_with_property_which_is_not_a_record.id = 2006
+ record_with_property_which_is_not_a_record.add_property(
+ property_which_is_not_a_record)
+ record_with_property_which_is_not_a_record.add_property(
+ property_which_shall_be_deleted)
+
+ container = db.Container()
+ container.extend([
+ rt,
+ rt_record_with_parent, # 1005, dependency
+ record_without_dependencies,
+ property_which_shall_be_deleted, # 1007, dependency
+ record_referenced, # 2002, dependency
+ record_with_dependencies,
+ record_with_parent,
+ record_with_property_which_is_not_a_record
+ ])
+ assert (db.Container()._test_dependencies_in_container(container)
+ == {2002, 1005, 1007})
+
+
+def test_container_dependencies_for_deletion_with_lists():
+ not_included_rt = 1000
+
+ record_referenced = db.Record().add_parent(not_included_rt)
+ record_referenced.id = 2001
+
+ record_with_list = db.Record().add_parent(not_included_rt)
+ record_with_list.id = 2002
+ record_with_list.add_property(not_included_rt, datatype=db.LIST(
+ not_included_rt), value=[record_referenced, 2003, 2004, 2005, 2006])
+
+ container = db.Container()
+ container.extend([record_with_list, record_referenced])
+
+ assert db.Container()._test_dependencies_in_container(container) == {2001}
diff --git a/unittests/test_entity.py b/unittests/test_entity.py
index e98dfbef5b6b5e5f691e8aecc2fa7d4a86991452..1e88702ac016d7dcfdf00919dd0f93b5d3345e00 100644
--- a/unittests/test_entity.py
+++ b/unittests/test_entity.py
@@ -24,6 +24,7 @@
"""Tests for the Entity class."""
# pylint: disable=missing-docstring
import unittest
+from lxml import etree
from caosdb import (INTEGER, Entity, Property, Record, RecordType,
configure_connection)
@@ -42,17 +43,54 @@ class TestEntity(unittest.TestCase):
def test_instance_variables(self):
entity = Entity()
self.assertTrue(hasattr(entity, "role"))
+ self.assertIsNone(entity.role)
self.assertTrue(hasattr(entity, "id"))
self.assertTrue(hasattr(entity, "name"))
self.assertTrue(hasattr(entity, "description"))
self.assertTrue(hasattr(entity, "parents"))
self.assertTrue(hasattr(entity, "properties"))
- def test_role(self):
+ def test_entity_role_1(self):
entity = Entity(role="TestRole")
self.assertEqual(entity.role, "TestRole")
entity.role = "TestRole2"
self.assertEqual(entity.role, "TestRole2")
- def test_instanciation(self):
+ def test_entity_role_2(self):
+ entity = Entity()
+
+ self.assertIsNone(entity.role)
+ self.assertEqual(entity.to_xml().tag, "Entity")
+
+ entity.role = "Record"
+ self.assertEqual(entity.role, "Record")
+ self.assertEqual(entity.to_xml().tag, "Record")
+
+ def test_recordtype_role(self):
+ entity = RecordType()
+
+ self.assertEqual(entity.role, "RecordType")
+ self.assertEqual(entity.to_xml().tag, "RecordType")
+
+ def test_property_role(self):
+ entity = Property()
+
+ self.assertEqual(entity.role, "Property")
+ self.assertEqual(entity.to_xml().tag, "Property")
+
+ def test_instantiation(self):
self.assertRaises(Exception, Entity())
+
+ def test_parse_role(self):
+ """During parsing, the role of an entity is set explicitely. All other
+ classes use the class name as a "natural" value for the role property.
+ """
+ parser = etree.XMLParser(remove_comments=True)
+ entity = Entity._from_xml(Entity(),
+ etree.parse("unittests/test_record.xml",
+ parser).getroot())
+
+ self.assertEqual(entity.role, "Record")
+ # test whether the __role property of this object has explicitely been
+ # set.
+ self.assertEqual(getattr(entity, "_Entity__role"), "Record")
diff --git a/unittests/test_property.py b/unittests/test_property.py
index 752ee01f0eafef14dbffd1e62c99d1c816c45d05..7c756117765e510587c00d818e39fb3945d44c53 100644
--- a/unittests/test_property.py
+++ b/unittests/test_property.py
@@ -24,14 +24,18 @@
# ** end header
#
"""Tests for the Property class."""
+import os
+
+import caosdb as db
+from caosdb import Entity, Property, Record
# pylint: disable=missing-docstring
from lxml import etree
-from caosdb import Entity, Property, Record
parser = etree.XMLParser(remove_comments=True)
-testrecord = Record._from_xml(Record(),
- etree.parse("unittests/test_record.xml",
- parser).getroot())
+testrecord = Record._from_xml(
+ Record(),
+ etree.parse(os.path.join(os.path.dirname(__file__), "test_record.xml"),
+ parser).getroot())
def test_is_entity():
@@ -47,7 +51,8 @@ def test_instance_variables():
def test_null_empty_text_value_1():
- assert testrecord.get_property("LISTofTEXT").value == ["One", "Two", "Three", None, ""]
+ assert testrecord.get_property("LISTofTEXT").value == ["One", "Two",
+ "Three", None, ""]
def test_null_empty_text_value_2():
@@ -89,3 +94,47 @@ def test_get_property_with_entity():
def test_selected_reference_list():
assert len(testrecord.get_property("Conductor").value) == 1
assert isinstance(testrecord.get_property("Conductor").value[0], Entity)
+
+
+def test_is_reference():
+ PROPS = {
+ 10: db.INTEGER,
+ 20: db.REFERENCE,
+ 30: "SomeRT",
+ }
+
+ def dummy_retrieve(self):
+ self.datatype = PROPS[self.id]
+ self.is_valid = lambda: True
+ # replace retrieve function by dummy
+ real_retrieve = Entity.retrieve
+ Entity.retrieve = dummy_retrieve
+
+ p1 = Property(id=1, datatype=db.INTEGER)
+ p2 = Property(id=2, datatype=db.DOUBLE)
+ p3 = Property(id=3, datatype=db.TEXT)
+ p4 = Property(id=4, datatype=db.DATETIME)
+ p5 = Property(id=5, datatype=db.BOOLEAN)
+ p6 = Property(id=6, datatype=db.REFERENCE)
+ assert p1.is_reference() is False
+ assert p2.is_reference() is False
+ assert p3.is_reference() is False
+ assert p4.is_reference() is False
+ assert p5.is_reference() is False
+ assert p6.is_reference() is True
+
+ p7 = Property(id=7)
+ p8 = Property(id=8, value=db.RecordType(id=1000))
+ p8.is_valid = lambda: True
+ assert p7.is_reference() is None # cannot be resolved without calling a server
+ assert p8.is_reference() is True
+
+ p10 = Property(id=10)
+ p20 = Property(id=20)
+ p30 = Property(id=30)
+ assert p10.is_reference(server_retrieval=True) is False
+ assert p20.is_reference(server_retrieval=True) is True
+ assert p30.is_reference(server_retrieval=True) is True
+
+ # restore retrieve function with original
+ Entity.retrieve = real_retrieve
diff --git a/unittests/test_query.py b/unittests/test_query.py
index f4b3ee9762d9d76b65ace9a5b9b3f4039c0c6919..12622ea486dda717ca1fbc1255510575c5e0c8e6 100644
--- a/unittests/test_query.py
+++ b/unittests/test_query.py
@@ -26,20 +26,23 @@ import caosdb as db
def test_query_parsing():
- s = '<Query string="FIND bla" results="0" cached="true"/>'
+ s = '<Query string="FIND bla" results="0" cached="true" etag="asdf"/>'
q = db.Query(etree.fromstring(s))
assert q.q == "FIND bla"
assert q.results == 0
assert q.cached is True
+ assert q.etag == "asdf"
- s = '<Query string="COUNT bla" results="1" cached="false"/>'
+ s = '<Query string="COUNT bla" results="1" cached="false" etag="asdf"/>'
q = db.Query(etree.fromstring(s))
assert q.q == "COUNT bla"
assert q.results == 1
assert q.cached is False
+ assert q.etag == "asdf"
s = '<Query string="COUNT blub" results="4"/>'
q = db.Query(etree.fromstring(s))
assert q.q == "COUNT blub"
assert q.results == 4
assert q.cached is False
+ assert q.etag is None
diff --git a/unittests/test_schema.py b/unittests/test_schema.py
new file mode 100644
index 0000000000000000000000000000000000000000..1552179a3e43dacb3ecca705466bb7ff84d330cf
--- /dev/null
+++ b/unittests/test_schema.py
@@ -0,0 +1,45 @@
+# -*- coding: utf-8 -*-
+#
+# This file is a part of the CaosDB Project.
+#
+# Copyright (C) 2021 Alexander Schlemmer
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see <https://www.gnu.org/licenses/>.
+#
+
+"""Test configuration schema.
+A. Schlemmer, 01/2021
+"""
+from jsonschema.exceptions import ValidationError
+from pytest import raises
+from glob import glob
+import os
+from caosdb.configuration import config_to_yaml, validate_yaml_schema
+from configparser import ConfigParser
+
+
+def test_config_files():
+ for fn in glob(os.path.join(os.path.dirname(__file__), "test_configs", "*.ini")):
+ c = ConfigParser()
+ c.read(fn)
+ validate_yaml_schema(config_to_yaml(c))
+
+
+def test_broken_config_files():
+ for fn in glob(os.path.join(os.path.dirname(__file__), "broken_configs", "*.ini")):
+ print(fn)
+ with raises(ValidationError):
+ c = ConfigParser()
+ c.read(fn)
+ validate_yaml_schema(config_to_yaml(c))
diff --git a/unittests/test_state.py b/unittests/test_state.py
new file mode 100644
index 0000000000000000000000000000000000000000..202c7a02af3db28434406626e5164def46febed7
--- /dev/null
+++ b/unittests/test_state.py
@@ -0,0 +1,77 @@
+import pytest
+import caosdb as db
+from caosdb import State, Transition
+from caosdb.common.models import parse_xml, ACL
+from lxml import etree
+
+
+def test_state_xml():
+ state = State(model="model1", name="state1")
+ xml = etree.tostring(state.to_xml())
+
+ assert xml == b'<State name="state1" model="model1"/>'
+ state = State.from_xml(etree.fromstring(xml))
+ assert state.name == "state1"
+ assert state.model == "model1"
+
+ assert xml == etree.tostring(state.to_xml())
+
+
+def test_entity_xml():
+ r = db.Record()
+ assert r.state is None
+ r.state = State(model="model1", name="state1")
+
+ xml = etree.tostring(r.to_xml())
+ assert xml == b'<Record><State name="state1" model="model1"/></Record>'
+
+ r = parse_xml(xml)
+ assert r.state == State(model="model1", name="state1")
+
+
+def test_description():
+ xml = b'<State name="state1" model="model1"/>'
+ state = State.from_xml(etree.fromstring(xml))
+ assert state.description is None
+
+ with pytest.raises(AttributeError):
+ state.description = "test"
+
+ xml = b'<State name="state1" model="model1" description="test2"/>'
+ state = State.from_xml(etree.fromstring(xml))
+ assert state.description == "test2"
+
+
+def test_id():
+ xml = b'<State name="state1" model="model1"/>'
+ state = State.from_xml(etree.fromstring(xml))
+ assert state.id is None
+
+ with pytest.raises(AttributeError):
+ state.id = "2345"
+
+ xml = b'<State name="state1" model="model1" id="1234"/>'
+ state = State.from_xml(etree.fromstring(xml))
+ assert state.id == "1234"
+
+
+def test_create_state_acl():
+ acl = ACL()
+ acl.grant(role="role1", permission="DO:IT")
+ acl.grant(role="?OWNER?", permission="DO:THAT")
+ state_acl = State.create_state_acl(acl)
+ assert state_acl.get_permissions_for_role("?STATE?role1?") == {"DO:IT"}
+ assert state_acl.get_permissions_for_role("?STATE??OWNER??") == {"DO:THAT"}
+
+
+def test_transitions():
+ xml = b'<State name="state1" model="model1"/>'
+ state = State.from_xml(etree.fromstring(xml))
+ assert state.transitions is None
+
+ with pytest.raises(AttributeError):
+ state.transitions = []
+
+ xml = b'<State name="state1" model="model1" id="1234"><Transition name="t1"><FromState name="state1"/><ToState name="state2"/></Transition></State>'
+ state = State.from_xml(etree.fromstring(xml))
+ assert state.transitions == set([Transition(name="t1", from_state="state1", to_state="state2")])
diff --git a/unittests/test_yamlapi.py b/unittests/test_yamlapi.py
new file mode 100644
index 0000000000000000000000000000000000000000..cdb1e0499890ee58d10ff7f102632e104ef60868
--- /dev/null
+++ b/unittests/test_yamlapi.py
@@ -0,0 +1,85 @@
+# -*- coding: utf-8 -*-
+#
+# This file is a part of the CaosDB Project.
+#
+# Copyright (C) 2021 IndiScale GmbH <info@indiscale.com>
+# Copyright (C) 2021 Alexander Kreft <akreft@trineo.org>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see <https://www.gnu.org/licenses/>.
+#
+
+import os
+import warnings
+import tempfile
+from caosdb.yamlapi import (append_sublist, kv_to_xml,
+ dict_to_xml, yaml_to_xml,
+ process, yaml_file_to_xml)
+
+with warnings.catch_warnings(record=True) as w:
+ # Cause all warnings to always be triggered.
+ warnings.simplefilter("always")
+
+ append_sublist(None, None, None)
+
+ assert issubclass(w[-1].category, DeprecationWarning)
+ assert "This function is deprecated" in str(w[-1].message)
+
+with warnings.catch_warnings(record=True) as w:
+ # Cause all warnings to always be triggered.
+ warnings.simplefilter("always")
+
+ kv_to_xml("None", "None")
+ assert len(w) == 1
+ assert issubclass(w[-1].category, DeprecationWarning)
+ assert "This function is deprecated" in str(w[-1].message)
+
+with warnings.catch_warnings(record=True) as w:
+ # Cause all warnings to always be triggered.
+ warnings.simplefilter("always")
+
+ dict_to_xml(None)
+
+ assert issubclass(w[-1].category, DeprecationWarning)
+ assert "This function is deprecated" in str(w[-1].message)
+
+with warnings.catch_warnings(record=True) as w:
+ # Cause all warnings to always be triggered.
+ warnings.simplefilter("always")
+
+ yaml_to_xml("None")
+
+ assert issubclass(w[-1].category, DeprecationWarning)
+ assert "This function is deprecated" in str(w[-1].message)
+
+with warnings.catch_warnings(record=True) as w:
+ # Cause all warnings to always be triggered.
+ warnings.simplefilter("always")
+
+ process("None")
+
+ assert issubclass(w[-1].category, DeprecationWarning)
+ assert "This function is deprecated" in str(w[-1].message)
+
+with warnings.catch_warnings(record=True) as w:
+ # Cause all warnings to always be triggered.
+ warnings.simplefilter("always")
+
+ with tempfile.TemporaryDirectory() as tmpdirname:
+ tmpfile = os.path.join(tmpdirname, 'yamlfile')
+ with open(tmpfile, 'w') as tf:
+ tf.write("")
+ yaml_file_to_xml(tmpfile)
+
+ assert issubclass(w[-1].category, DeprecationWarning)
+ assert "This function is deprecated" in str(w[-1].message)