diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index d99da405739bd301736a46941cda9cc2437462c7..749b9a6dd4b9dce8fd04dc3dfac48ff7c1ae9def 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -39,15 +39,18 @@ stages:
code_style:
tags: [ docker ]
stage: code_style
+ needs: [ ]
script:
- - pycodestyle --count ./
+ - make style
allow_failure: true
+# pylint tests for pycaosdb
pylint:
tags: [ docker ]
stage: linting
+ needs: [ ]
script:
- - pylint3 --unsafe-load-any-extension=y -d all -e E,F src/caosdb/common
+ - make lint
allow_failure: true
mypy:
@@ -57,26 +60,33 @@ mypy:
- mypy src/caosdb
allow_failure: true
-# pylint tests for pycaosdb
-test:
+# run unit tests
+unittest:
tags: [ docker ]
stage: test
+ needs: [ ]
script:
- touch ~/.pycaosdb.ini
- - tox -r
+ - make unittest
# Trigger building of server image and integration tests
trigger_build:
- tags: [ docker ]
stage: deploy
- script:
- - /usr/bin/curl -X POST
- -F token=$CI_JOB_TOKEN
- -F "variables[F_BRANCH]=$CI_COMMIT_REF_NAME"
- -F "variables[PYLIB]=$CI_COMMIT_REF_NAME"
- -F "variables[TriggerdBy]=PYLIB"
- -F "variables[TriggerdByHash]=$CI_COMMIT_SHORT_SHA"
- -F ref=$DEPLOY_REF https://gitlab.indiscale.com/api/v4/projects/14/trigger/pipeline
+ inherit:
+ variables:
+ - DEPLOY_REF
+ variables:
+ # Renaming variables.
+ F_BRANCH: $CI_COMMIT_REF_NAME
+ PYLIB: $CI_COMMIT_REF_NAME
+ TRIGGERED_BY_REPO: PYLIB
+ TRIGGERED_BY_REF: $CI_COMMIT_REF_NAME
+ TRIGGERED_BY_HASH: $CI_COMMIT_SHORT_SHA
+
+ trigger:
+ project: caosdb/src/caosdb-deploy
+ branch: $DEPLOY_REF
+ strategy: depend
# Build a docker image in which tests for this repository can run
build-testenv:
@@ -103,6 +113,7 @@ build-testenv:
pages_prepare: &pages_prepare
tags: [ cached-dind ]
stage: deploy
+ needs: [ code_style, pylint, unittest ]
only:
refs:
- /^release-.*$/i
diff --git a/.gitlab/merge_request_templates/Default.md b/.gitlab/merge_request_templates/Default.md
deleted file mode 100644
index 77a95da1cc40c815e4952a1283d345af56e80461..0000000000000000000000000000000000000000
--- a/.gitlab/merge_request_templates/Default.md
+++ /dev/null
@@ -1,49 +0,0 @@
-# Summary
-
- Insert a meaningful description for this merge request here. What is the
- new/changed behavior? Which bug has been fixed? Are there related Issues?
-
-# Focus
-
- Point the reviewer to the core of the code change. Where should they start
- reading? What should they focus on (e.g. security, performance,
- maintainability, user-friendliness, compliance with the specs, finding more
- corner cases, concrete questions)?
-
-# Test Environment
-
- How to set up a test environment for manual testing?
-
-# Check List for the Author
-
-Please, prepare your MR for a review. Be sure to write a summary and a
-focus and create gitlab comments for the reviewer. They should guide the
-reviewer through the changes, explain your changes and also point out open
-questions. For further good practices have a look at [our review
-guidelines](https://gitlab.com/caosdb/caosdb/-/blob/dev/REVIEW_GUIDELINES.md)
-
-- [ ] All automated tests pass
-- [ ] Reference related Issues
-- [ ] Up-to-date CHANGELOG.md
-- [ ] Annotations in code (Gitlab comments)
- - Intent of new code
- - Problems with old code
- - Why this implementation?
-
-
-# Check List for the Reviewer
-
-
-- [ ] I understand the intent of this MR
-- [ ] All automated tests pass
-- [ ] Up-to-date CHANGELOG.md
-- [ ] The test environment setup works and the intended behavior is
- reproducible in the test environment
-- [ ] In-code documentation and comments are up-to-date.
-- [ ] Check: Are there spezifications? Are they satisfied?
-
-For further good practices have a look at [our review guidelines](https://gitlab.com/caosdb/caosdb/-/blob/dev/REVIEW_GUIDELINES.md).
-
-
-/assign me
-/target_branch dev
diff --git a/CHANGELOG.md b/CHANGELOG.md
index f1fab72ed234c2e94fac2ded0a0a7be6e11fca03..aa036beccf9e7f0ab42bebd1fedc455d5ea250e1 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -5,26 +5,153 @@ All notable changes to this project will be documented in this file.
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
-## [Unreleased] ##
+## [Unreleased]
### Added ###
+### Changed ###
+
+* Set PyYAML dependency back to PyYaml>=5.4.1 (from 6.0) for better
+ compatibility with docker-compose)
+
+### Deprecated ###
+
+### Removed ###
+
+### Fixed ###
+
+### Security ###
+
+### Documentation ###
+
+## [0.8.0] - 2022-07-12
+(Timm Fitschen)
+
+### Removed ###
+
+* Support for Python 3.6 and Python 3.7
+
+### Fixed ###
+
+* `read()` of MockupResponse returns now an appropriate type on modern systems
+* [caosdb-server#142](https://gitlab.com/caosdb/caosdb-server/-/issues/142)
+ Can't create users with dots in their user names
+
+## [0.7.4] - 2022-05-31
+(Florian Spreckelsen)
+
+### Fixed ###
+
+* [#64](https://gitlab.com/caosdb/caosdb-pylib/-/issues/64) Use `Dict[]` and
+ `List[]` from `typing` for type hinting instead of `dict[]` and `list[]` for
+ compatibility with Python<3.9.
+
+## [0.7.3] - 2022-05-03
+(Henrik tom Wörden)
+
+### Added ###
+
+- New function in apiutils that copies an Entity.
+- New EXPERIMENTAL module `high_level_api` which is a completely refactored version of
+ the old `high_level_api` from apiutils. Please see the included documentation for details.
+- `to_graphics` now has `no_shadow` option.
+
+### Changed ###
+
+- Added additional customization options to the plantuml module.
+- The to_graphics function in the plantuml module uses a temporary directory now for creating the output files.
+
+### Fixed ###
+
+* [#75](https://gitlab.indiscale.com/caosdb/src/caosdb-pylib/-/issues/75), [#103](https://gitlab.indiscale.com/caosdb/src/caosdb-pylib/-/issues/103) Fixed JSON schema to allow more sections, and correct requirements for
+ password method.
+
+## [0.7.2] - 2022-03-25 ##
+(Timm Fitschen)
+
+### Deprecated ###
+
+* In module `caosdb.apiutils`:
+ * `CaosDBPythonEntity` class
+ * `convert_to_entity` function
+ * `convert_to_python_object` function
+
+### Fixed ###
+
+* [caosdb-pylib#106](https://gitlab.indiscale.com/caosdb/src/caosdb-pylib/-/issues/106)
+ Parsing Error in class caosdb.common.models.ACL. This may lead to the
+ unintentional revocation of permissions for some users or roles during
+ updates. However, no additional permissions are being granted.
+
+### Documentation ###
+
+## [0.7.1] - 2022-03-11 ##
+(Daniel Hornung)
+
+### Documentation ###
+
+- `timeout` option in example pycaosdb.ini
+
+## [0.7.0] - 2022-01-21 ##
+
+### Added ###
+
+- Function in administration that generates passwords that comply with the rules.
+
+### Fixed ###
+
+- #90 compare_entities function in apiutils does not check units
+- #94 some special properties were not checked in compare_entities
+
+### Security ###
+
+## [0.6.1] - 2021-12-03 ##
+
+### Fixed ###
+
+- #50 keyring can be used as password input method again
+* #81 compare_entities from apiutils does not compare entity values
+
+## [0.6.0] - 2021-10-19 ##
+
+### Added ###
+
+- It is possible now to supply a password for caosdb_admin on the command line
+ and also activate the user directly using "-c".
* Added examples for complex data models to documentation
* extended apiutils with `resolve_reference(Property)`
* is_reference function for Properties
+* function `retrieve_substructure` that recursively adds connected entities.
### Changed ###
-- It is possible now to supply a password for caosdb_admin on the command line and also activate the user directly using "-c".
+* Retrievals of entities where the class does not match the entity role raise
+ a ValueError now. See
+ [here](https://gitlab.indiscale.com/caosdb/src/caosdb-pylib/-/issues/66) for more
+ information. Updating a role is now being done by setting the `Entity.role`
+ to the new role (as string).
+* Entity.add_property and Entity.add_parent do not accept `**kwargs`-style
+ keywords anymore. Formerly known keywords have been refactored into named
+ parameters.
+* [#35](https://gitlab.com/caosdb/caosdb-pylib/-/issues/35) Loggers now use the
+ name of the unit where they are called instead of a static name
### Deprecated ###
-* `id_query(ids)` in apiutils
-
-### Removed ###
+* `id_query(ids)` in apiutils (to be removed with >=0.5.4)
+* The whole yamlapi with the following functions (to be removed with >=0.5.4):
+ * `append_sublist`
+ * `kv_to_xml`
+ * `dict_to_xml`
+ * `yaml_to_xml`
+ * `process`
+ * `yaml_file_to_xml`
+ * `insert_yaml_file`
### Fixed ###
+* #60 Unintuitive behavior of `Entity.role` after a `Entity(id).retrieve()`
+ Originally the role was `None`. The correct role is present now.
* #53 Documentation of inheritance
* #38 Dependencies in chunk-deletion of containers
@@ -45,10 +172,6 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
* Updated error-handling tutorial in documentation to reflect the new
error classes
-### Deprecated ###
-
-### Removed ###
-
### Fixed ###
* #45 - test_config_ini_via_envvar
@@ -141,14 +264,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
assumed that `password_method` is `auth_token` if the the `auth_token` is
set.
-### Removed ###
-
### Fixed ###
- Replaced deprecated Logger.warn() method.
-### Security ###
-
## [0.3.0] - 2020-04-24##
### Added ###
@@ -156,10 +275,6 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
* `apiutils.apply_to_ids` -- a helper which applies a function to all ids which
are used by an entity (own entity, parents, properties, references etc.).
-### Changed ###
-
-### Deprecated ###
-
### Fixed ###
* import bugs in apiutils
diff --git a/DEPENDENCIES.md b/DEPENDENCIES.md
index 04e783a7fa31b1d5c3a600a1009c8f040db1620d..cccdbcba9df2cc63e08a93c8b99dc13c92211f7f 100644
--- a/DEPENDENCIES.md
+++ b/DEPENDENCIES.md
@@ -1,6 +1,5 @@
-* caosdb-server == 0.3
-* Python >= 3.5
+* caosdb-server >= 0.8.0
+* Python >= 3.8
* pip >= 20.0.2
-
-Any other dependencies are being installed via pip
+Any other dependencies are defined in the setup.py and are being installed via pip
diff --git a/FEATURES.md b/FEATURES.md
new file mode 100644
index 0000000000000000000000000000000000000000..977d7a482279af00bc5e2b02a13d5d23564f1d04
--- /dev/null
+++ b/FEATURES.md
@@ -0,0 +1,8 @@
+# Experimental Features
+
+- High Level API in the module `high_level_api` is experimental and might be removed in future. It is for playing around with a possible future implementation of the Python client. See `src/doc/future_caosdb.md`
+
+
+# Features
+TODO: This is currently an incomplete list.
+- `to_graphics` defined in `caosdb.utils.plantuml` can be used to create an UML diagram of a data model
diff --git a/Makefile b/Makefile
index 4bc3459d209936c17a445c64e77180d9559e4653..0a0888ad0484c0307583e139e65058c38574ed3a 100644
--- a/Makefile
+++ b/Makefile
@@ -31,3 +31,18 @@ doc:
install:
@echo "Not implemented yet, use pip for installation."
+
+check: style lint
+.PHONY: check
+
+style:
+ pycodestyle --count examples src unittests
+.PHONY: style
+
+lint:
+ pylint --unsafe-load-any-extension=y -d all -e E,F src/caosdb/common
+.PHONY: lint
+
+unittest:
+ tox -r
+.PHONY: unittest
diff --git a/README.md b/README.md
index 04b34cbc07c98e73740b13200ed83fe067af99d2..602df33cecfc8ec37fd791e3257221e66f120cb3 100644
--- a/README.md
+++ b/README.md
@@ -47,7 +47,7 @@ However, you can also create an issue for it.
* Copyright (C) 2018 Research Group Biomedical Physics, Max Planck Institute
for Dynamics and Self-Organization Göttingen.
-* Copyright (C) 2020-2021 Indiscale GmbH <info@indiscale.com>
+* Copyright (C) 2020-2022 Indiscale GmbH <info@indiscale.com>
All files in this repository are licensed under a [GNU Affero General Public
License](LICENCE.md) (version 3 or later).
diff --git a/README_SETUP.md b/README_SETUP.md
index 9da548395073643c16539cef180c4d6412dd8d46..48928d6c3f2c878a8d8b268b36ed2cdeba7f8014 100644
--- a/README_SETUP.md
+++ b/README_SETUP.md
@@ -4,18 +4,22 @@
### Requirements ###
-PyCaosDB needs at least Python 3.6. Additionally, the following packages are required (they will
+PyCaosDB needs at least Python 3.8. Additionally, the following packages are required (they will
typically be installed automatically):
- `lxml`
- `PyYaml`
- `PySocks`
+Optional packages:
+- `keyring`
+- `jsonschema`
+
### How to install ###
#### Linux ####
-Make sure that Python (at least version 3.6) and pip is installed, using your system tools and
+Make sure that Python (at least version 3.8) and pip is installed, using your system tools and
documentation.
Then open a terminal and continue in the [Generic installation](#generic-installation) section.
@@ -30,12 +34,14 @@ packages you will ever need out of the box. If you prefer, you may also install
After installation, open an Anaconda prompt from the Windows menu and continue in the [Generic
installation](#generic-installation) section.
-#### iOS ####
+#### MacOS ####
-If there is no Python 3 installed yet, there are two main ways to obtain it: Either get the binary
-package from [python.org](https://www.python.org/downloads/) or, for advanced users, install via [Homebrew](https://brew.sh/). After installation
-from python.org, it is recommended to also update the TLS certificates for Python (this requires
-administrator rights for your user):
+If there is no Python 3 installed yet, there are two main ways to
+obtain it: Either get the binary package from
+[python.org](https://www.python.org/downloads/) or, for advanced
+users, install via [Homebrew](https://brew.sh/). After installation
+from python.org, it is recommended to also update the TLS certificates
+for Python (this requires administrator rights for your user):
```sh
# Replace this with your Python version number:
@@ -45,7 +51,8 @@ cd /Applications/Python\ 3.9/
sudo ./Install\ Certificates.command
```
-After these steps, you may continue with the [Generic installation](#generic-installation).
+After these steps, you may continue with the [Generic
+installation](#generic-installation).
#### Generic installation ####
@@ -66,62 +73,17 @@ cd caosdb-pylib
pip3 install --user .
```
-## Configuration ##
-
-The configuration is done using `ini` configuration files.
-PyCaosDB tries to read from the inifile specified in the environment variable `PYCAOSDBINI` or
-alternatively in `~/.pycaosdb.ini` upon import. After that, the ini file `pycaosdb.ini` in the
-current working directory will be read additionally, if it exists.
-
-Here, we will look at the most common configuration options. For a full and
-comprehensive description please check out
-[pycaosdb.ini file](https://gitlab.com/caosdb/caosdb-pylib/-/blob/main/examples/pycaosdb.ini)
-You can download this file and use it as a starting point.
-
-
-Typically, you need to change at least the `url` and `username` fields as required.
-(Ask your CaosDB administrator or IT crowd if
-you do not know what to put there, but for the demo instances https://demo.indiscale.com, `username=admin`
-and `password=caosdb` should work).
-
-### Authentication ###
-
-The default configuration (that your are asked for your password when ever a connection is created
-can be changed by setting `password_method`:
+For installation of optional packages, install with an additional option, e.g. for
+validating with the caosdb json schema:
-* with `password_method=input` password (and possibly user) will be queried on demand (**default**)
-* use the password manager [pass](https://www.passwordstore.org) by using `pass` as value, see also the [ArchWiki
- entry](https://wiki.archlinux.org/index.php/Pass#Basic_usage). This also requires `password_identifier` which refers to the identifier within pass
- for the desired password.
-* install the python package [keyring](https://pypi.org/project/keyring), to use the system keyring/wallet (macOS, GNOME, KDE,
- Windows). The password will be queried on first usage.
-* with `password_method=plain` (**strongly discouraged**)
-
-The following illustrates the recommended options:
-
-```ini
-[Connection]
-# using "pass" password manager
-#password_method=pass
-#password_identifier=...
-
-# using the system keyring/wallet (macOS, GNOME, KDE, Windows)
-#password_method=keyring
+```sh
+pip3 install --user .[jsonschema]
```
-### SSL Certificate ###
-In some cases (especially if you are testing CaosDB) you might need to supply
-an SSL certificate to allow SSL encryption.
-
-```ini
-[Connection]
-cacert=/path/to/caosdb.ca.pem
-```
+## Configuration ##
-### Further Settings ###
-As mentioned above, a complete list of options can be found in the
-[pycaosdb.ini file](https://gitlab.com/caosdb/caosdb-pylib/-/blob/main/examples/pycaosdb.ini) in
-the examples folder of the source code.
+The configuration is done using `ini` configuration files. The content of these configuration files
+is described in detail in the [configuration section of the documentation](https://docs.indiscale.com/caosdb-pylib/configuration.html).
## Try it out ##
@@ -141,7 +103,10 @@ like this, check out the "Authentication" section in the [configuration document
Now would be a good time to continue with the [tutorials](tutorials/index).
## Run Unit Tests
-tox
+
+- Run all tests: `tox` or `make unittest`
+- Run a specific test file: e.g. `tox -- unittests/test_schema.py`
+- Run a specific test function: e.g. `tox -- unittests/test_schema.py::test_config_files`
## Documentation ##
diff --git a/RELEASE_GUIDELINES.md b/RELEASE_GUIDELINES.md
index e015b598117abdcd575cf17e2f095fec459a4c4c..00d0362e6630267c135e2566b993a91fccf1fd91 100644
--- a/RELEASE_GUIDELINES.md
+++ b/RELEASE_GUIDELINES.md
@@ -1,6 +1,6 @@
# Release Guidelines for the CaosDB Python Client Library
-This document specifies release guidelines in addition to the generel release
+This document specifies release guidelines in addition to the general release
guidelines of the CaosDB Project
([RELEASE_GUIDELINES.md](https://gitlab.com/caosdb/caosdb/blob/dev/RELEASE_GUIDELINES.md))
@@ -20,9 +20,10 @@ guidelines of the CaosDB Project
3. Check all general prerequisites.
-4. Prepare [setup.py](./setup.py): Check the `MAJOR`, `MINOR`, `MICRO`, `PRE`
- variables and set `ISRELEASED` to `True`. Use the possibility to issue
- pre-release versions for testing.
+4. Update the version:
+ - `version` variables in `src/doc/conf.py`
+ - Version on [setup.py](./setup.py): Check the `MAJOR`, `MINOR`, `MICRO`, `PRE` variables and set
+ `ISRELEASED` to `True`. Use the possibility to issue pre-release versions for testing.
5. Merge the release branch into the main branch.
@@ -35,8 +36,12 @@ guidelines of the CaosDB Project
9. Publish the release by executing `./release.sh` with uploads the caosdb
module to the Python Package Index [pypi.org](https://pypi.org).
-10. Merge the main branch back into the dev branch.
+10. Create a gitlab release on gitlab.indiscale.com and gitlab.com
-11. After the merge of main to dev, start a new development version by
- setting `ISRELEASED` to `False` and by increasing at least the `MIRCO`
- version in [setup.py](./setup.py)
+11. Merge the main branch back into the dev branch.
+
+12. After the merge of main to dev, start a new development version by
+ setting `ISRELEASED` to `False` and by increasing at least the `MICRO`
+ version in [setup.py](./setup.py).
+ Also update CHANGELOG.md (new "Unreleased" section).
+ Also update `src/doc/conf.py`.
diff --git a/examples/pycaosdb.ini b/examples/pycaosdb.ini
index edc32195fbb364bb355d67b8733e8c7bccbb0d34..8cf74e43c5db32ed139c4fe371a6c2b3831b2ee1 100644
--- a/examples/pycaosdb.ini
+++ b/examples/pycaosdb.ini
@@ -67,3 +67,6 @@
# This option is used internally and for testing. Do not override.
# implementation=_DefaultCaosDBServerConnection
+
+# The timeout for requests to the server.
+# timeout=1000
diff --git a/setup.py b/setup.py
index e1d39458ea8d1b0b17ea12a82ebd7133b27b045a..93514b9b0c5f07b716bc5082e712fe1f101d8f36 100755
--- a/setup.py
+++ b/setup.py
@@ -45,11 +45,15 @@ from setuptools import find_packages, setup
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
########################################################################
-MAJOR = 0
-MINOR = 5
-MICRO = 3
-PRE = "" # e.g. rc0, alpha.1, 0.beta-23
ISRELEASED = False
+MAJOR = 0
+MINOR = 9
+MICRO = 0
+# Do not tag as pre-release until this commit
+# https://github.com/pypa/packaging/pull/515
+# has made it into a release. Probably we should wait for pypa/packaging>=21.4
+# https://github.com/pypa/packaging/releases
+PRE = "dev" # "dev" # e.g. rc0, alpha.1, 0.beta-23
if PRE:
VERSION = "{}.{}.{}-{}".format(MAJOR, MINOR, MICRO, PRE)
@@ -154,16 +158,27 @@ def setup_package():
long_description_content_type="text/markdown",
author='Timm Fitschen',
author_email='t.fitschen@indiscale.com',
+ url='https://www.caosdb.org',
+ license="AGPLv3+",
+ classifiers=[
+ "Programming Language :: Python :: 3",
+ "License :: OSI Approved :: GNU Affero General Public License v3 or later (AGPLv3+)",
+ "Operating System :: OS Independent",
+ "Topic :: Database",
+ "Topic :: Scientific/Engineering :: Information Analysis",
+ ],
packages=find_packages('src'),
- python_requires='>=3.6',
+ python_requires='>=3.8',
package_dir={'': 'src'},
- install_requires=['lxml>=3.6.4',
- 'PyYaml>=3.12', 'future', 'PySocks>=1.6.7'],
- extras_require={'keyring': ['keyring>=13.0.0']},
+ install_requires=['lxml>=4.6.3',
+ 'PyYAML>=5.4.1', 'future', 'PySocks>=1.6.7'],
+ extras_require={'keyring': ['keyring>=13.0.0'],
+ 'jsonschema': ['jsonschema>=4.4.0']},
setup_requires=["pytest-runner>=2.0,<3dev"],
- tests_require=["pytest", "pytest-cov", "coverage>=4.4.2"],
+ tests_require=["pytest", "pytest-cov", "coverage>=4.4.2",
+ "jsonschema>=4.4.0"],
package_data={
- 'caosdb': ['cert/indiscale.ca.crt'],
+ 'caosdb': ['cert/indiscale.ca.crt', 'schema-pycaosdb-ini.yml'],
},
scripts=["src/caosdb/utils/caosdb_admin.py"]
)
diff --git a/src/caosdb/apiutils.py b/src/caosdb/apiutils.py
index 9192289a0b6518a2d6ec3abcdeca9d1d2c063d6d..4c8393111bcbb4f9f91e309b81bebdcac55ba626 100644
--- a/src/caosdb/apiutils.py
+++ b/src/caosdb/apiutils.py
@@ -1,12 +1,11 @@
# -*- coding: utf-8 -*-
#
-# ** header v3.0
# This file is a part of the CaosDB Project.
#
# Copyright (C) 2018 Research Group Biomedical Physics,
# Max-Planck-Institute for Dynamics and Self-Organization Göttingen
# Copyright (C) 2020 Timm Fitschen <t.fitschen@indiscale.com>
-# Copyright (C) 2020 IndiScale GmbH <info@indiscale.com>
+# Copyright (C) 2020-2022 IndiScale GmbH <info@indiscale.com>
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as
@@ -30,15 +29,19 @@ Some simplified functions for generation of records etc.
import sys
import tempfile
-from collections.abc import Iterable
import warnings
+from collections.abc import Iterable
from subprocess import call
+from typing import Optional, Any, Dict, List
+
from caosdb.common.datatype import (BOOLEAN, DATETIME, DOUBLE, FILE, INTEGER,
REFERENCE, TEXT, is_reference)
from caosdb.common.models import (Container, Entity, File, Property, Query,
- Record, RecordType, get_config,
- execute_query)
+ Record, RecordType, execute_query,
+ get_config, SPECIAL_ATTRIBUTES)
+
+import logging
def new_record(record_type, name=None, description=None,
@@ -98,22 +101,6 @@ def create_id_query(ids):
["ID={}".format(id) for id in ids])
-def retrieve_entity_with_id(eid):
- return execute_query("FIND ENTITY WITH ID={}".format(eid), unique=True)
-
-
-def retrieve_entities_with_ids(entities):
- collection = Container()
- step = 20
-
- for i in range(len(entities)//step+1):
- collection.extend(
- execute_query(
- create_id_query(entities[i*step:(i+1)*step])))
-
- return collection
-
-
def get_type_of_entity_with(id_):
objs = retrieve_entities_with_ids([id_])
@@ -137,377 +124,20 @@ def get_type_of_entity_with(id_):
return Entity
-class CaosDBPythonEntity(object):
-
- _last_id = 0
-
- def __init__(self):
- # Save a copy of the dry state
- # of this object in order to be
- # able to detect conflicts.
- self.do_not_expand = False
- self._parents = []
- self._id = CaosDBPythonEntity._get_id()
- self._path = None
- self._file = None
- self.pickup = None
- # TODO:
- # 3.) resolve references up to a specific depth (including infinity)
- # 4.) resolve parents function -> partially implemented by
- # get_parent_names
- self._references = {}
- self._properties = set()
- self._datatypes = {}
- self._forbidden = dir(self)
-
- @staticmethod
- def _get_id():
- CaosDBPythonEntity._last_id -= 1
-
- return CaosDBPythonEntity._last_id
-
- def _set_property_from_entity(self, ent):
- name = ent.name
- val = ent.value
- pr = ent.datatype
- val, reference = self._type_converted_value(val, pr)
- self.set_property(name, val, reference, datatype=pr)
-
- def set_property(self, name, value, is_reference=False,
- overwrite=False, datatype=None):
- """
- overwrite: Use this if you definitely only want one property with that name (set to True).
- """
- self._datatypes[name] = datatype
-
- if isinstance(name, Entity):
- name = name.name
-
- if name in self._forbidden:
- raise RuntimeError("Entity cannot be converted to a corresponding "
- "Python representation. Name of property " +
- name + " is forbidden!")
- already_exists = (name in dir(self))
-
- if already_exists and not overwrite:
- # each call to _set_property checks first if it already exists
- # if yes: Turn the attribute into a list and
- # place all the elements into that list.
- att = self.__getattribute__(name)
-
- if isinstance(att, list):
- pass
- else:
- old_att = self.__getattribute__(name)
- self.__setattr__(name, [old_att])
-
- if is_reference:
- self._references[name] = [
- self._references[name]]
- att = self.__getattribute__(name)
- att.append(value)
-
- if is_reference:
- self._references[name].append(int(value))
- else:
- if is_reference:
- self._references[name] = value
- self.__setattr__(name, value)
-
- if not (already_exists and overwrite):
- self._properties.add(name)
-
- add_property = set_property
-
- def set_id(self, idx):
- self._id = idx
-
- def _type_converted_list(self, val, pr):
- """Convert a list to a python list of the correct type."""
- prrealpre = pr.replace("<", "<").replace(">", ">")
- prreal = prrealpre[prrealpre.index("<") + 1:prrealpre.rindex(">")]
- lst = [self._type_converted_value(i, prreal) for i in val]
-
- return ([i[0] for i in lst], lst[0][1])
-
- def _type_converted_value(self, val, pr):
- """Convert val to the correct type which is indicated by the database
- type string in pr.
-
- Returns a tuple with two entries:
- - the converted value
- - True if the value has to be interpreted as an id acting as a reference
- """
-
- if val is None:
- return (None, False)
- elif pr == DOUBLE:
- return (float(val), False)
- elif pr == BOOLEAN:
- return (bool(val), False)
- elif pr == INTEGER:
- return (int(val), False)
- elif pr == TEXT:
- return (val, False)
- elif pr == FILE:
- return (int(val), False)
- elif pr == REFERENCE:
- return (int(val), True)
- elif pr == DATETIME:
- return (val, False)
- elif pr[0:4] == "LIST":
- return self._type_converted_list(val, pr)
- elif isinstance(val, Entity):
- return (convert_to_python_object(val), False)
- else:
- return (int(val), True)
-
- def attribute_as_list(self, name):
- """This is a workaround for the problem that lists containing only one
- element are indistinguishable from simple types in this
- representation."""
- att = self.__getattribute__(name)
-
- if isinstance(att, list):
- return att
- else:
- return [att]
-
- def _add_parent(self, parent):
- self._parents.append(parent.id)
-
- def add_parent(self, parent_id=None, parent_name=None):
- if parent_id is not None:
- self._parents.append(parent_id)
- elif parent_name is not None:
- self._parents.append(parent_name)
- else:
- raise ValueError("no parent identifier supplied")
-
- def get_parent_names(self):
- new_plist = []
-
- for p in self._parents:
- obj_type = get_type_of_entity_with(p)
- ent = obj_type(id=p).retrieve()
- new_plist.append(ent.name)
-
- return new_plist
-
- def resolve_references(self, deep=False, visited=dict()):
- for i in self._references:
- if isinstance(self._references[i], list):
- for j in range(len(self._references[i])):
- new_id = self._references[i][j]
- obj_type = get_type_of_entity_with(new_id)
-
- if new_id in visited:
- new_object = visited[new_id]
- else:
- ent = obj_type(id=new_id).retrieve()
- new_object = convert_to_python_object(ent)
- visited[new_id] = new_object
-
- if deep:
- new_object.resolve_references(deep, visited)
- self.__getattribute__(i)[j] = new_object
- else:
- new_id = self._references[i]
- obj_type = get_type_of_entity_with(new_id)
-
- if new_id in visited:
- new_object = visited[new_id]
- else:
- ent = obj_type(id=new_id).retrieve()
- new_object = convert_to_python_object(ent)
- visited[new_id] = new_object
-
- if deep:
- new_object.resolve_references(deep, visited)
- self.__setattr__(i, new_object)
-
- def __str__(self, indent=1, name=None):
- if name is None:
- result = str(self.__class__.__name__) + "\n"
- else:
- result = name + "\n"
-
- for p in self._properties:
- value = self.__getattribute__(p)
-
- if isinstance(value, CaosDBPythonEntity):
- result += indent * "\t" + \
- value.__str__(indent=indent + 1, name=p)
- else:
- result += indent * "\t" + p + "\n"
-
- return result
-
-
-class CaosDBPythonRecord(CaosDBPythonEntity):
- pass
-
-
-class CaosDBPythonRecordType(CaosDBPythonEntity):
- pass
-
-
-class CaosDBPythonProperty(CaosDBPythonEntity):
- pass
-
-
-class CaosDBPythonFile(CaosDBPythonEntity):
- def get_File(self, target=None):
- f = File(id=self._id).retrieve()
- self._file = f.download(target)
-
-
-def _single_convert_to_python_object(robj, entity):
- robj._id = entity.id
-
- for i in entity.properties:
- robj._set_property_from_entity(i)
-
- for i in entity.parents:
- robj._add_parent(i)
-
- if entity.path is not None:
- robj._path = entity.path
-
- if entity.file is not None:
- robj._file = entity.file
- # if entity.pickup is not None:
- # robj.pickup = entity.pickup
-
- return robj
-
-
-def _single_convert_to_entity(entity, robj, **kwargs):
- if robj._id is not None:
- entity.id = robj._id
-
- if robj._path is not None:
- entity.path = robj._path
-
- if robj._file is not None:
- entity.file = robj._file
-
- if robj.pickup is not None:
- entity.pickup = robj.pickup
- children = []
+def retrieve_entity_with_id(eid):
+ return execute_query("FIND ENTITY WITH ID={}".format(eid), unique=True)
- for parent in robj._parents:
- if sys.version_info[0] < 3:
- if hasattr(parent, "encode"):
- entity.add_parent(name=parent)
- else:
- entity.add_parent(id=parent)
- else:
- if hasattr(parent, "encode"):
- entity.add_parent(name=parent)
- else:
- entity.add_parent(id=parent)
- def add_property(entity, prop, name, recursive=False, datatype=None):
- if datatype is None:
- raise ArgumentError("datatype must not be None")
+def retrieve_entities_with_ids(entities):
+ collection = Container()
+ step = 20
- if isinstance(prop, CaosDBPythonEntity):
- entity.add_property(name=name, value=str(
- prop._id), datatype=datatype)
+ for i in range(len(entities)//step+1):
+ collection.extend(
+ execute_query(
+ create_id_query(entities[i*step:(i+1)*step])))
- if recursive and not prop.do_not_expand:
- return convert_to_entity(prop, recursive=recursive)
- else:
- return []
- else:
- if isinstance(prop, float) or isinstance(prop, int):
- prop = str(prop)
- entity.add_property(name=name, value=prop, datatype=datatype)
-
- return []
-
- for prop in robj._properties:
- value = robj.__getattribute__(prop)
-
- if isinstance(value, list):
- if robj._datatypes[prop][0:4] == "LIST":
- lst = []
-
- for v in value:
- if isinstance(v, CaosDBPythonEntity):
- lst.append(v._id)
-
- if recursive and not v.do_not_expand:
- children.append(convert_to_entity(
- v, recursive=recursive))
- else:
- if isinstance(v, float) or isinstance(v, int):
- lst.append(str(v))
- else:
- lst.append(v)
- entity.add_property(name=prop, value=lst,
- datatype=robj._datatypes[prop])
- else:
- for v in value:
- children.extend(
- add_property(
- entity,
- v,
- prop,
- datatype=robj._datatypes[prop],
- **kwargs))
- else:
- children.extend(
- add_property(
- entity,
- value,
- prop,
- datatype=robj._datatypes[prop],
- **kwargs))
-
- return [entity] + children
-
-
-def convert_to_entity(python_object, **kwargs):
- if isinstance(python_object, Container):
- # Create a list of objects:
-
- return [convert_to_python_object(i, **kwargs) for i in python_object]
- elif isinstance(python_object, CaosDBPythonRecord):
- return _single_convert_to_entity(Record(), python_object, **kwargs)
- elif isinstance(python_object, CaosDBPythonFile):
- return _single_convert_to_entity(File(), python_object, **kwargs)
- elif isinstance(python_object, CaosDBPythonRecordType):
- return _single_convert_to_entity(RecordType(), python_object, **kwargs)
- elif isinstance(python_object, CaosDBPythonProperty):
- return _single_convert_to_entity(Property(), python_object, **kwargs)
- elif isinstance(python_object, CaosDBPythonEntity):
- return _single_convert_to_entity(Entity(), python_object, **kwargs)
- else:
- raise ValueError("Cannot convert an object of this type.")
-
-
-def convert_to_python_object(entity):
- """"""
-
- if isinstance(entity, Container):
- # Create a list of objects:
-
- return [convert_to_python_object(i) for i in entity]
- elif isinstance(entity, Record):
- return _single_convert_to_python_object(CaosDBPythonRecord(), entity)
- elif isinstance(entity, RecordType):
- return _single_convert_to_python_object(
- CaosDBPythonRecordType(), entity)
- elif isinstance(entity, File):
- return _single_convert_to_python_object(CaosDBPythonFile(), entity)
- elif isinstance(entity, Property):
- return _single_convert_to_python_object(CaosDBPythonProperty(), entity)
- elif isinstance(entity, Entity):
- return _single_convert_to_python_object(CaosDBPythonEntity(), entity)
- else:
- raise ValueError("Cannot convert an object of this type.")
+ return collection
def getOriginUrlIn(folder):
@@ -517,9 +147,9 @@ def getOriginUrlIn(folder):
with open(t.name, "r") as t:
urlString = "Fetch URL:"
- for l in t.readlines():
- if urlString in l:
- return l[l.find(urlString) + len(urlString):].strip()
+ for line in t.readlines():
+ if urlString in line:
+ return line[line.find(urlString) + len(urlString):].strip()
return None
@@ -558,17 +188,30 @@ def getCommitIn(folder):
return t.readline().strip()
-COMPARED = ["name", "role", "datatype", "description", "importance"]
-
-
-def compare_entities(old_entity, new_entity):
- olddiff = {"properties": {}, "parents": []}
- newdiff = {"properties": {}, "parents": []}
+def compare_entities(old_entity: Entity, new_entity: Entity):
+ """
+ Compare two entites.
+
+ Return a tuple of dictionaries, the first index belongs to additional information for old
+ entity, the second index belongs to additional information for new entity.
+
+ Additional information means in detail:
+ - Additional parents (a list under key "parents")
+ - Information about properties:
+ - Each property lists either an additional property or a property with a changed:
+ - ... datatype
+ - ... importance or
+ - ... value (not implemented yet)
+ In case of changed information the value listed under the respective key shows the
+ value that is stored in the respective entity.
+ """
+ olddiff: Dict[str, Any] = {"properties": {}, "parents": []}
+ newdiff: Dict[str, Any] = {"properties": {}, "parents": []}
if old_entity is new_entity:
return (olddiff, newdiff)
- for attr in COMPARED:
+ for attr in SPECIAL_ATTRIBUTES:
try:
oldattr = old_entity.__getattribute__(attr)
old_entity_attr_exists = True
@@ -616,20 +259,29 @@ def compare_entities(old_entity, new_entity):
newdiff["properties"][prop.name]["importance"] = \
new_entity.get_importance(prop.name)
- if ((prop.datatype is not None and
- matching[0].datatype is not None) and
- (prop.datatype != matching[0].datatype)):
+ if (prop.datatype != matching[0].datatype):
olddiff["properties"][prop.name]["datatype"] = prop.datatype
newdiff["properties"][prop.name]["datatype"] = \
matching[0].datatype
+ if (prop.unit != matching[0].unit):
+ olddiff["properties"][prop.name]["unit"] = prop.unit
+ newdiff["properties"][prop.name]["unit"] = \
+ matching[0].unit
+
+ if (prop.value != matching[0].value):
+ olddiff["properties"][prop.name]["value"] = prop.value
+ newdiff["properties"][prop.name]["value"] = \
+ matching[0].value
+
if (len(newdiff["properties"][prop.name]) == 0
and len(olddiff["properties"][prop.name]) == 0):
newdiff["properties"].pop(prop.name)
olddiff["properties"].pop(prop.name)
else:
- raise NotImplementedError()
+ raise NotImplementedError(
+ "Comparison not implemented for multi-properties.")
for prop in new_entity.properties:
if len([0 for p in old_entity.properties if p.name == prop.name]) == 0:
@@ -648,10 +300,77 @@ def compare_entities(old_entity, new_entity):
return (olddiff, newdiff)
+def merge_entities(entity_a: Entity, entity_b: Entity):
+ """
+ Merge entity_b into entity_a such that they have the same parents and properties.
+
+ datatype, unit, value, name and description will only be changed in entity_a if they
+ are None for entity_a and set for entity_b. If there is a corresponding value
+ for entity_a different from None a RuntimeError will be raised informing of an
+ unresolvable merge conflict.
+
+ The merge operation is done in place.
+
+ Returns entity_a.
+
+ WARNING: This function is currently experimental and insufficiently tested. Use with care.
+ """
+
+ logging.warning(
+ "This function is currently experimental and insufficiently tested. Use with care.")
+
+ # Compare both entities:
+ diff_r1, diff_r2 = compare_entities(entity_a, entity_b)
+
+ # Go through the comparison and try to apply changes to entity_a:
+ for key in diff_r2["parents"]:
+ entity_a.add_parent(entity_b.get_parent(key))
+
+ for key in diff_r2["properties"]:
+ if key in diff_r1["properties"]:
+ if ("importance" in diff_r1["properties"][key] and
+ "importance" in diff_r2["properties"][key]):
+ if (diff_r1["properties"][key]["importance"] !=
+ diff_r2["properties"][key]["importance"]):
+ raise NotImplementedError()
+ elif ("importance" in diff_r1["properties"][key] or
+ "importance" in diff_r2["properties"][key]):
+ raise NotImplementedError()
+
+ for attribute in ("datatype", "unit", "value"):
+ if diff_r1["properties"][key][attribute] is None:
+ setattr(entity_a.get_property(key), attribute,
+ diff_r2["properties"][key][attribute])
+ else:
+ raise RuntimeError("Merge conflict.")
+ else:
+ # TODO: This is a temporary FIX for
+ # https://gitlab.indiscale.com/caosdb/src/caosdb-pylib/-/issues/105
+ entity_a.add_property(id=entity_b.get_property(key).id,
+ name=entity_b.get_property(key).name,
+ datatype=entity_b.get_property(key).datatype,
+ value=entity_b.get_property(key).value,
+ unit=entity_b.get_property(key).unit,
+ importance=entity_b.get_importance(key))
+ # entity_a.add_property(
+ # entity_b.get_property(key),
+ # importance=entity_b.get_importance(key))
+
+ for special_attribute in ("name", "description"):
+ sa_a = getattr(entity_a, special_attribute)
+ sa_b = getattr(entity_b, special_attribute)
+ if sa_a != sa_b:
+ if sa_a is None:
+ setattr(entity_a, special_attribute, sa_b)
+ else:
+ raise RuntimeError("Merge conflict.")
+ return entity_a
+
+
def describe_diff(olddiff, newdiff, name=None, as_update=True):
description = ""
- for attr in list(set(list(olddiff.keys())+list(newdiff.keys()))):
+ for attr in list(set(list(olddiff.keys()) + list(newdiff.keys()))):
if attr == "parents" or attr == "properties":
continue
description += "{} differs:\n".format(attr)
@@ -700,6 +419,7 @@ def apply_to_ids(entities, func):
entities : list of Entity
func : function with one parameter.
"""
+
for entity in entities:
_apply_to_ids_of_entity(entity, func)
@@ -745,3 +465,28 @@ def resolve_reference(prop: Property):
else:
if isinstance(prop.value, int):
prop.value = retrieve_entity_with_id(prop.value)
+
+
+def create_flat_list(ent_list: List[Entity], flat: List[Entity]):
+ """
+ Recursively adds all properties contained in entities from ent_list to
+ the output list flat. Each element will only be added once to the list.
+
+ TODO: Currently this function is also contained in newcrawler module crawl.
+ We are planning to permanently move it to here.
+ """
+ for ent in ent_list:
+ for p in ent.properties:
+ # For lists append each element that is of type Entity to flat:
+ if isinstance(p.value, list):
+ for el in p.value:
+ if isinstance(el, Entity):
+ if el not in flat:
+ flat.append(el)
+ # TODO: move inside if block?
+ create_flat_list([el], flat)
+ elif isinstance(p.value, Entity):
+ if p.value not in flat:
+ flat.append(p.value)
+ # TODO: move inside if block?
+ create_flat_list([p.value], flat)
diff --git a/src/caosdb/common/administration.py b/src/caosdb/common/administration.py
index e7ba94182d7a4d8b60c6400cd1d804f62f7bf03c..98d4d2826da7131ef79b5c3cc9b3d9597abc0248 100644
--- a/src/caosdb/common/administration.py
+++ b/src/caosdb/common/administration.py
@@ -26,16 +26,15 @@
"""missing docstring."""
-from lxml import etree
-
+import re
+import string
+import random
from caosdb.common.utils import xml2str
from caosdb.connection.connection import get_connection
-from caosdb.exceptions import (HTTPClientError,
- HTTPForbiddenError,
- HTTPResourceNotFoundError,
- EntityDoesNotExistError,
- ServerConfigurationException,
- )
+from caosdb.exceptions import (EntityDoesNotExistError, HTTPClientError,
+ HTTPForbiddenError, HTTPResourceNotFoundError,
+ ServerConfigurationException)
+from lxml import etree
def set_server_property(key, value):
@@ -60,7 +59,8 @@ def set_server_property(key, value):
con._form_data_request(method="POST", path="_server_properties",
params={key: value}).read()
except EntityDoesNotExistError:
- raise ServerConfigurationException("Debug mode in server is probably disabled.") from None
+ raise ServerConfigurationException(
+ "Debug mode in server is probably disabled.") from None
def get_server_properties():
@@ -75,9 +75,11 @@ def get_server_properties():
"""
con = get_connection()
try:
- body = con._http_request(method="GET", path="_server_properties").response
+ body = con._http_request(
+ method="GET", path="_server_properties").response
except EntityDoesNotExistError:
- raise ServerConfigurationException("Debug mode in server is probably disabled.") from None
+ raise ServerConfigurationException(
+ "Debug mode in server is probably disabled.") from None
xml = etree.parse(body)
props = dict()
@@ -112,6 +114,39 @@ def get_server_property(key):
return get_server_properties()[key]
+def generate_password(length: int):
+ """Create a random password that fulfills the security requirements
+
+ Parameters
+ ----------
+ length : int
+ Length of the generated password. Has to be greater than 7.
+
+ Returns
+ -------
+ password : string
+ Generated random password of the given length
+
+ Raises
+ ------
+ ValueError:
+ If the length is less than 8.
+ """
+ minimum_password_length = 8
+ if length < minimum_password_length:
+ raise ValueError("CaosDB passwords have to be at least {} characters.".format(
+ minimum_password_length))
+ sample_letters = string.ascii_letters + string.digits + "!#$%*+-/:;?_"
+ password = ''.join((random.choice(sample_letters) for i in range(length)))
+
+ while not re.match(r"(?=.*[A-Z])(?=.*[a-z])(?=.*\d)(?=.*[\W_]).{8,}",
+ password):
+ password = ''.join((random.choice(sample_letters)
+ for i in range(length)))
+
+ return password
+
+
def _retrieve_user(name, realm=None, **kwargs):
con = get_connection()
try:
@@ -156,13 +191,14 @@ def _update_user(name, realm=None, password=None, status=None,
return con.put_form_data(entity_uri_segment="User/" + (realm + "/" + name if realm is not None else name), params=params, **kwargs).read()
except HTTPResourceNotFoundError as e:
e.msg = "User does not exist."
- raise
+ raise e
except HTTPForbiddenError as e:
e.msg = "You are not permitted to update this user."
- raise
+ raise e
except HTTPClientError as e:
- if e.status == 409:
- e.msg = "Entity does not exist."
+ for elem in etree.fromstring(e.body):
+ if elem.tag == "Error":
+ e.msg = elem.get("description")
raise
@@ -187,11 +223,9 @@ def _insert_user(name, password=None, status=None, email=None, entity=None, **kw
e.msg = "You are not permitted to insert a new user."
raise e
except HTTPClientError as e:
- if e.status == 409:
- e.msg = "User name is already in use."
-
- if e.status == 422:
- e.msg = "Maybe the password does not match the required standard?"
+ for elem in etree.fromstring(e.body):
+ if elem.tag == "Error":
+ e.msg = elem.get("description")
raise e
diff --git a/src/caosdb/common/datatype.py b/src/caosdb/common/datatype.py
index 5434f5b6556a13f65754eacc66cb32231366e5b3..03ff6d023ab0d3005c37d56c65353c1a1072518e 100644
--- a/src/caosdb/common/datatype.py
+++ b/src/caosdb/common/datatype.py
@@ -62,10 +62,21 @@ def is_list_datatype(datatype):
def is_reference(datatype):
- """ returns whether the value is a reference
+ """Returns whether the value is a reference
FILE and REFERENCE properties are examples, but also datatypes that are
- RecordTypes
+ RecordTypes.
+
+ Parameters
+ ----------
+ datatype : str
+ The datatype to check.
+
+ Returns
+ -------
+ bool
+ True if the datatype is a not base datatype or a list of a base datatype.
+ Otherwise False is returned.
"""
if datatype is None:
@@ -80,6 +91,41 @@ def is_reference(datatype):
return True
+def get_referenced_recordtype(datatype):
+ """Return the record type of the referenced datatype.
+
+ Raises
+ ------
+ ValueError
+ In cases where datatype is not a reference, the list does not have
+ a referenced record type or the datatype is a FILE.
+
+ Parameters
+ ----------
+ datatype : str
+ The datatype to check.
+
+ Returns
+ -------
+ str
+ String containing the name of the referenced datatype.
+ """
+
+ if not is_reference(datatype):
+ raise ValueError("datatype must be a reference")
+
+ if is_list_datatype(datatype):
+ datatype = get_list_datatype(datatype)
+ if datatype is None:
+ raise ValueError("list does not have a list datatype")
+
+ if datatype == FILE:
+ raise ValueError(
+ "FILE references are not considered references with a record type")
+
+ return datatype
+
+
def get_id_of_datatype(datatype):
""" returns the id of a Record Type
diff --git a/src/caosdb/common/models.py b/src/caosdb/common/models.py
index 23fe2e2d81c12b7ca9efb23e02ce381122a41ac1..f974060f4727e575a94a3afcdd2f86520e6123a9 100644
--- a/src/caosdb/common/models.py
+++ b/src/caosdb/common/models.py
@@ -5,9 +5,9 @@
#
# Copyright (C) 2018 Research Group Biomedical Physics,
# Max-Planck-Institute for Dynamics and Self-Organization Göttingen
-# Copyright (C) 2020 Indiscale GmbH <info@indiscale.com>
+# Copyright (C) 2020-2022 Indiscale GmbH <info@indiscale.com>
# Copyright (C) 2020 Florian Spreckelsen <f.spreckelsen@indiscale.com>
-# Copyright (C) 2020 Timm Fitschen <t.fitschen@indiscale.com>
+# Copyright (C) 2020-2022 Timm Fitschen <t.fitschen@indiscale.com>
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as
@@ -25,7 +25,14 @@
# ** end header
#
-"""missing docstring."""
+"""
+Collection of the central classes of the CaosDB client, namely the Entity class
+and all of its subclasses and the Container class which is used to carry out
+transactions.
+
+All additional classes are either important for the entities or the
+transactions.
+"""
from __future__ import print_function, unicode_literals
import re
@@ -37,7 +44,6 @@ from hashlib import sha512
from os import listdir
from os.path import isdir
from random import randint
-from sys import hexversion
from tempfile import NamedTemporaryFile
from warnings import warn
@@ -55,6 +61,7 @@ from caosdb.exceptions import (AmbiguousEntityError, AuthorizationError,
EntityDoesNotExistError, EntityError,
EntityHasNoDatatypeError, HTTPURITooLongError,
MismatchingEntitiesError, QueryNotUniqueError,
+ ServerConfigurationException,
TransactionError, UniqueNamesError,
UnqualifiedParentsError,
UnqualifiedPropertiesError)
@@ -71,6 +78,10 @@ ALL = "ALL"
NONE = "NONE"
+SPECIAL_ATTRIBUTES = ["name", "role", "datatype", "description",
+ "id", "path", "checksum", "size"]
+
+
class Entity(object):
"""Entity is a generic CaosDB object.
@@ -113,6 +124,47 @@ class Entity(object):
self.id = id
self.state = None
+ def copy(self):
+ """
+ Return a copy of entity.
+
+ If deep == True return a deep copy, recursively copying all sub entities.
+
+ Standard properties are copied using add_property.
+ Special attributes, as defined by the global variable SPECIAL_ATTRIBUTES and additionaly
+ the "value" are copied using setattr.
+ """
+ if self.role == "File":
+ new = File()
+ elif self.role == "Property":
+ new = Property()
+ elif self.role == "RecordType":
+ new = RecordType()
+ elif self.role == "Record":
+ new = Record()
+ elif self.role == "Entity":
+ new = Entity()
+ else:
+ raise RuntimeError("Unkonwn role.")
+
+ # Copy special attributes:
+ # TODO: this might rise an exception when copying
+ # special file attributes like checksum and size.
+ for attribute in SPECIAL_ATTRIBUTES + ["value"]:
+ val = getattr(self, attribute)
+ if val is not None:
+ setattr(new, attribute, val)
+
+ # Copy parents:
+ for p in self.parents:
+ new.add_parent(p)
+
+ # Copy properties:
+ for p in self.properties:
+ new.add_property(p, importance=self.get_importance(p))
+
+ return new
+
@property
def version(self):
if self._version is not None or self._wrapped_entity is None:
@@ -130,7 +182,10 @@ class Entity(object):
@role.setter
def role(self, role):
- self.__role = role
+ if role is not None and role.lower() == "entity":
+ self.__role = None
+ else:
+ self.__role = role
@property
def size(self):
@@ -265,14 +320,74 @@ class Entity(object):
self.__pickup = new_pickup
def grant(self, realm=None, username=None, role=None,
- permission=None, priority=False):
+ permission=None, priority=False, revoke_denial=True):
+ """Grant a permission to a user or role for this entity.
+
+ You must specify either only the username and the realm, or only the
+ role.
+
+ By default a previously existing denial rule would be revoked, because
+ otherwise this grant wouldn't have any effect. However, for keeping
+ contradicting rules pass revoke_denial=False.
+
+ Parameters
+ ----------
+ permission: str
+ The permission to be granted.
+ username : str, optional
+ The username. Exactly one is required, either the `username` or the
+ `role`.
+ realm: str, optional
+ The user's realm. Required when username is not None.
+ role: str, optional
+ The role (as in Role-Based Access Control). Exactly one is
+ required, either the `username` or the `role`.
+ priority: bool, default False
+ Whether this permission is granted with priority over non-priority
+ rules.
+ revoke_denial: bool, default True
+ Whether a contradicting denial (with same priority flag) in this
+ ACL will be revoked.
+ """
+ # @review Florian Spreckelsen 2022-03-17
self.acl.grant(realm=realm, username=username, role=role,
- permission=permission, priority=priority)
+ permission=permission, priority=priority,
+ revoke_denial=revoke_denial)
def deny(self, realm=None, username=None, role=None,
- permission=None, priority=False):
+ permission=None, priority=False, revoke_grant=True):
+ """Deny a permission to a user or role for this entity.
+
+ You must specify either only the username and the realm, or only the
+ role.
+
+ By default a previously existing grant rule would be revoked, because
+ otherwise this denial would override the grant rules anyways. However,
+ for keeping contradicting rules pass revoke_grant=False.
+
+ Parameters
+ ----------
+ permission: str
+ The permission to be denied.
+ username : str, optional
+ The username. Exactly one is required, either the `username` or the
+ `role`.
+ realm: str, optional
+ The user's realm. Required when username is not None.
+ role: str, optional
+ The role (as in Role-Based Access Control). Exactly one is
+ required, either the `username` or the `role`.
+ priority: bool, default False
+ Whether this permission is denied with priority over non-priority
+ rules.
+ revoke_grant: bool, default True
+ Whether a contradicting grant (with same priority flag) in this
+ ACL will be revoked.
+ """
+ # @review Florian Spreckelsen 2022-03-17
self.acl.deny(realm=realm, username=username, role=role,
- permission=permission, priority=priority)
+ permission=permission, priority=priority,
+ revoke_grant=revoke_grant)
def revoke_denial(self, realm=None, username=None,
role=None, permission=None, priority=False):
@@ -339,54 +454,80 @@ class Entity(object):
return self
- def add_property(self, property=None, value=None, **kwargs): # @ReservedAssignment
+ def add_property(self, property=None, value=None, id=None, name=None, description=None, datatype=None,
+ unit=None, importance=None, inheritance=None): # @ReservedAssignment
"""Add a property to this entity.
The first parameter is meant to identify the property entity. So the method expects an instance of
Entity, an integer or a string here. The second parameter is the value of the new property. Any
- other named parameter may be passed by means of the **kwargs. Accepted keywords are:
+ other named parameter may be passed by means of the keywwords. Accepted keywords are:
id, name, description, importance, inheritance, datatype, and unit. Any other keyword will be
ignored right now. But that may change in the future.
- @param property: An identifying parameter (name, id or abstract property).
- @param value: The value of the new property.
- @param **kwargs: Any other specification for this property. Accepted keywords: id, name, description, importance, inheritance, datatype, and unit.
- @raise UserWarning:
- If the first parameter is None then kwargs['id'] or kwargs['name'] must be defined and not be None.
- Otherwise a UserWarning is raised.
+ Parameters
+ ----------
+ property : int, str, Property, optional
+ An identifying parameter, by default None
+ value : int, str, Property, optional
+ The value of the new property, by default None
+ id : int, optional
+ Id of the property, by default None
+ name : str, optional
+ Name of the property, by default None
+ description : str, optional
+ Description of the property, by default None
+ datatype : str, optional
+ Datatype of the property, by default None
+ unit : str, optional
+ Unit of the property, by default None
+ importance :str, optional
+ Importance of the property, by default None
+ inheritance : str, optional
+ Inheritance of the property, by default None
- If the first parameter is an integer then it is interpreted as the id and kwargs['id'] must be
- undefined or None. Otherwise a UserWarning is raised.
+ Returns
+ -------
+ Entity
- If the first parameter is not None and neither an instance of Entity nor an integer it is
- interpreted as the name and kwargs['name'] must be undefined or None. Otherwise a UserWarning is
- raised.
+ Raises
+ ------
+ UserWarning
+ If the first parameter is None then id or name must be defined and not be None.
+ UserWarning
+ If the first parameter is an integer then it is interpreted as the id and id must be
+ undefined or None.
+ UserWarning
+ If the first parameter is not None and neither an instance of Entity nor an integer it is
+ interpreted as the name and name must be undefined or None.
"""
- copy_kwargs = kwargs.copy()
- name = (kwargs['name'] if 'name' in kwargs else None)
- pid = (kwargs['id'] if 'id' in kwargs else None)
+
+ pid = id
abstract_property = None
if isinstance(property, Entity):
+ if property.role is not None and property.role.lower() in ["record", "file"]:
+ raise ValueError("The property parameter is a {0}. This "
+ "is very unusual and probably not what you "
+ "want. Otherwise, construct a property from "
+ "a {0} using the Property class and add "
+ "that to this entity.".format(property.role))
abstract_property = property
elif isinstance(property, int):
if pid is not None:
raise UserWarning("The first parameter was an integer which would normally be interpreted as the id of the property which is to be added. But you have also specified a parameter 'id' in the method call. This is ambiguous and cannot be processed.")
pid = property
- copy_kwargs['id'] = pid
+ id = pid
elif property is not None:
if name is not None:
raise UserWarning("The first parameter was neither an instance of Entity nor an integer. Therefore the string representation of your first parameter would normally be interpreted name of the property which is to be added. But you have also specified a parameter 'name' in the method call. This is ambiguous and cannot be processed.")
name = str(property)
- copy_kwargs['name'] = name
if property is None and name is None and pid is None:
raise UserWarning(
"This method expects you to pass at least an entity, a name or an id.")
- del copy_kwargs['importance']
- del copy_kwargs['inheritance']
- new_property = Property(**copy_kwargs)
+ new_property = Property(name=name, id=id, description=description, datatype=datatype,
+ value=value, unit=unit)
if abstract_property is not None:
new_property._wrap(property)
@@ -399,9 +540,7 @@ class Entity(object):
new_property.value = value
self.properties.append(
- property=new_property, importance=(
- kwargs['importance'] if 'importance' in kwargs else None), inheritance=(
- kwargs['inheritance'] if 'inheritance' in kwargs else None))
+ property=new_property, importance=importance, inheritance=inheritance)
return self
@@ -426,7 +565,7 @@ class Entity(object):
return self
- def add_parent(self, parent=None, **kwargs): # @ReservedAssignment
+ def add_parent(self, parent=None, id=None, name=None, inheritance=None): # @ReservedAssignment
"""Add a parent to this entity.
Parameters
@@ -434,27 +573,23 @@ class Entity(object):
parent : Entity or int or str or None
The parent entity, either specified by the Entity object
itself, or its id or its name. Default is None.
- **kwargs : dict, optional
- Additional keyword arguments for specifying the parent by
- name or id, and for specifying the mode of inheritance.
-
- id : int
- Integer id of the parent entity. Ignored if `parent`
- is not None.
- name : str
- Name of the parent entity. Ignored if `parent is not
- none`.
- inheritance : str
- One of ``obligatory``, ``recommended``, ``suggested``, or ``fix``. Specifies the
- minimum importance which parent properties need to have to be inherited by this
- entity. If no `inheritance` is given, no properties will be inherited by the child.
- This parameter is case-insensitive.
-
- Note that the behaviour is currently not yet specified when assigning parents to
- Records, it only works for inheritance of RecordTypes (and Properties).
-
- For more information, it is recommended to look into the
- :ref:`data insertion tutorial<tutorial-inheritance-properties>`.
+ id : int
+ Integer id of the parent entity. Ignored if `parent`
+ is not None.
+ name : str
+ Name of the parent entity. Ignored if `parent is not
+ none`.
+ inheritance : str
+ One of ``obligatory``, ``recommended``, ``suggested``, or ``fix``. Specifies the
+ minimum importance which parent properties need to have to be inherited by this
+ entity. If no `inheritance` is given, no properties will be inherited by the child.
+ This parameter is case-insensitive.
+
+ Note that the behaviour is currently not yet specified when assigning parents to
+ Records, it only works for inheritance of RecordTypes (and Properties).
+
+ For more information, it is recommended to look into the
+ :ref:`data insertion tutorial<tutorial-inheritance-properties>`.
Raises
------
@@ -463,8 +598,8 @@ class Entity(object):
parameter is passed to this method.
"""
- name = (kwargs['name'] if 'name' in kwargs else None)
- pid = (kwargs['id'] if 'id' in kwargs else None)
+
+ pid = id
parent_entity = None
if isinstance(parent, Entity):
@@ -478,9 +613,6 @@ class Entity(object):
raise UserWarning(
"This method expects you to pass at least an entity, a name or an id.")
- inheritance = (kwargs['inheritance']
- if 'inheritance' in kwargs else None)
-
addp = Parent(id=pid, name=name, inheritance=inheritance)
if parent_entity is not None:
@@ -837,7 +969,9 @@ class Entity(object):
"""
if xml is None:
- xml = etree.Element("Entity")
+ # use role as xml tag name, fall-back to "Entity"
+ elem_tag = "Entity" if self.role is None else self.role
+ xml = etree.Element(elem_tag)
assert isinstance(xml, etree._Element)
# unwrap wrapped entity
@@ -956,6 +1090,8 @@ class Entity(object):
@param elem: the xml element
"""
+ if isinstance(entity, Entity):
+ entity.role = elem.tag
entity._cuid = elem.get("cuid")
entity.id = elem.get("id") # @ReservedAssignment
entity.name = elem.get("name")
@@ -1171,6 +1307,10 @@ class Entity(object):
def _parse_value(datatype, value):
+ """Parse the value (from XML input) according to the given datatype
+ """
+
+ # Simple values
if value is None:
return value
@@ -1191,12 +1331,12 @@ def _parse_value(datatype, value):
else:
raise ValueError("Boolean value was {}.".format(value))
+ # Datetime and text are returned as-is
if datatype in [DATETIME, TEXT]:
if isinstance(value, str):
return value
# deal with collections
-
if isinstance(datatype, str):
matcher = re.compile(r"^(?P<col>[^<]+)<(?P<dt>[^>]+)>$")
m = matcher.match(datatype)
@@ -1221,12 +1361,10 @@ def _parse_value(datatype, value):
# This is for a special case, where the xml parser could not differentiate
# between single values and lists with one element. As
-
if hasattr(value, "__len__") and len(value) == 1:
return _parse_value(datatype, value[0])
# deal with references
-
if isinstance(value, Entity):
return value
@@ -1242,6 +1380,12 @@ def _parse_value(datatype, value):
# reference via name
return str(value)
+ except TypeError:
+ # deal with invalid XML: List of values without appropriate datatype
+ if isinstance(value, list):
+ raise ServerConfigurationException(
+ "The server sent an invalid XML: List valued properties must be announced by "
+ "the datatype.\n" + f"Datatype: {datatype}\nvalue: {value}")
def _log_request(request, xml_body=None):
@@ -1257,11 +1401,7 @@ def _log_request(request, xml_body=None):
def _log_response(body):
if Container._debug() > 0:
print("\n======== Response body ========\n")
-
- if hexversion < 0x03000000:
- print(body)
- else:
- print(body.decode())
+ print(body.decode())
print("\n===============================\n")
@@ -1270,6 +1410,7 @@ class QueryTemplate():
def __init__(self, id=None, name=None, query=None, description=None): # @ReservedAssignment
self.id = (int(id) if id is not None else None)
+ self.role = "QueryTemplate"
self.name = name
self.description = description
self.query = query
@@ -1453,45 +1594,42 @@ class Property(Entity):
"""CaosDB's Property object."""
- def add_property(self, property=None, value=None, **kwargs): # @ReservedAssignment
- copy_kwargs = kwargs.copy()
+ def add_property(self, property=None, value=None, id=None, name=None, description=None, datatype=None,
+ unit=None, importance=FIX, inheritance=FIX): # @ReservedAssignment
- if 'importance' not in copy_kwargs:
- # set default importance
- copy_kwargs['importance'] = FIX
-
- if 'inheritance' not in copy_kwargs:
- # set default importance
- copy_kwargs['inheritance'] = FIX
-
- return super(Property, self).add_property(
- property=property, value=value, **copy_kwargs)
+ return super().add_property(
+ property=property, id=id, name=name, description=description, datatype=datatype,
+ value=value, unit=unit, importance=importance, inheritance=inheritance)
- def add_parent(self, parent=None, **kwargs):
+ def add_parent(self, parent=None, id=None, name=None, inheritance=FIX):
"""Add a parent Entity to this Property.
Parameters
----------
- parent : Entity or int or str or None, optional
- The parent entity
- **kwargs : dict, optional
- Additional keyword arguments specifying the parent Entity
- by id or name, and specifying the inheritance level. See
- :py:meth:`Entity.add_parent` for more information. Note
- that by default, `inheritance` is set to ``fix``.
+ Parameters
+ ----------
+ parent : Entity or int or str or None
+ The parent entity, either specified by the Entity object
+ itself, or its id or its name. Default is None.
+ id : int
+ Integer id of the parent entity. Ignored if `parent`
+ is not None.
+ name : str
+ Name of the parent entity. Ignored if `parent is not
+ none`.
+ inheritance : str, default: FIX
+ One of ``obligatory``, ``recommended``, ``suggested``, or ``fix``. Specifies the
+ minimum importance which parent properties need to have to be inherited by this
+ entity. If no `inheritance` is given, no properties will be inherited by the child.
+ This parameter is case-insensitive.
See Also
--------
Entity.add_parent
"""
- copy_kwargs = kwargs.copy()
- if 'inheritance' not in copy_kwargs:
- # set default importance
- copy_kwargs['inheritance'] = FIX
-
- return super(Property, self).add_parent(parent=parent, **copy_kwargs)
+ return super(Property, self).add_parent(parent=parent, id=id, name=name, inheritance=inheritance)
def __init__(self, name=None, id=None, description=None, datatype=None,
value=None, unit=None):
@@ -1529,6 +1667,11 @@ class Property(Entity):
if server_retrieval:
tmp_prop = deepcopy(self)
+ """
+ remove role to avoid unnessecary ValueError while
+ retrieving the Entity.
+ """
+ tmp_prop.role = None
tmp_prop.retrieve()
return tmp_prop.is_reference()
@@ -1584,21 +1727,14 @@ class RecordType(Entity):
"""This class represents CaosDB's RecordType entities."""
- def add_property(self, property=None, value=None, **kwargs): # @ReservedAssignment
- copy_kwargs = kwargs.copy()
-
- if 'importance' not in copy_kwargs:
- # set default importance
- copy_kwargs['importance'] = RECOMMENDED
+ def add_property(self, property=None, value=None, id=None, name=None, description=None, datatype=None,
+ unit=None, importance=RECOMMENDED, inheritance=FIX): # @ReservedAssignment
- if 'inheritance' not in copy_kwargs:
- # set default importance
- copy_kwargs['inheritance'] = FIX
-
- return super().add_property(property=property, value=value,
- **copy_kwargs)
+ return super().add_property(
+ property=property, id=id, name=name, description=description, datatype=datatype,
+ value=value, unit=unit, importance=importance, inheritance=inheritance)
- def add_parent(self, parent=None, **kwargs):
+ def add_parent(self, parent=None, id=None, name=None, inheritance=OBLIGATORY):
"""Add a parent to this RecordType
Parameters
@@ -1606,24 +1742,30 @@ class RecordType(Entity):
parent : Entity or int or str or None, optional
The parent entity, either specified by the Entity object
itself, or its id or its name. Default is None.
- **kwargs : dict, optional
- Additional keyword arguments specifying the parent Entity by id or
- name, and specifying the inheritance level. See
- :py:meth:`Entity.add_parent` for more information. Note
- that by default, `inheritance` is set to ``obligatory``.
+ Parameters
+ ----------
+ parent : Entity or int or str or None
+ The parent entity, either specified by the Entity object
+ itself, or its id or its name. Default is None.
+ id : int
+ Integer id of the parent entity. Ignored if `parent`
+ is not None.
+ name : str
+ Name of the parent entity. Ignored if `parent is not
+ none`.
+ inheritance : str, default OBLIGATORY
+ One of ``obligatory``, ``recommended``, ``suggested``, or ``fix``. Specifies the
+ minimum importance which parent properties need to have to be inherited by this
+ entity. If no `inheritance` is given, no properties will be inherited by the child.
+ This parameter is case-insensitive.
See Also
--------
Entity.add_parent
"""
- copy_kwargs = kwargs.copy()
- if 'inheritance' not in copy_kwargs:
- # set default importance
- copy_kwargs['inheritance'] = OBLIGATORY
-
- return super().add_parent(parent=parent, **copy_kwargs)
+ return super().add_parent(parent=parent, id=id, name=name, inheritance=inheritance)
def __init__(self, name=None, id=None, description=None, datatype=None): # @ReservedAssignment
Entity.__init__(self, name=name, id=id, description=description,
@@ -1640,19 +1782,12 @@ class Record(Entity):
"""This class represents CaosDB's Record entities."""
- def add_property(self, property=None, value=None, **kwargs): # @ReservedAssignment
- copy_kwargs = kwargs.copy()
-
- if 'importance' not in copy_kwargs:
- # set default importance
- copy_kwargs['importance'] = FIX
-
- if 'inheritance' not in copy_kwargs:
- # set default importance
- copy_kwargs['inheritance'] = FIX
+ def add_property(self, property=None, value=None, id=None, name=None, description=None, datatype=None,
+ unit=None, importance=FIX, inheritance=FIX): # @ReservedAssignment
return super().add_property(
- property=property, value=value, **copy_kwargs)
+ property=property, id=id, name=name, description=description, datatype=datatype,
+ value=value, unit=unit, importance=importance, inheritance=inheritance)
def __init__(self, name=None, id=None, description=None): # @ReservedAssignment
Entity.__init__(self, name=name, id=id, description=description,
@@ -1804,19 +1939,12 @@ class File(Record):
return checksum.hexdigest()
- def add_property(self, property=None, value=None, **kwargs): # @ReservedAssignment
- copy_kwargs = kwargs.copy()
-
- if 'importance' not in copy_kwargs:
- # set default importance
- copy_kwargs['importance'] = FIX
-
- if 'inheritance' not in copy_kwargs:
- # set default importance
- copy_kwargs['inheritance'] = FIX
+ def add_property(self, property=None, id=None, name=None, description=None, datatype=None,
+ value=None, unit=None, importance=FIX, inheritance=FIX): # @ReservedAssignment
return super().add_property(
- property=property, value=value, **copy_kwargs)
+ property=property, id=id, name=name, description=description, datatype=datatype,
+ value=value, unit=unit, importance=importance, inheritance=inheritance)
class _Properties(list):
@@ -2320,15 +2448,45 @@ class _Messages(dict):
def _basic_sync(e_local, e_remote):
+ '''Copy all state from a one entity to another.
+
+ This method is used to syncronize an entity with a remote (i.e. a newly
+ retrieved) one.
+
+ Any entity state of the local one will be overriden.
+
+ Parameters
+ ----------
+ e_local : Entity
+ Destination of the copy.
+ e_local : Entity
+ Source of the copy.
+
+
+ Returns
+ -------
+ e_local : Entity
+ The syncronized entity.
+ '''
if e_local is None or e_remote is None:
return None
+ if e_local.role is None:
+ e_local.role = e_remote.role
+ elif e_remote.role is not None and not e_local.role.lower() == e_remote.role.lower():
+ raise ValueError("The resulting entity had a different role ({0}) "
+ "than the local one ({1}). This probably means, that "
+ "the entity was intialized with a wrong class "
+ "by this client or it has changed in the past and "
+ "this client did't know about it yet.".format(
+ e_remote.role, e_local.role))
+
e_local.id = e_remote.id
e_local.name = e_remote.name
e_local.description = e_remote.description
e_local.path = e_remote.path
e_local._checksum = e_remote._checksum
e_local._size = e_remote._size
- e_local.datatype = e_remote.datatype # @ReservedAssignment
+ e_local.datatype = e_remote.datatype
e_local.unit = e_remote.unit
e_local.value = e_remote.value
e_local.properties = e_remote.properties
@@ -2934,7 +3092,9 @@ class Container(list):
if is_reference(references.datatype):
# add only if it is a reference, not a property
- if isinstance(references.value, int):
+ if references.value is None:
+ continue
+ elif isinstance(references.value, int):
is_being_referenced.add(references.value)
elif is_list_datatype(references.datatype):
for list_item in references.value:
@@ -3583,13 +3743,15 @@ class ACI():
self.permission = permission
def __hash__(self):
- return hash(str(self.realm) + ":" + str(self.username) +
- ":" + str(self.role) + ":" + str(self.permission))
+ return hash(self.__repr__())
def __eq__(self, other):
return isinstance(other, ACI) and (self.role is None and self.username == other.username and self.realm ==
other.realm) or self.role == other.role and self.permission == other.permission
+ def __repr__(self):
+ return str(self.realm) + ":" + str(self.username) + ":" + str(self.role) + ":" + str(self.permission)
+
def add_to_element(self, e):
if self.role is not None:
e.set("role", self.role)
@@ -3614,10 +3776,35 @@ class ACL():
self.clear()
def parse_xml(self, xml):
+ """Clear this ACL and parse the xml.
+
+ Iterate over the rules in the xml and add each rule to this ACL.
+
+ Contradicting rules will both be kept.
+
+ Parameters
+ ----------
+ xml : lxml.etree.Element
+ The xml element containing the ACL rules, i.e. <Grant> and <Deny>
+ rules.
+ """
self.clear()
self._parse_xml(xml)
def _parse_xml(self, xml):
+ """Parse the xml.
+
+ Iterate over the rules in the xml and add each rule to this ACL.
+
+ Contradicting rules will both be kept.
+
+ Parameters
+ ----------
+ xml : lxml.etree.Element
+ The xml element containing the ACL rules, i.e. <Grant> and <Deny>
+ rules.
+ """
+ # @review Florian Spreckelsen 2022-03-17
for e in xml:
role = e.get("role")
username = e.get("username")
@@ -3630,10 +3817,12 @@ class ACL():
if e.tag == "Grant":
self.grant(username=username, realm=realm, role=role,
- permission=permission, priority=priority)
+ permission=permission, priority=priority,
+ revoke_denial=False)
elif e.tag == "Deny":
self.deny(username=username, realm=realm, role=role,
- permission=permission, priority=priority)
+ permission=permission, priority=priority,
+ revoke_grant=False)
def combine(self, other):
""" Combine and return new instance."""
@@ -3711,12 +3900,42 @@ class ACL():
if item in self._denials:
self._denials.remove(item)
- def grant(self, username=None, realm=None, role=None,
- permission=None, priority=False):
+ def grant(self, permission, username=None, realm=None, role=None,
+ priority=False, revoke_denial=True):
+ """Grant a permission to a user or role.
+
+ You must specify either only the username and the realm, or only the
+ role.
+
+ By default a previously existing denial rule would be revoked, because
+ otherwise this grant wouldn't have any effect. However, for keeping
+ contradicting rules pass revoke_denial=False.
+
+ Parameters
+ ----------
+ permission: str
+ The permission to be granted.
+ username : str, optional
+ The username. Exactly one is required, either the `username` or the
+ `role`.
+ realm: str, optional
+ The user's realm. Required when username is not None.
+ role: str, optional
+ The role (as in Role-Based Access Control). Exactly one is
+ required, either the `username` or the `role`.
+ priority: bool, default False
+ Whether this permission is granted with priority over non-priority
+ rules.
+ revoke_denial: bool, default True
+ Whether a contradicting denial (with same priority flag) in this
+ ACL will be revoked.
+ """
+ # @review Florian Spreckelsen 2022-03-17
priority = self._get_boolean_priority(priority)
item = ACI(role=role, username=username,
realm=realm, permission=permission)
- self._remove_item(item, priority)
+ if revoke_denial:
+ self._remove_item(item, priority)
if priority is True:
self._priority_grants.add(item)
@@ -3724,11 +3943,41 @@ class ACL():
self._grants.add(item)
def deny(self, username=None, realm=None, role=None,
- permission=None, priority=False):
+ permission=None, priority=False, revoke_grant=True):
+ """Deny a permission to a user or role for this entity.
+
+ You must specify either only the username and the realm, or only the
+ role.
+
+ By default a previously existing grant rule would be revoked, because
+ otherwise this denial would override the grant rules anyways. However,
+ for keeping contradicting rules pass revoke_grant=False.
+
+ Parameters
+ ----------
+ permission: str
+ The permission to be denied.
+ username : str, optional
+ The username. Exactly one is required, either the `username` or the
+ `role`.
+ realm: str, optional
+ The user's realm. Required when username is not None.
+ role: str, optional
+ The role (as in Role-Based Access Control). Exactly one is
+ required, either the `username` or the `role`.
+ priority: bool, default False
+ Whether this permission is denied with priority over non-priority
+ rules.
+ revoke_grant: bool, default True
+ Whether a contradicting grant (with same priority flag) in this
+ ACL will be revoked.
+ """
+ # @review Florian Spreckelsen 2022-03-17
priority = self._get_boolean_priority(priority)
item = ACI(role=role, username=username,
realm=realm, permission=permission)
- self._remove_item(item, priority)
+ if revoke_grant:
+ self._remove_item(item, priority)
if priority is True:
self._priority_denials.add(item)
@@ -4222,7 +4471,7 @@ def _evaluate_and_add_error(parent_error, ent):
Parameters:
-----------
- parent_error : TrancactionError
+ parent_error : TransactionError
Parent error to which the new exception will be attached. This
exception will be a direct child.
ent : Entity
diff --git a/src/caosdb/common/utils.py b/src/caosdb/common/utils.py
index eabce3139ccabb35412b5dbe0fb83721fc18179a..f0ce740d38d90b0c7bb1031e808b83efb2207a43 100644
--- a/src/caosdb/common/utils.py
+++ b/src/caosdb/common/utils.py
@@ -26,7 +26,6 @@
from lxml import etree
from multiprocessing import Lock
from uuid import uuid4
-from sys import hexversion
_uuid_lock = Lock()
diff --git a/src/caosdb/configuration.py b/src/caosdb/configuration.py
index 842f1ee62a5b3178b7305e4c1e0c281a2dbd3b38..75827df0d00d6c82251c2c04fa47413ac2801928 100644
--- a/src/caosdb/configuration.py
+++ b/src/caosdb/configuration.py
@@ -21,6 +21,16 @@
#
# ** end header
#
+
+import os
+import yaml
+import warnings
+try:
+ optional_jsonschema_validate = None
+ from jsonschema import validate as optional_jsonschema_validate
+except ImportError:
+ pass
+
try:
# python2
from ConfigParser import ConfigParser
@@ -47,7 +57,9 @@ def configure(inifile):
_pycaosdbconf = None
if _pycaosdbconf is None:
_reset_config()
- return _pycaosdbconf.read(inifile)
+ read_config = _pycaosdbconf.read(inifile)
+ validate_yaml_schema(config_to_yaml(_pycaosdbconf))
+ return read_config
def get_config():
@@ -55,11 +67,45 @@ def get_config():
return _pycaosdbconf
+def config_to_yaml(config):
+ valobj = {}
+ for s in config.sections():
+ valobj[s] = {}
+ for key, value in config[s].items():
+ # TODO: Can the type be inferred from the config object?
+ if key in ["timeout", "debug"]:
+ valobj[s][key] = int(value)
+ elif key in ["ssl_insecure"]:
+ valobj[s][key] = bool(value)
+ else:
+ valobj[s][key] = value
+
+ return valobj
+
+
+def validate_yaml_schema(valobj):
+ if optional_jsonschema_validate:
+ with open(os.path.join(os.path.dirname(__file__), "schema-pycaosdb-ini.yml")) as f:
+ schema = yaml.load(f, Loader=yaml.SafeLoader)
+ optional_jsonschema_validate(instance=valobj, schema=schema["schema-pycaosdb-ini"])
+ else:
+ warnings.warn("""
+ Warning: The validation could not be performed because `jsonschema` is not installed.
+ """)
+
+
def _read_config_files():
- """Function to read config files from different paths. Checks for path in $PYCAOSDBINI or home directory (.pycaosdb.ini) and in the current working directory (pycaosdb.ini).
+ """Function to read config files from different paths.
+
+ Checks for path either in ``$PYCAOSDBINI`` or home directory (``.pycaosdb.ini``), and
+ additionally in the current working directory (``pycaosdb.ini``).
+
+ Returns
+ -------
+
+ ini files: list
+ The successfully parsed ini-files. Order: env_var or home directory, cwd. Used for testing the function.
- Returns:
- [list]: list with successfully parsed ini-files. Order: env_var or home directory, cwd. Used for testing the function.
"""
return_var = []
if "PYCAOSDBINI" in environ:
diff --git a/src/caosdb/connection/__init__.py b/src/caosdb/connection/__init__.py
index 638d4f0a8400f0ea1a2f197dcfdbe3fc933d4c10..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 100644
--- a/src/caosdb/connection/__init__.py
+++ b/src/caosdb/connection/__init__.py
@@ -1,29 +0,0 @@
-# -*- encoding: utf-8 -*-
-#
-# ** header v3.0
-# This file is a part of the CaosDB Project.
-#
-# Copyright (C) 2018 Research Group Biomedical Physics,
-# Max-Planck-Institute for Dynamics and Self-Organization Göttingen
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Affero General Public License as
-# published by the Free Software Foundation, either version 3 of the
-# License, or (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU Affero General Public License for more details.
-#
-# You should have received a copy of the GNU Affero General Public License
-# along with this program. If not, see <https://www.gnu.org/licenses/>.
-#
-# ** end header
-#
-from sys import hexversion
-from .utils import check_python_ssl_version
-
-check_python_ssl_version(hexversion)
-del check_python_ssl_version
-del hexversion
diff --git a/src/caosdb/connection/authentication/interface.py b/src/caosdb/connection/authentication/interface.py
index a364aeb564ee929d995b2f8098bd21e30e9733ab..f2cc5001cf8fa0f6d61ec65346f6a200ba0dfcd8 100644
--- a/src/caosdb/connection/authentication/interface.py
+++ b/src/caosdb/connection/authentication/interface.py
@@ -36,7 +36,7 @@ from caosdb.exceptions import LoginFailedError
# meta class compatible with Python 2 *and* 3:
ABC = ABCMeta('ABC', (object, ), {'__slots__': ()})
-_LOGGER = logging.getLogger("authentication")
+_LOGGER = logging.getLogger(__name__)
class AbstractAuthenticator(ABC):
diff --git a/src/caosdb/connection/authentication/keyring.py b/src/caosdb/connection/authentication/keyring.py
index d8be7ddf030577545230c9111fdad542b6d6e7e2..99d184136c20b23557efea0b54c648095a8d3ab2 100644
--- a/src/caosdb/connection/authentication/keyring.py
+++ b/src/caosdb/connection/authentication/keyring.py
@@ -28,7 +28,7 @@ retrieve the password.
"""
import sys
-import imp
+import importlib
from getpass import getpass
from caosdb.exceptions import ConfigurationError
from .external_credentials_provider import ExternalCredentialsProvider
@@ -52,17 +52,13 @@ def get_authentication_provider():
def _get_external_keyring():
try:
- fil, pathname, desc = imp.find_module("keyring", sys.path[1:])
- module = imp.load_module("external_keyring", fil, pathname, desc)
- return module
+ return importlib.import_module("keyring")
except ImportError:
raise RuntimeError(
"The keyring password method requires installation of the"
"keyring python package. On linux with python < 3.5, "
"this requires the installation of dbus-python as a "
"system package.")
- finally:
- fil.close()
def _call_keyring(**config):
@@ -74,7 +70,6 @@ def _call_keyring(**config):
url = config.get("url")
username = config.get("username")
app = "caosdb — {}".format(url)
- password = _call_keyring(app=app, username=username)
external_keyring = _get_external_keyring()
password = external_keyring.get_password(app, username)
if password is None:
diff --git a/src/caosdb/connection/connection.py b/src/caosdb/connection/connection.py
index 9e273a56778737033fda9f342f967f56946b501b..43eb3410d8d5bdc8323a811fd2b6424fb75f3fda 100644
--- a/src/caosdb/connection/connection.py
+++ b/src/caosdb/connection/connection.py
@@ -60,7 +60,7 @@ except ImportError:
# pylint: disable=missing-docstring
-_LOGGER = logging.getLogger("connection")
+_LOGGER = logging.getLogger(__name__)
class _WrappedHTTPResponse(CaosDBHTTPResponse):
@@ -176,16 +176,7 @@ class _DefaultCaosDBServerConnection(CaosDBServerConnection):
if "ssl_version" in config and config["cacert"] is not None:
ssl_version = getattr(ssl, config["ssl_version"])
else:
- try:
- ssl_version = ssl.PROTOCOL_TLS
- except AttributeError:
- # deprecated since Python 3.6, but necessary until there.
- try:
- ssl_version = ssl.PROTOCOL_TLSv1_2
- except AttributeError:
- print("It seems as if your Python version does not support current encryption"
- "standards (such as TLS > 1.0), please upgrade to at least Python 3.4.")
- sys.exit(1)
+ ssl_version = ssl.PROTOCOL_TLS
context = ssl.SSLContext(ssl_version)
context.verify_mode = ssl.CERT_REQUIRED
diff --git a/src/caosdb/connection/encode.py b/src/caosdb/connection/encode.py
index 970c4191e1defc55b2b4a6c7d01e0c9d4ba2952f..7b092aae784a76abec0104ef7269df7ae0111b3b 100644
--- a/src/caosdb/connection/encode.py
+++ b/src/caosdb/connection/encode.py
@@ -53,19 +53,9 @@ __all__ = [
'gen_boundary', 'encode_and_quote', 'MultipartParam', 'encode_string',
'encode_file_header', 'get_body_size', 'get_headers', 'multipart_encode'
]
-from sys import hexversion
-try:
- from urllib.parse import quote_plus
-except ImportError:
- from urllib import quote_plus
-
-try:
- from io import UnsupportedOperation
-except ImportError:
- UnsupportedOperation = None
-
+from urllib.parse import quote_plus
+from io import UnsupportedOperation
import uuid
-
import re
import os
import mimetypes
@@ -83,24 +73,9 @@ def encode_and_quote(data):
if data is None:
return None
- if hexversion < 0x03000000:
- if isinstance(data, unicode):
- data = data.encode("utf-8")
return quote_plus(data)
-def _strify(string):
- """If string is a unicode string, encode it to UTF-8 and return the
- results, otherwise return str(s), or None if s is None."""
- if string is None:
- return None
- if hexversion < 0x03000000:
- if isinstance(string, unicode):
- return string.encode("utf-8")
- return str(string)
- return str(string)
-
-
class MultipartParam(object):
"""Represents a single parameter in a multipart/form-data request.
@@ -143,22 +118,14 @@ class MultipartParam(object):
fileobj=None,
callback=None):
self.name = Header(name).encode()
- self.value = _strify(value)
+ self.value = value
if filename is None:
self.filename = None
- elif hexversion < 0x03000000:
- if isinstance(filename, unicode):
- # Encode with XML entities
- self.filename = filename.encode("ascii", "xmlcharrefreplace")
- else:
- self.filename = str(filename)
- self.filename = self.filename.encode("string_escape").\
- replace('"', '\\"')
else:
bfilename = filename.encode("ascii", "xmlcharrefreplace")
self.filename = bfilename.decode("UTF-8").replace('"', '\\"')
- self.filetype = _strify(filetype)
+ self.filetype = filetype
self.filesize = filesize
self.fileobj = fileobj
diff --git a/src/caosdb/connection/mockup.py b/src/caosdb/connection/mockup.py
index 692ec2b13f16556a9acfab2a377a04aaf27d650c..b37670b867cd88cf47e64084c6ccc802cad463b4 100644
--- a/src/caosdb/connection/mockup.py
+++ b/src/caosdb/connection/mockup.py
@@ -56,7 +56,7 @@ class MockUpResponse(CaosDBHTTPResponse):
def read(self, size=-1):
"""Return the body of the response."""
- return self.response.read(size)
+ return self.response.read(size).encode()
def getheader(self, name, default=None):
"""Get the contents of the header `name`, or `default` if there is no
diff --git a/src/caosdb/connection/utils.py b/src/caosdb/connection/utils.py
index 8c1518c1ba66b45c69d5b9fa0d137f0df633cd0c..9056bf9dea14fa2fa441fa13a5efe8e776990284 100644
--- a/src/caosdb/connection/utils.py
+++ b/src/caosdb/connection/utils.py
@@ -23,19 +23,9 @@
#
"""Utility functions for the connection module."""
from __future__ import unicode_literals, print_function
-try:
- from builtins import str, unicode # pylint: disable=redefined-builtin
-except ImportError:
- from builtins import str as unicode
-try: # pragma: no cover
- # python3
- from urllib.parse import (urlencode as _urlencode, quote as _quote,
- urlparse, urlunparse, unquote as _unquote)
-except ImportError: # pragma: no cover
- # python2
- from urllib import (urlencode as _urlencode, quote as _quote, unquote as
- _unquote)
- from urlparse import urlparse, urlunparse
+from builtins import str as unicode
+from urllib.parse import (urlencode as _urlencode, quote as _quote,
+ urlparse, urlunparse, unquote as _unquote)
import re
@@ -122,7 +112,7 @@ def make_uri_path(segments=None, query=None):
def quote(string):
- enc = unicode(string).encode('utf-8')
+ enc = string.encode('utf-8')
return _quote(enc).replace('/', '%2F')
@@ -136,35 +126,6 @@ def parse_url(url):
return fullurl
-def check_python_ssl_version(hexversion):
- """Check the python version.
-
- If `version < 2.7.9` or `3.0 <= version < 3.2` the ssl library does not
- actually verify the ssl certificates send by the server. That is evil and
- these versions shall not be used.
-
- Parameters
- ----------
- hexversion : int
- A python version.
-
- Raises
- ------
- Exception
- If the version does not fully support ssl encryption.
- """
- if hexversion < 0x02070900:
- raise Exception(
- "version " + str(hex(hexversion)) +
- "\nPython version is smaller than 2.7.9. It is not does not fully support SSL encryption. Please update your Python to 2.7.9 or greater, or 3.2 or greater."
- )
- elif hexversion >= 0x03000000 and hexversion < 0x03020000:
- raise Exception(
- "version " + str(hex(hexversion)) +
- "\nPython 3 version is smaller than 3.2. It is not does not fully support SSL encryption. Please update your Python to 2.7.9 or greater, or 3.2 or greater."
- )
-
-
_PATTERN = re.compile(r"^SessionToken=([^;]*);.*$")
diff --git a/src/caosdb/high_level_api.py b/src/caosdb/high_level_api.py
new file mode 100644
index 0000000000000000000000000000000000000000..427a095a4bafc0c372b0169298f2980dbd902c49
--- /dev/null
+++ b/src/caosdb/high_level_api.py
@@ -0,0 +1,1025 @@
+# -*- coding: utf-8 -*-
+#
+# This file is a part of the CaosDB Project.
+#
+# Copyright (C) 2018 Research Group Biomedical Physics,
+# Max-Planck-Institute for Dynamics and Self-Organization Göttingen
+# Copyright (C) 2020 Timm Fitschen <t.fitschen@indiscale.com>
+# Copyright (C) 2020-2022 IndiScale GmbH <info@indiscale.com>
+# Copyright (C) 2022 Alexander Schlemmer <alexander.schlemmer@ds.mpg.de>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see <https://www.gnu.org/licenses/>.
+#
+# ** end header
+#
+
+"""
+A high level API for accessing CaosDB entities from within python.
+
+This is refactored from apiutils.
+"""
+
+from caosdb.common.datatype import (BOOLEAN, DATETIME, DOUBLE, FILE, INTEGER,
+ REFERENCE, TEXT,
+ is_list_datatype,
+ get_list_datatype,
+ is_reference)
+import caosdb as db
+
+from .apiutils import get_type_of_entity_with, create_flat_list
+import warnings
+
+from typing import Any, Optional, List, Union, Dict
+
+import yaml
+
+from dataclasses import dataclass, fields
+from datetime import datetime
+from dateutil import parser
+
+warnings.warn("""EXPERIMENTAL! The high_level_api module is experimental and may be changed or
+removed in future. Its purpose is to give an impression on how the Python client user interface
+might be changed.""")
+
+
+def standard_type_for_high_level_type(high_level_record: "CaosDBPythonEntity",
+ return_string: bool = False):
+ """
+ For a given CaosDBPythonEntity either return the corresponding
+ class in the standard CaosDB API or - if return_string is True - return
+ the role as a string.
+ """
+ if type(high_level_record) == CaosDBPythonRecord:
+ if not return_string:
+ return db.Record
+ return "Record"
+ elif type(high_level_record) == CaosDBPythonFile:
+ if not return_string:
+ return db.File
+ return "File"
+ elif type(high_level_record) == CaosDBPythonProperty:
+ if not return_string:
+ return db.Property
+ return "Property"
+ elif type(high_level_record) == CaosDBPythonRecordType:
+ if not return_string:
+ return db.RecordType
+ return "RecordType"
+ elif type(high_level_record) == CaosDBPythonEntity:
+ if not return_string:
+ return db.Entity
+ return "Entity"
+ raise RuntimeError("Incompatible type.")
+
+
+def high_level_type_for_role(role: str):
+ if role == "Record":
+ return CaosDBPythonRecord
+ if role == "File":
+ return CaosDBPythonFile
+ if role == "Property":
+ return CaosDBPythonProperty
+ if role == "RecordType":
+ return CaosDBPythonRecordType
+ if role == "Entity":
+ return CaosDBPythonEntity
+ raise RuntimeError("Unknown role.")
+
+
+def high_level_type_for_standard_type(standard_record: db.Entity):
+ if not isinstance(standard_record, db.Entity):
+ raise ValueError()
+ role = standard_record.role
+ if role == "Record" or type(standard_record) == db.Record:
+ return CaosDBPythonRecord
+ elif role == "File" or type(standard_record) == db.File:
+ return CaosDBPythonFile
+ elif role == "Property" or type(standard_record) == db.Property:
+ return CaosDBPythonProperty
+ elif role == "RecordType" or type(standard_record) == db.RecordType:
+ return CaosDBPythonRecordType
+ elif role == "Entity" or type(standard_record) == db.Entity:
+ return CaosDBPythonEntity
+ raise RuntimeError("Incompatible type.")
+
+
+@dataclass
+class CaosDBPropertyMetaData:
+ # name is already the name of the attribute
+ unit: Optional[str] = None
+ datatype: Optional[str] = None
+ description: Optional[str] = None
+ id: Optional[int] = None
+ importance: Optional[str] = None
+
+
+class CaosDBPythonUnresolved:
+ pass
+
+
+@dataclass
+class CaosDBPythonUnresolvedParent(CaosDBPythonUnresolved):
+ """
+ Parents can be either given by name or by ID.
+
+ When resolved, both fields should be set.
+ """
+
+ id: Optional[int] = None
+ name: Optional[str] = None
+
+
+@dataclass
+class CaosDBPythonUnresolvedReference(CaosDBPythonUnresolved):
+
+ def __init__(self, id=None):
+ self.id = id
+
+
+class CaosDBPythonEntity(object):
+
+ def __init__(self):
+ """
+ Initialize a new CaosDBPythonEntity for the high level python api.
+
+ Parents are either unresolved references or CaosDB RecordTypes.
+
+ Properties are stored directly as attributes for the object.
+ Property metadata is maintained in a dctionary _properties_metadata that should
+ never be accessed directly, but only using the get_property_metadata function.
+ If property values are references to other objects, they will be stored as
+ CaosDBPythonUnresolvedReference objects that can be resolved later into
+ CaosDBPythonRecords.
+ """
+
+ # Parents are either unresolved references or CaosDB RecordTypes
+ self._parents: List[Union[
+ CaosDBPythonUnresolvedParent, CaosDBPythonRecordType]] = []
+ # self._id: int = CaosDBPythonEntity._get_new_id()
+ self._id: Optional[int] = None
+ self._name: Optional[str] = None
+ self._description: Optional[str] = None
+ self._version: Optional[str] = None
+
+ self._file: Optional[str] = None
+ self._path: Optional[str] = None
+
+ # name: name of property, value: property metadata
+ self._properties_metadata: Dict[CaosDBPropertyMetaData] = dict()
+
+ # Store all current attributes as forbidden attributes
+ # which must not be changed by the set_property function.
+ self._forbidden = dir(self) + ["_forbidden"]
+
+ def use_parameter(self, name, value):
+ self.__setattr__(name, value)
+ return value
+
+ @property
+ def id(self):
+ """
+ Getter for the id.
+ """
+ return self._id
+
+ @id.setter
+ def id(self, val: int):
+ self._id = val
+
+ @property
+ def name(self):
+ """
+ Getter for the name.
+ """
+ return self._name
+
+ @name.setter
+ def name(self, val: str):
+ self._name = val
+
+ @property
+ def file(self):
+ """
+ Getter for the file.
+ """
+ if type(self) != CaosDBPythonFile:
+ raise RuntimeError("Please don't use the file attribute for entities"
+ " that are no files.")
+ return self._file
+
+ @file.setter
+ def file(self, val: str):
+ if val is not None and type(self) != CaosDBPythonFile:
+ raise RuntimeError("Please don't use the file attribute for entities"
+ " that are no files.")
+ self._file = val
+
+ @property
+ def path(self):
+ """
+ Getter for the path.
+ """
+ if type(self) != CaosDBPythonFile:
+ raise RuntimeError("Please don't use the path attribute for entities"
+ " that are no files.")
+ return self._path
+
+ @path.setter
+ def path(self, val: str):
+ if val is not None and type(self) != CaosDBPythonFile:
+ raise RuntimeError("Please don't use the path attribute for entities"
+ " that are no files.")
+ self._path = val
+
+ @property
+ def description(self):
+ """
+ Getter for the description.
+ """
+ return self._description
+
+ @description.setter
+ def description(self, val: str):
+ self._description = val
+
+ @property
+ def version(self):
+ """
+ Getter for the version.
+ """
+ return self._version
+
+ @version.setter
+ def version(self, val: str):
+ self._version = val
+
+ def _set_property_from_entity(self, ent: db.Entity, importance: str,
+ references: Optional[db.Container]):
+ """
+ Set a new property using an entity from the normal python API.
+
+ ent : db.Entity
+ The entity to be set.
+ """
+
+ if ent.name is None:
+ raise RuntimeError("Setting properties without name is impossible.")
+
+ if ent.name in self.get_properties():
+ raise RuntimeError("Multiproperty not implemented yet.")
+
+ val = self._type_converted_value(ent.value, ent.datatype,
+ references)
+ self.set_property(
+ ent.name,
+ val,
+ datatype=ent.datatype)
+ metadata = self.get_property_metadata(ent.name)
+
+ for prop_name in fields(metadata):
+ k = prop_name.name
+ if k == "importance":
+ metadata.importance = importance
+ else:
+ metadata.__setattr__(k, ent.__getattribute__(k))
+
+ def get_property_metadata(self, prop_name: str) -> CaosDBPropertyMetaData:
+ """
+ Retrieve the property metadata for the property with name prop_name.
+
+ If the property with the given name does not exist or is forbidden, raise an exception.
+ Else return the metadata associated with this property.
+
+ If no metadata does exist yet for the given property, a new object will be created
+ and returned.
+
+ prop_name: str
+ Name of the property to retrieve metadata for.
+ """
+
+ if not self.property_exists(prop_name):
+ raise RuntimeError("The property with name {} does not exist.".format(prop_name))
+
+ if prop_name not in self._properties_metadata:
+ self._properties_metadata[prop_name] = CaosDBPropertyMetaData()
+
+ return self._properties_metadata[prop_name]
+
+ def property_exists(self, prop_name: str):
+ """
+ Check whether a property exists already.
+ """
+ return prop_name not in self._forbidden and prop_name in self.__dict__
+
+ def set_property(self,
+ name: str,
+ value: Any,
+ overwrite: bool = False,
+ datatype: Optional[str] = None):
+ """
+ Set a property for this entity with a name and a value.
+
+ If this property is already set convert the value into a list and append the value.
+ This behavior can be overwritten using the overwrite flag, which will just overwrite
+ the existing value.
+
+ name: str
+ Name of the property.
+
+ value: Any
+ Value of the property.
+
+ overwrite: bool
+ Use this if you definitely only want one property with
+ that name (set to True).
+ """
+
+ if name in self._forbidden:
+ raise RuntimeError("Entity cannot be converted to a corresponding "
+ "Python representation. Name of property " +
+ name + " is forbidden!")
+
+ already_exists = self.property_exists(name)
+
+ if already_exists and not overwrite:
+ # each call to set_property checks first if it already exists
+ # if yes: Turn the attribute into a list and
+ # place all the elements into that list.
+ att = self.__getattribute__(name)
+
+ if isinstance(att, list):
+ # just append, see below
+ pass
+ else:
+ old_att = self.__getattribute__(name)
+ self.__setattr__(name, [old_att])
+ att = self.__getattribute__(name)
+ att.append(value)
+ else:
+ self.__setattr__(name, value)
+
+ def __setattr__(self, name: str, val: Any):
+ """
+ Allow setting generic properties.
+ """
+
+ # TODO: implement checking the value to correspond to one of the datatypes
+ # known for conversion.
+
+ super().__setattr__(name, val)
+
+ def _type_converted_list(self,
+ val: List,
+ pr: str,
+ references: Optional[db.Container]):
+ """
+ Convert a list to a python list of the correct type.
+
+ val: List
+ The value of a property containing the list.
+
+ pr: str
+ The datatype according to the database entry.
+ """
+ if not is_list_datatype(pr) and not isinstance(val, list):
+ raise RuntimeError("Not a list.")
+
+ return [
+ self._type_converted_value(i, get_list_datatype(pr), references
+ ) for i in val]
+
+ def _type_converted_value(self,
+ val: Any,
+ pr: str,
+ references: Optional[db.Container]):
+ """
+ Convert val to the correct type which is indicated by the database
+ type string in pr.
+
+ References with ids will be turned into CaosDBPythonUnresolvedReference.
+ """
+
+ if val is None:
+ return None
+ elif isinstance(val, db.Entity):
+ # this needs to be checked as second case as it is the ONLY
+ # case which does not depend on pr
+ # TODO: we might need to pass through the reference container
+ return convert_to_python_object(val, references)
+ elif isinstance(val, list):
+ return self._type_converted_list(val, pr, references)
+ elif pr is None:
+ return val
+ elif pr == DOUBLE:
+ return float(val)
+ elif pr == BOOLEAN:
+ return bool(val)
+ elif pr == INTEGER:
+ return int(val)
+ elif pr == TEXT:
+ return str(val)
+ elif pr == FILE:
+ return CaosDBPythonUnresolvedReference(val)
+ elif pr == REFERENCE:
+ return CaosDBPythonUnresolvedReference(val)
+ elif pr == DATETIME:
+ return self._parse_datetime(val)
+ elif is_list_datatype(pr):
+ return self._type_converted_list(val, pr, references)
+ else:
+ # Generic references to entities:
+ return CaosDBPythonUnresolvedReference(val)
+
+ def _parse_datetime(self, val: Union[str, datetime]):
+ """
+ Convert val into a datetime object.
+ """
+ if isinstance(val, datetime):
+ return val
+ return parser.parse(val)
+
+ def get_property(self, name: str):
+ """
+ Return the value of the property with name name.
+
+ Raise an exception if the property does not exist.
+ """
+ if not self.property_exists(name):
+ raise RuntimeError("Property {} does not exist.".format(name))
+ att = self.__getattribute__(name)
+ return att
+
+ def attribute_as_list(self, name: str):
+ """
+ This is a workaround for the problem that lists containing only one
+ element are indistinguishable from simple types in this
+ representation.
+
+ TODO: still relevant? seems to be only a problem if LIST types are not used.
+ """
+ att = self.get_property(name)
+
+ if isinstance(att, list):
+ return att
+ else:
+ return [att]
+
+ def add_parent(self, parent: Union[
+ CaosDBPythonUnresolvedParent, "CaosDBPythonRecordType", str]):
+ """
+ Add a parent to this entity. Either using an unresolved parent or
+ using a real record type.
+
+ Strings as argument for parent will automatically be converted to an
+ unresolved parent. Likewise, integers as argument will be automatically converted
+ to unresolved parents with just an id.
+ """
+
+ if isinstance(parent, str):
+ parent = CaosDBPythonUnresolvedParent(name=parent)
+
+ if isinstance(parent, int):
+ parent = CaosDBPythonUnresolvedParent(id=parent)
+
+ if self.has_parent(parent):
+ raise RuntimeError("Duplicate parent.")
+ self._parents.append(parent)
+
+ def get_parents(self):
+ """
+ Returns all parents of this entity.
+
+ Use has_parent for checking for existence of parents
+ and add_parent for adding parents to this entity.
+ """
+ return self._parents
+
+ def has_parent(self, parent: Union[
+ CaosDBPythonUnresolvedParent, "CaosDBPythonRecordType"]):
+ """
+ Check whether this parent already exists for this entity.
+
+ Strings as argument for parent will automatically be converted to an
+ unresolved parent. Likewise, integers as argument will be automatically converted
+ to unresolved parents with just an id.
+ """
+
+ if isinstance(parent, str):
+ parent = CaosDBPythonUnresolvedParent(name=parent)
+
+ if isinstance(parent, int):
+ parent = CaosDBPythonUnresolvedParent(id=parent)
+
+ for p in self._parents:
+ if p.id is not None and p.id == parent.id:
+ return True
+ elif p.name is not None and p.name == parent.name:
+ return True
+ return False
+
+ def _resolve_caosdb_python_unresolved_reference(self, propval, deep,
+ references, visited):
+ # This does not make sense for unset ids:
+ if propval.id is None:
+ raise RuntimeError("Unresolved property reference without an ID.")
+ # have we encountered this id before:
+ if propval.id in visited:
+ # self.__setattr__(prop, visited[propval.id])
+ # don't do the lookup in the references container
+ return visited[propval.id]
+
+ if references is None:
+ ent = db.Entity(id=propval.id).retrieve()
+ obj = convert_to_python_object(ent, references)
+ visited[propval.id] = obj
+ if deep:
+ obj.resolve_references(deep, references, visited)
+ return obj
+
+ # lookup in container:
+ for ent in references:
+ # Entities in container without an ID will be skipped:
+ if ent.id is not None and ent.id == propval.id:
+ # resolve this entity:
+ obj = convert_to_python_object(ent, references)
+ visited[propval.id] = obj
+ # self.__setattr__(prop, visited[propval.id])
+ if deep:
+ obj.resolve_references(deep, references, visited)
+ return obj
+ return propval
+
+ def resolve_references(self, deep: bool, references: db.Container,
+ visited: Dict[Union[str, int],
+ "CaosDBPythonEntity"] = None):
+ """
+ Resolve this entity's references. This affects unresolved properties as well
+ as unresolved parents.
+
+ deep: bool
+ If True recursively resolve references also for all resolved references.
+
+ references: Optional[db.Container]
+ A container with references that might be resolved.
+ If None is passed as the container, this function tries to resolve entities from a running
+ CaosDB instance directly.
+ """
+
+ # This parameter is used in the recursion to keep track of already visited
+ # entites (in order to detect cycles).
+ if visited is None:
+ visited = dict()
+
+ for parent in self.get_parents():
+ # TODO
+ if isinstance(parent, CaosDBPythonUnresolvedParent):
+ pass
+
+ for prop in self.get_properties():
+ propval = self.__getattribute__(prop)
+ # Resolve all previously unresolved attributes that are entities:
+ if deep and isinstance(propval, CaosDBPythonEntity):
+ propval.resolve_references(deep, references)
+ elif isinstance(propval, list):
+ resolvedelements = []
+ for element in propval:
+ if deep and isinstance(element, CaosDBPythonEntity):
+ element.resolve_references(deep, references)
+ resolvedelements.append(element)
+ if isinstance(element, CaosDBPythonUnresolvedReference):
+ resolvedelements.append(
+ self._resolve_caosdb_python_unresolved_reference(element, deep,
+ references, visited))
+ else:
+ resolvedelements.append(element)
+ self.__setattr__(prop, resolvedelements)
+
+ elif isinstance(propval, CaosDBPythonUnresolvedReference):
+ val = self._resolve_caosdb_python_unresolved_reference(propval, deep,
+ references, visited)
+ self.__setattr__(prop, val)
+
+ def get_properties(self):
+ """
+ Return the names of all properties.
+ """
+
+ return [p for p in self.__dict__
+ if p not in self._forbidden]
+
+ @staticmethod
+ def deserialize(serialization: dict):
+ """
+ Deserialize a yaml representation of an entity in high level API form.
+ """
+
+ if "role" in serialization:
+ entity = high_level_type_for_role(serialization["role"])()
+ else:
+ entity = CaosDBPythonRecord()
+
+ for parent in serialization["parents"]:
+ if "unresolved" in parent:
+ id = None
+ name = None
+ if "id" in parent:
+ id = parent["id"]
+ if "name" in parent:
+ name = parent["name"]
+ entity.add_parent(CaosDBPythonUnresolvedParent(
+ id=id, name=name))
+ else:
+ raise NotImplementedError()
+
+ for baseprop in ("name", "id", "description", "version"):
+ if baseprop in serialization:
+ entity.__setattr__(baseprop, serialization[baseprop])
+
+ if type(entity) == CaosDBPythonFile:
+ entity.file = serialization["file"]
+ entity.path = serialization["path"]
+
+ for p in serialization["properties"]:
+ # The property needs to be set first:
+
+ prop = serialization["properties"][p]
+ if isinstance(prop, dict):
+ if "unresolved" in prop:
+ entity.__setattr__(p, CaosDBPythonUnresolvedReference(
+ id=prop["id"]))
+ else:
+ entity.__setattr__(p,
+ entity.deserialize(prop))
+ else:
+ entity.__setattr__(p, prop)
+
+ # if there is no metadata in the yaml file just initialize an empty metadata object
+ if "metadata" in serialization and p in serialization["metadata"]:
+ metadata = serialization["metadata"][p]
+ propmeta = entity.get_property_metadata(p)
+
+ for f in fields(propmeta):
+ if f.name in metadata:
+ propmeta.__setattr__(f.name, metadata[f.name])
+ else:
+ raise NotImplementedError()
+
+ return entity
+
+ def serialize(self, without_metadata: bool = False, visited: dict = None):
+ """
+ Serialize necessary information into a dict.
+
+ without_metadata: bool
+ If True don't set the metadata field in order to increase
+ readability. Not recommended if deserialization is needed.
+ """
+
+ if visited is None:
+ visited = dict()
+
+ if self in visited:
+ return visited[self]
+
+ metadata: Dict[str, Any] = dict()
+ properties = dict()
+ parents = list()
+
+ # The full information to be returned:
+ fulldict = dict()
+ visited[self] = fulldict
+
+ # Add CaosDB role:
+ fulldict["role"] = standard_type_for_high_level_type(self, True)
+
+ for parent in self._parents:
+ if isinstance(parent, CaosDBPythonEntity):
+ parents.append(parent.serialize(without_metadata, visited))
+ elif isinstance(parent, CaosDBPythonUnresolvedParent):
+ parents.append({"name": parent.name, "id": parent.id,
+ "unresolved": True})
+ else:
+ raise RuntimeError("Incompatible class used as parent.")
+
+ for baseprop in ("name", "id", "description", "version"):
+ val = self.__getattribute__(baseprop)
+ if val is not None:
+ fulldict[baseprop] = val
+
+ if type(self) == CaosDBPythonFile:
+ fulldict["file"] = self.file
+ fulldict["path"] = self.path
+
+ for p in self.get_properties():
+ m = self.get_property_metadata(p)
+ metadata[p] = dict()
+ for f in fields(m):
+ val = m.__getattribute__(f.name)
+ if val is not None:
+ metadata[p][f.name] = val
+
+ val = self.get_property(p)
+ if isinstance(val, CaosDBPythonUnresolvedReference):
+ properties[p] = {"id": val.id, "unresolved": True}
+ elif isinstance(val, CaosDBPythonEntity):
+ properties[p] = val.serialize(without_metadata, visited)
+ elif isinstance(val, list):
+ serializedelements = []
+ for element in val:
+ if isinstance(element, CaosDBPythonUnresolvedReference):
+ elm = dict()
+ elm["id"] = element.id
+ elm["unresolved"] = True
+ serializedelements.append(elm)
+ elif isinstance(element, CaosDBPythonEntity):
+ serializedelements.append(
+ element.serialize(without_metadata,
+ visited))
+ else:
+ serializedelements.append(element)
+ properties[p] = serializedelements
+ else:
+ properties[p] = val
+
+ fulldict["properties"] = properties
+ fulldict["parents"] = parents
+
+ if not without_metadata:
+ fulldict["metadata"] = metadata
+ return fulldict
+
+ def __str__(self):
+ return yaml.dump(self.serialize(False))
+
+ # This seemed like a good solution, but makes it difficult to
+ # compare python objects directly:
+ #
+ # def __repr__(self):
+ # return yaml.dump(self.serialize(True))
+
+
+class CaosDBPythonRecord(CaosDBPythonEntity):
+ pass
+
+
+class CaosDBPythonRecordType(CaosDBPythonEntity):
+ pass
+
+
+class CaosDBPythonProperty(CaosDBPythonEntity):
+ pass
+
+
+class CaosDBMultiProperty:
+ """
+ This implements a multi property using a python list.
+ """
+
+ def __init__(self):
+ raise NotImplementedError()
+
+
+class CaosDBPythonFile(CaosDBPythonEntity):
+ def download(self, target=None):
+ if self.id is None:
+ raise RuntimeError("Cannot download file when id is missing.")
+ f = db.File(id=self.id).retrieve()
+ return f.download(target)
+
+
+BASE_ATTRIBUTES = (
+ "id", "name", "description", "version", "path", "file")
+
+
+def _single_convert_to_python_object(robj: CaosDBPythonEntity,
+ entity: db.Entity,
+ references: Optional[db.Container] = None):
+ """
+ Convert a db.Entity from the standard API to a (previously created)
+ CaosDBPythonEntity from the high level API.
+
+ This method will not resolve any unresolved references, so reference properties
+ as well as parents will become unresolved references in the first place.
+
+ The optional third parameter can be used
+ to resolve references that occur in the converted entities and resolve them
+ to their correct representations. (Entities that are not found remain as
+ CaosDBPythonUnresolvedReferences.)
+
+ Returns the input object robj.
+ """
+ for base_attribute in BASE_ATTRIBUTES:
+ val = entity.__getattribute__(base_attribute)
+ if val is not None:
+ if isinstance(val, db.common.models.Version):
+ val = val.id
+ robj.__setattr__(base_attribute, val)
+
+ for prop in entity.properties:
+ robj._set_property_from_entity(prop, entity.get_importance(prop), references)
+
+ for parent in entity.parents:
+ robj.add_parent(CaosDBPythonUnresolvedParent(id=parent.id,
+ name=parent.name))
+
+ return robj
+
+
+def _convert_property_value(propval):
+ if isinstance(propval, CaosDBPythonUnresolvedReference):
+ propval = propval.id
+ elif isinstance(propval, CaosDBPythonEntity):
+ propval = _single_convert_to_entity(
+ standard_type_for_high_level_type(propval)(), propval)
+ elif isinstance(propval, list):
+ propval = [_convert_property_value(element) for element in propval]
+
+ # TODO: test case for list missing
+
+ return propval
+
+
+def _single_convert_to_entity(entity: db.Entity,
+ robj: CaosDBPythonEntity):
+ """
+ Convert a CaosDBPythonEntity to an entity in standard pylib format.
+
+ entity: db.Entity
+ An empty entity.
+
+ robj: CaosDBPythonEntity
+ The CaosDBPythonEntity that is supposed to be converted to the entity.
+ """
+
+ for base_attribute in BASE_ATTRIBUTES:
+ if base_attribute in ("file", "path") and not isinstance(robj, CaosDBPythonFile):
+ continue
+
+ # Skip version:
+ if base_attribute == "version":
+ continue
+
+ val = robj.__getattribute__(base_attribute)
+
+ if val is not None:
+ entity.__setattr__(base_attribute, val)
+
+ for parent in robj.get_parents():
+ if isinstance(parent, CaosDBPythonUnresolvedParent):
+ entity.add_parent(name=parent.name, id=parent.id)
+ elif isinstance(parent, CaosDBPythonRecordType):
+ raise NotImplementedError()
+ else:
+ raise RuntimeError("Incompatible class used as parent.")
+
+ for prop in robj.get_properties():
+ propval = robj.__getattribute__(prop)
+ metadata = robj.get_property_metadata(prop)
+
+ propval = _convert_property_value(propval)
+
+ entity.add_property(
+ name=prop,
+ value=propval,
+ unit=metadata.unit,
+ importance=metadata.importance,
+ datatype=metadata.datatype,
+ description=metadata.description,
+ id=metadata.id)
+
+ return entity
+
+
+def convert_to_entity(python_object):
+ if isinstance(python_object, db.Container):
+ # Create a list of objects:
+
+ return [convert_to_entity(i) for i in python_object]
+ elif isinstance(python_object, CaosDBPythonRecord):
+ return _single_convert_to_entity(db.Record(), python_object)
+ elif isinstance(python_object, CaosDBPythonFile):
+ return _single_convert_to_entity(db.File(), python_object)
+ elif isinstance(python_object, CaosDBPythonRecordType):
+ return _single_convert_to_entity(db.RecordType(), python_object)
+ elif isinstance(python_object, CaosDBPythonProperty):
+ return _single_convert_to_entity(db.Property(), python_object)
+ elif isinstance(python_object, CaosDBPythonEntity):
+ return _single_convert_to_entity(db.Entity(), python_object)
+ else:
+ raise ValueError("Cannot convert an object of this type.")
+
+
+def convert_to_python_object(entity: Union[db.Container, db.Entity],
+ references: Optional[db.Container] = None):
+ """
+ Convert either a container of CaosDB entities or a single CaosDB entity
+ into the high level representation.
+
+ The optional second parameter can be used
+ to resolve references that occur in the converted entities and resolve them
+ to their correct representations. (Entities that are not found remain as
+ CaosDBPythonUnresolvedReferences.)
+ """
+ if isinstance(entity, db.Container):
+ # Create a list of objects:
+ return [convert_to_python_object(i, references) for i in entity]
+
+ return _single_convert_to_python_object(
+ high_level_type_for_standard_type(entity)(), entity, references)
+
+
+def new_high_level_entity(entity: db.RecordType,
+ importance_level: str,
+ name: str = None):
+ """
+ Create an new record in high level format based on a record type in standard format.
+
+ entity: db.RecordType
+ The record type to initialize the new record from.
+
+ importance_level: str
+ None, obligatory, recommended or suggested
+ Initialize new properties up to this level.
+ Properties in the record type with no importance will be added
+ regardless of the importance_level.
+
+ name: str
+ Name of the new record.
+ """
+
+ r = db.Record(name=name)
+ r.add_parent(entity)
+
+ impmap = {
+ None: 0, "SUGGESTED": 3, "RECOMMENDED": 2, "OBLIGATORY": 1}
+
+ for prop in entity.properties:
+ imp = entity.get_importance(prop)
+ if imp is not None and impmap[importance_level] < impmap[imp]:
+ continue
+
+ r.add_property(prop)
+
+ return convert_to_python_object(r)
+
+
+def create_record(rtname: str, name: str = None, **kwargs):
+ """
+ Create a new record based on the name of a record type. The new record is returned.
+
+ rtname: str
+ The name of the record type.
+
+ name: str
+ This is optional. A name for the new record.
+
+ kwargs:
+ Additional arguments are used to set attributes of the
+ new record.
+ """
+ obj = new_high_level_entity(
+ db.RecordType(name=rtname).retrieve(), "SUGGESTED", name)
+ for key, value in kwargs.items():
+ obj.__setattr__(key, value)
+ return obj
+
+
+def load_external_record(record_name: str):
+ """
+ Retrieve a record by name and convert it to the high level API format.
+ """
+ return convert_to_python_object(db.Record(name=record_name).retrieve())
+
+
+def create_entity_container(record: CaosDBPythonEntity):
+ """
+ Convert this record into an entity container in standard format that can be used
+ to insert or update entities in a running CaosDB instance.
+ """
+ ent = convert_to_entity(record)
+ lse: List[db.Entity] = [ent]
+ create_flat_list([ent], lse)
+ return db.Container().extend(lse)
+
+
+def query(query: str, resolve_references: bool = True, references: db.Container = None):
+ """
+
+ """
+ res = db.execute_query(query)
+ objects = convert_to_python_object(res)
+ if resolve_references:
+ for obj in objects:
+ obj.resolve_references(True, references)
+ return objects
diff --git a/src/caosdb/schema-pycaosdb-ini.yml b/src/caosdb/schema-pycaosdb-ini.yml
new file mode 100644
index 0000000000000000000000000000000000000000..5dabdd89795e19a757209e03cc843776be705777
--- /dev/null
+++ b/src/caosdb/schema-pycaosdb-ini.yml
@@ -0,0 +1,103 @@
+schema-pycaosdb-ini:
+ type: object
+ additionalProperties: false
+ properties:
+ Container:
+ additionalProperties: false
+ properties:
+ debug:
+ default: 0
+ type: integer
+ enum: [0, 1, 2]
+ Connection:
+ description: Settings for the connection to the CaosDB server
+ additionalProperties: false
+ properties:
+ url:
+ description: URL of the CaosDB server
+ type: string
+ pattern: https://[-a-zA-Z0-9\.]+(:[0-9]+)?(/)?
+ examples: ["https://demo.indiscale.com/", "https://localhost:10443/"]
+ username:
+ type: string
+ description: User name used for authentication with the server
+ examples: [admin]
+ password_method:
+ description: The password input method defines how the password is supplied that is used for authentication with the server.
+ type: string
+ default: input
+ enum: [input, plain, pass, keyring]
+ password_identifier:
+ type: string
+ password:
+ type: string
+ examples: [caosdb]
+ auth_token:
+ type: string
+ description: Using an authentication token to connect with the server. This setting is not recommended for users.
+ cacert:
+ type: string
+ description: If the server's SSL certificate cannot be validated by your installed certificates (default or installed by your admins), you may also need to supply the matching key (pem file)
+ examples: [/path/to/caosdb.ca.pem]
+ ssl_insecure:
+ description: If this option is set, the SSL certificate of the server will not be validated. This has the potential of a man-in-the-middle attack. Use with care!
+ type: boolean
+ default: false
+ ssl_version:
+ description: You may define the ssl version to be used. It has to be the name of the corresponding attribute in the Python ssl module.
+ examples: [PROTOCOL_TLS]
+ debug:
+ default: 0
+ type: integer
+ enum: [0, 1, 2]
+ description: The debug key allows control the verbosity. Set it to 1 or 2 in case you want to see debugging output or if you want to learn more about the internals of the protocol. 0 disables debugging output.
+ socket_proxy:
+ examples: ["localhost:12345"]
+ type: string
+ description: You can define a socket proxy to be used. This is for the case that the server sits behind a firewall which is being tunnelled with a socket proxy (SOCKS4 or SOCKS5) (e.g. via ssh's -D option or a dedicated proxy server).
+ implementation:
+ description: This option is used internally and for testing. Do not override.
+ examples: [_DefaultCaosDBServerConnection]
+ timeout:
+ type: integer
+ allOf:
+ - if:
+ properties:
+ password_method:
+ const: input
+ required: [password_method]
+ then:
+ required: [url]
+ - if:
+ properties:
+ password_method:
+ const: plain
+ required: [password_method]
+ then:
+ required: [url, username, password]
+ - if:
+ properties:
+ password_method:
+ const: pass
+ required: [password_method]
+ then:
+ required: [url, username, password_identifier]
+ - if:
+ properties:
+ password_method:
+ const: keyring
+ required: [password_method]
+ then:
+ required: [url, username]
+ IntegrationTests:
+ description: "Used by the integration test suite from the caosdb-pyinttest repo."
+ additionalProperties: true
+ Misc:
+ description: "Some additional configuration settings."
+ additionalProperties: true
+ advancedtools:
+ description: "Configuration settings for the caosadvancedtools."
+ additionalProperties: true
+ sss_helper:
+ description: "Configuration settings for server-side scripting."
+ additionalProperties: true
diff --git a/src/caosdb/utils/caosdb_admin.py b/src/caosdb/utils/caosdb_admin.py
index 9c18f8962b3561999950059f23453d05edc0584d..9fb94f57683036f5432a40198cc4ae98893665fb 100755
--- a/src/caosdb/utils/caosdb_admin.py
+++ b/src/caosdb/utils/caosdb_admin.py
@@ -131,11 +131,17 @@ def do_create_user(args):
def do_activate_user(args):
- admin._update_user(name=args.user_name, status="ACTIVE")
+ try:
+ admin._update_user(name=args.user_name, status="ACTIVE")
+ except HTTPClientError as e:
+ print(e.msg)
def do_deactivate_user(args):
- admin._update_user(name=args.user_name, status="INACTIVE")
+ try:
+ admin._update_user(name=args.user_name, status="INACTIVE")
+ except HTTPClientError as e:
+ print(e.msg)
def do_set_user_password(args):
@@ -143,7 +149,10 @@ def do_set_user_password(args):
password = _promt_for_pw()
else:
password = args.user_password
- admin._update_user(name=args.user_name, password=password)
+ try:
+ admin._update_user(name=args.user_name, password=password)
+ except HTTPClientError as e:
+ print(e.msg)
def do_add_user_roles(args):
diff --git a/src/caosdb/utils/plantuml.py b/src/caosdb/utils/plantuml.py
index 75b96ee3aa28ba916adc69e418de398abfe23356..6252a48983c62e7a2f33113422205209d616b5b6 100644
--- a/src/caosdb/utils/plantuml.py
+++ b/src/caosdb/utils/plantuml.py
@@ -34,13 +34,31 @@ plantuml FILENAME.pu -> FILENAME.png
"""
import os
+import shutil
import caosdb as db
+from caosdb.common.datatype import is_reference, get_referenced_recordtype
+
+from typing import List, Optional
+
+import tempfile
REFERENCE = "REFERENCE"
def get_description(description_str):
+ """Extract and format a description string from a record type or property.
+
+ Parameters
+ ----------
+ description_str : str
+ The description string that is going to be formatted.
+
+ Returns
+ -------
+ str
+ The reformatted description ending in a line break.
+ """
words = description_str.split()
lines = []
lines.append("")
@@ -66,13 +84,24 @@ class Grouped(object):
return self.parents
-def recordtypes_to_plantuml_string(iterable):
+def recordtypes_to_plantuml_string(iterable,
+ add_properties: bool = True,
+ add_recordtypes: bool = True,
+ add_legend: bool = True,
+ no_shadow: bool = False,
+ style: str = "default"):
"""Converts RecordTypes into a string for PlantUML.
This function obtains an iterable and returns a string which can
be input into PlantUML for a representation of all RecordTypes in
the iterable.
+ Current options for style
+ -------------------------
+
+ "default" - Standard rectangles with uml class circle and methods section
+ "salexan" - Round rectangles, hide circle and methods section
+
Current limitations
-------------------
@@ -81,8 +110,24 @@ def recordtypes_to_plantuml_string(iterable):
either the "type" attribute is None or
type(element) == RecordType.
- Inheritance of Properties is not rendered nicely at the moment.
+
+ Parameters
+ ----------
+ iterable: iterable of caosdb.Entity
+ The objects to be rendered with plantuml.
+
+ no_shadow : bool, optional
+ If true, tell plantuml to use a skin without blurred shadows.
+
+
+ Returns
+ -------
+ out : str
+ The plantuml string for the given container.
"""
+ # TODO: This function needs a review of python type hints.
+
classes = [el for el in iterable
if isinstance(el, db.RecordType)]
dependencies = {}
@@ -127,74 +172,90 @@ def recordtypes_to_plantuml_string(iterable):
return result
result = "@startuml\n\n"
- result += "skinparam classAttributeIconSize 0\n"
- result += "package Properties #DDDDDD {\n"
+ if no_shadow:
+ result += "skinparam shadowing false\n"
+
+ if style == "default":
+ result += "skinparam classAttributeIconSize 0\n"
+ elif style == "salexan":
+ result += """skinparam roundcorner 20\n
+skinparam boxpadding 20\n
+\n
+hide methods\n
+hide circle\n
+"""
+ else:
+ raise ValueError("Unknown style.")
- for p in properties:
- inheritances[p] = p.get_parents()
- dependencies[p] = []
+ if add_properties:
+ result += "package Properties #DDDDDD {\n"
+ for p in properties:
+ inheritances[p] = p.get_parents()
+ dependencies[p] = []
- result += "class \"{klass}\" << (P,#008800) >> {{\n".format(klass=p.name)
+ result += "class \"{klass}\" << (P,#008800) >> {{\n".format(klass=p.name)
- if p.description is not None:
- result += get_description(p.description)
- result += "\n..\n"
+ if p.description is not None:
+ result += get_description(p.description)
+ result += "\n..\n"
- if isinstance(p.datatype, str):
- result += "datatype: " + p.datatype + "\n"
- elif isinstance(p.datatype, db.Entity):
- result += "datatype: " + p.datatype.name + "\n"
- else:
- result += "datatype: " + str(p.datatype) + "\n"
+ if isinstance(p.datatype, str):
+ result += "datatype: " + p.datatype + "\n"
+ elif isinstance(p.datatype, db.Entity):
+ result += "datatype: " + p.datatype.name + "\n"
+ else:
+ result += "datatype: " + str(p.datatype) + "\n"
+ result += "}\n\n"
result += "}\n\n"
- result += "}\n\n"
- result += "package RecordTypes #DDDDDD {\n"
+ if add_recordtypes:
+ result += "package RecordTypes #DDDDDD {\n"
- for c in classes:
- inheritances[c] = c.get_parents()
- dependencies[c] = []
- result += "class \"{klass}\" << (C,#FF1111) >> {{\n".format(klass=c.name)
+ for c in classes:
+ inheritances[c] = c.get_parents()
+ dependencies[c] = []
+ result += "class \"{klass}\" << (C,#FF1111) >> {{\n".format(klass=c.name)
- if c.description is not None:
- result += get_description(c.description)
+ if c.description is not None:
+ result += get_description(c.description)
- props = ""
- props += _add_properties(c, importance=db.FIX)
- props += _add_properties(c, importance=db.OBLIGATORY)
- props += _add_properties(c, importance=db.RECOMMENDED)
- props += _add_properties(c, importance=db.SUGGESTED)
+ props = ""
+ props += _add_properties(c, importance=db.FIX)
+ props += _add_properties(c, importance=db.OBLIGATORY)
+ props += _add_properties(c, importance=db.RECOMMENDED)
+ props += _add_properties(c, importance=db.SUGGESTED)
- if len(props) > 0:
- result += "__Properties__\n" + props
- else:
- result += "\n..\n"
- result += "}\n\n"
+ if len(props) > 0:
+ result += "__Properties__\n" + props
+ else:
+ result += "\n..\n"
+ result += "}\n\n"
- for g in grouped:
- inheritances[g] = g.get_parents()
- result += "class \"{klass}\" << (G,#0000FF) >> {{\n".format(klass=g.name)
- result += "}\n\n"
+ for g in grouped:
+ inheritances[g] = g.get_parents()
+ result += "class \"{klass}\" << (G,#0000FF) >> {{\n".format(klass=g.name)
+ result += "}\n\n"
- for c, parents in inheritances.items():
- for par in parents:
- result += "\"{par}\" <|-- \"{klass}\"\n".format(
- klass=c.name, par=par.name)
+ for c, parents in inheritances.items():
+ for par in parents:
+ result += "\"{par}\" <|-- \"{klass}\"\n".format(
+ klass=c.name, par=par.name)
- for c, deps in dependencies.items():
- for dep in deps:
- result += "\"{klass}\" *-- \"{dep}\"\n".format(
- klass=c.name, dep=dep)
+ for c, deps in dependencies.items():
+ for dep in deps:
+ result += "\"{klass}\" *-- \"{dep}\"\n".format(
+ klass=c.name, dep=dep)
- result += """
+ if add_legend:
+ result += """
package \"B is a subtype of A\" <<Rectangle>> {
A <|-right- B
note "This determines what you find when you query for the RecordType.\\n'FIND RECORD A' will provide Records which have a parent\\nA or B, while 'FIND RECORD B' will provide only Records which have a parent B." as N1
}
"""
- result += """
+ result += """
package \"The property P references an instance of D\" <<Rectangle>> {
class C {
@@ -211,25 +272,144 @@ package \"The property P references an instance of D\" <<Rectangle>> {
return result
-def to_graphics(recordtypes, filename):
- """ calls recordtypes_to_plantuml_string(), saves result to file and
+def retrieve_substructure(start_record_types, depth, result_id_set=None, result_container=None, cleanup=True):
+ """Recursively retrieves CaosDB record types and properties, starting
+ from given initial types up to a specific depth.
+
+ Parameters
+ ----------
+ start_record_types : Iterable[db.Entity]
+ Iterable with the entities to be displayed. Starting from these
+ entities more entities will be retrieved.
+ depth : int
+ The maximum depth up to which to retriev sub entities.
+ result_id_set : set[int]
+ Used by recursion. Filled with already visited ids.
+ result_container : db.Container
+ Used by recursion. Filled with already visited entities.
+ cleanup : bool
+ Used by recursion. If True return the resulting result_container.
+ Don't return anything otherwise.
+
+ Returns
+ -------
+ db.Container
+ A container containing all the retrieved entites
+ or None if cleanup is False.
+ """
+ # Initialize the id set and result container for level zero recursion depth:
+ if result_id_set is None:
+ result_id_set = set()
+ if result_container is None:
+ result_container = db.Container()
+
+ for entity in start_record_types:
+ entity.retrieve()
+ if entity.id not in result_id_set:
+ result_container.append(entity)
+ result_id_set.add(entity.id)
+ for prop in entity.properties:
+ if (is_reference(prop.datatype) and prop.datatype != db.FILE and depth > 0):
+ rt = db.RecordType(
+ name=get_referenced_recordtype(prop.datatype)).retrieve()
+ retrieve_substructure([rt], depth-1, result_id_set,
+ result_container, False)
+ # TODO: clean up this hack
+ # TODO: make it also work for files
+ if is_reference(prop.datatype) and prop.value is not None:
+ r = db.Record(id=prop.value).retrieve()
+ retrieve_substructure([r], depth-1, result_id_set, result_container, False)
+ if r.id not in result_id_set:
+ result_container.append(r)
+ result_id_set.add(r.id)
+
+ if prop.id not in result_id_set:
+ result_container.append(prop)
+ result_id_set.add(prop.id)
+
+ for parent in entity.parents:
+ rt = db.RecordType(id=parent.id).retrieve()
+ if parent.id not in result_id_set:
+ result_container.append(rt)
+ result_id_set.add(parent.id)
+ if depth > 0:
+ retrieve_substructure([rt], depth-1, result_id_set,
+ result_container, False)
+
+ if cleanup:
+ return result_container
+ return None
+
+
+def to_graphics(recordtypes: List[db.Entity], filename: str,
+ output_dirname: Optional[str] = None,
+ formats: List[str] = ["tsvg"],
+ silent: bool = True,
+ add_properties: bool = True,
+ add_recordtypes: bool = True,
+ add_legend: bool = True,
+ no_shadow: bool = False,
+ style: str = "default"):
+ """Calls recordtypes_to_plantuml_string(), saves result to file and
creates an svg image
- plantuml needs to be installed
- @params:
- recordtypes: itrable with the record types to be displayed
- filname: filename of the image (e.g. data_structure; data_structure.pu and
- data_structure.svg will be created.
+ plantuml needs to be installed.
+
+ Parameters
+ ----------
+ recordtypes : Iterable[db.Entity]
+ Iterable with the entities to be displayed.
+ filename : str
+ filename of the image without the extension(e.g. data_structure;
+ also without the preceeding path.
+ data_structure.pu and data_structure.svg will be created.)
+ output_dirname : str
+ the destination directory for the resulting images as defined by the "-o"
+ option by plantuml
+ default is to use current working dir
+ formats : List[str]
+ list of target formats as defined by the -t"..." options by plantuml, e.g. "tsvg"
+ silent : bool
+ Don't output messages.
+ no_shadow : bool, optional
+ If true, tell plantuml to use a skin without blurred shadows.
"""
- pu = recordtypes_to_plantuml_string(recordtypes)
-
- pu_filename = filename+".pu"
- with open(pu_filename, "w") as pu_file:
- pu_file.write(pu)
-
- cmd = "plantuml -tsvg %s" % pu_filename
- print("Executing:", cmd)
-
- if os.system(cmd) != 0:
- raise Exception("An error occured during the execution of plantuml. "
- "Is plantuml installed?")
+ pu = recordtypes_to_plantuml_string(iterable=recordtypes,
+ add_properties=add_properties,
+ add_recordtypes=add_recordtypes,
+ add_legend=add_legend,
+ no_shadow=no_shadow,
+ style=style)
+
+ if output_dirname is None:
+ output_dirname = os.getcwd()
+
+ allowed_formats = [
+ "tpng", "tsvg", "teps", "tpdf", "tvdx", "txmi",
+ "tscxml", "thtml", "ttxt", "tutxt", "tlatex", "tlatex:nopreamble"]
+
+ with tempfile.TemporaryDirectory() as td:
+
+ pu_filename = os.path.join(td, filename + ".pu")
+ with open(pu_filename, "w") as pu_file:
+ pu_file.write(pu)
+
+ for format in formats:
+ extension = format[1:]
+ if ":" in extension:
+ extension = extension[:extension.index(":")]
+
+ if format not in allowed_formats:
+ raise RuntimeError("Format not allowed.")
+ cmd = "plantuml -{} {}".format(format, pu_filename)
+ if not silent:
+ print("Executing:", cmd)
+
+ if os.system(cmd) != 0: # TODO: replace with subprocess.run
+ raise Exception("An error occured during the execution of "
+ "plantuml when using the format {}. "
+ "Is plantuml installed? "
+ "You might want to dry a different format.".format(format))
+ # copy only the final product into the target directory
+ shutil.copy(os.path.join(td, filename + "." + extension),
+ output_dirname)
diff --git a/src/caosdb/utils/register_tests.py b/src/caosdb/utils/register_tests.py
new file mode 100644
index 0000000000000000000000000000000000000000..9d0afcbb0845e1d8d31622e8ab9926f26f7e78f6
--- /dev/null
+++ b/src/caosdb/utils/register_tests.py
@@ -0,0 +1,136 @@
+#!/usr/bin/env python
+# encoding: utf-8
+#
+# This file is a part of the CaosDB Project.
+#
+# Copyright (C) 2022 Alexander Schlemmer <alexander.schlemmer@ds.mpg.de>
+# Copyright (C) 2022 Timm Fitschen <t.fitschen@indiscale.com>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+import caosdb as db
+from caosdb import administration as admin
+
+"""
+This module implements a registration procedure for integration tests which
+need a running CaosDB instance.
+
+It ensures that tests do not accidentally overwrite data in real CaosDB
+instances, as it checks whether the running CaosDB instance is actually the
+correct one, that
+should be used for these tests.
+
+The test files have to define a global variable TEST_KEY which must be unique
+for each test using
+
+set_test_key("ABCDE")
+
+The test procedure (invoked by pytest) checks whether a registration
+information is stored in one of the server properties or otherwise
+- offers to register this test in the currently running database ONLY if this
+ is empty.
+- fails otherwise with a RuntimeError
+
+NOTE: you probably need to use pytest with the -s option to be able to
+ register the test interactively. Otherwise, the server property has to be
+ set before server start-up in the server.conf of the CaosDB server.
+
+This module is intended to be used with pytest.
+
+There is a pytest fixture "clear_database" that performs the above mentioned
+checks and clears the database in case of success.
+"""
+
+TEST_KEY = None
+
+
+def set_test_key(KEY):
+ global TEST_KEY
+ TEST_KEY = KEY
+
+
+def _register_test():
+ res = db.execute_query("COUNT Entity")
+ if not isinstance(res, int):
+ raise RuntimeError("Response from server for Info could not be interpreted.")
+ if res > 0:
+ raise RuntimeError("This instance of CaosDB contains entities already."
+ "It must be empty in order to register a new test.")
+
+ print("Current host of CaosDB instance is: {}".format(
+ db.connection.connection.get_connection()._delegate_connection.setup_fields["host"]))
+ answer = input("This method will register your current test with key {} with the currently"
+ " running instance of CaosDB. Do you want to continue (y/N)?".format(
+ TEST_KEY))
+ if answer != "y":
+ raise RuntimeError("Test registration aborted by user.")
+
+ admin.set_server_property("_CAOSDB_INTEGRATION_TEST_SUITE_KEY",
+ TEST_KEY)
+
+
+def _get_registered_test_key():
+ try:
+ return admin.get_server_property("_CAOSDB_INTEGRATION_TEST_SUITE_KEY")
+ except KeyError:
+ return None
+
+
+def _is_registered():
+ registered_test_key = _get_registered_test_key()
+ if not registered_test_key:
+ return False
+ elif registered_test_key == TEST_KEY:
+ return True
+ else:
+ raise RuntimeError("The database has been setup for a different test.")
+
+
+def _assure_test_is_registered():
+ global TEST_KEY
+ if TEST_KEY is None:
+ raise RuntimeError("TEST_KEY is not defined.")
+ if not _is_registered():
+ answer = input("Do you want to register this instance of CaosDB"
+ " with the current test? Do you want to continue (y/N)?")
+ if answer == "y":
+ _register_test()
+ raise RuntimeError("Test has been registered. Please rerun tests.")
+ else:
+ raise RuntimeError("The database has not been setup for this test.")
+
+
+def _clear_database():
+ c = db.execute_query("FIND ENTITY WITH ID>99")
+ c.delete(raise_exception_on_error=False)
+ return None
+
+
+try:
+ import pytest
+
+ @pytest.fixture
+ def clear_database():
+ """Remove Records, RecordTypes, Properties, and Files ONLY IF the CaosDB
+ server the current connection points to was registered with the appropriate key.
+
+ PyTestInfo Records and the corresponding RecordType and Property are preserved.
+ """
+ _assure_test_is_registered()
+ yield _clear_database() # called before the test function
+ _clear_database() # called after the test function
+except ImportError:
+ raise Warning("""The register_tests module depends on pytest and is
+ intended to be used in integration test suites for the
+ caosdb-pylib library only.""")
diff --git a/src/caosdb/yamlapi.py b/src/caosdb/yamlapi.py
index 9a69a5276804727084af65c6568b22833e8be596..80bb4b13e4d1626c5d29c8950f3a22bbb73e0fdb 100644
--- a/src/caosdb/yamlapi.py
+++ b/src/caosdb/yamlapi.py
@@ -22,7 +22,7 @@
# ** end header
#
-"""YAML interface for the database (caosdb)"""
+"""!!! Deprecated !!! YAML interface for the database (caosdb)"""
import yaml
from lxml import etree
@@ -31,9 +31,14 @@ import re
import caosdb
import caosdb.common.utils as utils
from caosdb.connection.connection import get_connection
+import warnings
def append_sublist(v, newel, def_entity_type):
+ warnings.warn("""
+ This function is deprecated and will be removed with the next release.
+ Please use caosdb-advanced-user-tools/models/data_model.py for a
+ similar functionality.""", DeprecationWarning)
if v is None:
return
for i in v:
@@ -46,6 +51,10 @@ def append_sublist(v, newel, def_entity_type):
def kv_to_xml(k, v):
+ warnings.warn("""
+ This function is deprecated and will be removed with the next release.
+ Please use caosdb-advanced-user-tools/models/data_model.py for a
+ similar functionality.""", DeprecationWarning)
newel = Element(k)
# code.interact(local=locals())
if isinstance(v, list): # Top level loop
@@ -69,10 +78,18 @@ def dict_to_xml(d):
d: The dictionary (possibly loaded from yaml)
to convert to caosdb-xml.
"""
+ warnings.warn("""
+ This function is deprecated and will be removed with the next release.
+ Please use caosdb-advanced-user-tools/models/data_model.py for a
+ similar functionality.""", DeprecationWarning)
return kv_to_xml("Entities", d)
def yaml_to_xml(yamlstr):
+ warnings.warn("""
+ This function is deprecated and will be removed with the next release.
+ Please use caosdb-advanced-user-tools/models/data_model.py for a
+ similar functionality.""", DeprecationWarning)
"""Load a yaml document from yamlstr and converts it to XML.
Parameters
@@ -81,11 +98,15 @@ def yaml_to_xml(yamlstr):
The string to load the yaml document from.
"""
- return dict_to_xml(yaml.load(yamlstr))
+ return dict_to_xml(yaml.load(yamlstr, Loader=yaml.SafeLoader))
def process(text):
"""Do some replacements on the original file to obtain valid yaml."""
+ warnings.warn("""
+ This function is deprecated and will be removed with the next release.
+ Please use caosdb-advanced-user-tools/models/data_model.py for a
+ similar functionality.""", DeprecationWarning)
processed = re.sub(
"^(\\s*)-\\s*\\{?(.*)\\}?\\s*$",
"\\1- {\\2}",
@@ -98,6 +119,10 @@ def process(text):
def yaml_file_to_xml(yamlfilename):
+ warnings.warn("""
+ This function is deprecated and will be removed with the next release.
+ Please use caosdb-advanced-user-tools/models/data_model.py for a
+ similar functionality.""", DeprecationWarning)
with open(yamlfilename, "r") as f:
return yaml_to_xml(process(f.read()))
@@ -108,6 +133,10 @@ def insert_yaml_file(yamlfilename, simulate=False):
Set 'simulate' to True if you don't actually want to insert the xml,
but only receive what would be sent.
"""
+ warnings.warn("""
+ This function is deprecated and will be removed with the next release.
+ Please use caosdb-advanced-user-tools/models/data_model.py for a
+ similar functionality.""", DeprecationWarning)
con = get_connection()
prs = etree.XMLParser(remove_blank_text=True)
sent_xml = etree.tostring(
diff --git a/src/doc/conf.py b/src/doc/conf.py
index b05fa1c71c1dcd0b59916594818449d2ebc574bd..9e65bff1eba5d114a77d3ed9405e883df6ad7470 100644
--- a/src/doc/conf.py
+++ b/src/doc/conf.py
@@ -25,14 +25,14 @@ import sphinx_rtd_theme # noqa: E402
# -- Project information -----------------------------------------------------
project = 'pycaosdb'
-copyright = '2020, IndiScale GmbH'
+copyright = '2022, IndiScale GmbH'
author = 'Daniel Hornung'
# The short X.Y version
-version = '0.5.2'
+version = '0.9.0'
# The full version, including alpha/beta/rc tags
# release = '0.5.2-rc2'
-release = '0.5.2'
+release = '0.9.0-dev'
# -- General configuration ---------------------------------------------------
diff --git a/src/doc/configuration.md b/src/doc/configuration.md
index 6e53542f661dcae94622fef24a67cecf7491df9c..02cbbd7b13d916a676ad26c277e370ae76bf3725 100644
--- a/src/doc/configuration.md
+++ b/src/doc/configuration.md
@@ -4,6 +4,15 @@ PyCaosDB tries to read from the inifile specified in the environment variable `P
alternatively in `~/.pycaosdb.ini` upon import. After that, the ini file `pycaosdb.ini` in the
current working directory will be read additionally, if it exists.
+Here, we will look at the most common configuration options. For a full and comprehensive
+description please check out the [example pycaosdb.ini file](https://gitlab.com/caosdb/caosdb-pylib/-/blob/main/examples/pycaosdb.ini). You can download this file and use
+it as a starting point.
+
+
+Typically, you need to change at least the `url` and `username` fields as required. (Ask your
+CaosDB administrator or IT crowd if you do not know what to put there, but for the demo instance at
+https://demo.indiscale.com, `username=admin` and `password=caosdb` should work).
+
## Authentication ##
The default configuration (that your are asked for your password when ever a connection is created
@@ -17,6 +26,8 @@ can be changed by setting `password_method`:
Windows). The password will be queried on first usage.
* with `password_method=plain` (**strongly discouraged**)
+The following illustrates the recommended options:
+
```ini
[Connection]
username=YOUR_USERNAME
@@ -35,7 +46,10 @@ username=YOUR_USERNAME
## SSL Certificate ##
-You can set the pass to the ssl certificate to be used:
+In some cases (especially if you are testing CaosDB) you might need to supply an SSL certificate to
+allow SSL encryption.
+
+The `cacert` option sets the path to the ssl certificate for the connection:
```ini
[Connection]
@@ -49,6 +63,8 @@ with CaosDB which makes the experience much less verbose. Set it to 1 or 2 in ca
debugging (which I hope will not be necessary for this tutorial) or if you want to learn more about
the internals of the protocol.
+`timeout` sets the timeout for requests to the server.
+
A complete list of options can be found in the
[pycaosdb.ini file](https://gitlab.com/caosdb/caosdb-pylib/-/blob/main/examples/pycaosdb.ini) in
the examples folder of the source code.
diff --git a/src/doc/future_caosdb.md b/src/doc/future_caosdb.md
new file mode 100644
index 0000000000000000000000000000000000000000..de6170fa42674ed4e3161fb791a397a149dba659
--- /dev/null
+++ b/src/doc/future_caosdb.md
@@ -0,0 +1,193 @@
+# The future of the CaosDB Python Client
+
+The current Python client has done us great services but its structure and the
+way it is used sometimes feels outdated and clumsy. In this document we sketch
+how it might look different in future and invite everyone to comment or
+contribute to this development.
+
+At several locations in this document there will be links to discussion issues.
+If you want to discuss something new, you can create a new issue
+[here](https://gitlab.com/caosdb/caosdb-pylib/-/issues/new).
+
+## Overview
+Let's get a general impression before discussing single aspects.
+
+``` python
+import caosdb as db
+experiments = db.query("FIND Experiment")
+# print name and date for each `Experiment`
+for exp in experiments:
+ print(exp.name, exp.date)
+
+# suppose `Experiments` reference `Projects` which have a `Funding` Property
+one_exp = experiments[0]
+print(one_exp.Project.Funding)
+
+new_one = db.create_record("Experiment")
+new_one.date = "2022-01-01"
+new_one.name = "Needle Measurement"
+new_one.insert()
+```
+Related discussions:
+- [recursive retrieve in query](https://gitlab.com/caosdb/caosdb-pylib/-/issues/57)
+- [create_record function](https://gitlab.com/caosdb/caosdb-pylib/-/issues/58)
+- [data model utility](https://gitlab.com/caosdb/caosdb-pylib/-/issues/59)
+
+## Quickstart
+Note that you can try out one possible implementation using the
+`caosdb.high_level_api` module. It is experimental and might be removed in
+future!
+
+A `resolve_references` function allows to retrieve the referenced entities of
+an entity, container or a query result set (which is a container).
+It has the following parameters which can also be supplied to the `query`
+function:
+
+- `deep`: Whether to use recursive retrieval
+- `depth`: Maximum recursion depth
+- `references`: Whether to use the supplied db.Container to resolve
+ references. This allows offline usage. Set it to None if you want to
+ automatically retrieve entities from the current CaosDB connection.
+
+In order to allow a quick look at the object structures an easily readable
+serialization is provided by the `to_dict` function. It has the following
+argument:
+- `without_metadata`: Set this to True if you don\'t want to see
+ property metadata like \"unit\" or \"importance\".
+
+This function creates a simple dictionary containing a representation of
+the entity, which can be stored to disk and completely deserialized
+using the function `from_dict`.
+
+Furthermore, the `__str__` function uses this to display objects in yaml
+format by default statement
+
+## Design Decisions
+
+### Dot Notation
+Analogue, to what Pandas does. Provide bracket notation
+`rec.properties["test"]` for Properties with names that are in conflict with
+default attributes or contain spaces (or other forbidden characters).
+
+Entities can be initialized with a set of Propertynames. Those Propertynames will be used as
+attributes such that tab completion is possible in interactive use. The value however will be a special
+value (e.g. UnsetPropertyValue) and accessing it results in an Exception. Thus, tab completion can be used
+but no Properties are inserted unexpectedly with NULL values.
+
+- Raise Exception if attribute does not exist but is accessed?
+
+[Discussion](https://gitlab.com/caosdb/caosdb-pylib/-/issues/60)
+
+We aim for a distinction between "concrete" Properties of Records/RecordTypes and "abstract" Properties as part of the definition of a data model. Concrete properties are always "contained" in a record or record type while abstract properties stand for themselves.
+
+Draft:
+```
+class ConcreteProperty:
+ def __init__(self, v, u):
+ self.value = v
+ self.unit = u
+
+class Entity:
+ def __init__(self):
+ pass
+
+ def __setattr__(self, name, val):
+ if name not in dir(self):
+ # setattr(self, name, ConcreteProperty(val, None))
+ self.properties[name] = ConcreteProperty(val, None)
+ else:
+ # getattribute(self, name).value = val
+ self.properties[name].value = val
+```
+
+The old "get_property" functions serves the same purpose as the new "[]" notation.
+
+Instead of `get_property` / `add_property` etc. functions belonging to class Entity, we should refactor the list of properties (of an entity) to be a special kind of list, e.g. PropertyList.
+This list should enherit from a standard list, have all the known functions like "append", "extend", "__in__" and allow for all property-related functionality as part of its member functions (instead of access via Entity directly).
+Same story for the parents.
+
+**GET RID OF MULTI PROPERTIES!!!**
+
+#### how to deal with "property metadata"
+
+Current suggestion: stored in a special field "property_metadata" belonging to the object.
+`property_metadata` is a dict:
+- importance
+- unit
+- description
+- ...
+
+### Serialization
+What information needs to be contained in (meta)data? How compatible is it with
+GRPC json serialization?
+
+
+### Recursive Retrieval
+
+
+
+I can resolve later and end up with the same result:
+`recs =db.query("FIND Experiment", depth=2)` equals `recs = db.query("FIND Experiment"); recs = resolve_references(recs, depth=2)`
+
+[Discussion](https://gitlab.com/caosdb/caosdb-pylib/-/issues/57)
+
+
+#### Alternative
+
+`FIND Experiment` with `depth=2` will retrieve all referenced entities from any experiment found. A typical use case could also be:
+
+```python
+recs = db.query("FIND Experiment")
+recs[0].resolve_references(depth=2)
+```
+
+#### Idea
+
+Recursive retrievel as functionality of the server.
+
+retrieve and query commands should support the `depth` argument.
+
+### In-Place operations
+Default behavior is to return new objects instead of modifying them in-place.
+This can be changed with the argument `inplace=True`.
+Especially the following functions operate by default NOT in-place:
+- update
+- insert
+- retrieve
+- resolve_references
+[Discussion](https://gitlab.com/caosdb/caosdb-pylib/-/issues/61)
+
+## Extended Example
+``` python
+import caosdb as db
+
+dm = db.get_data_model()
+
+new_one = db.create_record(dm.Experiment)
+new_one.date = "2022-01-01"
+new_one.name = "Needle Measurement"
+new_one.dataset = db.create_record(dm.Dataset)
+new_one.dataset.voltage = (5, "V")
+new_one.dataset.pulses = [5, 5.3]
+inserted = new_one.insert()
+print("The new record has the ID:", inserted.id)
+```
+
+### Factory method
+While creating an Entity will not talk to a CaosDB server and can thus be done offline, the factory method
+`create_record` allows to
+1. Retrieve the parent and set attributes according to inheritance
+2. Use a container to resolve the parent and set attributes
+
+In general, more complex "magic" will be placed in the factory and only the straight forward version
+in the constructor.
+
+### References and sub entities
+
+Several possibilities exist for references:
+
+- value is the id of a referenced entity
+- value is a "sub object"
+- value is a reference to another (entity-)list element (similar to second variant, but with "sub object" always contained in container/entity-list)
+
+To be discussed: Which should be the obligatory/preferred variant?
diff --git a/src/doc/gallery/Makefile b/src/doc/gallery/Makefile
new file mode 100644
index 0000000000000000000000000000000000000000..658f9a6a93e23957b20aee5f38e5565bde35af80
--- /dev/null
+++ b/src/doc/gallery/Makefile
@@ -0,0 +1,23 @@
+# This file is a part of the CaosDB Project.
+#
+# Copyright (C) 2022 IndiScale GmbH <info@indiscale.com>
+# Copyright (C) 2022 Daniel Hornung <d.hornung@indiscale.com>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+
+# Run tests on the examples.
+test:
+ python3 -m doctest simulation.py
+.PHONY: test
diff --git a/src/doc/gallery/index.rst b/src/doc/gallery/index.rst
new file mode 100644
index 0000000000000000000000000000000000000000..a6ef53e4c7d1272c5dbc8c62b4d90a89591cac0f
--- /dev/null
+++ b/src/doc/gallery/index.rst
@@ -0,0 +1,16 @@
+
+PyCaosDB Code Gallery
+=====================
+
+This chapter collects code examples which can be immediately run against an empty CaosDB instance.
+
+.. note::
+
+ These examples require a configuration file with valid server and user/password settings. Refer
+ to the :ref:`Configuration <Configuration of PyCaosDB>` section for details.
+
+.. toctree::
+ :maxdepth: 2
+ :caption: The code examples:
+
+ simulation
diff --git a/src/doc/gallery/simulation.py b/src/doc/gallery/simulation.py
new file mode 100644
index 0000000000000000000000000000000000000000..342d5d980fc2b1a981f4a76d99e1954f8b2f5c2a
--- /dev/null
+++ b/src/doc/gallery/simulation.py
@@ -0,0 +1,134 @@
+"""
+Run a simulation and store the values into CaosDB.
+
+>>> main() # doctest: +ELLIPSIS
+These distances resulted in small x,y, values:
+[...]
+"""
+
+import numpy as np
+import scipy.integrate
+import caosdb as db
+from caosadvancedtools.table_converter import to_table
+
+
+def setup_caosdb():
+ """Create the data model and insert it into CaosDB
+
+ The data model consists of the following RecordTypes:
+
+ Software
+ with author and revision.
+
+ SoftwareRun
+ A specific run of the sofware, with input parameters, time of completion and a result.
+
+ State
+ An aggregate of x,y,z values.
+
+ Parameters
+ In this case the x,y,z initial values before the integration, so this is just a state.
+
+ Result
+ The x,y,z values at the end of the software run, the final state.
+
+ The data model of course also contains the corresponding properties for these RecordTypes.
+ """
+
+ cont = db.Container() # Container to insert all Entities at once into CaosDB
+ # create Properties
+ cont.append(db.Property("x", datatype=db.DOUBLE))
+ cont.append(db.Property("y", datatype=db.DOUBLE))
+ cont.append(db.Property("z", datatype=db.DOUBLE))
+ cont.append(db.Property("completed", datatype=db.DATETIME))
+ cont.append(db.Property("author", datatype=db.TEXT))
+ cont.append(db.Property("revision", datatype=db.TEXT))
+ # create RecordTypes
+ cont.append(db.RecordType("Software").add_property("author").add_property("revision"))
+ cont.append(db.RecordType("State").add_property("x", importance=db.OBLIGATORY)
+ .add_property("y").add_property("z"))
+ cont.append(db.RecordType("Parameters").add_parent("State", inheritance=db.ALL))
+ cont.append(db.RecordType("Result").add_parent("State", inheritance=db.RECOMMENDED))
+ cont.append(db.RecordType("SoftwareRun").add_property("Software").add_property("Parameters")
+ .add_property("completed").add_property("Result"))
+ cont.insert() # actually insert the Entities
+
+
+def simulations(n, t_max):
+ """Run the simulations.
+
+ Parameters
+ ----------
+ n : int
+ The number of runs.
+
+ t_max : float
+ The maximum time of integration.
+ """
+
+ software = (db.Record("simulator").add_parent("Software")
+ .add_property("author", value="IndiScale GmbH")
+ .add_property("revision", value="1234CDEF89AB"))
+ software.insert()
+ for i in range(n):
+ # Get the parameters and result
+ initial, result = run_simulation(run=i, t_max=t_max)
+
+ # Prepare CaosDB insertion
+ run = db.Record().add_parent("SoftwareRun").add_property("Software", value=software.id)
+ parameters = (db.Record().add_parent("Parameters").add_property("x", initial[0])
+ .add_property("y", initial[1]).add_property("z", initial[2]))
+ result_record = (db.Record().add_parent("Result").add_property("x", result[0])
+ .add_property("y", result[1]).add_property("z", result[2]))
+ run.add_property("Parameters", value=parameters).add_property("Result", value=result_record)
+ cont = db.Container()
+ cont.extend([run, parameters, result_record])
+ cont.insert() # Insert everything of this run into CaosDB.
+
+
+def run_simulation(run, t_max):
+ """Integrate the Rössler attractor from random initial values."""
+ a, b, c = (0.1, 0.1, 14)
+
+ def diff(t, x):
+ diff = np.array([-x[1] - x[2],
+ x[0] + a * x[1],
+ b + x[2] * (x[0] - c)])
+ return diff
+
+ x0 = np.random.uniform(-100, 100, 3)
+
+ result = scipy.integrate.solve_ivp(diff, [0, t_max], x0)
+ x = result.y[:, -1]
+ return (x0, x)
+
+
+def analyze():
+ """Find the initial conditions which produce the smalles x,y values after the given time."""
+ distance = 5
+ data = db.execute_query("""SELECT Parameters, Result FROM RECORD SoftwareRun WITH
+ (((Result.x < {dist}) AND (Result.x > -{dist}))
+ AND (Result.y < {dist})) AND Result.y > -{dist}""".format(dist=distance))
+ dataframe = to_table(data) # Convert into a Pandas DataFrame
+
+ parameters = db.Container().extend([db.Record(id=id) for id in dataframe.Parameters]).retrieve()
+
+ initial_distances = [np.linalg.norm([p.get_property(dim).value for dim in ["x", "y", "z"]])
+ for p in parameters]
+
+ print("These distances resulted in small x,y, values:\n{}".format(initial_distances))
+
+
+def main():
+ # 1. Set up the data model
+ setup_caosdb()
+
+ # 2. Run simulations
+ simulations(n=200, t_max=5)
+
+ # 3. Find initial conditions with interesting results
+ analyze()
+
+
+if __name__ == '__main__':
+ main()
diff --git a/src/doc/gallery/simulation.rst b/src/doc/gallery/simulation.rst
new file mode 100644
index 0000000000000000000000000000000000000000..ce1a7f457a142e36ef9f2b0cfe6a4df0b9fcedf6
--- /dev/null
+++ b/src/doc/gallery/simulation.rst
@@ -0,0 +1,14 @@
+========================================
+Managing data from numerical simulations
+========================================
+
+This code example
+
+1. sets up the data model
+2. runs simulations
+3. stores the simulation parameters and results into CaosDB
+4. retrieves the parameters for interesting results.
+
+:download:`Download code<simulation.py>`
+
+.. literalinclude:: simulation.py
diff --git a/src/doc/high_level_api.org b/src/doc/high_level_api.org
new file mode 100644
index 0000000000000000000000000000000000000000..516df1b41d500fab000a72517fd2d12ba61753b7
--- /dev/null
+++ b/src/doc/high_level_api.org
@@ -0,0 +1,171 @@
+* High Level API
+
+In addition to the old standard pylib API, new versions of pylib ship with a high level API
+that facilitates usage of CaosDB entities within data analysis scripts. In a nutshell that
+API exposes all properties of CaosDB Records as standard python attributes making their
+access easier.
+
+Or to spell it out directly in Python:
+#+BEGIN_SRC python
+
+ import caosdb as db
+ # Old API:
+ r = db.Record()
+ r.add_parent("Experiment")
+ r.add_property(name="alpha", value=5)
+ r.get_property("alpha").value = 25 # setting properties (old api)
+ print(r.get_property("alpha").value + 25) # getting properties (old api)
+
+ from caosdb.high_level_api import convert_to_python_entity
+ obj = convert_to_python_object(r) # create a high level entity
+ obj.r = 25 # setting properties (new api)
+ print(obj.r + 25) # getting properties (new api)
+
+#+END_SRC
+
+
+** Quickstart
+
+The module, needed for the high level API is called:
+~caosdb.high_level_api~
+
+There are two functions converting entities between the two representation (old API and new API):
+- ~convert_to_python_object~: Convert entities from **old** into **new** representation.
+- ~convert_to_entity~: Convert entities from **new** into **old** representation.
+
+Furthermore there are a few utility functions which expose very practical shorthands:
+- ~new_high_level_entity~: Retrieve a record type and create a new high level entity which contains properties of a certain importance level preset.
+- ~create_record~: Create a new high level entity using the name of a record type and a list of key value pairs as properties.
+- ~load_external_record~: Retrieve a record with a specific name and return it as high level entity.
+- ~create_entity_container~: Convert a high level entity into a standard entity including all sub entities.
+- ~query~: Do a CaosDB query and return the result as a container of high level objects.
+
+So as a first example, you could retrieve any record from CaosDB and use it using its high level representation:
+#+BEGIN_SRC python
+ from caosdb.high_level_api import query
+
+ res = query("FIND Record Experiment")
+ experiment = res[0]
+ # Use a property:
+ print(experiment.date)
+
+ # Use sub properties:
+ print(experiment.output[0].path)
+#+END_SRC
+
+The latter example demonstrates, that the function query is very powerful. For its default parameter
+values it automatically resolves and retrieves references recursively, so that sub properties,
+like the list of output files "output", become immediately available.
+
+**Note** that for the old API you were supposed to run the following series of commands
+to achieve the same result:
+#+BEGIN_SRC python
+ import caosdb as db
+
+ res = db.execute_query("FIND Record Experiment")
+ output = res.get_property("output")
+ output_file = db.File(id=output.value[0].id).retrieve()
+ print(output_file.path)
+#+END_SRC
+
+Resolving subproperties makes use of the "resolve_reference" function provided by the high level
+entity class (~CaosDBPythonEntity~), with the following parameters:
+- ~deep~: Whether to use recursive retrieval
+- ~references~: Whether to use the supplied db.Container to resolve references. This allows offline usage. Set it to None if you want to automatically retrieve entities from the current CaosDB connection.
+- ~visited~: Needed for recursion, set this to None.
+
+Objects in the high level representation can be serialized to a simple yaml form using the function
+~serialize~ with the following parameters:
+- ~without_metadata~: Set this to True if you don't want to see property metadata like "unit" or "importance".
+- ~visited~: Needed for recursion, set this to None.
+
+This function creates a simple dictionary containing a representation of the entity, which can be
+stored to disk and completely deserialized using the function ~deserialize~.
+
+Furthermore the "__str__" function is overloaded, so that you can use print to directly inspect
+high level objects using the following statement:
+#+BEGIN_SRC python
+print(str(obj))
+#+END_SRC
+
+
+** Concepts
+
+As described in the section [[Quickstart]] the two functions ~convert_to_python_object~ and ~convert_to_entity~ convert
+entities beetween the high level and the standard representation.
+
+The high level entities are represented using the following classes from the module ~caosdb.high_level_api~:
+- ~CaosDBPythonEntity~: Base class of the following entity classes.
+- ~CaosDBPythonRecord~
+- ~CaosDBPythonRecordType~
+- ~CaosDBPythonProperty~
+- ~CaosDBPythonMultiProperty~: **WARNING** Not implemented yet.
+- ~CaosDBPythonFile~: Used for file entities and provides an additional ~download~ function for being able to directly retrieve files from CaosDB.
+
+In addition, there are the following helper structures which are realized as Python data classes:
+- ~CaosDBPropertyMetaData~: For storing meta data about properties.
+- ~CaosDBPythonUnresolved~: The base class of unresolved "things".
+- ~CaosDBPythonUnresolvedParent~: Parents of entities are stored as unresolved parents by default, storing an id or a name of a parent (or both).
+- ~CaosDBPythonUnresolvedReference~: An unresolved reference is a reference property with an id which has not (yet) been resolved to an Entity.
+
+The function "resolve_references" can be used to recursively replace ~CaosDBPythonUnresolvedReferences~ into members of type ~CaosDBPythonRecords~
+or ~CaosDBPythonFile~.
+
+Each property stored in a CaosDB record corresponds to:
+- a member attribute of ~CaosDBPythonRecord~ **and**
+- an entry in a dict called "metadata" storing a CaosDBPropertyMetadata object with the following information about proeprties:
+ - ~unit~
+ - ~datatype~
+ - ~description~
+ - ~id~
+ - ~importance~
+
+
+* Example
+
+The following shows a more complex example taken from a real world use case:
+A numerical experiment is created to simulate cardiac electric dynamics. The physical problem
+is modelled using the monodomain equation with the local current term given by the Mitchell
+Schaeffer Model.
+
+The data model for the numerical experiment consists of multiple record types which stores assosciated paremeters:
+- `MonodomainTissueSimulation`
+- `MitchellSchaefferModel`
+- `SpatialExtent2d`
+- `SpatialDimension`
+- `ConstantTimestep`
+- `ConstantDiffusion`
+
+First, the data model will be filled with the parameter values for this specific simulation run. It will be stored in the python variable `MonodomainRecord`. Passing the `MonodomainRecord` through the python functions, the simulation parameters can be easily accessed everywhere in the code when needed.
+
+Records are created by the `create_record` function. Parameter values can be set at record creation and also after creation as python properties of the corresponding record instance. The following example shows how to create a record, how to set the parameter at creation and how to set them as python properties
+
+#+BEGIN_SRC python
+ from caosdb.high_level_api import create_record
+
+ MonodomainRecord = create_record("MonodomainTissueSimulation")
+ MonodomainRecord.LocalModel = create_record("MitchellSchaefferModel")
+ MonodomainRecord.SpatialExtent = create_record(
+ "SpatialExtent2d", spatial_extent_x=100, spatial_extent_y=100)
+ MonodomainRecord.SpatialExtent.cell_sizes = [0.1, 0.1] # parameters can be set as properties
+ MonodomainRecord.SpatialDimension = create_record("SpatialDimension",
+ num_dim=2)
+
+ MonodomainRecord.TimestepInformation = create_record("ConstantTimestep")
+ MonodomainRecord.TimestepInformation.DeltaT = 0.1
+
+ D = create_record("ConstantDiffusion", diffusion_constant=0.1)
+ MonodomainRecord.DiffusionConstantType = D
+ model = MonodomainRecord.LocalModel
+ model.t_close = 150
+ model.t_open = 120
+ model.t_out = 6
+ model.t_in = 0.3
+ model.v_gate = 0.13
+ model.nvars = 2
+#+END_SRC
+
+At any position in the algorithm you are free to:
+- Convert this model to the standard python API and insert or update the records in a running instance of CaosDB.
+- Serialize this model in the high level API yaml format. This enables the CaosDB crawler to pickup the file and synchronize it with a running instance
+of CaosDB.
diff --git a/src/doc/high_level_api.rst b/src/doc/high_level_api.rst
new file mode 100644
index 0000000000000000000000000000000000000000..603052b135ad2289caea7e3bed59ae9d3301f811
--- /dev/null
+++ b/src/doc/high_level_api.rst
@@ -0,0 +1,163 @@
+High Level API
+==============
+
+In addition to the old standard pylib API, new versions of pylib ship
+with a high level API that facilitates usage of CaosDB entities within
+data analysis scripts. In a nutshell that API exposes all properties of
+CaosDB Records as standard python attributes making their access easier.
+
+Or to speak it out directly in Python:
+
+.. code:: python
+
+
+ import caosdb as db
+ # Old API:
+ r = db.Record()
+ r.add_parent("Experiment")
+ r.add_property(name="alpha", value=5)
+ r.get_property("alpha").value = 25 # setting properties (old api)
+ print(r.get_property("alpha").value + 25) # getting properties (old api)
+
+ from caosdb.high_level_api import convert_to_python_entity
+ obj = convert_to_python_object(r) # create a high level entity
+ obj.r = 25 # setting properties (new api)
+ print(obj.r + 25) # getting properties (new api)
+
+Quickstart
+----------
+
+The module, needed for the high level API is called:
+``caosdb.high_level_api``
+
+There are two functions converting entities between the two
+representation (old API and new API):
+
+- ``convert_to_python_object``: Convert entities from **old** into
+ **new** representation.
+- ``convert_to_entity``: Convert entities from **new** into **old**
+ representation.
+
+Furthermore there are a few utility functions which expose very
+practical shorthands:
+
+- ``new_high_level_entity``: Retrieve a record type and create a new
+ high level entity which contains properties of a certain importance
+ level preset.
+- ``create_record``: Create a new high level entity using the name of a
+ record type and a list of key value pairs as properties.
+- ``load_external_record``: Retrieve a record with a specific name and
+ return it as high level entity.
+- ``create_entity_container``: Convert a high level entity into a
+ standard entity including all sub entities.
+- ``query``: Do a CaosDB query and return the result as a container of
+ high level objects.
+
+So as a first example, you could retrieve any record from CaosDB and use
+it using its high level representation:
+
+.. code:: python
+
+ from caosdb.high_level_api import query
+
+ res = query("FIND Record Experiment")
+ experiment = res[0]
+ # Use a property:
+ print(experiment.date)
+
+ # Use sub properties:
+ print(experiment.output[0].path)
+
+The latter example demonstrates, that the function query is very
+powerful. For its default parameter values it automatically resolves and
+retrieves references recursively, so that sub properties, like the list
+of output files "output", become immediately available.
+
+**Note** that for the old API you were supposed to run the following
+series of commands to achieve the same result:
+
+.. code:: python
+
+ import caosdb as db
+
+ res = db.execute_query("FIND Record Experiment")
+ output = res.get_property("output")
+ output_file = db.File(id=output.value[0].id).retrieve()
+ print(output_file.path)
+
+Resolving subproperties makes use of the "resolve\ :sub:`reference`"
+function provided by the high level entity class
+(``CaosDBPythonEntity``), with the following parameters:
+
+- ``deep``: Whether to use recursive retrieval
+- ``references``: Whether to use the supplied db.Container to resolve
+ references. This allows offline usage. Set it to None if you want to
+ automatically retrieve entities from the current CaosDB connection.
+- ``visited``: Needed for recursion, set this to None.
+
+Objects in the high level representation can be serialized to a simple
+yaml form using the function ``serialize`` with the following
+parameters:
+
+- ``without_metadata``: Set this to True if you don't want to see
+ property metadata like "unit" or "importance".
+- ``visited``: Needed for recursion, set this to None.
+
+This function creates a simple dictionary containing a representation of
+the entity, which can be stored to disk and completely deserialized
+using the function ``deserialize``.
+
+Furthermore the "*str*" function is overloaded, so that you can use
+print to directly inspect high level objects using the following
+statement:
+
+.. code:: python
+
+ print(str(obj))
+
+Concepts
+--------
+
+As described in the section Quickstart the two functions
+``convert_to_python_object`` and ``convert_to_entity`` convert entities
+beetween the high level and the standard representation.
+
+The high level entities are represented using the following classes from
+the module ``caosdb.high_level_api``:
+
+- ``CaosDBPythonEntity``: Base class of the following entity classes.
+- ``CaosDBPythonRecord``
+- ``CaosDBPythonRecordType``
+- ``CaosDBPythonProperty``
+- ``CaosDBPythonMultiProperty``: **WARNING** Not implemented yet.
+- ``CaosDBPythonFile``: Used for file entities and provides an
+ additional ``download`` function for being able to directly retrieve
+ files from CaosDB.
+
+In addition, there are the following helper structures which are
+realized as Python data classes:
+
+- ``CaosDBPropertyMetaData``: For storing meta data about properties.
+- ``CaosDBPythonUnresolved``: The base class of unresolved "things".
+- ``CaosDBPythonUnresolvedParent``: Parents of entities are stored as
+ unresolved parents by default, storing an id or a name of a parent
+ (or both).
+- ``CaosDBPythonUnresolvedReference``: An unresolved reference is a
+ reference property with an id which has not (yet) been resolved to an
+ Entity.
+
+The function "resolve\ :sub:`references`" can be used to recursively
+replace ``CaosDBPythonUnresolvedReferences`` into members of type
+``CaosDBPythonRecords`` or ``CaosDBPythonFile``.
+
+Each property stored in a CaosDB record corresponds to:
+
+- a member attribute of ``CaosDBPythonRecord`` **and**
+- an entry in a dict called "metadata" storing a CaosDBPropertyMetadata
+ object with the following information about proeprties:
+
+ - ``unit``
+ - ``datatype``
+ - ``description``
+ - ``id``
+ - ``importance``
diff --git a/src/doc/index.rst b/src/doc/index.rst
index bd29c6c56acf5c173e94ae6471a6aeba56ea4b93..7344b6aacdd55fd75f4940d834104faa00c33069 100644
--- a/src/doc/index.rst
+++ b/src/doc/index.rst
@@ -12,6 +12,8 @@ Welcome to PyCaosDB's documentation!
Concepts <concepts>
Configuration <configuration>
Administration <administration>
+ High Level API <high_level_api>
+ Code gallery <gallery/index>
API documentation<_apidoc/caosdb>
This is the documentation for the Python client library for CaosDB, ``PyCaosDB``.
diff --git a/src/doc/tutorials/Data-Insertion.rst b/src/doc/tutorials/Data-Insertion.rst
index 22fb9461d6916003b2dad496ff3487df335c8dcc..f2c7f830d1403fbdf45354d1f36a4ea339759058 100644
--- a/src/doc/tutorials/Data-Insertion.rst
+++ b/src/doc/tutorials/Data-Insertion.rst
@@ -10,15 +10,18 @@ Model<caosdb-server:Data-Model>` in the CaosDB server documentation.
In order to insert some actual data, we need to create a data model
using RecordTypes and Properties (You may skip this if you use a CaosDB
-instance that already has the required types). So, let’s create a simple
+instance that already has the required types). When you create a new Property
+you must supply a datatype. So, let’s create a simple
Property called “a” of datatype double. This is very easy in pylib:
.. code:: python
a = db.Property(name="a", datatype=db.DOUBLE)
-There are a few basic datatypes: db.INTEGER, db.TEXT. See `data
-type <Specification/Datatype>`__ for a full list.
+There are a few basic datatypes like db.INTEGER, db.DOUBLE, or db.TEXT. See the
+`data types
+<https://docs.indiscale.com/caosdb-server/specification/Datatype.html>`_ for a
+full list.
We can create our own small data model for e.g. a simulation by adding
two more Properties and a RecordType:
@@ -49,11 +52,11 @@ resolution, but we'll omit this for the sake of brevity for now.
.. code:: python
rt = db.RecordType(name="2D_BarkleySimulation",
- description="Spatially extended Barkley simulation")
+ description="Spatially extended Barkley simulation")
# inherit all properties from the BarkleySimulation RecordType
rt.add_parent(name="BarkleySimulation", inheritance="all")
rt.insert()
-
+
print(rt.get_property(name="epsilon").importance) ### rt has a "epsilon" property with the same importance as "BarkleySimulation"
The parameter ``inheritance=(obligatory|recommended|fix|all|none)`` of
@@ -135,15 +138,37 @@ Useful is also, that you can insert directly tabular data.
.. code:: python
- from caosadvancedtools.table_converter import from_tsv
-
- recs = from_tsv("test.csv", "Experiment")
- print(recs)
- recs.insert()
+ from caosadvancedtools.table_converter import from_tsv
+
+ recs = from_tsv("test.csv", "Experiment")
+ print(recs)
+ recs.insert()
With this example file
`test.csv <uploads/4f2c8756a26a3984c0af09d206d583e5/test.csv>`__.
+List Properties
+---------------
+
+As you may already know, properties can also have list values instead of scalar
+values. They can be accessed, set, and updated as you would expect from any
+list-valued attribute in Python, as the following example illustrates.
+
+.. code:: python
+
+ import caosdb as db
+ db.Property(name="TestList", datatype=db.LIST(db.DOUBLE)).insert()
+ db.RecordType(name="TestType").add_property(name="TestList").insert()
+ db.Record(name="TestRec").add_parent("TestType").add_property(
+ name="TestList", value=[1,2,3]).insert()
+ retrieved = db.Record(name="TestRec").retrieve()
+ retrieved.get_property("TestList").value += [4,5]
+ retrieved.update()
+
+ # Check update
+ retrieved = db.Record(name="TestRec").retrieve()
+ print(retrieved.get_property("TestList").value)
+
File Update
-----------
diff --git a/src/doc/tutorials/index.rst b/src/doc/tutorials/index.rst
index 79068e9201498c87b2eb61b4ffbea0969845b404..0b08d0b4fe153d803a780bd144787819b827db78 100644
--- a/src/doc/tutorials/index.rst
+++ b/src/doc/tutorials/index.rst
@@ -16,4 +16,5 @@ advanced usage of the Python client.
errors
data-model-interface
complex_data_models
+ serverside
diff --git a/src/doc/tutorials/serverside.rst b/src/doc/tutorials/serverside.rst
new file mode 100644
index 0000000000000000000000000000000000000000..93f0fdcf742efc70bc80f5113eb7c6ddbbf87cde
--- /dev/null
+++ b/src/doc/tutorials/serverside.rst
@@ -0,0 +1,61 @@
+
+Server Side Scripting
+=====================
+
+The administrator may store regularly needed scripts, e.g. for computing a
+standardized analysis, on the same machine as the CaosDB server, "on the server
+side", where they can be run directly by the server.
+
+The execution of those scripts can be initiated using the Python client, or the
+web interface.
+
+Call a Script
+~~~~~~~~~~~~~
+
+If you have access to the server and sufficient permissions to run the script,
+execution is fairly simple:
+
+.. code:: python
+
+ from caosdb.utils.server_side_scripting import run_server_side_script
+ response = run_server_side_script('scriptname.py')
+ print(response.stderr,response.stdout)
+
+
+This makes the server run the script ``scriptname.py``. The output of the
+script (``stderr`` and ``stdout``) is returned within an response object.
+
+
+If the script requires additional arguments, those can be provided after the
+script's name.
+
+Note that by default the script runs with your CaosDB account. It has your
+permissions and changes are logged as if they were done by you directly.
+
+
+Testing it
+~~~~~~~~~~
+
+You can try this out using for example the ``diagnostics.py`` script (it is part
+of the `CaosDB server repository
+<https://gitlab.indiscale.com/caosdb/src/caosdb-server/-/blob/main/scripting/bin/administration/diagnostics.py>`_
+and is also available on https://demo.indiscale.com). The script returns
+information about the server in JSON format. You can do for example the
+following:
+
+.. code:: python
+
+ import json
+ from caosdb.utils.server_side_scripting import run_server_side_script
+ response = run_server_side_script('administration/diagnostics.py')
+ print("JSON content:")
+ print(json.loads(response.stdout))
+ print("stderr:")
+ print(response.stderr)
+
+
+Further Information
+~~~~~~~~~~~~~~~~~~~
+
+Additionally, you might want to have a look at
+https://docs.indiscale.com/caosdb-server/specification/Server-side-scripting.html
diff --git a/tox.ini b/tox.ini
index 94c2dc8affb280d3e7f6cff4536636432c9f7749..e321891883b9425d24543a41f2d1283e0be52109 100644
--- a/tox.ini
+++ b/tox.ini
@@ -1,5 +1,5 @@
[tox]
-envlist=py36, py37, py38, py39
+envlist=py38, py39, py310
skip_missing_interpreters = true
[testenv]
@@ -7,4 +7,9 @@ deps = .
nose
pytest
pytest-cov
+ python-dateutil
+ jsonschema==4.0.1
commands=py.test --cov=caosdb -vv {posargs}
+
+[flake8]
+max-line-length=100
diff --git a/unittests/broken_configs/pycaosdb1.ini b/unittests/broken_configs/pycaosdb1.ini
new file mode 100644
index 0000000000000000000000000000000000000000..71180286881399c35e251bba89a54a345cd948ac
--- /dev/null
+++ b/unittests/broken_configs/pycaosdb1.ini
@@ -0,0 +1,5 @@
+[Connection]
+cacert=/very/long/path/to/self/signed/pem/file/caosdb.ca.pem
+url=https://hostname:8833/playground
+username=username
+password_method=pass
diff --git a/unittests/broken_configs/pycaosdb2.ini b/unittests/broken_configs/pycaosdb2.ini
new file mode 100644
index 0000000000000000000000000000000000000000..6bdf58ea812b83a622e647b2a18b01bb1a1e3099
--- /dev/null
+++ b/unittests/broken_configs/pycaosdb2.ini
@@ -0,0 +1,9 @@
+[Connection]
+url=https://0.0.0.0/
+username=username
+password_identifier=SECTION/SUBSECTION/identifier
+password_method=pass
+cacert=/etc/ssl/cert.pem
+ssl_insecure=true
+timeout=10000
+debug=9
diff --git a/unittests/broken_configs/pycaosdb3.ini b/unittests/broken_configs/pycaosdb3.ini
new file mode 100644
index 0000000000000000000000000000000000000000..62d1fed9497c0c258c13aa2ed8fecb23a3006849
--- /dev/null
+++ b/unittests/broken_configs/pycaosdb3.ini
@@ -0,0 +1,12 @@
+[connection]
+ssl_insecure=true
+url=https://localhost:10443/
+password=caosdb
+username=admin
+password_method=plain
+
+timeout=10000
+debug=0
+
+[Container]
+debug=0
diff --git a/unittests/broken_configs/pycaosdb4.ini b/unittests/broken_configs/pycaosdb4.ini
new file mode 100644
index 0000000000000000000000000000000000000000..e96604ac453ff2be4e2b419aa9ccbbf3598fa231
--- /dev/null
+++ b/unittests/broken_configs/pycaosdb4.ini
@@ -0,0 +1,10 @@
+[Connection]
+ssl_insecure=true
+url=https://localhost:10443/
+password=caosdb
+username=admin
+password_method=plain
+
+timeout=10000
+debug=0
+key=bla
\ No newline at end of file
diff --git a/unittests/data/list_in_value.xml b/unittests/data/list_in_value.xml
new file mode 100644
index 0000000000000000000000000000000000000000..0f92610d82caa5ced443b2f437f35da05b9e121a
--- /dev/null
+++ b/unittests/data/list_in_value.xml
@@ -0,0 +1,12 @@
+<Record id="1002" description="A description of this example experiment.">
+ <Version id="945c6858819d2609a5475ee4df64571984acd039" head="true">
+ <Predecessor id="0df3cfe164fbafe9777f9356d0be2403890c54cd" />
+ </Version>
+ <Parent id="1001" name="Experiment" />
+ <Property datatype="SomeRecordType" id="1003" name="DepthTest" importance="FIX">
+ <Value>1004</Value>
+ <Value>1005</Value>
+ </Property>
+</Record>
+
+<!-- Note: This XML is invalid, because list-valued Properties must have a LIST-Datatype -->
diff --git a/unittests/test_acl.py b/unittests/test_acl.py
new file mode 100644
index 0000000000000000000000000000000000000000..633c25ad5c4046c0fa41b66049bdf56aa695f482
--- /dev/null
+++ b/unittests/test_acl.py
@@ -0,0 +1,55 @@
+# -*- encoding: utf-8 -*-
+#
+# This file is a part of the CaosDB Project.
+#
+# Copyright (C) 2022 Indiscale GmbH <info@indiscale.com>
+# Copyright (C) 2022 Timm Fitschen <f.fitschen@indiscale.com>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see <https://www.gnu.org/licenses/>.
+#
+import caosdb as db
+from lxml import etree
+
+
+def test_parse_xml():
+ # @review Florian Spreckelsen 2022-03-17
+ xml_str = """
+ <EntityACL>
+ <Grant priority="False" role="role1">
+ <Permission name="RETRIEVE:ENTITY"/>
+ </Grant>
+ <Deny priority="False" role="role1">
+ <Permission name="RETRIEVE:ENTITY"/>
+ </Deny>
+ <Grant priority="True" role="role1">
+ <Permission name="RETRIEVE:ENTITY"/>
+ </Grant>
+ <Deny priority="True" role="role1">
+ <Permission name="RETRIEVE:ENTITY"/>
+ </Deny>
+ </EntityACL>"""
+ xml = etree.fromstring(xml_str)
+ left_acl = db.ACL(xml)
+
+ right_acl = db.ACL()
+ right_acl.grant(role="role1", permission="RETRIEVE:ENTITY",
+ revoke_denial=False)
+ right_acl.deny(role="role1", permission="RETRIEVE:ENTITY",
+ revoke_grant=False)
+ right_acl.grant(role="role1", permission="RETRIEVE:ENTITY",
+ priority=True, revoke_denial=False)
+ right_acl.deny(role="role1", permission="RETRIEVE:ENTITY",
+ priority=True, revoke_grant=False)
+
+ assert left_acl == right_acl
diff --git a/unittests/test_add_property.py b/unittests/test_add_property.py
index 5bae6c219732f0170f5c351eae58148c9d3d065a..0d3183b4c0ca5517ecea68d0e49bbf335bb2a13e 100644
--- a/unittests/test_add_property.py
+++ b/unittests/test_add_property.py
@@ -264,3 +264,33 @@ def test_add_list_of_entitities():
for e in rec.get_property("listOfEntities").value:
assert i == e.id
i += 1
+
+
+def test_add_property_with_wrong_role():
+ entity = db.Entity()
+
+ r = db.Record()
+ rt = db.RecordType()
+ p = db.Property()
+ f = db.File()
+ e = db.Entity()
+
+ entity.add_property(rt)
+ entity.add_property(p)
+ entity.add_property(e)
+
+ with raises(ValueError) as cm:
+ entity.add_property(r)
+ assert cm.value.args[0] == ("The property parameter is a Record. This is "
+ "very unusual and probably not what you want. "
+ "Otherwise, construct a property from a "
+ "Record using the Property class and add that "
+ "to this entity.")
+
+ with raises(ValueError) as cm:
+ entity.add_property(f)
+ assert cm.value.args[0] == ("The property parameter is a File. This is "
+ "very unusual and probably not what you want. "
+ "Otherwise, construct a property from a File "
+ "using the Property class and add that to "
+ "this entity.")
diff --git a/unittests/test_apiutils.py b/unittests/test_apiutils.py
index 264b4c880022e6fd135426864bf9c5084c047eca..43ab8107183f16bf8df1d0ea8e447b378bcf8123 100644
--- a/unittests/test_apiutils.py
+++ b/unittests/test_apiutils.py
@@ -1,12 +1,11 @@
# -*- encoding: utf-8 -*-
#
-# ** header v3.0
# This file is a part of the CaosDB Project.
#
# Copyright (C) 2018 Research Group Biomedical Physics,
# Max-Planck-Institute for Dynamics and Self-Organization Göttingen
# Copyright (C) 2020 Timm Fitschen <t.fitschen@indiscale.com>
-# Copyright (C) 2020 IndiScale GmbH <info@indiscale.com>
+# Copyright (C) 2020-2022 IndiScale GmbH <info@indiscale.com>
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as
@@ -26,26 +25,14 @@
# Test apiutils
# A. Schlemmer, 02/2018
+
+import pytest
import caosdb as db
-import pickle
-import tempfile
-from caosdb.apiutils import apply_to_ids, create_id_query, resolve_reference
import caosdb.apiutils
-from .test_property import testrecord
-
-
-def test_convert_object():
- r2 = db.apiutils.convert_to_python_object(testrecord)
- assert r2.species == "Rabbit"
-
+from caosdb.apiutils import (apply_to_ids, compare_entities, create_id_query,
+ resolve_reference, merge_entities)
-def test_pickle_object():
- r2 = db.apiutils.convert_to_python_object(testrecord)
- with tempfile.TemporaryFile() as f:
- pickle.dump(r2, f)
- f.seek(0)
- rn2 = pickle.load(f)
- assert r2.date == rn2.date
+from caosdb.common.models import SPECIAL_ATTRIBUTES
def test_apply_to_ids():
@@ -67,7 +54,8 @@ def test_apply_to_ids():
def test_id_query():
ids = [1, 2, 3, 4, 5]
- assert create_id_query(ids) == 'FIND ENTITY WITH ID=1 OR ID=2 OR ID=3 OR ID=4 OR ID=5'
+ assert create_id_query(ids) == 'FIND ENTITY WITH ID=1 OR ID=2 OR ID=3 OR '\
+ 'ID=4 OR ID=5'
def test_resolve_reference():
@@ -77,8 +65,10 @@ def test_resolve_reference():
prop = db.Property(id=1, datatype=db.REFERENCE, value=100)
prop.is_valid = lambda: True
items = [200, 300, 400]
- prop_list = db.Property(datatype=db.LIST(db.REFERENCE), value=items)
- prop_list2 = db.Property(datatype=db.LIST(db.REFERENCE), value=[db.Record(id=500)])
+ prop_list = db.Property(datatype=db.LIST(db.REFERENCE),
+ value=items)
+ prop_list2 = db.Property(datatype=db.LIST(db.REFERENCE),
+ value=[db.Record(id=500)])
resolve_reference(prop)
resolve_reference(prop_list)
resolve_reference(prop_list2)
@@ -86,6 +76,7 @@ def test_resolve_reference():
assert isinstance(prop.value, db.Entity)
prop_list_ids = []
+
for i in prop_list.value:
prop_list_ids.append(i.id)
assert isinstance(i, db.Entity)
@@ -102,3 +93,250 @@ def test_resolve_reference():
# restore retrive_entity_with_id
caosdb.apiutils.retrieve_entity_with_id = original_retrieve_entity_with_id
+
+
+def test_compare_entities():
+ r1 = db.Record()
+ r2 = db.Record()
+ r1.add_parent("bla")
+ r2.add_parent("bla")
+ r1.add_parent("lopp")
+ r1.add_property("test", value=2)
+ r2.add_property("test", value=2)
+ r1.add_property("tests", value=3)
+ r2.add_property("tests", value=45)
+ r1.add_property("tester", value=3)
+ r2.add_property("tester", )
+ r1.add_property("tests_234234", value=45)
+ r2.add_property("tests_TT", value=45)
+
+ diff_r1, diff_r2 = compare_entities(r1, r2)
+
+ assert len(diff_r1["parents"]) == 1
+ assert len(diff_r2["parents"]) == 0
+ assert len(diff_r1["properties"]) == 3
+ assert len(diff_r2["properties"]) == 3
+
+ assert "test" not in diff_r1["properties"]
+ assert "test" not in diff_r2["properties"]
+
+ assert "tests" in diff_r1["properties"]
+ assert "tests" in diff_r2["properties"]
+
+ assert "tester" in diff_r1["properties"]
+ assert "tester" in diff_r2["properties"]
+
+ assert "tests_234234" in diff_r1["properties"]
+ assert "tests_TT" in diff_r2["properties"]
+
+
+def test_compare_entities_units():
+ r1 = db.Record()
+ r2 = db.Record()
+ r1.add_parent("bla")
+ r2.add_parent("bla")
+ r1.add_parent("lopp")
+ r1.add_property("test", value=2, unit="cm")
+ r2.add_property("test", value=2, unit="m")
+ r1.add_property("tests", value=3, unit="cm")
+ r2.add_property("tests", value=45, unit="cm")
+ r1.add_property("tester", value=3)
+ r2.add_property("tester", )
+ r1.add_property("tests_234234", value=45, unit="cm")
+ r2.add_property("tests_TT", value=45, unit="cm")
+
+ diff_r1, diff_r2 = compare_entities(r1, r2)
+
+ assert len(diff_r1["parents"]) == 1
+ assert len(diff_r2["parents"]) == 0
+ assert len(diff_r1["properties"]) == 4
+ assert len(diff_r2["properties"]) == 4
+
+ assert "tests" in diff_r1["properties"]
+ assert "tests" in diff_r2["properties"]
+
+ assert "tester" in diff_r1["properties"]
+ assert "tester" in diff_r2["properties"]
+
+ assert "tests_234234" in diff_r1["properties"]
+ assert "tests_TT" in diff_r2["properties"]
+
+ assert diff_r1["properties"]["test"]["unit"] == "cm"
+ assert diff_r2["properties"]["test"]["unit"] == "m"
+
+
+def test_compare_special_properties():
+ # Test for all known special properties:
+ SPECIAL_PROPERTIES = ("description", "name",
+ "checksum", "size", "path", "id")
+ INTS = ("size", "id")
+ HIDDEN = ("checksum", "size")
+
+ for key in SPECIAL_PROPERTIES:
+ set_key = key
+ if key in HIDDEN:
+ set_key = "_" + key
+ r1 = db.Record()
+ r2 = db.Record()
+ if key not in INTS:
+ setattr(r1, set_key, "bla 1")
+ setattr(r2, set_key, "bla 1")
+ else:
+ setattr(r1, set_key, 1)
+ setattr(r2, set_key, 1)
+
+ diff_r1, diff_r2 = compare_entities(r1, r2)
+ assert key not in diff_r1
+ assert key not in diff_r2
+ assert len(diff_r1["parents"]) == 0
+ assert len(diff_r2["parents"]) == 0
+ assert len(diff_r1["properties"]) == 0
+ assert len(diff_r2["properties"]) == 0
+
+ if key not in INTS:
+ setattr(r2, set_key, "bla test")
+ else:
+ setattr(r2, set_key, 2)
+
+ diff_r1, diff_r2 = compare_entities(r1, r2)
+ assert key in diff_r1
+ assert key in diff_r2
+ if key not in INTS:
+ assert diff_r1[key] == "bla 1"
+ assert diff_r2[key] == "bla test"
+ else:
+ assert diff_r1[key] == 1
+ assert diff_r2[key] == 2
+ assert len(diff_r1["properties"]) == 0
+ assert len(diff_r2["properties"]) == 0
+
+
+@pytest.mark.xfail
+def test_compare_properties():
+ p1 = db.Property()
+ p2 = db.Property()
+
+ diff_r1, diff_r2 = compare_entities(p1, p2)
+ assert len(diff_r1["parents"]) == 0
+ assert len(diff_r2["parents"]) == 0
+ assert len(diff_r1["properties"]) == 0
+ assert len(diff_r2["properties"]) == 0
+
+ p1.importance = "SUGGESTED"
+ diff_r1, diff_r2 = compare_entities(p1, p2)
+ assert len(diff_r1["parents"]) == 0
+ assert len(diff_r2["parents"]) == 0
+ assert len(diff_r1["properties"]) == 0
+ assert len(diff_r2["properties"]) == 0
+ assert "importance" in diff_r1
+ assert diff_r1["importance"] == "SUGGESTED"
+
+ # TODO: I'm not sure why it is not like this:
+ # assert diff_r2["importance"] is None
+ # ... but:
+ assert "importance" not in diff_r2
+
+ p2.importance = "SUGGESTED"
+ p1.value = 42
+ p2.value = 4
+
+ diff_r1, diff_r2 = compare_entities(p1, p2)
+ assert len(diff_r1["parents"]) == 0
+ assert len(diff_r2["parents"]) == 0
+ assert len(diff_r1["properties"]) == 0
+ assert len(diff_r2["properties"]) == 0
+
+ # Comparing values currently does not seem to be implemented:
+ assert "value" in diff_r1
+ assert diff_r1["value"] == 42
+ assert "value" in diff_r2
+ assert diff_r2["value"] == 4
+
+
+def test_copy_entities():
+ r = db.Record(name="A")
+ r.add_parent(name="B")
+ r.add_property(name="C", value=4, importance="OBLIGATORY")
+ r.add_property(name="D", value=[3, 4, 7], importance="OBLIGATORY")
+ r.description = "A fancy test record"
+
+ c = r.copy()
+
+ assert c is not r
+ assert c.name == "A"
+ assert c.role == r.role
+ assert c.parents[0].name == "B"
+ # parent and property objects are not shared among copy and original:
+ assert c.parents[0] is not r.parents[0]
+
+ for i in [0, 1]:
+ assert c.properties[i] is not r.properties[i]
+ for special in SPECIAL_ATTRIBUTES:
+ assert getattr(c.properties[i], special) == getattr(r.properties[i], special)
+ assert c.get_importance(c.properties[i]) == r.get_importance(r.properties[i])
+
+
+def test_merge_entities():
+ r = db.Record(name="A")
+ r.add_parent(name="B")
+ r.add_property(name="C", value=4, importance="OBLIGATORY")
+ r.add_property(name="D", value=[3, 4, 7], importance="OBLIGATORY")
+ r.description = "A fancy test record"
+
+ r2 = db.Record()
+ r2.add_property(name="F", value="text")
+ merge_entities(r2, r)
+ assert r2.get_parents()[0].name == "B"
+ assert r2.get_property("C").name == "C"
+ assert r2.get_property("C").value == 4
+ assert r2.get_property("D").name == "D"
+ assert r2.get_property("D").value == [3, 4, 7]
+
+ assert r2.get_property("F").name == "F"
+ assert r2.get_property("F").value == "text"
+
+
+def test_merge_bug_109():
+ rt = db.RecordType(name="TestBug")
+ p = db.Property(name="test_bug_property", datatype=db.LIST(db.INTEGER))
+
+ r_b = db.Record(name="TestRecord")
+ r_b.add_parent(rt)
+ r_b.add_property(p, value=[18, 19])
+
+ r_a = db.Record(name="TestRecord")
+ r_a.add_parent(rt)
+
+ merge_entities(r_a, r_b)
+
+ assert r_b.get_property("test_bug_property").value == [18, 19]
+ assert r_a.get_property("test_bug_property").value == [18, 19]
+
+ assert "<Value>18</Value>\n <Value>19</Value>" in str(r_b)
+ assert "<Value>18</Value>\n <Value>19</Value>\n <Value>18</Value>\n <Value>19</Value>" not in str(r_b)
+
+ assert "<Value>18</Value>\n <Value>19</Value>" in str(r_a)
+ assert "<Value>18</Value>\n <Value>19</Value>\n <Value>18</Value>\n <Value>19</Value>" not in str(r_a)
+
+
+@pytest.mark.xfail
+def test_bug_109():
+ rt = db.RecordType(name="TestBug")
+ p = db.Property(name="test_bug_property", datatype=db.LIST(db.INTEGER))
+
+ r_b = db.Record(name="TestRecord")
+ r_b.add_parent(rt)
+ r_b.add_property(p, value=[18, 19])
+
+ r_a = db.Record(name="TestRecord")
+ r_a.add_parent(rt)
+ r_a.add_property(r_b.get_property("test_bug_property"))
+
+ assert r_b.get_property("test_bug_property").value == [18, 19]
+ assert r_a.get_property("test_bug_property").value == [18, 19]
+
+ assert "<Value>18</Value>\n <Value>19</Value>" in str(r_b)
+ assert "<Value>18</Value>\n <Value>19</Value>\n <Value>18</Value>\n <Value>19</Value>" not in str(r_b)
+
+ assert "<Value>18</Value>\n <Value>19</Value>" in str(r_a)
+ assert "<Value>18</Value>\n <Value>19</Value>\n <Value>18</Value>\n <Value>19</Value>" not in str(r_a)
diff --git a/unittests/test_authentication_auth_token.py b/unittests/test_authentication_auth_token.py
index 15e54121fc0d7b5c2be645cdb88bc20804a10980..d0eb6b90883951af584d42a80e319c14891f6e50 100644
--- a/unittests/test_authentication_auth_token.py
+++ b/unittests/test_authentication_auth_token.py
@@ -65,7 +65,7 @@ def test_configure_connection():
c._delegate_connection.resources.append(request_has_auth_token)
assert c._authenticator.auth_token == "[request token]"
response = c._http_request(method="GET", path="test")
- assert response.read() == "ok"
+ assert response.read().decode() == "ok"
assert c._authenticator.auth_token == "[response token]"
diff --git a/unittests/test_authentication_plain.py b/unittests/test_authentication_plain.py
index 10cbc418df8bd81c81568a4df0cf1e8a4ac498f8..146b59889c71c86ea77fb4ae962118cdda1afb06 100644
--- a/unittests/test_authentication_plain.py
+++ b/unittests/test_authentication_plain.py
@@ -65,4 +65,6 @@ def test_subclass_configure():
def test_plain_has_logger():
p = PlainTextCredentialsProvider()
assert hasattr(p, "logger")
- assert p.logger.name == "authentication"
+ assert "authentication" in p.logger.name
+ assert "connection" in p.logger.name
+ assert "caosdb" in p.logger.name
diff --git a/unittests/test_authentication_unauthenticated.py b/unittests/test_authentication_unauthenticated.py
index 52146b08ed4e1026660eebacedf348aeb2ff2721..45a709fcc62b609a97de7e87dd6c6f6ac94a55a1 100644
--- a/unittests/test_authentication_unauthenticated.py
+++ b/unittests/test_authentication_unauthenticated.py
@@ -59,7 +59,7 @@ def test_configure_connection():
assert c._authenticator.auth_token is None
response = c._http_request(method="GET", path="test")
- assert response.read() == "ok"
+ assert response.read().decode() == "ok"
mock.method.assert_called_once()
assert c._authenticator.auth_token is None
diff --git a/unittests/test_configs/pycaosdb-IntegrationTests.ini b/unittests/test_configs/pycaosdb-IntegrationTests.ini
new file mode 100644
index 0000000000000000000000000000000000000000..cb9871708f7f23c489de0cbc8f4fbda15dfa6ad0
--- /dev/null
+++ b/unittests/test_configs/pycaosdb-IntegrationTests.ini
@@ -0,0 +1,37 @@
+# -*- mode:conf; -*-
+## This sections needs to exist in addition to the usual section
+[IntegrationTests]
+# test_server_side_scripting.bin_dir.local=/path/to/scripting/bin
+test_server_side_scripting.bin_dir.local=/home/myself/test/caosdb-server/scripting/bin
+# test_server_side_scripting.bin_dir.server=/opt/caosdb/git/caosdb-server/scripting/bin
+
+# # location of the files from the pyinttest perspective
+# test_files.test_insert_files_in_dir.local=/extroot/test_insert_files_in_dir/
+test_files.test_insert_files_in_dir.local=/home/myself/test/debug_advanced/paths/extroot/test_insert_files_in_dir
+# # location of the files from the caosdb_servers perspective
+test_files.test_insert_files_in_dir.server=/opt/caosdb/mnt/extroot/test_insert_files_in_dir/
+
+########## Files ##################
+## Used by tests of file handling. Specify the path to an existing
+## directory in which file tests are performed, once as seen by the
+## host and once as seen by the server.
+
+# location of the files from the pyinttest (i.e. host) perspective
+#test_files.test_insert_files_in_dir.local=/extroot/test_insert_files_in_dir/
+
+# location of the files from the caosdb server's perspective
+#test_files.test_insert_files_in_dir.server=/opt/caosdb/mnt/extroot/test_insert_files_in_dir/
+
+# # location of the one-time tokens from the pyinttest's perspective
+# test_authentication.admin_token_crud = /authtoken/admin_token_crud.txt
+# test_authentication.admin_token_expired = /authtoken/admin_token_expired.txt
+# test_authentication.admin_token_3_attempts = /authtoken/admin_token_3_attempts.txt
+
+
+## Insert your usual settings here
+[Connection]
+url=https://localhost:10443/
+username=admin
+password_method=plain
+password=caosdb
+
diff --git a/unittests/test_configs/pycaosdb-empty.ini b/unittests/test_configs/pycaosdb-empty.ini
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/unittests/test_configs/pycaosdb-indiscale-demo.ini b/unittests/test_configs/pycaosdb-indiscale-demo.ini
new file mode 100644
index 0000000000000000000000000000000000000000..9010343467f78c7c9c3e25ea3a57520deac18c8e
--- /dev/null
+++ b/unittests/test_configs/pycaosdb-indiscale-demo.ini
@@ -0,0 +1,12 @@
+[Connection]
+url=https://demo.indiscale.com/
+username=admin
+password=caosdb
+password_method=plain
+cacert=/etc/ssl/cert.pem
+
+timeout=10000
+debug=0
+
+[Container]
+debug=0
\ No newline at end of file
diff --git a/unittests/test_configs/pycaosdb-real-world-1.ini b/unittests/test_configs/pycaosdb-real-world-1.ini
new file mode 100644
index 0000000000000000000000000000000000000000..e524f1d3465c61d89ae4a4dda54536a722f99837
--- /dev/null
+++ b/unittests/test_configs/pycaosdb-real-world-1.ini
@@ -0,0 +1,17 @@
+[Connection]
+url = https://localhost:10443
+cacert = /opt/caosdb/cert/caosdb.cert.pem
+debug = 0
+timeout = 5000
+
+[Misc]
+sendmail = /usr/local/bin/sendmail_to_file
+entity_loan.curator_mail_from=crawler-test@example.com
+entity_loan.curator_mail_to=crawler-test@example.com
+
+[sss_helper]
+external_uri = https://caosdb.example.com:443
+
+[advancedtools]
+crawler.from_mail=admin@example.com
+crawler.to_mail=admin@example.com
diff --git a/unittests/test_configs/pycaosdb-real-world-2.ini b/unittests/test_configs/pycaosdb-real-world-2.ini
new file mode 100644
index 0000000000000000000000000000000000000000..5ebd115a4a4de189d22180130acca2a4b78b6daf
--- /dev/null
+++ b/unittests/test_configs/pycaosdb-real-world-2.ini
@@ -0,0 +1,15 @@
+[Connection]
+url = https://samplemanager.example.com:443
+cacert = /opt/caosdb/cert/caosdb.cert.pem
+debug = 0
+timeout = 5000
+[Misc]
+sendmail = /usr/local/bin/sendmail_to_file
+entity_loan.curator_mail_from=crawler-test@example.com
+entity_loan.curator_mail_to=crawler-test@example.com
+[sss_helper]
+external_uri = https://localhost:10443
+[advancedtools]
+crawler.from_mail=crawler-test@example.com
+crawler.to_mail=crawler-test@example.com
+
diff --git a/unittests/test_configs/pycaosdb-server-side-scripting.ini b/unittests/test_configs/pycaosdb-server-side-scripting.ini
new file mode 100644
index 0000000000000000000000000000000000000000..de2867f8dc66b3e81f10f35e40c36f9cb8591604
--- /dev/null
+++ b/unittests/test_configs/pycaosdb-server-side-scripting.ini
@@ -0,0 +1,9 @@
+; this is the pycaosdb.ini for the server-side-scripting home.
+[Connection]
+url = https://caosdb-server:10443
+cacert = /opt/caosdb/cert/caosdb.cert.pem
+debug = 0
+timeout = 5000
+
+[Misc]
+sendmail = /usr/local/bin/sendmail_to_file
diff --git a/unittests/test_configs/pycaosdb1.ini b/unittests/test_configs/pycaosdb1.ini
new file mode 100644
index 0000000000000000000000000000000000000000..dcfa7c21fac735d81ab92b33f0abd31df25fc1ad
--- /dev/null
+++ b/unittests/test_configs/pycaosdb1.ini
@@ -0,0 +1,6 @@
+[Connection]
+cacert=/very/long/path/to/self/signed/pem/file/caosdb.ca.pem
+url=https://hostname:8833/playground
+password_identifier=SECTION/caosdb
+username=username
+password_method=pass
diff --git a/unittests/test_configs/pycaosdb2.ini b/unittests/test_configs/pycaosdb2.ini
new file mode 100644
index 0000000000000000000000000000000000000000..e5493bde3be0f84f38e427edb4d42fba9c75482d
--- /dev/null
+++ b/unittests/test_configs/pycaosdb2.ini
@@ -0,0 +1,9 @@
+[Connection]
+url=https://0.0.0.0/
+username=username
+password_identifier=SECTION/SUBSECTION/identifier
+password_method=pass
+cacert=/etc/ssl/cert.pem
+ssl_insecure=true
+timeout=10000
+debug=0
diff --git a/unittests/test_configs/pycaosdb3.ini b/unittests/test_configs/pycaosdb3.ini
new file mode 100644
index 0000000000000000000000000000000000000000..6c4934039c99855566f38c69f7511d774f81efbd
--- /dev/null
+++ b/unittests/test_configs/pycaosdb3.ini
@@ -0,0 +1,12 @@
+[Connection]
+ssl_insecure=true
+url=https://localhost:10443/
+password=caosdb
+username=admin
+password_method=plain
+
+timeout=10000
+debug=0
+
+[Container]
+debug=0
diff --git a/unittests/test_configs/pycaosdb4.ini b/unittests/test_configs/pycaosdb4.ini
new file mode 100644
index 0000000000000000000000000000000000000000..ddbc7ca6f969e55ea6131d96f091177a13687ece
--- /dev/null
+++ b/unittests/test_configs/pycaosdb4.ini
@@ -0,0 +1,4 @@
+[Connection]
+url=https://localhost:10443/
+username=admin
+password_method=input
diff --git a/unittests/test_configs/pycaosdb5.ini b/unittests/test_configs/pycaosdb5.ini
new file mode 100644
index 0000000000000000000000000000000000000000..3f365efdd92641a39b742e22f825033a69e12dc5
--- /dev/null
+++ b/unittests/test_configs/pycaosdb5.ini
@@ -0,0 +1,4 @@
+[Connection]
+url=https://localhost:10443/
+username=admin
+# No password method: should be "input" by default
diff --git a/unittests/test_connection.py b/unittests/test_connection.py
index 16370f00b7d5e3389582befaac1762b1d2992fcf..ee564ea033f9afc80522d75a85557f70819ece1e 100644
--- a/unittests/test_connection.py
+++ b/unittests/test_connection.py
@@ -169,7 +169,7 @@ def test_getter_status():
def test_read():
response = test_init_response()
tru(hasattr(response, "read"))
- eq(response.read(), "Body")
+ eq(response.read().decode(), "Body")
def test_getter_session_token():
diff --git a/unittests/test_entity.py b/unittests/test_entity.py
index e98dfbef5b6b5e5f691e8aecc2fa7d4a86991452..f2891fda266e1d62139b4cb2667c31b090ca6498 100644
--- a/unittests/test_entity.py
+++ b/unittests/test_entity.py
@@ -24,11 +24,15 @@
"""Tests for the Entity class."""
# pylint: disable=missing-docstring
import unittest
+from lxml import etree
+import os
from caosdb import (INTEGER, Entity, Property, Record, RecordType,
configure_connection)
from caosdb.connection.mockup import MockUpServerConnection
+UNITTESTDIR = os.path.dirname(os.path.abspath(__file__))
+
class TestEntity(unittest.TestCase):
@@ -42,17 +46,54 @@ class TestEntity(unittest.TestCase):
def test_instance_variables(self):
entity = Entity()
self.assertTrue(hasattr(entity, "role"))
+ self.assertIsNone(entity.role)
self.assertTrue(hasattr(entity, "id"))
self.assertTrue(hasattr(entity, "name"))
self.assertTrue(hasattr(entity, "description"))
self.assertTrue(hasattr(entity, "parents"))
self.assertTrue(hasattr(entity, "properties"))
- def test_role(self):
+ def test_entity_role_1(self):
entity = Entity(role="TestRole")
self.assertEqual(entity.role, "TestRole")
entity.role = "TestRole2"
self.assertEqual(entity.role, "TestRole2")
- def test_instanciation(self):
+ def test_entity_role_2(self):
+ entity = Entity()
+
+ self.assertIsNone(entity.role)
+ self.assertEqual(entity.to_xml().tag, "Entity")
+
+ entity.role = "Record"
+ self.assertEqual(entity.role, "Record")
+ self.assertEqual(entity.to_xml().tag, "Record")
+
+ def test_recordtype_role(self):
+ entity = RecordType()
+
+ self.assertEqual(entity.role, "RecordType")
+ self.assertEqual(entity.to_xml().tag, "RecordType")
+
+ def test_property_role(self):
+ entity = Property()
+
+ self.assertEqual(entity.role, "Property")
+ self.assertEqual(entity.to_xml().tag, "Property")
+
+ def test_instantiation(self):
self.assertRaises(Exception, Entity())
+
+ def test_parse_role(self):
+ """During parsing, the role of an entity is set explicitely. All other
+ classes use the class name as a "natural" value for the role property.
+ """
+ parser = etree.XMLParser(remove_comments=True)
+ entity = Entity._from_xml(Entity(),
+ etree.parse(os.path.join(UNITTESTDIR, "test_record.xml"),
+ parser).getroot())
+
+ self.assertEqual(entity.role, "Record")
+ # test whether the __role property of this object has explicitely been
+ # set.
+ self.assertEqual(getattr(entity, "_Entity__role"), "Record")
diff --git a/unittests/test_high_level_api.py b/unittests/test_high_level_api.py
new file mode 100644
index 0000000000000000000000000000000000000000..a9e55c9c2a79f7ead8bbb3fb652c1b81427e69e9
--- /dev/null
+++ b/unittests/test_high_level_api.py
@@ -0,0 +1,643 @@
+# This file is a part of the CaosDB Project.
+#
+# Copyright (C) 2018 Research Group Biomedical Physics,
+# Max-Planck-Institute for Dynamics and Self-Organization Göttingen
+# Copyright (C) 2022 Alexander Schlemmer <alexander.schlemmer@ds.mpg.de>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see <https://www.gnu.org/licenses/>.
+#
+# ** end header
+#
+# Test high level api module
+# A. Schlemmer, 02/2022
+
+
+import caosdb as db
+from caosdb.high_level_api import (convert_to_entity, convert_to_python_object,
+ new_high_level_entity)
+from caosdb.high_level_api import (CaosDBPythonUnresolvedParent,
+ CaosDBPythonUnresolvedReference,
+ CaosDBPythonRecord, CaosDBPythonFile,
+ high_level_type_for_standard_type,
+ standard_type_for_high_level_type,
+ high_level_type_for_role,
+ CaosDBPythonEntity)
+from caosdb.apiutils import compare_entities
+
+from caosdb.common.datatype import (is_list_datatype,
+ get_list_datatype,
+ is_reference)
+
+import pytest
+from lxml import etree
+import os
+import tempfile
+import pickle
+
+import sys
+import traceback
+import pdb
+
+
+@pytest.fixture
+def testrecord():
+ parser = etree.XMLParser(remove_comments=True)
+ testrecord = db.Record._from_xml(
+ db.Record(),
+ etree.parse(os.path.join(os.path.dirname(__file__), "test_record.xml"),
+ parser).getroot())
+ return testrecord
+
+
+def test_convert_object(testrecord):
+ r2 = convert_to_python_object(testrecord)
+ assert r2.species == "Rabbit"
+
+
+def test_pickle_object(testrecord):
+ r2 = convert_to_python_object(testrecord)
+ with tempfile.TemporaryFile() as f:
+ pickle.dump(r2, f)
+ f.seek(0)
+ rn2 = pickle.load(f)
+ assert r2.date == rn2.date
+
+
+def test_convert_record():
+ """
+ Test the high level python API.
+ """
+ r = db.Record()
+ r.add_parent("bla")
+ r.add_property(name="a", value=42)
+ r.add_property(name="b", value="test")
+
+ obj = convert_to_python_object(r)
+ assert obj.a == 42
+ assert obj.b == "test"
+
+ # There is no such property
+ with pytest.raises(AttributeError):
+ assert obj.c == 18
+
+ assert obj.has_parent("bla") is True
+ assert obj.has_parent(CaosDBPythonUnresolvedParent(name="bla")) is True
+
+ # Check the has_parent function:
+ assert obj.has_parent("test") is False
+ assert obj.has_parent(CaosDBPythonUnresolvedParent(name="test")) is False
+
+ # duplicate parent
+ with pytest.raises(RuntimeError):
+ obj.add_parent("bla")
+
+ # add parent with just an id:
+ obj.add_parent(CaosDBPythonUnresolvedParent(id=225))
+ assert obj.has_parent(225) is True
+ assert obj.has_parent(CaosDBPythonUnresolvedParent(id=225)) is True
+ assert obj.has_parent(226) is False
+ assert obj.has_parent(CaosDBPythonUnresolvedParent(id=228)) is False
+
+ # same with just a name:
+ obj.add_parent(CaosDBPythonUnresolvedParent(name="another"))
+ assert obj.has_parent("another") is True
+
+
+def test_convert_with_references():
+ r_ref = db.Record()
+ r_ref.add_property(name="a", value=42)
+
+ r = db.Record()
+ r.add_property(name="ref", value=r_ref)
+
+ # try:
+ obj = convert_to_python_object(r)
+ # except:
+ # extype, value, tb = sys.exc_info()
+ # traceback.print_exc()
+ # pdb.post_mortem(tb)
+ assert obj.ref.a == 42
+
+ # With datatype:
+ r_ref = db.Record()
+ r_ref.add_parent("bla")
+ r_ref.add_property(name="a", value=42)
+
+ r = db.Record()
+ r.add_property(name="ref", value=r_ref)
+
+ obj = convert_to_python_object(r)
+ assert obj.ref.a == 42
+ # Parent does not automatically lead to a datatype:
+ assert obj.get_property_metadata("ref").datatype is None
+ assert obj.ref.has_parent("bla") is True
+
+ # Add datatype explicitely:
+ r_ref = db.Record()
+ r_ref.add_parent("bla")
+ r_ref.add_property(name="a", value=42)
+
+ r = db.Record()
+ r.add_property(name="ref", value=r_ref, datatype="bla")
+
+ obj = convert_to_python_object(r)
+ assert obj.ref.a == 42
+ # Parent does not automatically lead to a datatype:
+ assert obj.get_property_metadata("ref").datatype is "bla"
+ assert obj.ref.has_parent("bla") is True
+
+ # Unresolved Reference:
+ r = db.Record()
+ r.add_property(name="ref", value=27, datatype="bla")
+
+ obj = convert_to_python_object(r)
+ # Parent does not automatically lead to a datatype:
+ assert obj.get_property_metadata("ref").datatype is "bla"
+ assert isinstance(obj.ref, CaosDBPythonUnresolvedReference)
+ assert obj.ref.id == 27
+
+
+def test_resolve_references():
+ r = db.Record()
+ r.add_property(name="ref", value=27, datatype="bla")
+ r.add_property(name="ref_false", value=27) # this should be interpreted as integer property
+ obj = convert_to_python_object(r)
+
+ ref = db.Record(id=27)
+ ref.add_property(name="a", value=57)
+
+ unused_ref1 = db.Record(id=28)
+ unused_ref2 = db.Record(id=29)
+ unused_ref3 = db.Record(name="bla")
+
+ references = db.Container().extend([
+ unused_ref1, ref, unused_ref2, unused_ref3])
+
+ # Nothing is going to be resolved:
+ obj.resolve_references(False, db.Container())
+ assert isinstance(obj.ref, CaosDBPythonUnresolvedReference)
+ assert obj.ref.id == 27
+ assert obj.ref_false == 27
+
+ # deep == True does not help:
+ obj.resolve_references(True, db.Container())
+ assert isinstance(obj.ref, CaosDBPythonUnresolvedReference)
+ assert obj.ref.id == 27
+
+ # But adding the reference container will do:
+ obj.resolve_references(False, references)
+ assert not isinstance(obj.ref, CaosDBPythonUnresolvedReference)
+ assert isinstance(obj.ref, CaosDBPythonRecord)
+ assert obj.ref.id == 27
+ assert obj.ref.a == 57
+ # Datatypes will not automatically be set:
+ assert obj.ref.get_property_metadata("a").datatype is None
+
+ # Test deep resolve:
+ ref2 = db.Record(id=225)
+ ref2.add_property(name="c", value="test")
+ ref.add_property(name="ref", value=225, datatype="bla")
+
+ obj = convert_to_python_object(r)
+ assert isinstance(obj.ref, CaosDBPythonUnresolvedReference)
+ obj.resolve_references(False, references)
+ assert not isinstance(obj.ref, CaosDBPythonUnresolvedReference)
+ assert isinstance(obj.ref.ref, CaosDBPythonUnresolvedReference)
+ assert obj.ref.ref.id == 225
+
+ # Will not help, because ref2 is missing in container:
+ obj.resolve_references(True, references)
+ assert not isinstance(obj.ref, CaosDBPythonUnresolvedReference)
+ assert isinstance(obj.ref.ref, CaosDBPythonUnresolvedReference)
+ assert obj.ref.ref.id == 225
+
+ references.append(ref2)
+ obj.resolve_references(False, references)
+ assert not isinstance(obj.ref, CaosDBPythonUnresolvedReference)
+ assert isinstance(obj.ref.ref, CaosDBPythonUnresolvedReference)
+ assert obj.ref.ref.id == 225
+
+ obj.resolve_references(True, references)
+ assert not isinstance(obj.ref, CaosDBPythonUnresolvedReference)
+ assert not isinstance(obj.ref.ref, CaosDBPythonUnresolvedReference)
+ assert obj.ref.ref.c == "test"
+
+ # Test circular dependencies:
+ ref2.add_property(name="ref", value=27, datatype="bla")
+ obj = convert_to_python_object(r)
+ obj.resolve_references(True, references)
+ assert obj.ref.ref.ref == obj.ref
+
+
+def equal_entities(r1, r2):
+ res = compare_entities(r1, r2)
+ if len(res) != 2:
+ return False
+ for i in range(2):
+ if len(res[i]["parents"]) != 0 or len(res[i]["properties"]) != 0:
+ return False
+ return True
+
+
+def test_conversion_to_entity():
+ r = db.Record()
+ r.add_parent("bla")
+ r.add_property(name="a", value=42)
+ r.add_property(name="b", value="test")
+ obj = convert_to_python_object(r)
+ rconv = convert_to_entity(obj)
+ assert equal_entities(r, rconv)
+
+ # With a reference:
+ r_ref = db.Record()
+ r_ref.add_parent("bla")
+ r_ref.add_property(name="a", value=42)
+
+ r = db.Record()
+ r.add_property(name="ref", value=r_ref)
+ obj = convert_to_python_object(r)
+ rconv = convert_to_entity(obj)
+ assert (rconv.get_property("ref").value.get_property("a").value
+ == r.get_property("ref").value.get_property("a").value)
+ # TODO: add more tests here
+
+
+def test_base_properties():
+ r = db.Record(id=5, name="test", description="ok")
+ r.add_property(name="v", value=15, datatype=db.INTEGER, unit="kpx",
+ importance="RECOMMENDED", description="description")
+ obj = convert_to_python_object(r)
+ assert obj.name == "test"
+ assert obj.id == 5
+ assert obj.description == "ok"
+ metadata = obj.get_property_metadata("v")
+ assert metadata.id is None
+ assert metadata.datatype == db.INTEGER
+ assert metadata.unit == "kpx"
+ assert metadata.importance == "RECOMMENDED"
+ assert metadata.description == "description"
+
+ rconv = convert_to_entity(obj)
+ assert rconv.name == "test"
+ assert rconv.id == 5
+ assert rconv.description == "ok"
+ prop = rconv.get_property("v")
+ assert prop.value == 15
+ assert prop.datatype == db.INTEGER
+ assert prop.unit == "kpx"
+ assert prop.description == "description"
+ assert rconv.get_importance("v") == "RECOMMENDED"
+
+
+def test_empty():
+ r = db.Record()
+ obj = convert_to_python_object(r)
+ assert isinstance(obj, CaosDBPythonRecord)
+ assert len(obj.get_properties()) == 0
+ assert len(obj.get_parents()) == 0
+
+ rconv = convert_to_entity(obj)
+ assert len(rconv.properties) == 0
+
+
+def test_wrong_entity_for_file():
+ r = db.Record()
+ r.path = "test.dat"
+ r.file = "/local/path/test.dat"
+ assert r.path == "test.dat"
+ assert r.file == "/local/path/test.dat"
+ with pytest.raises(RuntimeError):
+ obj = convert_to_python_object(r)
+
+
+def test_serialization():
+ r = db.Record(id=5, name="test", description="ok")
+ r.add_property(name="v", value=15, datatype=db.INTEGER, unit="kpx",
+ importance="RECOMMENDED")
+
+ obj = convert_to_python_object(r)
+ text = str(obj)
+ teststrs = ["description: ok", "id: 5", "datatype: INTEGER",
+ "importance: RECOMMENDED", "unit: kpx", "name: test", "v: 15"]
+ for teststr in teststrs:
+ assert teststr in text
+
+ r = db.Record(description="ok")
+ r.add_property(name="v", value=15, datatype=db.INTEGER, unit="kpx",
+ importance="RECOMMENDED")
+ obj = convert_to_python_object(r)
+ text = str(obj)
+ assert "name" not in text
+ assert "id" not in text
+
+
+def test_files():
+ # empty file:
+ r = db.File()
+ obj = convert_to_python_object(r)
+ print(type(obj))
+ assert isinstance(obj, CaosDBPythonFile)
+ assert len(obj.get_properties()) == 0
+ assert len(obj.get_parents()) == 0
+
+ rconv = convert_to_entity(obj)
+ assert len(rconv.properties) == 0
+
+ r.path = "test.dat"
+ r.file = "/local/path/test.dat"
+ obj = convert_to_python_object(r)
+ assert r.path == "test.dat"
+ assert r.file == "/local/path/test.dat"
+ assert isinstance(obj, CaosDBPythonFile)
+
+ assert obj.path == "test.dat"
+ assert obj.file == "/local/path/test.dat"
+
+ assert "path: test.dat" in str(obj)
+ assert "file: /local/path/test.dat" in str(obj)
+
+ # record with file property:
+ rec = db.Record()
+ rec.add_property(name="testfile", value=r)
+ assert rec.get_property("testfile").value.file == "/local/path/test.dat"
+ assert rec.get_property("testfile").value.path == "test.dat"
+
+ obj = convert_to_python_object(rec)
+ assert obj.testfile.file == "/local/path/test.dat"
+ assert obj.testfile.path == "test.dat"
+
+ rconv = convert_to_entity(obj)
+ assert rconv.get_property("testfile").value.file == "/local/path/test.dat"
+ assert rconv.get_property("testfile").value.path == "test.dat"
+
+ # record with file property as reference:
+ rec = db.Record()
+ rec.add_property(name="testfile", value=2, datatype=db.FILE)
+ obj = convert_to_python_object(rec)
+ assert type(obj.testfile) == CaosDBPythonUnresolvedReference
+ assert obj.testfile.id == 2
+ assert obj.get_property_metadata("testfile").datatype == db.FILE
+
+ # without resolving references:
+ rconv = convert_to_entity(obj)
+ p = rconv.get_property("testfile")
+ assert p.value == 2
+ assert p.datatype == db.FILE
+
+ # with previously resolved reference (should not work here, because id is missing):
+ obj.resolve_references(True, db.Container().extend(r))
+ rconv = convert_to_entity(obj)
+ p = rconv.get_property("testfile")
+ assert p.value == 2
+ assert p.datatype == db.FILE
+
+ # this time it must work:
+ r.id = 2
+ obj.resolve_references(True, db.Container().extend(r))
+ rconv = convert_to_entity(obj)
+ p = rconv.get_property("testfile")
+ assert type(p.value) == db.File
+ assert p.datatype == db.FILE
+ assert p.value.file == "/local/path/test.dat"
+ assert p.value.path == "test.dat"
+
+
+@pytest.mark.xfail
+def test_record_generator():
+ rt = db.RecordType(name="Simulation")
+ rt.add_property(name="a", datatype=db.INTEGER)
+ rt.add_property(name="b", datatype=db.DOUBLE)
+ rt.add_property(name="inputfile", datatype=db.FILE)
+
+ simrt = db.RecordType(name="SimOutput")
+ rt.add_property(name="outputfile", datatype="SimOutput")
+
+ obj = new_high_level_entity(
+ rt, "SUGGESTED", "", True)
+ print(obj)
+ assert False
+
+
+def test_list_types():
+ r = db.Record()
+ r.add_property(name="a", value=[1, 2, 4])
+
+ assert get_list_datatype(r.get_property("a").datatype) is None
+
+ obj = convert_to_python_object(r)
+ assert type(obj.a) == list
+ assert len(obj.a) == 3
+ assert 4 in obj.a
+ assert obj.get_property_metadata("a").datatype is None
+
+ conv = convert_to_entity(obj)
+ prop = r.get_property("a")
+ assert prop.value == [1, 2, 4]
+ assert prop.datatype is None
+
+ r.get_property("a").datatype = db.LIST(db.INTEGER)
+ assert r.get_property("a").datatype == "LIST<INTEGER>"
+ obj = convert_to_python_object(r)
+ assert type(obj.a) == list
+ assert len(obj.a) == 3
+ assert 4 in obj.a
+ assert obj.get_property_metadata("a").datatype == "LIST<INTEGER>"
+
+ conv = convert_to_entity(obj)
+ prop = r.get_property("a")
+ assert prop.value == [1, 2, 4]
+ assert obj.get_property_metadata("a").datatype == "LIST<INTEGER>"
+
+ # List of referenced objects:
+ r = db.Record()
+ r.add_property(name="a", value=[1, 2, 4], datatype="LIST<TestReference>")
+ obj = convert_to_python_object(r)
+ assert type(obj.a) == list
+ assert len(obj.a) == 3
+ assert obj.get_property_metadata("a").datatype == "LIST<TestReference>"
+ for i in range(3):
+ assert type(obj.a[i]) == CaosDBPythonUnresolvedReference
+ assert obj.a == [CaosDBPythonUnresolvedReference(id=i) for i in [1, 2, 4]]
+
+ # Try resolving:
+
+ # Should not work:
+ obj.resolve_references(False, db.Container())
+ assert type(obj.a) == list
+ assert len(obj.a) == 3
+ assert obj.get_property_metadata("a").datatype == "LIST<TestReference>"
+ for i in range(3):
+ assert type(obj.a[i]) == CaosDBPythonUnresolvedReference
+ assert obj.a == [CaosDBPythonUnresolvedReference(id=i) for i in [1, 2, 4]]
+
+ references = db.Container()
+ for i in [1, 2, 4]:
+ ref = db.Record(id=i)
+ ref.add_property(name="val", value=str(i) + " bla")
+ references.append(ref)
+
+ obj.resolve_references(False, references)
+ assert type(obj.a) == list
+ assert len(obj.a) == 3
+ assert obj.get_property_metadata("a").datatype == "LIST<TestReference>"
+ for i in range(3):
+ assert type(obj.a[i]) == CaosDBPythonRecord
+
+ assert obj.a[0].val == "1 bla"
+
+ # Conversion with embedded records:
+ r2 = db.Record()
+ r2.add_property(name="a", value=4)
+ r3 = db.Record()
+ r3.add_property(name="b", value=8)
+
+ r = db.Record()
+ r.add_property(name="a", value=[r2, r3])
+
+ obj = convert_to_python_object(r)
+ assert type(obj.a) == list
+ assert len(obj.a) == 2
+ assert obj.a[0].a == 4
+ assert obj.a[1].b == 8
+
+ # Serialization
+ text = str(obj)
+ text2 = str(convert_to_python_object(r2)).split("\n")
+ print(text)
+ # cut away first two characters in text
+ text = [line[4:] for line in text.split("\n")]
+ for line in text2:
+ assert line in text
+
+
+# Test utility functions:
+def test_type_conversion():
+ assert high_level_type_for_standard_type(db.Record()) == CaosDBPythonRecord
+ assert high_level_type_for_standard_type(db.Entity()) == CaosDBPythonEntity
+ assert standard_type_for_high_level_type(CaosDBPythonRecord()) == db.Record
+ assert standard_type_for_high_level_type(CaosDBPythonEntity()) == db.Entity
+ assert standard_type_for_high_level_type(CaosDBPythonFile(), True) == "File"
+ assert standard_type_for_high_level_type(CaosDBPythonRecord(), True) == "Record"
+ assert high_level_type_for_role("Record") == CaosDBPythonRecord
+ assert high_level_type_for_role("Entity") == CaosDBPythonEntity
+ assert high_level_type_for_role("File") == CaosDBPythonFile
+ with pytest.raises(RuntimeError, match="Unknown role."):
+ high_level_type_for_role("jkaldjfkaldsjf")
+
+ with pytest.raises(RuntimeError, match="Incompatible type."):
+ standard_type_for_high_level_type(42, True)
+
+ with pytest.raises(ValueError):
+ high_level_type_for_standard_type("ajsdkfjasfkj")
+
+ with pytest.raises(RuntimeError, match="Incompatible type."):
+ class IncompatibleType(db.Entity):
+ pass
+ high_level_type_for_standard_type(IncompatibleType())
+
+
+def test_deserialization():
+ r = db.Record(id=17, name="test")
+ r.add_parent("bla")
+ r.add_property(name="a", value=42)
+ r.add_property(name="b", value="test")
+
+ obj = convert_to_python_object(r)
+
+ serial = obj.serialize()
+ obj_des = CaosDBPythonEntity.deserialize(serial)
+
+ assert obj_des.name == "test"
+ assert obj_des.id == 17
+ assert obj_des.has_parent(CaosDBPythonUnresolvedParent(name="bla"))
+ print(obj)
+ print(obj_des)
+
+ # This test is very strict, and might fail if order in dictionary is not preserved:
+ assert obj.serialize() == obj_des.serialize()
+
+ f = db.File()
+ f.file = "bla.test"
+ f.path = "/test/n/bla.test"
+
+ obj = convert_to_python_object(f)
+
+ serial = obj.serialize()
+ obj_des = CaosDBPythonEntity.deserialize(serial)
+ assert obj_des.file == "bla.test"
+ assert obj_des.path == "/test/n/bla.test"
+
+ r = db.Record(id=17, name="test")
+ r.add_parent("bla")
+ r.add_property(name="a", value=42)
+ r.add_property(name="b", value="test")
+
+ ref = db.Record(id=28)
+ ref.add_parent("bla1")
+ ref.add_parent("bla2")
+ ref.add_property(name="c", value=5,
+ unit="s", description="description missing")
+ r.add_property(name="ref", value=ref)
+
+ obj = convert_to_python_object(r)
+
+ serial = obj.serialize()
+ obj_des = CaosDBPythonEntity.deserialize(serial)
+ assert obj.serialize() == obj_des.serialize()
+
+
+@pytest.fixture
+def get_record_container():
+ record_xml = """
+<Entities>
+ <Record id="109">
+ <Version id="da669fce50554b2835c3826cf717d6a4532f02de" head="true">
+ <Predecessor id="68534369c5fd05e5bb1d37801a3dbc1532a8e094"/>
+ </Version>
+ <Parent id="103" name="Experiment" description="General type for all experiments in our lab"/>
+ <Property id="104" name="alpha" description="A fictitious measurement" datatype="DOUBLE" unit="km" importance="FIX" flag="inheritance:FIX">16.0</Property>
+ <Property id="107" name="date" datatype="DATETIME" importance="FIX" flag="inheritance:FIX">2022-03-16</Property>
+ <Property id="108" name="identifier" datatype="TEXT" importance="FIX" flag="inheritance:FIX">Demonstration</Property>
+ <Property id="111" name="sources" description="The elements of this lists are scientific activities that this scientific activity is based on." datatype="LIST<ScientificActivity>" importance="FIX" flag="inheritance:FIX">
+ <Value>109</Value>
+ </Property>
+ </Record>
+</Entities>"""
+
+ c = db.Container.from_xml(record_xml)
+ return c
+
+
+def test_recursion(get_record_container):
+ r = convert_to_python_object(get_record_container[0])
+ r.resolve_references(r, get_record_container)
+ assert r.id == 109
+ assert r.sources[0].id == 109
+ assert r.sources[0].sources[0].id == 109
+ assert "&id001" in str(r)
+ assert "*id001" in str(r)
+
+ d = r.serialize(True)
+ assert r.sources[0] == r.sources[0].sources[0]
+
+
+@pytest.mark.xfail
+def test_recursion_advanced(get_record_container):
+ # TODO:
+ # This currently fails, because resolve is done in a second step
+ # and therefore a new python object is created for the reference.
+ r = convert_to_python_object(get_record_container[0])
+ r.resolve_references(r, get_record_container)
+ d = r.serialize(True)
+ assert r == r.sources[0]
diff --git a/unittests/test_issues.py b/unittests/test_issues.py
new file mode 100644
index 0000000000000000000000000000000000000000..1e649db4f23de67e55301e0a053fba70d14680b4
--- /dev/null
+++ b/unittests/test_issues.py
@@ -0,0 +1,39 @@
+# This file is a part of the CaosDB Project.
+#
+# Copyright (c) 2022 IndiScale GmbH
+# Copyright (c) 2022 Daniel Hornung (d.hornung@indiscale.com)
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+"""Test known issues to prevent regressions.
+"""
+
+import os
+
+import lxml
+import caosdb as db
+
+from pytest import raises
+
+
+def test_issue_100():
+ """_parse_value() fails for some list-valued content
+ """
+
+ # Parse from (invalid) XML file
+ filename = os.path.join(os.path.dirname(__file__), "data", "list_in_value.xml")
+ xml_el = lxml.etree.parse(filename).getroot()
+ with raises(db.ServerConfigurationException) as exc_info:
+ db.common.models._parse_single_xml_element(xml_el)
+ assert "invalid XML: List valued properties" in exc_info.value.msg
diff --git a/unittests/test_plantuml.py b/unittests/test_plantuml.py
new file mode 100644
index 0000000000000000000000000000000000000000..a507c36b2d3a4246205fc7507cb05119c575084c
--- /dev/null
+++ b/unittests/test_plantuml.py
@@ -0,0 +1,61 @@
+#!/usr/bin/env python3
+# encoding: utf-8
+#
+# ** header v3.0
+# This file is a part of the CaosDB Project.
+#
+# Copyright (C) 2022 Indiscale GmbH <info@indiscale.com>
+# Copyright (C) 2022 Henrik tom Wörden <h.tomwoerden@indiscale.com>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see <https://www.gnu.org/licenses/>.
+#
+# ** end header
+#
+
+"""
+test plantuml utility
+"""
+
+import tempfile
+import pytest
+import caosdb as db
+import shutil
+from caosdb.utils.plantuml import to_graphics
+
+
+@pytest.fixture
+def setup_n_teardown(autouse=True):
+
+ with tempfile.TemporaryDirectory() as td:
+ global output
+ output = td
+ yield
+
+
+@pytest.fixture
+def entities():
+ return [db.RecordType(name="TestRT1").add_property("testprop"),
+ db.RecordType(name="TestRT2").add_property("testprop2"),
+ db.Property("testprop")]
+
+
+@pytest.mark.skipif(shutil.which('plantuml') is None, reason="No plantuml found")
+def test_to_graphics1(entities, setup_n_teardown):
+ to_graphics(entities, "data_model", output_dirname=output)
+
+
+@pytest.mark.skipif(shutil.which('plantuml') is None, reason="No plantuml found")
+def test_to_graphics2(entities, setup_n_teardown):
+ to_graphics(entities, "data_model", output_dirname=output, formats=["tpng", "tsvg"],
+ add_properties=False, add_legend=False, style="salexan")
diff --git a/unittests/test_property.py b/unittests/test_property.py
index 834b1be582c58c60f70331de9cb0d0d6414fd6c9..7c756117765e510587c00d818e39fb3945d44c53 100644
--- a/unittests/test_property.py
+++ b/unittests/test_property.py
@@ -24,15 +24,18 @@
# ** end header
#
"""Tests for the Property class."""
+import os
+
import caosdb as db
from caosdb import Entity, Property, Record
# pylint: disable=missing-docstring
from lxml import etree
parser = etree.XMLParser(remove_comments=True)
-testrecord = Record._from_xml(Record(),
- etree.parse("unittests/test_record.xml",
- parser).getroot())
+testrecord = Record._from_xml(
+ Record(),
+ etree.parse(os.path.join(os.path.dirname(__file__), "test_record.xml"),
+ parser).getroot())
def test_is_entity():
@@ -48,7 +51,8 @@ def test_instance_variables():
def test_null_empty_text_value_1():
- assert testrecord.get_property("LISTofTEXT").value == ["One", "Two", "Three", None, ""]
+ assert testrecord.get_property("LISTofTEXT").value == ["One", "Two",
+ "Three", None, ""]
def test_null_empty_text_value_2():
diff --git a/unittests/test_schema.py b/unittests/test_schema.py
new file mode 100644
index 0000000000000000000000000000000000000000..fc3f63a4cbaeadcac3c1cb9be2d861a0688fe4b0
--- /dev/null
+++ b/unittests/test_schema.py
@@ -0,0 +1,50 @@
+# -*- coding: utf-8 -*-
+#
+# This file is a part of the CaosDB Project.
+#
+# Copyright (C) 2022 Indiscale GmbH <info@indiscale.com>
+# Copyright (C) 2021 Alexander Schlemmer
+# Copyright (C) 2022 Daniel Hornung <d.hornung@indiscale.com>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see <https://www.gnu.org/licenses/>.
+#
+
+"""Test configuration schema.
+A. Schlemmer, 01/2021
+"""
+from jsonschema.exceptions import ValidationError
+from pytest import raises
+from glob import glob
+import os
+from caosdb.configuration import config_to_yaml, validate_yaml_schema
+from configparser import ConfigParser
+
+
+def test_config_files():
+ for fn in glob(os.path.join(os.path.dirname(__file__), "test_configs", "*.ini")):
+ print(f"Testing {fn}.")
+ c = ConfigParser()
+ c.read(fn)
+ print(config_to_yaml(c))
+ validate_yaml_schema(config_to_yaml(c))
+
+
+def test_broken_config_files():
+ for fn in glob(os.path.join(os.path.dirname(__file__), "broken_configs", "*.ini")):
+ print(f"Testing {fn}.")
+ with raises(ValidationError):
+ c = ConfigParser()
+ c.read(fn)
+ print(config_to_yaml(c))
+ validate_yaml_schema(config_to_yaml(c))
diff --git a/unittests/test_yamlapi.py b/unittests/test_yamlapi.py
new file mode 100644
index 0000000000000000000000000000000000000000..cdb1e0499890ee58d10ff7f102632e104ef60868
--- /dev/null
+++ b/unittests/test_yamlapi.py
@@ -0,0 +1,85 @@
+# -*- coding: utf-8 -*-
+#
+# This file is a part of the CaosDB Project.
+#
+# Copyright (C) 2021 IndiScale GmbH <info@indiscale.com>
+# Copyright (C) 2021 Alexander Kreft <akreft@trineo.org>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see <https://www.gnu.org/licenses/>.
+#
+
+import os
+import warnings
+import tempfile
+from caosdb.yamlapi import (append_sublist, kv_to_xml,
+ dict_to_xml, yaml_to_xml,
+ process, yaml_file_to_xml)
+
+with warnings.catch_warnings(record=True) as w:
+ # Cause all warnings to always be triggered.
+ warnings.simplefilter("always")
+
+ append_sublist(None, None, None)
+
+ assert issubclass(w[-1].category, DeprecationWarning)
+ assert "This function is deprecated" in str(w[-1].message)
+
+with warnings.catch_warnings(record=True) as w:
+ # Cause all warnings to always be triggered.
+ warnings.simplefilter("always")
+
+ kv_to_xml("None", "None")
+ assert len(w) == 1
+ assert issubclass(w[-1].category, DeprecationWarning)
+ assert "This function is deprecated" in str(w[-1].message)
+
+with warnings.catch_warnings(record=True) as w:
+ # Cause all warnings to always be triggered.
+ warnings.simplefilter("always")
+
+ dict_to_xml(None)
+
+ assert issubclass(w[-1].category, DeprecationWarning)
+ assert "This function is deprecated" in str(w[-1].message)
+
+with warnings.catch_warnings(record=True) as w:
+ # Cause all warnings to always be triggered.
+ warnings.simplefilter("always")
+
+ yaml_to_xml("None")
+
+ assert issubclass(w[-1].category, DeprecationWarning)
+ assert "This function is deprecated" in str(w[-1].message)
+
+with warnings.catch_warnings(record=True) as w:
+ # Cause all warnings to always be triggered.
+ warnings.simplefilter("always")
+
+ process("None")
+
+ assert issubclass(w[-1].category, DeprecationWarning)
+ assert "This function is deprecated" in str(w[-1].message)
+
+with warnings.catch_warnings(record=True) as w:
+ # Cause all warnings to always be triggered.
+ warnings.simplefilter("always")
+
+ with tempfile.TemporaryDirectory() as tmpdirname:
+ tmpfile = os.path.join(tmpdirname, 'yamlfile')
+ with open(tmpfile, 'w') as tf:
+ tf.write("")
+ yaml_file_to_xml(tmpfile)
+
+ assert issubclass(w[-1].category, DeprecationWarning)
+ assert "This function is deprecated" in str(w[-1].message)