diff --git a/.gitignore b/.gitignore
index b522b1da9176e59756bffe89cd4eafe0d751a23c..55fb3f0d1bc6c101704557da8f35d6e784b5ea89 100644
--- a/.gitignore
+++ b/.gitignore
@@ -15,4 +15,5 @@ build/
src/caosdb/version.py
# documentation
-_apidoc
\ No newline at end of file
+_apidoc
+*~
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 72b4381aebcaf5edd16c10c479df6ca908128ec6..0430a4f6b5ac08d4ab38f00bff78b845e11fb97e 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -39,38 +39,85 @@ stages:
code_style:
tags: [ docker ]
stage: code_style
+ needs: [ ]
script:
- - pycodestyle --count ./
+ - make style
allow_failure: true
+# pylint tests for pycaosdb
pylint:
tags: [ docker ]
stage: linting
+ needs: [ ]
script:
- - pylint --unsafe-load-any-extension=y -d all -e E,F src/caosdb/common
+ - make lint
allow_failure: true
+# run unit tests
+unittest_py3.7:
+ tags: [ docker ]
+ stage: test
+ needs: [ ]
+ image: python:3.7
+ script: &python_test_script
+ # Python docker has problems with tox and pip so use plain pytest here
+ - touch ~/.pycaosdb.ini
+ - pip install nose pytest pytest-cov python-dateutil jsonschema>=4.4.0
+ - pip install .
+ - python -m pytest unittests
-# pylint tests for pycaosdb
-test:
+unittest_py3.8:
tags: [ docker ]
stage: test
+ needs: [ ]
+ image: python:3.8
+ script: *python_test_script
+
+# This needs to be changed once Python 3.9 isn't the standard Python in Debian
+# anymore.
+unittest_py3.9:
+ tags: [ docker ]
+ stage: test
+ needs: [ ]
script:
+ # verify that this actually is Python 3.9
+ - python3 -c "import sys; assert sys.version.startswith('3.9')"
- touch ~/.pycaosdb.ini
- - tox -r
+ - make unittest
+
+
+unittest_py3.10:
+ tags: [ docker ]
+ stage: test
+ needs: [ ]
+ image: python:3.10
+ script: *python_test_script
+
+unittest_py3.11:
+ tags: [ docker ]
+ stage: test
+ needs: [ ]
+ image: python:3.11
+ script: *python_test_script
# Trigger building of server image and integration tests
trigger_build:
- tags: [ docker ]
stage: deploy
- script:
- - /usr/bin/curl -X POST
- -F token=$CI_JOB_TOKEN
- -F "variables[F_BRANCH]=$CI_COMMIT_REF_NAME"
- -F "variables[PYLIB]=$CI_COMMIT_REF_NAME"
- -F "variables[TriggerdBy]=PYLIB"
- -F "variables[TriggerdByHash]=$CI_COMMIT_SHORT_SHA"
- -F ref=$DEPLOY_REF https://gitlab.indiscale.com/api/v4/projects/14/trigger/pipeline
+ inherit:
+ variables:
+ - DEPLOY_REF
+ variables:
+ # Renaming variables.
+ F_BRANCH: $CI_COMMIT_REF_NAME
+ PYLIB: $CI_COMMIT_REF_NAME
+ TRIGGERED_BY_REPO: PYLIB
+ TRIGGERED_BY_REF: $CI_COMMIT_REF_NAME
+ TRIGGERED_BY_HASH: $CI_COMMIT_SHORT_SHA
+
+ trigger:
+ project: caosdb/src/caosdb-deploy
+ branch: $DEPLOY_REF
+ strategy: depend
# Build a docker image in which tests for this repository can run
build-testenv:
@@ -79,12 +126,12 @@ build-testenv:
stage: setup
only:
- schedules
- script:
+ script:
- cd unittests/docker
- docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY
# use here general latest or specific branch latest...
- docker pull $CI_REGISTRY_IMAGE|| true
- - docker build
+ - docker build
--pull
--build-arg COMMIT=$CI_COMMIT_SHORT_SHA
--cache-from $CI_REGISTRY_IMAGE
@@ -96,6 +143,7 @@ build-testenv:
pages_prepare: &pages_prepare
tags: [ cached-dind ]
stage: deploy
+ needs: [ code_style, pylint, unittest_py3.8, unittest_py3.9, unittest_py3.10 ]
only:
refs:
- /^release-.*$/i
diff --git a/.gitlab/issue_templates/Default.md b/.gitlab/issue_templates/Default.md
new file mode 100644
index 0000000000000000000000000000000000000000..aa1a65aca363b87aff50280e1a86824009d2098b
--- /dev/null
+++ b/.gitlab/issue_templates/Default.md
@@ -0,0 +1,28 @@
+## Summary
+
+*Please give a short summary of what the issue is.*
+
+## Expected Behavior
+
+*What did you expect how the software should behave?*
+
+## Actual Behavior
+
+*What did the software actually do?*
+
+## Steps to Reproduce the Problem
+
+*Please describe, step by step, how others can reproduce the problem. Please try these steps for yourself on a clean system.*
+
+1.
+2.
+3.
+
+## Specifications
+
+- Version: *Which version of this software?*
+- Platform: *Which operating system, which other relevant software versions?*
+
+## Possible fixes
+
+*Do you have ideas how the issue can be resolved?*
diff --git a/.gitlab/merge_request_templates/Default.md b/.gitlab/merge_request_templates/Default.md
index e43435cb31f415fc4f9c8447983b411627612ad7..3629e0ca3695000863d8c254516f64bf59a7bf60 100644
--- a/.gitlab/merge_request_templates/Default.md
+++ b/.gitlab/merge_request_templates/Default.md
@@ -1,31 +1,36 @@
# Summary
- Insert a meaningful description for this merge request here. What is the
- new/changed behavior? Which bug has been fixed? Are there related Issues?
+*Insert a meaningful description for this merge request here: What is the new/changed behavior?
+Which bug has been fixed? Are there related issues?*
+
# Focus
- Point the reviewer to the core of the code change. Where should they start
- reading? What should they focus on (e.g. security, performance,
- maintainability, user-friendliness, compliance with the specs, finding more
- corner cases, concrete questions)?
+*Point the reviewer to the core of the code change. Where should they start reading? What should
+they focus on (e.g. security, performance, maintainability, user-friendliness, compliance with the
+specs, finding more corner cases, concrete questions)?*
+
# Test Environment
- How to set up a test environment for manual testing?
+*How to set up a test environment for manual testing?*
+
# Check List for the Author
-Please, prepare your MR for a review. Be sure to write a summary and a
-focus and create gitlab comments for the reviewer. They should guide the
-reviewer through the changes, explain your changes and also point out open
-questions. For further good practices have a look at [our review
+Please, prepare your MR for a review. Be sure to write a summary and a focus and create gitlab
+comments for the reviewer. They should guide the reviewer through the changes, explain your changes
+and also point out open questions. For further good practices have a look at [our review
guidelines](https://gitlab.com/caosdb/caosdb/-/blob/dev/REVIEW_GUIDELINES.md)
- [ ] All automated tests pass
-- [ ] Reference related Issues
-- [ ] Up-to-date CHANGELOG.md
-- [ ] Add type hints in created/changed code
+- [ ] Reference related issues
+- [ ] Up-to-date CHANGELOG.md (or not necessary)
+- [ ] Up-to-date JSON schema (or not necessary)
+- [ ] Appropriate user and developer documentation (or not necessary)
+ - Update / write published documentation (`make doc`).
+ - How do I use the software? Assume "stupid" users.
+ - How do I develop or debug the software? Assume novice developers.
- [ ] Annotations in code (Gitlab comments)
- Intent of new code
- Problems with old code
@@ -34,14 +39,15 @@ guidelines](https://gitlab.com/caosdb/caosdb/-/blob/dev/REVIEW_GUIDELINES.md)
# Check List for the Reviewer
-
- [ ] I understand the intent of this MR
- [ ] All automated tests pass
-- [ ] Up-to-date CHANGELOG.md
-- [ ] The test environment setup works and the intended behavior is
- reproducible in the test environment
+- [ ] Up-to-date CHANGELOG.md (or not necessary)
+- [ ] Appropriate user and developer documentation (or not necessary), also in published
+ documentation.
+- [ ] The test environment setup works and the intended behavior is reproducible in the test
+ environment
- [ ] In-code documentation and comments are up-to-date.
-- [ ] Check: Are there spezifications? Are they satisfied?
+- [ ] Check: Are there specifications? Are they satisfied?
For further good practices have a look at [our review guidelines](https://gitlab.com/caosdb/caosdb/-/blob/dev/REVIEW_GUIDELINES.md).
diff --git a/CHANGELOG.md b/CHANGELOG.md
index e109ba5a37dcccabcaf2b341212def848e676a99..ebdfab4bc64e8640207e7e678cedb4bd1698fb98 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -9,10 +9,258 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
### Added ###
-- It is possible now to supply a password for caosdb_admin on the command line and also activate the user directly using "-c".
+* `Entity.remove_value_from_property` function that removes a given value from a
+ property and optionally removes the property if it is empty afterwards.
+
+### Changed ###
+
+* `_Messages` is now `Messages` and inherits from list instead of dict
+* `Message.__init__` signature changed and `type` defaults to "Info" now.
+* `Message.__eq__` changed. Equality is equality of `type`, `code`, and
+ `description` now.
+
+### Deprecated ###
+
+* The API of Messages has been simplified and some ways to interact with
+ messages have been deprecated. Warnings are raised correspondingly.
+* `Message.get_code`. Use the `code` property instead.
+
+### Removed ###
+
+### Fixed ###
+
+- Detection for cyclic references when converting entites using the high level API.
+
+### Security ###
+
+### Documentation ###
+
+## [0.12.0] - 2023-06-02 ##
+
+### Added ###
+
+- Added location argument to `src/caosdb/utils/checkFileSystemConsistency.py`
+- Entity getters: `get_entity_by_<name/id/path>`
+- Cached versions of entity getters and of `execute_query` (`cached_query`)
+
+### Deprecated ###
+
+- getOriginUrlIn, getDiffIn, getBranchIn, getCommitIn (formerly apiutils) have been
+ moved to caosdb.utils.git_utils
+
+### Fixed ###
+
+- Fixed `src/caosdb/utils/checkFileSystemConsistency.py`
+
+### Documentation ###
+
+* [#83](https://gitlab.com/caosdb/caosdb-pylib/-/issues/83) - Improved
+ documentation on adding REFERENCE properties, both in the docstring of
+ `Entity.add_property` and in the data-insertion tutorial.
+
+## [0.11.2] - 2023-03-14 ##
+
+### Fixed ###
+- root logger is no longer used to create warnings. Fixes undesired output in
+ stderr
+
+## [0.11.1] - 2023-03-07 ##
+(Florian Spreckelsen)
+
+### Changed ###
+
+* Renamed `caosdb.common.models._Parents` to `caosdb.common.models._ParentList`.
+
+### Fixed ###
+
+* [caosdb-pylib#90](https://gitlab.com/caosdb/caosdb-pylib/-/issues/90): `Entity.get_parents_recursively()` did not work for unretrieved parents.
+
+## [0.11.0] - 2023-01-19 ##
+(Florian Spreckelsen)
+
+### Added ###
+
+* `apiutils.EntityMergeConflictError` class for unresesolvable merge conflicts
+ when merging two entities
+* Re-introduced support for Python 3.7
+
+### Changed ###
+
+* `apiutils.merge_entities` now raises an `EntityMergeConflictError` in case of
+ unresolvable merge conflicts.
+
+### Fixed ###
+
+* [#82](https://gitlab.com/caosdb/caosdb-pylib/-/issues/82) Merging an entity
+ with properties with missing datatype leads to Exception - The correct
+ exception is raised in case of a missing LIST datatype.
+
+### Documentation ###
+
+* [Fixed](https://gitlab.com/caosdb/caosdb-pylib/-/issues/79)
+ `{action}_entity_permissions` help line.
+
+## [0.10.0] - 2022-11-14
+(Florian Spreckelsen)
+
+### Added ###
+
+* HTTP connections are allowed additionally to HTTPS connections.
+* Dependency on the `requests` package.
+* Dependency on the `python-dateutil` package.
+* `Connection.https_proxy` and `Connection.http_proxy` option of the
+ pycaosdb.ini and the `https_proxy` and `http_proxy` parameter of the
+ `configure_connection` function. See the documentation of the
+ latter for more information.
+ Note that the `HTTP_PROXY` and `HTTPS_PROXY` environment variables are
+ respected as well, unless overridden programmatically.
+* `apiutils.empty_diff` function that returns `True` if the diffs of two
+ entities found with the `compare_entitis` function are empty, `False`
+ otherwise.
+
+### Changed ###
+
+* `apiutils.compare_entities` now has an optional `compare_referenced_records`
+ argument to compare referenced Entities recursively (fomerly, only the
+ referenced Python objects would be compared). The default is `False` to
+ recover the original behavior.
+* `apiutils.merge_entities` now has an optional
+ `merge_references_with_empty_diffs` argument that determines whether a merge
+ of two entities will be performed if they reference identical records (w.r.t
+ th above `empty_diff` function). Formerly this would have caused a merge
+ conflict if the referenced record(s) were identical, but stored in different
+ Python objects.
+* `apiutils.merge_entities` now has an optional `force` argument (defaults to
+ `False`, i.e., the old behavior) which determines whether in case of merge
+ conflicts errors will be raised or the properties and attributes of entity A
+ will be overwritten by entity B.
+
+### Deprecated ###
+
+* `Connection.socket_proxy` option of the pycaosdb.ini. Please use
+ `Connection.https_proxy` or `Connection.http_proxy` instead. The deprecated
+ option will be removed with the next minor release.
+
+### Fixed ###
+
+* handling of special attributes (name, id, ...) in `apiutils.empty_diff`
+
+## [0.9.0] - 2022-10-24
+(Florian Spreckelsen)
+
+### Added ###
+
+* Add TimeZone class and parse the server's time zone in the Info response.
+
+### Fixed ###
+
+* [#141](https://gitlab.indiscale.com/caosdb/src/caosdb-pylib/-/issues/141)
+ `password_method = unauthenticated` not allowed by schema
+* Set PyYAML dependency back to PyYaml>=5.4.1 (from 6.0) for better
+ compatibility with docker-compose
+
+### Documentation ###
+
+* Added curator role permissions example to code gallery
+
+## [0.8.0] - 2022-07-12
+(Timm Fitschen)
+
+### Removed ###
+
+* Support for Python 3.6 and Python 3.7
+
+### Fixed ###
+
+* `read()` of MockupResponse returns now an appropriate type on modern systems
+* [caosdb-server#142](https://gitlab.com/caosdb/caosdb-server/-/issues/142)
+ Can't create users with dots in their user names
+
+## [0.7.4] - 2022-05-31
+(Florian Spreckelsen)
+
+### Fixed ###
+
+* [#64](https://gitlab.com/caosdb/caosdb-pylib/-/issues/64) Use `Dict[]` and
+ `List[]` from `typing` for type hinting instead of `dict[]` and `list[]` for
+ compatibility with Python<3.9.
+
+## [0.7.3] - 2022-05-03
+(Henrik tom Wörden)
+
+### Added ###
+
+- New function in apiutils that copies an Entity.
+- New EXPERIMENTAL module `high_level_api` which is a completely refactored version of
+ the old `high_level_api` from apiutils. Please see the included documentation for details.
+- `to_graphics` now has `no_shadow` option.
+
+### Changed ###
+
+- Added additional customization options to the plantuml module.
+- The to_graphics function in the plantuml module uses a temporary directory now for creating the output files.
+
+### Fixed ###
+
+* [#75](https://gitlab.indiscale.com/caosdb/src/caosdb-pylib/-/issues/75), [#103](https://gitlab.indiscale.com/caosdb/src/caosdb-pylib/-/issues/103) Fixed JSON schema to allow more sections, and correct requirements for
+ password method.
+
+## [0.7.2] - 2022-03-25 ##
+(Timm Fitschen)
+
+### Deprecated ###
+
+* In module `caosdb.apiutils`:
+ * `CaosDBPythonEntity` class
+ * `convert_to_entity` function
+ * `convert_to_python_object` function
+
+### Fixed ###
+
+* [caosdb-pylib#106](https://gitlab.indiscale.com/caosdb/src/caosdb-pylib/-/issues/106)
+ Parsing Error in class caosdb.common.models.ACL. This may lead to the
+ unintentional revocation of permissions for some users or roles during
+ updates. However, no additional permissions are being granted.
+
+### Documentation ###
+
+## [0.7.1] - 2022-03-11 ##
+(Daniel Hornung)
+
+### Documentation ###
+
+- `timeout` option in example pycaosdb.ini
+
+## [0.7.0] - 2022-01-21 ##
+
+### Added ###
+
+- Function in administration that generates passwords that comply with the rules.
+
+### Fixed ###
+
+- #90 compare_entities function in apiutils does not check units
+- #94 some special properties were not checked in compare_entities
+
+### Security ###
+
+## [0.6.1] - 2021-12-03 ##
+
+### Fixed ###
+
+- #50 keyring can be used as password input method again
+* #81 compare_entities from apiutils does not compare entity values
+
+## [0.6.0] - 2021-10-19 ##
+
+### Added ###
+
+- It is possible now to supply a password for caosdb_admin on the command line
+ and also activate the user directly using "-c".
* Added examples for complex data models to documentation
* extended apiutils with `resolve_reference(Property)`
* is_reference function for Properties
+* function `retrieve_substructure` that recursively adds connected entities.
### Changed ###
@@ -24,14 +272,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
* Entity.add_property and Entity.add_parent do not accept `**kwargs`-style
keywords anymore. Formerly known keywords have been refactored into named
parameters.
-* [#35](https://gitlab.com/caosdb/caosdb-pylib/-/issues/35) Loggers now use the name of
- the unit where they are called instead of a static name
+* [#35](https://gitlab.com/caosdb/caosdb-pylib/-/issues/35) Loggers now use the
+ name of the unit where they are called instead of a static name
### Deprecated ###
* `id_query(ids)` in apiutils (to be removed with >=0.5.4)
* The whole yamlapi with the following functions (to be removed with >=0.5.4):
- * `append_sublist`
+ * `append_sublist`
* `kv_to_xml`
* `dict_to_xml`
* `yaml_to_xml`
@@ -39,8 +287,6 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
* `yaml_file_to_xml`
* `insert_yaml_file`
-### Removed ###
-
### Fixed ###
* #60 Unintuitive behavior of `Entity.role` after a `Entity(id).retrieve()`
@@ -59,17 +305,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
* `etag` property for the `caosdb.Query` class. The etag allows to debug the
caching and to decide whether the server has changed between queries.
* function `_read_config_files` to read `pycaosdb.ini` files from different paths.
-* function `retrieve_substructure` that recursively adds connected entities.
### Changed ###
* Updated error-handling tutorial in documentation to reflect the new
error classes
-### Deprecated ###
-
-### Removed ###
-
### Fixed ###
* #45 - test_config_ini_via_envvar
@@ -115,7 +356,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
### Removed ###
-* Dynamic exception type `EntityMultiError`.
+* Dynamic exception type `EntityMultiError`.
* `get_something` functions from all error object in `exceptions.py`
* `AmbiguityException`
@@ -162,14 +403,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
assumed that `password_method` is `auth_token` if the the `auth_token` is
set.
-### Removed ###
-
### Fixed ###
- Replaced deprecated Logger.warn() method.
-### Security ###
-
## [0.3.0] - 2020-04-24##
### Added ###
@@ -177,10 +414,6 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
* `apiutils.apply_to_ids` -- a helper which applies a function to all ids which
are used by an entity (own entity, parents, properties, references etc.).
-### Changed ###
-
-### Deprecated ###
-
### Fixed ###
* import bugs in apiutils
diff --git a/CITATION.cff b/CITATION.cff
new file mode 100644
index 0000000000000000000000000000000000000000..d9126aae6483459f8c8f248ed6a4fdf859f24e45
--- /dev/null
+++ b/CITATION.cff
@@ -0,0 +1,25 @@
+cff-version: 1.2.0
+message: "If you use this software, please cite it as below."
+authors:
+ - family-names: Fitschen
+ given-names: Timm
+ orcid: https://orcid.org/0000-0002-4022-432X
+ - family-names: Schlemmer
+ given-names: Alexander
+ orcid: https://orcid.org/0000-0003-4124-9649
+ - family-names: Hornung
+ given-names: Daniel
+ orcid: https://orcid.org/0000-0002-7846-6375
+ - family-names: tom Wörden
+ given-names: Henrik
+ orcid: https://orcid.org/0000-0002-5549-578X
+ - family-names: Parlitz
+ given-names: Ulrich
+ orcid: https://orcid.org/0000-0003-3058-1435
+ - family-names: Luther
+ given-names: Stefan
+ orcid: https://orcid.org/0000-0001-7214-8125
+title: CaosDB - Pylib
+version: 0.12.0
+doi: 10.3390/data4020083
+date-released: 2023-06-02
\ No newline at end of file
diff --git a/DEPENDENCIES.md b/DEPENDENCIES.md
index 04e783a7fa31b1d5c3a600a1009c8f040db1620d..cccdbcba9df2cc63e08a93c8b99dc13c92211f7f 100644
--- a/DEPENDENCIES.md
+++ b/DEPENDENCIES.md
@@ -1,6 +1,5 @@
-* caosdb-server == 0.3
-* Python >= 3.5
+* caosdb-server >= 0.8.0
+* Python >= 3.8
* pip >= 20.0.2
-
-Any other dependencies are being installed via pip
+Any other dependencies are defined in the setup.py and are being installed via pip
diff --git a/FEATURES.md b/FEATURES.md
new file mode 100644
index 0000000000000000000000000000000000000000..977d7a482279af00bc5e2b02a13d5d23564f1d04
--- /dev/null
+++ b/FEATURES.md
@@ -0,0 +1,8 @@
+# Experimental Features
+
+- High Level API in the module `high_level_api` is experimental and might be removed in future. It is for playing around with a possible future implementation of the Python client. See `src/doc/future_caosdb.md`
+
+
+# Features
+TODO: This is currently an incomplete list.
+- `to_graphics` defined in `caosdb.utils.plantuml` can be used to create an UML diagram of a data model
diff --git a/Makefile b/Makefile
index 4bc3459d209936c17a445c64e77180d9559e4653..0a0888ad0484c0307583e139e65058c38574ed3a 100644
--- a/Makefile
+++ b/Makefile
@@ -31,3 +31,18 @@ doc:
install:
@echo "Not implemented yet, use pip for installation."
+
+check: style lint
+.PHONY: check
+
+style:
+ pycodestyle --count examples src unittests
+.PHONY: style
+
+lint:
+ pylint --unsafe-load-any-extension=y -d all -e E,F src/caosdb/common
+.PHONY: lint
+
+unittest:
+ tox -r
+.PHONY: unittest
diff --git a/README.md b/README.md
index 04b34cbc07c98e73740b13200ed83fe067af99d2..7215591a4f31f1946029442de291eb9ccf9beea1 100644
--- a/README.md
+++ b/README.md
@@ -39,7 +39,7 @@ By participating, you are expected to uphold our [Code of Conduct](https://gitla
* If you have a suggestion for the [documentation](https://docs.indiscale.com/caosdb-pylib/),
the preferred way is also a merge request as describe above (the documentation resides in `src/doc`).
However, you can also create an issue for it.
-* You can also contact us at **info (AT) caosdb.de** and join the
+* You can also contact us at **info (AT) caosdb.org** and join the
CaosDB community on
[#caosdb:matrix.org](https://matrix.to/#/!unwwlTfOznjEnMMXxf:matrix.org).
@@ -47,7 +47,7 @@ However, you can also create an issue for it.
* Copyright (C) 2018 Research Group Biomedical Physics, Max Planck Institute
for Dynamics and Self-Organization Göttingen.
-* Copyright (C) 2020-2021 Indiscale GmbH <info@indiscale.com>
+* Copyright (C) 2020-2022 Indiscale GmbH <info@indiscale.com>
All files in this repository are licensed under a [GNU Affero General Public
License](LICENCE.md) (version 3 or later).
diff --git a/README_SETUP.md b/README_SETUP.md
index e58f934ceba176e4b5ba42239565f8e3bd48171a..01eea85188078ae6f2fe226e89e5c227497b4bd0 100644
--- a/README_SETUP.md
+++ b/README_SETUP.md
@@ -4,7 +4,7 @@
### Requirements ###
-PyCaosDB needs at least Python 3.6. Additionally, the following packages are required (they will
+PyCaosDB needs at least Python 3.8. Additionally, the following packages are required (they will
typically be installed automatically):
- `lxml`
@@ -19,7 +19,7 @@ Optional packages:
#### Linux ####
-Make sure that Python (at least version 3.6) and pip is installed, using your system tools and
+Make sure that Python (at least version 3.8) and pip is installed, using your system tools and
documentation.
Then open a terminal and continue in the [Generic installation](#generic-installation) section.
@@ -82,60 +82,8 @@ pip3 install --user .[jsonschema]
## Configuration ##
-The configuration is done using `ini` configuration files.
-PyCaosDB tries to read from the inifile specified in the environment variable `PYCAOSDBINI` or
-alternatively in `~/.pycaosdb.ini` upon import. After that, the ini file `pycaosdb.ini` in the
-current working directory will be read additionally, if it exists.
-
-Here, we will look at the most common configuration options. For a full and
-comprehensive description please check out
-[pycaosdb.ini file](https://gitlab.com/caosdb/caosdb-pylib/-/blob/main/examples/pycaosdb.ini)
-You can download this file and use it as a starting point.
-
-
-Typically, you need to change at least the `url` and `username` fields as required.
-(Ask your CaosDB administrator or IT crowd if
-you do not know what to put there, but for the demo instances https://demo.indiscale.com, `username=admin`
-and `password=caosdb` should work).
-
-### Authentication ###
-
-The default configuration (that your are asked for your password when ever a connection is created
-can be changed by setting `password_method`:
-
-* with `password_method=input` password (and possibly user) will be queried on demand (**default**)
-* use the password manager [pass](https://www.passwordstore.org) by using `pass` as value, see also the [ArchWiki
- entry](https://wiki.archlinux.org/index.php/Pass#Basic_usage). This also requires `password_identifier` which refers to the identifier within pass
- for the desired password.
-* install the python package [keyring](https://pypi.org/project/keyring), to use the system keyring/wallet (macOS, GNOME, KDE,
- Windows). The password will be queried on first usage.
-* with `password_method=plain` (**strongly discouraged**)
-
-The following illustrates the recommended options:
-
-```ini
-[Connection]
-# using "pass" password manager
-#password_method=pass
-#password_identifier=...
-
-# using the system keyring/wallet (macOS, GNOME, KDE, Windows)
-#password_method=keyring
-```
-
-### SSL Certificate ###
-In some cases (especially if you are testing CaosDB) you might need to supply
-an SSL certificate to allow SSL encryption.
-
-```ini
-[Connection]
-cacert=/path/to/caosdb.ca.pem
-```
-
-### Further Settings ###
-As mentioned above, a complete list of options can be found in the
-[pycaosdb.ini file](https://gitlab.com/caosdb/caosdb-pylib/-/blob/main/examples/pycaosdb.ini) in
-the examples folder of the source code.
+The configuration is done using `ini` configuration files. The content of these configuration files
+is described in detail in the [configuration section of the documentation](https://docs.indiscale.com/caosdb-pylib/configuration.html).
## Try it out ##
@@ -155,9 +103,14 @@ like this, check out the "Authentication" section in the [configuration document
Now would be a good time to continue with the [tutorials](tutorials/index).
## Run Unit Tests
-tox
+
+- Run all tests: `tox` or `make unittest`
+- Run a specific test file: e.g. `tox -- unittests/test_schema.py`
+- Run a specific test function: e.g. `tox -- unittests/test_schema.py::test_config_files`
## Documentation ##
+We use sphinx to create the documentation. Docstrings in the code should comply
+with the Googly style (see link below).
Build documentation in `build/` with `make doc`.
@@ -167,5 +120,11 @@ Build documentation in `build/` with `make doc`.
- `sphinx-autoapi`
- `recommonmark`
+### How to contribute ###
+
+- [Google Style Python Docstrings](https://sphinxcontrib-napoleon.readthedocs.io/en/latest/example_google.html)
+- [Google Style Python Docstrings 2nd reference](https://github.com/google/styleguide/blob/gh-pages/pyguide.md#38-comments-and-docstrings)
+- [References to other documentation](https://www.sphinx-doc.org/en/master/usage/extensions/intersphinx.html#role-external)
+
### Troubleshooting ###
If the client is to be executed directly from the `/src` folder, an initial `.\setup.py install --user` must be called.
diff --git a/RELEASE_GUIDELINES.md b/RELEASE_GUIDELINES.md
index e015b598117abdcd575cf17e2f095fec459a4c4c..95ee8e314871153476c30790a456242e38dcaf9e 100644
--- a/RELEASE_GUIDELINES.md
+++ b/RELEASE_GUIDELINES.md
@@ -1,6 +1,6 @@
# Release Guidelines for the CaosDB Python Client Library
-This document specifies release guidelines in addition to the generel release
+This document specifies release guidelines in addition to the general release
guidelines of the CaosDB Project
([RELEASE_GUIDELINES.md](https://gitlab.com/caosdb/caosdb/blob/dev/RELEASE_GUIDELINES.md))
@@ -20,9 +20,11 @@ guidelines of the CaosDB Project
3. Check all general prerequisites.
-4. Prepare [setup.py](./setup.py): Check the `MAJOR`, `MINOR`, `MICRO`, `PRE`
- variables and set `ISRELEASED` to `True`. Use the possibility to issue
- pre-release versions for testing.
+4. Update the version:
+ - `version` variables in `src/doc/conf.py`
+ - Version on [setup.py](./setup.py): Check the `MAJOR`, `MINOR`, `MICRO`, `PRE` variables and set
+ `ISRELEASED` to `True`. Use the possibility to issue pre-release versions for testing.
+ - `CITATION.cff` (update version and date)
5. Merge the release branch into the main branch.
@@ -35,8 +37,13 @@ guidelines of the CaosDB Project
9. Publish the release by executing `./release.sh` with uploads the caosdb
module to the Python Package Index [pypi.org](https://pypi.org).
-10. Merge the main branch back into the dev branch.
+10. Create a gitlab release on gitlab.indiscale.com and gitlab.com
-11. After the merge of main to dev, start a new development version by
- setting `ISRELEASED` to `False` and by increasing at least the `MIRCO`
- version in [setup.py](./setup.py)
+11. Merge the main branch back into the dev branch.
+
+12. After the merge of main to dev, start a new development version by setting
+ `ISRELEASED` to `False` and by increasing at least the `MICRO` version in
+ [setup.py](./setup.py). Please note that due to a bug in pip, the `PRE`
+ version has to remain empty in the setup.py.
+ Also update CHANGELOG.md (new "Unreleased" section). Also update
+ `src/doc/conf.py`.
diff --git a/examples/pycaosdb.ini b/examples/pycaosdb.ini
index edc32195fbb364bb355d67b8733e8c7bccbb0d34..8cf74e43c5db32ed139c4fe371a6c2b3831b2ee1 100644
--- a/examples/pycaosdb.ini
+++ b/examples/pycaosdb.ini
@@ -67,3 +67,6 @@
# This option is used internally and for testing. Do not override.
# implementation=_DefaultCaosDBServerConnection
+
+# The timeout for requests to the server.
+# timeout=1000
diff --git a/pytest.ini b/pytest.ini
deleted file mode 100644
index ca6aad829a3e0607292cf69b8b1d4b7f7758993e..0000000000000000000000000000000000000000
--- a/pytest.ini
+++ /dev/null
@@ -1,3 +0,0 @@
-[pytest]
-testpaths=unittests
-addopts=-x -vv --cov=caosdb
diff --git a/release.sh b/release.sh
index 1af097f014de6cd9eb3d3e8ba5da34aea0fe1671..f6335ae20d0c29e760b508aac831a35460a59ef3 100755
--- a/release.sh
+++ b/release.sh
@@ -1,4 +1,4 @@
#!/bin/bash
rm -rf dist/ build/ .eggs/
python setup.py sdist bdist_wheel
-python -m twine upload -s dist/*
+python -m twine upload dist/*
diff --git a/setup.py b/setup.py
index d0e12f1a88f43883ede5e53c90512968ca310e93..8fdf3b1c63322ec48af398d1dcb1c4028355d473 100755
--- a/setup.py
+++ b/setup.py
@@ -45,11 +45,15 @@ from setuptools import find_packages, setup
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
########################################################################
-MAJOR = 0
-MINOR = 5
-MICRO = 3
-PRE = "" # e.g. rc0, alpha.1, 0.beta-23
ISRELEASED = False
+MAJOR = 0
+MINOR = 12
+MICRO = 1
+# Do not tag as pre-release until this commit
+# https://github.com/pypa/packaging/pull/515
+# has made it into a release. Probably we should wait for pypa/packaging>=21.4
+# https://github.com/pypa/packaging/releases
+PRE = "" # "dev" # e.g. rc0, alpha.1, 0.beta-23
if PRE:
VERSION = "{}.{}.{}-{}".format(MAJOR, MINOR, MICRO, PRE)
@@ -93,6 +97,9 @@ def get_version_info():
if os.path.exists('.git'):
GIT_REVISION = git_version()
+ elif os.path.exists('caosdb_pylib_commit'):
+ with open('caosdb_pylib_commit', 'r') as f:
+ GIT_REVISION = f.read().strip()
elif os.path.exists('src/caosdb/version.py'):
# must be a source distribution, use existing version file
try:
@@ -154,15 +161,29 @@ def setup_package():
long_description_content_type="text/markdown",
author='Timm Fitschen',
author_email='t.fitschen@indiscale.com',
+ url='https://www.caosdb.org',
+ license="AGPLv3+",
+ classifiers=[
+ "Programming Language :: Python :: 3",
+ "License :: OSI Approved :: GNU Affero General Public License v3 or later (AGPLv3+)",
+ "Operating System :: OS Independent",
+ "Topic :: Database",
+ "Topic :: Scientific/Engineering :: Information Analysis",
+ ],
packages=find_packages('src'),
- python_requires='>=3.6',
+ python_requires='>=3.7',
package_dir={'': 'src'},
- install_requires=['lxml>=3.6.4',
- 'PyYaml>=3.12', 'future', 'PySocks>=1.6.7'],
+ install_requires=['lxml>=4.6.3',
+ "requests[socks]>=2.26",
+ "python-dateutil>=2.8.2",
+ 'PyYAML>=5.4.1',
+ 'future',
+ ],
extras_require={'keyring': ['keyring>=13.0.0'],
- 'jsonschema': ['jsonschema==4.0.1']},
+ 'jsonschema': ['jsonschema>=4.4.0']},
setup_requires=["pytest-runner>=2.0,<3dev"],
- tests_require=["pytest", "pytest-cov", "coverage>=4.4.2", "jsonschema==4.0.1"],
+ tests_require=["pytest", "pytest-cov", "coverage>=4.4.2",
+ "jsonschema>=4.4.0"],
package_data={
'caosdb': ['cert/indiscale.ca.crt', 'schema-pycaosdb-ini.yml'],
},
diff --git a/src/caosdb/__init__.py b/src/caosdb/__init__.py
index 7e06885fe495c1e8c4ccc99b7d0c0f8ff8c34b5b..acf323e860a93753b57f2e104531383b412f3fa0 100644
--- a/src/caosdb/__init__.py
+++ b/src/caosdb/__init__.py
@@ -46,6 +46,7 @@ from caosdb.common.models import (ACL, ALL, FIX, NONE, OBLIGATORY, RECOMMENDED,
Query, QueryTemplate, Record, RecordType,
delete, execute_query, get_global_acl,
get_known_permissions, raise_errors)
+from caosdb.utils.get_entity import get_entity_by_name, get_entity_by_path, get_entity_by_id
from caosdb.configuration import _read_config_files, configure, get_config
from caosdb.connection.connection import configure_connection, get_connection
from caosdb.exceptions import *
diff --git a/src/caosdb/apiutils.py b/src/caosdb/apiutils.py
index b3d5df698baceacf671c5171430dfa48b3c881b5..a46e30375b924d358448e73aece61562c36c700b 100644
--- a/src/caosdb/apiutils.py
+++ b/src/caosdb/apiutils.py
@@ -1,12 +1,11 @@
# -*- coding: utf-8 -*-
#
-# ** header v3.0
# This file is a part of the CaosDB Project.
#
# Copyright (C) 2018 Research Group Biomedical Physics,
# Max-Planck-Institute for Dynamics and Self-Organization Göttingen
# Copyright (C) 2020 Timm Fitschen <t.fitschen@indiscale.com>
-# Copyright (C) 2020 IndiScale GmbH <info@indiscale.com>
+# Copyright (C) 2020-2022 IndiScale GmbH <info@indiscale.com>
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as
@@ -23,22 +22,32 @@
#
# ** end header
#
-"""API-Utils:
+"""API-Utils: Some simplified functions for generation of records etc.
-Some simplified functions for generation of records etc.
"""
-import sys
-import tempfile
-from collections.abc import Iterable
+import logging
import warnings
-from subprocess import call
-from caosdb.common.datatype import (BOOLEAN, DATETIME, DOUBLE, FILE, INTEGER,
- REFERENCE, TEXT, is_reference)
-from caosdb.common.models import (Container, Entity, File, Property, Query,
- Record, RecordType, get_config,
- execute_query)
+from collections.abc import Iterable
+from typing import Any, Dict, List
+
+from caosdb.common.datatype import is_reference
+from caosdb.common.models import (Container, Entity, File, Property,
+ Record, RecordType, execute_query,
+ SPECIAL_ATTRIBUTES)
+from caosdb.exceptions import CaosDBException
+
+from caosdb.utils.git_utils import (get_origin_url_in, get_diff_in,
+ get_branch_in, get_commit_in)
+
+logger = logging.getLogger(__name__)
+
+
+class EntityMergeConflictError(CaosDBException):
+ """An error that is raised in case of an unresolvable conflict when merging
+ two entities.
+ """
def new_record(record_type, name=None, description=None,
@@ -98,22 +107,6 @@ def create_id_query(ids):
["ID={}".format(id) for id in ids])
-def retrieve_entity_with_id(eid):
- return execute_query("FIND ENTITY WITH ID={}".format(eid), unique=True)
-
-
-def retrieve_entities_with_ids(entities):
- collection = Container()
- step = 20
-
- for i in range(len(entities)//step+1):
- collection.extend(
- execute_query(
- create_id_query(entities[i*step:(i+1)*step])))
-
- return collection
-
-
def get_type_of_entity_with(id_):
objs = retrieve_entities_with_ids([id_])
@@ -137,438 +130,95 @@ def get_type_of_entity_with(id_):
return Entity
-class CaosDBPythonEntity(object):
-
- _last_id = 0
-
- def __init__(self):
- # Save a copy of the dry state
- # of this object in order to be
- # able to detect conflicts.
- self.do_not_expand = False
- self._parents = []
- self._id = CaosDBPythonEntity._get_id()
- self._path = None
- self._file = None
- self.pickup = None
- # TODO:
- # 3.) resolve references up to a specific depth (including infinity)
- # 4.) resolve parents function -> partially implemented by
- # get_parent_names
- self._references = {}
- self._properties = set()
- self._datatypes = {}
- self._forbidden = dir(self)
-
- @staticmethod
- def _get_id():
- CaosDBPythonEntity._last_id -= 1
-
- return CaosDBPythonEntity._last_id
-
- def _set_property_from_entity(self, ent):
- name = ent.name
- val = ent.value
- pr = ent.datatype
- val, reference = self._type_converted_value(val, pr)
- self.set_property(name, val, reference, datatype=pr)
-
- def set_property(self, name, value, is_reference=False,
- overwrite=False, datatype=None):
- """
- overwrite: Use this if you definitely only want one property with that name (set to True).
- """
- self._datatypes[name] = datatype
-
- if isinstance(name, Entity):
- name = name.name
-
- if name in self._forbidden:
- raise RuntimeError("Entity cannot be converted to a corresponding "
- "Python representation. Name of property " +
- name + " is forbidden!")
- already_exists = (name in dir(self))
-
- if already_exists and not overwrite:
- # each call to _set_property checks first if it already exists
- # if yes: Turn the attribute into a list and
- # place all the elements into that list.
- att = self.__getattribute__(name)
-
- if isinstance(att, list):
- pass
- else:
- old_att = self.__getattribute__(name)
- self.__setattr__(name, [old_att])
-
- if is_reference:
- self._references[name] = [
- self._references[name]]
- att = self.__getattribute__(name)
- att.append(value)
-
- if is_reference:
- self._references[name].append(int(value))
- else:
- if is_reference:
- self._references[name] = value
- self.__setattr__(name, value)
-
- if not (already_exists and overwrite):
- self._properties.add(name)
-
- add_property = set_property
-
- def set_id(self, idx):
- self._id = idx
-
- def _type_converted_list(self, val, pr):
- """Convert a list to a python list of the correct type."""
- prrealpre = pr.replace("<", "<").replace(">", ">")
- prreal = prrealpre[prrealpre.index("<") + 1:prrealpre.rindex(">")]
- lst = [self._type_converted_value(i, prreal) for i in val]
-
- return ([i[0] for i in lst], lst[0][1])
-
- def _type_converted_value(self, val, pr):
- """Convert val to the correct type which is indicated by the database
- type string in pr.
-
- Returns a tuple with two entries:
- - the converted value
- - True if the value has to be interpreted as an id acting as a reference
- """
-
- if val is None:
- return (None, False)
- elif pr == DOUBLE:
- return (float(val), False)
- elif pr == BOOLEAN:
- return (bool(val), False)
- elif pr == INTEGER:
- return (int(val), False)
- elif pr == TEXT:
- return (val, False)
- elif pr == FILE:
- return (int(val), False)
- elif pr == REFERENCE:
- return (int(val), True)
- elif pr == DATETIME:
- return (val, False)
- elif pr[0:4] == "LIST":
- return self._type_converted_list(val, pr)
- elif isinstance(val, Entity):
- return (convert_to_python_object(val), False)
- else:
- return (int(val), True)
-
- def attribute_as_list(self, name):
- """This is a workaround for the problem that lists containing only one
- element are indistinguishable from simple types in this
- representation."""
- att = self.__getattribute__(name)
-
- if isinstance(att, list):
- return att
- else:
- return [att]
-
- def _add_parent(self, parent):
- self._parents.append(parent.id)
-
- def add_parent(self, parent_id=None, parent_name=None):
- if parent_id is not None:
- self._parents.append(parent_id)
- elif parent_name is not None:
- self._parents.append(parent_name)
- else:
- raise ValueError("no parent identifier supplied")
-
- def get_parent_names(self):
- new_plist = []
-
- for p in self._parents:
- obj_type = get_type_of_entity_with(p)
- ent = obj_type(id=p).retrieve()
- new_plist.append(ent.name)
-
- return new_plist
-
- def resolve_references(self, deep=False, visited=dict()):
- for i in self._references:
- if isinstance(self._references[i], list):
- for j in range(len(self._references[i])):
- new_id = self._references[i][j]
- obj_type = get_type_of_entity_with(new_id)
-
- if new_id in visited:
- new_object = visited[new_id]
- else:
- ent = obj_type(id=new_id).retrieve()
- new_object = convert_to_python_object(ent)
- visited[new_id] = new_object
-
- if deep:
- new_object.resolve_references(deep, visited)
- self.__getattribute__(i)[j] = new_object
- else:
- new_id = self._references[i]
- obj_type = get_type_of_entity_with(new_id)
-
- if new_id in visited:
- new_object = visited[new_id]
- else:
- ent = obj_type(id=new_id).retrieve()
- new_object = convert_to_python_object(ent)
- visited[new_id] = new_object
-
- if deep:
- new_object.resolve_references(deep, visited)
- self.__setattr__(i, new_object)
-
- def __str__(self, indent=1, name=None):
- if name is None:
- result = str(self.__class__.__name__) + "\n"
- else:
- result = name + "\n"
-
- for p in self._properties:
- value = self.__getattribute__(p)
-
- if isinstance(value, CaosDBPythonEntity):
- result += indent * "\t" + \
- value.__str__(indent=indent + 1, name=p)
- else:
- result += indent * "\t" + p + "\n"
-
- return result
-
-
-class CaosDBPythonRecord(CaosDBPythonEntity):
- pass
-
-
-class CaosDBPythonRecordType(CaosDBPythonEntity):
- pass
-
-
-class CaosDBPythonProperty(CaosDBPythonEntity):
- pass
-
-
-class CaosDBPythonFile(CaosDBPythonEntity):
- def get_File(self, target=None):
- f = File(id=self._id).retrieve()
- self._file = f.download(target)
-
-
-def _single_convert_to_python_object(robj, entity):
- robj._id = entity.id
-
- for i in entity.properties:
- robj._set_property_from_entity(i)
-
- for i in entity.parents:
- robj._add_parent(i)
-
- if entity.path is not None:
- robj._path = entity.path
-
- if entity.file is not None:
- robj._file = entity.file
- # if entity.pickup is not None:
- # robj.pickup = entity.pickup
-
- return robj
-
-
-def _single_convert_to_entity(entity, robj, **kwargs):
- if robj._id is not None:
- entity.id = robj._id
-
- if robj._path is not None:
- entity.path = robj._path
-
- if robj._file is not None:
- entity.file = robj._file
-
- if robj.pickup is not None:
- entity.pickup = robj.pickup
- children = []
-
- for parent in robj._parents:
- if sys.version_info[0] < 3:
- if hasattr(parent, "encode"):
- entity.add_parent(name=parent)
- else:
- entity.add_parent(id=parent)
- else:
- if hasattr(parent, "encode"):
- entity.add_parent(name=parent)
- else:
- entity.add_parent(id=parent)
-
- def add_property(entity, prop, name, recursive=False, datatype=None):
- if datatype is None:
- raise ArgumentError("datatype must not be None")
-
- if isinstance(prop, CaosDBPythonEntity):
- entity.add_property(name=name, value=str(
- prop._id), datatype=datatype)
-
- if recursive and not prop.do_not_expand:
- return convert_to_entity(prop, recursive=recursive)
- else:
- return []
- else:
- if isinstance(prop, float) or isinstance(prop, int):
- prop = str(prop)
- entity.add_property(name=name, value=prop, datatype=datatype)
-
- return []
+def retrieve_entity_with_id(eid):
+ return execute_query("FIND ENTITY WITH ID={}".format(eid), unique=True)
- for prop in robj._properties:
- value = robj.__getattribute__(prop)
- if isinstance(value, list):
- if robj._datatypes[prop][0:4] == "LIST":
- lst = []
+def retrieve_entities_with_ids(entities):
+ collection = Container()
+ step = 20
- for v in value:
- if isinstance(v, CaosDBPythonEntity):
- lst.append(v._id)
+ for i in range(len(entities)//step+1):
+ collection.extend(
+ execute_query(
+ create_id_query(entities[i*step:(i+1)*step])))
- if recursive and not v.do_not_expand:
- children.append(convert_to_entity(
- v, recursive=recursive))
- else:
- if isinstance(v, float) or isinstance(v, int):
- lst.append(str(v))
- else:
- lst.append(v)
- entity.add_property(name=prop, value=lst,
- datatype=robj._datatypes[prop])
- else:
- for v in value:
- children.extend(
- add_property(
- entity,
- v,
- prop,
- datatype=robj._datatypes[prop],
- **kwargs))
- else:
- children.extend(
- add_property(
- entity,
- value,
- prop,
- datatype=robj._datatypes[prop],
- **kwargs))
-
- return [entity] + children
-
-
-def convert_to_entity(python_object, **kwargs):
- if isinstance(python_object, Container):
- # Create a list of objects:
-
- return [convert_to_python_object(i, **kwargs) for i in python_object]
- elif isinstance(python_object, CaosDBPythonRecord):
- return _single_convert_to_entity(Record(), python_object, **kwargs)
- elif isinstance(python_object, CaosDBPythonFile):
- return _single_convert_to_entity(File(), python_object, **kwargs)
- elif isinstance(python_object, CaosDBPythonRecordType):
- return _single_convert_to_entity(RecordType(), python_object, **kwargs)
- elif isinstance(python_object, CaosDBPythonProperty):
- return _single_convert_to_entity(Property(), python_object, **kwargs)
- elif isinstance(python_object, CaosDBPythonEntity):
- return _single_convert_to_entity(Entity(), python_object, **kwargs)
- else:
- raise ValueError("Cannot convert an object of this type.")
-
-
-def convert_to_python_object(entity):
- """"""
-
- if isinstance(entity, Container):
- # Create a list of objects:
-
- return [convert_to_python_object(i) for i in entity]
- elif isinstance(entity, Record):
- return _single_convert_to_python_object(CaosDBPythonRecord(), entity)
- elif isinstance(entity, RecordType):
- return _single_convert_to_python_object(
- CaosDBPythonRecordType(), entity)
- elif isinstance(entity, File):
- return _single_convert_to_python_object(CaosDBPythonFile(), entity)
- elif isinstance(entity, Property):
- return _single_convert_to_python_object(CaosDBPythonProperty(), entity)
- elif isinstance(entity, Entity):
- return _single_convert_to_python_object(CaosDBPythonEntity(), entity)
- else:
- raise ValueError("Cannot convert an object of this type.")
+ return collection
def getOriginUrlIn(folder):
- """return the Fetch URL of the git repository in the given folder."""
- with tempfile.NamedTemporaryFile(delete=False, mode="w") as t:
- call(["git", "remote", "show", "origin"], stdout=t, cwd=folder)
- with open(t.name, "r") as t:
- urlString = "Fetch URL:"
-
- for line in t.readlines():
- if urlString in line:
- return line[line.find(urlString) + len(urlString):].strip()
-
- return None
+ warnings.warn("""
+ This function is deprecated and will be removed with the next release.
+ Please use the caosdb.utils.git_utils.get_origin_url_in instead.""",
+ DeprecationWarning)
+ return get_origin_url_in(folder)
def getDiffIn(folder, save_dir=None):
- """returns the name of a file where the out put of "git diff" in the given
- folder is stored."""
- with tempfile.NamedTemporaryFile(delete=False, mode="w", dir=save_dir) as t:
- call(["git", "diff"], stdout=t, cwd=folder)
-
- return t.name
+ warnings.warn("""
+ This function is deprecated and will be removed with the next release.
+ Please use the caosdb.utils.git_utils.get_diff_in instead.""",
+ DeprecationWarning)
+ return get_diff_in(folder, save_dir)
def getBranchIn(folder):
- """returns the current branch of the git repository in the given folder.
-
- The command "git branch" is called in the given folder and the
- output is returned
- """
- with tempfile.NamedTemporaryFile(delete=False, mode="w") as t:
- call(["git", "rev-parse", "--abbrev-ref", "HEAD"], stdout=t, cwd=folder)
- with open(t.name, "r") as t:
- return t.readline().strip()
+ warnings.warn("""
+ This function is deprecated and will be removed with the next release.
+ Please use the caosdb.utils.git_utils.get_branch_in instead.""",
+ DeprecationWarning)
+ return get_branch_in(folder)
def getCommitIn(folder):
- """returns the commit hash in of the git repository in the given folder.
+ warnings.warn("""
+ This function is deprecated and will be removed with the next release.
+ Please use the caosdb.utils.git_utils.get_commit_in instead.""",
+ DeprecationWarning)
+ return get_commit_in(folder)
- The command "git log -1 --format=%h" is called in the given folder
- and the output is returned
- """
- with tempfile.NamedTemporaryFile(delete=False, mode="w") as t:
- call(["git", "log", "-1", "--format=%h"], stdout=t, cwd=folder)
- with open(t.name, "r") as t:
- return t.readline().strip()
+def compare_entities(old_entity: Entity, new_entity: Entity, compare_referenced_records: bool = False):
+ """Compare two entites.
+ Return a tuple of dictionaries, the first index belongs to additional information for old
+ entity, the second index belongs to additional information for new entity.
-COMPARED = ["name", "role", "datatype", "description", "importance"]
+ Additional information means in detail:
+ - Additional parents (a list under key "parents")
+ - Information about properties:
+ - Each property lists either an additional property or a property with a changed:
+ - datatype
+ - importance or
+ - value (not implemented yet)
+ In case of changed information the value listed under the respective key shows the
+ value that is stored in the respective entity.
-def compare_entities(old_entity, new_entity):
- olddiff = {"properties": {}, "parents": []}
- newdiff = {"properties": {}, "parents": []}
+ If `compare_referenced_records` is `True`, also referenced entities will be
+ compared using this function (which is then called with
+ `compare_referenced_records = False` to prevent infinite recursion in case
+ of circular references).
+
+ Parameters
+ ----------
+ old_entity, new_entity : Entity
+ Entities to be compared
+ compare_referenced_records : bool, optional
+ Whether to compare referenced records in case of both, `old_entity` and
+ `new_entity`, have the same reference properties and both have a Record
+ object as value. If set to `False`, only the corresponding Python
+ objects are compared which may lead to unexpected behavior when
+ identical records are stored in different objects. Default is False.
+
+ """
+ olddiff: Dict[str, Any] = {"properties": {}, "parents": []}
+ newdiff: Dict[str, Any] = {"properties": {}, "parents": []}
if old_entity is new_entity:
return (olddiff, newdiff)
- for attr in COMPARED:
+ for attr in SPECIAL_ATTRIBUTES:
try:
oldattr = old_entity.__getattribute__(attr)
old_entity_attr_exists = True
@@ -616,20 +266,49 @@ def compare_entities(old_entity, new_entity):
newdiff["properties"][prop.name]["importance"] = \
new_entity.get_importance(prop.name)
- if ((prop.datatype is not None and
- matching[0].datatype is not None) and
- (prop.datatype != matching[0].datatype)):
+ if (prop.datatype != matching[0].datatype):
olddiff["properties"][prop.name]["datatype"] = prop.datatype
newdiff["properties"][prop.name]["datatype"] = \
matching[0].datatype
+ if (prop.unit != matching[0].unit):
+ olddiff["properties"][prop.name]["unit"] = prop.unit
+ newdiff["properties"][prop.name]["unit"] = \
+ matching[0].unit
+
+ if (prop.value != matching[0].value):
+ # basic comparison of value objects says they are different
+ same_value = False
+ if compare_referenced_records:
+ # scalar reference
+ if isinstance(prop.value, Entity) and isinstance(matching[0].value, Entity):
+ # explicitely not recursive to prevent infinite recursion
+ same_value = empty_diff(
+ prop.value, matching[0].value, compare_referenced_records=False)
+ # list of references
+ elif isinstance(prop.value, list) and isinstance(matching[0].value, list):
+ # all elements in both lists actually are entity objects
+ # TODO: check, whether mixed cases can be allowed or should lead to an error
+ if all([isinstance(x, Entity) for x in prop.value]) and all([isinstance(x, Entity) for x in matching[0].value]):
+ # can't be the same if the lengths are different
+ if len(prop.value) == len(matching[0].value):
+ # do a one-by-one comparison; the values are the same, if all diffs are empty
+ same_value = all(
+ [empty_diff(x, y, False) for x, y in zip(prop.value, matching[0].value)])
+
+ if not same_value:
+ olddiff["properties"][prop.name]["value"] = prop.value
+ newdiff["properties"][prop.name]["value"] = \
+ matching[0].value
+
if (len(newdiff["properties"][prop.name]) == 0
and len(olddiff["properties"][prop.name]) == 0):
newdiff["properties"].pop(prop.name)
olddiff["properties"].pop(prop.name)
else:
- raise NotImplementedError()
+ raise NotImplementedError(
+ "Comparison not implemented for multi-properties.")
for prop in new_entity.properties:
if len([0 for p in old_entity.properties if p.name == prop.name]) == 0:
@@ -648,10 +327,149 @@ def compare_entities(old_entity, new_entity):
return (olddiff, newdiff)
+def empty_diff(old_entity: Entity, new_entity: Entity, compare_referenced_records: bool = False):
+ """Check whether the `compare_entities` found any differences between
+ old_entity and new_entity.
+
+ Parameters
+ ----------
+ old_entity, new_entity : Entity
+ Entities to be compared
+ compare_referenced_records : bool, optional
+ Whether to compare referenced records in case of both, `old_entity` and
+ `new_entity`, have the same reference properties and both have a Record
+ object as value.
+
+ """
+ olddiff, newdiff = compare_entities(
+ old_entity, new_entity, compare_referenced_records)
+ for diff in [olddiff, newdiff]:
+ for key in ["parents", "properties"]:
+ if len(diff[key]) > 0:
+ # There is a difference somewhere in the diff
+ return False
+ for key in SPECIAL_ATTRIBUTES:
+ if key in diff and diff[key]:
+ # There is a difference in at least one special attribute
+ return False
+ # all elements of the two diffs were empty
+ return True
+
+
+def merge_entities(entity_a: Entity, entity_b: Entity, merge_references_with_empty_diffs=True,
+ force=False):
+ """Merge entity_b into entity_a such that they have the same parents and properties.
+
+ datatype, unit, value, name and description will only be changed in entity_a
+ if they are None for entity_a and set for entity_b. If there is a
+ corresponding value for entity_a different from None, an
+ EntityMergeConflictError will be raised to inform about an unresolvable merge
+ conflict.
+
+ The merge operation is done in place.
+
+ Returns entity_a.
+
+ WARNING: This function is currently experimental and insufficiently tested. Use with care.
+
+ Parameters
+ ----------
+ entity_a, entity_b : Entity
+ The entities to be merged. entity_b will be merged into entity_a in place
+ merge_references_with_empty_diffs : bool, optional
+ Whether the merge is performed if entity_a and entity_b both reference
+ record(s) that may be different Python objects but have empty diffs. If
+ set to `False` a merge conflict will be raised in this case
+ instead. Default is True.
+ force : bool, optional
+ If True, in case `entity_a` and `entity_b` have the same properties, the
+ values of `entity_a` are replaced by those of `entity_b` in the merge.
+ If `False`, an EntityMergeConflictError is raised instead. Default is False.
+
+ Returns
+ -------
+ entity_a : Entity
+ The initial entity_a after the in-place merge
+
+ Raises
+ ------
+ EntityMergeConflictError
+ In case of an unresolvable merge conflict.
+
+ """
+
+ logger.warning(
+ "This function is currently experimental and insufficiently tested. Use with care.")
+
+ # Compare both entities:
+ diff_r1, diff_r2 = compare_entities(
+ entity_a, entity_b, compare_referenced_records=merge_references_with_empty_diffs)
+
+ # Go through the comparison and try to apply changes to entity_a:
+ for key in diff_r2["parents"]:
+ entity_a.add_parent(entity_b.get_parent(key))
+
+ for key in diff_r2["properties"]:
+ if key in diff_r1["properties"]:
+ if ("importance" in diff_r1["properties"][key] and
+ "importance" in diff_r2["properties"][key]):
+ if (diff_r1["properties"][key]["importance"] !=
+ diff_r2["properties"][key]["importance"]):
+ raise NotImplementedError()
+ elif ("importance" in diff_r1["properties"][key] or
+ "importance" in diff_r2["properties"][key]):
+ raise NotImplementedError()
+
+ for attribute in ("datatype", "unit", "value"):
+ if (attribute in diff_r2["properties"][key] and
+ diff_r2["properties"][key][attribute] is not None):
+ if (diff_r1["properties"][key][attribute] is None):
+ setattr(entity_a.get_property(key), attribute,
+ diff_r2["properties"][key][attribute])
+ elif force:
+ setattr(entity_a.get_property(key), attribute,
+ diff_r2["properties"][key][attribute])
+ else:
+ raise EntityMergeConflictError(
+ f"Entity a ({entity_a.id}, {entity_a.name}) "
+ f"has a Property '{key}' with {attribute}="
+ f"{diff_r2['properties'][key][attribute]}\n"
+ f"Entity b ({entity_b.id}, {entity_b.name}) "
+ f"has a Property '{key}' with {attribute}="
+ f"{diff_r1['properties'][key][attribute]}")
+ else:
+ # TODO: This is a temporary FIX for
+ # https://gitlab.indiscale.com/caosdb/src/caosdb-pylib/-/issues/105
+ entity_a.add_property(id=entity_b.get_property(key).id,
+ name=entity_b.get_property(key).name,
+ datatype=entity_b.get_property(key).datatype,
+ value=entity_b.get_property(key).value,
+ unit=entity_b.get_property(key).unit,
+ importance=entity_b.get_importance(key))
+ # entity_a.add_property(
+ # entity_b.get_property(key),
+ # importance=entity_b.get_importance(key))
+
+ for special_attribute in ("name", "description"):
+ sa_a = getattr(entity_a, special_attribute)
+ sa_b = getattr(entity_b, special_attribute)
+ if sa_a != sa_b:
+ if sa_a is None:
+ setattr(entity_a, special_attribute, sa_b)
+ elif force:
+ # force overwrite
+ setattr(entity_a, special_attribute, sa_b)
+ else:
+ raise EntityMergeConflictError(
+ f"Conflict in special attribute {special_attribute}:\n"
+ f"A: {sa_a}\nB: {sa_b}")
+ return entity_a
+
+
def describe_diff(olddiff, newdiff, name=None, as_update=True):
description = ""
- for attr in list(set(list(olddiff.keys())+list(newdiff.keys()))):
+ for attr in list(set(list(olddiff.keys()) + list(newdiff.keys()))):
if attr == "parents" or attr == "properties":
continue
description += "{} differs:\n".format(attr)
@@ -700,6 +518,7 @@ def apply_to_ids(entities, func):
entities : list of Entity
func : function with one parameter.
"""
+
for entity in entities:
_apply_to_ids_of_entity(entity, func)
@@ -745,3 +564,28 @@ def resolve_reference(prop: Property):
else:
if isinstance(prop.value, int):
prop.value = retrieve_entity_with_id(prop.value)
+
+
+def create_flat_list(ent_list: List[Entity], flat: List[Entity]):
+ """
+ Recursively adds all properties contained in entities from ent_list to
+ the output list flat. Each element will only be added once to the list.
+
+ TODO: Currently this function is also contained in newcrawler module crawl.
+ We are planning to permanently move it to here.
+ """
+ for ent in ent_list:
+ for p in ent.properties:
+ # For lists append each element that is of type Entity to flat:
+ if isinstance(p.value, list):
+ for el in p.value:
+ if isinstance(el, Entity):
+ if el not in flat:
+ flat.append(el)
+ # TODO: move inside if block?
+ create_flat_list([el], flat)
+ elif isinstance(p.value, Entity):
+ if p.value not in flat:
+ flat.append(p.value)
+ # TODO: move inside if block?
+ create_flat_list([p.value], flat)
diff --git a/src/caosdb/cached.py b/src/caosdb/cached.py
new file mode 100644
index 0000000000000000000000000000000000000000..131526674d7df97d598a6d1bfbc2af7805c63a03
--- /dev/null
+++ b/src/caosdb/cached.py
@@ -0,0 +1,184 @@
+# -*- coding: utf-8 -*-
+#
+# This file is a part of the CaosDB Project.
+#
+# Copyright (C) 2023 IndiScale GmbH <info@indiscale.com>
+# Copyright (C) 2023 Henrik tom Wörden <h.tomwoerden@indiscale.com>
+# Copyright (C) 2023 Daniel Hornung <d.hornung@indiscale.com>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see <https://www.gnu.org/licenses/>.
+#
+
+"""
+This module provides some cached versions of functions that retrieve Entities from a remote server.
+
+See also
+========
+
+- ``cache_initialize(...)`` : Re-initialize the cache.
+- ``cache_clear()`` : Clear the cache.
+- ``cached_query(query)`` : A cached version of ``execute_query(query)``.
+- ``cached_get_entity_by(...)`` : Get an Entity by name, id, ...
+"""
+
+from enum import Enum
+from functools import lru_cache
+from typing import Union
+
+from .utils import get_entity
+from .common.models import execute_query, Entity, Container
+
+
+# roughly 1GB for typical entity sizes
+DEFAULT_SIZE = 33333
+
+# This dict cache is solely for filling the real cache manually (e.g. to reuse older query results)
+_DUMMY_CACHE = {}
+
+
+class AccessType(Enum):
+ """Different access types for cached queries. Needed for filling the cache manually with
+:func:`cache_fill` .
+
+ """
+ QUERY = 1
+ PATH = 2
+ EID = 3
+ NAME = 4
+
+
+def cached_get_entity_by(eid: Union[str, int] = None, name: str = None, path: str = None, query:
+ str = None) -> Entity:
+ """Return a single entity that is identified uniquely by one argument.
+
+You must supply exactly one argument.
+
+If a query phrase is given, the result must be unique. If this is not what you need, use
+:func:`cached_query` instead.
+
+ """
+ count = 0
+ if eid is not None:
+ count += 1
+ if name is not None:
+ count += 1
+ if path is not None:
+ count += 1
+ if query is not None:
+ count += 1
+ if count != 1:
+ raise ValueError("You must supply exactly one argument.")
+
+ if eid is not None:
+ return _cached_access(AccessType.EID, eid, unique=True)
+ if name is not None:
+ return _cached_access(AccessType.NAME, name, unique=True)
+ if path is not None:
+ return _cached_access(AccessType.PATH, path, unique=True)
+ if query is not None:
+ return _cached_access(AccessType.QUERY, query, unique=True)
+
+ raise ValueError("Not all arguments may be None.")
+
+
+def cached_query(query_string) -> Container:
+ """A cached version of :func:`caosdb.execute_query<caosdb.common.models.execute_query>`.
+
+All additional arguments are at their default values.
+
+ """
+ return _cached_access(AccessType.QUERY, query_string, unique=False)
+
+
+@lru_cache(maxsize=DEFAULT_SIZE)
+def _cached_access(kind: AccessType, value: Union[str, int], unique=True):
+ # This is the function that is actually cached.
+ # Due to the arguments, the cache has kind of separate sections for cached_query and
+ # cached_get_entity_by with the different AccessTypes. However, there is only one cache size.
+
+ # The dummy dict cache is only for filling the cache manually, it is deleted afterwards.
+ if value in _DUMMY_CACHE:
+ return _DUMMY_CACHE[value]
+
+ if kind == AccessType.QUERY:
+ return execute_query(value, unique=unique)
+ if kind == AccessType.NAME:
+ return get_entity.get_entity_by_name(value)
+ if kind == AccessType.EID:
+ return get_entity.get_entity_by_id(value)
+ if kind == AccessType.PATH:
+ return get_entity.get_entity_by_path(value)
+
+ raise ValueError(f"Unknown AccessType: {kind}")
+
+
+def cache_clear() -> None:
+ """Empty the cache that is used by `cached_query` and `cached_get_entity_by`."""
+ _cached_access.cache_clear()
+
+
+def cache_info():
+ """Return info about the cache that is used by `cached_query` and `cached_get_entity_by`.
+
+Returns
+-------
+
+out: named tuple
+ See the standard library :func:`functools.lru_cache` for details."""
+ return _cached_access.cache_info()
+
+
+def cache_initialize(maxsize=DEFAULT_SIZE) -> None:
+ """Create a new cache with the given size for `cached_query` and `cached_get_entity_by`.
+
+ This implies a call of :func:`cache_clear`, the old cache is emptied.
+
+ """
+ cache_clear()
+ global _cached_access
+ _cached_access = lru_cache(maxsize=maxsize)(_cached_access.__wrapped__)
+
+
+def cache_fill(items: dict, kind: AccessType = AccessType.EID, unique: bool = True) -> None:
+ """Add entries to the cache manually.
+
+ This allows to fill the cache without actually submitting queries. Note that this does not
+ overwrite existing entries with the same keys.
+
+Parameters
+----------
+
+items: dict
+ A dictionary with the entries to go into the cache. The keys must be compatible with the
+ AccessType given in ``kind``
+
+kind: AccessType, optional
+ The AccessType, for example ID, name, path or query.
+
+unique: bool, optional
+ If True, fills the cache for :func:`cached_get_entity_by`, presumably with
+ :class:`caosdb.Entity<caosdb.common.models.Entity>` objects. If False, the cache should be filled
+ with :class:`caosdb.Container<caosdb.common.models.Container>` objects, for use with
+ :func:`cached_query`.
+
+ """
+ # 1. add the given items to the corresponding dummy dict cache
+ _DUMMY_CACHE.update(items)
+
+ # 2. call the cache function with each key (this only results in a dict look up)
+ for key in items.keys():
+ _cached_access(kind, key, unique=unique)
+
+ # 3. empty the dummy dict cache again
+ _DUMMY_CACHE.clear()
diff --git a/src/caosdb/cert/indiscale.ca.crt b/src/caosdb/cert/indiscale.ca.crt
deleted file mode 100644
index 08a79d60c5d34626eb96f1a92e33d0ac22494f3c..0000000000000000000000000000000000000000
--- a/src/caosdb/cert/indiscale.ca.crt
+++ /dev/null
@@ -1,55 +0,0 @@
------BEGIN CERTIFICATE-----
-MIIJ6TCCBdGgAwIBAgIIFVYzdrEDk6EwDQYJKoZIhvcNAQENBQAwgZMxCzAJBgNV
-BAYTAkRFMRUwEwYDVQQIEwxMb3dlciBTYXhvbnkxEzARBgNVBAcTCkdvZXR0aW5n
-ZW4xFzAVBgNVBAoTDkluZGlTY2FsZSBHbWJIMRwwGgYDVQQDExNJbmRpU2NhbGUg
-Um9vdCBDQSAxMSEwHwYJKoZIhvcNAQkBFhJpbmZvQGluZGlzY2FsZS5jb20wHhcN
-MTkwODA3MDAwMDAwWhcNMzQwODA2MjM1OTU5WjCBkzELMAkGA1UEBhMCREUxFTAT
-BgNVBAgTDExvd2VyIFNheG9ueTETMBEGA1UEBxMKR29ldHRpbmdlbjEXMBUGA1UE
-ChMOSW5kaVNjYWxlIEdtYkgxHDAaBgNVBAMTE0luZGlTY2FsZSBSb290IENBIDEx
-ITAfBgkqhkiG9w0BCQEWEmluZm9AaW5kaXNjYWxlLmNvbTCCBCIwDQYJKoZIhvcN
-AQEBBQADggQPADCCBAoCggQBAKxJO3XOqrUxFU3qdVyk9tmZEHwhwntcLO+kRR5t
-64/1Z/+VIPSgVN5phkSCukj2BPJITWKplWzJDAYWSvA/7cqavCtx8yP+m3AHWrRa
-CeHbtkGZ1nzwyFel3GIr93e65REeWqBE3knzem+qxTlZ2hp8/w3oxUlhy7tGxjBs
-JlekgLRDrnj4Opyb4GVjcVfcELmu3sLrrPX1wdYJrqaMQUR4BKZnbXxKdOYyX+kR
-/W2P4sihCCJh7Wy29VXHwSSCM1qEkU3REjvPEmEElCG7UpqOfg+3jaNZDqnvfskf
-okU4GuFCxSWQituyP9jm/hFVEhz59tUMYCllcjEi2jGmD2DBKpiru4t4/z0Aymf4
-Pep9hNtH1yhZMxpQeCYK9ESEE5d7do0bu/4YFp7jAg5vWZ8KlILZakmypVBFUw8I
-U/QJoJ55j95vIp+kjFdXelIVcr5La/zOR82JldaoPfyoBKObzwpwqaWQwYm8pj4p
-XkUdJTf8rpW21SSGWZm8JoFSYDfGvI61rPEjl/ohKhlG0tV6E2tCc406HNo/7pPe
-pmx/v9ZWLbYDAH7MVMB4tv6zDRE/c4KTbh5/s70VbXbAeOG6DNwegdDLDYZOv6Yw
-YQMz9NWtKGzvoFehP2vY5nGK95JVUcd90jaNaoURLB102VtxAjPIEQA1PjbQxLvC
-7A6kshlpQiN7zS/R9IgiEkYP/9gjy6mMuQVxH7C+9cqmCnXvVmpHmxXGUqk61r/B
-h12htsx5qjbbkToZYhUXBmwRq4LDtyoxNeaF2Jc+gE762obbHsSYMuSuh0kTFUUd
-uqfrI8OyzX4r1w5dYf2FEetZTT2Obyxb3Cy0btJF5+zEerBX44RulkdC+TPTMhJw
-b1jrPCACKywy9b6vJcSQ2V1+uLk7rH2JKD+fQRIKUqZZkhNKFYz5dnYYTgS45M0/
-C+vIvRnhgNSNb4efG6wyFvWEF8poDSPnJ4mM+0jHG/+cLqF/M2CMFvC+yU8Hj9YH
-B+H2L6V1QlCkpw5Ai4ji6OaQmnrsjE8EJj58vwYKsjmLGuf4j5AivkQTxfgCPGrT
-6CxSesoFmYDPSg/2eO+IfYEwnd7Rbs4aAhW8eo+lGpmK0DQxNjlejYt/Cgp7HWCq
-m/VNqWPIDMSTTqyk1GTmp67NjEZKt2ukJxI2CpL8s/9x4f3GTjNyI750pKM/uzMk
-OBKTMuWJQ6xeMR3h9RQlqlmwcErLXoUGInOTHHjRGXDI+ZBeLqT5DikcFiwbHG3+
-6FOuxXO0eqqg2tBW8cQ5kuRI0YFznipDUcfgDZt0JEkEXmRuL0nxYO35WKKdpGcF
-xFRJtO4FRB4nVWekVRuK9m47IPm6vC4eo+pCNPPoQ+FjyQ8CAwEAAaM/MD0wDAYD
-VR0TBAUwAwEB/zAdBgNVHQ4EFgQUFjE2TLaKASKEJ0LKOO+37/Hu7qowDgYDVR0P
-AQH/BAQDAgEGMA0GCSqGSIb3DQEBDQUAA4IEAQB2j1GL1G0ferWp9wmuDdF2oumn
-k+JIctRaGHaSrqHy4zjwj3Oqm2JA1ds+WfWozz+d38ZcLqSHo+g9ku5h/XOogQEU
-O4/y7j44pxIUg0EcIpMHtf7KPocPfvi9lw/4vE/3V/WKh4E09SXWgyY5tMUlEMaB
-6t8n7gg943HY2MJE1QU2wOPMXB1krFbunYxJcrUMs21w9jRWVv/wvaj6rkdvvSbU
-Yre11J+VlhC6fxx+STohQopzE6jnsaHile56b9xAmCCKcPEpWeKKBFS7pVNHEIHF
-uHWpgVjhoheEMMbYgu6l5E5K32TNYCKU49jNRWEKETjmYQSNl9dsSip+XlvaU8wQ
-VRR8UMHZPiJDW/AAHCr+bXEarZ9mSj/y+R512YtVw95zCnGUtzOJViThoIk/IAOR
-AJdnvsFmZSIKtFHpSEFYlTDq2yr1ulzbaDhuPRzita8b0cP27UvqRebZw5CvHN48
-B9a9tTYowKuJqmtjE6D00QA4xS8fRizLnx54uNmDbwf/8WavVk6MzDERwRE3OsSy
-D0dV6gy3t2AqEpVBrICrFqvgAQa4fcFcIwz3Qbt5o5uEi7acRomY57YrxrlfNTwh
-2oDQz+HQ/ZTDwZ3DrIgel7GrQ5fXrXDLL3ebtsbuIeBx8crOWQask832HcLtDVpu
-E/FdJEMMjglzIcy2dHpuODIGFmgEVfHR4DOOSBl0hfNdlrYnhC0h8/6QFswtlYFF
-8aQbGX7inK8L2in5wQ7ypeoMuXkQVYxlU1TEGmgB8aDke47MuX1FH+clsCaZ3s1E
-ka6lV6cjNYcosS718B6b2JgDUzmGBn2Sdm1xFmJM16dXp7TSmC5/fYxXuE/CynDs
-PmaUb9Ms6XUYSwKKhZ5HZdeRoNz8w62WNAeF7o7iX6IVrd/G1bJnSBN01istckyR
-BDuIkaoBQ9yvHN6Bo/J3KR08ixF1dHFPo/oSgkBxkLakb/yeslBTP/oISiFeQ4+q
-Gld1mhAvmG99dVZfoysrMjZSyghNbqwScjbYYN115lExV5ZeRtSwA7JCYE2lBjmB
-vocmz/hh/ifbmmqIvSv0NtiBnM6mNqngZEWD/rAloVOQoq0KVJJ5lUCQrBSFtR4+
-G1JGMX6b7uRp4mfdqqDE62KxxfkWBUwzUTIKGb5K42ji1Gy5li/TIWJtLNGNNQ2A
-0ui2RhwioaGGfYyomSFuAo5IPE/NF0ASjrTDW6GoNxypTSYE4/7oSoxeryafVnqN
-S0fRyrgSLiuT5tAiZ3b5Q3EFYUM2OcU3ezr/ZUabf9qIsqOnCi91SqE88BQbenot
-0HyUMdp/7QX9SyWM/azhcRiReAtkmq9pgeQA2TTZADDNTkKRljG9VeFDSwl7
------END CERTIFICATE-----
diff --git a/src/caosdb/common/administration.py b/src/caosdb/common/administration.py
index dff461e7fb0ed5270119907bd4ad859503b3ce21..a27aaf0406c83ac33c37b676a9cdeab812bf2f7a 100644
--- a/src/caosdb/common/administration.py
+++ b/src/caosdb/common/administration.py
@@ -26,6 +26,9 @@
"""missing docstring."""
+import re
+import string
+import random
from caosdb.common.utils import xml2str
from caosdb.connection.connection import get_connection
from caosdb.exceptions import (EntityDoesNotExistError, HTTPClientError,
@@ -56,7 +59,8 @@ def set_server_property(key, value):
con._form_data_request(method="POST", path="_server_properties",
params={key: value}).read()
except EntityDoesNotExistError:
- raise ServerConfigurationException("Debug mode in server is probably disabled.") from None
+ raise ServerConfigurationException(
+ "Debug mode in server is probably disabled.") from None
def get_server_properties():
@@ -71,9 +75,11 @@ def get_server_properties():
"""
con = get_connection()
try:
- body = con._http_request(method="GET", path="_server_properties").response
+ body = con._http_request(
+ method="GET", path="_server_properties")
except EntityDoesNotExistError:
- raise ServerConfigurationException("Debug mode in server is probably disabled.") from None
+ raise ServerConfigurationException(
+ "Debug mode in server is probably disabled.") from None
xml = etree.parse(body)
props = dict()
@@ -108,6 +114,39 @@ def get_server_property(key):
return get_server_properties()[key]
+def generate_password(length: int):
+ """Create a random password that fulfills the security requirements
+
+ Parameters
+ ----------
+ length : int
+ Length of the generated password. Has to be greater than 7.
+
+ Returns
+ -------
+ password : string
+ Generated random password of the given length
+
+ Raises
+ ------
+ ValueError:
+ If the length is less than 8.
+ """
+ minimum_password_length = 8
+ if length < minimum_password_length:
+ raise ValueError("CaosDB passwords have to be at least {} characters.".format(
+ minimum_password_length))
+ sample_letters = string.ascii_letters + string.digits + "!#$%*+-/:;?_"
+ password = ''.join((random.choice(sample_letters) for i in range(length)))
+
+ while not re.match(r"(?=.*[A-Z])(?=.*[a-z])(?=.*\d)(?=.*[\W_]).{8,}",
+ password):
+ password = ''.join((random.choice(sample_letters)
+ for i in range(length)))
+
+ return password
+
+
def _retrieve_user(name, realm=None, **kwargs):
con = get_connection()
try:
@@ -157,17 +196,9 @@ def _update_user(name, realm=None, password=None, status=None,
e.msg = "You are not permitted to update this user."
raise e
except HTTPClientError as e:
- if e.status == 409:
- e.msg = "Entity does not exist."
-
- if e.status == 422:
- e.msg = """Maybe the password does not match the required standard?
- The current requirements are:
- - at least 8 characters
- - at least 1 number
- - at least 1 lower case character
- - at least 1 upper case character
- - at least 1 special character"""
+ for elem in etree.fromstring(e.body):
+ if elem.tag == "Error":
+ e.msg = elem.get("description")
raise
@@ -192,17 +223,9 @@ def _insert_user(name, password=None, status=None, email=None, entity=None, **kw
e.msg = "You are not permitted to insert a new user."
raise e
except HTTPClientError as e:
- if e.status == 409:
- e.msg = "User name is already in use."
-
- if e.status == 422:
- e.msg = """Maybe the password does not match the required standard?
- The current requirements are:
- - at least 8 characters
- - at least 1 number
- - at least 1 lower case character
- - at least 1 upper case character
- - at least 1 special character"""
+ for elem in etree.fromstring(e.body):
+ if elem.tag == "Error":
+ e.msg = elem.get("description")
raise e
@@ -362,7 +385,7 @@ action : str
Either "grant" or "deny"
permission : str
- For example "RETRIEVE:*".
+ For example ``RETRIEVE:*``.
priority : bool, optional
Whether the priority shall be set, defaults is False.
diff --git a/src/caosdb/common/models.py b/src/caosdb/common/models.py
index 15c67fc96d099e3e4d5a7e06364330decab9a25f..67282a93f185db4a9f2346e4bfba5913270bd451 100644
--- a/src/caosdb/common/models.py
+++ b/src/caosdb/common/models.py
@@ -1,13 +1,12 @@
# -*- coding: utf-8 -*-
#
-# ** header v3.0
# This file is a part of the CaosDB Project.
#
# Copyright (C) 2018 Research Group Biomedical Physics,
# Max-Planck-Institute for Dynamics and Self-Organization Göttingen
-# Copyright (C) 2020 Indiscale GmbH <info@indiscale.com>
-# Copyright (C) 2020 Florian Spreckelsen <f.spreckelsen@indiscale.com>
-# Copyright (C) 2020 Timm Fitschen <t.fitschen@indiscale.com>
+# Copyright (C) 2020-2023 Indiscale GmbH <info@indiscale.com>
+# Copyright (C) 2020-2023 Florian Spreckelsen <f.spreckelsen@indiscale.com>
+# Copyright (C) 2020-2022 Timm Fitschen <t.fitschen@indiscale.com>
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as
@@ -22,11 +21,19 @@
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <https://www.gnu.org/licenses/>.
#
-# ** end header
#
-"""missing docstring."""
-from __future__ import annotations, print_function, unicode_literals
+"""
+Collection of the central classes of the CaosDB client, namely the Entity class
+and all of its subclasses and the Container class which is used to carry out
+transactions.
+
+All additional classes are either important for the entities or the
+transactions.
+"""
+
+from __future__ import annotations # Can be removed with 3.10.
+from __future__ import print_function, unicode_literals
import re
import sys
@@ -37,7 +44,6 @@ from hashlib import sha512
from os import listdir
from os.path import isdir
from random import randint
-from sys import hexversion
from tempfile import NamedTemporaryFile
from typing import Any, Literal, Optional, Type, Union
from warnings import warn
@@ -45,6 +51,7 @@ from warnings import warn
from caosdb.common.datatype import (BOOLEAN, DATETIME, DOUBLE, INTEGER, TEXT, REFERENCE,
is_list_datatype, is_reference)
from caosdb.common.state import State
+from caosdb.common.timezone import TimeZone
from caosdb.common.utils import uuid, xml2str
from caosdb.common.versioning import Version
from caosdb.configuration import get_config
@@ -74,7 +81,11 @@ NONE: Literal["NONE"] = "NONE"
TA_add_properties = Union[Literal['ALL', 'FIX', 'NONE'], None]
-class Entity(object):
+SPECIAL_ATTRIBUTES = ["name", "role", "datatype", "description",
+ "id", "path", "checksum", "size"]
+
+
+class Entity:
"""Entity is a generic CaosDB object.
@@ -94,6 +105,8 @@ class Entity(object):
self._checksum = None
self._size = None
self._upload = None
+ # If an entity is used (e.g. as parent), it is wrapped instead of being used directly.
+ # see Entity._wrap()
self._wrapped_entity = None
self._version = None
self._cuid = None
@@ -102,9 +115,9 @@ class Entity(object):
self.__datatype = None
self.datatype = datatype
self.value = value
- self.messages = _Messages()
+ self.messages = Messages()
self.properties = _Properties()
- self.parents = _Parents()
+ self.parents = _ParentList()
self.path = None
self.file = None
self.unit = None
@@ -117,6 +130,47 @@ class Entity(object):
self.id = id
self.state = None
+ def copy(self):
+ """
+ Return a copy of entity.
+
+ If deep == True return a deep copy, recursively copying all sub entities.
+
+ Standard properties are copied using add_property.
+ Special attributes, as defined by the global variable SPECIAL_ATTRIBUTES and additionaly
+ the "value" are copied using setattr.
+ """
+ if self.role == "File":
+ new = File()
+ elif self.role == "Property":
+ new = Property()
+ elif self.role == "RecordType":
+ new = RecordType()
+ elif self.role == "Record":
+ new = Record()
+ elif self.role == "Entity":
+ new = Entity()
+ else:
+ raise RuntimeError("Unkonwn role.")
+
+ # Copy special attributes:
+ # TODO: this might rise an exception when copying
+ # special file attributes like checksum and size.
+ for attribute in SPECIAL_ATTRIBUTES + ["value"]:
+ val = getattr(self, attribute)
+ if val is not None:
+ setattr(new, attribute, val)
+
+ # Copy parents:
+ for p in self.parents:
+ new.add_parent(p)
+
+ # Copy properties:
+ for p in self.properties:
+ new.add_property(p, importance=self.get_importance(p))
+
+ return new
+
@property
def version(self):
if self._version is not None or self._wrapped_entity is None:
@@ -276,14 +330,74 @@ class Entity(object):
self.__pickup = new_pickup
def grant(self, realm=None, username=None, role=None,
- permission=None, priority=False):
+ permission=None, priority=False, revoke_denial=True):
+ """Grant a permission to a user or role for this entity.
+
+ You must specify either only the username and the realm, or only the
+ role.
+
+ By default a previously existing denial rule would be revoked, because
+ otherwise this grant wouldn't have any effect. However, for keeping
+ contradicting rules pass revoke_denial=False.
+
+ Parameters
+ ----------
+ permission: str
+ The permission to be granted.
+ username : str, optional
+ The username. Exactly one is required, either the `username` or the
+ `role`.
+ realm: str, optional
+ The user's realm. Required when username is not None.
+ role: str, optional
+ The role (as in Role-Based Access Control). Exactly one is
+ required, either the `username` or the `role`.
+ priority: bool, default False
+ Whether this permission is granted with priority over non-priority
+ rules.
+ revoke_denial: bool, default True
+ Whether a contradicting denial (with same priority flag) in this
+ ACL will be revoked.
+ """
+ # @review Florian Spreckelsen 2022-03-17
self.acl.grant(realm=realm, username=username, role=role,
- permission=permission, priority=priority)
+ permission=permission, priority=priority,
+ revoke_denial=revoke_denial)
def deny(self, realm=None, username=None, role=None,
- permission=None, priority=False):
+ permission=None, priority=False, revoke_grant=True):
+ """Deny a permission to a user or role for this entity.
+
+ You must specify either only the username and the realm, or only the
+ role.
+
+ By default a previously existing grant rule would be revoked, because
+ otherwise this denial would override the grant rules anyways. However,
+ for keeping contradicting rules pass revoke_grant=False.
+
+ Parameters
+ ----------
+ permission: str
+ The permission to be denied.
+ username : str, optional
+ The username. Exactly one is required, either the `username` or the
+ `role`.
+ realm: str, optional
+ The user's realm. Required when username is not None.
+ role: str, optional
+ The role (as in Role-Based Access Control). Exactly one is
+ required, either the `username` or the `role`.
+ priority: bool, default False
+ Whether this permission is denied with priority over non-priority
+ rules.
+ revoke_grant: bool, default True
+ Whether a contradicting grant (with same priority flag) in this
+ ACL will be revoked.
+ """
+ # @review Florian Spreckelsen 2022-03-17
self.acl.deny(realm=realm, username=username, role=role,
- permission=permission, priority=priority)
+ permission=permission, priority=priority,
+ revoke_grant=revoke_grant)
def revoke_denial(self, realm=None, username=None,
role=None, permission=None, priority=False):
@@ -312,7 +426,7 @@ class Entity(object):
self.acl.is_permitted(permission=permission)
def get_all_messages(self):
- ret = _Messages()
+ ret = Messages()
ret.append(self.messages)
for p in self.properties:
@@ -345,6 +459,66 @@ class Entity(object):
return self
+ def remove_value_from_property(self, property_name: str, value: Any,
+ remove_if_empty_afterwards: Optional[bool] = True):
+ """Remove a value from a property given by name.
+
+ Do nothing if this entity does not have a property of this
+ ``property_name`` or if the property value is different of the given
+ ``value``. By default, the property is removed from this entity if it
+ becomes empty (i.e., value=None) through removal of the value. This
+ behavior can be changed by setting ``remove_if_empty_afterwards`` to
+ ``False`` in which case the property remains.
+
+ Notes
+ -----
+ If the property value is a list and the value to be removed occurs more
+ than once in this list, only its first occurrance is deleted (similar
+ to the behavior of Python's ``list.remove()``.)
+
+ If the property was empty (prop.value == None) before, the property is
+ not removed afterwards even if ``remove_if_empty_afterwards`` is set to
+ ``True``. Rationale: the property being empty is not an effect of
+ calling this function.
+
+ Parameters
+ ----------
+ property_name : str
+ Name of the property from which the ``value`` will be removed.
+
+ value
+ Value that is to be removed.
+
+ remove_if_empty_afterwards : bool, optional
+ Whether the property shall be removed from this entity if it is
+ emptied by removing the ``value``. Default is ``True``.
+
+ Returns
+ -------
+ self
+ This entity.
+
+ """
+
+ if self.get_property(property_name) is None:
+ return self
+ if self.get_property(property_name).value is None:
+ remove_if_empty_afterwards = False
+ empty_afterwards = False
+ if isinstance(self.get_property(property_name).value, list):
+ if value in self.get_property(property_name).value:
+ self.get_property(property_name).value.remove(value)
+ if self.get_property(property_name).value == []:
+ self.get_property(property_name).value = None
+ empty_afterwards = True
+ elif self.get_property(property_name).value == value:
+ self.get_property(property_name).value = None
+ empty_afterwards = True
+ if remove_if_empty_afterwards and empty_afterwards:
+ self.remove_property(property_name)
+
+ return self
+
def remove_parent(self, parent):
self.parents.remove(parent)
@@ -358,18 +532,30 @@ class Entity(object):
inheritance: Optional[str] = None) -> Entity: # @ReservedAssignment
"""Add a property to this entity.
- The first parameter is meant to identify the property entity. So the method expects an instance of
- Entity, an integer or a string here. The second parameter is the value of the new property. Any
- other named parameter may be passed by means of the keywwords. Accepted keywords are:
- id, name, description, importance, inheritance, datatype, and unit. Any other keyword will be
- ignored right now. But that may change in the future.
+ The first parameter is meant to identify the property entity either via
+ its id or name, or by providing the corresponding ``Entity`` Python
+ object. The second parameter is the value of the new property. Any other
+ named parameter may be passed by means of the keywwords. Accepted
+ keywords are: id, name, description, importance, inheritance, datatype,
+ and unit.
+
+ Notes
+ -----
+ If you want to add a property to an already existing entity, the
+ property ``id`` of that property needs to be specified before you send
+ the updated entity to the server.
Parameters
----------
property : Union[int, str, Entity, None], optional
- An identifying parameter, by default None
- value : Union[str, int, Entity, None], optional
- The value of the new property, by default None
+ An identifier for the property to be added, either its name, its id,
+ or the corresponding Entity Python object. If ``None``, either the
+ `name` or the `id` argument have to be specified explicitly. Default
+ is ``None``.
+ value : Union[str, int, Entity, bool,list[str], list[int], list[Entity], list[bool]], optional
+ The value of the new property. In case of a reference to another
+ entity, this value may be the referenced entities id or the
+ ``Entity`` as a Python object. Default is None.
id : Optional[int], optional
Id of the property, by default None
name : Optional[str], optional
@@ -388,13 +574,16 @@ class Entity(object):
Returns
-------
Entity
- The entity passed
+ This Entity object to which the new property has been added.
Raises
------
ValueError
Unusual Property role
+
+ Warns
+ -----
UserWarning
If the first parameter is None then id or name must be defined and not be None.
UserWarning
@@ -403,6 +592,52 @@ class Entity(object):
UserWarning
If the first parameter is not None and neither an instance of Entity nor an integer it is
interpreted as the name and name must be undefined or None.
+
+ Raises
+ ------
+ ValueError:
+ If you try to add an ``Entity`` object with File or Record role (or,
+ equivalently, a ``File`` or ``Record`` object) as a property, a
+ ``ValueError`` is raised.
+
+ Examples
+ --------
+ Add a simple integer property with the name ``TestProp`` and the value
+ 27 to a Record:
+
+ >>> import caosdb as db
+ >>> rec = db.Record(name="TestRec").add_parent(name="TestType")
+ >>> rec.add_property("TestProp", value=27) # specified by name, you could equally use the property's id if it is known
+
+ You can also use the Python object:
+
+ >>> prop = db.Property(name="TestProp", datatype=db.INTEGER)
+ >>> rec.add_property(prop, value=27) # specified via the Python object
+
+ In case of updating an existing Record, the Property needs to be
+ specified by id:
+
+ >>> rec = db.Record(name="TestRec").retrieve()
+ >>> prop2 = db.Property(name="OtherTestProp").retrieve()
+ >>> rec.add_property(id=prop2.id, value="My new value")
+ >>> rec.update()
+
+ Let's look at the more advanced example of adding a list of integers as
+ value of the above integer ``TestProp``:
+
+ >>> rec.add_property("TestProp", value=[27,28,29], datatype=db.LIST(db.INTEGER))
+
+ Note that since `TestProp` is a scalar integer Property, the datatype
+ `LIST<INTEGER>` has to be specified explicitly.
+
+ Finally, we can also add reference properties, specified by the RecordType of the referenced entity.
+
+ >>> ref_rec = db.Record(name="ReferencedRecord").add_parent(name="OtherRT")
+ >>> rec.add_property(name="OtherRT", value=ref_rec) # or value=ref_rec.id if ref_rec has one set by the server
+
+ See more on adding properties and inserting data in
+ https://docs.indiscale.com/caosdb-pylib/tutorials/Data-Insertion.html.
+
"""
pid = id
@@ -434,7 +669,7 @@ class Entity(object):
value=value, unit=unit)
if abstract_property is not None:
- new_property._wrap(property)
+ new_property._wrap(abstract_property)
# FIXME: this really necessary?
@@ -476,7 +711,8 @@ class Entity(object):
if msg is not None:
pass
else:
- msg = Message(type, code, description, body)
+ msg = Message(description=description, type=type, code=code,
+ body=body)
self.messages.append(msg)
return self
@@ -503,21 +739,18 @@ class Entity(object):
entity. If no `inheritance` is given, no properties will be inherited by the child.
This parameter is case-insensitive.
- Note that the behaviour is currently not yet specified when assigning parents to
- Records, it only works for inheritance of RecordTypes (and Properties).
-
- For more information, it is recommended to look into the
- :ref:`data insertion tutorial<tutorial-inheritance-properties>`, by default None
-
- Returns
- -------
- Entity
- The entity passed
Returns
-------
Entity
The entity passed
+ Notes
+ -----
+ Note that the behaviour of the `inheritance` argument currently has not
+ yet been specified when assigning parents to Records, it only works for
+ inheritance of RecordTypes (and Properties). For more information, it is
+ recommended to look into the :ref:`data insertion
+ tutorial<tutorial-inheritance-properties>`.
Raises
------
@@ -548,10 +781,9 @@ class Entity(object):
return self
- def has_parent(self, parent: Union[Entity, str, int],
- recursive: bool = True, check_name: bool = True,
- check_id: bool = False) -> bool:
- """Checks if this entity has a given parent.
+ def has_parent(self, parent: Union[Entity, str, int], recursive: bool = True, retrieve: bool = True,
+ check_name: bool = True, check_id: bool = False):
+ """Check if this entity has a given parent.
If 'check_name' and 'check_id' are both False, test for identity
on the Python level. Otherwise use the name and/or ID for the
@@ -560,24 +792,34 @@ class Entity(object):
Parameters
----------
- parent : Entity, str, int
- Check for this parent
- recursive : bool, optional
- Whether to check recursively, by default True
- check_name : bool, optional
- Whether to use the name for ancestry check, by default True
- check_id : bool, optional
- Whether to use the ID for ancestry check, by default False
+
+ parent: Entity, str, int
+ Check for this parent.
+
+ recursive: bool, optional
+ Whether to check recursively.
+
+ check_name: bool, optional
+ Whether to use the name for ancestry check.
+
+ check_id: bool, optional
+ Whether to use the ID for ancestry check.
+
+ retrieve: bool, optional
+ If False, do not retrieve parents from the server.
Returns
-------
- bool
- True if 'parent' is a true parent, False otherwise
+ out: bool
+ True if ``parent`` is a true parent, False otherwise.
"""
if recursive:
- parents = self.get_parents_recursively()
+ parents = self.get_parents_recursively(retrieve=retrieve)
else:
- parents = [pp._wrapped_entity for pp in self.parents]
+ if retrieve:
+ parents = [pp.retrieve()._wrapped_entity for pp in self.parents]
+ else:
+ parents = [pp._wrapped_entity for pp in self.parents]
if not (check_name or check_id):
return parent in parents
@@ -604,41 +846,59 @@ class Entity(object):
Returns
-------
- list
- Parents
+ list[ Parents]
"""
return self.parents
- def get_parents_recursively(self) -> list:
+ def get_parents_recursively(self, retrieve: bool = True):
"""Get all ancestors of this entity.
- Returns
- -------
- list
- list of Entities
- """
- all_parents = _Parents()
- self._get_parent_recursively(all_parents)
+Parameters
+----------
+
+retrieve: bool, optional
+ If False, do not retrieve parents from the server.
+
+Returns
+-------
+out: List[Entity]
+ The parents of this Entity
+"""
+
+ all_parents = []
+ self._get_parent_recursively(all_parents, retrieve=retrieve)
return all_parents
- def _get_parent_recursively(self, all_parents: list) -> None:
+ def _get_parent_recursively(self, all_parents: list, retrieve: bool = True):
"""Get all ancestors with a little helper.
Important: As a side effect of this method, the ancestors are
added to all_parents.
- Parameters
- ----------
- all_parents : list
- The added parents so far
+ @param all_parents: list, The added parents so far.
+
+ @return: None, but see side effects.
"""
for parent in self.parents:
+ # TODO:
+ # Comment on _wrap and _wrapped_entity
+ # Currently, I (henrik) do not why the wrapping is necessary (and it is not
+ # documented). However, the following illustrates, why I think, it is a bad idea.
+ # First you add a parent with rec.add_parent(parent), but then you cannot access
+ # attributes of parent when you use rec.parents[0] for example becasue you do not get
+ # the same object but a wrapping object and you need to know that you only get the
+ # original by accessing the private (!) _wrapped_entity object.
w_parent = parent._wrapped_entity
+ if retrieve:
+ parent.retrieve()
+ for next_parent in parent.parents:
+ w_parent.add_parent(next_parent)
- if w_parent not in all_parents:
+ if (w_parent.id, w_parent.name) not in [
+ (all_p.id, all_p.name) for all_p in all_parents]:
all_parents.append(w_parent)
- w_parent._get_parent_recursively(all_parents)
+ w_parent._get_parent_recursively(all_parents, retrieve=retrieve)
def get_parent(self, key: Union[Entity, int, str]) -> Optional[Entity]:
"""Return the first parent matching the key or None if no match exists.
@@ -851,10 +1111,7 @@ class Entity(object):
def get_messages(self) -> dict:
"""Get all messages of this entity.
- Returns
- -------
- list
- _Messages(list)
+ @return: Messages(list)
"""
return self.messages
@@ -866,7 +1123,7 @@ class Entity(object):
list
_Messages(list): Warning messages.
"""
- ret = _Messages()
+ ret = Messages()
for m in self.messages:
if m.type.lower() == "warning":
@@ -882,7 +1139,7 @@ class Entity(object):
list
_Messages(list): Error messages.
"""
- ret = _Messages()
+ ret = Messages()
for m in self.messages:
if m.type.lower() == "error":
@@ -1145,7 +1402,7 @@ class Entity(object):
else:
raise TypeError(
'Child was neither a Property, nor a Parent, nor a Message.\
- Was ' + str(type(child)))
+ Was ' + str(type(child)) + "\n" + str(child))
# add VALUE
value = None
@@ -1280,15 +1537,18 @@ class Entity(object):
Parameters
----------
raise_exception_on_error : bool, optional
- Flag to raise an exception when an error occurs, by default True
- unique : bool, optional
- Flag to only allow insertion of elements with unique name, by default True
- sync : bool, optional
- Flag for sync, by default True
strict : bool, optional
- Flag for strict mode, by default False
+ Flag for strict mode. Default is False.
+ raise_exception_on_error : bool, optional
+ Flag to raise an exception when an error occurs. Default is True.
+ unique : bool, optional
+ Flag to only allow insertion of elements with unique names. Default
+ is True.
flags : dict, optional
- A dictionary of flags to be send with the insertion, by default None
+ A dictionary of flags to be send with the insertion. Default is
+ None.
+
+ """
Returns
-------
@@ -1312,15 +1572,15 @@ class Entity(object):
sync: bool = True) -> Container:
"""Update this entity.
- There are two possible work-flows to perform this update:
- First:
- 1) retrieve an entity
- 2) do changes
- 3) call update method
+There are two possible work-flows to perform this update:
+First:
+ 1) retrieve an entity
+ 2) do changes
+ 3) call update method
- Second:
- 1) construct entity with id
- 2) call update method.
+Second:
+ 1) construct entity with id
+ 2) call update method.
For slight changes the second one it is more comfortable. Furthermore, it is possible to stay
off-line until calling the update method. The name, description, unit, datatype, path,
@@ -1369,6 +1629,12 @@ class Entity(object):
flags=flags)[0]
def _wrap(self, entity):
+ """
+ When entity shall be used as parent or property it is not added to the corresponding list
+ (such as the parent list) directly, but another Entity object is created and the original
+ Entity is wrapped using this function
+ TODO: document here and in dev docs why this is done.
+ """
self._wrapped_entity = entity
return self
@@ -1380,6 +1646,10 @@ class Entity(object):
def _parse_value(datatype, value):
+ """Parse the value (from XML input) according to the given datatype
+ """
+
+ # Simple values
if value is None:
return value
@@ -1400,12 +1670,12 @@ def _parse_value(datatype, value):
else:
raise ValueError("Boolean value was {}.".format(value))
+ # Datetime and text are returned as-is
if datatype in [DATETIME, TEXT]:
if isinstance(value, str):
return value
# deal with collections
-
if isinstance(datatype, str):
matcher = re.compile(r"^(?P<col>[^<]+)<(?P<dt>[^>]+)>$")
m = matcher.match(datatype)
@@ -1430,12 +1700,10 @@ def _parse_value(datatype, value):
# This is for a special case, where the xml parser could not differentiate
# between single values and lists with one element. As
-
if hasattr(value, "__len__") and len(value) == 1:
return _parse_value(datatype, value[0])
# deal with references
-
if isinstance(value, Entity):
return value
@@ -1451,6 +1719,15 @@ def _parse_value(datatype, value):
# reference via name
return str(value)
+ except TypeError as te:
+ # deal with invalid XML: List of values without appropriate datatype
+ if isinstance(value, list):
+ raise TypeError(
+ "Invalid datatype: List valued properties must be announced by "
+ "the datatype.\n" + f"Datatype: {datatype}\nvalue: {value}")
+ else:
+ # Everything else that's not related to wrong list assignments
+ raise te
def _log_request(request, xml_body=None):
@@ -1466,11 +1743,7 @@ def _log_request(request, xml_body=None):
def _log_response(body):
if Container._debug() > 0:
print("\n======== Response body ========\n")
-
- if hexversion < 0x03000000:
- print(body)
- else:
- print(body.decode())
+ print(body.decode())
print("\n===============================\n")
@@ -1488,7 +1761,7 @@ class QueryTemplate():
self._cuid = None
self.value = None
self.datatype = None
- self.messages = _Messages()
+ self.messages = Messages()
self.properties = None
self.parents = None
self.path = None
@@ -1735,8 +2008,8 @@ class QueryTemplate():
def has_id(self) -> bool:
return self.id is not None
- def get_errors(self) -> _Messages:
- ret = _Messages()
+ def get_errors(self):
+ ret = Messages()
for m in self.messages:
if m.type.lower() == "error":
@@ -1796,10 +2069,9 @@ class Property(Entity):
"""CaosDB's Property object."""
- def add_property(self, property: Union[int, str, Entity, None] = None,
- value: Union[str, int, Entity, None] = None,
- id=None, name=None, description=None, datatype=None,
- unit=None, importance=FIX, inheritance=FIX) -> Entity: # @ReservedAssignment
+ def add_property(self, property: Union[int, str, Entity, None]=None, value: Union[str, int, Entity, None]=None, id=None, name=None, description=None, datatype=None,
+ unit=None, importance=FIX, inheritance=FIX): # @ReservedAssignment
+ """See ``Entity.add_property``."""
return super().add_property(
property=property, id=id, name=name, description=description, datatype=datatype,
@@ -1893,12 +2165,10 @@ class Property(Entity):
class Message(object):
- # @ReservedAssignment
-
- def __init__(self, type, code=None, description=None, body=None): # @ReservedAssignment
- self.type = type
- self.code = code
+ def __init__(self, type=None, code=None, description=None, body=None): # @ReservedAssignment
self.description = description
+ self.type = type if type is not None else "Info"
+ self.code = int(code) if code is not None else None
self.body = body
def to_xml(self, xml=None):
@@ -1921,11 +2191,13 @@ class Message(object):
def __eq__(self, obj):
if isinstance(obj, Message):
- return self.type == obj.type and self.code == obj.code
+ return self.type == obj.type and self.code == obj.code and self.description == obj.description
return False
def get_code(self):
+ warn(("get_code is deprecated and will be removed in future. "
+ "Use self.code instead."), DeprecationWarning)
return int(self.code)
@@ -1935,6 +2207,7 @@ class RecordType(Entity):
def add_property(self, property=None, value=None, id=None, name=None, description=None, datatype=None,
unit=None, importance=RECOMMENDED, inheritance=FIX): # @ReservedAssignment
+ """See ``Entity.add_property``."""
return super().add_property(
property=property, id=id, name=name, description=description, datatype=datatype,
@@ -1990,6 +2263,7 @@ class Record(Entity):
def add_property(self, property=None, value=None, id=None, name=None, description=None, datatype=None,
unit=None, importance=FIX, inheritance=FIX): # @ReservedAssignment
+ """See ``Entity.add_property``."""
return super().add_property(
property=property, id=id, name=name, description=description, datatype=datatype,
@@ -2147,6 +2421,7 @@ class File(Record):
def add_property(self, property=None, id=None, name=None, description=None, datatype=None,
value=None, unit=None, importance=FIX, inheritance=FIX): # @ReservedAssignment
+ """See ``Entity.add_property``."""
return super().add_property(
property=property, id=id, name=name, description=description, datatype=datatype,
@@ -2297,7 +2572,8 @@ class _Properties(list):
raise KeyError(str(prop) + " not found.")
-class _Parents(list):
+class _ParentList(list):
+ # TODO unclear why this class is private. Isn't it use full for users?
def _get_entity_by_cuid(self, cuid):
'''
@@ -2418,10 +2694,9 @@ class _Parents(list):
raise KeyError(str(parent) + " not found.")
-class _Messages(dict):
-
- """This 'kind of dictionary' stores error, warning, info, and other
- messages. The mentioned three messages types are messages of special use.
+class Messages(list):
+ """This specialization of list stores error, warning, info, and other
+ messages. The mentioned three messages types play a special role.
They are generated by the client and the server while processing the entity
to which the message in question belongs. It is RECOMMENDED NOT to specify
such messages manually. The other messages are ignored by the server unless
@@ -2432,25 +2707,18 @@ class _Messages(dict):
<$Type code=$code description=$description>$body</$Type>
- Messages are treated as 'equal' if and only if both they have the same type (case-insensitive),
- and the same code (or no code). Every message
- MUST NOT occur more than once per entity (to which the message in question belongs).
-
- If a message m2 is added while a messages m1 is already in this _Message object m2 will
- OVERRIDE m1.
-
Error, warning, and info messages will be deleted before any transaction.
Examples:
- <<< msgs = _Messages()
+ <<< msgs = Messages()
<<< # create Message
<<< msg = Message(type="HelloWorld", code=1, description="Greeting the world", body="Hello, world!")
- <<< # append it to the _Messages
+ <<< # append it to the Messages
<<< msgs.append(msg)
- <<< # use _Messages as list of Message objects
+ <<< # use Messages as list of Message objects
<<< for m in msgs:
... assert isinstance(m,Message)
@@ -2461,29 +2729,12 @@ class _Messages(dict):
<<< msgs.append(msg)
<<< # get it back via get(...) and the key tuple (type, code)
<<< assert id(msgs.get("HelloWorld",1))==id(msg)
-
- <<< # delete Message via remove and the (type,code) tuple
- <<< msgs.remove("HelloWorld",1)
- <<< assert msgs.get("HelloWorld",1) == None
-
- <<< # short version of adding/setting/resetting a new Message
- <<< msgs["HelloWorld",2] = "Greeting the world in German", "Hallo, Welt!"
- <<< assert msgs["HelloWorld",2] == ("Greeting the world in German","Hallo, Welt!")
- <<< msgs["HelloWorld",2] = "Greeting the world in German", "Huhu, Welt!"
- <<< assert msgs["HelloWorld",2] == ("Greeting the world in German","Huhu, Welt!")
- <<< del msgs["HelloWorld",2]
- <<< assert msgs.get("HelloWorld",2) == None
-
- # this Message has no code and no description (make easy things easy...)
- <<<
- <<< msgs["HelloWorld"] = "Hello!"
- <<< assert msgs["HelloWorld"] == "Hello!"
-
- (to be continued...)
"""
def clear_server_messages(self):
- """Removes all error, warning and info messages."""
+ """Removes all messages of type error, warning and info. All other
+ messages types are custom types which should be handled by custom
+ code."""
rem = []
for m in self:
@@ -2493,9 +2744,18 @@ class _Messages(dict):
for m in rem:
self.remove(m)
- return self
-
+ #######################################################################
+ # can be removed after 01.07.24
+ # default implementation of list is sufficient
def __setitem__(self, key, value): # @ReservedAssignment
+ if not isinstance(value, Message):
+ warn("__setitem__ will in future only accept Message objects as second argument. "
+ "You will no longe be"
+ " able to pass bodys such that Message object is created on the fly",
+ DeprecationWarning)
+ if not isinstance(key, int):
+ warn("__setitem__ will in future only accept int as first argument",
+ DeprecationWarning)
if isinstance(key, tuple):
if len(key) == 2:
type = key[0] # @ReservedAssignment
@@ -2506,7 +2766,7 @@ class _Messages(dict):
else:
raise TypeError(
"('type', 'code'), ('type'), or 'type' expected.")
- elif isinstance(key, _Messages._msg_key):
+ elif isinstance(key, Messages._msg_key):
type = key._type # @ReservedAssignment
code = key._code
else:
@@ -2527,13 +2787,19 @@ class _Messages(dict):
if isinstance(value, Message):
body = value.body
description = value.description
+ m = Message
else:
body = value
description = None
- m = Message(type=type, code=code, description=description, body=body)
- dict.__setitem__(self, _Messages._msg_key(type, code), m)
+ m = Message(type=type, code=code, description=description, body=body)
+ if isinstance(key, int):
+ super().__setitem__(key, m)
+ else:
+ self.append(m)
def __getitem__(self, key):
+ if not isinstance(key, int):
+ warn("__getitem__ only supports integer keys in future.", DeprecationWarning)
if isinstance(key, tuple):
if len(key) == 2:
type = key[0] # @ReservedAssignment
@@ -2544,113 +2810,118 @@ class _Messages(dict):
else:
raise TypeError(
"('type', 'code'), ('type'), or 'type' expected.")
- elif isinstance(key, int) and int(key) >= 0:
- for m in self.values():
- if key == 0:
- return m
- else:
- key -= 1
- type = key # @ReservedAssignment
- code = None
+ elif isinstance(key, int) and key >= 0:
+ return super().__getitem__(key)
else:
type = key # @ReservedAssignment
code = None
- m = dict.__getitem__(self, _Messages._msg_key(type, code))
-
+ m = self.get(type, code)
+ if m is None:
+ raise KeyError()
if m.description:
return (m.description, m.body)
else:
return m.body
- def __init__(self):
- dict.__init__(self)
-
def __delitem__(self, key):
if isinstance(key, tuple):
- if len(key) == 2:
- type = key[0] # @ReservedAssignment
- code = key[1]
- elif len(key) == 1:
- type = key[0] # @ReservedAssignment
- code = None
- else:
- raise TypeError(
- "('type', 'code'), ('type'), or 'type' expected.")
+ warn("__delitem__ only supports integer keys in future.", DeprecationWarning)
+ if self.get(key[0], key[1]) is not None:
+ self.remove(self.get(key[0], key[1]))
else:
- type = key # @ReservedAssignment
- code = None
-
- return dict.__delitem__(self, _Messages._msg_key(type, code))
+ super().__delitem__(key)
def remove(self, obj, obj2=None):
- if isinstance(obj, Message):
- return dict.__delitem__(self, _Messages._msg_key.get(obj))
+ if obj2 is not None:
+ warn("Supplying a second argument to remove is deprecated.",
+ DeprecationWarning)
+ super().remove(self.get(obj, obj2))
+ else:
+ super().remove(obj)
- return self.__delitem__((obj, obj2))
+ def append(self, msg):
+ if isinstance(msg, Messages) or isinstance(msg, list):
+ warn("Supplying a list-like object to append is deprecated. Please use extend"
+ " instead.", DeprecationWarning)
+ for m in msg:
+ self.append(m)
+ return
- def get(self, type, code=None, default=None): # @ReservedAssignment
- try:
- return dict.__getitem__(self, _Messages._msg_key(type, code))
- except KeyError:
- return default
+ super().append(msg)
- def extend(self, messages):
- self.append(messages)
+ @staticmethod
+ def _hash(t, c):
+ return hash(str(t).lower() + (str(",") + str(c) if c is not None else ''))
+ # end remove
+ #######################################################################
- return self
+ def get(self, type, code=None, default=None, exact=False): # @ReservedAssignment
+ """
+ returns a message from the list that kind of matches type and code
- def append(self, msg):
- if hasattr(msg, "__iter__"):
- for m in msg:
- self.append(m)
+ case and types (str/int) are ignored
- return self
+ If no suitable message is found, the default argument is returned
+ If exact=True, the message has to match code and type exactly
+ """
+ if not exact:
+ warn("The fuzzy mode (exact=False) is deprecated. Please use exact in future.",
+ DeprecationWarning)
+
+ for msg in self:
+ if exact:
+ if msg.type == type and msg.code == code:
+ return msg
+ else:
+ if self._hash(msg.type, msg.code) == self._hash(type, code):
+ return msg
- if isinstance(msg, Message):
- dict.__setitem__(self, _Messages._msg_key.get(msg), msg)
+ return default
- return self
- else:
- raise TypeError("Argument was not a Message")
+ def to_xml(self, add_to_element):
+ for m in self:
+ melem = m.to_xml()
+ add_to_element.append(melem)
- return self
+ def __repr__(self):
+ xml = etree.Element("Messages")
+ self.to_xml(xml)
- def __iter__(self):
- return dict.values(self).__iter__()
+ return xml2str(xml)
+ #######################################################################
+ # can be removed after 01.07.24
class _msg_key:
def __init__(self, type, code): # @ReservedAssignment
+ warn("This class is deprecated.", DeprecationWarning)
self._type = type
self._code = code
@staticmethod
def get(msg):
- return _Messages._msg_key(msg.type, msg.code)
+ return Messages._msg_key(msg.type, msg.code)
def __eq__(self, obj):
return self.__hash__() == obj.__hash__()
def __hash__(self):
- return hash(str(self._type).lower() + (str(",") +
- str(self._code) if self._code is not None else ''))
+ return hash(str(self._type).lower() + (str(",") + str(self._code)
+ if self._code is not None else ''))
def __repr__(self):
return str(self._type) + (str(",") + str(self._code)
if self._code is not None else '')
+ # end remove
+ #######################################################################
- def to_xml(self, add_to_element):
- for m in self:
- melem = m.to_xml()
- add_to_element.append(melem)
-
- return self
- def __repr__(self):
- xml = etree.Element("Messages")
- self.to_xml(xml)
-
- return xml2str(xml)
+class _Messages(Messages):
+ def __init__(self, *args, **kwargs):
+ warn("_Messages is deprecated. "
+ "Use class Messages instead and beware of the slightly different API of the new"
+ " Messages class", DeprecationWarning)
+ super().__init__(*args, **kwargs)
def _basic_sync(e_local, e_remote):
@@ -2853,7 +3124,7 @@ class Container(list):
list.__init__(self)
self._timestamp = None
self._srid = None
- self.messages = _Messages()
+ self.messages = Messages()
def extend(self, entities):
"""Extend this Container by appending all single entities in the given
@@ -2900,9 +3171,11 @@ class Container(list):
elif isinstance(entity, QueryTemplate):
super().append(entity)
else:
- raise TypeError(
- "Entity was neither an id nor a name nor an entity." +
- " (was " + str(type(entity)) + ")")
+ warn("Entity was neither an id nor a name nor an entity." +
+ " (was " + str(type(entity)) + ":\n" + str(entity) + ")")
+ # raise TypeError(
+ # "Entity was neither an id nor a name nor an entity." +
+ # " (was " + str(type(entity)) + "\n" + str(entity) + ")")
return self
@@ -2944,11 +3217,11 @@ class Container(list):
def get_errors(self):
"""Get all error messages of this container.
- @return _Messages: Error messages.
+ @return Messages: Error messages.
"""
if self.has_errors():
- ret = _Messages()
+ ret = Messages()
for m in self.messages:
if m.type.lower() == "error":
@@ -2961,11 +3234,11 @@ class Container(list):
def get_warnings(self):
"""Get all warning messages of this container.
- @return _Messages: Warning messages.
+ @return Messages: Warning messages.
"""
if self.has_warnings():
- ret = _Messages()
+ ret = Messages()
for m in self.messages:
if m.type.lower() == "warning":
@@ -2976,7 +3249,7 @@ class Container(list):
return None
def get_all_messages(self):
- ret = _Messages()
+ ret = Messages()
for e in self:
ret.extend(e.get_all_messages())
@@ -3159,7 +3432,7 @@ class Container(list):
msg = "Request was not unique. CUID " + \
str(local_entity._cuid) + " was found " + \
str(len(sync_remote_entities)) + " times."
- local_entity.add_message(Message("Error", None, msg))
+ local_entity.add_message(Message(description=msg, type="Error"))
if raise_exception_on_error:
raise MismatchingEntitiesError(msg)
@@ -3184,7 +3457,7 @@ class Container(list):
msg = "Request was not unique. ID " + \
str(local_entity.id) + " was found " + \
str(len(sync_remote_entities)) + " times."
- local_entity.add_message(Message("Error", None, msg))
+ local_entity.add_message(Message(description=msg, type="Error"))
if raise_exception_on_error:
raise MismatchingEntitiesError(msg)
@@ -3214,7 +3487,7 @@ class Container(list):
msg = "Request was not unique. Path " + \
str(local_entity.path) + " was found " + \
str(len(sync_remote_entities)) + " times."
- local_entity.add_message(Message("Error", None, msg))
+ local_entity.add_message(Message(description=msg, type="Error"))
if raise_exception_on_error:
raise MismatchingEntitiesError(msg)
@@ -3244,7 +3517,7 @@ class Container(list):
msg = "Request was not unique. Name " + \
str(local_entity.name) + " was found " + \
str(len(sync_remote_entities)) + " times."
- local_entity.add_message(Message("Error", None, msg))
+ local_entity.add_message(Message(description=msg, type="Error"))
if raise_exception_on_error:
raise MismatchingEntitiesError(msg)
@@ -3263,7 +3536,7 @@ class Container(list):
msg = "Request was not unique. There are " + \
str(len(sync_remote_entities)) + \
" entities which could not be matched to one of the requested ones."
- remote_container.add_message(Message("Error", None, msg))
+ remote_container.add_message(Message(description=msg, type="Error"))
if raise_exception_on_error:
raise MismatchingEntitiesError(msg)
@@ -3850,6 +4123,7 @@ class Container(list):
for p in e.get_properties():
if p.id is None:
if p.name is not None:
+ # TODO using try except for normal execution flow is bad style
try:
w = self.get_entity_by_name(p.name)
p._wrap(w)
@@ -3861,6 +4135,7 @@ class Container(list):
for p in e.get_parents():
if p.id is None:
if p.name is not None:
+ # TODO using try except for normal execution flow is bad style
try:
p._wrap(self.get_entity_by_name(p.name))
except KeyError:
@@ -3949,13 +4224,15 @@ class ACI():
self.permission = permission
def __hash__(self):
- return hash(str(self.realm) + ":" + str(self.username) +
- ":" + str(self.role) + ":" + str(self.permission))
+ return hash(self.__repr__())
def __eq__(self, other):
return isinstance(other, ACI) and (self.role is None and self.username == other.username and self.realm ==
other.realm) or self.role == other.role and self.permission == other.permission
+ def __repr__(self):
+ return str(self.realm) + ":" + str(self.username) + ":" + str(self.role) + ":" + str(self.permission)
+
def add_to_element(self, e):
if self.role is not None:
e.set("role", self.role)
@@ -3980,10 +4257,35 @@ class ACL():
self.clear()
def parse_xml(self, xml):
+ """Clear this ACL and parse the xml.
+
+ Iterate over the rules in the xml and add each rule to this ACL.
+
+ Contradicting rules will both be kept.
+
+ Parameters
+ ----------
+ xml : lxml.etree.Element
+ The xml element containing the ACL rules, i.e. <Grant> and <Deny>
+ rules.
+ """
self.clear()
self._parse_xml(xml)
def _parse_xml(self, xml):
+ """Parse the xml.
+
+ Iterate over the rules in the xml and add each rule to this ACL.
+
+ Contradicting rules will both be kept.
+
+ Parameters
+ ----------
+ xml : lxml.etree.Element
+ The xml element containing the ACL rules, i.e. <Grant> and <Deny>
+ rules.
+ """
+ # @review Florian Spreckelsen 2022-03-17
for e in xml:
role = e.get("role")
username = e.get("username")
@@ -3996,10 +4298,12 @@ class ACL():
if e.tag == "Grant":
self.grant(username=username, realm=realm, role=role,
- permission=permission, priority=priority)
+ permission=permission, priority=priority,
+ revoke_denial=False)
elif e.tag == "Deny":
self.deny(username=username, realm=realm, role=role,
- permission=permission, priority=priority)
+ permission=permission, priority=priority,
+ revoke_grant=False)
def combine(self, other):
""" Combine and return new instance."""
@@ -4077,12 +4381,42 @@ class ACL():
if item in self._denials:
self._denials.remove(item)
- def grant(self, username=None, realm=None, role=None,
- permission=None, priority=False):
+ def grant(self, permission, username=None, realm=None, role=None,
+ priority=False, revoke_denial=True):
+ """Grant a permission to a user or role.
+
+ You must specify either only the username and the realm, or only the
+ role.
+
+ By default a previously existing denial rule would be revoked, because
+ otherwise this grant wouldn't have any effect. However, for keeping
+ contradicting rules pass revoke_denial=False.
+
+ Parameters
+ ----------
+ permission: str
+ The permission to be granted.
+ username : str, optional
+ The username. Exactly one is required, either the `username` or the
+ `role`.
+ realm: str, optional
+ The user's realm. Required when username is not None.
+ role: str, optional
+ The role (as in Role-Based Access Control). Exactly one is
+ required, either the `username` or the `role`.
+ priority: bool, default False
+ Whether this permission is granted with priority over non-priority
+ rules.
+ revoke_denial: bool, default True
+ Whether a contradicting denial (with same priority flag) in this
+ ACL will be revoked.
+ """
+ # @review Florian Spreckelsen 2022-03-17
priority = self._get_boolean_priority(priority)
item = ACI(role=role, username=username,
realm=realm, permission=permission)
- self._remove_item(item, priority)
+ if revoke_denial:
+ self._remove_item(item, priority)
if priority is True:
self._priority_grants.add(item)
@@ -4090,11 +4424,41 @@ class ACL():
self._grants.add(item)
def deny(self, username=None, realm=None, role=None,
- permission=None, priority=False):
+ permission=None, priority=False, revoke_grant=True):
+ """Deny a permission to a user or role for this entity.
+
+ You must specify either only the username and the realm, or only the
+ role.
+
+ By default a previously existing grant rule would be revoked, because
+ otherwise this denial would override the grant rules anyways. However,
+ for keeping contradicting rules pass revoke_grant=False.
+
+ Parameters
+ ----------
+ permission: str
+ The permission to be denied.
+ username : str, optional
+ The username. Exactly one is required, either the `username` or the
+ `role`.
+ realm: str, optional
+ The user's realm. Required when username is not None.
+ role: str, optional
+ The role (as in Role-Based Access Control). Exactly one is
+ required, either the `username` or the `role`.
+ priority: bool, default False
+ Whether this permission is denied with priority over non-priority
+ rules.
+ revoke_grant: bool, default True
+ Whether a contradicting grant (with same priority flag) in this
+ ACL will be revoked.
+ """
+ # @review Florian Spreckelsen 2022-03-17
priority = self._get_boolean_priority(priority)
item = ACI(role=role, username=username,
realm=realm, permission=permission)
- self._remove_item(item, priority)
+ if revoke_grant:
+ self._remove_item(item, priority)
if priority is True:
self._priority_denials.add(item)
@@ -4235,7 +4599,7 @@ class Query():
The query string.
flags : dict of str
A dictionary of flags to be send with the query request.
- messages : _Messages()
+ messages : Messages()
A container of messages included in the last query response.
cached : bool
indicates whether the server used the query cache for the execution of
@@ -4258,7 +4622,7 @@ class Query():
def __init__(self, q):
self.flags = dict()
- self.messages = _Messages()
+ self.messages = Messages()
self.cached = None
self.etag = None
@@ -4357,7 +4721,7 @@ def execute_query(q, unique=False, raise_exception_on_error=True, cache=True, fl
Whether the query is expected to have only one entity as result.
Defaults to False.
raise_exception_on_error : bool
- Whether an exception should be raises when there are errors in the
+ Whether an exception should be raised when there are errors in the
resulting entities. Defaults to True.
cache : bool
Whether to use the query cache (equivalent to adding a "cache" flag).
@@ -4382,6 +4746,10 @@ def execute_query(q, unique=False, raise_exception_on_error=True, cache=True, fl
class DropOffBox(list):
+ def __init__(self, *args, **kwargs):
+ warn(DeprecationWarning(
+ "The DropOffBox is deprecated and will be removed in future."))
+ super().__init__(*args, **kwargs)
path = None
@@ -4425,7 +4793,7 @@ class UserInfo():
class Info():
def __init__(self):
- self.messages = _Messages()
+ self.messages = Messages()
self.sync()
def sync(self):
@@ -4444,6 +4812,8 @@ class Info():
if isinstance(m, UserInfo):
self.user_info = m
+ elif isinstance(m, TimeZone):
+ self.time_zone = m
else:
self.messages.append(m)
@@ -4570,13 +4940,16 @@ def _parse_single_xml_element(elem):
elif elem.tag.lower() == 'stats':
counts = elem.find("counts")
- return Message(type="Counts", body=counts.attrib)
+ return Message(type="Counts", description=None, body=counts.attrib)
elif elem.tag == "EntityACL":
return ACL(xml=elem)
elif elem.tag == "Permissions":
return Permissions(xml=elem)
elif elem.tag == "UserInfo":
return UserInfo(xml=elem)
+ elif elem.tag == "TimeZone":
+ return TimeZone(zone_id=elem.get("id"), offset=elem.get("offset"),
+ display_name=elem.text.strip())
else:
return Message(type=elem.tag, code=elem.get(
"code"), description=elem.get("description"), body=elem.text)
diff --git a/src/caosdb/common/timezone.py b/src/caosdb/common/timezone.py
new file mode 100644
index 0000000000000000000000000000000000000000..8fc5e710d3cbf6f20cf81397573f972db3b22f12
--- /dev/null
+++ b/src/caosdb/common/timezone.py
@@ -0,0 +1,19 @@
+class TimeZone():
+ """
+ TimeZone, e.g. CEST, Europe/Berlin, UTC+4.
+
+
+ Attributes
+ ----------
+ zone_id : string
+ ID of the time zone.
+ offset : int
+ Offset to UTC in seconds.
+ display_name : string
+ A human-friendly name of the time zone:
+ """
+
+ def __init__(self, zone_id, offset, display_name):
+ self.zone_id = zone_id
+ self.offset = offset
+ self.display_name = display_name
diff --git a/src/caosdb/common/utils.py b/src/caosdb/common/utils.py
index eabce3139ccabb35412b5dbe0fb83721fc18179a..f0ce740d38d90b0c7bb1031e808b83efb2207a43 100644
--- a/src/caosdb/common/utils.py
+++ b/src/caosdb/common/utils.py
@@ -26,7 +26,6 @@
from lxml import etree
from multiprocessing import Lock
from uuid import uuid4
-from sys import hexversion
_uuid_lock = Lock()
diff --git a/src/caosdb/configuration.py b/src/caosdb/configuration.py
index 51e3749aaca3045afec9334ef987a174d5d19f26..1c108ac1d39c135dbc90f477be8b8f2f630391ce 100644
--- a/src/caosdb/configuration.py
+++ b/src/caosdb/configuration.py
@@ -31,12 +31,7 @@ try:
except ImportError:
pass
-try:
- # python2
- from ConfigParser import ConfigParser
-except ImportError:
- # python3
- from configparser import ConfigParser
+from configparser import ConfigParser
from os import environ, getcwd
from os.path import expanduser, join, isfile
@@ -59,6 +54,11 @@ def configure(inifile):
_reset_config()
read_config = _pycaosdbconf.read(inifile)
validate_yaml_schema(config_to_yaml(_pycaosdbconf))
+
+ if "HTTPS_PROXY" in environ:
+ _pycaosdbconf["Connection"]["https_proxy"] = environ["HTTPS_PROXY"]
+ if "HTTP_PROXY" in environ:
+ _pycaosdbconf["Connection"]["http_proxy"] = environ["HTTP_PROXY"]
return read_config
@@ -84,23 +84,28 @@ def config_to_yaml(config):
def validate_yaml_schema(valobj):
- # TODO: Re-enable warning once the schema has been extended to also cover
- # SSS pycaosdb.inis and integration tests.
if optional_jsonschema_validate:
with open(os.path.join(os.path.dirname(__file__), "schema-pycaosdb-ini.yml")) as f:
schema = yaml.load(f, Loader=yaml.SafeLoader)
optional_jsonschema_validate(instance=valobj, schema=schema["schema-pycaosdb-ini"])
- # else:
- # warnings.warn("""
- # Warning: The validation could not be performed because `jsonschema` is not installed.
- # """)
+ else:
+ warnings.warn("""
+ Warning: The validation could not be performed because `jsonschema` is not installed.
+ """)
def _read_config_files():
- """Function to read config files from different paths. Checks for path in $PYCAOSDBINI or home directory (.pycaosdb.ini) and in the current working directory (pycaosdb.ini).
+ """Function to read config files from different paths.
+
+ Checks for path either in ``$PYCAOSDBINI`` or home directory (``.pycaosdb.ini``), and
+ additionally in the current working directory (``pycaosdb.ini``).
+
+ Returns
+ -------
+
+ ini files: list
+ The successfully parsed ini-files. Order: env_var or home directory, cwd. Used for testing the function.
- Returns:
- [list]: list with successfully parsed ini-files. Order: env_var or home directory, cwd. Used for testing the function.
"""
return_var = []
if "PYCAOSDBINI" in environ:
diff --git a/src/caosdb/connection/SocksiPy.zip b/src/caosdb/connection/SocksiPy.zip
deleted file mode 100644
index e81f1f9393c766a3acd41b44245f9e17f090cbe5..0000000000000000000000000000000000000000
Binary files a/src/caosdb/connection/SocksiPy.zip and /dev/null differ
diff --git a/src/caosdb/connection/__init__.py b/src/caosdb/connection/__init__.py
index 638d4f0a8400f0ea1a2f197dcfdbe3fc933d4c10..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 100644
--- a/src/caosdb/connection/__init__.py
+++ b/src/caosdb/connection/__init__.py
@@ -1,29 +0,0 @@
-# -*- encoding: utf-8 -*-
-#
-# ** header v3.0
-# This file is a part of the CaosDB Project.
-#
-# Copyright (C) 2018 Research Group Biomedical Physics,
-# Max-Planck-Institute for Dynamics and Self-Organization Göttingen
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Affero General Public License as
-# published by the Free Software Foundation, either version 3 of the
-# License, or (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU Affero General Public License for more details.
-#
-# You should have received a copy of the GNU Affero General Public License
-# along with this program. If not, see <https://www.gnu.org/licenses/>.
-#
-# ** end header
-#
-from sys import hexversion
-from .utils import check_python_ssl_version
-
-check_python_ssl_version(hexversion)
-del check_python_ssl_version
-del hexversion
diff --git a/src/caosdb/connection/authentication/keyring.py b/src/caosdb/connection/authentication/keyring.py
index d8be7ddf030577545230c9111fdad542b6d6e7e2..99d184136c20b23557efea0b54c648095a8d3ab2 100644
--- a/src/caosdb/connection/authentication/keyring.py
+++ b/src/caosdb/connection/authentication/keyring.py
@@ -28,7 +28,7 @@ retrieve the password.
"""
import sys
-import imp
+import importlib
from getpass import getpass
from caosdb.exceptions import ConfigurationError
from .external_credentials_provider import ExternalCredentialsProvider
@@ -52,17 +52,13 @@ def get_authentication_provider():
def _get_external_keyring():
try:
- fil, pathname, desc = imp.find_module("keyring", sys.path[1:])
- module = imp.load_module("external_keyring", fil, pathname, desc)
- return module
+ return importlib.import_module("keyring")
except ImportError:
raise RuntimeError(
"The keyring password method requires installation of the"
"keyring python package. On linux with python < 3.5, "
"this requires the installation of dbus-python as a "
"system package.")
- finally:
- fil.close()
def _call_keyring(**config):
@@ -74,7 +70,6 @@ def _call_keyring(**config):
url = config.get("url")
username = config.get("username")
app = "caosdb — {}".format(url)
- password = _call_keyring(app=app, username=username)
external_keyring = _get_external_keyring()
password = external_keyring.get_password(app, username)
if password is None:
diff --git a/src/caosdb/connection/connection.py b/src/caosdb/connection/connection.py
index 6c3946ee639872b0edb0b5b3c30a808cf8c028d4..46dadea9dfcfa6e614493b75d709f604aa188ef6 100644
--- a/src/caosdb/connection/connection.py
+++ b/src/caosdb/connection/connection.py
@@ -28,9 +28,15 @@ from __future__ import absolute_import, print_function, unicode_literals
import logging
import ssl
import sys
+import warnings
from builtins import str # pylint: disable=redefined-builtin
from errno import EPIPE as BrokenPipe
from socket import error as SocketError
+from urllib.parse import quote, urlparse
+from requests import Session as HTTPSession
+from requests.exceptions import ConnectionError as HTTPConnectionError
+from urllib3.poolmanager import PoolManager
+from requests.adapters import HTTPAdapter
from caosdb.configuration import get_config
from caosdb.exceptions import (CaosDBException, HTTPClientError,
@@ -49,16 +55,8 @@ except ModuleNotFoundError:
from pkg_resources import resource_filename
from .interface import CaosDBHTTPResponse, CaosDBServerConnection
-from .streaminghttp import StreamingHTTPSConnection
from .utils import make_uri_path, parse_url, urlencode
-
-try:
- from urllib.parse import quote, urlparse
-except ImportError:
- from urllib import quote
- from urlparse import urlparse
-
-# pylint: disable=missing-docstring
+from .encode import MultipartYielder, ReadableMultiparts
_LOGGER = logging.getLogger(__name__)
@@ -67,6 +65,9 @@ class _WrappedHTTPResponse(CaosDBHTTPResponse):
def __init__(self, response):
self.response = response
+ self._generator = None
+ self._buffer = b''
+ self._stream_consumed = False
@property
def reason(self):
@@ -74,21 +75,71 @@ class _WrappedHTTPResponse(CaosDBHTTPResponse):
@property
def status(self):
- return self.response.status
+ return self.response.status_code
def read(self, size=None):
- return self.response.read(size)
+ if self._stream_consumed is True:
+ raise RuntimeError("Stream is consumed")
+
+ if self._buffer is None:
+ # the buffer has been drained in the previous call.
+ self._stream_consumed = True
+ return b''
+
+ if self._generator is None and (size is None or size == 0):
+ # return full content at once
+ self._stream_consumed = True
+ return self.response.content
+
+ if len(self._buffer) >= size:
+ # still enough bytes in the buffer
+ result = chunk[:size]
+ self._buffer = chunk[size:]
+ return result
+
+ if self._generator is None:
+ # first call to this method
+ if size is None or size == 0:
+ size = 512
+ self._generator = self.response.iter_content(size)
+
+ try:
+ # read new data into the buffer
+ chunk = self._buffer + next(self._generator)
+ result = chunk[:size]
+ if len(result) == 0:
+ self._stream_consumed = True
+ self._buffer = chunk[size:]
+ return result
+ except StopIteration:
+ # drain buffer
+ result = self._buffer
+ self._buffer = None
+ return result
def getheader(self, name, default=None):
- return self.response.getheader(name=name, default=default)
+ return self.response.headers[name] if name in self.response.headers else default
def getheaders(self):
- return self.response.getheaders()
+ return self.response.headers.items()
def close(self):
self.response.close()
+class _SSLAdapter(HTTPAdapter):
+ """Transport adapter that allows us to use different SSL versions."""
+
+ def __init__(self, ssl_version):
+ self.ssl_version = ssl_version
+ super().__init__()
+
+ def init_poolmanager(self, connections, maxsize, block=False):
+ self.poolmanager = PoolManager(
+ num_pools=connections, maxsize=maxsize,
+ block=block, ssl_version=self.ssl_version)
+
+
class _DefaultCaosDBServerConnection(CaosDBServerConnection):
"""_DefaultCaosDBServerConnection.
@@ -101,10 +152,11 @@ class _DefaultCaosDBServerConnection(CaosDBServerConnection):
def __init__(self):
self._useragent = ("caosdb-pylib/{version} - {implementation}".format(
version=version, implementation=type(self).__name__))
- self._http_con = None
self._base_path = None
+ self._session = None
+ self._timeout = None
- def request(self, method, path, headers=None, body=None, **kwargs):
+ def request(self, method, path, headers=None, body=None):
"""request.
Send a HTTP request to the server.
@@ -118,38 +170,40 @@ class _DefaultCaosDBServerConnection(CaosDBServerConnection):
including query and frament segments.
headers : dict of str -> str, optional
HTTP request headers. (Defautl: None)
- body : str or bytes or readable, opional
+ body : str or bytes or readable, optional
The body of the HTTP request. Bytes should be a utf-8 encoded
string.
- **kwargs :
- Any keyword arguments will be ignored.
- TODO: Why are they allowed then?
Returns
-------
- TODO: What?
+ response : CaosDBHTTPResponse
"""
if headers is None:
headers = {}
headers["User-Agent"] = self._useragent
+
+ if path.endswith("/."):
+ path = path[:-1] + "%2E"
+
+ if isinstance(body, MultipartYielder):
+ body = ReadableMultiparts(body)
+
try:
- self._http_con = StreamingHTTPSConnection(
- # TODO looks as if configure needs to be done first.
- # That is however not assured.
- host=self.setup_fields["host"],
- timeout=self.setup_fields["timeout"],
- context=self.setup_fields["context"],
- socket_proxy=self.setup_fields["socket_proxy"])
- self._http_con.request(method=method, url=self._base_path + path,
- headers=headers, body=body)
- except SocketError as socket_err:
+ response = self._session.request(
+ method=method,
+ url=self._base_path + path,
+ headers=headers,
+ data=body,
+ timeout=self._timeout,
+ stream=True)
+
+ return _WrappedHTTPResponse(response)
+ except HTTPConnectionError as conn_err:
raise CaosDBConnectionError(
- "Connection failed. Network or server down? " + str(socket_err)
+ "Connection failed. Network or server down? " + str(conn_err)
)
- return _WrappedHTTPResponse(self._http_con.getresponse())
-
def configure(self, **config):
"""configure.
@@ -173,64 +227,71 @@ class _DefaultCaosDBServerConnection(CaosDBServerConnection):
loaded.
"""
+ if "url" not in config:
+ raise CaosDBConnectionError(
+ "No connection url specified. Please "
+ "do so via caosdb.configure_connection(...) or in a config "
+ "file.")
+ if (not config["url"].lower().startswith("https://") and not config["url"].lower().startswith("http://")):
+ raise CaosDBConnectionError("The connection url is expected "
+ "to be a http or https url and "
+ "must include the url scheme "
+ "(i.e. start with https:// or "
+ "http://).")
+
+ url = urlparse(config["url"])
+ path = url.path.strip("/")
+ if len(path) > 0:
+ path = path + "/"
+ self._base_path = url.scheme + "://" + url.netloc + "/" + path
+
+ self._session = HTTPSession()
+
+ if url.scheme == "https":
+ self._setup_ssl(config)
+
+ # TODO(tf) remove in next release
+ socket_proxy = config["socket_proxy"] if "socket_proxy" in config else None
+ if socket_proxy is not None:
+ self._session.proxies = {
+ "https": "socks5://" + socket_proxy,
+ "http": "socks5://" + socket_proxy,
+ }
+
+ if "https_proxy" in config:
+ if self._session.proxies is None:
+ self._session.proxies = {}
+ self._session.proxies["https"] = config["https_proxy"]
+
+ if "http_proxy" in config:
+ if self._session.proxies is None:
+ self._session.proxies = {}
+ self._session.proxies["http"] = config["http_proxy"]
+
+ if "timeout" in config:
+ self._timeout = config["timeout"]
+
+ def _setup_ssl(self, config):
if "ssl_version" in config and config["cacert"] is not None:
ssl_version = getattr(ssl, config["ssl_version"])
else:
- try:
- ssl_version = ssl.PROTOCOL_TLS
- except AttributeError:
- # deprecated since Python 3.6, but necessary until there.
- try:
- ssl_version = ssl.PROTOCOL_TLSv1_2
- except AttributeError:
- print("It seems as if your Python version does not support current encryption"
- "standards (such as TLS > 1.0), please upgrade to at least Python 3.4.")
- sys.exit(1)
- context = ssl.SSLContext(ssl_version)
- context.verify_mode = ssl.CERT_REQUIRED
-
- if config.get("ssl_insecure"):
+ ssl_version = ssl.PROTOCOL_TLS
+
+ self._session.mount(self._base_path, _SSLAdapter(ssl_version))
+
+ verify = True
+ if "cacert" in config:
+ verify = config["cacert"]
+ if "ssl_insecure" in config and config["ssl_insecure"]:
_LOGGER.warning("*** Warning! ***\n"
"Insecure SSL mode, certificate will not be checked! "
"Please consider removing the `ssl_insecure` configuration option.\n"
"****************")
- context.verify_mode = ssl.CERT_NONE
-
- if (not context.verify_mode == ssl.CERT_NONE and
- hasattr(context, "check_hostname")):
- context.check_hostname = True
-
- if ("cacert" in config and config["cacert"] is not None and
- config["cacert"]):
- try:
- context.load_verify_locations(config["cacert"])
- except Exception as exc:
- raise CaosDBConnectionError("Could not load the cacert in"
- "`{}`: {}".format(config["cacert"],
- exc))
-
- context.load_default_certs()
-
- if "url" in config:
- parsed_url = parse_url(config["url"])
- host = parsed_url.netloc
- self._base_path = parsed_url.path
- else:
- raise CaosDBConnectionError(
- "No connection url specified. Please "
- "do so via caosdb.configure_connection(...) or in a config "
- "file.")
-
- socket_proxy = None
-
- if "socket_proxy" in config:
- socket_proxy = config["socket_proxy"]
-
- self.setup_fields = {
- "host": host,
- "timeout": int(config.get("timeout")),
- "context": context,
- "socket_proxy": socket_proxy}
+ warnings.filterwarnings(action="ignore", module="urllib3",
+ message="Unverified HTTPS request is being made")
+ verify = False
+ if verify is not None:
+ self._session.verify = verify
def _make_conf(*conf):
@@ -261,7 +322,6 @@ _DEFAULT_CONF = {
"password_method": "input",
"implementation": _DefaultCaosDBServerConnection,
"timeout": 210,
- "cacert": resource_filename("caosdb", 'cert/indiscale.ca.crt')
}
@@ -323,6 +383,10 @@ def configure_connection(**kwargs):
Parameters
----------
+ url : str
+ The url of the CaosDB Server. HTTP and HTTPS urls are allowed. However,
+ it is **highly** recommend to avoid HTTP because passwords and
+ authentication token are send over the network in plain text.
username : str
Username for login; e.g. 'admin'.
@@ -351,6 +415,24 @@ def configure_connection(**kwargs):
An authentication token which has been issued by the CaosDB Server.
Implies `password_method="auth_token"` if set. An example token string would be `["O","OneTimeAuthenticationToken","anonymous",["administration"],[],1592995200000,604800000,"3ZZ4WKRB-5I7DG2Q6-ZZE6T64P-VQ","197d0d081615c52dc18fb323c300d7be077beaad4020773bb58920b55023fa6ee49355e35754a4277b9ac525c882bcd3a22e7227ba36dfcbbdbf8f15f19d1ee9",1,30000]`.
+ https_proxy : str, optional
+ Define a proxy for the https connections, e.g. `http://localhost:8888`,
+ `socks5://localhost:8888`, or `socks4://localhost:8888`. These are
+ either (non-TLS) HTTP proxies, SOCKS4 proxies, or SOCKS5 proxies. HTTPS
+ proxies are not supported. However, the connection will be secured
+ using TLS in the tunneled connection nonetheless. Only the connection
+ to the proxy is insecure which is why it is not recommended to use HTTP
+ proxies when authentication against the proxy is necessary. If
+ unspecified, the https_proxy option of the pycaosdb.ini or the HTTPS_PROXY
+ environment variable are being used. Use `None` to override these
+ options with a no-proxy setting.
+
+ http_proxy : str, optional
+ Define a proxy for the http connections, e.g. `http://localhost:8888`.
+ If unspecified, the http_proxy option of the pycaosdb.ini or the
+ HTTP_PROXY environment variable are being used. Use `None` to override
+ these options with a no-proxy setting.
+
implementation : CaosDBServerConnection
The class which implements the connection. (Default:
_DefaultCaosDBServerConnection)
@@ -381,6 +463,11 @@ def configure_connection(**kwargs):
local_conf = _make_conf(_DEFAULT_CONF, global_conf, kwargs)
connection = _Connection.get_instance()
+
+ if "socket_proxy" in local_conf:
+ warnings.warn("Deprecated configuration option: socket_proxy. Use "
+ "the new https_proxy option instead",
+ DeprecationWarning, stacklevel=1)
connection.configure(**local_conf)
return connection
@@ -608,7 +695,7 @@ class _Connection(object): # pylint: disable=useless-object-inheritance
method=method,
path=path,
headers=headers,
- body=body, **kwargs)
+ body=body)
_LOGGER.debug("response: %s %s", str(http_response.status),
str(http_response.getheaders()))
self._authenticator.on_response(http_response)
diff --git a/src/caosdb/connection/encode.py b/src/caosdb/connection/encode.py
index 970c4191e1defc55b2b4a6c7d01e0c9d4ba2952f..0b826cc4400275a2374308ee104cdbdabb619b75 100644
--- a/src/caosdb/connection/encode.py
+++ b/src/caosdb/connection/encode.py
@@ -51,21 +51,12 @@ multipart/form-data is the standard way to upload files over HTTP
__all__ = [
'gen_boundary', 'encode_and_quote', 'MultipartParam', 'encode_string',
- 'encode_file_header', 'get_body_size', 'get_headers', 'multipart_encode'
+ 'encode_file_header', 'get_body_size', 'get_headers', 'multipart_encode',
+ 'ReadableMultiparts',
]
-from sys import hexversion
-try:
- from urllib.parse import quote_plus
-except ImportError:
- from urllib import quote_plus
-
-try:
- from io import UnsupportedOperation
-except ImportError:
- UnsupportedOperation = None
-
+from urllib.parse import quote_plus
+from io import UnsupportedOperation
import uuid
-
import re
import os
import mimetypes
@@ -83,24 +74,9 @@ def encode_and_quote(data):
if data is None:
return None
- if hexversion < 0x03000000:
- if isinstance(data, unicode):
- data = data.encode("utf-8")
return quote_plus(data)
-def _strify(string):
- """If string is a unicode string, encode it to UTF-8 and return the
- results, otherwise return str(s), or None if s is None."""
- if string is None:
- return None
- if hexversion < 0x03000000:
- if isinstance(string, unicode):
- return string.encode("utf-8")
- return str(string)
- return str(string)
-
-
class MultipartParam(object):
"""Represents a single parameter in a multipart/form-data request.
@@ -143,22 +119,14 @@ class MultipartParam(object):
fileobj=None,
callback=None):
self.name = Header(name).encode()
- self.value = _strify(value)
+ self.value = value
if filename is None:
self.filename = None
- elif hexversion < 0x03000000:
- if isinstance(filename, unicode):
- # Encode with XML entities
- self.filename = filename.encode("ascii", "xmlcharrefreplace")
- else:
- self.filename = str(filename)
- self.filename = self.filename.encode("string_escape").\
- replace('"', '\\"')
else:
bfilename = filename.encode("ascii", "xmlcharrefreplace")
self.filename = bfilename.decode("UTF-8").replace('"', '\\"')
- self.filetype = _strify(filetype)
+ self.filetype = filetype
self.filesize = filesize
self.fileobj = fileobj
@@ -508,3 +476,40 @@ def multipart_encode(params, boundary=None, callback=None):
params = MultipartParam.from_params(params)
return MultipartYielder(params, boundary, callback), headers
+
+
+class ReadableMultiparts(object):
+ """Wraps instances of the MultipartYielder class as a readable and withable
+ object."""
+
+ def __init__(self, multipart_yielder):
+ self.multipart_yielder = multipart_yielder
+ self.current_block = None
+ self.left_over = b''
+
+ def read(self, size=-1):
+ result = self.left_over
+ while size == -1 or len(result) < size:
+ try:
+ next_chunk = self.multipart_yielder.next()
+ if hasattr(next_chunk, "encode"):
+ next_chunk = next_chunk.encode("utf8")
+ result += next_chunk
+ except StopIteration:
+ break
+
+ if size == -1:
+ self.left_over = b''
+ return result
+
+ self.left_over = result[size:]
+ return result[:size]
+
+ def __enter__(self):
+ pass
+
+ def __exit__(self, type, value, traceback):
+ self.close()
+
+ def close(self):
+ self.multipart_yielder.reset()
diff --git a/src/caosdb/connection/mockup.py b/src/caosdb/connection/mockup.py
index 692ec2b13f16556a9acfab2a377a04aaf27d650c..b37670b867cd88cf47e64084c6ccc802cad463b4 100644
--- a/src/caosdb/connection/mockup.py
+++ b/src/caosdb/connection/mockup.py
@@ -56,7 +56,7 @@ class MockUpResponse(CaosDBHTTPResponse):
def read(self, size=-1):
"""Return the body of the response."""
- return self.response.read(size)
+ return self.response.read(size).encode()
def getheader(self, name, default=None):
"""Get the contents of the header `name`, or `default` if there is no
diff --git a/src/caosdb/connection/streaminghttp.py b/src/caosdb/connection/streaminghttp.py
deleted file mode 100644
index 01774301b9bdb55bdbf6b56695042aaf354dba97..0000000000000000000000000000000000000000
--- a/src/caosdb/connection/streaminghttp.py
+++ /dev/null
@@ -1,152 +0,0 @@
-# -*- encoding: utf-8 -*-
-#
-# ** header v3.0
-# This file is a part of the CaosDB Project.
-#
-# Copyright (C) 2018 Research Group Biomedical Physics,
-# Max-Planck-Institute for Dynamics and Self-Organization Göttingen
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Affero General Public License as
-# published by the Free Software Foundation, either version 3 of the
-# License, or (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU Affero General Public License for more details.
-#
-# You should have received a copy of the GNU Affero General Public License
-# along with this program. If not, see <https://www.gnu.org/licenses/>.
-#
-# ** end header
-#
-# Original work Copyright (c) 2011 Chris AtLee
-# Modified work Copyright (c) 2017 Biomedical Physics, MPI for Dynamics and Self-Organization
-#
-# Permission is hereby granted, free of charge, to any person obtaining a copy
-# of this software and associated documentation files (the "Software"), to deal
-# in the Software without restriction, including without limitation the rights
-# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-# copies of the Software, and to permit persons to whom the Software is
-# furnished to do so, subject to the following conditions:
-#
-# The above copyright notice and this permission notice shall be included in
-# all copies or substantial portions of the Software.
-#
-# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
-# THE SOFTWARE.
-"""Streaming HTTP uploads module.
-
-This module extends the standard httplib and http.client HTTPConnection so that
-iterable objects can be used in the body of HTTP requests.
-
-**N.B.** You must specify a Content-Length header if using an iterable object
-since there is no way to determine in advance the total size that will be
-yielded, and there is no way to reset an interator.
-"""
-
-from __future__ import unicode_literals, print_function, absolute_import
-import socks
-import socket
-try:
- # python3
- from http import client as client
-except ImportError:
- # python2
- import httplib as client
-
-
-__all__ = ['StreamingHTTPSConnection']
-
-
-class StreamingHTTPSConnection(client.HTTPSConnection, object):
- """Subclass of `http.client.HTTSConnection` or `httplib.HTTPSConnection`
- that overrides the `send()` method to support iterable body objects."""
- # pylint: disable=unused-argument, arguments-differ
-
- def __init__(self, socket_proxy=None, **kwargs):
- if socket_proxy is not None:
- host, port = socket_proxy.split(":")
- socks.setdefaultproxy(socks.PROXY_TYPE_SOCKS5, host,
- int(port))
- socket.socket = socks.socksocket
- super(StreamingHTTPSConnection, self).__init__(**kwargs)
-
- def _send_output(self, body, **kwargs):
- """Send the currently buffered request and clear the buffer.
-
- Appends an extra \\r\\n to the buffer.
- A message_body may be specified, to be appended to the request.
-
- This method is implemented in differently in the various python
- versions (which is extremely annoying). So we provide a unified but
- relatively dumb implementaion which only serves our needs.
- """
- self._buffer.extend(("".encode("utf-8"), "".encode("utf-8")))
- headers = "\r\n".encode("utf-8").join(self._buffer)
- del self._buffer[:]
-
- self.send(headers)
- if body is not None:
- self.send(body)
-
- # pylint: disable=too-many-branches
-
- def send(self, value):
- """Send ``value`` to the server.
-
- ``value`` can be a string-like object which supports a 'encode' method,
- a file-like object that supports a .read() method, or an iterable object
- that supports a .next() method.
-
- An encode()able ``value`` will be utf-8 encoded before sending.
- """
- # Based on python 2.6's httplib.HTTPConnection.send()
- if self.sock is None:
- if self.auto_open:
- self.connect()
- else:
- raise client.NotConnected()
-
- # send the data to the server. if we get a broken pipe, then close
- # the socket. we want to reconnect when somebody tries to send again.
- #
- # NOTE: we DO propagate the error, though, because we cannot simply
- # ignore the error... the caller will know if they can retry.
- if self.debuglevel > 0:
- print("send: ", repr(value))
- try:
- blocksize = 8192
- if hasattr(value, 'read'):
- if hasattr(value, 'seek'):
- value.seek(0)
- if self.debuglevel > 0:
- print("sendIng a read()able")
- data = value.read(blocksize)
- while data:
- self.sock.sendall(data)
- data = value.read(blocksize)
- elif hasattr(value, 'next'):
- if hasattr(value, 'reset'):
- value.reset()
- if self.debuglevel > 0:
- print("sendIng an iterable")
- for data in value:
- if hasattr(data, "encode"):
- self.sock.sendall(data.encode('utf-8'))
- else:
- self.sock.sendall(data)
- else:
- if self.debuglevel > 0:
- print("sendIng a byte-like")
- self.sock.sendall(value)
- except socket.error as err:
- if err.args[0] == 32: # Broken pipe
- self.close()
- raise
diff --git a/src/caosdb/connection/utils.py b/src/caosdb/connection/utils.py
index 8c1518c1ba66b45c69d5b9fa0d137f0df633cd0c..095d47035e24dad5b6d7041f5d3b8a739652f271 100644
--- a/src/caosdb/connection/utils.py
+++ b/src/caosdb/connection/utils.py
@@ -23,19 +23,9 @@
#
"""Utility functions for the connection module."""
from __future__ import unicode_literals, print_function
-try:
- from builtins import str, unicode # pylint: disable=redefined-builtin
-except ImportError:
- from builtins import str as unicode
-try: # pragma: no cover
- # python3
- from urllib.parse import (urlencode as _urlencode, quote as _quote,
- urlparse, urlunparse, unquote as _unquote)
-except ImportError: # pragma: no cover
- # python2
- from urllib import (urlencode as _urlencode, quote as _quote, unquote as
- _unquote)
- from urlparse import urlparse, urlunparse
+from builtins import str as unicode
+from urllib.parse import (urlencode as _urlencode, quote as _quote,
+ urlparse, urlunparse, unquote as _unquote)
import re
@@ -55,13 +45,16 @@ def urlencode(query):
3) All other parameters which can be passed to the respective functions are
not implemented here and the default parameters will be used.
+
+.. code::
+
>>> urlencode({'key': ['val1', 'val2']}, doseq=True)
Traceback (most recent call last):
...
TypeError: urlencode() got an unexpected keyword argument 'doseq'
- Otherwise, this functions works exactly as its counterparts in the urllib
- modules when they are called with only the query parameter.
+Otherwise, this functions works exactly as its counterparts in the urllib
+modules when they are called with only the query parameter.
Parameters
----------
@@ -122,7 +115,7 @@ def make_uri_path(segments=None, query=None):
def quote(string):
- enc = unicode(string).encode('utf-8')
+ enc = string.encode('utf-8')
return _quote(enc).replace('/', '%2F')
@@ -136,35 +129,6 @@ def parse_url(url):
return fullurl
-def check_python_ssl_version(hexversion):
- """Check the python version.
-
- If `version < 2.7.9` or `3.0 <= version < 3.2` the ssl library does not
- actually verify the ssl certificates send by the server. That is evil and
- these versions shall not be used.
-
- Parameters
- ----------
- hexversion : int
- A python version.
-
- Raises
- ------
- Exception
- If the version does not fully support ssl encryption.
- """
- if hexversion < 0x02070900:
- raise Exception(
- "version " + str(hex(hexversion)) +
- "\nPython version is smaller than 2.7.9. It is not does not fully support SSL encryption. Please update your Python to 2.7.9 or greater, or 3.2 or greater."
- )
- elif hexversion >= 0x03000000 and hexversion < 0x03020000:
- raise Exception(
- "version " + str(hex(hexversion)) +
- "\nPython 3 version is smaller than 3.2. It is not does not fully support SSL encryption. Please update your Python to 2.7.9 or greater, or 3.2 or greater."
- )
-
-
_PATTERN = re.compile(r"^SessionToken=([^;]*);.*$")
diff --git a/src/caosdb/high_level_api.py b/src/caosdb/high_level_api.py
new file mode 100644
index 0000000000000000000000000000000000000000..3509a7b6bfe7ec322f2e0d2590334c6fc6f02cf8
--- /dev/null
+++ b/src/caosdb/high_level_api.py
@@ -0,0 +1,1053 @@
+# -*- coding: utf-8 -*-
+#
+# This file is a part of the CaosDB Project.
+#
+# Copyright (C) 2018 Research Group Biomedical Physics,
+# Max-Planck-Institute for Dynamics and Self-Organization Göttingen
+# Copyright (C) 2020 Timm Fitschen <t.fitschen@indiscale.com>
+# Copyright (C) 2020-2022 IndiScale GmbH <info@indiscale.com>
+# Copyright (C) 2022 Alexander Schlemmer <alexander.schlemmer@ds.mpg.de>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see <https://www.gnu.org/licenses/>.
+#
+# ** end header
+#
+
+"""
+A high level API for accessing CaosDB entities from within python.
+
+This is refactored from apiutils.
+"""
+
+from caosdb.common.datatype import (BOOLEAN, DATETIME, DOUBLE, FILE, INTEGER,
+ REFERENCE, TEXT,
+ is_list_datatype,
+ get_list_datatype,
+ is_reference)
+import caosdb as db
+
+from .apiutils import get_type_of_entity_with, create_flat_list
+import warnings
+
+from typing import Any, Optional, List, Union, Dict
+
+import yaml
+
+from dataclasses import dataclass, fields
+from datetime import datetime
+from dateutil import parser
+
+warnings.warn("""EXPERIMENTAL! The high_level_api module is experimental and may be changed or
+removed in the future. Its purpose is to give an impression on how the Python client user interface
+might be changed.""")
+
+
+def standard_type_for_high_level_type(high_level_record: "CaosDBPythonEntity",
+ return_string: bool = False):
+ """
+ For a given CaosDBPythonEntity either return the corresponding
+ class in the standard CaosDB API or - if return_string is True - return
+ the role as a string.
+ """
+ if type(high_level_record) == CaosDBPythonRecord:
+ if not return_string:
+ return db.Record
+ return "Record"
+ elif type(high_level_record) == CaosDBPythonFile:
+ if not return_string:
+ return db.File
+ return "File"
+ elif type(high_level_record) == CaosDBPythonProperty:
+ if not return_string:
+ return db.Property
+ return "Property"
+ elif type(high_level_record) == CaosDBPythonRecordType:
+ if not return_string:
+ return db.RecordType
+ return "RecordType"
+ elif type(high_level_record) == CaosDBPythonEntity:
+ if not return_string:
+ return db.Entity
+ return "Entity"
+ raise RuntimeError("Incompatible type.")
+
+
+def high_level_type_for_role(role: str):
+ if role == "Record":
+ return CaosDBPythonRecord
+ if role == "File":
+ return CaosDBPythonFile
+ if role == "Property":
+ return CaosDBPythonProperty
+ if role == "RecordType":
+ return CaosDBPythonRecordType
+ if role == "Entity":
+ return CaosDBPythonEntity
+ raise RuntimeError("Unknown role.")
+
+
+def high_level_type_for_standard_type(standard_record: db.Entity):
+ if not isinstance(standard_record, db.Entity):
+ raise ValueError()
+ role = standard_record.role
+ if role == "Record" or type(standard_record) == db.Record:
+ return CaosDBPythonRecord
+ elif role == "File" or type(standard_record) == db.File:
+ return CaosDBPythonFile
+ elif role == "Property" or type(standard_record) == db.Property:
+ return CaosDBPythonProperty
+ elif role == "RecordType" or type(standard_record) == db.RecordType:
+ return CaosDBPythonRecordType
+ elif role == "Entity" or type(standard_record) == db.Entity:
+ return CaosDBPythonEntity
+ raise RuntimeError("Incompatible type.")
+
+
+@dataclass
+class CaosDBPropertyMetaData:
+ # name is already the name of the attribute
+ unit: Optional[str] = None
+ datatype: Optional[str] = None
+ description: Optional[str] = None
+ id: Optional[int] = None
+ importance: Optional[str] = None
+
+
+class CaosDBPythonUnresolved:
+ pass
+
+
+@dataclass
+class CaosDBPythonUnresolvedParent(CaosDBPythonUnresolved):
+ """
+ Parents can be either given by name or by ID.
+
+ When resolved, both fields should be set.
+ """
+
+ id: Optional[int] = None
+ name: Optional[str] = None
+
+
+@dataclass
+class CaosDBPythonUnresolvedReference(CaosDBPythonUnresolved):
+
+ def __init__(self, id=None):
+ self.id = id
+
+
+class CaosDBPythonEntity(object):
+
+ def __init__(self):
+ """
+ Initialize a new CaosDBPythonEntity for the high level python api.
+
+ Parents are either unresolved references or CaosDB RecordTypes.
+
+ Properties are stored directly as attributes for the object.
+ Property metadata is maintained in a dctionary _properties_metadata that should
+ never be accessed directly, but only using the get_property_metadata function.
+ If property values are references to other objects, they will be stored as
+ CaosDBPythonUnresolvedReference objects that can be resolved later into
+ CaosDBPythonRecords.
+ """
+
+ # Parents are either unresolved references or CaosDB RecordTypes
+ self._parents: List[Union[
+ CaosDBPythonUnresolvedParent, CaosDBPythonRecordType]] = []
+ # self._id: int = CaosDBPythonEntity._get_new_id()
+ self._id: Optional[int] = None
+ self._name: Optional[str] = None
+ self._description: Optional[str] = None
+ self._version: Optional[str] = None
+
+ self._file: Optional[str] = None
+ self._path: Optional[str] = None
+
+ # name: name of property, value: property metadata
+ self._properties_metadata: Dict[CaosDBPropertyMetaData] = dict()
+
+ # Store all current attributes as forbidden attributes
+ # which must not be changed by the set_property function.
+ self._forbidden = dir(self) + ["_forbidden"]
+
+ def use_parameter(self, name, value):
+ self.__setattr__(name, value)
+ return value
+
+ @property
+ def id(self):
+ """
+ Getter for the id.
+ """
+ return self._id
+
+ @id.setter
+ def id(self, val: int):
+ self._id = val
+
+ @property
+ def name(self):
+ """
+ Getter for the name.
+ """
+ return self._name
+
+ @name.setter
+ def name(self, val: str):
+ self._name = val
+
+ @property
+ def file(self):
+ """
+ Getter for the file.
+ """
+ if type(self) != CaosDBPythonFile:
+ raise RuntimeError("Please don't use the file attribute for entities"
+ " that are no files.")
+ return self._file
+
+ @file.setter
+ def file(self, val: str):
+ if val is not None and type(self) != CaosDBPythonFile:
+ raise RuntimeError("Please don't use the file attribute for entities"
+ " that are no files.")
+ self._file = val
+
+ @property
+ def path(self):
+ """
+ Getter for the path.
+ """
+ if type(self) != CaosDBPythonFile:
+ raise RuntimeError("Please don't use the path attribute for entities"
+ " that are no files.")
+ return self._path
+
+ @path.setter
+ def path(self, val: str):
+ if val is not None and type(self) != CaosDBPythonFile:
+ raise RuntimeError("Please don't use the path attribute for entities"
+ " that are no files.")
+ self._path = val
+
+ @property
+ def description(self):
+ """
+ Getter for the description.
+ """
+ return self._description
+
+ @description.setter
+ def description(self, val: str):
+ self._description = val
+
+ @property
+ def version(self):
+ """
+ Getter for the version.
+ """
+ return self._version
+
+ @version.setter
+ def version(self, val: str):
+ self._version = val
+
+ def _set_property_from_entity(self, ent: db.Entity, importance: str,
+ references: Optional[db.Container],
+ visited: Dict[int, "CaosDBPythonEntity"]):
+ """
+ Set a new property using an entity from the normal python API.
+
+ ent : db.Entity
+ The entity to be set.
+ """
+
+ if ent.name is None:
+ raise RuntimeError("Setting properties without name is impossible.")
+
+ if ent.name in self.get_properties():
+ raise RuntimeError("Multiproperty not implemented yet.")
+
+ val = self._type_converted_value(ent.value, ent.datatype,
+ references, visited)
+ self.set_property(
+ ent.name,
+ val,
+ datatype=ent.datatype)
+ metadata = self.get_property_metadata(ent.name)
+
+ for prop_name in fields(metadata):
+ k = prop_name.name
+ if k == "importance":
+ metadata.importance = importance
+ else:
+ metadata.__setattr__(k, ent.__getattribute__(k))
+
+ def get_property_metadata(self, prop_name: str) -> CaosDBPropertyMetaData:
+ """
+ Retrieve the property metadata for the property with name prop_name.
+
+ If the property with the given name does not exist or is forbidden, raise an exception.
+ Else return the metadata associated with this property.
+
+ If no metadata does exist yet for the given property, a new object will be created
+ and returned.
+
+ prop_name: str
+ Name of the property to retrieve metadata for.
+ """
+
+ if not self.property_exists(prop_name):
+ raise RuntimeError("The property with name {} does not exist.".format(prop_name))
+
+ if prop_name not in self._properties_metadata:
+ self._properties_metadata[prop_name] = CaosDBPropertyMetaData()
+
+ return self._properties_metadata[prop_name]
+
+ def property_exists(self, prop_name: str):
+ """
+ Check whether a property exists already.
+ """
+ return prop_name not in self._forbidden and prop_name in self.__dict__
+
+ def set_property(self,
+ name: str,
+ value: Any,
+ overwrite: bool = False,
+ datatype: Optional[str] = None):
+ """
+ Set a property for this entity with a name and a value.
+
+ If this property is already set convert the value into a list and append the value.
+ This behavior can be overwritten using the overwrite flag, which will just overwrite
+ the existing value.
+
+ name: str
+ Name of the property.
+
+ value: Any
+ Value of the property.
+
+ overwrite: bool
+ Use this if you definitely only want one property with
+ that name (set to True).
+ """
+
+ if name in self._forbidden:
+ raise RuntimeError("Entity cannot be converted to a corresponding "
+ "Python representation. Name of property " +
+ name + " is forbidden!")
+
+ already_exists = self.property_exists(name)
+
+ if already_exists and not overwrite:
+ # each call to set_property checks first if it already exists
+ # if yes: Turn the attribute into a list and
+ # place all the elements into that list.
+ att = self.__getattribute__(name)
+
+ if isinstance(att, list):
+ # just append, see below
+ pass
+ else:
+ old_att = self.__getattribute__(name)
+ self.__setattr__(name, [old_att])
+ att = self.__getattribute__(name)
+ att.append(value)
+ else:
+ self.__setattr__(name, value)
+
+ def __setattr__(self, name: str, val: Any):
+ """
+ Allow setting generic properties.
+ """
+
+ # TODO: implement checking the value to correspond to one of the datatypes
+ # known for conversion.
+
+ super().__setattr__(name, val)
+
+ def _type_converted_list(self,
+ val: List,
+ pr: str,
+ references: Optional[db.Container],
+ visited: Dict[int, "CaosDBPythonEntity"]):
+ """
+ Convert a list to a python list of the correct type.
+
+ val: List
+ The value of a property containing the list.
+
+ pr: str
+ The datatype according to the database entry.
+ """
+ if not is_list_datatype(pr) and not isinstance(val, list):
+ raise RuntimeError("Not a list.")
+
+ return [
+ self._type_converted_value(i, get_list_datatype(pr), references,
+ visited) for i in val]
+
+ def _type_converted_value(self,
+ val: Any,
+ pr: str,
+ references: Optional[db.Container],
+ visited: Dict[int, "CaosDBPythonEntity"]):
+ """
+ Convert val to the correct type which is indicated by the database
+ type string in pr.
+
+ References with ids will be turned into CaosDBPythonUnresolvedReference.
+ """
+
+ if val is None:
+ return None
+ elif isinstance(val, db.Entity):
+ # this needs to be checked as second case as it is the ONLY
+ # case which does not depend on pr
+ # TODO: we might need to pass through the reference container
+ return convert_to_python_object(val, references, visited)
+ elif isinstance(val, list):
+ return self._type_converted_list(val, pr, references, visited)
+ elif pr is None:
+ return val
+ elif pr == DOUBLE:
+ return float(val)
+ elif pr == BOOLEAN:
+ return bool(val)
+ elif pr == INTEGER:
+ return int(val)
+ elif pr == TEXT:
+ return str(val)
+ elif pr == FILE:
+ return CaosDBPythonUnresolvedReference(val)
+ elif pr == REFERENCE:
+ return CaosDBPythonUnresolvedReference(val)
+ elif pr == DATETIME:
+ return self._parse_datetime(val)
+ elif is_list_datatype(pr):
+ return self._type_converted_list(val, pr, references, visited)
+ else:
+ # Generic references to entities:
+ return CaosDBPythonUnresolvedReference(val)
+
+ def _parse_datetime(self, val: Union[str, datetime]):
+ """
+ Convert val into a datetime object.
+ """
+ if isinstance(val, datetime):
+ return val
+ return parser.parse(val)
+
+ def get_property(self, name: str):
+ """
+ Return the value of the property with name name.
+
+ Raise an exception if the property does not exist.
+ """
+ if not self.property_exists(name):
+ raise RuntimeError("Property {} does not exist.".format(name))
+ att = self.__getattribute__(name)
+ return att
+
+ def attribute_as_list(self, name: str):
+ """
+ This is a workaround for the problem that lists containing only one
+ element are indistinguishable from simple types in this
+ representation.
+
+ TODO: still relevant? seems to be only a problem if LIST types are not used.
+ """
+ att = self.get_property(name)
+
+ if isinstance(att, list):
+ return att
+ else:
+ return [att]
+
+ def add_parent(self, parent: Union[
+ CaosDBPythonUnresolvedParent, "CaosDBPythonRecordType", str]):
+ """
+ Add a parent to this entity. Either using an unresolved parent or
+ using a real record type.
+
+ Strings as argument for parent will automatically be converted to an
+ unresolved parent. Likewise, integers as argument will be automatically converted
+ to unresolved parents with just an id.
+ """
+
+ if isinstance(parent, str):
+ parent = CaosDBPythonUnresolvedParent(name=parent)
+
+ if isinstance(parent, int):
+ parent = CaosDBPythonUnresolvedParent(id=parent)
+
+ if self.has_parent(parent):
+ raise RuntimeError("Duplicate parent.")
+ self._parents.append(parent)
+
+ def get_parents(self):
+ """
+ Returns all parents of this entity.
+
+ Use has_parent for checking for existence of parents
+ and add_parent for adding parents to this entity.
+ """
+ return self._parents
+
+ def has_parent(self, parent: Union[
+ CaosDBPythonUnresolvedParent, "CaosDBPythonRecordType"]):
+ """
+ Check whether this parent already exists for this entity.
+
+ Strings as argument for parent will automatically be converted to an
+ unresolved parent. Likewise, integers as argument will be automatically converted
+ to unresolved parents with just an id.
+ """
+
+ if isinstance(parent, str):
+ parent = CaosDBPythonUnresolvedParent(name=parent)
+
+ if isinstance(parent, int):
+ parent = CaosDBPythonUnresolvedParent(id=parent)
+
+ for p in self._parents:
+ if p.id is not None and p.id == parent.id:
+ return True
+ elif p.name is not None and p.name == parent.name:
+ return True
+ return False
+
+ def _resolve_caosdb_python_unresolved_reference(self, propval, deep,
+ references, visited):
+ # This does not make sense for unset ids:
+ if propval.id is None:
+ raise RuntimeError("Unresolved property reference without an ID.")
+ # have we encountered this id before:
+ if propval.id in visited:
+ # self.__setattr__(prop, visited[propval.id])
+ # don't do the lookup in the references container
+ return visited[propval.id]
+
+ if references is None:
+ ent = db.Entity(id=propval.id).retrieve()
+ obj = convert_to_python_object(ent, references)
+ visited[propval.id] = obj
+ if deep:
+ obj.resolve_references(deep, references, visited)
+ return obj
+
+ # lookup in container:
+ for ent in references:
+ # Entities in container without an ID will be skipped:
+ if ent.id is not None and ent.id == propval.id:
+ # resolve this entity:
+ obj = convert_to_python_object(ent, references)
+ visited[propval.id] = obj
+ # self.__setattr__(prop, visited[propval.id])
+ if deep:
+ obj.resolve_references(deep, references, visited)
+ return obj
+ return propval
+
+ def resolve_references(self, deep: bool, references: db.Container,
+ visited: Optional[Dict[Union[str, int],
+ "CaosDBPythonEntity"]] = None):
+ """
+ Resolve this entity's references. This affects unresolved properties as well
+ as unresolved parents.
+
+ deep: bool
+ If True recursively resolve references also for all resolved references.
+
+ references: Optional[db.Container]
+ A container with references that might be resolved.
+ If None is passed as the container, this function tries to resolve entities from a running
+ CaosDB instance directly.
+ """
+
+ # This parameter is used in the recursion to keep track of already visited
+ # entites (in order to detect cycles).
+ if visited is None:
+ visited = dict()
+
+ for parent in self.get_parents():
+ # TODO
+ if isinstance(parent, CaosDBPythonUnresolvedParent):
+ pass
+
+ for prop in self.get_properties():
+ propval = self.__getattribute__(prop)
+ # Resolve all previously unresolved attributes that are entities:
+ if deep and isinstance(propval, CaosDBPythonEntity):
+ propval.resolve_references(deep, references)
+ elif isinstance(propval, list):
+ resolvedelements = []
+ for element in propval:
+ if deep and isinstance(element, CaosDBPythonEntity):
+ element.resolve_references(deep, references)
+ resolvedelements.append(element)
+ if isinstance(element, CaosDBPythonUnresolvedReference):
+ resolvedelements.append(
+ self._resolve_caosdb_python_unresolved_reference(element, deep,
+ references, visited))
+ else:
+ resolvedelements.append(element)
+ self.__setattr__(prop, resolvedelements)
+
+ elif isinstance(propval, CaosDBPythonUnresolvedReference):
+ val = self._resolve_caosdb_python_unresolved_reference(propval, deep,
+ references, visited)
+ self.__setattr__(prop, val)
+
+ def get_properties(self):
+ """
+ Return the names of all properties.
+ """
+
+ return [p for p in self.__dict__
+ if p not in self._forbidden]
+
+ @staticmethod
+ def deserialize(serialization: dict):
+ """
+ Deserialize a yaml representation of an entity in high level API form.
+ """
+
+ if "role" in serialization:
+ entity = high_level_type_for_role(serialization["role"])()
+ else:
+ entity = CaosDBPythonRecord()
+
+ if "parents" in serialization:
+ for parent in serialization["parents"]:
+ if "unresolved" in parent:
+ id = None
+ name = None
+ if "id" in parent:
+ id = parent["id"]
+ if "name" in parent:
+ name = parent["name"]
+ entity.add_parent(CaosDBPythonUnresolvedParent(
+ id=id, name=name))
+ else:
+ raise NotImplementedError(
+ "Currently, only unresolved parents can be deserialized.")
+
+ for baseprop in ("name", "id", "description", "version"):
+ if baseprop in serialization:
+ entity.__setattr__(baseprop, serialization[baseprop])
+
+ if type(entity) == CaosDBPythonFile:
+ entity.file = serialization["file"]
+ entity.path = serialization["path"]
+
+ for p in serialization["properties"]:
+ # The property needs to be set first:
+
+ prop = serialization["properties"][p]
+ if isinstance(prop, dict):
+ if "unresolved" in prop:
+ entity.__setattr__(p, CaosDBPythonUnresolvedReference(
+ id=prop["id"]))
+ else:
+ entity.__setattr__(p,
+ entity.deserialize(prop))
+ else:
+ entity.__setattr__(p, prop)
+
+ # if there is no metadata in the yaml file just initialize an empty metadata object
+ if "metadata" in serialization and p in serialization["metadata"]:
+ metadata = serialization["metadata"][p]
+ propmeta = entity.get_property_metadata(p)
+
+ for f in fields(propmeta):
+ if f.name in metadata:
+ propmeta.__setattr__(f.name, metadata[f.name])
+ else:
+ pass
+ # raise NotImplementedError()
+
+ return entity
+
+ def serialize(self, without_metadata: bool = False, visited: dict = None):
+ """
+ Serialize necessary information into a dict.
+
+ without_metadata: bool
+ If True don't set the metadata field in order to increase
+ readability. Not recommended if deserialization is needed.
+ """
+
+ if visited is None:
+ visited = dict()
+
+ if self in visited:
+ return visited[self]
+
+ metadata: Dict[str, Any] = dict()
+ properties = dict()
+ parents = list()
+
+ # The full information to be returned:
+ fulldict = dict()
+ visited[self] = fulldict
+
+ # Add CaosDB role:
+ fulldict["role"] = standard_type_for_high_level_type(self, True)
+
+ for parent in self._parents:
+ if isinstance(parent, CaosDBPythonEntity):
+ parents.append(parent.serialize(without_metadata, visited))
+ elif isinstance(parent, CaosDBPythonUnresolvedParent):
+ parents.append({"name": parent.name, "id": parent.id,
+ "unresolved": True})
+ else:
+ raise RuntimeError("Incompatible class used as parent.")
+
+ for baseprop in ("name", "id", "description", "version"):
+ val = self.__getattribute__(baseprop)
+ if val is not None:
+ fulldict[baseprop] = val
+
+ if type(self) == CaosDBPythonFile:
+ fulldict["file"] = self.file
+ fulldict["path"] = self.path
+
+ for p in self.get_properties():
+ m = self.get_property_metadata(p)
+ metadata[p] = dict()
+ for f in fields(m):
+ val = m.__getattribute__(f.name)
+ if val is not None:
+ metadata[p][f.name] = val
+
+ val = self.get_property(p)
+ if isinstance(val, CaosDBPythonUnresolvedReference):
+ properties[p] = {"id": val.id, "unresolved": True}
+ elif isinstance(val, CaosDBPythonEntity):
+ properties[p] = val.serialize(without_metadata, visited)
+ elif isinstance(val, list):
+ serializedelements = []
+ for element in val:
+ if isinstance(element, CaosDBPythonUnresolvedReference):
+ elm = dict()
+ elm["id"] = element.id
+ elm["unresolved"] = True
+ serializedelements.append(elm)
+ elif isinstance(element, CaosDBPythonEntity):
+ serializedelements.append(
+ element.serialize(without_metadata,
+ visited))
+ else:
+ serializedelements.append(element)
+ properties[p] = serializedelements
+ else:
+ properties[p] = val
+
+ fulldict["properties"] = properties
+ fulldict["parents"] = parents
+
+ if not without_metadata:
+ fulldict["metadata"] = metadata
+ return fulldict
+
+ def __str__(self):
+ return yaml.dump(self.serialize(False))
+
+ # This seemed like a good solution, but makes it difficult to
+ # compare python objects directly:
+ #
+ # def __repr__(self):
+ # return yaml.dump(self.serialize(True))
+
+
+class CaosDBPythonRecord(CaosDBPythonEntity):
+ pass
+
+
+class CaosDBPythonRecordType(CaosDBPythonEntity):
+ pass
+
+
+class CaosDBPythonProperty(CaosDBPythonEntity):
+ pass
+
+
+class CaosDBMultiProperty:
+ """
+ This implements a multi property using a python list.
+ """
+
+ def __init__(self):
+ raise NotImplementedError()
+
+
+class CaosDBPythonFile(CaosDBPythonEntity):
+ def download(self, target=None):
+ if self.id is None:
+ raise RuntimeError("Cannot download file when id is missing.")
+ f = db.File(id=self.id).retrieve()
+ return f.download(target)
+
+
+BASE_ATTRIBUTES = (
+ "id", "name", "description", "version", "path", "file")
+
+
+def _single_convert_to_python_object(robj: CaosDBPythonEntity,
+ entity: db.Entity,
+ references: Optional[db.Container] = None,
+ visited: Optional[Dict[int,
+ "CaosDBPythonEntity"]] = None):
+ """
+ Convert a db.Entity from the standard API to a (previously created)
+ CaosDBPythonEntity from the high level API.
+
+ This method will not resolve any unresolved references, so reference properties
+ as well as parents will become unresolved references in the first place.
+
+ The optional third parameter can be used
+ to resolve references that occur in the converted entities and resolve them
+ to their correct representations. (Entities that are not found remain as
+ CaosDBPythonUnresolvedReferences.)
+
+ Returns the input object robj.
+ """
+
+ # This parameter is used in the recursion to keep track of already visited
+ # entites (in order to detect cycles).
+ if visited is None:
+ visited = dict()
+
+ if id(entity) in visited:
+ return visited[id(entity)]
+ else:
+ visited[id(entity)] = robj
+
+ for base_attribute in BASE_ATTRIBUTES:
+ val = entity.__getattribute__(base_attribute)
+ if val is not None:
+ if isinstance(val, db.common.models.Version):
+ val = val.id
+ robj.__setattr__(base_attribute, val)
+
+ for prop in entity.properties:
+ robj._set_property_from_entity(prop, entity.get_importance(prop), references,
+ visited)
+
+ for parent in entity.parents:
+ robj.add_parent(CaosDBPythonUnresolvedParent(id=parent.id,
+ name=parent.name))
+
+ return robj
+
+
+def _convert_property_value(propval):
+ if isinstance(propval, CaosDBPythonUnresolvedReference):
+ propval = propval.id
+ elif isinstance(propval, CaosDBPythonEntity):
+ propval = _single_convert_to_entity(
+ standard_type_for_high_level_type(propval)(), propval)
+ elif isinstance(propval, list):
+ propval = [_convert_property_value(element) for element in propval]
+
+ # TODO: test case for list missing
+
+ return propval
+
+
+def _single_convert_to_entity(entity: db.Entity,
+ robj: CaosDBPythonEntity):
+ """
+ Convert a CaosDBPythonEntity to an entity in standard pylib format.
+
+ entity: db.Entity
+ An empty entity.
+
+ robj: CaosDBPythonEntity
+ The CaosDBPythonEntity that is supposed to be converted to the entity.
+ """
+
+ for base_attribute in BASE_ATTRIBUTES:
+ if base_attribute in ("file", "path") and not isinstance(robj, CaosDBPythonFile):
+ continue
+
+ # Skip version:
+ if base_attribute == "version":
+ continue
+
+ val = robj.__getattribute__(base_attribute)
+
+ if val is not None:
+ entity.__setattr__(base_attribute, val)
+
+ for parent in robj.get_parents():
+ if isinstance(parent, CaosDBPythonUnresolvedParent):
+ entity.add_parent(name=parent.name, id=parent.id)
+ elif isinstance(parent, CaosDBPythonRecordType):
+ raise NotImplementedError()
+ else:
+ raise RuntimeError("Incompatible class used as parent.")
+
+ for prop in robj.get_properties():
+ propval = robj.__getattribute__(prop)
+ metadata = robj.get_property_metadata(prop)
+
+ propval = _convert_property_value(propval)
+
+ entity.add_property(
+ name=prop,
+ value=propval,
+ unit=metadata.unit,
+ importance=metadata.importance,
+ datatype=metadata.datatype,
+ description=metadata.description,
+ id=metadata.id)
+
+ return entity
+
+
+def convert_to_entity(python_object):
+ if isinstance(python_object, db.Container):
+ # Create a list of objects:
+
+ return [convert_to_entity(i) for i in python_object]
+ elif isinstance(python_object, CaosDBPythonRecord):
+ return _single_convert_to_entity(db.Record(), python_object)
+ elif isinstance(python_object, CaosDBPythonFile):
+ return _single_convert_to_entity(db.File(), python_object)
+ elif isinstance(python_object, CaosDBPythonRecordType):
+ return _single_convert_to_entity(db.RecordType(), python_object)
+ elif isinstance(python_object, CaosDBPythonProperty):
+ return _single_convert_to_entity(db.Property(), python_object)
+ elif isinstance(python_object, CaosDBPythonEntity):
+ return _single_convert_to_entity(db.Entity(), python_object)
+ else:
+ raise ValueError("Cannot convert an object of this type.")
+
+
+def convert_to_python_object(entity: Union[db.Container, db.Entity],
+ references: Optional[db.Container] = None,
+ visited: Optional[Dict[int,
+ "CaosDBPythonEntity"]] = None):
+ """
+ Convert either a container of CaosDB entities or a single CaosDB entity
+ into the high level representation.
+
+ The optional second parameter can be used
+ to resolve references that occur in the converted entities and resolve them
+ to their correct representations. (Entities that are not found remain as
+ CaosDBPythonUnresolvedReferences.)
+ """
+ if isinstance(entity, db.Container):
+ # Create a list of objects:
+ return [convert_to_python_object(i, references, visited) for i in entity]
+
+ # TODO: recursion problems?
+ return _single_convert_to_python_object(
+ high_level_type_for_standard_type(entity)(),
+ entity,
+ references,
+ visited)
+
+
+def new_high_level_entity(entity: db.RecordType,
+ importance_level: str,
+ name: Optional[str] = None):
+ """
+ Create an new record in high level format based on a record type in standard format.
+
+ entity: db.RecordType
+ The record type to initialize the new record from.
+
+ importance_level: str
+ None, obligatory, recommended or suggested
+ Initialize new properties up to this level.
+ Properties in the record type with no importance will be added
+ regardless of the importance_level.
+
+ name: str
+ Name of the new record.
+ """
+
+ r = db.Record(name=name)
+ r.add_parent(entity)
+
+ impmap = {
+ None: 0, "SUGGESTED": 3, "RECOMMENDED": 2, "OBLIGATORY": 1}
+
+ for prop in entity.properties:
+ imp = entity.get_importance(prop)
+ if imp is not None and impmap[importance_level] < impmap[imp]:
+ continue
+
+ r.add_property(prop)
+
+ return convert_to_python_object(r)
+
+
+def create_record(rtname: str, name: Optional[str] = None, **kwargs):
+ """
+ Create a new record based on the name of a record type. The new record is returned.
+
+ rtname: str
+ The name of the record type.
+
+ name: str
+ This is optional. A name for the new record.
+
+ kwargs:
+ Additional arguments are used to set attributes of the
+ new record.
+ """
+ obj = new_high_level_entity(
+ db.RecordType(name=rtname).retrieve(), "SUGGESTED", name)
+ for key, value in kwargs.items():
+ obj.__setattr__(key, value)
+ return obj
+
+
+def load_external_record(record_name: str):
+ """
+ Retrieve a record by name and convert it to the high level API format.
+ """
+ return convert_to_python_object(db.Record(name=record_name).retrieve())
+
+
+def create_entity_container(record: CaosDBPythonEntity):
+ """
+ Convert this record into an entity container in standard format that can be used
+ to insert or update entities in a running CaosDB instance.
+ """
+ ent = convert_to_entity(record)
+ lse: List[db.Entity] = [ent]
+ create_flat_list([ent], lse)
+ return db.Container().extend(lse)
+
+
+def query(query: str,
+ resolve_references: Optional[bool] = True,
+ references: Optional[db.Container] = None):
+ """
+
+ """
+ res = db.execute_query(query)
+ objects = convert_to_python_object(res)
+ if resolve_references:
+ for obj in objects:
+ obj.resolve_references(True, references)
+ return objects
diff --git a/src/caosdb/schema-pycaosdb-ini.yml b/src/caosdb/schema-pycaosdb-ini.yml
index fd49c2ffa63e7bc334f8177ff79575f46718f38c..cb07dfeb84bc16e212100232403b0f66543c73e9 100644
--- a/src/caosdb/schema-pycaosdb-ini.yml
+++ b/src/caosdb/schema-pycaosdb-ini.yml
@@ -14,10 +14,10 @@ schema-pycaosdb-ini:
additionalProperties: false
properties:
url:
- description: URL of the CaosDB server
+ description: "URL of the CaosDB server. Allowed are HTTP and HTTPS connections. However, since authentication tokens and sometimes even passwords are send in plain text to the server it is **highly** recommended to use HTTPS connections whenever possible. HTTP is ok for testing and debugging."
type: string
- pattern: https://[-a-zA-Z0-9\.]+(:[0-9]+)?(/)?
- examples: [https://demo.indiscale.com/, https://localhost:10443/]
+ pattern: http(s)?://[-a-zA-Z0-9\.]+(:[0-9]+)?(/)?
+ examples: ["https://demo.indiscale.com/", "http://localhost:10080/"]
username:
type: string
description: User name used for authentication with the server
@@ -26,7 +26,7 @@ schema-pycaosdb-ini:
description: The password input method defines how the password is supplied that is used for authentication with the server.
type: string
default: input
- enum: [input, plain, pass, keyring]
+ enum: [input, unauthenticated, plain, pass, keyring]
password_identifier:
type: string
password:
@@ -52,9 +52,17 @@ schema-pycaosdb-ini:
enum: [0, 1, 2]
description: The debug key allows control the verbosity. Set it to 1 or 2 in case you want to see debugging output or if you want to learn more about the internals of the protocol. 0 disables debugging output.
socket_proxy:
- examples: [localhost:12345]
+ examples: ["localhost:12345"]
type: string
- description: You can define a socket proxy to be used. This is for the case that the server sits behind a firewall which is being tunnelled with a socket proxy (SOCKS4 or SOCKS5) (e.g. via ssh's -D option or a dedicated proxy server).
+ description: Deprecated. Please use https_proxy instead.
+ https_proxy:
+ examples: ["http://localhost:8888", "socks5://localhost:8888", "socks4://localhost:8888"]
+ type: string
+ description: "Define a proxy for the https connections. These are either (non-TLS) HTTP proxies, SOCKS4 proxies, or SOCKS5 proxies. HTTPS proxies are not supported. However, the connection will be secured using TLS in the tunneled connection nonetheless. Only the connection to the proxy is insecure which is why it is not recommended to use HTTP proxies when authentication against the proxy is necessary. Note: this option is overridden by the HTTPS_PROXY environment variable, if present."
+ http_proxy:
+ examples: ["http://localhost:8888", "socks5://localhost:8888", "socks4://localhost:8888"]
+ type: string
+ description: "Define a proxy for the http connections. These are either (non-TLS) HTTP proxies, SOCKS4 proxies, or SOCKS5 proxies. HTTPS proxies are not supported. Note: this option is overridden by the HTTP_PROXY environment variable, if present."
implementation:
description: This option is used internally and for testing. Do not override.
examples: [_DefaultCaosDBServerConnection]
@@ -65,23 +73,42 @@ schema-pycaosdb-ini:
properties:
password_method:
const: input
+ required: [password_method]
then:
required: [url]
- if:
properties:
password_method:
const: plain
+ required: [password_method]
then:
required: [url, username, password]
- if:
properties:
password_method:
const: pass
+ required: [password_method]
then:
required: [url, username, password_identifier]
- if:
properties:
password_method:
const: keyring
+ required: [password_method]
then:
required: [url, username]
+ IntegrationTests:
+ description: "Used by the integration test suite from the caosdb-pyinttest repo."
+ additionalProperties: true
+ Misc:
+ description: "Some additional configuration settings."
+ additionalProperties: true
+ advancedtools:
+ description: "Configuration settings for the caosadvancedtools."
+ additionalProperties: true
+ caoscrawler:
+ description: "Configuration settings for the CaosDB Crawler."
+ additionalProperties: true
+ sss_helper:
+ description: "Configuration settings for server-side scripting."
+ additionalProperties: true
diff --git a/src/caosdb/utils/caosdb_admin.py b/src/caosdb/utils/caosdb_admin.py
index 9fb94f57683036f5432a40198cc4ae98893665fb..09a8f64a3c6b9f0825089949840a8791604d1ded 100755
--- a/src/caosdb/utils/caosdb_admin.py
+++ b/src/caosdb/utils/caosdb_admin.py
@@ -621,8 +621,8 @@ USAGE
for action in ["grant", "deny", "revoke_denial", "revoke_grant"]:
action_entity_permissions_parser = subparsers.add_parser(
- "{}_entity_permissions".format(action),
- help="{} entity permissions to a role.".format(action))
+ f"{action}_entity_permissions",
+ help=f"{action} entity permissions to one or more Entities.")
action_entity_permissions_parser.set_defaults(
call=do_action_entity_permissions, action=action)
action_entity_permissions_parser.add_argument(dest="query", metavar="QUERY",
diff --git a/src/caosdb/utils/checkFileSystemConsistency.py b/src/caosdb/utils/checkFileSystemConsistency.py
index 6dd35f8a6f699a2c74ff41a9924cd65c436efd42..6c053fdca6acb3a6585589c0e6298ba0704ea590 100755
--- a/src/caosdb/utils/checkFileSystemConsistency.py
+++ b/src/caosdb/utils/checkFileSystemConsistency.py
@@ -30,7 +30,6 @@ import caosdb as db
from argparse import ArgumentParser
from argparse import RawDescriptionHelpFormatter
-from _testcapi import raise_exception
__all__ = []
__version__ = 0.1
@@ -82,17 +81,15 @@ def main(argv=None):
program_build_date = str(__updated__)
program_version_message = '%%(prog)s %s (%s)' % (
program_version, program_build_date)
- program_shortdesc = __import__('__main__').__doc__.split("\n")[1]
- program_license = '''%s
+ program_license = '''
- Created by timm fitschen on %s.
Copyright 2016 BMPG. All rights reserved.
Distributed on an "AS IS" basis without warranties
or conditions of any kind, either express or implied.
USAGE
-''' % (program_shortdesc, str(__date__))
+'''
# Setup argument parser
parser = ArgumentParser(description=program_license,
@@ -113,6 +110,7 @@ USAGE
help="timeout in seconds for the database requests. [default: %(default)s]",
metavar="TIMEOUT",
default="200")
+ parser.add_argument('location')
# Process arguments
args = parser.parse_args()
@@ -121,7 +119,7 @@ USAGE
VERBOSITY = args.verbose
TIMEOUT = args.timeout
- runCheck(TIMEOUT)
+ print(runCheck(TIMEOUT, args.location).messages)
return 0
diff --git a/src/caosdb/utils/create_revision.py b/src/caosdb/utils/create_revision.py
index 0b7ce996311a96a6a0fe89935de729f07b67a353..419e1c9f2b97171be0dccf1bc772ae5db679c0b7 100644
--- a/src/caosdb/utils/create_revision.py
+++ b/src/caosdb/utils/create_revision.py
@@ -34,13 +34,15 @@ def bend_references(from_id, to_id, except_for=None):
and those references are changed to point to to_id.
entities having an id listed in except_for are excluded.
- params:
- from_id : int
- the old object to which references where pointing
- to_id : int
- the new object to which references will be pointing
- except_for : list of int
- entities with id of this list will not be changed
+Parameters
+----------
+
+from_id : int
+ the old object to which references where pointing
+to_id : int
+ the new object to which references will be pointing
+except_for : list of int
+ entities with id of this list will not be changed
"""
if except_for is None:
except_for = [to_id]
@@ -71,14 +73,16 @@ def create_revision(old_id, prop, value):
This function changes the record with id old_id. The value of the
propertye prop is changed to value.
- params:
- old_id : int
- id of the record to be changed
- prop : string
- name of the property to be changed
- value : type of corresponding property
- the new value of the corresponding property
- """
+Parameters
+----------
+
+old_id : int
+ id of the record to be changed
+prop : string
+ name of the property to be changed
+value : type of corresponding property
+ the new value of the corresponding property
+"""
record = db.execute_query("FIND {}".format(old_id))[0]
new_rec = record.copy()
new_rec.get_property(prop).value = value
diff --git a/src/caosdb/utils/get_entity.py b/src/caosdb/utils/get_entity.py
new file mode 100644
index 0000000000000000000000000000000000000000..a27aafa99ffe3759a46876a5bcd5e686d631b1dc
--- /dev/null
+++ b/src/caosdb/utils/get_entity.py
@@ -0,0 +1,49 @@
+# -*- coding: utf-8 -*-
+#
+# This file is a part of the CaosDB Project.
+#
+# Copyright (C) 2023 Henrik tom Wörden <h.tomwoerden@indiscale.com>
+# Copyright (C) 2023 IndiScale GmbH <info@indiscale.com>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see <https://www.gnu.org/licenses/>.
+#
+
+"""Convenience functions to retrieve a specific entity."""
+
+from typing import Union
+from ..common.models import execute_query, Entity
+
+
+def get_entity_by_name(name: str) -> Entity:
+ """Return the result of a unique query that uses the name to find the correct entity.
+
+ Submits the query "FIND ENTITY WITH name='{name}'".
+ """
+ return execute_query(f"FIND ENTITY WITH name='{name}'", unique=True)
+
+
+def get_entity_by_id(eid: Union[str, int]) -> Entity:
+ """Return the result of a unique query that uses the id to find the correct entity.
+
+ Submits the query "FIND ENTITY WITH id='{eid}'".
+ """
+ return execute_query(f"FIND ENTITY WITH id='{eid}'", unique=True)
+
+
+def get_entity_by_path(path: str) -> Entity:
+ """Return the result of a unique query that uses the path to find the correct file.
+
+ Submits the query "FIND FILE WHICH IS STORED AT '{path}'".
+ """
+ return execute_query(f"FIND FILE WHICH IS STORED AT '{path}'", unique=True)
diff --git a/src/caosdb/utils/git_utils.py b/src/caosdb/utils/git_utils.py
new file mode 100644
index 0000000000000000000000000000000000000000..7a58272a3bef1930f75a1e08364349388e2bb89f
--- /dev/null
+++ b/src/caosdb/utils/git_utils.py
@@ -0,0 +1,82 @@
+# -*- coding: utf-8 -*-
+#
+# This file is a part of the CaosDB Project.
+#
+# Copyright (C) 2018 Research Group Biomedical Physics,
+# Max-Planck-Institute for Dynamics and Self-Organization Göttingen
+# Copyright (C) 2020 Timm Fitschen <t.fitschen@indiscale.com>
+# Copyright (C) 2020-2022 IndiScale GmbH <info@indiscale.com>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see <https://www.gnu.org/licenses/>.
+#
+# ** end header
+#
+"""git-utils: Some functions for retrieving information about git repositories.
+
+"""
+
+import logging
+import tempfile
+
+from subprocess import call
+
+logger = logging.getLogger(__name__)
+
+
+def get_origin_url_in(folder: str):
+ """return the Fetch URL of the git repository in the given folder."""
+ with tempfile.NamedTemporaryFile(delete=False, mode="w") as t:
+ call(["git", "remote", "show", "origin"], stdout=t, cwd=folder)
+ with open(t.name, "r") as t:
+ urlString = "Fetch URL:"
+
+ for line in t.readlines():
+ if urlString in line:
+ return line[line.find(urlString) + len(urlString):].strip()
+
+ return None
+
+
+def get_diff_in(folder: str, save_dir=None):
+ """returns the name of a file where the out put of "git diff" in the given
+ folder is stored."""
+ with tempfile.NamedTemporaryFile(delete=False, mode="w", dir=save_dir) as t:
+ call(["git", "diff"], stdout=t, cwd=folder)
+
+ return t.name
+
+
+def get_branch_in(folder: str):
+ """returns the current branch of the git repository in the given folder.
+
+ The command "git branch" is called in the given folder and the
+ output is returned
+ """
+ with tempfile.NamedTemporaryFile(delete=False, mode="w") as t:
+ call(["git", "rev-parse", "--abbrev-ref", "HEAD"], stdout=t, cwd=folder)
+ with open(t.name, "r") as t:
+ return t.readline().strip()
+
+
+def get_commit_in(folder: str):
+ """returns the commit hash in of the git repository in the given folder.
+
+ The command "git log -1 --format=%h" is called in the given folder
+ and the output is returned
+ """
+
+ with tempfile.NamedTemporaryFile(delete=False, mode="w") as t:
+ call(["git", "log", "-1", "--format=%h"], stdout=t, cwd=folder)
+ with open(t.name, "r") as t:
+ return t.readline().strip()
diff --git a/src/caosdb/utils/plantuml.py b/src/caosdb/utils/plantuml.py
index be34b2604f3682bb71b48bbd73e00fe854b3af51..6252a48983c62e7a2f33113422205209d616b5b6 100644
--- a/src/caosdb/utils/plantuml.py
+++ b/src/caosdb/utils/plantuml.py
@@ -34,10 +34,15 @@ plantuml FILENAME.pu -> FILENAME.png
"""
import os
+import shutil
import caosdb as db
from caosdb.common.datatype import is_reference, get_referenced_recordtype
+from typing import List, Optional
+
+import tempfile
+
REFERENCE = "REFERENCE"
@@ -79,13 +84,24 @@ class Grouped(object):
return self.parents
-def recordtypes_to_plantuml_string(iterable):
+def recordtypes_to_plantuml_string(iterable,
+ add_properties: bool = True,
+ add_recordtypes: bool = True,
+ add_legend: bool = True,
+ no_shadow: bool = False,
+ style: str = "default"):
"""Converts RecordTypes into a string for PlantUML.
This function obtains an iterable and returns a string which can
be input into PlantUML for a representation of all RecordTypes in
the iterable.
+ Current options for style
+ -------------------------
+
+ "default" - Standard rectangles with uml class circle and methods section
+ "salexan" - Round rectangles, hide circle and methods section
+
Current limitations
-------------------
@@ -94,8 +110,24 @@ def recordtypes_to_plantuml_string(iterable):
either the "type" attribute is None or
type(element) == RecordType.
- Inheritance of Properties is not rendered nicely at the moment.
+
+ Parameters
+ ----------
+ iterable: iterable of caosdb.Entity
+ The objects to be rendered with plantuml.
+
+ no_shadow : bool, optional
+ If true, tell plantuml to use a skin without blurred shadows.
+
+
+ Returns
+ -------
+ out : str
+ The plantuml string for the given container.
"""
+ # TODO: This function needs a review of python type hints.
+
classes = [el for el in iterable
if isinstance(el, db.RecordType)]
dependencies = {}
@@ -140,74 +172,90 @@ def recordtypes_to_plantuml_string(iterable):
return result
result = "@startuml\n\n"
- result += "skinparam classAttributeIconSize 0\n"
- result += "package Properties #DDDDDD {\n"
+ if no_shadow:
+ result += "skinparam shadowing false\n"
+
+ if style == "default":
+ result += "skinparam classAttributeIconSize 0\n"
+ elif style == "salexan":
+ result += """skinparam roundcorner 20\n
+skinparam boxpadding 20\n
+\n
+hide methods\n
+hide circle\n
+"""
+ else:
+ raise ValueError("Unknown style.")
- for p in properties:
- inheritances[p] = p.get_parents()
- dependencies[p] = []
+ if add_properties:
+ result += "package Properties #DDDDDD {\n"
+ for p in properties:
+ inheritances[p] = p.get_parents()
+ dependencies[p] = []
- result += "class \"{klass}\" << (P,#008800) >> {{\n".format(klass=p.name)
+ result += "class \"{klass}\" << (P,#008800) >> {{\n".format(klass=p.name)
- if p.description is not None:
- result += get_description(p.description)
- result += "\n..\n"
+ if p.description is not None:
+ result += get_description(p.description)
+ result += "\n..\n"
- if isinstance(p.datatype, str):
- result += "datatype: " + p.datatype + "\n"
- elif isinstance(p.datatype, db.Entity):
- result += "datatype: " + p.datatype.name + "\n"
- else:
- result += "datatype: " + str(p.datatype) + "\n"
+ if isinstance(p.datatype, str):
+ result += "datatype: " + p.datatype + "\n"
+ elif isinstance(p.datatype, db.Entity):
+ result += "datatype: " + p.datatype.name + "\n"
+ else:
+ result += "datatype: " + str(p.datatype) + "\n"
+ result += "}\n\n"
result += "}\n\n"
- result += "}\n\n"
- result += "package RecordTypes #DDDDDD {\n"
+ if add_recordtypes:
+ result += "package RecordTypes #DDDDDD {\n"
- for c in classes:
- inheritances[c] = c.get_parents()
- dependencies[c] = []
- result += "class \"{klass}\" << (C,#FF1111) >> {{\n".format(klass=c.name)
+ for c in classes:
+ inheritances[c] = c.get_parents()
+ dependencies[c] = []
+ result += "class \"{klass}\" << (C,#FF1111) >> {{\n".format(klass=c.name)
- if c.description is not None:
- result += get_description(c.description)
+ if c.description is not None:
+ result += get_description(c.description)
- props = ""
- props += _add_properties(c, importance=db.FIX)
- props += _add_properties(c, importance=db.OBLIGATORY)
- props += _add_properties(c, importance=db.RECOMMENDED)
- props += _add_properties(c, importance=db.SUGGESTED)
+ props = ""
+ props += _add_properties(c, importance=db.FIX)
+ props += _add_properties(c, importance=db.OBLIGATORY)
+ props += _add_properties(c, importance=db.RECOMMENDED)
+ props += _add_properties(c, importance=db.SUGGESTED)
- if len(props) > 0:
- result += "__Properties__\n" + props
- else:
- result += "\n..\n"
- result += "}\n\n"
+ if len(props) > 0:
+ result += "__Properties__\n" + props
+ else:
+ result += "\n..\n"
+ result += "}\n\n"
- for g in grouped:
- inheritances[g] = g.get_parents()
- result += "class \"{klass}\" << (G,#0000FF) >> {{\n".format(klass=g.name)
- result += "}\n\n"
+ for g in grouped:
+ inheritances[g] = g.get_parents()
+ result += "class \"{klass}\" << (G,#0000FF) >> {{\n".format(klass=g.name)
+ result += "}\n\n"
- for c, parents in inheritances.items():
- for par in parents:
- result += "\"{par}\" <|-- \"{klass}\"\n".format(
- klass=c.name, par=par.name)
+ for c, parents in inheritances.items():
+ for par in parents:
+ result += "\"{par}\" <|-- \"{klass}\"\n".format(
+ klass=c.name, par=par.name)
- for c, deps in dependencies.items():
- for dep in deps:
- result += "\"{klass}\" *-- \"{dep}\"\n".format(
- klass=c.name, dep=dep)
+ for c, deps in dependencies.items():
+ for dep in deps:
+ result += "\"{klass}\" *-- \"{dep}\"\n".format(
+ klass=c.name, dep=dep)
- result += """
+ if add_legend:
+ result += """
package \"B is a subtype of A\" <<Rectangle>> {
A <|-right- B
note "This determines what you find when you query for the RecordType.\\n'FIND RECORD A' will provide Records which have a parent\\nA or B, while 'FIND RECORD B' will provide only Records which have a parent B." as N1
}
"""
- result += """
+ result += """
package \"The property P references an instance of D\" <<Rectangle>> {
class C {
@@ -246,7 +294,8 @@ def retrieve_substructure(start_record_types, depth, result_id_set=None, result_
Returns
-------
db.Container
- A container containing all the retrieved entites or None if cleanup is False.
+ A container containing all the retrieved entites
+ or None if cleanup is False.
"""
# Initialize the id set and result container for level zero recursion depth:
if result_id_set is None:
@@ -260,9 +309,19 @@ def retrieve_substructure(start_record_types, depth, result_id_set=None, result_
result_container.append(entity)
result_id_set.add(entity.id)
for prop in entity.properties:
- if is_reference(prop.datatype) and prop.datatype != db.FILE and depth > 0:
- rt = db.RecordType(name=get_referenced_recordtype(prop.datatype)).retrieve()
- retrieve_substructure([rt], depth-1, result_id_set, result_container, False)
+ if (is_reference(prop.datatype) and prop.datatype != db.FILE and depth > 0):
+ rt = db.RecordType(
+ name=get_referenced_recordtype(prop.datatype)).retrieve()
+ retrieve_substructure([rt], depth-1, result_id_set,
+ result_container, False)
+ # TODO: clean up this hack
+ # TODO: make it also work for files
+ if is_reference(prop.datatype) and prop.value is not None:
+ r = db.Record(id=prop.value).retrieve()
+ retrieve_substructure([r], depth-1, result_id_set, result_container, False)
+ if r.id not in result_id_set:
+ result_container.append(r)
+ result_id_set.add(r.id)
if prop.id not in result_id_set:
result_container.append(prop)
@@ -274,14 +333,23 @@ def retrieve_substructure(start_record_types, depth, result_id_set=None, result_
result_container.append(rt)
result_id_set.add(parent.id)
if depth > 0:
- retrieve_substructure([rt], depth-1, result_id_set, result_container, False)
+ retrieve_substructure([rt], depth-1, result_id_set,
+ result_container, False)
if cleanup:
return result_container
return None
-def to_graphics(recordtypes, filename):
+def to_graphics(recordtypes: List[db.Entity], filename: str,
+ output_dirname: Optional[str] = None,
+ formats: List[str] = ["tsvg"],
+ silent: bool = True,
+ add_properties: bool = True,
+ add_recordtypes: bool = True,
+ add_legend: bool = True,
+ no_shadow: bool = False,
+ style: str = "default"):
"""Calls recordtypes_to_plantuml_string(), saves result to file and
creates an svg image
@@ -293,17 +361,55 @@ def to_graphics(recordtypes, filename):
Iterable with the entities to be displayed.
filename : str
filename of the image without the extension(e.g. data_structure;
+ also without the preceeding path.
data_structure.pu and data_structure.svg will be created.)
+ output_dirname : str
+ the destination directory for the resulting images as defined by the "-o"
+ option by plantuml
+ default is to use current working dir
+ formats : List[str]
+ list of target formats as defined by the -t"..." options by plantuml, e.g. "tsvg"
+ silent : bool
+ Don't output messages.
+ no_shadow : bool, optional
+ If true, tell plantuml to use a skin without blurred shadows.
"""
- pu = recordtypes_to_plantuml_string(recordtypes)
-
- pu_filename = filename+".pu"
- with open(pu_filename, "w") as pu_file:
- pu_file.write(pu)
-
- cmd = "plantuml -tsvg %s" % pu_filename
- print("Executing:", cmd)
-
- if os.system(cmd) != 0:
- raise Exception("An error occured during the execution of plantuml. "
- "Is plantuml installed?")
+ pu = recordtypes_to_plantuml_string(iterable=recordtypes,
+ add_properties=add_properties,
+ add_recordtypes=add_recordtypes,
+ add_legend=add_legend,
+ no_shadow=no_shadow,
+ style=style)
+
+ if output_dirname is None:
+ output_dirname = os.getcwd()
+
+ allowed_formats = [
+ "tpng", "tsvg", "teps", "tpdf", "tvdx", "txmi",
+ "tscxml", "thtml", "ttxt", "tutxt", "tlatex", "tlatex:nopreamble"]
+
+ with tempfile.TemporaryDirectory() as td:
+
+ pu_filename = os.path.join(td, filename + ".pu")
+ with open(pu_filename, "w") as pu_file:
+ pu_file.write(pu)
+
+ for format in formats:
+ extension = format[1:]
+ if ":" in extension:
+ extension = extension[:extension.index(":")]
+
+ if format not in allowed_formats:
+ raise RuntimeError("Format not allowed.")
+ cmd = "plantuml -{} {}".format(format, pu_filename)
+ if not silent:
+ print("Executing:", cmd)
+
+ if os.system(cmd) != 0: # TODO: replace with subprocess.run
+ raise Exception("An error occured during the execution of "
+ "plantuml when using the format {}. "
+ "Is plantuml installed? "
+ "You might want to dry a different format.".format(format))
+ # copy only the final product into the target directory
+ shutil.copy(os.path.join(td, filename + "." + extension),
+ output_dirname)
diff --git a/src/caosdb/utils/register_tests.py b/src/caosdb/utils/register_tests.py
new file mode 100644
index 0000000000000000000000000000000000000000..9d0afcbb0845e1d8d31622e8ab9926f26f7e78f6
--- /dev/null
+++ b/src/caosdb/utils/register_tests.py
@@ -0,0 +1,136 @@
+#!/usr/bin/env python
+# encoding: utf-8
+#
+# This file is a part of the CaosDB Project.
+#
+# Copyright (C) 2022 Alexander Schlemmer <alexander.schlemmer@ds.mpg.de>
+# Copyright (C) 2022 Timm Fitschen <t.fitschen@indiscale.com>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+import caosdb as db
+from caosdb import administration as admin
+
+"""
+This module implements a registration procedure for integration tests which
+need a running CaosDB instance.
+
+It ensures that tests do not accidentally overwrite data in real CaosDB
+instances, as it checks whether the running CaosDB instance is actually the
+correct one, that
+should be used for these tests.
+
+The test files have to define a global variable TEST_KEY which must be unique
+for each test using
+
+set_test_key("ABCDE")
+
+The test procedure (invoked by pytest) checks whether a registration
+information is stored in one of the server properties or otherwise
+- offers to register this test in the currently running database ONLY if this
+ is empty.
+- fails otherwise with a RuntimeError
+
+NOTE: you probably need to use pytest with the -s option to be able to
+ register the test interactively. Otherwise, the server property has to be
+ set before server start-up in the server.conf of the CaosDB server.
+
+This module is intended to be used with pytest.
+
+There is a pytest fixture "clear_database" that performs the above mentioned
+checks and clears the database in case of success.
+"""
+
+TEST_KEY = None
+
+
+def set_test_key(KEY):
+ global TEST_KEY
+ TEST_KEY = KEY
+
+
+def _register_test():
+ res = db.execute_query("COUNT Entity")
+ if not isinstance(res, int):
+ raise RuntimeError("Response from server for Info could not be interpreted.")
+ if res > 0:
+ raise RuntimeError("This instance of CaosDB contains entities already."
+ "It must be empty in order to register a new test.")
+
+ print("Current host of CaosDB instance is: {}".format(
+ db.connection.connection.get_connection()._delegate_connection.setup_fields["host"]))
+ answer = input("This method will register your current test with key {} with the currently"
+ " running instance of CaosDB. Do you want to continue (y/N)?".format(
+ TEST_KEY))
+ if answer != "y":
+ raise RuntimeError("Test registration aborted by user.")
+
+ admin.set_server_property("_CAOSDB_INTEGRATION_TEST_SUITE_KEY",
+ TEST_KEY)
+
+
+def _get_registered_test_key():
+ try:
+ return admin.get_server_property("_CAOSDB_INTEGRATION_TEST_SUITE_KEY")
+ except KeyError:
+ return None
+
+
+def _is_registered():
+ registered_test_key = _get_registered_test_key()
+ if not registered_test_key:
+ return False
+ elif registered_test_key == TEST_KEY:
+ return True
+ else:
+ raise RuntimeError("The database has been setup for a different test.")
+
+
+def _assure_test_is_registered():
+ global TEST_KEY
+ if TEST_KEY is None:
+ raise RuntimeError("TEST_KEY is not defined.")
+ if not _is_registered():
+ answer = input("Do you want to register this instance of CaosDB"
+ " with the current test? Do you want to continue (y/N)?")
+ if answer == "y":
+ _register_test()
+ raise RuntimeError("Test has been registered. Please rerun tests.")
+ else:
+ raise RuntimeError("The database has not been setup for this test.")
+
+
+def _clear_database():
+ c = db.execute_query("FIND ENTITY WITH ID>99")
+ c.delete(raise_exception_on_error=False)
+ return None
+
+
+try:
+ import pytest
+
+ @pytest.fixture
+ def clear_database():
+ """Remove Records, RecordTypes, Properties, and Files ONLY IF the CaosDB
+ server the current connection points to was registered with the appropriate key.
+
+ PyTestInfo Records and the corresponding RecordType and Property are preserved.
+ """
+ _assure_test_is_registered()
+ yield _clear_database() # called before the test function
+ _clear_database() # called after the test function
+except ImportError:
+ raise Warning("""The register_tests module depends on pytest and is
+ intended to be used in integration test suites for the
+ caosdb-pylib library only.""")
diff --git a/src/caosdb/utils/server_side_scripting.py b/src/caosdb/utils/server_side_scripting.py
index 663178dcbda4293cb30dff88efbfb7b7302df70d..7e5ee4390ae3314792d12fd2942980aa3d9c9773 100644
--- a/src/caosdb/utils/server_side_scripting.py
+++ b/src/caosdb/utils/server_side_scripting.py
@@ -30,7 +30,8 @@ from lxml import etree
from caosdb.connection.connection import get_connection
from caosdb.connection.utils import urlencode
-from caosdb.connection.encode import MultipartParam, multipart_encode
+from caosdb.connection.encode import (MultipartParam, multipart_encode,
+ ReadableMultiparts)
def _make_params(pos_args, opts):
@@ -63,6 +64,7 @@ def _make_multipart_request(call, pos_args, opts, files):
filename=filename))
body, headers = multipart_encode(parts)
+ body = ReadableMultiparts(body)
return body, headers
diff --git a/src/caosdb/yamlapi.py b/src/caosdb/yamlapi.py
index 69928af1568bf2150288844d74d606e39d598d0b..80bb4b13e4d1626c5d29c8950f3a22bbb73e0fdb 100644
--- a/src/caosdb/yamlapi.py
+++ b/src/caosdb/yamlapi.py
@@ -98,7 +98,7 @@ def yaml_to_xml(yamlstr):
The string to load the yaml document from.
"""
- return dict_to_xml(yaml.safe_load(yamlstr))
+ return dict_to_xml(yaml.load(yamlstr, Loader=yaml.SafeLoader))
def process(text):
diff --git a/src/doc/administration.rst b/src/doc/administration.rst
index 061acc8364d2ef62f743a20d7b9e6562baac0fc5..eab02e43a833559dc21ea7a9fa5edfaf6431facf 100644
--- a/src/doc/administration.rst
+++ b/src/doc/administration.rst
@@ -5,10 +5,12 @@ The Python script ``caosdb_admin.py`` should be used for administrative tasks.
Call ``caosdb_admin.py --help`` to see how to use it.
The most common task is to create a new user (in the CaosDB realm) and set a
-password for the user (note that a user typically needs to be activated)::
+password for the user (note that a user typically needs to be activated):
- caosdb_admin.py create_user anna
- caosdb_admin.py set_user_password anna
- caosdb_admin.py add_user_roles anna administration
- caosdb_admin.py activate_user anna
+.. code:: console
+
+ $ caosdb_admin.py create_user anna
+ $ caosdb_admin.py set_user_password anna
+ $ caosdb_admin.py add_user_roles anna administration
+ $ caosdb_admin.py activate_user anna
diff --git a/src/doc/conf.py b/src/doc/conf.py
index b05fa1c71c1dcd0b59916594818449d2ebc574bd..0fa5de575f5424e267cad8ecc193cca8230faa8b 100644
--- a/src/doc/conf.py
+++ b/src/doc/conf.py
@@ -25,14 +25,14 @@ import sphinx_rtd_theme # noqa: E402
# -- Project information -----------------------------------------------------
project = 'pycaosdb'
-copyright = '2020, IndiScale GmbH'
+copyright = '2023, IndiScale GmbH'
author = 'Daniel Hornung'
# The short X.Y version
-version = '0.5.2'
+version = '0.12.1'
# The full version, including alpha/beta/rc tags
# release = '0.5.2-rc2'
-release = '0.5.2'
+release = '0.12.1-dev'
# -- General configuration ---------------------------------------------------
@@ -78,6 +78,9 @@ exclude_patterns = []
# The name of the Pygments (syntax highlighting) style to use.
pygments_style = None
+suppress_warnings = [
+ "autosectionlabel.*", # duplicate labels
+]
# -- Options for HTML output -------------------------------------------------
diff --git a/src/doc/configuration.md b/src/doc/configuration.md
index 6e53542f661dcae94622fef24a67cecf7491df9c..02cbbd7b13d916a676ad26c277e370ae76bf3725 100644
--- a/src/doc/configuration.md
+++ b/src/doc/configuration.md
@@ -4,6 +4,15 @@ PyCaosDB tries to read from the inifile specified in the environment variable `P
alternatively in `~/.pycaosdb.ini` upon import. After that, the ini file `pycaosdb.ini` in the
current working directory will be read additionally, if it exists.
+Here, we will look at the most common configuration options. For a full and comprehensive
+description please check out the [example pycaosdb.ini file](https://gitlab.com/caosdb/caosdb-pylib/-/blob/main/examples/pycaosdb.ini). You can download this file and use
+it as a starting point.
+
+
+Typically, you need to change at least the `url` and `username` fields as required. (Ask your
+CaosDB administrator or IT crowd if you do not know what to put there, but for the demo instance at
+https://demo.indiscale.com, `username=admin` and `password=caosdb` should work).
+
## Authentication ##
The default configuration (that your are asked for your password when ever a connection is created
@@ -17,6 +26,8 @@ can be changed by setting `password_method`:
Windows). The password will be queried on first usage.
* with `password_method=plain` (**strongly discouraged**)
+The following illustrates the recommended options:
+
```ini
[Connection]
username=YOUR_USERNAME
@@ -35,7 +46,10 @@ username=YOUR_USERNAME
## SSL Certificate ##
-You can set the pass to the ssl certificate to be used:
+In some cases (especially if you are testing CaosDB) you might need to supply an SSL certificate to
+allow SSL encryption.
+
+The `cacert` option sets the path to the ssl certificate for the connection:
```ini
[Connection]
@@ -49,6 +63,8 @@ with CaosDB which makes the experience much less verbose. Set it to 1 or 2 in ca
debugging (which I hope will not be necessary for this tutorial) or if you want to learn more about
the internals of the protocol.
+`timeout` sets the timeout for requests to the server.
+
A complete list of options can be found in the
[pycaosdb.ini file](https://gitlab.com/caosdb/caosdb-pylib/-/blob/main/examples/pycaosdb.ini) in
the examples folder of the source code.
diff --git a/src/doc/future_caosdb.md b/src/doc/future_caosdb.md
new file mode 100644
index 0000000000000000000000000000000000000000..de6170fa42674ed4e3161fb791a397a149dba659
--- /dev/null
+++ b/src/doc/future_caosdb.md
@@ -0,0 +1,193 @@
+# The future of the CaosDB Python Client
+
+The current Python client has done us great services but its structure and the
+way it is used sometimes feels outdated and clumsy. In this document we sketch
+how it might look different in future and invite everyone to comment or
+contribute to this development.
+
+At several locations in this document there will be links to discussion issues.
+If you want to discuss something new, you can create a new issue
+[here](https://gitlab.com/caosdb/caosdb-pylib/-/issues/new).
+
+## Overview
+Let's get a general impression before discussing single aspects.
+
+``` python
+import caosdb as db
+experiments = db.query("FIND Experiment")
+# print name and date for each `Experiment`
+for exp in experiments:
+ print(exp.name, exp.date)
+
+# suppose `Experiments` reference `Projects` which have a `Funding` Property
+one_exp = experiments[0]
+print(one_exp.Project.Funding)
+
+new_one = db.create_record("Experiment")
+new_one.date = "2022-01-01"
+new_one.name = "Needle Measurement"
+new_one.insert()
+```
+Related discussions:
+- [recursive retrieve in query](https://gitlab.com/caosdb/caosdb-pylib/-/issues/57)
+- [create_record function](https://gitlab.com/caosdb/caosdb-pylib/-/issues/58)
+- [data model utility](https://gitlab.com/caosdb/caosdb-pylib/-/issues/59)
+
+## Quickstart
+Note that you can try out one possible implementation using the
+`caosdb.high_level_api` module. It is experimental and might be removed in
+future!
+
+A `resolve_references` function allows to retrieve the referenced entities of
+an entity, container or a query result set (which is a container).
+It has the following parameters which can also be supplied to the `query`
+function:
+
+- `deep`: Whether to use recursive retrieval
+- `depth`: Maximum recursion depth
+- `references`: Whether to use the supplied db.Container to resolve
+ references. This allows offline usage. Set it to None if you want to
+ automatically retrieve entities from the current CaosDB connection.
+
+In order to allow a quick look at the object structures an easily readable
+serialization is provided by the `to_dict` function. It has the following
+argument:
+- `without_metadata`: Set this to True if you don\'t want to see
+ property metadata like \"unit\" or \"importance\".
+
+This function creates a simple dictionary containing a representation of
+the entity, which can be stored to disk and completely deserialized
+using the function `from_dict`.
+
+Furthermore, the `__str__` function uses this to display objects in yaml
+format by default statement
+
+## Design Decisions
+
+### Dot Notation
+Analogue, to what Pandas does. Provide bracket notation
+`rec.properties["test"]` for Properties with names that are in conflict with
+default attributes or contain spaces (or other forbidden characters).
+
+Entities can be initialized with a set of Propertynames. Those Propertynames will be used as
+attributes such that tab completion is possible in interactive use. The value however will be a special
+value (e.g. UnsetPropertyValue) and accessing it results in an Exception. Thus, tab completion can be used
+but no Properties are inserted unexpectedly with NULL values.
+
+- Raise Exception if attribute does not exist but is accessed?
+
+[Discussion](https://gitlab.com/caosdb/caosdb-pylib/-/issues/60)
+
+We aim for a distinction between "concrete" Properties of Records/RecordTypes and "abstract" Properties as part of the definition of a data model. Concrete properties are always "contained" in a record or record type while abstract properties stand for themselves.
+
+Draft:
+```
+class ConcreteProperty:
+ def __init__(self, v, u):
+ self.value = v
+ self.unit = u
+
+class Entity:
+ def __init__(self):
+ pass
+
+ def __setattr__(self, name, val):
+ if name not in dir(self):
+ # setattr(self, name, ConcreteProperty(val, None))
+ self.properties[name] = ConcreteProperty(val, None)
+ else:
+ # getattribute(self, name).value = val
+ self.properties[name].value = val
+```
+
+The old "get_property" functions serves the same purpose as the new "[]" notation.
+
+Instead of `get_property` / `add_property` etc. functions belonging to class Entity, we should refactor the list of properties (of an entity) to be a special kind of list, e.g. PropertyList.
+This list should enherit from a standard list, have all the known functions like "append", "extend", "__in__" and allow for all property-related functionality as part of its member functions (instead of access via Entity directly).
+Same story for the parents.
+
+**GET RID OF MULTI PROPERTIES!!!**
+
+#### how to deal with "property metadata"
+
+Current suggestion: stored in a special field "property_metadata" belonging to the object.
+`property_metadata` is a dict:
+- importance
+- unit
+- description
+- ...
+
+### Serialization
+What information needs to be contained in (meta)data? How compatible is it with
+GRPC json serialization?
+
+
+### Recursive Retrieval
+
+
+
+I can resolve later and end up with the same result:
+`recs =db.query("FIND Experiment", depth=2)` equals `recs = db.query("FIND Experiment"); recs = resolve_references(recs, depth=2)`
+
+[Discussion](https://gitlab.com/caosdb/caosdb-pylib/-/issues/57)
+
+
+#### Alternative
+
+`FIND Experiment` with `depth=2` will retrieve all referenced entities from any experiment found. A typical use case could also be:
+
+```python
+recs = db.query("FIND Experiment")
+recs[0].resolve_references(depth=2)
+```
+
+#### Idea
+
+Recursive retrievel as functionality of the server.
+
+retrieve and query commands should support the `depth` argument.
+
+### In-Place operations
+Default behavior is to return new objects instead of modifying them in-place.
+This can be changed with the argument `inplace=True`.
+Especially the following functions operate by default NOT in-place:
+- update
+- insert
+- retrieve
+- resolve_references
+[Discussion](https://gitlab.com/caosdb/caosdb-pylib/-/issues/61)
+
+## Extended Example
+``` python
+import caosdb as db
+
+dm = db.get_data_model()
+
+new_one = db.create_record(dm.Experiment)
+new_one.date = "2022-01-01"
+new_one.name = "Needle Measurement"
+new_one.dataset = db.create_record(dm.Dataset)
+new_one.dataset.voltage = (5, "V")
+new_one.dataset.pulses = [5, 5.3]
+inserted = new_one.insert()
+print("The new record has the ID:", inserted.id)
+```
+
+### Factory method
+While creating an Entity will not talk to a CaosDB server and can thus be done offline, the factory method
+`create_record` allows to
+1. Retrieve the parent and set attributes according to inheritance
+2. Use a container to resolve the parent and set attributes
+
+In general, more complex "magic" will be placed in the factory and only the straight forward version
+in the constructor.
+
+### References and sub entities
+
+Several possibilities exist for references:
+
+- value is the id of a referenced entity
+- value is a "sub object"
+- value is a reference to another (entity-)list element (similar to second variant, but with "sub object" always contained in container/entity-list)
+
+To be discussed: Which should be the obligatory/preferred variant?
diff --git a/src/doc/gallery/Makefile b/src/doc/gallery/Makefile
new file mode 100644
index 0000000000000000000000000000000000000000..658f9a6a93e23957b20aee5f38e5565bde35af80
--- /dev/null
+++ b/src/doc/gallery/Makefile
@@ -0,0 +1,23 @@
+# This file is a part of the CaosDB Project.
+#
+# Copyright (C) 2022 IndiScale GmbH <info@indiscale.com>
+# Copyright (C) 2022 Daniel Hornung <d.hornung@indiscale.com>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+
+# Run tests on the examples.
+test:
+ python3 -m doctest simulation.py
+.PHONY: test
diff --git a/src/doc/gallery/curator-permissions.rst b/src/doc/gallery/curator-permissions.rst
new file mode 100644
index 0000000000000000000000000000000000000000..fa6b4022b7fbc1d042ed00f265e63a2675794a21
--- /dev/null
+++ b/src/doc/gallery/curator-permissions.rst
@@ -0,0 +1,123 @@
+
+Setting permissions for a curator role
+======================================
+
+The following example shows how to create and set permissions for a ``curator``
+role that is allowed to insert, update, or delete any entity apart from a set of
+RecordTypes and properties that define a "core data model" which can only be
+altered with administration permissions.
+
+In the following, you'll learn how to
+
+1. create the ``curator`` role.
+2. configure the ``global_entity_permissions.xml`` s.th. the ``curator`` role is
+ allowed to insert, update, or delete any entity by default.
+3. use a Python script to override the above configuration for the entities in
+ the externally defined core data model.
+
+Prerequisites
+-------------
+
+This example needs some preparations regarding your CaosDB setup that have to
+(or, for the sake of simplicity, should) be done outside the actual Python
+example script.
+
+The curator role
+~~~~~~~~~~~~~~~~
+
+First, a ``curator`` role is created with a meaningful description. We'll use
+``caosdb_admin.py`` for this which leads to the following command:
+
+.. code:: console
+
+ $ caosdb_admin.py create_role "curator" "A user who is permitted to create new Records, Properties, and RecordTypes but who is not allowed to change the core data model."
+
+To actually see how this role's permissions change, we also need a user with
+this role. Assume you already have created and activated (see
+:doc:`Administration <../administration>`) a ``test_curator`` user, then
+``caosdb_admin.py`` is used again to assign it the correct role:
+
+.. code:: console
+
+ $ caosdb_admin.py add_user_roles test_curator curator
+
+.. note::
+
+ The ``test_curator`` user shouldn't have administration privileges, otherwise
+ the below changes won't have any effect.
+
+The core data model and caosdb-advanced-user-tools
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+In principle, the following script works with any data model defined in a json
+or yaml file (just adapt lines 39-42 accordingly). In this example, we'll use the
+`metadata schema <https://github.com/leibniz-zmt/zmt-metadata-schema>`_ that was
+developed by J. Schmidt at the `Leibniz Centre for Tropical Marine Research
+<https://www.leibniz-zmt.de/en/>`_.
+
+Clone the schemata into the same directory containing the below script via
+
+.. code:: console
+
+ $ git clone https://github.com/leibniz-zmt/zmt-metadata-schema.git
+
+Furthermore, we'll need the `CaosDB Advanced User Tools
+<https://gitlab.com/caosdb/caosdb-advanced-user-tools>`_ for loading the
+metadata schemata from the json files, so install them via
+
+.. code:: console
+
+ $ pip install caosadvancedtools
+
+The global entity permissions file
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Users with the ``curator`` role should be able to have any permission for all
+entities by default. The exceptions for the core data model entities will be set
+with the script below. These default settings are best done via the
+``global_entities_permissions.xml`` config file (see the `server documentation
+<https://docs.indiscale.com/caosdb-server/permissions.html#how-to-set-permissions>`_). Simply
+add the following line to the file
+
+.. code:: xml
+
+ <Grant priority="true" role="curator"><Permission name="*"/></Grant>
+
+This means that, by default, all users with the ``curator`` role are **granted**
+all entity permissions (including insert, update, and delete as specified in the
+beginning) **with priority**. This ensures, that no normal user is allowed to
+overrule these permissions (since it is granted with priority), but it can still
+be denied for the core data model entities by a **deny** rule with priority. See
+the server documentation on `permission
+calculation <https://docs.indiscale.com/caosdb-server/permissions.html#permission-calculation>`_
+for more information on which permission rules can or can't be overruled.
+
+Your complete ``global_entities_permissions.xml`` might then look like
+
+.. code:: xml
+
+ <globalPermissions>
+ <Grant priority="false" role="?OWNER?"><Permission name="*"/></Grant>
+ <Grant priority="false" role="?OTHER?"><Permission name="RETRIEVE:*"/></Grant>
+ <Grant priority="false" role="?OTHER?"><Permission name="USE:*"/></Grant>
+ <Grant priority="false" role="anonymous"><Permission name="RETRIEVE:*"/></Grant>
+ <Grant priority="true" role="curator"><Permission name="*"/></Grant>
+ <Deny priority="false" role="?OTHER?"><Permission name="UPDATE:*"/></Deny>
+ <Deny priority="false" role="?OTHER?"><Permission name="DELETE"/></Deny>
+ <Deny priority="true" role="?OTHER?"><Permission name="EDIT:ACL"/></Deny>
+ </globalPermissions>
+
+.. note::
+
+ Note that you have to restart your CaosDB server after modifying the
+ ``global_entities_permissions.xml``.
+
+The code
+--------
+
+After having applied all of the above prerequisites and restarting your CaosDB
+server, execute the following code.
+
+:download:`Download full code<curator_permissions.py>`
+
+.. literalinclude:: curator_permissions.py
diff --git a/src/doc/gallery/curator_permissions.py b/src/doc/gallery/curator_permissions.py
new file mode 100644
index 0000000000000000000000000000000000000000..16b4b7f6f1bb9abfb7e191c6a1101181984bce9a
--- /dev/null
+++ b/src/doc/gallery/curator_permissions.py
@@ -0,0 +1,100 @@
+#!/usr/bin/env python3
+# encoding: utf-8
+#
+# This file is a part of the CaosDB Project.
+#
+# Copyright (C) 2022 Indiscale GmbH <info@indiscale.com>
+# Copyright (C) 2022 Florian Spreckelsen <f.spreckelsen@indiscale.com>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see <https://www.gnu.org/licenses/>.
+#
+
+import os
+import sys
+
+import caosdb as db
+from caosadvancedtools.models.parser import parse_model_from_json_schema
+from caosdb import administration as admin
+
+CURATOR = "curator"
+
+
+def main():
+ """Set curator role permissions: Is allowed to edit all Records; is allowed
+ to create new RTs and Properties and change them, but is not allowed to
+ change anything defined in the core data model, i.e., in the schemas.
+
+ """
+ dataspace_definitions = parse_model_from_json_schema(
+ "zmt-metadata-schema/schemas/dataspace.schema.json")
+ dataset_definitions = parse_model_from_json_schema(
+ "zmt-metadata-schema/schemas/dataset.schema.json")
+
+ # Set general permissions. The curator users should be allowed to perform
+ # any transaction.
+ perms = admin._get_permissions(CURATOR)
+ general_grant_perms = [
+ "TRANSACTION:*"
+ ]
+
+ for p in general_grant_perms:
+
+ g = admin.PermissionRule(action="Grant", permission=p, priority=True)
+ d = admin.PermissionRule(action="Deny", permission=p, priority=True)
+
+ if g in perms:
+ perms.remove(g)
+ if d in perms:
+ perms.remove(d)
+ perms.add(g)
+
+ admin._set_permissions(CURATOR, permission_rules=perms)
+
+ # Deny all permissions that could change the data model ...
+ core_model_deny_permissions = [
+ "DELETE",
+ "UPDATE:*",
+ "EDIT:ACL"
+ ]
+ # ... but allow read-access and of course using the entities as parents,
+ # properties, ...
+ core_model_grant_permissions = [
+ "RETRIEVE:*",
+ "USE:*",
+ ]
+
+ # Iterate over all entities defined in the schemas and update their access control list (ACL) accordingly.
+ updates = db.Container()
+ for model in [dataspace_definitions, dataset_definitions]:
+
+ for ent in model.values():
+ if ent.name in [u.name for u in updates]:
+ # Skip entities that have been updated already
+ continue
+ # The entity needs to be retrieved with the ACL flag to update the
+ # ACL down the road
+ ent.retrieve(flags={"ACL": None})
+ for d in core_model_deny_permissions:
+ ent.deny(role=CURATOR, priority=True, permission=d)
+ ent.update_acl()
+ ent.retrieve(flags={"ACL": None})
+ for g in core_model_grant_permissions:
+ ent.grant(role=CURATOR, priority=True, permission=g)
+ updates.append(ent)
+ ent.update_acl()
+
+
+if __name__ == "__main__":
+
+ sys.exit(main())
diff --git a/src/doc/gallery/index.rst b/src/doc/gallery/index.rst
new file mode 100644
index 0000000000000000000000000000000000000000..bfba4317c3556d0692eb402f42ba3699be586d5a
--- /dev/null
+++ b/src/doc/gallery/index.rst
@@ -0,0 +1,17 @@
+
+PyCaosDB Code Gallery
+=====================
+
+This chapter collects code examples which can be immediately run against an empty CaosDB instance.
+
+.. note::
+
+ These examples require a configuration file with valid server and user/password settings. Refer
+ to the :ref:`Configuration <Configuration of PyCaosDB>` section for details.
+
+.. toctree::
+ :maxdepth: 2
+ :caption: The code examples:
+
+ simulation
+ curator-permissions
diff --git a/src/doc/gallery/simulation.py b/src/doc/gallery/simulation.py
new file mode 100644
index 0000000000000000000000000000000000000000..342d5d980fc2b1a981f4a76d99e1954f8b2f5c2a
--- /dev/null
+++ b/src/doc/gallery/simulation.py
@@ -0,0 +1,134 @@
+"""
+Run a simulation and store the values into CaosDB.
+
+>>> main() # doctest: +ELLIPSIS
+These distances resulted in small x,y, values:
+[...]
+"""
+
+import numpy as np
+import scipy.integrate
+import caosdb as db
+from caosadvancedtools.table_converter import to_table
+
+
+def setup_caosdb():
+ """Create the data model and insert it into CaosDB
+
+ The data model consists of the following RecordTypes:
+
+ Software
+ with author and revision.
+
+ SoftwareRun
+ A specific run of the sofware, with input parameters, time of completion and a result.
+
+ State
+ An aggregate of x,y,z values.
+
+ Parameters
+ In this case the x,y,z initial values before the integration, so this is just a state.
+
+ Result
+ The x,y,z values at the end of the software run, the final state.
+
+ The data model of course also contains the corresponding properties for these RecordTypes.
+ """
+
+ cont = db.Container() # Container to insert all Entities at once into CaosDB
+ # create Properties
+ cont.append(db.Property("x", datatype=db.DOUBLE))
+ cont.append(db.Property("y", datatype=db.DOUBLE))
+ cont.append(db.Property("z", datatype=db.DOUBLE))
+ cont.append(db.Property("completed", datatype=db.DATETIME))
+ cont.append(db.Property("author", datatype=db.TEXT))
+ cont.append(db.Property("revision", datatype=db.TEXT))
+ # create RecordTypes
+ cont.append(db.RecordType("Software").add_property("author").add_property("revision"))
+ cont.append(db.RecordType("State").add_property("x", importance=db.OBLIGATORY)
+ .add_property("y").add_property("z"))
+ cont.append(db.RecordType("Parameters").add_parent("State", inheritance=db.ALL))
+ cont.append(db.RecordType("Result").add_parent("State", inheritance=db.RECOMMENDED))
+ cont.append(db.RecordType("SoftwareRun").add_property("Software").add_property("Parameters")
+ .add_property("completed").add_property("Result"))
+ cont.insert() # actually insert the Entities
+
+
+def simulations(n, t_max):
+ """Run the simulations.
+
+ Parameters
+ ----------
+ n : int
+ The number of runs.
+
+ t_max : float
+ The maximum time of integration.
+ """
+
+ software = (db.Record("simulator").add_parent("Software")
+ .add_property("author", value="IndiScale GmbH")
+ .add_property("revision", value="1234CDEF89AB"))
+ software.insert()
+ for i in range(n):
+ # Get the parameters and result
+ initial, result = run_simulation(run=i, t_max=t_max)
+
+ # Prepare CaosDB insertion
+ run = db.Record().add_parent("SoftwareRun").add_property("Software", value=software.id)
+ parameters = (db.Record().add_parent("Parameters").add_property("x", initial[0])
+ .add_property("y", initial[1]).add_property("z", initial[2]))
+ result_record = (db.Record().add_parent("Result").add_property("x", result[0])
+ .add_property("y", result[1]).add_property("z", result[2]))
+ run.add_property("Parameters", value=parameters).add_property("Result", value=result_record)
+ cont = db.Container()
+ cont.extend([run, parameters, result_record])
+ cont.insert() # Insert everything of this run into CaosDB.
+
+
+def run_simulation(run, t_max):
+ """Integrate the Rössler attractor from random initial values."""
+ a, b, c = (0.1, 0.1, 14)
+
+ def diff(t, x):
+ diff = np.array([-x[1] - x[2],
+ x[0] + a * x[1],
+ b + x[2] * (x[0] - c)])
+ return diff
+
+ x0 = np.random.uniform(-100, 100, 3)
+
+ result = scipy.integrate.solve_ivp(diff, [0, t_max], x0)
+ x = result.y[:, -1]
+ return (x0, x)
+
+
+def analyze():
+ """Find the initial conditions which produce the smalles x,y values after the given time."""
+ distance = 5
+ data = db.execute_query("""SELECT Parameters, Result FROM RECORD SoftwareRun WITH
+ (((Result.x < {dist}) AND (Result.x > -{dist}))
+ AND (Result.y < {dist})) AND Result.y > -{dist}""".format(dist=distance))
+ dataframe = to_table(data) # Convert into a Pandas DataFrame
+
+ parameters = db.Container().extend([db.Record(id=id) for id in dataframe.Parameters]).retrieve()
+
+ initial_distances = [np.linalg.norm([p.get_property(dim).value for dim in ["x", "y", "z"]])
+ for p in parameters]
+
+ print("These distances resulted in small x,y, values:\n{}".format(initial_distances))
+
+
+def main():
+ # 1. Set up the data model
+ setup_caosdb()
+
+ # 2. Run simulations
+ simulations(n=200, t_max=5)
+
+ # 3. Find initial conditions with interesting results
+ analyze()
+
+
+if __name__ == '__main__':
+ main()
diff --git a/src/doc/gallery/simulation.rst b/src/doc/gallery/simulation.rst
new file mode 100644
index 0000000000000000000000000000000000000000..ce1a7f457a142e36ef9f2b0cfe6a4df0b9fcedf6
--- /dev/null
+++ b/src/doc/gallery/simulation.rst
@@ -0,0 +1,14 @@
+========================================
+Managing data from numerical simulations
+========================================
+
+This code example
+
+1. sets up the data model
+2. runs simulations
+3. stores the simulation parameters and results into CaosDB
+4. retrieves the parameters for interesting results.
+
+:download:`Download code<simulation.py>`
+
+.. literalinclude:: simulation.py
diff --git a/src/doc/high_level_api.org b/src/doc/high_level_api.org
new file mode 100644
index 0000000000000000000000000000000000000000..516df1b41d500fab000a72517fd2d12ba61753b7
--- /dev/null
+++ b/src/doc/high_level_api.org
@@ -0,0 +1,171 @@
+* High Level API
+
+In addition to the old standard pylib API, new versions of pylib ship with a high level API
+that facilitates usage of CaosDB entities within data analysis scripts. In a nutshell that
+API exposes all properties of CaosDB Records as standard python attributes making their
+access easier.
+
+Or to spell it out directly in Python:
+#+BEGIN_SRC python
+
+ import caosdb as db
+ # Old API:
+ r = db.Record()
+ r.add_parent("Experiment")
+ r.add_property(name="alpha", value=5)
+ r.get_property("alpha").value = 25 # setting properties (old api)
+ print(r.get_property("alpha").value + 25) # getting properties (old api)
+
+ from caosdb.high_level_api import convert_to_python_entity
+ obj = convert_to_python_object(r) # create a high level entity
+ obj.r = 25 # setting properties (new api)
+ print(obj.r + 25) # getting properties (new api)
+
+#+END_SRC
+
+
+** Quickstart
+
+The module, needed for the high level API is called:
+~caosdb.high_level_api~
+
+There are two functions converting entities between the two representation (old API and new API):
+- ~convert_to_python_object~: Convert entities from **old** into **new** representation.
+- ~convert_to_entity~: Convert entities from **new** into **old** representation.
+
+Furthermore there are a few utility functions which expose very practical shorthands:
+- ~new_high_level_entity~: Retrieve a record type and create a new high level entity which contains properties of a certain importance level preset.
+- ~create_record~: Create a new high level entity using the name of a record type and a list of key value pairs as properties.
+- ~load_external_record~: Retrieve a record with a specific name and return it as high level entity.
+- ~create_entity_container~: Convert a high level entity into a standard entity including all sub entities.
+- ~query~: Do a CaosDB query and return the result as a container of high level objects.
+
+So as a first example, you could retrieve any record from CaosDB and use it using its high level representation:
+#+BEGIN_SRC python
+ from caosdb.high_level_api import query
+
+ res = query("FIND Record Experiment")
+ experiment = res[0]
+ # Use a property:
+ print(experiment.date)
+
+ # Use sub properties:
+ print(experiment.output[0].path)
+#+END_SRC
+
+The latter example demonstrates, that the function query is very powerful. For its default parameter
+values it automatically resolves and retrieves references recursively, so that sub properties,
+like the list of output files "output", become immediately available.
+
+**Note** that for the old API you were supposed to run the following series of commands
+to achieve the same result:
+#+BEGIN_SRC python
+ import caosdb as db
+
+ res = db.execute_query("FIND Record Experiment")
+ output = res.get_property("output")
+ output_file = db.File(id=output.value[0].id).retrieve()
+ print(output_file.path)
+#+END_SRC
+
+Resolving subproperties makes use of the "resolve_reference" function provided by the high level
+entity class (~CaosDBPythonEntity~), with the following parameters:
+- ~deep~: Whether to use recursive retrieval
+- ~references~: Whether to use the supplied db.Container to resolve references. This allows offline usage. Set it to None if you want to automatically retrieve entities from the current CaosDB connection.
+- ~visited~: Needed for recursion, set this to None.
+
+Objects in the high level representation can be serialized to a simple yaml form using the function
+~serialize~ with the following parameters:
+- ~without_metadata~: Set this to True if you don't want to see property metadata like "unit" or "importance".
+- ~visited~: Needed for recursion, set this to None.
+
+This function creates a simple dictionary containing a representation of the entity, which can be
+stored to disk and completely deserialized using the function ~deserialize~.
+
+Furthermore the "__str__" function is overloaded, so that you can use print to directly inspect
+high level objects using the following statement:
+#+BEGIN_SRC python
+print(str(obj))
+#+END_SRC
+
+
+** Concepts
+
+As described in the section [[Quickstart]] the two functions ~convert_to_python_object~ and ~convert_to_entity~ convert
+entities beetween the high level and the standard representation.
+
+The high level entities are represented using the following classes from the module ~caosdb.high_level_api~:
+- ~CaosDBPythonEntity~: Base class of the following entity classes.
+- ~CaosDBPythonRecord~
+- ~CaosDBPythonRecordType~
+- ~CaosDBPythonProperty~
+- ~CaosDBPythonMultiProperty~: **WARNING** Not implemented yet.
+- ~CaosDBPythonFile~: Used for file entities and provides an additional ~download~ function for being able to directly retrieve files from CaosDB.
+
+In addition, there are the following helper structures which are realized as Python data classes:
+- ~CaosDBPropertyMetaData~: For storing meta data about properties.
+- ~CaosDBPythonUnresolved~: The base class of unresolved "things".
+- ~CaosDBPythonUnresolvedParent~: Parents of entities are stored as unresolved parents by default, storing an id or a name of a parent (or both).
+- ~CaosDBPythonUnresolvedReference~: An unresolved reference is a reference property with an id which has not (yet) been resolved to an Entity.
+
+The function "resolve_references" can be used to recursively replace ~CaosDBPythonUnresolvedReferences~ into members of type ~CaosDBPythonRecords~
+or ~CaosDBPythonFile~.
+
+Each property stored in a CaosDB record corresponds to:
+- a member attribute of ~CaosDBPythonRecord~ **and**
+- an entry in a dict called "metadata" storing a CaosDBPropertyMetadata object with the following information about proeprties:
+ - ~unit~
+ - ~datatype~
+ - ~description~
+ - ~id~
+ - ~importance~
+
+
+* Example
+
+The following shows a more complex example taken from a real world use case:
+A numerical experiment is created to simulate cardiac electric dynamics. The physical problem
+is modelled using the monodomain equation with the local current term given by the Mitchell
+Schaeffer Model.
+
+The data model for the numerical experiment consists of multiple record types which stores assosciated paremeters:
+- `MonodomainTissueSimulation`
+- `MitchellSchaefferModel`
+- `SpatialExtent2d`
+- `SpatialDimension`
+- `ConstantTimestep`
+- `ConstantDiffusion`
+
+First, the data model will be filled with the parameter values for this specific simulation run. It will be stored in the python variable `MonodomainRecord`. Passing the `MonodomainRecord` through the python functions, the simulation parameters can be easily accessed everywhere in the code when needed.
+
+Records are created by the `create_record` function. Parameter values can be set at record creation and also after creation as python properties of the corresponding record instance. The following example shows how to create a record, how to set the parameter at creation and how to set them as python properties
+
+#+BEGIN_SRC python
+ from caosdb.high_level_api import create_record
+
+ MonodomainRecord = create_record("MonodomainTissueSimulation")
+ MonodomainRecord.LocalModel = create_record("MitchellSchaefferModel")
+ MonodomainRecord.SpatialExtent = create_record(
+ "SpatialExtent2d", spatial_extent_x=100, spatial_extent_y=100)
+ MonodomainRecord.SpatialExtent.cell_sizes = [0.1, 0.1] # parameters can be set as properties
+ MonodomainRecord.SpatialDimension = create_record("SpatialDimension",
+ num_dim=2)
+
+ MonodomainRecord.TimestepInformation = create_record("ConstantTimestep")
+ MonodomainRecord.TimestepInformation.DeltaT = 0.1
+
+ D = create_record("ConstantDiffusion", diffusion_constant=0.1)
+ MonodomainRecord.DiffusionConstantType = D
+ model = MonodomainRecord.LocalModel
+ model.t_close = 150
+ model.t_open = 120
+ model.t_out = 6
+ model.t_in = 0.3
+ model.v_gate = 0.13
+ model.nvars = 2
+#+END_SRC
+
+At any position in the algorithm you are free to:
+- Convert this model to the standard python API and insert or update the records in a running instance of CaosDB.
+- Serialize this model in the high level API yaml format. This enables the CaosDB crawler to pickup the file and synchronize it with a running instance
+of CaosDB.
diff --git a/src/doc/high_level_api.rst b/src/doc/high_level_api.rst
new file mode 100644
index 0000000000000000000000000000000000000000..e92f2dd5e326b14222ad3c776ce5f5ed1ed31536
--- /dev/null
+++ b/src/doc/high_level_api.rst
@@ -0,0 +1,163 @@
+High Level API
+==============
+
+In addition to the old standard pylib API, new versions of pylib ship
+with a high level API that facilitates usage of CaosDB entities within
+data analysis scripts. In a nutshell that API exposes all properties of
+CaosDB Records as standard python attributes making their access easier.
+
+Or to speak it out directly in Python:
+
+.. code:: python
+
+
+ import caosdb as db
+ # Old API:
+ r = db.Record()
+ r.add_parent("Experiment")
+ r.add_property(name="alpha", value=5)
+ r.get_property("alpha").value = 25 # setting properties (old api)
+ print(r.get_property("alpha").value + 25) # getting properties (old api)
+
+ from caosdb.high_level_api import convert_to_python_entity
+ obj = convert_to_python_object(r) # create a high level entity
+ obj.r = 25 # setting properties (new api)
+ print(obj.r + 25) # getting properties (new api)
+
+Quickstart
+----------
+
+The module, needed for the high level API is called:
+``caosdb.high_level_api``
+
+There are two functions converting entities between the two
+representation (old API and new API):
+
+- ``convert_to_python_object``: Convert entities from **old** into
+ **new** representation.
+- ``convert_to_entity``: Convert entities from **new** into **old**
+ representation.
+
+Furthermore there are a few utility functions which expose very
+practical shorthands:
+
+- ``new_high_level_entity``: Retrieve a record type and create a new
+ high level entity which contains properties of a certain importance
+ level preset.
+- ``create_record``: Create a new high level entity using the name of a
+ record type and a list of key value pairs as properties.
+- ``load_external_record``: Retrieve a record with a specific name and
+ return it as high level entity.
+- ``create_entity_container``: Convert a high level entity into a
+ standard entity including all sub entities.
+- ``query``: Do a CaosDB query and return the result as a container of
+ high level objects.
+
+So as a first example, you could retrieve any record from CaosDB and use
+it using its high level representation:
+
+.. code:: python
+
+ from caosdb.high_level_api import query
+
+ res = query("FIND Experiment")
+ experiment = res[0]
+ # Use a property:
+ print(experiment.date)
+
+ # Use sub properties:
+ print(experiment.output[0].path)
+
+The latter example demonstrates, that the function query is very
+powerful. For its default parameter values it automatically resolves and
+retrieves references recursively, so that sub properties, like the list
+of output files "output", become immediately available.
+
+**Note** that for the old API you were supposed to run the following
+series of commands to achieve the same result:
+
+.. code:: python
+
+ import caosdb as db
+
+ res = db.execute_query("FIND Experiment")
+ output = res.get_property("output")
+ output_file = db.File(id=output.value[0].id).retrieve()
+ print(output_file.path)
+
+Resolving subproperties makes use of the "resolve\ :sub:`reference`"
+function provided by the high level entity class
+(``CaosDBPythonEntity``), with the following parameters:
+
+- ``deep``: Whether to use recursive retrieval
+- ``references``: Whether to use the supplied db.Container to resolve
+ references. This allows offline usage. Set it to None if you want to
+ automatically retrieve entities from the current CaosDB connection.
+- ``visited``: Needed for recursion, set this to None.
+
+Objects in the high level representation can be serialized to a simple
+yaml form using the function ``serialize`` with the following
+parameters:
+
+- ``without_metadata``: Set this to True if you don't want to see
+ property metadata like "unit" or "importance".
+- ``visited``: Needed for recursion, set this to None.
+
+This function creates a simple dictionary containing a representation of
+the entity, which can be stored to disk and completely deserialized
+using the function ``deserialize``.
+
+Furthermore the "*str*" function is overloaded, so that you can use
+print to directly inspect high level objects using the following
+statement:
+
+.. code:: python
+
+ print(str(obj))
+
+Concepts
+--------
+
+As described in the section Quickstart the two functions
+``convert_to_python_object`` and ``convert_to_entity`` convert entities
+beetween the high level and the standard representation.
+
+The high level entities are represented using the following classes from
+the module ``caosdb.high_level_api``:
+
+- ``CaosDBPythonEntity``: Base class of the following entity classes.
+- ``CaosDBPythonRecord``
+- ``CaosDBPythonRecordType``
+- ``CaosDBPythonProperty``
+- ``CaosDBPythonMultiProperty``: **WARNING** Not implemented yet.
+- ``CaosDBPythonFile``: Used for file entities and provides an
+ additional ``download`` function for being able to directly retrieve
+ files from CaosDB.
+
+In addition, there are the following helper structures which are
+realized as Python data classes:
+
+- ``CaosDBPropertyMetaData``: For storing meta data about properties.
+- ``CaosDBPythonUnresolved``: The base class of unresolved "things".
+- ``CaosDBPythonUnresolvedParent``: Parents of entities are stored as
+ unresolved parents by default, storing an id or a name of a parent
+ (or both).
+- ``CaosDBPythonUnresolvedReference``: An unresolved reference is a
+ reference property with an id which has not (yet) been resolved to an
+ Entity.
+
+The function "resolve\ :sub:`references`" can be used to recursively
+replace ``CaosDBPythonUnresolvedReferences`` into members of type
+``CaosDBPythonRecords`` or ``CaosDBPythonFile``.
+
+Each property stored in a CaosDB record corresponds to:
+
+- a member attribute of ``CaosDBPythonRecord`` **and**
+- an entry in a dict called "metadata" storing a CaosDBPropertyMetadata
+ object with the following information about proeprties:
+
+ - ``unit``
+ - ``datatype``
+ - ``description``
+ - ``id``
+ - ``importance``
diff --git a/src/doc/index.rst b/src/doc/index.rst
index bd29c6c56acf5c173e94ae6471a6aeba56ea4b93..7344b6aacdd55fd75f4940d834104faa00c33069 100644
--- a/src/doc/index.rst
+++ b/src/doc/index.rst
@@ -12,6 +12,8 @@ Welcome to PyCaosDB's documentation!
Concepts <concepts>
Configuration <configuration>
Administration <administration>
+ High Level API <high_level_api>
+ Code gallery <gallery/index>
API documentation<_apidoc/caosdb>
This is the documentation for the Python client library for CaosDB, ``PyCaosDB``.
diff --git a/src/doc/tutorials/Data-Insertion.rst b/src/doc/tutorials/Data-Insertion.rst
index 84223db3615688a1fb900eefa2b86115b266b307..82df07691f7c78a2787d67463ca222d2e68249ca 100644
--- a/src/doc/tutorials/Data-Insertion.rst
+++ b/src/doc/tutorials/Data-Insertion.rst
@@ -10,7 +10,8 @@ Model<caosdb-server:Data-Model>` in the CaosDB server documentation.
In order to insert some actual data, we need to create a data model
using RecordTypes and Properties (You may skip this if you use a CaosDB
-instance that already has the required types). So, let’s create a simple
+instance that already has the required types). When you create a new Property
+you must supply a datatype. So, let’s create a simple
Property called “a” of datatype double. This is very easy in pylib:
.. code:: python
@@ -82,33 +83,86 @@ corresponding python class:
.. code:: python
- rec = db.Record()
+ rec = db.Record() # rec.id is None
rec.add_parent(name="Experiment")
rec.add_property(name="date", value="2020-01-07")
rec.insert()
+ print(rec.id) # rec.id set by the server
+
+Here, the record has a parent, the RecordType “Experiment”, and a Property date
+with a value ``"2020-01-07"``. After the successful insertion, our new Record is
+assigned an ``id`` by the server. In the following, let's assume this id to be
+``256``.
+
+Reference Properties
+--------------------
+
+Now suppose we want to insert an analysis that references the above experiment
+record as its source data. Since we know that the id of the experiment record is
+256, we can do the following:
+
+.. code:: python
+
+ ana = db.Record().add_parent(name="Analysis") # Create record and assign parent in one line
+ ana.add_property(name="Experiment", value=256)
+ ana.add_propertt(name="date", value="2020-01-08")
+ # possibly, add more properties here ...
+ ana.insert()
+
+The experiment record's id is used as the value of the ``Experiment`` property
+of the analysis Record (note how we use the RecordType ``Experiment`` as a
+``REFERENCE`` property here). Sending a CaosDB query like ``FIND RECORD
+Experiment WHICH IS REFERENCED BY A Analysis WITH date=2020-01-08`` would now
+return our original experiment record.
-Here, the record has a parent: The RecordType “Experiment”. And a
-Property: date.
+Equivalently, we can also use the Python object of the experiment record, i.e.,
+``rec`` as the value of the ``Experiment`` property:
-Note, that if you want to use a property that is not a primitive
-datatype like db.INTEGER and so on, you need to use the ID of the Entity
-that you are referencing.
+
+.. code:: python
+
+ ana = db.Record().add_parent(name="Analysis")
+ ana.add_property(name="Experiment", value=rec)
+ ana.add_propertt(name="date", value="2020-01-08")
+ # possibly, add more properties here ...
+ ana.insert()
+
+Finally, we can also insert both records at the same time using a
+``db.Container``:
.. code:: python
rec = db.Record()
rec.add_parent(name="Experiment")
- rec.add_property(name="report", value=235507)
- rec.add_property(name="Analysis", value=230007)
- rec.insert()
+ rec.add_property(name="date", value="2020-01-07")
+ ana = db.Record().add_parent(name="Analysis")
+ ana.add_property(name="Experiment", value=rec)
+ ana.add_propertt(name="date", value="2020-01-08")
+
+ cont = db.Container().extend([rec, ana]) # Add experiment and analysis
+ # records to our container
+ cont.insert() # Insert both at the same time, the CaosDB server will
+ # resolve the reference upon insertion.
-Of course, the IDs 235507 and 230007 need to exist in CaosDB. The first
-example shows how to use a db.REFERENCE Property (report) and the second
-shows that you can use any RecordType as Property to reference a Record
-that has such a parent.
+All three ways result in an Analysis record which references an Experiment
+record.
-Most Records do not have name however it can absolutely make sense. In
-that case use the name argument when creating it. Another useful feature
+.. note::
+
+ Instead of using the ``Experiment`` RecordType as a ``REFERENCE`` porperty,
+ we can also create an actual property with data type ``Experiment``:
+ ``db.property(name="source", datatype="Experiment")``. Now you can add this
+ property to the analysis record with the experiment record as a value as
+ explained above. As a rule of thumbs, using a separate property for these
+ references is meaningful whenever you want to highlight that, e.g., this
+ particular experiment provided the source data for your analysis (as opposed
+ to another experiment that was used for validation).
+
+Advanced insertions
+-------------------
+
+Most Records do not have a name, however it can absolutely make sense to assign
+one. In that case use the name argument when creating it. Another useful feature
is the fact that properties can have units:
.. code:: python
@@ -133,7 +187,7 @@ container. E.g. if you have a python list ``analysis_results``:
cont.insert()
-Useful is also, that you can insert directly tabular data.
+It may also be usefull to know that you can insert directly tabular data.
.. code:: python
@@ -143,8 +197,8 @@ Useful is also, that you can insert directly tabular data.
print(recs)
recs.insert()
-With this example file
-`test.csv <uploads/4f2c8756a26a3984c0af09d206d583e5/test.csv>`__.
+Try it yourself with this example file
+`test.csv <uploads/4f2c8756a26a3984c0af09d206d583e5/test.csv>`__!
List Properties
---------------
@@ -169,6 +223,28 @@ list-valued attribute in Python, as the following example illustrates.
print(retrieved.get_property("TestList").value)
+.. note::
+ Properties of Entities that shall be updated need to have IDs. Let's look at an
+ example:
+
+.. code:: python
+
+ experiment = db.Record(id=1111).retrieve()
+ experiment.add_property(name='date', value="2020-01-01")
+ retrieved.update() # Fails! The 'date' Property needs to have an ID.
+
+The easiest way to get around this is to use the corresponding entity getter:
+
+.. code:: python
+
+ experiment = db.Record(id=1111).retrieve()
+ experiment.add_property(db.get_entity_by_name('date'), value="2020-01-01")
+ retrieved.update() # Works!
+
+There also are the functions ``get_entity_by_path`` and ``get_entity_by_id``. You can easily use
+cached versions of those functions (see :doc:`caching options<caching>`).
+
+
File Update
-----------
diff --git a/src/doc/tutorials/Entity-Getters.rst b/src/doc/tutorials/Entity-Getters.rst
new file mode 100644
index 0000000000000000000000000000000000000000..50ed13201e5720de22bf0b605bc5162834a458a8
--- /dev/null
+++ b/src/doc/tutorials/Entity-Getters.rst
@@ -0,0 +1,16 @@
+
+Entity Getters
+==============
+
+There is a very frequent situation when working with PyCaosDB: You need to get a specific Entity
+from the remote server. For example, you need the Property Entity in order to make an update. Sure,
+you can do a ``db.Entity().retrieve()`` or submit a query, but there is an even faster way which
+also helps preventing errors:
+
+- ``get_entity_by_name``
+- ``get_entity_by_id``
+- ``get_entity_by_path``
+
+You can call these functions with a single argument (name/id/path). Since these functions are
+frequently used with the same arguments over and over again, you might want to look at the
+:doc:`caching options<caching>`.
diff --git a/src/doc/tutorials/basic_analysis.rst b/src/doc/tutorials/basic_analysis.rst
index cc185e0ee08f9e5ee0f890c0ab55f52972882d17..c40cad28b8c9a3be537c641b9614da2eb4df8dd9 100644
--- a/src/doc/tutorials/basic_analysis.rst
+++ b/src/doc/tutorials/basic_analysis.rst
@@ -34,7 +34,7 @@ Often we are interested in table like data for our processing. And the disentang
>>> from caosadvancedtools.table_converter import to_table
>>> # Let us retrieve the data in a table like form using `SELECT`
->>> data = db.execute_query("SELECT quality_factor FROM RECORD Analysis with quality_factor" )
+>>> data = db.execute_query("SELECT quality_factor FROM Analysis with quality_factor" )
>>> table = to_table(data)
>>> print(table)
quality_factor
diff --git a/src/doc/tutorials/caching.rst b/src/doc/tutorials/caching.rst
new file mode 100644
index 0000000000000000000000000000000000000000..aad9a1ddbd9e93a3cd06887eaffcf956c3c5bea6
--- /dev/null
+++ b/src/doc/tutorials/caching.rst
@@ -0,0 +1,58 @@
+
+Caching
+=======
+
+.. note::
+
+ Caching is great, because it can speed up things considerably. But it can also create dangerous
+ pitfalls if the cache is not cleared when needed and you work with outdated data. Thus, please use
+ the cache with care and make sure to clear it when needed.
+
+Python provides great tools for caching. For example, you could define a ``cached_get_by_name``
+function, easily created from ``get_entity_by_name`` using Python's ``lru_cache``:
+
+.. code:: python
+
+ @lru_cache(maxsize=1000)
+ def cached_get_by_name(name):
+ return db.get_entity_by_name(name)
+
+ exp = cached_get_by_name('Experiment')
+ # reset the cache with
+ cached_get_by_name.cache_clear()
+
+For convenience, PyCaosDB provides the ``caosdb.cached`` module that defines the functions
+``cached_query`` and ``cached_get_entity_by``, they use a shared cache. Let's have a look:
+
+.. code:: python
+
+ from caosdb.cached import cached_query, cached_get_entity_by, cache_clear, cache_info, cache_initialize
+ rt1 = cached_get_entity_by(name='RT1')
+ qresult = cached_query('FIND Experiment WITH parameter=1')
+ # you can inspect the cache
+ print(cache_info())
+ # this will not cause a server request since it is cached
+ rt1 = cached_get_entity_by(name='RT1')
+ # you can clear the cache with
+ cache_clear()
+ # If you want to have a cache with a custom size, you can initialize it (again). Old cached
+ # data is lost.
+ cache_initialize(maxsize=10)
+
+
+If you want to manually add entities to the cache, you can do it yourself. This is useful when you
+have entities on hand from previous queries that you want to add.
+
+.. code:: python
+
+ from caosdb.cached import cache_fill, AccessType
+ # Here, items must be a dict with Entity IDs as keys and the Entities as values.
+ cache_fill(items, AccessType.EID, unique=True)
+ # If you now use IDs that were in items, they are taken from the cache.
+ e1 = cached_get_entity_by(eid=10001)
+
+When filling the cache with Entity objects for ``cached_get_entity_by``, you need to set
+``unique=True``, whereas the cache for ``cached_query`` should be filled with Container object and
+``unique=False``.
+
+
diff --git a/src/doc/tutorials/complex_data_models.rst b/src/doc/tutorials/complex_data_models.rst
index 0fa868e78bb45a2905dc99392a3a28a9832d369e..7b45b6a2681bcf781fd4acc9329ffada28d4e01c 100644
--- a/src/doc/tutorials/complex_data_models.rst
+++ b/src/doc/tutorials/complex_data_models.rst
@@ -69,8 +69,9 @@ Examples
c.insert()
# Useful for testing: wait until the user presses a key
- # Meanwhile have a look at the WebUI: You can e.g. query "FIND Test*" to view
- # all the entities created here and see the relations and links between them.
+ # Meanwhile have a look at the WebUI: You can e.g. query "FIND ENTITY Test*"
+ # to view all the entities created here and see the relations and links
+ # between them.
b = input("Press any key to cleanup.")
# cleanup everything after the user presses any button.
c.delete()
diff --git a/src/doc/tutorials/first_steps.rst b/src/doc/tutorials/first_steps.rst
index 34b96bbeca416107fb34feb4707b9ef46fc49fe7..c84ec52aa63f0563b22c698081e89600c7af6122 100644
--- a/src/doc/tutorials/first_steps.rst
+++ b/src/doc/tutorials/first_steps.rst
@@ -25,7 +25,7 @@ However, you can also translate the examples to the data model that you have at
Let's start with a simple query.
->>> response = db.execute_query("FIND RECORD Guitar")
+>>> response = db.execute_query("FIND Guitar")
Queries work the same way as in the web interface. You simply provide the
query string to the corresponding function (``db.execute_query``). However, the result is not
@@ -52,6 +52,7 @@ Let's look at the first element:
<Record ...
.. The above example needs doctest ELLIPSIS
+
You see that the object is a Record. It has a Parent and two Properties.
.. note::
@@ -87,7 +88,7 @@ Ids can also come in handy when searching. Suppose you have some complicated con
>>> # This condition is not that complicated and long but let's suppose it was.
->>> record = db.execute_query("FIND Analysis with quality_factor=0.08", unique=True)
+>>> record = db.execute_query("FIND MusicalAnalysis with quality_factor=0.08", unique=True)
>>> # You can use unique=True when you only expect one result Entity. An error will be
>>> # thrown if the number of results is unequal to 1 and the resulting object will be
>>> # an Entity and not a Container
@@ -119,7 +120,7 @@ If the files are large data files, it is often a better idea to only retrieve th
Summary
-------
-Now you know, how you can use Python to send queries to CaosDB and you can access
+Now you know how to use Python to send queries to CaosDB and you can access
the result Records and their properties.
The next tutorial shows how to make some meaningful use of this.
diff --git a/src/doc/tutorials/index.rst b/src/doc/tutorials/index.rst
index 79068e9201498c87b2eb61b4ffbea0969845b404..ce37993d7ec5e0888da8a2b4c58904bcbdc43bb4 100644
--- a/src/doc/tutorials/index.rst
+++ b/src/doc/tutorials/index.rst
@@ -14,6 +14,9 @@ advanced usage of the Python client.
basic_analysis
Data-Insertion
errors
+ Entity-Getters
+ caching
data-model-interface
complex_data_models
+ serverside
diff --git a/src/doc/tutorials/serverside.rst b/src/doc/tutorials/serverside.rst
new file mode 100644
index 0000000000000000000000000000000000000000..93f0fdcf742efc70bc80f5113eb7c6ddbbf87cde
--- /dev/null
+++ b/src/doc/tutorials/serverside.rst
@@ -0,0 +1,61 @@
+
+Server Side Scripting
+=====================
+
+The administrator may store regularly needed scripts, e.g. for computing a
+standardized analysis, on the same machine as the CaosDB server, "on the server
+side", where they can be run directly by the server.
+
+The execution of those scripts can be initiated using the Python client, or the
+web interface.
+
+Call a Script
+~~~~~~~~~~~~~
+
+If you have access to the server and sufficient permissions to run the script,
+execution is fairly simple:
+
+.. code:: python
+
+ from caosdb.utils.server_side_scripting import run_server_side_script
+ response = run_server_side_script('scriptname.py')
+ print(response.stderr,response.stdout)
+
+
+This makes the server run the script ``scriptname.py``. The output of the
+script (``stderr`` and ``stdout``) is returned within an response object.
+
+
+If the script requires additional arguments, those can be provided after the
+script's name.
+
+Note that by default the script runs with your CaosDB account. It has your
+permissions and changes are logged as if they were done by you directly.
+
+
+Testing it
+~~~~~~~~~~
+
+You can try this out using for example the ``diagnostics.py`` script (it is part
+of the `CaosDB server repository
+<https://gitlab.indiscale.com/caosdb/src/caosdb-server/-/blob/main/scripting/bin/administration/diagnostics.py>`_
+and is also available on https://demo.indiscale.com). The script returns
+information about the server in JSON format. You can do for example the
+following:
+
+.. code:: python
+
+ import json
+ from caosdb.utils.server_side_scripting import run_server_side_script
+ response = run_server_side_script('administration/diagnostics.py')
+ print("JSON content:")
+ print(json.loads(response.stdout))
+ print("stderr:")
+ print(response.stderr)
+
+
+Further Information
+~~~~~~~~~~~~~~~~~~~
+
+Additionally, you might want to have a look at
+https://docs.indiscale.com/caosdb-server/specification/Server-side-scripting.html
diff --git a/tox.ini b/tox.ini
index 22c89f765c612ff78572ee2cab20dfab2e740e84..8212226eef2759c1864a86b8a3ad8f926480db4a 100644
--- a/tox.ini
+++ b/tox.ini
@@ -1,5 +1,5 @@
[tox]
-envlist=py36, py37, py38, py39
+envlist=py37, py38, py39, py310, py311
skip_missing_interpreters = true
[testenv]
@@ -7,5 +7,13 @@ deps = .
nose
pytest
pytest-cov
- jsonschema==4.0.1
+ jsonschema>=4.4.0
commands=py.test --cov=caosdb -vv {posargs}
+
+[flake8]
+max-line-length=100
+
+[pytest]
+testpaths = unittests
+xfail_strict = True
+addopts = -x -vv --cov=caosdb
diff --git a/unittests/data/list_in_value.xml b/unittests/data/list_in_value.xml
new file mode 100644
index 0000000000000000000000000000000000000000..0f92610d82caa5ced443b2f437f35da05b9e121a
--- /dev/null
+++ b/unittests/data/list_in_value.xml
@@ -0,0 +1,12 @@
+<Record id="1002" description="A description of this example experiment.">
+ <Version id="945c6858819d2609a5475ee4df64571984acd039" head="true">
+ <Predecessor id="0df3cfe164fbafe9777f9356d0be2403890c54cd" />
+ </Version>
+ <Parent id="1001" name="Experiment" />
+ <Property datatype="SomeRecordType" id="1003" name="DepthTest" importance="FIX">
+ <Value>1004</Value>
+ <Value>1005</Value>
+ </Property>
+</Record>
+
+<!-- Note: This XML is invalid, because list-valued Properties must have a LIST-Datatype -->
diff --git a/unittests/docker/Dockerfile b/unittests/docker/Dockerfile
index 7fa3f75bd198724628dee48ab328829fa071a639..7c84050b0a55ae6e1e8f2e2583f894a69f691193 100644
--- a/unittests/docker/Dockerfile
+++ b/unittests/docker/Dockerfile
@@ -1,4 +1,8 @@
-FROM debian:latest
+FROM debian:bullseye
+# Use local package repository
+COPY sources.list.local /etc/apt/
+RUN mv /etc/apt/sources.list /etc/apt/sources.list.orig
+RUN cat /etc/apt/sources.list.local /etc/apt/sources.list.orig > /etc/apt/sources.list
RUN apt-get update && \
apt-get install -y \
pylint3 python3-pip tox git \
diff --git a/unittests/docker/sources.list.local b/unittests/docker/sources.list.local
new file mode 100644
index 0000000000000000000000000000000000000000..c0b4107350ba37e77aa95d5a56c31976979e51e1
--- /dev/null
+++ b/unittests/docker/sources.list.local
@@ -0,0 +1,6 @@
+# Local repositories at Netcup
+deb http://debian.netcup.net/debian/ buster main
+deb http://mirrors.n-ix.net/debian-security buster/updates main
+deb http://debian.netcup.net/debian/ buster-updates main
+
+# The original content follows here:
\ No newline at end of file
diff --git a/unittests/test_acl.py b/unittests/test_acl.py
new file mode 100644
index 0000000000000000000000000000000000000000..633c25ad5c4046c0fa41b66049bdf56aa695f482
--- /dev/null
+++ b/unittests/test_acl.py
@@ -0,0 +1,55 @@
+# -*- encoding: utf-8 -*-
+#
+# This file is a part of the CaosDB Project.
+#
+# Copyright (C) 2022 Indiscale GmbH <info@indiscale.com>
+# Copyright (C) 2022 Timm Fitschen <f.fitschen@indiscale.com>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see <https://www.gnu.org/licenses/>.
+#
+import caosdb as db
+from lxml import etree
+
+
+def test_parse_xml():
+ # @review Florian Spreckelsen 2022-03-17
+ xml_str = """
+ <EntityACL>
+ <Grant priority="False" role="role1">
+ <Permission name="RETRIEVE:ENTITY"/>
+ </Grant>
+ <Deny priority="False" role="role1">
+ <Permission name="RETRIEVE:ENTITY"/>
+ </Deny>
+ <Grant priority="True" role="role1">
+ <Permission name="RETRIEVE:ENTITY"/>
+ </Grant>
+ <Deny priority="True" role="role1">
+ <Permission name="RETRIEVE:ENTITY"/>
+ </Deny>
+ </EntityACL>"""
+ xml = etree.fromstring(xml_str)
+ left_acl = db.ACL(xml)
+
+ right_acl = db.ACL()
+ right_acl.grant(role="role1", permission="RETRIEVE:ENTITY",
+ revoke_denial=False)
+ right_acl.deny(role="role1", permission="RETRIEVE:ENTITY",
+ revoke_grant=False)
+ right_acl.grant(role="role1", permission="RETRIEVE:ENTITY",
+ priority=True, revoke_denial=False)
+ right_acl.deny(role="role1", permission="RETRIEVE:ENTITY",
+ priority=True, revoke_grant=False)
+
+ assert left_acl == right_acl
diff --git a/unittests/test_apiutils.py b/unittests/test_apiutils.py
index 264b4c880022e6fd135426864bf9c5084c047eca..bda381cf6427377194e272dfa14b83399b6f012f 100644
--- a/unittests/test_apiutils.py
+++ b/unittests/test_apiutils.py
@@ -1,12 +1,12 @@
-# -*- encoding: utf-8 -*-
#
-# ** header v3.0
# This file is a part of the CaosDB Project.
#
+# Copyright (C) 2020 Timm Fitschen <t.fitschen@indiscale.com>
+# Copyright (C) 2022 Florian Spreckelsen <f.spreckelsen@indiscale.com>
+# Copyright (C) 2022 Daniel Hornung <d.hornung@indiscale.com>
+# Copyright (C) 2020-2022 IndiScale GmbH <info@indiscale.com>
# Copyright (C) 2018 Research Group Biomedical Physics,
# Max-Planck-Institute for Dynamics and Self-Organization Göttingen
-# Copyright (C) 2020 Timm Fitschen <t.fitschen@indiscale.com>
-# Copyright (C) 2020 IndiScale GmbH <info@indiscale.com>
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as
@@ -21,31 +21,19 @@
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <https://www.gnu.org/licenses/>.
#
-# ** end header
#
# Test apiutils
# A. Schlemmer, 02/2018
+
+import pytest
import caosdb as db
-import pickle
-import tempfile
-from caosdb.apiutils import apply_to_ids, create_id_query, resolve_reference
import caosdb.apiutils
-from .test_property import testrecord
-
+from caosdb.apiutils import (apply_to_ids, compare_entities, create_id_query,
+ empty_diff, EntityMergeConflictError,
+ resolve_reference, merge_entities)
-def test_convert_object():
- r2 = db.apiutils.convert_to_python_object(testrecord)
- assert r2.species == "Rabbit"
-
-
-def test_pickle_object():
- r2 = db.apiutils.convert_to_python_object(testrecord)
- with tempfile.TemporaryFile() as f:
- pickle.dump(r2, f)
- f.seek(0)
- rn2 = pickle.load(f)
- assert r2.date == rn2.date
+from caosdb.common.models import SPECIAL_ATTRIBUTES
def test_apply_to_ids():
@@ -67,7 +55,8 @@ def test_apply_to_ids():
def test_id_query():
ids = [1, 2, 3, 4, 5]
- assert create_id_query(ids) == 'FIND ENTITY WITH ID=1 OR ID=2 OR ID=3 OR ID=4 OR ID=5'
+ assert create_id_query(ids) == 'FIND ENTITY WITH ID=1 OR ID=2 OR ID=3 OR '\
+ 'ID=4 OR ID=5'
def test_resolve_reference():
@@ -77,8 +66,10 @@ def test_resolve_reference():
prop = db.Property(id=1, datatype=db.REFERENCE, value=100)
prop.is_valid = lambda: True
items = [200, 300, 400]
- prop_list = db.Property(datatype=db.LIST(db.REFERENCE), value=items)
- prop_list2 = db.Property(datatype=db.LIST(db.REFERENCE), value=[db.Record(id=500)])
+ prop_list = db.Property(datatype=db.LIST(db.REFERENCE),
+ value=items)
+ prop_list2 = db.Property(datatype=db.LIST(db.REFERENCE),
+ value=[db.Record(id=500)])
resolve_reference(prop)
resolve_reference(prop_list)
resolve_reference(prop_list2)
@@ -86,6 +77,7 @@ def test_resolve_reference():
assert isinstance(prop.value, db.Entity)
prop_list_ids = []
+
for i in prop_list.value:
prop_list_ids.append(i.id)
assert isinstance(i, db.Entity)
@@ -102,3 +94,483 @@ def test_resolve_reference():
# restore retrive_entity_with_id
caosdb.apiutils.retrieve_entity_with_id = original_retrieve_entity_with_id
+
+
+def test_compare_entities():
+ r1 = db.Record()
+ r2 = db.Record()
+ r1.add_parent("bla")
+ r2.add_parent("bla")
+ r1.add_parent("lopp")
+ r1.add_property("test", value=2)
+ r2.add_property("test", value=2)
+ r1.add_property("tests", value=3)
+ r2.add_property("tests", value=45)
+ r1.add_property("tester", value=3)
+ r2.add_property("tester", )
+ r1.add_property("tests_234234", value=45)
+ r2.add_property("tests_TT", value=45)
+
+ diff_r1, diff_r2 = compare_entities(r1, r2)
+
+ assert len(diff_r1["parents"]) == 1
+ assert len(diff_r2["parents"]) == 0
+ assert len(diff_r1["properties"]) == 3
+ assert len(diff_r2["properties"]) == 3
+
+ assert "test" not in diff_r1["properties"]
+ assert "test" not in diff_r2["properties"]
+
+ assert "tests" in diff_r1["properties"]
+ assert "tests" in diff_r2["properties"]
+
+ assert "tester" in diff_r1["properties"]
+ assert "tester" in diff_r2["properties"]
+
+ assert "tests_234234" in diff_r1["properties"]
+ assert "tests_TT" in diff_r2["properties"]
+
+
+def test_compare_entities_units():
+ r1 = db.Record()
+ r2 = db.Record()
+ r1.add_parent("bla")
+ r2.add_parent("bla")
+ r1.add_parent("lopp")
+ r1.add_property("test", value=2, unit="cm")
+ r2.add_property("test", value=2, unit="m")
+ r1.add_property("tests", value=3, unit="cm")
+ r2.add_property("tests", value=45, unit="cm")
+ r1.add_property("tester", value=3)
+ r2.add_property("tester", )
+ r1.add_property("tests_234234", value=45, unit="cm")
+ r2.add_property("tests_TT", value=45, unit="cm")
+
+ diff_r1, diff_r2 = compare_entities(r1, r2)
+
+ assert len(diff_r1["parents"]) == 1
+ assert len(diff_r2["parents"]) == 0
+ assert len(diff_r1["properties"]) == 4
+ assert len(diff_r2["properties"]) == 4
+
+ assert "tests" in diff_r1["properties"]
+ assert "tests" in diff_r2["properties"]
+
+ assert "tester" in diff_r1["properties"]
+ assert "tester" in diff_r2["properties"]
+
+ assert "tests_234234" in diff_r1["properties"]
+ assert "tests_TT" in diff_r2["properties"]
+
+ assert diff_r1["properties"]["test"]["unit"] == "cm"
+ assert diff_r2["properties"]["test"]["unit"] == "m"
+
+
+def test_compare_special_properties():
+ # Test for all known special properties:
+ SPECIAL_PROPERTIES = ("description", "name",
+ "checksum", "size", "path", "id")
+ INTS = ("size", "id")
+ HIDDEN = ("checksum", "size")
+
+ for key in SPECIAL_PROPERTIES:
+ set_key = key
+ if key in HIDDEN:
+ set_key = "_" + key
+ r1 = db.Record()
+ r2 = db.Record()
+ if key not in INTS:
+ setattr(r1, set_key, "bla 1")
+ setattr(r2, set_key, "bla 1")
+ else:
+ setattr(r1, set_key, 1)
+ setattr(r2, set_key, 1)
+
+ diff_r1, diff_r2 = compare_entities(r1, r2)
+ assert key not in diff_r1
+ assert key not in diff_r2
+ assert len(diff_r1["parents"]) == 0
+ assert len(diff_r2["parents"]) == 0
+ assert len(diff_r1["properties"]) == 0
+ assert len(diff_r2["properties"]) == 0
+
+ if key not in INTS:
+ setattr(r2, set_key, "bla test")
+ else:
+ setattr(r2, set_key, 2)
+
+ diff_r1, diff_r2 = compare_entities(r1, r2)
+ assert key in diff_r1
+ assert key in diff_r2
+ if key not in INTS:
+ assert diff_r1[key] == "bla 1"
+ assert diff_r2[key] == "bla test"
+ else:
+ assert diff_r1[key] == 1
+ assert diff_r2[key] == 2
+ assert len(diff_r1["properties"]) == 0
+ assert len(diff_r2["properties"]) == 0
+
+
+@pytest.mark.xfail
+def test_compare_properties():
+ p1 = db.Property()
+ p2 = db.Property()
+
+ diff_r1, diff_r2 = compare_entities(p1, p2)
+ assert len(diff_r1["parents"]) == 0
+ assert len(diff_r2["parents"]) == 0
+ assert len(diff_r1["properties"]) == 0
+ assert len(diff_r2["properties"]) == 0
+
+ p1.importance = "SUGGESTED"
+ diff_r1, diff_r2 = compare_entities(p1, p2)
+ assert len(diff_r1["parents"]) == 0
+ assert len(diff_r2["parents"]) == 0
+ assert len(diff_r1["properties"]) == 0
+ assert len(diff_r2["properties"]) == 0
+ assert "importance" in diff_r1
+ assert diff_r1["importance"] == "SUGGESTED"
+
+ # TODO: I'm not sure why it is not like this:
+ # assert diff_r2["importance"] is None
+ # ... but:
+ assert "importance" not in diff_r2
+
+ p2.importance = "SUGGESTED"
+ p1.value = 42
+ p2.value = 4
+
+ diff_r1, diff_r2 = compare_entities(p1, p2)
+ assert len(diff_r1["parents"]) == 0
+ assert len(diff_r2["parents"]) == 0
+ assert len(diff_r1["properties"]) == 0
+ assert len(diff_r2["properties"]) == 0
+
+ # Comparing values currently does not seem to be implemented:
+ assert "value" in diff_r1
+ assert diff_r1["value"] == 42
+ assert "value" in diff_r2
+ assert diff_r2["value"] == 4
+
+
+def test_copy_entities():
+ r = db.Record(name="A")
+ r.add_parent(name="B")
+ r.add_property(name="C", value=4, importance="OBLIGATORY")
+ r.add_property(name="D", value=[3, 4, 7], importance="OBLIGATORY")
+ r.description = "A fancy test record"
+
+ c = r.copy()
+
+ assert c is not r
+ assert c.name == "A"
+ assert c.role == r.role
+ assert c.parents[0].name == "B"
+ # parent and property objects are not shared among copy and original:
+ assert c.parents[0] is not r.parents[0]
+
+ for i in [0, 1]:
+ assert c.properties[i] is not r.properties[i]
+ for special in SPECIAL_ATTRIBUTES:
+ assert getattr(c.properties[i], special) == getattr(
+ r.properties[i], special)
+ assert c.get_importance(
+ c.properties[i]) == r.get_importance(r.properties[i])
+
+
+def test_merge_entities():
+ r = db.Record(name="A")
+ r.add_parent(name="B")
+ r.add_property(name="C", value=4, importance="OBLIGATORY")
+ r.add_property(name="D", value=[3, 4, 7], importance="OBLIGATORY")
+ r.description = "A fancy test record"
+
+ r2 = db.Record()
+ r2.add_property(name="F", value="text")
+ merge_entities(r2, r)
+ assert r2.get_parents()[0].name == "B"
+ assert r2.get_property("C").name == "C"
+ assert r2.get_property("C").value == 4
+ assert r2.get_property("D").name == "D"
+ assert r2.get_property("D").value == [3, 4, 7]
+
+ assert r2.get_property("F").name == "F"
+ assert r2.get_property("F").value == "text"
+
+
+def test_merge_bug_conflict():
+ r = db.Record()
+ r.add_property(name="C", value=4)
+ r2 = db.Record()
+ r2.add_property(name="C", value=4, datatype="TEXT")
+ merge_entities(r, r2)
+
+ r3 = db.Record()
+ r3.add_property(name="C", value=4, datatype="INTEGER")
+ with pytest.raises(EntityMergeConflictError):
+ merge_entities(r3, r2)
+
+
+def test_merge_bug_109():
+ rt = db.RecordType(name="TestBug")
+ p = db.Property(name="test_bug_property", datatype=db.LIST(db.INTEGER))
+
+ r_b = db.Record(name="TestRecord")
+ r_b.add_parent(rt)
+ r_b.add_property(p, value=[18, 19])
+
+ r_a = db.Record(name="TestRecord")
+ r_a.add_parent(rt)
+
+ merge_entities(r_a, r_b)
+
+ assert r_b.get_property("test_bug_property").value == [18, 19]
+ assert r_a.get_property("test_bug_property").value == [18, 19]
+
+ assert "<Value>18</Value>\n <Value>19</Value>" in str(r_b)
+ assert "<Value>18</Value>\n <Value>19</Value>\n <Value>18</Value>\n <Value>19</Value>" not in str(
+ r_b)
+
+ assert "<Value>18</Value>\n <Value>19</Value>" in str(r_a)
+ assert "<Value>18</Value>\n <Value>19</Value>\n <Value>18</Value>\n <Value>19</Value>" not in str(
+ r_a)
+
+
+@pytest.mark.xfail
+def test_bug_109():
+ rt = db.RecordType(name="TestBug")
+ p = db.Property(name="test_bug_property", datatype=db.LIST(db.INTEGER))
+
+ r_b = db.Record(name="TestRecord")
+ r_b.add_parent(rt)
+ r_b.add_property(p, value=[18, 19])
+
+ r_a = db.Record(name="TestRecord")
+ r_a.add_parent(rt)
+ r_a.add_property(r_b.get_property("test_bug_property"))
+
+ assert r_b.get_property("test_bug_property").value == [18, 19]
+ assert r_a.get_property("test_bug_property").value == [18, 19]
+
+ assert "<Value>18</Value>\n <Value>19</Value>" in str(r_b)
+ assert "<Value>18</Value>\n <Value>19</Value>\n <Value>18</Value>\n <Value>19</Value>" not in str(
+ r_b)
+
+ assert "<Value>18</Value>\n <Value>19</Value>" in str(r_a)
+ assert "<Value>18</Value>\n <Value>19</Value>\n <Value>18</Value>\n <Value>19</Value>" not in str(
+ r_a)
+
+
+def test_wrong_merge_conflict_reference():
+ """Test a wrongly detected merge conflict in case of two records referencing
+ two different, but identical objects.
+
+ """
+ # Two identical license records will be referenced from both records to be
+ # merged
+ license_rt = db.RecordType(name="license")
+ license_rec_a = db.Record(name="CC-BY-3.0").add_parent(license_rt)
+ license_rec_b = db.Record(name="CC-BY-3.0").add_parent(license_rt)
+
+ # two referencing records
+ dataset_rt = db.RecordType(name="Dataset")
+ title_prop = db.Property(name="title", datatype=db.TEXT)
+ doi_prop = db.Property(name="DOI", datatype=db.TEXT)
+ rec_a = db.Record().add_parent(dataset_rt)
+ rec_a.add_property(name=license_rt.name,
+ datatype=license_rt.name, value=license_rec_a)
+ rec_a.add_property(name=title_prop.name, value="Some dataset title")
+
+ rec_b = db.Record().add_parent(dataset_rt)
+ rec_b.add_property(name=license_rt.name,
+ datatype=license_rt.name, value=license_rec_b)
+ rec_b.add_property(name=doi_prop.name, value="https://doi.org/12345.678")
+
+ merge_entities(rec_a, rec_b)
+ assert rec_a.get_property(license_rt.name) is not None
+ assert rec_a.get_property(license_rt.name).value is not None
+ assert isinstance(rec_a.get_property(license_rt.name).value, db.Record)
+ assert rec_a.get_property(license_rt.name).value.name == license_rec_a.name
+ assert rec_a.get_property(license_rt.name).value.name == license_rec_b.name
+ assert rec_a.get_property("title").value == "Some dataset title"
+ assert rec_a.get_property("doi").value == "https://doi.org/12345.678"
+
+ # Reset rec_a
+ rec_a = db.Record().add_parent(dataset_rt)
+ rec_a.add_property(name=license_rt.name,
+ datatype=license_rt.name, value=license_rec_a)
+ rec_a.add_property(name=title_prop.name, value="Some dataset title")
+
+ # this does not compare referenced records, so it will fail
+ with pytest.raises(EntityMergeConflictError):
+ merge_entities(rec_a, rec_b, merge_references_with_empty_diffs=False)
+
+ # ... as should this, of course
+ rec_b.get_property(license_rt.name).value.name = "Another license"
+ with pytest.raises(EntityMergeConflictError) as re:
+ merge_entities(rec_a, rec_b)
+
+
+def test_empty_diff():
+
+ rec_a = db.Record(name="A")
+ rec_b = db.Record(name="B")
+
+ assert empty_diff(rec_a, rec_a)
+ assert not empty_diff(rec_a, rec_b)
+
+ rec_a.add_parent(name="RT")
+ rec_b.add_parent(name="RT")
+ assert empty_diff(rec_a, rec_a)
+ assert not empty_diff(rec_a, rec_b)
+
+ rec_b.name = "A"
+ assert empty_diff(rec_a, rec_b)
+
+ rec_a.add_property(name="some_prop", value=1)
+ assert not empty_diff(rec_a, rec_b)
+
+ rec_b.add_property(name="some_prop", value=1)
+ assert empty_diff(rec_a, rec_b)
+
+ rec_b.get_property("some_prop").value = 2
+ assert not empty_diff(rec_a, rec_b)
+
+ rec_b.get_property("some_prop").value = 1
+ rec_b.add_property(name="some_other_prop", value="Test")
+ assert not empty_diff(rec_a, rec_b)
+
+ rec_a.add_property(name="some_other_prop", value="Test")
+ assert empty_diff(rec_a, rec_b)
+
+ # reference identical records, but different Python Record objects
+ ref_rec_a = db.Record(name="Ref").add_parent(name="RefType")
+ ref_rec_b = db.Record(name="Ref").add_parent(name="RefType")
+ rec_a.add_property(name="RefType", datatype="RefType", value=ref_rec_a)
+ rec_b.add_property(name="RefType", datatype="RefType", value=ref_rec_b)
+ # the default is `compare_referenced_records=False`, so the diff shouldn't
+ # be empty (different Python objects are referenced.)
+ assert not empty_diff(rec_a, rec_b)
+ # when looking into the referenced record, the diffs should be empty again
+ assert empty_diff(rec_a, rec_b, compare_referenced_records=True)
+
+ # The same for lists of references
+ rec_a.remove_property("RefType")
+ rec_b.remove_property("RefType")
+ assert empty_diff(rec_a, rec_b)
+ rec_a.add_property(name="RefType", datatype=db.LIST(
+ "RefType"), value=[ref_rec_a, ref_rec_a])
+ rec_b.add_property(name="RefType", datatype=db.LIST(
+ "RefType"), value=[ref_rec_b, ref_rec_b])
+ assert not empty_diff(rec_a, rec_b)
+ assert empty_diff(rec_a, rec_b, compare_referenced_records=True)
+
+ # special case of ids
+ rec_a = db.Record(id=12)
+ rec_b = db.Record()
+ assert not empty_diff(rec_a, rec_b)
+ rec_b.id = 13
+ assert not empty_diff(rec_a, rec_b)
+ rec_b.id = 12
+ assert empty_diff(rec_a, rec_b)
+
+
+def test_force_merge():
+ """Test whether a forced merge overwrites existing properties correctly."""
+
+ # name overwrite
+ recA = db.Record(name="A")
+ recB = db.Record(name="B")
+
+ with pytest.raises(EntityMergeConflictError):
+ merge_entities(recA, recB)
+
+ merge_entities(recA, recB, force=True)
+ assert "B" == recA.name
+ # unchanged
+ assert "B" == recB.name
+
+ # description overwrite
+ recA = db.Record()
+ recA.description = "something"
+ recB = db.Record()
+ recB.description = "something else"
+
+ with pytest.raises(EntityMergeConflictError) as emce:
+ merge_entities(recA, recB)
+ assert str(emce.value) == """Conflict in special attribute description:
+A: something
+B: something else"""
+
+ merge_entities(recA, recB, force=True)
+ assert recA.description == "something else"
+ # unchanged
+ assert recB.description == "something else"
+
+ # property overwrite
+ recA = db.Record()
+ recA.add_property(name="propA", value="something")
+ recB = db.Record()
+ recB.add_property(name="propA", value="something else")
+
+ with pytest.raises(EntityMergeConflictError):
+ merge_entities(recA, recB)
+
+ merge_entities(recA, recB, force=True)
+ assert recA.get_property("propA").value == "something else"
+ # unchanged
+ assert recB.get_property("propA").value == "something else"
+
+ # don't remove a property that's not in recB
+ recA = db.Record()
+ recA.add_property(name="propA", value="something")
+ recA.add_property(name="propB", value=5.0)
+ recB = db.Record()
+ recB.add_property(name="propA", value="something else")
+
+ merge_entities(recA, recB, force=True)
+ assert recA.get_property("propA").value == "something else"
+ assert recA.get_property("propB").value == 5.0
+
+ # also overwrite datatypes ...
+ rtA = db.RecordType()
+ rtA.add_property(name="propA", datatype=db.INTEGER)
+ rtB = db.RecordType()
+ rtB.add_property(name="propA", datatype=db.TEXT)
+
+ with pytest.raises(EntityMergeConflictError):
+ merge_entities(rtA, rtB)
+
+ merge_entities(rtA, rtB, force=True)
+ assert rtA.get_property("propA").datatype == db.TEXT
+ # unchanged
+ assert rtB.get_property("propA").datatype == db.TEXT
+
+ # ... and units
+ recA = db.Record()
+ recA.add_property(name="propA", value=5, unit="m")
+ recB = db.Record()
+ recB.add_property(name="propA", value=5, unit="cm")
+
+ with pytest.raises(EntityMergeConflictError):
+ merge_entities(recA, recB)
+ merge_entities(recA, recB, force=True)
+ assert recA.get_property("propA").unit == "cm"
+ # unchanged
+ assert recB.get_property("propA").unit == "cm"
+
+
+def test_merge_missing_list_datatype_82():
+ """Merging two properties, where the list-valued one has no datatype."""
+
+ recA = db.Record().add_property("a", 5, datatype="B")
+ recB_with_DT = db.Record().add_property("a", [1, 2], datatype=f"LIST<{db.DOUBLE}>")
+ merge_entities(recA, recB_with_DT, force=True)
+ assert recA.get_property("a").datatype == f"LIST<{db.DOUBLE}>"
+
+ recA = db.Record().add_property("a", 5, datatype="B")
+ recB_without_DT = db.Record().add_property("a", [1, 2])
+ with pytest.raises(TypeError) as te:
+ merge_entities(recA, recB_without_DT, force=True)
+ assert "Invalid datatype: List valued properties" in str(te.value)
diff --git a/unittests/test_authentication_auth_token.py b/unittests/test_authentication_auth_token.py
index 15e54121fc0d7b5c2be645cdb88bc20804a10980..d0eb6b90883951af584d42a80e319c14891f6e50 100644
--- a/unittests/test_authentication_auth_token.py
+++ b/unittests/test_authentication_auth_token.py
@@ -65,7 +65,7 @@ def test_configure_connection():
c._delegate_connection.resources.append(request_has_auth_token)
assert c._authenticator.auth_token == "[request token]"
response = c._http_request(method="GET", path="test")
- assert response.read() == "ok"
+ assert response.read().decode() == "ok"
assert c._authenticator.auth_token == "[response token]"
diff --git a/unittests/test_authentication_unauthenticated.py b/unittests/test_authentication_unauthenticated.py
index 52146b08ed4e1026660eebacedf348aeb2ff2721..45a709fcc62b609a97de7e87dd6c6f6ac94a55a1 100644
--- a/unittests/test_authentication_unauthenticated.py
+++ b/unittests/test_authentication_unauthenticated.py
@@ -59,7 +59,7 @@ def test_configure_connection():
assert c._authenticator.auth_token is None
response = c._http_request(method="GET", path="test")
- assert response.read() == "ok"
+ assert response.read().decode() == "ok"
mock.method.assert_called_once()
assert c._authenticator.auth_token is None
diff --git a/unittests/test_cached.py b/unittests/test_cached.py
new file mode 100644
index 0000000000000000000000000000000000000000..ce302d671d6077aed7d8457e70da2076ebe65d50
--- /dev/null
+++ b/unittests/test_cached.py
@@ -0,0 +1,295 @@
+# -*- coding: utf-8 -*-
+#
+# This file is a part of the CaosDB Project.
+#
+# Copyright (C) 2023 Henrik tom Wörden <h.tomwoerden@indiscale.com>
+# Copyright (C) 2023 IndiScale GmbH <info@indiscale.com>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see <https://www.gnu.org/licenses/>.
+#
+
+""" Test the caosdb.cached module """
+
+from caosdb.cached import (cached_get_entity_by, cache_clear, cache_info, cache_fill,
+ AccessType, cache_initialize, cached_query)
+from unittest.mock import patch
+import caosdb as db
+from copy import deepcopy
+import pytest
+
+
+DUMMY_SERVER_CONTENT = [
+ db.Record(name='a', id=101),
+ db.Record(name='b', id=102),
+ db.Record(name='c', id=103),
+ db.File(path='p', id=104),
+ db.File(path='pp', id=105),
+]
+
+
+@pytest.fixture(autouse=True)
+def cache_clean_up():
+ cache_clear()
+ yield
+ cache_clear()
+
+
+def mocked_name_query(name):
+ # copy the object, because Entities would normally be created from XML response
+ return deepcopy([el for el in DUMMY_SERVER_CONTENT if el.name == name][0])
+
+
+def mocked_id_query(eid):
+ # copy the object, because Entities would normally be created from XML response
+ return deepcopy([el for el in DUMMY_SERVER_CONTENT if el.id == eid][0])
+
+
+def mocked_path_query(path):
+ # copy the object, because Entities would normally be created from XML response
+ return deepcopy([el for el in DUMMY_SERVER_CONTENT if el.path == path][0])
+
+
+def mocked_gen_query(q, unique):
+ if unique:
+ if q == 'a':
+ return DUMMY_SERVER_CONTENT[0]
+ else:
+ return None
+ else:
+ if q == 'a':
+ return db.Container().extend([DUMMY_SERVER_CONTENT[0]])
+ else:
+ return db.Container().extend(DUMMY_SERVER_CONTENT)
+
+
+@patch("caosdb.utils.get_entity.get_entity_by_name")
+def test_get_by_name(mocked_get_by_name):
+ mocked_get_by_name.side_effect = mocked_name_query
+ # first call; not in cache -> mocked_execute is touched
+ a = cached_get_entity_by(name='a')
+ assert a.id == 101
+ assert mocked_get_by_name.call_count == 1
+ # second call; in cache -> mocked_execute is NOT touched (count is still 1)
+ b = cached_get_entity_by(name='a')
+ assert mocked_get_by_name.call_count == 1
+ # the cache returned the same object
+ assert a is b
+ # check the info
+ assert cache_info().hits == 1
+ assert cache_info().currsize == 1
+ # after clearing the test, the mock is used again
+ cache_clear()
+ cached_get_entity_by(name='a')
+ assert mocked_get_by_name.call_count == 2
+ # we fill the cache manually and make sure the element is used
+ cache_fill({'lol': db.Entity(id=10001, name='lol')}, AccessType.NAME, unique=True)
+ # there are now two elements in the cache: a and lol
+ assert cache_info().currsize == 2
+ # we can retrieve the inserted element
+ lol = cached_get_entity_by(name='lol')
+ assert lol.id == 10001
+ # this did not touch the mocked function
+ assert mocked_get_by_name.call_count == 2
+ # make sure normal retrieval still works (count +1)
+ c = cached_get_entity_by(name='c')
+ assert mocked_get_by_name.call_count == 3
+ assert c.id == 103
+
+
+@patch("caosdb.utils.get_entity.get_entity_by_id")
+def test_get_by_id(mocked_get_by_id):
+ mocked_get_by_id.side_effect = mocked_id_query
+ # first call; not in cache -> mocked_execute is touched
+ b = cached_get_entity_by(eid=102)
+ assert b.id == 102
+ assert b.name == 'b'
+ assert mocked_get_by_id.call_count == 1
+ # second call; in cache -> mocked_execute is NOT touched (count is still 1)
+ a = cached_get_entity_by(eid=102)
+ assert mocked_get_by_id.call_count == 1
+ # the cache returned the same object
+ assert a is b
+ # check the info
+ assert cache_info().hits == 1
+ assert cache_info().currsize == 1
+ # after clearing the test, the mock is used again
+ cache_clear()
+ cached_get_entity_by(eid=102)
+ assert mocked_get_by_id.call_count == 2
+ # we fill the cache manually and make sure the element is used
+ cache_fill({10001: db.Entity(id=10001, name='lol')}, AccessType.EID, unique=True)
+ # there are now two elements in the cache: a and lol
+ assert cache_info().currsize == 2
+ # we can retrieve the inserted element
+ lol = cached_get_entity_by(eid=10001)
+ assert lol.name == 'lol'
+ # this did not touch the mocked function
+ assert mocked_get_by_id.call_count == 2
+ # make sure normal retrieval still works (count +1)
+ c = cached_get_entity_by(eid=103)
+ assert mocked_get_by_id.call_count == 3
+ assert c.name == 'c'
+
+
+@patch("caosdb.cached.get_entity.get_entity_by_path")
+def test_get_by_path(mocked_get_by_path):
+ mocked_get_by_path.side_effect = mocked_path_query
+ # first call; not in cache -> mocked_execute is touched
+ b = cached_get_entity_by(path='p')
+ assert b.id == 104
+ assert mocked_get_by_path.call_count == 1
+ # second call; in cache -> mocked_execute is NOT touched (count is still 1)
+ a = cached_get_entity_by(path='p')
+ assert mocked_get_by_path.call_count == 1
+ # the cache returned the same object
+ assert a is b
+ # check the info
+ assert cache_info().hits == 1
+ assert cache_info().currsize == 1
+ # after clearing the test, the mock is used again
+ cache_clear()
+ cached_get_entity_by(path='p')
+ assert mocked_get_by_path.call_count == 2
+ # we fill the cache manually and make sure the element is used
+ cache_fill({'lol': db.File(id=10001, path='lol')}, AccessType.PATH, unique=True)
+ # there are now two elements in the cache: a and lol
+ assert cache_info().currsize == 2
+ # we can retrieve the inserted element
+ lol = cached_get_entity_by(path='lol')
+ assert lol.id == 10001
+ # this did not touch the mocked function
+ assert mocked_get_by_path.call_count == 2
+ # make sure normal retrieval still works (count +1)
+ c = cached_get_entity_by(path='pp')
+ assert mocked_get_by_path.call_count == 3
+ assert c.id == 105
+
+
+@patch("caosdb.cached.execute_query")
+def test_get_by_query(mocked_query):
+ mocked_query.side_effect = mocked_gen_query
+ # test cache initialization
+ cache_initialize(maxsize=10)
+ assert cache_info().currsize == 0
+
+ # Non-existent entity
+ res = cached_get_entity_by(query='stuff')
+ assert res is None
+ assert cache_info().currsize == 1
+ assert cache_info().hits == 0
+ assert cache_info().misses == 1
+
+ res = cached_get_entity_by(query='stuff')
+ assert res is None
+ assert cache_info().currsize == 1
+ assert cache_info().hits == 1
+ assert cache_info().misses == 1
+
+ # Existent entity
+ a = cached_get_entity_by(query='a')
+ assert a is not None
+ assert a.id == 101
+ assert cache_info().currsize == 2
+ assert cache_info().hits == 1
+ assert cache_info().misses == 2
+
+
+@patch("caosdb.cached.execute_query")
+def test_cached_query(mocked_query):
+ mocked_query.side_effect = mocked_gen_query
+ # test cache initialization
+ cache_initialize(maxsize=10)
+ assert cache_info().maxsize == 10
+ # first call; not in cache -> mocked_execute is touched
+ res = cached_query('stuff')
+ assert len(res) == len(DUMMY_SERVER_CONTENT)
+ assert mocked_query.call_count == 1
+ # second call; in cache -> mocked_execute is NOT touched (count is still 1)
+ a = cached_query('stuff')
+ assert mocked_query.call_count == 1
+ # the cache returned the same object
+ assert a is res
+ # check the info
+ assert cache_info().hits == 1
+ assert cache_info().currsize == 1
+ # after clearing the test, the mock is used again
+ cache_clear()
+ cached_query('stuff')
+ assert mocked_query.call_count == 2
+ # we fill the cache manually and make sure the element is used
+ cache_fill({'lol': db.Container().extend([db.Entity(id=10001, name='lol')])},
+ AccessType.QUERY, unique=False)
+ # there are now two elements in the cache: a and lol
+ assert cache_info().currsize == 2
+ # we can retrieve the inserted element
+ lol = cached_query('lol')
+ assert lol[0].id == 10001
+ # this did not touch the mocked function
+ assert mocked_query.call_count == 2
+ # make sure normal retrieval still works (count +1)
+ c = cached_query('a')
+ assert mocked_query.call_count == 3
+ assert c[0].id == 101
+
+
+@patch("caosdb.utils.get_entity.get_entity_by_name")
+def test_cache_size(mocked_get_by_name):
+ mocked_get_by_name.side_effect = lambda x: x
+ # first call; not in cache -> mocked_execute is touched
+ maxsize = 5
+ cache_initialize(maxsize=maxsize)
+ assert cache_info().currsize == 0
+
+ names_first = ("a", "b", "c", "d", "e")
+ names_later = ("A", "B", "C", "D", "E")
+ names_fill = {"X": None, "Y": None, "Z": None}
+
+ # Use the first batch of names
+ for ii, name in enumerate(names_first, start=1):
+ cached_get_entity_by(name=name)
+ assert cache_info().currsize == ii
+ assert cache_info().hits == 0
+ assert cache_info().misses == ii
+ for ii, name in enumerate(names_first, start=1):
+ cached_get_entity_by(name=name)
+ assert cache_info().currsize == maxsize
+ assert cache_info().hits == ii
+ assert cache_info().misses == maxsize
+
+ # use the second batch of names
+ for ii, name in enumerate(names_later, start=1):
+ cached_get_entity_by(name=name)
+ assert cache_info().currsize == maxsize
+ assert cache_info().hits == len(names_first)
+ assert cache_info().misses == len(names_first) + ii
+ for ii, name in enumerate(names_later, start=1):
+ cached_get_entity_by(name=name)
+ assert cache_info().currsize == maxsize
+ assert cache_info().hits == len(names_first) + ii
+ assert cache_info().misses == len(names_first) + len(names_later)
+
+ # The cache is now filled with A,B,C,D,E (oldest to least recently used).
+ # Let's fill it with X,Y,Z.
+ cache_fill(names_fill, kind=AccessType.NAME)
+
+ # Now, the cache should be: D,E,X,Y,Z
+ current_misses = cache_info().misses
+
+ for name in ("Z", "Y", "X", "E", "D"):
+ cached_get_entity_by(name=name)
+ assert cache_info().misses == current_misses
+
+ for ii, name in enumerate(("A", "B", "C"), start=1):
+ cached_get_entity_by(name=name)
+ assert cache_info().misses == current_misses + ii
diff --git a/unittests/test_configs/pycaosdb-IntegrationTests.ini b/unittests/test_configs/pycaosdb-IntegrationTests.ini
new file mode 100644
index 0000000000000000000000000000000000000000..cb9871708f7f23c489de0cbc8f4fbda15dfa6ad0
--- /dev/null
+++ b/unittests/test_configs/pycaosdb-IntegrationTests.ini
@@ -0,0 +1,37 @@
+# -*- mode:conf; -*-
+## This sections needs to exist in addition to the usual section
+[IntegrationTests]
+# test_server_side_scripting.bin_dir.local=/path/to/scripting/bin
+test_server_side_scripting.bin_dir.local=/home/myself/test/caosdb-server/scripting/bin
+# test_server_side_scripting.bin_dir.server=/opt/caosdb/git/caosdb-server/scripting/bin
+
+# # location of the files from the pyinttest perspective
+# test_files.test_insert_files_in_dir.local=/extroot/test_insert_files_in_dir/
+test_files.test_insert_files_in_dir.local=/home/myself/test/debug_advanced/paths/extroot/test_insert_files_in_dir
+# # location of the files from the caosdb_servers perspective
+test_files.test_insert_files_in_dir.server=/opt/caosdb/mnt/extroot/test_insert_files_in_dir/
+
+########## Files ##################
+## Used by tests of file handling. Specify the path to an existing
+## directory in which file tests are performed, once as seen by the
+## host and once as seen by the server.
+
+# location of the files from the pyinttest (i.e. host) perspective
+#test_files.test_insert_files_in_dir.local=/extroot/test_insert_files_in_dir/
+
+# location of the files from the caosdb server's perspective
+#test_files.test_insert_files_in_dir.server=/opt/caosdb/mnt/extroot/test_insert_files_in_dir/
+
+# # location of the one-time tokens from the pyinttest's perspective
+# test_authentication.admin_token_crud = /authtoken/admin_token_crud.txt
+# test_authentication.admin_token_expired = /authtoken/admin_token_expired.txt
+# test_authentication.admin_token_3_attempts = /authtoken/admin_token_3_attempts.txt
+
+
+## Insert your usual settings here
+[Connection]
+url=https://localhost:10443/
+username=admin
+password_method=plain
+password=caosdb
+
diff --git a/unittests/test_configs/pycaosdb-empty.ini b/unittests/test_configs/pycaosdb-empty.ini
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/unittests/test_configs/pycaosdb-real-world-1.ini b/unittests/test_configs/pycaosdb-real-world-1.ini
new file mode 100644
index 0000000000000000000000000000000000000000..e524f1d3465c61d89ae4a4dda54536a722f99837
--- /dev/null
+++ b/unittests/test_configs/pycaosdb-real-world-1.ini
@@ -0,0 +1,17 @@
+[Connection]
+url = https://localhost:10443
+cacert = /opt/caosdb/cert/caosdb.cert.pem
+debug = 0
+timeout = 5000
+
+[Misc]
+sendmail = /usr/local/bin/sendmail_to_file
+entity_loan.curator_mail_from=crawler-test@example.com
+entity_loan.curator_mail_to=crawler-test@example.com
+
+[sss_helper]
+external_uri = https://caosdb.example.com:443
+
+[advancedtools]
+crawler.from_mail=admin@example.com
+crawler.to_mail=admin@example.com
diff --git a/unittests/test_configs/pycaosdb-real-world-2.ini b/unittests/test_configs/pycaosdb-real-world-2.ini
new file mode 100644
index 0000000000000000000000000000000000000000..5ebd115a4a4de189d22180130acca2a4b78b6daf
--- /dev/null
+++ b/unittests/test_configs/pycaosdb-real-world-2.ini
@@ -0,0 +1,15 @@
+[Connection]
+url = https://samplemanager.example.com:443
+cacert = /opt/caosdb/cert/caosdb.cert.pem
+debug = 0
+timeout = 5000
+[Misc]
+sendmail = /usr/local/bin/sendmail_to_file
+entity_loan.curator_mail_from=crawler-test@example.com
+entity_loan.curator_mail_to=crawler-test@example.com
+[sss_helper]
+external_uri = https://localhost:10443
+[advancedtools]
+crawler.from_mail=crawler-test@example.com
+crawler.to_mail=crawler-test@example.com
+
diff --git a/unittests/test_configs/pycaosdb-server-side-scripting.ini b/unittests/test_configs/pycaosdb-server-side-scripting.ini
new file mode 100644
index 0000000000000000000000000000000000000000..de2867f8dc66b3e81f10f35e40c36f9cb8591604
--- /dev/null
+++ b/unittests/test_configs/pycaosdb-server-side-scripting.ini
@@ -0,0 +1,9 @@
+; this is the pycaosdb.ini for the server-side-scripting home.
+[Connection]
+url = https://caosdb-server:10443
+cacert = /opt/caosdb/cert/caosdb.cert.pem
+debug = 0
+timeout = 5000
+
+[Misc]
+sendmail = /usr/local/bin/sendmail_to_file
diff --git a/unittests/test_configs/pycaosdb4.ini b/unittests/test_configs/pycaosdb4.ini
new file mode 100644
index 0000000000000000000000000000000000000000..ddbc7ca6f969e55ea6131d96f091177a13687ece
--- /dev/null
+++ b/unittests/test_configs/pycaosdb4.ini
@@ -0,0 +1,4 @@
+[Connection]
+url=https://localhost:10443/
+username=admin
+password_method=input
diff --git a/unittests/test_configs/pycaosdb5.ini b/unittests/test_configs/pycaosdb5.ini
new file mode 100644
index 0000000000000000000000000000000000000000..3f365efdd92641a39b742e22f825033a69e12dc5
--- /dev/null
+++ b/unittests/test_configs/pycaosdb5.ini
@@ -0,0 +1,4 @@
+[Connection]
+url=https://localhost:10443/
+username=admin
+# No password method: should be "input" by default
diff --git a/unittests/test_configs/pycaosdb6.ini b/unittests/test_configs/pycaosdb6.ini
new file mode 100644
index 0000000000000000000000000000000000000000..3826564f043c5702385a3d093cb4ebb8d4c24cd2
--- /dev/null
+++ b/unittests/test_configs/pycaosdb6.ini
@@ -0,0 +1,4 @@
+[Connection]
+url=https://localhost:10443/
+# No username, unauthenticated connection
+password_method = unauthenticated
diff --git a/unittests/test_connection.py b/unittests/test_connection.py
index 16370f00b7d5e3389582befaac1762b1d2992fcf..6cc23d87c5cdcf639709a444849a856a8c70af5f 100644
--- a/unittests/test_connection.py
+++ b/unittests/test_connection.py
@@ -37,7 +37,8 @@ from caosdb.connection.connection import (CaosDBServerConnection,
from caosdb.connection.mockup import (MockUpResponse, MockUpServerConnection,
_request_log_message)
from caosdb.connection.utils import make_uri_path, quote, urlencode
-from caosdb.exceptions import ConfigurationError, LoginFailedError
+from caosdb.exceptions import (ConfigurationError, LoginFailedError,
+ CaosDBConnectionError)
from nose.tools import assert_equal as eq
from nose.tools import assert_false as falz
from nose.tools import assert_is_not_none as there
@@ -46,6 +47,13 @@ from nose.tools import assert_true as tru
from pytest import raises
+def setup_function(function):
+ configure_connection(url="http://localhost:8888/some/path",
+ password_method="plain", username="test",
+ password="blub",
+ implementation=MockUpServerConnection)
+
+
def setup_module():
_reset_config()
@@ -103,6 +111,7 @@ def test_configure_connection():
get_config().set("Connection", "password_method", "plain")
get_config().set("Connection", "password", "test_password")
get_config().set("Connection", "timeout", "200")
+ get_config().set("Connection", "ssl_insecure", "True")
there(configure_connection)
tru(hasattr(configure_connection, "__call__"))
@@ -116,6 +125,18 @@ def test_configure_connection():
tru(isinstance(c._delegate_connection, MockUpServerConnection))
+def test_configure_connection_bad_url():
+ configure_connection(url="https://localhost:8888")
+ with raises(CaosDBConnectionError) as exc_info:
+ configure_connection(url="ftp://localhost:8888")
+ assert exc_info.value.args[0].startswith(
+ "The connection url is expected to be a http or https url")
+ with raises(CaosDBConnectionError) as exc_info:
+ configure_connection(url="localhost:8888")
+ assert exc_info.value.args[0].startswith(
+ "The connection url is expected to be a http or https url")
+
+
def test_connection_interface():
with raiz(TypeError) as cm:
CaosDBServerConnection()
@@ -169,7 +190,7 @@ def test_getter_status():
def test_read():
response = test_init_response()
tru(hasattr(response, "read"))
- eq(response.read(), "Body")
+ eq(response.read().decode(), "Body")
def test_getter_session_token():
diff --git a/unittests/test_entity.py b/unittests/test_entity.py
index 1e88702ac016d7dcfdf00919dd0f93b5d3345e00..f2891fda266e1d62139b4cb2667c31b090ca6498 100644
--- a/unittests/test_entity.py
+++ b/unittests/test_entity.py
@@ -26,10 +26,13 @@
import unittest
from lxml import etree
+import os
from caosdb import (INTEGER, Entity, Property, Record, RecordType,
configure_connection)
from caosdb.connection.mockup import MockUpServerConnection
+UNITTESTDIR = os.path.dirname(os.path.abspath(__file__))
+
class TestEntity(unittest.TestCase):
@@ -87,7 +90,7 @@ class TestEntity(unittest.TestCase):
"""
parser = etree.XMLParser(remove_comments=True)
entity = Entity._from_xml(Entity(),
- etree.parse("unittests/test_record.xml",
+ etree.parse(os.path.join(UNITTESTDIR, "test_record.xml"),
parser).getroot())
self.assertEqual(entity.role, "Record")
diff --git a/unittests/test_high_level_api.py b/unittests/test_high_level_api.py
new file mode 100644
index 0000000000000000000000000000000000000000..ea5e635eadaa849480de5f3ece10b813a538a1b0
--- /dev/null
+++ b/unittests/test_high_level_api.py
@@ -0,0 +1,654 @@
+# This file is a part of the CaosDB Project.
+#
+# Copyright (C) 2018 Research Group Biomedical Physics,
+# Max-Planck-Institute for Dynamics and Self-Organization Göttingen
+# Copyright (C) 2022 Alexander Schlemmer <alexander.schlemmer@ds.mpg.de>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see <https://www.gnu.org/licenses/>.
+#
+# ** end header
+#
+# Test high level api module
+# A. Schlemmer, 02/2022
+
+
+import caosdb as db
+from caosdb.high_level_api import (convert_to_entity, convert_to_python_object,
+ new_high_level_entity)
+from caosdb.high_level_api import (CaosDBPythonUnresolvedParent,
+ CaosDBPythonUnresolvedReference,
+ CaosDBPythonRecord, CaosDBPythonFile,
+ high_level_type_for_standard_type,
+ standard_type_for_high_level_type,
+ high_level_type_for_role,
+ CaosDBPythonEntity)
+from caosdb.apiutils import compare_entities
+
+from caosdb.common.datatype import (is_list_datatype,
+ get_list_datatype,
+ is_reference)
+
+import pytest
+from lxml import etree
+import os
+import tempfile
+import pickle
+
+import sys
+import traceback
+import pdb
+
+
+@pytest.fixture
+def testrecord():
+ parser = etree.XMLParser(remove_comments=True)
+ testrecord = db.Record._from_xml(
+ db.Record(),
+ etree.parse(os.path.join(os.path.dirname(__file__), "test_record.xml"),
+ parser).getroot())
+ return testrecord
+
+
+def test_convert_object(testrecord):
+ r2 = convert_to_python_object(testrecord)
+ assert r2.species == "Rabbit"
+
+
+def test_pickle_object(testrecord):
+ r2 = convert_to_python_object(testrecord)
+ with tempfile.TemporaryFile() as f:
+ pickle.dump(r2, f)
+ f.seek(0)
+ rn2 = pickle.load(f)
+ assert r2.date == rn2.date
+
+
+def test_convert_record():
+ """
+ Test the high level python API.
+ """
+ r = db.Record()
+ r.add_parent("bla")
+ r.add_property(name="a", value=42)
+ r.add_property(name="b", value="test")
+
+ obj = convert_to_python_object(r)
+ assert obj.a == 42
+ assert obj.b == "test"
+
+ # There is no such property
+ with pytest.raises(AttributeError):
+ assert obj.c == 18
+
+ assert obj.has_parent("bla") is True
+ assert obj.has_parent(CaosDBPythonUnresolvedParent(name="bla")) is True
+
+ # Check the has_parent function:
+ assert obj.has_parent("test") is False
+ assert obj.has_parent(CaosDBPythonUnresolvedParent(name="test")) is False
+
+ # duplicate parent
+ with pytest.raises(RuntimeError):
+ obj.add_parent("bla")
+
+ # add parent with just an id:
+ obj.add_parent(CaosDBPythonUnresolvedParent(id=225))
+ assert obj.has_parent(225) is True
+ assert obj.has_parent(CaosDBPythonUnresolvedParent(id=225)) is True
+ assert obj.has_parent(226) is False
+ assert obj.has_parent(CaosDBPythonUnresolvedParent(id=228)) is False
+
+ # same with just a name:
+ obj.add_parent(CaosDBPythonUnresolvedParent(name="another"))
+ assert obj.has_parent("another") is True
+
+
+def test_convert_with_references():
+ r_ref = db.Record()
+ r_ref.add_property(name="a", value=42)
+
+ r = db.Record()
+ r.add_property(name="ref", value=r_ref)
+
+ # try:
+ obj = convert_to_python_object(r)
+ # except:
+ # extype, value, tb = sys.exc_info()
+ # traceback.print_exc()
+ # pdb.post_mortem(tb)
+ assert obj.ref.a == 42
+
+ # With datatype:
+ r_ref = db.Record()
+ r_ref.add_parent("bla")
+ r_ref.add_property(name="a", value=42)
+
+ r = db.Record()
+ r.add_property(name="ref", value=r_ref)
+
+ obj = convert_to_python_object(r)
+ assert obj.ref.a == 42
+ # Parent does not automatically lead to a datatype:
+ assert obj.get_property_metadata("ref").datatype is None
+ assert obj.ref.has_parent("bla") is True
+
+ # Add datatype explicitely:
+ r_ref = db.Record()
+ r_ref.add_parent("bla")
+ r_ref.add_property(name="a", value=42)
+
+ r = db.Record()
+ r.add_property(name="ref", value=r_ref, datatype="bla")
+
+ obj = convert_to_python_object(r)
+ assert obj.ref.a == 42
+ # Parent does not automatically lead to a datatype:
+ assert obj.get_property_metadata("ref").datatype == "bla"
+ assert obj.ref.has_parent("bla") is True
+
+ # Unresolved Reference:
+ r = db.Record()
+ r.add_property(name="ref", value=27, datatype="bla")
+
+ obj = convert_to_python_object(r)
+ # Parent does not automatically lead to a datatype:
+ assert obj.get_property_metadata("ref").datatype == "bla"
+ assert isinstance(obj.ref, CaosDBPythonUnresolvedReference)
+ assert obj.ref.id == 27
+
+
+def test_resolve_references():
+ r = db.Record()
+ r.add_property(name="ref", value=27, datatype="bla")
+ r.add_property(name="ref_false", value=27) # this should be interpreted as integer property
+ obj = convert_to_python_object(r)
+
+ ref = db.Record(id=27)
+ ref.add_property(name="a", value=57)
+
+ unused_ref1 = db.Record(id=28)
+ unused_ref2 = db.Record(id=29)
+ unused_ref3 = db.Record(name="bla")
+
+ references = db.Container().extend([
+ unused_ref1, ref, unused_ref2, unused_ref3])
+
+ # Nothing is going to be resolved:
+ obj.resolve_references(False, db.Container())
+ assert isinstance(obj.ref, CaosDBPythonUnresolvedReference)
+ assert obj.ref.id == 27
+ assert obj.ref_false == 27
+
+ # deep == True does not help:
+ obj.resolve_references(True, db.Container())
+ assert isinstance(obj.ref, CaosDBPythonUnresolvedReference)
+ assert obj.ref.id == 27
+
+ # But adding the reference container will do:
+ obj.resolve_references(False, references)
+ assert not isinstance(obj.ref, CaosDBPythonUnresolvedReference)
+ assert isinstance(obj.ref, CaosDBPythonRecord)
+ assert obj.ref.id == 27
+ assert obj.ref.a == 57
+ # Datatypes will not automatically be set:
+ assert obj.ref.get_property_metadata("a").datatype is None
+
+ # Test deep resolve:
+ ref2 = db.Record(id=225)
+ ref2.add_property(name="c", value="test")
+ ref.add_property(name="ref", value=225, datatype="bla")
+
+ obj = convert_to_python_object(r)
+ assert isinstance(obj.ref, CaosDBPythonUnresolvedReference)
+ obj.resolve_references(False, references)
+ assert not isinstance(obj.ref, CaosDBPythonUnresolvedReference)
+ assert isinstance(obj.ref.ref, CaosDBPythonUnresolvedReference)
+ assert obj.ref.ref.id == 225
+
+ # Will not help, because ref2 is missing in container:
+ obj.resolve_references(True, references)
+ assert not isinstance(obj.ref, CaosDBPythonUnresolvedReference)
+ assert isinstance(obj.ref.ref, CaosDBPythonUnresolvedReference)
+ assert obj.ref.ref.id == 225
+
+ references.append(ref2)
+ obj.resolve_references(False, references)
+ assert not isinstance(obj.ref, CaosDBPythonUnresolvedReference)
+ assert isinstance(obj.ref.ref, CaosDBPythonUnresolvedReference)
+ assert obj.ref.ref.id == 225
+
+ obj.resolve_references(True, references)
+ assert not isinstance(obj.ref, CaosDBPythonUnresolvedReference)
+ assert not isinstance(obj.ref.ref, CaosDBPythonUnresolvedReference)
+ assert obj.ref.ref.c == "test"
+
+ # Test circular dependencies:
+ ref2.add_property(name="ref", value=27, datatype="bla")
+ obj = convert_to_python_object(r)
+ obj.resolve_references(True, references)
+ assert obj.ref.ref.ref == obj.ref
+
+
+def equal_entities(r1, r2):
+ res = compare_entities(r1, r2)
+ if len(res) != 2:
+ return False
+ for i in range(2):
+ if len(res[i]["parents"]) != 0 or len(res[i]["properties"]) != 0:
+ return False
+ return True
+
+
+def test_conversion_to_entity():
+ r = db.Record()
+ r.add_parent("bla")
+ r.add_property(name="a", value=42)
+ r.add_property(name="b", value="test")
+ obj = convert_to_python_object(r)
+ rconv = convert_to_entity(obj)
+ assert equal_entities(r, rconv)
+
+ # With a reference:
+ r_ref = db.Record()
+ r_ref.add_parent("bla")
+ r_ref.add_property(name="a", value=42)
+
+ r = db.Record()
+ r.add_property(name="ref", value=r_ref)
+ obj = convert_to_python_object(r)
+ rconv = convert_to_entity(obj)
+ assert (rconv.get_property("ref").value.get_property("a").value
+ == r.get_property("ref").value.get_property("a").value)
+ # TODO: add more tests here
+
+
+def test_base_properties():
+ r = db.Record(id=5, name="test", description="ok")
+ r.add_property(name="v", value=15, datatype=db.INTEGER, unit="kpx",
+ importance="RECOMMENDED", description="description")
+ obj = convert_to_python_object(r)
+ assert obj.name == "test"
+ assert obj.id == 5
+ assert obj.description == "ok"
+ metadata = obj.get_property_metadata("v")
+ assert metadata.id is None
+ assert metadata.datatype == db.INTEGER
+ assert metadata.unit == "kpx"
+ assert metadata.importance == "RECOMMENDED"
+ assert metadata.description == "description"
+
+ rconv = convert_to_entity(obj)
+ assert rconv.name == "test"
+ assert rconv.id == 5
+ assert rconv.description == "ok"
+ prop = rconv.get_property("v")
+ assert prop.value == 15
+ assert prop.datatype == db.INTEGER
+ assert prop.unit == "kpx"
+ assert prop.description == "description"
+ assert rconv.get_importance("v") == "RECOMMENDED"
+
+
+def test_empty():
+ r = db.Record()
+ obj = convert_to_python_object(r)
+ assert isinstance(obj, CaosDBPythonRecord)
+ assert len(obj.get_properties()) == 0
+ assert len(obj.get_parents()) == 0
+
+ rconv = convert_to_entity(obj)
+ assert len(rconv.properties) == 0
+
+
+def test_wrong_entity_for_file():
+ r = db.Record()
+ r.path = "test.dat"
+ r.file = "/local/path/test.dat"
+ assert r.path == "test.dat"
+ assert r.file == "/local/path/test.dat"
+ with pytest.raises(RuntimeError):
+ obj = convert_to_python_object(r)
+
+
+def test_serialization():
+ r = db.Record(id=5, name="test", description="ok")
+ r.add_property(name="v", value=15, datatype=db.INTEGER, unit="kpx",
+ importance="RECOMMENDED")
+
+ obj = convert_to_python_object(r)
+ text = str(obj)
+ teststrs = ["description: ok", "id: 5", "datatype: INTEGER",
+ "importance: RECOMMENDED", "unit: kpx", "name: test", "v: 15"]
+ for teststr in teststrs:
+ assert teststr in text
+
+ r = db.Record(description="ok")
+ r.add_property(name="v", value=15, datatype=db.INTEGER, unit="kpx",
+ importance="RECOMMENDED")
+ obj = convert_to_python_object(r)
+ text = str(obj)
+ assert "name" not in text
+ assert "id" not in text
+
+
+def test_files():
+ # empty file:
+ r = db.File()
+ obj = convert_to_python_object(r)
+ print(type(obj))
+ assert isinstance(obj, CaosDBPythonFile)
+ assert len(obj.get_properties()) == 0
+ assert len(obj.get_parents()) == 0
+
+ rconv = convert_to_entity(obj)
+ assert len(rconv.properties) == 0
+
+ r.path = "test.dat"
+ r.file = "/local/path/test.dat"
+ obj = convert_to_python_object(r)
+ assert r.path == "test.dat"
+ assert r.file == "/local/path/test.dat"
+ assert isinstance(obj, CaosDBPythonFile)
+
+ assert obj.path == "test.dat"
+ assert obj.file == "/local/path/test.dat"
+
+ assert "path: test.dat" in str(obj)
+ assert "file: /local/path/test.dat" in str(obj)
+
+ # record with file property:
+ rec = db.Record()
+ rec.add_property(name="testfile", value=r)
+ assert rec.get_property("testfile").value.file == "/local/path/test.dat"
+ assert rec.get_property("testfile").value.path == "test.dat"
+
+ obj = convert_to_python_object(rec)
+ assert obj.testfile.file == "/local/path/test.dat"
+ assert obj.testfile.path == "test.dat"
+
+ rconv = convert_to_entity(obj)
+ assert rconv.get_property("testfile").value.file == "/local/path/test.dat"
+ assert rconv.get_property("testfile").value.path == "test.dat"
+
+ # record with file property as reference:
+ rec = db.Record()
+ rec.add_property(name="testfile", value=2, datatype=db.FILE)
+ obj = convert_to_python_object(rec)
+ assert type(obj.testfile) == CaosDBPythonUnresolvedReference
+ assert obj.testfile.id == 2
+ assert obj.get_property_metadata("testfile").datatype == db.FILE
+
+ # without resolving references:
+ rconv = convert_to_entity(obj)
+ p = rconv.get_property("testfile")
+ assert p.value == 2
+ assert p.datatype == db.FILE
+
+ # with previously resolved reference (should not work here, because id is missing):
+ obj.resolve_references(True, db.Container().extend(r))
+ rconv = convert_to_entity(obj)
+ p = rconv.get_property("testfile")
+ assert p.value == 2
+ assert p.datatype == db.FILE
+
+ # this time it must work:
+ r.id = 2
+ obj.resolve_references(True, db.Container().extend(r))
+ rconv = convert_to_entity(obj)
+ p = rconv.get_property("testfile")
+ assert type(p.value) == db.File
+ assert p.datatype == db.FILE
+ assert p.value.file == "/local/path/test.dat"
+ assert p.value.path == "test.dat"
+
+
+@pytest.mark.xfail
+def test_record_generator():
+ rt = db.RecordType(name="Simulation")
+ rt.add_property(name="a", datatype=db.INTEGER)
+ rt.add_property(name="b", datatype=db.DOUBLE)
+ rt.add_property(name="inputfile", datatype=db.FILE)
+
+ simrt = db.RecordType(name="SimOutput")
+ rt.add_property(name="outputfile", datatype="SimOutput")
+
+ obj = new_high_level_entity(
+ rt, "SUGGESTED", "", True)
+ print(obj)
+ assert False
+
+
+def test_list_types():
+ r = db.Record()
+ r.add_property(name="a", value=[1, 2, 4])
+
+ assert get_list_datatype(r.get_property("a").datatype) is None
+
+ obj = convert_to_python_object(r)
+ assert type(obj.a) == list
+ assert len(obj.a) == 3
+ assert 4 in obj.a
+ assert obj.get_property_metadata("a").datatype is None
+
+ conv = convert_to_entity(obj)
+ prop = r.get_property("a")
+ assert prop.value == [1, 2, 4]
+ assert prop.datatype is None
+
+ r.get_property("a").datatype = db.LIST(db.INTEGER)
+ assert r.get_property("a").datatype == "LIST<INTEGER>"
+ obj = convert_to_python_object(r)
+ assert type(obj.a) == list
+ assert len(obj.a) == 3
+ assert 4 in obj.a
+ assert obj.get_property_metadata("a").datatype == "LIST<INTEGER>"
+
+ conv = convert_to_entity(obj)
+ prop = r.get_property("a")
+ assert prop.value == [1, 2, 4]
+ assert obj.get_property_metadata("a").datatype == "LIST<INTEGER>"
+
+ # List of referenced objects:
+ r = db.Record()
+ r.add_property(name="a", value=[1, 2, 4], datatype="LIST<TestReference>")
+ obj = convert_to_python_object(r)
+ assert type(obj.a) == list
+ assert len(obj.a) == 3
+ assert obj.get_property_metadata("a").datatype == "LIST<TestReference>"
+ for i in range(3):
+ assert type(obj.a[i]) == CaosDBPythonUnresolvedReference
+ assert obj.a == [CaosDBPythonUnresolvedReference(id=i) for i in [1, 2, 4]]
+
+ # Try resolving:
+
+ # Should not work:
+ obj.resolve_references(False, db.Container())
+ assert type(obj.a) == list
+ assert len(obj.a) == 3
+ assert obj.get_property_metadata("a").datatype == "LIST<TestReference>"
+ for i in range(3):
+ assert type(obj.a[i]) == CaosDBPythonUnresolvedReference
+ assert obj.a == [CaosDBPythonUnresolvedReference(id=i) for i in [1, 2, 4]]
+
+ references = db.Container()
+ for i in [1, 2, 4]:
+ ref = db.Record(id=i)
+ ref.add_property(name="val", value=str(i) + " bla")
+ references.append(ref)
+
+ obj.resolve_references(False, references)
+ assert type(obj.a) == list
+ assert len(obj.a) == 3
+ assert obj.get_property_metadata("a").datatype == "LIST<TestReference>"
+ for i in range(3):
+ assert type(obj.a[i]) == CaosDBPythonRecord
+
+ assert obj.a[0].val == "1 bla"
+
+ # Conversion with embedded records:
+ r2 = db.Record()
+ r2.add_property(name="a", value=4)
+ r3 = db.Record()
+ r3.add_property(name="b", value=8)
+
+ r = db.Record()
+ r.add_property(name="a", value=[r2, r3])
+
+ obj = convert_to_python_object(r)
+ assert type(obj.a) == list
+ assert len(obj.a) == 2
+ assert obj.a[0].a == 4
+ assert obj.a[1].b == 8
+
+ # Serialization
+ text = str(obj)
+ text2 = str(convert_to_python_object(r2)).split("\n")
+ print(text)
+ # cut away first two characters in text
+ text = [line[4:] for line in text.split("\n")]
+ for line in text2:
+ assert line in text
+
+
+# Test utility functions:
+def test_type_conversion():
+ assert high_level_type_for_standard_type(db.Record()) == CaosDBPythonRecord
+ assert high_level_type_for_standard_type(db.Entity()) == CaosDBPythonEntity
+ assert standard_type_for_high_level_type(CaosDBPythonRecord()) == db.Record
+ assert standard_type_for_high_level_type(CaosDBPythonEntity()) == db.Entity
+ assert standard_type_for_high_level_type(CaosDBPythonFile(), True) == "File"
+ assert standard_type_for_high_level_type(CaosDBPythonRecord(), True) == "Record"
+ assert high_level_type_for_role("Record") == CaosDBPythonRecord
+ assert high_level_type_for_role("Entity") == CaosDBPythonEntity
+ assert high_level_type_for_role("File") == CaosDBPythonFile
+ with pytest.raises(RuntimeError, match="Unknown role."):
+ high_level_type_for_role("jkaldjfkaldsjf")
+
+ with pytest.raises(RuntimeError, match="Incompatible type."):
+ standard_type_for_high_level_type(42, True)
+
+ with pytest.raises(ValueError):
+ high_level_type_for_standard_type("ajsdkfjasfkj")
+
+ with pytest.raises(RuntimeError, match="Incompatible type."):
+ class IncompatibleType(db.Entity):
+ pass
+ high_level_type_for_standard_type(IncompatibleType())
+
+
+def test_deserialization():
+ r = db.Record(id=17, name="test")
+ r.add_parent("bla")
+ r.add_property(name="a", value=42)
+ r.add_property(name="b", value="test")
+
+ obj = convert_to_python_object(r)
+
+ serial = obj.serialize()
+ obj_des = CaosDBPythonEntity.deserialize(serial)
+
+ assert obj_des.name == "test"
+ assert obj_des.id == 17
+ assert obj_des.has_parent(CaosDBPythonUnresolvedParent(name="bla"))
+ print(obj)
+ print(obj_des)
+
+ # This test is very strict, and might fail if order in dictionary is not preserved:
+ assert obj.serialize() == obj_des.serialize()
+
+ f = db.File()
+ f.file = "bla.test"
+ f.path = "/test/n/bla.test"
+
+ obj = convert_to_python_object(f)
+
+ serial = obj.serialize()
+ obj_des = CaosDBPythonEntity.deserialize(serial)
+ assert obj_des.file == "bla.test"
+ assert obj_des.path == "/test/n/bla.test"
+
+ r = db.Record(id=17, name="test")
+ r.add_parent("bla")
+ r.add_property(name="a", value=42)
+ r.add_property(name="b", value="test")
+
+ ref = db.Record(id=28)
+ ref.add_parent("bla1")
+ ref.add_parent("bla2")
+ ref.add_property(name="c", value=5,
+ unit="s", description="description missing")
+ r.add_property(name="ref", value=ref)
+
+ obj = convert_to_python_object(r)
+
+ serial = obj.serialize()
+ obj_des = CaosDBPythonEntity.deserialize(serial)
+ assert obj.serialize() == obj_des.serialize()
+
+
+@pytest.fixture
+def get_record_container():
+ record_xml = """
+<Entities>
+ <Record id="109">
+ <Version id="da669fce50554b2835c3826cf717d6a4532f02de" head="true">
+ <Predecessor id="68534369c5fd05e5bb1d37801a3dbc1532a8e094"/>
+ </Version>
+ <Parent id="103" name="Experiment" description="General type for all experiments in our lab"/>
+ <Property id="104" name="alpha" description="A fictitious measurement" datatype="DOUBLE" unit="km" importance="FIX" flag="inheritance:FIX">16.0</Property>
+ <Property id="107" name="date" datatype="DATETIME" importance="FIX" flag="inheritance:FIX">2022-03-16</Property>
+ <Property id="108" name="identifier" datatype="TEXT" importance="FIX" flag="inheritance:FIX">Demonstration</Property>
+ <Property id="111" name="sources" description="The elements of this lists are scientific activities that this scientific activity is based on." datatype="LIST<ScientificActivity>" importance="FIX" flag="inheritance:FIX">
+ <Value>109</Value>
+ </Property>
+ </Record>
+</Entities>"""
+
+ c = db.Container.from_xml(record_xml)
+ return c
+
+
+def test_recursion(get_record_container):
+ r = convert_to_python_object(get_record_container[0])
+ r.resolve_references(r, get_record_container)
+ assert r.id == 109
+ assert r.sources[0].id == 109
+ assert r.sources[0].sources[0].id == 109
+ assert "&id001" in str(r)
+ assert "*id001" in str(r)
+
+ d = r.serialize(True)
+ assert r.sources[0] == r.sources[0].sources[0]
+
+
+@pytest.mark.xfail
+def test_recursion_advanced(get_record_container):
+ # TODO:
+ # This currently fails, because resolve is done in a second step
+ # and therefore a new python object is created for the reference.
+ r = convert_to_python_object(get_record_container[0])
+ r.resolve_references(r, get_record_container)
+ d = r.serialize(True)
+ assert r == r.sources[0]
+
+
+def test_cyclic_references():
+ r1 = db.Record()
+ r2 = db.Record()
+ r1.add_property(name="ref_to_two", value=r2)
+ r2.add_property(name="ref_to_one", value=r1)
+
+ # This would have lead to a recursion error before adding the detection for
+ # cyclic references:
+ r = convert_to_python_object(r1)
diff --git a/unittests/test_issues.py b/unittests/test_issues.py
new file mode 100644
index 0000000000000000000000000000000000000000..3fb48416511ba654d6f998442319c4ff29ac2956
--- /dev/null
+++ b/unittests/test_issues.py
@@ -0,0 +1,66 @@
+# This file is a part of the CaosDB Project.
+#
+# Copyright (c) 2022 IndiScale GmbH
+# Copyright (c) 2022 Daniel Hornung (d.hornung@indiscale.com)
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+"""Test known issues to prevent regressions.
+"""
+
+import os
+
+import lxml
+import caosdb as db
+
+from pytest import raises
+
+
+def test_issue_100():
+ """_parse_value() fails for some list-valued content
+ """
+
+ # Parse from (invalid) XML file
+ filename = os.path.join(os.path.dirname(__file__), "data", "list_in_value.xml")
+ xml_el = lxml.etree.parse(filename).getroot()
+ with raises(TypeError) as exc_info:
+ db.common.models._parse_single_xml_element(xml_el)
+ assert "Invalid datatype: List valued properties" in str(exc_info.value)
+
+
+def test_issue_156():
+ """Does parse_value make a mistake with entities?
+
+ https://gitlab.indiscale.com/caosdb/src/caosdb-pylib/-/issues/156
+ """
+ project = db.Record(name="foo")
+ project.add_parent(name="RTName")
+ # <Record name="foo">
+ # <Parent name="RTName"/>
+ # </Record>
+ experiment = db.Record()
+ experiment.add_property(name="RTName", value=project)
+ # <Record>
+ # <Property name="RTName" importance="FIX" flag="inheritance:FIX">foo</Property>
+ # </Record>
+ value = experiment.get_property("RTName").value
+ # <Record name="foo">
+ # <Parent name="RTName"/>
+ # </Record>
+ parents = value.get_parents()
+ # <ParentList>
+ # <Parent name="RTName"/>
+ # </ParentList>
+ assert value is project
+ assert parents[0].name == "RTName"
diff --git a/unittests/test_message.py b/unittests/test_message.py
index 5e1003056c1b606a004b63bb7618e5e0474952bc..440e7169501afb0a35acb78df95cefae01bd9426 100644
--- a/unittests/test_message.py
+++ b/unittests/test_message.py
@@ -27,11 +27,14 @@ import caosdb as db
from copy import deepcopy
+import pytest
+
+
def test_messages_dict_behavior():
from caosdb.common.models import Message
- from caosdb.common.models import _Messages
+ from caosdb.common.models import Messages
- msgs = _Messages()
+ msgs = Messages()
# create Message
msg = Message(
@@ -40,12 +43,12 @@ def test_messages_dict_behavior():
description="Greeting the world",
body="Hello, world!")
- # append it to the _Messages
+ # append it to the Messages
assert repr(msg) == '<HelloWorld code="1" description="Greeting the world">Hello, world!</HelloWorld>\n'
msgs.append(msg)
assert len(msgs) == 1
- # use _Messages as list of Message objects
+ # use Messages as list of Message objects
for m in msgs:
assert isinstance(m, Message)
@@ -70,10 +73,6 @@ def test_messages_dict_behavior():
assert msgs["HelloWorld", 2] == (
"Greeting the world in German", "Hallo, Welt!")
- msgs["HelloWorld", 2] = "Greeting the world in German", "Huhu, Welt!"
- assert len(msgs) == 1
- assert msgs["HelloWorld", 2] == (
- "Greeting the world in German", "Huhu, Welt!")
del msgs["HelloWorld", 2]
assert msgs.get("HelloWorld", 2) is None
@@ -83,11 +82,11 @@ def test_messages_dict_behavior():
def test_deepcopy():
- """Test whether deepcopy of _Messages objects doesn't mess up
+ """Test whether deepcopy of Messages objects doesn't mess up
contained Messages objects.
"""
- msgs = db.common.models._Messages()
+ msgs = db.common.models.Messages()
msg = db.Message(type="bla", code=1234, description="desc", body="blabla")
msgs.append(msg)
msg_copy = deepcopy(msgs)[0]
@@ -102,7 +101,7 @@ def test_deepcopy():
def test_deepcopy_clear_server():
- msgs = db.common.models._Messages()
+ msgs = db.common.models.Messages()
msg = db.Message(type="bla", code=1234, description="desc", body="blabla")
err_msg = db.Message(type="Error", code=1357, description="error")
msgs.extend([msg, err_msg])
@@ -116,3 +115,18 @@ def test_deepcopy_clear_server():
copied_msgs.clear_server_messages()
assert len(copied_msgs) == 1
assert copied_msgs[0].code == msg.code
+
+
+def test_list_behavior():
+ msgs = db.common.models.Messages()
+ msgs.append(db.Message("test"))
+ assert len(msgs) == 1
+ assert msgs[0] == db.Message("test")
+ assert msgs[0] != db.Message("test2")
+
+ msgs.append(db.Message("test"))
+ assert len(msgs) == 2
+ assert msgs[0] == msgs[1]
+
+ with pytest.raises(IndexError):
+ msgs[3]
diff --git a/unittests/test_plantuml.py b/unittests/test_plantuml.py
new file mode 100644
index 0000000000000000000000000000000000000000..a507c36b2d3a4246205fc7507cb05119c575084c
--- /dev/null
+++ b/unittests/test_plantuml.py
@@ -0,0 +1,61 @@
+#!/usr/bin/env python3
+# encoding: utf-8
+#
+# ** header v3.0
+# This file is a part of the CaosDB Project.
+#
+# Copyright (C) 2022 Indiscale GmbH <info@indiscale.com>
+# Copyright (C) 2022 Henrik tom Wörden <h.tomwoerden@indiscale.com>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see <https://www.gnu.org/licenses/>.
+#
+# ** end header
+#
+
+"""
+test plantuml utility
+"""
+
+import tempfile
+import pytest
+import caosdb as db
+import shutil
+from caosdb.utils.plantuml import to_graphics
+
+
+@pytest.fixture
+def setup_n_teardown(autouse=True):
+
+ with tempfile.TemporaryDirectory() as td:
+ global output
+ output = td
+ yield
+
+
+@pytest.fixture
+def entities():
+ return [db.RecordType(name="TestRT1").add_property("testprop"),
+ db.RecordType(name="TestRT2").add_property("testprop2"),
+ db.Property("testprop")]
+
+
+@pytest.mark.skipif(shutil.which('plantuml') is None, reason="No plantuml found")
+def test_to_graphics1(entities, setup_n_teardown):
+ to_graphics(entities, "data_model", output_dirname=output)
+
+
+@pytest.mark.skipif(shutil.which('plantuml') is None, reason="No plantuml found")
+def test_to_graphics2(entities, setup_n_teardown):
+ to_graphics(entities, "data_model", output_dirname=output, formats=["tpng", "tsvg"],
+ add_properties=False, add_legend=False, style="salexan")
diff --git a/unittests/test_property.py b/unittests/test_property.py
index 834b1be582c58c60f70331de9cb0d0d6414fd6c9..84f89b5a959192d7831e1bb3eab3a441912afe7e 100644
--- a/unittests/test_property.py
+++ b/unittests/test_property.py
@@ -1,11 +1,11 @@
# -*- encoding: utf-8 -*-
#
-# ** header v3.0
# This file is a part of the CaosDB Project.
#
# Copyright (C) 2018 Research Group Biomedical Physics,
# Max-Planck-Institute for Dynamics and Self-Organization Göttingen
-# Copyright (C) 2020 IndiScale GmbH <info@indiscale.com>
+# Copyright (C) 2020 - 2023 IndiScale GmbH <info@indiscale.com>
+# Copyright (C) 2023 Florian Spreckelsen <f.spreckelsen@indiscale.com>
# Copyright (C) 2020 Timm Fitschen <t.fitschen@indiscale.com>
#
# This program is free software: you can redistribute it and/or modify
@@ -21,18 +21,19 @@
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <https://www.gnu.org/licenses/>.
#
-# ** end header
-#
"""Tests for the Property class."""
+import os
+
import caosdb as db
from caosdb import Entity, Property, Record
# pylint: disable=missing-docstring
from lxml import etree
parser = etree.XMLParser(remove_comments=True)
-testrecord = Record._from_xml(Record(),
- etree.parse("unittests/test_record.xml",
- parser).getroot())
+testrecord = Record._from_xml(
+ Record(),
+ etree.parse(os.path.join(os.path.dirname(__file__), "test_record.xml"),
+ parser).getroot())
def test_is_entity():
@@ -48,7 +49,8 @@ def test_instance_variables():
def test_null_empty_text_value_1():
- assert testrecord.get_property("LISTofTEXT").value == ["One", "Two", "Three", None, ""]
+ assert testrecord.get_property("LISTofTEXT").value == ["One", "Two",
+ "Three", None, ""]
def test_null_empty_text_value_2():
@@ -134,3 +136,87 @@ def test_is_reference():
# restore retrieve function with original
Entity.retrieve = real_retrieve
+
+
+def test_remove_value_from_property():
+
+ rec = Record()
+ names_values_dtypes = [
+ ("testListProp1", [1, 2, 3], db.LIST(db.INTEGER)),
+ ("testListProp2", ["a", "b", "a"], db.LIST(db.TEXT)),
+ ("testScalarProp1", "bla", db.TEXT),
+ ("testScalarProp2", False, db.BOOLEAN),
+ ("testEmptyProp", None, db.REFERENCE),
+ ("testNoneListProp", [None, None], db.LIST(db.REFERENCE)),
+ ]
+ for name, value, dtype in names_values_dtypes:
+ rec.add_property(name=name, value=value, datatype=dtype)
+
+ # property doesn't exist, so do nothing
+ returned = rec.remove_value_from_property("nonexisting", "some_value")
+ assert returned is rec
+ for name, value, dtype in names_values_dtypes:
+ assert rec.get_property(name).value == value
+ assert rec.get_property(name).datatype == dtype
+
+ # value doesn't exist so nothing changes either
+ rec.remove_value_from_property("testListProp1", 0)
+ assert rec.get_property("testListProp1").value == [1, 2, 3]
+ assert rec.get_property("testListProp1").datatype == db.LIST(db.INTEGER)
+
+ returned = rec.remove_value_from_property("testScalarProp2", True)
+ assert returned is rec
+ assert rec.get_property("testScalarProp2").value is False
+ assert rec.get_property("testScalarProp2").datatype == db.BOOLEAN
+
+ # Simple removals from lists without emptying them
+ rec.remove_value_from_property("testListProp1", 1)
+ assert rec.get_property("testListProp1").value == [2, 3]
+
+ rec.remove_value_from_property("testListProp1", 2)
+ assert rec.get_property("testListProp1").value == [3]
+
+ # similarly to Python's `list.remove()`, only remove first occurrance
+ rec.remove_value_from_property("testListProp2", "a")
+ assert rec.get_property("testListProp2").value == ["b", "a"]
+
+ # default is to remove an empty property:
+ rec.remove_value_from_property("testListProp1", 3)
+ assert rec.get_property("testListProp1") is None
+
+ rec.remove_value_from_property("testScalarProp1", "bla")
+ assert rec.get_property("testScalarProp1") is None
+
+ # don't remove if `remove_if_empty_afterwards=False`
+ rec.remove_value_from_property("testListProp2", "b")
+ rec.remove_value_from_property("testListProp2", "a", remove_if_empty_afterwards=False)
+ assert rec.get_property("testListProp2") is not None
+ assert rec.get_property("testListProp2").value is None
+ assert rec.get_property("testListProp2").datatype == db.LIST(db.TEXT)
+
+ rec.remove_value_from_property("testScalarProp2", False, remove_if_empty_afterwards=False)
+ assert rec.get_property("testScalarProp2") is not None
+ assert rec.get_property("testScalarProp2").value is None
+ assert rec.get_property("testScalarProp2").datatype == db.BOOLEAN
+
+ # Special case of an already empty property: It is not empty because a value
+ # was removed by `remove_value_from_property` but never had a value in the
+ # first place. So even `remove_if_empty_afterwards=True` should not lead to
+ # its removal.
+ rec.remove_value_from_property("testEmptyProp", 1234, remove_if_empty_afterwards=True)
+ assert rec.get_property("testEmptyProp") is not None
+ assert rec.get_property("testEmptyProp").value is None
+ assert rec.get_property("testEmptyProp").datatype == db.REFERENCE
+
+ # Corner case of corner case: remove with `value=None` and
+ # `remove_if_empty_afterwards=True` keeps the empty property.
+ rec.remove_value_from_property("testEmptyProp", None, remove_if_empty_afterwards=True)
+ assert rec.get_property("testEmptyProp") is not None
+ assert rec.get_property("testEmptyProp").value is None
+ assert rec.get_property("testEmptyProp").datatype == db.REFERENCE
+
+ # Remove `None` from list `[None, None]`
+ rec.remove_value_from_property("testNoneListProp", None, remove_if_empty_afterwards=True)
+ assert rec.get_property("testNoneListProp") is not None
+ assert rec.get_property("testNoneListProp").value == [None]
+ assert rec.get_property("testNoneListProp").datatype == db.LIST(db.REFERENCE)
diff --git a/unittests/test_schema.py b/unittests/test_schema.py
index 387518c9a30aac45facc24d77ea8c8532a4a8b16..fc3f63a4cbaeadcac3c1cb9be2d861a0688fe4b0 100644
--- a/unittests/test_schema.py
+++ b/unittests/test_schema.py
@@ -1,7 +1,28 @@
-#!/bin/python
-# Test configuration schema
-# A. Schlemmer, 01/2021
+# -*- coding: utf-8 -*-
+#
+# This file is a part of the CaosDB Project.
+#
+# Copyright (C) 2022 Indiscale GmbH <info@indiscale.com>
+# Copyright (C) 2021 Alexander Schlemmer
+# Copyright (C) 2022 Daniel Hornung <d.hornung@indiscale.com>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see <https://www.gnu.org/licenses/>.
+#
+"""Test configuration schema.
+A. Schlemmer, 01/2021
+"""
from jsonschema.exceptions import ValidationError
from pytest import raises
from glob import glob
@@ -12,15 +33,18 @@ from configparser import ConfigParser
def test_config_files():
for fn in glob(os.path.join(os.path.dirname(__file__), "test_configs", "*.ini")):
+ print(f"Testing {fn}.")
c = ConfigParser()
c.read(fn)
+ print(config_to_yaml(c))
validate_yaml_schema(config_to_yaml(c))
def test_broken_config_files():
for fn in glob(os.path.join(os.path.dirname(__file__), "broken_configs", "*.ini")):
- print(fn)
+ print(f"Testing {fn}.")
with raises(ValidationError):
c = ConfigParser()
c.read(fn)
+ print(config_to_yaml(c))
validate_yaml_schema(config_to_yaml(c))
diff --git a/unittests/test_server_side_scripting.py b/unittests/test_server_side_scripting.py
index 1fb24d7e40bb843391a971c5f69680b541e1de0e..b699c4482d02972282167eb9683a956097ebc5e9 100644
--- a/unittests/test_server_side_scripting.py
+++ b/unittests/test_server_side_scripting.py
@@ -46,7 +46,7 @@ def setup_module():
content_type = kwargs["headers"]["Content-Type"]
if content_type.startswith("multipart/form-data; boundary"):
- parts = kwargs["body"]
+ parts = kwargs["body"].multipart_yielder
stdout = []
for part in parts:
if hasattr(part, "decode"):
diff --git a/unittests/test_yamlapi.py b/unittests/test_yamlapi.py
index 4af8a53b1172be0ddac71444724dc4b69119d998..cdb1e0499890ee58d10ff7f102632e104ef60868 100644
--- a/unittests/test_yamlapi.py
+++ b/unittests/test_yamlapi.py
@@ -1,6 +1,5 @@
# -*- coding: utf-8 -*-
#
-# ** header v3.0
# This file is a part of the CaosDB Project.
#
# Copyright (C) 2021 IndiScale GmbH <info@indiscale.com>
@@ -19,8 +18,6 @@
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <https://www.gnu.org/licenses/>.
#
-# ** end header
-#
import os
import warnings