diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 9f8c131968c050fb18001b5dc7c5468d0ed26dae..1dc09269a92c486e9d80a8ae5ceb0e51dc50bd17 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -39,25 +39,28 @@ stages:
code_style:
tags: [ docker ]
stage: code_style
+ needs: [ ]
script:
- - pycodestyle --count ./
+ - make style
allow_failure: true
+# pylint tests for pycaosdb
pylint:
tags: [ docker ]
stage: linting
+ needs: [ ]
script:
- - pylint3 --unsafe-load-any-extension=y -d all -e E,F src/caosdb/common
+ - make lint
allow_failure: true
-
-# pylint tests for pycaosdb
-test:
+# run unit tests
+unittest:
tags: [ docker ]
stage: test
+ needs: [ ]
script:
- touch ~/.pycaosdb.ini
- - tox -r
+ - make unittest
# Trigger building of server image and integration tests
trigger_build:
@@ -96,6 +99,7 @@ build-testenv:
pages_prepare: &pages_prepare
tags: [ cached-dind ]
stage: deploy
+ needs: [ code_style, pylint, unittest ]
only:
refs:
- /^release-.*$/i
diff --git a/.gitlab/merge_request_templates/Default.md b/.gitlab/merge_request_templates/Default.md
deleted file mode 100644
index 77a95da1cc40c815e4952a1283d345af56e80461..0000000000000000000000000000000000000000
--- a/.gitlab/merge_request_templates/Default.md
+++ /dev/null
@@ -1,49 +0,0 @@
-# Summary
-
- Insert a meaningful description for this merge request here. What is the
- new/changed behavior? Which bug has been fixed? Are there related Issues?
-
-# Focus
-
- Point the reviewer to the core of the code change. Where should they start
- reading? What should they focus on (e.g. security, performance,
- maintainability, user-friendliness, compliance with the specs, finding more
- corner cases, concrete questions)?
-
-# Test Environment
-
- How to set up a test environment for manual testing?
-
-# Check List for the Author
-
-Please, prepare your MR for a review. Be sure to write a summary and a
-focus and create gitlab comments for the reviewer. They should guide the
-reviewer through the changes, explain your changes and also point out open
-questions. For further good practices have a look at [our review
-guidelines](https://gitlab.com/caosdb/caosdb/-/blob/dev/REVIEW_GUIDELINES.md)
-
-- [ ] All automated tests pass
-- [ ] Reference related Issues
-- [ ] Up-to-date CHANGELOG.md
-- [ ] Annotations in code (Gitlab comments)
- - Intent of new code
- - Problems with old code
- - Why this implementation?
-
-
-# Check List for the Reviewer
-
-
-- [ ] I understand the intent of this MR
-- [ ] All automated tests pass
-- [ ] Up-to-date CHANGELOG.md
-- [ ] The test environment setup works and the intended behavior is
- reproducible in the test environment
-- [ ] In-code documentation and comments are up-to-date.
-- [ ] Check: Are there spezifications? Are they satisfied?
-
-For further good practices have a look at [our review guidelines](https://gitlab.com/caosdb/caosdb/-/blob/dev/REVIEW_GUIDELINES.md).
-
-
-/assign me
-/target_branch dev
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 38b470db9d36675ffcef0b7e1434a08c3be7f407..57a664e7e3d2986d56e44ce6c57234502105b87e 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -17,7 +17,80 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
### Fixed ###
+### Security ###
+
+### Documentation ###
+
+## [0.7.1] - 2022-03-11 ##
+
+### Documentation ###
+
+- `timeout` option in example pycaosdb.ini
+
+## [0.7.0] - 2022-01-21 ##
+
+### Added ###
+
+- Function in administration that generates passwords that comply with the rules.
+
+### Fixed ###
+
+- #90 compare_entities function in apiutils does not check units
+- #94 some special properties were not checked in compare_entities
+
+### Security ###
+
+## [0.6.1] - 2021-12-03 ##
+
+### Fixed ###
+
+- #50 keyring can be used as password input method again
+* #81 compare_entities from apiutils does not compare entity values
+
+## [0.6.0] - 2021-10-19 ##
+
+### Added ###
+
+- It is possible now to supply a password for caosdb_admin on the command line
+ and also activate the user directly using "-c".
+* Added examples for complex data models to documentation
+* extended apiutils with `resolve_reference(Property)`
+* is_reference function for Properties
+* function `retrieve_substructure` that recursively adds connected entities.
+
+### Changed ###
+
+* Retrievals of entities where the class does not match the entity role raise
+ a ValueError now. See
+ [here](https://gitlab.indiscale.com/caosdb/src/caosdb-pylib/-/issues/66) for more
+ information. Updating a role is now being done by setting the `Entity.role`
+ to the new role (as string).
+* Entity.add_property and Entity.add_parent do not accept `**kwargs`-style
+ keywords anymore. Formerly known keywords have been refactored into named
+ parameters.
+* [#35](https://gitlab.com/caosdb/caosdb-pylib/-/issues/35) Loggers now use the
+ name of the unit where they are called instead of a static name
+
+### Deprecated ###
+
+* `id_query(ids)` in apiutils (to be removed with >=0.5.4)
+* The whole yamlapi with the following functions (to be removed with >=0.5.4):
+ * `append_sublist`
+ * `kv_to_xml`
+ * `dict_to_xml`
+ * `yaml_to_xml`
+ * `process`
+ * `yaml_file_to_xml`
+ * `insert_yaml_file`
+
+### Removed ###
+
+### Fixed ###
+
+* #60 Unintuitive behavior of `Entity.role` after a `Entity(id).retrieve()`
+ Originally the role was `None`. The correct role is present now.
* #53 Documentation of inheritance
+* #38 Dependencies in chunk-deletion of containers
### Security ###
diff --git a/Makefile b/Makefile
index 4bc3459d209936c17a445c64e77180d9559e4653..0a0888ad0484c0307583e139e65058c38574ed3a 100644
--- a/Makefile
+++ b/Makefile
@@ -31,3 +31,18 @@ doc:
install:
@echo "Not implemented yet, use pip for installation."
+
+check: style lint
+.PHONY: check
+
+style:
+ pycodestyle --count examples src unittests
+.PHONY: style
+
+lint:
+ pylint --unsafe-load-any-extension=y -d all -e E,F src/caosdb/common
+.PHONY: lint
+
+unittest:
+ tox -r
+.PHONY: unittest
diff --git a/README_SETUP.md b/README_SETUP.md
index 9da548395073643c16539cef180c4d6412dd8d46..e58f934ceba176e4b5ba42239565f8e3bd48171a 100644
--- a/README_SETUP.md
+++ b/README_SETUP.md
@@ -11,6 +11,10 @@ typically be installed automatically):
- `PyYaml`
- `PySocks`
+Optional packages:
+- `keyring`
+- `jsonschema`
+
### How to install ###
#### Linux ####
@@ -30,12 +34,14 @@ packages you will ever need out of the box. If you prefer, you may also install
After installation, open an Anaconda prompt from the Windows menu and continue in the [Generic
installation](#generic-installation) section.
-#### iOS ####
+#### MacOS ####
-If there is no Python 3 installed yet, there are two main ways to obtain it: Either get the binary
-package from [python.org](https://www.python.org/downloads/) or, for advanced users, install via [Homebrew](https://brew.sh/). After installation
-from python.org, it is recommended to also update the TLS certificates for Python (this requires
-administrator rights for your user):
+If there is no Python 3 installed yet, there are two main ways to
+obtain it: Either get the binary package from
+[python.org](https://www.python.org/downloads/) or, for advanced
+users, install via [Homebrew](https://brew.sh/). After installation
+from python.org, it is recommended to also update the TLS certificates
+for Python (this requires administrator rights for your user):
```sh
# Replace this with your Python version number:
@@ -45,7 +51,8 @@ cd /Applications/Python\ 3.9/
sudo ./Install\ Certificates.command
```
-After these steps, you may continue with the [Generic installation](#generic-installation).
+After these steps, you may continue with the [Generic
+installation](#generic-installation).
#### Generic installation ####
@@ -66,6 +73,13 @@ cd caosdb-pylib
pip3 install --user .
```
+For installation of optional packages, install with an additional option, e.g. for
+validating with the caosdb json schema:
+
+```sh
+pip3 install --user .[jsonschema]
+```
+
## Configuration ##
The configuration is done using `ini` configuration files.
diff --git a/RELEASE_GUIDELINES.md b/RELEASE_GUIDELINES.md
index e015b598117abdcd575cf17e2f095fec459a4c4c..e71234b8e2bc95f954ffbebdc26acf6edd8e0b2d 100644
--- a/RELEASE_GUIDELINES.md
+++ b/RELEASE_GUIDELINES.md
@@ -1,6 +1,6 @@
# Release Guidelines for the CaosDB Python Client Library
-This document specifies release guidelines in addition to the generel release
+This document specifies release guidelines in addition to the general release
guidelines of the CaosDB Project
([RELEASE_GUIDELINES.md](https://gitlab.com/caosdb/caosdb/blob/dev/RELEASE_GUIDELINES.md))
@@ -20,9 +20,10 @@ guidelines of the CaosDB Project
3. Check all general prerequisites.
-4. Prepare [setup.py](./setup.py): Check the `MAJOR`, `MINOR`, `MICRO`, `PRE`
- variables and set `ISRELEASED` to `True`. Use the possibility to issue
- pre-release versions for testing.
+4. Update the version:
+ - `version` variables in `src/doc/conf.py`
+ - Version on [setup.py](./setup.py): Check the `MAJOR`, `MINOR`, `MICRO`, `PRE` variables and set
+ `ISRELEASED` to `True`. Use the possibility to issue pre-release versions for testing.
5. Merge the release branch into the main branch.
@@ -38,5 +39,5 @@ guidelines of the CaosDB Project
10. Merge the main branch back into the dev branch.
11. After the merge of main to dev, start a new development version by
- setting `ISRELEASED` to `False` and by increasing at least the `MIRCO`
- version in [setup.py](./setup.py)
+ setting `ISRELEASED` to `False` and by increasing at least the `MICRO`
+ version in [setup.py](./setup.py) and preparing CHANGELOG.md.
diff --git a/examples/pycaosdb.ini b/examples/pycaosdb.ini
index edc32195fbb364bb355d67b8733e8c7bccbb0d34..8cf74e43c5db32ed139c4fe371a6c2b3831b2ee1 100644
--- a/examples/pycaosdb.ini
+++ b/examples/pycaosdb.ini
@@ -67,3 +67,6 @@
# This option is used internally and for testing. Do not override.
# implementation=_DefaultCaosDBServerConnection
+
+# The timeout for requests to the server.
+# timeout=1000
diff --git a/setup.py b/setup.py
index e1d39458ea8d1b0b17ea12a82ebd7133b27b045a..cdb10b6b41b5c466187a4bc63a77f41bd04ec454 100755
--- a/setup.py
+++ b/setup.py
@@ -45,11 +45,11 @@ from setuptools import find_packages, setup
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
########################################################################
-MAJOR = 0
-MINOR = 5
-MICRO = 3
-PRE = "" # e.g. rc0, alpha.1, 0.beta-23
ISRELEASED = False
+MAJOR = 0
+MINOR = 7
+MICRO = 2
+PRE = "dev" # e.g. rc0, alpha.1, 0.beta-23
if PRE:
VERSION = "{}.{}.{}-{}".format(MAJOR, MINOR, MICRO, PRE)
@@ -154,16 +154,26 @@ def setup_package():
long_description_content_type="text/markdown",
author='Timm Fitschen',
author_email='t.fitschen@indiscale.com',
+ url='https://www.caosdb.org',
+ license="AGPLv3+",
+ classifiers=[
+ "Programming Language :: Python :: 3",
+ "License :: OSI Approved :: GNU Affero General Public License v3 or later (AGPLv3+)",
+ "Operating System :: OS Independent",
+ "Topic :: Database",
+ "Topic :: Scientific/Engineering :: Information Analysis",
+ ],
packages=find_packages('src'),
python_requires='>=3.6',
package_dir={'': 'src'},
install_requires=['lxml>=3.6.4',
'PyYaml>=3.12', 'future', 'PySocks>=1.6.7'],
- extras_require={'keyring': ['keyring>=13.0.0']},
+ extras_require={'keyring': ['keyring>=13.0.0'],
+ 'jsonschema': ['jsonschema==4.0.1']},
setup_requires=["pytest-runner>=2.0,<3dev"],
- tests_require=["pytest", "pytest-cov", "coverage>=4.4.2"],
+ tests_require=["pytest", "pytest-cov", "coverage>=4.4.2", "jsonschema==4.0.1"],
package_data={
- 'caosdb': ['cert/indiscale.ca.crt'],
+ 'caosdb': ['cert/indiscale.ca.crt', 'schema-pycaosdb-ini.yml'],
},
scripts=["src/caosdb/utils/caosdb_admin.py"]
)
diff --git a/src/caosdb/apiutils.py b/src/caosdb/apiutils.py
index 73074efc3057e0548c5abfd56ef3cf1ac9e9bf47..dc9209b58c8163da552f29e7a4435a0c640b1ecf 100644
--- a/src/caosdb/apiutils.py
+++ b/src/caosdb/apiutils.py
@@ -1,12 +1,11 @@
# -*- coding: utf-8 -*-
#
-# ** header v3.0
# This file is a part of the CaosDB Project.
#
# Copyright (C) 2018 Research Group Biomedical Physics,
# Max-Planck-Institute for Dynamics and Self-Organization Göttingen
# Copyright (C) 2020 Timm Fitschen <t.fitschen@indiscale.com>
-# Copyright (C) 2020 IndiScale GmbH <info@indiscale.com>
+# Copyright (C) 2020-2022 IndiScale GmbH <info@indiscale.com>
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as
@@ -30,14 +29,15 @@ Some simplified functions for generation of records etc.
import sys
import tempfile
+import warnings
from collections.abc import Iterable
from subprocess import call
from caosdb.common.datatype import (BOOLEAN, DATETIME, DOUBLE, FILE, INTEGER,
REFERENCE, TEXT, is_reference)
from caosdb.common.models import (Container, Entity, File, Property, Query,
- Record, RecordType, get_config,
- execute_query)
+ Record, RecordType, execute_query,
+ get_config)
def new_record(record_type, name=None, description=None,
@@ -86,9 +86,19 @@ def new_record(record_type, name=None, description=None,
def id_query(ids):
- q = "FIND Entity with " + " OR ".join(["id={}".format(id) for id in ids])
+ warnings.warn("Please use 'create_id_query', which only creates"
+ "the string.", DeprecationWarning)
- return execute_query(q)
+ return execute_query(create_id_query(ids))
+
+
+def create_id_query(ids):
+ return "FIND ENTITY WITH " + " OR ".join(
+ ["ID={}".format(id) for id in ids])
+
+
+def retrieve_entity_with_id(eid):
+ return execute_query("FIND ENTITY WITH ID={}".format(eid), unique=True)
def retrieve_entities_with_ids(entities):
@@ -96,7 +106,9 @@ def retrieve_entities_with_ids(entities):
step = 20
for i in range(len(entities)//step+1):
- collection.extend(id_query(entities[i*step:(i+1)*step]))
+ collection.extend(
+ execute_query(
+ create_id_query(entities[i*step:(i+1)*step])))
return collection
@@ -129,6 +141,9 @@ class CaosDBPythonEntity(object):
_last_id = 0
def __init__(self):
+ warnings.warn("The CaosDBPythonEntity class is deprecated, replacements will be provided by"
+ " the high_level_api module.",
+ DeprecationWarning, stacklevel=2)
# Save a copy of the dry state
# of this object in order to be
# able to detect conflicts.
@@ -457,6 +472,9 @@ def _single_convert_to_entity(entity, robj, **kwargs):
def convert_to_entity(python_object, **kwargs):
+ warnings.warn("The convert_to_entity function is deprecated, replacement will be provided by "
+ "the high_level_api module.", DeprecationWarning, stacklevel=2)
+
if isinstance(python_object, Container):
# Create a list of objects:
@@ -478,6 +496,8 @@ def convert_to_entity(python_object, **kwargs):
def convert_to_python_object(entity):
""""""
+ warnings.warn("The convert_to_python_object function is deprecated, replacement will be "
+ "provided by the high_level_api module.", DeprecationWarning, stacklevel=2)
if isinstance(entity, Container):
# Create a list of objects:
@@ -504,9 +524,9 @@ def getOriginUrlIn(folder):
with open(t.name, "r") as t:
urlString = "Fetch URL:"
- for l in t.readlines():
- if urlString in l:
- return l[l.find(urlString) + len(urlString):].strip()
+ for line in t.readlines():
+ if urlString in line:
+ return line[line.find(urlString) + len(urlString):].strip()
return None
@@ -545,10 +565,27 @@ def getCommitIn(folder):
return t.readline().strip()
-COMPARED = ["name", "role", "datatype", "description", "importance"]
+COMPARED = ["name", "role", "datatype", "description", "importance",
+ "id", "path", "checksum", "size"]
-def compare_entities(old_entity, new_entity):
+def compare_entities(old_entity: Entity, new_entity: Entity):
+ """
+ Compare two entites.
+
+ Return a tuple of dictionaries, the first index belongs to additional information for old
+ entity, the second index belongs to additional information for new entity.
+
+ Additional information means in detail:
+ - Additional parents (a list under key "parents")
+ - Information about properties:
+ - Each property lists either an additional property or a property with a changed:
+ - ... datatype
+ - ... importance or
+ - ... value (not implemented yet)
+ In case of changed information the value listed under the respective key shows the
+ value that is stored in the respective entity.
+ """
olddiff = {"properties": {}, "parents": []}
newdiff = {"properties": {}, "parents": []}
@@ -603,20 +640,29 @@ def compare_entities(old_entity, new_entity):
newdiff["properties"][prop.name]["importance"] = \
new_entity.get_importance(prop.name)
- if ((prop.datatype is not None and
- matching[0].datatype is not None) and
- (prop.datatype != matching[0].datatype)):
+ if (prop.datatype != matching[0].datatype):
olddiff["properties"][prop.name]["datatype"] = prop.datatype
newdiff["properties"][prop.name]["datatype"] = \
matching[0].datatype
+ if (prop.unit != matching[0].unit):
+ olddiff["properties"][prop.name]["unit"] = prop.unit
+ newdiff["properties"][prop.name]["unit"] = \
+ matching[0].unit
+
+ if (prop.value != matching[0].value):
+ olddiff["properties"][prop.name]["value"] = prop.value
+ newdiff["properties"][prop.name]["value"] = \
+ matching[0].value
+
if (len(newdiff["properties"][prop.name]) == 0
and len(olddiff["properties"][prop.name]) == 0):
newdiff["properties"].pop(prop.name)
olddiff["properties"].pop(prop.name)
else:
- raise NotImplementedError()
+ raise NotImplementedError(
+ "Comparison not implemented for multi-properties.")
for prop in new_entity.properties:
if len([0 for p in old_entity.properties if p.name == prop.name]) == 0:
@@ -687,6 +733,7 @@ def apply_to_ids(entities, func):
entities : list of Entity
func : function with one parameter.
"""
+
for entity in entities:
_apply_to_ids_of_entity(entity, func)
@@ -707,3 +754,28 @@ def _apply_to_ids_of_entity(entity, func):
else:
if prop.value is not None:
prop.value = func(prop.value)
+
+
+def resolve_reference(prop: Property):
+ """resolves the value of a reference property
+
+ The integer value is replaced with the entity object.
+ If the property is not a reference, then the function returns without
+ change.
+ """
+
+ if not prop.is_reference(server_retrieval=True):
+ return
+
+ if isinstance(prop.value, list):
+ referenced = []
+
+ for val in prop.value:
+ if isinstance(val, int):
+ referenced.append(retrieve_entity_with_id(val))
+ else:
+ referenced.append(val)
+ prop.value = referenced
+ else:
+ if isinstance(prop.value, int):
+ prop.value = retrieve_entity_with_id(prop.value)
diff --git a/src/caosdb/common/administration.py b/src/caosdb/common/administration.py
index e7ba94182d7a4d8b60c6400cd1d804f62f7bf03c..cef0bd1cf6fceb9d8ec89324ba9ca540b79889cb 100644
--- a/src/caosdb/common/administration.py
+++ b/src/caosdb/common/administration.py
@@ -26,16 +26,15 @@
"""missing docstring."""
-from lxml import etree
-
+import re
+import string
+import random
from caosdb.common.utils import xml2str
from caosdb.connection.connection import get_connection
-from caosdb.exceptions import (HTTPClientError,
- HTTPForbiddenError,
- HTTPResourceNotFoundError,
- EntityDoesNotExistError,
- ServerConfigurationException,
- )
+from caosdb.exceptions import (EntityDoesNotExistError, HTTPClientError,
+ HTTPForbiddenError, HTTPResourceNotFoundError,
+ ServerConfigurationException)
+from lxml import etree
def set_server_property(key, value):
@@ -60,7 +59,8 @@ def set_server_property(key, value):
con._form_data_request(method="POST", path="_server_properties",
params={key: value}).read()
except EntityDoesNotExistError:
- raise ServerConfigurationException("Debug mode in server is probably disabled.") from None
+ raise ServerConfigurationException(
+ "Debug mode in server is probably disabled.") from None
def get_server_properties():
@@ -75,9 +75,11 @@ def get_server_properties():
"""
con = get_connection()
try:
- body = con._http_request(method="GET", path="_server_properties").response
+ body = con._http_request(
+ method="GET", path="_server_properties").response
except EntityDoesNotExistError:
- raise ServerConfigurationException("Debug mode in server is probably disabled.") from None
+ raise ServerConfigurationException(
+ "Debug mode in server is probably disabled.") from None
xml = etree.parse(body)
props = dict()
@@ -112,6 +114,39 @@ def get_server_property(key):
return get_server_properties()[key]
+def generate_password(length: int):
+ """Create a random password that fulfills the security requirements
+
+ Parameters
+ ----------
+ length : int
+ Length of the generated password. Has to be greater than 7.
+
+ Returns
+ -------
+ password : string
+ Generated random password of the given length
+
+ Raises
+ ------
+ ValueError:
+ If the length is less than 8.
+ """
+ minimum_password_length = 8
+ if length < minimum_password_length:
+ raise ValueError("CaosDB passwords have to be at least {} characters.".format(
+ minimum_password_length))
+ sample_letters = string.ascii_letters + string.digits + "!#$%*+-/:;?_"
+ password = ''.join((random.choice(sample_letters) for i in range(length)))
+
+ while not re.match(r"(?=.*[A-Z])(?=.*[a-z])(?=.*\d)(?=.*[\W_]).{8,}",
+ password):
+ password = ''.join((random.choice(sample_letters)
+ for i in range(length)))
+
+ return password
+
+
def _retrieve_user(name, realm=None, **kwargs):
con = get_connection()
try:
@@ -156,13 +191,22 @@ def _update_user(name, realm=None, password=None, status=None,
return con.put_form_data(entity_uri_segment="User/" + (realm + "/" + name if realm is not None else name), params=params, **kwargs).read()
except HTTPResourceNotFoundError as e:
e.msg = "User does not exist."
- raise
+ raise e
except HTTPForbiddenError as e:
e.msg = "You are not permitted to update this user."
- raise
+ raise e
except HTTPClientError as e:
if e.status == 409:
e.msg = "Entity does not exist."
+
+ if e.status == 422:
+ e.msg = """Maybe the password does not match the required standard?
+ The current requirements are:
+ - at least 8 characters
+ - at least 1 number
+ - at least 1 lower case character
+ - at least 1 upper case character
+ - at least 1 special character"""
raise
@@ -191,7 +235,13 @@ def _insert_user(name, password=None, status=None, email=None, entity=None, **kw
e.msg = "User name is already in use."
if e.status == 422:
- e.msg = "Maybe the password does not match the required standard?"
+ e.msg = """Maybe the password does not match the required standard?
+ The current requirements are:
+ - at least 8 characters
+ - at least 1 number
+ - at least 1 lower case character
+ - at least 1 upper case character
+ - at least 1 special character"""
raise e
diff --git a/src/caosdb/common/datatype.py b/src/caosdb/common/datatype.py
index eb8c1e4e0088f1924940a104ec3916b9d5d40f99..03ff6d023ab0d3005c37d56c65353c1a1072518e 100644
--- a/src/caosdb/common/datatype.py
+++ b/src/caosdb/common/datatype.py
@@ -45,6 +45,8 @@ def LIST(datatype):
def get_list_datatype(datatype):
""" returns the datatype of the elements in the list """
+ if not isinstance(datatype, str):
+ return None
match = re.match("LIST(<|<)(?P<datatype>.*)(>|>)", datatype)
if match is not None:
@@ -60,12 +62,27 @@ def is_list_datatype(datatype):
def is_reference(datatype):
- """ returns whether the value is a reference
+ """Returns whether the value is a reference
FILE and REFERENCE properties are examples, but also datatypes that are
- RecordTypes
+ RecordTypes.
+
+ Parameters
+ ----------
+ datatype : str
+ The datatype to check.
+
+ Returns
+ -------
+ bool
+ True if the datatype is a not base datatype or a list of a base datatype.
+ Otherwise False is returned.
"""
+ if datatype is None:
+ raise ValueError("Cannot decide whether datatype is reference if None"
+ " is supplied")
+
if datatype in [DOUBLE, BOOLEAN, INTEGER, TEXT, DATETIME]:
return False
elif is_list_datatype(datatype):
@@ -74,6 +91,41 @@ def is_reference(datatype):
return True
+def get_referenced_recordtype(datatype):
+ """Return the record type of the referenced datatype.
+
+ Raises
+ ------
+ ValueError
+ In cases where datatype is not a reference, the list does not have
+ a referenced record type or the datatype is a FILE.
+
+ Parameters
+ ----------
+ datatype : str
+ The datatype to check.
+
+ Returns
+ -------
+ str
+ String containing the name of the referenced datatype.
+ """
+
+ if not is_reference(datatype):
+ raise ValueError("datatype must be a reference")
+
+ if is_list_datatype(datatype):
+ datatype = get_list_datatype(datatype)
+ if datatype is None:
+ raise ValueError("list does not have a list datatype")
+
+ if datatype == FILE:
+ raise ValueError(
+ "FILE references are not considered references with a record type")
+
+ return datatype
+
+
def get_id_of_datatype(datatype):
""" returns the id of a Record Type
diff --git a/src/caosdb/common/models.py b/src/caosdb/common/models.py
index 6ec49df2722170805fb6230753f36503870a8821..80a6ee11e707fb3776fc96b42a16b649ac575f66 100644
--- a/src/caosdb/common/models.py
+++ b/src/caosdb/common/models.py
@@ -31,6 +31,7 @@ from __future__ import print_function, unicode_literals
import re
import sys
from builtins import str
+from copy import deepcopy
from functools import cmp_to_key
from hashlib import sha512
from os import listdir
@@ -40,26 +41,24 @@ from sys import hexversion
from tempfile import NamedTemporaryFile
from warnings import warn
-from caosdb.common.datatype import (BOOLEAN, DATETIME, DOUBLE, INTEGER, TEXT)
-from caosdb.common.versioning import Version
+from caosdb.common.datatype import (BOOLEAN, DATETIME, DOUBLE, INTEGER, TEXT,
+ is_list_datatype, is_reference)
from caosdb.common.state import State
from caosdb.common.utils import uuid, xml2str
+from caosdb.common.versioning import Version
from caosdb.configuration import get_config
from caosdb.connection.connection import get_connection
from caosdb.connection.encode import MultipartParam, multipart_encode
-from caosdb.exceptions import (AmbiguousEntityError,
- AuthorizationError,
- CaosDBException, CaosDBConnectionError,
- ConsistencyError,
- EmptyUniqueQueryError,
+from caosdb.exceptions import (AmbiguousEntityError, AuthorizationError,
+ CaosDBConnectionError, CaosDBException,
+ ConsistencyError, EmptyUniqueQueryError,
EntityDoesNotExistError, EntityError,
- EntityHasNoDatatypeError,
- MismatchingEntitiesError,
- QueryNotUniqueError, TransactionError,
- UniqueNamesError,
+ EntityHasNoDatatypeError, HTTPURITooLongError,
+ MismatchingEntitiesError, QueryNotUniqueError,
+ ServerConfigurationException,
+ TransactionError, UniqueNamesError,
UnqualifiedParentsError,
- UnqualifiedPropertiesError,
- HTTPURITooLongError)
+ UnqualifiedPropertiesError)
from lxml import etree
_ENTITY_URI_SEGMENT = "Entity"
@@ -132,7 +131,10 @@ class Entity(object):
@role.setter
def role(self, role):
- self.__role = role
+ if role is not None and role.lower() == "entity":
+ self.__role = None
+ else:
+ self.__role = role
@property
def size(self):
@@ -341,54 +343,80 @@ class Entity(object):
return self
- def add_property(self, property=None, value=None, **kwargs): # @ReservedAssignment
+ def add_property(self, property=None, value=None, id=None, name=None, description=None, datatype=None,
+ unit=None, importance=None, inheritance=None): # @ReservedAssignment
"""Add a property to this entity.
The first parameter is meant to identify the property entity. So the method expects an instance of
Entity, an integer or a string here. The second parameter is the value of the new property. Any
- other named parameter may be passed by means of the **kwargs. Accepted keywords are:
+ other named parameter may be passed by means of the keywwords. Accepted keywords are:
id, name, description, importance, inheritance, datatype, and unit. Any other keyword will be
ignored right now. But that may change in the future.
- @param property: An identifying parameter (name, id or abstract property).
- @param value: The value of the new property.
- @param **kwargs: Any other specification for this property. Accepted keywords: id, name, description, importance, inheritance, datatype, and unit.
- @raise UserWarning:
- If the first parameter is None then kwargs['id'] or kwargs['name'] must be defined and not be None.
- Otherwise a UserWarning is raised.
+ Parameters
+ ----------
+ property : int, str, Property, optional
+ An identifying parameter, by default None
+ value : int, str, Property, optional
+ The value of the new property, by default None
+ id : int, optional
+ Id of the property, by default None
+ name : str, optional
+ Name of the property, by default None
+ description : str, optional
+ Description of the property, by default None
+ datatype : str, optional
+ Datatype of the property, by default None
+ unit : str, optional
+ Unit of the property, by default None
+ importance :str, optional
+ Importance of the property, by default None
+ inheritance : str, optional
+ Inheritance of the property, by default None
- If the first parameter is an integer then it is interpreted as the id and kwargs['id'] must be
- undefined or None. Otherwise a UserWarning is raised.
+ Returns
+ -------
+ Entity
- If the first parameter is not None and neither an instance of Entity nor an integer it is
- interpreted as the name and kwargs['name'] must be undefined or None. Otherwise a UserWarning is
- raised.
+ Raises
+ ------
+ UserWarning
+ If the first parameter is None then id or name must be defined and not be None.
+ UserWarning
+ If the first parameter is an integer then it is interpreted as the id and id must be
+ undefined or None.
+ UserWarning
+ If the first parameter is not None and neither an instance of Entity nor an integer it is
+ interpreted as the name and name must be undefined or None.
"""
- copy_kwargs = kwargs.copy()
- name = (kwargs['name'] if 'name' in kwargs else None)
- pid = (kwargs['id'] if 'id' in kwargs else None)
+
+ pid = id
abstract_property = None
if isinstance(property, Entity):
+ if property.role is not None and property.role.lower() in ["record", "file"]:
+ raise ValueError("The property parameter is a {0}. This "
+ "is very unusual and probably not what you "
+ "want. Otherwise, construct a property from "
+ "a {0} using the Property class and add "
+ "that to this entity.".format(property.role))
abstract_property = property
elif isinstance(property, int):
if pid is not None:
raise UserWarning("The first parameter was an integer which would normally be interpreted as the id of the property which is to be added. But you have also specified a parameter 'id' in the method call. This is ambiguous and cannot be processed.")
pid = property
- copy_kwargs['id'] = pid
+ id = pid
elif property is not None:
if name is not None:
raise UserWarning("The first parameter was neither an instance of Entity nor an integer. Therefore the string representation of your first parameter would normally be interpreted name of the property which is to be added. But you have also specified a parameter 'name' in the method call. This is ambiguous and cannot be processed.")
name = str(property)
- copy_kwargs['name'] = name
if property is None and name is None and pid is None:
raise UserWarning(
"This method expects you to pass at least an entity, a name or an id.")
- del copy_kwargs['importance']
- del copy_kwargs['inheritance']
- new_property = Property(**copy_kwargs)
+ new_property = Property(name=name, id=id, description=description, datatype=datatype,
+ value=value, unit=unit)
if abstract_property is not None:
new_property._wrap(property)
@@ -401,9 +429,7 @@ class Entity(object):
new_property.value = value
self.properties.append(
- property=new_property, importance=(
- kwargs['importance'] if 'importance' in kwargs else None), inheritance=(
- kwargs['inheritance'] if 'inheritance' in kwargs else None))
+ property=new_property, importance=importance, inheritance=inheritance)
return self
@@ -428,7 +454,7 @@ class Entity(object):
return self
- def add_parent(self, parent=None, **kwargs): # @ReservedAssignment
+ def add_parent(self, parent=None, id=None, name=None, inheritance=None): # @ReservedAssignment
"""Add a parent to this entity.
Parameters
@@ -436,27 +462,23 @@ class Entity(object):
parent : Entity or int or str or None
The parent entity, either specified by the Entity object
itself, or its id or its name. Default is None.
- **kwargs : dict, optional
- Additional keyword arguments for specifying the parent by
- name or id, and for specifying the mode of inheritance.
-
- id : int
- Integer id of the parent entity. Ignored if `parent`
- is not None.
- name : str
- Name of the parent entity. Ignored if `parent is not
- none`.
- inheritance : str
- One of ``obligatory``, ``recommended``, ``suggested``, or ``fix``. Specifies the
- minimum importance which parent properties need to have to be inherited by this
- entity. If no `inheritance` is given, no properties will be inherited by the child.
- This parameter is case-insensitive.
-
- Note that the behaviour is currently not yet specified when assigning parents to
- Records, it only works for inheritance of RecordTypes (and Properties).
-
- For more information, it is recommended to look into the
- :ref:`data insertion tutorial<tutorial-inheritance-properties>`.
+ id : int
+ Integer id of the parent entity. Ignored if `parent`
+ is not None.
+ name : str
+ Name of the parent entity. Ignored if `parent is not
+ none`.
+ inheritance : str
+ One of ``obligatory``, ``recommended``, ``suggested``, or ``fix``. Specifies the
+ minimum importance which parent properties need to have to be inherited by this
+ entity. If no `inheritance` is given, no properties will be inherited by the child.
+ This parameter is case-insensitive.
+
+ Note that the behaviour is currently not yet specified when assigning parents to
+ Records, it only works for inheritance of RecordTypes (and Properties).
+
+ For more information, it is recommended to look into the
+ :ref:`data insertion tutorial<tutorial-inheritance-properties>`.
Raises
------
@@ -465,8 +487,8 @@ class Entity(object):
parameter is passed to this method.
"""
- name = (kwargs['name'] if 'name' in kwargs else None)
- pid = (kwargs['id'] if 'id' in kwargs else None)
+
+ pid = id
parent_entity = None
if isinstance(parent, Entity):
@@ -480,9 +502,6 @@ class Entity(object):
raise UserWarning(
"This method expects you to pass at least an entity, a name or an id.")
- inheritance = (kwargs['inheritance']
- if 'inheritance' in kwargs else None)
-
addp = Parent(id=pid, name=name, inheritance=inheritance)
if parent_entity is not None:
@@ -580,6 +599,7 @@ class Entity(object):
entity, or name.
"""
+
if isinstance(key, int):
for p in self.parents:
if p.id is not None and int(p.id) == int(key):
@@ -588,14 +608,17 @@ class Entity(object):
if key.id is not None:
# first try by id
found = self.get_parent(int(key.id))
+
if found is not None:
return found
# otherwise by name
+
return self.get_parent(key.name)
else:
for p in self.parents:
if (p.name is not None
and str(p.name).lower() == str(key).lower()):
+
return p
return None
@@ -680,6 +703,7 @@ class Entity(object):
special_selector = None
# iterating through the entity tree according to the selector
+
for subselector in selector:
# selector does not match the structure, we cannot get a
# property of non-entity
@@ -691,11 +715,13 @@ class Entity(object):
# selector does not match the structure, we did not get a
# property
+
if prop is None:
return None
# if the property is a reference, we are interested in the
# corresponding entities attributes
+
if isinstance(prop.value, Entity):
ref = prop.value
@@ -704,6 +730,7 @@ class Entity(object):
ref = prop
# if we saved a special selector before, apply it
+
if special_selector is None:
return prop.value
else:
@@ -831,10 +858,13 @@ class Entity(object):
"""
if xml is None:
- xml = etree.Element("Entity")
+ # use role as xml tag name, fall-back to "Entity"
+ elem_tag = "Entity" if self.role is None else self.role
+ xml = etree.Element(elem_tag)
assert isinstance(xml, etree._Element)
# unwrap wrapped entity
+
if self._wrapped_entity is not None:
xml = self._wrapped_entity.to_xml(xml, add_properties)
@@ -949,6 +979,8 @@ class Entity(object):
@param elem: the xml element
"""
+ if isinstance(entity, Entity):
+ entity.role = elem.tag
entity._cuid = elem.get("cuid")
entity.id = elem.get("id") # @ReservedAssignment
entity.name = elem.get("name")
@@ -1074,6 +1106,7 @@ class Entity(object):
if len(c) == 1:
c[0].messages.extend(c.messages)
+
return c[0]
raise QueryNotUniqueError("This retrieval was not unique!!!")
@@ -1163,6 +1196,10 @@ class Entity(object):
def _parse_value(datatype, value):
+ """Parse the value (from XML input) according to the given datatype
+ """
+
+ # Simple values
if value is None:
return value
@@ -1183,12 +1220,12 @@ def _parse_value(datatype, value):
else:
raise ValueError("Boolean value was {}.".format(value))
+ # Datetime and text are returned as-is
if datatype in [DATETIME, TEXT]:
if isinstance(value, str):
return value
# deal with collections
-
if isinstance(datatype, str):
matcher = re.compile(r"^(?P<col>[^<]+)<(?P<dt>[^>]+)>$")
m = matcher.match(datatype)
@@ -1213,12 +1250,10 @@ def _parse_value(datatype, value):
# This is for a special case, where the xml parser could not differentiate
# between single values and lists with one element. As
-
if hasattr(value, "__len__") and len(value) == 1:
return _parse_value(datatype, value[0])
# deal with references
-
if isinstance(value, Entity):
return value
@@ -1234,6 +1269,12 @@ def _parse_value(datatype, value):
# reference via name
return str(value)
+ except TypeError:
+ # deal with invalid XML: List of values without appropriate datatype
+ if isinstance(value, list):
+ raise ServerConfigurationException(
+ "The server sent an invalid XML: List valued properties must be announced by "
+ "the datatype.\n" + f"Datatype: {datatype}\nvalue: {value}")
def _log_request(request, xml_body=None):
@@ -1262,6 +1303,7 @@ class QueryTemplate():
def __init__(self, id=None, name=None, query=None, description=None): # @ReservedAssignment
self.id = (int(id) if id is not None else None)
+ self.role = "QueryTemplate"
self.name = name
self.description = description
self.query = query
@@ -1445,45 +1487,42 @@ class Property(Entity):
"""CaosDB's Property object."""
- def add_property(self, property=None, value=None, **kwargs): # @ReservedAssignment
- copy_kwargs = kwargs.copy()
-
- if 'importance' not in copy_kwargs:
- # set default importance
- copy_kwargs['importance'] = FIX
-
- if 'inheritance' not in copy_kwargs:
- # set default importance
- copy_kwargs['inheritance'] = FIX
+ def add_property(self, property=None, value=None, id=None, name=None, description=None, datatype=None,
+ unit=None, importance=FIX, inheritance=FIX): # @ReservedAssignment
- return super(Property, self).add_property(
- property=property, value=value, **copy_kwargs)
+ return super().add_property(
+ property=property, id=id, name=name, description=description, datatype=datatype,
+ value=value, unit=unit, importance=importance, inheritance=inheritance)
- def add_parent(self, parent=None, **kwargs):
+ def add_parent(self, parent=None, id=None, name=None, inheritance=FIX):
"""Add a parent Entity to this Property.
Parameters
----------
- parent : Entity or int or str or None, optional
- The parent entity
- **kwargs : dict, optional
- Additional keyword arguments specifying the parent Entity
- by id or name, and specifying the inheritance level. See
- :py:meth:`Entity.add_parent` for more information. Note
- that by default, `inheritance` is set to ``fix``.
+ Parameters
+ ----------
+ parent : Entity or int or str or None
+ The parent entity, either specified by the Entity object
+ itself, or its id or its name. Default is None.
+ id : int
+ Integer id of the parent entity. Ignored if `parent`
+ is not None.
+ name : str
+ Name of the parent entity. Ignored if `parent is not
+ none`.
+ inheritance : str, default: FIX
+ One of ``obligatory``, ``recommended``, ``suggested``, or ``fix``. Specifies the
+ minimum importance which parent properties need to have to be inherited by this
+ entity. If no `inheritance` is given, no properties will be inherited by the child.
+ This parameter is case-insensitive.
See Also
--------
Entity.add_parent
"""
- copy_kwargs = kwargs.copy()
- if 'inheritance' not in copy_kwargs:
- # set default importance
- copy_kwargs['inheritance'] = FIX
-
- return super(Property, self).add_parent(parent=parent, **copy_kwargs)
+ return super(Property, self).add_parent(parent=parent, id=id, name=name, inheritance=inheritance)
def __init__(self, name=None, id=None, description=None, datatype=None,
value=None, unit=None):
@@ -1497,6 +1536,47 @@ class Property(Entity):
return super(Property, self).to_xml(xml, add_properties)
+ def is_reference(self, server_retrieval=False):
+ """Returns whether this Property is a reference
+
+ Parameters
+ ----------
+ server_retrieval : bool, optional
+ If True and the datatype is not set, the Property is retrieved from the server, by default False
+
+ Returns
+ -------
+ bool, NoneType
+ Returns whether this Property is a reference or None if a server call is needed to
+ check correctly, but server_retrieval is set to False.
+
+ """
+
+ if self.datatype is None:
+
+ if not self.is_valid():
+ # this is a workaround to prevent side effects
+ # since retrieve currently changes the object
+
+ if server_retrieval:
+ tmp_prop = deepcopy(self)
+ """
+ remove role to avoid unnessecary ValueError while
+ retrieving the Entity.
+ """
+ tmp_prop.role = None
+ tmp_prop.retrieve()
+
+ return tmp_prop.is_reference()
+ else:
+ return None
+ else:
+ # a valid property without datatype has to be an RT
+
+ return True
+ else:
+ return is_reference(self.datatype)
+
class Message(object):
@@ -1540,21 +1620,14 @@ class RecordType(Entity):
"""This class represents CaosDB's RecordType entities."""
- def add_property(self, property=None, value=None, **kwargs): # @ReservedAssignment
- copy_kwargs = kwargs.copy()
-
- if 'importance' not in copy_kwargs:
- # set default importance
- copy_kwargs['importance'] = RECOMMENDED
+ def add_property(self, property=None, value=None, id=None, name=None, description=None, datatype=None,
+ unit=None, importance=RECOMMENDED, inheritance=FIX): # @ReservedAssignment
- if 'inheritance' not in copy_kwargs:
- # set default importance
- copy_kwargs['inheritance'] = FIX
-
- return super().add_property(property=property, value=value,
- **copy_kwargs)
+ return super().add_property(
+ property=property, id=id, name=name, description=description, datatype=datatype,
+ value=value, unit=unit, importance=importance, inheritance=inheritance)
- def add_parent(self, parent=None, **kwargs):
+ def add_parent(self, parent=None, id=None, name=None, inheritance=OBLIGATORY):
"""Add a parent to this RecordType
Parameters
@@ -1562,24 +1635,30 @@ class RecordType(Entity):
parent : Entity or int or str or None, optional
The parent entity, either specified by the Entity object
itself, or its id or its name. Default is None.
- **kwargs : dict, optional
- Additional keyword arguments specifying the parent Entity by id or
- name, and specifying the inheritance level. See
- :py:meth:`Entity.add_parent` for more information. Note
- that by default, `inheritance` is set to ``obligatory``.
+ Parameters
+ ----------
+ parent : Entity or int or str or None
+ The parent entity, either specified by the Entity object
+ itself, or its id or its name. Default is None.
+ id : int
+ Integer id of the parent entity. Ignored if `parent`
+ is not None.
+ name : str
+ Name of the parent entity. Ignored if `parent is not
+ none`.
+ inheritance : str, default OBLIGATORY
+ One of ``obligatory``, ``recommended``, ``suggested``, or ``fix``. Specifies the
+ minimum importance which parent properties need to have to be inherited by this
+ entity. If no `inheritance` is given, no properties will be inherited by the child.
+ This parameter is case-insensitive.
See Also
--------
Entity.add_parent
"""
- copy_kwargs = kwargs.copy()
-
- if 'inheritance' not in copy_kwargs:
- # set default importance
- copy_kwargs['inheritance'] = OBLIGATORY
- return super().add_parent(parent=parent, **copy_kwargs)
+ return super().add_parent(parent=parent, id=id, name=name, inheritance=inheritance)
def __init__(self, name=None, id=None, description=None, datatype=None): # @ReservedAssignment
Entity.__init__(self, name=name, id=id, description=description,
@@ -1596,19 +1675,12 @@ class Record(Entity):
"""This class represents CaosDB's Record entities."""
- def add_property(self, property=None, value=None, **kwargs): # @ReservedAssignment
- copy_kwargs = kwargs.copy()
-
- if 'importance' not in copy_kwargs:
- # set default importance
- copy_kwargs['importance'] = FIX
-
- if 'inheritance' not in copy_kwargs:
- # set default importance
- copy_kwargs['inheritance'] = FIX
+ def add_property(self, property=None, value=None, id=None, name=None, description=None, datatype=None,
+ unit=None, importance=FIX, inheritance=FIX): # @ReservedAssignment
return super().add_property(
- property=property, value=value, **copy_kwargs)
+ property=property, id=id, name=name, description=description, datatype=datatype,
+ value=value, unit=unit, importance=importance, inheritance=inheritance)
def __init__(self, name=None, id=None, description=None): # @ReservedAssignment
Entity.__init__(self, name=name, id=id, description=description,
@@ -1760,19 +1832,12 @@ class File(Record):
return checksum.hexdigest()
- def add_property(self, property=None, value=None, **kwargs): # @ReservedAssignment
- copy_kwargs = kwargs.copy()
-
- if 'importance' not in copy_kwargs:
- # set default importance
- copy_kwargs['importance'] = FIX
-
- if 'inheritance' not in copy_kwargs:
- # set default importance
- copy_kwargs['inheritance'] = FIX
+ def add_property(self, property=None, id=None, name=None, description=None, datatype=None,
+ value=None, unit=None, importance=FIX, inheritance=FIX): # @ReservedAssignment
return super().add_property(
- property=property, value=value, **copy_kwargs)
+ property=property, id=id, name=name, description=description, datatype=datatype,
+ value=value, unit=unit, importance=importance, inheritance=inheritance)
class _Properties(list):
@@ -2145,6 +2210,7 @@ class _Messages(dict):
else:
raise TypeError(
"('description', 'body'), ('body'), or 'body' expected.")
+
if isinstance(value, Message):
body = value.body
description = value.description
@@ -2275,15 +2341,45 @@ class _Messages(dict):
def _basic_sync(e_local, e_remote):
+ '''Copy all state from a one entity to another.
+
+ This method is used to syncronize an entity with a remote (i.e. a newly
+ retrieved) one.
+
+ Any entity state of the local one will be overriden.
+
+ Parameters
+ ----------
+ e_local : Entity
+ Destination of the copy.
+ e_local : Entity
+ Source of the copy.
+
+
+ Returns
+ -------
+ e_local : Entity
+ The syncronized entity.
+ '''
if e_local is None or e_remote is None:
return None
+ if e_local.role is None:
+ e_local.role = e_remote.role
+ elif e_remote.role is not None and not e_local.role.lower() == e_remote.role.lower():
+ raise ValueError("The resulting entity had a different role ({0}) "
+ "than the local one ({1}). This probably means, that "
+ "the entity was intialized with a wrong class "
+ "by this client or it has changed in the past and "
+ "this client did't know about it yet.".format(
+ e_remote.role, e_local.role))
+
e_local.id = e_remote.id
e_local.name = e_remote.name
e_local.description = e_remote.description
e_local.path = e_remote.path
e_local._checksum = e_remote._checksum
e_local._size = e_remote._size
- e_local.datatype = e_remote.datatype # @ReservedAssignment
+ e_local.datatype = e_remote.datatype
e_local.unit = e_remote.unit
e_local.value = e_remote.value
e_local.properties = e_remote.properties
@@ -2314,6 +2410,7 @@ def _deletion_sync(e_local, e_remote):
except KeyError:
# deletion info wasn't there
e_local.messages = e_remote.messages
+
return
_basic_sync(e_local, e_remote)
@@ -2506,6 +2603,7 @@ class Container(list):
tmpid = 0
# users might already have specified some tmpids. -> look for smallest.
+
for e in self:
tmpid = min(tmpid, Container._get_smallest_tmpid(e))
tmpid -= 1
@@ -2725,6 +2823,7 @@ class Container(list):
used_remote_entities = []
# match by cuid
+
for local_entity in self:
sync_dict[local_entity] = None
@@ -2753,6 +2852,7 @@ class Container(list):
raise MismatchingEntitiesError(msg)
# match by id
+
for local_entity in self:
if sync_dict[local_entity] is None and local_entity.id is not None:
sync_remote_entities = []
@@ -2777,6 +2877,7 @@ class Container(list):
raise MismatchingEntitiesError(msg)
# match by path
+
for local_entity in self:
if (sync_dict[local_entity] is None
and local_entity.path is not None):
@@ -2806,6 +2907,7 @@ class Container(list):
raise MismatchingEntitiesError(msg)
# match by name
+
for local_entity in self:
if (sync_dict[local_entity] is None
and local_entity.name is not None):
@@ -2855,7 +2957,62 @@ class Container(list):
return sync_dict
- def delete(self, raise_exception_on_error=True, flags=None):
+ def _test_dependencies_in_container(self, container):
+ """This function returns those elements of a given container that are a dependency of another element of the same container.
+
+ Args:
+ container (Container): a caosdb container
+
+ Returns:
+ [set]: a set of unique elements that are a dependency of another element of `container`
+ """
+ item_id = set()
+ is_parent = set()
+ is_property = set()
+ is_being_referenced = set()
+ dependent_parents = set()
+ dependent_properties = set()
+ dependent_references = set()
+ dependencies = set()
+
+ for container_item in container:
+ item_id.add(container_item.id)
+
+ for parents in container_item.get_parents():
+ is_parent.add(parents.id)
+
+ for references in container_item.get_properties():
+ if is_reference(references.datatype):
+ # add only if it is a reference, not a property
+
+ if references.value is None:
+ continue
+ elif isinstance(references.value, int):
+ is_being_referenced.add(references.value)
+ elif is_list_datatype(references.datatype):
+ for list_item in references.value:
+ if isinstance(list_item, int):
+ is_being_referenced.add(list_item)
+ else:
+ is_being_referenced.add(list_item.id)
+ else:
+ try:
+ is_being_referenced.add(references.value.id)
+ except AttributeError:
+ pass
+
+ if hasattr(references, 'id'):
+ is_property.add(references.id)
+
+ dependent_parents = item_id.intersection(is_parent)
+ dependent_properties = item_id.intersection(is_property)
+ dependent_references = item_id.intersection(is_being_referenced)
+ dependencies = dependent_parents.union(dependent_references)
+ dependencies = dependencies.union(dependent_properties)
+
+ return dependencies
+
+ def delete(self, raise_exception_on_error=True, flags=None, chunk_size=100):
"""Delete all entities in this container.
Entities are identified via their id if present and via their
@@ -2866,16 +3023,45 @@ class Container(list):
this happens, none of them will be deleted. It occurs an error
instead.
"""
- chunk_size = 100
item_count = len(self)
# Split Container in 'chunk_size'-sized containers (if necessary) to avoid error 414 Request-URI Too Long
+
if item_count > chunk_size:
+ dependencies = self._test_dependencies_in_container(self)
+ '''
+ If there are as many dependencies as entities in the container and it is larger than chunk_size it cannot be split and deleted.
+ This case cannot be handled at the moment.
+ '''
+
+ if len(dependencies) == item_count:
+ if raise_exception_on_error:
+ te = TransactionError(
+ msg="The container is too large and with too many dependencies within to be deleted.",
+ container=self)
+ raise te
+
+ return self
+
+ # items which have to be deleted later because of dependencies.
+ dependencies_delete = Container()
+
for i in range(0, int(item_count/chunk_size)+1):
chunk = Container()
+
for j in range(i*chunk_size, min(item_count, (i+1)*chunk_size)):
- chunk.append(self[j])
+ if len(dependencies):
+ if self[j].id in dependencies:
+ dependencies_delete.append(self[j])
+ else:
+ chunk.append(self[j])
+ else:
+ chunk.append(self[j])
+
if len(chunk):
chunk.delete()
+ if len(dependencies_delete):
+ dependencies_delete.delete()
+
return self
if len(self) == 0:
@@ -3513,6 +3699,7 @@ class ACL():
result._priority_grants.update(self._priority_grants)
result._priority_denials.update(other._priority_denials)
result._priority_denials.update(self._priority_denials)
+
return result
def __eq__(self, other):
@@ -3803,6 +3990,7 @@ class Query():
connection = get_connection()
flags = self.flags
+
if cache is False:
flags["cache"] = "false"
query_dict = dict(flags)
@@ -3829,9 +4017,11 @@ class Query():
if len(cresp) > 1 and raise_exception_on_error:
raise QueryNotUniqueError(
"Query '{}' wasn't unique.".format(self.q))
+
if len(cresp) == 0 and raise_exception_on_error:
raise EmptyUniqueQueryError(
"Query '{}' found no results.".format(self.q))
+
if len(cresp) == 1:
r = cresp[0]
r.messages.extend(cresp.messages)
@@ -3938,6 +4128,7 @@ class Info():
for e in xml:
m = _parse_single_xml_element(e)
+
if isinstance(m, UserInfo):
self.user_info = m
else:
@@ -4084,7 +4275,7 @@ def _evaluate_and_add_error(parent_error, ent):
Parameters:
-----------
- parent_error : TrancactionError
+ parent_error : TransactionError
Parent error to which the new exception will be attached. This
exception will be a direct child.
ent : Entity
@@ -4097,13 +4288,16 @@ def _evaluate_and_add_error(parent_error, ent):
Parent error with new exception(s) attached to it.
"""
+
if isinstance(ent, (Entity, QueryTemplate)):
# Check all error messages
found114 = False
found116 = False
+
for err in ent.get_errors():
# Evaluate specific EntityErrors depending on the error
# code
+
if err.code is not None:
if int(err.code) == 101: # ent doesn't exist
new_exc = EntityDoesNotExistError(entity=ent,
@@ -4120,6 +4314,7 @@ def _evaluate_and_add_error(parent_error, ent):
found114 = True
new_exc = UnqualifiedPropertiesError(entity=ent,
error=err)
+
for prop in ent.get_properties():
new_exc = _evaluate_and_add_error(new_exc,
prop)
@@ -4127,6 +4322,7 @@ def _evaluate_and_add_error(parent_error, ent):
found116 = True
new_exc = UnqualifiedParentsError(entity=ent,
error=err)
+
for par in ent.get_parents():
new_exc = _evaluate_and_add_error(new_exc,
par)
@@ -4137,21 +4333,28 @@ def _evaluate_and_add_error(parent_error, ent):
parent_error.add_error(new_exc)
# Check for possible errors in parents and properties that
# weren't detected up to here
+
if not found114:
dummy_err = EntityError(entity=ent)
+
for prop in ent.get_properties():
dummy_err = _evaluate_and_add_error(dummy_err, prop)
+
if dummy_err.errors:
parent_error.add_error(dummy_err)
+
if not found116:
dummy_err = EntityError(entity=ent)
+
for par in ent.get_parents():
dummy_err = _evaluate_and_add_error(dummy_err, par)
+
if dummy_err.errors:
parent_error.add_error(dummy_err)
elif isinstance(ent, Container):
parent_error.container = ent
+
if ent.get_errors() is not None:
parent_error.code = ent.get_errors()[0].code
# In the highly unusual case of more than one error
@@ -4159,6 +4362,7 @@ def _evaluate_and_add_error(parent_error, ent):
parent_error.msg = '\n'.join(
[x.description for x in ent.get_errors()])
# Go through all container elements and add them:
+
for elt in ent:
parent_error = _evaluate_and_add_error(parent_error, elt)
@@ -4184,10 +4388,12 @@ def raise_errors(arg0):
transaction_error = _evaluate_and_add_error(TransactionError(),
arg0)
# Raise if any error was found
+
if len(transaction_error.all_errors) > 0:
raise transaction_error
# Cover the special case of an empty container with error
# message(s) (e.g. query syntax error)
+
if (transaction_error.container is not None and
transaction_error.container.has_errors()):
raise transaction_error
diff --git a/src/caosdb/configuration.py b/src/caosdb/configuration.py
index 6e8a9c6ff2083b0c30324722003fb3c08a592191..51e3749aaca3045afec9334ef987a174d5d19f26 100644
--- a/src/caosdb/configuration.py
+++ b/src/caosdb/configuration.py
@@ -21,6 +21,16 @@
#
# ** end header
#
+
+import os
+import yaml
+import warnings
+try:
+ optional_jsonschema_validate = None
+ from jsonschema import validate as optional_jsonschema_validate
+except ImportError:
+ pass
+
try:
# python2
from ConfigParser import ConfigParser
@@ -47,13 +57,45 @@ def configure(inifile):
_pycaosdbconf = None
if _pycaosdbconf is None:
_reset_config()
- return _pycaosdbconf.read(inifile)
+ read_config = _pycaosdbconf.read(inifile)
+ validate_yaml_schema(config_to_yaml(_pycaosdbconf))
+ return read_config
def get_config():
+ global _pycaosdbconf
return _pycaosdbconf
+def config_to_yaml(config):
+ valobj = {}
+ for s in config.sections():
+ valobj[s] = {}
+ for key, value in config[s].items():
+ # TODO: Can the type be inferred from the config object?
+ if key in ["timeout", "debug"]:
+ valobj[s][key] = int(value)
+ elif key in ["ssl_insecure"]:
+ valobj[s][key] = bool(value)
+ else:
+ valobj[s][key] = value
+
+ return valobj
+
+
+def validate_yaml_schema(valobj):
+ # TODO: Re-enable warning once the schema has been extended to also cover
+ # SSS pycaosdb.inis and integration tests.
+ if optional_jsonschema_validate:
+ with open(os.path.join(os.path.dirname(__file__), "schema-pycaosdb-ini.yml")) as f:
+ schema = yaml.load(f, Loader=yaml.SafeLoader)
+ optional_jsonschema_validate(instance=valobj, schema=schema["schema-pycaosdb-ini"])
+ # else:
+ # warnings.warn("""
+ # Warning: The validation could not be performed because `jsonschema` is not installed.
+ # """)
+
+
def _read_config_files():
"""Function to read config files from different paths. Checks for path in $PYCAOSDBINI or home directory (.pycaosdb.ini) and in the current working directory (pycaosdb.ini).
diff --git a/src/caosdb/connection/authentication/interface.py b/src/caosdb/connection/authentication/interface.py
index a364aeb564ee929d995b2f8098bd21e30e9733ab..f2cc5001cf8fa0f6d61ec65346f6a200ba0dfcd8 100644
--- a/src/caosdb/connection/authentication/interface.py
+++ b/src/caosdb/connection/authentication/interface.py
@@ -36,7 +36,7 @@ from caosdb.exceptions import LoginFailedError
# meta class compatible with Python 2 *and* 3:
ABC = ABCMeta('ABC', (object, ), {'__slots__': ()})
-_LOGGER = logging.getLogger("authentication")
+_LOGGER = logging.getLogger(__name__)
class AbstractAuthenticator(ABC):
diff --git a/src/caosdb/connection/authentication/keyring.py b/src/caosdb/connection/authentication/keyring.py
index d8be7ddf030577545230c9111fdad542b6d6e7e2..99d184136c20b23557efea0b54c648095a8d3ab2 100644
--- a/src/caosdb/connection/authentication/keyring.py
+++ b/src/caosdb/connection/authentication/keyring.py
@@ -28,7 +28,7 @@ retrieve the password.
"""
import sys
-import imp
+import importlib
from getpass import getpass
from caosdb.exceptions import ConfigurationError
from .external_credentials_provider import ExternalCredentialsProvider
@@ -52,17 +52,13 @@ def get_authentication_provider():
def _get_external_keyring():
try:
- fil, pathname, desc = imp.find_module("keyring", sys.path[1:])
- module = imp.load_module("external_keyring", fil, pathname, desc)
- return module
+ return importlib.import_module("keyring")
except ImportError:
raise RuntimeError(
"The keyring password method requires installation of the"
"keyring python package. On linux with python < 3.5, "
"this requires the installation of dbus-python as a "
"system package.")
- finally:
- fil.close()
def _call_keyring(**config):
@@ -74,7 +70,6 @@ def _call_keyring(**config):
url = config.get("url")
username = config.get("username")
app = "caosdb — {}".format(url)
- password = _call_keyring(app=app, username=username)
external_keyring = _get_external_keyring()
password = external_keyring.get_password(app, username)
if password is None:
diff --git a/src/caosdb/connection/connection.py b/src/caosdb/connection/connection.py
index 9e273a56778737033fda9f342f967f56946b501b..6c3946ee639872b0edb0b5b3c30a808cf8c028d4 100644
--- a/src/caosdb/connection/connection.py
+++ b/src/caosdb/connection/connection.py
@@ -60,7 +60,7 @@ except ImportError:
# pylint: disable=missing-docstring
-_LOGGER = logging.getLogger("connection")
+_LOGGER = logging.getLogger(__name__)
class _WrappedHTTPResponse(CaosDBHTTPResponse):
diff --git a/src/caosdb/schema-pycaosdb-ini.yml b/src/caosdb/schema-pycaosdb-ini.yml
new file mode 100644
index 0000000000000000000000000000000000000000..bfe8fe7c63679507bba795bb45d7afa2b097f07b
--- /dev/null
+++ b/src/caosdb/schema-pycaosdb-ini.yml
@@ -0,0 +1,90 @@
+schema-pycaosdb-ini:
+ type: object
+ additionalProperties: false
+ properties:
+ Container:
+ additionalProperties: false
+ properties:
+ debug:
+ default: 0
+ type: integer
+ enum: [0, 1, 2]
+ Connection:
+ description: Settings for the connection to the CaosDB server
+ additionalProperties: false
+ properties:
+ url:
+ description: URL of the CaosDB server
+ type: string
+ pattern: https://[-a-zA-Z0-9\.]+(:[0-9]+)?(/)?
+ examples: ["https://demo.indiscale.com/", "https://localhost:10443/"]
+ username:
+ type: string
+ description: User name used for authentication with the server
+ examples: [admin]
+ password_method:
+ description: The password input method defines how the password is supplied that is used for authentication with the server.
+ type: string
+ default: input
+ enum: [input, plain, pass, keyring]
+ password_identifier:
+ type: string
+ password:
+ type: string
+ examples: [caosdb]
+ auth_token:
+ type: string
+ description: Using an authentication token to connect with the server. This setting is not recommended for users.
+ cacert:
+ type: string
+ description: If the server's SSL certificate cannot be validated by your installed certificates (default or installed by your admins), you may also need to supply the matching key (pem file)
+ examples: [/path/to/caosdb.ca.pem]
+ ssl_insecure:
+ description: If this option is set, the SSL certificate of the server will not be validated. This has the potential of a man-in-the-middle attack. Use with care!
+ type: boolean
+ default: false
+ ssl_version:
+ description: You may define the ssl version to be used. It has to be the name of the corresponding attribute in the Python ssl module.
+ examples: [PROTOCOL_TLS]
+ debug:
+ default: 0
+ type: integer
+ enum: [0, 1, 2]
+ description: The debug key allows control the verbosity. Set it to 1 or 2 in case you want to see debugging output or if you want to learn more about the internals of the protocol. 0 disables debugging output.
+ socket_proxy:
+ examples: ["localhost:12345"]
+ type: string
+ description: You can define a socket proxy to be used. This is for the case that the server sits behind a firewall which is being tunnelled with a socket proxy (SOCKS4 or SOCKS5) (e.g. via ssh's -D option or a dedicated proxy server).
+ implementation:
+ description: This option is used internally and for testing. Do not override.
+ examples: [_DefaultCaosDBServerConnection]
+ timeout:
+ type: integer
+ allOf:
+ - if:
+ properties:
+ password_method:
+ const: input
+ then:
+ required: [url]
+ - if:
+ properties:
+ password_method:
+ const: plain
+ then:
+ required: [url, username, password]
+ - if:
+ properties:
+ password_method:
+ const: pass
+ then:
+ required: [url, username, password_identifier]
+ - if:
+ properties:
+ password_method:
+ const: keyring
+ then:
+ required: [url, username]
+ IntegrationTests:
+ description: "Used by the integration test suite from the caosdb-pyinttest repo."
+ additionalProperties: true
diff --git a/src/caosdb/utils/caosdb_admin.py b/src/caosdb/utils/caosdb_admin.py
index 7fd4a141a86642ff43cac859830c96317356d63d..9fb94f57683036f5432a40198cc4ae98893665fb 100755
--- a/src/caosdb/utils/caosdb_admin.py
+++ b/src/caosdb/utils/caosdb_admin.py
@@ -116,28 +116,43 @@ def _promt_for_pw():
def do_create_user(args):
- password = None
+ password = args.user_password
if args.ask_password is True:
password = _promt_for_pw()
try:
admin._insert_user(name=args.user_name,
email=args.user_email, password=password)
+
+ if args.activate_user:
+ do_activate_user(args)
except HTTPClientError as e:
print(e.msg)
def do_activate_user(args):
- admin._update_user(name=args.user_name, status="ACTIVE")
+ try:
+ admin._update_user(name=args.user_name, status="ACTIVE")
+ except HTTPClientError as e:
+ print(e.msg)
def do_deactivate_user(args):
- admin._update_user(name=args.user_name, status="INACTIVE")
+ try:
+ admin._update_user(name=args.user_name, status="INACTIVE")
+ except HTTPClientError as e:
+ print(e.msg)
def do_set_user_password(args):
- password = _promt_for_pw()
- admin._update_user(name=args.user_name, password=password)
+ if args.user_password is None:
+ password = _promt_for_pw()
+ else:
+ password = args.user_password
+ try:
+ admin._update_user(name=args.user_name, password=password)
+ except HTTPClientError as e:
+ print(e.msg)
def do_add_user_roles(args):
@@ -304,8 +319,18 @@ USAGE
help="Create a new user in caosdb's internal user database. You need "
" to activate the user before use.")
subparser.set_defaults(call=do_create_user)
- subparser.add_argument("-a", "--ask-password",
- help="Prompt for a password.", action="store_true")
+ mg = subparser.add_mutually_exclusive_group()
+ mg.add_argument("-a", "--ask-password",
+ help="Prompt for a password.", action="store_true")
+ mg.add_argument(
+ "--password",
+ dest="user_password",
+ default=None,
+ help="Alternative way to provide the new user's password. Please "
+ "consider to use the more secure, interactive way (-a option).")
+ subparser.add_argument("-c", "--activate-user",
+ help="Activate the user after creation.",
+ action="store_true")
subparser.add_argument(
metavar='USERNAME',
dest="user_name",
@@ -332,14 +357,20 @@ USAGE
subparser = subparsers.add_parser(
"set_user_password",
- help="Set a new password for a user. The user needs to be reactivated "
- "afterwards! The password is not to be given on the command line for "
- "security reasons. You will be prompted for the password.")
+ help="Set a new password for a user. "
+ "By default, you will be prompted for the password.")
subparser.set_defaults(call=do_set_user_password)
subparser.add_argument(
metavar='USERNAME',
dest="user_name",
help="The name of the user who's password is to be set.")
+ subparser.add_argument(
+ metavar='PASSWORD',
+ nargs="?",
+ dest="user_password",
+ default=None,
+ help="Alternative way to provide the user's new password. "
+ "The more secure (and default way) is to provide it interactively.")
subparser = subparsers.add_parser(
"set_user_entity",
diff --git a/src/caosdb/utils/plantuml.py b/src/caosdb/utils/plantuml.py
index 75b96ee3aa28ba916adc69e418de398abfe23356..be34b2604f3682bb71b48bbd73e00fe854b3af51 100644
--- a/src/caosdb/utils/plantuml.py
+++ b/src/caosdb/utils/plantuml.py
@@ -36,11 +36,24 @@ plantuml FILENAME.pu -> FILENAME.png
import os
import caosdb as db
+from caosdb.common.datatype import is_reference, get_referenced_recordtype
REFERENCE = "REFERENCE"
def get_description(description_str):
+ """Extract and format a description string from a record type or property.
+
+ Parameters
+ ----------
+ description_str : str
+ The description string that is going to be formatted.
+
+ Returns
+ -------
+ str
+ The reformatted description ending in a line break.
+ """
words = description_str.split()
lines = []
lines.append("")
@@ -211,15 +224,76 @@ package \"The property P references an instance of D\" <<Rectangle>> {
return result
+def retrieve_substructure(start_record_types, depth, result_id_set=None, result_container=None, cleanup=True):
+ """Recursively retrieves CaosDB record types and properties, starting
+ from given initial types up to a specific depth.
+
+ Parameters
+ ----------
+ start_record_types : Iterable[db.Entity]
+ Iterable with the entities to be displayed. Starting from these
+ entities more entities will be retrieved.
+ depth : int
+ The maximum depth up to which to retriev sub entities.
+ result_id_set : set[int]
+ Used by recursion. Filled with already visited ids.
+ result_container : db.Container
+ Used by recursion. Filled with already visited entities.
+ cleanup : bool
+ Used by recursion. If True return the resulting result_container.
+ Don't return anything otherwise.
+
+ Returns
+ -------
+ db.Container
+ A container containing all the retrieved entites or None if cleanup is False.
+ """
+ # Initialize the id set and result container for level zero recursion depth:
+ if result_id_set is None:
+ result_id_set = set()
+ if result_container is None:
+ result_container = db.Container()
+
+ for entity in start_record_types:
+ entity.retrieve()
+ if entity.id not in result_id_set:
+ result_container.append(entity)
+ result_id_set.add(entity.id)
+ for prop in entity.properties:
+ if is_reference(prop.datatype) and prop.datatype != db.FILE and depth > 0:
+ rt = db.RecordType(name=get_referenced_recordtype(prop.datatype)).retrieve()
+ retrieve_substructure([rt], depth-1, result_id_set, result_container, False)
+
+ if prop.id not in result_id_set:
+ result_container.append(prop)
+ result_id_set.add(prop.id)
+
+ for parent in entity.parents:
+ rt = db.RecordType(id=parent.id).retrieve()
+ if parent.id not in result_id_set:
+ result_container.append(rt)
+ result_id_set.add(parent.id)
+ if depth > 0:
+ retrieve_substructure([rt], depth-1, result_id_set, result_container, False)
+
+ if cleanup:
+ return result_container
+ return None
+
+
def to_graphics(recordtypes, filename):
- """ calls recordtypes_to_plantuml_string(), saves result to file and
+ """Calls recordtypes_to_plantuml_string(), saves result to file and
creates an svg image
- plantuml needs to be installed
- @params:
- recordtypes: itrable with the record types to be displayed
- filname: filename of the image (e.g. data_structure; data_structure.pu and
- data_structure.svg will be created.
+ plantuml needs to be installed.
+
+ Parameters
+ ----------
+ recordtypes : Iterable[db.Entity]
+ Iterable with the entities to be displayed.
+ filename : str
+ filename of the image without the extension(e.g. data_structure;
+ data_structure.pu and data_structure.svg will be created.)
"""
pu = recordtypes_to_plantuml_string(recordtypes)
diff --git a/src/caosdb/yamlapi.py b/src/caosdb/yamlapi.py
index b2e527983cbbeca2ee71a63e87f487d24d7c0301..80bb4b13e4d1626c5d29c8950f3a22bbb73e0fdb 100644
--- a/src/caosdb/yamlapi.py
+++ b/src/caosdb/yamlapi.py
@@ -22,7 +22,7 @@
# ** end header
#
-"""YAML interface for the database (caosdb)"""
+"""!!! Deprecated !!! YAML interface for the database (caosdb)"""
import yaml
from lxml import etree
@@ -31,9 +31,14 @@ import re
import caosdb
import caosdb.common.utils as utils
from caosdb.connection.connection import get_connection
+import warnings
def append_sublist(v, newel, def_entity_type):
+ warnings.warn("""
+ This function is deprecated and will be removed with the next release.
+ Please use caosdb-advanced-user-tools/models/data_model.py for a
+ similar functionality.""", DeprecationWarning)
if v is None:
return
for i in v:
@@ -46,6 +51,10 @@ def append_sublist(v, newel, def_entity_type):
def kv_to_xml(k, v):
+ warnings.warn("""
+ This function is deprecated and will be removed with the next release.
+ Please use caosdb-advanced-user-tools/models/data_model.py for a
+ similar functionality.""", DeprecationWarning)
newel = Element(k)
# code.interact(local=locals())
if isinstance(v, list): # Top level loop
@@ -69,15 +78,35 @@ def dict_to_xml(d):
d: The dictionary (possibly loaded from yaml)
to convert to caosdb-xml.
"""
+ warnings.warn("""
+ This function is deprecated and will be removed with the next release.
+ Please use caosdb-advanced-user-tools/models/data_model.py for a
+ similar functionality.""", DeprecationWarning)
return kv_to_xml("Entities", d)
def yaml_to_xml(yamlstr):
- return dict_to_xml(yaml.load(yamlstr))
+ warnings.warn("""
+ This function is deprecated and will be removed with the next release.
+ Please use caosdb-advanced-user-tools/models/data_model.py for a
+ similar functionality.""", DeprecationWarning)
+ """Load a yaml document from yamlstr and converts it to XML.
+
+ Parameters
+ ----------
+ yamlstr : str
+ The string to load the yaml document from.
+
+ """
+ return dict_to_xml(yaml.load(yamlstr, Loader=yaml.SafeLoader))
def process(text):
"""Do some replacements on the original file to obtain valid yaml."""
+ warnings.warn("""
+ This function is deprecated and will be removed with the next release.
+ Please use caosdb-advanced-user-tools/models/data_model.py for a
+ similar functionality.""", DeprecationWarning)
processed = re.sub(
"^(\\s*)-\\s*\\{?(.*)\\}?\\s*$",
"\\1- {\\2}",
@@ -90,6 +119,10 @@ def process(text):
def yaml_file_to_xml(yamlfilename):
+ warnings.warn("""
+ This function is deprecated and will be removed with the next release.
+ Please use caosdb-advanced-user-tools/models/data_model.py for a
+ similar functionality.""", DeprecationWarning)
with open(yamlfilename, "r") as f:
return yaml_to_xml(process(f.read()))
@@ -100,6 +133,10 @@ def insert_yaml_file(yamlfilename, simulate=False):
Set 'simulate' to True if you don't actually want to insert the xml,
but only receive what would be sent.
"""
+ warnings.warn("""
+ This function is deprecated and will be removed with the next release.
+ Please use caosdb-advanced-user-tools/models/data_model.py for a
+ similar functionality.""", DeprecationWarning)
con = get_connection()
prs = etree.XMLParser(remove_blank_text=True)
sent_xml = etree.tostring(
diff --git a/src/doc/conf.py b/src/doc/conf.py
index b05fa1c71c1dcd0b59916594818449d2ebc574bd..b4dfcc9925eb8100b957b6c8c2c06c855e8d3ff0 100644
--- a/src/doc/conf.py
+++ b/src/doc/conf.py
@@ -25,14 +25,14 @@ import sphinx_rtd_theme # noqa: E402
# -- Project information -----------------------------------------------------
project = 'pycaosdb'
-copyright = '2020, IndiScale GmbH'
+copyright = '2022, IndiScale GmbH'
author = 'Daniel Hornung'
# The short X.Y version
-version = '0.5.2'
+version = '0.7'
# The full version, including alpha/beta/rc tags
# release = '0.5.2-rc2'
-release = '0.5.2'
+release = '0.7.2-dev'
# -- General configuration ---------------------------------------------------
diff --git a/src/doc/configuration.md b/src/doc/configuration.md
index 6e53542f661dcae94622fef24a67cecf7491df9c..802da4e91818ba65bd0184a9a5ac49f5c2ba02d2 100644
--- a/src/doc/configuration.md
+++ b/src/doc/configuration.md
@@ -49,6 +49,8 @@ with CaosDB which makes the experience much less verbose. Set it to 1 or 2 in ca
debugging (which I hope will not be necessary for this tutorial) or if you want to learn more about
the internals of the protocol.
+`timeout` sets the timeout for requests to the server.
+
A complete list of options can be found in the
[pycaosdb.ini file](https://gitlab.com/caosdb/caosdb-pylib/-/blob/main/examples/pycaosdb.ini) in
the examples folder of the source code.
diff --git a/src/doc/gallery/Makefile b/src/doc/gallery/Makefile
new file mode 100644
index 0000000000000000000000000000000000000000..658f9a6a93e23957b20aee5f38e5565bde35af80
--- /dev/null
+++ b/src/doc/gallery/Makefile
@@ -0,0 +1,23 @@
+# This file is a part of the CaosDB Project.
+#
+# Copyright (C) 2022 IndiScale GmbH <info@indiscale.com>
+# Copyright (C) 2022 Daniel Hornung <d.hornung@indiscale.com>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+
+# Run tests on the examples.
+test:
+ python3 -m doctest simulation.py
+.PHONY: test
diff --git a/src/doc/gallery/index.rst b/src/doc/gallery/index.rst
new file mode 100644
index 0000000000000000000000000000000000000000..a6ef53e4c7d1272c5dbc8c62b4d90a89591cac0f
--- /dev/null
+++ b/src/doc/gallery/index.rst
@@ -0,0 +1,16 @@
+
+PyCaosDB Code Gallery
+=====================
+
+This chapter collects code examples which can be immediately run against an empty CaosDB instance.
+
+.. note::
+
+ These examples require a configuration file with valid server and user/password settings. Refer
+ to the :ref:`Configuration <Configuration of PyCaosDB>` section for details.
+
+.. toctree::
+ :maxdepth: 2
+ :caption: The code examples:
+
+ simulation
diff --git a/src/doc/gallery/simulation.py b/src/doc/gallery/simulation.py
new file mode 100644
index 0000000000000000000000000000000000000000..342d5d980fc2b1a981f4a76d99e1954f8b2f5c2a
--- /dev/null
+++ b/src/doc/gallery/simulation.py
@@ -0,0 +1,134 @@
+"""
+Run a simulation and store the values into CaosDB.
+
+>>> main() # doctest: +ELLIPSIS
+These distances resulted in small x,y, values:
+[...]
+"""
+
+import numpy as np
+import scipy.integrate
+import caosdb as db
+from caosadvancedtools.table_converter import to_table
+
+
+def setup_caosdb():
+ """Create the data model and insert it into CaosDB
+
+ The data model consists of the following RecordTypes:
+
+ Software
+ with author and revision.
+
+ SoftwareRun
+ A specific run of the sofware, with input parameters, time of completion and a result.
+
+ State
+ An aggregate of x,y,z values.
+
+ Parameters
+ In this case the x,y,z initial values before the integration, so this is just a state.
+
+ Result
+ The x,y,z values at the end of the software run, the final state.
+
+ The data model of course also contains the corresponding properties for these RecordTypes.
+ """
+
+ cont = db.Container() # Container to insert all Entities at once into CaosDB
+ # create Properties
+ cont.append(db.Property("x", datatype=db.DOUBLE))
+ cont.append(db.Property("y", datatype=db.DOUBLE))
+ cont.append(db.Property("z", datatype=db.DOUBLE))
+ cont.append(db.Property("completed", datatype=db.DATETIME))
+ cont.append(db.Property("author", datatype=db.TEXT))
+ cont.append(db.Property("revision", datatype=db.TEXT))
+ # create RecordTypes
+ cont.append(db.RecordType("Software").add_property("author").add_property("revision"))
+ cont.append(db.RecordType("State").add_property("x", importance=db.OBLIGATORY)
+ .add_property("y").add_property("z"))
+ cont.append(db.RecordType("Parameters").add_parent("State", inheritance=db.ALL))
+ cont.append(db.RecordType("Result").add_parent("State", inheritance=db.RECOMMENDED))
+ cont.append(db.RecordType("SoftwareRun").add_property("Software").add_property("Parameters")
+ .add_property("completed").add_property("Result"))
+ cont.insert() # actually insert the Entities
+
+
+def simulations(n, t_max):
+ """Run the simulations.
+
+ Parameters
+ ----------
+ n : int
+ The number of runs.
+
+ t_max : float
+ The maximum time of integration.
+ """
+
+ software = (db.Record("simulator").add_parent("Software")
+ .add_property("author", value="IndiScale GmbH")
+ .add_property("revision", value="1234CDEF89AB"))
+ software.insert()
+ for i in range(n):
+ # Get the parameters and result
+ initial, result = run_simulation(run=i, t_max=t_max)
+
+ # Prepare CaosDB insertion
+ run = db.Record().add_parent("SoftwareRun").add_property("Software", value=software.id)
+ parameters = (db.Record().add_parent("Parameters").add_property("x", initial[0])
+ .add_property("y", initial[1]).add_property("z", initial[2]))
+ result_record = (db.Record().add_parent("Result").add_property("x", result[0])
+ .add_property("y", result[1]).add_property("z", result[2]))
+ run.add_property("Parameters", value=parameters).add_property("Result", value=result_record)
+ cont = db.Container()
+ cont.extend([run, parameters, result_record])
+ cont.insert() # Insert everything of this run into CaosDB.
+
+
+def run_simulation(run, t_max):
+ """Integrate the Rössler attractor from random initial values."""
+ a, b, c = (0.1, 0.1, 14)
+
+ def diff(t, x):
+ diff = np.array([-x[1] - x[2],
+ x[0] + a * x[1],
+ b + x[2] * (x[0] - c)])
+ return diff
+
+ x0 = np.random.uniform(-100, 100, 3)
+
+ result = scipy.integrate.solve_ivp(diff, [0, t_max], x0)
+ x = result.y[:, -1]
+ return (x0, x)
+
+
+def analyze():
+ """Find the initial conditions which produce the smalles x,y values after the given time."""
+ distance = 5
+ data = db.execute_query("""SELECT Parameters, Result FROM RECORD SoftwareRun WITH
+ (((Result.x < {dist}) AND (Result.x > -{dist}))
+ AND (Result.y < {dist})) AND Result.y > -{dist}""".format(dist=distance))
+ dataframe = to_table(data) # Convert into a Pandas DataFrame
+
+ parameters = db.Container().extend([db.Record(id=id) for id in dataframe.Parameters]).retrieve()
+
+ initial_distances = [np.linalg.norm([p.get_property(dim).value for dim in ["x", "y", "z"]])
+ for p in parameters]
+
+ print("These distances resulted in small x,y, values:\n{}".format(initial_distances))
+
+
+def main():
+ # 1. Set up the data model
+ setup_caosdb()
+
+ # 2. Run simulations
+ simulations(n=200, t_max=5)
+
+ # 3. Find initial conditions with interesting results
+ analyze()
+
+
+if __name__ == '__main__':
+ main()
diff --git a/src/doc/gallery/simulation.rst b/src/doc/gallery/simulation.rst
new file mode 100644
index 0000000000000000000000000000000000000000..ce1a7f457a142e36ef9f2b0cfe6a4df0b9fcedf6
--- /dev/null
+++ b/src/doc/gallery/simulation.rst
@@ -0,0 +1,14 @@
+========================================
+Managing data from numerical simulations
+========================================
+
+This code example
+
+1. sets up the data model
+2. runs simulations
+3. stores the simulation parameters and results into CaosDB
+4. retrieves the parameters for interesting results.
+
+:download:`Download code<simulation.py>`
+
+.. literalinclude:: simulation.py
diff --git a/src/doc/index.rst b/src/doc/index.rst
index bd29c6c56acf5c173e94ae6471a6aeba56ea4b93..004ae3a9926ed7a9a27720db1f3c28e72f1f3f28 100644
--- a/src/doc/index.rst
+++ b/src/doc/index.rst
@@ -12,6 +12,7 @@ Welcome to PyCaosDB's documentation!
Concepts <concepts>
Configuration <configuration>
Administration <administration>
+ Code gallery <gallery/index>
API documentation<_apidoc/caosdb>
This is the documentation for the Python client library for CaosDB, ``PyCaosDB``.
diff --git a/src/doc/tutorials/Data-Insertion.rst b/src/doc/tutorials/Data-Insertion.rst
index 22fb9461d6916003b2dad496ff3487df335c8dcc..f2c7f830d1403fbdf45354d1f36a4ea339759058 100644
--- a/src/doc/tutorials/Data-Insertion.rst
+++ b/src/doc/tutorials/Data-Insertion.rst
@@ -10,15 +10,18 @@ Model<caosdb-server:Data-Model>` in the CaosDB server documentation.
In order to insert some actual data, we need to create a data model
using RecordTypes and Properties (You may skip this if you use a CaosDB
-instance that already has the required types). So, let’s create a simple
+instance that already has the required types). When you create a new Property
+you must supply a datatype. So, let’s create a simple
Property called “a” of datatype double. This is very easy in pylib:
.. code:: python
a = db.Property(name="a", datatype=db.DOUBLE)
-There are a few basic datatypes: db.INTEGER, db.TEXT. See `data
-type <Specification/Datatype>`__ for a full list.
+There are a few basic datatypes like db.INTEGER, db.DOUBLE, or db.TEXT. See the
+`data types
+<https://docs.indiscale.com/caosdb-server/specification/Datatype.html>`_ for a
+full list.
We can create our own small data model for e.g. a simulation by adding
two more Properties and a RecordType:
@@ -49,11 +52,11 @@ resolution, but we'll omit this for the sake of brevity for now.
.. code:: python
rt = db.RecordType(name="2D_BarkleySimulation",
- description="Spatially extended Barkley simulation")
+ description="Spatially extended Barkley simulation")
# inherit all properties from the BarkleySimulation RecordType
rt.add_parent(name="BarkleySimulation", inheritance="all")
rt.insert()
-
+
print(rt.get_property(name="epsilon").importance) ### rt has a "epsilon" property with the same importance as "BarkleySimulation"
The parameter ``inheritance=(obligatory|recommended|fix|all|none)`` of
@@ -135,15 +138,37 @@ Useful is also, that you can insert directly tabular data.
.. code:: python
- from caosadvancedtools.table_converter import from_tsv
-
- recs = from_tsv("test.csv", "Experiment")
- print(recs)
- recs.insert()
+ from caosadvancedtools.table_converter import from_tsv
+
+ recs = from_tsv("test.csv", "Experiment")
+ print(recs)
+ recs.insert()
With this example file
`test.csv <uploads/4f2c8756a26a3984c0af09d206d583e5/test.csv>`__.
+List Properties
+---------------
+
+As you may already know, properties can also have list values instead of scalar
+values. They can be accessed, set, and updated as you would expect from any
+list-valued attribute in Python, as the following example illustrates.
+
+.. code:: python
+
+ import caosdb as db
+ db.Property(name="TestList", datatype=db.LIST(db.DOUBLE)).insert()
+ db.RecordType(name="TestType").add_property(name="TestList").insert()
+ db.Record(name="TestRec").add_parent("TestType").add_property(
+ name="TestList", value=[1,2,3]).insert()
+ retrieved = db.Record(name="TestRec").retrieve()
+ retrieved.get_property("TestList").value += [4,5]
+ retrieved.update()
+
+ # Check update
+ retrieved = db.Record(name="TestRec").retrieve()
+ print(retrieved.get_property("TestList").value)
+
File Update
-----------
diff --git a/src/doc/tutorials/complex_data_models.rst b/src/doc/tutorials/complex_data_models.rst
new file mode 100644
index 0000000000000000000000000000000000000000..0fa868e78bb45a2905dc99392a3a28a9832d369e
--- /dev/null
+++ b/src/doc/tutorials/complex_data_models.rst
@@ -0,0 +1,76 @@
+Complex Data Models
+-------------------
+
+With CaosDB it is possible to create very complex data models.
+
+E.g. it is possible to add properties to properties to cover complex relations
+in data management workflows.
+
+One example for a use case is meta data that is added to very specific properties of
+datasets, e.g. data privacy information can be added to properties which themselves
+could already be considered meta data of a dataset.
+
+The example below tries to cover some complex cases for data models:
+
+Examples
+--------
+
+.. code-block:: python3
+
+ import caosdb as db
+
+ # Create two record types with descriptions:
+ rt1 = db.RecordType(name="TypeA", description="The first type")
+ rt2 = db.RecordType(name="TypeB", description="The second type")
+
+ # Create a record using the first record type as parent:
+ r1 = db.Record(name="Test_R_1", description="A record")
+ r1.add_parent(rt1)
+
+ # Create two files (the files named test.txt and testfile2.txt should exist in the
+ # current working directory:
+ f1 = db.File(name="Test file", path="/test.txt", file="test.txt")
+ f2 = db.File(name="Test file 2", path="/testfile2.txt", file="testfile2.txt")
+
+ # Create two properties with different data types:
+ p1 = db.Property(name="TestFileProperty", datatype=db.FILE)
+ p2 = db.Property(name="TestDoubleProperty", datatype=db.DOUBLE, unit="m")
+ p3 = db.Property(name="TestIntegerProperty", datatype=db.INTEGER, unit="s")
+
+ # Create a reference property that points to records of record type 2:
+ p4 = db.Property(name="TestReferenceProperty", datatype=rt2)
+
+ # Create a complex record:
+ r2 = db.Record(name="Test_R_2", description="A second record")
+ r2.add_parent(rt2)
+ r2.add_property(rt1, value=r1) # this is a reference to the first record type
+ r2.add_property(p1, value=f1) # this is a reference to the first file
+ r2.add_property(p2, value=24.8) # this is a double property with a value
+ r2.add_property(p3, value=1) # this is an integer property with a value
+
+ # Very complex part of the data model:
+ # Case 1: File added to another file
+ f2.add_property(p1, value=f1) # this adds a file property with value first file
+ # to the second file
+
+ # Case 2: Property added to a property
+ p2.add_property(p3, value=27) # this adds an integer property with value 27 to the
+ # double property
+
+ # Case 3: Reference property added to a property
+ # The property p2 now has two sub properties, one is pointing to
+ # record p2 which itself has the property p2, therefore this can be
+ # considered a loop in the data model.
+ p2.add_property(p4, value=r2) # this adds a reference property pointing to
+ # record 2 to the double property
+
+ # Insert a container containing all the newly created entities:
+ c = db.Container().extend([rt1, rt2, r1, r2, f1, p1, p2, p3, f2, p4])
+ c.insert()
+
+ # Useful for testing: wait until the user presses a key
+ # Meanwhile have a look at the WebUI: You can e.g. query "FIND Test*" to view
+ # all the entities created here and see the relations and links between them.
+ b = input("Press any key to cleanup.")
+ # cleanup everything after the user presses any button.
+ c.delete()
diff --git a/src/doc/tutorials/index.rst b/src/doc/tutorials/index.rst
index 3889edb8f47e973cc7ae25c9134d75cfeab95f65..0b08d0b4fe153d803a780bd144787819b827db78 100644
--- a/src/doc/tutorials/index.rst
+++ b/src/doc/tutorials/index.rst
@@ -15,4 +15,6 @@ advanced usage of the Python client.
Data-Insertion
errors
data-model-interface
-
+ complex_data_models
+ serverside
+
diff --git a/src/doc/tutorials/serverside.rst b/src/doc/tutorials/serverside.rst
new file mode 100644
index 0000000000000000000000000000000000000000..93f0fdcf742efc70bc80f5113eb7c6ddbbf87cde
--- /dev/null
+++ b/src/doc/tutorials/serverside.rst
@@ -0,0 +1,61 @@
+
+Server Side Scripting
+=====================
+
+The administrator may store regularly needed scripts, e.g. for computing a
+standardized analysis, on the same machine as the CaosDB server, "on the server
+side", where they can be run directly by the server.
+
+The execution of those scripts can be initiated using the Python client, or the
+web interface.
+
+Call a Script
+~~~~~~~~~~~~~
+
+If you have access to the server and sufficient permissions to run the script,
+execution is fairly simple:
+
+.. code:: python
+
+ from caosdb.utils.server_side_scripting import run_server_side_script
+ response = run_server_side_script('scriptname.py')
+ print(response.stderr,response.stdout)
+
+
+This makes the server run the script ``scriptname.py``. The output of the
+script (``stderr`` and ``stdout``) is returned within an response object.
+
+
+If the script requires additional arguments, those can be provided after the
+script's name.
+
+Note that by default the script runs with your CaosDB account. It has your
+permissions and changes are logged as if they were done by you directly.
+
+
+Testing it
+~~~~~~~~~~
+
+You can try this out using for example the ``diagnostics.py`` script (it is part
+of the `CaosDB server repository
+<https://gitlab.indiscale.com/caosdb/src/caosdb-server/-/blob/main/scripting/bin/administration/diagnostics.py>`_
+and is also available on https://demo.indiscale.com). The script returns
+information about the server in JSON format. You can do for example the
+following:
+
+.. code:: python
+
+ import json
+ from caosdb.utils.server_side_scripting import run_server_side_script
+ response = run_server_side_script('administration/diagnostics.py')
+ print("JSON content:")
+ print(json.loads(response.stdout))
+ print("stderr:")
+ print(response.stderr)
+
+
+Further Information
+~~~~~~~~~~~~~~~~~~~
+
+Additionally, you might want to have a look at
+https://docs.indiscale.com/caosdb-server/specification/Server-side-scripting.html
diff --git a/tox.ini b/tox.ini
index 94c2dc8affb280d3e7f6cff4536636432c9f7749..0d245e03ef9c8fe2151e173cd1a10964d47ef82b 100644
--- a/tox.ini
+++ b/tox.ini
@@ -1,5 +1,5 @@
[tox]
-envlist=py36, py37, py38, py39
+envlist=py36, py37, py38, py39, py310
skip_missing_interpreters = true
[testenv]
@@ -7,4 +7,8 @@ deps = .
nose
pytest
pytest-cov
+ jsonschema==4.0.1
commands=py.test --cov=caosdb -vv {posargs}
+
+[flake8]
+max-line-length=100
diff --git a/unittests/broken_configs/pycaosdb1.ini b/unittests/broken_configs/pycaosdb1.ini
new file mode 100644
index 0000000000000000000000000000000000000000..71180286881399c35e251bba89a54a345cd948ac
--- /dev/null
+++ b/unittests/broken_configs/pycaosdb1.ini
@@ -0,0 +1,5 @@
+[Connection]
+cacert=/very/long/path/to/self/signed/pem/file/caosdb.ca.pem
+url=https://hostname:8833/playground
+username=username
+password_method=pass
diff --git a/unittests/broken_configs/pycaosdb2.ini b/unittests/broken_configs/pycaosdb2.ini
new file mode 100644
index 0000000000000000000000000000000000000000..6bdf58ea812b83a622e647b2a18b01bb1a1e3099
--- /dev/null
+++ b/unittests/broken_configs/pycaosdb2.ini
@@ -0,0 +1,9 @@
+[Connection]
+url=https://0.0.0.0/
+username=username
+password_identifier=SECTION/SUBSECTION/identifier
+password_method=pass
+cacert=/etc/ssl/cert.pem
+ssl_insecure=true
+timeout=10000
+debug=9
diff --git a/unittests/broken_configs/pycaosdb3.ini b/unittests/broken_configs/pycaosdb3.ini
new file mode 100644
index 0000000000000000000000000000000000000000..62d1fed9497c0c258c13aa2ed8fecb23a3006849
--- /dev/null
+++ b/unittests/broken_configs/pycaosdb3.ini
@@ -0,0 +1,12 @@
+[connection]
+ssl_insecure=true
+url=https://localhost:10443/
+password=caosdb
+username=admin
+password_method=plain
+
+timeout=10000
+debug=0
+
+[Container]
+debug=0
diff --git a/unittests/broken_configs/pycaosdb4.ini b/unittests/broken_configs/pycaosdb4.ini
new file mode 100644
index 0000000000000000000000000000000000000000..e96604ac453ff2be4e2b419aa9ccbbf3598fa231
--- /dev/null
+++ b/unittests/broken_configs/pycaosdb4.ini
@@ -0,0 +1,10 @@
+[Connection]
+ssl_insecure=true
+url=https://localhost:10443/
+password=caosdb
+username=admin
+password_method=plain
+
+timeout=10000
+debug=0
+key=bla
\ No newline at end of file
diff --git a/unittests/data/list_in_value.xml b/unittests/data/list_in_value.xml
new file mode 100644
index 0000000000000000000000000000000000000000..0f92610d82caa5ced443b2f437f35da05b9e121a
--- /dev/null
+++ b/unittests/data/list_in_value.xml
@@ -0,0 +1,12 @@
+<Record id="1002" description="A description of this example experiment.">
+ <Version id="945c6858819d2609a5475ee4df64571984acd039" head="true">
+ <Predecessor id="0df3cfe164fbafe9777f9356d0be2403890c54cd" />
+ </Version>
+ <Parent id="1001" name="Experiment" />
+ <Property datatype="SomeRecordType" id="1003" name="DepthTest" importance="FIX">
+ <Value>1004</Value>
+ <Value>1005</Value>
+ </Property>
+</Record>
+
+<!-- Note: This XML is invalid, because list-valued Properties must have a LIST-Datatype -->
diff --git a/unittests/test_add_property.py b/unittests/test_add_property.py
index 5bae6c219732f0170f5c351eae58148c9d3d065a..0d3183b4c0ca5517ecea68d0e49bbf335bb2a13e 100644
--- a/unittests/test_add_property.py
+++ b/unittests/test_add_property.py
@@ -264,3 +264,33 @@ def test_add_list_of_entitities():
for e in rec.get_property("listOfEntities").value:
assert i == e.id
i += 1
+
+
+def test_add_property_with_wrong_role():
+ entity = db.Entity()
+
+ r = db.Record()
+ rt = db.RecordType()
+ p = db.Property()
+ f = db.File()
+ e = db.Entity()
+
+ entity.add_property(rt)
+ entity.add_property(p)
+ entity.add_property(e)
+
+ with raises(ValueError) as cm:
+ entity.add_property(r)
+ assert cm.value.args[0] == ("The property parameter is a Record. This is "
+ "very unusual and probably not what you want. "
+ "Otherwise, construct a property from a "
+ "Record using the Property class and add that "
+ "to this entity.")
+
+ with raises(ValueError) as cm:
+ entity.add_property(f)
+ assert cm.value.args[0] == ("The property parameter is a File. This is "
+ "very unusual and probably not what you want. "
+ "Otherwise, construct a property from a File "
+ "using the Property class and add that to "
+ "this entity.")
diff --git a/unittests/test_apiutils.py b/unittests/test_apiutils.py
index c560b5e3c7c424b762bc8381c7cc9f42617288d0..0294646f6c526230a8e9fb722d56aa23a8f9285c 100644
--- a/unittests/test_apiutils.py
+++ b/unittests/test_apiutils.py
@@ -1,12 +1,11 @@
# -*- encoding: utf-8 -*-
#
-# ** header v3.0
# This file is a part of the CaosDB Project.
#
# Copyright (C) 2018 Research Group Biomedical Physics,
# Max-Planck-Institute for Dynamics and Self-Organization Göttingen
# Copyright (C) 2020 Timm Fitschen <t.fitschen@indiscale.com>
-# Copyright (C) 2020 IndiScale GmbH <info@indiscale.com>
+# Copyright (C) 2020-2022 IndiScale GmbH <info@indiscale.com>
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as
@@ -26,10 +25,14 @@
# Test apiutils
# A. Schlemmer, 02/2018
-import caosdb as db
import pickle
import tempfile
-from caosdb.apiutils import apply_to_ids
+
+import caosdb as db
+import caosdb.apiutils
+from caosdb.apiutils import (apply_to_ids, compare_entities, create_id_query,
+ resolve_reference)
+
from .test_property import testrecord
@@ -62,3 +65,168 @@ def test_apply_to_ids():
assert rec.parents[0].id == -3456
assert rec.properties[0].id == -23345
assert rec.id == -23
+
+
+def test_id_query():
+ ids = [1, 2, 3, 4, 5]
+ assert create_id_query(ids) == 'FIND ENTITY WITH ID=1 OR ID=2 OR ID=3 OR '\
+ 'ID=4 OR ID=5'
+
+
+def test_resolve_reference():
+ original_retrieve_entity_with_id = caosdb.apiutils.retrieve_entity_with_id
+ caosdb.apiutils.retrieve_entity_with_id = lambda eid: db.Record(id=eid)
+
+ prop = db.Property(id=1, datatype=db.REFERENCE, value=100)
+ prop.is_valid = lambda: True
+ items = [200, 300, 400]
+ prop_list = db.Property(datatype=db.LIST(db.REFERENCE),
+ value=items)
+ prop_list2 = db.Property(datatype=db.LIST(db.REFERENCE),
+ value=[db.Record(id=500)])
+ resolve_reference(prop)
+ resolve_reference(prop_list)
+ resolve_reference(prop_list2)
+ assert prop.value.id == 100
+ assert isinstance(prop.value, db.Entity)
+
+ prop_list_ids = []
+
+ for i in prop_list.value:
+ prop_list_ids.append(i.id)
+ assert isinstance(i, db.Entity)
+ assert prop_list_ids == items
+
+ for i in prop_list2.value:
+ assert i.id == 500
+ assert isinstance(i, db.Entity)
+
+ no_reference = db.Property(id=5000, datatype=db.INTEGER, value=2)
+ resolve_reference(no_reference)
+ assert no_reference.value == 2
+ assert no_reference.datatype is db.INTEGER
+
+ # restore retrive_entity_with_id
+ caosdb.apiutils.retrieve_entity_with_id = original_retrieve_entity_with_id
+
+
+def test_compare_entities():
+ r1 = db.Record()
+ r2 = db.Record()
+ r1.add_parent("bla")
+ r2.add_parent("bla")
+ r1.add_parent("lopp")
+ r1.add_property("test", value=2)
+ r2.add_property("test", value=2)
+ r1.add_property("tests", value=3)
+ r2.add_property("tests", value=45)
+ r1.add_property("tester", value=3)
+ r2.add_property("tester", )
+ r1.add_property("tests_234234", value=45)
+ r2.add_property("tests_TT", value=45)
+
+ diff_r1, diff_r2 = compare_entities(r1, r2)
+
+ assert len(diff_r1["parents"]) == 1
+ assert len(diff_r2["parents"]) == 0
+ assert len(diff_r1["properties"]) == 3
+ assert len(diff_r2["properties"]) == 3
+
+ assert "test" not in diff_r1["properties"]
+ assert "test" not in diff_r2["properties"]
+
+ assert "tests" in diff_r1["properties"]
+ assert "tests" in diff_r2["properties"]
+
+ assert "tester" in diff_r1["properties"]
+ assert "tester" in diff_r2["properties"]
+
+ assert "tests_234234" in diff_r1["properties"]
+ assert "tests_TT" in diff_r2["properties"]
+
+
+def test_compare_entities_units():
+ r1 = db.Record()
+ r2 = db.Record()
+ r1.add_parent("bla")
+ r2.add_parent("bla")
+ r1.add_parent("lopp")
+ r1.add_property("test", value=2, unit="cm")
+ r2.add_property("test", value=2, unit="m")
+ r1.add_property("tests", value=3, unit="cm")
+ r2.add_property("tests", value=45, unit="cm")
+ r1.add_property("tester", value=3)
+ r2.add_property("tester", )
+ r1.add_property("tests_234234", value=45, unit="cm")
+ r2.add_property("tests_TT", value=45, unit="cm")
+
+ diff_r1, diff_r2 = compare_entities(r1, r2)
+
+ assert len(diff_r1["parents"]) == 1
+ assert len(diff_r2["parents"]) == 0
+ assert len(diff_r1["properties"]) == 4
+ assert len(diff_r2["properties"]) == 4
+
+ assert "tests" in diff_r1["properties"]
+ assert "tests" in diff_r2["properties"]
+
+ assert "tester" in diff_r1["properties"]
+ assert "tester" in diff_r2["properties"]
+
+ assert "tests_234234" in diff_r1["properties"]
+ assert "tests_TT" in diff_r2["properties"]
+
+ assert diff_r1["properties"]["test"]["unit"] == "cm"
+ assert diff_r2["properties"]["test"]["unit"] == "m"
+
+
+def test_compare_special_properties():
+ # Test for all known special properties:
+ SPECIAL_PROPERTIES = ("description", "name",
+ "checksum", "size", "path", "id")
+ INTS = ("size", "id")
+ HIDDEN = ("checksum", "size")
+
+ for key in SPECIAL_PROPERTIES:
+ set_key = key
+ if key in HIDDEN:
+ set_key = "_" + key
+ r1 = db.Record()
+ r2 = db.Record()
+ if key not in INTS:
+ setattr(r1, set_key, "bla 1")
+ setattr(r2, set_key, "bla 1")
+ else:
+ setattr(r1, set_key, 1)
+ setattr(r2, set_key, 1)
+
+ diff_r1, diff_r2 = compare_entities(r1, r2)
+ print(diff_r1)
+ print(diff_r2)
+ assert key not in diff_r1
+ assert key not in diff_r2
+ assert len(diff_r1["parents"]) == 0
+ assert len(diff_r2["parents"]) == 0
+ assert len(diff_r1["properties"]) == 0
+ assert len(diff_r2["properties"]) == 0
+
+ if key not in INTS:
+ setattr(r2, set_key, "bla test")
+ else:
+ setattr(r2, set_key, 2)
+
+ diff_r1, diff_r2 = compare_entities(r1, r2)
+ print(r1)
+ print(r2)
+ print(diff_r1)
+ print(diff_r2)
+ assert key in diff_r1
+ assert key in diff_r2
+ if key not in INTS:
+ assert diff_r1[key] == "bla 1"
+ assert diff_r2[key] == "bla test"
+ else:
+ assert diff_r1[key] == 1
+ assert diff_r2[key] == 2
+ assert len(diff_r1["properties"]) == 0
+ assert len(diff_r2["properties"]) == 0
diff --git a/unittests/test_authentication_plain.py b/unittests/test_authentication_plain.py
index 10cbc418df8bd81c81568a4df0cf1e8a4ac498f8..146b59889c71c86ea77fb4ae962118cdda1afb06 100644
--- a/unittests/test_authentication_plain.py
+++ b/unittests/test_authentication_plain.py
@@ -65,4 +65,6 @@ def test_subclass_configure():
def test_plain_has_logger():
p = PlainTextCredentialsProvider()
assert hasattr(p, "logger")
- assert p.logger.name == "authentication"
+ assert "authentication" in p.logger.name
+ assert "connection" in p.logger.name
+ assert "caosdb" in p.logger.name
diff --git a/unittests/test_configs/pycaosdb-indiscale-demo.ini b/unittests/test_configs/pycaosdb-indiscale-demo.ini
new file mode 100644
index 0000000000000000000000000000000000000000..9010343467f78c7c9c3e25ea3a57520deac18c8e
--- /dev/null
+++ b/unittests/test_configs/pycaosdb-indiscale-demo.ini
@@ -0,0 +1,12 @@
+[Connection]
+url=https://demo.indiscale.com/
+username=admin
+password=caosdb
+password_method=plain
+cacert=/etc/ssl/cert.pem
+
+timeout=10000
+debug=0
+
+[Container]
+debug=0
\ No newline at end of file
diff --git a/unittests/test_configs/pycaosdb1.ini b/unittests/test_configs/pycaosdb1.ini
new file mode 100644
index 0000000000000000000000000000000000000000..dcfa7c21fac735d81ab92b33f0abd31df25fc1ad
--- /dev/null
+++ b/unittests/test_configs/pycaosdb1.ini
@@ -0,0 +1,6 @@
+[Connection]
+cacert=/very/long/path/to/self/signed/pem/file/caosdb.ca.pem
+url=https://hostname:8833/playground
+password_identifier=SECTION/caosdb
+username=username
+password_method=pass
diff --git a/unittests/test_configs/pycaosdb2.ini b/unittests/test_configs/pycaosdb2.ini
new file mode 100644
index 0000000000000000000000000000000000000000..e5493bde3be0f84f38e427edb4d42fba9c75482d
--- /dev/null
+++ b/unittests/test_configs/pycaosdb2.ini
@@ -0,0 +1,9 @@
+[Connection]
+url=https://0.0.0.0/
+username=username
+password_identifier=SECTION/SUBSECTION/identifier
+password_method=pass
+cacert=/etc/ssl/cert.pem
+ssl_insecure=true
+timeout=10000
+debug=0
diff --git a/unittests/test_configs/pycaosdb3.ini b/unittests/test_configs/pycaosdb3.ini
new file mode 100644
index 0000000000000000000000000000000000000000..6c4934039c99855566f38c69f7511d774f81efbd
--- /dev/null
+++ b/unittests/test_configs/pycaosdb3.ini
@@ -0,0 +1,12 @@
+[Connection]
+ssl_insecure=true
+url=https://localhost:10443/
+password=caosdb
+username=admin
+password_method=plain
+
+timeout=10000
+debug=0
+
+[Container]
+debug=0
diff --git a/unittests/test_container.py b/unittests/test_container.py
index b34055372fc83a5608ffcf54423a601001add12b..0ac4be44826825aa3302119c8bca08f335ab68d3 100644
--- a/unittests/test_container.py
+++ b/unittests/test_container.py
@@ -25,28 +25,28 @@
"""Tests for the Container class."""
from __future__ import absolute_import
-import caosdb as c
+import caosdb as db
def test_get_property_values():
- rt_house = c.RecordType("House")
- rt_window = c.RecordType("Window")
- rt_owner = c.RecordType("Owner")
- p_height = c.Property("Height", datatype=c.DOUBLE)
+ rt_house = db.RecordType("House")
+ rt_window = db.RecordType("Window")
+ rt_owner = db.RecordType("Owner")
+ p_height = db.Property("Height", datatype=db.DOUBLE)
- window = c.Record().add_parent(rt_window)
+ window = db.Record().add_parent(rt_window)
window.id = 1001
window.add_property(p_height, 20.5, unit="m")
- owner = c.Record("The Queen").add_parent(rt_owner)
+ owner = db.Record("The Queen").add_parent(rt_owner)
- house = c.Record("Buckingham Palace")
+ house = db.Record("Buckingham Palace")
house.add_parent(rt_house)
house.add_property(rt_owner, owner)
house.add_property(rt_window, window)
house.add_property(p_height, 40.2, unit="ft")
- container = c.Container()
+ container = db.Container()
container.extend([
house,
owner
@@ -77,3 +77,70 @@ def test_get_property_values():
assert container.get_property_values("non-existing") == [(None,), (None,)]
assert container.get_property_values("name") == [(house.name,),
(owner.name,)]
+
+
+def test_container_dependencies_for_deletion():
+ not_included_rt = 1000
+ rt = db.RecordType("Just a RecordType")
+ rt.id = 1001
+ rt_record_with_parent = db.RecordType("Records with parent")
+ rt_record_with_parent.id = 1005
+ property_which_is_not_a_record = db.Property(
+ "Normal Property", datatype=db.DOUBLE, value=1006)
+ property_which_is_not_a_record.id = 1006
+ property_which_shall_be_deleted = db.Property(
+ "Normal Property 2", datatype=db.DOUBLE, value=1006)
+ property_which_shall_be_deleted .id = 1007
+
+ record_without_dependencies = db.Record().add_parent(not_included_rt)
+ record_without_dependencies.id = 2003
+
+ record_referenced = db.Record().add_parent(not_included_rt)
+ record_referenced.id = 2002
+ record_with_dependencies = db.Record().add_parent(not_included_rt)
+ record_with_dependencies.id = 2004
+ record_with_dependencies.add_property(not_included_rt,
+ record_referenced,
+ datatype="not_included_rt")
+
+ record_with_parent = db.Record().add_parent(rt_record_with_parent)
+ record_with_parent.id = 2005
+
+ record_with_property_which_is_not_a_record = db.Record(
+ ).add_parent(not_included_rt)
+ record_with_property_which_is_not_a_record.id = 2006
+ record_with_property_which_is_not_a_record.add_property(
+ property_which_is_not_a_record)
+ record_with_property_which_is_not_a_record.add_property(
+ property_which_shall_be_deleted)
+
+ container = db.Container()
+ container.extend([
+ rt,
+ rt_record_with_parent, # 1005, dependency
+ record_without_dependencies,
+ property_which_shall_be_deleted, # 1007, dependency
+ record_referenced, # 2002, dependency
+ record_with_dependencies,
+ record_with_parent,
+ record_with_property_which_is_not_a_record
+ ])
+ assert (db.Container()._test_dependencies_in_container(container)
+ == {2002, 1005, 1007})
+
+
+def test_container_dependencies_for_deletion_with_lists():
+ not_included_rt = 1000
+
+ record_referenced = db.Record().add_parent(not_included_rt)
+ record_referenced.id = 2001
+
+ record_with_list = db.Record().add_parent(not_included_rt)
+ record_with_list.id = 2002
+ record_with_list.add_property(not_included_rt, datatype=db.LIST(
+ not_included_rt), value=[record_referenced, 2003, 2004, 2005, 2006])
+
+ container = db.Container()
+ container.extend([record_with_list, record_referenced])
+
+ assert db.Container()._test_dependencies_in_container(container) == {2001}
diff --git a/unittests/test_entity.py b/unittests/test_entity.py
index e98dfbef5b6b5e5f691e8aecc2fa7d4a86991452..1e88702ac016d7dcfdf00919dd0f93b5d3345e00 100644
--- a/unittests/test_entity.py
+++ b/unittests/test_entity.py
@@ -24,6 +24,7 @@
"""Tests for the Entity class."""
# pylint: disable=missing-docstring
import unittest
+from lxml import etree
from caosdb import (INTEGER, Entity, Property, Record, RecordType,
configure_connection)
@@ -42,17 +43,54 @@ class TestEntity(unittest.TestCase):
def test_instance_variables(self):
entity = Entity()
self.assertTrue(hasattr(entity, "role"))
+ self.assertIsNone(entity.role)
self.assertTrue(hasattr(entity, "id"))
self.assertTrue(hasattr(entity, "name"))
self.assertTrue(hasattr(entity, "description"))
self.assertTrue(hasattr(entity, "parents"))
self.assertTrue(hasattr(entity, "properties"))
- def test_role(self):
+ def test_entity_role_1(self):
entity = Entity(role="TestRole")
self.assertEqual(entity.role, "TestRole")
entity.role = "TestRole2"
self.assertEqual(entity.role, "TestRole2")
- def test_instanciation(self):
+ def test_entity_role_2(self):
+ entity = Entity()
+
+ self.assertIsNone(entity.role)
+ self.assertEqual(entity.to_xml().tag, "Entity")
+
+ entity.role = "Record"
+ self.assertEqual(entity.role, "Record")
+ self.assertEqual(entity.to_xml().tag, "Record")
+
+ def test_recordtype_role(self):
+ entity = RecordType()
+
+ self.assertEqual(entity.role, "RecordType")
+ self.assertEqual(entity.to_xml().tag, "RecordType")
+
+ def test_property_role(self):
+ entity = Property()
+
+ self.assertEqual(entity.role, "Property")
+ self.assertEqual(entity.to_xml().tag, "Property")
+
+ def test_instantiation(self):
self.assertRaises(Exception, Entity())
+
+ def test_parse_role(self):
+ """During parsing, the role of an entity is set explicitely. All other
+ classes use the class name as a "natural" value for the role property.
+ """
+ parser = etree.XMLParser(remove_comments=True)
+ entity = Entity._from_xml(Entity(),
+ etree.parse("unittests/test_record.xml",
+ parser).getroot())
+
+ self.assertEqual(entity.role, "Record")
+ # test whether the __role property of this object has explicitely been
+ # set.
+ self.assertEqual(getattr(entity, "_Entity__role"), "Record")
diff --git a/unittests/test_issues.py b/unittests/test_issues.py
new file mode 100644
index 0000000000000000000000000000000000000000..1e649db4f23de67e55301e0a053fba70d14680b4
--- /dev/null
+++ b/unittests/test_issues.py
@@ -0,0 +1,39 @@
+# This file is a part of the CaosDB Project.
+#
+# Copyright (c) 2022 IndiScale GmbH
+# Copyright (c) 2022 Daniel Hornung (d.hornung@indiscale.com)
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+"""Test known issues to prevent regressions.
+"""
+
+import os
+
+import lxml
+import caosdb as db
+
+from pytest import raises
+
+
+def test_issue_100():
+ """_parse_value() fails for some list-valued content
+ """
+
+ # Parse from (invalid) XML file
+ filename = os.path.join(os.path.dirname(__file__), "data", "list_in_value.xml")
+ xml_el = lxml.etree.parse(filename).getroot()
+ with raises(db.ServerConfigurationException) as exc_info:
+ db.common.models._parse_single_xml_element(xml_el)
+ assert "invalid XML: List valued properties" in exc_info.value.msg
diff --git a/unittests/test_property.py b/unittests/test_property.py
index 752ee01f0eafef14dbffd1e62c99d1c816c45d05..7c756117765e510587c00d818e39fb3945d44c53 100644
--- a/unittests/test_property.py
+++ b/unittests/test_property.py
@@ -24,14 +24,18 @@
# ** end header
#
"""Tests for the Property class."""
+import os
+
+import caosdb as db
+from caosdb import Entity, Property, Record
# pylint: disable=missing-docstring
from lxml import etree
-from caosdb import Entity, Property, Record
parser = etree.XMLParser(remove_comments=True)
-testrecord = Record._from_xml(Record(),
- etree.parse("unittests/test_record.xml",
- parser).getroot())
+testrecord = Record._from_xml(
+ Record(),
+ etree.parse(os.path.join(os.path.dirname(__file__), "test_record.xml"),
+ parser).getroot())
def test_is_entity():
@@ -47,7 +51,8 @@ def test_instance_variables():
def test_null_empty_text_value_1():
- assert testrecord.get_property("LISTofTEXT").value == ["One", "Two", "Three", None, ""]
+ assert testrecord.get_property("LISTofTEXT").value == ["One", "Two",
+ "Three", None, ""]
def test_null_empty_text_value_2():
@@ -89,3 +94,47 @@ def test_get_property_with_entity():
def test_selected_reference_list():
assert len(testrecord.get_property("Conductor").value) == 1
assert isinstance(testrecord.get_property("Conductor").value[0], Entity)
+
+
+def test_is_reference():
+ PROPS = {
+ 10: db.INTEGER,
+ 20: db.REFERENCE,
+ 30: "SomeRT",
+ }
+
+ def dummy_retrieve(self):
+ self.datatype = PROPS[self.id]
+ self.is_valid = lambda: True
+ # replace retrieve function by dummy
+ real_retrieve = Entity.retrieve
+ Entity.retrieve = dummy_retrieve
+
+ p1 = Property(id=1, datatype=db.INTEGER)
+ p2 = Property(id=2, datatype=db.DOUBLE)
+ p3 = Property(id=3, datatype=db.TEXT)
+ p4 = Property(id=4, datatype=db.DATETIME)
+ p5 = Property(id=5, datatype=db.BOOLEAN)
+ p6 = Property(id=6, datatype=db.REFERENCE)
+ assert p1.is_reference() is False
+ assert p2.is_reference() is False
+ assert p3.is_reference() is False
+ assert p4.is_reference() is False
+ assert p5.is_reference() is False
+ assert p6.is_reference() is True
+
+ p7 = Property(id=7)
+ p8 = Property(id=8, value=db.RecordType(id=1000))
+ p8.is_valid = lambda: True
+ assert p7.is_reference() is None # cannot be resolved without calling a server
+ assert p8.is_reference() is True
+
+ p10 = Property(id=10)
+ p20 = Property(id=20)
+ p30 = Property(id=30)
+ assert p10.is_reference(server_retrieval=True) is False
+ assert p20.is_reference(server_retrieval=True) is True
+ assert p30.is_reference(server_retrieval=True) is True
+
+ # restore retrieve function with original
+ Entity.retrieve = real_retrieve
diff --git a/unittests/test_schema.py b/unittests/test_schema.py
new file mode 100644
index 0000000000000000000000000000000000000000..1552179a3e43dacb3ecca705466bb7ff84d330cf
--- /dev/null
+++ b/unittests/test_schema.py
@@ -0,0 +1,45 @@
+# -*- coding: utf-8 -*-
+#
+# This file is a part of the CaosDB Project.
+#
+# Copyright (C) 2021 Alexander Schlemmer
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see <https://www.gnu.org/licenses/>.
+#
+
+"""Test configuration schema.
+A. Schlemmer, 01/2021
+"""
+from jsonschema.exceptions import ValidationError
+from pytest import raises
+from glob import glob
+import os
+from caosdb.configuration import config_to_yaml, validate_yaml_schema
+from configparser import ConfigParser
+
+
+def test_config_files():
+ for fn in glob(os.path.join(os.path.dirname(__file__), "test_configs", "*.ini")):
+ c = ConfigParser()
+ c.read(fn)
+ validate_yaml_schema(config_to_yaml(c))
+
+
+def test_broken_config_files():
+ for fn in glob(os.path.join(os.path.dirname(__file__), "broken_configs", "*.ini")):
+ print(fn)
+ with raises(ValidationError):
+ c = ConfigParser()
+ c.read(fn)
+ validate_yaml_schema(config_to_yaml(c))
diff --git a/unittests/test_yamlapi.py b/unittests/test_yamlapi.py
new file mode 100644
index 0000000000000000000000000000000000000000..cdb1e0499890ee58d10ff7f102632e104ef60868
--- /dev/null
+++ b/unittests/test_yamlapi.py
@@ -0,0 +1,85 @@
+# -*- coding: utf-8 -*-
+#
+# This file is a part of the CaosDB Project.
+#
+# Copyright (C) 2021 IndiScale GmbH <info@indiscale.com>
+# Copyright (C) 2021 Alexander Kreft <akreft@trineo.org>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see <https://www.gnu.org/licenses/>.
+#
+
+import os
+import warnings
+import tempfile
+from caosdb.yamlapi import (append_sublist, kv_to_xml,
+ dict_to_xml, yaml_to_xml,
+ process, yaml_file_to_xml)
+
+with warnings.catch_warnings(record=True) as w:
+ # Cause all warnings to always be triggered.
+ warnings.simplefilter("always")
+
+ append_sublist(None, None, None)
+
+ assert issubclass(w[-1].category, DeprecationWarning)
+ assert "This function is deprecated" in str(w[-1].message)
+
+with warnings.catch_warnings(record=True) as w:
+ # Cause all warnings to always be triggered.
+ warnings.simplefilter("always")
+
+ kv_to_xml("None", "None")
+ assert len(w) == 1
+ assert issubclass(w[-1].category, DeprecationWarning)
+ assert "This function is deprecated" in str(w[-1].message)
+
+with warnings.catch_warnings(record=True) as w:
+ # Cause all warnings to always be triggered.
+ warnings.simplefilter("always")
+
+ dict_to_xml(None)
+
+ assert issubclass(w[-1].category, DeprecationWarning)
+ assert "This function is deprecated" in str(w[-1].message)
+
+with warnings.catch_warnings(record=True) as w:
+ # Cause all warnings to always be triggered.
+ warnings.simplefilter("always")
+
+ yaml_to_xml("None")
+
+ assert issubclass(w[-1].category, DeprecationWarning)
+ assert "This function is deprecated" in str(w[-1].message)
+
+with warnings.catch_warnings(record=True) as w:
+ # Cause all warnings to always be triggered.
+ warnings.simplefilter("always")
+
+ process("None")
+
+ assert issubclass(w[-1].category, DeprecationWarning)
+ assert "This function is deprecated" in str(w[-1].message)
+
+with warnings.catch_warnings(record=True) as w:
+ # Cause all warnings to always be triggered.
+ warnings.simplefilter("always")
+
+ with tempfile.TemporaryDirectory() as tmpdirname:
+ tmpfile = os.path.join(tmpdirname, 'yamlfile')
+ with open(tmpfile, 'w') as tf:
+ tf.write("")
+ yaml_file_to_xml(tmpfile)
+
+ assert issubclass(w[-1].category, DeprecationWarning)
+ assert "This function is deprecated" in str(w[-1].message)