diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 9bb5adb6ed4fedf57694d6ec7843942c46296269..1dc09269a92c486e9d80a8ae5ceb0e51dc50bd17 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -39,6 +39,7 @@ stages:
code_style:
tags: [ docker ]
stage: code_style
+ needs: [ ]
script:
- make style
allow_failure: true
@@ -47,17 +48,19 @@ code_style:
pylint:
tags: [ docker ]
stage: linting
+ needs: [ ]
script:
- make lint
allow_failure: true
-# run tests
-test:
+# run unit tests
+unittest:
tags: [ docker ]
stage: test
+ needs: [ ]
script:
- touch ~/.pycaosdb.ini
- - tox -r
+ - make unittest
# Trigger building of server image and integration tests
trigger_build:
@@ -96,6 +99,7 @@ build-testenv:
pages_prepare: &pages_prepare
tags: [ cached-dind ]
stage: deploy
+ needs: [ code_style, pylint, unittest ]
only:
refs:
- /^release-.*$/i
diff --git a/.gitlab/merge_request_templates/Default.md b/.gitlab/merge_request_templates/Default.md
deleted file mode 100644
index e43435cb31f415fc4f9c8447983b411627612ad7..0000000000000000000000000000000000000000
--- a/.gitlab/merge_request_templates/Default.md
+++ /dev/null
@@ -1,50 +0,0 @@
-# Summary
-
- Insert a meaningful description for this merge request here. What is the
- new/changed behavior? Which bug has been fixed? Are there related Issues?
-
-# Focus
-
- Point the reviewer to the core of the code change. Where should they start
- reading? What should they focus on (e.g. security, performance,
- maintainability, user-friendliness, compliance with the specs, finding more
- corner cases, concrete questions)?
-
-# Test Environment
-
- How to set up a test environment for manual testing?
-
-# Check List for the Author
-
-Please, prepare your MR for a review. Be sure to write a summary and a
-focus and create gitlab comments for the reviewer. They should guide the
-reviewer through the changes, explain your changes and also point out open
-questions. For further good practices have a look at [our review
-guidelines](https://gitlab.com/caosdb/caosdb/-/blob/dev/REVIEW_GUIDELINES.md)
-
-- [ ] All automated tests pass
-- [ ] Reference related Issues
-- [ ] Up-to-date CHANGELOG.md
-- [ ] Add type hints in created/changed code
-- [ ] Annotations in code (Gitlab comments)
- - Intent of new code
- - Problems with old code
- - Why this implementation?
-
-
-# Check List for the Reviewer
-
-
-- [ ] I understand the intent of this MR
-- [ ] All automated tests pass
-- [ ] Up-to-date CHANGELOG.md
-- [ ] The test environment setup works and the intended behavior is
- reproducible in the test environment
-- [ ] In-code documentation and comments are up-to-date.
-- [ ] Check: Are there spezifications? Are they satisfied?
-
-For further good practices have a look at [our review guidelines](https://gitlab.com/caosdb/caosdb/-/blob/dev/REVIEW_GUIDELINES.md).
-
-
-/assign me
-/target_branch dev
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 11212ffbf578160298c3abc72fe9aa366b4bb164..0d5a5ac2ef93edca05c8e977b4ebb99f0dd3008e 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -5,7 +5,33 @@ All notable changes to this project will be documented in this file.
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
-## [Unreleased] ##
+## [Unreleased]
+
+### Added ###
+
+- New function in apiutils that copies an Entity.
+
+### Changed ###
+
+- Added additional customization options to the plantuml module.
+- The to_graphics function in the plantuml module uses a temporary directory now for creating the output files.
+
+### Deprecated ###
+
+### Removed ###
+
+### Fixed ###
+
+* [#75](https://gitlab.indiscale.com/caosdb/src/caosdb-pylib/-/issues/75), [#103](https://gitlab.indiscale.com/caosdb/src/caosdb-pylib/-/issues/103) Fixed JSON schema to allow more sections, and correct requirements for
+ password method.
+
+### Security ###
+
+### Documentation ###
+
+
+## [0.7.2] - 2022-03-25 ##
+(Timm Fitschen)
### Added ###
@@ -13,12 +39,31 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
### Deprecated ###
+* In module `caosdb.apiutils`:
+ * `CaosDBPythonEntity` class
+ * `convert_to_entity` function
+ * `convert_to_python_object` function
+
### Removed ###
### Fixed ###
+* [caosdb-pylib#106](https://gitlab.indiscale.com/caosdb/src/caosdb-pylib/-/issues/106)
+ Parsing Error in class caosdb.common.models.ACL. This may lead to the
+ unintentional revocation of permissions for some users or roles during
+ updates. However, no additional permissions are being granted.
+
### Security ###
+### Documentation ###
+
+## [0.7.1] - 2022-03-11 ##
+(Daniel Hornung)
+
+### Documentation ###
+
+- `timeout` option in example pycaosdb.ini
+
## [0.7.0] - 2022-01-21 ##
### Added ###
diff --git a/DEPENDENCIES.md b/DEPENDENCIES.md
index 04e783a7fa31b1d5c3a600a1009c8f040db1620d..d6f91cb9e15982bd1b7b0366f94bd9b37286f85a 100644
--- a/DEPENDENCIES.md
+++ b/DEPENDENCIES.md
@@ -1,6 +1,5 @@
-* caosdb-server == 0.3
-* Python >= 3.5
+* caosdb-server >= 0.7.2
+* Python >= 3.6
* pip >= 20.0.2
-
-Any other dependencies are being installed via pip
+Any other dependencies are defined in the setup.py and are being installed via pip
diff --git a/Makefile b/Makefile
index 192337853d8db8812e14f75fca8986006de82180..0a0888ad0484c0307583e139e65058c38574ed3a 100644
--- a/Makefile
+++ b/Makefile
@@ -42,3 +42,7 @@ style:
lint:
pylint --unsafe-load-any-extension=y -d all -e E,F src/caosdb/common
.PHONY: lint
+
+unittest:
+ tox -r
+.PHONY: unittest
diff --git a/README_SETUP.md b/README_SETUP.md
index e58f934ceba176e4b5ba42239565f8e3bd48171a..dc667da8aa5877132c1212d2ddd2827e85992118 100644
--- a/README_SETUP.md
+++ b/README_SETUP.md
@@ -82,60 +82,8 @@ pip3 install --user .[jsonschema]
## Configuration ##
-The configuration is done using `ini` configuration files.
-PyCaosDB tries to read from the inifile specified in the environment variable `PYCAOSDBINI` or
-alternatively in `~/.pycaosdb.ini` upon import. After that, the ini file `pycaosdb.ini` in the
-current working directory will be read additionally, if it exists.
-
-Here, we will look at the most common configuration options. For a full and
-comprehensive description please check out
-[pycaosdb.ini file](https://gitlab.com/caosdb/caosdb-pylib/-/blob/main/examples/pycaosdb.ini)
-You can download this file and use it as a starting point.
-
-
-Typically, you need to change at least the `url` and `username` fields as required.
-(Ask your CaosDB administrator or IT crowd if
-you do not know what to put there, but for the demo instances https://demo.indiscale.com, `username=admin`
-and `password=caosdb` should work).
-
-### Authentication ###
-
-The default configuration (that your are asked for your password when ever a connection is created
-can be changed by setting `password_method`:
-
-* with `password_method=input` password (and possibly user) will be queried on demand (**default**)
-* use the password manager [pass](https://www.passwordstore.org) by using `pass` as value, see also the [ArchWiki
- entry](https://wiki.archlinux.org/index.php/Pass#Basic_usage). This also requires `password_identifier` which refers to the identifier within pass
- for the desired password.
-* install the python package [keyring](https://pypi.org/project/keyring), to use the system keyring/wallet (macOS, GNOME, KDE,
- Windows). The password will be queried on first usage.
-* with `password_method=plain` (**strongly discouraged**)
-
-The following illustrates the recommended options:
-
-```ini
-[Connection]
-# using "pass" password manager
-#password_method=pass
-#password_identifier=...
-
-# using the system keyring/wallet (macOS, GNOME, KDE, Windows)
-#password_method=keyring
-```
-
-### SSL Certificate ###
-In some cases (especially if you are testing CaosDB) you might need to supply
-an SSL certificate to allow SSL encryption.
-
-```ini
-[Connection]
-cacert=/path/to/caosdb.ca.pem
-```
-
-### Further Settings ###
-As mentioned above, a complete list of options can be found in the
-[pycaosdb.ini file](https://gitlab.com/caosdb/caosdb-pylib/-/blob/main/examples/pycaosdb.ini) in
-the examples folder of the source code.
+The configuration is done using `ini` configuration files. The content of these configuration files
+is described in detail in the [configuration section of the documentation](https://docs.indiscale.com/caosdb-pylib/configuration.html).
## Try it out ##
@@ -155,7 +103,10 @@ like this, check out the "Authentication" section in the [configuration document
Now would be a good time to continue with the [tutorials](tutorials/index).
## Run Unit Tests
-tox
+
+- Run all tests: `tox` or `make unittest`
+- Run a specific test file: e.g. `tox -- unittests/test_schema.py`
+- Run a specific test function: e.g. `tox -- unittests/test_schema.py::test_config_files`
## Documentation ##
diff --git a/RELEASE_GUIDELINES.md b/RELEASE_GUIDELINES.md
index e71234b8e2bc95f954ffbebdc26acf6edd8e0b2d..b4e38d643756798f0ba8b07d6eceec529cbb3054 100644
--- a/RELEASE_GUIDELINES.md
+++ b/RELEASE_GUIDELINES.md
@@ -36,8 +36,10 @@ guidelines of the CaosDB Project
9. Publish the release by executing `./release.sh` with uploads the caosdb
module to the Python Package Index [pypi.org](https://pypi.org).
-10. Merge the main branch back into the dev branch.
+10. Create a gitlab release on gitlab.indiscale.com and gitlab.com
-11. After the merge of main to dev, start a new development version by
+11. Merge the main branch back into the dev branch.
+
+12. After the merge of main to dev, start a new development version by
setting `ISRELEASED` to `False` and by increasing at least the `MICRO`
version in [setup.py](./setup.py) and preparing CHANGELOG.md.
diff --git a/examples/pycaosdb.ini b/examples/pycaosdb.ini
index edc32195fbb364bb355d67b8733e8c7bccbb0d34..8cf74e43c5db32ed139c4fe371a6c2b3831b2ee1 100644
--- a/examples/pycaosdb.ini
+++ b/examples/pycaosdb.ini
@@ -67,3 +67,6 @@
# This option is used internally and for testing. Do not override.
# implementation=_DefaultCaosDBServerConnection
+
+# The timeout for requests to the server.
+# timeout=1000
diff --git a/setup.py b/setup.py
index 310d77786949a9bc2982936457a90d73047f2b0c..def734ac1b5b4a648df58a6904d51835229dbb86 100755
--- a/setup.py
+++ b/setup.py
@@ -47,9 +47,13 @@ from setuptools import find_packages, setup
ISRELEASED = False
MAJOR = 0
-MINOR = 7
-MICRO = 1
-PRE = "" # e.g. rc0, alpha.1, 0.beta-23
+MINOR = 8
+MICRO = 0
+# Do not tag as pre-release until this commit
+# https://github.com/pypa/packaging/pull/515
+# has made it into a release. Probably we should wait for pypa/packaging>=21.4
+# https://github.com/pypa/packaging/releases
+PRE = "" # "dev" # e.g. rc0, alpha.1, 0.beta-23
if PRE:
VERSION = "{}.{}.{}-{}".format(MAJOR, MINOR, MICRO, PRE)
@@ -166,12 +170,13 @@ def setup_package():
packages=find_packages('src'),
python_requires='>=3.6',
package_dir={'': 'src'},
- install_requires=['lxml>=3.6.4',
- 'PyYaml>=3.12', 'future', 'PySocks>=1.6.7'],
+ install_requires=['lxml>=4.6.3',
+ 'PyYAML>=6.0', 'future', 'PySocks>=1.6.7'],
extras_require={'keyring': ['keyring>=13.0.0'],
- 'jsonschema': ['jsonschema==4.0.1']},
+ 'jsonschema': ['jsonschema>=4.4.0']},
setup_requires=["pytest-runner>=2.0,<3dev"],
- tests_require=["pytest", "pytest-cov", "coverage>=4.4.2", "jsonschema==4.0.1"],
+ tests_require=["pytest", "pytest-cov", "coverage>=4.4.2",
+ "jsonschema>=4.4.0"],
package_data={
'caosdb': ['cert/indiscale.ca.crt', 'schema-pycaosdb-ini.yml'],
},
diff --git a/src/caosdb/apiutils.py b/src/caosdb/apiutils.py
index a8256976b79d71f514fcc1dc9c868f1eeebc76e9..08f31daad56c0ab471322197cadc1a1378267f35 100644
--- a/src/caosdb/apiutils.py
+++ b/src/caosdb/apiutils.py
@@ -37,7 +37,9 @@ from caosdb.common.datatype import (BOOLEAN, DATETIME, DOUBLE, FILE, INTEGER,
REFERENCE, TEXT, is_reference)
from caosdb.common.models import (Container, Entity, File, Property, Query,
Record, RecordType, execute_query,
- get_config)
+ get_config, SPECIAL_ATTRIBUTES)
+
+import logging
def new_record(record_type, name=None, description=None,
@@ -141,6 +143,9 @@ class CaosDBPythonEntity(object):
_last_id = 0
def __init__(self):
+ warnings.warn("The CaosDBPythonEntity class is deprecated, replacements will be provided by"
+ " the high_level_api module.",
+ DeprecationWarning, stacklevel=2)
# Save a copy of the dry state
# of this object in order to be
# able to detect conflicts.
@@ -469,6 +474,9 @@ def _single_convert_to_entity(entity, robj, **kwargs):
def convert_to_entity(python_object, **kwargs):
+ warnings.warn("The convert_to_entity function is deprecated, replacement will be provided by "
+ "the high_level_api module.", DeprecationWarning, stacklevel=2)
+
if isinstance(python_object, Container):
# Create a list of objects:
@@ -490,6 +498,8 @@ def convert_to_entity(python_object, **kwargs):
def convert_to_python_object(entity):
""""""
+ warnings.warn("The convert_to_python_object function is deprecated, replacement will be "
+ "provided by the high_level_api module.", DeprecationWarning, stacklevel=2)
if isinstance(entity, Container):
# Create a list of objects:
@@ -557,10 +567,6 @@ def getCommitIn(folder):
return t.readline().strip()
-COMPARED = ["name", "role", "datatype", "description", "importance",
- "id", "path", "checksum", "size"]
-
-
def compare_entities(old_entity: Entity, new_entity: Entity):
"""
Compare two entites.
@@ -584,7 +590,7 @@ def compare_entities(old_entity: Entity, new_entity: Entity):
if old_entity is new_entity:
return (olddiff, newdiff)
- for attr in COMPARED:
+ for attr in SPECIAL_ATTRIBUTES:
try:
oldattr = old_entity.__getattribute__(attr)
old_entity_attr_exists = True
@@ -673,10 +679,77 @@ def compare_entities(old_entity: Entity, new_entity: Entity):
return (olddiff, newdiff)
+def merge_entities(entity_a: Entity, entity_b: Entity):
+ """
+ Merge entity_b into entity_a such that they have the same parents and properties.
+
+ datatype, unit, value, name and description will only be changed in entity_a if they
+ are None for entity_a and set for entity_b. If there is a corresponding value
+ for entity_a different from None a RuntimeError will be raised informing of an
+ unresolvable merge conflict.
+
+ The merge operation is done in place.
+
+ Returns entity_a.
+
+ WARNING: This function is currently experimental and insufficiently tested. Use with care.
+ """
+
+ logging.warning(
+ "This function is currently experimental and insufficiently tested. Use with care.")
+
+ # Compare both entities:
+ diff_r1, diff_r2 = compare_entities(entity_a, entity_b)
+
+ # Go through the comparison and try to apply changes to entity_a:
+ for key in diff_r2["parents"]:
+ entity_a.add_parent(entity_b.get_parent(key))
+
+ for key in diff_r2["properties"]:
+ if key in diff_r1["properties"]:
+ if ("importance" in diff_r1["properties"][key] and
+ "importance" in diff_r2["properties"][key]):
+ if (diff_r1["properties"][key]["importance"] !=
+ diff_r2["properties"][key]["importance"]):
+ raise NotImplementedError()
+ elif ("importance" in diff_r1["properties"][key] or
+ "importance" in diff_r2["properties"][key]):
+ raise NotImplementedError()
+
+ for attribute in ("datatype", "unit", "value"):
+ if diff_r1["properties"][key][attribute] is None:
+ setattr(entity_a.get_property(key), attribute,
+ diff_r2["properties"][key][attribute])
+ else:
+ raise RuntimeError("Merge conflict.")
+ else:
+ # TODO: This is a temporary FIX for
+ # https://gitlab.indiscale.com/caosdb/src/caosdb-pylib/-/issues/105
+ entity_a.add_property(id=entity_b.get_property(key).id,
+ name=entity_b.get_property(key).name,
+ datatype=entity_b.get_property(key).datatype,
+ value=entity_b.get_property(key).value,
+ unit=entity_b.get_property(key).unit,
+ importance=entity_b.get_importance(key))
+ # entity_a.add_property(
+ # entity_b.get_property(key),
+ # importance=entity_b.get_importance(key))
+
+ for special_attribute in ("name", "description"):
+ sa_a = getattr(entity_a, special_attribute)
+ sa_b = getattr(entity_b, special_attribute)
+ if sa_a != sa_b:
+ if sa_a is None:
+ setattr(entity_a, special_attribute, sa_b)
+ else:
+ raise RuntimeError("Merge conflict.")
+ return entity_a
+
+
def describe_diff(olddiff, newdiff, name=None, as_update=True):
description = ""
- for attr in list(set(list(olddiff.keys())+list(newdiff.keys()))):
+ for attr in list(set(list(olddiff.keys()) + list(newdiff.keys()))):
if attr == "parents" or attr == "properties":
continue
description += "{} differs:\n".format(attr)
diff --git a/src/caosdb/common/models.py b/src/caosdb/common/models.py
index ec45dec070d1ea731648fe8d45e44ba89e393f76..3421f9ce39fc848f774b5d5d38280434354da8de 100644
--- a/src/caosdb/common/models.py
+++ b/src/caosdb/common/models.py
@@ -5,9 +5,9 @@
#
# Copyright (C) 2018 Research Group Biomedical Physics,
# Max-Planck-Institute for Dynamics and Self-Organization Göttingen
-# Copyright (C) 2020 Indiscale GmbH <info@indiscale.com>
+# Copyright (C) 2020-2022 Indiscale GmbH <info@indiscale.com>
# Copyright (C) 2020 Florian Spreckelsen <f.spreckelsen@indiscale.com>
-# Copyright (C) 2020 Timm Fitschen <t.fitschen@indiscale.com>
+# Copyright (C) 2020-2022 Timm Fitschen <t.fitschen@indiscale.com>
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as
@@ -25,7 +25,14 @@
# ** end header
#
-"""missing docstring."""
+"""
+Collection of the central classes of the CaosDB client, namely the Entity class
+and all of its subclasses and the Container class which is used to carry out
+transactions.
+
+All additional classes are either important for the entities or the
+transactions.
+"""
from __future__ import print_function, unicode_literals
import re
@@ -55,6 +62,7 @@ from caosdb.exceptions import (AmbiguousEntityError, AuthorizationError,
EntityDoesNotExistError, EntityError,
EntityHasNoDatatypeError, HTTPURITooLongError,
MismatchingEntitiesError, QueryNotUniqueError,
+ ServerConfigurationException,
TransactionError, UniqueNamesError,
UnqualifiedParentsError,
UnqualifiedPropertiesError)
@@ -71,6 +79,10 @@ ALL = "ALL"
NONE = "NONE"
+SPECIAL_ATTRIBUTES = ["name", "role", "datatype", "description",
+ "id", "path", "checksum", "size"]
+
+
class Entity(object):
"""Entity is a generic CaosDB object.
@@ -113,6 +125,47 @@ class Entity(object):
self.id = id
self.state = None
+ def copy(self):
+ """
+ Return a copy of entity.
+
+ If deep == True return a deep copy, recursively copying all sub entities.
+
+ Standard properties are copied using add_property.
+ Special attributes, as defined by the global variable SPECIAL_ATTRIBUTES and additionaly
+ the "value" are copied using setattr.
+ """
+ if self.role == "File":
+ new = File()
+ elif self.role == "Property":
+ new = Property()
+ elif self.role == "RecordType":
+ new = RecordType()
+ elif self.role == "Record":
+ new = Record()
+ elif self.role == "Entity":
+ new = Entity()
+ else:
+ raise RuntimeError("Unkonwn role.")
+
+ # Copy special attributes:
+ # TODO: this might rise an exception when copying
+ # special file attributes like checksum and size.
+ for attribute in SPECIAL_ATTRIBUTES + ["value"]:
+ val = getattr(self, attribute)
+ if val is not None:
+ setattr(new, attribute, val)
+
+ # Copy parents:
+ for p in self.parents:
+ new.add_parent(p)
+
+ # Copy properties:
+ for p in self.properties:
+ new.add_property(p, importance=self.get_importance(p))
+
+ return new
+
@property
def version(self):
if self._version is not None or self._wrapped_entity is None:
@@ -268,14 +321,74 @@ class Entity(object):
self.__pickup = new_pickup
def grant(self, realm=None, username=None, role=None,
- permission=None, priority=False):
+ permission=None, priority=False, revoke_denial=True):
+ """Grant a permission to a user or role for this entity.
+
+ You must specify either only the username and the realm, or only the
+ role.
+
+ By default a previously existing denial rule would be revoked, because
+ otherwise this grant wouldn't have any effect. However, for keeping
+ contradicting rules pass revoke_denial=False.
+
+ Parameters
+ ----------
+ permission: str
+ The permission to be granted.
+ username : str, optional
+ The username. Exactly one is required, either the `username` or the
+ `role`.
+ realm: str, optional
+ The user's realm. Required when username is not None.
+ role: str, optional
+ The role (as in Role-Based Access Control). Exactly one is
+ required, either the `username` or the `role`.
+ priority: bool, default False
+ Whether this permission is granted with priority over non-priority
+ rules.
+ revoke_denial: bool, default True
+ Whether a contradicting denial (with same priority flag) in this
+ ACL will be revoked.
+ """
+ # @review Florian Spreckelsen 2022-03-17
self.acl.grant(realm=realm, username=username, role=role,
- permission=permission, priority=priority)
+ permission=permission, priority=priority,
+ revoke_denial=revoke_denial)
def deny(self, realm=None, username=None, role=None,
- permission=None, priority=False):
+ permission=None, priority=False, revoke_grant=True):
+ """Deny a permission to a user or role for this entity.
+
+ You must specify either only the username and the realm, or only the
+ role.
+
+ By default a previously existing grant rule would be revoked, because
+ otherwise this denial would override the grant rules anyways. However,
+ for keeping contradicting rules pass revoke_grant=False.
+
+ Parameters
+ ----------
+ permission: str
+ The permission to be denied.
+ username : str, optional
+ The username. Exactly one is required, either the `username` or the
+ `role`.
+ realm: str, optional
+ The user's realm. Required when username is not None.
+ role: str, optional
+ The role (as in Role-Based Access Control). Exactly one is
+ required, either the `username` or the `role`.
+ priority: bool, default False
+ Whether this permission is denied with priority over non-priority
+ rules.
+ revoke_grant: bool, default True
+ Whether a contradicting grant (with same priority flag) in this
+ ACL will be revoked.
+ """
+ # @review Florian Spreckelsen 2022-03-17
self.acl.deny(realm=realm, username=username, role=role,
- permission=permission, priority=priority)
+ permission=permission, priority=priority,
+ revoke_grant=revoke_grant)
def revoke_denial(self, realm=None, username=None,
role=None, permission=None, priority=False):
@@ -1195,6 +1308,10 @@ class Entity(object):
def _parse_value(datatype, value):
+ """Parse the value (from XML input) according to the given datatype
+ """
+
+ # Simple values
if value is None:
return value
@@ -1215,12 +1332,12 @@ def _parse_value(datatype, value):
else:
raise ValueError("Boolean value was {}.".format(value))
+ # Datetime and text are returned as-is
if datatype in [DATETIME, TEXT]:
if isinstance(value, str):
return value
# deal with collections
-
if isinstance(datatype, str):
matcher = re.compile(r"^(?P<col>[^<]+)<(?P<dt>[^>]+)>$")
m = matcher.match(datatype)
@@ -1245,12 +1362,10 @@ def _parse_value(datatype, value):
# This is for a special case, where the xml parser could not differentiate
# between single values and lists with one element. As
-
if hasattr(value, "__len__") and len(value) == 1:
return _parse_value(datatype, value[0])
# deal with references
-
if isinstance(value, Entity):
return value
@@ -1266,6 +1381,12 @@ def _parse_value(datatype, value):
# reference via name
return str(value)
+ except TypeError:
+ # deal with invalid XML: List of values without appropriate datatype
+ if isinstance(value, list):
+ raise ServerConfigurationException(
+ "The server sent an invalid XML: List valued properties must be announced by "
+ "the datatype.\n" + f"Datatype: {datatype}\nvalue: {value}")
def _log_request(request, xml_body=None):
@@ -3627,13 +3748,15 @@ class ACI():
self.permission = permission
def __hash__(self):
- return hash(str(self.realm) + ":" + str(self.username) +
- ":" + str(self.role) + ":" + str(self.permission))
+ return hash(self.__repr__())
def __eq__(self, other):
return isinstance(other, ACI) and (self.role is None and self.username == other.username and self.realm ==
other.realm) or self.role == other.role and self.permission == other.permission
+ def __repr__(self):
+ return str(self.realm) + ":" + str(self.username) + ":" + str(self.role) + ":" + str(self.permission)
+
def add_to_element(self, e):
if self.role is not None:
e.set("role", self.role)
@@ -3658,10 +3781,35 @@ class ACL():
self.clear()
def parse_xml(self, xml):
+ """Clear this ACL and parse the xml.
+
+ Iterate over the rules in the xml and add each rule to this ACL.
+
+ Contradicting rules will both be kept.
+
+ Parameters
+ ----------
+ xml : lxml.etree.Element
+ The xml element containing the ACL rules, i.e. <Grant> and <Deny>
+ rules.
+ """
self.clear()
self._parse_xml(xml)
def _parse_xml(self, xml):
+ """Parse the xml.
+
+ Iterate over the rules in the xml and add each rule to this ACL.
+
+ Contradicting rules will both be kept.
+
+ Parameters
+ ----------
+ xml : lxml.etree.Element
+ The xml element containing the ACL rules, i.e. <Grant> and <Deny>
+ rules.
+ """
+ # @review Florian Spreckelsen 2022-03-17
for e in xml:
role = e.get("role")
username = e.get("username")
@@ -3674,10 +3822,12 @@ class ACL():
if e.tag == "Grant":
self.grant(username=username, realm=realm, role=role,
- permission=permission, priority=priority)
+ permission=permission, priority=priority,
+ revoke_denial=False)
elif e.tag == "Deny":
self.deny(username=username, realm=realm, role=role,
- permission=permission, priority=priority)
+ permission=permission, priority=priority,
+ revoke_grant=False)
def combine(self, other):
""" Combine and return new instance."""
@@ -3755,12 +3905,42 @@ class ACL():
if item in self._denials:
self._denials.remove(item)
- def grant(self, username=None, realm=None, role=None,
- permission=None, priority=False):
+ def grant(self, permission, username=None, realm=None, role=None,
+ priority=False, revoke_denial=True):
+ """Grant a permission to a user or role.
+
+ You must specify either only the username and the realm, or only the
+ role.
+
+ By default a previously existing denial rule would be revoked, because
+ otherwise this grant wouldn't have any effect. However, for keeping
+ contradicting rules pass revoke_denial=False.
+
+ Parameters
+ ----------
+ permission: str
+ The permission to be granted.
+ username : str, optional
+ The username. Exactly one is required, either the `username` or the
+ `role`.
+ realm: str, optional
+ The user's realm. Required when username is not None.
+ role: str, optional
+ The role (as in Role-Based Access Control). Exactly one is
+ required, either the `username` or the `role`.
+ priority: bool, default False
+ Whether this permission is granted with priority over non-priority
+ rules.
+ revoke_denial: bool, default True
+ Whether a contradicting denial (with same priority flag) in this
+ ACL will be revoked.
+ """
+ # @review Florian Spreckelsen 2022-03-17
priority = self._get_boolean_priority(priority)
item = ACI(role=role, username=username,
realm=realm, permission=permission)
- self._remove_item(item, priority)
+ if revoke_denial:
+ self._remove_item(item, priority)
if priority is True:
self._priority_grants.add(item)
@@ -3768,11 +3948,41 @@ class ACL():
self._grants.add(item)
def deny(self, username=None, realm=None, role=None,
- permission=None, priority=False):
+ permission=None, priority=False, revoke_grant=True):
+ """Deny a permission to a user or role for this entity.
+
+ You must specify either only the username and the realm, or only the
+ role.
+
+ By default a previously existing grant rule would be revoked, because
+ otherwise this denial would override the grant rules anyways. However,
+ for keeping contradicting rules pass revoke_grant=False.
+
+ Parameters
+ ----------
+ permission: str
+ The permission to be denied.
+ username : str, optional
+ The username. Exactly one is required, either the `username` or the
+ `role`.
+ realm: str, optional
+ The user's realm. Required when username is not None.
+ role: str, optional
+ The role (as in Role-Based Access Control). Exactly one is
+ required, either the `username` or the `role`.
+ priority: bool, default False
+ Whether this permission is denied with priority over non-priority
+ rules.
+ revoke_grant: bool, default True
+ Whether a contradicting grant (with same priority flag) in this
+ ACL will be revoked.
+ """
+ # @review Florian Spreckelsen 2022-03-17
priority = self._get_boolean_priority(priority)
item = ACI(role=role, username=username,
realm=realm, permission=permission)
- self._remove_item(item, priority)
+ if revoke_grant:
+ self._remove_item(item, priority)
if priority is True:
self._priority_denials.add(item)
diff --git a/src/caosdb/configuration.py b/src/caosdb/configuration.py
index 51e3749aaca3045afec9334ef987a174d5d19f26..75827df0d00d6c82251c2c04fa47413ac2801928 100644
--- a/src/caosdb/configuration.py
+++ b/src/caosdb/configuration.py
@@ -84,23 +84,28 @@ def config_to_yaml(config):
def validate_yaml_schema(valobj):
- # TODO: Re-enable warning once the schema has been extended to also cover
- # SSS pycaosdb.inis and integration tests.
if optional_jsonschema_validate:
with open(os.path.join(os.path.dirname(__file__), "schema-pycaosdb-ini.yml")) as f:
schema = yaml.load(f, Loader=yaml.SafeLoader)
optional_jsonschema_validate(instance=valobj, schema=schema["schema-pycaosdb-ini"])
- # else:
- # warnings.warn("""
- # Warning: The validation could not be performed because `jsonschema` is not installed.
- # """)
+ else:
+ warnings.warn("""
+ Warning: The validation could not be performed because `jsonschema` is not installed.
+ """)
def _read_config_files():
- """Function to read config files from different paths. Checks for path in $PYCAOSDBINI or home directory (.pycaosdb.ini) and in the current working directory (pycaosdb.ini).
+ """Function to read config files from different paths.
+
+ Checks for path either in ``$PYCAOSDBINI`` or home directory (``.pycaosdb.ini``), and
+ additionally in the current working directory (``pycaosdb.ini``).
+
+ Returns
+ -------
+
+ ini files: list
+ The successfully parsed ini-files. Order: env_var or home directory, cwd. Used for testing the function.
- Returns:
- [list]: list with successfully parsed ini-files. Order: env_var or home directory, cwd. Used for testing the function.
"""
return_var = []
if "PYCAOSDBINI" in environ:
diff --git a/src/caosdb/schema-pycaosdb-ini.yml b/src/caosdb/schema-pycaosdb-ini.yml
index bfe8fe7c63679507bba795bb45d7afa2b097f07b..5dabdd89795e19a757209e03cc843776be705777 100644
--- a/src/caosdb/schema-pycaosdb-ini.yml
+++ b/src/caosdb/schema-pycaosdb-ini.yml
@@ -65,26 +65,39 @@ schema-pycaosdb-ini:
properties:
password_method:
const: input
+ required: [password_method]
then:
required: [url]
- if:
properties:
password_method:
const: plain
+ required: [password_method]
then:
required: [url, username, password]
- if:
properties:
password_method:
const: pass
+ required: [password_method]
then:
required: [url, username, password_identifier]
- if:
properties:
password_method:
const: keyring
+ required: [password_method]
then:
required: [url, username]
IntegrationTests:
description: "Used by the integration test suite from the caosdb-pyinttest repo."
additionalProperties: true
+ Misc:
+ description: "Some additional configuration settings."
+ additionalProperties: true
+ advancedtools:
+ description: "Configuration settings for the caosadvancedtools."
+ additionalProperties: true
+ sss_helper:
+ description: "Configuration settings for server-side scripting."
+ additionalProperties: true
diff --git a/src/caosdb/utils/plantuml.py b/src/caosdb/utils/plantuml.py
index be34b2604f3682bb71b48bbd73e00fe854b3af51..acf218c8297028acba1cbcceabd9ba4398b0e7aa 100644
--- a/src/caosdb/utils/plantuml.py
+++ b/src/caosdb/utils/plantuml.py
@@ -34,10 +34,15 @@ plantuml FILENAME.pu -> FILENAME.png
"""
import os
+import shutil
import caosdb as db
from caosdb.common.datatype import is_reference, get_referenced_recordtype
+from typing import Optional
+
+import tempfile
+
REFERENCE = "REFERENCE"
@@ -79,13 +84,23 @@ class Grouped(object):
return self.parents
-def recordtypes_to_plantuml_string(iterable):
+def recordtypes_to_plantuml_string(iterable,
+ add_properties: bool = True,
+ add_recordtypes: bool = True,
+ add_legend: bool = True,
+ style: str = "default"):
"""Converts RecordTypes into a string for PlantUML.
This function obtains an iterable and returns a string which can
be input into PlantUML for a representation of all RecordTypes in
the iterable.
+ Current options for style
+ -------------------------
+
+ "default" - Standard rectangles with uml class circle and methods section
+ "salexan" - Round rectangles, hide circle and methods section
+
Current limitations
-------------------
@@ -96,6 +111,8 @@ def recordtypes_to_plantuml_string(iterable):
- Inheritance of Properties is not rendered nicely at the moment.
"""
+ # TODO: This function needs a review of python type hints.
+
classes = [el for el in iterable
if isinstance(el, db.RecordType)]
dependencies = {}
@@ -140,74 +157,87 @@ def recordtypes_to_plantuml_string(iterable):
return result
result = "@startuml\n\n"
- result += "skinparam classAttributeIconSize 0\n"
- result += "package Properties #DDDDDD {\n"
+ if style == "default":
+ result += "skinparam classAttributeIconSize 0\n"
+ elif style == "salexan":
+ result += """skinparam roundcorner 20\n
+skinparam boxpadding 20\n
+\n
+hide methods\n
+hide circle\n
+"""
+ else:
+ raise ValueError("Unknown style.")
- for p in properties:
- inheritances[p] = p.get_parents()
- dependencies[p] = []
+ if add_properties:
+ result += "package Properties #DDDDDD {\n"
+ for p in properties:
+ inheritances[p] = p.get_parents()
+ dependencies[p] = []
- result += "class \"{klass}\" << (P,#008800) >> {{\n".format(klass=p.name)
+ result += "class \"{klass}\" << (P,#008800) >> {{\n".format(klass=p.name)
- if p.description is not None:
- result += get_description(p.description)
- result += "\n..\n"
+ if p.description is not None:
+ result += get_description(p.description)
+ result += "\n..\n"
- if isinstance(p.datatype, str):
- result += "datatype: " + p.datatype + "\n"
- elif isinstance(p.datatype, db.Entity):
- result += "datatype: " + p.datatype.name + "\n"
- else:
- result += "datatype: " + str(p.datatype) + "\n"
+ if isinstance(p.datatype, str):
+ result += "datatype: " + p.datatype + "\n"
+ elif isinstance(p.datatype, db.Entity):
+ result += "datatype: " + p.datatype.name + "\n"
+ else:
+ result += "datatype: " + str(p.datatype) + "\n"
+ result += "}\n\n"
result += "}\n\n"
- result += "}\n\n"
- result += "package RecordTypes #DDDDDD {\n"
+ if add_recordtypes:
+ result += "package RecordTypes #DDDDDD {\n"
- for c in classes:
- inheritances[c] = c.get_parents()
- dependencies[c] = []
- result += "class \"{klass}\" << (C,#FF1111) >> {{\n".format(klass=c.name)
+ for c in classes:
+ inheritances[c] = c.get_parents()
+ dependencies[c] = []
+ result += "class \"{klass}\" << (C,#FF1111) >> {{\n".format(klass=c.name)
- if c.description is not None:
- result += get_description(c.description)
+ if c.description is not None:
+ result += get_description(c.description)
- props = ""
- props += _add_properties(c, importance=db.FIX)
- props += _add_properties(c, importance=db.OBLIGATORY)
- props += _add_properties(c, importance=db.RECOMMENDED)
- props += _add_properties(c, importance=db.SUGGESTED)
+ props = ""
+ props += _add_properties(c, importance=db.FIX)
+ props += _add_properties(c, importance=db.OBLIGATORY)
+ props += _add_properties(c, importance=db.RECOMMENDED)
+ props += _add_properties(c, importance=db.SUGGESTED)
- if len(props) > 0:
- result += "__Properties__\n" + props
- else:
- result += "\n..\n"
- result += "}\n\n"
+ if len(props) > 0:
+ result += "__Properties__\n" + props
+ else:
+ result += "\n..\n"
+ result += "}\n\n"
- for g in grouped:
- inheritances[g] = g.get_parents()
- result += "class \"{klass}\" << (G,#0000FF) >> {{\n".format(klass=g.name)
- result += "}\n\n"
+ for g in grouped:
+ inheritances[g] = g.get_parents()
+ result += "class \"{klass}\" << (G,#0000FF) >> {{\n".format(klass=g.name)
+ result += "}\n\n"
- for c, parents in inheritances.items():
- for par in parents:
- result += "\"{par}\" <|-- \"{klass}\"\n".format(
- klass=c.name, par=par.name)
+ for c, parents in inheritances.items():
+ for par in parents:
+ result += "\"{par}\" <|-- \"{klass}\"\n".format(
+ klass=c.name, par=par.name)
- for c, deps in dependencies.items():
- for dep in deps:
- result += "\"{klass}\" *-- \"{dep}\"\n".format(
- klass=c.name, dep=dep)
+ for c, deps in dependencies.items():
+ for dep in deps:
+ result += "\"{klass}\" *-- \"{dep}\"\n".format(
+ klass=c.name, dep=dep)
- result += """
+ if add_legend:
+ result += """
package \"B is a subtype of A\" <<Rectangle>> {
A <|-right- B
note "This determines what you find when you query for the RecordType.\\n'FIND RECORD A' will provide Records which have a parent\\nA or B, while 'FIND RECORD B' will provide only Records which have a parent B." as N1
}
"""
- result += """
+ result += """
package \"The property P references an instance of D\" <<Rectangle>> {
class C {
@@ -246,7 +276,8 @@ def retrieve_substructure(start_record_types, depth, result_id_set=None, result_
Returns
-------
db.Container
- A container containing all the retrieved entites or None if cleanup is False.
+ A container containing all the retrieved entites
+ or None if cleanup is False.
"""
# Initialize the id set and result container for level zero recursion depth:
if result_id_set is None:
@@ -260,9 +291,11 @@ def retrieve_substructure(start_record_types, depth, result_id_set=None, result_
result_container.append(entity)
result_id_set.add(entity.id)
for prop in entity.properties:
- if is_reference(prop.datatype) and prop.datatype != db.FILE and depth > 0:
- rt = db.RecordType(name=get_referenced_recordtype(prop.datatype)).retrieve()
- retrieve_substructure([rt], depth-1, result_id_set, result_container, False)
+ if (is_reference(prop.datatype) and prop.datatype != db.FILE and depth > 0):
+ rt = db.RecordType(
+ name=get_referenced_recordtype(prop.datatype)).retrieve()
+ retrieve_substructure([rt], depth-1, result_id_set,
+ result_container, False)
if prop.id not in result_id_set:
result_container.append(prop)
@@ -274,14 +307,22 @@ def retrieve_substructure(start_record_types, depth, result_id_set=None, result_
result_container.append(rt)
result_id_set.add(parent.id)
if depth > 0:
- retrieve_substructure([rt], depth-1, result_id_set, result_container, False)
+ retrieve_substructure([rt], depth-1, result_id_set,
+ result_container, False)
if cleanup:
return result_container
return None
-def to_graphics(recordtypes, filename):
+def to_graphics(recordtypes: list[db.Entity], filename: str,
+ output_dirname: Optional[str] = None,
+ formats: list[str] = ["tsvg"],
+ silent: bool = True,
+ add_properties: bool = True,
+ add_recordtypes: bool = True,
+ add_legend: bool = True,
+ style: str = "default"):
"""Calls recordtypes_to_plantuml_string(), saves result to file and
creates an svg image
@@ -293,17 +334,52 @@ def to_graphics(recordtypes, filename):
Iterable with the entities to be displayed.
filename : str
filename of the image without the extension(e.g. data_structure;
+ also without the preceeding path.
data_structure.pu and data_structure.svg will be created.)
+ output_dirname : str
+ the destination directory for the resulting images as defined by the "-o"
+ option by plantuml
+ default is to use current working dir
+ formats : list[str]
+ list of target formats as defined by the -t"..." options by plantuml, e.g. "tsvg"
+ silent : bool
+ Don't output messages.
"""
- pu = recordtypes_to_plantuml_string(recordtypes)
-
- pu_filename = filename+".pu"
- with open(pu_filename, "w") as pu_file:
- pu_file.write(pu)
-
- cmd = "plantuml -tsvg %s" % pu_filename
- print("Executing:", cmd)
-
- if os.system(cmd) != 0:
- raise Exception("An error occured during the execution of plantuml. "
- "Is plantuml installed?")
+ pu = recordtypes_to_plantuml_string(recordtypes,
+ add_properties,
+ add_recordtypes,
+ add_legend,
+ style)
+
+ if output_dirname is None:
+ output_dirname = os.getcwd()
+
+ allowed_formats = [
+ "tpng", "tsvg", "teps", "tpdf", "tvdx", "txmi",
+ "tscxml", "thtml", "ttxt", "tutxt", "tlatex", "tlatex:nopreamble"]
+
+ with tempfile.TemporaryDirectory() as td:
+
+ pu_filename = os.path.join(td, filename + ".pu")
+ with open(pu_filename, "w") as pu_file:
+ pu_file.write(pu)
+
+ for format in formats:
+ extension = format[1:]
+ if ":" in extension:
+ extension = extension[:extension.index(":")]
+
+ if format not in allowed_formats:
+ raise RuntimeError("Format not allowed.")
+ cmd = "plantuml -{} {}".format(format, pu_filename)
+ if not silent:
+ print("Executing:", cmd)
+
+ if os.system(cmd) != 0: # TODO: replace with subprocess.run
+ raise Exception("An error occured during the execution of "
+ "plantuml when using the format {}. "
+ "Is plantuml installed? "
+ "You might want to dry a different format.".format(format))
+ # copy only the final product into the target directory
+ shutil.copy(os.path.join(td, filename + "." + extension),
+ output_dirname)
diff --git a/src/doc/conf.py b/src/doc/conf.py
index b05fa1c71c1dcd0b59916594818449d2ebc574bd..ce1cfd261cb8b8d5aac8022d969c765b1c45fae3 100644
--- a/src/doc/conf.py
+++ b/src/doc/conf.py
@@ -25,14 +25,14 @@ import sphinx_rtd_theme # noqa: E402
# -- Project information -----------------------------------------------------
project = 'pycaosdb'
-copyright = '2020, IndiScale GmbH'
+copyright = '2022, IndiScale GmbH'
author = 'Daniel Hornung'
# The short X.Y version
-version = '0.5.2'
+version = '0.8.0'
# The full version, including alpha/beta/rc tags
# release = '0.5.2-rc2'
-release = '0.5.2'
+release = '0.8.0-dev'
# -- General configuration ---------------------------------------------------
diff --git a/src/doc/configuration.md b/src/doc/configuration.md
index 6e53542f661dcae94622fef24a67cecf7491df9c..02cbbd7b13d916a676ad26c277e370ae76bf3725 100644
--- a/src/doc/configuration.md
+++ b/src/doc/configuration.md
@@ -4,6 +4,15 @@ PyCaosDB tries to read from the inifile specified in the environment variable `P
alternatively in `~/.pycaosdb.ini` upon import. After that, the ini file `pycaosdb.ini` in the
current working directory will be read additionally, if it exists.
+Here, we will look at the most common configuration options. For a full and comprehensive
+description please check out the [example pycaosdb.ini file](https://gitlab.com/caosdb/caosdb-pylib/-/blob/main/examples/pycaosdb.ini). You can download this file and use
+it as a starting point.
+
+
+Typically, you need to change at least the `url` and `username` fields as required. (Ask your
+CaosDB administrator or IT crowd if you do not know what to put there, but for the demo instance at
+https://demo.indiscale.com, `username=admin` and `password=caosdb` should work).
+
## Authentication ##
The default configuration (that your are asked for your password when ever a connection is created
@@ -17,6 +26,8 @@ can be changed by setting `password_method`:
Windows). The password will be queried on first usage.
* with `password_method=plain` (**strongly discouraged**)
+The following illustrates the recommended options:
+
```ini
[Connection]
username=YOUR_USERNAME
@@ -35,7 +46,10 @@ username=YOUR_USERNAME
## SSL Certificate ##
-You can set the pass to the ssl certificate to be used:
+In some cases (especially if you are testing CaosDB) you might need to supply an SSL certificate to
+allow SSL encryption.
+
+The `cacert` option sets the path to the ssl certificate for the connection:
```ini
[Connection]
@@ -49,6 +63,8 @@ with CaosDB which makes the experience much less verbose. Set it to 1 or 2 in ca
debugging (which I hope will not be necessary for this tutorial) or if you want to learn more about
the internals of the protocol.
+`timeout` sets the timeout for requests to the server.
+
A complete list of options can be found in the
[pycaosdb.ini file](https://gitlab.com/caosdb/caosdb-pylib/-/blob/main/examples/pycaosdb.ini) in
the examples folder of the source code.
diff --git a/tox.ini b/tox.ini
index b1061a57c6a136cb29f77a1d0c03383ab82ecf8b..0d245e03ef9c8fe2151e173cd1a10964d47ef82b 100644
--- a/tox.ini
+++ b/tox.ini
@@ -9,3 +9,6 @@ deps = .
pytest-cov
jsonschema==4.0.1
commands=py.test --cov=caosdb -vv {posargs}
+
+[flake8]
+max-line-length=100
diff --git a/unittests/data/list_in_value.xml b/unittests/data/list_in_value.xml
new file mode 100644
index 0000000000000000000000000000000000000000..0f92610d82caa5ced443b2f437f35da05b9e121a
--- /dev/null
+++ b/unittests/data/list_in_value.xml
@@ -0,0 +1,12 @@
+<Record id="1002" description="A description of this example experiment.">
+ <Version id="945c6858819d2609a5475ee4df64571984acd039" head="true">
+ <Predecessor id="0df3cfe164fbafe9777f9356d0be2403890c54cd" />
+ </Version>
+ <Parent id="1001" name="Experiment" />
+ <Property datatype="SomeRecordType" id="1003" name="DepthTest" importance="FIX">
+ <Value>1004</Value>
+ <Value>1005</Value>
+ </Property>
+</Record>
+
+<!-- Note: This XML is invalid, because list-valued Properties must have a LIST-Datatype -->
diff --git a/unittests/test_acl.py b/unittests/test_acl.py
new file mode 100644
index 0000000000000000000000000000000000000000..633c25ad5c4046c0fa41b66049bdf56aa695f482
--- /dev/null
+++ b/unittests/test_acl.py
@@ -0,0 +1,55 @@
+# -*- encoding: utf-8 -*-
+#
+# This file is a part of the CaosDB Project.
+#
+# Copyright (C) 2022 Indiscale GmbH <info@indiscale.com>
+# Copyright (C) 2022 Timm Fitschen <f.fitschen@indiscale.com>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see <https://www.gnu.org/licenses/>.
+#
+import caosdb as db
+from lxml import etree
+
+
+def test_parse_xml():
+ # @review Florian Spreckelsen 2022-03-17
+ xml_str = """
+ <EntityACL>
+ <Grant priority="False" role="role1">
+ <Permission name="RETRIEVE:ENTITY"/>
+ </Grant>
+ <Deny priority="False" role="role1">
+ <Permission name="RETRIEVE:ENTITY"/>
+ </Deny>
+ <Grant priority="True" role="role1">
+ <Permission name="RETRIEVE:ENTITY"/>
+ </Grant>
+ <Deny priority="True" role="role1">
+ <Permission name="RETRIEVE:ENTITY"/>
+ </Deny>
+ </EntityACL>"""
+ xml = etree.fromstring(xml_str)
+ left_acl = db.ACL(xml)
+
+ right_acl = db.ACL()
+ right_acl.grant(role="role1", permission="RETRIEVE:ENTITY",
+ revoke_denial=False)
+ right_acl.deny(role="role1", permission="RETRIEVE:ENTITY",
+ revoke_grant=False)
+ right_acl.grant(role="role1", permission="RETRIEVE:ENTITY",
+ priority=True, revoke_denial=False)
+ right_acl.deny(role="role1", permission="RETRIEVE:ENTITY",
+ priority=True, revoke_grant=False)
+
+ assert left_acl == right_acl
diff --git a/unittests/test_apiutils.py b/unittests/test_apiutils.py
index f815fcdc28449bc40127efae0c9a75de24f5df3e..956c0a6b371a11ea98de1beb2a82c160b79d357b 100644
--- a/unittests/test_apiutils.py
+++ b/unittests/test_apiutils.py
@@ -31,7 +31,9 @@ import tempfile
import caosdb as db
import caosdb.apiutils
from caosdb.apiutils import (apply_to_ids, compare_entities, create_id_query,
- resolve_reference)
+ resolve_reference, merge_entities)
+
+from caosdb.common.models import SPECIAL_ATTRIBUTES
from .test_property import testrecord
@@ -268,3 +270,92 @@ def test_compare_properties():
assert diff_r1["value"] == 42
assert "value" in diff_r2
assert diff_r2["value"] == 4
+
+
+def test_copy_entities():
+ r = db.Record(name="A")
+ r.add_parent(name="B")
+ r.add_property(name="C", value=4, importance="OBLIGATORY")
+ r.add_property(name="D", value=[3, 4, 7], importance="OBLIGATORY")
+ r.description = "A fancy test record"
+
+ c = r.copy()
+
+ assert c is not r
+ assert c.name == "A"
+ assert c.role == r.role
+ assert c.parents[0].name == "B"
+ # parent and property objects are not shared among copy and original:
+ assert c.parents[0] is not r.parents[0]
+
+ for i in [0, 1]:
+ assert c.properties[i] is not r.properties[i]
+ for special in SPECIAL_ATTRIBUTES:
+ assert getattr(c.properties[i], special) == getattr(r.properties[i], special)
+ assert c.get_importance(c.properties[i]) == r.get_importance(r.properties[i])
+
+
+def test_merge_entities():
+ r = db.Record(name="A")
+ r.add_parent(name="B")
+ r.add_property(name="C", value=4, importance="OBLIGATORY")
+ r.add_property(name="D", value=[3, 4, 7], importance="OBLIGATORY")
+ r.description = "A fancy test record"
+
+ r2 = db.Record()
+ r2.add_property(name="F", value="text")
+ merge_entities(r2, r)
+ assert r2.get_parents()[0].name == "B"
+ assert r2.get_property("C").name == "C"
+ assert r2.get_property("C").value == 4
+ assert r2.get_property("D").name == "D"
+ assert r2.get_property("D").value == [3, 4, 7]
+
+ assert r2.get_property("F").name == "F"
+ assert r2.get_property("F").value == "text"
+
+
+def test_merge_bug_109():
+ rt = db.RecordType(name="TestBug")
+ p = db.Property(name="test_bug_property", datatype=db.LIST(db.INTEGER))
+
+ r_b = db.Record(name="TestRecord")
+ r_b.add_parent(rt)
+ r_b.add_property(p, value=[18, 19])
+
+ r_a = db.Record(name="TestRecord")
+ r_a.add_parent(rt)
+
+ merge_entities(r_a, r_b)
+
+ assert r_b.get_property("test_bug_property").value == [18, 19]
+ assert r_a.get_property("test_bug_property").value == [18, 19]
+
+ assert "<Value>18</Value>\n <Value>19</Value>" in str(r_b)
+ assert "<Value>18</Value>\n <Value>19</Value>\n <Value>18</Value>\n <Value>19</Value>" not in str(r_b)
+
+ assert "<Value>18</Value>\n <Value>19</Value>" in str(r_a)
+ assert "<Value>18</Value>\n <Value>19</Value>\n <Value>18</Value>\n <Value>19</Value>" not in str(r_a)
+
+
+@pytest.mark.xfail
+def test_bug_109():
+ rt = db.RecordType(name="TestBug")
+ p = db.Property(name="test_bug_property", datatype=db.LIST(db.INTEGER))
+
+ r_b = db.Record(name="TestRecord")
+ r_b.add_parent(rt)
+ r_b.add_property(p, value=[18, 19])
+
+ r_a = db.Record(name="TestRecord")
+ r_a.add_parent(rt)
+ r_a.add_property(r_b.get_property("test_bug_property"))
+
+ assert r_b.get_property("test_bug_property").value == [18, 19]
+ assert r_a.get_property("test_bug_property").value == [18, 19]
+
+ assert "<Value>18</Value>\n <Value>19</Value>" in str(r_b)
+ assert "<Value>18</Value>\n <Value>19</Value>\n <Value>18</Value>\n <Value>19</Value>" not in str(r_b)
+
+ assert "<Value>18</Value>\n <Value>19</Value>" in str(r_a)
+ assert "<Value>18</Value>\n <Value>19</Value>\n <Value>18</Value>\n <Value>19</Value>" not in str(r_a)
diff --git a/unittests/test_configs/pycaosdb-IntegrationTests.ini b/unittests/test_configs/pycaosdb-IntegrationTests.ini
new file mode 100644
index 0000000000000000000000000000000000000000..cb9871708f7f23c489de0cbc8f4fbda15dfa6ad0
--- /dev/null
+++ b/unittests/test_configs/pycaosdb-IntegrationTests.ini
@@ -0,0 +1,37 @@
+# -*- mode:conf; -*-
+## This sections needs to exist in addition to the usual section
+[IntegrationTests]
+# test_server_side_scripting.bin_dir.local=/path/to/scripting/bin
+test_server_side_scripting.bin_dir.local=/home/myself/test/caosdb-server/scripting/bin
+# test_server_side_scripting.bin_dir.server=/opt/caosdb/git/caosdb-server/scripting/bin
+
+# # location of the files from the pyinttest perspective
+# test_files.test_insert_files_in_dir.local=/extroot/test_insert_files_in_dir/
+test_files.test_insert_files_in_dir.local=/home/myself/test/debug_advanced/paths/extroot/test_insert_files_in_dir
+# # location of the files from the caosdb_servers perspective
+test_files.test_insert_files_in_dir.server=/opt/caosdb/mnt/extroot/test_insert_files_in_dir/
+
+########## Files ##################
+## Used by tests of file handling. Specify the path to an existing
+## directory in which file tests are performed, once as seen by the
+## host and once as seen by the server.
+
+# location of the files from the pyinttest (i.e. host) perspective
+#test_files.test_insert_files_in_dir.local=/extroot/test_insert_files_in_dir/
+
+# location of the files from the caosdb server's perspective
+#test_files.test_insert_files_in_dir.server=/opt/caosdb/mnt/extroot/test_insert_files_in_dir/
+
+# # location of the one-time tokens from the pyinttest's perspective
+# test_authentication.admin_token_crud = /authtoken/admin_token_crud.txt
+# test_authentication.admin_token_expired = /authtoken/admin_token_expired.txt
+# test_authentication.admin_token_3_attempts = /authtoken/admin_token_3_attempts.txt
+
+
+## Insert your usual settings here
+[Connection]
+url=https://localhost:10443/
+username=admin
+password_method=plain
+password=caosdb
+
diff --git a/unittests/test_configs/pycaosdb-empty.ini b/unittests/test_configs/pycaosdb-empty.ini
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/unittests/test_configs/pycaosdb-real-world-1.ini b/unittests/test_configs/pycaosdb-real-world-1.ini
new file mode 100644
index 0000000000000000000000000000000000000000..e524f1d3465c61d89ae4a4dda54536a722f99837
--- /dev/null
+++ b/unittests/test_configs/pycaosdb-real-world-1.ini
@@ -0,0 +1,17 @@
+[Connection]
+url = https://localhost:10443
+cacert = /opt/caosdb/cert/caosdb.cert.pem
+debug = 0
+timeout = 5000
+
+[Misc]
+sendmail = /usr/local/bin/sendmail_to_file
+entity_loan.curator_mail_from=crawler-test@example.com
+entity_loan.curator_mail_to=crawler-test@example.com
+
+[sss_helper]
+external_uri = https://caosdb.example.com:443
+
+[advancedtools]
+crawler.from_mail=admin@example.com
+crawler.to_mail=admin@example.com
diff --git a/unittests/test_configs/pycaosdb-real-world-2.ini b/unittests/test_configs/pycaosdb-real-world-2.ini
new file mode 100644
index 0000000000000000000000000000000000000000..5ebd115a4a4de189d22180130acca2a4b78b6daf
--- /dev/null
+++ b/unittests/test_configs/pycaosdb-real-world-2.ini
@@ -0,0 +1,15 @@
+[Connection]
+url = https://samplemanager.example.com:443
+cacert = /opt/caosdb/cert/caosdb.cert.pem
+debug = 0
+timeout = 5000
+[Misc]
+sendmail = /usr/local/bin/sendmail_to_file
+entity_loan.curator_mail_from=crawler-test@example.com
+entity_loan.curator_mail_to=crawler-test@example.com
+[sss_helper]
+external_uri = https://localhost:10443
+[advancedtools]
+crawler.from_mail=crawler-test@example.com
+crawler.to_mail=crawler-test@example.com
+
diff --git a/unittests/test_configs/pycaosdb-server-side-scripting.ini b/unittests/test_configs/pycaosdb-server-side-scripting.ini
new file mode 100644
index 0000000000000000000000000000000000000000..de2867f8dc66b3e81f10f35e40c36f9cb8591604
--- /dev/null
+++ b/unittests/test_configs/pycaosdb-server-side-scripting.ini
@@ -0,0 +1,9 @@
+; this is the pycaosdb.ini for the server-side-scripting home.
+[Connection]
+url = https://caosdb-server:10443
+cacert = /opt/caosdb/cert/caosdb.cert.pem
+debug = 0
+timeout = 5000
+
+[Misc]
+sendmail = /usr/local/bin/sendmail_to_file
diff --git a/unittests/test_configs/pycaosdb4.ini b/unittests/test_configs/pycaosdb4.ini
new file mode 100644
index 0000000000000000000000000000000000000000..ddbc7ca6f969e55ea6131d96f091177a13687ece
--- /dev/null
+++ b/unittests/test_configs/pycaosdb4.ini
@@ -0,0 +1,4 @@
+[Connection]
+url=https://localhost:10443/
+username=admin
+password_method=input
diff --git a/unittests/test_configs/pycaosdb5.ini b/unittests/test_configs/pycaosdb5.ini
new file mode 100644
index 0000000000000000000000000000000000000000..3f365efdd92641a39b742e22f825033a69e12dc5
--- /dev/null
+++ b/unittests/test_configs/pycaosdb5.ini
@@ -0,0 +1,4 @@
+[Connection]
+url=https://localhost:10443/
+username=admin
+# No password method: should be "input" by default
diff --git a/unittests/test_issues.py b/unittests/test_issues.py
new file mode 100644
index 0000000000000000000000000000000000000000..1e649db4f23de67e55301e0a053fba70d14680b4
--- /dev/null
+++ b/unittests/test_issues.py
@@ -0,0 +1,39 @@
+# This file is a part of the CaosDB Project.
+#
+# Copyright (c) 2022 IndiScale GmbH
+# Copyright (c) 2022 Daniel Hornung (d.hornung@indiscale.com)
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+"""Test known issues to prevent regressions.
+"""
+
+import os
+
+import lxml
+import caosdb as db
+
+from pytest import raises
+
+
+def test_issue_100():
+ """_parse_value() fails for some list-valued content
+ """
+
+ # Parse from (invalid) XML file
+ filename = os.path.join(os.path.dirname(__file__), "data", "list_in_value.xml")
+ xml_el = lxml.etree.parse(filename).getroot()
+ with raises(db.ServerConfigurationException) as exc_info:
+ db.common.models._parse_single_xml_element(xml_el)
+ assert "invalid XML: List valued properties" in exc_info.value.msg
diff --git a/unittests/test_plantuml.py b/unittests/test_plantuml.py
new file mode 100644
index 0000000000000000000000000000000000000000..a507c36b2d3a4246205fc7507cb05119c575084c
--- /dev/null
+++ b/unittests/test_plantuml.py
@@ -0,0 +1,61 @@
+#!/usr/bin/env python3
+# encoding: utf-8
+#
+# ** header v3.0
+# This file is a part of the CaosDB Project.
+#
+# Copyright (C) 2022 Indiscale GmbH <info@indiscale.com>
+# Copyright (C) 2022 Henrik tom Wörden <h.tomwoerden@indiscale.com>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see <https://www.gnu.org/licenses/>.
+#
+# ** end header
+#
+
+"""
+test plantuml utility
+"""
+
+import tempfile
+import pytest
+import caosdb as db
+import shutil
+from caosdb.utils.plantuml import to_graphics
+
+
+@pytest.fixture
+def setup_n_teardown(autouse=True):
+
+ with tempfile.TemporaryDirectory() as td:
+ global output
+ output = td
+ yield
+
+
+@pytest.fixture
+def entities():
+ return [db.RecordType(name="TestRT1").add_property("testprop"),
+ db.RecordType(name="TestRT2").add_property("testprop2"),
+ db.Property("testprop")]
+
+
+@pytest.mark.skipif(shutil.which('plantuml') is None, reason="No plantuml found")
+def test_to_graphics1(entities, setup_n_teardown):
+ to_graphics(entities, "data_model", output_dirname=output)
+
+
+@pytest.mark.skipif(shutil.which('plantuml') is None, reason="No plantuml found")
+def test_to_graphics2(entities, setup_n_teardown):
+ to_graphics(entities, "data_model", output_dirname=output, formats=["tpng", "tsvg"],
+ add_properties=False, add_legend=False, style="salexan")
diff --git a/unittests/test_schema.py b/unittests/test_schema.py
index 1552179a3e43dacb3ecca705466bb7ff84d330cf..fc3f63a4cbaeadcac3c1cb9be2d861a0688fe4b0 100644
--- a/unittests/test_schema.py
+++ b/unittests/test_schema.py
@@ -2,7 +2,9 @@
#
# This file is a part of the CaosDB Project.
#
+# Copyright (C) 2022 Indiscale GmbH <info@indiscale.com>
# Copyright (C) 2021 Alexander Schlemmer
+# Copyright (C) 2022 Daniel Hornung <d.hornung@indiscale.com>
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as
@@ -31,15 +33,18 @@ from configparser import ConfigParser
def test_config_files():
for fn in glob(os.path.join(os.path.dirname(__file__), "test_configs", "*.ini")):
+ print(f"Testing {fn}.")
c = ConfigParser()
c.read(fn)
+ print(config_to_yaml(c))
validate_yaml_schema(config_to_yaml(c))
def test_broken_config_files():
for fn in glob(os.path.join(os.path.dirname(__file__), "broken_configs", "*.ini")):
- print(fn)
+ print(f"Testing {fn}.")
with raises(ValidationError):
c = ConfigParser()
c.read(fn)
+ print(config_to_yaml(c))
validate_yaml_schema(config_to_yaml(c))