diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 9bb5adb6ed4fedf57694d6ec7843942c46296269..1dc09269a92c486e9d80a8ae5ceb0e51dc50bd17 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -39,6 +39,7 @@ stages: code_style: tags: [ docker ] stage: code_style + needs: [ ] script: - make style allow_failure: true @@ -47,17 +48,19 @@ code_style: pylint: tags: [ docker ] stage: linting + needs: [ ] script: - make lint allow_failure: true -# run tests -test: +# run unit tests +unittest: tags: [ docker ] stage: test + needs: [ ] script: - touch ~/.pycaosdb.ini - - tox -r + - make unittest # Trigger building of server image and integration tests trigger_build: @@ -96,6 +99,7 @@ build-testenv: pages_prepare: &pages_prepare tags: [ cached-dind ] stage: deploy + needs: [ code_style, pylint, unittest ] only: refs: - /^release-.*$/i diff --git a/.gitlab/merge_request_templates/Default.md b/.gitlab/merge_request_templates/Default.md deleted file mode 100644 index e43435cb31f415fc4f9c8447983b411627612ad7..0000000000000000000000000000000000000000 --- a/.gitlab/merge_request_templates/Default.md +++ /dev/null @@ -1,50 +0,0 @@ -# Summary - - Insert a meaningful description for this merge request here. What is the - new/changed behavior? Which bug has been fixed? Are there related Issues? - -# Focus - - Point the reviewer to the core of the code change. Where should they start - reading? What should they focus on (e.g. security, performance, - maintainability, user-friendliness, compliance with the specs, finding more - corner cases, concrete questions)? - -# Test Environment - - How to set up a test environment for manual testing? - -# Check List for the Author - -Please, prepare your MR for a review. Be sure to write a summary and a -focus and create gitlab comments for the reviewer. They should guide the -reviewer through the changes, explain your changes and also point out open -questions. For further good practices have a look at [our review -guidelines](https://gitlab.com/caosdb/caosdb/-/blob/dev/REVIEW_GUIDELINES.md) - -- [ ] All automated tests pass -- [ ] Reference related Issues -- [ ] Up-to-date CHANGELOG.md -- [ ] Add type hints in created/changed code -- [ ] Annotations in code (Gitlab comments) - - Intent of new code - - Problems with old code - - Why this implementation? - - -# Check List for the Reviewer - - -- [ ] I understand the intent of this MR -- [ ] All automated tests pass -- [ ] Up-to-date CHANGELOG.md -- [ ] The test environment setup works and the intended behavior is - reproducible in the test environment -- [ ] In-code documentation and comments are up-to-date. -- [ ] Check: Are there spezifications? Are they satisfied? - -For further good practices have a look at [our review guidelines](https://gitlab.com/caosdb/caosdb/-/blob/dev/REVIEW_GUIDELINES.md). - - -/assign me -/target_branch dev diff --git a/CHANGELOG.md b/CHANGELOG.md index 8771150978588aa848103fe7f330f8b9c8796a8d..206d8b3f7fb4a3e9be0766924fc7f3534f81284a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -21,6 +21,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Security ### +### Documentation ### + +## [0.7.1] - 2022-03-11 ## + +### Documentation ### + +- `timeout` option in example pycaosdb.ini + ## [0.7.0] - 2022-01-21 ## ### Added ### diff --git a/Makefile b/Makefile index 192337853d8db8812e14f75fca8986006de82180..0a0888ad0484c0307583e139e65058c38574ed3a 100644 --- a/Makefile +++ b/Makefile @@ -42,3 +42,7 @@ style: lint: pylint --unsafe-load-any-extension=y -d all -e E,F src/caosdb/common .PHONY: lint + +unittest: + tox -r +.PHONY: unittest diff --git a/examples/pycaosdb.ini b/examples/pycaosdb.ini index edc32195fbb364bb355d67b8733e8c7bccbb0d34..8cf74e43c5db32ed139c4fe371a6c2b3831b2ee1 100644 --- a/examples/pycaosdb.ini +++ b/examples/pycaosdb.ini @@ -67,3 +67,6 @@ # This option is used internally and for testing. Do not override. # implementation=_DefaultCaosDBServerConnection + +# The timeout for requests to the server. +# timeout=1000 diff --git a/setup.py b/setup.py index 310d77786949a9bc2982936457a90d73047f2b0c..cdb10b6b41b5c466187a4bc63a77f41bd04ec454 100755 --- a/setup.py +++ b/setup.py @@ -48,8 +48,8 @@ from setuptools import find_packages, setup ISRELEASED = False MAJOR = 0 MINOR = 7 -MICRO = 1 -PRE = "" # e.g. rc0, alpha.1, 0.beta-23 +MICRO = 2 +PRE = "dev" # e.g. rc0, alpha.1, 0.beta-23 if PRE: VERSION = "{}.{}.{}-{}".format(MAJOR, MINOR, MICRO, PRE) diff --git a/src/caosdb/apiutils.py b/src/caosdb/apiutils.py index e126b843a062a9681e48a5c9238764b27a2b0732..516603fceab1861100825a3d3eb9e0e0c94af56f 100644 --- a/src/caosdb/apiutils.py +++ b/src/caosdb/apiutils.py @@ -141,6 +141,9 @@ class CaosDBPythonEntity(object): _last_id = 0 def __init__(self): + warnings.warn("The CaosDBPythonEntity class is deprecated, replacements will be provided by" + " the high_level_api module.", + DeprecationWarning, stacklevel=2) # Save a copy of the dry state # of this object in order to be # able to detect conflicts. @@ -469,6 +472,9 @@ def _single_convert_to_entity(entity, robj, **kwargs): def convert_to_entity(python_object, **kwargs): + warnings.warn("The convert_to_entity function is deprecated, replacement will be provided by " + "the high_level_api module.", DeprecationWarning, stacklevel=2) + if isinstance(python_object, Container): # Create a list of objects: @@ -490,6 +496,8 @@ def convert_to_entity(python_object, **kwargs): def convert_to_python_object(entity): """""" + warnings.warn("The convert_to_python_object function is deprecated, replacement will be " + "provided by the high_level_api module.", DeprecationWarning, stacklevel=2) if isinstance(entity, Container): # Create a list of objects: diff --git a/src/caosdb/common/models.py b/src/caosdb/common/models.py index ec45dec070d1ea731648fe8d45e44ba89e393f76..80a6ee11e707fb3776fc96b42a16b649ac575f66 100644 --- a/src/caosdb/common/models.py +++ b/src/caosdb/common/models.py @@ -55,6 +55,7 @@ from caosdb.exceptions import (AmbiguousEntityError, AuthorizationError, EntityDoesNotExistError, EntityError, EntityHasNoDatatypeError, HTTPURITooLongError, MismatchingEntitiesError, QueryNotUniqueError, + ServerConfigurationException, TransactionError, UniqueNamesError, UnqualifiedParentsError, UnqualifiedPropertiesError) @@ -1195,6 +1196,10 @@ class Entity(object): def _parse_value(datatype, value): + """Parse the value (from XML input) according to the given datatype + """ + + # Simple values if value is None: return value @@ -1215,12 +1220,12 @@ def _parse_value(datatype, value): else: raise ValueError("Boolean value was {}.".format(value)) + # Datetime and text are returned as-is if datatype in [DATETIME, TEXT]: if isinstance(value, str): return value # deal with collections - if isinstance(datatype, str): matcher = re.compile(r"^(?P<col>[^<]+)<(?P<dt>[^>]+)>$") m = matcher.match(datatype) @@ -1245,12 +1250,10 @@ def _parse_value(datatype, value): # This is for a special case, where the xml parser could not differentiate # between single values and lists with one element. As - if hasattr(value, "__len__") and len(value) == 1: return _parse_value(datatype, value[0]) # deal with references - if isinstance(value, Entity): return value @@ -1266,6 +1269,12 @@ def _parse_value(datatype, value): # reference via name return str(value) + except TypeError: + # deal with invalid XML: List of values without appropriate datatype + if isinstance(value, list): + raise ServerConfigurationException( + "The server sent an invalid XML: List valued properties must be announced by " + "the datatype.\n" + f"Datatype: {datatype}\nvalue: {value}") def _log_request(request, xml_body=None): diff --git a/src/doc/conf.py b/src/doc/conf.py index b05fa1c71c1dcd0b59916594818449d2ebc574bd..b4dfcc9925eb8100b957b6c8c2c06c855e8d3ff0 100644 --- a/src/doc/conf.py +++ b/src/doc/conf.py @@ -25,14 +25,14 @@ import sphinx_rtd_theme # noqa: E402 # -- Project information ----------------------------------------------------- project = 'pycaosdb' -copyright = '2020, IndiScale GmbH' +copyright = '2022, IndiScale GmbH' author = 'Daniel Hornung' # The short X.Y version -version = '0.5.2' +version = '0.7' # The full version, including alpha/beta/rc tags # release = '0.5.2-rc2' -release = '0.5.2' +release = '0.7.2-dev' # -- General configuration --------------------------------------------------- diff --git a/src/doc/configuration.md b/src/doc/configuration.md index 6e53542f661dcae94622fef24a67cecf7491df9c..802da4e91818ba65bd0184a9a5ac49f5c2ba02d2 100644 --- a/src/doc/configuration.md +++ b/src/doc/configuration.md @@ -49,6 +49,8 @@ with CaosDB which makes the experience much less verbose. Set it to 1 or 2 in ca debugging (which I hope will not be necessary for this tutorial) or if you want to learn more about the internals of the protocol. +`timeout` sets the timeout for requests to the server. + A complete list of options can be found in the [pycaosdb.ini file](https://gitlab.com/caosdb/caosdb-pylib/-/blob/main/examples/pycaosdb.ini) in the examples folder of the source code. diff --git a/tox.ini b/tox.ini index b1061a57c6a136cb29f77a1d0c03383ab82ecf8b..0d245e03ef9c8fe2151e173cd1a10964d47ef82b 100644 --- a/tox.ini +++ b/tox.ini @@ -9,3 +9,6 @@ deps = . pytest-cov jsonschema==4.0.1 commands=py.test --cov=caosdb -vv {posargs} + +[flake8] +max-line-length=100 diff --git a/unittests/data/list_in_value.xml b/unittests/data/list_in_value.xml new file mode 100644 index 0000000000000000000000000000000000000000..0f92610d82caa5ced443b2f437f35da05b9e121a --- /dev/null +++ b/unittests/data/list_in_value.xml @@ -0,0 +1,12 @@ +<Record id="1002" description="A description of this example experiment."> + <Version id="945c6858819d2609a5475ee4df64571984acd039" head="true"> + <Predecessor id="0df3cfe164fbafe9777f9356d0be2403890c54cd" /> + </Version> + <Parent id="1001" name="Experiment" /> + <Property datatype="SomeRecordType" id="1003" name="DepthTest" importance="FIX"> + <Value>1004</Value> + <Value>1005</Value> + </Property> +</Record> + +<!-- Note: This XML is invalid, because list-valued Properties must have a LIST-Datatype --> diff --git a/unittests/test_issues.py b/unittests/test_issues.py new file mode 100644 index 0000000000000000000000000000000000000000..1e649db4f23de67e55301e0a053fba70d14680b4 --- /dev/null +++ b/unittests/test_issues.py @@ -0,0 +1,39 @@ +# This file is a part of the CaosDB Project. +# +# Copyright (c) 2022 IndiScale GmbH +# Copyright (c) 2022 Daniel Hornung (d.hornung@indiscale.com) +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see <https://www.gnu.org/licenses/>. + +"""Test known issues to prevent regressions. +""" + +import os + +import lxml +import caosdb as db + +from pytest import raises + + +def test_issue_100(): + """_parse_value() fails for some list-valued content + """ + + # Parse from (invalid) XML file + filename = os.path.join(os.path.dirname(__file__), "data", "list_in_value.xml") + xml_el = lxml.etree.parse(filename).getroot() + with raises(db.ServerConfigurationException) as exc_info: + db.common.models._parse_single_xml_element(xml_el) + assert "invalid XML: List valued properties" in exc_info.value.msg