diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 792ab22f1524f8d4dc1db90f2b65c7f8f28f90ed..0430a4f6b5ac08d4ab38f00bff78b845e11fb97e 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -54,18 +54,25 @@ pylint: allow_failure: true # run unit tests -unittest_py3.8: +unittest_py3.7: tags: [ docker ] stage: test needs: [ ] - image: python:3.8 + image: python:3.7 script: &python_test_script # Python docker has problems with tox and pip so use plain pytest here - touch ~/.pycaosdb.ini - - pip install nose pytest pytest-cov python-dateutil jsonschema==4.0.1 + - pip install nose pytest pytest-cov python-dateutil jsonschema>=4.4.0 - pip install . - python -m pytest unittests +unittest_py3.8: + tags: [ docker ] + stage: test + needs: [ ] + image: python:3.8 + script: *python_test_script + # This needs to be changed once Python 3.9 isn't the standard Python in Debian # anymore. unittest_py3.9: @@ -86,6 +93,13 @@ unittest_py3.10: image: python:3.10 script: *python_test_script +unittest_py3.11: + tags: [ docker ] + stage: test + needs: [ ] + image: python:3.11 + script: *python_test_script + # Trigger building of server image and integration tests trigger_build: stage: deploy diff --git a/CHANGELOG.md b/CHANGELOG.md index f3f29f875aae54312cd6ddd890604e185d02de02..b7e37c52b9d734da4e877fe6456fb5dee6e02a37 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,7 +5,33 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [0.11.0] - 2023-01-19 ## +(Florian Spreckelsen) + +### Added ### + +* `apiutils.EntityMergeConflictError` class for unresesolvable merge conflicts + when merging two entities +* Re-introduced support for Python 3.7 + +### Changed ### + +* `apiutils.merge_entities` now raises an `EntityMergeConflictError` in case of + unresolvable merge conflicts. + +### Fixed ### + +* [#82](https://gitlab.com/caosdb/caosdb-pylib/-/issues/82) Merging an entity + with properties with missing datatype leads to Exception - The correct + exception is raised in case of a missing LIST datatype. + +### Documentation ### + +* [Fixed](https://gitlab.com/caosdb/caosdb-pylib/-/issues/79) + `{action}_entity_permissions` help line. + ## [0.10.0] - 2022-11-14 +(Florian Spreckelsen) ### Added ### diff --git a/README.md b/README.md index 602df33cecfc8ec37fd791e3257221e66f120cb3..7215591a4f31f1946029442de291eb9ccf9beea1 100644 --- a/README.md +++ b/README.md @@ -39,7 +39,7 @@ By participating, you are expected to uphold our [Code of Conduct](https://gitla * If you have a suggestion for the [documentation](https://docs.indiscale.com/caosdb-pylib/), the preferred way is also a merge request as describe above (the documentation resides in `src/doc`). However, you can also create an issue for it. -* You can also contact us at **info (AT) caosdb.de** and join the +* You can also contact us at **info (AT) caosdb.org** and join the CaosDB community on [#caosdb:matrix.org](https://matrix.to/#/!unwwlTfOznjEnMMXxf:matrix.org). diff --git a/README_SETUP.md b/README_SETUP.md index 48928d6c3f2c878a8d8b268b36ed2cdeba7f8014..01eea85188078ae6f2fe226e89e5c227497b4bd0 100644 --- a/README_SETUP.md +++ b/README_SETUP.md @@ -109,6 +109,8 @@ Now would be a good time to continue with the [tutorials](tutorials/index). - Run a specific test function: e.g. `tox -- unittests/test_schema.py::test_config_files` ## Documentation ## +We use sphinx to create the documentation. Docstrings in the code should comply +with the Googly style (see link below). Build documentation in `build/` with `make doc`. @@ -118,5 +120,11 @@ Build documentation in `build/` with `make doc`. - `sphinx-autoapi` - `recommonmark` +### How to contribute ### + +- [Google Style Python Docstrings](https://sphinxcontrib-napoleon.readthedocs.io/en/latest/example_google.html) +- [Google Style Python Docstrings 2nd reference](https://github.com/google/styleguide/blob/gh-pages/pyguide.md#38-comments-and-docstrings) +- [References to other documentation](https://www.sphinx-doc.org/en/master/usage/extensions/intersphinx.html#role-external) + ### Troubleshooting ### If the client is to be executed directly from the `/src` folder, an initial `.\setup.py install --user` must be called. diff --git a/pytest.ini b/pytest.ini deleted file mode 100644 index ca6aad829a3e0607292cf69b8b1d4b7f7758993e..0000000000000000000000000000000000000000 --- a/pytest.ini +++ /dev/null @@ -1,3 +0,0 @@ -[pytest] -testpaths=unittests -addopts=-x -vv --cov=caosdb diff --git a/setup.py b/setup.py index 187b04f01773280c42711c1a4b80f5eddc91eae1..0d2305f0b59453c0d209918fc74b991f87bcbda0 100755 --- a/setup.py +++ b/setup.py @@ -47,7 +47,7 @@ from setuptools import find_packages, setup ISRELEASED = True MAJOR = 0 -MINOR = 10 +MINOR = 11 MICRO = 0 # Do not tag as pre-release until this commit # https://github.com/pypa/packaging/pull/515 @@ -171,10 +171,10 @@ def setup_package(): "Topic :: Scientific/Engineering :: Information Analysis", ], packages=find_packages('src'), - python_requires='>=3.8', + python_requires='>=3.7', package_dir={'': 'src'}, install_requires=['lxml>=4.6.3', - "requests[socks]>=2.28.1", + "requests[socks]>=2.26", "python-dateutil>=2.8.2", 'PyYAML>=5.4.1', 'future', diff --git a/src/caosdb/apiutils.py b/src/caosdb/apiutils.py index 0862cf9f32575b9773bc16d845bb459d67b0140c..f3195b8e152f0cb13e5dab3e3a449b7bb36623b4 100644 --- a/src/caosdb/apiutils.py +++ b/src/caosdb/apiutils.py @@ -27,12 +27,13 @@ Some simplified functions for generation of records etc. """ +import logging import sys import tempfile import warnings + from collections.abc import Iterable from subprocess import call - from typing import Optional, Any, Dict, List from caosdb.common.datatype import (BOOLEAN, DATETIME, DOUBLE, FILE, INTEGER, @@ -40,8 +41,13 @@ from caosdb.common.datatype import (BOOLEAN, DATETIME, DOUBLE, FILE, INTEGER, from caosdb.common.models import (Container, Entity, File, Property, Query, Record, RecordType, execute_query, get_config, SPECIAL_ATTRIBUTES) +from caosdb.exceptions import CaosDBException -import logging + +class EntityMergeConflictError(CaosDBException): + """An error that is raised in case of an unresolvable conflict when merging + two entities. + """ def new_record(record_type, name=None, description=None, @@ -365,14 +371,15 @@ def empty_diff(old_entity: Entity, new_entity: Entity, compare_referenced_record return True -def merge_entities(entity_a: Entity, entity_b: Entity, merge_references_with_empty_diffs=True, force=False): - """ - Merge entity_b into entity_a such that they have the same parents and properties. +def merge_entities(entity_a: Entity, entity_b: Entity, merge_references_with_empty_diffs=True, + force=False): + """Merge entity_b into entity_a such that they have the same parents and properties. - datatype, unit, value, name and description will only be changed in entity_a if they - are None for entity_a and set for entity_b. If there is a corresponding value - for entity_a different from None a RuntimeError will be raised informing of an - unresolvable merge conflict. + datatype, unit, value, name and description will only be changed in entity_a + if they are None for entity_a and set for entity_b. If there is a + corresponding value for entity_a different from None, an + EntityMergeConflictError will be raised to inform about an unresolvable merge + conflict. The merge operation is done in place. @@ -392,13 +399,18 @@ def merge_entities(entity_a: Entity, entity_b: Entity, merge_references_with_emp force : bool, optional If True, in case `entity_a` and `entity_b` have the same properties, the values of `entity_a` are replaced by those of `entity_b` in the merge. - If `False`, a RuntimeError is raised instead. Default is False. + If `False`, an EntityMergeConflictError is raised instead. Default is False. Returns ------- entity_a : Entity The initial entity_a after the in-place merge + Raises + ------ + EntityMergeConflictError + In case of an unresolvable merge conflict. + """ logging.warning( @@ -433,8 +445,8 @@ def merge_entities(entity_a: Entity, entity_b: Entity, merge_references_with_emp setattr(entity_a.get_property(key), attribute, diff_r2["properties"][key][attribute]) else: - raise RuntimeError( - f"Merge conflict:\nEntity a ({entity_a.id}, {entity_a.name}) " + raise EntityMergeConflictError( + f"Entity a ({entity_a.id}, {entity_a.name}) " f"has a Property '{key}' with {attribute}=" f"{diff_r2['properties'][key][attribute]}\n" f"Entity b ({entity_b.id}, {entity_b.name}) " @@ -463,7 +475,9 @@ def merge_entities(entity_a: Entity, entity_b: Entity, merge_references_with_emp # force overwrite setattr(entity_a, special_attribute, sa_b) else: - raise RuntimeError("Merge conflict.") + raise EntityMergeConflictError( + f"Conflict in special attribute {special_attribute}:\n" + f"A: {sa_a}\nB: {sa_b}") return entity_a diff --git a/src/caosdb/common/models.py b/src/caosdb/common/models.py index 7000ede917995c6c01b78a822c2d39ac626fcc23..83359ac847fa62de976208f9af023a2cf2a73af6 100644 --- a/src/caosdb/common/models.py +++ b/src/caosdb/common/models.py @@ -62,7 +62,6 @@ from caosdb.exceptions import (AmbiguousEntityError, AuthorizationError, EntityDoesNotExistError, EntityError, EntityHasNoDatatypeError, HTTPURITooLongError, MismatchingEntitiesError, QueryNotUniqueError, - ServerConfigurationException, TransactionError, UniqueNamesError, UnqualifiedParentsError, UnqualifiedPropertiesError) @@ -1381,12 +1380,15 @@ def _parse_value(datatype, value): # reference via name return str(value) - except TypeError: + except TypeError as te: # deal with invalid XML: List of values without appropriate datatype if isinstance(value, list): - raise ServerConfigurationException( - "The server sent an invalid XML: List valued properties must be announced by " + raise TypeError( + "Invalid datatype: List valued properties must be announced by " "the datatype.\n" + f"Datatype: {datatype}\nvalue: {value}") + else: + # Everything else that's not related to wrong list assignments + raise te def _log_request(request, xml_body=None): diff --git a/src/caosdb/utils/caosdb_admin.py b/src/caosdb/utils/caosdb_admin.py index 9fb94f57683036f5432a40198cc4ae98893665fb..09a8f64a3c6b9f0825089949840a8791604d1ded 100755 --- a/src/caosdb/utils/caosdb_admin.py +++ b/src/caosdb/utils/caosdb_admin.py @@ -621,8 +621,8 @@ USAGE for action in ["grant", "deny", "revoke_denial", "revoke_grant"]: action_entity_permissions_parser = subparsers.add_parser( - "{}_entity_permissions".format(action), - help="{} entity permissions to a role.".format(action)) + f"{action}_entity_permissions", + help=f"{action} entity permissions to one or more Entities.") action_entity_permissions_parser.set_defaults( call=do_action_entity_permissions, action=action) action_entity_permissions_parser.add_argument(dest="query", metavar="QUERY", diff --git a/src/doc/conf.py b/src/doc/conf.py index 8f3dfbdbdf80307b8bd68c068360aca4b7e7a2c7..b6d82cd82ee62e6a03c2fdea8c35129adb6096a9 100644 --- a/src/doc/conf.py +++ b/src/doc/conf.py @@ -29,10 +29,10 @@ copyright = '2022, IndiScale GmbH' author = 'Daniel Hornung' # The short X.Y version -version = '0.10.0' +version = '0.11.0' # The full version, including alpha/beta/rc tags # release = '0.5.2-rc2' -release = '0.10.0' +release = '0.10.1' # -- General configuration --------------------------------------------------- diff --git a/src/doc/tutorials/basic_analysis.rst b/src/doc/tutorials/basic_analysis.rst index cc185e0ee08f9e5ee0f890c0ab55f52972882d17..c40cad28b8c9a3be537c641b9614da2eb4df8dd9 100644 --- a/src/doc/tutorials/basic_analysis.rst +++ b/src/doc/tutorials/basic_analysis.rst @@ -34,7 +34,7 @@ Often we are interested in table like data for our processing. And the disentang >>> from caosadvancedtools.table_converter import to_table >>> # Let us retrieve the data in a table like form using `SELECT` ->>> data = db.execute_query("SELECT quality_factor FROM RECORD Analysis with quality_factor" ) +>>> data = db.execute_query("SELECT quality_factor FROM Analysis with quality_factor" ) >>> table = to_table(data) >>> print(table) quality_factor diff --git a/tox.ini b/tox.ini index 50c22d5716769ef2ec818f6c8fb94491ea372434..8212226eef2759c1864a86b8a3ad8f926480db4a 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist=py38, py39, py310 +envlist=py37, py38, py39, py310, py311 skip_missing_interpreters = true [testenv] @@ -7,8 +7,13 @@ deps = . nose pytest pytest-cov - jsonschema==4.0.1 + jsonschema>=4.4.0 commands=py.test --cov=caosdb -vv {posargs} [flake8] max-line-length=100 + +[pytest] +testpaths = unittests +xfail_strict = True +addopts = -x -vv --cov=caosdb diff --git a/unittests/test_apiutils.py b/unittests/test_apiutils.py index f9de6d1a037667d0ead0f02439bde13ac4f14f60..bda381cf6427377194e272dfa14b83399b6f012f 100644 --- a/unittests/test_apiutils.py +++ b/unittests/test_apiutils.py @@ -3,6 +3,7 @@ # # Copyright (C) 2020 Timm Fitschen <t.fitschen@indiscale.com> # Copyright (C) 2022 Florian Spreckelsen <f.spreckelsen@indiscale.com> +# Copyright (C) 2022 Daniel Hornung <d.hornung@indiscale.com> # Copyright (C) 2020-2022 IndiScale GmbH <info@indiscale.com> # Copyright (C) 2018 Research Group Biomedical Physics, # Max-Planck-Institute for Dynamics and Self-Organization Göttingen @@ -29,7 +30,8 @@ import pytest import caosdb as db import caosdb.apiutils from caosdb.apiutils import (apply_to_ids, compare_entities, create_id_query, - empty_diff, resolve_reference, merge_entities) + empty_diff, EntityMergeConflictError, + resolve_reference, merge_entities) from caosdb.common.models import SPECIAL_ATTRIBUTES @@ -306,7 +308,7 @@ def test_merge_bug_conflict(): r3 = db.Record() r3.add_property(name="C", value=4, datatype="INTEGER") - with pytest.raises(RuntimeError) as excinfo: + with pytest.raises(EntityMergeConflictError): merge_entities(r3, r2) @@ -401,15 +403,13 @@ def test_wrong_merge_conflict_reference(): rec_a.add_property(name=title_prop.name, value="Some dataset title") # this does not compare referenced records, so it will fail - with pytest.raises(RuntimeError) as re: + with pytest.raises(EntityMergeConflictError): merge_entities(rec_a, rec_b, merge_references_with_empty_diffs=False) - assert "Merge conflict" in str(re.value) # ... as should this, of course rec_b.get_property(license_rt.name).value.name = "Another license" - with pytest.raises(RuntimeError) as re: + with pytest.raises(EntityMergeConflictError) as re: merge_entities(rec_a, rec_b) - assert "Merge conflict" in str(re.value) def test_empty_diff(): @@ -483,9 +483,8 @@ def test_force_merge(): recA = db.Record(name="A") recB = db.Record(name="B") - with pytest.raises(RuntimeError) as re: + with pytest.raises(EntityMergeConflictError): merge_entities(recA, recB) - assert "Merge conflict" in str(re.value) merge_entities(recA, recB, force=True) assert "B" == recA.name @@ -498,9 +497,11 @@ def test_force_merge(): recB = db.Record() recB.description = "something else" - with pytest.raises(RuntimeError) as re: + with pytest.raises(EntityMergeConflictError) as emce: merge_entities(recA, recB) - assert "Merge conflict" in str(re.value) + assert str(emce.value) == """Conflict in special attribute description: +A: something +B: something else""" merge_entities(recA, recB, force=True) assert recA.description == "something else" @@ -513,9 +514,8 @@ def test_force_merge(): recB = db.Record() recB.add_property(name="propA", value="something else") - with pytest.raises(RuntimeError) as re: + with pytest.raises(EntityMergeConflictError): merge_entities(recA, recB) - assert "Merge conflict" in str(re.value) merge_entities(recA, recB, force=True) assert recA.get_property("propA").value == "something else" @@ -539,9 +539,8 @@ def test_force_merge(): rtB = db.RecordType() rtB.add_property(name="propA", datatype=db.TEXT) - with pytest.raises(RuntimeError) as re: + with pytest.raises(EntityMergeConflictError): merge_entities(rtA, rtB) - assert "Merge conflict" in str(re.value) merge_entities(rtA, rtB, force=True) assert rtA.get_property("propA").datatype == db.TEXT @@ -554,10 +553,24 @@ def test_force_merge(): recB = db.Record() recB.add_property(name="propA", value=5, unit="cm") - with pytest.raises(RuntimeError) as re: + with pytest.raises(EntityMergeConflictError): merge_entities(recA, recB) - assert "Merge conflict" in str(re.value) merge_entities(recA, recB, force=True) assert recA.get_property("propA").unit == "cm" # unchanged assert recB.get_property("propA").unit == "cm" + + +def test_merge_missing_list_datatype_82(): + """Merging two properties, where the list-valued one has no datatype.""" + + recA = db.Record().add_property("a", 5, datatype="B") + recB_with_DT = db.Record().add_property("a", [1, 2], datatype=f"LIST<{db.DOUBLE}>") + merge_entities(recA, recB_with_DT, force=True) + assert recA.get_property("a").datatype == f"LIST<{db.DOUBLE}>" + + recA = db.Record().add_property("a", 5, datatype="B") + recB_without_DT = db.Record().add_property("a", [1, 2]) + with pytest.raises(TypeError) as te: + merge_entities(recA, recB_without_DT, force=True) + assert "Invalid datatype: List valued properties" in str(te.value) diff --git a/unittests/test_issues.py b/unittests/test_issues.py index 1e649db4f23de67e55301e0a053fba70d14680b4..2c45a6d77ba61c3f948e403f708994c0fe31481a 100644 --- a/unittests/test_issues.py +++ b/unittests/test_issues.py @@ -34,6 +34,6 @@ def test_issue_100(): # Parse from (invalid) XML file filename = os.path.join(os.path.dirname(__file__), "data", "list_in_value.xml") xml_el = lxml.etree.parse(filename).getroot() - with raises(db.ServerConfigurationException) as exc_info: + with raises(TypeError) as exc_info: db.common.models._parse_single_xml_element(xml_el) - assert "invalid XML: List valued properties" in exc_info.value.msg + assert "Invalid datatype: List valued properties" in str(exc_info.value)