diff --git a/.docker/Dockerfile b/.docker/Dockerfile index 8e18096490632372d589749867c41e2244a67c11..dd4f3d258443dc1f8b2bacb8d535780e8e37e5e8 100644 --- a/.docker/Dockerfile +++ b/.docker/Dockerfile @@ -7,8 +7,10 @@ RUN apt-get update && \ python3-autopep8 \ python3-pip \ python3-pytest \ + python3-sphinx \ tox \ -y +RUN pip3 install recommonmark sphinx-rtd-theme COPY .docker/wait-for-it.sh /wait-for-it.sh ARG PYLIB ADD https://gitlab.indiscale.com/api/v4/projects/97/repository/commits/${PYLIB} \ diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 67415c1b3a7da52e3179bec8463cd69ac3c667aa..8840e613f1e1eb86f30779b8b3535e2ff97ad0cc 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -296,8 +296,9 @@ style: pages_prepare: &pages_prepare tags: [ cached-dind ] stage: deploy - needs: [] - image: $CI_REGISTRY/caosdb/src/caosdb-pylib/testenv:latest + needs: + - job: build-testenv + image: $CI_REGISTRY_IMAGE only: refs: - /^release-.*$/i diff --git a/CHANGELOG.md b/CHANGELOG.md index accc64bce8c03a1f27cbd616ab96f02a38ecf4b5..29013891d7f53dd4c4d8164e79eecfa169fcb289 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 already has a Parent (because it was explicitly given or the default Parent), the old Parent(s) are now overwritten with the value belonging to the `parents` key. +- If a registered identifiable states, that a reference by a Record with parent + RT1 is needed, then now also references from Records that have a child of RT1 + as parent are accepted. ### Deprecated ### @@ -22,6 +25,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed ### - Empty Records can now be created (https://gitlab.com/caosdb/caosdb-crawler/-/issues/27) +* [#58](https://gitlab.com/caosdb/caosdb-crawler/-/issues/58) Documentation builds API docs in pipeline now. + ### Security ### ### Documentation ### diff --git a/src/caoscrawler/identifiable_adapters.py b/src/caoscrawler/identifiable_adapters.py index 776baeaeac7caf961d6dba97641804c9e1608114..d9c9c00b22443121b4989bf988a40308b143dbf1 100644 --- a/src/caoscrawler/identifiable_adapters.py +++ b/src/caoscrawler/identifiable_adapters.py @@ -40,6 +40,12 @@ from .utils import has_parent logger = logging.getLogger(__name__) +def get_children_of_rt(rtname): + """Supply the name of a recordtype. This name and the name of all children RTs are returned in + a list""" + return [p.name for p in db.execute_query(f"FIND RECORDTYPE {rtname}")] + + def convert_value(value: Any): """ Returns a string representation of the value that is suitable to be used in the query @@ -212,11 +218,16 @@ identifiabel, identifiable and identified record) for a Record. # TODO: similar to the Identifiable class, Registred Identifiable should be a # separate class too if prop.name.lower() == "is_referenced_by": - for rtname in prop.value: - if (id(record) in referencing_entities - and rtname in referencing_entities[id(record)]): - identifiable_backrefs.extend(referencing_entities[id(record)][rtname]) - else: + for givenrt in prop.value: + rt_and_children = get_children_of_rt(givenrt) + found = False + for rtname in rt_and_children: + if (id(record) in referencing_entities + and rtname in referencing_entities[id(record)]): + identifiable_backrefs.extend( + referencing_entities[id(record)][rtname]) + found = True + if not found: # TODO: is this the appropriate error? raise NotImplementedError( f"The following record is missing an identifying property:" diff --git a/unittests/test_crawler.py b/unittests/test_crawler.py index dc53cb099eb5e4b225b13461176504a003f2d2ba..91e0e86a6d6cf2967ab3567a2ef93b7ccde56e64 100644 --- a/unittests/test_crawler.py +++ b/unittests/test_crawler.py @@ -607,7 +607,7 @@ def test_create_flat_list(): assert c in flat -@ pytest.fixture +@pytest.fixture def crawler_mocked_for_backref_test(): crawler = Crawler() # mock retrieval of registered identifiabls: return Record with just a parent @@ -651,6 +651,8 @@ def test_validation_error_print(caplog): caplog.clear() +@patch("caoscrawler.identifiable_adapters.get_children_of_rt", + new=Mock(side_effect=lambda x: [x])) def test_split_into_inserts_and_updates_backref(crawler_mocked_for_backref_test): crawler = crawler_mocked_for_backref_test identlist = [Identifiable(name="A", record_type="BR"), @@ -685,6 +687,8 @@ def test_split_into_inserts_and_updates_backref(crawler_mocked_for_backref_test) assert insert[0].name == "B" +@patch("caoscrawler.identifiable_adapters.get_children_of_rt", + new=Mock(side_effect=lambda x: [x])) def test_split_into_inserts_and_updates_mult_backref(crawler_mocked_for_backref_test): # test whether multiple references of the same record type are correctly used crawler = crawler_mocked_for_backref_test @@ -705,6 +709,8 @@ def test_split_into_inserts_and_updates_mult_backref(crawler_mocked_for_backref_ assert len(insert) == 2 +@patch("caoscrawler.identifiable_adapters.get_children_of_rt", + new=Mock(side_effect=lambda x: [x])) def test_split_into_inserts_and_updates_diff_backref(crawler_mocked_for_backref_test): # test whether multiple references of the different record types are correctly used crawler = crawler_mocked_for_backref_test diff --git a/unittests/test_file_identifiables.py b/unittests/test_file_identifiables.py index 2852b40ffde98180d5dd7b11b9109cc5875502da..4ec02aa3fc497f8dc35adc709533ef5b35066f3a 100644 --- a/unittests/test_file_identifiables.py +++ b/unittests/test_file_identifiables.py @@ -20,6 +20,8 @@ def clear_cache(): cache_clear() +@patch("caoscrawler.identifiable_adapters.get_children_of_rt", + new=Mock(side_effect=id)) @patch("caoscrawler.identifiable_adapters.cached_get_entity_by", new=Mock(side_effect=mock_get_entity_by)) def test_file_identifiable(): diff --git a/unittests/test_scanner.py b/unittests/test_scanner.py index e99de65289cf4cc7b65f609955905c9c44efdd66..21b7da10ef860378a4779f34f8f1b26c6a54f359 100644 --- a/unittests/test_scanner.py +++ b/unittests/test_scanner.py @@ -1,3 +1,25 @@ +# encoding: utf-8 +# +# This file is a part of the CaosDB Project. +# +# Copyright (C) 2021 Henrik tom Wörden <h.tomwoerden@indiscale.com> +# 2021-2023 Research Group Biomedical Physics, +# Max-Planck-Institute for Dynamics and Self-Organization Göttingen +# Alexander Schlemmer <alexander.schlemmer@ds.mpg.de> +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see <https://www.gnu.org/licenses/>. +# import json import logging