Merge branch 'dev' into f-empty

.docker/Dockerfile

@@ -7,8 +7,10 @@ RUN apt-get update && \
     python3-autopep8 \
     python3-pip \
     python3-pytest \
+    python3-sphinx \
     tox \
     -y
+RUN pip3 install recommonmark sphinx-rtd-theme
 COPY .docker/wait-for-it.sh /wait-for-it.sh
 ARG PYLIB
 ADD https://gitlab.indiscale.com/api/v4/projects/97/repository/commits/${PYLIB} \

.gitlab-ci.yml

@@ -296,8 +296,9 @@ style:
 pages_prepare: &pages_prepare
   tags: [ cached-dind ]
   stage: deploy
-  needs: []
-  image: $CI_REGISTRY/caosdb/src/caosdb-pylib/testenv:latest
+  needs:
+    - job: build-testenv
+  image: $CI_REGISTRY_IMAGE
   only:
     refs:
       - /^release-.*$/i

CHANGELOG.md

@@ -14,6 +14,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
   already has a Parent (because it was explicitly given or the default Parent),
   the old Parent(s) are now overwritten with the value belonging to the 
   `parents` key.
+- If a registered identifiable states, that a reference by a Record with parent
+  RT1 is needed, then now also references from Records that have a child of RT1
+  as parent are accepted.
 
 ### Deprecated ###
 
@@ -22,6 +25,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 ### Fixed ###
 - Empty Records can now be created (https://gitlab.com/caosdb/caosdb-crawler/-/issues/27)
 
+* [#58](https://gitlab.com/caosdb/caosdb-crawler/-/issues/58) Documentation builds API docs in pipeline now.
+
 ### Security ###
 
 ### Documentation ###

src/caoscrawler/identifiable_adapters.py

@@ -40,6 +40,12 @@ from .utils import has_parent
 logger = logging.getLogger(__name__)
 
 
+def get_children_of_rt(rtname):
+    """Supply the name of a recordtype. This name and the name of all children RTs are returned in
+    a list"""
+    return [p.name for p in db.execute_query(f"FIND RECORDTYPE {rtname}")]
+
+
 def convert_value(value: Any):
     """ Returns a string representation of the value that is suitable
     to be used in the query
@@ -212,11 +218,16 @@ identifiabel, identifiable and identified record) for a Record.
                 # TODO: similar to the Identifiable class, Registred Identifiable should be a
                 # separate class too
                 if prop.name.lower() == "is_referenced_by":
-                    for rtname in prop.value:
-                        if (id(record) in referencing_entities
-                                and rtname in referencing_entities[id(record)]):
-                            identifiable_backrefs.extend(referencing_entities[id(record)][rtname])
-                        else:
+                    for givenrt in prop.value:
+                        rt_and_children = get_children_of_rt(givenrt)
+                        found = False
+                        for rtname in rt_and_children:
+                            if (id(record) in referencing_entities
+                                    and rtname in referencing_entities[id(record)]):
+                                identifiable_backrefs.extend(
+                                    referencing_entities[id(record)][rtname])
+                                found = True
+                        if not found:
                             # TODO: is this the appropriate error?
                             raise NotImplementedError(
                                 f"The following record is missing an identifying property:"

unittests/test_crawler.py

@@ -607,7 +607,7 @@ def test_create_flat_list():
     assert c in flat
 
 
-@ pytest.fixture
+@pytest.fixture
 def crawler_mocked_for_backref_test():
     crawler = Crawler()
     # mock retrieval of registered identifiabls: return Record with just a parent
@@ -651,6 +651,8 @@ def test_validation_error_print(caplog):
         caplog.clear()
 
 
+@patch("caoscrawler.identifiable_adapters.get_children_of_rt",
+       new=Mock(side_effect=lambda x: [x]))
 def test_split_into_inserts_and_updates_backref(crawler_mocked_for_backref_test):
     crawler = crawler_mocked_for_backref_test
     identlist = [Identifiable(name="A", record_type="BR"),
@@ -685,6 +687,8 @@ def test_split_into_inserts_and_updates_backref(crawler_mocked_for_backref_test)
     assert insert[0].name == "B"
 
 
+@patch("caoscrawler.identifiable_adapters.get_children_of_rt",
+       new=Mock(side_effect=lambda x: [x]))
 def test_split_into_inserts_and_updates_mult_backref(crawler_mocked_for_backref_test):
     # test whether multiple references of the same record type are correctly used
     crawler = crawler_mocked_for_backref_test
@@ -705,6 +709,8 @@ def test_split_into_inserts_and_updates_mult_backref(crawler_mocked_for_backref_
     assert len(insert) == 2
 
 
+@patch("caoscrawler.identifiable_adapters.get_children_of_rt",
+       new=Mock(side_effect=lambda x: [x]))
 def test_split_into_inserts_and_updates_diff_backref(crawler_mocked_for_backref_test):
     # test whether multiple references of the different record types are correctly used
     crawler = crawler_mocked_for_backref_test

unittests/test_file_identifiables.py

@@ -20,6 +20,8 @@ def clear_cache():
     cache_clear()
 
 
+@patch("caoscrawler.identifiable_adapters.get_children_of_rt",
+       new=Mock(side_effect=id))
 @patch("caoscrawler.identifiable_adapters.cached_get_entity_by",
        new=Mock(side_effect=mock_get_entity_by))
 def test_file_identifiable():

unittests/test_scanner.py

+# encoding: utf-8
+#
+# This file is a part of the CaosDB Project.
+#
+# Copyright (C) 2021      Henrik tom Wörden <h.tomwoerden@indiscale.com>
+#               2021-2023 Research Group Biomedical Physics,
+# Max-Planck-Institute for Dynamics and Self-Organization Göttingen
+# Alexander Schlemmer <alexander.schlemmer@ds.mpg.de>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see <https://www.gnu.org/licenses/>.
+#
 
 import json
 import logging