Merge branch 'dev' into f-output

.docker-base/Dockerfile

@@ -9,6 +9,34 @@ RUN apk add --no-cache py3-pip python3 python3-dev gcc make \
     git bash curl gettext  py3-requests 
 RUN apk add --no-cache libffi-dev openssl-dev libc-dev libxslt libxslt-dev \
     libxml2 libxml2-dev
+
+# install rust (needed for compiling a docker-compose dependency)
+# This is necessary until alpine comes with an up to date RUST
+# copied from https://github.com/rust-lang/docker-rust/blob/bbc7feb12033da3909dced4e88ddbb6964fbc328/1.50.0/alpine3.13/Dockerfile
+
+ENV RUSTUP_HOME=/usr/local/rustup \
+    CARGO_HOME=/usr/local/cargo \
+    PATH=/usr/local/cargo/bin:$PATH \
+    RUST_VERSION=1.50.0
+
+RUN set -eux; \
+    apkArch="$(apk --print-arch)"; \
+    case "$apkArch" in \
+        x86_64) rustArch='x86_64-unknown-linux-musl'; rustupSha256='05c5c05ec76671d73645aac3afbccf2187352fce7e46fc85be859f52a42797f6' ;; \
+        aarch64) rustArch='aarch64-unknown-linux-musl'; rustupSha256='6a8a480d8d9e7f8c6979d7f8b12bc59da13db67970f7b13161ff409f0a771213' ;; \
+        *) echo >&2 "unsupported architecture: $apkArch"; exit 1 ;; \
+    esac; \
+    url="https://static.rust-lang.org/rustup/archive/1.23.1/${rustArch}/rustup-init"; \
+    wget "$url"; \
+    echo "${rustupSha256} *rustup-init" | sha256sum -c -; \
+    chmod +x rustup-init; \
+    ./rustup-init -y --no-modify-path --profile minimal --default-toolchain $RUST_VERSION --default-host ${rustArch}; \
+    rm rustup-init; \
+    chmod -R a+w $RUSTUP_HOME $CARGO_HOME; \
+    rustup --version; \
+    cargo --version; \
+    rustc --version;
+
 RUN pip3 install docker-compose==1.25
 
 # Script for waiting on LA server

.docker/Dockerfile

@@ -15,16 +15,18 @@ RUN apt-get update && \
     python3-pytest \
     libxml2 \
     -y
+
+
 COPY .docker/wait-for-it.sh /wait-for-it.sh
 ADD https://gitlab.com/api/v4/projects/13656973/repository/branches/dev \
    pylib_version.json
 RUN git clone https://gitlab.com/caosdb/caosdb-pylib.git && \
    cd caosdb-pylib && git checkout dev && pip3 install .
+# At least recommonmark 0.6 required.
+RUN pip3 install recommonmark sphinx-rtd-theme
 COPY . /git
 RUN rm -r /git/.git \
     && mv /git/.docker/pycaosdb.ini /git/integrationtests
 RUN cd /git && pip3 install .
 WORKDIR /git/integrationtests
 CMD /wait-for-it.sh caosdb-server:10443 -t 500 -- ./test.sh
-# At least recommonmark 0.6 required.
-RUN pip3 install recommonmark sphinx-rtd-theme

.docker/docker-compose.yml

@@ -7,7 +7,7 @@ services:
     networks:
       - caosnet
   caosdb-server:
-    image: "$CI_REGISTRY_INDISCALE/caosdb/src/caosdb-deploy:$CAOSDB_TAG"
+    image: "$CI_REGISTRY/caosdb/src/caosdb-deploy:$CAOSDB_TAG"
     user: 999:999
     depends_on:
       - sqldb

.gitlab-ci.yml

@@ -21,13 +21,9 @@
 # along with this program. If not, see <https://www.gnu.org/licenses/>.
 
 variables:
-   CI_REGISTRY_IMAGE: $CI_REGISTRY/caosdb/caosdb-advanced-user-tools/testenv:latest
-   CI_REGISTRY_IMAGE_BASE: $CI_REGISTRY/caosdb/caosdb-advanced-user-tools/base:latest
-   # When using dind, it's wise to use the overlayfs driver for
-   # improved performance.
+   CI_REGISTRY_IMAGE: $CI_REGISTRY/caosdb/src/caosdb-advanced-user-tools/testenv:latest
+   CI_REGISTRY_IMAGE_BASE: $CI_REGISTRY/caosdb/src/caosdb-advanced-user-tools/base:latest
 
-services:
-  - docker:19.03.0-dind
 
 stages: 
   - setup
@@ -38,7 +34,14 @@ stages:
   - deploy
 
 test:
-  tags: [cached-dind]
+  tags: [docker]
+  services:
+    - docker:20.10.5-dind
+  variables:
+    # This is a workaround for the gitlab-runner health check mechanism when
+    # using docker-dind service.  The runner will otherwise guess the port
+    # wrong and the health check will timeout.
+    SERVICE_PORT_2376_TCP_PORT: 2375
   stage: integrationtest
   image: $CI_REGISTRY_IMAGE_BASE
   script:
@@ -48,12 +51,8 @@ test:
       - echo $CAOSDB_TAG
       - time docker load < /image-cache/caosdb-advanced-testenv.tar || true
       - time docker load < /image-cache/mariadb.tar || true
-      - time docker load < /image-cache/caosdb.tar || true
+      - time docker load < /image-cache/caosdb-dev.tar || true
       - docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY
-      - docker login -u gitlab+deploy-token-ci-pull -p $TOKEN_CI_PULL $CI_REGISTRY_INDISCALE
-      - time docker pull $CI_REGISTRY_IMAGE
-      - time docker pull mariadb:10.4
-      - time docker pull $CI_REGISTRY_INDISCALE/caosdb/src/caosdb-deploy:$CAOSDB_TAG
       - EXEPATH=`pwd` CAOSDB_TAG=$CAOSDB_TAG docker-compose 
         -f .docker/docker-compose.yml up -d
       - cd .docker 
@@ -76,11 +75,10 @@ build-testenv:
   tags: [cached-dind]
   image: docker:18.09
   stage: setup
-  only:
-      - schedules
-      - web
+  # Hint: do not use only here; the image needs always to be build since it 
+  # contains the repo code
+  #only:
   script: 
-      - df -h
       - docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY
         # use here general latest or specific branch latest...
       - docker build 

CHANGELOG.md

@@ -29,6 +29,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Changed ###
 
+- identifiables must have at least one property or a name
 * `caosadvancedtools.serverside.helper.init_data_model` also checks the role
   and data type of entities.
 * The `caosadvancedtools.table_importer.date_converter` now actually returns

README.md

-[![pipeline status](https://gitlab.com/caosdb/caosdb-advanced-user-tools/badges/master/pipeline.svg)](https://gitlab.com/caosdb/caosdb-advanced-user-tools/commits/master)
+# README
 
-Project migrated to https://gitlab.com/caosdb
-
-# Welcome
+## Welcome
 
 This is the **CaosDB Advanced User Tools** repository and a part of the
-CaosDB project. This project contains tools that are beyond the typical use of
+CaosDB project.
+This project contains tools that are beyond the typical use of
 the CaosDB python client. Especially, this includes the crawler which will
 typically be used by a data curator.
 
-# Setup
+## Setup
 
 Please read the [README_SETUP.md](README_SETUP.md) for instructions on how to
 setup this code.
 
 
-# Further Reading
+## Further Reading
+
+Please refer to the [official documentation](https://docs.indiscale.com/caosdb-advanced-user-tools/) for more information.
+
+## Contributing
+
+Thank you very much to all contributers—[past, present](https://gitlab.com/caosdb/caosdb/-/blob/dev/HUMANS.md), and prospective ones.
 
-Please refer to the [official gitlab repository of the CaosDB
-project](https://gitlab.com/caosdb/caosdb) for more information.
+### Code of Conduct
 
-# License
+By participating, you are expected to uphold our [Code of Conduct](https://gitlab.com/caosdb/caosdb/-/blob/dev/CODE_OF_CONDUCT.md).
 
-Copyright (C) 2018 Research Group Biomedical Physics, Max Planck Institute for
-Dynamics and Self-Organization Göttingen.
+### How to Contribute
+
+* You found a bug, have a question, or want to request a feature? Please 
+[create an issue](https://gitlab.com/caosdb/caosdb-advanced-user-tools/-/issues).
+* You want to contribute code? Please fork the repository and create a merge 
+request in GitLab and choose this repository as target. Make sure to select
+"Allow commits from members who can merge the target branch" under Contribution
+when creating the merge request. This allows our team to work with you on your request.
+- If you have a suggestion for the [documentation](https://docs.indiscale.com/caosdb-advanced-user-tools/), 
+the preferred way is also a merge request as describe above (the documentation resides in `src/doc`).
+However, you can also create an issue for it. 
+- You can also contact us at **info (AT) caosdb.de**.
+
+## License
+
+* Copyright (C) 2018 Research Group Biomedical Physics, Max Planck Institute
+  for Dynamics and Self-Organization Göttingen.
+* Copyright (C) 2020-2021 Indiscale GmbH <info@indiscale.com>
 
 All files in this repository are licensed under a [GNU Affero General Public
 License](LICENCE.md) (version 3 or later).
-

README_SETUP.md

@@ -14,7 +14,7 @@ Dependencies will be installed automatically if you use the below described proc
 - `xlrd>=1.2.0`
 
 For testing:
-- `tox` 
+- `tox`
 
 
 ## Installation
@@ -33,7 +33,7 @@ For testing:
    the extroot of the empty profile to be used is located at).
 3. Start an empty (!) CaosDB instance (with the mounted extroot). The
    database will be cleared during testing, so it's important to use
-   an empty insctance.
+   an empty instance.
 4. Run `test.sh`.
 
 ## Code Formatting

integrationtests/test_crawler_basics.py

@@ -65,6 +65,7 @@ class CrawlerTest(unittest.TestCase):
         self.rec2.add_parent(name="Test_Type_2")
         self.rec3 = db.Record()
         self.rec3.add_parent(name="Test_Type_3")
+        self.rec3.add_property(name="Test_Prop", value="Test")
 
     def test_check_existence(self):
         # This hasn't been inserted yet:
@@ -92,6 +93,7 @@ class CrawlerTest(unittest.TestCase):
         old_id = id(identifiables[0])
         reference_to_first = identifiables[0]
         assert reference_to_first is identifiables[0]
+
         Crawler.find_or_insert_identifiables(identifiables)
 
         for el in identifiables:
@@ -107,6 +109,7 @@ class CrawlerTest(unittest.TestCase):
 
     def tearDown(self):
         setup_module()
+
         # Delete nameless entities
         for el in [self.rec1, self.rec2, self.rec3]:
             try:

integrationtests/test_im_und_export.py

@@ -13,13 +13,8 @@ if __name__ == "__main__":
     directory = TemporaryDirectory()
     export(rec.id, directory=directory.name)
     # delete everything
-    rec = db.execute_query("FIND record which was inserted by me")
-    prop = db.execute_query("FIND property which was inserted by me")
-    rt = db.execute_query("FIND recordtype which was inserted by me")
-    fi = db.execute_query("FIND file which was inserted by me")
-    c = db.Container()
-    c.extend(rec+prop+rt+fi)
-    c.delete()
+    recs = db.execute_query("FIND entity with id>99")
+    recs.delete()
     assert 0 == len(db.execute_query("FIND File which is stored at "
                                      "**/poster.pdf"))
     import_xml(os.path.join(directory.name, "caosdb_data.xml"), interactive=False)

src/caosadvancedtools/crawler.py

@@ -629,6 +629,10 @@ carefully and if the changes are ok, click on the following link:
             raise ValueError("The identifiable must have at least one parent.")
         query_string = "FIND Record " + ident.get_parents()[0].name
         query_string += " WITH "
+        if ident.name is None and len(ident.get_properties()) == 0:
+            raise ValueError(
+                "The identifiable must have features to identify it.")
+
         if ident.name is not None:
             query_string += "name='{}' AND".format(ident.name)
 

src/caosadvancedtools/export_related.py

@@ -47,6 +47,9 @@ def get_ids_of_related_entities(entity):
     """
     entities = []
 
+    if isinstance(entity, int):
+        entity = db.Entity(id=entity).retrieve()
+
     for par in entity.parents:
         entities.append(par.id)
 
@@ -76,20 +79,17 @@ def recursively_collect_related(entity):
     """
     all_entities = db.Container()
     all_entities.append(entity)
-    ids = set([entity.id])
-    new_entities = [entity]
+    ids = set()
+    new_ids = set([entity.id])
 
-    while new_entities:
-        new_ids = set()
+    while new_ids:
+        ids.update(new_ids)
 
-        for ent in new_entities:
-            new_ids.update(get_ids_of_related_entities(ent))
+        for eid in list(new_ids):
+            new_ids.update(get_ids_of_related_entities(eid))
         new_ids = new_ids - ids
-        new_entities = retrieve_entities_with_ids(list(new_ids))
-        ids.update([e.id for e in new_entities])
-        all_entities.extend(new_entities)
 
-    return all_entities
+    return retrieve_entities_with_ids(list(ids))
 
 
 def invert_ids(entities):

src/caosadvancedtools/import_from_xml.py

@@ -57,7 +57,7 @@ def import_xml(filename, rerun=False, interactive=True):
     tmpfile = create_dummy_file()
     model = []
 
-    files = []
+    files = {}
 
     # add files to files list and properties and record types to model
 
@@ -70,19 +70,19 @@ def import_xml(filename, rerun=False, interactive=True):
                 el.file = target
             else:
                 el.file = tmpfile
-            files.append(el)
+            files[el.path] = el
 
         if (isinstance(el, db.Property) or isinstance(el, db.RecordType)):
             model.append(el)
 
     # remove entities of the model from the container
 
-    for el in model+files:
+    for el in model+list(files.values()):
         cont.remove(el)
 
     id_mapping = {}
 
-    for el in model+files:
+    for el in model+list(files.values()):
         id_mapping[el.id] = el
 
     # insert/update the model
@@ -93,10 +93,10 @@ def import_xml(filename, rerun=False, interactive=True):
     # insert files
 
     if not rerun:
-        for _, el in enumerate(files):
+        for _, el in enumerate(files.values()):
             r = el.insert(unique=False)
     else:
-        for _, el in enumerate(files):
+        for _, el in enumerate(files.values()):
             el.id = None
             el.retrieve()
 

unittests/test_crawler.py

@@ -45,3 +45,7 @@ class CrawlerTest(unittest.TestCase):
                            datatype=db.LIST("RT2"))
         qs = Crawler.create_query_for_identifiable(ident)
         assert qs == "FIND Record RT WITH references 2345 AND references 234567 "
+        ident = db.Record()
+        ident.add_parent(name="RT")
+        self.assertRaises(ValueError, Crawler.create_query_for_identifiable,
+                          ident)