Merge branch 'f-fix-ci' into 'dev'

FIX CI

See merge request !1

.docker/Dockerfile

@@ -22,11 +22,11 @@ ADD https://gitlab.com/api/v4/projects/13656973/repository/branches/dev \
    pylib_version.json
 RUN git clone https://gitlab.com/caosdb/caosdb-pylib.git && \
    cd caosdb-pylib && git checkout dev && pip3 install .
+# At least recommonmark 0.6 required.
+RUN pip3 install recommonmark sphinx-rtd-theme
 COPY . /git
 RUN rm -r /git/.git \
     && mv /git/.docker/pycaosdb.ini /git/integrationtests
 RUN cd /git && pip3 install .
 WORKDIR /git/integrationtests
 CMD /wait-for-it.sh caosdb-server:10443 -t 500 -- ./test.sh
-# At least recommonmark 0.6 required.
-RUN pip3 install recommonmark sphinx-rtd-theme

.gitlab-ci.yml

@@ -23,11 +23,7 @@
 variables:
    CI_REGISTRY_IMAGE: $CI_REGISTRY/caosdb/src/caosdb-advanced-user-tools/testenv:latest
    CI_REGISTRY_IMAGE_BASE: $CI_REGISTRY/caosdb/src/caosdb-advanced-user-tools/base:latest
-   # When using dind, it's wise to use the overlayfs driver for
-   # improved performance.
 
-services:
-  - docker:19.03.0-dind
 
 stages: 
   - setup
@@ -38,7 +34,14 @@ stages:
   - deploy
 
 test:
-  tags: [cached-dind]
+  tags: [docker]
+  services:
+    - docker:20.10.5-dind
+  variables:
+    # This is a workaround for the gitlab-runner health check mechanism when
+    # using docker-dind service.  The runner will otherwise guess the port
+    # wrong and the health check will timeout.
+    SERVICE_PORT_2376_TCP_PORT: 2375
   stage: integrationtest
   image: $CI_REGISTRY_IMAGE_BASE
   script:
@@ -48,11 +51,8 @@ test:
       - echo $CAOSDB_TAG
       - time docker load < /image-cache/caosdb-advanced-testenv.tar || true
       - time docker load < /image-cache/mariadb.tar || true
-      - time docker load < /image-cache/caosdb.tar || true
+      - time docker load < /image-cache/caosdb-dev.tar || true
       - docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY
-      - time docker pull $CI_REGISTRY_IMAGE
-      - time docker pull mariadb:10.4
-      - time docker pull $CI_REGISTRY/caosdb/src/caosdb-deploy:$CAOSDB_TAG
       - EXEPATH=`pwd` CAOSDB_TAG=$CAOSDB_TAG docker-compose 
         -f .docker/docker-compose.yml up -d
       - cd .docker 
@@ -75,11 +75,10 @@ build-testenv:
   tags: [cached-dind]
   image: docker:18.09
   stage: setup
-  only:
-      - schedules
-      - web
+  # Hint: do not use only here; the image needs always to be build since it 
+  # contains the repo code
+  #only:
   script: 
-      - df -h
       - docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY
         # use here general latest or specific branch latest...
       - docker build 

CHANGELOG.md

@@ -29,6 +29,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Changed ###
 
+- identifiables must have at least one property or a name
 * `caosadvancedtools.serverside.helper.init_data_model` also checks the role
   and data type of entities.
 * The `caosadvancedtools.table_importer.date_converter` now actually returns

README_SETUP.md

@@ -33,7 +33,7 @@ For testing:
    the extroot of the empty profile to be used is located at).
 3. Start an empty (!) CaosDB instance (with the mounted extroot). The
    database will be cleared during testing, so it's important to use
-   an empty insctance.
+   an empty instance.
 4. Run `test.sh`.
 
 ## Code Formatting

integrationtests/test_crawler_basics.py

@@ -65,6 +65,7 @@ class CrawlerTest(unittest.TestCase):
         self.rec2.add_parent(name="Test_Type_2")
         self.rec3 = db.Record()
         self.rec3.add_parent(name="Test_Type_3")
+        self.rec3.add_property(name="Test_Prop", value="Test")
 
     def test_check_existence(self):
         # This hasn't been inserted yet:
@@ -92,6 +93,7 @@ class CrawlerTest(unittest.TestCase):
         old_id = id(identifiables[0])
         reference_to_first = identifiables[0]
         assert reference_to_first is identifiables[0]
+
         Crawler.find_or_insert_identifiables(identifiables)
 
         for el in identifiables:
@@ -107,6 +109,7 @@ class CrawlerTest(unittest.TestCase):
 
     def tearDown(self):
         setup_module()
+
         # Delete nameless entities
         for el in [self.rec1, self.rec2, self.rec3]:
             try:

integrationtests/test_im_und_export.py

@@ -13,13 +13,8 @@ if __name__ == "__main__":
     directory = TemporaryDirectory()
     export(rec.id, directory=directory.name)
     # delete everything
-    rec = db.execute_query("FIND record which was inserted by me")
-    prop = db.execute_query("FIND property which was inserted by me")
-    rt = db.execute_query("FIND recordtype which was inserted by me")
-    fi = db.execute_query("FIND file which was inserted by me")
-    c = db.Container()
-    c.extend(rec+prop+rt+fi)
-    c.delete()
+    recs = db.execute_query("FIND entity with id>99")
+    recs.delete()
     assert 0 == len(db.execute_query("FIND File which is stored at "
                                      "**/poster.pdf"))
     import_xml(os.path.join(directory.name, "caosdb_data.xml"), interactive=False)

src/caosadvancedtools/crawler.py

@@ -623,6 +623,10 @@ carefully and if the changes are ok, click on the following link:
             raise ValueError("The identifiable must have at least one parent.")
         query_string = "FIND Record " + ident.get_parents()[0].name
         query_string += " WITH "
+        if ident.name is None and len(ident.get_properties()) == 0:
+            raise ValueError(
+                "The identifiable must have features to identify it.")
+
         if ident.name is not None:
             query_string += "name='{}' AND".format(ident.name)
 

src/caosadvancedtools/export_related.py

@@ -47,6 +47,9 @@ def get_ids_of_related_entities(entity):
     """
     entities = []
 
+    if isinstance(entity, int):
+        entity = db.Entity(id=entity).retrieve()
+
     for par in entity.parents:
         entities.append(par.id)
 
@@ -76,20 +79,17 @@ def recursively_collect_related(entity):
     """
     all_entities = db.Container()
     all_entities.append(entity)
-    ids = set([entity.id])
-    new_entities = [entity]
+    ids = set()
+    new_ids = set([entity.id])
 
-    while new_entities:
-        new_ids = set()
+    while new_ids:
+        ids.update(new_ids)
 
-        for ent in new_entities:
-            new_ids.update(get_ids_of_related_entities(ent))
+        for eid in list(new_ids):
+            new_ids.update(get_ids_of_related_entities(eid))
         new_ids = new_ids - ids
-        new_entities = retrieve_entities_with_ids(list(new_ids))
-        ids.update([e.id for e in new_entities])
-        all_entities.extend(new_entities)
 
-    return all_entities
+    return retrieve_entities_with_ids(list(ids))
 
 
 def invert_ids(entities):

src/caosadvancedtools/import_from_xml.py

@@ -57,7 +57,7 @@ def import_xml(filename, rerun=False, interactive=True):
     tmpfile = create_dummy_file()
     model = []
 
-    files = []
+    files = {}
 
     # add files to files list and properties and record types to model
 
@@ -70,19 +70,19 @@ def import_xml(filename, rerun=False, interactive=True):
                 el.file = target
             else:
                 el.file = tmpfile
-            files.append(el)
+            files[el.path] = el
 
         if (isinstance(el, db.Property) or isinstance(el, db.RecordType)):
             model.append(el)
 
     # remove entities of the model from the container
 
-    for el in model+files:
+    for el in model+list(files.values()):
         cont.remove(el)
 
     id_mapping = {}
 
-    for el in model+files:
+    for el in model+list(files.values()):
         id_mapping[el.id] = el
 
     # insert/update the model
@@ -93,10 +93,10 @@ def import_xml(filename, rerun=False, interactive=True):
     # insert files
 
     if not rerun:
-        for _, el in enumerate(files):
+        for _, el in enumerate(files.values()):
             r = el.insert(unique=False)
     else:
-        for _, el in enumerate(files):
+        for _, el in enumerate(files.values()):
             el.id = None
             el.retrieve()
 

unittests/test_crawler.py

@@ -45,3 +45,7 @@ class CrawlerTest(unittest.TestCase):
                            datatype=db.LIST("RT2"))
         qs = Crawler.create_query_for_identifiable(ident)
         assert qs == "FIND Record RT WITH references 2345 AND references 234567 "
+        ident = db.Record()
+        ident.add_parent(name="RT")
+        self.assertRaises(ValueError, Crawler.create_query_for_identifiable,
+                          ident)