From 2aaa75e769c34faceab16f699c3df97b54d25376 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Henrik=20tom=20W=C3=B6rden?= <h.tomwoerden@indiscale.com>
Date: Tue, 23 Mar 2021 14:59:49 +0000
Subject: [PATCH] FIX CI
---
.docker/Dockerfile | 4 ++--
.gitlab-ci.yml | 25 ++++++++++++------------
CHANGELOG.md | 1 +
README_SETUP.md | 4 ++--
integrationtests/test_crawler_basics.py | 3 +++
integrationtests/test_im_und_export.py | 9 ++-------
src/caosadvancedtools/crawler.py | 4 ++++
src/caosadvancedtools/export_related.py | 20 +++++++++----------
src/caosadvancedtools/import_from_xml.py | 12 ++++++------
unittests/test_crawler.py | 4 ++++
10 files changed, 46 insertions(+), 40 deletions(-)
diff --git a/.docker/Dockerfile b/.docker/Dockerfile
index d00258d9..1651fa08 100644
--- a/.docker/Dockerfile
+++ b/.docker/Dockerfile
@@ -22,11 +22,11 @@ ADD https://gitlab.com/api/v4/projects/13656973/repository/branches/dev \
pylib_version.json
RUN git clone https://gitlab.com/caosdb/caosdb-pylib.git && \
cd caosdb-pylib && git checkout dev && pip3 install .
+# At least recommonmark 0.6 required.
+RUN pip3 install recommonmark sphinx-rtd-theme
COPY . /git
RUN rm -r /git/.git \
&& mv /git/.docker/pycaosdb.ini /git/integrationtests
RUN cd /git && pip3 install .
WORKDIR /git/integrationtests
CMD /wait-for-it.sh caosdb-server:10443 -t 500 -- ./test.sh
-# At least recommonmark 0.6 required.
-RUN pip3 install recommonmark sphinx-rtd-theme
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 87d80317..60a80123 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -23,11 +23,7 @@
variables:
CI_REGISTRY_IMAGE: $CI_REGISTRY/caosdb/src/caosdb-advanced-user-tools/testenv:latest
CI_REGISTRY_IMAGE_BASE: $CI_REGISTRY/caosdb/src/caosdb-advanced-user-tools/base:latest
- # When using dind, it's wise to use the overlayfs driver for
- # improved performance.
-services:
- - docker:19.03.0-dind
stages:
- setup
@@ -38,7 +34,14 @@ stages:
- deploy
test:
- tags: [cached-dind]
+ tags: [docker]
+ services:
+ - docker:20.10.5-dind
+ variables:
+ # This is a workaround for the gitlab-runner health check mechanism when
+ # using docker-dind service. The runner will otherwise guess the port
+ # wrong and the health check will timeout.
+ SERVICE_PORT_2376_TCP_PORT: 2375
stage: integrationtest
image: $CI_REGISTRY_IMAGE_BASE
script:
@@ -48,11 +51,8 @@ test:
- echo $CAOSDB_TAG
- time docker load < /image-cache/caosdb-advanced-testenv.tar || true
- time docker load < /image-cache/mariadb.tar || true
- - time docker load < /image-cache/caosdb.tar || true
+ - time docker load < /image-cache/caosdb-dev.tar || true
- docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY
- - time docker pull $CI_REGISTRY_IMAGE
- - time docker pull mariadb:10.4
- - time docker pull $CI_REGISTRY/caosdb/src/caosdb-deploy:$CAOSDB_TAG
- EXEPATH=`pwd` CAOSDB_TAG=$CAOSDB_TAG docker-compose
-f .docker/docker-compose.yml up -d
- cd .docker
@@ -75,11 +75,10 @@ build-testenv:
tags: [cached-dind]
image: docker:18.09
stage: setup
- only:
- - schedules
- - web
+ # Hint: do not use only here; the image needs always to be build since it
+ # contains the repo code
+ #only:
script:
- - df -h
- docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY
# use here general latest or specific branch latest...
- docker build
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 3973a4b3..95772fc8 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -29,6 +29,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
### Changed ###
+- identifiables must have at least one property or a name
* `caosadvancedtools.serverside.helper.init_data_model` also checks the role
and data type of entities.
* The `caosadvancedtools.table_importer.date_converter` now actually returns
diff --git a/README_SETUP.md b/README_SETUP.md
index 243fba2d..9b7b27ec 100644
--- a/README_SETUP.md
+++ b/README_SETUP.md
@@ -14,7 +14,7 @@ Dependencies will be installed automatically if you use the below described proc
- `xlrd>=1.2.0`
For testing:
-- `tox`
+- `tox`
## Installation
@@ -33,7 +33,7 @@ For testing:
the extroot of the empty profile to be used is located at).
3. Start an empty (!) CaosDB instance (with the mounted extroot). The
database will be cleared during testing, so it's important to use
- an empty insctance.
+ an empty instance.
4. Run `test.sh`.
## Code Formatting
diff --git a/integrationtests/test_crawler_basics.py b/integrationtests/test_crawler_basics.py
index 85fca282..7da90844 100644
--- a/integrationtests/test_crawler_basics.py
+++ b/integrationtests/test_crawler_basics.py
@@ -65,6 +65,7 @@ class CrawlerTest(unittest.TestCase):
self.rec2.add_parent(name="Test_Type_2")
self.rec3 = db.Record()
self.rec3.add_parent(name="Test_Type_3")
+ self.rec3.add_property(name="Test_Prop", value="Test")
def test_check_existence(self):
# This hasn't been inserted yet:
@@ -92,6 +93,7 @@ class CrawlerTest(unittest.TestCase):
old_id = id(identifiables[0])
reference_to_first = identifiables[0]
assert reference_to_first is identifiables[0]
+
Crawler.find_or_insert_identifiables(identifiables)
for el in identifiables:
@@ -107,6 +109,7 @@ class CrawlerTest(unittest.TestCase):
def tearDown(self):
setup_module()
+
# Delete nameless entities
for el in [self.rec1, self.rec2, self.rec3]:
try:
diff --git a/integrationtests/test_im_und_export.py b/integrationtests/test_im_und_export.py
index 5c7584e6..db26249b 100644
--- a/integrationtests/test_im_und_export.py
+++ b/integrationtests/test_im_und_export.py
@@ -13,13 +13,8 @@ if __name__ == "__main__":
directory = TemporaryDirectory()
export(rec.id, directory=directory.name)
# delete everything
- rec = db.execute_query("FIND record which was inserted by me")
- prop = db.execute_query("FIND property which was inserted by me")
- rt = db.execute_query("FIND recordtype which was inserted by me")
- fi = db.execute_query("FIND file which was inserted by me")
- c = db.Container()
- c.extend(rec+prop+rt+fi)
- c.delete()
+ recs = db.execute_query("FIND entity with id>99")
+ recs.delete()
assert 0 == len(db.execute_query("FIND File which is stored at "
"**/poster.pdf"))
import_xml(os.path.join(directory.name, "caosdb_data.xml"), interactive=False)
diff --git a/src/caosadvancedtools/crawler.py b/src/caosadvancedtools/crawler.py
index 5dccdd8c..1684c970 100644
--- a/src/caosadvancedtools/crawler.py
+++ b/src/caosadvancedtools/crawler.py
@@ -623,6 +623,10 @@ carefully and if the changes are ok, click on the following link:
raise ValueError("The identifiable must have at least one parent.")
query_string = "FIND Record " + ident.get_parents()[0].name
query_string += " WITH "
+ if ident.name is None and len(ident.get_properties()) == 0:
+ raise ValueError(
+ "The identifiable must have features to identify it.")
+
if ident.name is not None:
query_string += "name='{}' AND".format(ident.name)
diff --git a/src/caosadvancedtools/export_related.py b/src/caosadvancedtools/export_related.py
index 47fe2f49..00f440d2 100755
--- a/src/caosadvancedtools/export_related.py
+++ b/src/caosadvancedtools/export_related.py
@@ -47,6 +47,9 @@ def get_ids_of_related_entities(entity):
"""
entities = []
+ if isinstance(entity, int):
+ entity = db.Entity(id=entity).retrieve()
+
for par in entity.parents:
entities.append(par.id)
@@ -76,20 +79,17 @@ def recursively_collect_related(entity):
"""
all_entities = db.Container()
all_entities.append(entity)
- ids = set([entity.id])
- new_entities = [entity]
+ ids = set()
+ new_ids = set([entity.id])
- while new_entities:
- new_ids = set()
+ while new_ids:
+ ids.update(new_ids)
- for ent in new_entities:
- new_ids.update(get_ids_of_related_entities(ent))
+ for eid in list(new_ids):
+ new_ids.update(get_ids_of_related_entities(eid))
new_ids = new_ids - ids
- new_entities = retrieve_entities_with_ids(list(new_ids))
- ids.update([e.id for e in new_entities])
- all_entities.extend(new_entities)
- return all_entities
+ return retrieve_entities_with_ids(list(ids))
def invert_ids(entities):
diff --git a/src/caosadvancedtools/import_from_xml.py b/src/caosadvancedtools/import_from_xml.py
index 0bf9b1c0..9d0e03f6 100755
--- a/src/caosadvancedtools/import_from_xml.py
+++ b/src/caosadvancedtools/import_from_xml.py
@@ -57,7 +57,7 @@ def import_xml(filename, rerun=False, interactive=True):
tmpfile = create_dummy_file()
model = []
- files = []
+ files = {}
# add files to files list and properties and record types to model
@@ -70,19 +70,19 @@ def import_xml(filename, rerun=False, interactive=True):
el.file = target
else:
el.file = tmpfile
- files.append(el)
+ files[el.path] = el
if (isinstance(el, db.Property) or isinstance(el, db.RecordType)):
model.append(el)
# remove entities of the model from the container
- for el in model+files:
+ for el in model+list(files.values()):
cont.remove(el)
id_mapping = {}
- for el in model+files:
+ for el in model+list(files.values()):
id_mapping[el.id] = el
# insert/update the model
@@ -93,10 +93,10 @@ def import_xml(filename, rerun=False, interactive=True):
# insert files
if not rerun:
- for _, el in enumerate(files):
+ for _, el in enumerate(files.values()):
r = el.insert(unique=False)
else:
- for _, el in enumerate(files):
+ for _, el in enumerate(files.values()):
el.id = None
el.retrieve()
diff --git a/unittests/test_crawler.py b/unittests/test_crawler.py
index f603031e..64bf291c 100644
--- a/unittests/test_crawler.py
+++ b/unittests/test_crawler.py
@@ -45,3 +45,7 @@ class CrawlerTest(unittest.TestCase):
datatype=db.LIST("RT2"))
qs = Crawler.create_query_for_identifiable(ident)
assert qs == "FIND Record RT WITH references 2345 AND references 234567 "
+ ident = db.Record()
+ ident.add_parent(name="RT")
+ self.assertRaises(ValueError, Crawler.create_query_for_identifiable,
+ ident)
--
GitLab