diff --git a/.docker-base/Dockerfile b/.docker-base/Dockerfile
index 2152183a410302df34d35ec6f514399678e0baaf..923924e75e03c6ca8346b17cdf87eda78efd766f 100644
--- a/.docker-base/Dockerfile
+++ b/.docker-base/Dockerfile
@@ -9,6 +9,34 @@ RUN apk add --no-cache py3-pip python3 python3-dev gcc make \
git bash curl gettext py3-requests
RUN apk add --no-cache libffi-dev openssl-dev libc-dev libxslt libxslt-dev \
libxml2 libxml2-dev
+
+# install rust (needed for compiling a docker-compose dependency)
+# This is necessary until alpine comes with an up to date RUST
+# copied from https://github.com/rust-lang/docker-rust/blob/bbc7feb12033da3909dced4e88ddbb6964fbc328/1.50.0/alpine3.13/Dockerfile
+
+ENV RUSTUP_HOME=/usr/local/rustup \
+ CARGO_HOME=/usr/local/cargo \
+ PATH=/usr/local/cargo/bin:$PATH \
+ RUST_VERSION=1.50.0
+
+RUN set -eux; \
+ apkArch="$(apk --print-arch)"; \
+ case "$apkArch" in \
+ x86_64) rustArch='x86_64-unknown-linux-musl'; rustupSha256='05c5c05ec76671d73645aac3afbccf2187352fce7e46fc85be859f52a42797f6' ;; \
+ aarch64) rustArch='aarch64-unknown-linux-musl'; rustupSha256='6a8a480d8d9e7f8c6979d7f8b12bc59da13db67970f7b13161ff409f0a771213' ;; \
+ *) echo >&2 "unsupported architecture: $apkArch"; exit 1 ;; \
+ esac; \
+ url="https://static.rust-lang.org/rustup/archive/1.23.1/${rustArch}/rustup-init"; \
+ wget "$url"; \
+ echo "${rustupSha256} *rustup-init" | sha256sum -c -; \
+ chmod +x rustup-init; \
+ ./rustup-init -y --no-modify-path --profile minimal --default-toolchain $RUST_VERSION --default-host ${rustArch}; \
+ rm rustup-init; \
+ chmod -R a+w $RUSTUP_HOME $CARGO_HOME; \
+ rustup --version; \
+ cargo --version; \
+ rustc --version;
+
RUN pip3 install docker-compose==1.25
# Script for waiting on LA server
diff --git a/.docker/Dockerfile b/.docker/Dockerfile
index d5d2fe66770b2d37f7ecbb718a2260cdd7f501c1..1651fa08f7fb157e007cf5c4a992f548b7d411ba 100644
--- a/.docker/Dockerfile
+++ b/.docker/Dockerfile
@@ -15,16 +15,18 @@ RUN apt-get update && \
python3-pytest \
libxml2 \
-y
+
+
COPY .docker/wait-for-it.sh /wait-for-it.sh
ADD https://gitlab.com/api/v4/projects/13656973/repository/branches/dev \
pylib_version.json
RUN git clone https://gitlab.com/caosdb/caosdb-pylib.git && \
cd caosdb-pylib && git checkout dev && pip3 install .
+# At least recommonmark 0.6 required.
+RUN pip3 install recommonmark sphinx-rtd-theme
COPY . /git
RUN rm -r /git/.git \
&& mv /git/.docker/pycaosdb.ini /git/integrationtests
RUN cd /git && pip3 install .
WORKDIR /git/integrationtests
CMD /wait-for-it.sh caosdb-server:10443 -t 500 -- ./test.sh
-# At least recommonmark 0.6 required.
-RUN pip3 install recommonmark sphinx-rtd-theme
diff --git a/.docker/docker-compose.yml b/.docker/docker-compose.yml
index e859e4337653a41dd2e17a819760b18fe2185c5e..36964ee68b7e384267a08484524de1f72cdfad6d 100644
--- a/.docker/docker-compose.yml
+++ b/.docker/docker-compose.yml
@@ -7,7 +7,7 @@ services:
networks:
- caosnet
caosdb-server:
- image: "$CI_REGISTRY_INDISCALE/caosdb/src/caosdb-deploy:$CAOSDB_TAG"
+ image: "$CI_REGISTRY/caosdb/src/caosdb-deploy:$CAOSDB_TAG"
user: 999:999
depends_on:
- sqldb
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 8ad682bf6bac6b50ed6a98ffe42b94f2c96aabb0..60a80123632e0797e7e83d73a7306ced100937c1 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -21,13 +21,9 @@
# along with this program. If not, see <https://www.gnu.org/licenses/>.
variables:
- CI_REGISTRY_IMAGE: $CI_REGISTRY/caosdb/caosdb-advanced-user-tools/testenv:latest
- CI_REGISTRY_IMAGE_BASE: $CI_REGISTRY/caosdb/caosdb-advanced-user-tools/base:latest
- # When using dind, it's wise to use the overlayfs driver for
- # improved performance.
+ CI_REGISTRY_IMAGE: $CI_REGISTRY/caosdb/src/caosdb-advanced-user-tools/testenv:latest
+ CI_REGISTRY_IMAGE_BASE: $CI_REGISTRY/caosdb/src/caosdb-advanced-user-tools/base:latest
-services:
- - docker:19.03.0-dind
stages:
- setup
@@ -38,7 +34,14 @@ stages:
- deploy
test:
- tags: [cached-dind]
+ tags: [docker]
+ services:
+ - docker:20.10.5-dind
+ variables:
+ # This is a workaround for the gitlab-runner health check mechanism when
+ # using docker-dind service. The runner will otherwise guess the port
+ # wrong and the health check will timeout.
+ SERVICE_PORT_2376_TCP_PORT: 2375
stage: integrationtest
image: $CI_REGISTRY_IMAGE_BASE
script:
@@ -48,12 +51,8 @@ test:
- echo $CAOSDB_TAG
- time docker load < /image-cache/caosdb-advanced-testenv.tar || true
- time docker load < /image-cache/mariadb.tar || true
- - time docker load < /image-cache/caosdb.tar || true
+ - time docker load < /image-cache/caosdb-dev.tar || true
- docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY
- - docker login -u gitlab+deploy-token-ci-pull -p $TOKEN_CI_PULL $CI_REGISTRY_INDISCALE
- - time docker pull $CI_REGISTRY_IMAGE
- - time docker pull mariadb:10.4
- - time docker pull $CI_REGISTRY_INDISCALE/caosdb/src/caosdb-deploy:$CAOSDB_TAG
- EXEPATH=`pwd` CAOSDB_TAG=$CAOSDB_TAG docker-compose
-f .docker/docker-compose.yml up -d
- cd .docker
@@ -76,11 +75,10 @@ build-testenv:
tags: [cached-dind]
image: docker:18.09
stage: setup
- only:
- - schedules
- - web
+ # Hint: do not use only here; the image needs always to be build since it
+ # contains the repo code
+ #only:
script:
- - df -h
- docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY
# use here general latest or specific branch latest...
- docker build
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 3973a4b3f6b0098b871abf6394e5b9158b3e43c2..95772fc8a58ca39ad1af4cf26fd32b4c52b31d7d 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -29,6 +29,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
### Changed ###
+- identifiables must have at least one property or a name
* `caosadvancedtools.serverside.helper.init_data_model` also checks the role
and data type of entities.
* The `caosadvancedtools.table_importer.date_converter` now actually returns
diff --git a/README.md b/README.md
index 5208a711f72a3daa919e9195a5a0b05413e3de3a..2a29b8cfec187c483eb6d420dcaa5d8c7a1b3ad4 100644
--- a/README.md
+++ b/README.md
@@ -1,30 +1,49 @@
-[](https://gitlab.com/caosdb/caosdb-advanced-user-tools/commits/master)
+# README
-Project migrated to https://gitlab.com/caosdb
-
-# Welcome
+## Welcome
This is the **CaosDB Advanced User Tools** repository and a part of the
-CaosDB project. This project contains tools that are beyond the typical use of
+CaosDB project.
+This project contains tools that are beyond the typical use of
the CaosDB python client. Especially, this includes the crawler which will
typically be used by a data curator.
-# Setup
+## Setup
Please read the [README_SETUP.md](README_SETUP.md) for instructions on how to
setup this code.
-# Further Reading
+## Further Reading
+
+Please refer to the [official documentation](https://docs.indiscale.com/caosdb-advanced-user-tools/) for more information.
+
+## Contributing
+
+Thank you very much to all contributers—[past, present](https://gitlab.com/caosdb/caosdb/-/blob/dev/HUMANS.md), and prospective ones.
-Please refer to the [official gitlab repository of the CaosDB
-project](https://gitlab.com/caosdb/caosdb) for more information.
+### Code of Conduct
-# License
+By participating, you are expected to uphold our [Code of Conduct](https://gitlab.com/caosdb/caosdb/-/blob/dev/CODE_OF_CONDUCT.md).
-Copyright (C) 2018 Research Group Biomedical Physics, Max Planck Institute for
-Dynamics and Self-Organization Göttingen.
+### How to Contribute
+
+* You found a bug, have a question, or want to request a feature? Please
+[create an issue](https://gitlab.com/caosdb/caosdb-advanced-user-tools/-/issues).
+* You want to contribute code? Please fork the repository and create a merge
+request in GitLab and choose this repository as target. Make sure to select
+"Allow commits from members who can merge the target branch" under Contribution
+when creating the merge request. This allows our team to work with you on your request.
+- If you have a suggestion for the [documentation](https://docs.indiscale.com/caosdb-advanced-user-tools/),
+the preferred way is also a merge request as describe above (the documentation resides in `src/doc`).
+However, you can also create an issue for it.
+- You can also contact us at **info (AT) caosdb.de**.
+
+## License
+
+* Copyright (C) 2018 Research Group Biomedical Physics, Max Planck Institute
+ for Dynamics and Self-Organization Göttingen.
+* Copyright (C) 2020-2021 Indiscale GmbH <info@indiscale.com>
All files in this repository are licensed under a [GNU Affero General Public
License](LICENCE.md) (version 3 or later).
-
diff --git a/README_SETUP.md b/README_SETUP.md
index 243fba2dd1259aaefbe6c7163a242b700eb5a66e..9b7b27ec056583708a8773ebac49f37ff45d9fd4 100644
--- a/README_SETUP.md
+++ b/README_SETUP.md
@@ -14,7 +14,7 @@ Dependencies will be installed automatically if you use the below described proc
- `xlrd>=1.2.0`
For testing:
-- `tox`
+- `tox`
## Installation
@@ -33,7 +33,7 @@ For testing:
the extroot of the empty profile to be used is located at).
3. Start an empty (!) CaosDB instance (with the mounted extroot). The
database will be cleared during testing, so it's important to use
- an empty insctance.
+ an empty instance.
4. Run `test.sh`.
## Code Formatting
diff --git a/integrationtests/test_crawler_basics.py b/integrationtests/test_crawler_basics.py
index 85fca282c8546ad1e7f6a708a2eaf46e374a528f..7da90844f14cf0d1eaded9d4fc8f37320da46aad 100644
--- a/integrationtests/test_crawler_basics.py
+++ b/integrationtests/test_crawler_basics.py
@@ -65,6 +65,7 @@ class CrawlerTest(unittest.TestCase):
self.rec2.add_parent(name="Test_Type_2")
self.rec3 = db.Record()
self.rec3.add_parent(name="Test_Type_3")
+ self.rec3.add_property(name="Test_Prop", value="Test")
def test_check_existence(self):
# This hasn't been inserted yet:
@@ -92,6 +93,7 @@ class CrawlerTest(unittest.TestCase):
old_id = id(identifiables[0])
reference_to_first = identifiables[0]
assert reference_to_first is identifiables[0]
+
Crawler.find_or_insert_identifiables(identifiables)
for el in identifiables:
@@ -107,6 +109,7 @@ class CrawlerTest(unittest.TestCase):
def tearDown(self):
setup_module()
+
# Delete nameless entities
for el in [self.rec1, self.rec2, self.rec3]:
try:
diff --git a/integrationtests/test_im_und_export.py b/integrationtests/test_im_und_export.py
index 5c7584e6f98ee792789f144d89f13ef84a7467fc..db26249b14d3d547db8dcea4e49de2aa07479e5b 100644
--- a/integrationtests/test_im_und_export.py
+++ b/integrationtests/test_im_und_export.py
@@ -13,13 +13,8 @@ if __name__ == "__main__":
directory = TemporaryDirectory()
export(rec.id, directory=directory.name)
# delete everything
- rec = db.execute_query("FIND record which was inserted by me")
- prop = db.execute_query("FIND property which was inserted by me")
- rt = db.execute_query("FIND recordtype which was inserted by me")
- fi = db.execute_query("FIND file which was inserted by me")
- c = db.Container()
- c.extend(rec+prop+rt+fi)
- c.delete()
+ recs = db.execute_query("FIND entity with id>99")
+ recs.delete()
assert 0 == len(db.execute_query("FIND File which is stored at "
"**/poster.pdf"))
import_xml(os.path.join(directory.name, "caosdb_data.xml"), interactive=False)
diff --git a/src/caosadvancedtools/crawler.py b/src/caosadvancedtools/crawler.py
index a01b13ca5b5ef61163e77063e68a3ca938bfbf84..c26c681e8e0231c63ac105af0642a387b2e3e0c5 100644
--- a/src/caosadvancedtools/crawler.py
+++ b/src/caosadvancedtools/crawler.py
@@ -629,6 +629,10 @@ carefully and if the changes are ok, click on the following link:
raise ValueError("The identifiable must have at least one parent.")
query_string = "FIND Record " + ident.get_parents()[0].name
query_string += " WITH "
+ if ident.name is None and len(ident.get_properties()) == 0:
+ raise ValueError(
+ "The identifiable must have features to identify it.")
+
if ident.name is not None:
query_string += "name='{}' AND".format(ident.name)
diff --git a/src/caosadvancedtools/export_related.py b/src/caosadvancedtools/export_related.py
index 47fe2f4900add818e940fa81466bb9c98a2f0223..00f440d28a2ae1da14132083e4b8d3c5003d1b65 100755
--- a/src/caosadvancedtools/export_related.py
+++ b/src/caosadvancedtools/export_related.py
@@ -47,6 +47,9 @@ def get_ids_of_related_entities(entity):
"""
entities = []
+ if isinstance(entity, int):
+ entity = db.Entity(id=entity).retrieve()
+
for par in entity.parents:
entities.append(par.id)
@@ -76,20 +79,17 @@ def recursively_collect_related(entity):
"""
all_entities = db.Container()
all_entities.append(entity)
- ids = set([entity.id])
- new_entities = [entity]
+ ids = set()
+ new_ids = set([entity.id])
- while new_entities:
- new_ids = set()
+ while new_ids:
+ ids.update(new_ids)
- for ent in new_entities:
- new_ids.update(get_ids_of_related_entities(ent))
+ for eid in list(new_ids):
+ new_ids.update(get_ids_of_related_entities(eid))
new_ids = new_ids - ids
- new_entities = retrieve_entities_with_ids(list(new_ids))
- ids.update([e.id for e in new_entities])
- all_entities.extend(new_entities)
- return all_entities
+ return retrieve_entities_with_ids(list(ids))
def invert_ids(entities):
diff --git a/src/caosadvancedtools/import_from_xml.py b/src/caosadvancedtools/import_from_xml.py
index 0bf9b1c0cbb478bb75687f9f3e41ca2d4960d2c0..9d0e03f649db771147915740cabf201fae910760 100755
--- a/src/caosadvancedtools/import_from_xml.py
+++ b/src/caosadvancedtools/import_from_xml.py
@@ -57,7 +57,7 @@ def import_xml(filename, rerun=False, interactive=True):
tmpfile = create_dummy_file()
model = []
- files = []
+ files = {}
# add files to files list and properties and record types to model
@@ -70,19 +70,19 @@ def import_xml(filename, rerun=False, interactive=True):
el.file = target
else:
el.file = tmpfile
- files.append(el)
+ files[el.path] = el
if (isinstance(el, db.Property) or isinstance(el, db.RecordType)):
model.append(el)
# remove entities of the model from the container
- for el in model+files:
+ for el in model+list(files.values()):
cont.remove(el)
id_mapping = {}
- for el in model+files:
+ for el in model+list(files.values()):
id_mapping[el.id] = el
# insert/update the model
@@ -93,10 +93,10 @@ def import_xml(filename, rerun=False, interactive=True):
# insert files
if not rerun:
- for _, el in enumerate(files):
+ for _, el in enumerate(files.values()):
r = el.insert(unique=False)
else:
- for _, el in enumerate(files):
+ for _, el in enumerate(files.values()):
el.id = None
el.retrieve()
diff --git a/unittests/test_crawler.py b/unittests/test_crawler.py
index f603031eddbcf1e10c2842ec4e89ca591700b94f..64bf291c1181d901ac39a4d2535dcd6eddf39f70 100644
--- a/unittests/test_crawler.py
+++ b/unittests/test_crawler.py
@@ -45,3 +45,7 @@ class CrawlerTest(unittest.TestCase):
datatype=db.LIST("RT2"))
qs = Crawler.create_query_for_identifiable(ident)
assert qs == "FIND Record RT WITH references 2345 AND references 234567 "
+ ident = db.Record()
+ ident.add_parent(name="RT")
+ self.assertRaises(ValueError, Crawler.create_query_for_identifiable,
+ ident)