diff --git a/.docker/Dockerfile b/.docker/Dockerfile
new file mode 100644
index 0000000000000000000000000000000000000000..1ba73a5b3ea0e4969e882ab11990303aa432f3eb
--- /dev/null
+++ b/.docker/Dockerfile
@@ -0,0 +1,37 @@
+FROM debian:latest
+RUN apt-get update && \
+ apt-get install \
+ curl \
+ git \
+ openjdk-11-jdk-headless \
+ python3-autopep8 \
+ python3-pip \
+ python3-pytest \
+ tox \
+ -y
+COPY .docker/wait-for-it.sh /wait-for-it.sh
+ADD https://gitlab.indiscale.com/api/v4/projects/97/repository/commits/${PYLIB} \
+ pylib_version.json
+RUN git clone https://gitlab.indiscale.com/caosdb/src/caosdb-pylib.git && \
+ cd caosdb-pylib && git checkout ${PYLIB} && pip3 install .
+ADD https://gitlab.indiscale.com/api/v4/projects/104/repository/commits/${ADVANCED} \
+ advanced_version.json
+RUN git clone https://gitlab.indiscale.com/caosdb/src/caosdb-advanced-user-tools.git && \
+ cd caosdb-advanced-user-tools && git checkout ${ADVANCED} && pip3 install .
+COPY . /git
+
+# Delete .git because it is huge.
+RUN rm -r /git/.git
+
+# Install pycaosdb.ini for the tests
+RUN mv /git/.docker/tester_pycaosdb.ini /git/pycaosdb.ini
+
+RUN cd /git/ && pip3 install .
+
+WORKDIR /git/integrationtests
+# wait for server,
+CMD /wait-for-it.sh caosdb-server:10443 -t 500 -- \
+ # ... install pycaosdb.ini the server-side scripts
+ cp /git/.docker/sss_pycaosdb.ini /scripting/home/.pycaosdb.ini && \
+ # ... and run tests
+ pytest .
diff --git a/.docker/cert.sh b/.docker/cert.sh
new file mode 100755
index 0000000000000000000000000000000000000000..e22cfba2995b5fd9d812232f562b7254233fe5b0
--- /dev/null
+++ b/.docker/cert.sh
@@ -0,0 +1,58 @@
+#!/bin/bash
+
+# ** header v3.0
+# This file is a part of the CaosDB Project.
+#
+# Copyright (C) 2019 Daniel Hornung, Göttingen
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see <https://www.gnu.org/licenses/>.
+#
+# ** end header
+
+
+# Creates a directory `cert` and certificates in this directory.
+#
+# The hostname for which the certificate is created can be changed by setting
+# the environment variable CAOSHOSTNAME.
+#
+# ## Overview of variables ##
+#
+# - CAOSHOSTNAME :: Hostname for the key (localhost)
+# - KEYPW :: Password for the key (default ist CaosDBSecret)
+# - KEYSTOREPW :: Password for the key store (same as KEYPW)
+function cert() {
+ mkdir -p cert
+ cd cert
+ KEYPW="${KEYPW:-CaosDBSecret}"
+ CAOSHOSTNAME="${CAOSHOSTNAME:-localhost}"
+ KEYSTOREPW="${KEYPW:-}"
+ # NOTE: KEYPW and KEYSTOREPW are the same, due to Java limitations.
+ KEYPW="${KEYPW}" openssl genrsa -aes256 -out caosdb.key.pem \
+ -passout env:KEYPW 2048
+ # Certificate is for localhost
+ KEYPW="${KEYPW}" openssl req -new -x509 -key caosdb.key.pem \
+ -out caosdb.cert.pem -passin env:KEYPW \
+ -subj "/C=/ST=/L=/O=/OU=/CN=${CAOSHOSTNAME}"
+ KEYPW="${KEYPW}" KEYSTOREPW="$KEYSTOREPW" openssl pkcs12 -export \
+ -inkey caosdb.key.pem -in caosdb.cert.pem -out all-certs.pkcs12 \
+ -passin env:KEYPW -passout env:KEYPW
+
+ keytool -importkeystore -srckeystore all-certs.pkcs12 -srcstoretype PKCS12 \
+ -deststoretype pkcs12 -destkeystore caosdb.jks \
+ -srcstorepass "${KEYPW}" \
+ -destkeypass "${KEYPW}" -deststorepass "$KEYSTOREPW"
+ echo "Certificates successfuly created."
+}
+
+cert
diff --git a/.docker/docker-compose.yml b/.docker/docker-compose.yml
new file mode 100644
index 0000000000000000000000000000000000000000..bbee24fbd8c898c479a0fafa13000ddf506d00eb
--- /dev/null
+++ b/.docker/docker-compose.yml
@@ -0,0 +1,43 @@
+version: '3.7'
+services:
+ sqldb:
+ image: mariadb:10.4
+ environment:
+ MYSQL_ROOT_PASSWORD: caosdb1234
+ networks:
+ - caosnet
+ caosdb-server:
+ image: "$CI_REGISTRY/caosdb/src/caosdb-deploy:$CAOSDB_TAG"
+ user: 999:999
+ depends_on:
+ - sqldb
+ networks:
+ - caosnet
+ volumes:
+ - type: bind
+ source: ./cert
+ target: /opt/caosdb/cert
+ - type: volume
+ source: extroot
+ target: /opt/caosdb/mnt/extroot
+ - type: volume
+ source: scripting
+ target: /opt/caosdb/git/caosdb-server/scripting
+ - type: volume
+ source: authtoken
+ target: /opt/caosdb/git/caosdb-server/authtoken
+ ports:
+ # - "from_outside:from_inside"
+ - "10443:10443"
+ - "10080:10080"
+ environment:
+ DEBUG: 1
+ CAOSDB_CONFIG_AUTHTOKEN_CONFIG: "conf/core/authtoken.example.yaml"
+ CAOSDB_CONFIG_TRANSACTION_BENCHMARK_ENABLED: "TRUE"
+volumes:
+ scripting:
+ extroot:
+ authtoken:
+networks:
+ caosnet:
+ driver: bridge
diff --git a/.docker/run.sh b/.docker/run.sh
new file mode 100755
index 0000000000000000000000000000000000000000..b0e1a716f28516b83043fb3fdb6594515a0bafd4
--- /dev/null
+++ b/.docker/run.sh
@@ -0,0 +1,5 @@
+#!/bin/sh
+
+docker-compose -f tester.yml run tester
+rv=$?
+echo $rv > result
diff --git a/.docker/sss_pycaosdb.ini b/.docker/sss_pycaosdb.ini
new file mode 100644
index 0000000000000000000000000000000000000000..de2867f8dc66b3e81f10f35e40c36f9cb8591604
--- /dev/null
+++ b/.docker/sss_pycaosdb.ini
@@ -0,0 +1,9 @@
+; this is the pycaosdb.ini for the server-side-scripting home.
+[Connection]
+url = https://caosdb-server:10443
+cacert = /opt/caosdb/cert/caosdb.cert.pem
+debug = 0
+timeout = 5000
+
+[Misc]
+sendmail = /usr/local/bin/sendmail_to_file
diff --git a/.docker/tester.yml b/.docker/tester.yml
new file mode 100644
index 0000000000000000000000000000000000000000..83db879c6072bfdea7b3212c833116b96bb54d0c
--- /dev/null
+++ b/.docker/tester.yml
@@ -0,0 +1,26 @@
+version: '3.7'
+services:
+ tester:
+ image: "$CI_REGISTRY_IMAGE"
+ networks:
+ - docker_caosnet
+ volumes:
+ - type: bind
+ source: ./cert
+ target: /cert
+ - type: volume
+ source: extroot
+ target: /extroot
+ - type: volume
+ source: scripting
+ target: /scripting
+ - type: volume
+ source: authtoken
+ target: /authtoken
+networks:
+ docker_caosnet:
+ external: true
+volumes:
+ scripting:
+ extroot:
+ authtoken:
diff --git a/.docker/tester_pycaosdb.ini b/.docker/tester_pycaosdb.ini
new file mode 100644
index 0000000000000000000000000000000000000000..2159dec250b3dcb2f16043d12bdbe73675e4d75c
--- /dev/null
+++ b/.docker/tester_pycaosdb.ini
@@ -0,0 +1,31 @@
+; pycaosdb.ini for pytest test suites.
+
+[IntegrationTests]
+; location of the scripting bin dir which is used for the test scripts from the
+; server's perspective.
+test_server_side_scripting.bin_dir.server = scripting/bin-debug/
+; location of the scripting bin dir which is used for the test scripts from the
+; pyinttest's perspective.
+test_server_side_scripting.bin_dir.local = /scripting/bin-debug/
+
+; location of the files from the pyinttest perspective
+test_files.test_insert_files_in_dir.local = /extroot/test_insert_files_in_dir/
+; location of the files from the caosdb_servers perspective
+test_files.test_insert_files_in_dir.server = /opt/caosdb/mnt/extroot/test_insert_files_in_dir/
+
+; location of the one-time tokens from the pyinttest's perspective
+test_authentication.admin_token_crud = /authtoken/admin_token_crud.txt
+test_authentication.admin_token_expired = /authtoken/admin_token_expired.txt
+test_authentication.admin_token_3_attempts = /authtoken/admin_token_3_attempts.txt
+
+
+[Connection]
+url = https://caosdb-server:10443/
+username = admin
+cacert = /cert/caosdb.cert.pem
+debug = 0
+
+password_method = plain
+password = caosdb
+
+timeout = 500
diff --git a/.docker/wait-for-it.sh b/.docker/wait-for-it.sh
new file mode 100755
index 0000000000000000000000000000000000000000..d69e99f1f13257b559dce2433de0515379663efa
--- /dev/null
+++ b/.docker/wait-for-it.sh
@@ -0,0 +1,182 @@
+#!/usr/bin/env bash
+# License:
+# From https://github.com/vishnubob/wait-for-it
+# The MIT License (MIT)
+# Use this script to test if a given TCP host/port are available
+
+WAITFORIT_cmdname=${0##*/}
+
+echoerr() { if [[ $WAITFORIT_QUIET -ne 1 ]]; then echo "$@" 1>&2; fi }
+
+usage()
+{
+ cat << USAGE >&2
+Usage:
+ $WAITFORIT_cmdname host:port [-s] [-t timeout] [-- command args]
+ -h HOST | --host=HOST Host or IP under test
+ -p PORT | --port=PORT TCP port under test
+ Alternatively, you specify the host and port as host:port
+ -s | --strict Only execute subcommand if the test succeeds
+ -q | --quiet Don't output any status messages
+ -t TIMEOUT | --timeout=TIMEOUT
+ Timeout in seconds, zero for no timeout
+ -- COMMAND ARGS Execute command with args after the test finishes
+USAGE
+ exit 1
+}
+
+wait_for()
+{
+ if [[ $WAITFORIT_TIMEOUT -gt 0 ]]; then
+ echoerr "$WAITFORIT_cmdname: waiting $WAITFORIT_TIMEOUT seconds for $WAITFORIT_HOST:$WAITFORIT_PORT"
+ else
+ echoerr "$WAITFORIT_cmdname: waiting for $WAITFORIT_HOST:$WAITFORIT_PORT without a timeout"
+ fi
+ WAITFORIT_start_ts=$(date +%s)
+ while :
+ do
+ if [[ $WAITFORIT_ISBUSY -eq 1 ]]; then
+ nc -z $WAITFORIT_HOST $WAITFORIT_PORT
+ WAITFORIT_result=$?
+ else
+ (echo > /dev/tcp/$WAITFORIT_HOST/$WAITFORIT_PORT) >/dev/null 2>&1
+ WAITFORIT_result=$?
+ fi
+ if [[ $WAITFORIT_result -eq 0 ]]; then
+ WAITFORIT_end_ts=$(date +%s)
+ echoerr "$WAITFORIT_cmdname: $WAITFORIT_HOST:$WAITFORIT_PORT is available after $((WAITFORIT_end_ts - WAITFORIT_start_ts)) seconds"
+ break
+ fi
+ sleep 1
+ done
+ return $WAITFORIT_result
+}
+
+wait_for_wrapper()
+{
+ # In order to support SIGINT during timeout: http://unix.stackexchange.com/a/57692
+ if [[ $WAITFORIT_QUIET -eq 1 ]]; then
+ timeout $WAITFORIT_BUSYTIMEFLAG $WAITFORIT_TIMEOUT $0 --quiet --child --host=$WAITFORIT_HOST --port=$WAITFORIT_PORT --timeout=$WAITFORIT_TIMEOUT &
+ else
+ timeout $WAITFORIT_BUSYTIMEFLAG $WAITFORIT_TIMEOUT $0 --child --host=$WAITFORIT_HOST --port=$WAITFORIT_PORT --timeout=$WAITFORIT_TIMEOUT &
+ fi
+ WAITFORIT_PID=$!
+ trap "kill -INT -$WAITFORIT_PID" INT
+ wait $WAITFORIT_PID
+ WAITFORIT_RESULT=$?
+ if [[ $WAITFORIT_RESULT -ne 0 ]]; then
+ echoerr "$WAITFORIT_cmdname: timeout occurred after waiting $WAITFORIT_TIMEOUT seconds for $WAITFORIT_HOST:$WAITFORIT_PORT"
+ fi
+ return $WAITFORIT_RESULT
+}
+
+# process arguments
+while [[ $# -gt 0 ]]
+do
+ case "$1" in
+ *:* )
+ WAITFORIT_hostport=(${1//:/ })
+ WAITFORIT_HOST=${WAITFORIT_hostport[0]}
+ WAITFORIT_PORT=${WAITFORIT_hostport[1]}
+ shift 1
+ ;;
+ --child)
+ WAITFORIT_CHILD=1
+ shift 1
+ ;;
+ -q | --quiet)
+ WAITFORIT_QUIET=1
+ shift 1
+ ;;
+ -s | --strict)
+ WAITFORIT_STRICT=1
+ shift 1
+ ;;
+ -h)
+ WAITFORIT_HOST="$2"
+ if [[ $WAITFORIT_HOST == "" ]]; then break; fi
+ shift 2
+ ;;
+ --host=*)
+ WAITFORIT_HOST="${1#*=}"
+ shift 1
+ ;;
+ -p)
+ WAITFORIT_PORT="$2"
+ if [[ $WAITFORIT_PORT == "" ]]; then break; fi
+ shift 2
+ ;;
+ --port=*)
+ WAITFORIT_PORT="${1#*=}"
+ shift 1
+ ;;
+ -t)
+ WAITFORIT_TIMEOUT="$2"
+ if [[ $WAITFORIT_TIMEOUT == "" ]]; then break; fi
+ shift 2
+ ;;
+ --timeout=*)
+ WAITFORIT_TIMEOUT="${1#*=}"
+ shift 1
+ ;;
+ --)
+ shift
+ WAITFORIT_CLI=("$@")
+ break
+ ;;
+ --help)
+ usage
+ ;;
+ *)
+ echoerr "Unknown argument: $1"
+ usage
+ ;;
+ esac
+done
+
+if [[ "$WAITFORIT_HOST" == "" || "$WAITFORIT_PORT" == "" ]]; then
+ echoerr "Error: you need to provide a host and port to test."
+ usage
+fi
+
+WAITFORIT_TIMEOUT=${WAITFORIT_TIMEOUT:-15}
+WAITFORIT_STRICT=${WAITFORIT_STRICT:-0}
+WAITFORIT_CHILD=${WAITFORIT_CHILD:-0}
+WAITFORIT_QUIET=${WAITFORIT_QUIET:-0}
+
+# check to see if timeout is from busybox?
+WAITFORIT_TIMEOUT_PATH=$(type -p timeout)
+WAITFORIT_TIMEOUT_PATH=$(realpath $WAITFORIT_TIMEOUT_PATH 2>/dev/null || readlink -f $WAITFORIT_TIMEOUT_PATH)
+if [[ $WAITFORIT_TIMEOUT_PATH =~ "busybox" ]]; then
+ WAITFORIT_ISBUSY=1
+ WAITFORIT_BUSYTIMEFLAG="-t"
+
+else
+ WAITFORIT_ISBUSY=0
+ WAITFORIT_BUSYTIMEFLAG=""
+fi
+
+if [[ $WAITFORIT_CHILD -gt 0 ]]; then
+ wait_for
+ WAITFORIT_RESULT=$?
+ exit $WAITFORIT_RESULT
+else
+ if [[ $WAITFORIT_TIMEOUT -gt 0 ]]; then
+ wait_for_wrapper
+ WAITFORIT_RESULT=$?
+ else
+ wait_for
+ WAITFORIT_RESULT=$?
+ fi
+fi
+
+if [[ $WAITFORIT_CLI != "" ]]; then
+ if [[ $WAITFORIT_RESULT -ne 0 && $WAITFORIT_STRICT -eq 1 ]]; then
+ echoerr "$WAITFORIT_cmdname: strict mode, refusing to execute subprocess"
+ exit $WAITFORIT_RESULT
+ fi
+ exec "${WAITFORIT_CLI[@]}"
+else
+ exit $WAITFORIT_RESULT
+fi
+
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
new file mode 100644
index 0000000000000000000000000000000000000000..861e25e1a2c1603f1cf9e733f21ad531f7614676
--- /dev/null
+++ b/.gitlab-ci.yml
@@ -0,0 +1,248 @@
+#
+# This file is a part of the CaosDB Project.
+#
+# Copyright (C) 2018 Research Group Biomedical Physics,
+# Max-Planck-Institute for Dynamics and Self-Organization Göttingen
+# Copyright (C) 2019 Henrik tom Wörden
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+variables:
+ CI_REGISTRY_IMAGE: $CI_REGISTRY/caosdb/src/crawler2.0/testenv:$CI_COMMIT_REF_NAME
+ CI_REGISTRY_IMAGE_BASE: $CI_REGISTRY/caosdb/src/caosdb-pyinttest/base:latest
+
+stages:
+ - info
+ - setup
+ - cert
+ - style
+ - test
+ - deploy
+
+
+# During the test stage the CI pipeline (which runs in a "root" docker) starts
+# two docker containers with docker-compose (one for the caosdb-server, another
+# for the mysql-backend). Then a third docker is being started which contains
+# the test suite and executes it.
+#
+# +-------------(root docker)-------------------------+
+# | |
+# | +-(caosdb_mysqlbackend)-------------+ |
+# | | | |
+# | +-----------------------------------+ |
+# | +-(caosdb-server)-------------------+ |
+# | | | |
+# | | /opt/caosdb | |
+# | .------->| + /git/caosdb-server/scripting/ | |
+# | | .----->| + /git/caosdb-server/authtoken/ | |
+# | | | .--->| + /mnt/extroot | |
+# | | | | .->| + /cert | |
+# | | | | | | | |
+# | | | | | +-----------------------------------+ |
+# | | | | | |
+# | | | | | filesystem: |
+# | | | | *--- /cert -----------. |
+# | | | | | |
+# | | | | volumes: | |
+# | | | *----- extroot ------. | |
+# | | *------- scripting --. | | |
+# | *--------- authtoken -. | | | |
+# | | | | | |
+# | +-(crawler tests)---+ | | | | |
+# | | | | | | | |
+# | | /authtoken |<---* | | | |
+# | | /scripting |<----* | | |
+# | | /extroot |<------* | |
+# | | /cert |<--------* |
+# | | | |
+# | +----------------------+ |
+# +---------------------------------------------------+
+#
+# In the root docker, the directory /cert is mounted to .docker/cert relative
+# to this repository. The directory is created during the cert stage of this
+# pipeline and a certificate is created in there. The certificat is then
+# available in mounted directories in the server and crawler containers.
+#
+# Additional volumes in the root docker are shared by the caosdb-server and the crawler
+# containers. These volumes are intended to be used for testing server-side scripting and
+# file-system features.
+#
+
+.env: &env
+ - echo "Pipeline triggered by $TRIGGERED_BY_REPO@$TRIGGERED_BY_REF ($TRIGGERED_BY_HASH)"
+ - echo "CI_REGISTRY_IMAGE_BASE = $CI_REGISTRY_IMAGE_BASE"
+ - echo "CI_REGISTRY_IMAGE = $CI_REGISTRY_IMAGE"
+ - echo "CAOSDB_TAG = $CAOSDB_TAG"
+ - echo "REFTAG = $REFTAG"
+ - echo "F_BRANCH = $F_BRANCH"
+ - echo "CI_COMMIT_REF_NAME = $CI_COMMIT_REF_NAME"
+ - ls -lah /image-cache/
+
+ - F_BRANCH=${F_BRANCH:-$CI_COMMIT_REF_NAME}
+ - echo $F_BRANCH
+ - if [[ "$REFTAG" == "" ]] ; then
+ if [[ "$F_BRANCH" == "dev" ]] ; then
+ REFTAG=dev;
+ fi;
+ fi
+ - REFTAG=${REFTAG:-dev_F_${F_BRANCH}}
+
+ - echo $F_BRANCH
+
+ - if [[ "$CAOSDB_TAG" == "" ]]; then
+ CAOSDB_TAG=${REFTAG};
+ fi
+ - echo $CAOSDB_TAG
+
+info:
+ tags: [cached-dind]
+ image: docker:20.10
+ stage: info
+ needs: []
+ script:
+ - *env
+
+unittest:
+ tags: [cached-dind]
+ image: docker:20.10
+ stage: test
+ image: $CI_REGISTRY_IMAGE
+ script:
+ - tox
+
+inttest:
+ tags: [docker]
+ services:
+ - docker:20.10-dind
+ variables:
+ # This is a workaround for the gitlab-runner health check mechanism when
+ # using docker-dind service. The runner will otherwise guess the port
+ # wrong and the health check will timeout.
+ SERVICE_PORT_2376_TCP_PORT: 2375
+ stage: test
+ image: $CI_REGISTRY_IMAGE_BASE
+ needs: [cert]
+ script:
+ - *env
+ - docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY
+ - echo $CAOSDB_TAG
+
+ - cd .docker
+ # Store mariadb version
+ - MARIADBVERSION=$(grep mariadb docker-compose.yml | awk '{print $2}')
+ - echo "mariadb image:"$MARIADBVERSION
+ - time docker load < /image-cache/caosdb-crawler-testenv-${CI_COMMIT_REF_NAME}.tar || true
+ - time docker load < /image-cache/caosdb-${REFTAG}.tar || time docker load < /image-cache/caosdb-dev.tar || true
+ - time docker load < /image-cache/$MARIADBVERSION.tar || true
+ - docker pull $CI_REGISTRY/caosdb/src/caosdb-deploy:$CAOSDB_TAG || CAOSDB_TAG=dev
+ - docker pull $CI_REGISTRY_IMAGE
+
+ # Here, the server and the mysql backend docker are being started
+ - CAOSDB_TAG=$CAOSDB_TAG docker-compose up -d
+
+ # Store versions of CaosDB parts
+ - docker exec -u 0 -t docker_caosdb-server_1 cat /opt/caosdb/git/caosdb_pylib_commit > hash_pylib
+ - docker exec -u 0 -t docker_caosdb-server_1 cat /opt/caosdb/git/caosdb_webui_commit > hash_webui
+ - docker exec -u 0 -t docker_caosdb-server_1 cat /opt/caosdb/git/caosdb_server_commit > hash_server
+ - docker exec -u 0 -t docker_caosdb-server_1 cat /opt/caosdb/git/caosdb_mysqlbackend_commit > hash_mysql
+ - docker exec -u 0 -t docker_caosdb-server_1 cat /opt/caosdb/git/caosdb_proto_commit > hash_proto
+ - cat hash_server
+ - cat hash_proto
+ - cat hash_mysql
+ - cat hash_webui
+ - cat hash_pylib
+ # Run the actual tests. This starts a new docker container within which
+ # the tests run. The return value is stored in .docker/result
+ - /bin/sh ./run.sh
+
+ # Save logs
+ - docker logs docker_caosdb-server_1 &> ../caosdb_log.txt
+ - docker logs docker_sqldb_1 &> ../mariadb_log.txt
+ - cd ..
+
+ # Stop the server
+ - docker-compose -f .docker/docker-compose.yml down
+
+ # the crawler docker writes the return value of the tests into the
+ # file result
+ - rc=`cat .docker/result`
+ - exit $rc
+ dependencies: [cert]
+ timeout: 3h
+ artifacts:
+ paths:
+ - caosdb_log.txt
+ - mariadb_log.txt
+ - .docker/hash_*
+ expire_in: 1 week
+
+build-testenv:
+ tags: [cached-dind]
+ image: docker:20.10
+ stage: setup
+ timeout: 2h
+ only:
+ - schedules
+ - web
+ - pushes
+ needs: []
+ script:
+ - df -h
+ - command -v wget
+ - if [ -z "$PYLIB" ]; then
+ if echo "$CI_COMMIT_REF_NAME" | grep -c "^f-" ; then
+ echo "Check if pylib has branch $CI_COMMIT_REF_NAME" ;
+ if wget https://gitlab.indiscale.com/api/v4/projects/97/repository/branches/${CI_COMMIT_REF_NAME} ; then
+ PYLIB=$CI_COMMIT_REF_NAME ;
+ fi;
+ fi;
+ fi;
+ - PYLIB=${PYLIB:-dev}
+ - echo $PYLIB
+
+ - docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY
+ # use here general latest or specific branch latest...
+ - docker build
+ --build-arg PYLIB=${PYLIB}
+ --build-arg ADVANCED=${ADVANCED:dev}
+ --file .docker/Dockerfile
+ -t $CI_REGISTRY_IMAGE .
+ - docker push $CI_REGISTRY_IMAGE
+ - docker save $CI_REGISTRY_IMAGE > /image-cache/caosdb-crawler-testenv-${CI_COMMIT_REF_NAME}.tar
+
+cert:
+ tags: [docker]
+ stage: cert
+ image: $CI_REGISTRY_IMAGE
+ needs:
+ - job: build-testenv
+ optional: true
+ artifacts:
+ paths:
+ - .docker/cert/
+ expire_in: 1 week
+ script:
+ - cd .docker
+ - CAOSHOSTNAME=caosdb-server ./cert.sh
+
+style:
+ tags: [docker]
+ stage: style
+ image: $CI_REGISTRY_IMAGE
+ needs:
+ - job: build-testenv
+ optional: true
+ script:
+ - autopep8 -r --diff --exit-code .
+ allow_failure: true
diff --git a/integrationtests/test.py b/integrationtests/test.py
index efff64305bbc9dd24ebf7817fb9d10d0523c9f5b..2d8818005ee4350bd3b1403ae71c315c6d5c1a14 100755
--- a/integrationtests/test.py
+++ b/integrationtests/test.py
@@ -28,6 +28,7 @@
module description
"""
+import os
from caosdb import EmptyUniqueQueryError
import argparse
import sys
@@ -42,7 +43,6 @@ import yaml
from caosadvancedtools.testutils import clear_database, set_test_key
set_test_key("10b128cf8a1372f30aa3697466bb55e76974e0c16a599bb44ace88f19c8f61e2")
-import os
def rfp(*pathcomponents):
"""
@@ -52,14 +52,12 @@ def rfp(*pathcomponents):
return os.path.join(os.path.dirname(__file__), *pathcomponents)
-
-
@pytest.fixture
def usemodel():
model = parse_model_from_yaml(rfp("model.yml"))
model.sync_data_model(noquestion=True, verbose=False)
-
+
@pytest.fixture
def ident():
ident = CaosDBIdentifiableAdapter()
@@ -68,12 +66,12 @@ def ident():
ident.register_identifiable(
"Person", db.RecordType()
.add_parent(name="Person")
- #.add_property(name="first_name")
+ # .add_property(name="first_name")
.add_property(name="last_name"))
ident.register_identifiable(
"Measurement", db.RecordType()
.add_parent(name="Measurement")
- #.add_property(name="identifier")
+ # .add_property(name="identifier")
.add_property(name="date")
.add_property(name="project"))
ident.register_identifiable(
@@ -150,6 +148,7 @@ def test_multiple_insertions(clear_database, usemodel, ident, crawler):
assert len(ins) == 0
assert len(ups) == 0
+
def test_insertion(clear_database, usemodel, ident, crawler):
ins, ups = crawler.synchronize()
@@ -169,6 +168,7 @@ def test_insertion(clear_database, usemodel, ident, crawler):
assert len(ins) == 0
assert len(ups) == 0
+
def test_insertion_and_update(clear_database, usemodel, ident, crawler):
ins, ups = crawler.synchronize()
@@ -184,7 +184,8 @@ def test_insertion_and_update(clear_database, usemodel, ident, crawler):
ins, ups = cr.synchronize()
assert len(ins) == 0
assert len(ups) == 1
-
+
+
def test_identifiable_update(clear_database, usemodel, ident, crawler):
ins, ups = crawler.synchronize()
@@ -197,7 +198,7 @@ def test_identifiable_update(clear_database, usemodel, ident, crawler):
l = cr.updateList
for record in l:
if (record.parents[0].name == "Measurement" and
- record.get_property("date").value == "2020-01-03"):
+ record.get_property("date").value == "2020-01-03"):
# maybe a bit weird, but add an email address to a measurement
record.add_property(name="email", value="testperson@testaccount.test")
print("one change")
@@ -206,14 +207,13 @@ def test_identifiable_update(clear_database, usemodel, ident, crawler):
assert len(ins) == 0
assert len(ups) == 1
-
# Test the change within one property:
cr = Crawler(debug=True, identifiableAdapter=ident)
crawl_standard_test_directory(cr)
l = cr.updateList
for record in l:
if (record.parents[0].name == "Measurement" and
- record.get_property("date").value == "2020-01-03"):
+ record.get_property("date").value == "2020-01-03"):
record.add_property(name="email", value="testperson@coolmail.test")
print("one change")
break
@@ -227,7 +227,7 @@ def test_identifiable_update(clear_database, usemodel, ident, crawler):
l = cr.updateList
for record in l:
if (record.parents[0].name == "Measurement" and
- record.get_property("date").value == "2020-01-03"):
+ record.get_property("date").value == "2020-01-03"):
record.add_property(name="email", value="testperson@coolmail.test")
record.get_property("date").value = "2012-01-02"
print("one change")
@@ -269,6 +269,7 @@ def test_file_insertion(clear_database, usemodel, ident, crawler_extended):
assert len(r) == 1
assert r[0].get_property("ReadmeFile").value == f.id
+
def test_file_update(clear_database, usemodel, ident, crawler_extended):
ins1, ups1 = crawler_extended.synchronize(commit_changes=True)
fileList_ins = [r for r in ins1 if r.role == "File"]
@@ -288,7 +289,7 @@ def test_file_update(clear_database, usemodel, ident, crawler_extended):
res = db.execute_query("Find File")
assert len(res) == 11
assert len(res[0].parents) == 0
-
+
cr2 = Crawler(debug=True, identifiableAdapter=ident)
crawl_standard_test_directory(cr2, cfood="scifolder_extended2.yml")
@@ -302,7 +303,6 @@ def test_file_update(clear_database, usemodel, ident, crawler_extended):
res = db.execute_query("Find File")
assert len(res) == 11
assert res[0].parents[0].name == "ProjectMarkdownReadme"
-
# TODO: Implement file update checks (based on checksum)
# Add test with actual file update:
diff --git a/setup.cfg b/setup.cfg
index b89b07543d91dd35e2238aaddd363e85dd45f2d2..0d5048c2c130ff9e33d0186f87ee3916af53ee64 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -16,7 +16,6 @@ classifiers =
[options]
package_dir =
= src
-
packages = find:
python_requires = >=3.6
install_requires =
@@ -24,6 +23,7 @@ install_requires =
caosdb
caosadvancedtools
yaml-header-tools
+ pyyaml
[options.packages.find]
where = src
diff --git a/src/doc/conf.py b/src/doc/conf.py
index 75731285a77f8a30fcb4bfc6be0483c4bba0052a..825f878b2d52947978e2163b5bcff327e5da009d 100644
--- a/src/doc/conf.py
+++ b/src/doc/conf.py
@@ -3,7 +3,7 @@
# Configuration file for the Sphinx documentation builder.
#
# Based on the configuration for caosdb-pylib.
-#
+#
# # Copyright (C) 2021 Alexander Schlemmer <alexander.schlemmer@ds.mpg.de>
#
# This file only contains a selection of the most common options. For a full
diff --git a/src/newcrawler/identified_cache.py b/src/newcrawler/identified_cache.py
index cba00dd2bfff8a0f886878f532133bb18b1a20de..0b9d7a47bdecc4094edb1296f4c04dfa083a2436 100644
--- a/src/newcrawler/identified_cache.py
+++ b/src/newcrawler/identified_cache.py
@@ -66,7 +66,6 @@ def _create_hashable_string(identifiable: db.Record):
else:
tmplist.append(val)
value = str(tmplist)
-
rec_string += "{}:".format(pname) + value
return rec_string
diff --git a/tox.ini b/tox.ini
index 8b5ad34fb1583790de5365f2bfa4ff7b3704574c..2cf966fb5b80e62cb7f216b0785ba567e13ee3ff 100644
--- a/tox.ini
+++ b/tox.ini
@@ -1,5 +1,5 @@
[tox]
-envlist=py36, py37, py38, py39
+envlist=py36, py37, py38, py39, py310
skip_missing_interpreters = true
[testenv]
diff --git a/unittests/test_converters.py b/unittests/test_converters.py
index fcc89f2e1467f23accfe47bd52a51d934ccecf91..166f7d516687b5e2d8e8e60321de5084b38e060a 100644
--- a/unittests/test_converters.py
+++ b/unittests/test_converters.py
@@ -177,7 +177,8 @@ def test_json_converter(converter_registry):
test_json = File("testjson.json", rfp(
"test_directories", "examples_json", "testjson.json"))
- schema_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), "test_directories", "examples_json", "testjson.schema.json")
+ schema_path = os.path.join(os.path.dirname(os.path.abspath(__file__)),
+ "test_directories", "examples_json", "testjson.schema.json")
jsonconverter = JSONFileConverter(
definition={"match": "(.*)", "validate": schema_path},
name="TestJSONFileConverter",
@@ -228,7 +229,7 @@ def test_json_converter(converter_registry):
broken_json = File("brokenjson.json", rfp(
"test_directories", "examples_json", "brokenjson.json"))
m = jsonconverter.match(broken_json)
-
+
# Doesn't validate because of missing required 'name' property
with pytest.raises(ConverterValidationError) as err:
children = jsonconverter.create_children(None, broken_json)
diff --git a/unittests/test_entity_comparison.py b/unittests/test_entity_comparison.py
index 1e22280d893ae7bba301baa61213b5e49eaaba6c..750828bbd00a7142040c95a532246adbc97ca630 100644
--- a/unittests/test_entity_comparison.py
+++ b/unittests/test_entity_comparison.py
@@ -13,13 +13,13 @@ from newcrawler.crawl import check_identical
def test_compare_entities():
record1 = db.Record()
record2 = db.Record()
-
+
assert check_identical(record1, record2)
record1.add_property(name="type", value="int")
assert not check_identical(record1, record2)
assert not check_identical(record2, record1)
-
+
record2.add_property(name="type", value="int")
assert check_identical(record1, record2)
record2.get_property("type").value = "int2"
@@ -79,7 +79,8 @@ def test_compare_entities():
for attribute, values in zip(("_checksum", "_size"),
(vals[0], vals[1])):
setattr(record1, attribute, values[0])
- assert not check_identical(record1, record2) # not identical, because record1 sets the datatype
+ # not identical, because record1 sets the datatype
+ assert not check_identical(record1, record2)
assert check_identical(record2, record1) # identical, because record2 sets the datatype
setattr(record2, attribute, values[1])
@@ -89,5 +90,3 @@ def test_compare_entities():
setattr(record2, attribute, values[0])
assert check_identical(record1, record2)
assert check_identical(record2, record1)
-
-
diff --git a/unittests/test_file_identifiables.py b/unittests/test_file_identifiables.py
index 234fae20c53e137bf049e496dbe178a30e5de833..ae5c536a8d55ba345f2708625d04fce3a5fff5c2 100644
--- a/unittests/test_file_identifiables.py
+++ b/unittests/test_file_identifiables.py
@@ -71,5 +71,3 @@ def test_file_identifiable():
with raises(RuntimeError, match=".*unambigiously.*"):
records.append(test_record_alsocorrect_path)
identified_file = ident.get_file(file_obj)
-
-
diff --git a/unittests/test_schema.py b/unittests/test_schema.py
index cac37c758aa838d78eb24435db55b099258900ac..4af29e28782a29360ecc10f342ae2e09ea7ce88a 100644
--- a/unittests/test_schema.py
+++ b/unittests/test_schema.py
@@ -13,6 +13,7 @@ from pytest import raises
from jsonschema.exceptions import ValidationError
+
def rfp(*pathcomponents):
"""
Return full path.
diff --git a/unittests/test_tool_extended.py b/unittests/test_tool_extended.py
index 2361e99373042a5f5ce73b8eb98083f7431d4836..d9f37064589a9d40e2e1e5be8952c430f058b65e 100644
--- a/unittests/test_tool_extended.py
+++ b/unittests/test_tool_extended.py
@@ -47,7 +47,7 @@ def crawler():
# def ident(crawler):
# ident = LocalStorageIdentifiableAdapter()
# crawler.identifiableAdapter = ident
-
+
# ident.restore_state(rfp("records.xml"))
# ident.register_identifiable(