Merge branch 'f-unittest' into 'dev'

Add CI pipelines

See merge request caosdb/src/crawler2.0!16

.docker/Dockerfile

+FROM debian:latest
+RUN apt-get update && \
+    apt-get install \
+    curl \
+    git \
+    openjdk-11-jdk-headless \
+    python3-autopep8 \
+    python3-pip \
+    python3-pytest \
+    tox \
+    -y
+COPY .docker/wait-for-it.sh /wait-for-it.sh
+ADD https://gitlab.indiscale.com/api/v4/projects/97/repository/commits/${PYLIB} \
+    pylib_version.json
+RUN git clone https://gitlab.indiscale.com/caosdb/src/caosdb-pylib.git && \
+    cd caosdb-pylib && git checkout ${PYLIB} && pip3 install .
+ADD https://gitlab.indiscale.com/api/v4/projects/104/repository/commits/${ADVANCED} \
+    advanced_version.json
+RUN git clone https://gitlab.indiscale.com/caosdb/src/caosdb-advanced-user-tools.git && \
+    cd caosdb-advanced-user-tools && git checkout ${ADVANCED} && pip3 install .
+COPY . /git
+
+# Delete .git because it is huge.
+RUN rm -r /git/.git
+
+# Install pycaosdb.ini for the tests
+RUN mv /git/.docker/tester_pycaosdb.ini /git/pycaosdb.ini
+
+RUN cd /git/ && pip3 install .
+
+WORKDIR /git/integrationtests
+# wait for server,
+CMD /wait-for-it.sh caosdb-server:10443 -t 500 -- \
+    # ... install pycaosdb.ini the server-side scripts
+    cp /git/.docker/sss_pycaosdb.ini /scripting/home/.pycaosdb.ini && \
+    # ... and run tests
+    pytest .

.docker/cert.sh

+#!/bin/bash
+
+# ** header v3.0
+# This file is a part of the CaosDB Project.
+#
+# Copyright (C) 2019 Daniel Hornung, Göttingen
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see <https://www.gnu.org/licenses/>.
+#
+# ** end header
+
+
+# Creates a directory `cert` and certificates in this directory.
+#
+# The hostname for which the certificate is created can be changed by setting
+# the environment variable CAOSHOSTNAME.
+#
+# ## Overview of variables ##
+#
+# - CAOSHOSTNAME :: Hostname for the key (localhost)
+# - KEYPW :: Password for the key (default ist CaosDBSecret)
+# - KEYSTOREPW :: Password for the key store (same as KEYPW)
+function cert() {
+    mkdir -p cert
+    cd cert
+    KEYPW="${KEYPW:-CaosDBSecret}"
+    CAOSHOSTNAME="${CAOSHOSTNAME:-localhost}"
+    KEYSTOREPW="${KEYPW:-}"
+    # NOTE: KEYPW and KEYSTOREPW are the same, due to Java limitations.
+    KEYPW="${KEYPW}" openssl genrsa -aes256 -out caosdb.key.pem \
+         -passout env:KEYPW 2048
+    # Certificate is for localhost
+    KEYPW="${KEYPW}" openssl req -new -x509 -key caosdb.key.pem \
+         -out caosdb.cert.pem -passin env:KEYPW \
+         -subj "/C=/ST=/L=/O=/OU=/CN=${CAOSHOSTNAME}"
+    KEYPW="${KEYPW}" KEYSTOREPW="$KEYSTOREPW" openssl pkcs12 -export \
+         -inkey caosdb.key.pem -in caosdb.cert.pem -out all-certs.pkcs12 \
+         -passin env:KEYPW -passout env:KEYPW
+
+    keytool -importkeystore -srckeystore all-certs.pkcs12 -srcstoretype PKCS12 \
+            -deststoretype pkcs12 -destkeystore caosdb.jks \
+            -srcstorepass "${KEYPW}" \
+            -destkeypass "${KEYPW}" -deststorepass "$KEYSTOREPW"
+    echo "Certificates successfuly created."
+}
+
+cert

.docker/docker-compose.yml

+version: '3.7'
+services:
+  sqldb:
+    image: mariadb:10.4
+    environment:
+      MYSQL_ROOT_PASSWORD: caosdb1234
+    networks:
+      - caosnet
+  caosdb-server:
+    image: "$CI_REGISTRY/caosdb/src/caosdb-deploy:$CAOSDB_TAG"
+    user: 999:999
+    depends_on:
+      - sqldb
+    networks:
+      - caosnet
+    volumes:
+      - type: bind
+        source: ./cert
+        target: /opt/caosdb/cert
+      - type: volume
+        source: extroot
+        target: /opt/caosdb/mnt/extroot
+      - type: volume
+        source: scripting
+        target: /opt/caosdb/git/caosdb-server/scripting
+      - type: volume
+        source: authtoken
+        target: /opt/caosdb/git/caosdb-server/authtoken
+    ports:
+      # - "from_outside:from_inside"
+      - "10443:10443"
+      - "10080:10080"
+    environment:
+      DEBUG: 1
+      CAOSDB_CONFIG_AUTHTOKEN_CONFIG: "conf/core/authtoken.example.yaml"
+      CAOSDB_CONFIG_TRANSACTION_BENCHMARK_ENABLED: "TRUE"
+volumes:
+  scripting:
+  extroot:
+  authtoken:
+networks:
+  caosnet:
+    driver: bridge

.docker/run.sh

+#!/bin/sh
+
+docker-compose -f tester.yml run tester
+rv=$?
+echo $rv > result

.docker/sss_pycaosdb.ini

+; this is the pycaosdb.ini for the server-side-scripting home.
+[Connection]
+url = https://caosdb-server:10443
+cacert = /opt/caosdb/cert/caosdb.cert.pem
+debug = 0
+timeout = 5000
+
+[Misc]
+sendmail = /usr/local/bin/sendmail_to_file

.docker/tester.yml

+version: '3.7'
+services:
+  tester:
+    image: "$CI_REGISTRY_IMAGE"
+    networks:
+      - docker_caosnet
+    volumes:
+      - type: bind
+        source: ./cert
+        target: /cert
+      - type: volume
+        source: extroot
+        target: /extroot
+      - type: volume
+        source: scripting
+        target: /scripting
+      - type: volume
+        source: authtoken
+        target: /authtoken
+networks:
+  docker_caosnet:
+    external: true
+volumes:
+  scripting:
+  extroot:
+  authtoken:

.docker/tester_pycaosdb.ini

+; pycaosdb.ini for pytest test suites.
+
+[IntegrationTests]
+; location of the scripting bin dir which is used for the test scripts from the
+; server's perspective.
+test_server_side_scripting.bin_dir.server = scripting/bin-debug/
+; location of the scripting bin dir which is used for the test scripts from the
+; pyinttest's perspective.
+test_server_side_scripting.bin_dir.local = /scripting/bin-debug/
+
+; location of the files from the pyinttest perspective
+test_files.test_insert_files_in_dir.local = /extroot/test_insert_files_in_dir/
+; location of the files from the caosdb_servers perspective
+test_files.test_insert_files_in_dir.server = /opt/caosdb/mnt/extroot/test_insert_files_in_dir/
+
+; location of the one-time tokens from the pyinttest's perspective
+test_authentication.admin_token_crud = /authtoken/admin_token_crud.txt
+test_authentication.admin_token_expired = /authtoken/admin_token_expired.txt
+test_authentication.admin_token_3_attempts = /authtoken/admin_token_3_attempts.txt
+
+
+[Connection]
+url = https://caosdb-server:10443/
+username = admin
+cacert = /cert/caosdb.cert.pem
+debug = 0
+
+password_method = plain
+password = caosdb
+
+timeout = 500

.docker/wait-for-it.sh

+#!/usr/bin/env bash
+# License: 
+# From https://github.com/vishnubob/wait-for-it
+# The MIT License (MIT)
+# Use this script to test if a given TCP host/port are available
+
+WAITFORIT_cmdname=${0##*/}
+
+echoerr() { if [[ $WAITFORIT_QUIET -ne 1 ]]; then echo "$@" 1>&2; fi }
+
+usage()
+{
+    cat << USAGE >&2
+Usage:
+    $WAITFORIT_cmdname host:port [-s] [-t timeout] [-- command args]
+    -h HOST | --host=HOST       Host or IP under test
+    -p PORT | --port=PORT       TCP port under test
+                                Alternatively, you specify the host and port as host:port
+    -s | --strict               Only execute subcommand if the test succeeds
+    -q | --quiet                Don't output any status messages
+    -t TIMEOUT | --timeout=TIMEOUT
+                                Timeout in seconds, zero for no timeout
+    -- COMMAND ARGS             Execute command with args after the test finishes
+USAGE
+    exit 1
+}
+
+wait_for()
+{
+    if [[ $WAITFORIT_TIMEOUT -gt 0 ]]; then
+        echoerr "$WAITFORIT_cmdname: waiting $WAITFORIT_TIMEOUT seconds for $WAITFORIT_HOST:$WAITFORIT_PORT"
+    else
+        echoerr "$WAITFORIT_cmdname: waiting for $WAITFORIT_HOST:$WAITFORIT_PORT without a timeout"
+    fi
+    WAITFORIT_start_ts=$(date +%s)
+    while :
+    do
+        if [[ $WAITFORIT_ISBUSY -eq 1 ]]; then
+            nc -z $WAITFORIT_HOST $WAITFORIT_PORT
+            WAITFORIT_result=$?
+        else
+            (echo > /dev/tcp/$WAITFORIT_HOST/$WAITFORIT_PORT) >/dev/null 2>&1
+            WAITFORIT_result=$?
+        fi
+        if [[ $WAITFORIT_result -eq 0 ]]; then
+            WAITFORIT_end_ts=$(date +%s)
+            echoerr "$WAITFORIT_cmdname: $WAITFORIT_HOST:$WAITFORIT_PORT is available after $((WAITFORIT_end_ts - WAITFORIT_start_ts)) seconds"
+            break
+        fi
+        sleep 1
+    done
+    return $WAITFORIT_result
+}
+
+wait_for_wrapper()
+{
+    # In order to support SIGINT during timeout: http://unix.stackexchange.com/a/57692
+    if [[ $WAITFORIT_QUIET -eq 1 ]]; then
+        timeout $WAITFORIT_BUSYTIMEFLAG $WAITFORIT_TIMEOUT $0 --quiet --child --host=$WAITFORIT_HOST --port=$WAITFORIT_PORT --timeout=$WAITFORIT_TIMEOUT &
+    else
+        timeout $WAITFORIT_BUSYTIMEFLAG $WAITFORIT_TIMEOUT $0 --child --host=$WAITFORIT_HOST --port=$WAITFORIT_PORT --timeout=$WAITFORIT_TIMEOUT &
+    fi
+    WAITFORIT_PID=$!
+    trap "kill -INT -$WAITFORIT_PID" INT
+    wait $WAITFORIT_PID
+    WAITFORIT_RESULT=$?
+    if [[ $WAITFORIT_RESULT -ne 0 ]]; then
+        echoerr "$WAITFORIT_cmdname: timeout occurred after waiting $WAITFORIT_TIMEOUT seconds for $WAITFORIT_HOST:$WAITFORIT_PORT"
+    fi
+    return $WAITFORIT_RESULT
+}
+
+# process arguments
+while [[ $# -gt 0 ]]
+do
+    case "$1" in
+        *:* )
+        WAITFORIT_hostport=(${1//:/ })
+        WAITFORIT_HOST=${WAITFORIT_hostport[0]}
+        WAITFORIT_PORT=${WAITFORIT_hostport[1]}
+        shift 1
+        ;;
+        --child)
+        WAITFORIT_CHILD=1
+        shift 1
+        ;;
+        -q | --quiet)
+        WAITFORIT_QUIET=1
+        shift 1
+        ;;
+        -s | --strict)
+        WAITFORIT_STRICT=1
+        shift 1
+        ;;
+        -h)
+        WAITFORIT_HOST="$2"
+        if [[ $WAITFORIT_HOST == "" ]]; then break; fi
+        shift 2
+        ;;
+        --host=*)
+        WAITFORIT_HOST="${1#*=}"
+        shift 1
+        ;;
+        -p)
+        WAITFORIT_PORT="$2"
+        if [[ $WAITFORIT_PORT == "" ]]; then break; fi
+        shift 2
+        ;;
+        --port=*)
+        WAITFORIT_PORT="${1#*=}"
+        shift 1
+        ;;
+        -t)
+        WAITFORIT_TIMEOUT="$2"
+        if [[ $WAITFORIT_TIMEOUT == "" ]]; then break; fi
+        shift 2
+        ;;
+        --timeout=*)
+        WAITFORIT_TIMEOUT="${1#*=}"
+        shift 1
+        ;;
+        --)
+        shift
+        WAITFORIT_CLI=("$@")
+        break
+        ;;
+        --help)
+        usage
+        ;;
+        *)
+        echoerr "Unknown argument: $1"
+        usage
+        ;;
+    esac
+done
+
+if [[ "$WAITFORIT_HOST" == "" || "$WAITFORIT_PORT" == "" ]]; then
+    echoerr "Error: you need to provide a host and port to test."
+    usage
+fi
+
+WAITFORIT_TIMEOUT=${WAITFORIT_TIMEOUT:-15}
+WAITFORIT_STRICT=${WAITFORIT_STRICT:-0}
+WAITFORIT_CHILD=${WAITFORIT_CHILD:-0}
+WAITFORIT_QUIET=${WAITFORIT_QUIET:-0}
+
+# check to see if timeout is from busybox?
+WAITFORIT_TIMEOUT_PATH=$(type -p timeout)
+WAITFORIT_TIMEOUT_PATH=$(realpath $WAITFORIT_TIMEOUT_PATH 2>/dev/null || readlink -f $WAITFORIT_TIMEOUT_PATH)
+if [[ $WAITFORIT_TIMEOUT_PATH =~ "busybox" ]]; then
+        WAITFORIT_ISBUSY=1
+        WAITFORIT_BUSYTIMEFLAG="-t"
+
+else
+        WAITFORIT_ISBUSY=0
+        WAITFORIT_BUSYTIMEFLAG=""
+fi
+
+if [[ $WAITFORIT_CHILD -gt 0 ]]; then
+    wait_for
+    WAITFORIT_RESULT=$?
+    exit $WAITFORIT_RESULT
+else
+    if [[ $WAITFORIT_TIMEOUT -gt 0 ]]; then
+        wait_for_wrapper
+        WAITFORIT_RESULT=$?
+    else
+        wait_for
+        WAITFORIT_RESULT=$?
+    fi
+fi
+
+if [[ $WAITFORIT_CLI != "" ]]; then
+    if [[ $WAITFORIT_RESULT -ne 0 && $WAITFORIT_STRICT -eq 1 ]]; then
+        echoerr "$WAITFORIT_cmdname: strict mode, refusing to execute subprocess"
+        exit $WAITFORIT_RESULT
+    fi
+    exec "${WAITFORIT_CLI[@]}"
+else
+    exit $WAITFORIT_RESULT
+fi
+

.gitlab-ci.yml

+#
+# This file is a part of the CaosDB Project.
+#
+# Copyright (C) 2018 Research Group Biomedical Physics,
+# Max-Planck-Institute for Dynamics and Self-Organization Göttingen
+# Copyright (C) 2019 Henrik tom Wörden 
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+variables:
+   CI_REGISTRY_IMAGE: $CI_REGISTRY/caosdb/src/crawler2.0/testenv:$CI_COMMIT_REF_NAME
+   CI_REGISTRY_IMAGE_BASE: $CI_REGISTRY/caosdb/src/caosdb-pyinttest/base:latest
+
+stages:
+  - info
+  - setup
+  - cert
+  - style
+  - test
+  - deploy
+
+
+# During the test stage the CI pipeline (which runs in a "root" docker) starts
+# two docker containers with docker-compose (one for the caosdb-server, another
+# for the mysql-backend). Then a third docker is being started which contains
+# the test suite and executes it.
+#
+# +-------------(root docker)-------------------------+
+# |                                                   |
+# |             +-(caosdb_mysqlbackend)-------------+ |
+# |             |                                   | |
+# |             +-----------------------------------+ |
+# |             +-(caosdb-server)-------------------+ |
+# |             |                                   | |
+# |             | /opt/caosdb                       | |
+# |    .------->|  + /git/caosdb-server/scripting/  | |
+# |    | .----->|  + /git/caosdb-server/authtoken/  | |
+# |    | | .--->|  + /mnt/extroot                   | |
+# |    | | | .->|  + /cert                          | |
+# |    | | | |  |                                   | |
+# |    | | | |  +-----------------------------------+ |
+# |    | | | |                                        |
+# |    | | | |    filesystem:                         |
+# |    | | |  *---  /cert -----------.                |
+# |    | | |                          |               |
+# |    | | |      volumes:            |               |
+# |    | |  *-----  extroot  ------.  |               |
+# |    |  *-------  scripting  --.  | |               |
+# |     *---------  authtoken  -. | | |               |
+# |                             | | | |               |
+# | +-(crawler tests)---+       | | | |               |
+# | |                      |    | | | |               |
+# | | /authtoken           |<---* | | |               |
+# | | /scripting           |<----*  | |               |
+# | | /extroot             |<------*  |               |
+# | | /cert                |<--------*                |
+# | |                      |                          |
+# | +----------------------+                          |
+# +---------------------------------------------------+
+#
+# In the root docker, the directory /cert is mounted to .docker/cert relative
+# to this repository. The directory is created during the cert stage of this
+# pipeline and a certificate is created in there. The certificat is then
+# available in mounted directories in the server and crawler containers.
+#
+# Additional volumes in the root docker are shared by the caosdb-server and the crawler
+# containers.  These volumes are intended to be used for testing server-side scripting and
+# file-system features.
+#
+
+.env: &env
+  - echo "Pipeline triggered by $TRIGGERED_BY_REPO@$TRIGGERED_BY_REF ($TRIGGERED_BY_HASH)"
+  - echo "CI_REGISTRY_IMAGE_BASE = $CI_REGISTRY_IMAGE_BASE"
+  - echo "CI_REGISTRY_IMAGE = $CI_REGISTRY_IMAGE"
+  - echo "CAOSDB_TAG = $CAOSDB_TAG"
+  - echo "REFTAG = $REFTAG"
+  - echo "F_BRANCH = $F_BRANCH"
+  - echo "CI_COMMIT_REF_NAME = $CI_COMMIT_REF_NAME"
+  - ls -lah /image-cache/
+
+  - F_BRANCH=${F_BRANCH:-$CI_COMMIT_REF_NAME}
+  - echo $F_BRANCH
+  - if [[ "$REFTAG" == "" ]] ; then
+      if [[ "$F_BRANCH" == "dev" ]] ; then
+        REFTAG=dev;
+      fi;
+    fi
+  - REFTAG=${REFTAG:-dev_F_${F_BRANCH}}
+
+  - echo $F_BRANCH
+
+  - if [[ "$CAOSDB_TAG" == "" ]]; then
+      CAOSDB_TAG=${REFTAG};
+    fi
+  - echo $CAOSDB_TAG
+
+info:
+  tags: [cached-dind]
+  image: docker:20.10
+  stage: info
+  needs: []
+  script:
+    - *env
+
+unittest:
+  tags: [cached-dind]
+  image: docker:20.10
+  stage: test
+  image: $CI_REGISTRY_IMAGE
+  script:
+      - tox
+
+inttest:
+  tags: [docker]
+  services:
+    - docker:20.10-dind
+  variables:
+    # This is a workaround for the gitlab-runner health check mechanism when
+    # using docker-dind service.  The runner will otherwise guess the port
+    # wrong and the health check will timeout.
+    SERVICE_PORT_2376_TCP_PORT: 2375
+  stage: test
+  image: $CI_REGISTRY_IMAGE_BASE
+  needs: [cert]
+  script:
+      - *env
+      - docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY
+      - echo $CAOSDB_TAG
+
+      - cd .docker
+        # Store mariadb  version
+      - MARIADBVERSION=$(grep mariadb docker-compose.yml | awk '{print $2}')
+      - echo "mariadb image:"$MARIADBVERSION
+      - time docker load < /image-cache/caosdb-crawler-testenv-${CI_COMMIT_REF_NAME}.tar || true
+      - time docker load < /image-cache/caosdb-${REFTAG}.tar || time docker load < /image-cache/caosdb-dev.tar || true
+      - time docker load < /image-cache/$MARIADBVERSION.tar || true
+      - docker pull $CI_REGISTRY/caosdb/src/caosdb-deploy:$CAOSDB_TAG || CAOSDB_TAG=dev
+      - docker pull $CI_REGISTRY_IMAGE
+
+        # Here, the server and the mysql backend docker are being started
+      - CAOSDB_TAG=$CAOSDB_TAG docker-compose  up -d
+
+        # Store versions of CaosDB parts
+      - docker exec -u 0 -t docker_caosdb-server_1 cat /opt/caosdb/git/caosdb_pylib_commit > hash_pylib
+      - docker exec -u 0 -t docker_caosdb-server_1 cat /opt/caosdb/git/caosdb_webui_commit > hash_webui
+      - docker exec -u 0 -t docker_caosdb-server_1 cat /opt/caosdb/git/caosdb_server_commit > hash_server
+      - docker exec -u 0 -t docker_caosdb-server_1 cat /opt/caosdb/git/caosdb_mysqlbackend_commit > hash_mysql
+      - docker exec -u 0 -t docker_caosdb-server_1 cat /opt/caosdb/git/caosdb_proto_commit > hash_proto
+      - cat hash_server
+      - cat hash_proto
+      - cat hash_mysql
+      - cat hash_webui
+      - cat hash_pylib
+        # Run the actual tests. This starts a new docker container within which
+        # the tests run. The return value is stored in .docker/result
+      - /bin/sh ./run.sh
+
+        # Save logs
+      - docker logs docker_caosdb-server_1 &> ../caosdb_log.txt
+      - docker logs docker_sqldb_1 &> ../mariadb_log.txt
+      - cd ..
+
+        # Stop the server
+      - docker-compose -f .docker/docker-compose.yml down
+
+        # the crawler docker writes the return value of the tests into the
+        # file result
+      - rc=`cat .docker/result`
+      - exit $rc
+  dependencies: [cert]
+  timeout: 3h
+  artifacts:
+    paths:
+      - caosdb_log.txt
+      - mariadb_log.txt
+      - .docker/hash_*
+    expire_in: 1 week
+
+build-testenv:
+  tags: [cached-dind]
+  image: docker:20.10
+  stage: setup
+  timeout: 2h
+  only:
+    - schedules
+    - web
+    - pushes
+  needs: []
+  script:
+      - df -h
+      - command -v wget
+      - if [ -z "$PYLIB" ]; then
+          if echo "$CI_COMMIT_REF_NAME" | grep -c "^f-" ; then
+            echo "Check if pylib has branch $CI_COMMIT_REF_NAME" ;
+            if wget https://gitlab.indiscale.com/api/v4/projects/97/repository/branches/${CI_COMMIT_REF_NAME} ; then
+              PYLIB=$CI_COMMIT_REF_NAME ;
+            fi;
+          fi;
+        fi;
+      - PYLIB=${PYLIB:-dev}
+      - echo $PYLIB
+
+      - docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY
+        # use here general latest or specific branch latest...
+      - docker build 
+        --build-arg PYLIB=${PYLIB}
+        --build-arg ADVANCED=${ADVANCED:dev}
+        --file .docker/Dockerfile
+        -t $CI_REGISTRY_IMAGE .
+      - docker push $CI_REGISTRY_IMAGE
+      - docker save $CI_REGISTRY_IMAGE > /image-cache/caosdb-crawler-testenv-${CI_COMMIT_REF_NAME}.tar
+
+cert:
+  tags: [docker]
+  stage: cert
+  image: $CI_REGISTRY_IMAGE
+  needs:
+    - job: build-testenv
+      optional: true
+  artifacts:
+    paths:
+      - .docker/cert/
+    expire_in: 1 week
+  script:
+      - cd .docker
+      - CAOSHOSTNAME=caosdb-server ./cert.sh
+
+style:
+  tags: [docker]
+  stage: style
+  image: $CI_REGISTRY_IMAGE
+  needs:
+    - job: build-testenv
+      optional: true
+  script:
+      - autopep8 -r --diff --exit-code .
+  allow_failure: true

integrationtests/test.py

@@ -28,6 +28,7 @@
 module description
 """
 
+import os
 from caosdb import EmptyUniqueQueryError
 import argparse
 import sys
@@ -42,7 +43,6 @@ import yaml
 from caosadvancedtools.testutils import clear_database, set_test_key
 set_test_key("10b128cf8a1372f30aa3697466bb55e76974e0c16a599bb44ace88f19c8f61e2")
 
-import os
 
 def rfp(*pathcomponents):
     """
@@ -52,8 +52,6 @@ def rfp(*pathcomponents):
     return os.path.join(os.path.dirname(__file__), *pathcomponents)
 
 
-
-
 @pytest.fixture
 def usemodel():
     model = parse_model_from_yaml(rfp("model.yml"))
@@ -150,6 +148,7 @@ def test_multiple_insertions(clear_database, usemodel, ident, crawler):
     assert len(ins) == 0
     assert len(ups) == 0
 
+
 def test_insertion(clear_database, usemodel, ident, crawler):
     ins, ups = crawler.synchronize()
 
@@ -169,6 +168,7 @@ def test_insertion(clear_database, usemodel, ident, crawler):
     assert len(ins) == 0
     assert len(ups) == 0
 
+
 def test_insertion_and_update(clear_database, usemodel, ident, crawler):
     ins, ups = crawler.synchronize()
 
@@ -185,6 +185,7 @@ def test_insertion_and_update(clear_database, usemodel, ident, crawler):
     assert len(ins) == 0
     assert len(ups) == 1
 
+
 def test_identifiable_update(clear_database, usemodel, ident, crawler):
     ins, ups = crawler.synchronize()
 
@@ -206,7 +207,6 @@ def test_identifiable_update(clear_database, usemodel, ident, crawler):
     assert len(ins) == 0
     assert len(ups) == 1
 
-    
     # Test the change within one property:
     cr = Crawler(debug=True, identifiableAdapter=ident)
     crawl_standard_test_directory(cr)
@@ -269,6 +269,7 @@ def test_file_insertion(clear_database, usemodel, ident, crawler_extended):
         assert len(r) == 1
         assert r[0].get_property("ReadmeFile").value == f.id
 
+
 def test_file_update(clear_database, usemodel, ident, crawler_extended):
     ins1, ups1 = crawler_extended.synchronize(commit_changes=True)
     fileList_ins = [r for r in ins1 if r.role == "File"]
@@ -303,7 +304,6 @@ def test_file_update(clear_database, usemodel, ident, crawler_extended):
     assert len(res) == 11
     assert res[0].parents[0].name == "ProjectMarkdownReadme"
 
-
     # TODO: Implement file update checks (based on checksum)
     # Add test with actual file update:
     # assert len(ins2) == 0

setup.cfg

@@ -16,7 +16,6 @@ classifiers =
 [options]
 package_dir =
             = src
-
 packages = find:
 python_requires = >=3.6
 install_requires =
@@ -24,6 +23,7 @@ install_requires =
 	caosdb
 	caosadvancedtools
     yaml-header-tools
+    pyyaml
 
 [options.packages.find]
 where = src

src/newcrawler/identified_cache.py

@@ -67,7 +67,6 @@ def _create_hashable_string(identifiable: db.Record):
                     tmplist.append(val)
             value = str(tmplist)
 
-
         rec_string += "{}:".format(pname) + value
     return rec_string
 

tox.ini

 [tox]
-envlist=py36, py37, py38, py39
+envlist=py36, py37, py38, py39, py310
 skip_missing_interpreters = true
 
 [testenv]

unittests/test_converters.py

@@ -177,7 +177,8 @@ def test_json_converter(converter_registry):
     test_json = File("testjson.json", rfp(
         "test_directories", "examples_json", "testjson.json"))
 
-    schema_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), "test_directories", "examples_json", "testjson.schema.json")
+    schema_path = os.path.join(os.path.dirname(os.path.abspath(__file__)),
+                               "test_directories", "examples_json", "testjson.schema.json")
     jsonconverter = JSONFileConverter(
         definition={"match": "(.*)", "validate": schema_path},
         name="TestJSONFileConverter",

unittests/test_entity_comparison.py

@@ -79,7 +79,8 @@ def test_compare_entities():
     for attribute, values in zip(("_checksum", "_size"),
                                  (vals[0], vals[1])):
         setattr(record1, attribute, values[0])
-        assert not check_identical(record1, record2)  # not identical, because record1 sets the datatype
+        # not identical, because record1 sets the datatype
+        assert not check_identical(record1, record2)
         assert check_identical(record2, record1)  # identical, because record2 sets the datatype
 
         setattr(record2, attribute, values[1])
@@ -89,5 +90,3 @@ def test_compare_entities():
         setattr(record2, attribute, values[0])
         assert check_identical(record1, record2)
         assert check_identical(record2, record1)
-
-    

unittests/test_file_identifiables.py

@@ -71,5 +71,3 @@ def test_file_identifiable():
     with raises(RuntimeError, match=".*unambigiously.*"):
         records.append(test_record_alsocorrect_path)
         identified_file = ident.get_file(file_obj)
-    
-    

unittests/test_schema.py

@@ -13,6 +13,7 @@ from pytest import raises
 
 from jsonschema.exceptions import ValidationError
 
+
 def rfp(*pathcomponents):
     """
     Return full path.