Skip to content
Snippets Groups Projects
Commit a8298556 authored by Henrik tom Wörden's avatar Henrik tom Wörden
Browse files

MAINT: set up CI pipelines

parent 03ac6656
No related branches found
No related tags found
2 merge requests!53Release 0.1,!16Add CI pipelines
Pipeline #21418 failed
FROM debian:latest
RUN apt-get update && \
apt-get install \
curl \
git \
openjdk-11-jdk-headless \
python3-autopep8 \
python3-pip \
tox \
-y
COPY .docker/wait-for-it.sh /wait-for-it.sh
ADD https://gitlab.indiscale.com/api/v4/projects/97/repository/commits/${PYLIB} \
pylib_version.json
RUN git clone https://gitlab.indiscale.com/caosdb/src/caosdb-pylib.git && \
cd caosdb-pylib && git checkout ${PYLIB} && pip3 install .
ADD https://gitlab.indiscale.com/api/v4/projects/104/repository/commits/${ADVANCED} \
advanced_version.json
RUN git clone https://gitlab.indiscale.com/caosdb/src/caosdb-advanced-user-tools.git && \
cd caosdb-advanced-user-tools && git checkout ${ADVANCED} && pip3 install .
COPY . /git
# Delete .git because it is huge.
RUN rm -r /git/.git
# Install pycaosdb.ini for the tests
RUN mv /git/.docker/tester_pycaosdb.ini /git/pycaosdb.ini
WORKDIR /git/integrationtests
# wait for server,
CMD /wait-for-it.sh caosdb-server:10443 -t 500 -- \
# ... install pycaosdb.ini the server-side scripts
cp /git/.docker/sss_pycaosdb.ini /scripting/home/.pycaosdb.ini && \
# ... and run tests
./test.py
#!/bin/bash
# ** header v3.0
# This file is a part of the CaosDB Project.
#
# Copyright (C) 2019 Daniel Hornung, Göttingen
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as
# published by the Free Software Foundation, either version 3 of the
# License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <https://www.gnu.org/licenses/>.
#
# ** end header
# Creates a directory `cert` and certificates in this directory.
#
# The hostname for which the certificate is created can be changed by setting
# the environment variable CAOSHOSTNAME.
#
# ## Overview of variables ##
#
# - CAOSHOSTNAME :: Hostname for the key (localhost)
# - KEYPW :: Password for the key (default ist CaosDBSecret)
# - KEYSTOREPW :: Password for the key store (same as KEYPW)
function cert() {
mkdir -p cert
cd cert
KEYPW="${KEYPW:-CaosDBSecret}"
CAOSHOSTNAME="${CAOSHOSTNAME:-localhost}"
KEYSTOREPW="${KEYPW:-}"
# NOTE: KEYPW and KEYSTOREPW are the same, due to Java limitations.
KEYPW="${KEYPW}" openssl genrsa -aes256 -out caosdb.key.pem \
-passout env:KEYPW 2048
# Certificate is for localhost
KEYPW="${KEYPW}" openssl req -new -x509 -key caosdb.key.pem \
-out caosdb.cert.pem -passin env:KEYPW \
-subj "/C=/ST=/L=/O=/OU=/CN=${CAOSHOSTNAME}"
KEYPW="${KEYPW}" KEYSTOREPW="$KEYSTOREPW" openssl pkcs12 -export \
-inkey caosdb.key.pem -in caosdb.cert.pem -out all-certs.pkcs12 \
-passin env:KEYPW -passout env:KEYPW
keytool -importkeystore -srckeystore all-certs.pkcs12 -srcstoretype PKCS12 \
-deststoretype pkcs12 -destkeystore caosdb.jks \
-srcstorepass "${KEYPW}" \
-destkeypass "${KEYPW}" -deststorepass "$KEYSTOREPW"
echo "Certificates successfuly created."
}
cert
version: '3.7'
services:
sqldb:
image: mariadb:10.4
environment:
MYSQL_ROOT_PASSWORD: caosdb1234
networks:
- caosnet
caosdb-server:
image: "$CI_REGISTRY/caosdb/src/caosdb-deploy:$CAOSDB_TAG"
user: 999:999
depends_on:
- sqldb
networks:
- caosnet
volumes:
- type: bind
source: ./cert
target: /opt/caosdb/cert
- type: volume
source: extroot
target: /opt/caosdb/mnt/extroot
- type: volume
source: scripting
target: /opt/caosdb/git/caosdb-server/scripting
- type: volume
source: authtoken
target: /opt/caosdb/git/caosdb-server/authtoken
ports:
# - "from_outside:from_inside"
- "10443:10443"
- "10080:10080"
environment:
DEBUG: 1
CAOSDB_CONFIG_AUTHTOKEN_CONFIG: "conf/core/authtoken.example.yaml"
CAOSDB_CONFIG_TRANSACTION_BENCHMARK_ENABLED: "TRUE"
volumes:
scripting:
extroot:
authtoken:
networks:
caosnet:
driver: bridge
#!/bin/sh
docker-compose -f tester.yml run tester
rv=$?
echo $rv > result
; this is the pycaosdb.ini for the server-side-scripting home.
[Connection]
url = https://caosdb-server:10443
cacert = /opt/caosdb/cert/caosdb.cert.pem
debug = 0
timeout = 5000
[Misc]
sendmail = /usr/local/bin/sendmail_to_file
version: '3.7'
services:
tester:
image: "$CI_REGISTRY_IMAGE"
networks:
- docker_caosnet
volumes:
- type: bind
source: ./cert
target: /cert
- type: volume
source: extroot
target: /extroot
- type: volume
source: scripting
target: /scripting
- type: volume
source: authtoken
target: /authtoken
networks:
docker_caosnet:
external: true
volumes:
scripting:
extroot:
authtoken:
; pycaosdb.ini for pytest test suites.
[IntegrationTests]
; location of the scripting bin dir which is used for the test scripts from the
; server's perspective.
test_server_side_scripting.bin_dir.server = scripting/bin-debug/
; location of the scripting bin dir which is used for the test scripts from the
; pyinttest's perspective.
test_server_side_scripting.bin_dir.local = /scripting/bin-debug/
; location of the files from the pyinttest perspective
test_files.test_insert_files_in_dir.local = /extroot/test_insert_files_in_dir/
; location of the files from the caosdb_servers perspective
test_files.test_insert_files_in_dir.server = /opt/caosdb/mnt/extroot/test_insert_files_in_dir/
; location of the one-time tokens from the pyinttest's perspective
test_authentication.admin_token_crud = /authtoken/admin_token_crud.txt
test_authentication.admin_token_expired = /authtoken/admin_token_expired.txt
test_authentication.admin_token_3_attempts = /authtoken/admin_token_3_attempts.txt
[Connection]
url = https://caosdb-server:10443/
username = admin
cacert = /cert/caosdb.cert.pem
debug = 0
password_method = plain
password = caosdb
timeout = 500
#!/usr/bin/env bash
# License:
# From https://github.com/vishnubob/wait-for-it
# The MIT License (MIT)
# Use this script to test if a given TCP host/port are available
WAITFORIT_cmdname=${0##*/}
echoerr() { if [[ $WAITFORIT_QUIET -ne 1 ]]; then echo "$@" 1>&2; fi }
usage()
{
cat << USAGE >&2
Usage:
$WAITFORIT_cmdname host:port [-s] [-t timeout] [-- command args]
-h HOST | --host=HOST Host or IP under test
-p PORT | --port=PORT TCP port under test
Alternatively, you specify the host and port as host:port
-s | --strict Only execute subcommand if the test succeeds
-q | --quiet Don't output any status messages
-t TIMEOUT | --timeout=TIMEOUT
Timeout in seconds, zero for no timeout
-- COMMAND ARGS Execute command with args after the test finishes
USAGE
exit 1
}
wait_for()
{
if [[ $WAITFORIT_TIMEOUT -gt 0 ]]; then
echoerr "$WAITFORIT_cmdname: waiting $WAITFORIT_TIMEOUT seconds for $WAITFORIT_HOST:$WAITFORIT_PORT"
else
echoerr "$WAITFORIT_cmdname: waiting for $WAITFORIT_HOST:$WAITFORIT_PORT without a timeout"
fi
WAITFORIT_start_ts=$(date +%s)
while :
do
if [[ $WAITFORIT_ISBUSY -eq 1 ]]; then
nc -z $WAITFORIT_HOST $WAITFORIT_PORT
WAITFORIT_result=$?
else
(echo > /dev/tcp/$WAITFORIT_HOST/$WAITFORIT_PORT) >/dev/null 2>&1
WAITFORIT_result=$?
fi
if [[ $WAITFORIT_result -eq 0 ]]; then
WAITFORIT_end_ts=$(date +%s)
echoerr "$WAITFORIT_cmdname: $WAITFORIT_HOST:$WAITFORIT_PORT is available after $((WAITFORIT_end_ts - WAITFORIT_start_ts)) seconds"
break
fi
sleep 1
done
return $WAITFORIT_result
}
wait_for_wrapper()
{
# In order to support SIGINT during timeout: http://unix.stackexchange.com/a/57692
if [[ $WAITFORIT_QUIET -eq 1 ]]; then
timeout $WAITFORIT_BUSYTIMEFLAG $WAITFORIT_TIMEOUT $0 --quiet --child --host=$WAITFORIT_HOST --port=$WAITFORIT_PORT --timeout=$WAITFORIT_TIMEOUT &
else
timeout $WAITFORIT_BUSYTIMEFLAG $WAITFORIT_TIMEOUT $0 --child --host=$WAITFORIT_HOST --port=$WAITFORIT_PORT --timeout=$WAITFORIT_TIMEOUT &
fi
WAITFORIT_PID=$!
trap "kill -INT -$WAITFORIT_PID" INT
wait $WAITFORIT_PID
WAITFORIT_RESULT=$?
if [[ $WAITFORIT_RESULT -ne 0 ]]; then
echoerr "$WAITFORIT_cmdname: timeout occurred after waiting $WAITFORIT_TIMEOUT seconds for $WAITFORIT_HOST:$WAITFORIT_PORT"
fi
return $WAITFORIT_RESULT
}
# process arguments
while [[ $# -gt 0 ]]
do
case "$1" in
*:* )
WAITFORIT_hostport=(${1//:/ })
WAITFORIT_HOST=${WAITFORIT_hostport[0]}
WAITFORIT_PORT=${WAITFORIT_hostport[1]}
shift 1
;;
--child)
WAITFORIT_CHILD=1
shift 1
;;
-q | --quiet)
WAITFORIT_QUIET=1
shift 1
;;
-s | --strict)
WAITFORIT_STRICT=1
shift 1
;;
-h)
WAITFORIT_HOST="$2"
if [[ $WAITFORIT_HOST == "" ]]; then break; fi
shift 2
;;
--host=*)
WAITFORIT_HOST="${1#*=}"
shift 1
;;
-p)
WAITFORIT_PORT="$2"
if [[ $WAITFORIT_PORT == "" ]]; then break; fi
shift 2
;;
--port=*)
WAITFORIT_PORT="${1#*=}"
shift 1
;;
-t)
WAITFORIT_TIMEOUT="$2"
if [[ $WAITFORIT_TIMEOUT == "" ]]; then break; fi
shift 2
;;
--timeout=*)
WAITFORIT_TIMEOUT="${1#*=}"
shift 1
;;
--)
shift
WAITFORIT_CLI=("$@")
break
;;
--help)
usage
;;
*)
echoerr "Unknown argument: $1"
usage
;;
esac
done
if [[ "$WAITFORIT_HOST" == "" || "$WAITFORIT_PORT" == "" ]]; then
echoerr "Error: you need to provide a host and port to test."
usage
fi
WAITFORIT_TIMEOUT=${WAITFORIT_TIMEOUT:-15}
WAITFORIT_STRICT=${WAITFORIT_STRICT:-0}
WAITFORIT_CHILD=${WAITFORIT_CHILD:-0}
WAITFORIT_QUIET=${WAITFORIT_QUIET:-0}
# check to see if timeout is from busybox?
WAITFORIT_TIMEOUT_PATH=$(type -p timeout)
WAITFORIT_TIMEOUT_PATH=$(realpath $WAITFORIT_TIMEOUT_PATH 2>/dev/null || readlink -f $WAITFORIT_TIMEOUT_PATH)
if [[ $WAITFORIT_TIMEOUT_PATH =~ "busybox" ]]; then
WAITFORIT_ISBUSY=1
WAITFORIT_BUSYTIMEFLAG="-t"
else
WAITFORIT_ISBUSY=0
WAITFORIT_BUSYTIMEFLAG=""
fi
if [[ $WAITFORIT_CHILD -gt 0 ]]; then
wait_for
WAITFORIT_RESULT=$?
exit $WAITFORIT_RESULT
else
if [[ $WAITFORIT_TIMEOUT -gt 0 ]]; then
wait_for_wrapper
WAITFORIT_RESULT=$?
else
wait_for
WAITFORIT_RESULT=$?
fi
fi
if [[ $WAITFORIT_CLI != "" ]]; then
if [[ $WAITFORIT_RESULT -ne 0 && $WAITFORIT_STRICT -eq 1 ]]; then
echoerr "$WAITFORIT_cmdname: strict mode, refusing to execute subprocess"
exit $WAITFORIT_RESULT
fi
exec "${WAITFORIT_CLI[@]}"
else
exit $WAITFORIT_RESULT
fi
#
# This file is a part of the CaosDB Project.
#
# Copyright (C) 2018 Research Group Biomedical Physics,
# Max-Planck-Institute for Dynamics and Self-Organization Göttingen
# Copyright (C) 2019 Henrik tom Wörden
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as
# published by the Free Software Foundation, either version 3 of the
# License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <https://www.gnu.org/licenses/>.
variables:
CI_REGISTRY_IMAGE: $CI_REGISTRY/caosdb/src/crawler2.0/testenv:$CI_COMMIT_REF_NAME
CI_REGISTRY_IMAGE_BASE: $CI_REGISTRY/caosdb/src/caosdb-pyinttest/base:latest
stages:
- info
- setup
- cert
- style
- test
- deploy
# During the test stage the CI pipeline (which runs in a "root" docker) starts
# two docker containers with docker-compose (one for the caosdb-server, another
# for the mysql-backend). Then a third docker is being started which contains
# the test suite and executes it.
#
# +-------------(root docker)-------------------------+
# | |
# | +-(caosdb_mysqlbackend)-------------+ |
# | | | |
# | +-----------------------------------+ |
# | +-(caosdb-server)-------------------+ |
# | | | |
# | | /opt/caosdb | |
# | .------->| + /git/caosdb-server/scripting/ | |
# | | .----->| + /git/caosdb-server/authtoken/ | |
# | | | .--->| + /mnt/extroot | |
# | | | | .->| + /cert | |
# | | | | | | | |
# | | | | | +-----------------------------------+ |
# | | | | | |
# | | | | | filesystem: |
# | | | | *--- /cert -----------. |
# | | | | | |
# | | | | volumes: | |
# | | | *----- extroot ------. | |
# | | *------- scripting --. | | |
# | *--------- authtoken -. | | | |
# | | | | | |
# | +-(crawler tests)---+ | | | | |
# | | | | | | | |
# | | /authtoken |<---* | | | |
# | | /scripting |<----* | | |
# | | /extroot |<------* | |
# | | /cert |<--------* |
# | | | |
# | +----------------------+ |
# +---------------------------------------------------+
#
# In the root docker, the directory /cert is mounted to .docker/cert relative
# to this repository. The directory is created during the cert stage of this
# pipeline and a certificate is created in there. The certificat is then
# available in mounted directories in the server and crawler containers.
#
# Additional volumes in the root docker are shared by the caosdb-server and the crawler
# containers. These volumes are intended to be used for testing server-side scripting and
# file-system features.
#
.env: &env
- echo "Pipeline triggered by $TRIGGERED_BY_REPO@$TRIGGERED_BY_REF ($TRIGGERED_BY_HASH)"
- echo "CI_REGISTRY_IMAGE_BASE = $CI_REGISTRY_IMAGE_BASE"
- echo "CI_REGISTRY_IMAGE = $CI_REGISTRY_IMAGE"
- echo "CAOSDB_TAG = $CAOSDB_TAG"
- echo "REFTAG = $REFTAG"
- echo "F_BRANCH = $F_BRANCH"
- echo "CI_COMMIT_REF_NAME = $CI_COMMIT_REF_NAME"
- ls -lah /image-cache/
- F_BRANCH=${F_BRANCH:-$CI_COMMIT_REF_NAME}
- echo $F_BRANCH
- if [[ "$REFTAG" == "" ]] ; then
if [[ "$F_BRANCH" == "dev" ]] ; then
REFTAG=dev;
fi;
fi
- REFTAG=${REFTAG:-dev_F_${F_BRANCH}}
- echo $F_BRANCH
- if [[ "$CAOSDB_TAG" == "" ]]; then
CAOSDB_TAG=${REFTAG};
fi
- echo $CAOSDB_TAG
info:
tags: [cached-dind]
image: docker:20.10
stage: info
needs: []
script:
- *env
unittest:
tags: [cached-dind]
image: docker:20.10
stage: test
image: $CI_REGISTRY_IMAGE
script:
- tox
inttest:
tags: [docker]
services:
- docker:20.10-dind
variables:
# This is a workaround for the gitlab-runner health check mechanism when
# using docker-dind service. The runner will otherwise guess the port
# wrong and the health check will timeout.
SERVICE_PORT_2376_TCP_PORT: 2375
stage: test
image: $CI_REGISTRY_IMAGE_BASE
needs: [cert]
script:
- *env
- docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY
- echo $CAOSDB_TAG
- cd .docker
# Store mariadb version
- MARIADBVERSION=$(grep mariadb docker-compose.yml | awk '{print $2}')
- echo "mariadb image:"$MARIADBVERSION
- time docker load < /image-cache/caosdb-crawler-testenv-${CI_COMMIT_REF_NAME}.tar || true
- time docker load < /image-cache/caosdb-${REFTAG}.tar || time docker load < /image-cache/caosdb-dev.tar || true
- time docker load < /image-cache/$MARIADBVERSION.tar || true
- docker pull $CI_REGISTRY/caosdb/src/caosdb-deploy:$CAOSDB_TAG || CAOSDB_TAG=dev
- docker pull $CI_REGISTRY_IMAGE
# Here, the server and the mysql backend docker are being started
- CAOSDB_TAG=$CAOSDB_TAG docker-compose up -d
# Store versions of CaosDB parts
- docker exec -u 0 -t docker_caosdb-server_1 cat /opt/caosdb/git/caosdb_pylib_commit > hash_pylib
- docker exec -u 0 -t docker_caosdb-server_1 cat /opt/caosdb/git/caosdb_webui_commit > hash_webui
- docker exec -u 0 -t docker_caosdb-server_1 cat /opt/caosdb/git/caosdb_server_commit > hash_server
- docker exec -u 0 -t docker_caosdb-server_1 cat /opt/caosdb/git/caosdb_mysqlbackend_commit > hash_mysql
- docker exec -u 0 -t docker_caosdb-server_1 cat /opt/caosdb/git/caosdb_proto_commit > hash_proto
- cat hash_server
- cat hash_proto
- cat hash_mysql
- cat hash_webui
- cat hash_pylib
# Run the actual tests. This starts a new docker container within which
# the tests run. The return value is stored in .docker/result
- /bin/sh ./run.sh
# Save logs
- docker logs docker_caosdb-server_1 &> ../caosdb_log.txt
- docker logs docker_sqldb_1 &> ../mariadb_log.txt
- cd ..
# Stop the server
- docker-compose -f .docker/docker-compose.yml down
# the crawler docker writes the return value of the tests into the
# file result
- rc=`cat .docker/result`
- exit $rc
dependencies: [cert]
timeout: 3h
artifacts:
paths:
- caosdb_log.txt
- mariadb_log.txt
- .docker/hash_*
expire_in: 1 week
build-testenv:
tags: [cached-dind]
image: docker:20.10
stage: setup
timeout: 2h
only:
- schedules
- web
- pushes
needs: []
script:
- df -h
- command -v wget
- if [ -z "$PYLIB" ]; then
if echo "$CI_COMMIT_REF_NAME" | grep -c "^f-" ; then
echo "Check if pylib has branch $CI_COMMIT_REF_NAME" ;
if wget https://gitlab.indiscale.com/api/v4/projects/97/repository/branches/${CI_COMMIT_REF_NAME} ; then
PYLIB=$CI_COMMIT_REF_NAME ;
fi;
fi;
fi;
- PYLIB=${PYLIB:-dev}
- echo $PYLIB
- docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY
# use here general latest or specific branch latest...
- docker build
--build-arg PYLIB=${PYLIB}
--build-arg ADVANCED=${ADVANCED:dev}
--file .docker/Dockerfile
-t $CI_REGISTRY_IMAGE .
- docker push $CI_REGISTRY_IMAGE
- docker save $CI_REGISTRY_IMAGE > /image-cache/caosdb-crawler-testenv-${CI_COMMIT_REF_NAME}.tar
cert:
tags: [docker]
stage: cert
image: $CI_REGISTRY_IMAGE
needs:
- job: build-testenv
optional: true
artifacts:
paths:
- .docker/cert/
expire_in: 1 week
script:
- cd .docker
- CAOSHOSTNAME=caosdb-server ./cert.sh
style:
tags: [docker]
stage: style
image: $CI_REGISTRY_IMAGE
needs:
- job: build-testenv
optional: true
script:
- autopep8 -r --diff --exit-code .
allow_failure: true
[tox]
envlist=py36, py37, py38, py39
envlist=py36, py37, py38, py39, py310
skip_missing_interpreters = true
[testenv]
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment