WIP: pipeline

.docker/Dockerfile-outer

+# Use docker as parent image
+FROM docker:20.10.3
+
+# http://bugs.python.org/issue19846
+ENV LANG C.UTF-8
+
+# install dependencies
+RUN apk add --no-cache py3-pip python3 python3-dev gcc make \
+    git bash curl gettext  py3-requests 
+RUN apk add --no-cache libffi-dev openssl-dev libc-dev libxslt libxslt-dev \
+    libxml2 libxml2-dev
+RUN apk add --no-cache ca-certificates
+
+# install rust (needed for compiling a docker-compose dependency)
+# This is necessary until alpine comes with an up to date RUST
+# copied from https://github.com/rust-lang/docker-rust/blob/bbc7feb12033da3909dced4e88ddbb6964fbc328/1.50.0/alpine3.13/Dockerfile
+
+ENV RUSTUP_HOME=/usr/local/rustup \
+    CARGO_HOME=/usr/local/cargo \
+    PATH=/usr/local/cargo/bin:$PATH \
+    RUST_VERSION=1.50.0
+
+RUN set -eux; \
+    apkArch="$(apk --print-arch)"; \
+    case "$apkArch" in \
+        x86_64) rustArch='x86_64-unknown-linux-musl'; rustupSha256='05c5c05ec76671d73645aac3afbccf2187352fce7e46fc85be859f52a42797f6' ;; \
+        aarch64) rustArch='aarch64-unknown-linux-musl'; rustupSha256='6a8a480d8d9e7f8c6979d7f8b12bc59da13db67970f7b13161ff409f0a771213' ;; \
+        *) echo >&2 "unsupported architecture: $apkArch"; exit 1 ;; \
+    esac; \
+    url="https://static.rust-lang.org/rustup/archive/1.23.1/${rustArch}/rustup-init"; \
+    wget "$url"; \
+    echo "${rustupSha256} *rustup-init" | sha256sum -c -; \
+    chmod +x rustup-init; \
+    ./rustup-init -y --no-modify-path --profile minimal --default-toolchain $RUST_VERSION --default-host ${rustArch}; \
+    rm rustup-init; \
+    chmod -R a+w $RUSTUP_HOME $CARGO_HOME; \
+    rustup --version; \
+    cargo --version; \
+    rustc --version;
+
+
+RUN pip3 install wheel
+RUN pip3 install docker-compose==1.25
+
+# Script for waiting on LA server
+COPY wait-for-it.sh /opt/caosdb/wait-for-it.sh
+
+# Runtime settings
+WORKDIR /opt/caosdb
+RUN mkdir -p /opt/caosdb/build_docker/
+CMD /bin/bash

.docker/cert.sh

+#!/bin/bash
+
+# ** header v3.0
+# This file is a part of the CaosDB Project.
+#
+# Copyright (C) 2019 Daniel Hornung, Göttingen
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see <https://www.gnu.org/licenses/>.
+#
+# ** end header
+
+
+# Creates a directory `cert` and certificates in this directory.
+#
+# The hostname for which the certificate is created can be changed by setting
+# the environment variable CAOSHOSTNAME.
+#
+# ## Overview of variables ##
+#
+# - CAOSHOSTNAME :: Hostname for the key (localhost)
+# - KEYPW :: Password for the key (default ist CaosDBSecret)
+# - KEYSTOREPW :: Password for the key store (same as KEYPW)
+function cert() {
+    mkdir -p cert
+    cd cert
+    KEYPW="${KEYPW:-CaosDBSecret}"
+    CAOSHOSTNAME="${CAOSHOSTNAME:-localhost}"
+    KEYSTOREPW="${KEYPW:-}"
+    # NOTE: KEYPW and KEYSTOREPW are the same, due to Java limitations.
+    KEYPW="${KEYPW}" openssl genrsa -aes256 -out caosdb.key.pem \
+         -passout env:KEYPW 2048
+    # Certificate is for localhost
+    KEYPW="${KEYPW}" openssl req -new -x509 -key caosdb.key.pem \
+         -out caosdb.cert.pem -passin env:KEYPW \
+         -subj "/C=/ST=/L=/O=/OU=/CN=${CAOSHOSTNAME}"
+    KEYPW="${KEYPW}" KEYSTOREPW="$KEYSTOREPW" openssl pkcs12 -export \
+         -inkey caosdb.key.pem -in caosdb.cert.pem -out all-certs.pkcs12 \
+         -passin env:KEYPW -passout env:KEYPW
+
+    keytool -importkeystore -srckeystore all-certs.pkcs12 -srcstoretype PKCS12 \
+            -deststoretype pkcs12 -destkeystore caosdb.jks \
+            -srcstorepass "${KEYPW}" \
+            -destkeypass "${KEYPW}" -deststorepass "$KEYSTOREPW"
+    echo "Certificates successfuly created."
+}
+
+cert

.docker/docker-compose.yml

+version: '3.7'
+services:
+  sqldb:
+    image: mariadb:10.4
+    environment:
+      MYSQL_ROOT_PASSWORD: caosdb1234
+    networks:
+      - caosnet
+  caosdb-server:
+    image: "$CI_REGISTRY/caosdb/src/caosdb-deploy:$CAOSDB_TAG"
+    user: 999:999
+    depends_on:
+      - sqldb
+    networks:
+      - caosnet
+    volumes:
+      - type: bind
+        source: ./cert
+        target: /opt/caosdb/cert
+      - type: volume
+        source: extroot
+        target: /opt/caosdb/mnt/extroot
+      - type: volume
+        source: scripting
+        target: /opt/caosdb/git/caosdb-server/scripting
+      - type: volume
+        source: authtoken
+        target: /opt/caosdb/git/caosdb-server/authtoken
+    ports:
+      # - "from_outside:from_inside"
+      - "10443:10443"
+      - "10080:10080"
+    environment:
+      DEBUG: 1
+      CAOSDB_CONFIG_AUTHTOKEN_CONFIG: "conf/core/authtoken.example.yaml"
+volumes:
+  scripting:
+  extroot:
+  authtoken:
+networks:
+  caosnet:
+    driver: bridge

.docker/run.sh

+#!/bin/sh
+
+docker-compose -f tester.yml run tester
+rv=$?
+echo $rv > result

.docker/tester.yml

+version: '3.7'
+services:
+  tester:
+    image: "$CI_REGISTRY_IMAGE"
+    networks:
+      - docker_caosnet
+    volumes:
+      - type: bind
+        source: ./cert
+        target: /cert
+      - type: volume
+        source: extroot
+        target: /extroot
+      - type: volume
+        source: scripting
+        target: /scripting
+      - type: volume
+        source: authtoken
+        target: /authtoken
+networks:
+  docker_caosnet:
+    external: true
+volumes:
+  scripting:
+  extroot:
+  authtoken:

.docker/wait-for-it.sh

+#!/usr/bin/env bash
+# License: 
+# From https://github.com/vishnubob/wait-for-it
+# The MIT License (MIT)
+# Use this script to test if a given TCP host/port are available
+
+WAITFORIT_cmdname=${0##*/}
+
+echoerr() { if [[ $WAITFORIT_QUIET -ne 1 ]]; then echo "$@" 1>&2; fi }
+
+usage()
+{
+    cat << USAGE >&2
+Usage:
+    $WAITFORIT_cmdname host:port [-s] [-t timeout] [-- command args]
+    -h HOST | --host=HOST       Host or IP under test
+    -p PORT | --port=PORT       TCP port under test
+                                Alternatively, you specify the host and port as host:port
+    -s | --strict               Only execute subcommand if the test succeeds
+    -q | --quiet                Don't output any status messages
+    -t TIMEOUT | --timeout=TIMEOUT
+                                Timeout in seconds, zero for no timeout
+    -- COMMAND ARGS             Execute command with args after the test finishes
+USAGE
+    exit 1
+}
+
+wait_for()
+{
+    if [[ $WAITFORIT_TIMEOUT -gt 0 ]]; then
+        echoerr "$WAITFORIT_cmdname: waiting $WAITFORIT_TIMEOUT seconds for $WAITFORIT_HOST:$WAITFORIT_PORT"
+    else
+        echoerr "$WAITFORIT_cmdname: waiting for $WAITFORIT_HOST:$WAITFORIT_PORT without a timeout"
+    fi
+    WAITFORIT_start_ts=$(date +%s)
+    while :
+    do
+        if [[ $WAITFORIT_ISBUSY -eq 1 ]]; then
+            nc -z $WAITFORIT_HOST $WAITFORIT_PORT
+            WAITFORIT_result=$?
+        else
+            (echo > /dev/tcp/$WAITFORIT_HOST/$WAITFORIT_PORT) >/dev/null 2>&1
+            WAITFORIT_result=$?
+        fi
+        if [[ $WAITFORIT_result -eq 0 ]]; then
+            WAITFORIT_end_ts=$(date +%s)
+            echoerr "$WAITFORIT_cmdname: $WAITFORIT_HOST:$WAITFORIT_PORT is available after $((WAITFORIT_end_ts - WAITFORIT_start_ts)) seconds"
+            break
+        fi
+        sleep 1
+    done
+    return $WAITFORIT_result
+}
+
+wait_for_wrapper()
+{
+    # In order to support SIGINT during timeout: http://unix.stackexchange.com/a/57692
+    if [[ $WAITFORIT_QUIET -eq 1 ]]; then
+        timeout $WAITFORIT_BUSYTIMEFLAG $WAITFORIT_TIMEOUT $0 --quiet --child --host=$WAITFORIT_HOST --port=$WAITFORIT_PORT --timeout=$WAITFORIT_TIMEOUT &
+    else
+        timeout $WAITFORIT_BUSYTIMEFLAG $WAITFORIT_TIMEOUT $0 --child --host=$WAITFORIT_HOST --port=$WAITFORIT_PORT --timeout=$WAITFORIT_TIMEOUT &
+    fi
+    WAITFORIT_PID=$!
+    trap "kill -INT -$WAITFORIT_PID" INT
+    wait $WAITFORIT_PID
+    WAITFORIT_RESULT=$?
+    if [[ $WAITFORIT_RESULT -ne 0 ]]; then
+        echoerr "$WAITFORIT_cmdname: timeout occurred after waiting $WAITFORIT_TIMEOUT seconds for $WAITFORIT_HOST:$WAITFORIT_PORT"
+    fi
+    return $WAITFORIT_RESULT
+}
+
+# process arguments
+while [[ $# -gt 0 ]]
+do
+    case "$1" in
+        *:* )
+        WAITFORIT_hostport=(${1//:/ })
+        WAITFORIT_HOST=${WAITFORIT_hostport[0]}
+        WAITFORIT_PORT=${WAITFORIT_hostport[1]}
+        shift 1
+        ;;
+        --child)
+        WAITFORIT_CHILD=1
+        shift 1
+        ;;
+        -q | --quiet)
+        WAITFORIT_QUIET=1
+        shift 1
+        ;;
+        -s | --strict)
+        WAITFORIT_STRICT=1
+        shift 1
+        ;;
+        -h)
+        WAITFORIT_HOST="$2"
+        if [[ $WAITFORIT_HOST == "" ]]; then break; fi
+        shift 2
+        ;;
+        --host=*)
+        WAITFORIT_HOST="${1#*=}"
+        shift 1
+        ;;
+        -p)
+        WAITFORIT_PORT="$2"
+        if [[ $WAITFORIT_PORT == "" ]]; then break; fi
+        shift 2
+        ;;
+        --port=*)
+        WAITFORIT_PORT="${1#*=}"
+        shift 1
+        ;;
+        -t)
+        WAITFORIT_TIMEOUT="$2"
+        if [[ $WAITFORIT_TIMEOUT == "" ]]; then break; fi
+        shift 2
+        ;;
+        --timeout=*)
+        WAITFORIT_TIMEOUT="${1#*=}"
+        shift 1
+        ;;
+        --)
+        shift
+        WAITFORIT_CLI=("$@")
+        break
+        ;;
+        --help)
+        usage
+        ;;
+        *)
+        echoerr "Unknown argument: $1"
+        usage
+        ;;
+    esac
+done
+
+if [[ "$WAITFORIT_HOST" == "" || "$WAITFORIT_PORT" == "" ]]; then
+    echoerr "Error: you need to provide a host and port to test."
+    usage
+fi
+
+WAITFORIT_TIMEOUT=${WAITFORIT_TIMEOUT:-15}
+WAITFORIT_STRICT=${WAITFORIT_STRICT:-0}
+WAITFORIT_CHILD=${WAITFORIT_CHILD:-0}
+WAITFORIT_QUIET=${WAITFORIT_QUIET:-0}
+
+# check to see if timeout is from busybox?
+WAITFORIT_TIMEOUT_PATH=$(type -p timeout)
+WAITFORIT_TIMEOUT_PATH=$(realpath $WAITFORIT_TIMEOUT_PATH 2>/dev/null || readlink -f $WAITFORIT_TIMEOUT_PATH)
+if [[ $WAITFORIT_TIMEOUT_PATH =~ "busybox" ]]; then
+        WAITFORIT_ISBUSY=1
+        WAITFORIT_BUSYTIMEFLAG="-t"
+
+else
+        WAITFORIT_ISBUSY=0
+        WAITFORIT_BUSYTIMEFLAG=""
+fi
+
+if [[ $WAITFORIT_CHILD -gt 0 ]]; then
+    wait_for
+    WAITFORIT_RESULT=$?
+    exit $WAITFORIT_RESULT
+else
+    if [[ $WAITFORIT_TIMEOUT -gt 0 ]]; then
+        wait_for_wrapper
+        WAITFORIT_RESULT=$?
+    else
+        wait_for
+        WAITFORIT_RESULT=$?
+    fi
+fi
+
+if [[ $WAITFORIT_CLI != "" ]]; then
+    if [[ $WAITFORIT_RESULT -ne 0 && $WAITFORIT_STRICT -eq 1 ]]; then
+        echoerr "$WAITFORIT_cmdname: strict mode, refusing to execute subprocess"
+        exit $WAITFORIT_RESULT
+    fi
+    exec "${WAITFORIT_CLI[@]}"
+else
+    exit $WAITFORIT_RESULT
+fi
+