diff --git a/.docker/Dockerfile b/.docker/Dockerfile new file mode 100644 index 0000000000000000000000000000000000000000..97b8fdf2b5ae43dc96726e16ea21a2c6a1883fdb --- /dev/null +++ b/.docker/Dockerfile @@ -0,0 +1,37 @@ +FROM debian:bookworm +RUN apt-get update && \ + apt-get install \ + curl \ + git \ + openjdk-17-jdk-headless \ + python3-autopep8 \ + python3-pip \ + python3-pytest \ + python3-sphinx \ + -y +RUN pip3 install --break-system-packages pylint recommonmark sphinx-rtd-theme tox +COPY .docker/wait-for-it.sh /wait-for-it.sh +ARG PYLIB +ADD https://gitlab.indiscale.com/api/v4/projects/97/repository/commits/${PYLIB} \ + pylib_version.json +RUN git clone https://gitlab.indiscale.com/caosdb/src/caosdb-pylib.git && \ + cd caosdb-pylib && git checkout ${PYLIB} && pip3 install --break-system-packages . +ARG ADVANCED +ADD https://gitlab.indiscale.com/api/v4/projects/104/repository/commits/${ADVANCED} \ + advanced_version.json +RUN git clone https://gitlab.indiscale.com/caosdb/src/caosdb-advanced-user-tools.git && \ + cd caosdb-advanced-user-tools && git checkout ${ADVANCED} && pip3 install --break-system-packages .[h5-crawler] +COPY . /git + +# Delete .git because it is huge. +RUN rm -r /git/.git + +RUN cd /git/ && pip3 install --break-system-packages . + +WORKDIR /git/integrationtests +# wait for server, +CMD /wait-for-it.sh caosdb-server:10443 -t 500 -- \ + # ... install pycaosdb.ini the server-side scripts + cp /git/.docker/sss_pycaosdb.ini /scripting/home/.pycaosdb.ini && \ + # ... and run tests + pytest-3 . diff --git a/.docker/cert.sh b/.docker/cert.sh new file mode 100755 index 0000000000000000000000000000000000000000..628ba8dd9cc19f85a515a75cebd03b8981337bfd --- /dev/null +++ b/.docker/cert.sh @@ -0,0 +1,61 @@ +#!/bin/bash + +# ** header v3.0 +# This file is a part of the CaosDB Project. +# +# Copyright (C) 2019 Daniel Hornung, Göttingen +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see <https://www.gnu.org/licenses/>. +# +# ** end header + + +# Creates a directory `cert` and certificates in this directory. +# +# The hostname for which the certificate is created can be changed by setting +# the environment variable CAOSHOSTNAME. +# +# ## Overview of variables ## +# +# - CAOSHOSTNAME :: Hostname for the key (localhost) +# - KEYPW :: Password for the key (default ist CaosDBSecret) +# - KEYSTOREPW :: Password for the key store (same as KEYPW) +function cert() { + mkdir -p cert + cd cert + KEYPW="${KEYPW:-CaosDBSecret}" + CAOSHOSTNAME="${CAOSHOSTNAME:-localhost}" + KEYSTOREPW="${KEYPW:-}" + # NOTE: KEYPW and KEYSTOREPW are the same, due to Java limitations. + KEYPW="${KEYPW}" openssl genrsa -aes256 -out caosdb.key.pem \ + -passout env:KEYPW 2048 + # Certificate is for localhost + KEYPW="${KEYPW}" openssl req -new -x509 -key caosdb.key.pem \ + -out caosdb.cert.pem -passin env:KEYPW \ + -subj "/C=/ST=/L=/O=example/OU=example/CN=${CAOSHOSTNAME}" \ + -days 365 \ + -addext "subjectAltName = DNS:${CAOSHOSTNAME}" \ + -addext "certificatePolicies = 1.2.3.4" + KEYPW="${KEYPW}" KEYSTOREPW="$KEYSTOREPW" openssl pkcs12 -export \ + -inkey caosdb.key.pem -in caosdb.cert.pem -out all-certs.pkcs12 \ + -passin env:KEYPW -passout env:KEYPW + + keytool -importkeystore -srckeystore all-certs.pkcs12 -srcstoretype PKCS12 \ + -deststoretype pkcs12 -destkeystore caosdb.jks \ + -srcstorepass "${KEYPW}" \ + -destkeypass "${KEYPW}" -deststorepass "$KEYSTOREPW" + echo "Certificates successfuly created." +} + +cert diff --git a/.docker/docker-compose.yml b/.docker/docker-compose.yml new file mode 100644 index 0000000000000000000000000000000000000000..02ccac5c48e039a3374a0d169f3b355f897e45fc --- /dev/null +++ b/.docker/docker-compose.yml @@ -0,0 +1,43 @@ +version: '3.7' +services: + sqldb: + image: mariadb:10.4 + environment: + MYSQL_ROOT_PASSWORD: caosdb1234 + networks: + - caosnet + caosdb-server: + image: "$CI_REGISTRY/caosdb/src/caosdb-deploy:$CAOSDB_TAG" + user: 999:999 + depends_on: + - sqldb + networks: + - caosnet + volumes: + - type: bind + source: ./cert + target: /opt/caosdb/cert + - type: bind + source: "../integrationtests/test_data/extroot" + target: /opt/caosdb/mnt/extroot + - type: volume + source: scripting + target: /opt/caosdb/git/caosdb-server/scripting + - type: volume + source: authtoken + target: /opt/caosdb/git/caosdb-server/authtoken + ports: + # - "from_outside:from_inside" + - "10443:10443" + - "10080:10080" + environment: + DEBUG: 1 + CAOSDB_CONFIG_AUTHTOKEN_CONFIG: "conf/core/authtoken.example.yaml" + CAOSDB_CONFIG_TRANSACTION_BENCHMARK_ENABLED: "TRUE" + CAOSDB_CONFIG__CAOSDB_INTEGRATION_TEST_SUITE_KEY: 10b128cf8a1372f30aa3697466bb55e76974e0c16a599bb44ace88f19c8f61e2 +volumes: + scripting: + authtoken: +networks: + caosnet: + driver: bridge diff --git a/.docker/run.sh b/.docker/run.sh new file mode 100755 index 0000000000000000000000000000000000000000..b0e1a716f28516b83043fb3fdb6594515a0bafd4 --- /dev/null +++ b/.docker/run.sh @@ -0,0 +1,5 @@ +#!/bin/sh + +docker-compose -f tester.yml run tester +rv=$? +echo $rv > result diff --git a/.docker/sss_pycaosdb.ini b/.docker/sss_pycaosdb.ini new file mode 100644 index 0000000000000000000000000000000000000000..de2867f8dc66b3e81f10f35e40c36f9cb8591604 --- /dev/null +++ b/.docker/sss_pycaosdb.ini @@ -0,0 +1,9 @@ +; this is the pycaosdb.ini for the server-side-scripting home. +[Connection] +url = https://caosdb-server:10443 +cacert = /opt/caosdb/cert/caosdb.cert.pem +debug = 0 +timeout = 5000 + +[Misc] +sendmail = /usr/local/bin/sendmail_to_file diff --git a/.docker/tester.yml b/.docker/tester.yml new file mode 100644 index 0000000000000000000000000000000000000000..83db879c6072bfdea7b3212c833116b96bb54d0c --- /dev/null +++ b/.docker/tester.yml @@ -0,0 +1,26 @@ +version: '3.7' +services: + tester: + image: "$CI_REGISTRY_IMAGE" + networks: + - docker_caosnet + volumes: + - type: bind + source: ./cert + target: /cert + - type: volume + source: extroot + target: /extroot + - type: volume + source: scripting + target: /scripting + - type: volume + source: authtoken + target: /authtoken +networks: + docker_caosnet: + external: true +volumes: + scripting: + extroot: + authtoken: diff --git a/.docker/tester_pycaosdb.ini b/.docker/tester_pycaosdb.ini new file mode 100644 index 0000000000000000000000000000000000000000..2159dec250b3dcb2f16043d12bdbe73675e4d75c --- /dev/null +++ b/.docker/tester_pycaosdb.ini @@ -0,0 +1,31 @@ +; pycaosdb.ini for pytest test suites. + +[IntegrationTests] +; location of the scripting bin dir which is used for the test scripts from the +; server's perspective. +test_server_side_scripting.bin_dir.server = scripting/bin-debug/ +; location of the scripting bin dir which is used for the test scripts from the +; pyinttest's perspective. +test_server_side_scripting.bin_dir.local = /scripting/bin-debug/ + +; location of the files from the pyinttest perspective +test_files.test_insert_files_in_dir.local = /extroot/test_insert_files_in_dir/ +; location of the files from the caosdb_servers perspective +test_files.test_insert_files_in_dir.server = /opt/caosdb/mnt/extroot/test_insert_files_in_dir/ + +; location of the one-time tokens from the pyinttest's perspective +test_authentication.admin_token_crud = /authtoken/admin_token_crud.txt +test_authentication.admin_token_expired = /authtoken/admin_token_expired.txt +test_authentication.admin_token_3_attempts = /authtoken/admin_token_3_attempts.txt + + +[Connection] +url = https://caosdb-server:10443/ +username = admin +cacert = /cert/caosdb.cert.pem +debug = 0 + +password_method = plain +password = caosdb + +timeout = 500 diff --git a/.docker/wait-for-it.sh b/.docker/wait-for-it.sh new file mode 100755 index 0000000000000000000000000000000000000000..d69e99f1f13257b559dce2433de0515379663efa --- /dev/null +++ b/.docker/wait-for-it.sh @@ -0,0 +1,182 @@ +#!/usr/bin/env bash +# License: +# From https://github.com/vishnubob/wait-for-it +# The MIT License (MIT) +# Use this script to test if a given TCP host/port are available + +WAITFORIT_cmdname=${0##*/} + +echoerr() { if [[ $WAITFORIT_QUIET -ne 1 ]]; then echo "$@" 1>&2; fi } + +usage() +{ + cat << USAGE >&2 +Usage: + $WAITFORIT_cmdname host:port [-s] [-t timeout] [-- command args] + -h HOST | --host=HOST Host or IP under test + -p PORT | --port=PORT TCP port under test + Alternatively, you specify the host and port as host:port + -s | --strict Only execute subcommand if the test succeeds + -q | --quiet Don't output any status messages + -t TIMEOUT | --timeout=TIMEOUT + Timeout in seconds, zero for no timeout + -- COMMAND ARGS Execute command with args after the test finishes +USAGE + exit 1 +} + +wait_for() +{ + if [[ $WAITFORIT_TIMEOUT -gt 0 ]]; then + echoerr "$WAITFORIT_cmdname: waiting $WAITFORIT_TIMEOUT seconds for $WAITFORIT_HOST:$WAITFORIT_PORT" + else + echoerr "$WAITFORIT_cmdname: waiting for $WAITFORIT_HOST:$WAITFORIT_PORT without a timeout" + fi + WAITFORIT_start_ts=$(date +%s) + while : + do + if [[ $WAITFORIT_ISBUSY -eq 1 ]]; then + nc -z $WAITFORIT_HOST $WAITFORIT_PORT + WAITFORIT_result=$? + else + (echo > /dev/tcp/$WAITFORIT_HOST/$WAITFORIT_PORT) >/dev/null 2>&1 + WAITFORIT_result=$? + fi + if [[ $WAITFORIT_result -eq 0 ]]; then + WAITFORIT_end_ts=$(date +%s) + echoerr "$WAITFORIT_cmdname: $WAITFORIT_HOST:$WAITFORIT_PORT is available after $((WAITFORIT_end_ts - WAITFORIT_start_ts)) seconds" + break + fi + sleep 1 + done + return $WAITFORIT_result +} + +wait_for_wrapper() +{ + # In order to support SIGINT during timeout: http://unix.stackexchange.com/a/57692 + if [[ $WAITFORIT_QUIET -eq 1 ]]; then + timeout $WAITFORIT_BUSYTIMEFLAG $WAITFORIT_TIMEOUT $0 --quiet --child --host=$WAITFORIT_HOST --port=$WAITFORIT_PORT --timeout=$WAITFORIT_TIMEOUT & + else + timeout $WAITFORIT_BUSYTIMEFLAG $WAITFORIT_TIMEOUT $0 --child --host=$WAITFORIT_HOST --port=$WAITFORIT_PORT --timeout=$WAITFORIT_TIMEOUT & + fi + WAITFORIT_PID=$! + trap "kill -INT -$WAITFORIT_PID" INT + wait $WAITFORIT_PID + WAITFORIT_RESULT=$? + if [[ $WAITFORIT_RESULT -ne 0 ]]; then + echoerr "$WAITFORIT_cmdname: timeout occurred after waiting $WAITFORIT_TIMEOUT seconds for $WAITFORIT_HOST:$WAITFORIT_PORT" + fi + return $WAITFORIT_RESULT +} + +# process arguments +while [[ $# -gt 0 ]] +do + case "$1" in + *:* ) + WAITFORIT_hostport=(${1//:/ }) + WAITFORIT_HOST=${WAITFORIT_hostport[0]} + WAITFORIT_PORT=${WAITFORIT_hostport[1]} + shift 1 + ;; + --child) + WAITFORIT_CHILD=1 + shift 1 + ;; + -q | --quiet) + WAITFORIT_QUIET=1 + shift 1 + ;; + -s | --strict) + WAITFORIT_STRICT=1 + shift 1 + ;; + -h) + WAITFORIT_HOST="$2" + if [[ $WAITFORIT_HOST == "" ]]; then break; fi + shift 2 + ;; + --host=*) + WAITFORIT_HOST="${1#*=}" + shift 1 + ;; + -p) + WAITFORIT_PORT="$2" + if [[ $WAITFORIT_PORT == "" ]]; then break; fi + shift 2 + ;; + --port=*) + WAITFORIT_PORT="${1#*=}" + shift 1 + ;; + -t) + WAITFORIT_TIMEOUT="$2" + if [[ $WAITFORIT_TIMEOUT == "" ]]; then break; fi + shift 2 + ;; + --timeout=*) + WAITFORIT_TIMEOUT="${1#*=}" + shift 1 + ;; + --) + shift + WAITFORIT_CLI=("$@") + break + ;; + --help) + usage + ;; + *) + echoerr "Unknown argument: $1" + usage + ;; + esac +done + +if [[ "$WAITFORIT_HOST" == "" || "$WAITFORIT_PORT" == "" ]]; then + echoerr "Error: you need to provide a host and port to test." + usage +fi + +WAITFORIT_TIMEOUT=${WAITFORIT_TIMEOUT:-15} +WAITFORIT_STRICT=${WAITFORIT_STRICT:-0} +WAITFORIT_CHILD=${WAITFORIT_CHILD:-0} +WAITFORIT_QUIET=${WAITFORIT_QUIET:-0} + +# check to see if timeout is from busybox? +WAITFORIT_TIMEOUT_PATH=$(type -p timeout) +WAITFORIT_TIMEOUT_PATH=$(realpath $WAITFORIT_TIMEOUT_PATH 2>/dev/null || readlink -f $WAITFORIT_TIMEOUT_PATH) +if [[ $WAITFORIT_TIMEOUT_PATH =~ "busybox" ]]; then + WAITFORIT_ISBUSY=1 + WAITFORIT_BUSYTIMEFLAG="-t" + +else + WAITFORIT_ISBUSY=0 + WAITFORIT_BUSYTIMEFLAG="" +fi + +if [[ $WAITFORIT_CHILD -gt 0 ]]; then + wait_for + WAITFORIT_RESULT=$? + exit $WAITFORIT_RESULT +else + if [[ $WAITFORIT_TIMEOUT -gt 0 ]]; then + wait_for_wrapper + WAITFORIT_RESULT=$? + else + wait_for + WAITFORIT_RESULT=$? + fi +fi + +if [[ $WAITFORIT_CLI != "" ]]; then + if [[ $WAITFORIT_RESULT -ne 0 && $WAITFORIT_STRICT -eq 1 ]]; then + echoerr "$WAITFORIT_cmdname: strict mode, refusing to execute subprocess" + exit $WAITFORIT_RESULT + fi + exec "${WAITFORIT_CLI[@]}" +else + exit $WAITFORIT_RESULT +fi + diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml new file mode 100644 index 0000000000000000000000000000000000000000..1944b7062da4fab02dffe185215115e039514d92 --- /dev/null +++ b/.gitlab-ci.yml @@ -0,0 +1,216 @@ +# +# This file is a part of the CaosDB Project. +# +# Copyright (C) 2018 Research Group Biomedical Physics, +# Max-Planck-Institute for Dynamics and Self-Organization Göttingen +# Copyright (C) 2019 Henrik tom Wörden +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see <https://www.gnu.org/licenses/>. + +variables: + CI_REGISTRY_IMAGE: $CI_REGISTRY/caosdb/src/linkahead-python-package-template/testenv:$CI_COMMIT_REF_NAME + CI_REGISTRY_IMAGE_BASE: $CI_REGISTRY/caosdb/src/caosdb-pyinttest/base:latest + +stages: + - info + - setup + - cert + - style + - test + - deploy + +.env: &env + - echo "Pipeline triggered by $TRIGGERED_BY_REPO@$TRIGGERED_BY_REF ($TRIGGERED_BY_HASH)" + - echo "CI_REGISTRY_IMAGE_BASE = $CI_REGISTRY_IMAGE_BASE" + - echo "CI_REGISTRY_IMAGE = $CI_REGISTRY_IMAGE" + - echo "CAOSDB_TAG = $CAOSDB_TAG" + - echo "REFTAG = $REFTAG" + - echo "F_BRANCH = $F_BRANCH" + - echo "CI_COMMIT_REF_NAME = $CI_COMMIT_REF_NAME" + - ls -lah /image-cache/ + + - F_BRANCH=${F_BRANCH:-$CI_COMMIT_REF_NAME} + - echo $F_BRANCH + - if [[ "$REFTAG" == "" ]] ; then + if [[ "$F_BRANCH" == "dev" ]] ; then + REFTAG=dev; + fi; + fi + - REFTAG=${REFTAG:-dev_F_${F_BRANCH}} + + - echo $F_BRANCH + + - if [[ "$CAOSDB_TAG" == "" ]]; then + CAOSDB_TAG=${REFTAG}; + fi + - echo $CAOSDB_TAG + +info: + tags: [cached-dind] + image: docker:20.10 + stage: info + needs: [] + script: + - *env + +unittest_py3.11: + tags: [cached-dind] + stage: test + image: $CI_REGISTRY_IMAGE + script: + - python3 -c "import sys; assert sys.version.startswith('3.11')" + - tox + +unittest_py3.8: + tags: [cached-dind] + stage: test + image: python:3.8 + script: &python_test_script + # install dependencies + - pip install pytest pytest-cov + - pip install . + # actual test + - pytest --cov=linkahead_python_package_template -vv ./unittests + +unittest_py3.9: + tags: [cached-dind] + stage: test + image: python:3.11 + script: *python_test_script + +unittest_py3.10: + tags: [cached-dind] + stage: test + image: python:3.10 + script: *python_test_script + +unittest_py3.12: + tags: [cached-dind] + stage: test + image: python:3.12 + script: *python_test_script + +unittest_py3.13: + allow_failure: true + tags: [cached-dind] + stage: test + image: python:3.13-rc + script: *python_test_script + +build-testenv: + tags: [cached-dind] + image: docker:20.10 + stage: setup + timeout: 2h + only: + - schedules + - web + - pushes + needs: [] + script: + - df -h + - command -v wget + - if [ -z "$PYLIB" ]; then + if echo "$CI_COMMIT_REF_NAME" | grep -c "^f-" ; then + echo "Check if pylib has branch $CI_COMMIT_REF_NAME" ; + if wget https://gitlab.indiscale.com/api/v4/projects/97/repository/branches/${CI_COMMIT_REF_NAME} ; then + PYLIB=$CI_COMMIT_REF_NAME ; + fi; + fi; + fi; + - PYLIB=${PYLIB:-dev} + - echo $PYLIB + + - if [ -z "$ADVANCED" ]; then + if echo "$CI_COMMIT_REF_NAME" | grep -c "^f-" ; then + echo "Check if advanced user tools have branch $CI_COMMIT_REF_NAME" ; + if wget https://gitlab.indiscale.com/api/v4/projects/104/repository/branches/${CI_COMMIT_REF_NAME} ; then + ADVANCED=$CI_COMMIT_REF_NAME ; + fi; + fi; + fi; + - ADVANCED=${ADVANCED:-dev} + - echo $ADVANCED + + - docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY + # use here general latest or specific branch latest... + - docker build + --build-arg PYLIB=${PYLIB} + --build-arg ADVANCED=${ADVANCED:dev} + --file .docker/Dockerfile + -t $CI_REGISTRY_IMAGE . + - docker push $CI_REGISTRY_IMAGE + - docker save $CI_REGISTRY_IMAGE > /image-cache/linkahead-python-package-template-testenv-${CI_COMMIT_REF_NAME}.tar + +cert: + tags: [docker] + stage: cert + image: $CI_REGISTRY_IMAGE + needs: + - job: build-testenv + optional: true + artifacts: + paths: + - .docker/cert/ + expire_in: 1 week + script: + - cd .docker + - CAOSHOSTNAME=caosdb-server ./cert.sh + +code-style: + tags: [docker] + stage: style + image: $CI_REGISTRY_IMAGE + needs: + - job: build-testenv + optional: true + script: + - autopep8 -r --diff --exit-code . + allow_failure: true + +pylint: + tags: [docker] + stage: style + image: $CI_REGISTRY_IMAGE + needs: + - job: build-testenv + optional: true + allow_failure: true + script: + - pylint --unsafe-load-any-extension=y -d all -e E,F src/linkahead_python_package_template + +# Build the sphinx documentation and make it ready for deployment by Gitlab Pages +# Special job for serving a static website. See https://docs.gitlab.com/ee/ci/yaml/README.html#pages +# Based on: https://gitlab.indiscale.com/caosdb/src/caosdb-pylib/-/ci/editor?branch_name=main +pages_prepare: &pages_prepare + tags: [ cached-dind ] + stage: deploy + needs: + - job: build-testenv + image: $CI_REGISTRY_IMAGE + only: + refs: + - /^release-.*$/i + script: + - echo "Deploying documentation" + - make doc + - cp -r build/doc/html public + artifacts: + paths: + - public +pages: + <<: *pages_prepare + only: + refs: + - main diff --git a/README.md b/README.md index b2fead9206f770cf9d897cc454582e3ab99aa1be..4e457007fa2dc3a3b3d241183ef7d2e7e5b69e79 100644 --- a/README.md +++ b/README.md @@ -46,3 +46,5 @@ For linting and code-style we additionally require For building the documentation we require - `sphinx` +- `recommonmark` +- `sphinx-rtd-theme`