diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml new file mode 100644 index 0000000000000000000000000000000000000000..be73ae362e2f2041ba8219d4632d43b42b663108 --- /dev/null +++ b/.gitlab-ci.yml @@ -0,0 +1,447 @@ +# This file is a part of the CaosDB Project. +# +# Copyright (C) 2022 Indiscale GmbH <info@indiscale.com> +# Copyright (C) 2019 Henrik tom Wörden +# Copyright (C) 2022 Daniel Hornung <d.hornung@indiscale.com> +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see <https://www.gnu.org/licenses/>. + +variables: + CI_REGISTRY_IMAGE: $CI_REGISTRY/caosdb/src/caosdb-docker + CI_ENVIMAGE: $CI_REGISTRY/caosdb/src/caosdb-deploy/linkahead-buildenv + CI_BASE_IMAGE: $CI_REGISTRY/caosdb/src/caosdb-deploy/caosdb-run-base + # With dind the docker daemon is available on the network. + # Set versions here if you want to have non standard commits/branches, + # Also set fixed versions for releases. + + # For storing cross-job values + DOTENV: custom.env + + # Check out git submodules at start. + GIT_SUBMODULE_STRATEGY: normal + + TRIGGERED_BY_REPO: DEPLOY + TRIGGERED_BY_HASH: $CI_COMMIT_SHORT_SHA + TRIGGERED_BY_REF: $CI_COMMIT_REF_NAME + +image: docker:20.10.6 + +workflow: + rules: + - if: $CI_PIPELINE_SOURCE != "merge_request_event" + +stages: + - info + - setup + - cert + - unittest + - build + - test + - deploy + +.env: &env + - echo "Pipeline triggered by $TRIGGERED_BY_REPO@$TRIGGERED_BY_REF ($TRIGGERED_BY_HASH)" + - PYLIB=${TMP_PYLIB:-$PYLIB} + - PYINT=${TMP_PYINT:-$PYINT} + - CPPINT=${TMP_CPPINT:-$CPPINT} + - CPPLIB=${TMP_CPPLIB:-$CPPLIB} + - SERVER=${TMP_SERVER:-$SERVER} + - WEBUI=${TMP_WEBUI:-$WEBUI} + - MYSQLBACKEND=${TMP_MYSQLBACKEND:-$MYSQLBACKEND} + - ADVANCEDUSERTOOLS=${TMP_ADVANCEDUSERTOOLS:-$ADVANCEDUSERTOOLS} + - echo "F_BRANCH = $F_BRANCH" + - echo "SERVER = $SERVER" + - echo "MYSQLBACKEND = $MYSQLBACKEND" + - echo "CPPINT = $CPPINT" + - echo "CPPLIB = $CPPLIB" + - echo "PYINT = $PYINT" + - echo "PYLIB = $PYLIB" + - echo "ADVANCEDUSERTOOLS = $ADVANCEDUSERTOOLS" + + + + +info: + tags: [ cached-dind ] + stage: info + needs: [] + script: + - *env + +# ## Overview of setup-env ## +# +# Unless explicitly specified otherwise with variables, we want to test +# - dev branches with dev branches (in all repos) +# - main branches with main branches (in all repos) +# - f-branches with f-branches (in all repos) +# +# After this job has run, the dotenv will be set with the following variables: +# - DEFAULT_BRANCH: +# "main" (mainly if everything is on main) or "dev" (else) +# - F_BRANCH: +# "main" for main or release branches, "f-foo" if the branch is a real f-branch, unset else. +# - REFTAG: +# A long string identifying all the dependent repository branches. See below for details. +# - others: +# Variables for dependent repository references, e.g. "SERVER", "WEBUI", ... +# +# See [README_PIPELINE.md](./README_PIPELINE.md) for more details. +# +# See the Gitlab dotenv documentation for more details: +# https://docs.gitlab.com/ee/ci/variables/#pass-an-environment-variable-to-another-job +# +# ## Details ## +# +# If the f-branch does not exist or does not start with "f-", the fallback +# should be dev. +# +# Special case 1: If this is dev, but the F_BRANCH is main, all other repos +# should also be in main. +# +# Special case 2: If the f-branch starts with "release-", F_BRANCH is set to +# "main". +# +# The REFTAG should subsequently identify all used repositories. +# E.g. +# * REFTAG=dev means, everything is in dev +# * REFTAG=dev_F_f-blabla means, deploy is in dev, everything else is in +# f-blabla (with fallback to dev if f-blabla does not exist). +# * REFTAG=dev_F_main means, deploy is in dev, everything else is in main +# * REFTAG=dev_F_f-blabla_S_special-branch means, deploy is in dev, everything +# else f-blabla (with fallback to dev...) only caosdb-server is in +# "special-branch". +# * REFTAG=f-deploy means, everything is in f-deploy (with fallback to dev) +# +# First: determine REFTAG, DEFAULT_BRANCH and F_BRANCH applying the +# following algorithm: +# +# if deploy is in main branch: +# if F_BRANCH == dev: +# F_BRANCH= +# REFTAG=main_F_dev +# DEFAULT_BRANCH=dev +# elif F_BRANCH starts with "f-": +# REFTAG=main_F_f-blabla +# DEFAULT_BRANCH=dev +# else: +# F_BRANCH= +# REFTAG=main +# DEFAULT_BRANCH=main +# else: +# if F_BRANCH == main: +# F_BRANCH= +# REFTAG=dev_F_main +# DEFAULT_BRANCH=main +# elif F_BRANCH starts with "f-": +# REFTAG=dev_F_f-blabla +# DEFAULT_BRANCH=dev +# else: +# F_BRANCH= +# REFTAG=dev +# DEFAULT_BRANCH=dev +# +# Second: determine for every repository R the correct branch and extend the +# REFTAG applying the following algorithm: +# +# if $REPO_BRANCH is not empty: +# REFTAG="${REFTAG}_${R}_${REPO_BRANCH} +# else: +# if $F_BRANCH is not empty and $F_BRANCH exists for repo R: +# +setup-env: + stage: setup + tags: [ cached-dind ] + needs: [ ] + image: python:3 + script: + # Calculate the values first. + - *env + - pip install requests + - if [ "${CI_COMMIT_REF_NAME:0:8}" == release- ] ; then + echo "Release branch detected, setting default F_BRANCH to 'main'."; + F_BRANCH="${F_BRANCH:-main}"; + fi + - F_BRANCH=${F_BRANCH:-$CI_COMMIT_REF_NAME} + - if [ "$F_BRANCH" == "main" ] && [ "$CI_COMMIT_REF_NAME" != "main" ] ; then + DEFAULT_BRANCH=main + F_BRANCH=; + REFTAG="${CI_COMMIT_REF_NAME}_F_main"; + elif [ "$F_BRANCH" != "$CI_COMMIT_REF_NAME" ] && echo "$F_BRANCH" | grep -c "^f-" ; then + REFTAG="${CI_COMMIT_REF_NAME}_F_${F_BRANCH}"; + elif echo "$CI_COMMIT_REF_NAME" | grep -c "^f-" ; then + F_BRANCH="${CI_COMMIT_REF_NAME}"; + REFTAG="${CI_COMMIT_REF_NAME}"; + else + F_BRANCH=; + REFTAG="${CI_COMMIT_REF_NAME}"; + fi + + - if [ "${CI_COMMIT_REF_NAME}" == "main" ]; then + DEFAULT_BRANCH=main; + fi + - DEFAULT_BRANCH=${DEFAULT_BRANCH:-dev} + # if this is main, default to other main branches. + + + - echo "DEFAULT_BRANCH = $DEFAULT_BRANCH" + - echo "CI_COMMIT_REF_NAME = $CI_COMMIT_REF_NAME" + - echo "REFTAG = $REFTAG" + - echo "F_BRANCH = $F_BRANCH" + + # determine branch for mysql backend (specific branch, f-branch or dev) + - echo "MYSQLBACKEND REF = $MYSQLBACKEND" + - if [ "$MYSQLBACKEND" == "" ] && [ "$F_BRANCH" != "" ] && ./utils/branch_exists.py MYSQLBACKEND "$F_BRANCH" ; then + MYSQLBACKEND="$F_BRANCH"; + fi + - MYSQLBACKEND=${MYSQLBACKEND:-$DEFAULT_BRANCH} + - if [ "$MYSQLBACKEND" != "$DEFAULT_BRANCH" ] && [ "$MYSQLBACKEND" != "$F_BRANCH" ] ; then + REFTAG="${REFTAG}_M_${MYSQLBACKEND}"; + fi + - echo "MYSQLBACKEND REF = $MYSQLBACKEND" + + # convert branch to commit + - MYSQLBACKEND=`./utils/ref_to_commit.py MYSQLBACKEND $MYSQLBACKEND`; + - echo "MYSQLBACKEND COMMIT = $MYSQLBACKEND" + + + # determine branch for caosdb-server (specific branch, f-branch or dev) + - echo "SERVER REF = $SERVER" + - if [ "$SERVER" == "" ] && [ "$F_BRANCH" != "" ] && ./utils/branch_exists.py SERVER "$F_BRANCH" ; then + SERVER="$F_BRANCH"; + fi + - SERVER=${SERVER:-$DEFAULT_BRANCH} + - if [ "$SERVER" != "$DEFAULT_BRANCH" ] && [ "$SERVER" != "$F_BRANCH" ] ; then + REFTAG="${REFTAG}_S_${SERVER}"; + fi + - echo "SERVER REF = $SERVER" + + # convert branch to commit + - SERVER=`./utils/ref_to_commit.py SERVER $SERVER`; + - echo "SERVER COMMIT = $SERVER" + + + # determine branch for caosdb-webui (specific branch, f-branch or dev) + - echo "WEBUI REF = $WEBUI" + - if [ "$WEBUI" == "" ] && [ "$F_BRANCH" != "" ] && ./utils/branch_exists.py WEBUI "$F_BRANCH" ; then + WEBUI="$F_BRANCH"; + fi + - WEBUI=${WEBUI:-$DEFAULT_BRANCH} + - if [ "$WEBUI" != "$DEFAULT_BRANCH" ] && [ "$WEBUI" != "$F_BRANCH" ] ; then + REFTAG="${REFTAG}_W_${WEBUI}"; + fi + - echo "WEBUI REF = $WEBUI" + + # convert branch to commit + - WEBUI=`./utils/ref_to_commit.py WEBUI $WEBUI`; + - echo "WEBUI COMMIT = $WEBUI" + + + # determine branch for caosdb-pylib (specific branch, f-branch or dev) + - echo "PYLIB REF = $PYLIB" + - if [ "$PYLIB" == "" ] && [ "$F_BRANCH" != "" ] && ./utils/branch_exists.py PYLIB "$F_BRANCH" ; then + PYLIB="$F_BRANCH"; + fi + - PYLIB=${PYLIB:-$DEFAULT_BRANCH} + - if [ "$PYLIB" != "$DEFAULT_BRANCH" ] && [ "$PYLIB" != "$F_BRANCH" ] ; then + REFTAG="${REFTAG}_P_${PYLIB}"; + fi + - echo "PYLIB REF = $PYLIB" + + # convert branch to commit + - PYLIB=`./utils/ref_to_commit.py PYLIB $PYLIB`; + - echo "PYLIB COMMIT = $PYLIB" + + + # determine branch for caosdb-advancedusertools (specific branch, f-branch or dev) + - echo "ADVANCEDUSERTOOLS REF = $ADVANCEDUSERTOOLS" + - if [ "$ADVANCEDUSERTOOLS" == "" ] && [ "$F_BRANCH" != "" ] && ./utils/branch_exists.py ADVANCEDUSERTOOLS "$F_BRANCH" ; then + ADVANCEDUSERTOOLS="$F_BRANCH"; + fi + - ADVANCEDUSERTOOLS=${ADVANCEDUSERTOOLS:-$DEFAULT_BRANCH} + - if [ "$ADVANCEDUSERTOOLS" != "$DEFAULT_BRANCH" ] && [ "$ADVANCEDUSERTOOLS" != "$F_BRANCH" ] ; then + REFTAG="${REFTAG}_PA_${ADVANCEDUSERTOOLS}"; + fi + - echo "ADVANCEDUSERTOOLS REF = $ADVANCEDUSERTOOLS" + + # convert branch to commit + - ADVANCEDUSERTOOLS=`./utils/ref_to_commit.py ADVANCEDUSERTOOLS $ADVANCEDUSERTOOLS`; + - echo "ADVANCEDUSERTOOLS COMMIT = $ADVANCEDUSERTOOLS" + + + # determine branch for caosdb-pylib (specific branch, f-branch or dev) + - echo "CPPLIB REF = $CPPLIB" + - if [ "$CPPLIB" == "" ] && [ "$F_BRANCH" != "" ] && ./utils/branch_exists.py CPPLIB "$F_BRANCH" ; then + CPPLIB="$F_BRANCH"; + fi + - CPPLIB=${CPPLIB:-$DEFAULT_BRANCH} + - if [ "$CPPLIB" != "$DEFAULT_BRANCH" ] && [ "$CPPLIB" != "$F_BRANCH" ] ; then + REFTAG="${REFTAG}_C_${CPPLIB}"; + fi + - echo "CPPLIB REF = $CPPLIB" + + # fail if ref does not exist + - ./utils/ref_to_commit.py CPPLIB $CPPLIB + + + # determine branch for caosdb-cppinttest (specific branch, f-branch or dev) + - echo "CPPINT REF = $CPPINT" + - if [ "$CPPINT" == "" ] && [ "$F_BRANCH" != "" ] && ./utils/branch_exists.py CPPINT "$F_BRANCH" ; then + CPPINT="$F_BRANCH"; + fi + - CPPINT=${CPPINT:-$DEFAULT_BRANCH} + - if [ "$CPPINT" != "$DEFAULT_BRANCH" ] && [ "$CPPINT" != "$F_BRANCH" ] ; then + REFTAG="${REFTAG}_IC_${CPPINT}"; + fi + - echo "CPPINT REF = $CPPINT" + + + # determine branch for caosdb-pyinttest (specific branch, f-branch or dev) + - echo "PYINT REF = $PYINT" + - if [ "$PYINT" == "" ] && [ "$F_BRANCH" != "" ] && ./utils/branch_exists.py PYINT "$F_BRANCH" ; then + PYINT="$F_BRANCH"; + fi + - PYINT=${PYINT:-$DEFAULT_BRANCH} + - if [ "$PYINT" != "$DEFAULT_BRANCH" ] && [ "$PYINT" != "$F_BRANCH" ] ; then + REFTAG="${REFTAG}_IP_${PYINT}"; + fi + - echo "PYINT REF = $PYINT" + + - echo "REFTAG = $REFTAG" + + ######################################## + # Exporting values to environment file + #### + - echo "DEFAULT_BRANCH=${DEFAULT_BRANCH}" >> "$DOTENV" + - echo "F_BRANCH=${F_BRANCH}" >> "$DOTENV" + - echo "REFTAG=${REFTAG}" >> "$DOTENV" + - echo "MYSQLBACKEND=${MYSQLBACKEND}" >> "$DOTENV" + - echo "TMP_MYSQLBACKEND=${MYSQLBACKEND}" >> "$DOTENV" + - echo "SERVER=${SERVER}" >> "$DOTENV" + - echo "TMP_SERVER=${SERVER}" >> "$DOTENV" + - echo "WEBUI=${WEBUI}" >> "$DOTENV" + - echo "TMP_WEBUI=${WEBUI}" >> "$DOTENV" + - echo "PYLIB=${PYLIB}" >> "$DOTENV" + - echo "TMP_PYLIB=${PYLIB}" >> "$DOTENV" + - echo "ADVANCEDUSERTOOLS=${ADVANCEDUSERTOOLS}" >> "$DOTENV" + - echo "TMP_ADVANCEDUSERTOOLS=${ADVANCEDUSERTOOLS}" >> "$DOTENV" + - echo "CPPLIB=${CPPLIB}" >> "$DOTENV" + - echo "TMP_CPPLIB=${CPPLIB}" >> "$DOTENV" + - echo "PYINT=${PYINT}" >> "$DOTENV" + - echo "TMP_PYINT=${PYINT}" >> "$DOTENV" + - echo "CPPINT=${CPPINT}" >> "$DOTENV" + - echo "TMP_CPPINT=${CPPINT}" >> "$DOTENV" + - cat "$DOTENV" + artifacts: + reports: + dotenv: "$DOTENV" + +# is this job really necessary? for the tests +# TODO remove this job after 2023-01 if the disabling did not cause any problems +.create_cert: + tags: [docker] + stage: cert + image: "$CI_BASE_IMAGE:$CI_COMMIT_REF_NAME" + only: + - schedules + script: + - cd docker/transfer/build_docker + - CAOSHOSTNAME=caosdb-server ./cert.sh + artifacts: + paths: + - docker/transfer/build_docker/cert/ + expire_in: 1 week + +# runs the unittests, currently validation tests for the profile schema +unittest: + tags: [ cached-dind ] + image: "$CI_ENVIMAGE:$CI_COMMIT_REF_NAME" + stage: unittest + script: + - ls + #- make unittest + +# builds a docker image with the LinkAhead Server +build: + tags: [ cached-dind ] + image: "$CI_ENVIMAGE:$CI_COMMIT_REF_NAME" + stage: build + inherit: + variables: + - CI_ENVIMAGE + - CI_REGISTRY_IMAGE + - CI_COMMIT_REF_NAME + - CI_COMMIT_SHORT_SHA + - DOTENV + + script: + - *env + - pushd docker + - docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY + - echo "Attempting Docker pull for $CI_REGISTRY_IMAGE:$REFTAG ..." + - docker pull "$CI_REGISTRY_IMAGE:$REFTAG" || true + # - docker pull docker/dockerfile:experimental + - export DOCKER_BUILDKIT=1 + - docker build + --pull + --build-arg MYSQLBACKEND=$MYSQLBACKEND + --build-arg SERVER=$SERVER + --build-arg WEBUI=$WEBUI + --build-arg PYLIB=$PYLIB + --build-arg ADVANCEDUSERTOOLS=$ADVANCEDUSERTOOLS + -t "$CI_REGISTRY_IMAGE:$CI_COMMIT_SHORT_SHA" + -t "$CI_REGISTRY_IMAGE:$REFTAG" + . + - echo $CI_COMMIT_SHORT_SHA + - docker push "$CI_REGISTRY_IMAGE:$CI_COMMIT_SHORT_SHA" + - echo $REFTAG + - docker push "$CI_REGISTRY_IMAGE:$REFTAG" + - full_name=`docker inspect --format='{{index .RepoDigests 0}}' "$CI_REGISTRY_IMAGE:$REFTAG"` + - hash=`echo "$full_name" | awk -"F:" '{print substr($3,0,10)}'` + - docker tag "${CI_REGISTRY_IMAGE}:$REFTAG" "$CI_REGISTRY_IMAGE:$hash" + - docker push "$CI_REGISTRY_IMAGE:$hash" + - echo "Saving Docker image $CI_REGISTRY_IMAGE:$REFTAG ..." + - docker save "$CI_REGISTRY_IMAGE:$REFTAG" > /image-cache/caosdb-${REFTAG}.tar + - CAOSDB_TAG="$hash" + - echo "CAOSDB_TAG $CAOSDB_TAG" + - popd + + # Re-exporting values to dotenv (dotenvs are not cumulative, latest one wins) + - echo "CAOSDB_TAG=$CAOSDB_TAG" >> "$DOTENV" + - echo "TMP_CAOSDB_TAG=$CAOSDB_TAG" >> "$DOTENV" + - echo "DEFAULT_BRANCH=${DEFAULT_BRANCH}" >> "$DOTENV" + - echo "F_BRANCH=${F_BRANCH}" >> "$DOTENV" + - echo "REFTAG=${REFTAG}" >> "$DOTENV" + - echo "MYSQLBACKEND=${MYSQLBACKEND}" >> "$DOTENV" + - echo "TMP_MYSQLBACKEND=${MYSQLBACKEND}" >> "$DOTENV" + - echo "SERVER=${SERVER}" >> "$DOTENV" + - echo "TMP_SERVER=${SERVER}" >> "$DOTENV" + - echo "WEBUI=${WEBUI}" >> "$DOTENV" + - echo "TMP_WEBUI=${WEBUI}" >> "$DOTENV" + - echo "PYLIB=${PYLIB}" >> "$DOTENV" + - echo "TMP_PYLIB=${PYLIB}" >> "$DOTENV" + - echo "ADVANCEDUSERTOOLS=${ADVANCEDUSERTOOLS}" >> "$DOTENV" + - echo "TMP_ADVANCEDUSERTOOLS=${ADVANCEDUSERTOOLS}" >> "$DOTENV" + - echo "CPPLIB=${CPPLIB}" >> "$DOTENV" + - echo "TMP_CPPLIB=${CPPLIB}" >> "$DOTENV" + - echo "PYINT=${PYINT}" >> "$DOTENV" + - echo "TMP_PYINT=${PYINT}" >> "$DOTENV" + - echo "CPPINT=${CPPINT}" >> "$DOTENV" + - echo "TMP_CPPINT=${CPPINT}" >> "$DOTENV" + + - cat "$DOTENV" + + artifacts: + reports: + dotenv: "$DOTENV" + expire_in: 1 week diff --git a/docker/Dockerfile b/docker/Dockerfile new file mode 100644 index 0000000000000000000000000000000000000000..726141ed117e8382ae5dfc50d300203ab059a916 --- /dev/null +++ b/docker/Dockerfile @@ -0,0 +1,232 @@ +# syntax=docker/dockerfile:experimental + +####################################### +###### Get the sources from git ####### +####################################### +FROM debian:bullseye as git_base + +# Check for availability of DNS +RUN if getent hosts indiscale.com > /dev/null; \ + then echo "Connected to the internet and DNS available"; \ + else echo "No internet connection or DNS not available"; \ + fi +RUN apt-get update && apt-get install -y \ + git +RUN mkdir -p /opt/caosdb/git +WORKDIR /opt/caosdb/git + +##### caosdb-mysqlbackend ##### +FROM git_base as git_mysql +ARG MYSQLBACKEND="dev" +ARG REM_MYSQLBACKEND="https://gitlab.indiscale.com/caosdb/src/caosdb-mysqlbackend.git" +ENV REF_MYSQLBACKEND=$MYSQLBACKEND +ENV PROJECT_NO=101 +ENV REM_GITLAB_API="https://gitlab.indiscale.com/api/v4/projects/$PROJECT_NO/repository/commits/" +# Use the contents of .../repository/commits/dev as a condition whether to use the cache or not. +ADD ${REM_GITLAB_API}${MYSQLBACKEND} mysql_version.json +RUN git clone "$REM_MYSQLBACKEND" \ + && cd caosdb-mysqlbackend \ + && git checkout "$REF_MYSQLBACKEND" + +# Store hash of HEAD commit: +RUN git -C caosdb-mysqlbackend rev-parse HEAD >> caosdb_mysqlbackend_commit + +# Remove git objects +RUN rm -r caosdb-mysqlbackend/.git + +##### caosdb-webui ##### +FROM git_base as git_webui +ARG WEBUI="dev" +ARG REM_WEBUI="https://gitlab.indiscale.com/caosdb/src/caosdb-webui.git" +ENV REF_WEBUI=$WEBUI +ENV PROJECT_NO=98 +ENV REM_GITLAB_API="https://gitlab.indiscale.com/api/v4/projects/$PROJECT_NO/repository/commits/" +ADD ${REM_GITLAB_API}${WEBUI} webui_version.json + +RUN mkdir caosdb-server && cd caosdb-server \ + && git clone "$REM_WEBUI" \ + && cd caosdb-webui && git checkout "$REF_WEBUI" + +# Store hash of HEAD commit: +RUN git -C caosdb-server/caosdb-webui rev-parse HEAD >> caosdb_webui_commit + +# Remove git objects +RUN rm -r caosdb-server/caosdb-webui/.git + +##### caosdb-pylib ##### +FROM git_base as git_python +ARG PYLIB="dev" +ARG REM_PYLIB="https://gitlab.indiscale.com/caosdb/src/caosdb-pylib.git" +ENV REF_PYLIB=$PYLIB +ENV PROJECT_NO=97 +ENV REM_GITLAB_API="https://gitlab.indiscale.com/api/v4/projects/$PROJECT_NO/repository/commits/" +ADD ${REM_GITLAB_API}${PYLIB} pylib_version.json +# clone the required repos +RUN git clone "$REM_PYLIB" \ + && cd caosdb-pylib \ + && git checkout "$REF_PYLIB" + +# Store hash of HEAD commit: +RUN git -C caosdb-pylib rev-parse HEAD >> caosdb_pylib_commit + +# Remove git objects +RUN rm -r caosdb-pylib/.git + +##### caosdb-advanced-user-tools ##### +FROM git_base as git_advancedtools +ARG ADVANCEDUSERTOOLS="dev" +ARG REM_ADVANCEDUSERTOOLS="https://gitlab.indiscale.com/caosdb/src/caosdb-advanced-user-tools.git" +ENV REF_ADVANCEDUSERTOOLS=$ADVANCEDUSERTOOLS +ENV PROJECT_NO=104 +ENV REM_GITLAB_API="https://gitlab.indiscale.com/api/v4/projects/$PROJECT_NO/repository/commits/" +ADD ${REM_GITLAB_API}${ADVANCEDUSERTOOLS} advancedtools_version.json +# clone the required repos +RUN git clone "$REM_ADVANCEDUSERTOOLS" \ + && cd caosdb-advanced-user-tools \ + && git checkout "$REF_ADVANCEDUSERTOOLS" + +# Store hash of HEAD commit: +RUN git -C caosdb-advanced-user-tools rev-parse HEAD >> caosdb_advancedusertools_commit + +# Remove git objects +RUN rm -r caosdb-advanced-user-tools/.git + + +############################### +###### Java server build ###### +############################### +FROM git_base as jar + +RUN apt-get install -y \ + make \ + maven \ + openjdk-11-jdk-headless + +ARG SERVER="dev" +# Set to an empty string if the WebUI should be at the version set in the parent +ARG REM_SERVER="https://gitlab.indiscale.com/caosdb/src/caosdb-server.git" +ENV REF_SERVER=$SERVER +ENV PROJECT_NO=100 +ENV REM_GITLAB_API="https://gitlab.indiscale.com/api/v4/projects/$PROJECT_NO/repository/commits/" +ADD ${REM_GITLAB_API}${SERVER} server_version.json + +# Checkout set branch. +RUN git clone "$REM_SERVER" \ + && cd caosdb-server \ + && git checkout "$REF_SERVER" \ + # write commit hash to server.conf (which is readable by integration tests) + && echo "SERVER_COMMIT = $(git rev-parse HEAD)" >> conf/core/server.conf \ + && mkdir authtoken \ + && mkdir -p scripting/home/ + +RUN git -C caosdb-server submodule update --init caosdb-proto +RUN git -C caosdb-server/caosdb-proto rev-parse HEAD >> caosdb_proto_commit +# Remove git objects +RUN rm -r caosdb-server/caosdb-proto/.git + +RUN --mount=type=cache,target=/root/.m2,sharing=shared \ + cd /opt/caosdb/git/caosdb-server && \ + echo "Making CaosDB server jar" && \ + make jar && \ + # Copying the mvn repository cache for later use + cp -a /root/.m2 /opt/caosdb/m2 + +# Store hash of HEAD commit: +RUN git -C caosdb-server rev-parse HEAD >> caosdb_server_commit + +# Remove git objects +RUN rm -r caosdb-server/.git + +############################### +###### Main Image Build ####### +############################### +# Use Debian as parent image +FROM debian:bullseye + +# http://bugs.python.org/issue19846 +ENV LANG C.UTF-8 + +# Working directory is /opt/caosdb +WORKDIR /opt/caosdb + +# Set up the basic OS + +ENV DEBIAN_FRONTEND noninteractive +RUN apt-get update && apt-get install -y \ + libpam-ldapd \ + git \ + libhdf5-dev \ + pkgconf \ + python3-numpy \ + python3-openpyxl \ + python3-pandas \ + python3-pip \ + python3-xlrd \ + python3-yaml + +RUN pip3 install \ + jsonschema \ + pysocks \ + py3-validate-email + +RUN rm -r /var/cache/apt /var/lib/apt + +RUN mkdir -p /opt/caosdb/build_docker/ +COPY transfer/build_docker/setup_os.sh /opt/caosdb/build_docker/ +RUN --mount=type=cache,target=/var/cache/apt \ + --mount=type=cache,target=/var/lib/apt build_docker/setup_os.sh + +# Copy everything build-related there +COPY transfer/build_docker/* /opt/caosdb/build_docker/ + +RUN mkdir -p /opt/caosdb/mnt/caosroot && chmod a+w /opt/caosdb/mnt/caosroot && \ + mkdir -p /opt/caosdb/mnt/dropoffbox && \ + chmod a+w /opt/caosdb/mnt/dropoffbox && \ + mkdir -p /opt/caosdb/mnt/other && mkdir -p /opt/caosdb/mnt/extroot && \ + mkdir -p /opt/caosdb/mnt/caosdb-server && mkdir -p /opt/caosdb/mnt/tmpfiles + +COPY --from=git_python /opt/caosdb/git /opt/caosdb/git +COPY --from=git_advancedtools /opt/caosdb/git /opt/caosdb/git + +RUN cd git/caosdb-pylib \ + && echo "Installing pylib" \ + && pip3 install . + +# `bottleneck` is required for https://github.com/pandas-dev/pandas/issues/45753 +RUN cd git/caosdb-advanced-user-tools \ + && echo "Installing advanced user tools" \ + && pip3 install --upgrade numexpr \ + && pip3 install --upgrade pandas \ + && pip3 install --upgrade bottleneck \ + && pip3 install . + +COPY --from=jar /opt/caosdb/git /opt/caosdb/git +COPY --from=git_mysql /opt/caosdb/git /opt/caosdb/git +COPY --from=git_webui /opt/caosdb/git /opt/caosdb/git + +RUN build_docker/setup.sh prep_sql +RUN build_docker/setup.sh config +RUN build_docker/setup.sh make +COPY --from=jar /opt/caosdb/git/caosdb-server/target/caosdb-server.jar \ +/opt/caosdb/git/caosdb-server/target/caosdb-server.jar +COPY transfer/settings/maven_m2_settings.xml /etc/maven/settings.xml + +## Creates the TLS certificate. Note: This will then be the same certificate +## for all containers created from this image. +RUN build_docker/setup.sh cert +RUN build_docker/setup.sh user +RUN build_docker/setup.sh privilege + +# Finally: copy the runner environment +RUN mkdir /opt/caosdb/run_docker/ +COPY transfer/run_docker /opt/caosdb/run_docker/ +RUN build_docker/setup.sh make_run +RUN env > /.creation_environment + +RUN cp /opt/caosdb/run_docker/bin/* /usr/local/bin/ +RUN cp /opt/caosdb/run_docker/etc/* /etc/ # copy stuff like the /etc/nsswitch.conf + +CMD ["./run_docker/run"] +HEALTHCHECK --interval=30s --timeout=5s --start-period=1m \ + CMD curl --cacert /opt/caosdb/cert/caosdb.cert.pem \ + -I https://localhost:10443/Entities || exit 1 diff --git a/docker/transfer/build_docker/cert.sh b/docker/transfer/build_docker/cert.sh new file mode 100755 index 0000000000000000000000000000000000000000..e3cdaeae8a96925cd21dee945fcc031bd80048ef --- /dev/null +++ b/docker/transfer/build_docker/cert.sh @@ -0,0 +1,64 @@ +#!/bin/bash + +# ** header v3.0 +# This file is a part of the CaosDB Project. +# +# Copyright (C) 2019 Daniel Hornung, Göttingen +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see <https://www.gnu.org/licenses/>. +# +# ** end header + + +# Creates a directory `cert` and certificates in this directory. +# +# The hostname for which the certificate is created can be changed by setting +# the environment variable CAOSHOSTNAME. +# +# Any arguments will be passed on to `keytool`, you can use this for example for the +# `-noprompt` option. +# +# ## Overview of variables ## +# +# - CAOSHOSTNAME :: Hostname for the key (localhost) +# - KEYPW :: Password for the key (default ist CaosDBSecret) +# - KEYSTOREPW :: Password for the key store (same as KEYPW) + +function cert() { + mkdir -p cert + cd cert + KEYPW="${KEYPW:-CaosDBSecret}" + CAOSHOSTNAME="${CAOSHOSTNAME:-localhost}" + KEYSTOREPW="${KEYPW:-}" + # NOTE: KEYPW and KEYSTOREPW are the same, due to Java limitations. + KEYPW="${KEYPW}" openssl genrsa -aes256 -out caosdb.key.pem \ + -passout env:KEYPW 2048 + # Certificate is for localhost + # Some organization should be given, otherwise Firefox won't even display the certificate. + KEYPW="${KEYPW}" openssl req -new -x509 -key caosdb.key.pem \ + -out caosdb.cert.pem -passin env:KEYPW \ + -days 365 \ + -subj "/C=/ST=/L=/O=example/OU=example/CN=${CAOSHOSTNAME}" + KEYPW="${KEYPW}" KEYSTOREPW="$KEYSTOREPW" openssl pkcs12 -export \ + -inkey caosdb.key.pem -in caosdb.cert.pem -out all-certs.pkcs12 \ + -passin env:KEYPW -passout env:KEYPW + keytool -importkeystore "$@" \ + -srckeystore all-certs.pkcs12 -srcstoretype PKCS12 \ + -deststoretype pkcs12 -destkeystore caosdb.jks \ + -srcstorepass "${KEYPW}" \ + -destkeypass "${KEYPW}" -deststorepass "$KEYSTOREPW" + echo "Certificates successfuly created." +} + +cert "$@" diff --git a/docker/transfer/build_docker/create_server_conf.py b/docker/transfer/build_docker/create_server_conf.py new file mode 100755 index 0000000000000000000000000000000000000000..6b9921746cb131a20274cc34453a8a120ad3426e --- /dev/null +++ b/docker/transfer/build_docker/create_server_conf.py @@ -0,0 +1,87 @@ +#!/usr/bin/env python3 + +# ** header v3.0 +# This file is a part of the CaosDB Project. +# +# Copyright (C) 2019 Daniel Hornung, Göttingen +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see <https://www.gnu.org/licenses/>. +# +# ** end header + +import argparse +import os +import configparser + +# Defaults appropriate for a Docker image +default_settings = { + "MYSQL_USER_NAME": "caosdb", + # replace later for productive use + "MYSQL_USER_PASSWORD": "random1234", + "MYSQL_DATABASE_NAME": "caosdb", + "MYSQL_HOST": "sqldb", + + "SERVER_PORT_HTTP": "10080", + "SERVER_PORT_HTTPS": "10443", + + # Change later or create key now + "CERTIFICATES_KEY_PASSWORD": "CaosDBSecret", + "CERTIFICATES_KEY_STORE_PATH": "/opt/caosdb/cert/caosdb.jks", + # NOTE: Using the same PW for the moment, until the Java side is fixed. + # "CERTIFICATES_KEY_STORE_PASSWORD": "CaosDBKeyStore", + "CERTIFICATES_KEY_STORE_PASSWORD": "CaosDBSecret", + "CHECK_ENTITY_ACL_ROLES_MODE": "SHOULD", + + # Paths concerning file storage + "INSERT_FILES_IN_DIR_ALLOWED_DIRS": "/opt/caosdb/mnt/extroot", + "FILE_SYSTEM_ROOT": "/opt/caosdb/mnt/caosroot", + "DROP_OFF_BOX": "/opt/caosdb/mnt/dropoffbox", + "TMP_FILES": "/tmp/caosdb/tmpfiles", +} + +def _create_parser(): + """Creates the argument parser.""" + epilog =""" +Instead of command line arguments, environment variables are allowed as +well. In this case, use the UPPER_CASE version of the argument, with hyphens +(-) replaced by underscores (_). """ + parser = argparse.ArgumentParser(description='Create a config file.', + epilog=epilog) + parser.add_argument('conf_dir', type=str, help='The config directory') + parser.add_argument("-c", "--certificate", action="store_true", + help="Should certificates be created as well?") + + for setting, default in default_settings.items(): + # Order of precedence: local < environment < command line + parser.add_argument("--" + setting.lower().replace("_","-"), + dest=setting, default=os.environ.get( + setting, default)) + + return parser + + +def _create_config(args): + conf = "" + for setting, _ in default_settings.items(): + conf += "{key}={val}\n".format(key=setting, val=getattr(args, setting)) + return conf + +def main(): + parser = _create_parser() + args = parser.parse_args() + conf = _create_config(args) + print(conf) + +if __name__ == "__main__": + main() diff --git a/docker/transfer/build_docker/setup.sh b/docker/transfer/build_docker/setup.sh new file mode 100755 index 0000000000000000000000000000000000000000..f945cbf2b87e76f4e0d1d216efa0353fb3fc3a2a --- /dev/null +++ b/docker/transfer/build_docker/setup.sh @@ -0,0 +1,147 @@ +#!/bin/bash + +# ** header v3.0 +# This file is a part of the CaosDB Project. +# +# Copyright (C) 2019 Daniel Hornung, Göttingen +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see <https://www.gnu.org/licenses/>. +# +# ** end header + +# To fail fast, but beware https://mywiki.wooledge.org/BashFAQ/105 +set -e + +############# Remotes ################# +REM_MYSQLBACKEND="https://gitlab.indiscale.com/caosdb/src/caosdb-mysqlbackend.git" +REM_SERVER="https://gitlab.indiscale.com/caosdb/src/caosdb-server.git" +REM_PYLIB="https://gitlab.indiscale.com/caosdb/src/caosdb-pylib.git" +############# End of user configurables ################# + +function fail() { + echo "Some error occured, exiting." + exit 1 +} + +function make_caosdb() { + # python client + pushd git/caosdb-pylib + popd + # CaosDB itself + pushd git/caosdb-server + pushd caosdb-webui + echo "Making webui" + make + popd + pushd misc/pam_authentication/ + make + popd + popd +} + +function make_run() { + # Startup helper tools + pushd run_docker/utils + tools=" + start_nslcd + ypsetup + tzsetup + mail_setup + permission_setup + " + for tool in $tools; do + g++ -std=c++17 -o "$tool" src/"$tool".cpp \ + -lboost_system -lboost_filesystem -ldl -lstdc++fs + chown root "$tool" + chmod u+s "$tool" + done + popd + + # Modify pam files + nis_lines=" + passwd + group + shadow + " + for nis_line in $nis_lines; do + sed -i -e 's/\(^'"${nis_line}"':.*\)/\1 nis/' /etc/nsswitch.conf + done +} + +function config_caosdb() { + cd git/caosdb-server/conf/core + ../../../../build_docker/create_server_conf.py . > ../ext/server.conf + mkdir -p /tmp/caosdb/tmpfiles + mkdir -p /opt/caosdb/mnt/caosdb-server + mkdir -p /opt/caosdb/mnt/other +} + +function prepare_sql() { + cp build_docker/sql.config git/caosdb-mysqlbackend/.config +} + + +# Adds an admin user, including PAM setup in the usersources.ini +function user() { + PASSWORD="caosdb" + PASSWORDTU="caosdb" + useradd admin + useradd testuser + echo -e "${PASSWORD}\n${PASSWORD}" | passwd admin + echo -e "${PASSWORDTU}\n${PASSWORDTU}" | passwd testuser + echo "Adding CaosDB user 'admin' with password 'caosdb'." + INI_TXT=$(cat <<EOF +realms = PAM +defaultRealm = PAM +[PAM] +class = org.caosdb.server.accessControl.Pam +default_status = ACTIVE +include.user = admin +# , testuser +user.admin.roles = administration +# user.testuser.roles = administration +EOF + ) + echo "$INI_TXT" > git/caosdb-server/conf/ext/usersources.ini +} + +function cert() { + ./$(dirname $0)/cert.sh +} + +function drop_privilege() { + chmod -R a+rwX cert + cd git + chmod -R a+rwX caosdb-server caosdb-mysqlbackend + chmod -R a+rwX /tmp/caosdb/tmpfiles + #TODO should not be needed in the container + #chmod -R a+rwX /opt/caosdb/m2 + # We need this for the "PAM inside Docker" authentication + chown :shadow \ + caosdb-server/misc/pam_authentication/bin/pam_authentication + chmod g+s caosdb-server/misc/pam_authentication/bin/pam_authentication +} + +case $1 in + "os") echo "Not supported anymore, call setup_os.sh instead."; exit 1 ;; + "clone") clone_caosdb ;; + "config") config_caosdb ;; + "make") make_caosdb ;; + "make_run") make_run ;; + "cert") cert ;; + "prep_sql") prepare_sql ;; + "user") user ;; + "privilege") drop_privilege ;; + *) echo "Unknown action: $1"; exit 1 +esac diff --git a/docker/transfer/build_docker/setup_os.sh b/docker/transfer/build_docker/setup_os.sh new file mode 100755 index 0000000000000000000000000000000000000000..cc4b8e8650c3ab0c79497b2e1477262dec72f938 --- /dev/null +++ b/docker/transfer/build_docker/setup_os.sh @@ -0,0 +1,68 @@ +#!/bin/bash + +# ** header v3.0 +# This file is a part of the CaosDB Project. +# +# Copyright (C) 2019 Daniel Hornung, Göttingen +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see <https://www.gnu.org/licenses/>. +# +# ** end header + +# To fail fast, but beware https://mywiki.wooledge.org/BashFAQ/105 +set -e + +function setup_os() { + # - mariadb-client :: For SQL server configuration. + PACKAGES=" + build-essential + curl + gettext-base + ldap-utils + libboost-filesystem-dev + libboost-system-dev + libpam0g-dev + default-jre + make + mariadb-client + nis + postfix + rsync + unzip + " + # vim etc :: For debugging / development + PACKAGES+=" + bash-completion + byobu + emacs-nox + htop + procps + vim + " + export DEBIAN_FRONTEND="noninteractive" + + # Re-enable caching, since we have cache-mounted directories now: + # https://github.com/moby/buildkit/blob/master/frontend/dockerfile/docs/experimental.md + rm /etc/apt/apt.conf.d/docker-clean + + apt-get --allow-releaseinfo-change update + apt-get dist-upgrade -y + apt-get install -y $PACKAGES + + # For debugging / development + echo ". /etc/bash_completion" >> /etc/bash.bashrc +} + +# Just call the function +setup_os diff --git a/docker/transfer/build_docker/sql.config b/docker/transfer/build_docker/sql.config new file mode 100644 index 0000000000000000000000000000000000000000..83b7cd37b7fa7c18603783eb6a31c10dfe26a224 --- /dev/null +++ b/docker/transfer/build_docker/sql.config @@ -0,0 +1,45 @@ +# ** header v3.0 +# This file is a part of the CaosDB Project. +# +# Copyright (C) 2019 Daniel Hornung, Göttingen +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see <https://www.gnu.org/licenses/>. +# +# ** end header + +# Default settings for Docker image installation + +# Commands +MYSQL_CMD=/usr/bin/mysql +MYSQLADMIN_CMD=/usr/bin/mysqladmin +MYSQLDUMP_CMD=/usr/bin/mysqldump + +# MySQL Connection +MYSQL_HOST=sqldb +MYSQL_PORT=3306 +MYSQL_USER=root +MYSQL_USER_PASSWORD=caosdb1234 + +# DATABASE +DATABASE_NAME=caosdb +DATABASE_USER=caosdb +DATABASE_USER_PW=random1234 + +# A comma-separated list of hosts from which MySQL will accept logins. This +# option follows the MySQL-style for host-names and ip addresses. E.g. `%` is a +# wildcard for all hosts, so `192.168.0.%` permits logins from the local +# network. +DATABASE_USER_HOST_LIST=%, + + diff --git a/docker/transfer/run_docker/bin/backup b/docker/transfer/run_docker/bin/backup new file mode 100755 index 0000000000000000000000000000000000000000..87a68da914113483f3802400ea64f56a5c14cc66 --- /dev/null +++ b/docker/transfer/run_docker/bin/backup @@ -0,0 +1,3 @@ +#!/bin/bash + +/opt/caosdb/git/caosdb-mysqlbackend/utils/backup.py "$@" diff --git a/docker/transfer/run_docker/bin/cert_gen b/docker/transfer/run_docker/bin/cert_gen new file mode 100755 index 0000000000000000000000000000000000000000..d8a8fee7a6b95a6bd6949e6f9b2a95fb465cb17f --- /dev/null +++ b/docker/transfer/run_docker/bin/cert_gen @@ -0,0 +1,60 @@ +#!/usr/bin/env python3 + +# This file is a part of the CaosDB Project. +# +# Copyright (C) 2020 IndiScale GmbH <www.indiscale.com> +# Copyright (C) 2020 Daniel Hornung <d.hornung@indiscale.com> +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see <https://www.gnu.org/licenses/>. + +"""Generate a new SSL certificate.""" + +import argparse +import os +import subprocess + + +def _get_cert_dir(): + """Get the `cert` dir where certificates are stored.""" + return "/opt/caosdb/cert/" + + +def _get_cert_script(): + """Get the path to the certificate generation script.""" + return "/opt/caosdb/build_docker/cert.sh" + + +def _parse_arguments(): + """Parse the arguments.""" + parser = argparse.ArgumentParser(description='Generate a new TLS key.') + # No args at the moment. + parser.add_argument('-n', '--hostname', help="The hostname, default is localhost.", + default="localhost") + + return parser.parse_args() + + +def main(): + """The main function of this script.""" + args = _parse_arguments() + cert_dir = _get_cert_dir() + cert_script = _get_cert_script() + env = {"CAOSHOSTNAME": args.hostname} + subprocess.run([cert_script, "-noprompt"], + cwd=os.path.dirname(os.path.abspath(cert_dir)), check=True, + env=env) + + +if __name__ == "__main__": + main() diff --git a/docker/transfer/run_docker/bin/check_mail b/docker/transfer/run_docker/bin/check_mail new file mode 100755 index 0000000000000000000000000000000000000000..f34aab0f82d85a47991eb4e03d5ea42e1fb89f93 --- /dev/null +++ b/docker/transfer/run_docker/bin/check_mail @@ -0,0 +1,3 @@ +#!/bin/bash + +echo -e "[TESTMAIL]\nTest Mail due to Health Check" | $SENDMAIL -f testuser admin@caosdb diff --git a/docker/transfer/run_docker/bin/mysqllog b/docker/transfer/run_docker/bin/mysqllog new file mode 100755 index 0000000000000000000000000000000000000000..6cf1c6d354b70ce59f9ed53adb3d14497d2388b9 --- /dev/null +++ b/docker/transfer/run_docker/bin/mysqllog @@ -0,0 +1,3 @@ +#!/bin/bash + +/opt/caosdb/git/caosdb-mysqlbackend/utils/log.py "$@" diff --git a/docker/transfer/run_docker/bin/sendmail_to_file b/docker/transfer/run_docker/bin/sendmail_to_file new file mode 100755 index 0000000000000000000000000000000000000000..d6d5350a3c158c93b35648bae0b062bef78d0a19 --- /dev/null +++ b/docker/transfer/run_docker/bin/sendmail_to_file @@ -0,0 +1,39 @@ +#!/usr/bin/env python3 +""" +A dummy sendmail implementation. Instead of sending an email, it dumps the mail +text to a file with path `/tmp/mail/mail_<time>`. + +Warning: This is not a complete sendmail implementation. Only the `-f|--sender` +option and the `TO` header as a positional argument are implemented. +""" + +import argparse +import os +import sys +from datetime import datetime + +p = argparse.ArgumentParser() +p.add_argument("-f", "--sender", help="origin email address") +p.add_argument("-oi", action="store_true", help="""Ignore dots alone on lines by themselves in + incoming messages. This should be set if you are reading data + from a file. (This is the postfix option.)""") +p.add_argument("-i", action="store_true", help="""Ignore dots alone on lines by themselves in + incoming messages. This should be set if you are reading data + from a file. (This is the canonical sendmail way of -io.)""") +p.add_argument("-t", action="store_true", help="""Read message for recipients. To:, Cc:, and Bcc: + lines will be scanned for recipient addresses. The Bcc: line + will be deleted before transmission.""") +p.add_argument("to", metavar="TO", nargs="?", default=None) + +args = p.parse_args() + +text = sys.stdin.read() + +if not os.path.exists("/tmp/mail/"): + os.makedirs("/tmp/mail/") +with open("/tmp/mail/mail_"+datetime.today().isoformat(), "w") as f: + if args.sender: + f.write("From: " + args.sender + "\n") + if args.to: + f.write("To: " + args.to + "\n") + f.write(text) diff --git a/docker/transfer/run_docker/etc/nsswitch.conf b/docker/transfer/run_docker/etc/nsswitch.conf new file mode 100644 index 0000000000000000000000000000000000000000..77facac749ce020625af2134ca727d429e2134f9 --- /dev/null +++ b/docker/transfer/run_docker/etc/nsswitch.conf @@ -0,0 +1,20 @@ +# /etc/nsswitch.conf +# +# Example configuration of GNU Name Service Switch functionality. +# If you have the `glibc-doc-reference' and `info' packages installed, try: +# `info libc "Name Service Switch"' for information about this file. + +passwd: files nis ldap +group: files nis ldap +shadow: files nis ldap +gshadow: files nis ldap + +hosts: files dns +networks: files + +protocols: db files +services: db files +ethers: db files +rpc: db files + +netgroup: nis diff --git a/docker/transfer/run_docker/run b/docker/transfer/run_docker/run new file mode 100755 index 0000000000000000000000000000000000000000..536c7b5c3306a4ae8fab3b8a04e11f1aa1e7aff5 --- /dev/null +++ b/docker/transfer/run_docker/run @@ -0,0 +1,548 @@ +#!/usr/bin/env python3 + +# This file is a part of the CaosDB Project. +# +# Copyright (C) 2022 Indiscale GmbH <info@indiscale.com> +# Copyright (C) 2019-2022 Daniel Hornung <d.hornung@indiscale.com> +# Copyright (C) 2022 Timm Fitschen <t.fitschen@indiscale.com> +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see <https://www.gnu.org/licenses/>. + +""" +This script does the following: + +1) Wait for the SQL server to become available. +2) Configure the SQL database, if necessary. +3) Start the CaosDB server. + +""" + +# import configparser +import datetime +import glob +import os +import re +import subprocess +import sys +import time + +verbosity = 1 + + +def _log(msg, min_level=0, newline=True): + if min_level <= verbosity: + end = "\n" if newline else "" + print(msg, end=end, flush=True) + + +def sql_wait(sql_path, timeout=300, sleep=3): + """Waits for the SQL server to become ready. + + Arguments + --------- + sql_path: str + The path to the caosdb-mysql repository, where the makefile is located. + + timeout: int, optional + How long (in seconds) to wait for the SQL server to become available + before giving up. Default is 300. + + sleep: int, optional + How long (in seconds) to wait before retrying to connect to the SQL + server. Default is 3. + """ + # with open(sql_conf_file, 'r') as f: + # conf_str = '[config]\n' + f.read() + # config = configparser.ConfigParser() + # config.read(conf_str) + # print() + # config["config"]["mysql_host"] + start = datetime.datetime.now() + + _log("Trying to reach the SQL server ", 1) + + while (datetime.datetime.now() - start) <= datetime.timedelta(0, timeout+1): + _log("Trying...", 1) + + if verbosity >= 2: + out_err = None + else: + out_err = subprocess.PIPE + result = subprocess.run(["make", "test-connection"], cwd=sql_path, + stdout=out_err, stderr=out_err) + + if result.returncode == 0: + _log("", 1) + _log("SQL server successfully reached.", 1) + + return True + time.sleep(sleep) + + _log("", 1) + _log("Timeout when trying to connect to the SQL server.", 1) + + return False + + +def sql_configure(sql_path, custom_other_path): + """Configure the database for proper use by CaosDB. + + This function uses the same calls as the sql-backend make file to configure + the database as needed. This includes inserting data if commanded and `make + install`. + + Inserting data will take place if there's an environment variable named + `SQL_RESTORE` and its value is `1`. + + """ + sql_file_legacy = os.path.join(custom_other_path, "restore.sql") + + if os.path.exists(sql_file_legacy): + raise RuntimeError("You have a `restore.sql` at the old location. " + "Please put it in a restore folder in custom and " + "have it end on 'dump.sql'.") + + if os.getenv("SQL_RESTORE") == "1": + restore_dir = os.path.join(custom_other_path, "restore") + + if not os.path.exists(restore_dir): + raise RuntimeError( + "Trying to restore, but restore directory `{}` does not " + "exist".format(restore_dir)) + + sql_file = glob.glob(os.path.join(restore_dir, "*dump.sql")) + + if len(sql_file) != 1: + raise RuntimeError( + "Could not unambiguously identify restore file.") + sql_file = os.path.join(restore_dir, sql_file[0]) + + # Can we remove this if statement? + if not os.path.exists(sql_file): + _log("Trying to restore SQL from {path}, but file does not " + "exist.".format(path=sql_file), -1) + raise FileNotFoundError( + "This really should not happen! `{}`".format(sql_file)) + + _log("Importing old data into database.", 1) + # `make _install` for minimal database content + result_install = subprocess.run(["make", "_install"], cwd=sql_path) + result = subprocess.run(["./make_db", "restore_db", sql_file], + cwd=os.path.join(sql_path, "utils")) + + if result.returncode == 0 and result_install.returncode == 0: + _log("SQL server successfully configured.", 1) + _log("Data successfully inserted into SQL server.", 1) + else: + _log("Error while configuring SQL server or restoring data.", -1) + + return False + + result = subprocess.run(["make", "install"], cwd=sql_path) + + if result.returncode != 0: + return False + + if os.getenv("ANON_ADMIN") == "1": + anonymous_result = subprocess.run( + ["./make_db", "grant-permission", "anonymous", + '[{"grant":"true","priority":"true","permission":"*"}]'], + cwd=os.path.join(sql_path, "utils")) + + if anonymous_result.returncode == 0: + _log("Anonymous user successfully granted full " + "administrative rights.", 1) + else: + _log("Error while granting administrative rights to anonymous", -1) + + return False + + _log("SQL server successfully configured.", 1) + + return True + + +def initial_setup(custom_other_path, caosdb_path): + """Performs initial setup. + + This includes: + - time zone + - NIS + - email + - network config + - permissions of directories + """ + + if os.getenv("DOCKER_TZ"): + _log("Setting time zone", 0) + os.environ["CAOSDB_CONFIG_TIMEZONE"] = os.getenv("DOCKER_TZ") + subprocess.run(["./run_docker/utils/tzsetup", os.getenv("DOCKER_TZ")], + check=True) + + if os.getenv("LOCAL_USERS") == "1" and not local_users(caosdb_path=caosdb_path): + return False + + if os.getenv("NIS_ENABLE") == "1" and not nis_enable(): + return False + + if os.getenv("LDAP_ENABLE") == "1" and not ldap_enable(): + _log("Unsuccessfully tried to start LDAP client.", 0) + return False + + if not init_mail(custom_other_path=custom_other_path): + _log("Unsuccessfully tried to set up the email.", 0) + + return False + + if os.getenv("PORT_SSL") and not _create_port_config(caosdb_path): + _log("Unsuccessfully tried to set up the networking settings.", 0) + + return False + + subprocess.run(["./run_docker/utils/permission_setup", str(os.getuid()), + str(os.getgid())], check=True) + + return True + + +def init_mail(custom_other_path): + if os.getenv("DOCKER_MAIL") == "1": + _log("Email sending: Enabled", 0) + main_cf = os.path.join(custom_other_path, "main.cf") + + if os.path.exists(main_cf): + result = subprocess.run( + ["./run_docker/utils/mail_setup", main_cf], + stdout=subprocess.PIPE, stderr=subprocess.PIPE) + print(result.stdout.decode()) + print(result.stderr.decode()) + + if not result.returncode == 0: + return False + else: + _log("No configuration file for Email given", 0) + + else: + _log("Email sending: Disabled", 0) + + return True + + +def local_users(caosdb_path): + """Enable local users. + +More specifically, this modifies the default usersources.ini to include the users found in +/etc/passwd. + +These settings can be overridden by users if they provide custom `usersources.ini` and +`global_entity_permissions.xml` config files. + +Parameters +---------- +caosdb_path : str +CaosDB's base directory. +""" + local_ini_template = """ +realms = PAM +defaultRealm = PAM +[PAM] +class = org.caosdb.server.accessControl.Pam +default_status = ACTIVE +# user.admin.roles = administration # No user roles for default PAM users +# user.testuser.roles = administration +""" + # Get minimum user ID + uid_min = 1000 + try: + pat = re.compile( + "\\s*UID_MIN\\s+(?P<uid_min>\\d+)\\s*", flags=re.ASCII) + with open("/etc/login.defs", "r") as login_defs: + for line in login_defs.readlines(): + match = pat.match(line) + if match: + uid_min = int(match.group("uid_min")) + except (OSError, ValueError) as err: + print(err) + # read user names + users = [] + with open("/etc/passwd", "r") as passwd: + pat = re.compile( + "^(?P<username>[^:]+):x:(?P<uid>\\d+):.*", flags=re.ASCII) + for line in passwd.readlines(): + match = pat.match(line) + if not match: + continue + name, uid = match.groups() + uid = int(uid) + if uid >= uid_min: + users.append(name) + # Append users to ini + local_ini = local_ini_template + \ + "include.user = {}".format(", ".join(users)) + confdir = os.path.join(caosdb_path, "conf", "ext") + os.makedirs(confdir, exist_ok=True) + with open(os.path.join(confdir, "usersources.ini"), "w") as usersources: + usersources.write(local_ini) + return True + + +def ldap_enable(): + """Enable LDAP client daemon.""" + print("Start nslcd (LDAP client)") + try: + subprocess.run(["./run_docker/utils/start_nslcd"], check=True) + except subprocess.CalledProcessError: + return False + return True + + +def nis_enable(): + """Enable NIS name lookup for PAM.""" + + # Set up files and start yp service + nis_domain = os.getenv("NIS_DOMAIN") + nis_server = os.getenv("NIS_SERVER") + print("NIS domain: {domain}\nNIS server: {server}".format( + domain=nis_domain, server=nis_server)) + try: + subprocess.run(["./run_docker/utils/ypsetup", nis_domain, nis_server], + check=True) + except subprocess.CalledProcessError: + return False + + return True + + +def caosdb_run(caosdb_path, debug=False, test=False, no_tls=False): + """Start the CaosDB server.""" + webui_path = _get_path("caosdb-webui", base=os.getcwd(), + exclude_str="/opt/caosdb/mnt") + + if not webui_path: + _log("Could not find WebUI!", -1) + sys.exit(90) + res_webui = subprocess.run(["make"], cwd=webui_path) + + if not res_webui.returncode == 0: + _log("WebUI update failed!", -1) + sys.exit(91) + + if test: + subprocess.run(["mvn", "test"], cwd=caosdb_path) + # result = subprocess.run(['mvn', 'exec:java@run',], cwd=caosdb_path) + + env = dict(os.environ) + if no_tls: + env["CAOSDB_COMMAND_LINE_OPTIONS"] = " ".join( + os.environ.get("CAOSDB_COMMAND_LINE_OPTIONS", "").split() + + ["--no-tls"]) + if debug: + result = subprocess.run( + ['make', 'run-debug-single', ], cwd=caosdb_path, env=env) + else: + result = subprocess.run(['make', 'run-single', ], cwd=caosdb_path, env=env) + + if result.returncode == 0: + _log("CaosDB terminated gracefully.", 2) + + return True + + return False + + +def _get_path(dirname, base, exclude_str=None): + """Tries to find a directory, "close" to the current working directory. + + Returns the path of said directory, if found. + + Arguments + --------- + dirname: str + The name of the directory to be found. + + base: str + Where to start searching for the directory. + + exclude_str: str, optional + If given, this string must not be part of the result. + """ + + # glob cannot exclude natively, so we need to remimplement it + def search_in_dir(dirname, base, excludes=[], depth=0, max_depth=100, + visited=set()): + + if depth == max_depth: + return None + next_level = [] + + for entry in os.scandir(base): + if not entry.is_dir(): + continue + + if entry.inode() in visited: + _log("Visited before: {}".format(os.path.abspath(entry.path))) + + continue + abspath = os.path.abspath(entry.path) + + if any([excl in abspath for excl in excludes]): + _log("Blacklisted: {}".format(abspath), 3) + + continue + + if dirname == entry.name: + return abspath + # Else, continue deeper into the directory tree + visited.add(entry.inode()) + _log("Adding for next recursion: {}".format(abspath), 3) + next_level.append(abspath) + + for next_base in next_level: + result = search_in_dir(dirname, next_base, excludes=excludes, + depth=depth+1, max_depth=max_depth, + visited=visited) + + if result: + return result + + while not base == os.path.sep: + base = os.path.split(base)[0] + path = search_in_dir(dirname, base, excludes=[exclude_str]) + + if path: + return path + + +def _create_port_config(caosdb_path): + """E.g. adds a config to set the external SSL port. + """ + port_ssl_ext = os.getenv("PORT_SSL") + network_template = """ +# Additional network settings + +# Set the same ports as from the outside +REDIRECT_HTTP_TO_HTTPS_PORT={port} +""" + confdir = os.path.join(caosdb_path, "conf", "ext", "server.conf.d") + os.makedirs(confdir, exist_ok=True) + conffile = os.path.join(confdir, "80.network_docker.conf") + with open(conffile, mode="w") as fd_conf: + fd_conf.write(network_template.format(port=port_ssl_ext)) + return True + + +def import_custom(custom_server_path, base, custom_other_path): + """Imports folders and specialized settings etc. from the `custom` directory. + """ + ################# + # Server config and extensions + caosdb_source = os.path.join(custom_server_path, "") + caosdb_target = os.path.join(base, "git", "caosdb-server") + if os.path.exists(caosdb_source): + rsync = ["rsync", "-r", caosdb_source, caosdb_target] + _log("Rsyncing caosdb: {}".format(rsync), 2) + result = subprocess.run(rsync) + if not result.returncode == 0: + _log("Rsync of caosdb failed.", 0) + return False + + ################# + # TLS certificate + cert_source = os.path.join(custom_other_path, "cert", "") + cert_target = os.path.join(base, "cert") + if os.path.exists(cert_source): + rsync = ["rsync", "-r", cert_source, cert_target] + _log("Rsyncing cert: {}".format(rsync), 2) + result = subprocess.run(rsync) + if not result.returncode == 0: + _log("Rsync of cert failed.", 0) + return False + + return True + + +def main(): + # conf_file = os.path.join(_get_path("caosdb-mysqlbackend"), ".config") + global verbosity + verbosity = 1 + base = os.getcwd() + test = (os.getenv("MAKE_TEST") == "1") + debug = (os.getenv("DEBUG") == "1") + no_tls = (os.getenv("NO_TLS") == "1") + + if debug: + verbosity += 1 + _log("Getting paths, starting at {}...".format(base), 3) + sql_path = _get_path("caosdb-mysqlbackend", base=base, + exclude_str="/opt/caosdb/mnt/" + ) + _log("sql_path: {}".format(sql_path), 3) + caosdb_path = _get_path("caosdb-server", base=base, + exclude_str="/opt/caosdb/mnt/" + ) + _log("caosdb_path: {}".format(caosdb_path), 3) + custom_other_path = os.path.join(base, "mnt", "other") + if not sql_path: + _log("SQL configuration path not found, exiting.", -1) + sys.exit(1) + + if not caosdb_path: + _log("CaosDB path not found, exiting.", -1) + sys.exit(1) + + ######################### + # Initial setup procedures + ######################### + # This also modifies configuration files according to environment variables etc. + if not initial_setup(custom_other_path=custom_other_path, + caosdb_path=caosdb_path): + _log("Problem during initial setup of system.", 0) + sys.exit(10) + + ######################### + # Wait for SQL server + ######################### + if not sql_wait(sql_path, timeout=800): + _log("SQL server not reachable, giving up.", 0) + sys.exit(20) + + ######################### + # Look for mounted overwritables + ######################### + # This should happen after any local modification of the default files so that overwriting + # manually has a higher priority. + _log("Importing external material.", 1) + custom_server_path = os.path.join(base, "mnt", "caosdb-server") + if not import_custom(custom_server_path=custom_server_path, base=base, + custom_other_path=custom_other_path): + _log("Problem while importing external material.", 0) + sys.exit(30) + ######################### + # Configure SQL server + ######################### + if not sql_configure(sql_path, custom_other_path=custom_other_path): + _log("Unsuccessfully tried to configure SQL server.", 0) + sys.exit(40) + + ######################### + # Start CaosDB server + ######################### + _log("Starting CaosDB server...", 1) + if not caosdb_run(caosdb_path, test=test, debug=debug, no_tls=no_tls): + _log("CaosDB exited unsuccessfully.", 0) + sys.exit(50) + + +if __name__ == "__main__": + main() diff --git a/docker/transfer/run_docker/utils/src/mail_setup.cpp b/docker/transfer/run_docker/utils/src/mail_setup.cpp new file mode 100644 index 0000000000000000000000000000000000000000..edb06c16b37539d3669155d66704e23a5745c2ce --- /dev/null +++ b/docker/transfer/run_docker/utils/src/mail_setup.cpp @@ -0,0 +1,53 @@ +// ** header v3.0 +// This file is a part of the CaosDB Project. +// +// Copyright (C) 2019 Daniel Hornung +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see <https://www.gnu.org/licenses/>. +// +// ** end header + +// Compile it with (the order of the cpp and the libs matters: +// g++ -std=c++17 -o mail_setup src/mail_setup.cpp -lboost_system \ +// -lboost_filesystem -ldl -lstdc++fs + +#include <experimental/filesystem> +#include <fstream> +#include <iostream> +#include <unistd.h> +#include <boost/dll.hpp> + +namespace fs = std::experimental::filesystem; + +int main(int argc, char* argv[]) { + // Get the main.cf file + if (argc != 2) { + std::cerr << "`main.cf` file required as the argument!" << std::endl; + std::cerr << "You gave " << argc - 1 << " arguments instead." << std::endl; + return 1; + } + char* main_cf = argv[1]; + + fs::copy(main_cf, "/etc/postfix/", fs::copy_options::overwrite_existing); + // Change the real user ID + setuid(0); + int result = std::system("/usr/sbin/service postfix start"); + + // We trust the first run of this program since we call it ourselves here. + // Simply delete it after running. ;-) + std::string delete_me = "/bin/rm " + boost::dll::program_location().string(); + std::cout << delete_me << std::endl; + std::system(delete_me.c_str()); + return 0; +} diff --git a/docker/transfer/run_docker/utils/src/permission_setup.cpp b/docker/transfer/run_docker/utils/src/permission_setup.cpp new file mode 100644 index 0000000000000000000000000000000000000000..4800b12cd86c77afabc80911981379bd137b96af --- /dev/null +++ b/docker/transfer/run_docker/utils/src/permission_setup.cpp @@ -0,0 +1,72 @@ +// ** header v3.0 +// This file is a part of the CaosDB Project. +// +// Copyright (C) 2019 Daniel Hornung <d.hornung@indiscale.com> +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see <https://www.gnu.org/licenses/>. +// +// ** end header + +/** + * This program fixes incorrect permissions. + */ + +#include <fstream> +#include <iostream> +#include <unistd.h> +#include <pwd.h> +#include <unistd.h> +#include <sys/stat.h> + +// Compile it with (the order of the cpp and the libs matters): +// g++ -o permission_setup src/permission_setup.cpp + +void fix_permission(const std::string& dir, uid_t uid, uid_t gid, mode_t perms); + +/** + * Positional arguments: + * - uid : The UID to set the directories to. + * - gid : The GID to set the directories to. + */ +int main(int argc, char* argv[]) { + + if (argc != 3) { + std::cerr << "Target UID and GID required as arguments!" << std::endl; + std::cerr << "You gave " << argc - 1 << " arguments instead." << std::endl; + return 1; + } + uid_t uid = std::stoul(argv[1]); + uid_t gid = std::stoul(argv[2]); + + fix_permission("/opt/caosdb/mnt/caosroot", uid, -1, 0755) ; + fix_permission("/opt/caosdb/mnt/tmpfiles", uid, -1, 0700) ; + fix_permission("/opt/caosdb/mnt/dropoffbox", uid, -1, 0733); + fix_permission("/opt/caosdb/git/caosdb-server/authtoken", -1, gid, 0770) ; + + return 0; +} + +/** + * Arguments: + * ========== + * - dir + * - uid + * - gid + * - user perms + */ +void fix_permission(const std::string& dir, uid_t uid, uid_t gid, mode_t perms) { + // all in plain C, unfortunately + chown(dir.c_str(), uid, gid); + chmod(dir.c_str(), perms); +} diff --git a/docker/transfer/run_docker/utils/src/start_nslcd.cpp b/docker/transfer/run_docker/utils/src/start_nslcd.cpp new file mode 100644 index 0000000000000000000000000000000000000000..1915b5bde6e12ff706f1e6e5ba67f77fa17c4232 --- /dev/null +++ b/docker/transfer/run_docker/utils/src/start_nslcd.cpp @@ -0,0 +1,47 @@ +// ** header v3.0 +// This file is a part of the CaosDB Project. +// +// Copyright (C) 2019 Daniel Hornung +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see <https://www.gnu.org/licenses/>. +// +// ** end header + +/** + * This program sets the time zone to the given value and deletes itself afterwards + */ + +#include <iostream> +#include <string> +#include <unistd.h> +#include <boost/dll.hpp> + +// Compile it with (the order of the cpp and the libs matters: +// g++ -o start_nslcd src/start_nslcd.cpp -lboost_system -lboost_filesystem -ldl + +int main(int argc, char* argv[]) { + + // Change the real user ID + setuid(0); + std::string start_nslcd_str = std::string("/etc/init.d/nslcd start"); + std::cout << start_nslcd_str << std::endl; + std::system(start_nslcd_str.c_str()); + + // We trust the first run of this program since we call it ourselves here. + // Simply delete it after running. ;-) + std::string delete_me = "/bin/rm " + boost::dll::program_location().string(); + std::cout << delete_me << std::endl; + std::system(delete_me.c_str()); + return 0; +} diff --git a/docker/transfer/run_docker/utils/src/tzsetup.cpp b/docker/transfer/run_docker/utils/src/tzsetup.cpp new file mode 100644 index 0000000000000000000000000000000000000000..23f7df73173e970fb2e99c73dceb709729d18632 --- /dev/null +++ b/docker/transfer/run_docker/utils/src/tzsetup.cpp @@ -0,0 +1,57 @@ +// ** header v3.0 +// This file is a part of the CaosDB Project. +// +// Copyright (C) 2019 Daniel Hornung +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see <https://www.gnu.org/licenses/>. +// +// ** end header + +/** + * This program sets the time zone to the given value and deletes itself afterwards + */ + +#include <fstream> +#include <iostream> +#include <unistd.h> +#include <boost/dll.hpp> + +// Compile it with (the order of the cpp and the libs matters: +// g++ -o tzsetup src/tzsetup.cpp -lboost_system -lboost_filesystem -ldl + +int main(int argc, char* argv[]) { + // Set up the config files. + if (argc != 2) { + std::cerr << "Timezone info required as the argument!" << std::endl; + std::cerr << "You gave " << argc - 1 << " arguments instead." << std::endl; + return 1; + } + char* timezone = argv[1]; + + // Change the real user ID + setuid(0); + + // Create symlink for timezone + std::string link_str = std::string("ln -sf /usr/share/zoneinfo/") + + timezone + " /etc/localtime"; + std::cout << link_str << std::endl; + std::system(link_str.c_str()); + + // We trust the first run of this program since we call it ourselves here. + // Simply delete it after running. ;-) + std::string delete_me = "/bin/rm " + boost::dll::program_location().string(); + std::cout << delete_me << std::endl; + std::system(delete_me.c_str()); + return 0; +} diff --git a/docker/transfer/run_docker/utils/src/ypsetup.cpp b/docker/transfer/run_docker/utils/src/ypsetup.cpp new file mode 100644 index 0000000000000000000000000000000000000000..8fa6913617018eac401f107da20d272205d6f3fe --- /dev/null +++ b/docker/transfer/run_docker/utils/src/ypsetup.cpp @@ -0,0 +1,60 @@ +// ** header v3.0 +// This file is a part of the CaosDB Project. +// +// Copyright (C) 2019 Daniel Hornung +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see <https://www.gnu.org/licenses/>. +// +// ** end header + +#include <fstream> +#include <iostream> +#include <unistd.h> +#include <boost/dll.hpp> + +// Compile it with (the order of the cpp and the libs matters: +// g++ -o ypsetup src/ypsetup.cpp -lboost_system -lboost_filesystem -ldl + +int main(int argc, char* argv[]) { + // Check number of arguments. + if (argc != 3) { + std::cerr << "Exactly 2 arguments are required:" << std::endl; + std::cerr << "ypsetup <NIS_DOMAIN> <NIS_SERVER>" << std::endl; + return 1; + } + + // Set up the config files. + char* nis_domain = argv[1]; + char* nis_server = argv[2]; + + std::ofstream ofs; + ofs.open("/etc/yp.conf", std::ofstream::trunc); + ofs << "ypserver " << nis_server << std::endl; + ofs.close(); + + ofs.open("/etc/defaultdomain", std::ofstream::trunc); + ofs << nis_domain << std::endl; + ofs.close(); + + // Change the real user ID + setuid(0); + std::system("/sbin/rpcbind"); + std::system("/usr/sbin/ypbind"); + // We trust the first run of this program since we call it ourselves here. + // Simply delete it after running. ;-) + std::string delete_me = "/bin/rm " + boost::dll::program_location().string(); + std::cout << delete_me << std::endl; + std::system(delete_me.c_str()); + return 0; +} diff --git a/docker/transfer/settings/maven_m2_settings.xml b/docker/transfer/settings/maven_m2_settings.xml new file mode 100644 index 0000000000000000000000000000000000000000..e921845068a25e3d194f400b922d08806e3de5e2 --- /dev/null +++ b/docker/transfer/settings/maven_m2_settings.xml @@ -0,0 +1,257 @@ +<?xml version="1.0" encoding="UTF-8"?> + +<!-- +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. +--> + +<!-- + | This is the configuration file for Maven. It can be specified at two levels: + | + | 1. User Level. This settings.xml file provides configuration for a single user, + | and is normally provided in ${user.home}/.m2/settings.xml. + | + | NOTE: This location can be overridden with the CLI option: + | + | -s /path/to/user/settings.xml + | + | 2. Global Level. This settings.xml file provides configuration for all Maven + | users on a machine (assuming they're all using the same Maven + | installation). It's normally provided in + | ${maven.home}/conf/settings.xml. + | + | NOTE: This location can be overridden with the CLI option: + | + | -gs /path/to/global/settings.xml + | + | The sections in this sample file are intended to give you a running start at + | getting the most out of your Maven installation. Where appropriate, the default + | values (values used when the setting is not specified) are provided. + | + |--> +<settings xmlns="http://maven.apache.org/SETTINGS/1.0.0" + xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" + xsi:schemaLocation="http://maven.apache.org/SETTINGS/1.0.0 http://maven.apache.org/xsd/settings-1.0.0.xsd"> + <!-- localRepository + | The path to the local repository maven will use to store artifacts. + | + | Default: ${user.home}/.m2/repository + --> + <localRepository>/opt/caosdb/m2/repository</localRepository> + + <!-- interactiveMode + | This will determine whether maven prompts you when it needs input. If set to false, + | maven will use a sensible default value, perhaps based on some other setting, for + | the parameter in question. + | + | Default: true + <interactiveMode>true</interactiveMode> + --> + + <!-- offline + | Determines whether maven should attempt to connect to the network when executing a build. + | This will have an effect on artifact downloads, artifact deployment, and others. + | + | Default: false + <offline>false</offline> + --> + + <!-- pluginGroups + | This is a list of additional group identifiers that will be searched when resolving plugins by their prefix, i.e. + | when invoking a command line like "mvn prefix:goal". Maven will automatically add the group identifiers + | "org.apache.maven.plugins" and "org.codehaus.mojo" if these are not already contained in the list. + |--> + <pluginGroups> + <!-- pluginGroup + | Specifies a further group identifier to use for plugin lookup. + <pluginGroup>com.your.plugins</pluginGroup> + --> + </pluginGroups> + + <!-- proxies + | This is a list of proxies which can be used on this machine to connect to the network. + | Unless otherwise specified (by system property or command-line switch), the first proxy + | specification in this list marked as active will be used. + |--> + <proxies> + <!-- proxy + | Specification for one proxy, to be used in connecting to the network. + | + <proxy> + <id>optional</id> + <active>true</active> + <protocol>http</protocol> + <username>proxyuser</username> + <password>proxypass</password> + <host>proxy.host.net</host> + <port>80</port> + <nonProxyHosts>local.net|some.host.com</nonProxyHosts> + </proxy> + --> + </proxies> + + <!-- servers + | This is a list of authentication profiles, keyed by the server-id used within the system. + | Authentication profiles can be used whenever maven must make a connection to a remote server. + |--> + <servers> + <!-- server + | Specifies the authentication information to use when connecting to a particular server, identified by + | a unique name within the system (referred to by the 'id' attribute below). + | + | NOTE: You should either specify username/password OR privateKey/passphrase, since these pairings are + | used together. + | + <server> + <id>deploymentRepo</id> + <username>repouser</username> + <password>repopwd</password> + </server> + --> + + <!-- Another sample, using keys to authenticate. + <server> + <id>siteServer</id> + <privateKey>/path/to/private/key</privateKey> + <passphrase>optional; leave empty if not used.</passphrase> + </server> + --> + </servers> + + <!-- mirrors + | This is a list of mirrors to be used in downloading artifacts from remote repositories. + | + | It works like this: a POM may declare a repository to use in resolving certain artifacts. + | However, this repository may have problems with heavy traffic at times, so people have mirrored + | it to several places. + | + | That repository definition will have a unique id, so we can create a mirror reference for that + | repository, to be used as an alternate download site. The mirror site will be the preferred + | server for that repository. + |--> + <mirrors> + <!-- mirror + | Specifies a repository mirror site to use instead of a given repository. The repository that + | this mirror serves has an ID that matches the mirrorOf element of this mirror. IDs are used + | for inheritance and direct lookup purposes, and must be unique across the set of mirrors. + | + <mirror> + <id>mirrorId</id> + <mirrorOf>repositoryId</mirrorOf> + <name>Human Readable Name for this Mirror.</name> + <url>http://my.repository.com/repo/path</url> + </mirror> + --> + </mirrors> + + <!-- profiles + | This is a list of profiles which can be activated in a variety of ways, and which can modify + | the build process. Profiles provided in the settings.xml are intended to provide local machine- + | specific paths and repository locations which allow the build to work in the local environment. + | + | For example, if you have an integration testing plugin - like cactus - that needs to know where + | your Tomcat instance is installed, you can provide a variable here such that the variable is + | dereferenced during the build process to configure the cactus plugin. + | + | As noted above, profiles can be activated in a variety of ways. One way - the activeProfiles + | section of this document (settings.xml) - will be discussed later. Another way essentially + | relies on the detection of a system property, either matching a particular value for the property, + | or merely testing its existence. Profiles can also be activated by JDK version prefix, where a + | value of '1.4' might activate a profile when the build is executed on a JDK version of '1.4.2_07'. + | Finally, the list of active profiles can be specified directly from the command line. + | + | NOTE: For profiles defined in the settings.xml, you are restricted to specifying only artifact + | repositories, plugin repositories, and free-form properties to be used as configuration + | variables for plugins in the POM. + | + |--> + <profiles> + <!-- profile + | Specifies a set of introductions to the build process, to be activated using one or more of the + | mechanisms described above. For inheritance purposes, and to activate profiles via <activatedProfiles/> + | or the command line, profiles have to have an ID that is unique. + | + | An encouraged best practice for profile identification is to use a consistent naming convention + | for profiles, such as 'env-dev', 'env-test', 'env-production', 'user-jdcasey', 'user-brett', etc. + | This will make it more intuitive to understand what the set of introduced profiles is attempting + | to accomplish, particularly when you only have a list of profile id's for debug. + | + | This profile example uses the JDK version to trigger activation, and provides a JDK-specific repo. + <profile> + <id>jdk-1.4</id> + + <activation> + <jdk>1.4</jdk> + </activation> + + <repositories> + <repository> + <id>jdk14</id> + <name>Repository for JDK 1.4 builds</name> + <url>http://www.myhost.com/maven/jdk14</url> + <layout>default</layout> + <snapshotPolicy>always</snapshotPolicy> + </repository> + </repositories> + </profile> + --> + + <!-- + | Here is another profile, activated by the system property 'target-env' with a value of 'dev', + | which provides a specific path to the Tomcat instance. To use this, your plugin configuration + | might hypothetically look like: + | + | ... + | <plugin> + | <groupId>org.myco.myplugins</groupId> + | <artifactId>myplugin</artifactId> + | + | <configuration> + | <tomcatLocation>${tomcatPath}</tomcatLocation> + | </configuration> + | </plugin> + | ... + | + | NOTE: If you just wanted to inject this configuration whenever someone set 'target-env' to + | anything, you could just leave off the <value/> inside the activation-property. + | + <profile> + <id>env-dev</id> + + <activation> + <property> + <name>target-env</name> + <value>dev</value> + </property> + </activation> + + <properties> + <tomcatPath>/path/to/tomcat/instance</tomcatPath> + </properties> + </profile> + --> + </profiles> + + <!-- activeProfiles + | List of profiles that are active for all builds. + | + <activeProfiles> + <activeProfile>alwaysActiveProfile</activeProfile> + <activeProfile>anotherAlwaysActiveProfile</activeProfile> + </activeProfiles> + --> +</settings> diff --git a/utils/ref_to_commit.py b/utils/ref_to_commit.py new file mode 100755 index 0000000000000000000000000000000000000000..dbc5ab4f4c14b7a97fba833a8fff1bdff9c2b693 --- /dev/null +++ b/utils/ref_to_commit.py @@ -0,0 +1,47 @@ +#!/usr/bin/env python3 +""" +replaces git branch names with the newest commit hash using gitlab api +""" +import argparse + +import requests + + +_REPOS = { + "SERVER": "https://gitlab.indiscale.com/api/v4/projects/100", + "WEBUI": "https://gitlab.indiscale.com/api/v4/projects/98", + "PYLIB": "https://gitlab.indiscale.com/api/v4/projects/97", + "MYSQLBACKEND": "https://gitlab.indiscale.com/api/v4/projects/101", + "PYINT": "https://gitlab.indiscale.com/api/v4/projects/99", + "CPPLIB": "https://gitlab.indiscale.com/api/v4/projects/107", + "CPPINT": "https://gitlab.indiscale.com/api/v4/projects/111", + "ADVANCEDUSERTOOLS": "https://gitlab.indiscale.com/api/v4/projects/104" +} + +def get_remote(repository): + return _REPOS[repository] + + +def ref_to_commit(repository, reference): + remote = get_remote(repository) + r = requests.get(remote+"/repository/branches/"+reference).json() + + if "name" in r: + return r["commit"]["short_id"] + + return reference + + +def define_parser(): + parser = argparse.ArgumentParser() + parser.add_argument("repository") + parser.add_argument("reference") + + return parser + + +if __name__ == "__main__": + parser = define_parser() + args = parser.parse_args() + ret = ref_to_commit(repository=args.repository, reference=args.reference) + print(ret)