diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml new file mode 100644 index 0000000000000000000000000000000000000000..27dd8f427fc9267ffe317737201af680b41e40ea --- /dev/null +++ b/.gitlab-ci.yml @@ -0,0 +1,184 @@ +# +# This file is a part of the CaosDB Project. +# +# Copyright (C) 2018 Research Group Biomedical Physics, +# Max-Planck-Institute for Dynamics and Self-Organization Göttingen +# Copyright (C) 2019 Henrik tom Wörden +# Copyright (C) 2021 Timm Fitschen <t.fitschen@indiscale.com> +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see <https://www.gnu.org/licenses/>. + +variables: + OCTAVE_REGISTRY_IMAGE: $CI_REGISTRY_IMAGE/testenv:$CI_COMMIT_REF_NAME + # this is the image where all the docker containers (server, mysql and test + # suite) are started + CI_REGISTRY_IMAGE_BASE: $CI_REGISTRY/caosdb/src/caosdb-pyinttest/base:latest + + # this is the image where the octavelib is installed and where the testsuite runs + OCTAVEINTTEST_REGISTRY_IMAGE: $CI_REGISTRY/caosdb/src/caosdb-octaveinttest/testenv:$CI_COMMIT_REF_NAME + # this is the name of the image in the /image-cache/ directory + OCTAVEINTTEST_IMAGE_CACHE: caosdb-ocataveinttest-testenv:$CI_COMMIT_REF_NAME.tar + + ## FOR DEBUGGING + TRIGGERED_BY_REPO: OCTAVEINTTEST + TRIGGERED_BY_REF: $CI_COMMIT_REF_NAME + TRIGGERED_BY_HASH: $CI_COMMIT_SHORT_SHA + + + # This is the caosdb-deploy branch which build the images for caosdb-server and caosdb-mysql + # TODO change to dev after merge of f-grpc to dev + DEPLOY_REF: f-grpc + DEFAULT_CAOSDB_TAG: f-grpc + +stages: + - info + - setup + - build + - test + +info: + tags: [cached-dind] + image: docker:20.10 + stage: info + needs: [] + script: + - echo "Pipeline triggered by $TRIGGERED_BY_REPO@$TRIGGERED_BY_REF ($TRIGGERED_BY_HASH)" + - echo "OCTAVELIB_REGISTRY_IMAGE = $OCTAVELIB_REGISTRY_IMAGE" + - echo "CI_REGISTRY_IMAGE_BASE = $CI_REGISTRY_IMAGE_BASE" + - echo "OCTAVEINTTEST_REGISTRY_IMAGE = $OCTAVEINTTEST_REGISTRY_IMAGE" + - echo "OCTAVEINTTEST_IMAGE_CACHE = $OCTAVEINTTEST_IMAGE_CACHE" + - echo "CAOSDB_TAG = $CAOSDB_TAG" + - echo "REFTAG = $REFTAG" + - echo "F_BRANCH = $F_BRANCH" + - ls -lah /image-cache/ + + +# Build a docker image in which tests for this repository can run +build-testenv: &build-testenv + tags: [cached-dind] + image: docker:20.10.6 + stage: setup + timeout: 2h + needs: [] + script: + - docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY + + #build testenv image + - docker pull $OCTAVELIB_REGISTRY_IMAGE || true # remove??? docker build --pull??? + - docker build + --build-arg OCTAVELIB_REGISTRY_IMAGE=$OCTAVELIB_REGISTRY_IMAGE + --file .docker/Dockerfile + --pull + --cache-from $OCTAVEINTTEST_REGISTRY_IMAGE + --tag $OCTAVEINTTEST_REGISTRY_IMAGE . + - docker push $OCTAVEINTTEST_REGISTRY_IMAGE + - docker save $OCTAVEINTTEST_REGISTRY_IMAGE > /image-cache/${OCTAVEINTTEST_IMAGE_CACHE} + + +test: + tags: [docker] + services: + - docker:20.10.3-dind + variables: + # This is a workaround for the gitlab-runner health check mechanism when + # using docker-dind service. The runner will otherwise guess the port + # wrong and the health check will timeout. + SERVICE_PORT_2376_TCP_PORT: 2375 + stage: test + image: $CI_REGISTRY_IMAGE_BASE + needs: + - job: cert + artifacts: true + - job: build-test + script: + - echo $PWD + - ls -la + + - docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY + + - if [[ "$CAOSDB_TAG" == "" ]]; then + if echo "$F_BRANCH" | grep -c "^f-" ; then + CAOSDB_TAG=${DEPLOY_REF}_F_${F_BRANCH}; + docker pull $CI_REGISTRY/caosdb/src/caosdb-deploy:$CAOSDB_TAG || CAOSDB_TAG="${DEFAULT_CAOSDB_TAG}_F_dev" ; + elif [ "$F_BRANCH" == "main" ] ; then + CAOSDB_TAG=${DEFAULT_CAOSDB_TAG}_F_main ; + else + CAOSDB_TAG="${DEFAULT_CAOSDB_TAG}_F_dev" ; + fi; + fi + - echo "F_BRANCH = $F_BRANCH" + - echo "CAOSDB_TAG = $CAOSDB_TAG" + - echo "DEPLOY_REF = $DEPLOY_REF" + - docker load < /image-cache/caosdb-${F_BRANCH}.tar || true + - docker pull $CI_REGISTRY/caosdb/src/caosdb-deploy:$CAOSDB_TAG || CAOSDB_TAG=${DEFAULT_CAOSDB_TAG}_F_dev ; + - echo "CAOSDB_TAG = $CAOSDB_TAG" + + - docker load < /image-cache/${OCTAVEINTTEST_IMAGE_CACHE} || true + - docker pull $OCTAVEINTTEST_REGISTRY_IMAGE + + - docker load < /image-cache/mariadb.tar || true + # load any(!!!) mariab image + - cd .docker + - ls -la + - ls -la cert + + # here the server and the mysql backend docker are being started + - CAOSDB_TAG=$CAOSDB_TAG docker-compose up -d + # store versions of CaosDB parts + - docker exec -u 0 -t docker_caosdb-server_1 cat /opt/caosdb/git/caosdb_server_commit | tee hash_server + - docker exec -u 0 -t docker_caosdb-server_1 cat /opt/caosdb/git/caosdb_mysqlbackend_commit | tee hash_mysql + - docker exec -u 0 -t docker_caosdb-server_1 cat /opt/caosdb/git/caosdb_proto_commit | tee hash_proto + + # the pyinttest docker writes the return value of the tests into the + # file result + - /bin/sh ./run.sh + + # collect log files + - docker logs docker_caosdb-server_1 &> ../caosdb_log.txt + - docker logs docker_sqldb_1 &> ../mariadb_log.txt + + # shut down + - CAOSDB_TAG=$CAOSDB_TAG docker-compose -f docker-compose.yml down + + # collect results + - rc=`cat result` + + # renew mariadb image in the cache (why? to renew the timestamp???) + - echo mariadb:$(docker image ls mariadb | grep mariadb | awk '{print $2}') + - docker save mariadb:$(docker image ls mariadb | grep mariadb | awk '{print $2}') > /image-cache/mariadb.tar || true + + - cat ../caosdb_log.txt + + # exit with fail when errors exist + - exit $rc + timeout: 3h + artifacts: + when: on_failure + paths: + - caosdb_log.txt + - mariadb_log.txt + - .docker/hash_* + expire_in: 1 week + +cert: + tags: [docker] + stage: build + image: $OCTAVEINTTEST_REGISTRY_IMAGE + artifacts: + paths: + - .docker/cert/ + expire_in: 1 days + script: + - cd .docker + - CAOSHOSTNAME=caosdb-server ./cert.sh