PIPELINE add .gitlab-ci.yml

.gitlab-ci.yml

+#
+# This file is a part of the CaosDB Project.
+#
+# Copyright (C) 2018 Research Group Biomedical Physics,
+# Max-Planck-Institute for Dynamics and Self-Organization Göttingen
+# Copyright (C) 2019 Henrik tom Wörden
+# Copyright (C) 2021 Timm Fitschen <t.fitschen@indiscale.com>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+variables:
+  OCTAVE_REGISTRY_IMAGE: $CI_REGISTRY_IMAGE/testenv:$CI_COMMIT_REF_NAME
+  # this is the image where all the docker containers (server, mysql and test
+  # suite) are started
+  CI_REGISTRY_IMAGE_BASE: $CI_REGISTRY/caosdb/src/caosdb-pyinttest/base:latest
+
+  # this is the image where the octavelib is installed and where the testsuite runs
+  OCTAVEINTTEST_REGISTRY_IMAGE: $CI_REGISTRY/caosdb/src/caosdb-octaveinttest/testenv:$CI_COMMIT_REF_NAME
+  # this is the name of the image in the /image-cache/ directory
+  OCTAVEINTTEST_IMAGE_CACHE: caosdb-ocataveinttest-testenv:$CI_COMMIT_REF_NAME.tar
+
+  ## FOR DEBUGGING
+  TRIGGERED_BY_REPO: OCTAVEINTTEST
+  TRIGGERED_BY_REF: $CI_COMMIT_REF_NAME
+  TRIGGERED_BY_HASH: $CI_COMMIT_SHORT_SHA
+
+
+  # This is the caosdb-deploy branch which build the images for caosdb-server and caosdb-mysql
+  # TODO change to dev after merge of f-grpc to dev
+  DEPLOY_REF: f-grpc
+  DEFAULT_CAOSDB_TAG: f-grpc
+
+stages:
+  - info
+  - setup
+  - build
+  - test
+
+info:
+  tags: [cached-dind]
+  image: docker:20.10
+  stage: info
+  needs: []
+  script:
+    - echo "Pipeline triggered by $TRIGGERED_BY_REPO@$TRIGGERED_BY_REF ($TRIGGERED_BY_HASH)"
+    - echo "OCTAVELIB_REGISTRY_IMAGE = $OCTAVELIB_REGISTRY_IMAGE"
+    - echo "CI_REGISTRY_IMAGE_BASE = $CI_REGISTRY_IMAGE_BASE"
+    - echo "OCTAVEINTTEST_REGISTRY_IMAGE = $OCTAVEINTTEST_REGISTRY_IMAGE"
+    - echo "OCTAVEINTTEST_IMAGE_CACHE = $OCTAVEINTTEST_IMAGE_CACHE"
+    - echo "CAOSDB_TAG = $CAOSDB_TAG"
+    - echo "REFTAG = $REFTAG"
+    - echo "F_BRANCH = $F_BRANCH"
+    - ls -lah /image-cache/
+
+
+# Build a docker image in which tests for this repository can run
+build-testenv: &build-testenv
+  tags: [cached-dind]
+  image: docker:20.10.6
+  stage: setup
+  timeout: 2h
+  needs: []
+  script:
+    - docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY
+
+      #build testenv image
+    - docker pull $OCTAVELIB_REGISTRY_IMAGE || true # remove??? docker build --pull???
+    - docker build
+      --build-arg OCTAVELIB_REGISTRY_IMAGE=$OCTAVELIB_REGISTRY_IMAGE
+      --file .docker/Dockerfile
+      --pull
+      --cache-from $OCTAVEINTTEST_REGISTRY_IMAGE
+      --tag $OCTAVEINTTEST_REGISTRY_IMAGE .
+    - docker push $OCTAVEINTTEST_REGISTRY_IMAGE
+    - docker save $OCTAVEINTTEST_REGISTRY_IMAGE > /image-cache/${OCTAVEINTTEST_IMAGE_CACHE}
+
+
+test:
+  tags: [docker]
+  services:
+    - docker:20.10.3-dind
+  variables:
+    # This is a workaround for the gitlab-runner health check mechanism when
+    # using docker-dind service.  The runner will otherwise guess the port
+    # wrong and the health check will timeout.
+    SERVICE_PORT_2376_TCP_PORT: 2375
+  stage: test
+  image: $CI_REGISTRY_IMAGE_BASE
+  needs:
+    - job: cert
+      artifacts: true
+    - job: build-test
+  script:
+    - echo $PWD
+    - ls -la
+
+    - docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY
+
+    - if [[ "$CAOSDB_TAG" == "" ]]; then
+        if echo "$F_BRANCH" | grep -c "^f-" ; then
+          CAOSDB_TAG=${DEPLOY_REF}_F_${F_BRANCH};
+          docker pull $CI_REGISTRY/caosdb/src/caosdb-deploy:$CAOSDB_TAG || CAOSDB_TAG="${DEFAULT_CAOSDB_TAG}_F_dev" ;
+        elif [ "$F_BRANCH" == "main" ] ; then
+          CAOSDB_TAG=${DEFAULT_CAOSDB_TAG}_F_main ;
+        else
+          CAOSDB_TAG="${DEFAULT_CAOSDB_TAG}_F_dev" ;
+        fi;
+      fi
+    - echo "F_BRANCH = $F_BRANCH"
+    - echo "CAOSDB_TAG = $CAOSDB_TAG"
+    - echo "DEPLOY_REF = $DEPLOY_REF"
+    - docker load < /image-cache/caosdb-${F_BRANCH}.tar || true
+    - docker pull $CI_REGISTRY/caosdb/src/caosdb-deploy:$CAOSDB_TAG || CAOSDB_TAG=${DEFAULT_CAOSDB_TAG}_F_dev ;
+    - echo "CAOSDB_TAG = $CAOSDB_TAG"
+
+    - docker load < /image-cache/${OCTAVEINTTEST_IMAGE_CACHE} || true
+    - docker pull $OCTAVEINTTEST_REGISTRY_IMAGE
+
+    - docker load < /image-cache/mariadb.tar || true
+      # load any(!!!) mariab image
+    - cd .docker
+    - ls -la
+    - ls -la cert
+
+      # here the server and the mysql backend docker are being started
+    - CAOSDB_TAG=$CAOSDB_TAG docker-compose  up -d
+      # store versions of CaosDB parts
+    - docker exec -u 0 -t docker_caosdb-server_1 cat /opt/caosdb/git/caosdb_server_commit | tee hash_server
+    - docker exec -u 0 -t docker_caosdb-server_1 cat /opt/caosdb/git/caosdb_mysqlbackend_commit | tee hash_mysql
+    - docker exec -u 0 -t docker_caosdb-server_1 cat /opt/caosdb/git/caosdb_proto_commit | tee hash_proto
+
+      # the pyinttest docker writes the return value of the tests into the
+      # file result
+    - /bin/sh ./run.sh
+
+      # collect log files
+    - docker logs docker_caosdb-server_1 &> ../caosdb_log.txt
+    - docker logs docker_sqldb_1 &> ../mariadb_log.txt
+
+      # shut down
+    - CAOSDB_TAG=$CAOSDB_TAG docker-compose -f docker-compose.yml down
+
+      # collect results
+    - rc=`cat result`
+
+      # renew mariadb image in the cache (why? to renew the timestamp???)
+    - echo mariadb:$(docker image ls mariadb | grep mariadb | awk '{print $2}')
+    - docker save mariadb:$(docker image ls mariadb | grep mariadb | awk '{print $2}') > /image-cache/mariadb.tar || true
+
+    - cat ../caosdb_log.txt
+
+      # exit with fail when errors exist
+    - exit $rc
+  timeout: 3h
+  artifacts:
+    when: on_failure
+    paths:
+      - caosdb_log.txt
+      - mariadb_log.txt
+      - .docker/hash_*
+    expire_in: 1 week
+
+cert:
+  tags: [docker]
+  stage: build
+  image: $OCTAVEINTTEST_REGISTRY_IMAGE
+  artifacts:
+    paths:
+      - .docker/cert/
+    expire_in: 1 days
+  script:
+    - cd .docker
+    - CAOSHOSTNAME=caosdb-server ./cert.sh