From e987533067c6335cfcf16b9520f05a37c6cce259 Mon Sep 17 00:00:00 2001
From: Timm Fitschen <t.fitschen@indiscale.com>
Date: Mon, 7 Jun 2021 00:43:03 +0200
Subject: [PATCH] WIP: pipeline
---
.docker/Dockerfile | 8 ++-
.gitlab-ci.yml | 161 +++++++++++++++++++++++++++++++++++++++++----
2 files changed, 156 insertions(+), 13 deletions(-)
diff --git a/.docker/Dockerfile b/.docker/Dockerfile
index 16eda49..3230ad0 100644
--- a/.docker/Dockerfile
+++ b/.docker/Dockerfile
@@ -1,6 +1,9 @@
ARG CI_REGISTRY_IMAGE
FROM $CI_REGISTRY_IMAGE
+COPY .docker/wait-for-it.sh /wait-for-it.sh
+
+# build and install caosdb-cpplib
WORKDIR /libcaosdb/
RUN mkdir build
WORKDIR /libcaosdb/build
@@ -8,4 +11,7 @@ RUN cmake ..
RUN cmake --build .
RUN cmake --install . --prefix /usr/local/
-CMD /bin/bash
+
+CMD /wait-for-it.sh caosdb-server:10443 -t 500 -- \
+ # ... and run tests
+ ctest
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 8d3c6bc..15daf2e 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -20,17 +20,51 @@
# along with this program. If not, see <https://www.gnu.org/licenses/>.
variables:
- CI_REGISTRY_IMAGE_CPPLIB: $CI_REGISTRY/caosdb/src/caosdb-cpplib/testenv:latest
- CI_REGISTRY_IMAGE: $CI_REGISTRY/caosdb/src/caosdb-cppinttest/testenv:latest
+ # this is the image where cpplib repo is present (artifact from pipeline in
+ # caosdb-cpplib)
+ CI_REGISTRY_IMAGE_CPPLIB: $CI_REGISTRY/caosdb/src/caosdb-cpplib/testenv:$CPP
+ # this is the image where all the docker containers (server, mysql and test
+ # suite) are started
+ CI_REGISTRY_IMAGE_BASE: $CI_REGISTRY/caosdb/src/caosdb-pyinttest/base:latest
+
+ CPPINTTEST_TESTENV_IMAGE_SUFFIX: $CI_COMMIT_REF_NAME
+ CPPINTTEST_REPO_PATH: caosdb/src/caosdb-cppinttest
+ CPPINTTEST_TESTENV_IMAGE_TAG: ${CPPINTTEST_REPO_PATH}/testenv:${CPPINTTEST_IMAGE_SUFFIX}
+ # this is the image where the cpplib is installed and where the testsuite runs
+ CPPINTTEST_REGISTRY_IMAGE: ${CI_REGISTRY}/${CPPINTTEST_IMAGE_TAG}
+ # this is the name of the image in the /image-cache/ directory
+ CPPINTTEST_IMAGE_CACHE: ${CPPINTTEST_IMAGE_TAG}.tar
+
+
+ ## FOR DEBUGGING
TRIGGERED_BY_REPO: CPPINTTEST
TRIGGERED_BY_REF: $CI_COMMIT_REF_NAME
TRIGGERED_BY_HASH: $CI_COMMIT_SHORT_SHA
+
+ ## Run tests against this cpplib version
CPPLIB_VERSION: 0.0.1
+
+ # This is the caosdb-deploy branch which build the images for caosdb-server and caosdb-mysql
+ # TODO change to dev after merge of f-grpc to dev
+ DEPLOY_REF: f-grpc
+
stages:
+ - info
- setup
+ - build
- test
+info:
+ tags: [cached-dind]
+ image: docker:20.10
+ stage: setup_tests
+ timeout: 1 minutes
+ needs: []
+ script:
+ - echo "Pipeline triggered by $TRIGGERED_BY_REPO@$TRIGGERED_BY_REF ($TRIGGERED_BY_HASH)"
+
+
# Build a docker image in which tests for this repository can run
build-testenv:
tags: [cached-dind]
@@ -39,28 +73,131 @@ build-testenv:
timeout: 2h
needs: []
script:
- - echo "Pipeline triggered by $TRIGGERED_BY_REPO@$TRIGGERED_BY_REF ($TRIGGERED_BY_HASH)"
- docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY
- - docker pull $CI_REGISTRY_IMAGE_CPPLIB|| true
+
+ ## Determine the cppinttest branch...
+ # ... use an f-branch if posible...
+ - if echo "$CI_COMMIT_REF_NAME" | grep -c "^f-" ; then
+ # this is an f-branch
+ CPPINT_REF=$CI_COMMIT_REF_NAME ;
+ fi;
+ # ... or use main if possible...
+ - if [[ "$CI_COMMIT_REF_NAME" == "main" ]] ; then
+ CPPINT_REF=main ;
+ fi
+ # ... and fall-back to dev
+ - CPPINT_REF=${CPPINT_REF:-dev}
+
+ #build testenv image
+ - docker pull $CI_REGISTRY_IMAGE_CPPLIB || true # remove??? docker build --pull???
- docker build
--build-arg CI_REGISTRY_IMAGE=$CI_REGISTRY_IMAGE_CPPLIB
--file .docker/Dockerfile
--pull
- --cache-from $CI_REGISTRY_IMAGE
- --tag $CI_REGISTRY_IMAGE .
- - docker push $CI_REGISTRY_IMAGE
+ --cache-from $CPPINTTEST_REGISTRY_IMAGE
+ --tag $CPPINTTEST_REGISTRY_IMAGE .
+ - docker push $CPPINTTEST_REGISTRY_IMAGE
+ - docker save $CPPINTTEST_REGISTRY_IMAGE > /image-cache/${CPPINTTEST_IMAGE_CACHE}
-# run integration tests
-test:
- image: $CI_REGISTRY_IMAGE
+# build integration tests
+build-test:
+ image: $CPPINTTEST_REGISTRY_IMAGE
dependencies:
- build-testenv
tags: [ docker ]
- stage: test
+ stage: build
script:
- mkdir build
- cd build
- cmake -D CMAKE_PREFIX_PATH=/usr/local/lib/libcaosdb-${CPPLIB_VERSION} ..
- cmake --build .
- - ctest
+ artifacts:
+ path:
+ - build/
+ expire_in: 1 week
+
+
+test:
+ tags: [docker]
+ services:
+ - docker:20.10.3-dind
+ variables:
+ # This is a workaround for the gitlab-runner health check mechanism when
+ # using docker-dind service. The runner will otherwise guess the port
+ # wrong and the health check will timeout.
+ SERVICE_PORT_2376_TCP_PORT: 2375
+ stage: test
+ image: $CI_REGISTRY_IMAGE_BASE
+ needs:
+ - job: build-test
+ artifacts: true
+ script:
+ - docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY
+
+ - if [[ "$CAOSDB_TAG" == "" ]]; then
+ # the CAOSDB_TAG has to be guessed...
+ if echo "$F_BRANCH" | grep -c "^f-" ; then
+ CAOSDB_TAG=${DEPLOY_REF}_F_${F_BRANCH}-latest;
+ echo "Load deploy images with tag: $CAOSDB_TAG ..."
+ if ! docker pull $CI_REGISTRY/caosdb/src/caosdb-deploy:$CAOSDB_TAG ; then
+ echo "Load deploy images with tag: $CAOSDB_TAG ... error. Try another:" ;
+ CAOSDB_TAG=${DEPLOY_REF}-latest ;
+ echo "Load deploy images with tag: $CAOSDB_TAG ..."
+ docker pull $CI_REGISTRY/caosdb/src/caosdb-deploy:$CAOSDB_TAG ;
+ echo "Load deploy images with tag: $CAOSDB_TAG ... success"
+ fi
+
+ else
+ # no F_BRANCH -> use default DEPLOY_REF branch
+ CAOSDB_TAG=${DEPLOY_REF}-latest ;
+ echo "Load deploy images with tag: $CAOSDB_TAG ..."
+ docker pull $CI_REGISTRY/caosdb/src/caosdb-deploy:$CAOSDB_TAG ;
+ echo "Load deploy images with tag: $CAOSDB_TAG ... success"
+ fi;
+ else
+ # If the CAOSDB_TAG was given
+ echo "Load deploy images with tag: $CAOSDB_TAG ..."
+ docker pull $CI_REGISTRY/caosdb/src/caosdb-deploy:$CAOSDB_TAG ;
+ echo "Load deploy images with tag: $CAOSDB_TAG ... success"
+ fi
+
+ - docker load < /image-cache/${CPPINTTEST_IMAGE_CACHE} || true
+ - docker pull $CPPINTTEST_REGISTRY_IMAGE
+
+ # load any(!!!) mariab image
+ - docker load < /image-cache/mariadb.tar || true
+ - cd .docker
+
+ # here the server and the mysql backend docker are being started
+ - CAOSDB_TAG=$CAOSDB_TAG docker-compose up -d
+
+ # the pyinttest docker writes the return value of the tests into the
+ # file result
+ - /bin/sh ./run.sh
+
+ # collect log files
+ - cd ..
+ - docker logs docker_caosdb-server_1 &> ../caosdb_log.txt
+ - docker logs docker_sqldb_1 &> ../mariadb_log.txt
+
+ # shut down
+ - docker-compose -f .docker/docker-compose.yml down
+
+ # collect results
+ - rc=`cat .docker/result`
+
+ # renew mariadb image in the cache (why? to renew the timestamp???)
+ - echo mariadb:$(docker image ls mariadb | grep mariadb | awk '{print $2}')
+ - docker save mariadb:$(docker image ls mariadb | grep mariadb | awk '{print $2}') > /image-cache/mariadb.tar || true
+
+ # exit with fail when errors exist
+ - exit $rc
+ dependencies: [cert]
+ timeout: 3h
+ artifacts:
+ paths:
+ - caosdb_log.txt
+ - mariadb_log.txt
+ - .docker/hash_*
+ expire_in: 1 week
--
GitLab