WIP: pipeline

.docker/Dockerfile

 ARG CI_REGISTRY_IMAGE
 FROM $CI_REGISTRY_IMAGE
 
+COPY .docker/wait-for-it.sh /wait-for-it.sh
+
+# build and install caosdb-cpplib
 WORKDIR /libcaosdb/
 RUN mkdir build
 WORKDIR /libcaosdb/build
@@ -8,4 +11,7 @@ RUN cmake ..
 RUN cmake --build .
 RUN cmake --install . --prefix /usr/local/
 
-CMD /bin/bash
+
+CMD /wait-for-it.sh caosdb-server:10443 -t 500 -- \
+    # ... and run tests
+    ctest

.gitlab-ci.yml

@@ -20,17 +20,51 @@
 # along with this program. If not, see <https://www.gnu.org/licenses/>.
 
 variables:
-  CI_REGISTRY_IMAGE_CPPLIB: $CI_REGISTRY/caosdb/src/caosdb-cpplib/testenv:latest
-  CI_REGISTRY_IMAGE: $CI_REGISTRY/caosdb/src/caosdb-cppinttest/testenv:latest
+  # this is the image where cpplib repo is present (artifact from pipeline in
+  # caosdb-cpplib)
+  CI_REGISTRY_IMAGE_CPPLIB: $CI_REGISTRY/caosdb/src/caosdb-cpplib/testenv:$CPP
+  # this is the image where all the docker containers (server, mysql and test
+  # suite) are started
+  CI_REGISTRY_IMAGE_BASE: $CI_REGISTRY/caosdb/src/caosdb-pyinttest/base:latest
+
+  CPPINTTEST_TESTENV_IMAGE_SUFFIX: $CI_COMMIT_REF_NAME
+  CPPINTTEST_REPO_PATH: caosdb/src/caosdb-cppinttest
+  CPPINTTEST_TESTENV_IMAGE_TAG: ${CPPINTTEST_REPO_PATH}/testenv:${CPPINTTEST_IMAGE_SUFFIX}
+  # this is the image where the cpplib is installed and where the testsuite runs
+  CPPINTTEST_REGISTRY_IMAGE: ${CI_REGISTRY}/${CPPINTTEST_IMAGE_TAG}
+  # this is the name of the image in the /image-cache/ directory
+  CPPINTTEST_IMAGE_CACHE: ${CPPINTTEST_IMAGE_TAG}.tar
+
+
+  ## FOR DEBUGGING
   TRIGGERED_BY_REPO: CPPINTTEST
   TRIGGERED_BY_REF: $CI_COMMIT_REF_NAME
   TRIGGERED_BY_HASH: $CI_COMMIT_SHORT_SHA
+
+  ## Run tests against this cpplib version
   CPPLIB_VERSION: 0.0.1
 
+
+  # This is the caosdb-deploy branch which build the images for caosdb-server and caosdb-mysql
+  # TODO change to dev after merge of f-grpc to dev
+  DEPLOY_REF: f-grpc
+
 stages:
+  - info
   - setup
+  - build
   - test
 
+info:
+  tags: [cached-dind]
+  image: docker:20.10
+  stage: setup_tests
+  timeout: 1 minutes
+  needs: []
+  script:
+    - echo "Pipeline triggered by $TRIGGERED_BY_REPO@$TRIGGERED_BY_REF ($TRIGGERED_BY_HASH)"
+
+
 # Build a docker image in which tests for this repository can run
 build-testenv:
   tags: [cached-dind]
@@ -39,28 +73,131 @@ build-testenv:
   timeout: 2h
   needs: []
   script:
-    - echo "Pipeline triggered by $TRIGGERED_BY_REPO@$TRIGGERED_BY_REF ($TRIGGERED_BY_HASH)"
     - docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY
-    - docker pull $CI_REGISTRY_IMAGE_CPPLIB|| true
+
+    ## Determine the cppinttest branch...
+    # ... use an f-branch if posible...
+    - if echo "$CI_COMMIT_REF_NAME" | grep -c "^f-" ; then
+        # this is an f-branch
+        CPPINT_REF=$CI_COMMIT_REF_NAME ;
+      fi;
+    # ... or use main if possible...
+    - if [[ "$CI_COMMIT_REF_NAME" == "main" ]] ; then
+        CPPINT_REF=main ;
+      fi
+    # ... and fall-back to dev
+    - CPPINT_REF=${CPPINT_REF:-dev}
+
+      #build testenv image
+    - docker pull $CI_REGISTRY_IMAGE_CPPLIB || true # remove??? docker build --pull???
     - docker build
       --build-arg CI_REGISTRY_IMAGE=$CI_REGISTRY_IMAGE_CPPLIB
       --file .docker/Dockerfile
       --pull
-      --cache-from $CI_REGISTRY_IMAGE
-      --tag $CI_REGISTRY_IMAGE .
-    - docker push $CI_REGISTRY_IMAGE
+      --cache-from $CPPINTTEST_REGISTRY_IMAGE
+      --tag $CPPINTTEST_REGISTRY_IMAGE .
+    - docker push $CPPINTTEST_REGISTRY_IMAGE
+    - docker save $CPPINTTEST_REGISTRY_IMAGE > /image-cache/${CPPINTTEST_IMAGE_CACHE}
 
 
-# run integration tests
-test:
-  image: $CI_REGISTRY_IMAGE
+# build integration tests
+build-test:
+  image: $CPPINTTEST_REGISTRY_IMAGE
   dependencies:
     - build-testenv
   tags: [ docker ]
-  stage: test
+  stage: build
   script:
     - mkdir build
     - cd build
     - cmake -D CMAKE_PREFIX_PATH=/usr/local/lib/libcaosdb-${CPPLIB_VERSION} ..
     - cmake --build .
-    - ctest
+  artifacts:
+    path:
+      - build/
+    expire_in: 1 week
+
+
+test:
+  tags: [docker]
+  services:
+    - docker:20.10.3-dind
+  variables:
+    # This is a workaround for the gitlab-runner health check mechanism when
+    # using docker-dind service.  The runner will otherwise guess the port
+    # wrong and the health check will timeout.
+    SERVICE_PORT_2376_TCP_PORT: 2375
+  stage: test
+  image: $CI_REGISTRY_IMAGE_BASE
+  needs:
+    - job: build-test
+      artifacts: true
+  script:
+      - docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY
+
+      - if [[ "$CAOSDB_TAG" == "" ]]; then
+          # the CAOSDB_TAG has to be guessed...
+          if echo "$F_BRANCH" | grep -c "^f-" ; then
+            CAOSDB_TAG=${DEPLOY_REF}_F_${F_BRANCH}-latest;
+            echo "Load deploy images with tag: $CAOSDB_TAG ..."
+            if ! docker pull $CI_REGISTRY/caosdb/src/caosdb-deploy:$CAOSDB_TAG ; then
+              echo "Load deploy images with tag: $CAOSDB_TAG ... error. Try another:" ;
+              CAOSDB_TAG=${DEPLOY_REF}-latest ;
+              echo "Load deploy images with tag: $CAOSDB_TAG ..."
+              docker pull $CI_REGISTRY/caosdb/src/caosdb-deploy:$CAOSDB_TAG ;
+              echo "Load deploy images with tag: $CAOSDB_TAG ... success"
+            fi
+
+          else
+            # no F_BRANCH -> use default DEPLOY_REF branch
+            CAOSDB_TAG=${DEPLOY_REF}-latest ;
+            echo "Load deploy images with tag: $CAOSDB_TAG ..."
+            docker pull $CI_REGISTRY/caosdb/src/caosdb-deploy:$CAOSDB_TAG ;
+            echo "Load deploy images with tag: $CAOSDB_TAG ... success"
+          fi;
+        else
+          # If the CAOSDB_TAG was given
+          echo "Load deploy images with tag: $CAOSDB_TAG ..."
+          docker pull $CI_REGISTRY/caosdb/src/caosdb-deploy:$CAOSDB_TAG ;
+          echo "Load deploy images with tag: $CAOSDB_TAG ... success"
+        fi
+
+      - docker load < /image-cache/${CPPINTTEST_IMAGE_CACHE} || true
+      - docker pull $CPPINTTEST_REGISTRY_IMAGE
+
+        # load any(!!!) mariab image
+      - docker load < /image-cache/mariadb.tar || true
+      - cd .docker
+
+        # here the server and the mysql backend docker are being started
+      - CAOSDB_TAG=$CAOSDB_TAG docker-compose  up -d
+
+        # the pyinttest docker writes the return value of the tests into the
+        # file result
+      - /bin/sh ./run.sh
+
+        # collect log files
+      - cd ..
+      - docker logs docker_caosdb-server_1 &> ../caosdb_log.txt
+      - docker logs docker_sqldb_1 &> ../mariadb_log.txt
+
+        # shut down
+      - docker-compose -f .docker/docker-compose.yml down
+
+        # collect results
+      - rc=`cat .docker/result`
+
+        # renew mariadb image in the cache (why? to renew the timestamp???)
+      - echo mariadb:$(docker image ls mariadb | grep mariadb | awk '{print $2}')
+      - docker save mariadb:$(docker image ls mariadb | grep mariadb | awk '{print $2}') > /image-cache/mariadb.tar || true
+
+        # exit with fail when errors exist
+      - exit $rc
+  dependencies: [cert]
+  timeout: 3h
+  artifacts:
+    paths:
+      - caosdb_log.txt
+      - mariadb_log.txt
+      - .docker/hash_*
+    expire_in: 1 week