MAINT: refactor pipeleine script

.gitlab-ci.yml

@@ -104,6 +104,27 @@ stages:
       CAOSDB_TAG=${REFTAG};
     fi
   - echo $CAOSDB_TAG
+  - if [ -z "$PYLIB" ]; then
+      if echo "$CI_COMMIT_REF_NAME" | grep -c "^f-" ; then
+        echo "Check if pylib has branch $CI_COMMIT_REF_NAME" ;
+        if wget https://gitlab.indiscale.com/api/v4/projects/97/repository/branches/${CI_COMMIT_REF_NAME} ; then
+          PYLIB=$CI_COMMIT_REF_NAME ;
+        fi;
+      fi;
+    fi;
+  - PYLIB=${PYLIB:-dev}
+  - echo $PYLIB
+
+  - if [ -z "$ADVANCED" ]; then
+      if echo "$CI_COMMIT_REF_NAME" | grep -c "^f-" ; then
+        echo "Check if advanced user tools have branch $CI_COMMIT_REF_NAME" ;
+        if wget https://gitlab.indiscale.com/api/v4/projects/104/repository/branches/${CI_COMMIT_REF_NAME} ; then
+          ADVANCED=$CI_COMMIT_REF_NAME ;
+        fi;
+      fi;
+    fi;
+  - ADVANCED=${ADVANCED:-dev}
+  - echo $ADVANCED
 
 info:
   tags: [cached-dind]
@@ -113,47 +134,58 @@ info:
   script:
     - *env
 
-unittest_py3.11:
-  tags: [cached-dind]
-  stage: test
-  image: $CI_REGISTRY_IMAGE
-  script:
-    - python3 -c "import sys; assert sys.version.startswith('3.11')"
-    - tox
-
 unittest_py3.9:
   tags: [cached-dind]
   stage: test
+  variables:
+    PYVER: 3.9
   image: python:3.9
   script: &python_test_script
     # install dependencies
+    - *env
     - pip install pytest pytest-cov
-    # TODO: Use f-branch logic here
-    - pip install git+https://gitlab.indiscale.com/caosdb/src/caosdb-pylib.git@dev
-    - pip install git+https://gitlab.indiscale.com/caosdb/src/caosdb-advanced-user-tools.git@dev
+    - pip install git+https://gitlab.indiscale.com/caosdb/src/caosdb-pylib.git@${PYLIB}
+    - pip install git+https://gitlab.indiscale.com/caosdb/src/caosdb-advanced-user-tools.git@{ADVANCED}
     - pip install .[h5-crawler,spss,rocrate]
+    - echo "import sys; assert sys.version.startswith('"$PYVER"')"
+    - python3 -c "import sys; assert sys.version.startswith('"$PYVER"')"
     # actual test
     - caosdb-crawler --help
     - pytest --cov=caosdb -vv ./unittests
 
 unittest_py3.10:
-  tags: [cached-dind]
+  variables:
+    PYVER: 3.10
   stage: test
+  tags: [cached-dind]
   image: python:3.10
   script: *python_test_script
 
-unittest_py3.12:
+unittest_py3.11:
+  variables:
+    PYVER: 3.9
   tags: [cached-dind]
   stage: test
+  image: python:3.11
+  script: *python_test_script
+
+unittest_py3.12:
+  variables:
+    PYVER: 3.12
+  stage: test
+  tags: [cached-dind]
   image: python:3.12
   script: *python_test_script
 
 unittest_py3.13:
+  variables:
+    PYVER: 3.13
   tags: [cached-dind]
   stage: test
   image: python:3.13
   script: *python_test_script
 
+
 inttest:
   tags: [docker]
   services:
@@ -170,6 +202,8 @@ inttest:
       - *env
       - docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY
       - echo $CAOSDB_TAG
+      - echo $PYLIB
+      - echo $ADVANCED
 
       - cd .docker
         # Store mariadb  version
@@ -233,27 +267,6 @@ build-testenv:
   script:
       - df -h
       - command -v wget
-      - if [ -z "$PYLIB" ]; then
-          if echo "$CI_COMMIT_REF_NAME" | grep -c "^f-" ; then
-            echo "Check if pylib has branch $CI_COMMIT_REF_NAME" ;
-            if wget https://gitlab.indiscale.com/api/v4/projects/97/repository/branches/${CI_COMMIT_REF_NAME} ; then
-              PYLIB=$CI_COMMIT_REF_NAME ;
-            fi;
-          fi;
-        fi;
-      - PYLIB=${PYLIB:-dev}
-      - echo $PYLIB
-
-      - if [ -z "$ADVANCED" ]; then
-          if echo "$CI_COMMIT_REF_NAME" | grep -c "^f-" ; then
-            echo "Check if advanced user tools have branch $CI_COMMIT_REF_NAME" ;
-            if wget https://gitlab.indiscale.com/api/v4/projects/104/repository/branches/${CI_COMMIT_REF_NAME} ; then
-              ADVANCED=$CI_COMMIT_REF_NAME ;
-            fi;
-          fi;
-        fi;
-      - ADVANCED=${ADVANCED:-dev}
-      - echo $ADVANCED
 
       - docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY
         # use here general latest or specific branch latest...

tox.ini

@@ -6,9 +6,8 @@ skip_missing_interpreters = true
 deps = .[h5-crawler,spss,rocrate]
     pytest
     pytest-cov
-    # TODO: Make this f-branch sensitive
-    git+https://gitlab.indiscale.com/caosdb/src/caosdb-pylib.git@dev
-    git+https://gitlab.indiscale.com/caosdb/src/caosdb-advanced-user-tools.git@dev
+    caosdb-pylib
+    caosdb-advanced-user-tools
 commands = caosdb-crawler --help
     py.test --cov=caoscrawler -vv {posargs}