From 1460d5bfc0070e977c3f47bdbb6044397b6b447e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Henrik=20tom=20W=C3=B6rden?= <h.tomwoerden@indiscale.com>
Date: Fri, 10 Jan 2025 11:27:59 +0100
Subject: [PATCH 01/11] FIX: remove unused code
---
.docker/Dockerfile | 10 ----------
1 file changed, 10 deletions(-)
diff --git a/.docker/Dockerfile b/.docker/Dockerfile
index 1468a17f..e549f2f4 100644
--- a/.docker/Dockerfile
+++ b/.docker/Dockerfile
@@ -16,16 +16,6 @@ RUN pip3 install --break-system-packages \
sphinx-rtd-theme \
;
COPY .docker/wait-for-it.sh /wait-for-it.sh
-ARG PYLIB
-ADD https://gitlab.indiscale.com/api/v4/projects/97/repository/commits/${PYLIB} \
- pylib_version.json
-RUN git clone https://gitlab.indiscale.com/caosdb/src/caosdb-pylib.git && \
- cd caosdb-pylib && git checkout ${PYLIB} && pip3 install --break-system-packages .
-ARG ADVANCED
-ADD https://gitlab.indiscale.com/api/v4/projects/104/repository/commits/${ADVANCED} \
- advanced_version.json
-RUN git clone https://gitlab.indiscale.com/caosdb/src/caosdb-advanced-user-tools.git && \
- cd caosdb-advanced-user-tools && git checkout ${ADVANCED} && pip3 install --break-system-packages .[h5-crawler]
COPY . /git
# Delete .git because it is huge.
--
GitLab
From 9ea70aaff32ab117b5c3888a63340ede84d5e4a1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Henrik=20tom=20W=C3=B6rden?= <h.tomwoerden@indiscale.com>
Date: Fri, 10 Jan 2025 11:53:14 +0100
Subject: [PATCH 02/11] MAINT: refactor pipeleine script
---
.gitlab-ci.yml | 81 +++++++++++++++++++++++++++++---------------------
tox.ini | 5 ++--
2 files changed, 49 insertions(+), 37 deletions(-)
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index e4322356..1d454da0 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -104,6 +104,27 @@ stages:
CAOSDB_TAG=${REFTAG};
fi
- echo $CAOSDB_TAG
+ - if [ -z "$PYLIB" ]; then
+ if echo "$CI_COMMIT_REF_NAME" | grep -c "^f-" ; then
+ echo "Check if pylib has branch $CI_COMMIT_REF_NAME" ;
+ if wget https://gitlab.indiscale.com/api/v4/projects/97/repository/branches/${CI_COMMIT_REF_NAME} ; then
+ PYLIB=$CI_COMMIT_REF_NAME ;
+ fi;
+ fi;
+ fi;
+ - PYLIB=${PYLIB:-dev}
+ - echo $PYLIB
+
+ - if [ -z "$ADVANCED" ]; then
+ if echo "$CI_COMMIT_REF_NAME" | grep -c "^f-" ; then
+ echo "Check if advanced user tools have branch $CI_COMMIT_REF_NAME" ;
+ if wget https://gitlab.indiscale.com/api/v4/projects/104/repository/branches/${CI_COMMIT_REF_NAME} ; then
+ ADVANCED=$CI_COMMIT_REF_NAME ;
+ fi;
+ fi;
+ fi;
+ - ADVANCED=${ADVANCED:-dev}
+ - echo $ADVANCED
info:
tags: [cached-dind]
@@ -113,47 +134,58 @@ info:
script:
- *env
-unittest_py3.11:
- tags: [cached-dind]
- stage: test
- image: $CI_REGISTRY_IMAGE
- script:
- - python3 -c "import sys; assert sys.version.startswith('3.11')"
- - tox
-
unittest_py3.9:
tags: [cached-dind]
stage: test
+ variables:
+ PYVER: 3.9
image: python:3.9
script: &python_test_script
# install dependencies
+ - *env
- pip install pytest pytest-cov
- # TODO: Use f-branch logic here
- - pip install git+https://gitlab.indiscale.com/caosdb/src/caosdb-pylib.git@dev
- - pip install git+https://gitlab.indiscale.com/caosdb/src/caosdb-advanced-user-tools.git@dev
+ - pip install git+https://gitlab.indiscale.com/caosdb/src/caosdb-pylib.git@${PYLIB}
+ - pip install git+https://gitlab.indiscale.com/caosdb/src/caosdb-advanced-user-tools.git@{ADVANCED}
- pip install .[h5-crawler,spss,rocrate]
+ - echo "import sys; assert sys.version.startswith('"$PYVER"')"
+ - python3 -c "import sys; assert sys.version.startswith('"$PYVER"')"
# actual test
- caosdb-crawler --help
- pytest --cov=caosdb -vv ./unittests
unittest_py3.10:
- tags: [cached-dind]
+ variables:
+ PYVER: 3.10
stage: test
+ tags: [cached-dind]
image: python:3.10
script: *python_test_script
-unittest_py3.12:
+unittest_py3.11:
+ variables:
+ PYVER: 3.9
tags: [cached-dind]
stage: test
+ image: python:3.11
+ script: *python_test_script
+
+unittest_py3.12:
+ variables:
+ PYVER: 3.12
+ stage: test
+ tags: [cached-dind]
image: python:3.12
script: *python_test_script
unittest_py3.13:
+ variables:
+ PYVER: 3.13
tags: [cached-dind]
stage: test
image: python:3.13
script: *python_test_script
+
inttest:
tags: [docker]
services:
@@ -170,6 +202,8 @@ inttest:
- *env
- docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY
- echo $CAOSDB_TAG
+ - echo $PYLIB
+ - echo $ADVANCED
- cd .docker
# Store mariadb version
@@ -233,27 +267,6 @@ build-testenv:
script:
- df -h
- command -v wget
- - if [ -z "$PYLIB" ]; then
- if echo "$CI_COMMIT_REF_NAME" | grep -c "^f-" ; then
- echo "Check if pylib has branch $CI_COMMIT_REF_NAME" ;
- if wget https://gitlab.indiscale.com/api/v4/projects/97/repository/branches/${CI_COMMIT_REF_NAME} ; then
- PYLIB=$CI_COMMIT_REF_NAME ;
- fi;
- fi;
- fi;
- - PYLIB=${PYLIB:-dev}
- - echo $PYLIB
-
- - if [ -z "$ADVANCED" ]; then
- if echo "$CI_COMMIT_REF_NAME" | grep -c "^f-" ; then
- echo "Check if advanced user tools have branch $CI_COMMIT_REF_NAME" ;
- if wget https://gitlab.indiscale.com/api/v4/projects/104/repository/branches/${CI_COMMIT_REF_NAME} ; then
- ADVANCED=$CI_COMMIT_REF_NAME ;
- fi;
- fi;
- fi;
- - ADVANCED=${ADVANCED:-dev}
- - echo $ADVANCED
- docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY
# use here general latest or specific branch latest...
diff --git a/tox.ini b/tox.ini
index e003e26e..65c7549d 100644
--- a/tox.ini
+++ b/tox.ini
@@ -6,9 +6,8 @@ skip_missing_interpreters = true
deps = .[h5-crawler,spss,rocrate]
pytest
pytest-cov
- # TODO: Make this f-branch sensitive
- git+https://gitlab.indiscale.com/caosdb/src/caosdb-pylib.git@dev
- git+https://gitlab.indiscale.com/caosdb/src/caosdb-advanced-user-tools.git@dev
+ caosdb-pylib
+ caosdb-advanced-user-tools
commands = caosdb-crawler --help
py.test --cov=caoscrawler -vv {posargs}
--
GitLab
From 8d623f8933f6dc88602b1815c2208d841ba314df Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Henrik=20tom=20W=C3=B6rden?= <h.tomwoerden@indiscale.com>
Date: Fri, 10 Jan 2025 12:28:37 +0100
Subject: [PATCH 03/11] FIX: add missing dollar
---
.gitlab-ci.yml | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 1d454da0..91fd8fdf 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -145,7 +145,7 @@ unittest_py3.9:
- *env
- pip install pytest pytest-cov
- pip install git+https://gitlab.indiscale.com/caosdb/src/caosdb-pylib.git@${PYLIB}
- - pip install git+https://gitlab.indiscale.com/caosdb/src/caosdb-advanced-user-tools.git@{ADVANCED}
+ - pip install git+https://gitlab.indiscale.com/caosdb/src/caosdb-advanced-user-tools.git@${ADVANCED}
- pip install .[h5-crawler,spss,rocrate]
- echo "import sys; assert sys.version.startswith('"$PYVER"')"
- python3 -c "import sys; assert sys.version.startswith('"$PYVER"')"
@@ -271,7 +271,7 @@ build-testenv:
- docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY
# use here general latest or specific branch latest...
- docker build
- --build-arg PYLIB=${PYLIB}
+ --build-arg PYLIB=${PYLIB:dev}
--build-arg ADVANCED=${ADVANCED:dev}
--file .docker/Dockerfile
-t $CI_REGISTRY_IMAGE .
--
GitLab
From ad096e50d5dc74977713c2f92e2b1734893d95e2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Henrik=20tom=20W=C3=B6rden?= <h.tomwoerden@indiscale.com>
Date: Fri, 10 Jan 2025 12:40:11 +0100
Subject: [PATCH 04/11] FIX: string concatenation
---
.gitlab-ci.yml | 14 +++++++-------
1 file changed, 7 insertions(+), 7 deletions(-)
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 91fd8fdf..cd3e46b5 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -138,7 +138,7 @@ unittest_py3.9:
tags: [cached-dind]
stage: test
variables:
- PYVER: 3.9
+ PYVER: "3.9"
image: python:3.9
script: &python_test_script
# install dependencies
@@ -147,15 +147,15 @@ unittest_py3.9:
- pip install git+https://gitlab.indiscale.com/caosdb/src/caosdb-pylib.git@${PYLIB}
- pip install git+https://gitlab.indiscale.com/caosdb/src/caosdb-advanced-user-tools.git@${ADVANCED}
- pip install .[h5-crawler,spss,rocrate]
- - echo "import sys; assert sys.version.startswith('"$PYVER"')"
- - python3 -c "import sys; assert sys.version.startswith('"$PYVER"')"
+ - echo "import sys; assert sys.version.startswith('$PYVER')"
+ - python3 -c "import sys; assert sys.version.startswith('$PYVER')"
# actual test
- caosdb-crawler --help
- pytest --cov=caosdb -vv ./unittests
unittest_py3.10:
variables:
- PYVER: 3.10
+ PYVER: "3.10"
stage: test
tags: [cached-dind]
image: python:3.10
@@ -163,7 +163,7 @@ unittest_py3.10:
unittest_py3.11:
variables:
- PYVER: 3.9
+ PYVER: "3.11"
tags: [cached-dind]
stage: test
image: python:3.11
@@ -171,7 +171,7 @@ unittest_py3.11:
unittest_py3.12:
variables:
- PYVER: 3.12
+ PYVER: "3.12"
stage: test
tags: [cached-dind]
image: python:3.12
@@ -179,7 +179,7 @@ unittest_py3.12:
unittest_py3.13:
variables:
- PYVER: 3.13
+ PYVER: "3.13"
tags: [cached-dind]
stage: test
image: python:3.13
--
GitLab
From 08ad7cfea99bfc80eac64725159516df54ef7327 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Henrik=20tom=20W=C3=B6rden?= <h.tomwoerden@indiscale.com>
Date: Fri, 10 Jan 2025 13:25:44 +0100
Subject: [PATCH 05/11] MAINT: resturcture docker image
---
.docker/Dockerfile | 43 +++++++++++++++++++++++++++++++++++++------
1 file changed, 37 insertions(+), 6 deletions(-)
diff --git a/.docker/Dockerfile b/.docker/Dockerfile
index e549f2f4..42926cf7 100644
--- a/.docker/Dockerfile
+++ b/.docker/Dockerfile
@@ -1,3 +1,29 @@
+###############################
+###### Temporary Image ########
+###############################
+FROM debian:bookworm as git_base
+
+# Check for availability of DNS
+RUN if getent hosts indiscale.com > /dev/null; \
+ then echo "Connected to the internet and DNS available"; \
+ else echo "No internet connection or DNS not available"; \
+ fi
+
+RUN apt-get update && apt-get install -y \
+ git
+
+COPY . /git
+
+# Delete .git because it is huge.
+RUN rm -r /git/.git
+
+# Install pycaosdb.ini for the tests
+RUN mv /git/.docker/tester_pycaosdb.ini /git/integrationtests/pycaosdb.ini
+
+###############################
+###### Main Image Build #######
+###############################
+
FROM debian:bookworm
RUN apt-get update && \
apt-get install \
@@ -16,13 +42,18 @@ RUN pip3 install --break-system-packages \
sphinx-rtd-theme \
;
COPY .docker/wait-for-it.sh /wait-for-it.sh
-COPY . /git
+ARG PYLIB
+ADD https://gitlab.indiscale.com/api/v4/projects/97/repository/commits/${PYLIB} \
+ pylib_version.json
+RUN git clone https://gitlab.indiscale.com/caosdb/src/caosdb-pylib.git && \
+ cd caosdb-pylib && git checkout ${PYLIB} && pip3 install --break-system-packages .
+ARG ADVANCED
+ADD https://gitlab.indiscale.com/api/v4/projects/104/repository/commits/${ADVANCED} \
+ advanced_version.json
+RUN git clone https://gitlab.indiscale.com/caosdb/src/caosdb-advanced-user-tools.git && \
+ cd caosdb-advanced-user-tools && git checkout ${ADVANCED} && pip3 install --break-system-packages .[h5-crawler]
-# Delete .git because it is huge.
-RUN rm -r /git/.git
-
-# Install pycaosdb.ini for the tests
-RUN mv /git/.docker/tester_pycaosdb.ini /git/integrationtests/pycaosdb.ini
+COPY --from=git_base /git /git
RUN cd /git/ && pip3 install --break-system-packages .[h5-crawler,spss,rocrate]
--
GitLab
From 40a07e9c959d7278913b925de4d4d81ca9168384 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Henrik=20tom=20W=C3=B6rden?= <h.tomwoerden@indiscale.com>
Date: Fri, 10 Jan 2025 13:31:21 +0100
Subject: [PATCH 06/11] FIX: wget does not produce output
---
.gitlab-ci.yml | 5 +++--
1 file changed, 3 insertions(+), 2 deletions(-)
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index cd3e46b5..782768dd 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -107,7 +107,7 @@ stages:
- if [ -z "$PYLIB" ]; then
if echo "$CI_COMMIT_REF_NAME" | grep -c "^f-" ; then
echo "Check if pylib has branch $CI_COMMIT_REF_NAME" ;
- if wget https://gitlab.indiscale.com/api/v4/projects/97/repository/branches/${CI_COMMIT_REF_NAME} ; then
+ if wget -O /dev/null https://gitlab.indiscale.com/api/v4/projects/97/repository/branches/${CI_COMMIT_REF_NAME}>/dev/null ; then
PYLIB=$CI_COMMIT_REF_NAME ;
fi;
fi;
@@ -118,7 +118,7 @@ stages:
- if [ -z "$ADVANCED" ]; then
if echo "$CI_COMMIT_REF_NAME" | grep -c "^f-" ; then
echo "Check if advanced user tools have branch $CI_COMMIT_REF_NAME" ;
- if wget https://gitlab.indiscale.com/api/v4/projects/104/repository/branches/${CI_COMMIT_REF_NAME} ; then
+ if wget -O /dev/null https://gitlab.indiscale.com/api/v4/projects/104/repository/branches/${CI_COMMIT_REF_NAME} ; then
ADVANCED=$CI_COMMIT_REF_NAME ;
fi;
fi;
@@ -265,6 +265,7 @@ build-testenv:
- pushes
needs: []
script:
+ - *env
- df -h
- command -v wget
--
GitLab
From 9691b126d5ccbf07f61a10b707ad2bdb9107aab5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Henrik=20tom=20W=C3=B6rden?= <h.tomwoerden@indiscale.com>
Date: Fri, 10 Jan 2025 14:26:27 +0100
Subject: [PATCH 07/11] FIX: remove tox from makefile
---
Makefile | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/Makefile b/Makefile
index 95fc2bf6..1ba97ae3 100644
--- a/Makefile
+++ b/Makefile
@@ -44,5 +44,5 @@ lint:
.PHONY: lint
unittest:
- tox -r
+ pytest
.PHONY: unittest
--
GitLab
From 36de070666543047ed2e82b040ee29eae2c2ffe2 Mon Sep 17 00:00:00 2001
From: Florian Spreckelsen <f.spreckelsen@indiscale.com>
Date: Mon, 13 Jan 2025 14:57:15 +0100
Subject: [PATCH 08/11] PIPELINE: Reove unnessary git installation
---
.docker/Dockerfile | 3 ---
1 file changed, 3 deletions(-)
diff --git a/.docker/Dockerfile b/.docker/Dockerfile
index 42926cf7..14c3c1ef 100644
--- a/.docker/Dockerfile
+++ b/.docker/Dockerfile
@@ -9,9 +9,6 @@ RUN if getent hosts indiscale.com > /dev/null; \
else echo "No internet connection or DNS not available"; \
fi
-RUN apt-get update && apt-get install -y \
- git
-
COPY . /git
# Delete .git because it is huge.
--
GitLab
From 8367763330eb6fd6e333b9f0d4132bb98af090a2 Mon Sep 17 00:00:00 2001
From: Florian Spreckelsen <f.spreckelsen@indiscale.com>
Date: Mon, 13 Jan 2025 15:05:42 +0100
Subject: [PATCH 09/11] FIX: Use correct package names in tox.ini
---
tox.ini | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/tox.ini b/tox.ini
index 65c7549d..d44fbb6d 100644
--- a/tox.ini
+++ b/tox.ini
@@ -6,8 +6,8 @@ skip_missing_interpreters = true
deps = .[h5-crawler,spss,rocrate]
pytest
pytest-cov
- caosdb-pylib
- caosdb-advanced-user-tools
+ linkahead
+ caosadvancedtools
commands = caosdb-crawler --help
py.test --cov=caoscrawler -vv {posargs}
--
GitLab
From f9796f8f58247d721ad671236bfab8f0e4bd0623 Mon Sep 17 00:00:00 2001
From: Florian Spreckelsen <f.spreckelsen@indiscale.com>
Date: Mon, 13 Jan 2025 15:20:17 +0100
Subject: [PATCH 10/11] MAINT: Use cov in make unittest
---
.gitlab-ci.yml | 2 +-
Makefile | 2 +-
2 files changed, 2 insertions(+), 2 deletions(-)
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 782768dd..f295e0b9 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -151,7 +151,7 @@ unittest_py3.9:
- python3 -c "import sys; assert sys.version.startswith('$PYVER')"
# actual test
- caosdb-crawler --help
- - pytest --cov=caosdb -vv ./unittests
+ - make unittest
unittest_py3.10:
variables:
diff --git a/Makefile b/Makefile
index 1ba97ae3..d9b1354d 100644
--- a/Makefile
+++ b/Makefile
@@ -44,5 +44,5 @@ lint:
.PHONY: lint
unittest:
- pytest
+ pytest --cov=caosdb -vv ./unittests
.PHONY: unittest
--
GitLab
From ce66dcc26ea64de676da0e5bf739aed47336daba Mon Sep 17 00:00:00 2001
From: Florian Spreckelsen <f.spreckelsen@indiscale.com>
Date: Mon, 13 Jan 2025 15:24:17 +0100
Subject: [PATCH 11/11] MAINT: Use correct module for cov
---
Makefile | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/Makefile b/Makefile
index d9b1354d..7167ebfd 100644
--- a/Makefile
+++ b/Makefile
@@ -44,5 +44,5 @@ lint:
.PHONY: lint
unittest:
- pytest --cov=caosdb -vv ./unittests
+ pytest --cov=caoscrawler -vv ./unittests
.PHONY: unittest
--
GitLab