Merge branch 'f-deployment' into 'main'

Prepare deployment

See merge request caosdb/customers/f-fit/ruqad!2

.gitlab-ci.yml

@@ -19,8 +19,7 @@
 # along with this program. If not, see <https://www.gnu.org/licenses/>.
 
 variables:
-   CI_REGISTRY_IMAGE: $CI_REGISTRY/caosdb/src/linkahead-python-package-template/testenv:$CI_COMMIT_REF_NAME
-   CI_REGISTRY_IMAGE_BASE: $CI_REGISTRY/caosdb/src/caosdb-pyinttest/base:latest
+   CI_REGISTRY_IMAGE: $CI_REGISTRY/caosdb/customers/f-fit/ruqad/ruqad-demonstrator-fair-ds:$CI_COMMIT_REF_NAME
 
 stages:
   - info
@@ -32,30 +31,10 @@ stages:
 
 .env: &env
   - echo "Pipeline triggered by $TRIGGERED_BY_REPO@$TRIGGERED_BY_REF ($TRIGGERED_BY_HASH)"
-  - echo "CI_REGISTRY_IMAGE_BASE = $CI_REGISTRY_IMAGE_BASE"
   - echo "CI_REGISTRY_IMAGE = $CI_REGISTRY_IMAGE"
-  - echo "CAOSDB_TAG = $CAOSDB_TAG"
-  - echo "REFTAG = $REFTAG"
-  - echo "F_BRANCH = $F_BRANCH"
   - echo "CI_COMMIT_REF_NAME = $CI_COMMIT_REF_NAME"
   - ls -lah /image-cache/
 
-  - F_BRANCH=${F_BRANCH:-$CI_COMMIT_REF_NAME}
-  - echo $F_BRANCH
-  - if [[ "$REFTAG" == "" ]] ; then
-      if [[ "$F_BRANCH" == "dev" ]] ; then
-        REFTAG=dev;
-      fi;
-    fi
-  - REFTAG=${REFTAG:-dev_F_${F_BRANCH}}
-
-  - echo $F_BRANCH
-
-  - if [[ "$CAOSDB_TAG" == "" ]]; then
-      CAOSDB_TAG=${REFTAG};
-    fi
-  - echo $CAOSDB_TAG
-
 info:
   tags: [cached-dind]
   image: docker:20.10
@@ -64,13 +43,15 @@ info:
   script:
     - *env
 
-unittest_py3.11:
+e2e_tests:
   tags: [cached-dind]
   stage: test
   image: $CI_REGISTRY_IMAGE
+  #image: python:3.13
   script:
-    - python3 -c "import sys; assert sys.version.startswith('3.11')"
-    - tox
+    - echo $KADITOKEN
+    - python -m pytest end-to-end-tests/test_kadi.py
+
 
 unittest_py3.8:
   tags: [cached-dind]
@@ -81,12 +62,12 @@ unittest_py3.8:
     - pip install pytest pytest-cov
     - pip install .
     # actual test
-    - pytest --cov=linkahead_python_package_template -vv ./unittests
+    - pytest --cov=ruqad -vv ./unittests
 
 unittest_py3.9:
   tags: [cached-dind]
   stage: test
-  image: python:3.11
+  image: python:3.9
   script: *python_test_script
 
 unittest_py3.10:
@@ -95,6 +76,12 @@ unittest_py3.10:
   image: python:3.10
   script: *python_test_script
 
+unittest_py3.11:
+  tags: [cached-dind]
+  stage: test
+  image: python:3.11
+  script: *python_test_script
+
 unittest_py3.12:
   tags: [cached-dind]
   stage: test
@@ -102,10 +89,9 @@ unittest_py3.12:
   script: *python_test_script
 
 unittest_py3.13:
-  allow_failure: true
   tags: [cached-dind]
   stage: test
-  image: python:3.13-rc
+  image: python:3.13
   script: *python_test_script
 
 build-testenv:
@@ -121,52 +107,12 @@ build-testenv:
   script:
       - df -h
       - command -v wget
-      - if [ -z "$PYLIB" ]; then
-          if echo "$CI_COMMIT_REF_NAME" | grep -c "^f-" ; then
-            echo "Check if pylib has branch $CI_COMMIT_REF_NAME" ;
-            if wget https://gitlab.indiscale.com/api/v4/projects/97/repository/branches/${CI_COMMIT_REF_NAME} ; then
-              PYLIB=$CI_COMMIT_REF_NAME ;
-            fi;
-          fi;
-        fi;
-      - PYLIB=${PYLIB:-dev}
-      - echo $PYLIB
-
-      - if [ -z "$ADVANCED" ]; then
-          if echo "$CI_COMMIT_REF_NAME" | grep -c "^f-" ; then
-            echo "Check if advanced user tools have branch $CI_COMMIT_REF_NAME" ;
-            if wget https://gitlab.indiscale.com/api/v4/projects/104/repository/branches/${CI_COMMIT_REF_NAME} ; then
-              ADVANCED=$CI_COMMIT_REF_NAME ;
-            fi;
-          fi;
-        fi;
-      - ADVANCED=${ADVANCED:-dev}
-      - echo $ADVANCED
-
       - docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY
         # use here general latest or specific branch latest...
       - docker build
-        --build-arg PYLIB=${PYLIB}
-        --build-arg ADVANCED=${ADVANCED:dev}
-        --file .docker/Dockerfile
+        --file docker/Dockerfile
         -t $CI_REGISTRY_IMAGE .
       - docker push $CI_REGISTRY_IMAGE
-      - docker save $CI_REGISTRY_IMAGE > /image-cache/linkahead-python-package-template-testenv-${CI_COMMIT_REF_NAME}.tar
-
-cert:
-  tags: [docker]
-  stage: cert
-  image: $CI_REGISTRY_IMAGE
-  needs:
-    - job: build-testenv
-      optional: true
-  artifacts:
-    paths:
-      - .docker/cert/
-    expire_in: 1 week
-  script:
-      - cd .docker
-      - CAOSHOSTNAME=caosdb-server ./cert.sh
 
 code-style:
   tags: [docker]
@@ -188,29 +134,5 @@ pylint:
       optional: true
   allow_failure: true
   script:
-    - pylint --unsafe-load-any-extension=y -d all -e E,F src/linkahead_python_package_template
+    - pylint --unsafe-load-any-extension=y -d all -e E,F src/ruqad
 
-# Build the sphinx documentation and make it ready for deployment by Gitlab Pages
-# Special job for serving a static website. See https://docs.gitlab.com/ee/ci/yaml/README.html#pages
-# Based on: https://gitlab.indiscale.com/caosdb/src/caosdb-pylib/-/ci/editor?branch_name=main
-pages_prepare: &pages_prepare
-  tags: [ cached-dind ]
-  stage: deploy
-  needs:
-    - job: build-testenv
-  image: $CI_REGISTRY_IMAGE
-  only:
-    refs:
-      - /^release-.*$/i
-  script:
-    - echo "Deploying documentation"
-    - make doc
-    - cp -r build/doc/html public
-  artifacts:
-    paths:
-      - public
-pages:
-  <<: *pages_prepare
-  only:
-    refs:
-      - main

docker/Dockerfile

+FROM python:3.13
+
+RUN pip install pytest autopep8 pylint
+COPY ./src /ruqad/src
+COPY ./unittests /ruqad/unittests
+COPY ./end-to-end-tests /ruqad/end-to-end-tests
+COPY ./pyproject.toml /ruqad/
+RUN cd /ruqad && pip install .
+CMD python -m ruqad.monitor

end-to-end-tests/test_kadi.py

@@ -21,9 +21,11 @@ import os
 from datetime import datetime
 from tempfile import NamedTemporaryFile
 import zipfile
+from uuid import uuid1
 
 from ruqad.kadi import collect_records_created_after, download_eln_for
 from kadi_apy import KadiManager
+from time import sleep
 
 KADIARGS = {
     "host": "https://demo-kadi4mat.iam.kit.edu",
@@ -35,16 +37,22 @@ def test_collect():
     """
     queries data from the Kadi demo instance and checks whether the cut-off date is used correctly
     """
+    new1, new2, newest = None, None, None
+    with KadiManager(**KADIARGS) as manager:
+        query_params = {"per_page": 1, "sort": "-created_at"}
+        newest = manager.search.search_resources("record", **query_params).json()["items"][0]
+        cut_off_date = datetime.fromisoformat(newest["created_at"])
+        new1 = manager.record(identifier=str(uuid1()), create=True)
+        new2 = manager.record(identifier=str(uuid1()), create=True)
+    sleep(15)
     with KadiManager(**KADIARGS) as manager:
-        cut_off_date = datetime.fromisoformat(
-            "2024-10-01 02:34:42.484312+00:00")
         rec_ids = collect_records_created_after(manager, cut_off_date)
-        known_new_recs = [664, 656, 641, 640, 639, 638, 637]
+        known_new_recs = [int(newest["id"]), new1.id, new2.id]
         for knr in known_new_recs:
-            assert knr in rec_ids, "when the sample data changes, this test may fail"
-        known_old_recs = [158,636, 1]
+            assert knr in rec_ids
+        known_old_recs = [158, newest["id"]-1, 1]
         for knr in known_old_recs:
-            assert knr not in rec_ids, "when the sample data changes, this test may fail"
+            assert knr not in rec_ids
 
 
 def test_download():

src/ruqad/kadi.py

@@ -39,6 +39,9 @@ def _generate_pages(manager) -> dict:
     query_params = {"per_page": PAGE_SIZE, "sort": "-created_at"}
     # test search to get number of pages.
     response = manager.search.search_resources("record", **query_params).json()
+    if 'code' in response and response['code'] != 200:
+        raise RuntimeError("Could not search Kadi. Connection returned code:"
+                           + str(response['code']))
     n_pages = response["_pagination"]["total_pages"]
     for ii in range(n_pages):
         query_params.update({"page": ii+1})

src/ruqad/monitor.py

+"""
+monitor the kadi instance
+"""
+import os
+from time import sleep
+from tempfile import NamedTemporaryFile
+import traceback
+from datetime import datetime, timezone
+from kadi_apy import KadiManager
+
+from .kadi import collect_records_created_after, download_eln_for
+
+
+KADIARGS = {
+    "host": os.environ['KADIHOST'],
+    "pat": os.environ['KADITOKEN']
+}
+
+
+if __name__ == "__main__":
+    cut_off_date = datetime.fromisoformat("1990-01-01 02:34:42.484312+00:00")
+    while True:
+        try:
+            timestamp = datetime.now(timezone.utc)
+            with KadiManager(**KADIARGS) as manager:
+                print(f"Checking for records created after {cut_off_date}...")
+                rec_ids = collect_records_created_after(manager, cut_off_date)
+                cut_off_date = timestamp
+
+                if len(rec_ids) > 5:
+                    print("skipping, too many recs: ", len(rec_ids))
+                    continue
+                if len(rec_ids) == 0:
+                    print("no new recs")
+                for rid in rec_ids:
+                    temp = NamedTemporaryFile(delete=False)
+                    temp.close()
+                    download_eln_for(manager, rid, path=temp.name)
+                    print(temp.name)
+            sleep(5)
+
+        except KeyboardInterrupt as e:
+            raise e
+        except Exception as e:
+            print("ERROR")
+            print(traceback.format_exc())
+            print(e)

tox.ini

+[tox]
+envlist = py38, py39, py310, py311, py312, py313
+skip_missing_interpreters = true
+
+[testenv]
+deps = .
+
+[flake8]
+max-line-length = 100
+
+[pycodestyle]
+max-line-length = 100
+
+[pytest]
+testpaths = unittests
+xfail_strict = True