diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 1944b7062da4fab02dffe185215115e039514d92..d299aa7f830e460c3925c34db8a21187e6db69fc 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -19,8 +19,7 @@
# along with this program. If not, see <https://www.gnu.org/licenses/>.
variables:
- CI_REGISTRY_IMAGE: $CI_REGISTRY/caosdb/src/linkahead-python-package-template/testenv:$CI_COMMIT_REF_NAME
- CI_REGISTRY_IMAGE_BASE: $CI_REGISTRY/caosdb/src/caosdb-pyinttest/base:latest
+ CI_REGISTRY_IMAGE: $CI_REGISTRY/caosdb/customers/f-fit/ruqad/ruqad-demonstrator-fair-ds:$CI_COMMIT_REF_NAME
stages:
- info
@@ -32,30 +31,10 @@ stages:
.env: &env
- echo "Pipeline triggered by $TRIGGERED_BY_REPO@$TRIGGERED_BY_REF ($TRIGGERED_BY_HASH)"
- - echo "CI_REGISTRY_IMAGE_BASE = $CI_REGISTRY_IMAGE_BASE"
- echo "CI_REGISTRY_IMAGE = $CI_REGISTRY_IMAGE"
- - echo "CAOSDB_TAG = $CAOSDB_TAG"
- - echo "REFTAG = $REFTAG"
- - echo "F_BRANCH = $F_BRANCH"
- echo "CI_COMMIT_REF_NAME = $CI_COMMIT_REF_NAME"
- ls -lah /image-cache/
- - F_BRANCH=${F_BRANCH:-$CI_COMMIT_REF_NAME}
- - echo $F_BRANCH
- - if [[ "$REFTAG" == "" ]] ; then
- if [[ "$F_BRANCH" == "dev" ]] ; then
- REFTAG=dev;
- fi;
- fi
- - REFTAG=${REFTAG:-dev_F_${F_BRANCH}}
-
- - echo $F_BRANCH
-
- - if [[ "$CAOSDB_TAG" == "" ]]; then
- CAOSDB_TAG=${REFTAG};
- fi
- - echo $CAOSDB_TAG
-
info:
tags: [cached-dind]
image: docker:20.10
@@ -64,13 +43,15 @@ info:
script:
- *env
-unittest_py3.11:
+e2e_tests:
tags: [cached-dind]
stage: test
image: $CI_REGISTRY_IMAGE
+ #image: python:3.13
script:
- - python3 -c "import sys; assert sys.version.startswith('3.11')"
- - tox
+ - echo $KADITOKEN
+ - python -m pytest end-to-end-tests/test_kadi.py
+
unittest_py3.8:
tags: [cached-dind]
@@ -81,12 +62,12 @@ unittest_py3.8:
- pip install pytest pytest-cov
- pip install .
# actual test
- - pytest --cov=linkahead_python_package_template -vv ./unittests
+ - pytest --cov=ruqad -vv ./unittests
unittest_py3.9:
tags: [cached-dind]
stage: test
- image: python:3.11
+ image: python:3.9
script: *python_test_script
unittest_py3.10:
@@ -95,6 +76,12 @@ unittest_py3.10:
image: python:3.10
script: *python_test_script
+unittest_py3.11:
+ tags: [cached-dind]
+ stage: test
+ image: python:3.11
+ script: *python_test_script
+
unittest_py3.12:
tags: [cached-dind]
stage: test
@@ -102,10 +89,9 @@ unittest_py3.12:
script: *python_test_script
unittest_py3.13:
- allow_failure: true
tags: [cached-dind]
stage: test
- image: python:3.13-rc
+ image: python:3.13
script: *python_test_script
build-testenv:
@@ -121,52 +107,12 @@ build-testenv:
script:
- df -h
- command -v wget
- - if [ -z "$PYLIB" ]; then
- if echo "$CI_COMMIT_REF_NAME" | grep -c "^f-" ; then
- echo "Check if pylib has branch $CI_COMMIT_REF_NAME" ;
- if wget https://gitlab.indiscale.com/api/v4/projects/97/repository/branches/${CI_COMMIT_REF_NAME} ; then
- PYLIB=$CI_COMMIT_REF_NAME ;
- fi;
- fi;
- fi;
- - PYLIB=${PYLIB:-dev}
- - echo $PYLIB
-
- - if [ -z "$ADVANCED" ]; then
- if echo "$CI_COMMIT_REF_NAME" | grep -c "^f-" ; then
- echo "Check if advanced user tools have branch $CI_COMMIT_REF_NAME" ;
- if wget https://gitlab.indiscale.com/api/v4/projects/104/repository/branches/${CI_COMMIT_REF_NAME} ; then
- ADVANCED=$CI_COMMIT_REF_NAME ;
- fi;
- fi;
- fi;
- - ADVANCED=${ADVANCED:-dev}
- - echo $ADVANCED
-
- docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY
# use here general latest or specific branch latest...
- docker build
- --build-arg PYLIB=${PYLIB}
- --build-arg ADVANCED=${ADVANCED:dev}
- --file .docker/Dockerfile
+ --file docker/Dockerfile
-t $CI_REGISTRY_IMAGE .
- docker push $CI_REGISTRY_IMAGE
- - docker save $CI_REGISTRY_IMAGE > /image-cache/linkahead-python-package-template-testenv-${CI_COMMIT_REF_NAME}.tar
-
-cert:
- tags: [docker]
- stage: cert
- image: $CI_REGISTRY_IMAGE
- needs:
- - job: build-testenv
- optional: true
- artifacts:
- paths:
- - .docker/cert/
- expire_in: 1 week
- script:
- - cd .docker
- - CAOSHOSTNAME=caosdb-server ./cert.sh
code-style:
tags: [docker]
@@ -188,29 +134,5 @@ pylint:
optional: true
allow_failure: true
script:
- - pylint --unsafe-load-any-extension=y -d all -e E,F src/linkahead_python_package_template
-
-# Build the sphinx documentation and make it ready for deployment by Gitlab Pages
-# Special job for serving a static website. See https://docs.gitlab.com/ee/ci/yaml/README.html#pages
-# Based on: https://gitlab.indiscale.com/caosdb/src/caosdb-pylib/-/ci/editor?branch_name=main
-pages_prepare: &pages_prepare
- tags: [ cached-dind ]
- stage: deploy
- needs:
- - job: build-testenv
- image: $CI_REGISTRY_IMAGE
- only:
- refs:
- - /^release-.*$/i
- script:
- - echo "Deploying documentation"
- - make doc
- - cp -r build/doc/html public
- artifacts:
- paths:
- - public
-pages:
- <<: *pages_prepare
- only:
- refs:
- - main
+ - pylint --unsafe-load-any-extension=y -d all -e E,F src/ruqad
+
diff --git a/docker/Dockerfile b/docker/Dockerfile
new file mode 100644
index 0000000000000000000000000000000000000000..cab341deb1c6d513ed9d290aa2bd36bc7ee7c3e7
--- /dev/null
+++ b/docker/Dockerfile
@@ -0,0 +1,9 @@
+FROM python:3.13
+
+RUN pip install pytest autopep8 pylint
+COPY ./src /ruqad/src
+COPY ./unittests /ruqad/unittests
+COPY ./end-to-end-tests /ruqad/end-to-end-tests
+COPY ./pyproject.toml /ruqad/
+RUN cd /ruqad && pip install .
+CMD python -m ruqad.monitor
diff --git a/end-to-end-tests/test_kadi.py b/end-to-end-tests/test_kadi.py
index 16324ff552b0b02aca0acee0f5d60f9a72173ad4..30b89bd8b653923b603315ac4587ec92cce7a7f0 100644
--- a/end-to-end-tests/test_kadi.py
+++ b/end-to-end-tests/test_kadi.py
@@ -21,9 +21,11 @@ import os
from datetime import datetime
from tempfile import NamedTemporaryFile
import zipfile
+from uuid import uuid1
from ruqad.kadi import collect_records_created_after, download_eln_for
from kadi_apy import KadiManager
+from time import sleep
KADIARGS = {
"host": "https://demo-kadi4mat.iam.kit.edu",
@@ -35,16 +37,22 @@ def test_collect():
"""
queries data from the Kadi demo instance and checks whether the cut-off date is used correctly
"""
+ new1, new2, newest = None, None, None
+ with KadiManager(**KADIARGS) as manager:
+ query_params = {"per_page": 1, "sort": "-created_at"}
+ newest = manager.search.search_resources("record", **query_params).json()["items"][0]
+ cut_off_date = datetime.fromisoformat(newest["created_at"])
+ new1 = manager.record(identifier=str(uuid1()), create=True)
+ new2 = manager.record(identifier=str(uuid1()), create=True)
+ sleep(15)
with KadiManager(**KADIARGS) as manager:
- cut_off_date = datetime.fromisoformat(
- "2024-10-01 02:34:42.484312+00:00")
rec_ids = collect_records_created_after(manager, cut_off_date)
- known_new_recs = [664, 656, 641, 640, 639, 638, 637]
+ known_new_recs = [int(newest["id"]), new1.id, new2.id]
for knr in known_new_recs:
- assert knr in rec_ids, "when the sample data changes, this test may fail"
- known_old_recs = [158,636, 1]
+ assert knr in rec_ids
+ known_old_recs = [158, newest["id"]-1, 1]
for knr in known_old_recs:
- assert knr not in rec_ids, "when the sample data changes, this test may fail"
+ assert knr not in rec_ids
def test_download():
diff --git a/src/ruqad/kadi.py b/src/ruqad/kadi.py
index 4b6c95a10aa0fdb3421bff436c8d43e5e59cd267..f4ea8231df01c97b5b8b03dd5a61fd213ef60bfe 100644
--- a/src/ruqad/kadi.py
+++ b/src/ruqad/kadi.py
@@ -39,6 +39,9 @@ def _generate_pages(manager) -> dict:
query_params = {"per_page": PAGE_SIZE, "sort": "-created_at"}
# test search to get number of pages.
response = manager.search.search_resources("record", **query_params).json()
+ if 'code' in response and response['code'] != 200:
+ raise RuntimeError("Could not search Kadi. Connection returned code:"
+ + str(response['code']))
n_pages = response["_pagination"]["total_pages"]
for ii in range(n_pages):
query_params.update({"page": ii+1})
diff --git a/src/ruqad/monitor.py b/src/ruqad/monitor.py
new file mode 100644
index 0000000000000000000000000000000000000000..fca58580d7214729511bc4343e41636999c82ebe
--- /dev/null
+++ b/src/ruqad/monitor.py
@@ -0,0 +1,47 @@
+"""
+monitor the kadi instance
+"""
+import os
+from time import sleep
+from tempfile import NamedTemporaryFile
+import traceback
+from datetime import datetime, timezone
+from kadi_apy import KadiManager
+
+from .kadi import collect_records_created_after, download_eln_for
+
+
+KADIARGS = {
+ "host": os.environ['KADIHOST'],
+ "pat": os.environ['KADITOKEN']
+}
+
+
+if __name__ == "__main__":
+ cut_off_date = datetime.fromisoformat("1990-01-01 02:34:42.484312+00:00")
+ while True:
+ try:
+ timestamp = datetime.now(timezone.utc)
+ with KadiManager(**KADIARGS) as manager:
+ print(f"Checking for records created after {cut_off_date}...")
+ rec_ids = collect_records_created_after(manager, cut_off_date)
+ cut_off_date = timestamp
+
+ if len(rec_ids) > 5:
+ print("skipping, too many recs: ", len(rec_ids))
+ continue
+ if len(rec_ids) == 0:
+ print("no new recs")
+ for rid in rec_ids:
+ temp = NamedTemporaryFile(delete=False)
+ temp.close()
+ download_eln_for(manager, rid, path=temp.name)
+ print(temp.name)
+ sleep(5)
+
+ except KeyboardInterrupt as e:
+ raise e
+ except Exception as e:
+ print("ERROR")
+ print(traceback.format_exc())
+ print(e)
diff --git a/tox.ini b/tox.ini
new file mode 100644
index 0000000000000000000000000000000000000000..09fa08a0c811f4b251f09f50d481fc9486c94658
--- /dev/null
+++ b/tox.ini
@@ -0,0 +1,16 @@
+[tox]
+envlist = py38, py39, py310, py311, py312, py313
+skip_missing_interpreters = true
+
+[testenv]
+deps = .
+
+[flake8]
+max-line-length = 100
+
+[pycodestyle]
+max-line-length = 100
+
+[pytest]
+testpaths = unittests
+xfail_strict = True