Skip to content
Snippets Groups Projects
Commit b9f6b56a authored by Henrik tom Wörden's avatar Henrik tom Wörden
Browse files

Merge branch 'f-kadi' into 'main'

Add Kadi connection

See merge request caosdb/customers/f-fit/ruqad!1
parents 998eeccd 948cdb49
Branches
Tags
1 merge request!1Add Kadi connection
Pipeline #57000 failed
...@@ -8,7 +8,12 @@ TODO ...@@ -8,7 +8,12 @@ TODO
### Unit Tests ### Unit Tests
Run `tox` or alternatively `pytest unittests/`. Run `pytest unittests/`.
### E2E Tests
In order to run the E2E test, you need to create a personal access token (pat) in the public
[demo instance](https://demo-kadi4mat.iam.kit.edu). You can then run the test as follows:
`KADITOKEN=<token> python -m pytest end-to-end-tests/test_kadi.py`
### Code style and liniting ### Code style and liniting
......
# Copyright (C) 2024 IndiScale GmbH <info@indiscale.com>
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as
# published by the Free Software Foundation, either version 3 of the
# License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <https://www.gnu.org/licenses/>.
#
"""
tests the interaction with the Kadi API
"""
import os
from datetime import datetime
from tempfile import NamedTemporaryFile
import zipfile
from ruqad.kadi import collect_records_created_after, download_eln_for
from kadi_apy import KadiManager
KADIARGS = {
"host": "https://demo-kadi4mat.iam.kit.edu",
"pat": os.environ['KADITOKEN']
}
def test_collect():
"""
queries data from the Kadi demo instance and checks whether the cut-off date is used correctly
"""
with KadiManager(**KADIARGS) as manager:
cut_off_date = datetime.fromisoformat(
"2024-10-01 02:34:42.484312+00:00")
rec_ids = collect_records_created_after(manager, cut_off_date)
known_new_recs = [664, 656, 641, 640, 639, 638, 637]
for knr in known_new_recs:
assert knr in rec_ids, "when the sample data changes, this test may fail"
known_old_recs = [158,636, 1]
for knr in known_old_recs:
assert knr not in rec_ids, "when the sample data changes, this test may fail"
def test_download():
""" downloads a record from the demo instance and checks that the download occurred correctly"""
temp = NamedTemporaryFile(delete=False)
temp.close()
with KadiManager(**KADIARGS) as manager:
download_eln_for(manager, 664, temp.name)
assert os.path.getsize(temp.name) > 0
assert zipfile.is_zipfile(temp.name)
...@@ -3,7 +3,7 @@ requires = ["setuptools >= 61.0"] ...@@ -3,7 +3,7 @@ requires = ["setuptools >= 61.0"]
build-backend = "setuptools.build_meta" build-backend = "setuptools.build_meta"
[project] [project]
name = "fair-dataspaces-ruqad" name = "ruqad"
description = "A tool for quality assured data integration from Kadi to LinkAhead." description = "A tool for quality assured data integration from Kadi to LinkAhead."
version = "0.0.1" version = "0.0.1"
readme = "README.md" readme = "README.md"
...@@ -23,7 +23,9 @@ classifiers = [ ...@@ -23,7 +23,9 @@ classifiers = [
] ]
requires-python = ">= 3.8" requires-python = ">= 3.8"
dependencies = [ dependencies = [
"linkahead" "linkahead",
"kadi-apy"
] ]
[project.urls] [project.urls]
...@@ -33,3 +35,9 @@ Repository = "https://gitlab.indiscale.com/caosdb/src/linkahead-python-package-t ...@@ -33,3 +35,9 @@ Repository = "https://gitlab.indiscale.com/caosdb/src/linkahead-python-package-t
Issues = "https://gitlab.indiscale.com/caosdb/src/linkahead-python-package-template/-/issues" Issues = "https://gitlab.indiscale.com/caosdb/src/linkahead-python-package-template/-/issues"
Changelog = "https://gitlab.indiscale.com/caosdb/src/linkahead-python-package-template/-/blob/main/CHANGELOG.md?ref_type=heads" Changelog = "https://gitlab.indiscale.com/caosdb/src/linkahead-python-package-template/-/blob/main/CHANGELOG.md?ref_type=heads"
[project.optional-dependencies]
dev = [
]
test = [
"pytest",
]
LinkkAhead Python Package Template LinkAhead Python Package Template
================================== ==================================
**TODO**: Write your documentation here. **TODO**: Write your documentation here.
# Copyright (C) 2024 IndiScale GmbH <info@indiscale.com>
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as
# published by the Free Software Foundation, either version 3 of the
# License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <https://www.gnu.org/licenses/>.
#
"""
utilities to create .eln exports for certain records hosted in a Kadi instance
"""
from __future__ import annotations
from kadi_apy import KadiManager
from datetime import datetime
PAGE_SIZE = 100
def _generate_pages(manager) -> dict:
"""
Generates JSON responses (dict) that represent single pages returned by the Kadi API
Paremters
---------
manager: KadiManager, KadiManager instance used to connect to the Kadi API
Returns
-------
dict, the JSON response as dict
"""
query_params = {"per_page": PAGE_SIZE, "sort": "-created_at"}
# test search to get number of pages.
response = manager.search.search_resources("record", **query_params).json()
n_pages = response["_pagination"]["total_pages"]
for ii in range(n_pages):
query_params.update({"page": ii+1})
yield manager.search.search_resources("record", **query_params).json()
def collect_records_created_after(manager: KadiManager, cut_off_date: datetime.datetime) -> list(int):
"""
Iterates page-wise over the responses of the Kadi API until records are reached that are older
than the given cut_off_date.
Paremters
---------
manager: KadiManager, KadiManager instance used to connect to the Kadi API
cut_off_date: datetime, Records that were created after this date are included in the returned
list
Returns
-------
list(int), list of IDs of records that were created after the given cut_off_date
"""
record_ids = []
done = False
for response in _generate_pages(manager):
for el in response["items"]:
if cut_off_date > datetime.fromisoformat(el["created_at"]):
done = True
break
record_ids.append(el["id"])
if done:
break
return record_ids
def download_eln_for(manager: KadiManager, rid: int, path: str) -> None:
"""
Downloads the record with the given ID as '.eln' file and stores it in the given path.
Paremters
---------
manager: KadiManager, KadiManager instance used to connect to the Kadi API
rid: int, ID of the record to be exported
path: str, the path where the file will be stored
"""
rec = manager.record(id=rid)
rec.export(path=path, export_type='ro-crate')
def main():
with KadiManager(instance='demo') as manager:
cut_off_date = datetime.fromisoformat(
"2024-10-01 02:34:42.484312+00:00")
rec_ids = collect_records_created_after(manager, cut_off_date)
print(rec_ids)
if __name__ == "__main__":
main()
[tox]
envlist = py38, py39, py310, py311, py312, py313
skip_missing_interpreters = true
[testenv]
deps = .
pytest
pytest-cov
commands =
py.test --cov=ruqad -vv {posargs}
[flake8]
max-line-length = 100
[pycodestyle]
max-line-length = 100
[pytest]
testpaths = unittests
xfail_strict = True
#
# This file is a part of the LinkAhead Project.
#
# Copyright (C) 2024 IndiScale GmbH <info@indiscale.com> # Copyright (C) 2024 IndiScale GmbH <info@indiscale.com>
# #
# This program is free software: you can redistribute it and/or modify # This program is free software: you can redistribute it and/or modify
...@@ -16,7 +13,34 @@ ...@@ -16,7 +13,34 @@
# You should have received a copy of the GNU Affero General Public License # You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <https://www.gnu.org/licenses/>. # along with this program. If not, see <https://www.gnu.org/licenses/>.
# #
from ruqad import kadi
from datetime import datetime
from unittest.mock import patch, Mock
def mock_generator(manager):
""" Mocks the server response. We only need the keys: "items", "created_at", and "id"."""
pages = [
{"items": [
{"created_at": "2024-01-02 01:00:00.000000+00:00",
"id": 1},
{"created_at": "2024-01-02 01:00:00.000000+00:00",
"id": 2}
]}, {"items": [
{"created_at": "2024-01-01 01:00:00.000000+00:00",
"id": 3},
{"created_at": "2024-01-01 01:00:00.000000+00:00",
"id": 4}
]}
]
for el in pages:
yield el
def test_template():
assert True @patch("ruqad.kadi._generate_pages", new=Mock(side_effect=mock_generator))
def test_collect_records_created_after():
# we set a cut-off date after half of the records and check that we get the correct ids
assert [1, 2] == kadi.collect_records_created_after(manager=None,
cut_off_date=datetime.fromisoformat(
"2024-01-01 03:00:00.000000+00:00")
)
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment