Skip to content
Snippets Groups Projects
Commit 1cf97e10 authored by Alexander Schlemmer's avatar Alexander Schlemmer
Browse files

TST: end to end test for license check

parent 28f5a610
No related branches found
No related tags found
1 merge request!7FAIRness check
...@@ -18,6 +18,7 @@ ...@@ -18,6 +18,7 @@
tests the crawling of ELN files tests the crawling of ELN files
""" """
import os import os
import re
from pathlib import Path from pathlib import Path
from ruqad.crawler import trigger_crawler from ruqad.crawler import trigger_crawler
...@@ -30,5 +31,18 @@ def test_crawl(): ...@@ -30,5 +31,18 @@ def test_crawl():
crawl a directory as it would be created by export from kadi and running a data quality check crawl a directory as it would be created by export from kadi and running a data quality check
""" """
print(os.listdir(DATADIR)) print(os.listdir(DATADIR))
retval = trigger_crawler(os.fspath(DATADIR)) retval, ent_qc = trigger_crawler(os.fspath(DATADIR))
# Check that validation of metadata was successful:
assert retval assert retval
# Check that license was present in 1223 and absent in 1222:
qc = {}
for ent in ent_qc:
pth = ent.get_property("ELNFile").value.path
match = re.match("/.*/.*/(?P<folder>[0-9]+)/.*\.eln", pth)
assert match is not None
qc[match.group("folder")] = ent
assert qc["1223"].get_property("FAIRLicenseCheck").value
assert not qc["1222"].get_property("FAIRLicenseCheck").value
...@@ -18,7 +18,7 @@ from caoscrawler.validator import (load_json_schema_from_datamodel_yaml, ...@@ -18,7 +18,7 @@ from caoscrawler.validator import (load_json_schema_from_datamodel_yaml,
ruqad_crawler_settings = resources.files('ruqad').joinpath('resources/crawler-settings') ruqad_crawler_settings = resources.files('ruqad').joinpath('resources/crawler-settings')
def trigger_crawler(target_dir: str) -> bool: def trigger_crawler(target_dir: str) -> tuple[bool, list[db.Entity]]:
""" """
Trigger a standard crawler run equivalent to the command line: Trigger a standard crawler run equivalent to the command line:
...@@ -26,7 +26,10 @@ def trigger_crawler(target_dir: str) -> bool: ...@@ -26,7 +26,10 @@ def trigger_crawler(target_dir: str) -> bool:
caosdb-crawler -i crawler/identifiables.yaml -s update crawler/cfood.yaml <target_dir> caosdb-crawler -i crawler/identifiables.yaml -s update crawler/cfood.yaml <target_dir>
``` ```
Return False in case of unsuccessful metadata validation and True otherwise. A tuple:
- 1st element of tuple: Return False in case of unsuccessful metadata validation
and True otherwise.
- 2nd element of tuple: list of quality check records.
""" """
# insert all .zip and .eln files, if they do not yet exist # insert all .zip and .eln files, if they do not yet exist
...@@ -53,6 +56,17 @@ def trigger_crawler(target_dir: str) -> bool: ...@@ -53,6 +56,17 @@ def trigger_crawler(target_dir: str) -> bool:
entities = scan_directory(target_dir, entities = scan_directory(target_dir,
ruqad_crawler_settings.joinpath('cfood.yaml')) ruqad_crawler_settings.joinpath('cfood.yaml'))
ent_qc = []
# Show warning if license is not present in an eln file:
for ent in entities:
if not (len(ent.parents) == 1 and ent.parents[0].name == "QualityCheck"):
continue
ent_qc.append(ent)
if not ent.get_property("FAIRLicenseCheck").value:
print("{} does not contain a license.".format(ent.get_property("ELNFile").value.path))
# Remove files from entities: # Remove files from entities:
records = [r for r in entities if r.role == "Record"] records = [r for r in entities if r.role == "Record"]
validation = validate(records, schemas) validation = validate(records, schemas)
...@@ -62,7 +76,7 @@ def trigger_crawler(target_dir: str) -> bool: ...@@ -62,7 +76,7 @@ def trigger_crawler(target_dir: str) -> bool:
for v, recordtype in zip(validation, schemas.keys()): for v, recordtype in zip(validation, schemas.keys()):
if not v[0]: if not v[0]:
print("{}: {}".format(recordtype, v[1])) print("{}: {}".format(recordtype, v[1]))
return False return (False, ent_qc)
print("crawl", target_dir) print("crawl", target_dir)
crawler_main(crawled_directory_path=target_dir, crawler_main(crawled_directory_path=target_dir,
...@@ -71,4 +85,4 @@ def trigger_crawler(target_dir: str) -> bool: ...@@ -71,4 +85,4 @@ def trigger_crawler(target_dir: str) -> bool:
'identifiables.yaml'), 'identifiables.yaml'),
remove_prefix="/"+os.path.basename(target_dir)) remove_prefix="/"+os.path.basename(target_dir))
return True return (True, ent_qc)
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment