Skip to content
Snippets Groups Projects
Commit 7218f73c authored by Florian Spreckelsen's avatar Florian Spreckelsen
Browse files

Merge branch 'f-more-scifolder-scanner-unittests' into 'dev'

More scifolder scanner unittests

See merge request !124
parents bf50e29d f6b11c60
No related branches found
No related tags found
2 merge requests!160STY: styling,!124More scifolder scanner unittests
Pipeline #39803 passed
......@@ -35,15 +35,15 @@ from utils import dircheckstr as dircheck_base
UNITTESTDIR = Path(__file__).parent
dircheckstr = partial(dircheck_base, UNITTESTDIR/"test_directories" / "examples_article")
dircheckstr = partial(dircheck_base, UNITTESTDIR / "test_directories" / "examples_article")
def test_scan_structure_elements():
tmpfi = NamedTemporaryFile(delete=False)
with open(UNITTESTDIR/"example_datastructure.yml", "r") as f:
with open(UNITTESTDIR / "example_datastructure.yml", "r") as f:
data = yaml.load(f, Loader=yaml.SafeLoader)
crawler_definition = load_definition(UNITTESTDIR/"example_cfood.yml")
crawler_definition = load_definition(UNITTESTDIR / "example_cfood.yml")
converter_registry = create_converter_registry(crawler_definition)
recs = scan_structure_elements(DictElement(name="", value=data), crawler_definition,
converter_registry)
......@@ -54,10 +54,10 @@ def test_provenance_debug_data():
# TODO rewrite the test to use a smaller example setup
tmpfi = NamedTemporaryFile(delete=False)
debug_tree = DebugTree()
with open(UNITTESTDIR/"example_datastructure.yml", "r") as f:
with open(UNITTESTDIR / "example_datastructure.yml", "r") as f:
data = yaml.load(f, Loader=yaml.SafeLoader)
crawler_definition = load_definition(UNITTESTDIR/"example_cfood.yml")
crawler_definition = load_definition(UNITTESTDIR / "example_cfood.yml")
converter_registry = create_converter_registry(crawler_definition)
stuff = scan_structure_elements(DictElement(name="", value=data), crawler_definition,
converter_registry, debug_tree=debug_tree)
......@@ -83,8 +83,8 @@ def test_record_structure_generation():
# TODO test creation of debug information in a separate test
dbt = DebugTree()
scan_directory(UNITTESTDIR/"test_directories" / "examples_article",
UNITTESTDIR/"scifolder_cfood.yml",
scan_directory(UNITTESTDIR / "test_directories" / "examples_article",
UNITTESTDIR / "scifolder_cfood.yml",
debug_tree=dbt)
subd = dbt.debug_tree[dircheckstr("DataAnalysis")]
subc = dbt.debug_metadata["copied"][dircheckstr("DataAnalysis")]
......@@ -167,3 +167,81 @@ def test_record_structure_generation():
assert subc[0]["Measurement"] is False
assert subc[0]["date"] is False
assert subc[0]["identifier"] is False
def test_record_generation():
"""
Test the correct list of returned records by the scanner using the
scifolder example from the article.
"""
records = scan_directory(UNITTESTDIR / "test_directories" / "examples_article",
UNITTESTDIR / "scifolder_cfood.yml")
def parent_filter(parent_name):
return [p for p in records if len(p.parents) == 1 and p.parents[0].name == parent_name]
def check_properties(records, check_props, check_additional=True):
records_found = [0 for r in check_props]
for rec in records:
rec_found = 0
# Try each record to check
for i, check_prop in enumerate(check_props):
matches = True
# breakpoint()
# Verify that all props are in the record and have the right value
for pr in check_prop:
if rec.get_property(pr) is None:
matches = False
break
if check_prop[pr] is None:
if rec.get_property(pr).value is not None:
matches = False
break
else:
if rec.get_property(pr).value != check_prop[pr]:
matches = False
break
if check_additional:
# Verify that there are no additional props in the record
for rpr in rec.properties:
if rpr.name not in check_prop:
matches = False
break
if matches:
records_found[i] += 1
return records_found
# Check projects:
# Ther are two projects in mixed categories: climate_model_predict and SpeedOfLight
projects_found = check_properties(parent_filter("Project"), [
{"identifier": "climate-model-predict", "date": "2020"},
{"identifier": "SpeedOfLight", "date": "2020"}
])
assert projects_found == [3, 2]
measurements = parent_filter("Measurement")
assert len(measurements) == 11
measurements_found = check_properties(measurements, [
{"identifier": "prediction-errors", "date": "2020-02-08"},
{"identifier": "average-all-exp", "date": "2020-01-04"},
{"identifier": "average-all-exp-corr", "date": "2020-01-05"},
{"date": "1980-01-01", "identifier": None},
{"date": "1990-01-01", "identifier": None},
{"date": "2000-01-01", "identifier": None},
{"date": "2010-01-01", "identifier": None},
{"date": "2020-01-01", "identifier": "TimeOfFlight"},
{"date": "2020-01-02", "identifier": "Cavity"},
{"date": "2020-01-03", "identifier": None},
{"date": "2020-02-01", "identifier": None},
], False)
for f in measurements_found:
assert f == 1
persons = parent_filter("Person")
check_props = [
{"first_name": None, "last_name": "Author" + letter} for letter in
("A", "B", "C", "D", "E")]
persons_found = check_properties(persons, check_props)
for f in persons_found:
assert f > 0
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment