Skip to content
Snippets Groups Projects
Commit 6bdac91d authored by Alexander Schlemmer's avatar Alexander Schlemmer
Browse files

TST: update of file tests

parent e62921cc
No related branches found
No related tags found
1 merge request!53Release 0.1
...@@ -85,3 +85,4 @@ Measurement: ...@@ -85,3 +85,4 @@ Measurement:
date: date:
ReadmeFile: ReadmeFile:
datatype: REFERENCE datatype: REFERENCE
ProjectMarkdownReadme:
...@@ -102,6 +102,11 @@ def crawler(ident): ...@@ -102,6 +102,11 @@ def crawler(ident):
def crawler_extended(ident): def crawler_extended(ident):
cr = Crawler(debug=True, identifiableAdapter=ident) cr = Crawler(debug=True, identifiableAdapter=ident)
crawl_standard_test_directory(cr, cfood="scifolder_extended.yml") crawl_standard_test_directory(cr, cfood="scifolder_extended.yml")
# correct paths for current working directory
updateList = cr.updateList
fileList = [r for r in updateList if r.role == "File"]
for f in fileList:
f.file = rfp("..", "unittests", "test_directories", "examples_article", f.file)
return cr return cr
...@@ -208,7 +213,9 @@ def test_identifiable_update(clear_database, usemodel, ident, crawler): ...@@ -208,7 +213,9 @@ def test_identifiable_update(clear_database, usemodel, ident, crawler):
assert len(ups) == 0 assert len(ups) == 0
def test_file_insertion_dry(clear_database, usemodel, ident, crawler_extended): def test_file_insertion_dry(clear_database, usemodel, ident):
crawler_extended = Crawler(debug=True, identifiableAdapter=ident)
crawl_standard_test_directory(crawler_extended, cfood="scifolder_extended.yml")
updateList = crawler_extended.updateList updateList = crawler_extended.updateList
fileList = [r for r in updateList if r.role == "File"] fileList = [r for r in updateList if r.role == "File"]
assert len(fileList) == 11 assert len(fileList) == 11
...@@ -224,14 +231,56 @@ def test_file_insertion_dry(clear_database, usemodel, ident, crawler_extended): ...@@ -224,14 +231,56 @@ def test_file_insertion_dry(clear_database, usemodel, ident, crawler_extended):
def test_file_insertion(clear_database, usemodel, ident, crawler_extended): def test_file_insertion(clear_database, usemodel, ident, crawler_extended):
# correct paths for current working directory
updateList = crawler_extended.updateList
fileList = [r for r in updateList if r.role == "File"]
for f in fileList:
f.file = rfp("..", "unittests", "test_directories", "examples_article", f.file)
ins, ups = crawler_extended.synchronize(commit_changes=True) ins, ups = crawler_extended.synchronize(commit_changes=True)
fileList_ins = [r for r in ins if r.role == "File"] fileList_ins = [r for r in ins if r.role == "File"]
assert len(fileList_ins) == 11 assert len(fileList_ins) == 11
assert db.execute_query("COUNT File") > 0 assert db.execute_query("COUNT File") > 0
# find record which references File does not seem to be possible
# retrieve ids of files:
files = db.execute_query("FIND File")
for f in files:
r = db.execute_query("FIND Record which references {}".format(f.id))
assert len(r) == 1
assert r[0].get_property("ReadmeFile").value == f.id
def test_file_update(clear_database, usemodel, ident, crawler_extended):
ins1, ups1 = crawler_extended.synchronize(commit_changes=True)
fileList_ins = [r for r in ins1 if r.role == "File"]
cr = Crawler(debug=True, identifiableAdapter=ident)
crawl_standard_test_directory(cr, cfood="scifolder_extended.yml")
updateList = cr.updateList
fileList = [r for r in updateList if r.role == "File"]
for f in fileList:
f.file = rfp("..", "unittests", "test_directories", "examples_article", f.file)
ins2, ups2 = cr.synchronize(commit_changes=True)
assert len(ups1) == 0
assert len(ups2) == 0
# Try adding a parent:
res = db.execute_query("Find File")
assert len(res) == 11
assert len(res[0].parents) == 0
cr2 = Crawler(debug=True, identifiableAdapter=ident)
crawl_standard_test_directory(cr2, cfood="scifolder_extended2.yml")
updateList = cr2.updateList
fileList = [r for r in updateList if r.role == "File"]
for f in fileList:
f.file = rfp("..", "unittests", "test_directories", "examples_article", f.file)
ins3, ups3 = cr2.synchronize(commit_changes=True)
assert len(ups3) == 11
res = db.execute_query("Find File")
assert len(res) == 11
assert res[0].parents[0].name == "ProjectMarkdownReadme"
# TODO: Implement file update checks (based on checksum)
# Add test with actual file update:
# assert len(ins2) == 0
# assert len(ups2) == len(fileList_ins)
...@@ -183,6 +183,8 @@ class Converter(object): ...@@ -183,6 +183,8 @@ class Converter(object):
if key in SPECIAL_PROPERTIES: if key in SPECIAL_PROPERTIES:
# e.g. description, name, etc. # e.g. description, name, etc.
# list mode does not work for them # list mode does not work for them
if key == "path" and not propvalue.startswith(os.path.sep):
propvalue = os.path.sep + propvalue
setattr(c_record, key, propvalue) setattr(c_record, key, propvalue)
else: else:
......
Definitions:
type: Definitions
#include "description.yml"
# Converter-Provenance
# DataAnalysis/project_dir/measurement/match/identifier
# Structure-Element-Provenance
# DataAnalysis/2020_SpeedOflight/2020-11-10_kram
DataAnalysis: # name of the converter
type: Directory
match: DataAnalysis
subtree: &template
project_dir: # name of the first subtree element which is a converter
type: Directory
match: (?P<date>.*?)_(?P<identifier>.*)
records:
Project: # this is an identifiable in this case
parents:
- Project # not needed as the name is equivalent
date: $date
identifier: $identifier
subtree:
measurement: # new name for folders on the 3rd level
type: Directory
match: (?P<date>[0-9]{4,4}-[0-9]{2,2}-[0-9]{2,2})(_(?P<identifier>.*))?
records:
Measurement:
date: $date
identifier: $identifier
project: $Project
subtree:
README:
type: MarkdownFile # this is a subclass of converter File
# function signature: GeneralStore, StructureElement
# preprocessors: custom.caosdb.convert_values
match: ^README\.md$
# how to make match case insensitive?
records: # this block is very verbose and intended to make sure that this
# file is inserted correctly (and can be supplemented with properties
# and / or parents), TODO: maybe there should be a shorthand
ReadmeFile:
parents:
- ProjectMarkdownReadme
role: File
path: $README
file: $README # this is automatically the relative path
# starting from the top level structure element
# of this element
Measurement:
ReadmeFile: $ReadmeFile
subtree:
description:
type: DictTextElement
match_value: (?P<description>.*)
match_name: description
records:
Measurement:
description: $description
responsible_single:
type: DictTextElement
match_name: responsible
match_value: &person_regexp ((?P<first_name>.+) )?(?P<last_name>.+)
records: &responsible_records
Person:
first_name: $first_name
last_name: $last_name
Measurement: # this uses the reference to the above defined record
responsible: +$Person # each record also implicitely creates a variable
# with the same name. The "+" indicates, that
# this will become a list entry in list property
# "responsible" belonging to Measurement.
responsible_list:
type: DictListElement
match_name: responsible
subtree:
Person:
type: TextElement
match: *person_regexp
records: *responsible_records
# sources_list:
# type: DictListElement
# match_name: sources
# subtree:
# Source:
# type: TextElement
# match: &path ... ???
ExperimentalData: # name of the converter
type: Directory
match: ExperimentalData
subtree: *template
SimulationData: # name of the converter
type: Directory
match: SimulationData
subtree: *template
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment