diff --git a/setup.cfg b/setup.cfg
index 0dafd58aede11f5fc976f746f04738e49072a43b..7238216f8474443855a499b8da42b1b77bcd52d1 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -19,6 +19,10 @@ package_dir =
packages = find:
python_requires = >=3.6
+install_requires =
+ importlib-resources
+ caosdb
+ caosadvancedtools
[options.packages.find]
where = src
diff --git a/src/newcrawler/crawl.py b/src/newcrawler/crawl.py
index f69a920fd9c79f78f18c16fed1ad3b5e7ea7c207..b34e5436acc28864f943c6343c1dfd7896bf3e10 100644
--- a/src/newcrawler/crawl.py
+++ b/src/newcrawler/crawl.py
@@ -813,11 +813,17 @@ class Crawler(object):
scoped_records = recordStore.get_records_current_scope()
for record in scoped_records:
self.updateList.append(record)
+
+ # TODO: the scoped variables should be cleaned up as soon if the variables
+ # are no longer in the current scope. This can be implemented as follows,
+ # but this breaks the test "test_record_structure_generation", because
+ # some debug info is also deleted. This implementation can be used as soon
+ # as the remaining problems with the debug_tree are fixed.
# Delete the variables that are no longer needed:
- scoped_names = recordStore.get_names_current_scope()
- for name in scoped_names:
- del recordStore[name]
- del generalStore[name]
+ # scoped_names = recordStore.get_names_current_scope()
+ # for name in scoped_names:
+ # del recordStore[name]
+ # del generalStore[name]
return self.updateList
diff --git a/unittests/broken_cfoods/broken1.yml b/unittests/broken_cfoods/broken1.yml
new file mode 100644
index 0000000000000000000000000000000000000000..9fd4c52934c56512ada8ea564ccd540e07e25661
--- /dev/null
+++ b/unittests/broken_cfoods/broken1.yml
@@ -0,0 +1,79 @@
+Definitions:
+ type: Definitions
+ #include "description.yml"
+
+# Converter-Provenance
+# DataAnalysis/project_dir/measurement/match/identifier
+# Structure-Element-Provenance
+# DataAnalysis/2020_SpeedOflight/2020-11-10_kram
+
+DataAnalysis: # name of the converter
+ type: Directory_djskfj
+ match: DataAnalysis
+ subtree: &template
+ project_dir: # name of the first subtree element which is a converter
+ type: Directory
+ match: (?P<date>.*?)_(?P<identifier>.*)
+ records:
+ Project: # this is an identifiable in this case
+ parents:
+ - Project # not needed as the name is equivalent
+ date: $date
+ identifier: $identifier
+
+ subtree:
+ measurement: # new name for folders on the 3rd level
+ type: Directory
+ match: (?P<date>[0-9]{4,4}-[0-9]{2,2}-[0-9]{2,2})(_(?P<identifier>.*))?
+ records:
+ Measurement:
+ date: $date
+ identifier: $identifier
+ project: $Project
+ subtree:
+ README:
+ type: MarkdownFile # this is a subclass of converter File
+ # function signature: GeneralStore, StructureElement
+ # preprocessors: custom.caosdb.convert_values
+ match: ^README\.md$
+ # how to make match case insensitive?
+ subtree:
+ description:
+ type: DictTextElement
+ match_value: (?P<description>.*)
+ match_name: description
+ records:
+ Measurement:
+ description: $description
+ responsible_single:
+ type: DictTextElement
+ match_name: responsible
+ match_value: &person_regexp ((?P<first_name>.+) )?(?P<last_name>.+)
+ records: &responsible_records
+ Person:
+ first_name: $first_name
+ last_name: $last_name
+ Measurement: # this uses the reference to the above defined record
+ responsible: +$Person # each record also implicitely creates a variable
+ # with the same name. The "+" indicates, that
+ # this will become a list entry in list property
+ # "responsible" belonging to Measurement.
+
+ responsible_list:
+ type: DictListElement
+ match_name: responsible
+ subtree:
+ Person:
+ type: TextElement
+ match: *person_regexp
+ records: *responsible_records
+
+ExperimentalData: # name of the converter
+ type: Directory
+ match: ExperimentalData
+ subtree: *template
+
+SimulationData: # name of the converter
+ type: Directory
+ match: SimulationData
+ subtree: *template
diff --git a/unittests/records.xml b/unittests/records.xml
index 73f94a635dd8e6c435b39a122bd194134bb2015a..0ae34124a8875a723d7f0879687d8f0bdec51de0 100644
--- a/unittests/records.xml
+++ b/unittests/records.xml
@@ -138,6 +138,7 @@
<Property id="248" name="identifier" description="identifier of the experiment" datatype="TEXT" importance="FIX" flag="inheritance:FIX"/>
<Property id="250" name="project" datatype="Project" importance="FIX" flag="inheritance:FIX">287</Property>
<Property id="249" name="responsible" datatype="LIST<Person>" importance="FIX" flag="inheritance:FIX">
+ <Value>289</Value>
<Value>288</Value>
</Property>
</Record>
@@ -148,6 +149,7 @@
<Property id="248" name="identifier" description="identifier of the experiment" datatype="TEXT" importance="FIX" flag="inheritance:FIX">Cavity</Property>
<Property id="250" name="project" datatype="Project" importance="FIX" flag="inheritance:FIX">287</Property>
<Property id="249" name="responsible" datatype="LIST<Person>" importance="FIX" flag="inheritance:FIX">
+ <Value>289</Value>
<Value>290</Value>
</Property>
</Record>
diff --git a/unittests/test_tool_extended.py b/unittests/test_tool_extended.py
index 556ee7d599b845cd16e51eb4134fd677aede5879..2361e99373042a5f5ce73b8eb98083f7431d4836 100644
--- a/unittests/test_tool_extended.py
+++ b/unittests/test_tool_extended.py
@@ -69,8 +69,6 @@ def crawler():
# return ident
-# TODO fix
-@pytest.mark.xfail
def test_file_structure_generation(crawler):
sd = crawler.debug_tree[dircheckstr("SimulationData",
"2020_climate-model-predict", "2020-02-01",