diff --git a/integrationtests/basic_example/test_basic.py b/integrationtests/basic_example/test_basic.py
index ac8c62382b6b38a0686518932f5b851d60d584cd..b33974d9c2c5600bf2a91cbf14d7c8799ffc2644 100755
--- a/integrationtests/basic_example/test_basic.py
+++ b/integrationtests/basic_example/test_basic.py
@@ -27,6 +27,7 @@ an integration test module that does basic integration tests
"""
from caosadvancedtools.crawler import Crawler as OldCrawler
+from caoscrawler.debug_tree import DebugTree
import os
from caosdb import EmptyUniqueQueryError
import argparse
@@ -36,6 +37,7 @@ from caoscrawler import Crawler, SecurityMode
from caoscrawler.identifiable import Identifiable
import caosdb as db
from caoscrawler.identifiable_adapters import CaosDBIdentifiableAdapter
+from caoscrawler.scanner import scan_directory
import pytest
from caosadvancedtools.models.parser import parse_model_from_yaml
import yaml
@@ -82,24 +84,28 @@ def ident():
return ident
-def crawl_standard_test_directory(cr: Crawler,
- subdir: str = "examples_article",
- cfood: str = "scifolder_cfood.yml"):
- return cr.crawl_directory(rfp("..", "..", "unittests", "test_directories", subdir),
- rfp("..", "..", "unittests", cfood))
+def crawl_standard_test_directory(subdir: str = "examples_article",
+ cfood: str = "scifolder_cfood.yml",
+ debug_tree=None):
+ return scan_directory(rfp("..", "..", "unittests", "test_directories", subdir),
+ rfp("..", "..", "unittests", cfood),
+ debug_tree=debug_tree)
@pytest.fixture
def crawler(ident):
cr = Crawler(identifiableAdapter=ident)
- crawled_data, debug_tree = crawl_standard_test_directory(cr)
+ debug_tree = DebugTree()
+ crawled_data = crawl_standard_test_directory(debug_tree=debug_tree)
return cr, crawled_data, debug_tree
@pytest.fixture
def crawler_extended(ident):
cr = Crawler(identifiableAdapter=ident)
- crawled_data, debug_tree = crawl_standard_test_directory(cr, cfood="scifolder_extended.yml")
+ debug_tree = DebugTree()
+ crawled_data = crawl_standard_test_directory(
+ cfood="scifolder_extended.yml", debug_tree=debug_tree)
# correct paths for current working directory
file_list = [r for r in crawled_data if r.role == "File"]
for f in file_list:
@@ -108,7 +114,7 @@ def crawler_extended(ident):
def test_ambigious_lookup(clear_database, usemodel, crawler, ident):
- ins, ups = crawler[0].synchronize(crawler[1])
+ ins, ups = crawler[0].synchronize(crawled_data=crawler[1])
proj = db.execute_query("FIND Project WITH identifier='SpeedOfLight'", unique=True)
with pytest.raises(RuntimeError, match=".*unambigiously.*"):
@@ -117,7 +123,7 @@ def test_ambigious_lookup(clear_database, usemodel, crawler, ident):
def test_single_insertion(clear_database, usemodel, crawler, ident):
- ins, ups = crawler[0].synchronize(crawler[1])
+ ins, ups = crawler[0].synchronize(crawled_data=crawler[1])
# This test also generates the file records.xml used in some of the unittesets:
res = db.execute_query("FIND Record")
@@ -139,52 +145,52 @@ def test_single_insertion(clear_database, usemodel, crawler, ident):
# Do a second run on the same data, there should be no changes:
crawler = Crawler(identifiableAdapter=ident)
- crawled_data, debug_tree = crawler.crawl_directory(rfp("../../unittests/test_directories", "examples_article"),
- rfp("../../unittests/scifolder_cfood.yml"))
- ins, ups = crawler.synchronize(crawled_data)
+ crawled_data = scan_directory(rfp("../../unittests/test_directories", "examples_article"),
+ rfp("../../unittests/scifolder_cfood.yml"))
+ ins, ups = crawler.synchronize(crawled_data=crawled_data)
assert len(ins) == 0
assert len(ups) == 0
def test_multiple_insertions(clear_database, usemodel, ident, crawler):
- ins, ups = crawler[0].synchronize(crawler[1])
+ ins, ups = crawler[0].synchronize(crawled_data=crawler[1])
# Do a second run on the same data, there should be no changes:
cr = Crawler(identifiableAdapter=ident)
- crawled_data, debug_tree = crawl_standard_test_directory(cr)
- ins, ups = cr.synchronize(crawled_data)
+ crawled_data = crawl_standard_test_directory()
+ ins, ups = cr.synchronize(crawled_data=crawled_data)
assert len(ins) == 0
assert len(ups) == 0
def test_insertion(clear_database, usemodel, ident, crawler):
- ins, ups = crawler[0].synchronize(crawler[1])
+ ins, ups = crawler[0].synchronize(crawled_data=crawler[1])
# Do a second run on the same data, there should a new insert:
cr = Crawler(identifiableAdapter=ident)
- crawled_data, debug_tree = crawl_standard_test_directory(cr, "example_insert")
+ crawled_data = crawl_standard_test_directory("example_insert")
assert len(crawled_data) == 3
- ins, ups = cr.synchronize(crawled_data)
+ ins, ups = cr.synchronize(crawled_data=crawled_data)
assert len(ins) == 1
assert len(ups) == 0
# Do it again to check whether nothing is changed:
cr = Crawler(identifiableAdapter=ident)
- crawled_data, debug_tree = crawl_standard_test_directory(cr, "example_insert")
+ crawled_data = crawl_standard_test_directory("example_insert")
assert len(crawled_data) == 3
- ins, ups = cr.synchronize(crawled_data)
+ ins, ups = cr.synchronize(crawled_data=crawled_data)
assert len(ins) == 0
assert len(ups) == 0
def test_insert_auth(clear_database, usemodel, ident, crawler):
- ins, ups = crawler[0].synchronize(crawler[1])
+ ins, ups = crawler[0].synchronize(crawled_data=crawler[1])
# Do a second run on the same data, there should a new insert:
cr = Crawler(identifiableAdapter=ident, securityMode=SecurityMode.RETRIEVE)
- crawled_data, debug_tree = crawl_standard_test_directory(cr, "example_insert")
+ crawled_data = crawl_standard_test_directory("example_insert")
assert len(crawled_data) == 3
- ins, ups = cr.synchronize(crawled_data)
+ ins, ups = cr.synchronize(crawled_data=crawled_data)
assert len(ins) == 1
assert not ins[0].is_valid()
nins, nups = OldCrawler.update_authorized_changes(cr.run_id)
@@ -192,37 +198,36 @@ def test_insert_auth(clear_database, usemodel, ident, crawler):
# Do it again to check whether nothing is changed:
cr = Crawler(identifiableAdapter=ident)
- crawled_data, debug_tree = crawl_standard_test_directory(cr, "example_insert")
+ crawled_data = crawl_standard_test_directory("example_insert")
assert len(crawled_data) == 3
- ins, ups = cr.synchronize(crawled_data)
+ ins, ups = cr.synchronize(crawled_data=crawled_data)
assert len(ins) == 0
assert len(ups) == 0
def test_insertion_and_update(clear_database, usemodel, ident, crawler):
- ins, ups = crawler[0].synchronize(crawler[1])
+ ins, ups = crawler[0].synchronize(crawled_data=crawler[1])
cr = Crawler(identifiableAdapter=ident)
- crawled_data, debug_tree = crawl_standard_test_directory(cr, "example_insert")
- ins, ups = cr.synchronize(crawled_data)
+ crawled_data = crawl_standard_test_directory("example_insert")
+ ins, ups = cr.synchronize(crawled_data=crawled_data)
cr = Crawler(identifiableAdapter=ident)
- crawled_data, debug_tree = crawl_standard_test_directory(cr, "example_overwrite_1")
- # print(cr.crawled_data)
+ crawled_data = crawl_standard_test_directory("example_overwrite_1")
# cr.save_debug_data(rfp("provenance.yml"))
assert len(crawled_data) == 3
- ins, ups = cr.synchronize(crawled_data)
+ ins, ups = cr.synchronize(crawled_data=crawled_data)
assert len(ins) == 0
assert len(ups) == 1
def test_identifiable_update(clear_database, usemodel, ident, crawler):
- ins, ups = crawler[0].synchronize(crawler[1])
+ ins, ups = crawler[0].synchronize(crawled_data=crawler[1])
# Do a second run on the same data with a change in one
# of the identifiables:
cr = Crawler(identifiableAdapter=ident)
- crawled_data, debug_tree = crawl_standard_test_directory(cr)
+ crawled_data = crawl_standard_test_directory()
# Test the addition of a single property:
l = crawled_data
@@ -234,13 +239,13 @@ def test_identifiable_update(clear_database, usemodel, ident, crawler):
name="email", value="testperson@testaccount.test")
print("one change")
break
- ins, ups = cr.synchronize(crawled_data)
+ ins, ups = cr.synchronize(crawled_data=crawled_data)
assert len(ins) == 0
assert len(ups) == 1
# Test the change within one property:
cr = Crawler(identifiableAdapter=ident)
- crawled_data, debug_tree = crawl_standard_test_directory(cr)
+ crawled_data = crawl_standard_test_directory()
l = crawled_data
for record in l:
if (record.parents[0].name == "Measurement" and
@@ -248,13 +253,13 @@ def test_identifiable_update(clear_database, usemodel, ident, crawler):
record.add_property(name="email", value="testperson@coolmail.test")
print("one change")
break
- ins, ups = cr.synchronize(crawled_data)
+ ins, ups = cr.synchronize(crawled_data=crawled_data)
assert len(ins) == 0
assert len(ups) == 1
# Changing the date should result in a new insertion:
cr = Crawler(identifiableAdapter=ident)
- crawled_data, debug_tree = crawl_standard_test_directory(cr)
+ crawled_data = crawl_standard_test_directory()
l = crawled_data
for record in l:
if (record.parents[0].name == "Measurement" and
@@ -263,15 +268,15 @@ def test_identifiable_update(clear_database, usemodel, ident, crawler):
record.get_property("date").value = "2012-01-02"
print("one change")
break
- ins, ups = cr.synchronize(crawled_data)
+ ins, ups = cr.synchronize(crawled_data=crawled_data)
assert len(ins) == 1
assert len(ups) == 0
def test_file_insertion_dry(clear_database, usemodel, ident):
crawler_extended = Crawler(identifiableAdapter=ident)
- crawled_data, debug_tree = crawl_standard_test_directory(
- crawler_extended, cfood="scifolder_extended.yml")
+ crawled_data = crawl_standard_test_directory(
+ cfood="scifolder_extended.yml")
file_list = [r for r in crawled_data if r.role == "File"]
assert len(file_list) == 11
@@ -279,14 +284,15 @@ def test_file_insertion_dry(clear_database, usemodel, ident):
assert f.path.endswith("README.md")
assert f.path[1:] == f.file
- ins, ups = crawler_extended.synchronize(crawled_data, commit_changes=False)
+ ins, ups = crawler_extended.synchronize(crawled_data=crawled_data, commit_changes=False)
assert len(ups) == 0
file_list_ins = [r for r in ins if r.role == "File"]
assert len(file_list_ins) == 11
def test_file_insertion(clear_database, usemodel, ident, crawler_extended):
- ins, ups = crawler_extended[0].synchronize(crawler_extended[1], commit_changes=True)
+ ins, ups = crawler_extended[0].synchronize(
+ crawled_data=crawler_extended[1], commit_changes=True)
file_list_ins = [r for r in ins if r.role == "File"]
assert len(file_list_ins) == 11
@@ -302,16 +308,17 @@ def test_file_insertion(clear_database, usemodel, ident, crawler_extended):
def test_file_update(clear_database, usemodel, ident, crawler_extended):
- ins1, ups1 = crawler_extended[0].synchronize(crawler_extended[1], commit_changes=True)
+ ins1, ups1 = crawler_extended[0].synchronize(
+ crawled_data=crawler_extended[1], commit_changes=True)
file_list_ins = [r for r in ins1 if r.role == "File"]
cr = Crawler(identifiableAdapter=ident)
- crawled_data, debug_tree = crawl_standard_test_directory(cr, cfood="scifolder_extended.yml")
+ crawled_data = crawl_standard_test_directory(cfood="scifolder_extended.yml")
file_list = [r for r in crawled_data if r.role == "File"]
for f in file_list:
f.file = rfp("..", "..", "unittests", "test_directories", f.file)
- ins2, ups2 = cr.synchronize(crawled_data, commit_changes=True)
+ ins2, ups2 = cr.synchronize(crawled_data=crawled_data, commit_changes=True)
assert len(ups1) == 0
assert len(ups2) == 0
@@ -321,12 +328,12 @@ def test_file_update(clear_database, usemodel, ident, crawler_extended):
assert len(res[0].parents) == 0
cr2 = Crawler(identifiableAdapter=ident)
- crawled_data, debug_tree = crawl_standard_test_directory(cr2, cfood="scifolder_extended2.yml")
+ crawled_data = crawl_standard_test_directory(cfood="scifolder_extended2.yml")
file_list = [r for r in crawled_data if r.role == "File"]
for f in file_list:
f.file = rfp("..", "..", "unittests", "test_directories", f.file)
- ins3, ups3 = cr2.synchronize(crawled_data, commit_changes=True)
+ ins3, ups3 = cr2.synchronize(crawled_data=crawled_data, commit_changes=True)
assert len(ups3) == 11
res = db.execute_query("Find File")
diff --git a/integrationtests/test_issues.py b/integrationtests/test_issues.py
index d9dd4114eb089f53f83384c9b1a7670fd43984c6..08e254daf4052670fcec18760626c460604efe15 100644
--- a/integrationtests/test_issues.py
+++ b/integrationtests/test_issues.py
@@ -98,7 +98,7 @@ def test_issue_23(clear_database):
}
crawler.generate_run_id()
- records, debug_tree = scan_structure_elements(
+ records = scan_structure_elements(
DictElement("TestDict", test_dict), crawler_definition, converter_registry)
assert len(records) == 1
@@ -112,7 +112,7 @@ def test_issue_23(clear_database):
assert rec_crawled.get_property("prop_a") is None
# synchronize with database and update the record
- ins, ups = crawler.synchronize(records)
+ ins, ups = crawler.synchronize(crawled_data=records)
assert len(ins) == 0
assert len(ups) == 1
diff --git a/integrationtests/test_realworld_example.py b/integrationtests/test_realworld_example.py
index 9373989da6e10c07f072854f9ec67c06bad20e96..45873ddeb8b4f4a23fbcbc9225cbeea60b213cc4 100644
--- a/integrationtests/test_realworld_example.py
+++ b/integrationtests/test_realworld_example.py
@@ -105,12 +105,11 @@ def test_dataset(clear_database, usemodel, addfiles, caplog):
identifiable_path = os.path.join(DATADIR, "identifiables.yml")
crawler_definition_path = os.path.join(DATADIR, "dataset_cfoods.yml")
crawler_main(
- os.path.join(DATADIR, 'data'),
- crawler_definition_path,
- identifiable_path,
- True,
- os.path.join(DATADIR, "provenance.yml"),
- False,
+ crawled_directory_path=os.path.join(DATADIR, 'data'),
+ cfood_file_name=crawler_definition_path,
+ identifiables_definition_file=identifiable_path,
+ provenance_file=os.path.join(DATADIR, "provenance.yml"),
+ dry_run=False,
remove_prefix=DATADIR,
# this test will fail without this prefix since the crawler would try to create new files
add_prefix="/extroot/realworld_example"
@@ -145,12 +144,11 @@ def test_event_update(clear_database, usemodel, addfiles):
crawler_definition_path = os.path.join(DATADIR, "dataset_cfoods.yml")
crawler_main(
- os.path.join(DATADIR, 'data'),
- crawler_definition_path,
- identifiable_path,
- True,
- os.path.join(DATADIR, "provenance.yml"),
- False,
+ crawled_directory_path=os.path.join(DATADIR, 'data'),
+ cfood_file_name=crawler_definition_path,
+ identifiables_definition_file=identifiable_path,
+ provenance_file=os.path.join(DATADIR, "provenance.yml"),
+ dry_run=False,
remove_prefix=DATADIR,
# this test will fail without this prefix since the crawler would try to create new files
add_prefix="/extroot/realworld_example"
@@ -175,7 +173,7 @@ def test_event_update(clear_database, usemodel, addfiles):
crawler_definition = load_definition(
crawler_definition_path)
converter_registry = create_converter_registry(crawler_definition)
- records, debug_tree = scan_structure_elements(
+ records = scan_structure_elements(
Directory("data", os.path.join(DATADIR, "data")),
crawler_definition,
converter_registry
@@ -193,7 +191,7 @@ def test_event_update(clear_database, usemodel, addfiles):
"latitude").value = 0.0
rec.get_property("Event").value[0].get_property(
"location").value = "Origin"
- second_crawler.synchronize(records)
+ second_crawler.synchronize(crawled_data=records)
# Dataset is still the same Record, but with an updated event
new_dataset_rec = db.Record(id=old_dataset_rec.id).retrieve()
diff --git a/integrationtests/test_use_case_simple_presentation.py b/integrationtests/test_use_case_simple_presentation.py
index 5fc0f6c7d85a0fce4490c72952e711fe241a0099..0f48677d4bf64158374a0eb0865eb2b85ea715db 100644
--- a/integrationtests/test_use_case_simple_presentation.py
+++ b/integrationtests/test_use_case_simple_presentation.py
@@ -57,22 +57,24 @@ def test_complete_crawler(clear_database):
# test that a bad value for "remove_prefix" leads to runtime error
with pytest.raises(RuntimeError) as re:
- crawler_main(DATADIR,
- os.path.join(DATADIR, "cfood.yml"),
- os.path.join(DATADIR, "identifiables.yml"),
- True,
- os.path.join(DATADIR, "provenance.yml"),
- False,
- remove_prefix="sldkfjsldf")
+ crawler_main(
+ crawled_directory_path=os.path.join(DATADIR),
+ cfood_file_name=os.path.join(DATADIR, "cfood.yml"),
+ identifiables_definition_file=os.path.join(DATADIR, "identifiables.yml"),
+ provenance_file=os.path.join(DATADIR, "provenance.yml"),
+ dry_run=False,
+ remove_prefix="sldkfjsldf",
+ )
assert "path does not start with the prefix" in str(re.value)
- crawler_main(DATADIR,
- os.path.join(DATADIR, "cfood.yml"),
- os.path.join(DATADIR, "identifiables.yml"),
- True,
- os.path.join(DATADIR, "provenance.yml"),
- False,
- remove_prefix=os.path.abspath(DATADIR))
+ crawler_main(
+ crawled_directory_path=os.path.join(DATADIR),
+ cfood_file_name=os.path.join(DATADIR, "cfood.yml"),
+ identifiables_definition_file=os.path.join(DATADIR, "identifiables.yml"),
+ provenance_file=os.path.join(DATADIR, "provenance.yml"),
+ dry_run=False,
+ remove_prefix=os.path.abspath(DATADIR),
+ )
res = db.execute_query("FIND Record Experiment")
assert len(res) == 1
diff --git a/src/caoscrawler/__init__.py b/src/caoscrawler/__init__.py
index 044d8f0bf53c4c80dab9b492919fa64ab321a60d..05bad0b54d9098c0b7f165d8295a0faa2966fa32 100644
--- a/src/caoscrawler/__init__.py
+++ b/src/caoscrawler/__init__.py
@@ -1,2 +1,4 @@
from .crawl import Crawler, SecurityMode
-from .version import CfoodRequiredVersionError, version as __version__
+from .version import CfoodRequiredVersionError, get_caoscrawler_version
+
+__version__ = get_caoscrawler_version()
diff --git a/src/caoscrawler/crawl.py b/src/caoscrawler/crawl.py
index f93c0ec436e4106c22f3bb065137394c54f12754..bacc5356b7b14f43d44db25c461c717fa9c39bc9 100644
--- a/src/caoscrawler/crawl.py
+++ b/src/caoscrawler/crawl.py
@@ -184,7 +184,7 @@ class Crawler(object):
def __init__(self,
generalStore: Optional[GeneralStore] = None,
- debug: bool = False,
+ debug: Optional[bool] = None,
identifiableAdapter: Optional[IdentifiableAdapter] = None,
securityMode: SecurityMode = SecurityMode.UPDATE):
"""
@@ -204,6 +204,10 @@ class Crawler(object):
Please use SecurityMode Enum
"""
+ # Remove this once the property `crawled_data` is no longer needed for compatibility
+ # reasons
+ self._crawled_data = None
+
# The following caches store records, where we checked whether they exist on the remote
# server. Since, it is important to know whether they exist or not, we store them into two
# different caches.
@@ -215,8 +219,7 @@ class Crawler(object):
if identifiableAdapter is not None:
self.identifiableAdapter = identifiableAdapter
- if not debug:
- # TODO: not ideal, how to warn if debug is explicitely set to false?
+ if debug is not None:
warnings.warn(DeprecationWarning(
"The debug argument of the Crawler class is deprecated and has no effect."))
@@ -254,10 +257,22 @@ class Crawler(object):
"The function start_crawling in the crawl module is deprecated. "
"Please use scan_structure_elements from the scanner module."))
- self.generate_run_id()
+ data = scan_structure_elements(
+ items, crawler_definition, converter_registry, restricted_path)
+ self.crawled_data = data
+ return data
- return scan_structure_elements(
- items, crawler_definition, converter_registry, restrict_path)
+ @property
+ def crawled_data(self):
+ warnings.warn(DeprecationWarning(
+ "The use of self.crawled_data is depricated. You should not access this variable. "
+ "Instead, create the data with the scanner and then pass it as argument to Crawler "
+ "functions"))
+ return self._crawled_data
+
+ @crawled_data.setter
+ def crawled_data(self, arg):
+ self._crawled_data = arg
def crawl_directory(self,
crawled_directory: str,
@@ -267,15 +282,15 @@ class Crawler(object):
The new main function to run the crawler on a directory.
"""
- self.crawled_directory = crawled_directory
- self.generate_run_id()
-
- # TODO: This is not ideal yet, the data is just returned and needs to be
- # separately supplied to the synchronize function.
+ warnings.warn(DeprecationWarning(
+ "The function crawl_directory in the crawl module is deprecated. "
+ "Please use scan_directory from the scanner module."))
- return scan_directory(crawled_directory,
+ data = scan_directory(crawled_directory,
crawler_definition_path,
restricted_path)
+ self.crawled_data = data
+ return data
def _has_reference_value_without_id(self, ident: Identifiable) -> bool:
"""
@@ -791,9 +806,10 @@ class Crawler(object):
update_cache.insert(to_be_updated, run_id)
def synchronize(self,
- crawled_data: list[db.Record],
commit_changes: bool = True,
- unique_names=True):
+ unique_names: bool = True,
+ crawled_data: Optional[list[db.Record]] = None,
+ ):
"""
This function applies several stages:
1) Retrieve identifiables for all records in crawled_data.
@@ -808,6 +824,13 @@ class Crawler(object):
Return the final to_be_inserted and to_be_updated as tuple.
"""
+ if crawled_data is None:
+ warnings.warn(DeprecationWarning(
+ "Calling synchronize without the data to be synchronized is depricated. Please "
+ "use for example the Scanner to create this data."))
+ crawled_data = self.crawled_data
+
+ self.generate_run_id()
to_be_inserted, to_be_updated = self.split_into_inserts_and_updates(crawled_data)
referencing_entities = self.create_reference_mapping(to_be_updated + to_be_inserted)
@@ -846,12 +869,14 @@ class Crawler(object):
pending_inserts = update_cache.get_inserts(self.run_id)
if pending_inserts:
Crawler.inform_about_pending_changes(
- pending_inserts, self.run_id, self.crawled_directory)
+ # TODO crawled_directory is no longer available
+ pending_inserts, self.run_id, "missing crawled_directory")
pending_updates = update_cache.get_updates(self.run_id)
if pending_updates:
Crawler.inform_about_pending_changes(
- pending_updates, self.run_id, self.crawled_directory)
+ # TODO crawled_directory is no longer available
+ pending_updates, self.run_id, "missing crawled_directory")
return (to_be_inserted, to_be_updated)
@@ -915,7 +940,7 @@ ____________________\n""".format(i + 1, len(pending_changes)) + str(el[3]))
res[converter.name]["subtree"][k[0]] = d[k[0]]
return res
- def save_debug_data(self, debug_tree: DebugTree, filename: str):
+ def save_debug_data(self, filename: str, debug_tree: DebugTree = None):
"""
Save the information contained in a debug_tree to a file named filename.
"""
@@ -997,8 +1022,10 @@ def crawler_main(crawled_directory_path: str,
"""
crawler = Crawler(securityMode=securityMode)
try:
- crawled_data, debug_tree = crawler.crawl_directory(
- crawled_directory_path, cfood_file_name, restricted_path)
+
+ debug_tree = DebugTree()
+ crawled_data = scan_directory(
+ crawled_directory_path, cfood_file_name, restricted_path, debug_tree=debug_tree)
except ConverterValidationError as err:
logger.error(err)
return 1
@@ -1020,7 +1047,7 @@ def crawler_main(crawled_directory_path: str,
remove_prefix = prefix
if dry_run:
- ins, upd = crawler.synchronize(crawled_data, commit_changes=False)
+ ins, upd = crawler.synchronize(commit_changes=False, crawled_data=crawled_data)
inserts = [str(i) for i in ins]
updates = [str(i) for i in upd]
with open("dry.yml", "w") as f:
@@ -1066,7 +1093,8 @@ def crawler_main(crawled_directory_path: str,
raise RuntimeError("Missing RecordTypes: {}".
format(", ".join(notfound)))
- crawler.synchronize(crawled_data, commit_changes=True, unique_names=unique_names)
+ crawler.synchronize(commit_changes=True, unique_names=unique_names,
+ crawled_data=crawled_data)
return 0
diff --git a/src/caoscrawler/scanner.py b/src/caoscrawler/scanner.py
index 83cd6c9fbce63e0cd47fec90350576f09b378c4b..ff6156aed3bde639435219a705d6d7d2124f7f38 100644
--- a/src/caoscrawler/scanner.py
+++ b/src/caoscrawler/scanner.py
@@ -275,9 +275,6 @@ def scanner(items: list[StructureElement],
if converters_path is None:
converters_path = []
- if debug_tree is None:
- debug_tree = DebugTree()
-
for element in items:
for converter in converters:
@@ -303,25 +300,26 @@ def scanner(items: list[StructureElement],
children = converter.create_children(general_store_copy, element)
- # add provenance information for each variable
- debug_tree.debug_tree[str(element)] = (
- general_store_copy.get_storage(), record_store_copy.get_storage())
- debug_tree.debug_metadata["copied"][str(element)] = (
- general_store_copy.get_dict_copied(),
- record_store_copy.get_dict_copied())
- debug_tree.debug_metadata["usage"][str(element)].add(
- "/".join(converters_path + [converter.name]))
- mod_info = debug_tree.debug_metadata["provenance"]
- for record_name, prop_name in keys_modified:
- # TODO: check
- internal_id = record_store_copy.get_internal_id(
- record_name)
- record_identifier = record_name + \
- "_" + str(internal_id)
- converter.metadata["usage"].add(record_identifier)
- mod_info[record_identifier][prop_name] = (
- structure_elements_path + [element.get_name()],
- converters_path + [converter.name])
+ if debug_tree is not None:
+ # add provenance information for each variable
+ debug_tree.debug_tree[str(element)] = (
+ general_store_copy.get_storage(), record_store_copy.get_storage())
+ debug_tree.debug_metadata["copied"][str(element)] = (
+ general_store_copy.get_dict_copied(),
+ record_store_copy.get_dict_copied())
+ debug_tree.debug_metadata["usage"][str(element)].add(
+ "/".join(converters_path + [converter.name]))
+ mod_info = debug_tree.debug_metadata["provenance"]
+ for record_name, prop_name in keys_modified:
+ # TODO: check
+ internal_id = record_store_copy.get_internal_id(
+ record_name)
+ record_identifier = record_name + \
+ "_" + str(internal_id)
+ converter.metadata["usage"].add(record_identifier)
+ mod_info[record_identifier][prop_name] = (
+ structure_elements_path + [element.get_name()],
+ converters_path + [converter.name])
scanner(children, converter.converters,
general_store_copy, record_store_copy,
@@ -351,7 +349,7 @@ def scanner(items: list[StructureElement],
# del record_store[name]
# del general_store[name]
- return crawled_data, debug_tree
+ return crawled_data
# --------------------------------------------------------------------------------
@@ -360,7 +358,8 @@ def scanner(items: list[StructureElement],
def scan_directory(dirname: str, crawler_definition_path: str,
- restricted_path: Optional[list[str]] = None):
+ restricted_path: Optional[list[str]] = None,
+ debug_tree: Optional[DebugTree] = None):
""" Crawl a single directory.
Formerly known as "crawl_directory".
@@ -396,13 +395,16 @@ def scan_directory(dirname: str, crawler_definition_path: str,
dirname),
crawler_definition,
converter_registry,
- restricted_path=restricted_path)
+ restricted_path=restricted_path,
+ debug_tree=debug_tree
+ )
def scan_structure_elements(items: Union[list[StructureElement], StructureElement],
crawler_definition: dict,
converter_registry: dict,
- restricted_path: Optional[list[str]] = None):
+ restricted_path: Optional[list[str]] = None,
+ debug_tree: Optional[DebugTree] = None):
"""
Start point of the crawler recursion.
@@ -437,4 +439,6 @@ def scan_structure_elements(items: Union[list[StructureElement], StructureElemen
return scanner(
items=items,
converters=converters,
- restricted_path=restricted_path)
+ restricted_path=restricted_path,
+ debug_tree=debug_tree
+ )
diff --git a/src/caoscrawler/version.py b/src/caoscrawler/version.py
index e73905dcd25673eae88f718a7e45b7b4d0665e47..fdc8323452cd190cc3628efa57c15992f30fabeb 100644
--- a/src/caoscrawler/version.py
+++ b/src/caoscrawler/version.py
@@ -25,8 +25,10 @@ except ImportError: # Python<3.8 dowesn"t support this so use
from packaging.version import parse as parse_version
from warnings import warn
-# Read in version of locally installed caoscrawler package
-version = importlib_metadata.version("caoscrawler")
+
+def get_caoscrawler_version():
+ """ Read in version of locally installed caoscrawler package"""
+ return importlib_metadata.version("caoscrawler")
class CfoodRequiredVersionError(RuntimeError):
@@ -51,7 +53,7 @@ as expected with the installed version of the crawler.
warn(msg, UserWarning)
return
- installed_version = parse_version(version)
+ installed_version = parse_version(get_caoscrawler_version())
cfood_version = parse_version(metadata["crawler-version"])
if cfood_version > installed_version:
diff --git a/unittests/test_cfood_metadata.py b/unittests/test_cfood_metadata.py
index 2881a47df2418bd0d794f79291b166f091456cf3..494bd383d95b4a845b5ea6f86ccff0f9a1db257f 100644
--- a/unittests/test_cfood_metadata.py
+++ b/unittests/test_cfood_metadata.py
@@ -21,23 +21,13 @@ import pytest
import yaml
from tempfile import NamedTemporaryFile
+from unittest.mock import patch
+from unittest.mock import MagicMock, Mock
import caoscrawler
from caoscrawler.scanner import load_definition
-CRAWLER_VERSION = ""
-
-
-def setup_function(function):
- """Store original crawler version in case it is altered for tests."""
- CRAWLER_VERSION = caoscrawler.version.version
-
-
-def teardown_function(function):
- """Reset version"""
- caoscrawler.version.version = CRAWLER_VERSION
-
def _temp_file_load(txt: str):
"""
@@ -69,9 +59,12 @@ SimulationData:
with pytest.warns(UserWarning) as uw:
_temp_file_load(definition_text)
- assert len(uw) == 1
- assert "No crawler version specified in cfood definition" in uw[0].message.args[0]
- assert "Specifying a version is highly recommended" in uw[0].message.args[0]
+ found = False
+ for w in uw:
+ if ("No crawler version specified in cfood definition" in w.message.args[0] and
+ "Specifying a version is highly recommended" in w.message.args[0]):
+ found = True
+ assert found
# metadata section is missing alltogether
definition_text = """
@@ -83,12 +76,16 @@ SimulationData:
with pytest.warns(UserWarning) as uw:
_temp_file_load(definition_text)
- assert len(uw) == 1
- assert "No crawler version specified in cfood definition" in uw[0].message.args[0]
- assert "Specifying a version is highly recommended" in uw[0].message.args[0]
+ found = False
+ for w in uw:
+ if ("No crawler version specified in cfood definition" in w.message.args[0] and
+ "Specifying a version is highly recommended" in w.message.args[0]):
+ found = True
+ assert found
-def test_warning_if_version_too_old():
+@patch("caoscrawler.version.get_caoscrawler_version")
+def test_warning_if_version_too_old(get_version):
"""Warn if the cfood was written for an older crawler version."""
definition_text = """
@@ -103,31 +100,38 @@ SimulationData:
match: SimulationData
"""
- # higher minor
- caoscrawler.version.version = "0.3.0"
+ get_version.side_effect = lambda: "0.3.0"
with pytest.warns(UserWarning) as uw:
_temp_file_load(definition_text)
- assert len(uw) == 1
- assert "cfood was written for a previous crawler version" in uw[0].message.args[0]
- assert "version specified in cfood: 0.2.0" in uw[0].message.args[0]
- assert "version installed on your system: 0.3.0" in uw[0].message.args[0]
+ found = False
+ for w in uw:
+ if ("cfood was written for a previous crawler version" in w.message.args[0] and
+ "version specified in cfood: 0.2.0" in w.message.args[0] and
+ "version installed on your system: 0.3.0" in w.message.args[0]):
+ found = True
+ assert found
# higher major
- caoscrawler.version.version = "1.1.0"
+ get_version.side_effect = lambda: "1.1.0"
with pytest.warns(UserWarning) as uw:
_temp_file_load(definition_text)
- assert len(uw) == 1
- assert "cfood was written for a previous crawler version" in uw[0].message.args[0]
- assert "version specified in cfood: 0.2.0" in uw[0].message.args[0]
- assert "version installed on your system: 1.1.0" in uw[0].message.args[0]
+ found = False
+ for w in uw:
+ if ("cfood was written for a previous crawler version" in w.message.args[0] and
+ "version specified in cfood: 0.2.0" in w.message.args[0] and
+ "version installed on your system: 1.1.0" in w.message.args[0]):
+ found = True
+ assert found
-def test_error_if_version_too_new():
+@patch("caoscrawler.version.get_caoscrawler_version")
+def test_error_if_version_too_new(get_version):
"""Raise error if the cfood requires a newer crawler version."""
# minor too old
+ get_version.side_effect = lambda: "0.1.5"
definition_text = """
---
metadata:
@@ -139,7 +143,6 @@ SimulationData:
type: Directory
match: SimulationData
"""
- caoscrawler.version.version = "0.1.5"
with pytest.raises(caoscrawler.CfoodRequiredVersionError) as cre:
_temp_file_load(definition_text)
@@ -167,7 +170,7 @@ SimulationData:
assert "version installed on your system: 0.1.5" in str(cre.value)
# patch to old
- caoscrawler.version.version = "1.0.0"
+ get_version.side_effect = lambda: "1.0.0"
with pytest.raises(caoscrawler.CfoodRequiredVersionError) as cre:
_temp_file_load(definition_text)
@@ -177,7 +180,8 @@ SimulationData:
assert "version installed on your system: 1.0.0" in str(cre.value)
-def test_matching_version():
+@patch("caoscrawler.version.get_caoscrawler_version")
+def test_matching_version(get_version):
"""Test that there is no warning or error in case the version matches."""
definition_text = """
@@ -191,10 +195,10 @@ SimulationData:
type: Directory
match: SimulationData
"""
- caoscrawler.version.version = "0.2.1"
+ get_version.side_effect = lambda: "0.2.1"
assert _temp_file_load(definition_text)
# The version is also considered a match if the patch version of the
# installed crawler is newer than the one specified in the cfood metadata
- caoscrawler.version.version = "0.2.7"
+ get_version.side_effect = lambda: "0.2.7"
assert _temp_file_load(definition_text)
diff --git a/unittests/test_converters.py b/unittests/test_converters.py
index 25b8412126289b34b97dd83f6357964f62823a56..154724be6d126aefb430c7d0600b86a5ec721812 100644
--- a/unittests/test_converters.py
+++ b/unittests/test_converters.py
@@ -438,6 +438,7 @@ def test_filter_children_of_directory(converter_registry, capsys):
children = dc.create_children(None, test_dir)
+@pytest.mark.filterwarnings("ignore::UserWarning")
def test_validate_custom_converters():
one_doc_yaml = """
Converters:
@@ -457,6 +458,7 @@ MyElement:
two_doc_yaml = """
---
metadata:
+ crawler-version: 0.3.1
Converters:
MyNewType:
converter: MyNewTypeConverter
diff --git a/unittests/test_issues.py b/unittests/test_issues.py
index ef8c38afdd77c93ec2439157f6e7dacfa9c32c14..46157af9225c11b79e76dd3ef856d60519a6eb9d 100644
--- a/unittests/test_issues.py
+++ b/unittests/test_issues.py
@@ -63,7 +63,7 @@ def test_issue_10():
"float_value": 4
}
- records, debug_tree = scan_structure_elements(
+ records = scan_structure_elements(
DictElement("TestDict", test_dict), crawler_definition, converter_registry)
assert len(records) == 1
assert records[0].parents[0].name == "TestRec"
@@ -95,7 +95,7 @@ def test_list_datatypes():
}
}
- crawler = Crawler(debug=True)
+ crawler = Crawler()
converter_registry = crawler.load_converters(crawler_definition)
test_dict = {
diff --git a/unittests/test_json.py b/unittests/test_json.py
index 42841da7715da8b796cbb9b85ee831bf004f36b7..3c120be174ff819baeeaa49ddf142cf40dba751e 100644
--- a/unittests/test_json.py
+++ b/unittests/test_json.py
@@ -49,7 +49,7 @@ def test_json():
# Load and register converter packages:
converter_registry = create_converter_registry(crawler_definition)
- records, debug_tree = scan_structure_elements(
+ records = scan_structure_elements(
JSONFile(os.path.basename(json_file_path), json_file_path),
crawler_definition,
converter_registry
diff --git a/unittests/test_macros.py b/unittests/test_macros.py
index cc79731e7e8e52ab972a22151b580720c10e195c..5244307db8e694ffb4864380d33936ebb76ae715 100644
--- a/unittests/test_macros.py
+++ b/unittests/test_macros.py
@@ -142,6 +142,7 @@ def test_multi_macros_toplevel(register_macros, macro_store_reset):
dat_loader = list(yaml.safe_load_all("""
---
metadata:
+ crawler-version: 0.3.1
macros:
- !defmacro
name: test_one
@@ -168,6 +169,10 @@ testnode: !macro
def test_load_definition(register_macros, macro_store_reset):
txt = """
+---
+metadata:
+ crawler-version: 0.3.1
+---
extroot:
type: Directory
match: extroot
@@ -183,6 +188,7 @@ extroot:
cfood = _temp_file_load("""
---
metadata:
+ crawler-version: 0.3.1
macros:
- !defmacro
name: test_one
@@ -256,6 +262,7 @@ def test_circular_macro_definition(register_macros, macro_store_reset):
cfood = _temp_file_load("""
---
metadata:
+ crawler-version: 0.3.1
macros:
- !defmacro
name: test_one
@@ -304,6 +311,7 @@ def test_use_macro_twice():
cfood = _temp_file_load("""
---
metadata:
+ crawler-version: 0.3.1
macros:
- !defmacro
name: test_twice
@@ -337,6 +345,7 @@ def test_documentation_example_2():
cfood = _temp_file_load("""
---
metadata:
+ crawler-version: 0.3.1
macros:
- !defmacro
name: MarkdownFile
@@ -374,6 +383,7 @@ def test_documentation_example_1():
cfood = _temp_file_load("""
---
metadata:
+ crawler-version: 0.3.1
macros:
- !defmacro
name: SimulationDatasetFile
@@ -422,6 +432,7 @@ def test_def_replacements():
cfood = _temp_file_load("""
---
metadata:
+ crawler-version: 0.3.1
macros:
- !defmacro
name: test_def_replacements
diff --git a/unittests/test_scalars_cfood.py b/unittests/test_scalars_cfood.py
index 4fbeeed628101654a20de18210bdac683cf801f4..89d94fc74ebda6aedfbee422294e99eab2216d73 100644
--- a/unittests/test_scalars_cfood.py
+++ b/unittests/test_scalars_cfood.py
@@ -10,16 +10,11 @@ from caoscrawler.converters import handle_value
from caoscrawler.crawl import Crawler
# We need the store for the above function
from caoscrawler.stores import GeneralStore
-
-from test_tool import dircheckstr, rfp
+from caoscrawler.scanner import scan_directory
+from caoscrawler.debug_tree import DebugTree
-@pytest.fixture
-def crawler():
- crawler = Crawler()
- crawled_data, debug_tree = crawler.crawl_directory(rfp("test_directories", "examples_article"),
- rfp("cfoods_scalar.yml"))
- return crawler, crawled_data, debug_tree
+from test_tool import dircheckstr, rfp
def test_handle_value():
@@ -38,8 +33,11 @@ def test_handle_value():
assert handle_value([4, 3, 2], store) == ([4, 3, 2], "single")
-def test_record_structure_generation(crawler):
- subd = crawler[2].debug_tree[dircheckstr("DataAnalysis")]
+def test_record_structure_generation():
+ dbt = DebugTree()
+ scan_directory(rfp("test_directories", "examples_article"), rfp("cfoods_scalar.yml"),
+ debug_tree=dbt)
+ subd = dbt.debug_tree[dircheckstr("DataAnalysis")]
assert len(subd) == 2
# variables store on Data Analysis node of debug tree
if "Data" in subd[0]:
diff --git a/unittests/test_table_converter.py b/unittests/test_table_converter.py
index 6a776fa2a633ae5c7bb21163732abec467e09f8b..d739695fc4c6a019f28f3c3697e3f134e0f1755e 100644
--- a/unittests/test_table_converter.py
+++ b/unittests/test_table_converter.py
@@ -28,6 +28,8 @@ test the converters module
from caoscrawler.converters import Converter
from caoscrawler.stores import GeneralStore
+from caoscrawler.scanner import scan_directory
+from caoscrawler.debug_tree import DebugTree
from caoscrawler.converters import (ConverterValidationError,
DictConverter, XLSXTableConverter, CSVTableConverter)
from caoscrawler.structure_elements import Directory
@@ -91,14 +93,6 @@ def dircheckstr(*pathcomponents):
return "caoscrawler.structure_elements.File: " + basename(join(*pathcomponents)) + ", " + rfp("test_directories", "examples_tables", "ExperimentalData", *pathcomponents)
-@pytest.fixture
-def crawler():
- crawler = Crawler()
- crawled_data, debug_tree = crawler.crawl_directory(rfp("test_directories", "examples_tables", "ExperimentalData"),
- rfp("test_directories", "examples_tables", "crawler_for_tables.yml"))
- return crawler, crawled_data, debug_tree
-
-
def test_convert_table(converter_registry):
extentions = ["xlsx", "csv", "tsv"]
if importlib.util.find_spec("odf") is not None:
@@ -151,9 +145,13 @@ def test_convert_table(converter_registry):
assert res[0].name == "jdsfkljadskf"
-def test_crawl_csv_table(crawler):
+def test_crawl_csv_table():
+ dbt = DebugTree()
+ scan_directory(rfp("test_directories", "examples_tables", "ExperimentalData"),
+ rfp("test_directories", "examples_tables", "crawler_for_tables.yml"),
+ debug_tree=dbt)
for file_ext in ["xlsx", "csv"]:
- subd = crawler[2].debug_tree[dircheckstr("test1." + file_ext)]
+ subd = dbt.debug_tree[dircheckstr("test1." + file_ext)]
record_experiment = subd[1]["Experiment"]
assert isinstance(record_experiment, db.Record)
assert isinstance(record_experiment.get_property("Measurements").value, list)
diff --git a/unittests/test_tool.py b/unittests/test_tool.py
index d0bb108453f1e390dc72980baed99e4b469e3edb..94266278cd2186a7df3b40a7457cc8c8bfc54ce3 100755
--- a/unittests/test_tool.py
+++ b/unittests/test_tool.py
@@ -32,6 +32,8 @@ import os
from caoscrawler.crawl import Crawler, SecurityMode, split_restricted_path
from caoscrawler.identifiable import Identifiable
from caoscrawler.structure_elements import File, DictTextElement, DictListElement, DictElement
+from caoscrawler.scanner import scan_directory
+from caoscrawler.debug_tree import DebugTree
from caoscrawler.identifiable_adapters import IdentifiableAdapter, LocalStorageIdentifiableAdapter
from simulated_server_data import full_data
from functools import partial
@@ -77,9 +79,10 @@ def dircheckstr(*pathcomponents):
@pytest.fixture
def crawler():
crawler = Crawler()
- crawled_data, debug_tree = crawler.crawl_directory(
+ debug_tree = DebugTree()
+ crawled_data = scan_directory(
rfp("test_directories", "examples_article"),
- rfp("scifolder_cfood.yml"))
+ rfp("scifolder_cfood.yml"), debug_tree=debug_tree)
return crawler, crawled_data, debug_tree
@@ -112,14 +115,16 @@ def ident(crawler):
return ident
-def test_record_structure_generation(crawler):
+def test_record_structure_generation():
# TODO How does this test relate to the test function in test_scalars_cfood with the same name?
# There seems to be code duplication
- debug_tree = crawler[2]
-
- subd = debug_tree.debug_tree[dircheckstr("DataAnalysis")]
- subc = debug_tree.debug_metadata["copied"][dircheckstr("DataAnalysis")]
+ dbt = DebugTree()
+ scan_directory(rfp("test_directories", "examples_article"),
+ rfp("scifolder_cfood.yml"),
+ debug_tree=dbt)
+ subd = dbt.debug_tree[dircheckstr("DataAnalysis")]
+ subc = dbt.debug_metadata["copied"][dircheckstr("DataAnalysis")]
assert len(subd) == 2
# variables store on Data Analysis node of debug tree
assert len(subd[0]) == 4
@@ -133,9 +138,9 @@ def test_record_structure_generation(crawler):
assert subd[0]["DataAnalysis"] == "examples_article/DataAnalysis"
assert subc[0]["DataAnalysis"] is False
- subd = debug_tree.debug_tree[dircheckstr(
+ subd = dbt.debug_tree[dircheckstr(
"DataAnalysis", "2020_climate-model-predict")]
- subc = debug_tree.debug_metadata["copied"][dircheckstr(
+ subc = dbt.debug_metadata["copied"][dircheckstr(
"DataAnalysis", "2020_climate-model-predict")]
assert len(subd[1]) == 1
@@ -163,12 +168,12 @@ def test_record_structure_generation(crawler):
assert subc[0]["date"] is False
assert subc[0]["identifier"] is False
- subd = debug_tree.debug_tree[dircheckstr("DataAnalysis",
- "2020_climate-model-predict",
- "2020-02-08_prediction-errors")]
- subc = debug_tree.debug_metadata["copied"][dircheckstr("DataAnalysis",
- "2020_climate-model-predict",
- "2020-02-08_prediction-errors")]
+ subd = dbt.debug_tree[dircheckstr("DataAnalysis",
+ "2020_climate-model-predict",
+ "2020-02-08_prediction-errors")]
+ subc = dbt.debug_metadata["copied"][dircheckstr("DataAnalysis",
+ "2020_climate-model-predict",
+ "2020-02-08_prediction-errors")]
assert len(subd[0]) == 12
assert subd[0]["date"] == "2020-02-08"
assert subd[0]["identifier"] == "prediction-errors"
@@ -290,7 +295,7 @@ def test_crawler_update_list(crawler, ident):
def test_synchronization(crawler, ident):
- insl, updl = crawler[0].synchronize(crawler[1], commit_changes=False)
+ insl, updl = crawler[0].synchronize(commit_changes=False, crawled_data=crawler[1])
assert len(insl) == 0
assert len(updl) == 0
@@ -339,16 +344,16 @@ def test_remove_unnecessary_updates():
@pytest.mark.xfail
def test_identifiable_adapter_no_identifiable(crawler, ident):
del ident._registered_identifiables["Person"]
- insl, updl = crawler.synchronize()
+ insl, updl = crawler[0].synchronize()
assert len(updl) == 0
- pers = [r for r in crawler.crawled_data if r.parents[0].name == "Person"]
+ pers = [r for r in crawler[0].crawled_data if r.parents[0].name == "Person"]
# All persons are inserted, because they are not identifiable:
assert len(insl) == len(pers)
def test_provenance_debug_data(crawler):
- crawler[0].save_debug_data(crawler[2], rfp("provenance.yml"))
+ crawler[0].save_debug_data(rfp("provenance.yml"), debug_tree=crawler[2])
with open(rfp("provenance.yml"), "r") as f:
provenance = yaml.load(f, Loader=yaml.SafeLoader)
@@ -597,9 +602,10 @@ def mock_get_entity_by_name(name):
def prepare_crawler_with_sec_mode(mode, ident):
crawler = Crawler(securityMode=mode)
- crawled_data, debug_tree = crawler.crawl_directory(
+ debug_tree = DebugTree()
+ crawled_data = scan_directory(
rfp("test_directories", "examples_article"),
- rfp("scifolder_cfood.yml"))
+ rfp("scifolder_cfood.yml"), debug_tree=debug_tree)
crawler.identifiableAdapter = ident
return crawler, crawled_data, debug_tree
@@ -656,7 +662,7 @@ def test_security_mode(updateCacheMock, upmock, insmock, ident):
# trivial case: nothing to do
crawler, crawled_data, debug_tree = prepare_crawler_with_sec_mode(SecurityMode.RETRIEVE, ident)
- crawler.synchronize(crawled_data, commit_changes=True)
+ crawler.synchronize(commit_changes=True, crawled_data=crawled_data)
assert crawler.run_id is not None
insmock.assert_not_called()
upmock.assert_not_called()
@@ -667,7 +673,7 @@ def test_security_mode(updateCacheMock, upmock, insmock, ident):
# remove one element
del ident._records[-1]
# insert forbidden
- crawler.synchronize(crawled_data, commit_changes=True)
+ crawler.synchronize(commit_changes=True, crawled_data=crawled_data)
assert crawler.run_id is not None
insmock.assert_not_called()
upmock.assert_not_called()
@@ -681,7 +687,7 @@ def test_security_mode(updateCacheMock, upmock, insmock, ident):
crawler, crawled_data, debug_tree = prepare_crawler_with_sec_mode(SecurityMode.RETRIEVE, ident)
# change one element
change_non_identifiable_prop(ident)
- crawler.synchronize(crawled_data, commit_changes=True)
+ crawler.synchronize(commit_changes=True, crawled_data=crawled_data)
assert crawler.run_id is not None
insmock.assert_not_called()
upmock.assert_not_called()
@@ -695,7 +701,7 @@ def test_security_mode(updateCacheMock, upmock, insmock, ident):
crawler, crawled_data, debug_tree = prepare_crawler_with_sec_mode(SecurityMode.INSERT, ident)
# remove one element
del ident._records[-1]
- crawler.synchronize(crawled_data, commit_changes=True)
+ crawler.synchronize(commit_changes=True, crawled_data=crawled_data)
assert crawler.run_id is not None
insmock.assert_called_once()
upmock.assert_not_called()
@@ -709,7 +715,7 @@ def test_security_mode(updateCacheMock, upmock, insmock, ident):
crawler, crawled_data, debug_tree = prepare_crawler_with_sec_mode(SecurityMode.INSERT, ident)
# change one element
change_non_identifiable_prop(ident)
- crawler.synchronize(crawled_data, commit_changes=True)
+ crawler.synchronize(commit_changes=True, crawled_data=crawled_data)
assert crawler.run_id is not None
insmock.assert_not_called()
upmock.assert_not_called()
@@ -724,7 +730,7 @@ def test_security_mode(updateCacheMock, upmock, insmock, ident):
# change two elements
change_non_identifiable_prop(ident)
change_identifiable_prop(ident)
- crawler.synchronize(crawled_data, commit_changes=True)
+ crawler.synchronize(commit_changes=True, crawled_data=crawled_data)
assert crawler.run_id is not None
insmock.asser_called_once()
upmock.assert_not_called()
@@ -976,6 +982,7 @@ def test_split_restricted_path():
assert ["el", "el"] == split_restricted_path("/el/el")
+@pytest.mark.filterwarnings("ignore:The prefix:DeprecationWarning")
def test_deprecated_prefix_option():
"""Test that calling the crawler's main function with the deprecated
`prefix` option raises the correct errors and warnings.
diff --git a/unittests/test_tool_extended.py b/unittests/test_tool_extended.py
index 4cb7286f91df577e7f17a354bd54547b3802c55d..7dd4282e4c6d206c8c360424d865b9f736b5e582 100644
--- a/unittests/test_tool_extended.py
+++ b/unittests/test_tool_extended.py
@@ -6,7 +6,9 @@
from caoscrawler import Crawler
from caoscrawler.structure_elements import File, DictTextElement, DictListElement
from caoscrawler.identifiable_adapters import IdentifiableAdapter, LocalStorageIdentifiableAdapter
+from caoscrawler.scanner import scan_directory
from functools import partial
+from caoscrawler.debug_tree import DebugTree
from copy import deepcopy
from unittest.mock import MagicMock, Mock
from os.path import join, dirname, basename
@@ -37,10 +39,10 @@ def dircheckstr(*pathcomponents, structure_element_type="Directory"):
@pytest.fixture
def crawler():
- crawler = Crawler()
- crawled_data, debug_tree = crawler.crawl_directory(rfp("test_directories", "examples_article"),
- rfp("scifolder_extended.yml"))
- return crawler, crawled_data, debug_tree
+ crawler = Crawler(debug=True)
+ crawler.crawl_directory(rfp("test_directories", "examples_article"),
+ rfp("scifolder_extended.yml"))
+ return crawler
# @pytest.fixture
@@ -69,10 +71,14 @@ def crawler():
# return ident
-def test_file_structure_generation(crawler):
- sd = crawler[2].debug_tree[dircheckstr("SimulationData",
- "2020_climate-model-predict", "2020-02-01",
- "README.md", structure_element_type="File")]
+def test_file_structure_generation():
+ dbt = DebugTree()
+ scan_directory(rfp("test_directories", "examples_article"),
+ rfp("scifolder_extended.yml"),
+ debug_tree=dbt)
+ sd = dbt.debug_tree[dircheckstr("SimulationData",
+ "2020_climate-model-predict", "2020-02-01",
+ "README.md", structure_element_type="File")]
assert sd[1]["ReadmeFile"].role == "File"
assert len(sd[1]["ReadmeFile"].path) > 0
assert len(sd[1]["ReadmeFile"].file) > 0
diff --git a/unittests/test_variable_substitutions.py b/unittests/test_variable_substitutions.py
index 8680d792973d1f6c9aac2fe2ff0229edfa495d57..f13e759982e8102bbf37e65311ff4073ba52e5a2 100644
--- a/unittests/test_variable_substitutions.py
+++ b/unittests/test_variable_substitutions.py
@@ -2,7 +2,9 @@
# Tests for variable substitutions
# A. Schlemmer, 05/2022
+from caoscrawler.debug_tree import DebugTree
from caoscrawler import Crawler
+from caoscrawler.scanner import scan_directory
from caoscrawler.structure_elements import File, DictTextElement, DictListElement
from caoscrawler.identifiable_adapters import IdentifiableAdapter, LocalStorageIdentifiableAdapter
from functools import partial
@@ -32,33 +34,21 @@ def dircheckstr(element_type, *pathcomponents):
return "caoscrawler.structure_elements." + element_type + ": " + basename(join(*pathcomponents)) + ", " + rfp("test_directories", "example_substitutions", *pathcomponents)
-@pytest.fixture
-def crawler():
- crawler = Crawler()
- crawled_data, debug_tree = crawler.crawl_directory(rfp("test_directories", "example_substitutions", "ExperimentalData"),
- rfp("test_directories", "example_substitutions", "substitutions.yml"))
- return crawler, crawled_data, debug_tree
+def test_substitutions():
-
-@pytest.fixture
-def crawler_2():
- crawler = Crawler()
- crawled_data, debug_tree = crawler.crawl_directory(rfp("test_directories", "example_substitutions", "ExperimentalData"),
- rfp("test_directories", "example_substitutions",
- "substitutions_parents.yml"))
- return crawler, crawled_data, debug_tree
-
-
-def test_substitutions(crawler):
+ dbt = DebugTree()
+ scan_directory(rfp("test_directories", "example_substitutions", "ExperimentalData"),
+ rfp("test_directories", "example_substitutions", "substitutions.yml"),
+ debug_tree=dbt)
# @review Florian Spreckelsen 2022-05-13
for i in range(2):
- subd = crawler[2].debug_tree[dircheckstr(
+ subd = dbt.debug_tree[dircheckstr(
"File", "ExperimentalData", "220512_data.dat")]
assert subd[i]["Experiment"].get_property("date").value == "2022-05-12"
assert isinstance(subd[i]["ExperimentSeries"].get_property(
"Experiment").value, db.Record)
- subd = crawler[2].debug_tree[dircheckstr("Directory", "ExperimentalData")]
+ subd = dbt.debug_tree[dircheckstr("Directory", "ExperimentalData")]
assert subd[i]["Project"].name == "project"
assert isinstance(subd[i]["Project"].get_property(
"Experiments").value, list)
@@ -70,11 +60,16 @@ def test_substitutions(crawler):
"dates").value[0] == "2022-05-12"
-def test_substitutions_parents(crawler_2):
+def test_substitutions_parents():
+ dbt = DebugTree()
+ scan_directory(rfp("test_directories", "example_substitutions", "ExperimentalData"),
+ rfp("test_directories", "example_substitutions",
+ "substitutions_parents.yml"),
+ debug_tree=dbt)
# This is a test for:
# https://gitlab.indiscale.com/caosdb/src/caosdb-crawler/-/issues/35
# ... testing whether variable substitutions can be used in parent declarations.
- subd = crawler_2[2].debug_tree[dircheckstr(
+ subd = dbt.debug_tree[dircheckstr(
"File", "ExperimentalData", "220512_data.dat")]
# subd[0] <- generalStore
# subd[1] <- recordStore
@@ -85,11 +80,16 @@ def test_substitutions_parents(crawler_2):
assert parents[1].name == "Month_05"
-def test_empty_parents(crawler_2):
+def test_empty_parents():
+ dbt = DebugTree()
+ scan_directory(rfp("test_directories", "example_substitutions", "ExperimentalData"),
+ rfp("test_directories", "example_substitutions",
+ "substitutions_parents.yml"),
+ debug_tree=dbt)
# This is a test for:
# https://gitlab.com/caosdb/caosdb-crawler/-/issues/8
- subd = crawler_2[2].debug_tree[dircheckstr(
+ subd = dbt.debug_tree[dircheckstr(
"File", "ExperimentalData", "220512_data.dat")]
parents = subd[1]["RecordWithoutParents"].get_parents()