Suggestion htw

for !104 (comment 50361)

and for !104 (comment 50366) (i.e. it removes the debug_tree from return values and only creates the it if a debug tree is given as argument to e.g. scanner)

integrationtests/basic_example/test_basic.py

@@ -27,6 +27,7 @@ an integration test module that does basic integration tests
 """
 
 from caosadvancedtools.crawler import Crawler as OldCrawler
+from caoscrawler.debug_tree import DebugTree
 import os
 from caosdb import EmptyUniqueQueryError
 import argparse
@@ -36,6 +37,7 @@ from caoscrawler import Crawler, SecurityMode
 from caoscrawler.identifiable import Identifiable
 import caosdb as db
 from caoscrawler.identifiable_adapters import CaosDBIdentifiableAdapter
+from caoscrawler.scanner import scan_directory
 import pytest
 from caosadvancedtools.models.parser import parse_model_from_yaml
 import yaml
@@ -82,24 +84,28 @@ def ident():
     return ident
 
 
-def crawl_standard_test_directory(cr: Crawler,
-                                  subdir: str = "examples_article",
-                                  cfood: str = "scifolder_cfood.yml"):
-    return cr.crawl_directory(rfp("..", "..", "unittests", "test_directories", subdir),
-                              rfp("..", "..", "unittests", cfood))
+def crawl_standard_test_directory(subdir: str = "examples_article",
+                                  cfood: str = "scifolder_cfood.yml",
+                                  debug_tree=None):
+    return scan_directory(rfp("..", "..", "unittests", "test_directories", subdir),
+                          rfp("..", "..", "unittests", cfood),
+                          debug_tree=debug_tree)
 
 
 @pytest.fixture
 def crawler(ident):
     cr = Crawler(identifiableAdapter=ident)
-    crawled_data, debug_tree = crawl_standard_test_directory(cr)
+    debug_tree = DebugTree()
+    crawled_data = crawl_standard_test_directory(debug_tree=debug_tree)
     return cr, crawled_data, debug_tree
 
 
 @pytest.fixture
 def crawler_extended(ident):
     cr = Crawler(identifiableAdapter=ident)
-    crawled_data, debug_tree = crawl_standard_test_directory(cr, cfood="scifolder_extended.yml")
+    debug_tree = DebugTree()
+    crawled_data = crawl_standard_test_directory(
+        cfood="scifolder_extended.yml", debug_tree=debug_tree)
     # correct paths for current working directory
     file_list = [r for r in crawled_data if r.role == "File"]
     for f in file_list:
@@ -108,7 +114,7 @@ def crawler_extended(ident):
 
 
 def test_ambigious_lookup(clear_database, usemodel, crawler, ident):
-    ins, ups = crawler[0].synchronize(crawler[1])
+    ins, ups = crawler[0].synchronize(crawled_data=crawler[1])
 
     proj = db.execute_query("FIND Project WITH identifier='SpeedOfLight'", unique=True)
     with pytest.raises(RuntimeError, match=".*unambigiously.*"):
@@ -117,7 +123,7 @@ def test_ambigious_lookup(clear_database, usemodel, crawler, ident):
 
 
 def test_single_insertion(clear_database, usemodel, crawler, ident):
-    ins, ups = crawler[0].synchronize(crawler[1])
+    ins, ups = crawler[0].synchronize(crawled_data=crawler[1])
 
     # This test also generates the file records.xml used in some of the unittesets:
     res = db.execute_query("FIND Record")
@@ -139,52 +145,52 @@ def test_single_insertion(clear_database, usemodel, crawler, ident):
 
     # Do a second run on the same data, there should be no changes:
     crawler = Crawler(identifiableAdapter=ident)
-    crawled_data, debug_tree = crawler.crawl_directory(rfp("../../unittests/test_directories", "examples_article"),
-                                                       rfp("../../unittests/scifolder_cfood.yml"))
-    ins, ups = crawler.synchronize(crawled_data)
+    crawled_data = scan_directory(rfp("../../unittests/test_directories", "examples_article"),
+                                  rfp("../../unittests/scifolder_cfood.yml"))
+    ins, ups = crawler.synchronize(crawled_data=crawled_data)
     assert len(ins) == 0
     assert len(ups) == 0
 
 
 def test_multiple_insertions(clear_database, usemodel, ident, crawler):
-    ins, ups = crawler[0].synchronize(crawler[1])
+    ins, ups = crawler[0].synchronize(crawled_data=crawler[1])
 
     # Do a second run on the same data, there should be no changes:
     cr = Crawler(identifiableAdapter=ident)
-    crawled_data, debug_tree = crawl_standard_test_directory(cr)
-    ins, ups = cr.synchronize(crawled_data)
+    crawled_data = crawl_standard_test_directory()
+    ins, ups = cr.synchronize(crawled_data=crawled_data)
     assert len(ins) == 0
     assert len(ups) == 0
 
 
 def test_insertion(clear_database, usemodel, ident, crawler):
-    ins, ups = crawler[0].synchronize(crawler[1])
+    ins, ups = crawler[0].synchronize(crawled_data=crawler[1])
 
     # Do a second run on the same data, there should a new insert:
     cr = Crawler(identifiableAdapter=ident)
-    crawled_data, debug_tree = crawl_standard_test_directory(cr, "example_insert")
+    crawled_data = crawl_standard_test_directory("example_insert")
     assert len(crawled_data) == 3
-    ins, ups = cr.synchronize(crawled_data)
+    ins, ups = cr.synchronize(crawled_data=crawled_data)
     assert len(ins) == 1
     assert len(ups) == 0
 
     # Do it again to check whether nothing is changed:
     cr = Crawler(identifiableAdapter=ident)
-    crawled_data, debug_tree = crawl_standard_test_directory(cr, "example_insert")
+    crawled_data = crawl_standard_test_directory("example_insert")
     assert len(crawled_data) == 3
-    ins, ups = cr.synchronize(crawled_data)
+    ins, ups = cr.synchronize(crawled_data=crawled_data)
     assert len(ins) == 0
     assert len(ups) == 0
 
 
 def test_insert_auth(clear_database, usemodel, ident, crawler):
-    ins, ups = crawler[0].synchronize(crawler[1])
+    ins, ups = crawler[0].synchronize(crawled_data=crawler[1])
 
     # Do a second run on the same data, there should a new insert:
     cr = Crawler(identifiableAdapter=ident, securityMode=SecurityMode.RETRIEVE)
-    crawled_data, debug_tree = crawl_standard_test_directory(cr, "example_insert")
+    crawled_data = crawl_standard_test_directory("example_insert")
     assert len(crawled_data) == 3
-    ins, ups = cr.synchronize(crawled_data)
+    ins, ups = cr.synchronize(crawled_data=crawled_data)
     assert len(ins) == 1
     assert not ins[0].is_valid()
     nins, nups = OldCrawler.update_authorized_changes(cr.run_id)
@@ -192,37 +198,36 @@ def test_insert_auth(clear_database, usemodel, ident, crawler):
 
     # Do it again to check whether nothing is changed:
     cr = Crawler(identifiableAdapter=ident)
-    crawled_data, debug_tree = crawl_standard_test_directory(cr, "example_insert")
+    crawled_data = crawl_standard_test_directory("example_insert")
     assert len(crawled_data) == 3
-    ins, ups = cr.synchronize(crawled_data)
+    ins, ups = cr.synchronize(crawled_data=crawled_data)
     assert len(ins) == 0
     assert len(ups) == 0
 
 
 def test_insertion_and_update(clear_database, usemodel, ident, crawler):
-    ins, ups = crawler[0].synchronize(crawler[1])
+    ins, ups = crawler[0].synchronize(crawled_data=crawler[1])
 
     cr = Crawler(identifiableAdapter=ident)
-    crawled_data, debug_tree = crawl_standard_test_directory(cr, "example_insert")
-    ins, ups = cr.synchronize(crawled_data)
+    crawled_data = crawl_standard_test_directory("example_insert")
+    ins, ups = cr.synchronize(crawled_data=crawled_data)
 
     cr = Crawler(identifiableAdapter=ident)
-    crawled_data, debug_tree = crawl_standard_test_directory(cr, "example_overwrite_1")
-    # print(cr.crawled_data)
+    crawled_data = crawl_standard_test_directory("example_overwrite_1")
     # cr.save_debug_data(rfp("provenance.yml"))
     assert len(crawled_data) == 3
-    ins, ups = cr.synchronize(crawled_data)
+    ins, ups = cr.synchronize(crawled_data=crawled_data)
     assert len(ins) == 0
     assert len(ups) == 1
 
 
 def test_identifiable_update(clear_database, usemodel, ident, crawler):
-    ins, ups = crawler[0].synchronize(crawler[1])
+    ins, ups = crawler[0].synchronize(crawled_data=crawler[1])
 
     # Do a second run on the same data with a change in one
     # of the identifiables:
     cr = Crawler(identifiableAdapter=ident)
-    crawled_data, debug_tree = crawl_standard_test_directory(cr)
+    crawled_data = crawl_standard_test_directory()
 
     # Test the addition of a single property:
     l = crawled_data
@@ -234,13 +239,13 @@ def test_identifiable_update(clear_database, usemodel, ident, crawler):
                 name="email", value="testperson@testaccount.test")
             print("one change")
             break
-    ins, ups = cr.synchronize(crawled_data)
+    ins, ups = cr.synchronize(crawled_data=crawled_data)
     assert len(ins) == 0
     assert len(ups) == 1
 
     # Test the change within one property:
     cr = Crawler(identifiableAdapter=ident)
-    crawled_data, debug_tree = crawl_standard_test_directory(cr)
+    crawled_data = crawl_standard_test_directory()
     l = crawled_data
     for record in l:
         if (record.parents[0].name == "Measurement" and
@@ -248,13 +253,13 @@ def test_identifiable_update(clear_database, usemodel, ident, crawler):
             record.add_property(name="email", value="testperson@coolmail.test")
             print("one change")
             break
-    ins, ups = cr.synchronize(crawled_data)
+    ins, ups = cr.synchronize(crawled_data=crawled_data)
     assert len(ins) == 0
     assert len(ups) == 1
 
     # Changing the date should result in a new insertion:
     cr = Crawler(identifiableAdapter=ident)
-    crawled_data, debug_tree = crawl_standard_test_directory(cr)
+    crawled_data = crawl_standard_test_directory()
     l = crawled_data
     for record in l:
         if (record.parents[0].name == "Measurement" and
@@ -263,15 +268,15 @@ def test_identifiable_update(clear_database, usemodel, ident, crawler):
             record.get_property("date").value = "2012-01-02"
             print("one change")
             break
-    ins, ups = cr.synchronize(crawled_data)
+    ins, ups = cr.synchronize(crawled_data=crawled_data)
     assert len(ins) == 1
     assert len(ups) == 0
 
 
 def test_file_insertion_dry(clear_database, usemodel, ident):
     crawler_extended = Crawler(identifiableAdapter=ident)
-    crawled_data, debug_tree = crawl_standard_test_directory(
-        crawler_extended, cfood="scifolder_extended.yml")
+    crawled_data = crawl_standard_test_directory(
+        cfood="scifolder_extended.yml")
     file_list = [r for r in crawled_data if r.role == "File"]
     assert len(file_list) == 11
 
@@ -279,14 +284,15 @@ def test_file_insertion_dry(clear_database, usemodel, ident):
         assert f.path.endswith("README.md")
         assert f.path[1:] == f.file
 
-    ins, ups = crawler_extended.synchronize(crawled_data, commit_changes=False)
+    ins, ups = crawler_extended.synchronize(crawled_data=crawled_data, commit_changes=False)
     assert len(ups) == 0
     file_list_ins = [r for r in ins if r.role == "File"]
     assert len(file_list_ins) == 11
 
 
 def test_file_insertion(clear_database, usemodel, ident, crawler_extended):
-    ins, ups = crawler_extended[0].synchronize(crawler_extended[1], commit_changes=True)
+    ins, ups = crawler_extended[0].synchronize(
+        crawled_data=crawler_extended[1], commit_changes=True)
     file_list_ins = [r for r in ins if r.role == "File"]
     assert len(file_list_ins) == 11
 
@@ -302,16 +308,17 @@ def test_file_insertion(clear_database, usemodel, ident, crawler_extended):
 
 
 def test_file_update(clear_database, usemodel, ident, crawler_extended):
-    ins1, ups1 = crawler_extended[0].synchronize(crawler_extended[1], commit_changes=True)
+    ins1, ups1 = crawler_extended[0].synchronize(
+        crawled_data=crawler_extended[1], commit_changes=True)
     file_list_ins = [r for r in ins1 if r.role == "File"]
 
     cr = Crawler(identifiableAdapter=ident)
-    crawled_data, debug_tree = crawl_standard_test_directory(cr, cfood="scifolder_extended.yml")
+    crawled_data = crawl_standard_test_directory(cfood="scifolder_extended.yml")
 
     file_list = [r for r in crawled_data if r.role == "File"]
     for f in file_list:
         f.file = rfp("..", "..", "unittests", "test_directories", f.file)
-    ins2, ups2 = cr.synchronize(crawled_data, commit_changes=True)
+    ins2, ups2 = cr.synchronize(crawled_data=crawled_data, commit_changes=True)
     assert len(ups1) == 0
     assert len(ups2) == 0
 
@@ -321,12 +328,12 @@ def test_file_update(clear_database, usemodel, ident, crawler_extended):
     assert len(res[0].parents) == 0
 
     cr2 = Crawler(identifiableAdapter=ident)
-    crawled_data, debug_tree = crawl_standard_test_directory(cr2, cfood="scifolder_extended2.yml")
+    crawled_data = crawl_standard_test_directory(cfood="scifolder_extended2.yml")
 
     file_list = [r for r in crawled_data if r.role == "File"]
     for f in file_list:
         f.file = rfp("..", "..", "unittests", "test_directories", f.file)
-    ins3, ups3 = cr2.synchronize(crawled_data, commit_changes=True)
+    ins3, ups3 = cr2.synchronize(crawled_data=crawled_data, commit_changes=True)
     assert len(ups3) == 11
 
     res = db.execute_query("Find File")