From 2cc517ee59f7d48af45b490b598c6e90e85e2ee3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Henrik=20tom=20W=C3=B6rden?= <h.tomwoerden@indiscale.com>
Date: Thu, 5 May 2022 15:13:00 +0200
Subject: [PATCH] move
---
integrationtests/README.md | 5 +-
.../realworld_example/test_dataset_crawler.py | 51 +++++++++++++++----
.../realworld_example/data/35/.dataspace.json | 0
.../03_raw_data/001_dataset1/demo-dataset.csv | 0
.../35/03_raw_data/001_dataset1/metadata.json | 0
.../data/35/03_raw_data/README_RawData.md | 0
.../35/03_raw_data/metadata-template.json | 0
.../README_ProcessedData.md | 0
.../04_data_processing/metadata-template.json | 0
.../data/35/05_results/README_Results.md | 0
.../realworld_example/data/35/README.md | 0
.../realworld_example/dataset_cfoods.yml | 0
.../schema/dataset-inheritance.yml | 0
.../schema/dataset.schema.json | 0
.../schema/dataspace.schema.json | 0
.../schema/zmt-organisation.yml | 0
16 files changed, 42 insertions(+), 14 deletions(-)
rename integrationtests/test_data/{ => extroot}/realworld_example/data/35/.dataspace.json (100%)
rename integrationtests/test_data/{ => extroot}/realworld_example/data/35/03_raw_data/001_dataset1/demo-dataset.csv (100%)
rename integrationtests/test_data/{ => extroot}/realworld_example/data/35/03_raw_data/001_dataset1/metadata.json (100%)
rename integrationtests/test_data/{ => extroot}/realworld_example/data/35/03_raw_data/README_RawData.md (100%)
rename integrationtests/test_data/{ => extroot}/realworld_example/data/35/03_raw_data/metadata-template.json (100%)
rename integrationtests/test_data/{ => extroot}/realworld_example/data/35/04_data_processing/README_ProcessedData.md (100%)
rename integrationtests/test_data/{ => extroot}/realworld_example/data/35/04_data_processing/metadata-template.json (100%)
rename integrationtests/test_data/{ => extroot}/realworld_example/data/35/05_results/README_Results.md (100%)
rename integrationtests/test_data/{ => extroot}/realworld_example/data/35/README.md (100%)
rename integrationtests/test_data/{ => extroot}/realworld_example/dataset_cfoods.yml (100%)
rename integrationtests/test_data/{ => extroot}/realworld_example/schema/dataset-inheritance.yml (100%)
rename integrationtests/test_data/{ => extroot}/realworld_example/schema/dataset.schema.json (100%)
rename integrationtests/test_data/{ => extroot}/realworld_example/schema/dataspace.schema.json (100%)
rename integrationtests/test_data/{ => extroot}/realworld_example/schema/zmt-organisation.yml (100%)
diff --git a/integrationtests/README.md b/integrationtests/README.md
index 5c308f51..96789ed9 100644
--- a/integrationtests/README.md
+++ b/integrationtests/README.md
@@ -1,3 +1,2 @@
-1. Clear database (see clear_database.py)
-2. Insert model (see insert_model.py)
-3. Run test.py
+1. Mount test_data/extroot as extroot folder in the CaosDB server
+2. use an empty server
diff --git a/integrationtests/realworld_example/test_dataset_crawler.py b/integrationtests/realworld_example/test_dataset_crawler.py
index 3c8e486a..51f67330 100644
--- a/integrationtests/realworld_example/test_dataset_crawler.py
+++ b/integrationtests/realworld_example/test_dataset_crawler.py
@@ -34,10 +34,44 @@ from newcrawler.converters import JSONFileConverter, DictConverter
from newcrawler.identifiable_adapters import CaosDBIdentifiableAdapter
from newcrawler.structure_elements import File, JSONFile, Directory
+from caosadvancedtools.testutils import clear_database, set_test_key
+import sys
-def test_dataset():
- crawler_definition_path = "./dataset_cfoods.yml"
- #json_file_path = rfp("test_directories", "single_file_test_data", "testjson.json")
+from caosadvancedtools.models.parser import parse_model_from_json_schema, parse_model_from_yaml
+set_test_key("10b128cf8a1372f30aa3697466bb55e76974e0c16a599bb44ace88f19c8f61e2")
+
+
+def rfp(*pathcomponents):
+ """
+ Return full path.
+ Shorthand convenience function.
+ """
+ return os.path.join(os.path.dirname(__file__), *pathcomponents)
+
+
+DATADIR = rfp("..", "test_data", "extroot", "realworld_example")
+
+
+@pytest.fixture
+def usemodel():
+ # First load dataspace data model
+ dataspace_definitions = parse_model_from_json_schema(
+ os.path.join(DATADIR, "schema", "dataspace.schema.json"))
+ dataspace_definitions.sync_data_model(noquestion=True)
+
+ # Then general dataset definitions
+ dataset_definitions = parse_model_from_json_schema(
+ os.path.join(DATADIR, "schema", "dataset.schema.json"))
+ dataset_definitions.sync_data_model(noquestion=True)
+
+ # Finally, add inheritances as defined in yaml
+ dataset_inherits = parse_model_from_yaml(
+ os.path.join(DATADIR, "schema", "dataset-inheritance.yml"))
+ dataset_inherits.sync_data_model(noquestion=True)
+
+
+def test_dataset(clear_database, usemodel):
+ # json_file_path = rfp("test_directories", "single_file_test_data", "testjson.json")
ident = CaosDBIdentifiableAdapter()
ident.register_identifiable(
@@ -48,15 +82,14 @@ def test_dataset():
).add_parent("Person").add_property("full_name"))
crawler = Crawler(debug=True, identifiableAdapter=ident)
- crawler_definition = crawler.load_definition(crawler_definition_path)
- #print(json.dumps(crawler_definition, indent=3))
+ crawler_definition = crawler.load_definition(os.path.join(DATADIR, "dataset_cfoods.yml"))
+ # print(json.dumps(crawler_definition, indent=3))
# Load and register converter packages:
converter_registry = crawler.load_converters(crawler_definition)
# print("DictIntegerElement" in converter_registry)
records = crawler.start_crawling(
- Directory('data',
- "data"),
+ Directory(os.path.join(DATADIR, 'data'), "data"),
crawler_definition,
converter_registry
)
@@ -67,7 +100,3 @@ def test_dataset():
# print(subc)
# print(records)
ins, ups = crawler.synchronize()
-
-
-if __name__ == "__main__":
- test_dataset()
diff --git a/integrationtests/test_data/realworld_example/data/35/.dataspace.json b/integrationtests/test_data/extroot/realworld_example/data/35/.dataspace.json
similarity index 100%
rename from integrationtests/test_data/realworld_example/data/35/.dataspace.json
rename to integrationtests/test_data/extroot/realworld_example/data/35/.dataspace.json
diff --git a/integrationtests/test_data/realworld_example/data/35/03_raw_data/001_dataset1/demo-dataset.csv b/integrationtests/test_data/extroot/realworld_example/data/35/03_raw_data/001_dataset1/demo-dataset.csv
similarity index 100%
rename from integrationtests/test_data/realworld_example/data/35/03_raw_data/001_dataset1/demo-dataset.csv
rename to integrationtests/test_data/extroot/realworld_example/data/35/03_raw_data/001_dataset1/demo-dataset.csv
diff --git a/integrationtests/test_data/realworld_example/data/35/03_raw_data/001_dataset1/metadata.json b/integrationtests/test_data/extroot/realworld_example/data/35/03_raw_data/001_dataset1/metadata.json
similarity index 100%
rename from integrationtests/test_data/realworld_example/data/35/03_raw_data/001_dataset1/metadata.json
rename to integrationtests/test_data/extroot/realworld_example/data/35/03_raw_data/001_dataset1/metadata.json
diff --git a/integrationtests/test_data/realworld_example/data/35/03_raw_data/README_RawData.md b/integrationtests/test_data/extroot/realworld_example/data/35/03_raw_data/README_RawData.md
similarity index 100%
rename from integrationtests/test_data/realworld_example/data/35/03_raw_data/README_RawData.md
rename to integrationtests/test_data/extroot/realworld_example/data/35/03_raw_data/README_RawData.md
diff --git a/integrationtests/test_data/realworld_example/data/35/03_raw_data/metadata-template.json b/integrationtests/test_data/extroot/realworld_example/data/35/03_raw_data/metadata-template.json
similarity index 100%
rename from integrationtests/test_data/realworld_example/data/35/03_raw_data/metadata-template.json
rename to integrationtests/test_data/extroot/realworld_example/data/35/03_raw_data/metadata-template.json
diff --git a/integrationtests/test_data/realworld_example/data/35/04_data_processing/README_ProcessedData.md b/integrationtests/test_data/extroot/realworld_example/data/35/04_data_processing/README_ProcessedData.md
similarity index 100%
rename from integrationtests/test_data/realworld_example/data/35/04_data_processing/README_ProcessedData.md
rename to integrationtests/test_data/extroot/realworld_example/data/35/04_data_processing/README_ProcessedData.md
diff --git a/integrationtests/test_data/realworld_example/data/35/04_data_processing/metadata-template.json b/integrationtests/test_data/extroot/realworld_example/data/35/04_data_processing/metadata-template.json
similarity index 100%
rename from integrationtests/test_data/realworld_example/data/35/04_data_processing/metadata-template.json
rename to integrationtests/test_data/extroot/realworld_example/data/35/04_data_processing/metadata-template.json
diff --git a/integrationtests/test_data/realworld_example/data/35/05_results/README_Results.md b/integrationtests/test_data/extroot/realworld_example/data/35/05_results/README_Results.md
similarity index 100%
rename from integrationtests/test_data/realworld_example/data/35/05_results/README_Results.md
rename to integrationtests/test_data/extroot/realworld_example/data/35/05_results/README_Results.md
diff --git a/integrationtests/test_data/realworld_example/data/35/README.md b/integrationtests/test_data/extroot/realworld_example/data/35/README.md
similarity index 100%
rename from integrationtests/test_data/realworld_example/data/35/README.md
rename to integrationtests/test_data/extroot/realworld_example/data/35/README.md
diff --git a/integrationtests/test_data/realworld_example/dataset_cfoods.yml b/integrationtests/test_data/extroot/realworld_example/dataset_cfoods.yml
similarity index 100%
rename from integrationtests/test_data/realworld_example/dataset_cfoods.yml
rename to integrationtests/test_data/extroot/realworld_example/dataset_cfoods.yml
diff --git a/integrationtests/test_data/realworld_example/schema/dataset-inheritance.yml b/integrationtests/test_data/extroot/realworld_example/schema/dataset-inheritance.yml
similarity index 100%
rename from integrationtests/test_data/realworld_example/schema/dataset-inheritance.yml
rename to integrationtests/test_data/extroot/realworld_example/schema/dataset-inheritance.yml
diff --git a/integrationtests/test_data/realworld_example/schema/dataset.schema.json b/integrationtests/test_data/extroot/realworld_example/schema/dataset.schema.json
similarity index 100%
rename from integrationtests/test_data/realworld_example/schema/dataset.schema.json
rename to integrationtests/test_data/extroot/realworld_example/schema/dataset.schema.json
diff --git a/integrationtests/test_data/realworld_example/schema/dataspace.schema.json b/integrationtests/test_data/extroot/realworld_example/schema/dataspace.schema.json
similarity index 100%
rename from integrationtests/test_data/realworld_example/schema/dataspace.schema.json
rename to integrationtests/test_data/extroot/realworld_example/schema/dataspace.schema.json
diff --git a/integrationtests/test_data/realworld_example/schema/zmt-organisation.yml b/integrationtests/test_data/extroot/realworld_example/schema/zmt-organisation.yml
similarity index 100%
rename from integrationtests/test_data/realworld_example/schema/zmt-organisation.yml
rename to integrationtests/test_data/extroot/realworld_example/schema/zmt-organisation.yml
--
GitLab