diff --git a/.docker/docker-compose.yml b/.docker/docker-compose.yml
index bbee24fbd8c898c479a0fafa13000ddf506d00eb..e5bb4c9b8ca6ad1750922cb07c92cd6c5eb77c6b 100644
--- a/.docker/docker-compose.yml
+++ b/.docker/docker-compose.yml
@@ -17,8 +17,8 @@ services:
- type: bind
source: ./cert
target: /opt/caosdb/cert
- - type: volume
- source: extroot
+ - type: bind
+ source: "../integrationtests/test_data/extroot"
target: /opt/caosdb/mnt/extroot
- type: volume
source: scripting
@@ -36,7 +36,6 @@ services:
CAOSDB_CONFIG_TRANSACTION_BENCHMARK_ENABLED: "TRUE"
volumes:
scripting:
- extroot:
authtoken:
networks:
caosnet:
diff --git a/integrationtests/realworld_example/crawl.sh b/integrationtests/realworld_example/crawl.sh
deleted file mode 100755
index 55a2a331fe517a539e2dd937ac35605c72b496c9..0000000000000000000000000000000000000000
--- a/integrationtests/realworld_example/crawl.sh
+++ /dev/null
@@ -1,4 +0,0 @@
-#!/bin/bash
-python -m caosadvancedtools.loadFiles /opt/caosdb/mnt/extroot/data
-python load_and_insert_json_models.py
-python test_dataset_crawler.py
diff --git a/integrationtests/realworld_example/load_and_insert_json_models.py b/integrationtests/realworld_example/load_and_insert_json_models.py
deleted file mode 100644
index 682fd9c77531e63ed18dd13417399ad0d18a8de2..0000000000000000000000000000000000000000
--- a/integrationtests/realworld_example/load_and_insert_json_models.py
+++ /dev/null
@@ -1,47 +0,0 @@
-#!/usr/bin/env python3
-# encoding: utf-8
-#
-# This file is a part of the CaosDB Project.
-#
-# Copyright (C) 2022 Indiscale GmbH <info@indiscale.com>
-# Copyright (C) 2022 Henrik tom Wörden <h.tomwoerden@indiscale.com>
-# Copyright (C) 2022 Florian Spreckelsen <f.spreckelsen@indiscale.com>
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Affero General Public License as
-# published by the Free Software Foundation, either version 3 of the
-# License, or (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU Affero General Public License for more details.
-#
-# You should have received a copy of the GNU Affero General Public License
-# along with this program. If not, see <https://www.gnu.org/licenses/>.
-#
-import sys
-
-from caosadvancedtools.models.parser import parse_model_from_json_schema, parse_model_from_yaml
-
-
-def main():
- # First load dataspace data model
- dataspace_definitions = parse_model_from_json_schema(
- "schema/dataspace.schema.json")
- dataspace_definitions.sync_data_model(noquestion=True)
-
- # Then general dataset definitions
- dataset_definitions = parse_model_from_json_schema(
- "schema/dataset.schema.json")
- dataset_definitions.sync_data_model(noquestion=True)
-
- # Finally, add inheritances as defined in yaml
- dataset_inherits = parse_model_from_yaml(
- "schema/dataset-inheritance.yml")
- dataset_inherits.sync_data_model(noquestion=True)
-
-
-if __name__ == "__main__":
-
- sys.exit(main())
diff --git a/integrationtests/test-profile/profile.yml b/integrationtests/test-profile/profile.yml
index 4cbbe28c87d1146910479e13a23fa7c20933a926..f830a2fbe6c6a4ae35362676db310f3eadf6f4cc 100644
--- a/integrationtests/test-profile/profile.yml
+++ b/integrationtests/test-profile/profile.yml
@@ -21,7 +21,8 @@ default:
# extroot: From where files are copied/symlinked. This is a
# list of `NAME: PATH` pairs or a single path.
extroot:
- "": "paths/extroot"
+ # "": "paths/extroot"
+ "": "../test_data/extroot/"
#
# "base": "/path/to/base/dir"
# "other": "/path/to/other"
diff --git a/integrationtests/test_data/extroot/use_case_simple_presentation/DataAnalysis/results.md b/integrationtests/test_data/extroot/use_case_simple_presentation/DataAnalysis/results.md
new file mode 100644
index 0000000000000000000000000000000000000000..b867d778942ce5595286870bd6a92e53015be0e8
--- /dev/null
+++ b/integrationtests/test_data/extroot/use_case_simple_presentation/DataAnalysis/results.md
@@ -0,0 +1,8 @@
+---
+identifier: test analysis
+date: 2022-03-16
+source_identifier: crawlertest
+source_date: 2022-03-16
+
+frequency: 17
+---
diff --git a/integrationtests/test_data/extroot/use_case_simple_presentation/ExperimentalData/data.md b/integrationtests/test_data/extroot/use_case_simple_presentation/ExperimentalData/data.md
new file mode 100644
index 0000000000000000000000000000000000000000..60dcd78ed1f70428b18e8762a14dc3fe7f3fa5cd
--- /dev/null
+++ b/integrationtests/test_data/extroot/use_case_simple_presentation/ExperimentalData/data.md
@@ -0,0 +1,5 @@
+---
+date: "2022-03-16"
+identifier: crawlertest
+alpha: 16
+---
diff --git a/integrationtests/test_data/extroot/use_case_simple_presentation/cfood.yml b/integrationtests/test_data/extroot/use_case_simple_presentation/cfood.yml
new file mode 100644
index 0000000000000000000000000000000000000000..0b17073a823269c9eebbf53df6b9002ac8dbd45f
--- /dev/null
+++ b/integrationtests/test_data/extroot/use_case_simple_presentation/cfood.yml
@@ -0,0 +1,115 @@
+# This is only a scifolder test cfood with a limited functionality.
+# The full scifolder cfood will be developed here:
+# https://gitlab.indiscale.com/caosdb/src/crawler-cfoods/scifolder-cfood
+
+Definitions:
+ type: Definitions
+ #include "description.yml"
+
+Converters: {}
+
+extroot:
+ type: Directory
+ match: use_case_simple_presentation
+ subtree:
+ ExperimentalData: # name of the converter
+ type: Directory
+ match: ExperimentalData
+ subtree:
+ DataFile:
+ type: MarkdownFile
+ match: ^data\.md$
+
+ records:
+ mdfile:
+ parents: []
+ role: File
+ path: $DataFile
+ file: $DataFile
+
+ Experiment:
+ mdfile: $mdfile
+
+
+ subtree:
+ date:
+ type: DictTextElement
+ match_name: date
+ match_value: (?P<date>.+)
+ records:
+ Experiment:
+ date: $date
+ identifier:
+ type: DictTextElement
+ match_name: identifier
+ match_value: (?P<identifier>.+)
+ records:
+ Experiment:
+ identifier: $identifier
+ parameter_alpha:
+ type: DictTextElement
+ match_name: alpha
+ match_value: (?P<alpha>[0-9]+)
+ records:
+ Experiment:
+ alpha: $alpha
+
+ DataAnalysis:
+ type: Directory
+ match: DataAnalysis
+ subtree:
+ DataFile:
+ type: MarkdownFile
+ match: ^results\.md$
+
+ records:
+ mdfile:
+ parents: []
+ role: File
+ path: $DataFile
+ file: $DataFile
+
+ Experiment: {}
+
+ DataAnalysis:
+ mdfile: $mdfile
+ sources: +$Experiment
+
+ subtree:
+ date:
+ type: DictTextElement
+ match_name: date
+ match_value: (?P<date>.+)
+ records:
+ DataAnalysis:
+ date: $date
+ identifier:
+ type: DictTextElement
+ match_name: identifier
+ match_value: (?P<identifier>.+)
+ records:
+ DataAnalysis:
+ identifier: $identifier
+
+ frequency:
+ type: DictTextElement
+ match_name: frequency
+ match_value: (?P<frequency>[0-9]+)
+ records:
+ DataAnalysis:
+ frequency: $frequency
+
+ source_date:
+ type: DictTextElement
+ match_name: source_date
+ match_value: (?P<source_date>.+)
+ records:
+ Experiment:
+ date: $source_date
+ source_identifier:
+ type: DictTextElement
+ match_name: source_identifier
+ match_value: (?P<source_identifier>.+)
+ records:
+ Experiment:
+ identifier: $source_identifier
diff --git a/integrationtests/test_data/extroot/use_case_simple_presentation/identifiables.yml b/integrationtests/test_data/extroot/use_case_simple_presentation/identifiables.yml
new file mode 100644
index 0000000000000000000000000000000000000000..94b593bfb4c425ce71a4f94504d4f0033538cacb
--- /dev/null
+++ b/integrationtests/test_data/extroot/use_case_simple_presentation/identifiables.yml
@@ -0,0 +1,6 @@
+Experiment:
+- date
+- identifier
+DataAnalysis:
+- date
+- identifier
diff --git a/integrationtests/test_data/extroot/use_case_simple_presentation/model.yml b/integrationtests/test_data/extroot/use_case_simple_presentation/model.yml
new file mode 100644
index 0000000000000000000000000000000000000000..b1dcbd487b56519553a873fdd252c93ce21402a1
--- /dev/null
+++ b/integrationtests/test_data/extroot/use_case_simple_presentation/model.yml
@@ -0,0 +1,42 @@
+
+
+
+ScientificActivity:
+ description: |
+ The base record type for all scientific activities, like experiments,
+ data analysis records, simulations or publications.
+ recommended_properties:
+ sources:
+ description: This scientific activity is based on the activity referenced here.
+ datatype: LIST<ScientificActivity>
+ date:
+ description: The date according to https://doi.org/10.3390/data5020043
+ datatype: DATETIME
+ identifier:
+ description: An identifier according to https://doi.org/10.3390/data5020043
+ datatype: TEXT
+ mdfile:
+ description: The file storing information about this record.
+ datatype: FILE
+
+Experiment:
+ description: |
+ The base record type for all records containing data from experiments.
+ inherit_from_obligatory:
+ - ScientificActivity
+ obligatory_properties:
+ alpha:
+ description: A ficticious piece of data.
+ datatype: DOUBLE
+ unit: km
+
+DataAnalysis:
+ description: |
+ The base record type for all records containing results from data analysis.
+ inherit_from_obligatory:
+ - ScientificActivity
+ recommended_properties:
+ frequency:
+ description: A ficticious piece of data.
+ datatype: DOUBLE
+ unit: Hz
diff --git a/integrationtests/realworld_example/test_dataset_crawler.py b/integrationtests/test_realworld_example.py
similarity index 94%
rename from integrationtests/realworld_example/test_dataset_crawler.py
rename to integrationtests/test_realworld_example.py
index 250ee2122ab135317e236f3130b86e304d4208a7..c40a3c2403d5e4d81648fe964f6ae3d9f427ff47 100644
--- a/integrationtests/realworld_example/test_dataset_crawler.py
+++ b/integrationtests/test_realworld_example.py
@@ -51,7 +51,7 @@ def rfp(*pathcomponents):
return os.path.join(os.path.dirname(__file__), *pathcomponents)
-DATADIR = rfp("..", "test_data", "extroot", "realworld_example")
+DATADIR = rfp("test_data", "extroot", "realworld_example")
@pytest.fixture
@@ -72,8 +72,17 @@ def usemodel():
dataset_inherits.sync_data_model(noquestion=True)
+@pytest.fixture
+def clear_database():
+ # TODO(fspreck): Remove once the corresponding advancedtools function can
+ # be used.
+ ents = db.execute_query("FIND ENTITY WITH ID>99")
+ if ents:
+ ents.delete()
+
+
def test_dataset(
- # clear_database,
+ clear_database,
usemodel):
ident = CaosDBIdentifiableAdapter()
diff --git a/integrationtests/test_use_case_simple_presentation.py b/integrationtests/test_use_case_simple_presentation.py
new file mode 100644
index 0000000000000000000000000000000000000000..ba8009fcaab2696fb20970a2a6daaa8848d6d0a6
--- /dev/null
+++ b/integrationtests/test_use_case_simple_presentation.py
@@ -0,0 +1,99 @@
+#!/usr/bin/env python3
+# encoding: utf-8
+#
+# ** header v3.0
+# This file is a part of the CaosDB Project.
+#
+# Copyright (C) 2022 Alexander Schlemmer
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see <https://www.gnu.org/licenses/>.
+#
+# ** end header
+#
+
+"""
+module description
+"""
+import os
+import pytest
+from subprocess import run
+
+import caosdb as db
+from caosadvancedtools.loadFiles import loadpath
+from caosadvancedtools.models import parser as parser
+from caoscrawler.crawl import crawler_main
+
+
+# TODO(fspreck) Re-eneable once this is part of dev in advancedusertools.
+# from caosadvancedtools.testutils import clear_database, set_test_key
+# set_test_key("10b128cf8a1372f30aa3697466bb55e76974e0c16a599bb44ace88f19c8f61e2")
+
+DATADIR = os.path.join(os.path.dirname(__file__), "test_data",
+ "extroot", "use_case_simple_presentation")
+
+
+@pytest.fixture
+def clear_database():
+ # TODO(fspreck): Remove once the corresponding advancedtools function can be
+ # used.
+ ents = db.execute_query("FIND ENTITY WITH ID>99")
+ if ents:
+ ents.delete()
+
+
+def test_complete_crawler(
+ clear_database
+):
+ # Setup the data model:
+ model = parser.parse_model_from_yaml(os.path.join(DATADIR, "model.yml"))
+ model.sync_data_model(noquestion=True, verbose=False)
+
+ # Insert the data:
+ for path in [
+ "/opt/caosdb/mnt/extroot/use_case_simple_presentation/ExperimentalData",
+ "/opt/caosdb/mnt/extroot/use_case_simple_presentation/DataAnalysis"]:
+ loadpath(
+ path=path,
+ include=None,
+ exclude=None,
+ prefix="/",
+ dryrun=False,
+ forceAllowSymlinks=False)
+
+ crawler_main(DATADIR,
+ os.path.join(DATADIR, "cfood.yml"),
+ os.path.join(DATADIR, "identifiables.yml"),
+ True,
+ os.path.join(DATADIR, "provenance.yml"),
+ False,
+ True,
+ "/use_case_simple_presentation")
+
+ res = db.execute_query("FIND Record Experiment")
+ assert len(res) == 1
+ assert res[0].get_property("identifier").value == "crawlertest"
+ assert res[0].get_property("date").value == "2022-03-16"
+
+ lf = db.File(id=res[0].get_property("mdfile").value).retrieve()
+ assert lf.path == "/ExperimentalData/data.md"
+
+ assert res[0].get_property("alpha").value == 16.0
+ assert res[0].get_property("alpha").unit == "km"
+
+ res_da = db.execute_query("FIND Record DataAnalysis")
+ assert len(res_da) == 1
+ assert res_da[0].get_property("sources").value[0] == res[0].id
+
+ lf = db.File(id=res_da[0].get_property("mdfile").value).retrieve()
+ assert lf.path == "/DataAnalysis/results.md"