diff --git a/integrationtests/test_use_case_simple_presentation.py b/integrationtests/test_use_case_simple_presentation.py new file mode 100644 index 0000000000000000000000000000000000000000..17da89510be679966cb14914fac239b355797a69 --- /dev/null +++ b/integrationtests/test_use_case_simple_presentation.py @@ -0,0 +1,68 @@ +#!/usr/bin/env python3 +# encoding: utf-8 +# +# ** header v3.0 +# This file is a part of the CaosDB Project. +# +# Copyright (C) 2022 Alexander Schlemmer +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see <https://www.gnu.org/licenses/>. +# +# ** end header +# + +""" +module description +""" + +import caosdb as db +import pytest +from caosadvancedtools.models import parser as parser +from newcrawler.crawl import crawler_main +from caosadvancedtools.loadFiles import loadpath + +from subprocess import run + +from caosadvancedtools.testutils import clear_database, set_test_key +set_test_key("10b128cf8a1372f30aa3697466bb55e76974e0c16a599bb44ace88f19c8f61e2") + + +def test_complete_crawler(clear_database): + # Setup the data model: + model = parser.parse_model_from_yaml("use_case_simple_presentation/model.yml") + model.sync_data_model(noquestion=True, verbose=False) + + # Insert the data: + for path in [ + "/opt/caosdb/mnt/extroot/base/SimulationData", + "/opt/caosdb/mnt/extroot/base/DataAnalysis"]: + loadpath( + path=path, + include=None, + exclude=None, + prefix="/", + dryrun=False, + forceAllowSymlinks=False) + + crawler_main("use_case_simple_presentation/extroot/", + "use_case_simple_presentation/cfood.yml", + "use_case_simple_presentation/identifiables.yml", + True, + "use_case_simple_presentation/provenance.yml", + False, + True, + "/extroot") + + res = db.execute_query("FIND Record MonodomainTissueSimulationRun") + assert len(res) > 0 diff --git a/integrationtests/use_case_simple_presentation/cfood.yml b/integrationtests/use_case_simple_presentation/cfood.yml new file mode 100644 index 0000000000000000000000000000000000000000..840072fd16e5bac767c2fdc4bc2107bd0310dd23 --- /dev/null +++ b/integrationtests/use_case_simple_presentation/cfood.yml @@ -0,0 +1,141 @@ +# This is only a scifolder test cfood with a limited functionality. +# The full scifolder cfood will be developed here: +# https://gitlab.indiscale.com/caosdb/src/crawler-cfoods/scifolder-cfood + +Definitions: + type: Definitions + #include "description.yml" + +Converters: {} + +ExperimentalData: # name of the converter + type: Directory + match: ExperimentalData + subtree: + DataFile: + type: MarkdownFile + match: ^data\.md$ + + records: + mdfile: + parents: [] + role: File + path: $DataFile + file: $DataFile + + Experiment: + mdfile: $mdfile + + + subtree: + date: + type: DictTextElement + match_name: date + match_value: (?P<date>.+) + records: + Experiment: + date: $date + identifier: + type: DictTextElement + match_name: identifier + match_value: (?P<identifier>.+) + records: + Experiment: + identifier: $identifier + parameter_alpha: + type: DictTextElement + match_name: alpha + match_value: (?P<alpha>[0-9]+) + records: + Experiment: + alpha: $alpha + +# description: +# type: DictTextElement +# match_value: (?P<description>.*) +# match_name: description +# records: +# SimulationActivity: +# description: $description +# responsible_single: +# type: DictTextElement +# match_name: responsible +# match_value: &person_regexp ((?P<first_name>.+) )?(?P<last_name>.+) +# records: &responsible_records +# Person: +# first_name: $first_name +# last_name: $last_name +# SimulationActivity: +# responsible: +$Person + +# responsible_list: +# type: DictListElement +# match_name: responsible +# subtree: +# Person: +# type: TextElement +# match: *person_regexp +# records: *responsible_records + +DataAnalysis: + type: Directory + match: DataAnalysis + subtree: + DataFile: + type: MarkdownFile + match: ^results\.md$ + + records: + mdfile: + parents: [] + role: File + path: $DataFile + file: $DataFile + + Experiment: {} + + DataAnalysis: + mdfile: $mdfile + sources: +$Experiment + + + + + subtree: + date: + type: DictTextElement + match_name: date + match_value: (?P<date>.+) + records: + DataAnalysis: + date: $date + identifier: + type: DictTextElement + match_name: identifier + match_value: (?P<identifier>.+) + records: + DataAnalysis: + identifier: $identifier + + frequency: + type: DictTextElement + match_name: frequency + match_value: (?P<frequency>[0-9]+) + records: + DataAnalysis: + frequency: $frequency + + source_date: + type: DictTextElement + match_name: source_date + match_value: (?P<source_date>.+) + records: + Experiment: + date: $source_date + source_identifier: + type: DictTextElement + match_name: source_identifier + match_value: (?P<source_identifier>.+) + records: + Experiment: + identifier: $source_identifier diff --git a/integrationtests/use_case_simple_presentation/extroot/DataAnalysis/results.md b/integrationtests/use_case_simple_presentation/extroot/DataAnalysis/results.md new file mode 100644 index 0000000000000000000000000000000000000000..b867d778942ce5595286870bd6a92e53015be0e8 --- /dev/null +++ b/integrationtests/use_case_simple_presentation/extroot/DataAnalysis/results.md @@ -0,0 +1,8 @@ +--- +identifier: test analysis +date: 2022-03-16 +source_identifier: crawlertest +source_date: 2022-03-16 + +frequency: 17 +--- diff --git a/integrationtests/use_case_simple_presentation/extroot/ExperimentalData/data.md b/integrationtests/use_case_simple_presentation/extroot/ExperimentalData/data.md new file mode 100644 index 0000000000000000000000000000000000000000..60dcd78ed1f70428b18e8762a14dc3fe7f3fa5cd --- /dev/null +++ b/integrationtests/use_case_simple_presentation/extroot/ExperimentalData/data.md @@ -0,0 +1,5 @@ +--- +date: "2022-03-16" +identifier: crawlertest +alpha: 16 +--- diff --git a/integrationtests/use_case_simple_presentation/identifiables.yml b/integrationtests/use_case_simple_presentation/identifiables.yml new file mode 100644 index 0000000000000000000000000000000000000000..94b593bfb4c425ce71a4f94504d4f0033538cacb --- /dev/null +++ b/integrationtests/use_case_simple_presentation/identifiables.yml @@ -0,0 +1,6 @@ +Experiment: +- date +- identifier +DataAnalysis: +- date +- identifier diff --git a/integrationtests/use_case_simple_presentation/model.yml b/integrationtests/use_case_simple_presentation/model.yml new file mode 100644 index 0000000000000000000000000000000000000000..b1dcbd487b56519553a873fdd252c93ce21402a1 --- /dev/null +++ b/integrationtests/use_case_simple_presentation/model.yml @@ -0,0 +1,42 @@ + + + +ScientificActivity: + description: | + The base record type for all scientific activities, like experiments, + data analysis records, simulations or publications. + recommended_properties: + sources: + description: This scientific activity is based on the activity referenced here. + datatype: LIST<ScientificActivity> + date: + description: The date according to https://doi.org/10.3390/data5020043 + datatype: DATETIME + identifier: + description: An identifier according to https://doi.org/10.3390/data5020043 + datatype: TEXT + mdfile: + description: The file storing information about this record. + datatype: FILE + +Experiment: + description: | + The base record type for all records containing data from experiments. + inherit_from_obligatory: + - ScientificActivity + obligatory_properties: + alpha: + description: A ficticious piece of data. + datatype: DOUBLE + unit: km + +DataAnalysis: + description: | + The base record type for all records containing results from data analysis. + inherit_from_obligatory: + - ScientificActivity + recommended_properties: + frequency: + description: A ficticious piece of data. + datatype: DOUBLE + unit: Hz