TST: new integration test based on a simple use case that was used for a presentation

integrationtests/test_use_case_simple_presentation.py

+#!/usr/bin/env python3
+# encoding: utf-8
+#
+# ** header v3.0
+# This file is a part of the CaosDB Project.
+#
+# Copyright (C) 2022 Alexander Schlemmer
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see <https://www.gnu.org/licenses/>.
+#
+# ** end header
+#
+
+"""
+module description
+"""
+
+import caosdb as db
+import pytest
+from caosadvancedtools.models import parser as parser
+from newcrawler.crawl import crawler_main
+from caosadvancedtools.loadFiles import loadpath
+
+from subprocess import run
+
+from caosadvancedtools.testutils import clear_database, set_test_key
+set_test_key("10b128cf8a1372f30aa3697466bb55e76974e0c16a599bb44ace88f19c8f61e2")
+
+
+def test_complete_crawler(clear_database):
+    # Setup the data model:
+    model = parser.parse_model_from_yaml("use_case_simple_presentation/model.yml")
+    model.sync_data_model(noquestion=True, verbose=False)
+
+    # Insert the data:
+    for path in [
+            "/opt/caosdb/mnt/extroot/base/SimulationData",
+            "/opt/caosdb/mnt/extroot/base/DataAnalysis"]:
+        loadpath(
+            path=path,
+            include=None,
+            exclude=None,
+            prefix="/",
+            dryrun=False,
+            forceAllowSymlinks=False)
+
+    crawler_main("use_case_simple_presentation/extroot/",
+                 "use_case_simple_presentation/cfood.yml",
+                 "use_case_simple_presentation/identifiables.yml",
+                 True,
+                 "use_case_simple_presentation/provenance.yml",
+                 False,
+                 True,
+                 "/extroot")
+
+    res = db.execute_query("FIND Record MonodomainTissueSimulationRun")
+    assert len(res) > 0

integrationtests/use_case_simple_presentation/cfood.yml

+# This is only a scifolder test cfood with a limited functionality.
+# The full scifolder cfood will be developed here:
+# https://gitlab.indiscale.com/caosdb/src/crawler-cfoods/scifolder-cfood
+
+Definitions:
+  type: Definitions
+  #include "description.yml"
+
+Converters: {}
+  
+ExperimentalData:  # name of the converter
+  type: Directory
+  match: ExperimentalData
+  subtree:
+    DataFile:
+      type: MarkdownFile
+      match: ^data\.md$
+
+      records:
+        mdfile:
+          parents: []
+          role: File
+          path: $DataFile
+          file: $DataFile
+
+        Experiment:
+          mdfile: $mdfile
+
+            
+      subtree:
+        date:
+          type: DictTextElement
+          match_name: date
+          match_value: (?P<date>.+)
+          records:
+            Experiment:
+              date: $date
+        identifier:
+          type: DictTextElement
+          match_name: identifier
+          match_value: (?P<identifier>.+)
+          records:
+            Experiment:
+              identifier: $identifier
+        parameter_alpha:
+          type: DictTextElement
+          match_name: alpha
+          match_value: (?P<alpha>[0-9]+)
+          records:
+            Experiment:
+              alpha: $alpha
+        
+#           description:
+#             type: DictTextElement
+#             match_value: (?P<description>.*)
+#             match_name: description
+#             records:
+#               SimulationActivity:
+#                 description: $description
+#           responsible_single:
+#             type: DictTextElement
+#             match_name: responsible
+#             match_value: &person_regexp ((?P<first_name>.+) )?(?P<last_name>.+)
+#               records: &responsible_records
+#                 Person: 
+#                   first_name: $first_name
+#                   last_name: $last_name
+#                   SimulationActivity:
+#                     responsible: +$Person
+
+#             responsible_list:
+#               type: DictListElement
+#               match_name: responsible
+#               subtree:
+#                 Person:
+#                   type: TextElement
+#                   match: *person_regexp
+#                   records: *responsible_records
+
+DataAnalysis:
+  type: Directory
+  match: DataAnalysis
+  subtree:
+    DataFile:
+      type: MarkdownFile
+      match: ^results\.md$
+
+      records:
+        mdfile:
+          parents: []
+          role: File
+          path: $DataFile
+          file: $DataFile
+
+        Experiment: {}
+        
+        DataAnalysis:
+          mdfile: $mdfile
+          sources: +$Experiment
+          
+        
+
+            
+      subtree:
+        date:
+          type: DictTextElement
+          match_name: date
+          match_value: (?P<date>.+)
+          records:
+            DataAnalysis:
+              date: $date
+        identifier:
+          type: DictTextElement
+          match_name: identifier
+          match_value: (?P<identifier>.+)
+          records:
+            DataAnalysis:
+              identifier: $identifier
+
+        frequency:
+          type: DictTextElement
+          match_name: frequency
+          match_value: (?P<frequency>[0-9]+)
+          records:
+            DataAnalysis:
+              frequency: $frequency
+
+        source_date:
+          type: DictTextElement
+          match_name: source_date
+          match_value: (?P<source_date>.+)
+          records:
+            Experiment:
+              date: $source_date
+        source_identifier:
+          type: DictTextElement
+          match_name: source_identifier
+          match_value: (?P<source_identifier>.+)
+          records:
+            Experiment:
+              identifier: $source_identifier

integrationtests/use_case_simple_presentation/extroot/DataAnalysis/results.md

+---
+identifier: test analysis
+date: 2022-03-16
+source_identifier: crawlertest
+source_date: 2022-03-16
+
+frequency: 17
+---

integrationtests/use_case_simple_presentation/extroot/ExperimentalData/data.md

+---
+date: "2022-03-16"
+identifier: crawlertest
+alpha: 16
+---

integrationtests/use_case_simple_presentation/identifiables.yml

+Experiment:
+- date
+- identifier
+DataAnalysis:
+- date
+- identifier

integrationtests/use_case_simple_presentation/model.yml

+
+
+
+ScientificActivity:
+  description: |
+    The base record type for all scientific activities, like experiments,
+    data analysis records, simulations or publications.
+  recommended_properties:
+    sources:
+      description: This scientific activity is based on the activity referenced here.
+      datatype: LIST<ScientificActivity>
+    date:
+      description: The date according to https://doi.org/10.3390/data5020043
+      datatype: DATETIME
+    identifier:
+      description: An identifier according to https://doi.org/10.3390/data5020043
+      datatype: TEXT
+    mdfile:
+      description: The file storing information about this record.
+      datatype: FILE
+
+Experiment:
+  description: |
+    The base record type for all records containing data from experiments.
+  inherit_from_obligatory:
+    - ScientificActivity
+  obligatory_properties:
+    alpha:
+      description: A ficticious piece of data.
+      datatype: DOUBLE
+      unit: km
+
+DataAnalysis:
+  description: |
+    The base record type for all records containing results from data analysis.
+  inherit_from_obligatory:
+    - ScientificActivity
+  recommended_properties:
+    frequency:
+      description: A ficticious piece of data.
+      datatype: DOUBLE
+      unit: Hz