Merge branch 'dev' into f-identifiable-test

.docker/docker-compose.yml

@@ -17,8 +17,8 @@ services:
       - type: bind
         source: ./cert
         target: /opt/caosdb/cert
-      - type: volume
-        source: extroot
+      - type: bind
+        source: "../integrationtests/test_data/extroot"
         target: /opt/caosdb/mnt/extroot
       - type: volume
         source: scripting
@@ -36,7 +36,6 @@ services:
       CAOSDB_CONFIG_TRANSACTION_BENCHMARK_ENABLED: "TRUE"
 volumes:
   scripting:
-  extroot:
   authtoken:
 networks:
   caosnet:

integrationtests/realworld_example/crawl.sh

-#!/bin/bash
-python -m caosadvancedtools.loadFiles /opt/caosdb/mnt/extroot/data
-python load_and_insert_json_models.py
-python test_dataset_crawler.py

integrationtests/realworld_example/load_and_insert_json_models.py

-#!/usr/bin/env python3
-# encoding: utf-8
-#
-# This file is a part of the CaosDB Project.
-#
-# Copyright (C) 2022 Indiscale GmbH <info@indiscale.com>
-# Copyright (C) 2022 Henrik tom Wörden <h.tomwoerden@indiscale.com>
-# Copyright (C) 2022 Florian Spreckelsen <f.spreckelsen@indiscale.com>
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Affero General Public License as
-# published by the Free Software Foundation, either version 3 of the
-# License, or (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU Affero General Public License for more details.
-#
-# You should have received a copy of the GNU Affero General Public License
-# along with this program. If not, see <https://www.gnu.org/licenses/>.
-#
-import sys
-
-from caosadvancedtools.models.parser import parse_model_from_json_schema, parse_model_from_yaml
-
-
-def main():
-    # First load dataspace data model
-    dataspace_definitions = parse_model_from_json_schema(
-        "schema/dataspace.schema.json")
-    dataspace_definitions.sync_data_model(noquestion=True)
-
-    # Then general dataset definitions
-    dataset_definitions = parse_model_from_json_schema(
-        "schema/dataset.schema.json")
-    dataset_definitions.sync_data_model(noquestion=True)
-
-    # Finally, add inheritances as defined in yaml
-    dataset_inherits = parse_model_from_yaml(
-        "schema/dataset-inheritance.yml")
-    dataset_inherits.sync_data_model(noquestion=True)
-
-
-if __name__ == "__main__":
-
-    sys.exit(main())

integrationtests/test-profile/profile.yml

@@ -21,7 +21,8 @@ default:
     # extroot: From where files are copied/symlinked.  This is a
     # list of `NAME: PATH` pairs or a single path.  
     extroot:
-      "": "paths/extroot"
+    #  "": "paths/extroot"
+      "": "../test_data/extroot/"
     #
     #  "base": "/path/to/base/dir"
     #  "other": "/path/to/other"

integrationtests/test_data/extroot/use_case_simple_presentation/DataAnalysis/results.md

+---
+identifier: test analysis
+date: 2022-03-16
+source_identifier: crawlertest
+source_date: 2022-03-16
+
+frequency: 17
+---

integrationtests/test_data/extroot/use_case_simple_presentation/ExperimentalData/data.md

+---
+date: "2022-03-16"
+identifier: crawlertest
+alpha: 16
+---

integrationtests/test_data/extroot/use_case_simple_presentation/cfood.yml

+# This is only a scifolder test cfood with a limited functionality.
+# The full scifolder cfood will be developed here:
+# https://gitlab.indiscale.com/caosdb/src/crawler-cfoods/scifolder-cfood
+
+Definitions:
+  type: Definitions
+  #include "description.yml"
+
+Converters: {}
+
+extroot:
+  type: Directory
+  match: use_case_simple_presentation
+  subtree:
+    ExperimentalData:  # name of the converter
+      type: Directory
+      match: ExperimentalData
+      subtree:
+        DataFile:
+          type: MarkdownFile
+          match: ^data\.md$
+
+          records:
+            mdfile:
+              parents: []
+              role: File
+              path: $DataFile
+              file: $DataFile
+
+            Experiment:
+              mdfile: $mdfile
+
+
+          subtree:
+            date:
+              type: DictTextElement
+              match_name: date
+              match_value: (?P<date>.+)
+              records:
+                Experiment:
+                  date: $date
+            identifier:
+              type: DictTextElement
+              match_name: identifier
+              match_value: (?P<identifier>.+)
+              records:
+                Experiment:
+                  identifier: $identifier
+            parameter_alpha:
+              type: DictTextElement
+              match_name: alpha
+              match_value: (?P<alpha>[0-9]+)
+              records:
+                Experiment:
+                  alpha: $alpha
+
+    DataAnalysis:
+      type: Directory
+      match: DataAnalysis
+      subtree:
+        DataFile:
+          type: MarkdownFile
+          match: ^results\.md$
+
+          records:
+            mdfile:
+              parents: []
+              role: File
+              path: $DataFile
+              file: $DataFile
+
+            Experiment: {}
+
+            DataAnalysis:
+              mdfile: $mdfile
+              sources: +$Experiment
+
+          subtree:
+            date:
+              type: DictTextElement
+              match_name: date
+              match_value: (?P<date>.+)
+              records:
+                DataAnalysis:
+                  date: $date
+            identifier:
+              type: DictTextElement
+              match_name: identifier
+              match_value: (?P<identifier>.+)
+              records:
+                DataAnalysis:
+                  identifier: $identifier
+
+            frequency:
+              type: DictTextElement
+              match_name: frequency
+              match_value: (?P<frequency>[0-9]+)
+              records:
+                DataAnalysis:
+                  frequency: $frequency
+
+            source_date:
+              type: DictTextElement
+              match_name: source_date
+              match_value: (?P<source_date>.+)
+              records:
+                Experiment:
+                  date: $source_date
+            source_identifier:
+              type: DictTextElement
+              match_name: source_identifier
+              match_value: (?P<source_identifier>.+)
+              records:
+                Experiment:
+                  identifier: $source_identifier

integrationtests/test_data/extroot/use_case_simple_presentation/identifiables.yml

+Experiment:
+- date
+- identifier
+DataAnalysis:
+- date
+- identifier

integrationtests/test_data/extroot/use_case_simple_presentation/model.yml

+
+
+
+ScientificActivity:
+  description: |
+    The base record type for all scientific activities, like experiments,
+    data analysis records, simulations or publications.
+  recommended_properties:
+    sources:
+      description: This scientific activity is based on the activity referenced here.
+      datatype: LIST<ScientificActivity>
+    date:
+      description: The date according to https://doi.org/10.3390/data5020043
+      datatype: DATETIME
+    identifier:
+      description: An identifier according to https://doi.org/10.3390/data5020043
+      datatype: TEXT
+    mdfile:
+      description: The file storing information about this record.
+      datatype: FILE
+
+Experiment:
+  description: |
+    The base record type for all records containing data from experiments.
+  inherit_from_obligatory:
+    - ScientificActivity
+  obligatory_properties:
+    alpha:
+      description: A ficticious piece of data.
+      datatype: DOUBLE
+      unit: km
+
+DataAnalysis:
+  description: |
+    The base record type for all records containing results from data analysis.
+  inherit_from_obligatory:
+    - ScientificActivity
+  recommended_properties:
+    frequency:
+      description: A ficticious piece of data.
+      datatype: DOUBLE
+      unit: Hz

integrationtests/realworld_example/test_dataset_crawler.py → integrationtests/test_realworld_example.py

@@ -51,7 +51,7 @@ def rfp(*pathcomponents):
     return os.path.join(os.path.dirname(__file__), *pathcomponents)
 
 
-DATADIR = rfp("..", "test_data", "extroot", "realworld_example")
+DATADIR = rfp("test_data", "extroot", "realworld_example")
 
 
 @pytest.fixture
@@ -72,8 +72,17 @@ def usemodel():
     dataset_inherits.sync_data_model(noquestion=True)
 
 
+@pytest.fixture
+def clear_database():
+    # TODO(fspreck): Remove once the corresponding advancedtools function can
+    # be used.
+    ents = db.execute_query("FIND ENTITY WITH ID>99")
+    if ents:
+        ents.delete()
+
+
 def test_dataset(
-        # clear_database,
+        clear_database,
         usemodel):
 
     ident = CaosDBIdentifiableAdapter()

integrationtests/test_use_case_simple_presentation.py

+#!/usr/bin/env python3
+# encoding: utf-8
+#
+# ** header v3.0
+# This file is a part of the CaosDB Project.
+#
+# Copyright (C) 2022 Alexander Schlemmer
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see <https://www.gnu.org/licenses/>.
+#
+# ** end header
+#
+
+"""
+module description
+"""
+import os
+import pytest
+from subprocess import run
+
+import caosdb as db
+from caosadvancedtools.loadFiles import loadpath
+from caosadvancedtools.models import parser as parser
+from caoscrawler.crawl import crawler_main
+
+
+# TODO(fspreck) Re-eneable once this is part of dev in advancedusertools.
+# from caosadvancedtools.testutils import clear_database, set_test_key
+# set_test_key("10b128cf8a1372f30aa3697466bb55e76974e0c16a599bb44ace88f19c8f61e2")
+
+DATADIR = os.path.join(os.path.dirname(__file__), "test_data",
+                       "extroot", "use_case_simple_presentation")
+
+
+@pytest.fixture
+def clear_database():
+    # TODO(fspreck): Remove once the corresponding advancedtools function can be
+    # used.
+    ents = db.execute_query("FIND ENTITY WITH ID>99")
+    if ents:
+        ents.delete()
+
+
+def test_complete_crawler(
+        clear_database
+):
+    # Setup the data model:
+    model = parser.parse_model_from_yaml(os.path.join(DATADIR, "model.yml"))
+    model.sync_data_model(noquestion=True, verbose=False)
+
+    # Insert the data:
+    for path in [
+            "/opt/caosdb/mnt/extroot/use_case_simple_presentation/ExperimentalData",
+            "/opt/caosdb/mnt/extroot/use_case_simple_presentation/DataAnalysis"]:
+        loadpath(
+            path=path,
+            include=None,
+            exclude=None,
+            prefix="/",
+            dryrun=False,
+            forceAllowSymlinks=False)
+
+    crawler_main(DATADIR,
+                 os.path.join(DATADIR, "cfood.yml"),
+                 os.path.join(DATADIR, "identifiables.yml"),
+                 True,
+                 os.path.join(DATADIR, "provenance.yml"),
+                 False,
+                 True,
+                 "/use_case_simple_presentation")
+
+    res = db.execute_query("FIND Record Experiment")
+    assert len(res) == 1
+    assert res[0].get_property("identifier").value == "crawlertest"
+    assert res[0].get_property("date").value == "2022-03-16"
+
+    lf = db.File(id=res[0].get_property("mdfile").value).retrieve()
+    assert lf.path == "/ExperimentalData/data.md"
+
+    assert res[0].get_property("alpha").value == 16.0
+    assert res[0].get_property("alpha").unit == "km"
+
+    res_da = db.execute_query("FIND Record DataAnalysis")
+    assert len(res_da) == 1
+    assert res_da[0].get_property("sources").value[0] == res[0].id
+
+    lf = db.File(id=res_da[0].get_property("mdfile").value).retrieve()
+    assert lf.path == "/DataAnalysis/results.md"