Merge branch 'dev' into f-verbose-option

.gitlab-ci.yml

@@ -46,7 +46,7 @@ test:
   image: $CI_REGISTRY_IMAGE_BASE
   script:
       - if [[ "$CAOSDB_TAG" == "" ]]; then
-            CAOSDB_TAG=dev-latest;
+            CAOSDB_TAG=dev;
         fi
       - echo $CAOSDB_TAG
       - time docker load < /image-cache/caosdb-advanced-testenv.tar || true

CHANGELOG.md

@@ -7,10 +7,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 ## [Unreleased] ##
 
 ### Added ###
-- `check_reference_field` function to check whether entities with provided ids
-  exits (for example when importing data from a table)
-- added the `datatypes` argument to `TableImporter` for columns that do not 
-  need a special conversion function
+
+- CFood that creates a Record for each line in a csv file
+- `generic_analysis.py` allows to easily call scripts to perform analyses in
+  server side scripting [EXPERIMENTAL]
 
 ### Changed ###
 
@@ -22,6 +22,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Security ###
 
+## [0.3.1] - 2021-12-06  ##
+
+### Added ###
+- `check_reference_field` function to check whether entities with provided ids
+  exits (for example when importing data from a table)
+- added the `datatypes` argument to `TableImporter` for columns that do not 
+  need a special conversion function
+
 ## [0.3.0] - 2021-11-02 ##
 
 ### Added ###

integrationtests/crawl.py

@@ -34,7 +34,7 @@ from caosadvancedtools.crawler import FileCrawler
 from caosadvancedtools.guard import INSERT, UPDATE
 from caosadvancedtools.scifolder import (AnalysisCFood, ExperimentCFood,
                                          PublicationCFood, SimulationCFood,
-                                         SoftwareCFood)
+                                         SoftwareCFood, ResultTableCFood)
 
 from example_hdf5cfood import ExampleH5CFood
 
@@ -91,6 +91,7 @@ if __name__ == "__main__":
                     interactive=False, hideKnown=False,
                     cfood_types=[ExperimentCFood, AnalysisCFood, SoftwareCFood,
                                  PublicationCFood, SimulationCFood,
+                                 ResultTableCFood,
                                  ExampleH5CFood
                                  ])
 

integrationtests/create_analysis.py

+#!/usr/bin/env python3
+# encoding: utf-8
+#
+# ** header v3.0
+# This file is a part of the CaosDB Project.
+#
+# Copyright (C) 2021 Indiscale GmbH <info@indiscale.com>
+# Copyright (C) 2021 Henrik tom Wörden <h.tomwoerden@indiscale.com>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see <https://www.gnu.org/licenses/>.
+#
+# ** end header
+#
+
+"""
+A small script that creates an Analysis Record that can be used for testing the
+automated analysis pipeline.
+"""
+
+import sys
+from datetime import datetime
+
+import caosdb as db
+
+
+def main():
+    script = db.File(
+        file="../src/caosadvancedtools/serverside/examples/example_script.py",
+        path=("AutomatedAnalysis/scripts/"
+              + str(datetime.now())+"example_script.py"),
+    )
+    script.insert()
+
+    da = db.Record()
+    da.add_parent("Analysis")
+    da.add_property("scripts", value=[script], datatype=db.LIST(db.FILE))
+    da.add_property("sources",
+                    value=db.execute_query(
+                        "FIND FILE which is stored at '**/timeseries.npy'",
+                        unique=True),
+                    )
+    da.add_property("date", "2020-01-01")
+    da.add_property("identifier", "TEST")
+    only = db.execute_query(
+                        "FIND RECORD Person WITH firstname=Only",
+                        unique=True)
+    only.add_property(db.Property("Email").retrieve().id, "only@example.com")
+    only.update()
+    da.add_property("responsible", only)
+    da.insert()
+
+
+if __name__ == "__main__":
+    sys.exit(main())

integrationtests/example_script.py

+../src/caosadvancedtools/serverside/examples/example_script.py
\ No newline at end of file

integrationtests/extroot/DataAnalysis/2010_TestProject/2019-02-03/plot.py

+import plot

integrationtests/extroot/ExperimentalData/2010_TestProject/2019-02-03/datafile.dat

+datadatadata

integrationtests/extroot/ExperimentalData/2010_TestProject/2019-02-03/result_table_DepthTest.csv

+temperature [°C] ,depth 
+234.4,3.0
+344.6,5.1

integrationtests/extroot/Software/2010_TestSoftware/2019-02-03_v0.1/plot.py

+import plot

integrationtests/extroot/Software/2020NewProject0X/2020-02-03/plot.py

+import plot

integrationtests/extroot/Software/2020NewProject0X/2020-02-04/plot.py

+import plot

integrationtests/model.yml

@@ -19,6 +19,14 @@ SoftwareVersion:
     binaries:
     sourceCode:
     Software:
+DepthTest:
+  obligatory_properties:
+    temperature:
+      datatype: DOUBLE 
+      description: 'temp'
+    depth:
+      datatype: DOUBLE 
+      description: 'temp'
 Person:
   obligatory_properties:
     firstName:
@@ -51,6 +59,9 @@ Analysis:
     date:
     identifier:
     responsible:
+  suggested_properties:
+    mean_value:
+      datatype: DOUBLE
 Publication:
 Thesis:
   inherit_from_suggested:

integrationtests/test.sh

@@ -65,6 +65,12 @@ python3 test_table.py
 # TODO the following test deletes lots of the data inserted by the crawler
 echo "Testing im and export"
 python3 test_im_und_export.py
+
+# automated analysis
+# for some reason the loadFiles of sim data has to be called again
+python3 -m caosadvancedtools.loadFiles /opt/caosdb/mnt/extroot/SimulationData
+python3 create_analysis.py
+
 # Better safe than sorry:
 python3 clear_database.py
 

integrationtests/test_crawler_with_cfoods.py

@@ -66,6 +66,17 @@ class CrawlerTest(unittest.TestCase):
                          datfile.description)
         assert os.path.basename(datfile.path) == "datafile.dat"
 
+        # There should be two DepthTest Properties
+        depthtests = exp.get_property("DepthTest")
+        assert depthtests is not None
+        assert len(depthtests.value) == 2
+        depthtest = db.Record(id=depthtests.value[0])
+        depthtest.retrieve()
+        assert "DepthTest" in [p.name for p in depthtest.get_parents()]
+        assert 234.4 == depthtest.get_property("temperature").value
+        assert "°C" == depthtest.get_property("temperature").unit
+        assert 3.0 == depthtest.get_property("depth").value
+
         # Should have a responsible person
         self.assertIsNotNone(exp.get_property("responsible"))
         person = db.Record(id=exp.get_property("responsible").value[0])

integrationtests/test_im_und_export.py

@@ -12,10 +12,12 @@ if __name__ == "__main__":
     directory = TemporaryDirectory()
     export_related_to(rec.id, directory=directory.name)
     # delete everything
+    print("Clearing database")
     recs = db.execute_query("FIND entity with id>99")
     recs.delete()
     assert 0 == len(db.execute_query("FIND File which is stored at "
                                      "**/poster.pdf"))
+    print("Importing stored elements")
     import_xml(os.path.join(directory.name, "caosdb_data.xml"), interactive=False)
 
     # The following tests the existence of some required entities.
@@ -26,3 +28,4 @@ if __name__ == "__main__":
     db.execute_query("FIND RecordType Person", unique=True)
     db.execute_query("FIND Record Person with firstname=Only", unique=True)
     db.execute_query("FIND File which is stored at **/poster.pdf", unique=True)
+    print("Found all required Records and Files.")

integrationtests/update_analysis.py

+#!/usr/bin/env python3
+# encoding: utf-8
+#
+# ** header v3.0
+# This file is a part of the CaosDB Project.
+#
+# Copyright (C) 2021 Indiscale GmbH <info@indiscale.com>
+# Copyright (C) 2021 Henrik tom Wörden <h.tomwoerden@indiscale.com>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see <https://www.gnu.org/licenses/>.
+#
+# ** end header
+#
+
+"""Example update script. An anlysis Record is retrieved and passed to the
+generic run function which then calls the appropriate script based on the
+Record.
+
+The simple query here could be replaced with something that e.g. retrieves all
+entities that where changed within a certain period of time.
+
+"""
+
+import sys
+
+import caosdb as db
+from caosadvancedtools.serverside.generic_analysis import run
+
+
+def main():
+    da = db.execute_query("FIND Analysis with identifier=TEST", unique=True)
+    run(da)
+
+
+if __name__ == "__main__":
+    sys.exit(main())

setup.py

@@ -47,7 +47,7 @@ from setuptools import find_packages, setup
 
 MAJOR = 0
 MINOR = 3
-MICRO = 1
+MICRO = 2
 PRE = ""  # e.g. rc0, alpha.1, 0.beta-23
 ISRELEASED = False