From 3662b976ed385a450dac08a9f54f633088277867 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Henrik=20tom=20W=C3=B6rden?= <henrik@trineo.org>
Date: Tue, 23 Nov 2021 16:13:54 +0100
Subject: [PATCH] WIP
---
integrationtests/create_analysis.py | 66 ++++++++++++++++++
integrationtests/example_script.py | 1 +
.../2019-02-03_something/timeseries.npy | Bin 0 -> 200 bytes
integrationtests/model.yml | 3 +
integrationtests/test.sh | 4 ++
integrationtests/update_analysis.py | 42 +++++++++++
.../{ => examples}/example_script.py | 28 +++++---
.../serverside/generic_analysis.py | 26 ++++---
8 files changed, 150 insertions(+), 20 deletions(-)
create mode 100644 integrationtests/create_analysis.py
create mode 120000 integrationtests/example_script.py
create mode 100644 integrationtests/update_analysis.py
rename src/caosadvancedtools/serverside/{ => examples}/example_script.py (79%)
mode change 100644 => 100755
diff --git a/integrationtests/create_analysis.py b/integrationtests/create_analysis.py
new file mode 100644
index 00000000..44e99c98
--- /dev/null
+++ b/integrationtests/create_analysis.py
@@ -0,0 +1,66 @@
+#!/usr/bin/env python3
+# encoding: utf-8
+#
+# ** header v3.0
+# This file is a part of the CaosDB Project.
+#
+# Copyright (C) 2021 Indiscale GmbH <info@indiscale.com>
+# Copyright (C) 2021 Henrik tom Wörden <h.tomwoerden@indiscale.com>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see <https://www.gnu.org/licenses/>.
+#
+# ** end header
+#
+
+"""
+module description
+"""
+
+import sys
+
+import caosdb as db
+
+
+def main():
+ # TODO remove fixed path
+ script = db.File(
+ file="../src/caosadvancedtools/serverside/examples/example_script.py",
+ path="Analysis/scripts/example_script.py",
+ )
+ try:
+ script.insert()
+ except:
+ script = db.execute_query(
+ "FIND FILE which is stored at 'Analysis/scripts/example_script.py'",
+ unique=True)
+
+ da = db.Record()
+ da.add_parent("Analysis")
+ da.add_property("scripts", value=[script], datatype=db.LIST(db.FILE))
+ da.add_property("sources",
+ value=db.execute_query(
+ "FIND FILE which is stored at '**/timeseries.npy'",
+ unique=True),
+ # datatype=db.LIST(db.FILE)
+ )
+ da.add_property("date", "2020-01-01")
+ da.add_property("identifier", "TEST")
+ da.add_property("responsible", db.execute_query(
+ "FIND RECORD Person WITH firstname=Only",
+ unique=True))
+ da.insert()
+
+
+if __name__ == "__main__":
+ sys.exit(main())
diff --git a/integrationtests/example_script.py b/integrationtests/example_script.py
new file mode 120000
index 00000000..6a03bd10
--- /dev/null
+++ b/integrationtests/example_script.py
@@ -0,0 +1 @@
+../src/caosadvancedtools/examples/example_script.py
\ No newline at end of file
diff --git a/integrationtests/extroot/SimulationData/2010_TestProject/2019-02-03_something/timeseries.npy b/integrationtests/extroot/SimulationData/2010_TestProject/2019-02-03_something/timeseries.npy
index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..18da9b18cda23d411d0f2666629377dd7991ac8f 100644
GIT binary patch
literal 200
zcmbR27wQ`j$;eQ~P_3SlTAW;@Zl$1ZlWC!@qoAIaUsO_*m=~X4l#&V(cT3DEP6dh=
nXCxM+0{I%2I+{8PwF(pfE+z&DV20AHP#PwVPNS=1gsKMs5VRYB
literal 0
HcmV?d00001
diff --git a/integrationtests/model.yml b/integrationtests/model.yml
index cad79588..6c68a861 100644
--- a/integrationtests/model.yml
+++ b/integrationtests/model.yml
@@ -51,6 +51,9 @@ Analysis:
date:
identifier:
responsible:
+ suggested_properties:
+ mean_value:
+ datatype: DOUBLE
Publication:
Thesis:
inherit_from_suggested:
diff --git a/integrationtests/test.sh b/integrationtests/test.sh
index 71af5436..9cf17be4 100755
--- a/integrationtests/test.sh
+++ b/integrationtests/test.sh
@@ -65,6 +65,10 @@ python3 test_table.py
# TODO the following test deletes lots of the data inserted by the crawler
echo "Testing im and export"
python3 test_im_und_export.py
+
+# automated analysis
+python3 create_analysis.py
+
# Better safe than sorry:
python3 clear_database.py
diff --git a/integrationtests/update_analysis.py b/integrationtests/update_analysis.py
new file mode 100644
index 00000000..3fd9840a
--- /dev/null
+++ b/integrationtests/update_analysis.py
@@ -0,0 +1,42 @@
+#!/usr/bin/env python3
+# encoding: utf-8
+#
+# ** header v3.0
+# This file is a part of the CaosDB Project.
+#
+# Copyright (C) 2021 Indiscale GmbH <info@indiscale.com>
+# Copyright (C) 2021 Henrik tom Wörden <h.tomwoerden@indiscale.com>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see <https://www.gnu.org/licenses/>.
+#
+# ** end header
+#
+
+"""
+module description
+"""
+
+import sys
+
+import caosdb as db
+from caosadvancedtools.serverside.generic_analysis import run
+
+
+def main():
+ da = db.execute_query("FIND Analysis with identifier=TEST", unique=True)
+ run(da)
+
+
+if __name__ == "__main__":
+ sys.exit(main())
diff --git a/src/caosadvancedtools/serverside/example_script.py b/src/caosadvancedtools/serverside/examples/example_script.py
old mode 100644
new mode 100755
similarity index 79%
rename from src/caosadvancedtools/serverside/example_script.py
rename to src/caosadvancedtools/serverside/examples/example_script.py
index ff01ac30..148d8cba
--- a/src/caosadvancedtools/serverside/example_script.py
+++ b/src/caosadvancedtools/serverside/examples/example_script.py
@@ -32,14 +32,19 @@ import argparse
import logging
import sys
from argparse import RawTextHelpFormatter
+from datetime import datetime
import caosdb as db
import matplotlib.pyplot as plt
import numpy as np
from caosadvancedtools.cfood import assure_property_is
+from caosadvancedtools.guard import INSERT, UPDATE
+from caosadvancedtools.guard import global_guard as guard
logger = logging.getLogger(__name__)
+guard.set_level(level=UPDATE)
+
def main(args):
# TODO (maybe) can these checks be replaced by a more declaritive appoach?
@@ -51,32 +56,37 @@ def main(args):
))
# The script may require certain information to exist. Here, we expect that
- # a InputDataSet Property exists that references a numpy file.
+ # a sources Property exists that references a numpy file.
+ # Similarly an InputDataSet could be used.
- if (dataAnalysisRecord.get_property("InputDataSet") is None
- or db.apiutils.is_reference(
- dataAnalysisRecord.get_property("InputDataSet"))):
+ if (dataAnalysisRecord.get_property("sources") is None
+ or not db.apiutils.is_reference(
+ dataAnalysisRecord.get_property("sources"))):
- raise RuntimeError("InputDataSet Refenrence must exist.")
+ raise RuntimeError("sources Refenrence must exist.")
# ####### this core might be replaced by a call to another script ####### #
# Download the data
+ source_val = dataAnalysisRecord.get_property("sources").value
npobj = db.File(
- id=dataAnalysisRecord.get_property("InputDataSet")).retrieve()
+ id=(source_val[0]
+ if isinstance(source_val, list)
+ else source_val)).retrieve()
npfile = npobj.download()
data = np.load(npfile)
# Plot data
filename = "hist.png"
plt.hist(data)
- plt.savefig()
+ plt.savefig(filename)
mean = data.mean()
# ####################################################################### #
# Insert the result plot
# TODO (must): how do we find a good file path??
- fig = db.File(file=filename, path="/uploaded/something/"+filename)
+ fig = db.File(file=filename,
+ path="/uploaded/something3/"+str(datetime.now())+filename)
fig.insert()
# Add the result to the analysis Record
@@ -92,7 +102,7 @@ def main(args):
# be different.... Compare checksums of files?
assure_property_is(
dataAnalysisRecord,
- "result",
+ "results",
fig.id,
)
# TODO (must) what should be done with the old file? Removed if not referenced?
diff --git a/src/caosadvancedtools/serverside/generic_analysis.py b/src/caosadvancedtools/serverside/generic_analysis.py
index a9751da9..1d213702 100644
--- a/src/caosadvancedtools/serverside/generic_analysis.py
+++ b/src/caosadvancedtools/serverside/generic_analysis.py
@@ -111,15 +111,15 @@ def check_referenced_script(record: db.Record):
script_prop = record.get_property("scripts")
- if (not db.apiutils.is_reference(script_prop) or
- not isinstance(script_prop.value, int)):
+ if not db.apiutils.is_reference(script_prop):
logger.warning("The 'scripts' Property of the following Record should "
"reference a File:\n{}".format(str(record)))
return
script = db.execute_query("FIND ENTITY WITH id={}".format(
- script_prop.value), unique=True)
+ script_prop.value[0] if isinstance(script_prop.value, list)
+ else script_prop.value), unique=True)
if (not isinstance(script, db.File)):
logger.warning("The 'scripts' Property of the Record {} should "
@@ -140,10 +140,14 @@ def call_script(script_name, record_id):
return
- ret = subprocess.run([script_name, record_id], stdout=subprocess.PIPE,
- stderr=subprocess.PIPE)
+ cmd = ["./"+script_name, str(record_id)]
+ print("Running: "+" ".join(cmd))
+ logger.debug("Running: "+" ".join(cmd))
+ ret = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+ print(ret.stdout.decode())
+ print(ret.stderr.decode())
- if ret.returnvalue != 0:
+ if ret.returncode != 0:
logger.warning("Skript failed")
@@ -184,11 +188,9 @@ def _parse_arguments():
"""Parses the command line arguments.
"""
parser = argparse.ArgumentParser(description='__doc__')
- parser.add_argument("--entity", help="An id an input dataset.")
- parser.add_argument("--entity", help="An id of a parameter record.")
- parser.add_argument(
- '-a', '--auth-token', required=False,
- help="An authentication token (not needed, only for compatibility).")
+ parser.add_argument("--module", help="An id an input dataset.")
+ parser.add_argument("--inputset", help="An id an input dataset.")
+ parser.add_argument("--parameterset", help="An id of a parameter record.")
return parser.parse_args()
@@ -199,9 +201,11 @@ def main():
dataAnalysisRecord = db.Record()
dataAnalysisRecord.add_property(name="InputDataSet", value=args.entity)
dataAnalysisRecord.add_property(name="ParameterSet", value=args.parameter)
+ dataAnalysisRecord.add_property(name="Software", value=args.module)
# TODO: should this be done?
dataAnalysisRecord.insert()
+ run(dataAnalysisRecord)
if __name__ == "__main__":
--
GitLab