diff --git a/integrationtests/create_analysis.py b/integrationtests/create_analysis.py new file mode 100644 index 0000000000000000000000000000000000000000..44e99c9848a4c75fd80e419452be1eda087b3230 --- /dev/null +++ b/integrationtests/create_analysis.py @@ -0,0 +1,66 @@ +#!/usr/bin/env python3 +# encoding: utf-8 +# +# ** header v3.0 +# This file is a part of the CaosDB Project. +# +# Copyright (C) 2021 Indiscale GmbH <info@indiscale.com> +# Copyright (C) 2021 Henrik tom Wörden <h.tomwoerden@indiscale.com> +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see <https://www.gnu.org/licenses/>. +# +# ** end header +# + +""" +module description +""" + +import sys + +import caosdb as db + + +def main(): + # TODO remove fixed path + script = db.File( + file="../src/caosadvancedtools/serverside/examples/example_script.py", + path="Analysis/scripts/example_script.py", + ) + try: + script.insert() + except: + script = db.execute_query( + "FIND FILE which is stored at 'Analysis/scripts/example_script.py'", + unique=True) + + da = db.Record() + da.add_parent("Analysis") + da.add_property("scripts", value=[script], datatype=db.LIST(db.FILE)) + da.add_property("sources", + value=db.execute_query( + "FIND FILE which is stored at '**/timeseries.npy'", + unique=True), + # datatype=db.LIST(db.FILE) + ) + da.add_property("date", "2020-01-01") + da.add_property("identifier", "TEST") + da.add_property("responsible", db.execute_query( + "FIND RECORD Person WITH firstname=Only", + unique=True)) + da.insert() + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/integrationtests/example_script.py b/integrationtests/example_script.py new file mode 120000 index 0000000000000000000000000000000000000000..6a03bd102a8a2989cf70e6ed1954d7cc93d6130d --- /dev/null +++ b/integrationtests/example_script.py @@ -0,0 +1 @@ +../src/caosadvancedtools/examples/example_script.py \ No newline at end of file diff --git a/integrationtests/extroot/SimulationData/2010_TestProject/2019-02-03_something/timeseries.npy b/integrationtests/extroot/SimulationData/2010_TestProject/2019-02-03_something/timeseries.npy index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..18da9b18cda23d411d0f2666629377dd7991ac8f 100644 Binary files a/integrationtests/extroot/SimulationData/2010_TestProject/2019-02-03_something/timeseries.npy and b/integrationtests/extroot/SimulationData/2010_TestProject/2019-02-03_something/timeseries.npy differ diff --git a/integrationtests/model.yml b/integrationtests/model.yml index cad7958803edf1a01f0649353443ffab350cc5e5..6c68a86156c21c37abccee575de76486c44f3f06 100644 --- a/integrationtests/model.yml +++ b/integrationtests/model.yml @@ -51,6 +51,9 @@ Analysis: date: identifier: responsible: + suggested_properties: + mean_value: + datatype: DOUBLE Publication: Thesis: inherit_from_suggested: diff --git a/integrationtests/test.sh b/integrationtests/test.sh index 71af543643a35cb082f10a24440c5ea87df946c9..9cf17be452a77f2ad0cbc827b9d4c49382098111 100755 --- a/integrationtests/test.sh +++ b/integrationtests/test.sh @@ -65,6 +65,10 @@ python3 test_table.py # TODO the following test deletes lots of the data inserted by the crawler echo "Testing im and export" python3 test_im_und_export.py + +# automated analysis +python3 create_analysis.py + # Better safe than sorry: python3 clear_database.py diff --git a/integrationtests/update_analysis.py b/integrationtests/update_analysis.py new file mode 100644 index 0000000000000000000000000000000000000000..3fd9840ac6cce6844670f3147649c034c405ba7f --- /dev/null +++ b/integrationtests/update_analysis.py @@ -0,0 +1,42 @@ +#!/usr/bin/env python3 +# encoding: utf-8 +# +# ** header v3.0 +# This file is a part of the CaosDB Project. +# +# Copyright (C) 2021 Indiscale GmbH <info@indiscale.com> +# Copyright (C) 2021 Henrik tom Wörden <h.tomwoerden@indiscale.com> +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see <https://www.gnu.org/licenses/>. +# +# ** end header +# + +""" +module description +""" + +import sys + +import caosdb as db +from caosadvancedtools.serverside.generic_analysis import run + + +def main(): + da = db.execute_query("FIND Analysis with identifier=TEST", unique=True) + run(da) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/src/caosadvancedtools/serverside/example_script.py b/src/caosadvancedtools/serverside/examples/example_script.py old mode 100644 new mode 100755 similarity index 79% rename from src/caosadvancedtools/serverside/example_script.py rename to src/caosadvancedtools/serverside/examples/example_script.py index ff01ac30faef487bc476e23a8d3df99b83c4904d..148d8cbae22a62865c3c619f14e459c7556df957 --- a/src/caosadvancedtools/serverside/example_script.py +++ b/src/caosadvancedtools/serverside/examples/example_script.py @@ -32,14 +32,19 @@ import argparse import logging import sys from argparse import RawTextHelpFormatter +from datetime import datetime import caosdb as db import matplotlib.pyplot as plt import numpy as np from caosadvancedtools.cfood import assure_property_is +from caosadvancedtools.guard import INSERT, UPDATE +from caosadvancedtools.guard import global_guard as guard logger = logging.getLogger(__name__) +guard.set_level(level=UPDATE) + def main(args): # TODO (maybe) can these checks be replaced by a more declaritive appoach? @@ -51,32 +56,37 @@ def main(args): )) # The script may require certain information to exist. Here, we expect that - # a InputDataSet Property exists that references a numpy file. + # a sources Property exists that references a numpy file. + # Similarly an InputDataSet could be used. - if (dataAnalysisRecord.get_property("InputDataSet") is None - or db.apiutils.is_reference( - dataAnalysisRecord.get_property("InputDataSet"))): + if (dataAnalysisRecord.get_property("sources") is None + or not db.apiutils.is_reference( + dataAnalysisRecord.get_property("sources"))): - raise RuntimeError("InputDataSet Refenrence must exist.") + raise RuntimeError("sources Refenrence must exist.") # ####### this core might be replaced by a call to another script ####### # # Download the data + source_val = dataAnalysisRecord.get_property("sources").value npobj = db.File( - id=dataAnalysisRecord.get_property("InputDataSet")).retrieve() + id=(source_val[0] + if isinstance(source_val, list) + else source_val)).retrieve() npfile = npobj.download() data = np.load(npfile) # Plot data filename = "hist.png" plt.hist(data) - plt.savefig() + plt.savefig(filename) mean = data.mean() # ####################################################################### # # Insert the result plot # TODO (must): how do we find a good file path?? - fig = db.File(file=filename, path="/uploaded/something/"+filename) + fig = db.File(file=filename, + path="/uploaded/something3/"+str(datetime.now())+filename) fig.insert() # Add the result to the analysis Record @@ -92,7 +102,7 @@ def main(args): # be different.... Compare checksums of files? assure_property_is( dataAnalysisRecord, - "result", + "results", fig.id, ) # TODO (must) what should be done with the old file? Removed if not referenced? diff --git a/src/caosadvancedtools/serverside/generic_analysis.py b/src/caosadvancedtools/serverside/generic_analysis.py index a9751da9d0d824c8fab18d322881f371fa753f09..1d21370296b4b22ba644f5078a8d74b3305dab75 100644 --- a/src/caosadvancedtools/serverside/generic_analysis.py +++ b/src/caosadvancedtools/serverside/generic_analysis.py @@ -111,15 +111,15 @@ def check_referenced_script(record: db.Record): script_prop = record.get_property("scripts") - if (not db.apiutils.is_reference(script_prop) or - not isinstance(script_prop.value, int)): + if not db.apiutils.is_reference(script_prop): logger.warning("The 'scripts' Property of the following Record should " "reference a File:\n{}".format(str(record))) return script = db.execute_query("FIND ENTITY WITH id={}".format( - script_prop.value), unique=True) + script_prop.value[0] if isinstance(script_prop.value, list) + else script_prop.value), unique=True) if (not isinstance(script, db.File)): logger.warning("The 'scripts' Property of the Record {} should " @@ -140,10 +140,14 @@ def call_script(script_name, record_id): return - ret = subprocess.run([script_name, record_id], stdout=subprocess.PIPE, - stderr=subprocess.PIPE) + cmd = ["./"+script_name, str(record_id)] + print("Running: "+" ".join(cmd)) + logger.debug("Running: "+" ".join(cmd)) + ret = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + print(ret.stdout.decode()) + print(ret.stderr.decode()) - if ret.returnvalue != 0: + if ret.returncode != 0: logger.warning("Skript failed") @@ -184,11 +188,9 @@ def _parse_arguments(): """Parses the command line arguments. """ parser = argparse.ArgumentParser(description='__doc__') - parser.add_argument("--entity", help="An id an input dataset.") - parser.add_argument("--entity", help="An id of a parameter record.") - parser.add_argument( - '-a', '--auth-token', required=False, - help="An authentication token (not needed, only for compatibility).") + parser.add_argument("--module", help="An id an input dataset.") + parser.add_argument("--inputset", help="An id an input dataset.") + parser.add_argument("--parameterset", help="An id of a parameter record.") return parser.parse_args() @@ -199,9 +201,11 @@ def main(): dataAnalysisRecord = db.Record() dataAnalysisRecord.add_property(name="InputDataSet", value=args.entity) dataAnalysisRecord.add_property(name="ParameterSet", value=args.parameter) + dataAnalysisRecord.add_property(name="Software", value=args.module) # TODO: should this be done? dataAnalysisRecord.insert() + run(dataAnalysisRecord) if __name__ == "__main__":