diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 72b4381aebcaf5edd16c10c479df6ca908128ec6..9bb5adb6ed4fedf57694d6ec7843942c46296269 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -40,18 +40,18 @@ code_style: tags: [ docker ] stage: code_style script: - - pycodestyle --count ./ + - make style allow_failure: true +# pylint tests for pycaosdb pylint: tags: [ docker ] stage: linting script: - - pylint --unsafe-load-any-extension=y -d all -e E,F src/caosdb/common + - make lint allow_failure: true - -# pylint tests for pycaosdb +# run tests test: tags: [ docker ] stage: test diff --git a/CHANGELOG.md b/CHANGELOG.md index 757d47416cd2fdacda6d011acacd0fab4f6bad34..11212ffbf578160298c3abc72fe9aa366b4bb164 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,8 +8,6 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] ## ### Added ### -- function in administration that generates passwords that comply with the - rules ### Changed ### @@ -19,6 +17,16 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed ### +### Security ### + +## [0.7.0] - 2022-01-21 ## + +### Added ### + +- Function in administration that generates passwords that comply with the rules. + +### Fixed ### + - #90 compare_entities function in apiutils does not check units - #94 some special properties were not checked in compare_entities diff --git a/Makefile b/Makefile index 4bc3459d209936c17a445c64e77180d9559e4653..192337853d8db8812e14f75fca8986006de82180 100644 --- a/Makefile +++ b/Makefile @@ -31,3 +31,14 @@ doc: install: @echo "Not implemented yet, use pip for installation." + +check: style lint +.PHONY: check + +style: + pycodestyle --count examples src unittests +.PHONY: style + +lint: + pylint --unsafe-load-any-extension=y -d all -e E,F src/caosdb/common +.PHONY: lint diff --git a/RELEASE_GUIDELINES.md b/RELEASE_GUIDELINES.md index 68deedf263dd8431d4e37e8ea732af2dda3304f5..f2671ed5b35e04872e3081a3e65c51baf381410b 100644 --- a/RELEASE_GUIDELINES.md +++ b/RELEASE_GUIDELINES.md @@ -1,6 +1,6 @@ # Release Guidelines for the CaosDB Python Client Library -This document specifies release guidelines in addition to the generel release +This document specifies release guidelines in addition to the general release guidelines of the CaosDB Project ([RELEASE_GUIDELINES.md](https://gitlab.com/caosdb/caosdb/blob/dev/RELEASE_GUIDELINES.md)) @@ -38,5 +38,5 @@ guidelines of the CaosDB Project 10. Merge the main branch back into the dev branch. 11. After the merge of main to dev, start a new development version by - setting `ISRELEASED` to `False` and by increasing at least the `MIRCO` + setting `ISRELEASED` to `False` and by increasing at least the `MICRO` version in [setup.py](./setup.py) and preparing CHANGELOG.md. diff --git a/setup.py b/setup.py index b2a89cfe8dc26de874585cdd46c06e458d3533e9..083865555352a4fd06fb12b9da6f787d51510a4a 100755 --- a/setup.py +++ b/setup.py @@ -45,11 +45,11 @@ from setuptools import find_packages, setup # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ######################################################################## +ISRELEASED = False MAJOR = 0 -MINOR = 6 -MICRO = 2 +MINOR = 7 +MICRO = 1 PRE = "" # e.g. rc0, alpha.1, 0.beta-23 -ISRELEASED = False if PRE: VERSION = "{}.{}.{}-{}".format(MAJOR, MINOR, MICRO, PRE) diff --git a/src/doc/gallery/Makefile b/src/doc/gallery/Makefile new file mode 100644 index 0000000000000000000000000000000000000000..658f9a6a93e23957b20aee5f38e5565bde35af80 --- /dev/null +++ b/src/doc/gallery/Makefile @@ -0,0 +1,23 @@ +# This file is a part of the CaosDB Project. +# +# Copyright (C) 2022 IndiScale GmbH <info@indiscale.com> +# Copyright (C) 2022 Daniel Hornung <d.hornung@indiscale.com> +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see <https://www.gnu.org/licenses/>. + + +# Run tests on the examples. +test: + python3 -m doctest simulation.py +.PHONY: test diff --git a/src/doc/gallery/index.rst b/src/doc/gallery/index.rst new file mode 100644 index 0000000000000000000000000000000000000000..a6ef53e4c7d1272c5dbc8c62b4d90a89591cac0f --- /dev/null +++ b/src/doc/gallery/index.rst @@ -0,0 +1,16 @@ + +PyCaosDB Code Gallery +===================== + +This chapter collects code examples which can be immediately run against an empty CaosDB instance. + +.. note:: + + These examples require a configuration file with valid server and user/password settings. Refer + to the :ref:`Configuration <Configuration of PyCaosDB>` section for details. + +.. toctree:: + :maxdepth: 2 + :caption: The code examples: + + simulation diff --git a/src/doc/gallery/simulation.py b/src/doc/gallery/simulation.py new file mode 100644 index 0000000000000000000000000000000000000000..342d5d980fc2b1a981f4a76d99e1954f8b2f5c2a --- /dev/null +++ b/src/doc/gallery/simulation.py @@ -0,0 +1,134 @@ +""" +Run a simulation and store the values into CaosDB. + +>>> main() # doctest: +ELLIPSIS +These distances resulted in small x,y, values: +[...] +""" + +import numpy as np +import scipy.integrate +import caosdb as db +from caosadvancedtools.table_converter import to_table + + +def setup_caosdb(): + """Create the data model and insert it into CaosDB + + The data model consists of the following RecordTypes: + + Software + with author and revision. + + SoftwareRun + A specific run of the sofware, with input parameters, time of completion and a result. + + State + An aggregate of x,y,z values. + + Parameters + In this case the x,y,z initial values before the integration, so this is just a state. + + Result + The x,y,z values at the end of the software run, the final state. + + The data model of course also contains the corresponding properties for these RecordTypes. + """ + + cont = db.Container() # Container to insert all Entities at once into CaosDB + # create Properties + cont.append(db.Property("x", datatype=db.DOUBLE)) + cont.append(db.Property("y", datatype=db.DOUBLE)) + cont.append(db.Property("z", datatype=db.DOUBLE)) + cont.append(db.Property("completed", datatype=db.DATETIME)) + cont.append(db.Property("author", datatype=db.TEXT)) + cont.append(db.Property("revision", datatype=db.TEXT)) + # create RecordTypes + cont.append(db.RecordType("Software").add_property("author").add_property("revision")) + cont.append(db.RecordType("State").add_property("x", importance=db.OBLIGATORY) + .add_property("y").add_property("z")) + cont.append(db.RecordType("Parameters").add_parent("State", inheritance=db.ALL)) + cont.append(db.RecordType("Result").add_parent("State", inheritance=db.RECOMMENDED)) + cont.append(db.RecordType("SoftwareRun").add_property("Software").add_property("Parameters") + .add_property("completed").add_property("Result")) + cont.insert() # actually insert the Entities + + +def simulations(n, t_max): + """Run the simulations. + + Parameters + ---------- + n : int + The number of runs. + + t_max : float + The maximum time of integration. + """ + + software = (db.Record("simulator").add_parent("Software") + .add_property("author", value="IndiScale GmbH") + .add_property("revision", value="1234CDEF89AB")) + software.insert() + for i in range(n): + # Get the parameters and result + initial, result = run_simulation(run=i, t_max=t_max) + + # Prepare CaosDB insertion + run = db.Record().add_parent("SoftwareRun").add_property("Software", value=software.id) + parameters = (db.Record().add_parent("Parameters").add_property("x", initial[0]) + .add_property("y", initial[1]).add_property("z", initial[2])) + result_record = (db.Record().add_parent("Result").add_property("x", result[0]) + .add_property("y", result[1]).add_property("z", result[2])) + run.add_property("Parameters", value=parameters).add_property("Result", value=result_record) + cont = db.Container() + cont.extend([run, parameters, result_record]) + cont.insert() # Insert everything of this run into CaosDB. + + +def run_simulation(run, t_max): + """Integrate the Rössler attractor from random initial values.""" + a, b, c = (0.1, 0.1, 14) + + def diff(t, x): + diff = np.array([-x[1] - x[2], + x[0] + a * x[1], + b + x[2] * (x[0] - c)]) + return diff + + x0 = np.random.uniform(-100, 100, 3) + + result = scipy.integrate.solve_ivp(diff, [0, t_max], x0) + x = result.y[:, -1] + return (x0, x) + + +def analyze(): + """Find the initial conditions which produce the smalles x,y values after the given time.""" + distance = 5 + data = db.execute_query("""SELECT Parameters, Result FROM RECORD SoftwareRun WITH + (((Result.x < {dist}) AND (Result.x > -{dist})) + AND (Result.y < {dist})) AND Result.y > -{dist}""".format(dist=distance)) + dataframe = to_table(data) # Convert into a Pandas DataFrame + + parameters = db.Container().extend([db.Record(id=id) for id in dataframe.Parameters]).retrieve() + + initial_distances = [np.linalg.norm([p.get_property(dim).value for dim in ["x", "y", "z"]]) + for p in parameters] + + print("These distances resulted in small x,y, values:\n{}".format(initial_distances)) + + +def main(): + # 1. Set up the data model + setup_caosdb() + + # 2. Run simulations + simulations(n=200, t_max=5) + + # 3. Find initial conditions with interesting results + analyze() + + +if __name__ == '__main__': + main() diff --git a/src/doc/gallery/simulation.rst b/src/doc/gallery/simulation.rst new file mode 100644 index 0000000000000000000000000000000000000000..ce1a7f457a142e36ef9f2b0cfe6a4df0b9fcedf6 --- /dev/null +++ b/src/doc/gallery/simulation.rst @@ -0,0 +1,14 @@ +======================================== +Managing data from numerical simulations +======================================== + +This code example + +1. sets up the data model +2. runs simulations +3. stores the simulation parameters and results into CaosDB +4. retrieves the parameters for interesting results. + +:download:`Download code<simulation.py>` + +.. literalinclude:: simulation.py diff --git a/src/doc/index.rst b/src/doc/index.rst index bd29c6c56acf5c173e94ae6471a6aeba56ea4b93..004ae3a9926ed7a9a27720db1f3c28e72f1f3f28 100644 --- a/src/doc/index.rst +++ b/src/doc/index.rst @@ -12,6 +12,7 @@ Welcome to PyCaosDB's documentation! Concepts <concepts> Configuration <configuration> Administration <administration> + Code gallery <gallery/index> API documentation<_apidoc/caosdb> This is the documentation for the Python client library for CaosDB, ``PyCaosDB``. diff --git a/src/doc/tutorials/index.rst b/src/doc/tutorials/index.rst index 79068e9201498c87b2eb61b4ffbea0969845b404..0b08d0b4fe153d803a780bd144787819b827db78 100644 --- a/src/doc/tutorials/index.rst +++ b/src/doc/tutorials/index.rst @@ -16,4 +16,5 @@ advanced usage of the Python client. errors data-model-interface complex_data_models + serverside diff --git a/src/doc/tutorials/serverside.rst b/src/doc/tutorials/serverside.rst new file mode 100644 index 0000000000000000000000000000000000000000..93f0fdcf742efc70bc80f5113eb7c6ddbbf87cde --- /dev/null +++ b/src/doc/tutorials/serverside.rst @@ -0,0 +1,61 @@ + +Server Side Scripting +===================== + +The administrator may store regularly needed scripts, e.g. for computing a +standardized analysis, on the same machine as the CaosDB server, "on the server +side", where they can be run directly by the server. + +The execution of those scripts can be initiated using the Python client, or the +web interface. + +Call a Script +~~~~~~~~~~~~~ + +If you have access to the server and sufficient permissions to run the script, +execution is fairly simple: + +.. code:: python + + from caosdb.utils.server_side_scripting import run_server_side_script + response = run_server_side_script('scriptname.py') + print(response.stderr,response.stdout) + + +This makes the server run the script ``scriptname.py``. The output of the +script (``stderr`` and ``stdout``) is returned within an response object. + + +If the script requires additional arguments, those can be provided after the +script's name. + +Note that by default the script runs with your CaosDB account. It has your +permissions and changes are logged as if they were done by you directly. + + +Testing it +~~~~~~~~~~ + +You can try this out using for example the ``diagnostics.py`` script (it is part +of the `CaosDB server repository +<https://gitlab.indiscale.com/caosdb/src/caosdb-server/-/blob/main/scripting/bin/administration/diagnostics.py>`_ +and is also available on https://demo.indiscale.com). The script returns +information about the server in JSON format. You can do for example the +following: + +.. code:: python + + import json + from caosdb.utils.server_side_scripting import run_server_side_script + response = run_server_side_script('administration/diagnostics.py') + print("JSON content:") + print(json.loads(response.stdout)) + print("stderr:") + print(response.stderr) + + +Further Information +~~~~~~~~~~~~~~~~~~~ + +Additionally, you might want to have a look at +https://docs.indiscale.com/caosdb-server/specification/Server-side-scripting.html diff --git a/unittests/test_apiutils.py b/unittests/test_apiutils.py index b46d89986b843cef74492f38e3c069538a227872..99394348286b28f06ad9ce4d369edcdc8f9a3589 100644 --- a/unittests/test_apiutils.py +++ b/unittests/test_apiutils.py @@ -145,7 +145,7 @@ def test_compare_entities(): assert "tests_234234" in diff_r1["properties"] assert "tests_TT" in diff_r2["properties"] - + def test_compare_entities_units(): r1 = db.Record() r2 = db.Record()