.gitlab-ci.yml

@@ -40,18 +40,18 @@ code_style:
   tags: [ docker ]
   stage: code_style
   script:
-    - pycodestyle --count ./
+    - make style
   allow_failure: true
 
+# pylint tests for pycaosdb
 pylint:
   tags: [ docker ]
   stage: linting
   script:
-    - pylint --unsafe-load-any-extension=y -d all -e E,F src/caosdb/common
+    - make lint
   allow_failure: true
 
-
-# pylint tests for pycaosdb
+# run tests
 test:
   tags: [ docker ]
   stage: test

CHANGELOG.md

@@ -8,8 +8,6 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 ## [Unreleased] ##
 
 ### Added ###
-- function in administration that generates passwords that comply with the
-  rules
 
 ### Changed ###
 
@@ -19,6 +17,16 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Fixed ###
 
+### Security ###
+
+## [0.7.0] - 2022-01-21 ##
+
+### Added ###
+
+- Function in administration that generates passwords that comply with the rules.
+
+### Fixed ###
+
 - #90 compare_entities function in apiutils does not check units
 
 ### Security ###

Makefile

@@ -31,3 +31,14 @@ doc:
 
 install:
 	@echo "Not implemented yet, use pip for installation."
+
+check: style lint
+.PHONY: check
+
+style:
+	pycodestyle --count examples src unittests
+.PHONY: style
+
+lint:
+	pylint --unsafe-load-any-extension=y -d all -e E,F src/caosdb/common
+.PHONY: lint

RELEASE_GUIDELINES.md

 # Release Guidelines for the CaosDB Python Client Library
 
-This document specifies release guidelines in addition to the generel release
+This document specifies release guidelines in addition to the general release
 guidelines of the CaosDB Project
 ([RELEASE_GUIDELINES.md](https://gitlab.com/caosdb/caosdb/blob/dev/RELEASE_GUIDELINES.md))
 
@@ -38,5 +38,5 @@ guidelines of the CaosDB Project
 10. Merge the main branch back into the dev branch.
 
 11. After the merge of main to dev, start a new development version by
-    setting `ISRELEASED` to `False` and by increasing at least the `MIRCO`
+    setting `ISRELEASED` to `False` and by increasing at least the `MICRO`
     version in [setup.py](./setup.py) and preparing CHANGELOG.md.

setup.py

@@ -45,11 +45,11 @@ from setuptools import find_packages, setup
 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 ########################################################################
 
+ISRELEASED = False
 MAJOR = 0
-MINOR = 6
-MICRO = 2
+MINOR = 7
+MICRO = 1
 PRE = ""  # e.g. rc0, alpha.1, 0.beta-23
-ISRELEASED = False
 
 if PRE:
     VERSION = "{}.{}.{}-{}".format(MAJOR, MINOR, MICRO, PRE)

src/doc/gallery/Makefile

+# This file is a part of the CaosDB Project.
+#
+# Copyright (C) 2022 IndiScale GmbH <info@indiscale.com>
+# Copyright (C) 2022 Daniel Hornung <d.hornung@indiscale.com>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+
+# Run tests on the examples.
+test:
+	python3 -m doctest simulation.py
+.PHONY: test

src/doc/gallery/index.rst

+
+PyCaosDB Code Gallery
+=====================
+
+This chapter collects code examples which can be immediately run against an empty CaosDB instance.
+
+.. note::
+
+   These examples require a configuration file with valid server and user/password settings.  Refer
+   to the :ref:`Configuration <Configuration of PyCaosDB>` section for details.
+
+.. toctree::
+   :maxdepth: 2
+   :caption: The code examples:
+
+   simulation

src/doc/gallery/simulation.py

+"""
+Run a simulation and store the values into CaosDB.
+
+>>> main()              # doctest: +ELLIPSIS
+These distances resulted in small x,y, values:
+[...]
+"""
+
+import numpy as np
+import scipy.integrate
+import caosdb as db
+from caosadvancedtools.table_converter import to_table
+
+
+def setup_caosdb():
+    """Create the data model and insert it into CaosDB
+
+    The data model consists of the following RecordTypes:
+
+    Software
+      with author and revision.
+
+    SoftwareRun
+      A specific run of the sofware, with input parameters, time of completion and a result.
+
+    State
+      An aggregate of x,y,z values.
+
+    Parameters
+      In this case the x,y,z initial values before the integration, so this is just a state.
+
+    Result
+      The x,y,z values at the end of the software run, the final state.
+
+    The data model of course also contains the corresponding properties for these RecordTypes.
+    """
+
+    cont = db.Container()  # Container to insert all Entities at once into CaosDB
+    # create Properties
+    cont.append(db.Property("x", datatype=db.DOUBLE))
+    cont.append(db.Property("y", datatype=db.DOUBLE))
+    cont.append(db.Property("z", datatype=db.DOUBLE))
+    cont.append(db.Property("completed", datatype=db.DATETIME))
+    cont.append(db.Property("author", datatype=db.TEXT))
+    cont.append(db.Property("revision", datatype=db.TEXT))
+    # create RecordTypes
+    cont.append(db.RecordType("Software").add_property("author").add_property("revision"))
+    cont.append(db.RecordType("State").add_property("x", importance=db.OBLIGATORY)
+                .add_property("y").add_property("z"))
+    cont.append(db.RecordType("Parameters").add_parent("State", inheritance=db.ALL))
+    cont.append(db.RecordType("Result").add_parent("State", inheritance=db.RECOMMENDED))
+    cont.append(db.RecordType("SoftwareRun").add_property("Software").add_property("Parameters")
+                .add_property("completed").add_property("Result"))
+    cont.insert()  # actually insert the Entities
+
+
+def simulations(n, t_max):
+    """Run the simulations.
+
+    Parameters
+    ----------
+    n : int
+      The number of runs.
+
+    t_max : float
+      The maximum time of integration.
+    """
+
+    software = (db.Record("simulator").add_parent("Software")
+                .add_property("author", value="IndiScale GmbH")
+                .add_property("revision", value="1234CDEF89AB"))
+    software.insert()
+    for i in range(n):
+        # Get the parameters and result
+        initial, result = run_simulation(run=i, t_max=t_max)
+
+        # Prepare CaosDB insertion
+        run = db.Record().add_parent("SoftwareRun").add_property("Software", value=software.id)
+        parameters = (db.Record().add_parent("Parameters").add_property("x", initial[0])
+                      .add_property("y", initial[1]).add_property("z", initial[2]))
+        result_record = (db.Record().add_parent("Result").add_property("x", result[0])
+                         .add_property("y", result[1]).add_property("z", result[2]))
+        run.add_property("Parameters", value=parameters).add_property("Result", value=result_record)
+        cont = db.Container()
+        cont.extend([run, parameters, result_record])
+        cont.insert()           # Insert everything of this run into CaosDB.
+
+
+def run_simulation(run, t_max):
+    """Integrate the Rössler attractor from random initial values."""
+    a, b, c = (0.1, 0.1, 14)
+
+    def diff(t, x):
+        diff = np.array([-x[1] - x[2],
+                         x[0] + a * x[1],
+                         b + x[2] * (x[0] - c)])
+        return diff
+
+    x0 = np.random.uniform(-100, 100, 3)
+
+    result = scipy.integrate.solve_ivp(diff, [0, t_max], x0)
+    x = result.y[:, -1]
+    return (x0, x)
+
+
+def analyze():
+    """Find the initial conditions which produce the smalles x,y values after the given time."""
+    distance = 5
+    data = db.execute_query("""SELECT Parameters, Result FROM RECORD SoftwareRun WITH
+        (((Result.x < {dist}) AND (Result.x > -{dist}))
+        AND (Result.y < {dist})) AND Result.y > -{dist}""".format(dist=distance))
+    dataframe = to_table(data)  # Convert into a Pandas DataFrame
+
+    parameters = db.Container().extend([db.Record(id=id) for id in dataframe.Parameters]).retrieve()
+
+    initial_distances = [np.linalg.norm([p.get_property(dim).value for dim in ["x", "y", "z"]])
+                         for p in parameters]
+
+    print("These distances resulted in small x,y, values:\n{}".format(initial_distances))
+
+
+def main():
+    # 1. Set up the data model
+    setup_caosdb()
+
+    # 2. Run simulations
+    simulations(n=200, t_max=5)
+
+    # 3. Find initial conditions with interesting results
+    analyze()
+
+
+if __name__ == '__main__':
+    main()

src/doc/gallery/simulation.rst

+========================================
+Managing data from numerical simulations
+========================================
+
+This code example
+
+1. sets up the data model
+2. runs simulations
+3. stores the simulation parameters and results into CaosDB
+4. retrieves the parameters for interesting results.
+
+:download:`Download code<simulation.py>`
+
+.. literalinclude:: simulation.py

src/doc/index.rst

@@ -12,6 +12,7 @@ Welcome to PyCaosDB's documentation!
    Concepts <concepts>
    Configuration <configuration>
    Administration <administration>
+   Code gallery <gallery/index>
    API documentation<_apidoc/caosdb>
 
 This is the documentation for the Python client library for CaosDB, ``PyCaosDB``.

src/doc/tutorials/index.rst

@@ -16,4 +16,5 @@ advanced usage of the Python client.
    errors
    data-model-interface
    complex_data_models
+   serverside
       

src/doc/tutorials/serverside.rst

+
+Server Side Scripting
+=====================
+
+The administrator may store regularly needed scripts, e.g. for computing a
+standardized analysis, on the same machine as the CaosDB server, "on the server
+side", where they can be run directly by the server.
+
+The execution of those scripts can be initiated using the Python client, or the
+web interface.
+
+Call a Script
+~~~~~~~~~~~~~
+
+If you have access to the server and sufficient permissions to run the script,
+execution is fairly simple:
+
+.. code:: python
+
+    from caosdb.utils.server_side_scripting import run_server_side_script
+    response = run_server_side_script('scriptname.py')
+    print(response.stderr,response.stdout)
+
+
+This makes the server run the script ``scriptname.py``. The output of the
+script (``stderr`` and ``stdout``) is returned within an response object.
+
+
+If the script requires additional arguments, those can be provided after the 
+script's name.
+
+Note that by default the script runs with your CaosDB account. It has your
+permissions and changes are logged as if they were done by you directly.
+
+
+Testing it
+~~~~~~~~~~
+
+You can try this out using for example the ``diagnostics.py`` script (it is part
+of the `CaosDB server repository
+<https://gitlab.indiscale.com/caosdb/src/caosdb-server/-/blob/main/scripting/bin/administration/diagnostics.py>`_
+and is also available on https://demo.indiscale.com). The script returns
+information about the server in JSON format. You can do for example the
+following:
+
+.. code:: python
+
+    import json
+    from caosdb.utils.server_side_scripting import run_server_side_script
+    response = run_server_side_script('administration/diagnostics.py')
+    print("JSON content:")
+    print(json.loads(response.stdout))
+    print("stderr:")
+    print(response.stderr)
+
+
+Further Information
+~~~~~~~~~~~~~~~~~~~
+
+Additionally, you might want to have a look at
+https://docs.indiscale.com/caosdb-server/specification/Server-side-scripting.html

unittests/test_apiutils.py

@@ -145,7 +145,7 @@ def test_compare_entities():
     assert "tests_234234" in diff_r1["properties"]
     assert "tests_TT" in diff_r2["properties"]
 
-    
+
 def test_compare_entities_units():
     r1 = db.Record()
     r2 = db.Record()