diff --git a/.gitignore b/.gitignore
index ab11f8441c1690ede967897c0e64745f9ad30f86..c30895c42d4a4bae9f1bea8750f63899102b2fa9 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,3 +1,5 @@
+# -*- mode:conf; -*-
+
# configuration files
/conf/ext/*
!/conf/core/*.template
@@ -11,8 +13,11 @@
*.jks
# typical build dirs
+build/
bin/
target/
+_apidoc/
+
# But include server-side scripting
!/scripting/bin
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index c1afaf3f2a5274ff9e9a3083dbfd788e629709c0..0cf07b5cc6c376d27df7f1956af2e5ec8639be10 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -37,17 +37,19 @@ build-testenv:
tags: [ cached-dind ]
image: docker:19.03
stage: setup
+ timeout: 3h
only:
- schedules
script:
- cd src/test/docker
+ - time docker load < /image-cache/caosdb-server-testenv.tar || true
- docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY
# use here general latest or specific branch latest...
- - docker pull $CI_REGISTRY_IMAGE || true
- docker build
--pull
- --cache-from $CI_REGISTRY_IMAGE
-t $CI_REGISTRY_IMAGE .
+ - docker save $CI_REGISTRY_IMAGE > image.tar;
+ mv image.tar /image-cache/caosdb-server-testenv.tar;
- docker push $CI_REGISTRY_IMAGE
# Test: run unit tests of the server
@@ -74,3 +76,21 @@ trigger_build:
-F "variables[TriggerdBy]=SERVER"
-F "variables[TriggerdByHash]=$CI_COMMIT_SHORT_SHA"
-F ref=$DEPLOY_REF https://gitlab.indiscale.com/api/v4/projects/14/trigger/pipeline
+
+# Build the sphinx documentation and make it ready for deployment by Gitlab Pages
+# documentation:
+# stage: deploy
+
+# Special job for serving a static website. See https://docs.gitlab.com/ee/ci/yaml/README.html#pages
+pages:
+ tags: [ cached-dind ]
+ stage: deploy
+ only:
+ - dev
+ script:
+ - echo "Deploying"
+ - make doc
+ - cp -r build/doc/html public
+ artifacts:
+ paths:
+ - public
diff --git a/CHANGELOG.md b/CHANGELOG.md
index b70b2c179d9c340df77b3b42a103cbfd926b601f..e51cc4f004662b95d12abdb411043da60192b2de 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -23,6 +23,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
* New server property `SERVER_SIDE_SCRIPTING_BIN_DIRS` which accepts a comma or
space separated list as values. The server looks for scripts in all
directories in the order or the list and uses the first matching file.
+* Automated documentation builds: `make doc`
### Changed
diff --git a/makefile b/Makefile
similarity index 98%
rename from makefile
rename to Makefile
index 5c0e6cdda534c34f9f34109a008feac1d79312dc..d73f140f065ca0b3db62651e40162194f4ffb4eb 100644
--- a/makefile
+++ b/Makefile
@@ -128,3 +128,8 @@ stop-debug-screen:
easy-units: .m2-local
mvn clean
mvn deploy:deploy-file -DgroupId=de.timmfitschen -DartifactId=easy-units -Dversion=0.0.1-SNAPSHOT -Durl=file:./.m2-local/ -DrepositoryId=local-maven-repo -DupdateReleaseInfo=true -Dfile=./lib/easy-units-0.0.1-SNAPSHOT-jar-with-dependencies.jar
+
+# Compile the standalone documentation
+.PHONY: doc
+doc:
+ $(MAKE) -C src/doc html
diff --git a/README_SETUP.md b/README_SETUP.md
index 24fe8bde744585a8e050f1d2f11dc4c5e50c1b6e..5cbd85ab73bc200fe8534508be1f4538ac1dad88 100644
--- a/README_SETUP.md
+++ b/README_SETUP.md
@@ -1,25 +1,28 @@
-# Requirements
+# Getting Started with the CaosDB Server
+Here, you find information on requirements, the installation, configuration and more.
-## CaosDB Packages
+## Requirements
+
+### CaosDB Packages
* caosdb-webui=0.2.1
* caosdb-mysqlbackend=3.0
-## Third-party Software
-* >=Java 8
-* >=Apache Maven 3.0.4
-* >=Python 3.4
-* >=pip 9.0.1
-* >=git 1.9.1
-* >=Make 3.81
-* >=Screen 4.01
-* >=MySQL 5.5 (better >=5.6) or >=MariaDB 10.1
-* libpam (if PAM authentication is required)
-* unzip
-* openpyxl (for XLS/ODS export)
-* openssl (if a custom TLS certificate is required)
-
-### Install the requirements on Debian
+### Third-party Software
+* `>=Java 8`
+* `>=Apache Maven 3.0.4`
+* `>=Python 3.4`
+* `>=pip 9.0.1`
+* `>=git 1.9.1`
+* `>=Make 3.81`
+* `>=Screen 4.01`
+* `>=MySQL 5.5` (better `>=5.6`) or `>=MariaDB 10.1`
+* `libpam` (if PAM authentication is required)
+* `unzip`
+* `openpyxl` (for XLS/ODS export)
+* `openssl` (if a custom TLS certificate is required)
+
+#### Install the requirements on Debian
On Debian, the required packages can be installed with:
apt-get install git make mariadb-server maven openjdk-11-jdk-headless \
@@ -28,21 +31,21 @@ On Debian, the required packages can be installed with:
Note that installing MariaDB will uninstall existing MySQL packages and vice
versa.
-## System
+### System
-* >=Linux 4.0.0, x86\_64, e.g. Ubuntu 14.04.1
+* `>=Linux 4.0.0`, `x86_64`, e.g. Ubuntu 18.04
* Mounted filesytem(s) with enough space
* Working internet connection (for up-to-date python and java libraries)
-## Extensions ##
+### Extensions ##
-### Web UI ###
+#### Web UI ###
- If the WebUI shall run, check out the respective submodule: `git submodule
update --init caosdb-webui`
- Then configure and compile it according to its
- [documentation](caosdb-webui/README_SETUP.md).
+ [documentation](https://http://caosdb.gitlab.io/caosdb-webui/getting_started.html).
-### PAM ###
+#### PAM ###
Authentication via PAM is possible, for this the PAM development library must be
installed and the pam user tool must be compiled:
@@ -52,14 +55,14 @@ installed and the pam user tool must be compiled:
should print `[FAILED]` and return with a non-zero exit code. Unless there is
a user `asdf` with password `ghjk` on your system, of course.
-#### Troubleshooting ####
+##### Troubleshooting ####
If `make` fails with `pam_authentication.c:4:31: fatal error:
security/pam_appl.h: No such file or directory` the header files are probably
not installed. You can do so under Debian and Ubuntu with `apt-get install
libpam0g-dev`. Then try again.
-# First Setup
+## First Setup
After a fresh clone of the repository, this is what you need to setup the
server:
@@ -75,25 +78,28 @@ server:
- `keytool -genkey -keyalg RSA -alias selfsigned -keystore caosdb.jks -validity 375 -keysize 2048 -ext san=dns:localhost`
Replace `localhost` by your host name, if you want.
- `keytool -importkeystore -srckeystore caosdb.jks -destkeystore caosdb.p12 -deststoretype PKCS12 -srcalias selfsigned`
- - `openssl pkcs12 -in caosdb.p12 -nokeys -out cert.pem`
+ - Export the public part only: `openssl pkcs12 -in caosdb.p12 -nokeys -out cert.pem`.
+ The resulting ``cert.pem` can safely be given to users to allow ssl verification.
- You can check the content of the certificate with `openssl x509 -in cert.pem -text`
Alternatively, you can create a keystore from certificate files that you already have:
- `openssl pkcs12 -export -inkey privkey.pem -in fullchain.pem -out all-certs.pkcs12`
- `keytool -importkeystore -srckeystore all-certs.pkcs12 -srcstoretype PKCS12 -deststoretype pkcs12 -destkeystore caosdb.jks`
-
-3. Copy `conf/core/server.conf` to `conf/ext/server.conf` and change it
+3. Install/configure the MySQL back-end: see the `README_SETUP.md` of the
+ `caosdb-mysqlbackend` repository
+4. Create an authtoken config (e.g. copy `conf/core/authtoken.example.yaml` to `conf/ext/authtoken.yml` and change it)
+5. Copy `conf/core/server.conf` to `conf/ext/server.conf` and change it
appropriately:
- * Setup for MySQL back-end: Assuming that the mysql back-end is installed
- (see the `README_SETUP.md` of the `caosdb-mysqlbackend` repository),
+ * Setup for MySQL back-end:
specify the fields `MYSQL_USER_NAME`, `MYSQL_USER_PASSWORD`,
- `MYSQL_DATABASE_NAME`, and `MYSQL_HOST`.
+ `MYSQL_DATABASE_NAME`, and `MYSQL_HOST`.
* Choose the ports under which CaosDB will be accessible.
* Setup the SSL certificate: Assuming that there is an appropriate `Java Key
Store` file (see above), change the fields `CERTIFICATES_KEY_PASSWORD`,
`CERTIFICATES_KEY_STORE_PATH`, and `CERTIFICATES_KEY_STORE_PASSWORD`.
Make sure that the conf file is not readable by other users because the
certificate passwords are stored in plaintext.
+ - Set the path to the authtoken config (see step 4)
* Set the file system paths:
- `FILE_SYSTEM_ROOT`: The root for all the files managed by CaosDB.
- `DROP_OFF_BOX`: Files can be put here for insertion into CaosDB.
@@ -102,15 +108,17 @@ server:
- `SHARED_FOLDER`: Folder for sharing files via cryptographic tokens,
also those created by scripts.
- `SERVER_SIDE_SCRIPTING_BIN_DIRS`: A comma or white space separated list
- of directories (relative of absolute) where the server will be looking
+ of directories (relative or absolute) where the server will be looking
for executables which are then callable as server-side scripts. By
default this list only contains `./scripting/bin`. If you want to
include e.g. scripts which are maintained as part of the caosdb-webui
repository (because they are intended for usage by the webui), you
should add `./caosdb-webui/sss_bin/` as well.
+ - `INSERT_FILES_IN_DIR_ALLOWED_DIRS`: add mounted filesystems here that
+ shall be accessible by CaosDB
* Maybe set another `SESSION_TIMEOUT_MS`.
* See also [README_CONFIGURATION.md](README_CONFIGURATION.md)
-4. Copy `conf/core/usersources.ini.template` to `conf/ext/usersources.ini`.
+6. Copy `conf/core/usersources.ini.template` to `conf/ext/usersources.ini`.
* Define the users/groups who you want to include/exclude.
* Assign at least one user the `administration` role.
* For example, if the admin user is called `caosdb`, there should be the
@@ -123,11 +131,12 @@ server:
Especially that there are no `properties` (aka `keys`) without a
`value`. An emtpy value can be represented by `""`. Comments are
everything from `#` or `;` to the end of the line.
-5. Install the pam caller in `misc/pam_authentication/`. See
- [the pam authentication README](misc/pam_authentication/README.md)
+7. Possibly install the PAM caller in `misc/pam_authentication/` if you have
+ not do so already. See above.
+
Done!
-# Start Server
+## Start Server
`$ make run`
@@ -138,12 +147,12 @@ type `https://localhost:10443` in your Browser, assuming you used 10443 as port.
Note, that you will get a security warning if you are using a self-signed
certificate.
-# Run Unit Tests
+## Run Unit Tests
`$ make test`
-# Setup Eclipse
+## Setup Eclipse
1. Open Eclipse (recommended version: Oxygen.1a Release (4.7.1a))
2. `File > New > Java Project`: Choose a project name and specify the location
@@ -162,9 +171,9 @@ certificate.
Done!
-# Migration
+## Migration
-## From 0.1 to 0.2
+### From 0.1 to 0.2
A major change in the code is the renaming of the java packages (from
`caosdb.[...]` to `org.caosdb.[...]`).
@@ -179,3 +188,21 @@ before you execute it.
```sh
sed -i.bak -e "s/\(\s*\)\([^.]\)caosdb\.server/\1\2org.caosdb.server/g" FILE_TO_BE_CHANGED
```
+
+## Build the documentation #
+
+Stand-alone documentation is built using Sphinx: `make doc`
+
+### Requirements ##
+
+- sphinx
+- javasphinx :: `pip3 install --user javasphinx`
+ - Alternative, if javasphinx fails because python3-sphinx is too recent:
+ (`l_` not found):
+
+```sh
+git clone git@github.com:simgrid/javasphinx.git
+cd javasphinx
+git checkout 659209069603a
+pip3 install .
+```
diff --git a/conf/core/cache.ccf b/conf/core/cache.ccf
index ad73cb86d259a1de8ecd836f8e72de8aa254793e..b6a50bee08fdd8598dac3c9f3d7aa70f43190127 100644
--- a/conf/core/cache.ccf
+++ b/conf/core/cache.ccf
@@ -1,3 +1,8 @@
+# -*- mode:conf-javaprop; -*-
+
+# Configuration for the Java Caching System (JCS) which is used by the server. Please look at
+# http://commons.apache.org/proper/commons-jcs/getting_started/intro.html for further information.
+
# default caching options
jcs.default.cacheattributes=org.apache.commons.jcs.engine.CompositeCacheAttributes
jcs.default.cacheattributes.MaxObjects=1000
diff --git a/conf/core/global_entity_permissions.xml b/conf/core/global_entity_permissions.xml
index 3cebb79bd1c40564c99219519aa92b59a92e9dcd..3b0cf1b5ccb3f50c73f7a0ad8f4a2651c2dad221 100644
--- a/conf/core/global_entity_permissions.xml
+++ b/conf/core/global_entity_permissions.xml
@@ -1,4 +1,10 @@
<globalPermissions>
+ <!--
+ 4-store for permissions, implemented as a mapping:
+ {Grant/Deny, priority, role} -> List of permissions
+
+ Please look at the permission documentation for more information.
+ -->
<Grant priority="false" role="?OWNER?"><Permission name="*"/></Grant>
<Grant priority="false" role="?OTHER?"><Permission name="RETRIEVE:*"/></Grant>
<Grant priority="false" role="?OTHER?"><Permission name="USE:*"/></Grant>
diff --git a/conf/core/log4j2-debug.properties b/conf/core/log4j2-debug.properties
index 40ffe0fc02c16eb3c80a9216f58b317faecc41dc..3ae3ed266ef865fcffde45c21458650dc5ccc0c1 100644
--- a/conf/core/log4j2-debug.properties
+++ b/conf/core/log4j2-debug.properties
@@ -1,3 +1,5 @@
+# This log4j2-debug.properties file is only loaded when the server runs in debug mode.
+
# override location of log files for debugging and testing
property.LOG_DIR = testlog
diff --git a/conf/core/log4j2-default.properties b/conf/core/log4j2-default.properties
index b1697a73fe6703850865406e1441947614d00f47..974ce34df0f8290d763bf6a993554dab0ec7eeb2 100644
--- a/conf/core/log4j2-default.properties
+++ b/conf/core/log4j2-default.properties
@@ -1,3 +1,6 @@
+# This log4j2-default.properties file describes the logging settings. See
+# https://logging.apache.org/log4j/2.x/ for more information.
+
name = base_configuration
status = TRACE
verbose = true
diff --git a/conf/core/server.conf b/conf/core/server.conf
index 81b35a2a7498b0ad0f6feae7a20ce44dbe570865..2c9a50ced2600d837e933742e715624eeada6a4b 100644
--- a/conf/core/server.conf
+++ b/conf/core/server.conf
@@ -121,7 +121,7 @@ SESSION_TIMEOUT_MS=600000
# 7days
ONE_TIME_TOKEN_EXPIRES_MS=604800000
-# Path to config file for one time tokens, for example authtoken.yml.
+# Path to config file for one time tokens, see authtoken.example.yml.
AUTHTOKEN_CONFIG=
# Timeout after which a one-time token expires once it has been first consumed,
@@ -182,6 +182,11 @@ CHECK_ENTITY_ACL_ROLES_MODE=MUST
# part of any Entity ACL.
GLOBAL_ENTITY_PERMISSIONS_FILE=./conf/core/global_entity_permissions.xml
+# --------------------------------------------------
+# Extensions
+# --------------------------------------------------
+
+# If set to true, versioning of entities' history is enabled.
ENTITY_VERSIONING_ENABLED=true
diff --git a/conf/core/usersources.ini.template b/conf/core/usersources.ini.template
index 270a4ad069305ef47b7ad46b36bd69673860fb97..9053093eba21af0adcc5846112932a8a84b8250c 100644
--- a/conf/core/usersources.ini.template
+++ b/conf/core/usersources.ini.template
@@ -1,4 +1,5 @@
-#
+# -*- mode:conf; -*-
+
# ** header v3.0
# This file is a part of the CaosDB Project.
#
@@ -20,17 +21,30 @@
#
# ** end header
#
+
+# `realms` is a comma and/or space separated list of realms which users can use for authentication
realms = PAM
+
+# This is the default realm, to be used when no other realms is specified
defaultRealm = PAM
+# Each realm has one section with specific options. The options for a specific realm can be looked
+# up in that realm's documentation.
+#
+# Hint: Realms are implemented by classes which are typically in the
+# org.caosdb.server.accessControl.Pam package and implement the UserSource interface.
+
+# Options for authentication against Linux' PAM.
[PAM]
class = org.caosdb.server.accessControl.Pam
+# The script which does the actual checking.
; pam_script = ./misc/pam_authentication/pam_authentication.sh
default_status = ACTIVE
+# Only users which fulfill these criteria are accepted.
;include.user = [uncomment and put your users here]
;include.group = [uncomment and put your groups here]
;exclude.user = [uncomment and put excluded users here]
;exclude.group = [uncomment and put excluded groups here]
-;it is necessary to add at least one admin
+# It is typically necessary to add at least one admin
;user.[uncomment a set a username here].roles = administration
diff --git a/src/doc/CaosDB-Query-Language.md b/src/doc/CaosDB-Query-Language.md
new file mode 100644
index 0000000000000000000000000000000000000000..47a890d716564eb72c70de4f9d39d1cd00d92157
--- /dev/null
+++ b/src/doc/CaosDB-Query-Language.md
@@ -0,0 +1,377 @@
+# CaosDB Query Language
+**WIP This is going to be the specification. CQL tutorials are in the webui**
+
+## Example queries
+
+### Simple FIND Query
+The following query will return any entity which has the name _ename_ and all its children.
+`FIND ename`
+
+The following queries are equivalent and will return any entity which has the name _ename_ and all its children, but only if they are genuin records. Of course, the returned set of entities (henceforth referred to as _resultset_) can also be restricted to recordtypes, properties and files.
+
+`FIND RECORD ename`
+
+`FIND RECORDS ename`
+
+Wildcards use `*` for any characters or none at all. Wildcards for single characters (like the '_' wildcard from mysql) are not implemented yet.
+
+`FIND RECORD en*` returns any entity which has a name beginning with _en_.
+
+Regular expressions must be surrounded by _<<_ and '>>':
+
+`FIND RECORD <<e[aemn]{2,5}>>`
+
+`FIND RECORD <<[cC]am_[0-9]*>>`
+
+*TODO* (Timm):
+Describe escape sequences like `\\ `, `\*`, `\<<` and `\>>`.
+
+Currently, wildcards and regular expressions are only available for the _simple-find-part_ of the query, i. e. no wildcards/regexps for filters.
+
+### Simple COUNT Query
+
+This query counts entities which have certain properties.
+
+`COUNT ename`
+will return the number of entities which have the name _ename_ and all their children.
+
+The syntax of the COUNT queries is equivalent to the FIND queries in any respect (this also applies to wildcards and regular expressions) but one: The prefix is to be `COUNT` instead of `FIND`.
+
+Unlike the FIND queries, the COUNT queries do not return any entities. The result of the query is the number of entities which _would be_ returned if the query was a FIND query.
+
+## Filters
+
+### POV - Property-Operator-Value
+
+The following queries are equivalent and will restrict the result set to entities which have a property named _pname1_ that has a value _val1_.
+
+`FIND ename.pname1=val1`
+
+`FIND ename WITH pname1=val1`
+
+`FIND ename WHICH HAS A PROPERTY pname1=val1`
+
+`FIND ename WHICH HAS A pname1=val1`
+
+Again, the resultset can be restricted to records:
+
+`FIND RECORD ename WHICH HAS A pname1=val1`
+
+_currently known operators:_ `=, !=, <=, <, >=, >` (and cf. next paragraphes!)
+
+#### Special Operator: LIKE
+
+The _LIKE_ can be used with wildcards. The `*` is a wildcard for any (possibly empty) sequence of characters. Examples:
+
+`FIND RECORD ename WHICH HAS A pname1 LIKE va*`
+
+`FIND RECORD ename WHICH HAS A pname1 LIKE va*1`
+
+`FIND RECORD ename WHICH HAS A pname1 LIKE *al1`
+
+_Note:_ The _LIKE_ operator is will only produce expectable results with text properties.
+
+#### Special Case: References
+
+In general a reference can be addressed just like a POV filter. So
+
+`FIND ename1.pname1=ename2`
+
+will also return any entity named _ename1_ which references the entity with name or id _ename2_ via a reference property named _pname1_. However, it will also return any entity with a text property of that name with the string value _ename2_. In order to restrict the result set to reference properties one may make use of special reference operators:
+
+_reference operators:_ `->, REFERENCES, REFERENCE TO`
+
+
+The query looks like this:
+
+`FIND ename1 WHICH HAS A pname1 REFERENCE TO ename2`
+
+`FIND ename1 WHICH HAS A pname1->ename2`
+
+#### Time Special Case: DateTime
+
+_DateTime operators:_ `=, !=, <, >, IN, NOT IN`
+
+##### `d1=d2`: Equivalence relation.
+* ''True'' iff d1 and d2 are equal in every respect (same DateTime flavor, same fields are defined/undefined and all defined fields are equal respectively).
+* ''False'' iff they have the same DateTime flavor but have different fields defined or fields with differing values.
+* ''Undefined'' otherwise.
+
+Examples:
+* `2015-04-03=2015-04-03T00:00:00` is undefined.
+* `2015-04-03T00:00:00=2015-04-03T00:00:00.0` is undefined (second precision vs. nanosecond precision).
+* `2015-04-03T00:00:00.0=2015-04-03T00:00:00.0` is true.
+* `2015-04-03T00:00:00=2015-04-03T00:00:00` is true.
+* `2015-04=2015-05` is false.
+* `2015-04=2015-04` is true.
+
+##### `d1!=d2`: Intransitive, symmetric relation.
+* ''True'' iff `d1=d2` is false.
+* ''False'' iff `d1=d2` is true.
+* ''Undefined'' otherwise.
+
+Examples:
+* `2015-04-03!=2015-04-03T00:00:00` is undefined.
+* `2015-04-03T00:00:00!=2015-04-03T00:00:00.0` is undefined.
+* `2015-04-03T00:00:00.0!=2015-04-03T00:00:00.0` is false.
+* `2015-04-03T00:00:00!=2015-04-03T00:00:00` is false.
+* `2015-04!=2015-05` is true.
+* `2015-04!=2015-04` is false.
+
+##### `d1>d2`: Transitive, non-symmetric relation.
+Semantics depend on the flavors of d1 and d2. If both are...
+###### [UTCDateTime](Datatype#datetime)
+* ''True'' iff the time of d1 is after the the time of d2 according to [https://en.wikipedia.org/wiki/Coordinated_Universal_Time](UTC)
+* ''False'' otherwise.
+
+###### [SemiCompleteDateTime](Datatype#datetime)
+* ''True'' iff `d1.ILB>d2.EUB` is true or `d1.ILB=d2.EUB` is true.
+* ''False'' iff `d1.EUB<d2.ILB}} is true or {{{d1.EUB=d2.ILB` is true.
+* ''Undefined'' otherwise.
+
+Examples:
+* `2015>2014` is true.
+* `2015-04>2014` is true.
+* `2015-01-01T20:15.00>2015-01-01T20:14` is true.
+* `2015-04>2015` is undefined.
+* `2015>2015-04` is undefined.
+* `2015-01-01T20:15>2015-01-01T20:15:15` is undefined.
+* `2014>2015` is false.
+* `2014-04>2015` is false.
+* `2014-01-01>2015-01-01T20:15:30` is false.
+
+##### `d1<d2`: Transitive, non-symmetric relation.
+Semantics depend on the flavors of d1 and d2. If both are...
+###### [UTCDateTime](Datatype#datetime)
+* ''True'' iff the time of d1 is before the the time of d2 according to [UTC](https://en.wikipedia.org/wiki/Coordinated_Universal_Time)
+* ''False'' otherwise.
+
+###### [SemiCompleteDateTime](Datatype#datetime)
+* ''True'' iff `d1.EUB<d2.ILB` is true or `d1.EUB=d2.ILB` is true.
+* ''False'' iff `d1.ILB>d2.EUB}} is true or {{{d1.ILB=d2.EUB` is true.
+* ''Undefined'' otherwise.
+
+Examples:
+* `2014<2015` is true.
+* `2014-04<2015` is true.
+* `2014-01-01<2015-01-01T20:15:30` is true.
+* `2015-04<2015` is undefined.
+* `2015<2015-04` is undefined.
+* `2015-01-01T20:15<2015-01-01T20:15:15` is undefined.
+* `2015<2014` is false.
+* `2015-04<2014` is false.
+* `2015-01-01T20:15.00<2015-01-01T20:14` is false.
+
+##### `d1 IN d2`: Transitive, non-symmetric relation.
+Semantics depend on the flavors of d1 and d2. If both are...
+###### [SemiCompleteDateTime](Datatype#datetime)
+* ''True'' iff (`d1.ILB>d2.ILB` is true or `d1.ILB=d2.ILB` is true) and (`d1.EUB<d2.EUB` is true or `d1.EUB=d2.EUB` is true).
+* ''False'' otherwise.
+
+Examples:
+* `2015-01-01 IN 2015` is true.
+* `2015-01-01T20:15:30 IN 2015-01-01` is true.
+* `2015-01-01T20:15:30 IN 2015-01-01T20:15:30` is true.
+* `2015 IN 2015-01-01` is false.
+* `2015-01-01 IN 2015-01-01T20:15:30` is false.
+
+##### `d1 NOT IN d2`: Non-symmetric relation.
+Semantics depend on the flavors of d1 and d2. If both are...
+###### [SemiCompleteDateTime](Datatype#datetime)
+* ''True'' iff `d1.ILB IN d2.ILB` is false.
+* ''False'' otherwise.
+
+Examples:
+* `2015 NOT IN 2015-01-01` is true.
+* `2015-01-01 NOT IN 2015-01-01T20:15:30` is true.
+* `2015-01-01 NOT IN 2015` is false.
+* `2015-01-01T20:15:30 NOT IN 2015-01-01` is false.
+* `2015-01-01T20:15:30 NOT IN 2015-01-01T20:15:30` is false.
+
+##### Note
+These semantics follow a three-valued logic with ''true'', ''false'' and ''undefined'' as truth values. Only ''true'' is truth preserving. I.e. only those expressions which evaluate to ''true'' pass the POV filter. `FIND ... WHICH HAS A somedate=2015-01` only returns entities for which `somedate=2015-01` is true. On the other hand, `FIND ... WHICH DOESN'T HAVE A somedate=2015-01` returns entities for which `somedate=2015-01` is false or undefined. Shortly put, `NOT d1=d2` is not equivalent to `d1!=d2`. The latter assertion is stronger.
+
+#### Omitting the Property or the Value
+
+One doesn't have to specify the property or the value at all. The following query filters the result set for entities which have any property with a value greater than _val1_.
+
+`FIND ename WHICH HAS A PROPERTY > val1`
+
+`FIND ename . > val1`
+
+`FIND ename.>val1`
+
+
+And for references...
+
+`FIND ename1 WHICH HAS A REFERENCE TO ename2`
+
+`FIND ename1 WHICH REFERENCES ename2`
+
+`FIND ename1 . -> ename2`
+
+`FIND ename1.->ename2`
+
+
+The following query returns entities which have a _pname1_ property with any value.
+
+`FIND ename WHICH HAS A PROPERTY pname1`
+
+`FIND ename WHICH HAS A pname1`
+
+`FIND ename WITH pname1`
+
+`FIND ename . pname1`
+
+`FIND ename.pname1`
+
+### TransactionFilter
+
+*Definition*
+
+ sugar:: `HAS BEEN` | `HAVE BEEN` | `HAD BEEN` | `WAS` | `IS` |
+
+ negated_sugar:: `HAS NOT BEEN` | `HASN'T BEEN` | `WAS NOT` | `WASN'T` | `IS NOT` | `ISN'T` | `HAVN'T BEEN` |
+`HAVE NOT BEEN` | `HADN'T BEEN` | `HAD NOT BEEN`
+
+ by_clause:: `BY (ME | username | SOMEONE ELSE (BUT ME)? | SOMEONE ELSE BUT username)`
+
+ date:: A date string of the form `YYYY-MM-DD`
+
+ datetime:: A datetime string of the form `YYYY-MM-DD hh:mm:ss`
+
+ time_clause:: `ON ($date|$datetime) ` Here is plenty of room for more syntactic sugar, e.g. a `TODAY` keyword, and more funcionality, e.g. ranges.
+
+`FIND ename WHICH ($sugar|$negated_sugar)? (NOT)? (CREATED|INSERTED|UPDATED|DELETED) (by_clause time_clause?| time_clause by_clause?)`
+
+*Examples*
+
+`FIND ename WHICH HAS BEEN CREATED BY ME ON 2014-12-24`
+
+`FIND ename WHICH HAS BEEN CREATED BY SOMEONE ELSE ON 2014-12-24`
+
+`FIND ename WHICH HAS BEEN CREATED BY erwin ON 2014-12-24`
+
+`FIND ename WHICH HAS BEEN CREATED BY SOMEONE ELSE BUT erwin ON 2014-12-24`
+
+`FIND ename WHICH HAS BEEN CREATED BY erwin`
+
+`FIND ename . CREATED BY erwin ON `
+
+
+
+### File Location
+
+Search for file objects by their location:
+
+`FIND FILE WHICH IS STORED AT a/certain/path/`
+
+#### Wildcards
+
+_STORED AT_ can be used with wildcards similar to unix wildcards.
+ * `*` matches any characters or none at all, but not the directory separator `/`
+ * `**` matches any character or none at all.
+ * A leading `*` is a shortcut for `/**`
+ * Asterisks directly between two other asterisks are ignored: `***` is the same as `**`.
+ * Escape character: `\` (E.g. `\\` is a literal backslash. `\*` is a literal star. But `\\*` is a literal backslash followed by a wildcard.)
+
+Examples:
+
+Find any files ending with `.acq`:
+`FIND FILE WHICH IS STORED AT *.acq` or
+`FIND FILE WHICH IS STORED AT **.acq` or
+`FIND FILE WHICH IS STORED AT /**.acq`
+
+Find files stored one directory below `/data/`, ending with `.acq`:
+`FIND FILE WHICH IS STORED AT /data/*/*.acq`
+
+Find files stored in `/data/`, ending with `.acq`:
+`FIND FILE WHICH IS STORED AT /data/*.acq`
+
+Find files stored in a directory at any depth in the tree below `/data/`, ending with `.acq`:
+`FIND FILE WHICH IS STORED AT /data/**.acq`
+
+Find any file in a directory which begins with `2016-02`:
+`FIND FILE WHICH IS STORED AT */2016-02*/*`
+
+
+### Back References
+
+The back reference filters for entities that are referenced by another entity. The following query returns entities of the type _ename1_ which are referenced by _ename2_ entities via the reference property _pname1_.
+
+* `FIND ename1 WHICH IS REFERENCED BY ename2 AS A pname1`
+* `FIND ename1 WITH @ ename2 / pname1`
+* `FIND ename1 . @ ename2 / pname1`
+
+One may omit the property specification:
+
+* `FIND ename1 WHICH IS REFERENCED BY ename2`
+* `FIND ename1 WHICH HAS A PROPERTY @ ename2`
+* `FIND ename1 WITH @ ename2`
+* `FIND ename1 . @ ename2`
+
+### Combining Filters with Propositional Logic
+
+Any result set can be filtered by logically combining POV filters or back reference filters:
+
+#### Conjunction (AND)
+
+* `FIND ename1 WHICH HAS A PROPERTY pname1=val1 AND A PROPERTY pname2=val2 AND A PROPERTY...`
+* `FIND ename1 WHICH HAS A PROPERTY pname1=val1 AND A pname2=val2 AND ...`
+* `FIND ename1 . pname1=val1 & pname2=val2 & ...`
+
+#### Disjunction (OR)
+
+* `FIND ename1 WHICH HAS A PROPERTY pname1=val1 OR A PROPERTY pname2=val2 Or A PROPERTY...`
+* `FIND ename1 WHICH HAS A PROPERTY pname1=val1 OR A pname2=val2 OR ...`
+* `FIND ename1 . pname1=val1 | pname2=val2 | ...`
+
+#### Negation (NOT)
+
+* `FIND ename1 WHICH DOES NOT HAVE A PROPERTY pname1=val1`
+* `FIND ename1 WHICH DOESN'T HAVE A pname1=val1`
+* `FIND ename1 . NOT pname2=val2`
+* `FIND ename1 . !pname2=val2`
+
+#### ... and combinations with parentheses
+
+* `FIND ename1 WHICH HAS A pname1=val1 AND DOESN'T HAVE A pname2<val2 AND ((WHICH HAS A pname3=val3 AND A pname4=val4) OR DOES NOT HAVE A (pname5=val5 AND pname6=val6))`
+* `FIND ename1 . pname1=val1 & !pname2<val2 & ((pname3=val3 & pname4=val4) | !(pname5=val5 & pname6=val6))`
+* `FIND ename1.pname1=val1&!pname2<val2&((pname3=val3&pname4=val4)|!(pname5=val5&pname6=val6))`
+
+### A Few Important Expressions
+
+* A:: The indistinct article. This is only syntactic suger. Equivalent expressions: `A, AN`
+* AND:: The logical _and_. Equivalent expressions: `AND, &`
+* FIND:: The beginning of the query.
+* NOT:: The logical negation. Equivalent expressions: `NOT, DOESN'T HAVE A PROPERTY, DOES NOT HAVE A PROPERTY, DOESN'T HAVE A, DOES NOT HAVE A, DOES NOT, DOESN'T, IS NOT, ISN'T, !`
+* OR:: The logical _or_. Equivalent expressions: `OR, |`
+* RECORD,RECORDTYPE,FILE,PROPERTY:: Role expression for restricting the result set to a specific role.
+* WHICH:: The marker for the beginning of the filters. Equivalent expressions: `WHICH, WHICH HAS A, WHICH HAS A PROPERTY, WHERE, WITH, .`
+* REFERENCE:: This one is tricky: `REFERENCE TO` expresses a the state of _having_ a reference property. `REFERENCED BY` expresses the state of _being_ referenced by another entity.
+* COUNT:: `COUNT` works like `FIND` but doesn't return the entities.
+
+## Versioning
+
+Since Caosdb 0.2 entities are optionally version controlled. The query language will be extended to include versioning in the future. A current minimal implementation introduces the `ANY VERSION OF` modifier which can be used to return all matching versions in the results of `COUNT`, `FIND`, and `SELECT` queries.
+
+### Example
+
+* `FIND ANY VERSION OF RECORD WITH pname=value` returns the all past and present versions of records with `pname = value`.
+
+### Scope and current limitations
+
+* The `ANY VERSION OF` modifier currently the only expression for taking the versioning into account when using the query language.
+* Subproperties are not supported yet, e.g. `FIND ANY VERSION OF ENTITY WHICH IS REFERENCED BY ename WITH ...`. This applies to all cases where you specify properties of *referenced* entities or *referencing* entities.
+
+### Future
+
+* Add `(LATEST|LAST|OLDEST|NEWEST|FIRST) VERSION OF` modifiers.
+* Add `(ANY|LATEST|LAST|OLDEST|NEWEST|FIRST) VERSION (BEFORE|AFTER) (<timestamp>|<transaction id>|<entity@version>) OF` modifier.
+* Add support for subproperties, e.g. `FIND ANY VERSION OF ENTITY WHICH IS REFERENCED BY ename WITH ...`.
+
+## Future
+
+ * *Sub Queries* (or *Sub Properties*): `FIND ename WHICH HAS A pname WHICH HAS A subpname=val`. This is like: `FIND AN experiment WHICH HAS A camera WHICH HAS A 'serial number'= 1234567890`
+ * *More Logic*, especially `ANY`, `ALL`, `NONE`, and `SUCH THAT` key words (and equivalents) for logical quantisation: `FIND ename1 SUCH THAT ALL ename2 WHICH HAVE A REFERENCE TO ename1 HAVE A pname=val`. This is like `FIND experiment SUCH THAT ALL person WHICH ARE REFERENCED BY THIS experiment AS conductor HAVE AN 'academic title'=professor.`
+
diff --git a/src/doc/Makefile b/src/doc/Makefile
new file mode 100644
index 0000000000000000000000000000000000000000..4f177fd3850423c3005829a75a2b37625dc68a87
--- /dev/null
+++ b/src/doc/Makefile
@@ -0,0 +1,47 @@
+# ** header v3.0
+# This file is a part of the CaosDB Project.
+#
+# Copyright (C) 2020 IndiScale GmbH <info@indiscale.com>
+# Copyright (C) 2020 Daniel Hornung <d.hornung@indiscale.com>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see <https://www.gnu.org/licenses/>.
+#
+# ** end header
+
+# This Makefile is a wrapper for sphinx scripts.
+#
+# It is based upon the autocreated makefile for Sphinx documentation.
+
+# You can set these variables from the command line, and also
+# from the environment for the first two.
+SPHINXOPTS ?= -a
+SPHINXBUILD ?= sphinx-build
+SPHINXAPIDOC ?= javasphinx-apidoc
+SOURCEDIR = .
+BUILDDIR = ../../build/doc
+
+.PHONY: doc-help Makefile apidoc
+
+# Put it first so that "make" without argument is like "make help".
+doc-help:
+ @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
+
+# Catch-all target: route all unknown targets to Sphinx using the new
+# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS).
+%: Makefile apidoc
+ @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
+# sphinx-build -M html . ../../build/doc
+
+apidoc:
+ @$(SPHINXAPIDOC) -o _apidoc --update --title="CaosDB Server" ../main/
diff --git a/src/doc/README_SETUP.md b/src/doc/README_SETUP.md
new file mode 120000
index 0000000000000000000000000000000000000000..88332e357f5e06f3de522768ccdcd9e513c15f62
--- /dev/null
+++ b/src/doc/README_SETUP.md
@@ -0,0 +1 @@
+../../README_SETUP.md
\ No newline at end of file
diff --git a/src/doc/administration/configuration.rst b/src/doc/administration/configuration.rst
new file mode 100644
index 0000000000000000000000000000000000000000..4170fd0ed31c7a0ec5c1b4e589ff919267cf7de9
--- /dev/null
+++ b/src/doc/administration/configuration.rst
@@ -0,0 +1,62 @@
+Configuration
+=============
+
+The server is configured through configuration files. There are two directories with config files:
+
+``conf/core``
+ Upstream defaults are stored here.
+``conf/ext``
+ User specific configuration should be stored here, settings in ``ext`` override settings in
+ ``core``. Additionally, configuration files may be stored in ``*.d`` directories here, named
+ after the original config file name. For example, the general server configuration will be
+ assembled from ``conf/core/server.conf``, ``conf/ext/server.conf`` and any ``*.conf`` files found
+ in ``conf/ext/server.conf.d``.
+
+Configuration files
+-------------------
+
+In each of these directories, the server looks for the following files:
+
+``server.conf``
+ General server configuration options. The possible configuration options are documented inside
+ the `default file <https://gitlab.com/caosdb/caosdb-server/-/blob/dev/conf/core/server.conf>`_.
+
+``global_entity_permissions.xml``
+ :ref:`Permissions <concepts:Permissions>` which are automatically set, based on user roles.
+
+``usersources.ini``
+ This file defines possible sources which are checked when a user tries to authenticate. Each
+ defined source has a special section, the possible options are defined separately for each user
+ source. At the moment the best place to look for this specific documentation is at the API
+ documentation of :java:type:`UserSource` and its implementing classes. The provided `template
+ file <https://gitlab.com/caosdb/caosdb-server/-/blob/dev/conf/core/usersources.ini.template>`_
+ also has some information. The general concept about authentication realms is described in
+ :java:type:`UserSources`.
+
+``authtoken.yaml``
+ Configuration for dispensed authentication tokens, which can be used to authenticate to CaosDB
+ without the need of a user/password combination. Possible use cases are server-side scripts or
+ initial setup after the server start. There is more documentation inside the `template file
+ <https://gitlab.com/caosdb/caosdb-server/-/blob/dev/conf/core/authtoken.example.yaml>`_.
+
+``cache.ccf``
+ Configuration for the Java Caching System (JCS) which can be used by the server. More
+ documentation is `upstream
+ <http://commons.apache.org/proper/commons-jcs/getting_started/intro.html>`_ and inside `the file
+ <https://gitlab.com/caosdb/caosdb-server/-/blob/dev/conf/core/cache.ccf>`_.
+
+``log4j2-default.properties``, ``log4j2-debug.properties``
+ Configuration for logging, following the standard described by the `log4j library
+ <https://logging.apache.org/log4j/2.x/>`_. The ``default`` file is always loaded, in debug mode
+ the ``debug`` file iss added as well.
+
+
+Changing the configuration at runtime
+-------------------------------------
+
+Remark:
+ Only when the server is in debug mode, the general configuration can be changed at runtime.
+
+In the debug case, the server provides the ``_server_properties`` resource which allows the ``GET``
+and ``POST`` methods to access the server's properties. The Python client library conveniently
+wraps this in the :any:`caosdb-pylib:caosdb.common.administration` module.
diff --git a/src/doc/administration/maintenance.rst b/src/doc/administration/maintenance.rst
new file mode 100644
index 0000000000000000000000000000000000000000..8a3397614b409a4ad498d0d8898c79cd0eb69bd6
--- /dev/null
+++ b/src/doc/administration/maintenance.rst
@@ -0,0 +1,59 @@
+
+Maintenance of the CaosDB Server
+================================
+
+Creating a Backup
+-----------------
+
+In order to create a full backup of CaosDB, the state of the SQL-Backend (MySQL, MariaDB)
+has to be saved and the internal file system of CaosDB (symbolic links to
+file systems that are mounted and uploaded files) has to be saved.
+
+You find the documentation on how to backup the SQL-Backend :any:`caosdb-mysqlbackend:Maintenance`
+
+In order to save the file backend we recommend to tar the file system. However,
+you could use other backup methods that allow to restore the file system.
+The CaosDB internal file system is located at the path defined by the
+``FILE_SYSTEM_ROOT`` configuration variable (see :any:`configuration`).
+
+The command could look like::
+
+ tar czvf /path/to/new/backup /path/to/caosdb/filesystem.tar.gz
+
+
+You can also save the content of CaosDB using XML. This is **not recommended** since it is less reliable than a real SQL backup. However there may be cases in which an XML backup is desirable, e.g., when transferring entities between two different CaosDB instances.
+Collect the entities that you want to export in a :any:`caosdb-pylib:caosdb.common.models.Container`, here
+named ``cont``. Then you can export the XML with::
+
+ from caosadvancedtools.export_related import invert_ids
+ from lxml import etree
+ invert_ids(cont)
+ xml = etree.tounicode(cont.to_xml(
+ local_serialization=True), pretty_print=True)
+
+ with open("caosdb_data.xml"), "w") as fi:
+ fi.write(xml)
+
+
+Restoring a Backup
+------------------
+
+.. note :
+ CaosDB should be offline before restoring data.
+
+If you want to restore the internal file system, simply replace it. E.g. if your
+Backup is a tarball::
+
+ tar xvf /path/to/caosroot.tar.gz
+
+
+You find the documentation on how to restore the data in the SQL-Backend :any:`caosdb-mysqlbackend:Maintenance`
+
+
+If you want to restore the entities exported to XML, you can do::
+
+ cont = db.Container()
+ with open("caosdb_data.xml") as fi:
+ cont = cont.from_xml(fi.read())
+ cont.insert()
+
diff --git a/src/doc/administration/server_side_scripting.rst b/src/doc/administration/server_side_scripting.rst
new file mode 100644
index 0000000000000000000000000000000000000000..59bce98682c6d1190fc90681fc8e14fd483e891e
--- /dev/null
+++ b/src/doc/administration/server_side_scripting.rst
@@ -0,0 +1,85 @@
+Server-Side Scripting
+=====================
+
+Introduction
+------------
+
+Small computation task, like some visualization, might be easily implemented in Python or some other language, but cumbersome to integrate into the server. Furthermore, the CaosDB server should stay a general tool without burden from specific projects. Also, one might want to trigger some standardized processing task from the web interface for convenience. For these situations the "server side scripting" is intended.
+
+Concepts
+------------
+
+The basic idea is that a script or program (script in the following) can be called to run on the server (or elsewhere in future) to do some calculations. This triggering of the script is done over the API so it can be done with any client. Input arguments can be passed to the script and the STDOUT and STDERR are returned.
+
+Each script is executed in a temporary home directory, which is automatically clean up. However, scripts can store files in the "$SHARED" folder and for example provide users a link that allows them to download files.
+
+Write and Install a Script
+--------------------------
+
+A server-side script must accept at least the ``--auth-token=AUTH_TOKEN`` option. All other command-line parameters which are passed to the script are not specified by the API and maybe defined by the script itself.
+
+So a minimal bash script would be
+
+.. code-block:: sh
+
+ #!/bin/bash
+ echo Hello, World!
+
+thereby just ignoring the ``--auth-token`` option.
+
+The script has to be executable and must be placed somewhere in one of the directory trees which are configured by the server config :doc:`SERVER_SIDE_SCRIPTING_BIN_DIRS <configuration>`.
+
+Users will need the ``SCRIPTING:EXECUTE:path:to:the:script`` permission. Here the path to the script is of course relativet to the ``SERVER_SIDE_SCRIPTING_BIN_DIRS`` where it is located.
+
+For more information see the :doc:`specification of the API <../specs/Server-side-scripting>`
+
+Environment
+------------
+
+The script is called with several special environment variables to accommodate
+for its special location.
+
+`HOME`
+^^^^^^^^^^^^
+To be able to run with reduced privileges, the script has its `HOME` environment
+variable set to a special directory with write access. This directory will be
+deleted after the script has terminated. Its content is freshly copied for each
+script invocation from a skeleton directory, located in the server directory, in
+`scripting/home/`. By default, this directory contains the following:
+
+- `readme.md` :: A small text file describing the purpose of the directory.
+
+Users of CaosDB are invited to populate the directory with whatever their
+scripts need.
+
+Invocation
+------------
+
+Server side scripts are triggered by sending a POST to the `/scripting` resource. There are the following arguments that can be provided:
+
+- `call`: the name of the script to be called
+- `-pN`: positional arguments (e.g. `-p0`, `-p1` etc.)
+- `-ONAME`: named arguments (e.g. `-Otest`, `-Onumber` etc.)
+
+The arguments will be passed to the script.
+
+An invocation via a button in javascript could look like:
+
+.. code-block:: javascript
+
+ var _make_sss_button = function (entity) {
+ const script = "script.py";
+
+ const scripting_form = $(`
+ <form class="btn-group-xs ${_css_class_export_button}"
+ method="POST"
+ action="/scripting">
+ <input type="hidden" name="call" value="${script}"/>
+ <input name="-p0" value=""/>
+ <button type="submit" class="btn btn-link">Start script</button>
+ </form>`);
+ return scripting_form[0];
+ }
+
+For more information see the :doc:`specification of the API <../specs/Server-side-scripting>`
+
diff --git a/src/doc/concepts.rst b/src/doc/concepts.rst
new file mode 100644
index 0000000000000000000000000000000000000000..ec93719548a7d7c577ac62c6615b3fcf24111b42
--- /dev/null
+++ b/src/doc/concepts.rst
@@ -0,0 +1,29 @@
+===================================
+Basic concepts of the CaosDB server
+===================================
+
+The CaosDB server provides the HTTP API resources to users and client libraries. It uses a plain
+MariaDB/MySQL database as backend for data storage, raw files are stored separately on the file
+system.
+
+
+Configuration
+-------------
+
+Administrators may configure the server through :doc:`configuration
+files<administration/configuration>`. Additionally, configurations may be set via the API if the
+server is in debug mode.
+
+
+Permissions
+-----------
+
+CaosDB has a fine grained role based permission system. Each interaction with the server is
+governed by the current rules of the user, by default this is the ``anonymous`` role. The
+permissions for an action which involves one or more objects are set either manually or via default
+permissions which can be configured. For more detailed information, there is separate
+:doc:`documentation of the permission system<permissions>`.
+
+
+
+
diff --git a/src/doc/conf.py b/src/doc/conf.py
new file mode 100644
index 0000000000000000000000000000000000000000..84e676394d19e8ec35f241f1cf3cc202dafa0d4d
--- /dev/null
+++ b/src/doc/conf.py
@@ -0,0 +1,214 @@
+# -*- coding: utf-8 -*-
+#
+# Configuration file for the Sphinx documentation builder.
+#
+# This file only contains a selection of the most common options. For a full
+# list see the documentation:
+# https://www.sphinx-doc.org/en/master/usage/configuration.html
+
+# -- Path setup --------------------------------------------------------------
+
+# If extensions (or modules to document with autodoc) are in another directory,
+# add these directories to sys.path here. If the directory is relative to the
+# documentation root, use os.path.abspath to make it absolute, like shown here.
+#
+# import os
+# import sys
+# sys.path.insert(0, os.path.abspath('../caosdb'))
+
+import sphinx_rtd_theme
+
+# -- Project information -----------------------------------------------------
+
+project = 'caosdb-server'
+copyright = '2020, IndiScale GmbH'
+author = 'Daniel Hornung'
+
+# The short X.Y version
+version = '0.2'
+# The full version, including alpha/beta/rc tags
+release = '0.2'
+
+
+# -- General configuration ---------------------------------------------------
+
+# If your documentation needs a minimal Sphinx version, state it here.
+#
+# needs_sphinx = '1.0'
+
+# Add any Sphinx extension module names here, as strings. They can be
+# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
+# ones.
+extensions = [
+ 'javasphinx',
+ 'sphinx.ext.autodoc',
+ 'sphinx.ext.intersphinx',
+ 'sphinx.ext.napoleon', # For Google style docstrings
+ "recommonmark", # For markdown files.
+ "sphinx.ext.autosectionlabel", # Allow reference sections using its title
+ "sphinx_rtd_theme",
+]
+
+# Add any paths that contain templates here, relative to this directory.
+templates_path = ['_templates']
+
+# The suffix(es) of source filenames.
+# You can specify multiple suffix as a list of string:
+#
+# source_suffix = ['.rst', '.md']
+source_suffix = '.rst'
+
+# The master toctree document.
+master_doc = 'index'
+
+# The language for content autogenerated by Sphinx. Refer to documentation
+# for a list of supported languages.
+#
+# This is also used if you do content translation via gettext catalogs.
+# Usually you set "language" from the command line for these cases.
+language = None
+
+# List of patterns, relative to source directory, that match files and
+# directories to ignore when looking for source files.
+# This pattern also affects html_static_path and html_extra_path.
+exclude_patterns = []
+
+# The name of the Pygments (syntax highlighting) style to use.
+pygments_style = None
+
+
+# -- Options for HTML output -------------------------------------------------
+
+# The theme to use for HTML and HTML Help pages. See the documentation for
+# a list of builtin themes.
+#
+html_theme = 'sphinx_rtd_theme'
+
+# Theme options are theme-specific and customize the look and feel of a theme
+# further. For a list of options available for each theme, see the
+# documentation.
+#
+# html_theme_options = {}
+
+# Add any paths that contain custom static files (such as style sheets) here,
+# relative to this directory. They are copied after the builtin static files,
+# so a file named "default.css" will overwrite the builtin "default.css".
+# html_static_path = ['_static']
+
+# Custom sidebar templates, must be a dictionary that maps document names
+# to template names.
+#
+# The default sidebars (for documents that don't match any pattern) are
+# defined by theme itself. Builtin themes are using these templates by
+# default: ``['localtoc.html', 'relations.html', 'sourcelink.html',
+# 'searchbox.html']``.
+#
+# html_sidebars = {}
+
+
+# -- Options for HTMLHelp output ---------------------------------------------
+
+# Output file base name for HTML help builder.
+htmlhelp_basename = 'caosdb-serverdoc'
+
+
+# -- Options for LaTeX output ------------------------------------------------
+
+latex_elements = {
+ # The paper size ('letterpaper' or 'a4paper').
+ #
+ # 'papersize': 'letterpaper',
+
+ # The font size ('10pt', '11pt' or '12pt').
+ #
+ # 'pointsize': '10pt',
+
+ # Additional stuff for the LaTeX preamble.
+ #
+ # 'preamble': '',
+
+ # Latex figure (float) alignment
+ #
+ # 'figure_align': 'htbp',
+}
+
+# Grouping the document tree into LaTeX files. List of tuples
+# (source start file, target name, title,
+# author, documentclass [howto, manual, or own class]).
+latex_documents = [
+ (master_doc, 'caosdb-server.tex', 'caosdb-server Documentation',
+ 'IndiScale GmbH', 'manual'),
+]
+
+
+# -- Options for manual page output ------------------------------------------
+
+# One entry per manual page. List of tuples
+# (source start file, name, description, authors, manual section).
+man_pages = [
+ (master_doc, 'caosdb-server', 'caosdb-server Documentation',
+ [author], 1)
+]
+
+
+# -- Options for Texinfo output ----------------------------------------------
+
+# Grouping the document tree into Texinfo files. List of tuples
+# (source start file, target name, title, author,
+# dir menu entry, description, category)
+texinfo_documents = [
+ (master_doc, 'caosdb-server', 'caosdb-server Documentation',
+ author, 'caosdb-server', 'One line description of project.',
+ 'Miscellaneous'),
+]
+
+
+# -- Options for Epub output -------------------------------------------------
+
+# Bibliographic Dublin Core info.
+epub_title = project
+
+# The unique identifier of the text. This can be a ISBN number
+# or the project homepage.
+#
+# epub_identifier = ''
+
+# A unique identification for the text.
+#
+# epub_uid = ''
+
+# A list of files that should not be packed into the epub file.
+epub_exclude_files = ['search.html']
+
+
+# -- Extension configuration -------------------------------------------------
+
+# -- Options for javasphinx --------------------------------------------------
+# See also https://bronto-javasphinx.readthedocs.io/en/latest/
+
+# javadoc_url_map = {
+# '<namespace_here>' : ('<base_url_here>', 'javadoc'),
+# }
+
+
+# -- Options for intersphinx -------------------------------------------------
+
+# https://www.sphinx-doc.org/en/master/usage/extensions/intersphinx.html#confval-intersphinx_mapping
+intersphinx_mapping = {
+ "python": ("https://docs.python.org/", None),
+ "caosdb-pylib": ("https://caosdb.gitlab.io/caosdb-pylib/", None),
+ "caosdb-mysqlbackend": ("https://caosdb.gitlab.io/caosdb-mysqlbackend/", None),
+}
+
+
+# -- Options for autodoc -----------------------------------------------------
+# TODO Which options do we want?
+autodoc_default_options = {
+ 'members': None,
+ 'undoc-members': None,
+}
+
+
+# -- Options for autosectionlabel --------------------------------------------
+
+autosectionlabel_prefix_document = True
diff --git a/src/doc/development/benchmarking.md b/src/doc/development/benchmarking.md
new file mode 100644
index 0000000000000000000000000000000000000000..f2d663f6e69799c7a87a28fbdf32f15fab892ac1
--- /dev/null
+++ b/src/doc/development/benchmarking.md
@@ -0,0 +1,130 @@
+
+# Benchmarking CaosDB #
+
+Benchmarking CaosDB may encompass several distinct areas: How much time is spent in the server's
+Java code, how much time is spent inside the SQL backend, are the same costly methods clalled more
+than once? This documentation tries to answer some questions connected with these benchmarking
+aspects and give you the tools to answer your own questions.
+
+## Tools for the benchmarking ##
+
+For averaging over many runs of comparable requests and for putting the database into a
+representative state, Python scripts are used. The scripts can be found in the `caosdb-dev-tools`
+repository, located at [https://gitlab.indiscale.com/caosdb/src/caosdb-dev-tools](https://gitlab.indiscale.com/caosdb/src/caosdb-dev-tools) in the folder
+`benchmarking`:
+
+### `fill_database.py` ###
+
+This commandline script is meant for filling the database with enough data to represeny an actual
+real-life case, it can easily create hundreds of thousands of Entities.
+
+The script inserts predefined amounts of randomized Entities into the database, RecordTypes,
+Properties and Records. Each Record has a random (but with defined average) number of Properties,
+some of which may be references to other Records which have been inserted before. Actual insertion
+of the Entities into CaosDB is done in chunks of a defined size.
+
+Users can tell the script to store times needed for the insertion of each chunk into a tsv file.
+
+### `measure_execution_time.py` ###
+
+A somewhat outdated script which executes a given query a number of times and then save statistics
+about the `TransactionBenchmark` readings (see below for more information about the transaction
+benchmarks) delivered by the server.
+
+### Benchmarking SQL commands ###
+
+MariaDB and MySQL have a feature to enable the logging of SQL queries' times. This logging must be
+turned on on the SQL server as described in the [upstream documentation](https://mariadb.com/kb/en/general-query-log/). For the Docker
+environment LinkAhead, this can conveniently be done with `linkahead mysqllog {on,off,store}`.
+
+### External JVM profilers ###
+
+Additionally to the transaction benchmarks, it is possible to benchmark the server execution via
+external Java profilers. For example, [VisualVM](https://visualvm.github.io/) can connect to JVMs running locally or remotely
+(e.g. in a Docker container). To enable this in LinkAhead's Docker environment, set
+
+```yaml
+devel:
+ profiler: true
+```
+
+Most profilers, like as VisualVM, only gather cumulative data for call trees, they do not provide
+complete call graphs (as callgrind/kcachegrind would do). They also do not differentiate between
+calls with different query strings, as long as the Java process flow is the same (for example, `FIND
+Record 1234` and `FIND Record A WHICH HAS A Property B WHICH HAS A Property C>100` would be handled
+equally).
+
+## How to set up a representative database ##
+For reproducible results, it makes sense to start off with an empty database and fill it using the
+`fill_database.py` script, for example like this:
+
+```sh
+./fill_database.py -t 500 -p 700 -r 10000 -s 100 --clean
+```
+
+The `--clean` argument is not strictly necessary when the database was empty before, but it may make
+sense when there have been previous runs of the command. This example would create 500 RecordTypes,
+700 Properties and 10000 Records with randomized properties, everything is inserted in chunks of 100
+Entities.
+
+## How to measure request times ##
+
+If the execution of the Java components is of interest, the VisualVM profiler should be started and
+connected to the server before any requests to the server are started.
+
+When doing performance tests which are used for detailed analysis, it is important that
+
+1. CaosDB is in a reproducible state, which should be documented
+2. all measurements are repeated several times to account for inevitable variance in access (for
+ example file system caching, network variablity etc.)
+
+### Filling the database ###
+
+By simply adding the option `-T logfile.tsv` to the `fill_database.py` command above, the times for
+inserting the records are stored in a tsv file and can be analyzed later.
+
+### Obtain statistics about a query ###
+
+To repeat single queries a number of times, `measure_execution_time.py` can be used, for example:
+
+```sh
+./measure_execution_time.py -n 120 -q "FIND MusicalInstrument WHICH IS REFERENCED BY Analysis"
+```
+
+This command executes the query 120 times, additional arguments could even plot the
+TransactionBenchmark results directly.
+
+## What is measured ##
+
+For a consistent interpretation, the exact definitions of the measured times are as follows:
+
+### SQL logs ###
+
+As per https://mariadb.com/kb/en/general-query-log, the logs store only the time at which the SQL
+server received a query, not the duration of the query.
+
+#### Possible future enhancements ####
+
+- The `query_response_time` plugin may be additionally used in the future, see
+ https://mariadb.com/kb/en/query-response-time-plugin
+
+### Transaction benchmarks ###
+
+Transaction benchmarking manually collects timing information for each transaction. At defined
+points, different measurements can be made, accumulated and will finally be returned to the client.
+Benchmark objects may consist of sub benchmarks and have a number of measurement objects, which
+contain the actual statistics.
+
+Because transaction benchmarks must be manually added to the server code, they only monitor those
+code paths where they are added. On the other hand, their manual nature allows for a more
+abstracted analysis of performance bottlenecks.
+
+### Java profiler ###
+
+VisualVM records for each thread the call tree, specifically which methods were called how often and
+how much time was spent inside these methods.
+
+### Global requests ###
+
+Python scripts may measure the global time needed for the execution of each request.
+`fill_database.py` obtains its numbers this way.
diff --git a/src/doc/index.rst b/src/doc/index.rst
new file mode 100644
index 0000000000000000000000000000000000000000..e41d318a2dbf7323e5cb327be1c7a3fe0b5d61ad
--- /dev/null
+++ b/src/doc/index.rst
@@ -0,0 +1,29 @@
+
+Welcome to caosdb-server's documentation!
+=========================================
+
+.. toctree::
+ :maxdepth: 2
+ :caption: Contents:
+ :hidden:
+ :glob:
+
+ Getting started <README_SETUP>
+ Concepts <concepts>
+ Query Language <CaosDB-Query-Language>
+ administration/*
+ development/*
+ API documentation<_apidoc/packages>
+
+Welcome to the CaosDB, the flexible semantic data management toolkit!
+
+This documentation helps you to :doc:`get started<getting_started>`, explains the most important
+:doc:`concepts<concepts>` and offers a range of :doc:`tutorials<tutorials>`.
+
+
+Indices and tables
+==================
+
+* :ref:`genindex`
+* :ref:`modindex`
+* :ref:`search`
diff --git a/src/doc/specs/Server-side-scripting.md b/src/doc/specs/Server-side-scripting.md
new file mode 100644
index 0000000000000000000000000000000000000000..49f56af66a6b65855e36de12144955b2925a4f51
--- /dev/null
+++ b/src/doc/specs/Server-side-scripting.md
@@ -0,0 +1,187 @@
+# Server-Side Scripting API (v0.1)
+
+The CaosDB Server can execute scripts (bash/python/perl) and compiled executables. The scripts can be invoked by a remote-procedure-call (RPC) protocol. Both, the requirements for the scripts and the RPC are described in this document.
+
+## Configuration of the Server
+
+* The CaosDB Server has two relevant properties:
+ 1. `SERVER_SIDE_SCRIPTING_BIN_DIR` is the directory where executable scripts are to be placed. The server will not execute scripts which are out side of this directory. This directory must be readable and executable for the server. But it should not be readable or executable for anyone else.
+ 2. `SERVER_SIDE_SCRIPTING_WORKING_DIR` is the directory under which the server creates temporary working directories. The server needs writing, reading and executing permissions. The temporary working directories are deleted after the scripts have finished and the server has collected the results of the scripts.
+
+## Installing a Server-Side Script (SSS)
+
+* Put your script into the `SERVER_SIDE_SCRIPTING_BIN_DIR` or in any subdirectory and make it executable. Executable files below this directory are called `SSS`.
+* All other files in that directory MUST be ignored by the server, i.e. the server will never call them directly.
+ However, they MAY contain additional data, different implementations, libraries etc.
+* A symlink pointing to an executable MUST be treated as SSS, too.
+
+### Example SERVER\_SIDE\_SCRIPTING\_BIN\_DIR
+
+```
+/
++- script1.py (EXECUTABLE)
++- script2.sh (EXECUTABLE)
++- subdir1/
+ +-script3.pl (EXECUTABLE)
++- subdir2/
+ +- data_for_script2
+ +- another_script.py (EXECUTABLE)
++- script4 -> ./subdir2/another_script.py (SIMLINK to EXECUTABLE)
++- script5 -> /usr/local/bin/external (SIMLINK to EXECUTABLE)
+```
+
+The files `scripts1.py`, `scripts2.sh`, `scripts3.pl` and `another_script.py` are SSS.
+Also the `script4` can be called which would result in calling `another_script.py`
+The script5 points to an executable which is not stored in the `SSD_DIR`.
+
+## Calling a Server-Side Script Via Remote Procedure Call
+
+* Users can invoke scripts via HTTP under the uri `https://$HOST:$HTTPS_PORT/$CONTEXT_ROOT/scripting`. The server accepts POST requests with content types `application/x-www-form-urlencoded` and `multipart/form-data`.
+* There are 6 types of form fields which are processed by the server:
+ 1. A single parameter form field with name="call" (the path to script, relative to the `SERVER_SIDE_SCRIPTING_BIN_DIR`). If there are more than one fields with that name, the behavior is not defined.
+ 2. Zero or more parameter form fields with a unique name which starts with the string `-O` (command line options).
+ 3. Zero or more parameter form fields with a unique name which starts with the string `-p` (positional command line arguments).
+ 4. Zero or more file form fields which have a unique field name and a unique file name (upload files). If the field names or file names are not unique, the behavior is not defined.
+ 5. A parameter form field with name="timeout" (the request timeout)
+ 6. A parameter form field with name="auth-token" and a valid CaosDB AuthToken string or "generate" as value.
+
+## How Does the Server Call the Script?
+* The server executes the script in a temporary working directory, called *PWD* (which will be deleted afterwards).
+* The form parameter `call`, the options, the arguments and file fields construct the string which is executed in a shell. The value of *call* begins the command line.
+* For any option paramter with the name `-Ooption_name` and a value `option_value` a resulting `--option_name=option_value` is appended to the commandLine in no particular order.
+* The values of the positional arguments are appended sorted alphabetically by their field name. E.g. a parameter `-p0` with value `v0` and a parameter `-p1` with value `v1` result in appending `v0 v1` to the command line.
+* All files will be loaded into the directory `$PWD/.upload_files` with their file name (i.e. the form field property). If the file names contain slashes '/' they will build sub-directories in the `./upload_files`.
+* If a file form field has a field name which begins with either `-p` or `-O` the file name with prefix `.upload_files/` is passed as value of an option or as a positional argument to the script. Thus it is possible to distinguish several uploaded files from one another.
+* If there is a "auth-token" field present, another command line options `--auth-token=...` is appended. The value is either the string which was submitted with the POST request, or, if the value was "generate", a refreshed, valid AuthToken which authenticates as the user of the request (why? see below).
+
+### Example HTML Form
+
+```html
+<form action="/scripting" method="post" enctype="multipart/form-data">
+<input type="hidden" name="call" value="my/script.py"/>
+<input type="file" name="-Oconfig-file"/>
+<input type="file" name="-p1"/>
+<input type="text" name="-p0" value="analyze"/>
+<input type="text" name="user"/>
+<input type="text" name="-Oalgorithm" value="fast"/>
+<input type="submit" value="Submit">
+</form>
+```
+
+where the user uploads `my.conf` as `-Oconfig-file` and `my.input.tsv` as `-p1`, would result in this command line:
+
+```
+$SERVER_SIDE_SCRIPTING_BIN_DIR/my/script.py --config-file=.upload_files/my.conf --algorithm=fast analyze .upload_files/my.input.tsv
+```
+## CaosDB Server Response
+
+The CaosDB Server responds with an xml document. The root element is the usual `/Response`. If no errors occurred (which would be represented with `/Response/Error` elements) the result of the script execution is represented as a `/Response/script/` element.
+
+* It has a `code` attribute which contains the exit code value of the execution.
+* It has `stdout` and `stderr` children which contain the dump of the stdout and stderr file of the execution environment.
+* It has a `call` child which contains the command line which was executed (but without a possible `--auth-token` option and with a relative path to the executable).
+
+### Example XML Response
+
+```xml
+<Response>
+<script code="0">
+<call>my/script.py --config-file=.upload_files/my.conf --algorithm=fast analyze .upload_files/my.input.tsv</call>
+<stdout>Result: 0.5</stdout>
+<stderr>Warning: 8 Lines did not contain enough columns</stderr>
+</Response>
+```
+
+## CaosDB Clients and Authentication Token
+
+A special use case for server side scripting is the automated execution of CaosDB clients. These clients need to connect to the CaosDB Server and thus need a way to authenticate themselves.
+For this special case the server can pass an Authentication Token which can be used by the script to authenticate itself. If the invocation request send a particulare AuthToken in the form, this AuthToken will be passed to the script with the `--auth-token` option. Otherwise, if the `auth-token` field has "generate" as value, a fresh AuthToken is generated which belongs to the user who requested the script execution.
+Thus the script is executed (and connects back to the server) as the user who called the script in the first place.
+
+A CaosDB client might use the python client library to connect to the server with that AuthToken.
+
+### Example Script
+
+```python
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+#
+# ** header v3.0
+# This file is a part of the CaosDB Project.
+#
+# Copyright (C) 2018 Research Group Biomedical Physics,
+# Max-Planck-Institute for Dynamics and Self-Organization Göttingen
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see <https://www.gnu.org/licenses/>.
+#
+# ** end header
+#
+"""server_side_script.py.
+
+An example which implements a minimal server-side script.
+
+1) This script expects to find a *.txt file in the .upload_files dir which is
+printed to stdout.
+
+2) It executes a "Count stars" query and prints the result to stdout.
+
+3) It will return with code 0 if everything is ok, or with any code that is
+specified with the commandline option --exit
+"""
+
+import sys
+from os import listdir
+from caosdb import configure_connection, execute_query
+
+
+# parse --auth-token option and configure connection
+CODE = 0
+QUERY = "COUNT stars"
+for arg in sys.argv:
+ if arg.startswith("--auth-token="):
+ auth_token = arg[13:]
+ configure_connection(auth_token=auth_token)
+ if arg.startswith("--exit="):
+ CODE = int(arg[7:])
+ if arg.startswith("--query="):
+ QUERY = arg[8:]
+
+
+############################################################
+# 1 # find and print *.txt file ############################
+############################################################
+
+try:
+ for fname in listdir(".upload_files"):
+ if fname.endswith(".txt"):
+ with open(".upload_files/{}".format(fname)) as f:
+ print(f.read())
+except FileNotFoundError:
+ pass
+
+
+############################################################
+# 2 # query "COUNT stars" ##################################
+############################################################
+
+RESULT = execute_query(QUERY)
+print(RESULT)
+
+############################################################
+# 3 ########################################################
+############################################################
+
+sys.exit(CODE)
+```
+
diff --git a/src/main/java/org/caosdb/server/database/backend/transaction/VersionTransaction.java b/src/main/java/org/caosdb/server/database/backend/transaction/VersionTransaction.java
index 8a98564050f5fb860402aff0f55f1dd9de417a39..ee617a1c1110e9a01e9a6d6ee4c799e44cd66ab3 100644
--- a/src/main/java/org/caosdb/server/database/backend/transaction/VersionTransaction.java
+++ b/src/main/java/org/caosdb/server/database/backend/transaction/VersionTransaction.java
@@ -35,7 +35,7 @@ import org.caosdb.server.entity.EntityInterface;
import org.caosdb.server.entity.Version;
/**
- * Abstract base class which retrieves and caches the full, but flag version history. The
+ * Abstract base class which retrieves and caches the full, but flat version history. The
* implementations then use the flat version history to construct either single version information
* items (see {@link RetrieveVersionInfo}) or the complete history as a tree (see {@link
* RetrieveVersionHistory})
diff --git a/src/main/java/org/caosdb/server/jobs/Job.java b/src/main/java/org/caosdb/server/jobs/Job.java
index 17d4500870f2372f793480692c27505fc3f71e72..4d744383897a121032e14bd97a3d859f950feae9 100644
--- a/src/main/java/org/caosdb/server/jobs/Job.java
+++ b/src/main/java/org/caosdb/server/jobs/Job.java
@@ -54,6 +54,13 @@ import org.caosdb.server.utils.EntityStatus;
import org.caosdb.server.utils.ServerMessages;
import org.reflections.Reflections;
+
+/**
+ * This is a Job.
+ *
+ * @todo Describe me.
+ *
+ */
public abstract class Job {
private Transaction<? extends TransactionContainer> transaction = null;
private Mode mode = null;
diff --git a/src/test/docker/Dockerfile b/src/test/docker/Dockerfile
index ecc6b332b2d88e0d587fdb3df0fc13cdfd0c159a..08a2a0884d1f154bb941388075b2bdd915125f99 100644
--- a/src/test/docker/Dockerfile
+++ b/src/test/docker/Dockerfile
@@ -1,5 +1,16 @@
FROM debian:buster
RUN apt-get update && \
- apt-get install \
- git make mariadb-server maven openjdk-11-jdk-headless \
- python3-pip screen libpam0g-dev unzip curl shunit2 -y
+ apt-get install -y \
+ git make mariadb-server maven openjdk-11-jdk-headless \
+ python3-pip screen libpam0g-dev unzip curl shunit2 \
+ python3-sphinx \
+ && \
+ pip3 install javasphinx recommonmark sphinx-rtd-theme
+
+# Alternative, if javasphinx fails because python3-sphinx is too recent:
+# (`_l` not found):
+#
+# git clone git@github.com:simgrid/javasphinx.git
+# cd javasphinx
+# git checkout 659209069603a
+# pip3 install .