diff --git a/.gitignore b/.gitignore
index ab11f8441c1690ede967897c0e64745f9ad30f86..c30895c42d4a4bae9f1bea8750f63899102b2fa9 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,3 +1,5 @@
+# -*- mode:conf; -*-
+
# configuration files
/conf/ext/*
!/conf/core/*.template
@@ -11,8 +13,11 @@
*.jks
# typical build dirs
+build/
bin/
target/
+_apidoc/
+
# But include server-side scripting
!/scripting/bin
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index c1afaf3f2a5274ff9e9a3083dbfd788e629709c0..0cf07b5cc6c376d27df7f1956af2e5ec8639be10 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -37,17 +37,19 @@ build-testenv:
tags: [ cached-dind ]
image: docker:19.03
stage: setup
+ timeout: 3h
only:
- schedules
script:
- cd src/test/docker
+ - time docker load < /image-cache/caosdb-server-testenv.tar || true
- docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY
# use here general latest or specific branch latest...
- - docker pull $CI_REGISTRY_IMAGE || true
- docker build
--pull
- --cache-from $CI_REGISTRY_IMAGE
-t $CI_REGISTRY_IMAGE .
+ - docker save $CI_REGISTRY_IMAGE > image.tar;
+ mv image.tar /image-cache/caosdb-server-testenv.tar;
- docker push $CI_REGISTRY_IMAGE
# Test: run unit tests of the server
@@ -74,3 +76,21 @@ trigger_build:
-F "variables[TriggerdBy]=SERVER"
-F "variables[TriggerdByHash]=$CI_COMMIT_SHORT_SHA"
-F ref=$DEPLOY_REF https://gitlab.indiscale.com/api/v4/projects/14/trigger/pipeline
+
+# Build the sphinx documentation and make it ready for deployment by Gitlab Pages
+# documentation:
+# stage: deploy
+
+# Special job for serving a static website. See https://docs.gitlab.com/ee/ci/yaml/README.html#pages
+pages:
+ tags: [ cached-dind ]
+ stage: deploy
+ only:
+ - dev
+ script:
+ - echo "Deploying"
+ - make doc
+ - cp -r build/doc/html public
+ artifacts:
+ paths:
+ - public
diff --git a/CHANGELOG.md b/CHANGELOG.md
index fccd7c8ff5d78126d80692728d7941be1d142e29..ea88e321261634711f9e084f318adf497672d66c 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -9,14 +9,47 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
### Added
+* New version history feature. The "H" container flag retrieves the full
+ version history during a transaction (e.g. during Retrievals) and constructs
+ a tree of successors and predecessors of the requested entity version.
+* New query functionality: `ANY VERSION OF` modifier. E.g. `FIND ANY VERSION OF
+ RECORD WITH pname=val` returns all current and old versions of records where
+ `pname=val`. For further information, examples and limitations see the wiki
+ page on [CQL](https://gitlab.com/caosdb/caosdb/-/wikis/manuals/CQL/CaosDB%20Query%20Language)
+* New server property `SERVER_SIDE_SCRIPTING_BIN_DIRS` which accepts a comma or
+ space separated list as values. The server looks for scripts in all
+ directories in the order or the list and uses the first matching file.
+* Automated documentation builds: `make doc`
+
### Changed
+* Select queries would originally only select the returned properties by their
+ names and would not check if a property is a subtype of a selected property. This
+ has changed now and select queries will also return subtypes of selected
+ properties.
+
### Deprecated
+* `SERVER_SIDE_SCRIPTING_BIN_DIR` property is deprecated.
+ `SERVER_SIDE_SCRIPTING_BIN_DIRS` should be used instead (note the plural
+ form!)
+
### Removed
### Fixed
+* Bug: When the user password is updated the user is deactivated.
+* Semi-fixed a bug which occurs when retrieving old versions of entities which
+ reference entities which have been deleted in the mean time. The current fix
+ adds a warning message to the reference property in question and sets the
+ value to NULL. This might even be desired behavior, however this would have
+ to finally specified during the Delete/Forget phase of the implementation of
+ the versioning.
+- Inheritance job cannot handle inheritance from same container (!54)
+* Bug in the query parser (MR!56) - The parser would throw an error when the
+ query contains a conjunction or disjunction filter with a first element which
+ is another disjunction or conjunction and being wrapped into parenthesis.
+
### Security
## [0.2] - 2020-09-02
@@ -50,7 +83,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
package naming conventions while the old was not. This has some implications
for configuring the server. See [README_SETUP.md](./README_SETUP.md), section
"Migration" for additional information.
-- The sever by default now only serves TLS 1.2 and 1.3, all previous versions
+- The server by default now only serves TLS 1.2 and 1.3, all previous versions
have been disabled in the default settings. Make sure that your clients
(especially the Python client) are up to date.
diff --git a/makefile b/Makefile
similarity index 98%
rename from makefile
rename to Makefile
index 5c0e6cdda534c34f9f34109a008feac1d79312dc..d73f140f065ca0b3db62651e40162194f4ffb4eb 100644
--- a/makefile
+++ b/Makefile
@@ -128,3 +128,8 @@ stop-debug-screen:
easy-units: .m2-local
mvn clean
mvn deploy:deploy-file -DgroupId=de.timmfitschen -DartifactId=easy-units -Dversion=0.0.1-SNAPSHOT -Durl=file:./.m2-local/ -DrepositoryId=local-maven-repo -DupdateReleaseInfo=true -Dfile=./lib/easy-units-0.0.1-SNAPSHOT-jar-with-dependencies.jar
+
+# Compile the standalone documentation
+.PHONY: doc
+doc:
+ $(MAKE) -C src/doc html
diff --git a/README_SETUP.md b/README_SETUP.md
index 2d682c34f4065f15082507fc7c80b839a486dcc9..f47f5a08624c20de194829b3534d72c2e06b461c 100644
--- a/README_SETUP.md
+++ b/README_SETUP.md
@@ -1,25 +1,28 @@
-# Requirements
+# Getting Started with the CaosDB Server
+Here, you find information on requirements, the installation, configuration and more.
-## CaosDB Packages
+## Requirements
+
+### CaosDB Packages
* caosdb-webui=0.2.1
* caosdb-mysqlbackend=3.0
-## Third-party Software
-* >=Java 8
-* >=Apache Maven 3.0.4
-* >=Python 3.4
-* >=pip 9.0.1
-* >=git 1.9.1
-* >=Make 3.81
-* >=Screen 4.01
-* >=MySQL 5.5 (better >=5.6) or >=MariaDB 10.1
-* libpam (if PAM authentication is required)
-* unzip
-* openpyxl (for XLS/ODS export)
-* openssl (if a custom TLS certificate is required)
-
-### Install the requirements on Debian
+### Third-party Software
+* `>=Java 8`
+* `>=Apache Maven 3.0.4`
+* `>=Python 3.4`
+* `>=pip 9.0.1`
+* `>=git 1.9.1`
+* `>=Make 3.81`
+* `>=Screen 4.01`
+* `>=MySQL 5.5` (better `>=5.6`) or `>=MariaDB 10.1`
+* `libpam` (if PAM authentication is required)
+* `unzip`
+* `openpyxl` (for XLS/ODS export)
+* `openssl` (if a custom TLS certificate is required)
+
+#### Install the requirements on Debian
On Debian, the required packages can be installed with:
apt-get install git make mariadb-server maven openjdk-11-jdk-headless \
@@ -28,21 +31,21 @@ On Debian, the required packages can be installed with:
Note that installing MariaDB will uninstall existing MySQL packages and vice
versa.
-## System
+### System
-* >=Linux 4.0.0, x86\_64, e.g. Ubuntu 14.04.1
+* `>=Linux 4.0.0`, `x86_64`, e.g. Ubuntu 18.04
* Mounted filesytem(s) with enough space
* Working internet connection (for up-to-date python and java libraries)
-## Extensions ##
+### Extensions ##
-### Web UI ###
+#### Web UI ###
- If the WebUI shall run, check out the respective submodule: `git submodule
update --init caosdb-webui`
- Then configure and compile it according to its
- [documentation](caosdb-webui/README_SETUP.md).
+ [documentation](https://http://caosdb.gitlab.io/caosdb-webui/getting_started.html).
-### PAM ###
+#### PAM ###
Authentication via PAM is possible, for this the PAM development library must be
installed and the pam user tool must be compiled:
@@ -52,14 +55,14 @@ installed and the pam user tool must be compiled:
should print `[FAILED]` and return with a non-zero exit code. Unless there is
a user `asdf` with password `ghjk` on your system, of course.
-#### Troubleshooting ####
+##### Troubleshooting ####
If `make` fails with `pam_authentication.c:4:31: fatal error:
security/pam_appl.h: No such file or directory` the header files are probably
not installed. You can do so under Debian and Ubuntu with `apt-get install
libpam0g-dev`. Then try again.
-# First Setup
+## First Setup
After a fresh clone of the repository, this is what you need to setup the
server:
@@ -68,42 +71,56 @@ server:
needs to be an internet connection as packages are downloaded to be
integrated in the java file.
1. It is recommended to run the unit tests with `make test`. It may take a
- while.
+ while.
2. Create an SSL certificate somewhere with a `Java Key Store` file. For
self-signed certificates (not recommended for production use) you can do:
- `mkdir certificates; cd certificates`
- `keytool -genkey -keyalg RSA -alias selfsigned -keystore caosdb.jks -validity 375 -keysize 2048 -ext san=dns:localhost`
Replace `localhost` by your host name, if you want.
- `keytool -importkeystore -srckeystore caosdb.jks -destkeystore caosdb.p12 -deststoretype PKCS12 -srcalias selfsigned`
- - `openssl pkcs12 -in caosdb.p12 -nokeys -out cert.pem`
+ - Export the public part only: `openssl pkcs12 -in caosdb.p12 -nokeys -out cert.pem`.
+ The resulting ``cert.pem` can safely be given to users to allow ssl verification.
- You can check the content of the certificate with `openssl x509 -in cert.pem -text`
Alternatively, you can create a keystore from certificate files that you already have:
- `openssl pkcs12 -export -inkey privkey.pem -in fullchain.pem -out all-certs.pkcs12`
- `keytool -importkeystore -srckeystore all-certs.pkcs12 -srcstoretype PKCS12 -deststoretype pkcs12 -destkeystore caosdb.jks`
-
-3. Copy `conf/core/server.conf` to `conf/ext/server.conf` and change it
+3. Install/configure the MySQL back-end: see the `README_SETUP.md` of the
+ `caosdb-mysqlbackend` repository
+4. Create an authtoken config (e.g. copy `conf/core/authtoken.example.yaml` to `conf/ext/authtoken.yml` and change it)
+5. Copy `conf/core/server.conf` to `conf/ext/server.conf` and change it
appropriately:
- * Setup for MySQL back-end: Assuming that the mysql back-end is installed
- (see the `README_SETUP.md` of the `caosdb-mysqlbackend` repository),
+ * Setup for MySQL back-end:
specify the fields `MYSQL_USER_NAME`, `MYSQL_USER_PASSWORD`,
- `MYSQL_DATABASE_NAME`, and `MYSQL_HOST`.
+ `MYSQL_DATABASE_NAME`, and `MYSQL_HOST`.
* Choose the ports under which CaosDB will be accessible.
* Setup the SSL certificate: Assuming that there is an appropriate `Java Key
Store` file (see above), change the fields `CERTIFICATES_KEY_PASSWORD`,
`CERTIFICATES_KEY_STORE_PATH`, and `CERTIFICATES_KEY_STORE_PASSWORD`.
Make sure that the conf file is not readable by other users because the
certificate passwords are stored in plaintext.
+ - Set the path to the authtoken config (see step 4)
* Set the file system paths:
- - `FILE_SYSTEM_ROOT`: The root for all the files managed by CaosDB.
- - `DROP_OFF_BOX`: Files can be put here for insertion into CaosDB.
- - `TMP_FILES`: Temporary files go here, for example during script execution
- or when uploading or moving files.
- - `SHARED_FOLDER`: Folder for sharing files via cryptographic tokens, also
- those created by scripts.
+ - `FILE_SYSTEM_ROOT`: The root for all the files managed by CaosDB.
+ - `DROP_OFF_BOX`: Files can be put here for insertion into CaosDB.
+ - `TMP_FILES`: Temporary files go here, for example during script
+ execution or when uploading or moving files.
+ - `SHARED_FOLDER`: Folder for sharing files via cryptographic tokens,
+ also those created by scripts.
+ - `SERVER_SIDE_SCRIPTING_BIN_DIRS`: A comma or white space separated list
+ of directories (relative or absolute) where the server will be looking
+ for executables which are then callable as server-side scripts. By
+ default this list only contains `./scripting/bin`. If you want to
+ include e.g. scripts which are maintained as part of the caosdb-webui
+ repository (because they are intended for usage by the webui), you
+ should add `./caosdb-webui/sss_bin/` as well.
+ - `INSERT_FILES_IN_DIR_ALLOWED_DIRS`: add mounted filesystems here that
+ shall be accessible by CaosDB
* Maybe set another `SESSION_TIMEOUT_MS`.
* See also [README_CONFIGURATION.md](README_CONFIGURATION.md)
-4. Copy `conf/core/usersources.ini.template` to `conf/ext/usersources.ini`.
+6. Copy `conf/core/usersources.ini.template` to `conf/ext/usersources.ini`.
+ * You can skip this if you do not want to use an external authentication.
+ Local users (CaosDB realm) are always available.
* Define the users/groups who you want to include/exclude.
* Assign at least one user the `administration` role.
* For example, if the admin user is called `caosdb`, there should be the
@@ -116,11 +133,12 @@ server:
Especially that there are no `properties` (aka `keys`) without a
`value`. An emtpy value can be represented by `""`. Comments are
everything from `#` or `;` to the end of the line.
-5. Install the pam caller in `misc/pam_authentication/`. See
- [the pam authentication README](misc/pam_authentication/README.md)
+7. Possibly install the PAM caller in `misc/pam_authentication/` if you have
+ not do so already. See above.
+
Done!
-# Start Server
+## Start Server
`$ make run`
@@ -131,12 +149,12 @@ type `https://localhost:10443` in your Browser, assuming you used 10443 as port.
Note, that you will get a security warning if you are using a self-signed
certificate.
-# Run Unit Tests
+## Run Unit Tests
`$ make test`
-# Setup Eclipse
+## Setup Eclipse
1. Open Eclipse (recommended version: Oxygen.1a Release (4.7.1a))
2. `File > New > Java Project`: Choose a project name and specify the location
@@ -155,9 +173,9 @@ certificate.
Done!
-# Migration
+## Migration
-## From 0.1 to 0.2
+### From 0.1 to 0.2
A major change in the code is the renaming of the java packages (from
`caosdb.[...]` to `org.caosdb.[...]`).
@@ -172,3 +190,21 @@ before you execute it.
```sh
sed -i.bak -e "s/\(\s*\)\([^.]\)caosdb\.server/\1\2org.caosdb.server/g" FILE_TO_BE_CHANGED
```
+
+## Build the documentation #
+
+Stand-alone documentation is built using Sphinx: `make doc`
+
+### Requirements ##
+
+- sphinx
+- javasphinx :: `pip3 install --user javasphinx`
+ - Alternative, if javasphinx fails because python3-sphinx is too recent:
+ (`l_` not found):
+
+```sh
+git clone git@github.com:simgrid/javasphinx.git
+cd javasphinx
+git checkout 659209069603a
+pip3 install .
+```
diff --git a/conf/core/cache.ccf b/conf/core/cache.ccf
index b4e1f93596a6170ef04ec373dc7a8d70e51fedcc..b6a50bee08fdd8598dac3c9f3d7aa70f43190127 100644
--- a/conf/core/cache.ccf
+++ b/conf/core/cache.ccf
@@ -1,3 +1,8 @@
+# -*- mode:conf-javaprop; -*-
+
+# Configuration for the Java Caching System (JCS) which is used by the server. Please look at
+# http://commons.apache.org/proper/commons-jcs/getting_started/intro.html for further information.
+
# default caching options
jcs.default.cacheattributes=org.apache.commons.jcs.engine.CompositeCacheAttributes
jcs.default.cacheattributes.MaxObjects=1000
@@ -28,8 +33,8 @@ jcs.region.BACKEND_JobRules.cacheattributes.MaxObjects=103
jcs.region.BACKEND_SparseEntities
jcs.region.BACKEND_SparseEntities.cacheattributes.MaxObjects=1002
-jcs.region.BACKEND_RetrieveFullVersionInfo
-jcs.region.BACKEND_RetrieveFullVersionInfo.cacheattributes.MaxObjects=1006
+jcs.region.BACKEND_RetrieveVersionHistory
+jcs.region.BACKEND_RetrieveVersionHistory.cacheattributes.MaxObjects=1006
# PAM UserSource Caching: Cached Items expire after 60 seconds if they are not requested (idle) and after 600 seconds max.
# PAM_UnixUserGroups
diff --git a/conf/core/global_entity_permissions.xml b/conf/core/global_entity_permissions.xml
index 3cebb79bd1c40564c99219519aa92b59a92e9dcd..3b0cf1b5ccb3f50c73f7a0ad8f4a2651c2dad221 100644
--- a/conf/core/global_entity_permissions.xml
+++ b/conf/core/global_entity_permissions.xml
@@ -1,4 +1,10 @@
<globalPermissions>
+ <!--
+ 4-store for permissions, implemented as a mapping:
+ {Grant/Deny, priority, role} -> List of permissions
+
+ Please look at the permission documentation for more information.
+ -->
<Grant priority="false" role="?OWNER?"><Permission name="*"/></Grant>
<Grant priority="false" role="?OTHER?"><Permission name="RETRIEVE:*"/></Grant>
<Grant priority="false" role="?OTHER?"><Permission name="USE:*"/></Grant>
diff --git a/conf/core/log4j2-debug.properties b/conf/core/log4j2-debug.properties
index 40ffe0fc02c16eb3c80a9216f58b317faecc41dc..3ae3ed266ef865fcffde45c21458650dc5ccc0c1 100644
--- a/conf/core/log4j2-debug.properties
+++ b/conf/core/log4j2-debug.properties
@@ -1,3 +1,5 @@
+# This log4j2-debug.properties file is only loaded when the server runs in debug mode.
+
# override location of log files for debugging and testing
property.LOG_DIR = testlog
diff --git a/conf/core/log4j2-default.properties b/conf/core/log4j2-default.properties
index b1697a73fe6703850865406e1441947614d00f47..974ce34df0f8290d763bf6a993554dab0ec7eeb2 100644
--- a/conf/core/log4j2-default.properties
+++ b/conf/core/log4j2-default.properties
@@ -1,3 +1,6 @@
+# This log4j2-default.properties file describes the logging settings. See
+# https://logging.apache.org/log4j/2.x/ for more information.
+
name = base_configuration
status = TRACE
verbose = true
diff --git a/conf/core/server.conf b/conf/core/server.conf
index 73a7bc871c47409f48dd724988818ab210e0ab87..039284de44da219837e522b6f9282327562eb1f0 100644
--- a/conf/core/server.conf
+++ b/conf/core/server.conf
@@ -12,9 +12,10 @@ SERVER_NAME=CaosDB Server
# The following paths are relative to the working directory of the server.
# --------------------------------------------------
-# The location of the server side scripting binaries.
+# The location(s) of the server side scripting binaries.
# Put your executable python scripts here, if they need to be called from the scripting API.
-SERVER_SIDE_SCRIPTING_BIN_DIR=./scripting/bin/
+# The value is a comma or space separated list or a single directory
+SERVER_SIDE_SCRIPTING_BIN_DIRS=./scripting/bin/
# Working directory of the server side scripting API.
# On execution of binaries and scripts the server will create a corresponding working directory in this folder.
@@ -67,7 +68,7 @@ MYSQL_USER_NAME=caosdb
# Password for the user
MYSQL_USER_PASSWORD=caosdb
# Schema of mysql procedures and tables which is required by this CaosDB instance
-MYSQL_SCHEMA_VERSION=v3.0.0-rc2
+MYSQL_SCHEMA_VERSION=v4.0.0-rc2
# --------------------------------------------------
@@ -120,7 +121,7 @@ SESSION_TIMEOUT_MS=600000
# 7days
ONE_TIME_TOKEN_EXPIRES_MS=604800000
-# Path to config file for one time tokens, for example authtoken.yml.
+# Path to config file for one time tokens, see authtoken.example.yml.
AUTHTOKEN_CONFIG=
# Timeout after which a one-time token expires once it has been first consumed,
@@ -144,7 +145,7 @@ MAIL_TO_FILE_HANDLER_LOC=./
# --------------------------------------------------
# Admin settings
-# # --------------------------------------------------
+# --------------------------------------------------
# Name of the administrator of this instance
ADMIN_NAME=CaosDB Admin
# Email of the administrator of this instance
@@ -181,4 +182,9 @@ CHECK_ENTITY_ACL_ROLES_MODE=MUST
# part of any Entity ACL.
GLOBAL_ENTITY_PERMISSIONS_FILE=./conf/core/global_entity_permissions.xml
+# --------------------------------------------------
+# Extensions
+# --------------------------------------------------
+
+# If set to true, versioning of entities' history is enabled.
ENTITY_VERSIONING_ENABLED=true
diff --git a/conf/core/usersources.ini.template b/conf/core/usersources.ini.template
index 270a4ad069305ef47b7ad46b36bd69673860fb97..2e0fe2490a65b53022d8d0edff6495f3b92ec10a 100644
--- a/conf/core/usersources.ini.template
+++ b/conf/core/usersources.ini.template
@@ -1,4 +1,5 @@
-#
+# -*- mode:conf; -*-
+
# ** header v3.0
# This file is a part of the CaosDB Project.
#
@@ -20,17 +21,35 @@
#
# ** end header
#
+
+# This file configures external authentication providers. The CaosDB realm is
+# always available (without being defined here).
+
+# `realms` is a comma and/or space separated list of realms which users can
+# use for authentication
+# Currently available: PAM
realms = PAM
+
+# This is the default realm, to be used when no other realms is specified
defaultRealm = PAM
+# Each realm has one section with specific options. The options for a specific
+# realm can be looked up in that realm's documentation.
+#
+# Hint: Realms are implemented by classes which are typically in the
+# org.caosdb.server.accessControl.Pam package and implement the UserSource interface.
+
+# Options for authentication against Linux' PAM.
[PAM]
class = org.caosdb.server.accessControl.Pam
+# The script which does the actual checking.
; pam_script = ./misc/pam_authentication/pam_authentication.sh
default_status = ACTIVE
+# Only users which fulfill these criteria are accepted.
;include.user = [uncomment and put your users here]
;include.group = [uncomment and put your groups here]
;exclude.user = [uncomment and put excluded users here]
;exclude.group = [uncomment and put excluded groups here]
-;it is necessary to add at least one admin
+# It is typically necessary to add at least one admin
;user.[uncomment a set a username here].roles = administration
diff --git a/scripting/home/readme.md b/scripting/home/readme.md
index 9c9744a55cec41f33929a3953d9c582fa97f46e0..c79afce2a399089c797e0157d30d9908b33dd23e 100644
--- a/scripting/home/readme.md
+++ b/scripting/home/readme.md
@@ -1,3 +1,3 @@
# The `home` directory #
-This directory will be copied to a temporary directy for each server-side
+This directory will be copied to a temporary directly for each server-side
scripting invocation and set as the `HOME` environment variable.
diff --git a/src/doc/CaosDB-Query-Language.md b/src/doc/CaosDB-Query-Language.md
new file mode 100644
index 0000000000000000000000000000000000000000..07fbdbc310b8d22de4642f041918710e6e707488
--- /dev/null
+++ b/src/doc/CaosDB-Query-Language.md
@@ -0,0 +1,404 @@
+# CaosDB Query Language
+**WIP This is going to be the specification. CQL tutorials are in the webui**
+
+## Example queries
+
+### Simple FIND Query
+The following query will return any entity which has the name _ename_ and all its children.
+`FIND ename`
+
+The following queries are equivalent and will return any entity which has the name _ename_ and all its children, but only if they are genuin records. Of course, the returned set of entities (henceforth referred to as _resultset_) can also be restricted to recordtypes, properties and files.
+
+`FIND RECORD ename`
+
+`FIND RECORDS ename`
+
+Wildcards use `*` for any characters or none at all. Wildcards for single characters (like the '_' wildcard from mysql) are not implemented yet.
+
+`FIND RECORD en*` returns any entity which has a name beginning with _en_.
+
+Regular expressions must be surrounded by _<<_ and '>>':
+
+`FIND RECORD <<e[aemn]{2,5}>>`
+
+`FIND RECORD <<[cC]am_[0-9]*>>`
+
+*TODO* (Timm):
+Describe escape sequences like `\\ `, `\*`, `\<<` and `\>>`.
+
+Currently, wildcards and regular expressions are only available for the _simple-find-part_ of the query, i. e. no wildcards/regexps for filters.
+
+### Simple COUNT Query
+
+This query counts entities which have certain properties.
+
+`COUNT ename`
+will return the number of entities which have the name _ename_ and all their children.
+
+The syntax of the COUNT queries is equivalent to the FIND queries in any respect (this also applies to wildcards and regular expressions) but one: The prefix is to be `COUNT` instead of `FIND`.
+
+Unlike the FIND queries, the COUNT queries do not return any entities. The result of the query is the number of entities which _would be_ returned if the query was a FIND query.
+
+## Filters
+
+### POV - Property-Operator-Value
+
+The following queries are equivalent and will restrict the result set to entities which have a property named _pname1_ that has a value _val1_.
+
+`FIND ename.pname1=val1`
+
+`FIND ename WITH pname1=val1`
+
+`FIND ename WHICH HAS A PROPERTY pname1=val1`
+
+`FIND ename WHICH HAS A pname1=val1`
+
+Again, the resultset can be restricted to records:
+
+`FIND RECORD ename WHICH HAS A pname1=val1`
+
+_currently known operators:_ `=, !=, <=, <, >=, >` (and cf. next paragraphes!)
+
+#### Special Operator: LIKE
+
+The _LIKE_ can be used with wildcards. The `*` is a wildcard for any (possibly empty) sequence of characters. Examples:
+
+`FIND RECORD ename WHICH HAS A pname1 LIKE va*`
+
+`FIND RECORD ename WHICH HAS A pname1 LIKE va*1`
+
+`FIND RECORD ename WHICH HAS A pname1 LIKE *al1`
+
+_Note:_ The _LIKE_ operator is will only produce expectable results with text properties.
+
+#### Special Case: References
+
+In general a reference can be addressed just like a POV filter. So
+
+`FIND ename1.pname1=ename2`
+
+will also return any entity named _ename1_ which references the entity with name or id _ename2_ via a reference property named _pname1_. However, it will also return any entity with a text property of that name with the string value _ename2_. In order to restrict the result set to reference properties one may make use of special reference operators:
+
+_reference operators:_ `->, REFERENCES, REFERENCE TO`
+
+
+The query looks like this:
+
+`FIND ename1 WHICH HAS A pname1 REFERENCE TO ename2`
+
+`FIND ename1 WHICH HAS A pname1->ename2`
+
+#### Time Special Case: DateTime
+
+_DateTime operators:_ `=, !=, <, >, IN, NOT IN`
+
+##### `d1=d2`: Equivalence relation.
+* ''True'' iff d1 and d2 are equal in every respect (same DateTime flavor, same fields are defined/undefined and all defined fields are equal respectively).
+* ''False'' iff they have the same DateTime flavor but have different fields defined or fields with differing values.
+* ''Undefined'' otherwise.
+
+Examples:
+* `2015-04-03=2015-04-03T00:00:00` is undefined.
+* `2015-04-03T00:00:00=2015-04-03T00:00:00.0` is undefined (second precision vs. nanosecond precision).
+* `2015-04-03T00:00:00.0=2015-04-03T00:00:00.0` is true.
+* `2015-04-03T00:00:00=2015-04-03T00:00:00` is true.
+* `2015-04=2015-05` is false.
+* `2015-04=2015-04` is true.
+
+##### `d1!=d2`: Intransitive, symmetric relation.
+* ''True'' iff `d1=d2` is false.
+* ''False'' iff `d1=d2` is true.
+* ''Undefined'' otherwise.
+
+Examples:
+* `2015-04-03!=2015-04-03T00:00:00` is undefined.
+* `2015-04-03T00:00:00!=2015-04-03T00:00:00.0` is undefined.
+* `2015-04-03T00:00:00.0!=2015-04-03T00:00:00.0` is false.
+* `2015-04-03T00:00:00!=2015-04-03T00:00:00` is false.
+* `2015-04!=2015-05` is true.
+* `2015-04!=2015-04` is false.
+
+##### `d1>d2`: Transitive, non-symmetric relation.
+Semantics depend on the flavors of d1 and d2. If both are...
+###### [UTCDateTime](Datatype#datetime)
+* ''True'' iff the time of d1 is after the the time of d2 according to [https://en.wikipedia.org/wiki/Coordinated_Universal_Time](UTC)
+* ''False'' otherwise.
+
+###### [SemiCompleteDateTime](Datatype#datetime)
+* ''True'' iff `d1.ILB>d2.EUB` is true or `d1.ILB=d2.EUB` is true.
+* ''False'' iff `d1.EUB<d2.ILB}} is true or {{{d1.EUB=d2.ILB` is true.
+* ''Undefined'' otherwise.
+
+Examples:
+* `2015>2014` is true.
+* `2015-04>2014` is true.
+* `2015-01-01T20:15.00>2015-01-01T20:14` is true.
+* `2015-04>2015` is undefined.
+* `2015>2015-04` is undefined.
+* `2015-01-01T20:15>2015-01-01T20:15:15` is undefined.
+* `2014>2015` is false.
+* `2014-04>2015` is false.
+* `2014-01-01>2015-01-01T20:15:30` is false.
+
+##### `d1<d2`: Transitive, non-symmetric relation.
+Semantics depend on the flavors of d1 and d2. If both are...
+###### [UTCDateTime](Datatype#datetime)
+* ''True'' iff the time of d1 is before the the time of d2 according to [UTC](https://en.wikipedia.org/wiki/Coordinated_Universal_Time)
+* ''False'' otherwise.
+
+###### [SemiCompleteDateTime](Datatype#datetime)
+* ''True'' iff `d1.EUB<d2.ILB` is true or `d1.EUB=d2.ILB` is true.
+* ''False'' iff `d1.ILB>d2.EUB}} is true or {{{d1.ILB=d2.EUB` is true.
+* ''Undefined'' otherwise.
+
+Examples:
+* `2014<2015` is true.
+* `2014-04<2015` is true.
+* `2014-01-01<2015-01-01T20:15:30` is true.
+* `2015-04<2015` is undefined.
+* `2015<2015-04` is undefined.
+* `2015-01-01T20:15<2015-01-01T20:15:15` is undefined.
+* `2015<2014` is false.
+* `2015-04<2014` is false.
+* `2015-01-01T20:15.00<2015-01-01T20:14` is false.
+
+##### `d1 IN d2`: Transitive, non-symmetric relation.
+Semantics depend on the flavors of d1 and d2. If both are...
+###### [SemiCompleteDateTime](Datatype#datetime)
+* ''True'' iff (`d1.ILB>d2.ILB` is true or `d1.ILB=d2.ILB` is true) and (`d1.EUB<d2.EUB` is true or `d1.EUB=d2.EUB` is true).
+* ''False'' otherwise.
+
+Examples:
+* `2015-01-01 IN 2015` is true.
+* `2015-01-01T20:15:30 IN 2015-01-01` is true.
+* `2015-01-01T20:15:30 IN 2015-01-01T20:15:30` is true.
+* `2015 IN 2015-01-01` is false.
+* `2015-01-01 IN 2015-01-01T20:15:30` is false.
+
+##### `d1 NOT IN d2`: Non-symmetric relation.
+Semantics depend on the flavors of d1 and d2. If both are...
+###### [SemiCompleteDateTime](Datatype#datetime)
+* ''True'' iff `d1.ILB IN d2.ILB` is false.
+* ''False'' otherwise.
+
+Examples:
+* `2015 NOT IN 2015-01-01` is true.
+* `2015-01-01 NOT IN 2015-01-01T20:15:30` is true.
+* `2015-01-01 NOT IN 2015` is false.
+* `2015-01-01T20:15:30 NOT IN 2015-01-01` is false.
+* `2015-01-01T20:15:30 NOT IN 2015-01-01T20:15:30` is false.
+
+##### Note
+These semantics follow a three-valued logic with ''true'', ''false'' and ''undefined'' as truth values. Only ''true'' is truth preserving. I.e. only those expressions which evaluate to ''true'' pass the POV filter. `FIND ... WHICH HAS A somedate=2015-01` only returns entities for which `somedate=2015-01` is true. On the other hand, `FIND ... WHICH DOESN'T HAVE A somedate=2015-01` returns entities for which `somedate=2015-01` is false or undefined. Shortly put, `NOT d1=d2` is not equivalent to `d1!=d2`. The latter assertion is stronger.
+
+#### Omitting the Property or the Value
+
+One doesn't have to specify the property or the value at all. The following query filters the result set for entities which have any property with a value greater than _val1_.
+
+`FIND ename WHICH HAS A PROPERTY > val1`
+
+`FIND ename . > val1`
+
+`FIND ename.>val1`
+
+
+And for references...
+
+`FIND ename1 WHICH HAS A REFERENCE TO ename2`
+
+`FIND ename1 WHICH REFERENCES ename2`
+
+`FIND ename1 . -> ename2`
+
+`FIND ename1.->ename2`
+
+
+The following query returns entities which have a _pname1_ property with any value.
+
+`FIND ename WHICH HAS A PROPERTY pname1`
+
+`FIND ename WHICH HAS A pname1`
+
+`FIND ename WITH pname1`
+
+`FIND ename . pname1`
+
+`FIND ename.pname1`
+
+### TransactionFilter
+
+*Definition*
+
+ sugar:: `HAS BEEN` | `HAVE BEEN` | `HAD BEEN` | `WAS` | `IS` |
+
+ negated_sugar:: `HAS NOT BEEN` | `HASN'T BEEN` | `WAS NOT` | `WASN'T` | `IS NOT` | `ISN'T` | `HAVN'T BEEN` |
+`HAVE NOT BEEN` | `HADN'T BEEN` | `HAD NOT BEEN`
+
+ by_clause:: `BY (ME | username | SOMEONE ELSE (BUT ME)? | SOMEONE ELSE BUT username)`
+
+ date:: A date string of the form `YYYY-MM-DD`
+
+ datetime:: A datetime string of the form `YYYY-MM-DD hh:mm:ss`
+
+ time_clause:: `ON ($date|$datetime) ` Here is plenty of room for more syntactic sugar, e.g. a `TODAY` keyword, and more funcionality, e.g. ranges.
+
+`FIND ename WHICH ($sugar|$negated_sugar)? (NOT)? (CREATED|INSERTED|UPDATED|DELETED) (by_clause time_clause?| time_clause by_clause?)`
+
+*Examples*
+
+`FIND ename WHICH HAS BEEN CREATED BY ME ON 2014-12-24`
+
+`FIND ename WHICH HAS BEEN CREATED BY SOMEONE ELSE ON 2014-12-24`
+
+`FIND ename WHICH HAS BEEN CREATED BY erwin ON 2014-12-24`
+
+`FIND ename WHICH HAS BEEN CREATED BY SOMEONE ELSE BUT erwin ON 2014-12-24`
+
+`FIND ename WHICH HAS BEEN CREATED BY erwin`
+
+`FIND ename . CREATED BY erwin ON `
+
+
+
+### File Location
+
+Search for file objects by their location:
+
+`FIND FILE WHICH IS STORED AT a/certain/path/`
+
+#### Wildcards
+
+_STORED AT_ can be used with wildcards similar to unix wildcards.
+ * `*` matches any characters or none at all, but not the directory separator `/`
+ * `**` matches any character or none at all.
+ * A leading `*` is a shortcut for `/**`
+ * Asterisks directly between two other asterisks are ignored: `***` is the same as `**`.
+ * Escape character: `\` (E.g. `\\` is a literal backslash. `\*` is a literal star. But `\\*` is a literal backslash followed by a wildcard.)
+
+Examples:
+
+Find any files ending with `.acq`:
+`FIND FILE WHICH IS STORED AT *.acq` or
+`FIND FILE WHICH IS STORED AT **.acq` or
+`FIND FILE WHICH IS STORED AT /**.acq`
+
+Find files stored one directory below `/data/`, ending with `.acq`:
+`FIND FILE WHICH IS STORED AT /data/*/*.acq`
+
+Find files stored in `/data/`, ending with `.acq`:
+`FIND FILE WHICH IS STORED AT /data/*.acq`
+
+Find files stored in a directory at any depth in the tree below `/data/`, ending with `.acq`:
+`FIND FILE WHICH IS STORED AT /data/**.acq`
+
+Find any file in a directory which begins with `2016-02`:
+`FIND FILE WHICH IS STORED AT */2016-02*/*`
+
+
+### Back References
+
+The back reference filters for entities that are referenced by another entity. The following query returns entities of the type _ename1_ which are referenced by _ename2_ entities via the reference property _pname1_.
+
+* `FIND ename1 WHICH IS REFERENCED BY ename2 AS A pname1`
+* `FIND ename1 WITH @ ename2 / pname1`
+* `FIND ename1 . @ ename2 / pname1`
+
+One may omit the property specification:
+
+* `FIND ename1 WHICH IS REFERENCED BY ename2`
+* `FIND ename1 WHICH HAS A PROPERTY @ ename2`
+* `FIND ename1 WITH @ ename2`
+* `FIND ename1 . @ ename2`
+
+### Combining Filters with Propositional Logic
+
+Any result set can be filtered by logically combining POV filters or back reference filters:
+
+#### Conjunction (AND)
+
+* `FIND ename1 WHICH HAS A PROPERTY pname1=val1 AND A PROPERTY pname2=val2 AND A PROPERTY...`
+* `FIND ename1 WHICH HAS A PROPERTY pname1=val1 AND A pname2=val2 AND ...`
+* `FIND ename1 . pname1=val1 & pname2=val2 & ...`
+
+#### Disjunction (OR)
+
+* `FIND ename1 WHICH HAS A PROPERTY pname1=val1 OR A PROPERTY pname2=val2 Or A PROPERTY...`
+* `FIND ename1 WHICH HAS A PROPERTY pname1=val1 OR A pname2=val2 OR ...`
+* `FIND ename1 . pname1=val1 | pname2=val2 | ...`
+
+#### Negation (NOT)
+
+* `FIND ename1 WHICH DOES NOT HAVE A PROPERTY pname1=val1`
+* `FIND ename1 WHICH DOESN'T HAVE A pname1=val1`
+* `FIND ename1 . NOT pname2=val2`
+* `FIND ename1 . !pname2=val2`
+
+#### ... and combinations with parentheses
+
+* `FIND ename1 WHICH HAS A pname1=val1 AND DOESN'T HAVE A pname2<val2 AND ((WHICH HAS A pname3=val3 AND A pname4=val4) OR DOES NOT HAVE A (pname5=val5 AND pname6=val6))`
+* `FIND ename1 . pname1=val1 & !pname2<val2 & ((pname3=val3 & pname4=val4) | !(pname5=val5 & pname6=val6))`
+* `FIND ename1.pname1=val1&!pname2<val2&((pname3=val3&pname4=val4)|!(pname5=val5&pname6=val6))`
+
+### A Few Important Expressions
+
+* A:: The indistinct article. This is only syntactic suger. Equivalent expressions: `A, AN`
+* AND:: The logical _and_. Equivalent expressions: `AND, &`
+* FIND:: The beginning of the query.
+* NOT:: The logical negation. Equivalent expressions: `NOT, DOESN'T HAVE A PROPERTY, DOES NOT HAVE A PROPERTY, DOESN'T HAVE A, DOES NOT HAVE A, DOES NOT, DOESN'T, IS NOT, ISN'T, !`
+* OR:: The logical _or_. Equivalent expressions: `OR, |`
+* RECORD,RECORDTYPE,FILE,PROPERTY:: Role expression for restricting the result set to a specific role.
+* WHICH:: The marker for the beginning of the filters. Equivalent expressions: `WHICH, WHICH HAS A, WHICH HAS A PROPERTY, WHERE, WITH, .`
+* REFERENCE:: This one is tricky: `REFERENCE TO` expresses a the state of _having_ a reference property. `REFERENCED BY` expresses the state of _being_ referenced by another entity.
+* COUNT:: `COUNT` works like `FIND` but doesn't return the entities.
+
+
+## Select Queries
+
+In contrast to `FIND` queries, which always return the complete entity, there are `SELECT` queries which only return the entity with only those properties which are specified in the query. The syntax is very similar to `FIND` queries - just replace the `FIND` by `SELECT <comma separated list of selectors> FROM`:
+
+`SELECT p1, p2, p3 FROM Record ename`
+
+However, the `SELECT` query can also return properties of referenced entities and thereby are a means of joining entities together and return a custom view or projection:
+
+`SELECT Conductor.Last Name FROM Experiment`
+
+would return the conductor's last name, when `Conductor` is a reference property of `Experiment` and `Last Name` is a property of the `Conductor` records.
+
+### Selectors
+
+Selectors are strings of entity names which are separated by `.` (dot). E.g. `Conductor.Last Name` or `Conductor.Address` or even `Experiment.Conductor.Last name`. Selectors in a `SELECT` queries are separated by `,` (comma). E.g. `Conductor.First Name, Conductor.Last Name`.
+
+### Evaluation of Selectors
+
+The query will return all those properties which have the same name as
+specified by the selector (case-insensitive). However, `SELECT` queries are
+also capable of subtyping in the selectors:
+
+`SELECT Person FROM Experiment` would return all `Person` properties but all `Conductors` as well, if `Conductor` is a child of `Person`.
+
+Note: When a property `responsible` with data type `Person` exists, the above `SELECT` statement would not include records that use this property (since `responsible` is not a child of Person).
+
+## Versioning
+
+Since Caosdb 0.2 entities are optionally version controlled. The query language will be extended to include versioning in the future. A current minimal implementation introduces the `ANY VERSION OF` modifier which can be used to return all matching versions in the results of `COUNT`, `FIND`, and `SELECT` queries.
+
+### Example
+
+* `FIND ANY VERSION OF RECORD WITH pname=value` returns the all past and present versions of records with `pname = value`.
+
+### Scope and current limitations
+
+* The `ANY VERSION OF` modifier currently the only expression for taking the versioning into account when using the query language.
+* Subproperties are not supported yet, e.g. `FIND ANY VERSION OF ENTITY WHICH IS REFERENCED BY ename WITH ...`. This applies to all cases where you specify properties of *referenced* entities or *referencing* entities.
+
+### Future
+
+* Add `(LATEST|LAST|OLDEST|NEWEST|FIRST) VERSION OF` modifiers.
+* Add `(ANY|LATEST|LAST|OLDEST|NEWEST|FIRST) VERSION (BEFORE|AFTER) (<timestamp>|<transaction id>|<entity@version>) OF` modifier.
+* Add support for subproperties, e.g. `FIND ANY VERSION OF ENTITY WHICH IS REFERENCED BY ename WITH ...`.
+
+## Future
+
+ * *Sub Queries* (or *Sub Properties*): `FIND ename WHICH HAS A pname WHICH HAS A subpname=val`. This is like: `FIND AN experiment WHICH HAS A camera WHICH HAS A 'serial number'= 1234567890`
+ * *More Logic*, especially `ANY`, `ALL`, `NONE`, and `SUCH THAT` key words (and equivalents) for logical quantisation: `FIND ename1 SUCH THAT ALL ename2 WHICH HAVE A REFERENCE TO ename1 HAVE A pname=val`. This is like `FIND experiment SUCH THAT ALL person WHICH ARE REFERENCED BY THIS experiment AS conductor HAVE AN 'academic title'=professor.`
+
diff --git a/src/doc/Makefile b/src/doc/Makefile
new file mode 100644
index 0000000000000000000000000000000000000000..4f177fd3850423c3005829a75a2b37625dc68a87
--- /dev/null
+++ b/src/doc/Makefile
@@ -0,0 +1,47 @@
+# ** header v3.0
+# This file is a part of the CaosDB Project.
+#
+# Copyright (C) 2020 IndiScale GmbH <info@indiscale.com>
+# Copyright (C) 2020 Daniel Hornung <d.hornung@indiscale.com>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see <https://www.gnu.org/licenses/>.
+#
+# ** end header
+
+# This Makefile is a wrapper for sphinx scripts.
+#
+# It is based upon the autocreated makefile for Sphinx documentation.
+
+# You can set these variables from the command line, and also
+# from the environment for the first two.
+SPHINXOPTS ?= -a
+SPHINXBUILD ?= sphinx-build
+SPHINXAPIDOC ?= javasphinx-apidoc
+SOURCEDIR = .
+BUILDDIR = ../../build/doc
+
+.PHONY: doc-help Makefile apidoc
+
+# Put it first so that "make" without argument is like "make help".
+doc-help:
+ @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
+
+# Catch-all target: route all unknown targets to Sphinx using the new
+# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS).
+%: Makefile apidoc
+ @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
+# sphinx-build -M html . ../../build/doc
+
+apidoc:
+ @$(SPHINXAPIDOC) -o _apidoc --update --title="CaosDB Server" ../main/
diff --git a/src/doc/README_SETUP.md b/src/doc/README_SETUP.md
new file mode 120000
index 0000000000000000000000000000000000000000..88332e357f5e06f3de522768ccdcd9e513c15f62
--- /dev/null
+++ b/src/doc/README_SETUP.md
@@ -0,0 +1 @@
+../../README_SETUP.md
\ No newline at end of file
diff --git a/src/doc/administration/configuration.rst b/src/doc/administration/configuration.rst
new file mode 100644
index 0000000000000000000000000000000000000000..4170fd0ed31c7a0ec5c1b4e589ff919267cf7de9
--- /dev/null
+++ b/src/doc/administration/configuration.rst
@@ -0,0 +1,62 @@
+Configuration
+=============
+
+The server is configured through configuration files. There are two directories with config files:
+
+``conf/core``
+ Upstream defaults are stored here.
+``conf/ext``
+ User specific configuration should be stored here, settings in ``ext`` override settings in
+ ``core``. Additionally, configuration files may be stored in ``*.d`` directories here, named
+ after the original config file name. For example, the general server configuration will be
+ assembled from ``conf/core/server.conf``, ``conf/ext/server.conf`` and any ``*.conf`` files found
+ in ``conf/ext/server.conf.d``.
+
+Configuration files
+-------------------
+
+In each of these directories, the server looks for the following files:
+
+``server.conf``
+ General server configuration options. The possible configuration options are documented inside
+ the `default file <https://gitlab.com/caosdb/caosdb-server/-/blob/dev/conf/core/server.conf>`_.
+
+``global_entity_permissions.xml``
+ :ref:`Permissions <concepts:Permissions>` which are automatically set, based on user roles.
+
+``usersources.ini``
+ This file defines possible sources which are checked when a user tries to authenticate. Each
+ defined source has a special section, the possible options are defined separately for each user
+ source. At the moment the best place to look for this specific documentation is at the API
+ documentation of :java:type:`UserSource` and its implementing classes. The provided `template
+ file <https://gitlab.com/caosdb/caosdb-server/-/blob/dev/conf/core/usersources.ini.template>`_
+ also has some information. The general concept about authentication realms is described in
+ :java:type:`UserSources`.
+
+``authtoken.yaml``
+ Configuration for dispensed authentication tokens, which can be used to authenticate to CaosDB
+ without the need of a user/password combination. Possible use cases are server-side scripts or
+ initial setup after the server start. There is more documentation inside the `template file
+ <https://gitlab.com/caosdb/caosdb-server/-/blob/dev/conf/core/authtoken.example.yaml>`_.
+
+``cache.ccf``
+ Configuration for the Java Caching System (JCS) which can be used by the server. More
+ documentation is `upstream
+ <http://commons.apache.org/proper/commons-jcs/getting_started/intro.html>`_ and inside `the file
+ <https://gitlab.com/caosdb/caosdb-server/-/blob/dev/conf/core/cache.ccf>`_.
+
+``log4j2-default.properties``, ``log4j2-debug.properties``
+ Configuration for logging, following the standard described by the `log4j library
+ <https://logging.apache.org/log4j/2.x/>`_. The ``default`` file is always loaded, in debug mode
+ the ``debug`` file iss added as well.
+
+
+Changing the configuration at runtime
+-------------------------------------
+
+Remark:
+ Only when the server is in debug mode, the general configuration can be changed at runtime.
+
+In the debug case, the server provides the ``_server_properties`` resource which allows the ``GET``
+and ``POST`` methods to access the server's properties. The Python client library conveniently
+wraps this in the :any:`caosdb-pylib:caosdb.common.administration` module.
diff --git a/src/doc/administration/maintenance.rst b/src/doc/administration/maintenance.rst
new file mode 100644
index 0000000000000000000000000000000000000000..67d8475bf469957416a6f42e495db07b1533f151
--- /dev/null
+++ b/src/doc/administration/maintenance.rst
@@ -0,0 +1,67 @@
+
+Maintenance of the CaosDB Server
+================================
+
+Creating a Backup
+-----------------
+
+In order to create a full backup of CaosDB, the state of the SQL-Backend (MySQL, MariaDB)
+has to be saved and the internal file system of CaosDB (symbolic links to
+file systems that are mounted and uploaded files) has to be saved.
+
+You find the documentation on how to backup the SQL-Backend :any:`caosdb-mysqlbackend:Maintenance`
+
+In order to save the file backend we recommend to tar the file system. However,
+you could use other backup methods that allow to restore the file system.
+The CaosDB internal file system is located at the path defined by the
+``FILE_SYSTEM_ROOT`` configuration variable (see :any:`configuration`).
+
+The command could look like::
+
+ tar czvf /path/to/new/backup /path/to/caosdb/filesystem.tar.gz
+
+
+You can also save the content of CaosDB using XML. This is **not recommended** since it is less reliable than a real SQL backup. However there may be cases in which an XML backup is desirable, e.g., when transferring entities between two different CaosDB instances.
+Collect the entities that you want to export in a :any:`caosdb-pylib:caosdb.common.models.Container`, here
+named ``cont``. Then you can export the XML with::
+
+ from caosadvancedtools.export_related import invert_ids
+ from lxml import etree
+ invert_ids(cont)
+ xml = etree.tounicode(cont.to_xml(
+ local_serialization=True), pretty_print=True)
+
+ with open("caosdb_data.xml"), "w") as fi:
+ fi.write(xml)
+
+
+Restoring a Backup
+------------------
+
+.. note :
+ CaosDB should be offline before restoring data.
+
+If you want to restore the internal file system, simply replace it. E.g. if your
+Backup is a tarball::
+
+ tar xvf /path/to/caosroot.tar.gz
+
+
+You find the documentation on how to restore the data in the SQL-Backend :any:`caosdb-mysqlbackend:Maintenance`
+
+
+If you want to restore the entities exported to XML, you can do::
+
+ cont = db.Container()
+ with open("caosdb_data.xml") as fi:
+ cont = cont.from_xml(fi.read())
+ cont.insert()
+
+User Management
+---------------
+The configuration of authentication mechanisms is done via the
+``usersources.ini`` file (see :any:`configuration`).
+
+We recommend the Python tools (:any:`caosdb-pylib:Administration`) for further administrative tasks (e.g. setting
+user passwords).
+
diff --git a/src/doc/administration/server_side_scripting.rst b/src/doc/administration/server_side_scripting.rst
new file mode 100644
index 0000000000000000000000000000000000000000..59bce98682c6d1190fc90681fc8e14fd483e891e
--- /dev/null
+++ b/src/doc/administration/server_side_scripting.rst
@@ -0,0 +1,85 @@
+Server-Side Scripting
+=====================
+
+Introduction
+------------
+
+Small computation task, like some visualization, might be easily implemented in Python or some other language, but cumbersome to integrate into the server. Furthermore, the CaosDB server should stay a general tool without burden from specific projects. Also, one might want to trigger some standardized processing task from the web interface for convenience. For these situations the "server side scripting" is intended.
+
+Concepts
+------------
+
+The basic idea is that a script or program (script in the following) can be called to run on the server (or elsewhere in future) to do some calculations. This triggering of the script is done over the API so it can be done with any client. Input arguments can be passed to the script and the STDOUT and STDERR are returned.
+
+Each script is executed in a temporary home directory, which is automatically clean up. However, scripts can store files in the "$SHARED" folder and for example provide users a link that allows them to download files.
+
+Write and Install a Script
+--------------------------
+
+A server-side script must accept at least the ``--auth-token=AUTH_TOKEN`` option. All other command-line parameters which are passed to the script are not specified by the API and maybe defined by the script itself.
+
+So a minimal bash script would be
+
+.. code-block:: sh
+
+ #!/bin/bash
+ echo Hello, World!
+
+thereby just ignoring the ``--auth-token`` option.
+
+The script has to be executable and must be placed somewhere in one of the directory trees which are configured by the server config :doc:`SERVER_SIDE_SCRIPTING_BIN_DIRS <configuration>`.
+
+Users will need the ``SCRIPTING:EXECUTE:path:to:the:script`` permission. Here the path to the script is of course relativet to the ``SERVER_SIDE_SCRIPTING_BIN_DIRS`` where it is located.
+
+For more information see the :doc:`specification of the API <../specs/Server-side-scripting>`
+
+Environment
+------------
+
+The script is called with several special environment variables to accommodate
+for its special location.
+
+`HOME`
+^^^^^^^^^^^^
+To be able to run with reduced privileges, the script has its `HOME` environment
+variable set to a special directory with write access. This directory will be
+deleted after the script has terminated. Its content is freshly copied for each
+script invocation from a skeleton directory, located in the server directory, in
+`scripting/home/`. By default, this directory contains the following:
+
+- `readme.md` :: A small text file describing the purpose of the directory.
+
+Users of CaosDB are invited to populate the directory with whatever their
+scripts need.
+
+Invocation
+------------
+
+Server side scripts are triggered by sending a POST to the `/scripting` resource. There are the following arguments that can be provided:
+
+- `call`: the name of the script to be called
+- `-pN`: positional arguments (e.g. `-p0`, `-p1` etc.)
+- `-ONAME`: named arguments (e.g. `-Otest`, `-Onumber` etc.)
+
+The arguments will be passed to the script.
+
+An invocation via a button in javascript could look like:
+
+.. code-block:: javascript
+
+ var _make_sss_button = function (entity) {
+ const script = "script.py";
+
+ const scripting_form = $(`
+ <form class="btn-group-xs ${_css_class_export_button}"
+ method="POST"
+ action="/scripting">
+ <input type="hidden" name="call" value="${script}"/>
+ <input name="-p0" value=""/>
+ <button type="submit" class="btn btn-link">Start script</button>
+ </form>`);
+ return scripting_form[0];
+ }
+
+For more information see the :doc:`specification of the API <../specs/Server-side-scripting>`
+
diff --git a/src/doc/concepts.rst b/src/doc/concepts.rst
new file mode 100644
index 0000000000000000000000000000000000000000..ec93719548a7d7c577ac62c6615b3fcf24111b42
--- /dev/null
+++ b/src/doc/concepts.rst
@@ -0,0 +1,29 @@
+===================================
+Basic concepts of the CaosDB server
+===================================
+
+The CaosDB server provides the HTTP API resources to users and client libraries. It uses a plain
+MariaDB/MySQL database as backend for data storage, raw files are stored separately on the file
+system.
+
+
+Configuration
+-------------
+
+Administrators may configure the server through :doc:`configuration
+files<administration/configuration>`. Additionally, configurations may be set via the API if the
+server is in debug mode.
+
+
+Permissions
+-----------
+
+CaosDB has a fine grained role based permission system. Each interaction with the server is
+governed by the current rules of the user, by default this is the ``anonymous`` role. The
+permissions for an action which involves one or more objects are set either manually or via default
+permissions which can be configured. For more detailed information, there is separate
+:doc:`documentation of the permission system<permissions>`.
+
+
+
+
diff --git a/src/doc/conf.py b/src/doc/conf.py
new file mode 100644
index 0000000000000000000000000000000000000000..84e676394d19e8ec35f241f1cf3cc202dafa0d4d
--- /dev/null
+++ b/src/doc/conf.py
@@ -0,0 +1,214 @@
+# -*- coding: utf-8 -*-
+#
+# Configuration file for the Sphinx documentation builder.
+#
+# This file only contains a selection of the most common options. For a full
+# list see the documentation:
+# https://www.sphinx-doc.org/en/master/usage/configuration.html
+
+# -- Path setup --------------------------------------------------------------
+
+# If extensions (or modules to document with autodoc) are in another directory,
+# add these directories to sys.path here. If the directory is relative to the
+# documentation root, use os.path.abspath to make it absolute, like shown here.
+#
+# import os
+# import sys
+# sys.path.insert(0, os.path.abspath('../caosdb'))
+
+import sphinx_rtd_theme
+
+# -- Project information -----------------------------------------------------
+
+project = 'caosdb-server'
+copyright = '2020, IndiScale GmbH'
+author = 'Daniel Hornung'
+
+# The short X.Y version
+version = '0.2'
+# The full version, including alpha/beta/rc tags
+release = '0.2'
+
+
+# -- General configuration ---------------------------------------------------
+
+# If your documentation needs a minimal Sphinx version, state it here.
+#
+# needs_sphinx = '1.0'
+
+# Add any Sphinx extension module names here, as strings. They can be
+# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
+# ones.
+extensions = [
+ 'javasphinx',
+ 'sphinx.ext.autodoc',
+ 'sphinx.ext.intersphinx',
+ 'sphinx.ext.napoleon', # For Google style docstrings
+ "recommonmark", # For markdown files.
+ "sphinx.ext.autosectionlabel", # Allow reference sections using its title
+ "sphinx_rtd_theme",
+]
+
+# Add any paths that contain templates here, relative to this directory.
+templates_path = ['_templates']
+
+# The suffix(es) of source filenames.
+# You can specify multiple suffix as a list of string:
+#
+# source_suffix = ['.rst', '.md']
+source_suffix = '.rst'
+
+# The master toctree document.
+master_doc = 'index'
+
+# The language for content autogenerated by Sphinx. Refer to documentation
+# for a list of supported languages.
+#
+# This is also used if you do content translation via gettext catalogs.
+# Usually you set "language" from the command line for these cases.
+language = None
+
+# List of patterns, relative to source directory, that match files and
+# directories to ignore when looking for source files.
+# This pattern also affects html_static_path and html_extra_path.
+exclude_patterns = []
+
+# The name of the Pygments (syntax highlighting) style to use.
+pygments_style = None
+
+
+# -- Options for HTML output -------------------------------------------------
+
+# The theme to use for HTML and HTML Help pages. See the documentation for
+# a list of builtin themes.
+#
+html_theme = 'sphinx_rtd_theme'
+
+# Theme options are theme-specific and customize the look and feel of a theme
+# further. For a list of options available for each theme, see the
+# documentation.
+#
+# html_theme_options = {}
+
+# Add any paths that contain custom static files (such as style sheets) here,
+# relative to this directory. They are copied after the builtin static files,
+# so a file named "default.css" will overwrite the builtin "default.css".
+# html_static_path = ['_static']
+
+# Custom sidebar templates, must be a dictionary that maps document names
+# to template names.
+#
+# The default sidebars (for documents that don't match any pattern) are
+# defined by theme itself. Builtin themes are using these templates by
+# default: ``['localtoc.html', 'relations.html', 'sourcelink.html',
+# 'searchbox.html']``.
+#
+# html_sidebars = {}
+
+
+# -- Options for HTMLHelp output ---------------------------------------------
+
+# Output file base name for HTML help builder.
+htmlhelp_basename = 'caosdb-serverdoc'
+
+
+# -- Options for LaTeX output ------------------------------------------------
+
+latex_elements = {
+ # The paper size ('letterpaper' or 'a4paper').
+ #
+ # 'papersize': 'letterpaper',
+
+ # The font size ('10pt', '11pt' or '12pt').
+ #
+ # 'pointsize': '10pt',
+
+ # Additional stuff for the LaTeX preamble.
+ #
+ # 'preamble': '',
+
+ # Latex figure (float) alignment
+ #
+ # 'figure_align': 'htbp',
+}
+
+# Grouping the document tree into LaTeX files. List of tuples
+# (source start file, target name, title,
+# author, documentclass [howto, manual, or own class]).
+latex_documents = [
+ (master_doc, 'caosdb-server.tex', 'caosdb-server Documentation',
+ 'IndiScale GmbH', 'manual'),
+]
+
+
+# -- Options for manual page output ------------------------------------------
+
+# One entry per manual page. List of tuples
+# (source start file, name, description, authors, manual section).
+man_pages = [
+ (master_doc, 'caosdb-server', 'caosdb-server Documentation',
+ [author], 1)
+]
+
+
+# -- Options for Texinfo output ----------------------------------------------
+
+# Grouping the document tree into Texinfo files. List of tuples
+# (source start file, target name, title, author,
+# dir menu entry, description, category)
+texinfo_documents = [
+ (master_doc, 'caosdb-server', 'caosdb-server Documentation',
+ author, 'caosdb-server', 'One line description of project.',
+ 'Miscellaneous'),
+]
+
+
+# -- Options for Epub output -------------------------------------------------
+
+# Bibliographic Dublin Core info.
+epub_title = project
+
+# The unique identifier of the text. This can be a ISBN number
+# or the project homepage.
+#
+# epub_identifier = ''
+
+# A unique identification for the text.
+#
+# epub_uid = ''
+
+# A list of files that should not be packed into the epub file.
+epub_exclude_files = ['search.html']
+
+
+# -- Extension configuration -------------------------------------------------
+
+# -- Options for javasphinx --------------------------------------------------
+# See also https://bronto-javasphinx.readthedocs.io/en/latest/
+
+# javadoc_url_map = {
+# '<namespace_here>' : ('<base_url_here>', 'javadoc'),
+# }
+
+
+# -- Options for intersphinx -------------------------------------------------
+
+# https://www.sphinx-doc.org/en/master/usage/extensions/intersphinx.html#confval-intersphinx_mapping
+intersphinx_mapping = {
+ "python": ("https://docs.python.org/", None),
+ "caosdb-pylib": ("https://caosdb.gitlab.io/caosdb-pylib/", None),
+ "caosdb-mysqlbackend": ("https://caosdb.gitlab.io/caosdb-mysqlbackend/", None),
+}
+
+
+# -- Options for autodoc -----------------------------------------------------
+# TODO Which options do we want?
+autodoc_default_options = {
+ 'members': None,
+ 'undoc-members': None,
+}
+
+
+# -- Options for autosectionlabel --------------------------------------------
+
+autosectionlabel_prefix_document = True
diff --git a/src/doc/development/benchmarking.md b/src/doc/development/benchmarking.md
new file mode 100644
index 0000000000000000000000000000000000000000..f2d663f6e69799c7a87a28fbdf32f15fab892ac1
--- /dev/null
+++ b/src/doc/development/benchmarking.md
@@ -0,0 +1,130 @@
+
+# Benchmarking CaosDB #
+
+Benchmarking CaosDB may encompass several distinct areas: How much time is spent in the server's
+Java code, how much time is spent inside the SQL backend, are the same costly methods clalled more
+than once? This documentation tries to answer some questions connected with these benchmarking
+aspects and give you the tools to answer your own questions.
+
+## Tools for the benchmarking ##
+
+For averaging over many runs of comparable requests and for putting the database into a
+representative state, Python scripts are used. The scripts can be found in the `caosdb-dev-tools`
+repository, located at [https://gitlab.indiscale.com/caosdb/src/caosdb-dev-tools](https://gitlab.indiscale.com/caosdb/src/caosdb-dev-tools) in the folder
+`benchmarking`:
+
+### `fill_database.py` ###
+
+This commandline script is meant for filling the database with enough data to represeny an actual
+real-life case, it can easily create hundreds of thousands of Entities.
+
+The script inserts predefined amounts of randomized Entities into the database, RecordTypes,
+Properties and Records. Each Record has a random (but with defined average) number of Properties,
+some of which may be references to other Records which have been inserted before. Actual insertion
+of the Entities into CaosDB is done in chunks of a defined size.
+
+Users can tell the script to store times needed for the insertion of each chunk into a tsv file.
+
+### `measure_execution_time.py` ###
+
+A somewhat outdated script which executes a given query a number of times and then save statistics
+about the `TransactionBenchmark` readings (see below for more information about the transaction
+benchmarks) delivered by the server.
+
+### Benchmarking SQL commands ###
+
+MariaDB and MySQL have a feature to enable the logging of SQL queries' times. This logging must be
+turned on on the SQL server as described in the [upstream documentation](https://mariadb.com/kb/en/general-query-log/). For the Docker
+environment LinkAhead, this can conveniently be done with `linkahead mysqllog {on,off,store}`.
+
+### External JVM profilers ###
+
+Additionally to the transaction benchmarks, it is possible to benchmark the server execution via
+external Java profilers. For example, [VisualVM](https://visualvm.github.io/) can connect to JVMs running locally or remotely
+(e.g. in a Docker container). To enable this in LinkAhead's Docker environment, set
+
+```yaml
+devel:
+ profiler: true
+```
+
+Most profilers, like as VisualVM, only gather cumulative data for call trees, they do not provide
+complete call graphs (as callgrind/kcachegrind would do). They also do not differentiate between
+calls with different query strings, as long as the Java process flow is the same (for example, `FIND
+Record 1234` and `FIND Record A WHICH HAS A Property B WHICH HAS A Property C>100` would be handled
+equally).
+
+## How to set up a representative database ##
+For reproducible results, it makes sense to start off with an empty database and fill it using the
+`fill_database.py` script, for example like this:
+
+```sh
+./fill_database.py -t 500 -p 700 -r 10000 -s 100 --clean
+```
+
+The `--clean` argument is not strictly necessary when the database was empty before, but it may make
+sense when there have been previous runs of the command. This example would create 500 RecordTypes,
+700 Properties and 10000 Records with randomized properties, everything is inserted in chunks of 100
+Entities.
+
+## How to measure request times ##
+
+If the execution of the Java components is of interest, the VisualVM profiler should be started and
+connected to the server before any requests to the server are started.
+
+When doing performance tests which are used for detailed analysis, it is important that
+
+1. CaosDB is in a reproducible state, which should be documented
+2. all measurements are repeated several times to account for inevitable variance in access (for
+ example file system caching, network variablity etc.)
+
+### Filling the database ###
+
+By simply adding the option `-T logfile.tsv` to the `fill_database.py` command above, the times for
+inserting the records are stored in a tsv file and can be analyzed later.
+
+### Obtain statistics about a query ###
+
+To repeat single queries a number of times, `measure_execution_time.py` can be used, for example:
+
+```sh
+./measure_execution_time.py -n 120 -q "FIND MusicalInstrument WHICH IS REFERENCED BY Analysis"
+```
+
+This command executes the query 120 times, additional arguments could even plot the
+TransactionBenchmark results directly.
+
+## What is measured ##
+
+For a consistent interpretation, the exact definitions of the measured times are as follows:
+
+### SQL logs ###
+
+As per https://mariadb.com/kb/en/general-query-log, the logs store only the time at which the SQL
+server received a query, not the duration of the query.
+
+#### Possible future enhancements ####
+
+- The `query_response_time` plugin may be additionally used in the future, see
+ https://mariadb.com/kb/en/query-response-time-plugin
+
+### Transaction benchmarks ###
+
+Transaction benchmarking manually collects timing information for each transaction. At defined
+points, different measurements can be made, accumulated and will finally be returned to the client.
+Benchmark objects may consist of sub benchmarks and have a number of measurement objects, which
+contain the actual statistics.
+
+Because transaction benchmarks must be manually added to the server code, they only monitor those
+code paths where they are added. On the other hand, their manual nature allows for a more
+abstracted analysis of performance bottlenecks.
+
+### Java profiler ###
+
+VisualVM records for each thread the call tree, specifically which methods were called how often and
+how much time was spent inside these methods.
+
+### Global requests ###
+
+Python scripts may measure the global time needed for the execution of each request.
+`fill_database.py` obtains its numbers this way.
diff --git a/src/doc/index.rst b/src/doc/index.rst
new file mode 100644
index 0000000000000000000000000000000000000000..e41d318a2dbf7323e5cb327be1c7a3fe0b5d61ad
--- /dev/null
+++ b/src/doc/index.rst
@@ -0,0 +1,29 @@
+
+Welcome to caosdb-server's documentation!
+=========================================
+
+.. toctree::
+ :maxdepth: 2
+ :caption: Contents:
+ :hidden:
+ :glob:
+
+ Getting started <README_SETUP>
+ Concepts <concepts>
+ Query Language <CaosDB-Query-Language>
+ administration/*
+ development/*
+ API documentation<_apidoc/packages>
+
+Welcome to the CaosDB, the flexible semantic data management toolkit!
+
+This documentation helps you to :doc:`get started<getting_started>`, explains the most important
+:doc:`concepts<concepts>` and offers a range of :doc:`tutorials<tutorials>`.
+
+
+Indices and tables
+==================
+
+* :ref:`genindex`
+* :ref:`modindex`
+* :ref:`search`
diff --git a/src/doc/specs/Server-side-scripting.md b/src/doc/specs/Server-side-scripting.md
new file mode 100644
index 0000000000000000000000000000000000000000..49f56af66a6b65855e36de12144955b2925a4f51
--- /dev/null
+++ b/src/doc/specs/Server-side-scripting.md
@@ -0,0 +1,187 @@
+# Server-Side Scripting API (v0.1)
+
+The CaosDB Server can execute scripts (bash/python/perl) and compiled executables. The scripts can be invoked by a remote-procedure-call (RPC) protocol. Both, the requirements for the scripts and the RPC are described in this document.
+
+## Configuration of the Server
+
+* The CaosDB Server has two relevant properties:
+ 1. `SERVER_SIDE_SCRIPTING_BIN_DIR` is the directory where executable scripts are to be placed. The server will not execute scripts which are out side of this directory. This directory must be readable and executable for the server. But it should not be readable or executable for anyone else.
+ 2. `SERVER_SIDE_SCRIPTING_WORKING_DIR` is the directory under which the server creates temporary working directories. The server needs writing, reading and executing permissions. The temporary working directories are deleted after the scripts have finished and the server has collected the results of the scripts.
+
+## Installing a Server-Side Script (SSS)
+
+* Put your script into the `SERVER_SIDE_SCRIPTING_BIN_DIR` or in any subdirectory and make it executable. Executable files below this directory are called `SSS`.
+* All other files in that directory MUST be ignored by the server, i.e. the server will never call them directly.
+ However, they MAY contain additional data, different implementations, libraries etc.
+* A symlink pointing to an executable MUST be treated as SSS, too.
+
+### Example SERVER\_SIDE\_SCRIPTING\_BIN\_DIR
+
+```
+/
++- script1.py (EXECUTABLE)
++- script2.sh (EXECUTABLE)
++- subdir1/
+ +-script3.pl (EXECUTABLE)
++- subdir2/
+ +- data_for_script2
+ +- another_script.py (EXECUTABLE)
++- script4 -> ./subdir2/another_script.py (SIMLINK to EXECUTABLE)
++- script5 -> /usr/local/bin/external (SIMLINK to EXECUTABLE)
+```
+
+The files `scripts1.py`, `scripts2.sh`, `scripts3.pl` and `another_script.py` are SSS.
+Also the `script4` can be called which would result in calling `another_script.py`
+The script5 points to an executable which is not stored in the `SSD_DIR`.
+
+## Calling a Server-Side Script Via Remote Procedure Call
+
+* Users can invoke scripts via HTTP under the uri `https://$HOST:$HTTPS_PORT/$CONTEXT_ROOT/scripting`. The server accepts POST requests with content types `application/x-www-form-urlencoded` and `multipart/form-data`.
+* There are 6 types of form fields which are processed by the server:
+ 1. A single parameter form field with name="call" (the path to script, relative to the `SERVER_SIDE_SCRIPTING_BIN_DIR`). If there are more than one fields with that name, the behavior is not defined.
+ 2. Zero or more parameter form fields with a unique name which starts with the string `-O` (command line options).
+ 3. Zero or more parameter form fields with a unique name which starts with the string `-p` (positional command line arguments).
+ 4. Zero or more file form fields which have a unique field name and a unique file name (upload files). If the field names or file names are not unique, the behavior is not defined.
+ 5. A parameter form field with name="timeout" (the request timeout)
+ 6. A parameter form field with name="auth-token" and a valid CaosDB AuthToken string or "generate" as value.
+
+## How Does the Server Call the Script?
+* The server executes the script in a temporary working directory, called *PWD* (which will be deleted afterwards).
+* The form parameter `call`, the options, the arguments and file fields construct the string which is executed in a shell. The value of *call* begins the command line.
+* For any option paramter with the name `-Ooption_name` and a value `option_value` a resulting `--option_name=option_value` is appended to the commandLine in no particular order.
+* The values of the positional arguments are appended sorted alphabetically by their field name. E.g. a parameter `-p0` with value `v0` and a parameter `-p1` with value `v1` result in appending `v0 v1` to the command line.
+* All files will be loaded into the directory `$PWD/.upload_files` with their file name (i.e. the form field property). If the file names contain slashes '/' they will build sub-directories in the `./upload_files`.
+* If a file form field has a field name which begins with either `-p` or `-O` the file name with prefix `.upload_files/` is passed as value of an option or as a positional argument to the script. Thus it is possible to distinguish several uploaded files from one another.
+* If there is a "auth-token" field present, another command line options `--auth-token=...` is appended. The value is either the string which was submitted with the POST request, or, if the value was "generate", a refreshed, valid AuthToken which authenticates as the user of the request (why? see below).
+
+### Example HTML Form
+
+```html
+<form action="/scripting" method="post" enctype="multipart/form-data">
+<input type="hidden" name="call" value="my/script.py"/>
+<input type="file" name="-Oconfig-file"/>
+<input type="file" name="-p1"/>
+<input type="text" name="-p0" value="analyze"/>
+<input type="text" name="user"/>
+<input type="text" name="-Oalgorithm" value="fast"/>
+<input type="submit" value="Submit">
+</form>
+```
+
+where the user uploads `my.conf` as `-Oconfig-file` and `my.input.tsv` as `-p1`, would result in this command line:
+
+```
+$SERVER_SIDE_SCRIPTING_BIN_DIR/my/script.py --config-file=.upload_files/my.conf --algorithm=fast analyze .upload_files/my.input.tsv
+```
+## CaosDB Server Response
+
+The CaosDB Server responds with an xml document. The root element is the usual `/Response`. If no errors occurred (which would be represented with `/Response/Error` elements) the result of the script execution is represented as a `/Response/script/` element.
+
+* It has a `code` attribute which contains the exit code value of the execution.
+* It has `stdout` and `stderr` children which contain the dump of the stdout and stderr file of the execution environment.
+* It has a `call` child which contains the command line which was executed (but without a possible `--auth-token` option and with a relative path to the executable).
+
+### Example XML Response
+
+```xml
+<Response>
+<script code="0">
+<call>my/script.py --config-file=.upload_files/my.conf --algorithm=fast analyze .upload_files/my.input.tsv</call>
+<stdout>Result: 0.5</stdout>
+<stderr>Warning: 8 Lines did not contain enough columns</stderr>
+</Response>
+```
+
+## CaosDB Clients and Authentication Token
+
+A special use case for server side scripting is the automated execution of CaosDB clients. These clients need to connect to the CaosDB Server and thus need a way to authenticate themselves.
+For this special case the server can pass an Authentication Token which can be used by the script to authenticate itself. If the invocation request send a particulare AuthToken in the form, this AuthToken will be passed to the script with the `--auth-token` option. Otherwise, if the `auth-token` field has "generate" as value, a fresh AuthToken is generated which belongs to the user who requested the script execution.
+Thus the script is executed (and connects back to the server) as the user who called the script in the first place.
+
+A CaosDB client might use the python client library to connect to the server with that AuthToken.
+
+### Example Script
+
+```python
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+#
+# ** header v3.0
+# This file is a part of the CaosDB Project.
+#
+# Copyright (C) 2018 Research Group Biomedical Physics,
+# Max-Planck-Institute for Dynamics and Self-Organization Göttingen
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see <https://www.gnu.org/licenses/>.
+#
+# ** end header
+#
+"""server_side_script.py.
+
+An example which implements a minimal server-side script.
+
+1) This script expects to find a *.txt file in the .upload_files dir which is
+printed to stdout.
+
+2) It executes a "Count stars" query and prints the result to stdout.
+
+3) It will return with code 0 if everything is ok, or with any code that is
+specified with the commandline option --exit
+"""
+
+import sys
+from os import listdir
+from caosdb import configure_connection, execute_query
+
+
+# parse --auth-token option and configure connection
+CODE = 0
+QUERY = "COUNT stars"
+for arg in sys.argv:
+ if arg.startswith("--auth-token="):
+ auth_token = arg[13:]
+ configure_connection(auth_token=auth_token)
+ if arg.startswith("--exit="):
+ CODE = int(arg[7:])
+ if arg.startswith("--query="):
+ QUERY = arg[8:]
+
+
+############################################################
+# 1 # find and print *.txt file ############################
+############################################################
+
+try:
+ for fname in listdir(".upload_files"):
+ if fname.endswith(".txt"):
+ with open(".upload_files/{}".format(fname)) as f:
+ print(f.read())
+except FileNotFoundError:
+ pass
+
+
+############################################################
+# 2 # query "COUNT stars" ##################################
+############################################################
+
+RESULT = execute_query(QUERY)
+print(RESULT)
+
+############################################################
+# 3 ########################################################
+############################################################
+
+sys.exit(CODE)
+```
+
diff --git a/src/main/java/org/caosdb/server/ServerProperties.java b/src/main/java/org/caosdb/server/ServerProperties.java
index dd12add7f20de16d74b2cda907e543b7f5ddef16..3172a3e88e7790580af1b7865c024fa2b40c017b 100644
--- a/src/main/java/org/caosdb/server/ServerProperties.java
+++ b/src/main/java/org/caosdb/server/ServerProperties.java
@@ -117,6 +117,7 @@ public class ServerProperties extends Properties {
public static final String KEY_SERVER_OWNER = "SERVER_OWNER";
public static final String KEY_SERVER_SIDE_SCRIPTING_BIN_DIR = "SERVER_SIDE_SCRIPTING_BIN_DIR";
+ public static final String KEY_SERVER_SIDE_SCRIPTING_BIN_DIRS = "SERVER_SIDE_SCRIPTING_BIN_DIRS";
public static final String KEY_SERVER_SIDE_SCRIPTING_HOME_DIR = "SERVER_SIDE_SCRIPTING_HOME_DIR";
public static final String KEY_SERVER_SIDE_SCRIPTING_WORKING_DIR =
"SERVER_SIDE_SCRIPTING_WORKING_DIR";
diff --git a/src/main/java/org/caosdb/server/database/BackendTransaction.java b/src/main/java/org/caosdb/server/database/BackendTransaction.java
index 20ec5b71a7ae6f4dd908dc38145df1c41f035e8e..5bf978343d72272309d4f4ab0f039770419c87b8 100644
--- a/src/main/java/org/caosdb/server/database/BackendTransaction.java
+++ b/src/main/java/org/caosdb/server/database/BackendTransaction.java
@@ -59,7 +59,6 @@ import org.caosdb.server.database.backend.implementation.MySQL.MySQLRetrieveProp
import org.caosdb.server.database.backend.implementation.MySQL.MySQLRetrieveQueryTemplateDefinition;
import org.caosdb.server.database.backend.implementation.MySQL.MySQLRetrieveRole;
import org.caosdb.server.database.backend.implementation.MySQL.MySQLRetrieveSparseEntity;
-import org.caosdb.server.database.backend.implementation.MySQL.MySQLRetrieveTransactionHistory;
import org.caosdb.server.database.backend.implementation.MySQL.MySQLRetrieveUser;
import org.caosdb.server.database.backend.implementation.MySQL.MySQLRetrieveVersionHistory;
import org.caosdb.server.database.backend.implementation.MySQL.MySQLRuleLoader;
@@ -116,7 +115,6 @@ import org.caosdb.server.database.backend.interfaces.RetrievePropertiesImpl;
import org.caosdb.server.database.backend.interfaces.RetrieveQueryTemplateDefinitionImpl;
import org.caosdb.server.database.backend.interfaces.RetrieveRoleImpl;
import org.caosdb.server.database.backend.interfaces.RetrieveSparseEntityImpl;
-import org.caosdb.server.database.backend.interfaces.RetrieveTransactionHistoryImpl;
import org.caosdb.server.database.backend.interfaces.RetrieveUserImpl;
import org.caosdb.server.database.backend.interfaces.RetrieveVersionHistoryImpl;
import org.caosdb.server.database.backend.interfaces.RuleLoaderImpl;
@@ -177,7 +175,6 @@ public abstract class BackendTransaction implements Undoable {
setImpl(RetrieveAllImpl.class, MySQLRetrieveAll.class);
setImpl(RegisterSubDomainImpl.class, MySQLRegisterSubDomain.class);
setImpl(RetrieveDatatypesImpl.class, MySQLRetrieveDatatypes.class);
- setImpl(RetrieveTransactionHistoryImpl.class, MySQLRetrieveTransactionHistory.class);
setImpl(RetrieveUserImpl.class, MySQLRetrieveUser.class);
setImpl(RetrieveParentsImpl.class, MySQLRetrieveParents.class);
setImpl(GetFileRecordByPathImpl.class, MySQLGetFileRecordByPath.class);
diff --git a/src/main/java/org/caosdb/server/database/DatabaseUtils.java b/src/main/java/org/caosdb/server/database/DatabaseUtils.java
index c985c1b9bc62270ccf6c5904b1c7f0147bb20de0..4f6a58a4fc50fa91170fad6139384bccb413fa81 100644
--- a/src/main/java/org/caosdb/server/database/DatabaseUtils.java
+++ b/src/main/java/org/caosdb/server/database/DatabaseUtils.java
@@ -216,10 +216,7 @@ public class DatabaseUtils {
ret.fileSize = rs.getLong("FileSize");
ret.fileHash = bytes2UTF8(rs.getBytes("FileHash"));
- ret.version = bytes2UTF8(rs.getBytes("Version"));
- ret.versionSeconds = rs.getLong("VersionSeconds");
- ret.versionNanos = rs.getInt("VersionNanos");
-
+ ret.versionId = bytes2UTF8(rs.getBytes("Version"));
return ret;
}
@@ -258,7 +255,8 @@ public class DatabaseUtils {
}
}
- private static void replace(final Property p, final HashMap<Integer, Property> domainMap) {
+ private static void replace(
+ final Property p, final HashMap<Integer, Property> domainMap, boolean isHead) {
// ... find the corresponding domain and replace it
ReferenceValue ref;
try {
@@ -267,6 +265,17 @@ public class DatabaseUtils {
throw new RuntimeException("This should never happen.");
}
final EntityInterface replacement = domainMap.get((ref.getId()));
+ if (replacement == null) {
+ if (isHead) {
+ throw new NullPointerException("Replacement was null");
+ }
+ // entity has been deleted (we are processing properties of an old entity version)
+ p.setValue(null);
+ p.addWarning(
+ new Message(
+ "The referenced entity has been deleted in the mean time and is not longer available."));
+ return;
+ }
if (replacement.isDescOverride()) {
p.setDescOverride(true);
p.setDescription(replacement.getDescription());
@@ -314,15 +323,18 @@ public class DatabaseUtils {
}
// loop over all properties
+ boolean isHead =
+ e.getVersion().getSuccessors() == null || e.getVersion().getSuccessors().isEmpty();
for (final Property p : protoProperties) {
+
// if this is a replacement
if (p.getStatementStatus() == StatementStatus.REPLACEMENT) {
- replace(p, domainMap);
+ replace(p, domainMap, isHead);
}
for (final Property subP : p.getProperties()) {
if (subP.getStatementStatus() == StatementStatus.REPLACEMENT) {
- replace(subP, domainMap);
+ replace(subP, domainMap, isHead);
}
}
diff --git a/src/main/java/org/caosdb/server/database/backend/implementation/MySQL/MySQLInsertSparseEntity.java b/src/main/java/org/caosdb/server/database/backend/implementation/MySQL/MySQLInsertSparseEntity.java
index 1f1f32c69a260d374ea930ba4e1be63e34cf9ca9..1b945acf566db938f12847f758636dbfddeac3d1 100644
--- a/src/main/java/org/caosdb/server/database/backend/implementation/MySQL/MySQLInsertSparseEntity.java
+++ b/src/main/java/org/caosdb/server/database/backend/implementation/MySQL/MySQLInsertSparseEntity.java
@@ -57,7 +57,7 @@ public class MySQLInsertSparseEntity extends MySQLTransaction implements InsertS
try (final ResultSet rs = insertEntityStmt.executeQuery()) {
if (rs.next()) {
entity.id = rs.getInt("EntityID");
- entity.version = DatabaseUtils.bytes2UTF8(rs.getBytes("Version"));
+ entity.versionId = DatabaseUtils.bytes2UTF8(rs.getBytes("Version"));
} else {
throw new TransactionException("Didn't get new EntityID back.");
}
diff --git a/src/main/java/org/caosdb/server/database/backend/implementation/MySQL/MySQLRetrieveTransactionHistory.java b/src/main/java/org/caosdb/server/database/backend/implementation/MySQL/MySQLRetrieveTransactionHistory.java
deleted file mode 100644
index 3273f49aa85776c4fd323ed4c027f4d1fcd3fbb0..0000000000000000000000000000000000000000
--- a/src/main/java/org/caosdb/server/database/backend/implementation/MySQL/MySQLRetrieveTransactionHistory.java
+++ /dev/null
@@ -1,75 +0,0 @@
-/*
- * ** header v3.0
- * This file is a part of the CaosDB Project.
- *
- * Copyright (C) 2018 Research Group Biomedical Physics,
- * Max-Planck-Institute for Dynamics and Self-Organization Göttingen
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as
- * published by the Free Software Foundation, either version 3 of the
- * License, or (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <https://www.gnu.org/licenses/>.
- *
- * ** end header
- */
-package org.caosdb.server.database.backend.implementation.MySQL;
-
-import static org.caosdb.server.database.DatabaseUtils.bytes2UTF8;
-
-import java.sql.PreparedStatement;
-import java.sql.ResultSet;
-import java.sql.SQLException;
-import java.util.ArrayList;
-import org.caosdb.server.database.access.Access;
-import org.caosdb.server.database.backend.interfaces.RetrieveTransactionHistoryImpl;
-import org.caosdb.server.database.exceptions.TransactionException;
-import org.caosdb.server.database.proto.ProtoTransactionLogMessage;
-
-public class MySQLRetrieveTransactionHistory extends MySQLTransaction
- implements RetrieveTransactionHistoryImpl {
-
- public MySQLRetrieveTransactionHistory(final Access access) {
- super(access);
- }
-
- public static final String STMT_RETRIEVE_HISTORY =
- "SELECT transaction, realm, username, seconds, nanos FROM transaction_log WHERE entity_id=? ";
-
- @Override
- public ArrayList<ProtoTransactionLogMessage> execute(final Integer id)
- throws TransactionException {
- try {
- final PreparedStatement stmt = prepareStatement(STMT_RETRIEVE_HISTORY);
-
- final ArrayList<ProtoTransactionLogMessage> ret = new ArrayList<ProtoTransactionLogMessage>();
- stmt.setInt(1, id);
- final ResultSet rs = stmt.executeQuery();
- try {
- while (rs.next()) {
- final String transaction = bytes2UTF8(rs.getBytes("transaction"));
- final String realm = bytes2UTF8(rs.getBytes("realm"));
- final String username = bytes2UTF8(rs.getBytes("username"));
- final Integer seconds = rs.getInt("seconds");
- final Integer nanos = rs.getInt("nanos");
-
- ret.add(new ProtoTransactionLogMessage(transaction, realm, username, seconds, nanos));
- }
- return ret;
- } finally {
- rs.close();
- }
- } catch (final SQLException e) {
- throw new TransactionException(e);
- } catch (final ConnectionException e) {
- throw new TransactionException(e);
- }
- }
-}
diff --git a/src/main/java/org/caosdb/server/database/backend/implementation/MySQL/MySQLRetrieveVersionHistory.java b/src/main/java/org/caosdb/server/database/backend/implementation/MySQL/MySQLRetrieveVersionHistory.java
index a3c87ae6e97d3b26a99126b61d1d29db1ef6206a..7ecfbfb7ac70a4a5ec248eb0e2be2c590f0676aa 100644
--- a/src/main/java/org/caosdb/server/database/backend/implementation/MySQL/MySQLRetrieveVersionHistory.java
+++ b/src/main/java/org/caosdb/server/database/backend/implementation/MySQL/MySQLRetrieveVersionHistory.java
@@ -61,12 +61,16 @@ public class MySQLRetrieveVersionHistory extends MySQLTransaction
String parentId = DatabaseUtils.bytes2UTF8(rs.getBytes("parent"));
Long childSeconds = rs.getLong("child_seconds");
Integer childNanos = rs.getInt("child_nanos");
+ String childUsername = DatabaseUtils.bytes2UTF8(rs.getBytes("child_username"));
+ String childRealm = DatabaseUtils.bytes2UTF8(rs.getBytes("child_realm"));
VersionHistoryItem v = result.get(childId);
if (v == null) {
v = new VersionHistoryItem();
v.id = childId;
v.seconds = childSeconds;
v.nanos = childNanos;
+ v.username = childUsername;
+ v.realm = childRealm;
result.put(childId, v);
}
diff --git a/src/main/java/org/caosdb/server/database/backend/implementation/MySQL/MySQLUpdateSparseEntity.java b/src/main/java/org/caosdb/server/database/backend/implementation/MySQL/MySQLUpdateSparseEntity.java
index 18b59ef9121bd2f488efcb657a3bf158b6e14308..1c8039253b4c5e74a8054dfa937fec23c55440e8 100644
--- a/src/main/java/org/caosdb/server/database/backend/implementation/MySQL/MySQLUpdateSparseEntity.java
+++ b/src/main/java/org/caosdb/server/database/backend/implementation/MySQL/MySQLUpdateSparseEntity.java
@@ -80,7 +80,7 @@ public class MySQLUpdateSparseEntity extends MySQLTransaction implements UpdateS
ResultSet rs = updateEntityStmt.executeQuery();
if (rs.next()) {
- spe.version = DatabaseUtils.bytes2UTF8(rs.getBytes("Version"));
+ spe.versionId = DatabaseUtils.bytes2UTF8(rs.getBytes("Version"));
}
} catch (final SQLIntegrityConstraintViolationException e) {
diff --git a/src/main/java/org/caosdb/server/database/backend/interfaces/RetrieveTransactionHistoryImpl.java b/src/main/java/org/caosdb/server/database/backend/interfaces/RetrieveTransactionHistoryImpl.java
deleted file mode 100644
index b02e4d9347e572941ab31e3a1a371470e1d8ee5c..0000000000000000000000000000000000000000
--- a/src/main/java/org/caosdb/server/database/backend/interfaces/RetrieveTransactionHistoryImpl.java
+++ /dev/null
@@ -1,32 +0,0 @@
-/*
- * ** header v3.0
- * This file is a part of the CaosDB Project.
- *
- * Copyright (C) 2018 Research Group Biomedical Physics,
- * Max-Planck-Institute for Dynamics and Self-Organization Göttingen
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as
- * published by the Free Software Foundation, either version 3 of the
- * License, or (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <https://www.gnu.org/licenses/>.
- *
- * ** end header
- */
-package org.caosdb.server.database.backend.interfaces;
-
-import java.util.ArrayList;
-import org.caosdb.server.database.exceptions.TransactionException;
-import org.caosdb.server.database.proto.ProtoTransactionLogMessage;
-
-public interface RetrieveTransactionHistoryImpl extends BackendTransactionImpl {
-
- public ArrayList<ProtoTransactionLogMessage> execute(Integer id) throws TransactionException;
-}
diff --git a/src/main/java/org/caosdb/server/database/backend/transaction/InsertSparseEntity.java b/src/main/java/org/caosdb/server/database/backend/transaction/InsertSparseEntity.java
index 1800168ebd2cd6ef4180e23046f95066597086e5..22720f836e5cbd4912f3aedccd25398e80a19324 100644
--- a/src/main/java/org/caosdb/server/database/backend/transaction/InsertSparseEntity.java
+++ b/src/main/java/org/caosdb/server/database/backend/transaction/InsertSparseEntity.java
@@ -69,6 +69,7 @@ public class InsertSparseEntity extends BackendTransaction {
public void cleanUp() {}
});
this.entity.setId(e.id);
- this.entity.setVersion(new Version(e.version));
+ this.entity.setVersion(new Version(e.versionId));
+ this.entity.getVersion().setHead(true);
}
}
diff --git a/src/main/java/org/caosdb/server/database/backend/transaction/IsSubType.java b/src/main/java/org/caosdb/server/database/backend/transaction/IsSubType.java
index d75fb2c2453613c50915678d81f53f696589afd4..683903f396a186c8243b6bba20c0d19e29a61688 100644
--- a/src/main/java/org/caosdb/server/database/backend/transaction/IsSubType.java
+++ b/src/main/java/org/caosdb/server/database/backend/transaction/IsSubType.java
@@ -29,19 +29,36 @@ import org.caosdb.server.database.exceptions.TransactionException;
public class IsSubType extends BackendTransaction {
private final Integer child;
- private final Integer parent;
+ private Integer parent = null;
+ private String parentName = null;
public IsSubType(final Integer child, final Integer parent) {
this.child = child;
this.parent = parent;
}
- private boolean isSubType;
+ public IsSubType(Integer child, String parent) {
+ this.parentName = parent;
+ this.child = child;
+ }
+
+ private Boolean isSubType = null;
@Override
public void execute() throws TransactionException {
final IsSubTypeImpl t = getImplementation(IsSubTypeImpl.class);
- this.isSubType = t.execute(this.child, this.parent);
+
+ if (this.parent == null) {
+ this.isSubType = false;
+ for (Integer parent : execute(new GetIDByName(parentName, false)).getList()) {
+ this.isSubType = t.execute(this.child, parent);
+ if (this.isSubType) {
+ return;
+ }
+ }
+ } else {
+ this.isSubType = t.execute(this.child, this.parent);
+ }
}
public boolean isSubType() {
diff --git a/src/main/java/org/caosdb/server/database/backend/transaction/RetrieveFullEntity.java b/src/main/java/org/caosdb/server/database/backend/transaction/RetrieveFullEntity.java
index 2601367b7312331a981d2b352040962ac3a98a43..f9b6356adfcbe42760f272c4540c25489d060446 100644
--- a/src/main/java/org/caosdb/server/database/backend/transaction/RetrieveFullEntity.java
+++ b/src/main/java/org/caosdb/server/database/backend/transaction/RetrieveFullEntity.java
@@ -1,24 +1,20 @@
/*
- * ** header v3.0
- * This file is a part of the CaosDB Project.
+ * ** header v3.0 This file is a part of the CaosDB Project.
*
- * Copyright (C) 2018 Research Group Biomedical Physics,
- * Max-Planck-Institute for Dynamics and Self-Organization Göttingen
- * Copyright (C) 2020 Timm Fitschen <t.fitschen@indiscale.com>
- * Copyright (C) 2020 IndiScale GmbH <info@indiscale.com>
+ * Copyright (C) 2018 Research Group Biomedical Physics, Max-Planck-Institute for Dynamics and
+ * Self-Organization Göttingen Copyright (C) 2020 Timm Fitschen <t.fitschen@indiscale.com> Copyright
+ * (C) 2020 IndiScale GmbH <info@indiscale.com>
*
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as
- * published by the Free Software Foundation, either version 3 of the
- * License, or (at your option) any later version.
+ * This program is free software: you can redistribute it and/or modify it under the terms of the
+ * GNU Affero General Public License as published by the Free Software Foundation, either version 3
+ * of the License, or (at your option) any later version.
*
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU Affero General Public License for more details.
+ * This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without
+ * even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Affero General Public License for more details.
*
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <https://www.gnu.org/licenses/>.
+ * You should have received a copy of the GNU Affero General Public License along with this program.
+ * If not, see <https://www.gnu.org/licenses/>.
*
* ** end header
*/
@@ -27,6 +23,7 @@ package org.caosdb.server.database.backend.transaction;
import java.util.LinkedList;
import java.util.List;
import org.caosdb.server.database.BackendTransaction;
+import org.caosdb.server.database.exceptions.EntityDoesNotExistException;
import org.caosdb.server.datatype.ReferenceDatatype;
import org.caosdb.server.datatype.ReferenceValue;
import org.caosdb.server.entity.EntityInterface;
@@ -102,12 +99,12 @@ public class RetrieveFullEntity extends BackendTransaction {
execute(new RetrieveSparseEntity(e));
if (e.getEntityStatus() == EntityStatus.VALID) {
+ execute(new RetrieveVersionInfo(e));
if (e.getRole() == Role.QueryTemplate) {
execute(new RetrieveQueryTemplateDefinition(e));
}
execute(new RetrieveParents(e));
execute(new RetrieveProperties(e));
- execute(new RetrieveVersionInfo(e));
// recursion! retrieveSubEntities calls retrieveFull sometimes, but with reduced selectors.
if (selections != null && !selections.isEmpty()) {
@@ -124,10 +121,9 @@ public class RetrieveFullEntity extends BackendTransaction {
*/
public void retrieveSubEntities(EntityInterface e, List<Selection> selections) {
for (final Selection s : selections) {
+ String propertyName = s.getSelector();
if (s.getSubselection() != null) {
- String propertyName = s.getSelector();
-
// Find matching (i.e. referencing) Properties
for (Property p : e.getProperties()) {
// get reference properties by name.
@@ -142,10 +138,47 @@ public class RetrieveFullEntity extends BackendTransaction {
ReferenceValue value = (ReferenceValue) p.getValue();
RetrieveEntity ref = new RetrieveEntity(value.getId());
- // recursion! (Only for the matching selections)
+ // recursion! (Only for the matching selections)
retrieveFullEntity(ref, getSubSelects(selections, propertyName));
value.setEntity(ref, true);
}
+ continue;
+ }
+ try {
+ boolean isSubtype = execute(new IsSubType(p.getId(), propertyName)).isSubType();
+ if (isSubtype) {
+ if (p.getValue() != null) {
+ if (p.getDatatype() instanceof ReferenceDatatype) {
+ try {
+ p.parseValue();
+ } catch (Message m) {
+ p.addError(m);
+ }
+
+ ReferenceValue value = (ReferenceValue) p.getValue();
+ RetrieveEntity ref = new RetrieveEntity(value.getId());
+ // recursion! (Only for the matching selections)
+ retrieveFullEntity(ref, getSubSelects(selections, propertyName));
+ value.setEntity(ref, true);
+ p.setName(propertyName);
+ }
+ }
+ }
+ } catch (EntityDoesNotExistException exc) {
+ // unknown parent name.
+ }
+ }
+ } else {
+ for (Property p : e.getProperties()) {
+ if (!propertyName.equalsIgnoreCase(p.getName())) {
+ try {
+ boolean isSubtype = execute(new IsSubType(p.getId(), propertyName)).isSubType();
+ if (isSubtype) {
+ p.setName(propertyName);
+ }
+ } catch (EntityDoesNotExistException exc) {
+ // unknown parent name.
+ }
}
}
}
diff --git a/src/main/java/org/caosdb/server/database/backend/transaction/RetrieveTransactionHistory.java b/src/main/java/org/caosdb/server/database/backend/transaction/RetrieveTransactionHistory.java
deleted file mode 100644
index 632ce20cfe71d0e116bbe3987a60c45b11085eec..0000000000000000000000000000000000000000
--- a/src/main/java/org/caosdb/server/database/backend/transaction/RetrieveTransactionHistory.java
+++ /dev/null
@@ -1,59 +0,0 @@
-/*
- * ** header v3.0
- * This file is a part of the CaosDB Project.
- *
- * Copyright (C) 2018 Research Group Biomedical Physics,
- * Max-Planck-Institute for Dynamics and Self-Organization Göttingen
- * Copyright (C) 2019 IndiScale GmbH
- * Copyright (C) 2019 Timm Fitschen (t.fitschen@indiscale.com)
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as
- * published by the Free Software Foundation, either version 3 of the
- * License, or (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <https://www.gnu.org/licenses/>.
- *
- * ** end header
- */
-package org.caosdb.server.database.backend.transaction;
-
-import java.util.ArrayList;
-import org.caosdb.datetime.UTCDateTime;
-import org.caosdb.server.database.BackendTransaction;
-import org.caosdb.server.database.backend.interfaces.RetrieveTransactionHistoryImpl;
-import org.caosdb.server.database.exceptions.TransactionException;
-import org.caosdb.server.database.proto.ProtoTransactionLogMessage;
-import org.caosdb.server.entity.EntityInterface;
-import org.caosdb.server.utils.TransactionLogMessage;
-
-public class RetrieveTransactionHistory extends BackendTransaction {
-
- private final EntityInterface entity;
-
- public RetrieveTransactionHistory(final EntityInterface entity) {
- this.entity = entity;
- }
-
- @Override
- protected void execute() {
- final RetrieveTransactionHistoryImpl t =
- getImplementation(RetrieveTransactionHistoryImpl.class);
- process(t.execute(entity.getId()));
- }
-
- private void process(final ArrayList<ProtoTransactionLogMessage> l) throws TransactionException {
- for (final ProtoTransactionLogMessage t : l) {
- final UTCDateTime dateTime = UTCDateTime.UTCSeconds(t.seconds, t.nanos);
-
- this.entity.addTransactionLog(
- new TransactionLogMessage(t.transaction, this.entity, t.username, dateTime));
- }
- }
-}
diff --git a/src/main/java/org/caosdb/server/database/backend/transaction/RetrieveVersionHistory.java b/src/main/java/org/caosdb/server/database/backend/transaction/RetrieveVersionHistory.java
index 6077d6491fda93b96262b48783092578af37f4f8..a5079c4e033882c4b1068615551b5132e4ee0cba 100644
--- a/src/main/java/org/caosdb/server/database/backend/transaction/RetrieveVersionHistory.java
+++ b/src/main/java/org/caosdb/server/database/backend/transaction/RetrieveVersionHistory.java
@@ -22,58 +22,43 @@
*/
package org.caosdb.server.database.backend.transaction;
-import java.util.Collection;
import java.util.HashMap;
-import org.caosdb.server.database.CacheableBackendTransaction;
-import org.caosdb.server.database.backend.interfaces.RetrieveVersionHistoryImpl;
+import org.caosdb.datetime.UTCDateTime;
import org.caosdb.server.database.exceptions.TransactionException;
import org.caosdb.server.database.proto.VersionHistoryItem;
import org.caosdb.server.entity.EntityInterface;
+import org.caosdb.server.entity.Version;
-public abstract class RetrieveVersionHistory
- extends CacheableBackendTransaction<Integer, HashMap<String, VersionHistoryItem>> {
-
- // TODO
- // private static final ICacheAccess<String, Version> cache =
- // Cache.getCache("BACKEND_RetrieveVersionHistory");
- private EntityInterface entity;
- private HashMap<String, VersionHistoryItem> map;
-
- public static void removeCached(Integer entityId) {
- // TODO
- }
+public class RetrieveVersionHistory extends VersionTransaction {
public RetrieveVersionHistory(EntityInterface e) {
- super(null); // TODO caching
- this.entity = e;
- }
-
- @Override
- public HashMap<String, VersionHistoryItem> executeNoCache() throws TransactionException {
- RetrieveVersionHistoryImpl impl = getImplementation(RetrieveVersionHistoryImpl.class);
- return impl.execute(getKey());
+ super(e);
}
- /** After this method call, the version map is available to the object. */
@Override
protected void process(HashMap<String, VersionHistoryItem> map) throws TransactionException {
- this.map = map;
+ super.process(map);
+ if (!map.isEmpty()) getEntity().setVersion(getHistory());
}
@Override
- protected Integer getKey() {
- return entity.getId();
- }
-
- public HashMap<String, VersionHistoryItem> getMap() {
- return this.map;
- }
-
- public EntityInterface getEntity() {
- return this.entity;
+ protected Version getVersion(String id) {
+ Version v = new Version(id);
+ VersionHistoryItem i = getHistoryItems().get(v.getId());
+ if (i != null) {
+ v.setDate(UTCDateTime.UTCSeconds(i.seconds, i.nanos));
+ v.setUsername(i.username);
+ v.setRealm(i.realm);
+ }
+ return v;
}
- public Collection<VersionHistoryItem> getList() {
- return this.map.values();
+ private Version getHistory() {
+ Version v = getVersion(getEntity().getVersion().getId());
+ v.setSuccessors(getSuccessors(v.getId(), true));
+ v.setPredecessors(getPredecessors(v.getId(), true));
+ v.setHead(v.getSuccessors().isEmpty());
+ v.setCompleteHistory(true);
+ return v;
}
}
diff --git a/src/main/java/org/caosdb/server/database/backend/transaction/RetrieveVersionInfo.java b/src/main/java/org/caosdb/server/database/backend/transaction/RetrieveVersionInfo.java
index 7d9fe5fd7ea42cf3d6aa29a127a7f82f7ec1764c..8d0e54c95d17d0eb83af541b9fee2fb605b17ec9 100644
--- a/src/main/java/org/caosdb/server/database/backend/transaction/RetrieveVersionInfo.java
+++ b/src/main/java/org/caosdb/server/database/backend/transaction/RetrieveVersionInfo.java
@@ -23,14 +23,12 @@
package org.caosdb.server.database.backend.transaction;
import java.util.HashMap;
-import java.util.LinkedList;
-import org.caosdb.datetime.UTCDateTime;
import org.caosdb.server.database.exceptions.TransactionException;
import org.caosdb.server.database.proto.VersionHistoryItem;
import org.caosdb.server.entity.EntityInterface;
import org.caosdb.server.entity.Version;
-public class RetrieveVersionInfo extends RetrieveVersionHistory {
+public class RetrieveVersionInfo extends VersionTransaction {
public RetrieveVersionInfo(EntityInterface e) {
super(e);
@@ -39,46 +37,19 @@ public class RetrieveVersionInfo extends RetrieveVersionHistory {
@Override
protected void process(HashMap<String, VersionHistoryItem> map) throws TransactionException {
super.process(map); // Make the map available to the object.
- if (!map.isEmpty()) getVersion();
+ if (!map.isEmpty()) getEntity().setVersion(getVersion());
}
- public Version getVersion() {
- Version v = getEntity().getVersion();
- VersionHistoryItem i = getMap().get(v.getId());
- if (i != null) v.setDate(UTCDateTime.UTCSeconds(i.seconds, i.nanos));
-
- v.setPredecessors(getPredecessors(v.getId()));
- v.setSuccessors(getSuccessors(v.getId()));
- return v;
- }
-
- /** Return a list of direct children. */
- private LinkedList<Version> getSuccessors(String id) {
- LinkedList<Version> result = new LinkedList<>();
-
- outer:
- for (VersionHistoryItem i : getList()) {
- if (i.parents != null)
- for (String p : i.parents) {
- if (id.equals(p)) {
- Version successor = new Version(i.id, i.seconds, i.nanos);
- result.add(successor);
- continue outer;
- }
- }
- }
- return result;
+ @Override
+ protected Version getVersion(String id) {
+ return new Version(id);
}
- /** Return a list of direct parents. */
- private LinkedList<Version> getPredecessors(String id) {
- LinkedList<Version> result = new LinkedList<>();
- if (getMap().containsKey(id) && getMap().get(id).parents != null)
- for (String p : getMap().get(id).parents) {
- VersionHistoryItem i = getMap().get(p);
- Version predecessor = new Version(i.id, i.seconds, i.nanos);
- result.add(predecessor);
- }
- return result;
+ public Version getVersion() {
+ Version v = getVersion(getEntity().getVersion().getId());
+ v.setPredecessors(getPredecessors(v.getId(), false));
+ v.setSuccessors(getSuccessors(v.getId(), false));
+ v.setHead(v.getSuccessors().isEmpty());
+ return v;
}
}
diff --git a/src/main/java/org/caosdb/server/database/backend/transaction/UpdateEntity.java b/src/main/java/org/caosdb/server/database/backend/transaction/UpdateEntity.java
index 9c20cb1d2d112e8d64ce5be2cfa8a7f8da20fe10..2bf965575884f29640a80ec3d3ab284b7779c8f2 100644
--- a/src/main/java/org/caosdb/server/database/backend/transaction/UpdateEntity.java
+++ b/src/main/java/org/caosdb/server/database/backend/transaction/UpdateEntity.java
@@ -54,6 +54,7 @@ public class UpdateEntity extends BackendTransaction {
execute(new InsertEntityProperties(e));
+ VersionTransaction.removeCached(e.getId());
execute(new RetrieveVersionInfo(e));
}
}
diff --git a/src/main/java/org/caosdb/server/database/backend/transaction/UpdateSparseEntity.java b/src/main/java/org/caosdb/server/database/backend/transaction/UpdateSparseEntity.java
index 2385ec843a3d499d1184b58eca06f8ae05cbe4f8..8548fb0477de9d8b52b18b8132af15295dbc9353 100644
--- a/src/main/java/org/caosdb/server/database/backend/transaction/UpdateSparseEntity.java
+++ b/src/main/java/org/caosdb/server/database/backend/transaction/UpdateSparseEntity.java
@@ -52,6 +52,6 @@ public class UpdateSparseEntity extends BackendTransaction {
t.execute(spe);
- this.entity.setVersion(new Version(spe.version));
+ this.entity.setVersion(new Version(spe.versionId));
}
}
diff --git a/src/main/java/org/caosdb/server/database/backend/transaction/VersionTransaction.java b/src/main/java/org/caosdb/server/database/backend/transaction/VersionTransaction.java
new file mode 100644
index 0000000000000000000000000000000000000000..ee617a1c1110e9a01e9a6d6ee4c799e44cd66ab3
--- /dev/null
+++ b/src/main/java/org/caosdb/server/database/backend/transaction/VersionTransaction.java
@@ -0,0 +1,159 @@
+/*
+ * ** header v3.0
+ * This file is a part of the CaosDB Project.
+ *
+ * Copyright (C) 2020 IndiScale GmbH <info@indiscale.com>
+ * Copyright (C) 2020 Timm Fitschen <t.fitschen@indiscale.com>
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <https://www.gnu.org/licenses/>.
+ *
+ * ** end header
+ */
+package org.caosdb.server.database.backend.transaction;
+
+import java.util.HashMap;
+import java.util.LinkedList;
+import java.util.List;
+import org.apache.commons.jcs.access.behavior.ICacheAccess;
+import org.caosdb.server.caching.Cache;
+import org.caosdb.server.database.CacheableBackendTransaction;
+import org.caosdb.server.database.backend.interfaces.RetrieveVersionHistoryImpl;
+import org.caosdb.server.database.exceptions.TransactionException;
+import org.caosdb.server.database.proto.VersionHistoryItem;
+import org.caosdb.server.entity.EntityInterface;
+import org.caosdb.server.entity.Version;
+
+/**
+ * Abstract base class which retrieves and caches the full, but flat version history. The
+ * implementations then use the flat version history to construct either single version information
+ * items (see {@link RetrieveVersionInfo}) or the complete history as a tree (see {@link
+ * RetrieveVersionHistory})
+ *
+ * @author Timm Fitschen (t.fitschen@indiscale.com)
+ */
+public abstract class VersionTransaction
+ extends CacheableBackendTransaction<Integer, HashMap<String, VersionHistoryItem>> {
+
+ private static final ICacheAccess<Integer, HashMap<String, VersionHistoryItem>> cache =
+ Cache.getCache("BACKEND_RetrieveVersionHistory");
+ private EntityInterface entity;
+
+ /** A map of all history items which belong to this entity. The keys are the version ids. */
+ private HashMap<String, VersionHistoryItem> historyItems;
+
+ /**
+ * Invalidate a cache item. This should be called upon update of entities.
+ *
+ * @param entityId
+ */
+ public static void removeCached(Integer entityId) {
+ cache.remove(entityId);
+ }
+
+ public VersionTransaction(EntityInterface e) {
+ super(cache);
+ this.entity = e;
+ }
+
+ @Override
+ public HashMap<String, VersionHistoryItem> executeNoCache() throws TransactionException {
+ RetrieveVersionHistoryImpl impl = getImplementation(RetrieveVersionHistoryImpl.class);
+ return impl.execute(getKey());
+ }
+
+ /** After this method call, the version map is available to the object. */
+ @Override
+ protected void process(HashMap<String, VersionHistoryItem> historyItems)
+ throws TransactionException {
+ this.historyItems = historyItems;
+ }
+
+ @Override
+ protected Integer getKey() {
+ return entity.getId();
+ }
+
+ public HashMap<String, VersionHistoryItem> getHistoryItems() {
+ return this.historyItems;
+ }
+
+ public EntityInterface getEntity() {
+ return this.entity;
+ }
+
+ /**
+ * Return a list of direct predecessors. The predecessors are constructed by {@link
+ * #getVersion(String)}.
+ *
+ * <p>If transitive is true, this function is called recursively on the predecessors as well,
+ * resulting in a list of trees of predecessors, with the direct predecessors at the root(s).
+ *
+ * @param versionId
+ * @param transitive
+ * @return A list of predecessors.
+ */
+ protected List<Version> getPredecessors(String versionId, boolean transitive) {
+ LinkedList<Version> result = new LinkedList<>();
+ if (getHistoryItems().containsKey(versionId)
+ && getHistoryItems().get(versionId).parents != null)
+ for (String p : getHistoryItems().get(versionId).parents) {
+ Version predecessor = getVersion(p);
+ if (transitive) {
+ predecessor.setPredecessors(getPredecessors(p, transitive));
+ }
+ result.add(predecessor);
+ }
+ return result;
+ }
+
+ /**
+ * To be implemented by the base class. The idea is, that the base class decides which information
+ * is being included into the Version instance.
+ *
+ * @param versionId - the id of the version
+ * @return
+ */
+ protected abstract Version getVersion(String versionId);
+
+ /**
+ * Return a list of direct successors. The successors are constructed by {@link
+ * #getVersion(String)}.
+ *
+ * <p>If transitive is true, this function is called recursively on the successors as well,
+ * resulting in a list of trees of successors, with the direct successors at the root(s).
+ *
+ * @param versionId
+ * @param transitive
+ * @return A list of successors.
+ */
+ protected List<Version> getSuccessors(String versionId, boolean transitive) {
+ LinkedList<Version> result = new LinkedList<>();
+
+ outer:
+ for (VersionHistoryItem i : getHistoryItems().values()) {
+ if (i.parents != null)
+ for (String p : i.parents) {
+ if (versionId.equals(p)) {
+ Version successor = getVersion(i.id);
+ result.add(successor);
+ if (transitive) {
+ successor.setSuccessors(getSuccessors(i.id, transitive));
+ }
+ continue outer;
+ }
+ }
+ }
+ return result;
+ }
+}
diff --git a/src/main/java/org/caosdb/server/database/proto/ProtoTransactionLogMessage.java b/src/main/java/org/caosdb/server/database/proto/ProtoTransactionLogMessage.java
deleted file mode 100644
index dd5cdbebcda7dd2c5c7f9ca93be69972ecef0e2c..0000000000000000000000000000000000000000
--- a/src/main/java/org/caosdb/server/database/proto/ProtoTransactionLogMessage.java
+++ /dev/null
@@ -1,50 +0,0 @@
-/*
- * ** header v3.0
- * This file is a part of the CaosDB Project.
- *
- * Copyright (C) 2018 Research Group Biomedical Physics,
- * Max-Planck-Institute for Dynamics and Self-Organization Göttingen
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as
- * published by the Free Software Foundation, either version 3 of the
- * License, or (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <https://www.gnu.org/licenses/>.
- *
- * ** end header
- */
-package org.caosdb.server.database.proto;
-
-import java.io.Serializable;
-
-public class ProtoTransactionLogMessage implements Serializable {
-
- public ProtoTransactionLogMessage(
- final String transaction,
- final String realm,
- final String username,
- final long seconds,
- final int nanos) {
- this.transaction = transaction;
- this.realm = realm;
- this.username = username;
- this.seconds = seconds;
- this.nanos = nanos;
- }
-
- public ProtoTransactionLogMessage() {}
-
- private static final long serialVersionUID = -5856887517281480754L;
- public String transaction = null;
- public String realm;
- public String username = null;
- public Long seconds = null;
- public Integer nanos = null;
-}
diff --git a/src/main/java/org/caosdb/server/database/proto/SparseEntity.java b/src/main/java/org/caosdb/server/database/proto/SparseEntity.java
index 3d5a29c9c4a69670d49347864b7ed885e758192f..d2292c58825ac6807e24f3ad3d086b29dd8a6e7e 100644
--- a/src/main/java/org/caosdb/server/database/proto/SparseEntity.java
+++ b/src/main/java/org/caosdb/server/database/proto/SparseEntity.java
@@ -38,9 +38,7 @@ public class SparseEntity extends VerySparseEntity {
public String filePath = null;
public Long fileSize = null;
public Long fileChecked = null;
- public String version = null;
- public Long versionSeconds = null;
- public Integer versionNanos = null;
+ public String versionId = null;
@Override
public String toString() {
@@ -54,9 +52,7 @@ public class SparseEntity extends VerySparseEntity {
.append(this.fileHash)
.append(this.filePath)
.append(this.fileSize)
- .append(this.version)
- .append(this.versionSeconds)
- .append(this.versionNanos)
+ .append(this.versionId)
.toString();
}
}
diff --git a/src/main/java/org/caosdb/server/database/proto/VersionHistoryItem.java b/src/main/java/org/caosdb/server/database/proto/VersionHistoryItem.java
index 26109760ecde53ea20405bd02095cd951818677d..4c0a60ce57cce03cfaa7a2c4d41af7039c463838 100644
--- a/src/main/java/org/caosdb/server/database/proto/VersionHistoryItem.java
+++ b/src/main/java/org/caosdb/server/database/proto/VersionHistoryItem.java
@@ -1,13 +1,27 @@
package org.caosdb.server.database.proto;
import java.io.Serializable;
-import java.util.LinkedList;
+import java.util.List;
+/**
+ * This class is a flat, data-only representation of a single item of version information. This
+ * class is an intermediate representation which abstracts away the data base results and comes in a
+ * form which is easily cacheable.
+ *
+ * @author Timm Fitschen (t.fitschen@indiscale.com)
+ */
public class VersionHistoryItem implements Serializable {
- private static final long serialVersionUID = 6319362462701459355L;
+ private static final long serialVersionUID = 428030617967255942L;
public String id = null;
- public LinkedList<String> parents = null;
+ public List<String> parents = null;
public Long seconds = null;
public Integer nanos = null;
+ public String username = null;
+ public String realm = null;
+
+ @Override
+ public String toString() {
+ return id;
+ }
}
diff --git a/src/main/java/org/caosdb/server/entity/Entity.java b/src/main/java/org/caosdb/server/entity/Entity.java
index 28f42308e303c0315b432c017fd2147119c5b346..845bb25a3cee665fd8f0985c6969f53dfc868025 100644
--- a/src/main/java/org/caosdb/server/entity/Entity.java
+++ b/src/main/java/org/caosdb/server/entity/Entity.java
@@ -34,7 +34,6 @@ import org.apache.shiro.SecurityUtils;
import org.apache.shiro.authz.AuthorizationException;
import org.apache.shiro.authz.Permission;
import org.apache.shiro.subject.Subject;
-import org.caosdb.datetime.UTCDateTime;
import org.caosdb.server.CaosDBException;
import org.caosdb.server.database.proto.SparseEntity;
import org.caosdb.server.database.proto.VerySparseEntity;
@@ -957,7 +956,8 @@ public class Entity extends AbstractObservable implements EntityInterface {
@Override
public String toString() {
- return (hasId() ? "(" + getId().toString() + ")" : "()")
+ return (getRole().toString())
+ + (hasId() ? "(" + getId().toString() + ")" : "()")
+ (hasCuid() ? "[" + getCuid() + "]" : "[]")
+ (hasName() ? "(" + getName() + ")" : "()")
+ (hasDatatype() ? "{" + getDatatype().toString() + "}" : "{}");
@@ -1078,11 +1078,9 @@ public class Entity extends AbstractObservable implements EntityInterface {
setId(spe.id);
this.setRole(spe.role);
setEntityACL(spe.acl);
- UTCDateTime versionDate = null;
- if (spe.versionSeconds != null) {
- versionDate = UTCDateTime.UTCSeconds(spe.versionSeconds, spe.versionNanos);
+ if (spe.versionId != null) {
+ this.version = new Version(spe.versionId);
}
- this.version = new Version(spe.version, versionDate);
if (!isNameOverride()) {
setName(spe.name);
diff --git a/src/main/java/org/caosdb/server/entity/StatementStatus.java b/src/main/java/org/caosdb/server/entity/StatementStatus.java
index ec6b0b8ec8cc33fc0fa2c41119e82b2b807aa89f..b3d7a613d26eba432308272b3bd932d2fdb54b26 100644
--- a/src/main/java/org/caosdb/server/entity/StatementStatus.java
+++ b/src/main/java/org/caosdb/server/entity/StatementStatus.java
@@ -22,12 +22,21 @@
*/
package org.caosdb.server.entity;
+/**
+ * The statement status has two purposes.
+ *
+ * <p>1. Storing the importance of an entity (any of OBLIGATORY, RECOMMENDED, SUGGESTED, or FIX). 2.
+ * Marking an entity as a REPLACEMENT which is needed for flat representation of deeply nested
+ * properties. This constant is only used for internal processes and has no meaning in the API. That
+ * is also the reason why this enum is not called "Importance". Apart from that, in most cases its
+ * meaning is identical to the importance of an entity.
+ *
+ * @author Timm Fitschen (t.fitschen@indiscale.com)
+ */
public enum StatementStatus {
OBLIGATORY,
RECOMMENDED,
SUGGESTED,
FIX,
- SUBTYPING,
- INHERITANCE,
REPLACEMENT
}
diff --git a/src/main/java/org/caosdb/server/entity/Version.java b/src/main/java/org/caosdb/server/entity/Version.java
index 3f9e148583686887fab39038aa427bccf1c69b25..5529c7c5f5e8c83d95ad5afae7c1ae786806df89 100644
--- a/src/main/java/org/caosdb/server/entity/Version.java
+++ b/src/main/java/org/caosdb/server/entity/Version.java
@@ -20,7 +20,7 @@
package org.caosdb.server.entity;
-import java.util.LinkedList;
+import java.util.List;
import org.caosdb.datetime.UTCDateTime;
/**
@@ -31,25 +31,35 @@ import org.caosdb.datetime.UTCDateTime;
public class Version {
private String id = null;
- private LinkedList<Version> predecessors = null;
- private LinkedList<Version> successors = null;
+ private String username = null;
+ private String realm = null;
+ private List<Version> predecessors = null;
+ private List<Version> successors = null;
private UTCDateTime date = null;
+ private boolean isHead = false;
+ private boolean isCompleteHistory = false;
public Version(String id, long seconds, int nanos) {
- this(id, UTCDateTime.UTCSeconds(seconds, nanos));
+ this(id, UTCDateTime.UTCSeconds(seconds, nanos), null, null);
}
- public Version(String id, UTCDateTime date) {
+ public Version(String id, UTCDateTime date, String username, String realm) {
this.id = id;
this.date = date;
+ this.username = username;
+ this.realm = realm;
}
public Version(String id) {
- this(id, null);
+ this(id, null, null, null);
}
public Version() {}
+ public Version(String id, UTCDateTime timestamp) {
+ this(id, timestamp, null, null);
+ }
+
public UTCDateTime getDate() {
return date;
}
@@ -66,19 +76,55 @@ public class Version {
this.id = id;
}
- public LinkedList<Version> getSuccessors() {
+ public List<Version> getSuccessors() {
return successors;
}
- public void setSuccessors(LinkedList<Version> successors) {
+ public void setSuccessors(List<Version> successors) {
this.successors = successors;
}
- public LinkedList<Version> getPredecessors() {
+ public List<Version> getPredecessors() {
return predecessors;
}
- public void setPredecessors(LinkedList<Version> predecessors) {
+ public void setPredecessors(List<Version> predecessors) {
this.predecessors = predecessors;
}
+
+ public String getRealm() {
+ return realm;
+ }
+
+ public void setRealm(String realm) {
+ this.realm = realm;
+ }
+
+ public String getUsername() {
+ return username;
+ }
+
+ public void setUsername(String username) {
+ this.username = username;
+ }
+
+ public void setDate(Long timestamp) {
+ this.date = UTCDateTime.SystemMillisToUTCDateTime(timestamp);
+ }
+
+ public boolean isHead() {
+ return isHead;
+ }
+
+ public void setHead(boolean isHead) {
+ this.isHead = isHead;
+ }
+
+ public boolean isCompleteHistory() {
+ return isCompleteHistory;
+ }
+
+ public void setCompleteHistory(boolean isCompleteHistory) {
+ this.isCompleteHistory = isCompleteHistory;
+ }
}
diff --git a/src/main/java/org/caosdb/server/entity/wrapper/Property.java b/src/main/java/org/caosdb/server/entity/wrapper/Property.java
index 305a9d83cbfad835baa547670fae8e186417330d..b3ce45b56527e40382d3971a61b43f9bc054d21f 100644
--- a/src/main/java/org/caosdb/server/entity/wrapper/Property.java
+++ b/src/main/java/org/caosdb/server/entity/wrapper/Property.java
@@ -120,7 +120,7 @@ public class Property extends EntityWrapper {
@Override
public String toString() {
- return "IMPLPROPERTY " + this.entity.toString();
+ return "IMPLPROPERTY (" + this.getStatementStatus() + ")" + this.entity.toString();
}
public void setIsName(final boolean b) {
diff --git a/src/main/java/org/caosdb/server/entity/xml/EntityToElementStrategy.java b/src/main/java/org/caosdb/server/entity/xml/EntityToElementStrategy.java
index e6dc2b21d088ceeff9e0e7a9b60547b288893f18..a829d711aa6345a16c9c368db27cb2f146f044b8 100644
--- a/src/main/java/org/caosdb/server/entity/xml/EntityToElementStrategy.java
+++ b/src/main/java/org/caosdb/server/entity/xml/EntityToElementStrategy.java
@@ -87,7 +87,7 @@ public class EntityToElementStrategy implements ToElementStrategy {
if (setFieldStrategy.isToBeSet("id") && entity.hasId()) {
element.setAttribute("id", Integer.toString(entity.getId()));
}
- if (setFieldStrategy.isToBeSet("version") && entity.hasVersion()) {
+ if (entity.hasVersion()) {
Element v = new VersionXMLSerializer().toElement(entity.getVersion());
element.addContent(v);
}
diff --git a/src/main/java/org/caosdb/server/entity/xml/VersionXMLSerializer.java b/src/main/java/org/caosdb/server/entity/xml/VersionXMLSerializer.java
index 65ec9880dfef43a4e460bba692ece6c9f87faa7e..40d19a5f3a0ef6d9448ed8567840e610be712391 100644
--- a/src/main/java/org/caosdb/server/entity/xml/VersionXMLSerializer.java
+++ b/src/main/java/org/caosdb/server/entity/xml/VersionXMLSerializer.java
@@ -30,28 +30,43 @@ import org.jdom2.Element;
* @author Timm Fitschen <t.fitschen@indiscale.com>
*/
class VersionXMLSerializer {
+
public Element toElement(Version version) {
- Element result = new Element("Version");
- result.setAttribute("id", version.getId());
- if (version.getDate() != null) {
- result.setAttribute("date", version.getDate().toDateTimeString(TimeZone.getDefault()));
- }
+ return toElement(version, "Version");
+ }
+
+ private Element toElement(Version version, String tag) {
+ Element element = new Element(tag);
+ setAttributes(version, element);
if (version.getPredecessors() != null) {
for (Version p : version.getPredecessors()) {
- Element predecessor = new Element("Predecessor");
- predecessor.setAttribute("id", p.getId());
- predecessor.setAttribute("date", p.getDate().toDateTimeString(TimeZone.getDefault()));
- result.addContent(predecessor);
+ element.addContent(toElement(p, "Predecessor"));
}
}
if (version.getSuccessors() != null) {
for (Version s : version.getSuccessors()) {
- Element successor = new Element("Successor");
- successor.setAttribute("id", s.getId());
- successor.setAttribute("date", s.getDate().toDateTimeString(TimeZone.getDefault()));
- result.addContent(successor);
+ element.addContent(toElement(s, "Successor"));
}
}
- return result;
+ return element;
+ }
+
+ private void setAttributes(Version version, Element element) {
+ element.setAttribute("id", version.getId());
+ if (version.getUsername() != null) {
+ element.setAttribute("username", version.getUsername());
+ }
+ if (version.getRealm() != null) {
+ element.setAttribute("realm", version.getRealm());
+ }
+ if (version.getDate() != null) {
+ element.setAttribute("date", version.getDate().toDateTimeString(TimeZone.getDefault()));
+ }
+ if (version.isHead()) {
+ element.setAttribute("head", "true");
+ }
+ if (version.isCompleteHistory()) {
+ element.setAttribute("completeHistory", "true");
+ }
}
}
diff --git a/src/main/java/org/caosdb/server/jobs/Job.java b/src/main/java/org/caosdb/server/jobs/Job.java
index d1e3fef39fcd6a2fb5175e4dd396686bcf773b4e..5d8dd5d98a00960561c6b97f8e1f4fa118eafd0b 100644
--- a/src/main/java/org/caosdb/server/jobs/Job.java
+++ b/src/main/java/org/caosdb/server/jobs/Job.java
@@ -57,6 +57,11 @@ import org.caosdb.server.utils.Observer;
import org.caosdb.server.utils.ServerMessages;
import org.reflections.Reflections;
+/**
+ * This is a Job.
+ *
+ * @todo Describe me.
+ */
public abstract class Job extends AbstractObservable implements Observer {
private Transaction<? extends TransactionContainer> transaction = null;
private Mode mode = null;
diff --git a/src/main/java/org/caosdb/server/jobs/core/History.java b/src/main/java/org/caosdb/server/jobs/core/History.java
index a7c58979f49b2a7abdc313924dc8fb6b86bbffd4..5594b5afdb841436dbdd63f96b8e2c44faef51c6 100644
--- a/src/main/java/org/caosdb/server/jobs/core/History.java
+++ b/src/main/java/org/caosdb/server/jobs/core/History.java
@@ -23,7 +23,7 @@
package org.caosdb.server.jobs.core;
import org.apache.shiro.authz.AuthorizationException;
-import org.caosdb.server.database.backend.transaction.RetrieveTransactionHistory;
+import org.caosdb.server.database.backend.transaction.RetrieveVersionHistory;
import org.caosdb.server.entity.EntityInterface;
import org.caosdb.server.jobs.FlagJob;
import org.caosdb.server.jobs.JobAnnotation;
@@ -32,6 +32,11 @@ import org.caosdb.server.permissions.EntityPermission;
import org.caosdb.server.utils.EntityStatus;
import org.caosdb.server.utils.ServerMessages;
+/**
+ * Retrieves the complete version history of each entity and appends it to the entity.
+ *
+ * @author Timm Fitschen (t.fitschen@indiscale.com)
+ */
@JobAnnotation(time = JobExecutionTime.POST_TRANSACTION, flag = "H")
public class History extends FlagJob {
@@ -42,7 +47,7 @@ public class History extends FlagJob {
if (entity.getId() != null && entity.getId() > 0) {
try {
entity.checkPermission(EntityPermission.RETRIEVE_HISTORY);
- final RetrieveTransactionHistory t = new RetrieveTransactionHistory(entity);
+ final RetrieveVersionHistory t = new RetrieveVersionHistory(entity);
execute(t);
} catch (final AuthorizationException e) {
entity.setEntityStatus(EntityStatus.UNQUALIFIED);
diff --git a/src/main/java/org/caosdb/server/jobs/core/Inheritance.java b/src/main/java/org/caosdb/server/jobs/core/Inheritance.java
index 3c08f9f281335c64a2af28ae131ff99d2c5ed6ca..086787e8bd4752ccf0a25f8684afa5a5c4a33c82 100644
--- a/src/main/java/org/caosdb/server/jobs/core/Inheritance.java
+++ b/src/main/java/org/caosdb/server/jobs/core/Inheritance.java
@@ -23,26 +23,36 @@
package org.caosdb.server.jobs.core;
import java.util.ArrayList;
+import java.util.List;
import org.caosdb.server.database.backend.transaction.RetrieveFullEntity;
import org.caosdb.server.entity.Entity;
import org.caosdb.server.entity.EntityInterface;
import org.caosdb.server.entity.Message;
import org.caosdb.server.entity.Message.MessageType;
+import org.caosdb.server.entity.StatementStatus;
import org.caosdb.server.entity.wrapper.Property;
import org.caosdb.server.jobs.EntityJob;
import org.caosdb.server.transaction.Insert;
import org.caosdb.server.transaction.Update;
import org.caosdb.server.utils.EntityStatus;
+/**
+ * Add all those properties from the parent to the child which have the same importance (or higher).
+ *
+ * @author Timm Fitschen (t.fitschen@indiscale.com)
+ */
public class Inheritance extends EntityJob {
+ /*
+ * Storing which properties of the properties of the parents should be inherited by the child.
+ */
public enum INHERITANCE_MODE {
- NONE,
- ALL,
- OBLIGATORY,
- FIX,
- RECOMMENDED,
- DONE
+ NONE, // inherit no properties from this parent
+ ALL, // inherit all inheritable properties, alias for suggested
+ OBLIGATORY, // inherit only obligatory properties
+ RECOMMENDED, // inherit obligatory and recommended properties
+ SUGGESTED, // inherit all inheritable properties, alias for all
+ FIX, // inherit fix properties only (deprecated)
};
public static final Message ILLEGAL_INHERITANCE_MODE =
@@ -66,51 +76,22 @@ public class Inheritance extends EntityJob {
INHERITANCE_MODE.valueOf(parent.getFlags().get("inheritance").toUpperCase());
// mark inheritance flag as done
- parent.setFlag("inheritance", "done");
- if (inheritance == INHERITANCE_MODE.NONE || inheritance == INHERITANCE_MODE.DONE) {
+ parent.setFlag("inheritance", null);
+ if (inheritance == INHERITANCE_MODE.NONE) {
break parentLoop;
}
runJobFromSchedule(getEntity(), CheckParValid.class);
- execute(new RetrieveFullEntity(parent));
-
- if (parent.hasProperties()) {
- // loop over all properties of the parent and
- // collect
- // properties to be transfered
- for (final EntityInterface parProperty : parent.getProperties()) {
- switch (inheritance) {
- case ALL:
- transfer.add(parProperty);
- break;
- case RECOMMENDED:
- if (parProperty
- .getStatementStatus()
- .toString()
- .equalsIgnoreCase(INHERITANCE_MODE.RECOMMENDED.toString())) {
- transfer.add(parProperty);
- }
- case OBLIGATORY:
- if (parProperty
- .getStatementStatus()
- .toString()
- .equalsIgnoreCase(INHERITANCE_MODE.OBLIGATORY.toString())) {
- transfer.add(parProperty);
- }
- case FIX:
- if (parProperty
- .getStatementStatus()
- .toString()
- .equalsIgnoreCase(INHERITANCE_MODE.FIX.toString())) {
- transfer.add(parProperty);
- }
- break;
- default:
- break;
- }
- }
+ // try to get the parent entity from the current transaction container
+ EntityInterface foreign = getEntityByName(parent.getName());
+ if (foreign == null) {
+ // was not in container -> retrieve from database.
+ execute(new RetrieveFullEntity(parent));
+ foreign = parent;
}
+
+ collectInheritedProperties(transfer, foreign, inheritance);
} catch (final IllegalArgumentException e) {
parent.addWarning(ILLEGAL_INHERITANCE_MODE);
break parentLoop;
@@ -142,9 +123,10 @@ public class Inheritance extends EntityJob {
}
final INHERITANCE_MODE inheritance =
INHERITANCE_MODE.valueOf(property.getFlags().get("inheritance").toUpperCase());
+
// mark inheritance flag as done
- property.setFlag("inheritance", "done");
- if (inheritance == INHERITANCE_MODE.NONE || inheritance == INHERITANCE_MODE.DONE) {
+ property.setFlag("inheritance", null);
+ if (inheritance == INHERITANCE_MODE.NONE) {
break propertyLoop;
}
@@ -164,48 +146,14 @@ public class Inheritance extends EntityJob {
} else {
execute(new RetrieveFullEntity(validProperty));
}
-
- if (validProperty.getEntityStatus() == EntityStatus.VALID
- && validProperty.hasProperties()) {
- // loop over all properties of the property and
- // collect
- // properties to be transfered
- for (final EntityInterface propProperty : validProperty.getProperties()) {
- switch (inheritance) {
- case ALL:
- transfer.add(propProperty);
- break;
- case RECOMMENDED:
- if (propProperty
- .getStatementStatus()
- .toString()
- .equalsIgnoreCase(INHERITANCE_MODE.RECOMMENDED.toString())) {
- transfer.add(propProperty);
- }
- case OBLIGATORY:
- if (propProperty
- .getStatementStatus()
- .toString()
- .equalsIgnoreCase(INHERITANCE_MODE.OBLIGATORY.toString())) {
- transfer.add(propProperty);
- }
- case FIX:
- if (propProperty
- .getStatementStatus()
- .toString()
- .equalsIgnoreCase(INHERITANCE_MODE.FIX.toString())) {
- transfer.add(propProperty);
- }
- break;
- default:
- break;
- }
- }
+ if (validProperty.getEntityStatus() == EntityStatus.VALID) {
+ collectInheritedProperties(transfer, validProperty, inheritance);
}
} catch (final IllegalArgumentException e) {
property.addWarning(ILLEGAL_INHERITANCE_MODE);
break propertyLoop;
}
+
// transfer properties if they are not implemented yet
outerLoop:
for (final EntityInterface prop : transfer) {
@@ -222,4 +170,52 @@ public class Inheritance extends EntityJob {
}
}
}
+
+ /**
+ * Put all those properties from the `from` entity into the `transfer` List which match the
+ * INHERITANCE_MODE.
+ *
+ * <p>That means:
+ *
+ * @param transfer
+ * @param from
+ * @param inheritance
+ */
+ private void collectInheritedProperties(
+ List<EntityInterface> transfer, EntityInterface from, INHERITANCE_MODE inheritance) {
+ if (from.hasProperties()) {
+ for (final EntityInterface propProperty : from.getProperties()) {
+ switch (inheritance) {
+ // the following cases are ordered according to their importance level and use a
+ // fall-through.
+ case ALL:
+ case SUGGESTED:
+ if (propProperty.getStatementStatus() == StatementStatus.SUGGESTED) {
+ transfer.add(propProperty);
+ }
+ // fall-through!
+ case RECOMMENDED:
+ if (propProperty.getStatementStatus() == StatementStatus.RECOMMENDED) {
+ transfer.add(propProperty);
+ }
+ // fall-through!
+ case OBLIGATORY:
+ if (propProperty.getStatementStatus() == StatementStatus.OBLIGATORY) {
+ transfer.add(propProperty);
+ }
+ break;
+ case FIX:
+ if (propProperty.getStatementStatus() == StatementStatus.FIX) {
+ transfer.add(propProperty);
+ propProperty.addWarning(
+ new Message(
+ MessageType.Warning,
+ "DeprecationWarning: The inheritance of fix properties is deprecated and will be removed from the API in the near future. Clients have to copy fix properties by themselves, if necessary."));
+ }
+ default:
+ break;
+ }
+ }
+ }
+ }
}
diff --git a/src/main/java/org/caosdb/server/query/Backreference.java b/src/main/java/org/caosdb/server/query/Backreference.java
index 5f8105c823a231a14853e1db370649c7a7189649..b9114ad55147ca92163a55cf374d3b13093ead98 100644
--- a/src/main/java/org/caosdb/server/query/Backreference.java
+++ b/src/main/java/org/caosdb/server/query/Backreference.java
@@ -103,9 +103,12 @@ public class Backreference implements EntityFilterInterface, QueryInterface {
return "@(" + getEntity() + "," + getProperty() + ")";
}
- /** */
@Override
public void apply(final QueryInterface query) throws QueryException {
+ if (query.isVersioned() && hasSubProperty()) {
+ throw new UnsupportedOperationException(
+ "Versioned queries are not supported for subqueries yet. Please file a feature request.");
+ }
final long t1 = System.currentTimeMillis();
this.query = query;
this.targetSet = query.getTargetSet();
@@ -113,7 +116,7 @@ public class Backreference implements EntityFilterInterface, QueryInterface {
initBackRef(query);
final CallableStatement callApplyBackRef =
- getConnection().prepareCall("call applyBackReference(?,?,?,?,?)");
+ getConnection().prepareCall("call applyBackReference(?,?,?,?,?,?)");
callApplyBackRef.setString(1, getSourceSet()); // sourceSet
this.statistics.put("sourceSet", getSourceSet());
this.statistics.put(
@@ -145,6 +148,7 @@ public class Backreference implements EntityFilterInterface, QueryInterface {
callApplyBackRef.setNull(4, VARCHAR);
}
callApplyBackRef.setBoolean(5, hasSubProperty()); // subQuery?
+ callApplyBackRef.setBoolean(6, this.isVersioned());
executeStmt(query, callApplyBackRef);
callApplyBackRef.close();
@@ -214,7 +218,7 @@ public class Backreference implements EntityFilterInterface, QueryInterface {
final long t3 = System.currentTimeMillis();
try (final PreparedStatement callFinishSubProperty =
- getConnection().prepareCall("call finishSubProperty(?,?,?)")) {
+ getConnection().prepareCall("call finishSubProperty(?,?,?,?)")) {
callFinishSubProperty.setString(1, query.getSourceSet()); // sourceSet
if (query.getTargetSet() != null) { // targetSet
callFinishSubProperty.setString(2, query.getTargetSet());
@@ -222,6 +226,7 @@ public class Backreference implements EntityFilterInterface, QueryInterface {
callFinishSubProperty.setNull(2, VARCHAR);
}
callFinishSubProperty.setString(3, this.sourceSet); // list
+ callFinishSubProperty.setBoolean(4, this.isVersioned());
final ResultSet rs2 = callFinishSubProperty.executeQuery();
rs2.next();
this.statistics.put("finishSubPropertyStmt", rs2.getString("finishSubPropertyStmt"));
@@ -320,4 +325,9 @@ public class Backreference implements EntityFilterInterface, QueryInterface {
public void addBenchmark(final String str, final long time) {
this.query.addBenchmark(this.getClass().getSimpleName() + "." + str, time);
}
+
+ @Override
+ public boolean isVersioned() {
+ return this.query.isVersioned();
+ }
}
diff --git a/src/main/java/org/caosdb/server/query/CQLLexer.g4 b/src/main/java/org/caosdb/server/query/CQLLexer.g4
index 62cb12c09c00a2f0005a8b9e2b7d5cc8f6b753e5..ec363be59893ad72cf412dbdc9d1f6d6644da5be 100644
--- a/src/main/java/org/caosdb/server/query/CQLLexer.g4
+++ b/src/main/java/org/caosdb/server/query/CQLLexer.g4
@@ -34,6 +34,25 @@ BY:
[Bb][Yy]
;
+fragment
+OF_f:
+ [Oo][Ff]
+;
+
+fragment
+ANY_f:
+ [Aa][Nn][Yy]
+;
+
+fragment
+VERSION_f:
+ [Vv][Ee][Rr][Ss][Ii][Oo][Nn]
+;
+
+ANY_VERSION_OF:
+ (ANY_f EMPTY_SPACE VERSION_f EMPTY_SPACE OF_f)
+;
+
SELECT:
[Ss][Ee][Ll][Ee][Cc][Tt] -> pushMode(SELECT_MODE)
;
diff --git a/src/main/java/org/caosdb/server/query/CQLParser.g4 b/src/main/java/org/caosdb/server/query/CQLParser.g4
index 8b5e5a94aaea5d62ba60068a1ad652f7b51f1bd4..0daeff47a95ff38391ce3f9b8004bec642b0ccb2 100644
--- a/src/main/java/org/caosdb/server/query/CQLParser.g4
+++ b/src/main/java/org/caosdb/server/query/CQLParser.g4
@@ -31,11 +31,12 @@ options { tokenVocab = CQLLexer; }
import java.util.List;
}
-cq returns [Query.Type t, List<Query.Selection> s, Query.Pattern e, Query.Role r, EntityFilterInterface filter]
+cq returns [Query.Type t, List<Query.Selection> s, Query.Pattern e, Query.Role r, EntityFilterInterface filter, VersionFilter v]
@init{
$s = null;
$e = null;
$r = null;
+ $v = VersionFilter.UNVERSIONED;
$filter = null;
}
:
@@ -44,6 +45,7 @@ cq returns [Query.Type t, List<Query.Selection> s, Query.Pattern e, Query.Role r
SELECT prop_sel {$s = $prop_sel.s;} FROM {$t = Query.Type.FIND;}
| FIND {$t = Query.Type.FIND;}
| COUNT {$t = Query.Type.COUNT;})
+ (version {$v = $version.v;})?
(
(
role {$r = $role.r;}
@@ -56,6 +58,14 @@ cq returns [Query.Type t, List<Query.Selection> s, Query.Pattern e, Query.Role r
EOF
;
+version returns [VersionFilter v]
+ @init{
+ $v = null;
+ }
+:
+ ANY_VERSION_OF {$v = VersionFilter.ANY_VERSION;}
+;
+
prop_sel returns [List<Query.Selection> s]
@init{
$s = new LinkedList<Query.Selection>();
@@ -312,7 +322,11 @@ conjunction returns [Conjunction c] locals [Conjunction dummy]
f1 = filter_expression {$c.add($f1.efi);}
|
LPAREN
- f4 = filter_expression {$c.add($f4.efi);}
+ (
+ f4 = filter_expression {$c.add($f4.efi);}
+ | disjunction {$c.add($disjunction.d);}
+ | c3=conjunction {$c.addAll($c3.c);}
+ )
RPAREN
)
(
@@ -327,7 +341,7 @@ conjunction returns [Conjunction c] locals [Conjunction dummy]
(
f3 = filter_expression {$c.add($f3.efi);}
| disjunction {$c.add($disjunction.d);}
- | c2=conjunction {$c.addAll($c2.c);}
+ | c2=conjunction {$c.addAll($c2.c);}
)
RPAREN
)
@@ -344,7 +358,11 @@ disjunction returns [Disjunction d]
f1 = filter_expression {$d.add($f1.efi);}
|
LPAREN
- f4 = filter_expression {$d.add($f4.efi);}
+ (
+ f4 = filter_expression {$d.add($f4.efi);}
+ | conjunction {$d.add($conjunction.c);}
+ | d3 = disjunction {$d.addAll($d3.d);}
+ )
RPAREN
)
(
diff --git a/src/main/java/org/caosdb/server/query/Conjunction.java b/src/main/java/org/caosdb/server/query/Conjunction.java
index a8fd9cd16296dfdf846bd81d2c25676537cd590c..2e44aea4f64a43ebb2a1c1edb153c6f68c172f8e 100644
--- a/src/main/java/org/caosdb/server/query/Conjunction.java
+++ b/src/main/java/org/caosdb/server/query/Conjunction.java
@@ -63,8 +63,9 @@ public class Conjunction extends EntityFilterContainer implements QueryInterface
// generate empty temporary targetSet if query.getTargetSet() is not
// empty anyways.
final PreparedStatement callInitEmptyTarget =
- getConnection().prepareStatement("call initEmptyTargetSet(?)");
+ getConnection().prepareStatement("call initEmptyTargetSet(?,?)");
callInitEmptyTarget.setString(1, query.getTargetSet());
+ callInitEmptyTarget.setBoolean(2, query.isVersioned());
final ResultSet initEmptyTargetResultSet = callInitEmptyTarget.executeQuery();
if (initEmptyTargetResultSet.next()) {
this.targetSet = bytes2UTF8(initEmptyTargetResultSet.getBytes("newTableName"));
@@ -149,4 +150,9 @@ public class Conjunction extends EntityFilterContainer implements QueryInterface
public void addBenchmark(final String str, final long time) {
this.query.addBenchmark(this.getClass().getSimpleName() + "." + str, time);
}
+
+ @Override
+ public boolean isVersioned() {
+ return this.query.isVersioned();
+ }
}
diff --git a/src/main/java/org/caosdb/server/query/Disjunction.java b/src/main/java/org/caosdb/server/query/Disjunction.java
index 3bd2f32fdd3bfb64ad2a07c527b91136e965c22b..31288f2f169d4ebce1cf3d4da5b2ab369fcd5e1f 100644
--- a/src/main/java/org/caosdb/server/query/Disjunction.java
+++ b/src/main/java/org/caosdb/server/query/Disjunction.java
@@ -51,7 +51,8 @@ public class Disjunction extends EntityFilterContainer implements QueryInterface
// targetTable
// which will be used to collect the entities.
final PreparedStatement callInitDisjunctionFilter =
- getConnection().prepareStatement("call initDisjunctionFilter()");
+ getConnection().prepareStatement("call initDisjunctionFilter(?)");
+ callInitDisjunctionFilter.setBoolean(1, this.isVersioned());
final ResultSet initDisjuntionFilterResultSet = callInitDisjunctionFilter.executeQuery();
if (initDisjuntionFilterResultSet.next()) {
this.targetSet = bytes2UTF8(initDisjuntionFilterResultSet.getBytes("newTableName"));
@@ -69,11 +70,12 @@ public class Disjunction extends EntityFilterContainer implements QueryInterface
if (query.getTargetSet() == null) {
// calculate the difference and store to sourceSet
- final CallableStatement callFinishNegationFilter =
- getConnection().prepareCall("call calcIntersection(?,?)");
- callFinishNegationFilter.setString(1, getSourceSet());
- callFinishNegationFilter.setString(2, this.targetSet);
- callFinishNegationFilter.execute();
+ final CallableStatement callFinishDisjunctionFilter =
+ getConnection().prepareCall("call calcIntersection(?,?,?)");
+ callFinishDisjunctionFilter.setString(1, getSourceSet());
+ callFinishDisjunctionFilter.setString(2, this.targetSet);
+ callFinishDisjunctionFilter.setBoolean(3, this.isVersioned());
+ callFinishDisjunctionFilter.execute();
}
// ELSE: the query.getTargetSet() is identical to targetSet and is
// not
@@ -136,4 +138,9 @@ public class Disjunction extends EntityFilterContainer implements QueryInterface
public void addBenchmark(final String str, final long time) {
this.query.addBenchmark(this.getClass().getSimpleName() + "." + str, time);
}
+
+ @Override
+ public boolean isVersioned() {
+ return this.query.isVersioned();
+ }
}
diff --git a/src/main/java/org/caosdb/server/query/IDFilter.java b/src/main/java/org/caosdb/server/query/IDFilter.java
index 26054bb16f0eb39a581a4fa0da6c62a67c6fc385..12f53b44472d5b897c3d8b2a065388969c1edcbc 100644
--- a/src/main/java/org/caosdb/server/query/IDFilter.java
+++ b/src/main/java/org/caosdb/server/query/IDFilter.java
@@ -59,7 +59,7 @@ public class IDFilter implements EntityFilterInterface {
final Connection connection = query.getConnection();
// applyIDFilter(sourceSet, targetSet, o, vInt, agg)
final CallableStatement callIDFilter =
- connection.prepareCall("call applyIDFilter(?,?,?,?,?)");
+ connection.prepareCall("call applyIDFilter(?,?,?,?,?,?)");
callIDFilter.setString(1, query.getSourceSet()); // sourceSet
if (query.getTargetSet() != null) { // targetSet
@@ -89,6 +89,8 @@ public class IDFilter implements EntityFilterInterface {
callIDFilter.setString(5, getAggregate());
}
+ // versioning
+ callIDFilter.setBoolean(6, query.isVersioned());
callIDFilter.execute();
callIDFilter.close();
} catch (final SQLException e) {
diff --git a/src/main/java/org/caosdb/server/query/Negation.java b/src/main/java/org/caosdb/server/query/Negation.java
index d4243580bf52511d05699c7c80e69015ea2a0dac..981b4c8f2ca19a1a6ec47df9e85f108ab7bb01ac 100644
--- a/src/main/java/org/caosdb/server/query/Negation.java
+++ b/src/main/java/org/caosdb/server/query/Negation.java
@@ -90,8 +90,8 @@ public class Negation implements EntityFilterInterface, QueryInterface {
// generate empty temporary targetSet if query.getTargetSet() is not
// empty anyways.
final PreparedStatement callInitEmptyTarget =
- getConnection().prepareStatement("call initEmptyTargetSet(?)");
- callInitEmptyTarget.setString(1, query.getTargetSet());
+ getConnection().prepareStatement("call initDisjunctionFilter(?)");
+ callInitEmptyTarget.setBoolean(1, query.isVersioned());
final ResultSet initEmptyTargetResultSet = callInitEmptyTarget.executeQuery();
if (initEmptyTargetResultSet.next()) {
this.targetSet = bytes2UTF8(initEmptyTargetResultSet.getBytes("newTableName"));
@@ -100,19 +100,22 @@ public class Negation implements EntityFilterInterface, QueryInterface {
this.filter.apply(this);
- if (query.getTargetSet() != null && !this.targetSet.equalsIgnoreCase(query.getTargetSet())) {
- // merge temporary targetSet with query.getTargetSet()
+ if (query.getTargetSet() == null) {
+ // intersect temporary targetSet with query.getSourceSet()
final CallableStatement callFinishConjunctionFilter =
- query.getConnection().prepareCall("call calcUnion(?,?)");
- callFinishConjunctionFilter.setString(1, query.getTargetSet());
+ query.getConnection().prepareCall("call calcDifference(?,?,?)");
+ callFinishConjunctionFilter.setString(1, query.getSourceSet());
callFinishConjunctionFilter.setString(2, this.targetSet);
+ callFinishConjunctionFilter.setBoolean(3, this.isVersioned());
callFinishConjunctionFilter.execute();
- } else if (query.getTargetSet() == null) {
- // intersect temporary targetSet with query.getSourceSet()
+ } else {
+ // merge temporary targetSet with query.getTargetSet()
final CallableStatement callFinishConjunctionFilter =
- query.getConnection().prepareCall("call calcDifference(?,?)");
- callFinishConjunctionFilter.setString(1, query.getSourceSet());
+ query.getConnection().prepareCall("call calcComplementUnion(?,?,?,?)");
+ callFinishConjunctionFilter.setString(1, query.getTargetSet());
callFinishConjunctionFilter.setString(2, this.targetSet);
+ callFinishConjunctionFilter.setString(3, query.getSourceSet());
+ callFinishConjunctionFilter.setBoolean(4, this.isVersioned());
callFinishConjunctionFilter.execute();
}
} catch (final SQLException e) {
@@ -167,4 +170,9 @@ public class Negation implements EntityFilterInterface, QueryInterface {
public void addBenchmark(final String str, final long time) {
this.query.addBenchmark(this.getClass().getSimpleName() + "." + str, time);
}
+
+ @Override
+ public boolean isVersioned() {
+ return this.query.isVersioned();
+ }
}
diff --git a/src/main/java/org/caosdb/server/query/POV.java b/src/main/java/org/caosdb/server/query/POV.java
index 03b603d0df70d1159c1d78f1626e23faab40e033..fcc719e6ff24299d0b5a62a241442e08fa33962f 100644
--- a/src/main/java/org/caosdb/server/query/POV.java
+++ b/src/main/java/org/caosdb/server/query/POV.java
@@ -155,7 +155,7 @@ public class POV implements EntityFilterInterface {
this.unit = getUnit(unitStr);
} catch (final ParserException e) {
e.printStackTrace();
- throw new UnsupportedOperationException();
+ throw new UnsupportedOperationException("Could not parse the unit.");
}
this.stdUnitSig = this.unit.normalize().getSignature();
@@ -214,6 +214,10 @@ public class POV implements EntityFilterInterface {
@Override
public void apply(final QueryInterface query) throws QueryException {
+ if (query.isVersioned() && hasSubProperty()) {
+ throw new UnsupportedOperationException(
+ "Versioned queries are not supported for subqueries yet. Please file a feature request.");
+ }
final long t1 = System.currentTimeMillis();
try {
this.connection = query.getConnection();
@@ -226,9 +230,9 @@ public class POV implements EntityFilterInterface {
// applyPOV(sourceSet, targetSet, propertiesTable, refIdsTable, o,
// vText, vInt,
// vDouble,
- // vDatetime, vDateTimeDotNotation, agg, pname)
+ // vDatetime, vDateTimeDotNotation, agg, pname, versioned)
final CallableStatement callPOV =
- this.connection.prepareCall("call applyPOV(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)");
+ this.connection.prepareCall("call applyPOV(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)");
callPOV.setString(1, query.getSourceSet()); // sourceSet
this.statistics.put("sourceSet", query.getSourceSet());
this.statistics.put(
@@ -313,6 +317,10 @@ public class POV implements EntityFilterInterface {
}
if (getAggregate() != null) { // agg
+ if (query.isVersioned()) {
+ throw new UnsupportedOperationException(
+ "Versioned queries are not supported for aggregate functions like GREATES or SMALLEST in the filters.");
+ }
callPOV.setString(14, getAggregate());
} else {
callPOV.setNull(14, VARCHAR);
@@ -323,6 +331,7 @@ public class POV implements EntityFilterInterface {
} else {
callPOV.setNull(15, VARCHAR);
}
+ callPOV.setBoolean(16, query.isVersioned());
prefix.add("#executeStmt");
executeStmt(callPOV, query);
prefix.pop();
@@ -420,7 +429,9 @@ public class POV implements EntityFilterInterface {
if (hasSubProperty() && this.targetSet != null) {
try (PreparedStatement stmt =
- query.getConnection().prepareStatement("call initEmptyTargetSet(NULL)")) {
+ query.getConnection().prepareStatement("call initEmptyTargetSet(?, ?)")) {
+ stmt.setNull(1, VARCHAR);
+ stmt.setBoolean(2, query.isVersioned());
// generate new targetSet
final ResultSet rs = stmt.executeQuery();
if (rs.next()) {
diff --git a/src/main/java/org/caosdb/server/query/Query.java b/src/main/java/org/caosdb/server/query/Query.java
index c3873cf3b4ba2322373dde13f4dfd72e6a94c7e4..695a295da5f64ec788c4dfbb3332734ead44b289 100644
--- a/src/main/java/org/caosdb/server/query/Query.java
+++ b/src/main/java/org/caosdb/server/query/Query.java
@@ -169,12 +169,22 @@ public class Query implements QueryInterface, ToElementable, TransactionInterfac
}
}
+ public static class IdVersionPair {
+ public IdVersionPair(Integer id, String version) {
+ this.id = id;
+ this.version = version;
+ }
+
+ public Integer id;
+ public String version;
+ }
+
private static boolean filterEntitiesWithoutRetrievePermisions =
!CaosDBServer.getServerProperty(
ServerProperties.KEY_QUERY_FILTER_ENTITIES_WITHOUT_RETRIEVE_PERMISSIONS)
.equalsIgnoreCase("FALSE");
- List<Integer> resultSet = null;
+ List<IdVersionPair> resultSet = null;
private final String query;
private Pattern entity = null;
private Role role = null;
@@ -189,6 +199,7 @@ public class Query implements QueryInterface, ToElementable, TransactionInterfac
private Type type = null;
private final ArrayList<ToElementable> messages = new ArrayList<>();
private Access access;
+ private boolean versioned = false;
public Type getType() {
return this.type;
@@ -216,7 +227,7 @@ public class Query implements QueryInterface, ToElementable, TransactionInterfac
*/
private void initResultSetWithNameIDAndChildren() throws SQLException {
final CallableStatement callInitEntity =
- getConnection().prepareCall("call initEntity(?,?,?,?,?)");
+ getConnection().prepareCall("call initEntity(?,?,?,?,?,?)");
try {
callInitEntity.setInt(1, Integer.parseInt(this.entity.toString()));
@@ -243,6 +254,7 @@ public class Query implements QueryInterface, ToElementable, TransactionInterfac
break;
}
callInitEntity.setString(5, this.sourceSet);
+ callInitEntity.setBoolean(6, this.versioned);
callInitEntity.execute();
callInitEntity.close();
}
@@ -260,8 +272,8 @@ public class Query implements QueryInterface, ToElementable, TransactionInterfac
applyQueryTemplates(this, getSourceSet());
}
- if (this.role != null) {
- final RoleFilter roleFilter = new RoleFilter(this.role, "=");
+ if (this.role != null && this.role != Role.ENTITY) {
+ final RoleFilter roleFilter = new RoleFilter(this.role, "=", this.versioned);
roleFilter.apply(this);
}
@@ -312,7 +324,9 @@ public class Query implements QueryInterface, ToElementable, TransactionInterfac
final Query subQuery = new Query(q.getValue(), query.getUser());
subQuery.setAccess(query.getAccess());
subQuery.parse();
- final String subResultSet = subQuery.executeStrategy();
+
+ // versioning for QueryTemplates is not supported and probably never will.
+ final String subResultSet = subQuery.executeStrategy(false);
// ... and merge the resultSets.
union(query, resultSet, subResultSet);
@@ -384,7 +398,7 @@ public class Query implements QueryInterface, ToElementable, TransactionInterfac
// filter by role
if (this.role != null && this.role != Role.ENTITY) {
- final RoleFilter roleFilter = new RoleFilter(this.role, "=");
+ final RoleFilter roleFilter = new RoleFilter(this.role, "=", this.versioned);
roleFilter.apply(this);
}
@@ -402,8 +416,9 @@ public class Query implements QueryInterface, ToElementable, TransactionInterfac
}
}
- private String initQuery() throws QueryException {
- try (final CallableStatement callInitQuery = getConnection().prepareCall("call initQuery()")) {
+ private String initQuery(boolean versioned) throws QueryException {
+ String sql = "call initQuery(" + versioned + ")";
+ try (final CallableStatement callInitQuery = getConnection().prepareCall(sql)) {
ResultSet initQueryResult = null;
initQueryResult = callInitQuery.executeQuery();
if (!initQueryResult.next()) {
@@ -431,6 +446,7 @@ public class Query implements QueryInterface, ToElementable, TransactionInterfac
}
this.entity = cq.e;
+ this.versioned = cq.v != VersionFilter.UNVERSIONED;
this.role = cq.r;
this.parseTree = cq.toStringTree(parser);
this.type = cq.t;
@@ -442,23 +458,33 @@ public class Query implements QueryInterface, ToElementable, TransactionInterfac
}
}
- private String executeStrategy() throws QueryException {
+ private String executeStrategy(boolean versioned) throws QueryException {
if (this.entity != null) {
- return sourceStrategy(initQuery());
+ return sourceStrategy(initQuery(versioned));
} else {
- return targetStrategy(initQuery());
+ return targetStrategy(initQuery(versioned));
}
}
- private LinkedList<Integer> getResultSet(final String resultSetTableName) throws QueryException {
+ private List<IdVersionPair> getResultSet(final String resultSetTableName, boolean versioned)
+ throws QueryException {
ResultSet finishResultSet = null;
try {
- final PreparedStatement finish =
- getConnection().prepareStatement("Select id from `" + resultSetTableName + "`");
+ final String sql =
+ "Select results.id AS id"
+ + (versioned ? ", ev.version AS version" : "")
+ + " from `"
+ + resultSetTableName
+ + "` AS results"
+ + (versioned
+ ? " JOIN entity_version AS ev ON (results.id = ev.entity_id AND results._iversion = ev._iversion)"
+ : "");
+ final PreparedStatement finish = getConnection().prepareStatement(sql);
finishResultSet = finish.executeQuery();
- final LinkedList<Integer> rs = new LinkedList<Integer>();
+ final List<IdVersionPair> rs = new LinkedList<>();
while (finishResultSet.next()) {
- rs.add(finishResultSet.getInt("id"));
+ final String version = versioned ? finishResultSet.getString("version") : null;
+ rs.add(new IdVersionPair(finishResultSet.getInt("id"), version));
}
return rs;
} catch (final SQLException e) {
@@ -480,15 +506,15 @@ public class Query implements QueryInterface, ToElementable, TransactionInterfac
try {
- this.resultSet = getResultSet(executeStrategy());
+ this.resultSet = getResultSet(executeStrategy(this.versioned), this.versioned);
filterEntitiesWithoutRetrievePermission(this.resultSet);
// Fill resulting entities into container
if (this.container != null && this.type == Type.FIND) {
- for (final int id : this.resultSet) {
+ for (final IdVersionPair p : this.resultSet) {
- final Entity e = new RetrieveEntity(id);
+ final Entity e = new RetrieveEntity(p.id, p.version);
// if query has select-clause:
if (this.selections != null && !this.selections.isEmpty()) {
@@ -579,16 +605,16 @@ public class Query implements QueryInterface, ToElementable, TransactionInterfac
* @param entities
* @throws TransactionException
*/
- private void filterEntitiesWithoutRetrievePermission(final List<Integer> entities)
+ private void filterEntitiesWithoutRetrievePermission(final List<IdVersionPair> entities)
throws TransactionException {
if (!filterEntitiesWithoutRetrievePermisions) {
return;
}
- final Iterator<Integer> iterator = entities.iterator();
+ final Iterator<IdVersionPair> iterator = entities.iterator();
while (iterator.hasNext()) {
final long t1 = System.currentTimeMillis();
- final Integer id = iterator.next();
- if (!execute(new RetrieveSparseEntity(id, null), getAccess())
+ final IdVersionPair next = iterator.next();
+ if (!execute(new RetrieveSparseEntity(next.id, next.version), getAccess())
.getEntity()
.getEntityACL()
.isPermitted(getUser(), EntityPermission.RETRIEVE_ENTITY)) {
@@ -604,10 +630,6 @@ public class Query implements QueryInterface, ToElementable, TransactionInterfac
return this.query;
}
- public List<Integer> getResultSet() {
- return this.resultSet;
- }
-
@Override
public String getSourceSet() {
return this.sourceSet;
@@ -742,4 +764,9 @@ public class Query implements QueryInterface, ToElementable, TransactionInterfac
}
return benchmark;
}
+
+ @Override
+ public boolean isVersioned() {
+ return this.versioned;
+ }
}
diff --git a/src/main/java/org/caosdb/server/query/QueryInterface.java b/src/main/java/org/caosdb/server/query/QueryInterface.java
index 32cc1c7d8d98b56f80b159fc3f0004a491773563..0f2db9a97094ddc26062a9c9922907c183a26413 100644
--- a/src/main/java/org/caosdb/server/query/QueryInterface.java
+++ b/src/main/java/org/caosdb/server/query/QueryInterface.java
@@ -43,4 +43,6 @@ public interface QueryInterface {
public Subject getUser();
public void addBenchmark(final String str, final long time);
+
+ public boolean isVersioned();
}
diff --git a/src/main/java/org/caosdb/server/query/RoleFilter.java b/src/main/java/org/caosdb/server/query/RoleFilter.java
index 9055dccb358dd94ec17dee8b26bac97d2170b016..48c8372014c6e702be8b88113102da030dbc6046 100644
--- a/src/main/java/org/caosdb/server/query/RoleFilter.java
+++ b/src/main/java/org/caosdb/server/query/RoleFilter.java
@@ -33,6 +33,7 @@ public class RoleFilter implements EntityFilterInterface {
private final Role role;
private final String operator;
+ private boolean versioned;
/**
* Guarantees that all entities in the result set do have ("=") or do not have ("!=") the role in
@@ -43,7 +44,8 @@ public class RoleFilter implements EntityFilterInterface {
* @throws NullPointerException If role or operator is null.
* @throws IllegalArgumentException If operator is not "=" or "!=".
*/
- public RoleFilter(final Role role, final String operator) {
+ public RoleFilter(final Role role, final String operator, final boolean versioned) {
+ this.versioned = versioned;
if (role == null) {
throw new NullPointerException("The role must not be null.");
}
@@ -72,7 +74,8 @@ public class RoleFilter implements EntityFilterInterface {
getOperator(),
getRole(),
query.getSourceSet(),
- query.getTargetSet());
+ query.getTargetSet(),
+ this.versioned);
}
} catch (final SQLException e) {
throw new QueryException(e);
@@ -85,20 +88,28 @@ public class RoleFilter implements EntityFilterInterface {
final String operator,
final String role,
final String sourceSet,
- final String targetSet)
+ final String targetSet,
+ final boolean versioned)
throws SQLException {
- final PreparedStatement filterRoleStmt =
- connection.prepareCall(
- "INSERT IGNORE INTO `"
- + targetSet
- + "` (id) SELECT id FROM `"
- + sourceSet
- + (sourceSet.equals("entities")
- ? "` AS s WHERE EXISTS (SELECT * FROM entities AS e WHERE e.id=s.id AND e.role"
- : "` AS e WHERE NOT (e.role")
- + operator
- + "?);");
- filterRoleStmt.setString(1, role);
+ if (!sourceSet.equals("entities")) {
+ throw new UnsupportedOperationException("SourceSet is supposed to be the `entities` table.");
+ }
+ final String sql =
+ ("INSERT IGNORE INTO `"
+ + targetSet
+ + (versioned
+ ? "` (id, _iversion) SELECT e.id, _get_head_iversion(e.id) FROM `entities` AS e WHERE e.role "
+ + operator
+ + " ? UNION SELECT a.id, a._iversion FROM `archive_entities` AS a WHERE a.role"
+ + operator
+ + "?"
+ : "` (id) SELECT e.id FROM `entities` AS e WHERE e.role" + operator + "?"));
+ final PreparedStatement filterRoleStmt = connection.prepareCall(sql);
+ int params = (versioned ? 2 : 1);
+ while (params > 0) {
+ filterRoleStmt.setString(params, role);
+ params--;
+ }
filterRoleStmt.execute();
}
@@ -109,7 +120,7 @@ public class RoleFilter implements EntityFilterInterface {
connection.prepareCall(
"DELETE FROM `"
+ sourceSet
- + "` WHERE EXISTS (SELECT * FROM entities AS e WHERE e.id=`"
+ + "` WHERE EXISTS (SELECT 1 FROM entities AS e WHERE e.id=`"
+ sourceSet
+ "`.id AND NOT e.role"
+ operator
diff --git a/src/main/java/org/caosdb/server/query/SubProperty.java b/src/main/java/org/caosdb/server/query/SubProperty.java
index f8c22367c70f623903a3ee2f14a0af3cbbefd128..4668935e56f0057d3d0fe927aa0a729669e7e0f6 100644
--- a/src/main/java/org/caosdb/server/query/SubProperty.java
+++ b/src/main/java/org/caosdb/server/query/SubProperty.java
@@ -90,18 +90,15 @@ public class SubProperty implements QueryInterface, EntityFilterInterface {
getQuery().filterEntitiesWithoutRetrievePermission(this, this.sourceSet);
final CallableStatement callFinishSubProperty =
- getConnection().prepareCall("call finishSubProperty(?,?,?)");
- callFinishSubProperty.setString(1, query.getSourceSet()); // sourceSet
- // of
- // parent
- // query
+ getConnection().prepareCall("call finishSubProperty(?,?,?,?)");
+ callFinishSubProperty.setString(1, query.getSourceSet()); // sourceSet of parent query
if (query.getTargetSet() != null) { // targetSet
callFinishSubProperty.setString(2, query.getTargetSet());
} else {
callFinishSubProperty.setNull(2, VARCHAR);
}
- callFinishSubProperty.setString(3, this.sourceSet); // sub query
- // sourceSet
+ callFinishSubProperty.setString(3, this.sourceSet); // sub query sourceSet
+ callFinishSubProperty.setBoolean(4, this.isVersioned());
callFinishSubProperty.execute();
callFinishSubProperty.close();
} else {
@@ -165,4 +162,9 @@ public class SubProperty implements QueryInterface, EntityFilterInterface {
public void addBenchmark(final String str, final long time) {
this.query.addBenchmark(this.getClass().getSimpleName() + "." + str, time);
}
+
+ @Override
+ public boolean isVersioned() {
+ return this.query.isVersioned();
+ }
}
diff --git a/src/main/java/org/caosdb/server/query/VersionFilter.java b/src/main/java/org/caosdb/server/query/VersionFilter.java
new file mode 100644
index 0000000000000000000000000000000000000000..aa391778ce20c089a9fa6dae88caf77e0f797703
--- /dev/null
+++ b/src/main/java/org/caosdb/server/query/VersionFilter.java
@@ -0,0 +1,7 @@
+package org.caosdb.server.query;
+
+public class VersionFilter {
+
+ public static final VersionFilter ANY_VERSION = new VersionFilter();
+ public static final VersionFilter UNVERSIONED = new VersionFilter();
+}
diff --git a/src/main/java/org/caosdb/server/resource/UserResource.java b/src/main/java/org/caosdb/server/resource/UserResource.java
index 8ac0df1d8f8f018a07251e35a118ae944224dcc3..2c463e0e4bf7a640adc2a8368f74d798f69d0fd5 100644
--- a/src/main/java/org/caosdb/server/resource/UserResource.java
+++ b/src/main/java/org/caosdb/server/resource/UserResource.java
@@ -99,13 +99,11 @@ public class UserResource extends AbstractCaosDBServerResource {
: UserSources.guessRealm(username));
final String password = form.getFirstValue("password");
final String email = form.getFirstValue("email");
+
final UserStatus status =
- UserStatus.valueOf(
- form.getFirstValue(
- "status",
- CaosDBServer.getServerProperty(
- ServerProperties.KEY_NEW_USER_DEFAULT_ACTIVITY))
- .toUpperCase());
+ form.getFirstValue("status") != null
+ ? UserStatus.valueOf(form.getFirstValue("status").toUpperCase())
+ : null;
Integer userEntity = null;
if (form.getFirst("entity") != null) {
if (form.getFirstValue("entity").isEmpty()) {
diff --git a/src/main/java/org/caosdb/server/scripting/ScriptingUtils.java b/src/main/java/org/caosdb/server/scripting/ScriptingUtils.java
index 5bb9498944f02c3cc2bd79429dab10868299200f..c9a0b522d949a4ba3c36ae4f867661263a0185d3 100644
--- a/src/main/java/org/caosdb/server/scripting/ScriptingUtils.java
+++ b/src/main/java/org/caosdb/server/scripting/ScriptingUtils.java
@@ -23,7 +23,8 @@
package org.caosdb.server.scripting;
import java.io.File;
-import java.nio.file.Path;
+import java.io.IOException;
+import java.util.ArrayList;
import org.caosdb.server.CaosDBServer;
import org.caosdb.server.ServerProperties;
import org.caosdb.server.entity.Message;
@@ -33,25 +34,48 @@ import org.caosdb.server.utils.Utils;
public class ScriptingUtils {
- private File bin;
+ private File[] bin_dirs;
private File working;
public ScriptingUtils() {
- this.bin =
- new File(
- CaosDBServer.getServerProperty(ServerProperties.KEY_SERVER_SIDE_SCRIPTING_BIN_DIR));
+ ArrayList<File> new_bin_dirs = new ArrayList<>();
+ String bin_dirs_str =
+ CaosDBServer.getServerProperty(ServerProperties.KEY_SERVER_SIDE_SCRIPTING_BIN_DIRS);
+ if (bin_dirs_str == null) {
+ // fall-back for old server property
+ bin_dirs_str =
+ CaosDBServer.getServerProperty(ServerProperties.KEY_SERVER_SIDE_SCRIPTING_BIN_DIR);
+ }
+
+ // split and process
+ if (bin_dirs_str != null) {
+ for (String dir : bin_dirs_str.split("\\s+|\\s*,\\s*")) {
+
+ File bin;
+ try {
+ bin = new File(dir).getCanonicalFile();
+ } catch (IOException e) {
+ throw new ConfigurationException(
+ "Scripting bin dir `" + dir + "` cannot be resolved to a real path.");
+ }
+ if (!bin.exists()) {
+ bin.mkdirs();
+ }
+ if (!bin.isDirectory()) {
+ throw new ConfigurationException(
+ "The ServerProperty `"
+ + ServerProperties.KEY_SERVER_SIDE_SCRIPTING_BIN_DIRS
+ + "` must point to directories");
+ }
+ new_bin_dirs.add(bin);
+ }
+ }
+
+ bin_dirs = new_bin_dirs.toArray(new File[new_bin_dirs.size()]);
+
this.working =
new File(
CaosDBServer.getServerProperty(ServerProperties.KEY_SERVER_SIDE_SCRIPTING_WORKING_DIR));
- if (!bin.exists()) {
- bin.mkdirs();
- }
- if (!bin.isDirectory()) {
- throw new ConfigurationException(
- "The ServerProperty `"
- + ServerProperties.KEY_SERVER_SIDE_SCRIPTING_BIN_DIR
- + "` must point to a directory");
- }
if (!working.exists()) {
working.mkdirs();
@@ -64,21 +88,46 @@ public class ScriptingUtils {
}
}
- public File getScriptFile(final String command) {
- final Path script = bin.toPath().resolve(command);
- return script.toFile();
- }
+ /**
+ * Get the script file by the relative path.
+ *
+ * <p>Run through all registered bin_dirs and try to resolve the command relative to them. The
+ * first matching file is used. When it is not executable throw a
+ * SERVER_SIDE_SCRIPT_NOT_EXECUTABLE message. When no matching file exists throw a
+ * SERVER_SIDE_SCRIPT_DOES_NOT_EXIST message.
+ *
+ * @param command The relative path
+ * @return The script File object.
+ * @throws Message
+ */
+ public File getScriptFile(final String command) throws Message {
+ for (File bin_dir : bin_dirs) {
+ File script = bin_dir.toPath().resolve(command).toFile();
- public void checkScriptExists(final String command) throws Message {
- if (!getScriptFile(command).exists()) {
- throw ServerMessages.SERVER_SIDE_SCRIPT_DOES_NOT_EXIST;
- }
- }
+ try {
+ script = script.getCanonicalFile();
+ if (!script.toPath().startsWith(bin_dir.toPath())) {
+ // not below the allowed directory tree
+ continue;
+ }
+ } catch (IOException e) {
+ // cannot be resolved to canonical file - we treat it as non-existing.
+ continue;
+ }
+
+ if (!script.exists()) {
+ // doesn't exist.
+ continue;
+ }
+
+ if (!script.canExecute()) {
+ throw ServerMessages.SERVER_SIDE_SCRIPT_NOT_EXECUTABLE;
+ }
- public void checkScriptExecutable(final String command) throws Message {
- if (!getScriptFile(command).canExecute()) {
- throw ServerMessages.SERVER_SIDE_SCRIPT_NOT_EXECUTABLE;
+ // we found it!
+ return script;
}
+ throw ServerMessages.SERVER_SIDE_SCRIPT_DOES_NOT_EXIST;
}
public File getTmpWorkingDir() {
diff --git a/src/main/java/org/caosdb/server/scripting/ServerSideScriptingCaller.java b/src/main/java/org/caosdb/server/scripting/ServerSideScriptingCaller.java
index 2ec19a426fdd5a7f0cb6438b86f73a3dc93f55cd..4f5f81e248e17f6344d8523286828ce7142c529a 100644
--- a/src/main/java/org/caosdb/server/scripting/ServerSideScriptingCaller.java
+++ b/src/main/java/org/caosdb/server/scripting/ServerSideScriptingCaller.java
@@ -54,6 +54,7 @@ public class ServerSideScriptingCaller {
public static final Integer STARTED = -1;
private final String[] commandLine;
private final int timeoutMs;
+ private String absoluteScriptPath = null;
private ScriptingUtils utils;
private List<FileProperties> files;
private File workingDir;
@@ -122,7 +123,7 @@ public class ServerSideScriptingCaller {
/** Does some final preparation, then calls the script and cleans up. */
public int invoke() throws Message {
try {
- checkCommandLine(commandLine);
+ this.absoluteScriptPath = getAbsoluteScriptPath(commandLine);
try {
createWorkingDir();
putFilesInWorkingDir(files);
@@ -135,7 +136,8 @@ public class ServerSideScriptingCaller {
}
try {
- return callScript();
+ code = callScript(this.absoluteScriptPath, this.commandLine, this.authToken, this.env);
+ return code;
} catch (TimeoutException e) {
throw ServerMessages.SERVER_SIDE_SCRIPT_TIMEOUT;
} catch (final Throwable e) {
@@ -147,9 +149,17 @@ public class ServerSideScriptingCaller {
}
}
- void checkCommandLine(String[] commandLine) throws Message {
- utils.checkScriptExists(commandLine[0]);
- utils.checkScriptExecutable(commandLine[0]);
+ /**
+ * Returns the absolute script path.
+ *
+ * <p>Throws Message when the script does not exist or when the script is not executable.
+ *
+ * @param commandLine
+ * @return The absolute script path.
+ * @throws Message
+ */
+ String getAbsoluteScriptPath(String[] commandLine) throws Message {
+ return utils.getScriptFile(commandLine[0]).getAbsolutePath();
}
void putFilesInWorkingDir(final Collection<FileProperties> files)
@@ -243,10 +253,6 @@ public class ServerSideScriptingCaller {
if (pwd.exists()) FileUtils.forceDelete(pwd);
}
- String makeCallAbsolute(String call) {
- return utils.getScriptFile(call).getAbsolutePath();
- }
-
/** @fixme Should be injected into environment instead. Will be changed in v0.4 of SSS-API */
String[] injectAuthToken(String[] commandLine) {
String[] newCommandLine = new String[commandLine.length + 1];
@@ -258,14 +264,31 @@ public class ServerSideScriptingCaller {
return newCommandLine;
}
- int callScript() throws IOException, InterruptedException, TimeoutException {
+ /**
+ * Call the script.
+ *
+ * <p>The absoluteScriptPath is called with all remaining parameters from the commandLine arrays,
+ * an optional additional authToken and environment variables.
+ *
+ * @param absoluteScriptPath
+ * @param commandLine
+ * @param authToken
+ * @param env - environment variables
+ * @return the exit code of the script call
+ * @throws IOException
+ * @throws InterruptedException
+ * @throws TimeoutException
+ */
+ int callScript(
+ String absoluteScriptPath, String[] commandLine, Object authToken, Map<String, String> env)
+ throws IOException, InterruptedException, TimeoutException {
String[] effectiveCommandLine;
if (authToken != null) {
- effectiveCommandLine = injectAuthToken(getCommandLine());
+ effectiveCommandLine = injectAuthToken(commandLine);
} else {
- effectiveCommandLine = Arrays.copyOf(getCommandLine(), getCommandLine().length);
+ effectiveCommandLine = Arrays.copyOf(commandLine, commandLine.length);
}
- effectiveCommandLine[0] = makeCallAbsolute(effectiveCommandLine[0]);
+ effectiveCommandLine[0] = absoluteScriptPath;
final ProcessBuilder pb = new ProcessBuilder(effectiveCommandLine);
// inject environment variables
@@ -278,7 +301,7 @@ public class ServerSideScriptingCaller {
pb.redirectError(Redirect.to(getStdErrFile()));
pb.directory(getTmpWorkingDir());
- code = STARTED;
+ int code = STARTED;
final TimeoutProcess process = new TimeoutProcess(pb.start(), getTimeoutMs());
code = process.waitFor();
diff --git a/src/main/java/org/caosdb/server/transaction/Insert.java b/src/main/java/org/caosdb/server/transaction/Insert.java
index f60fc1286ae909eb7fd999aa3af1f8379dd331ff..d37f7ce516f36252466555ea18fe249f2730fea1 100644
--- a/src/main/java/org/caosdb/server/transaction/Insert.java
+++ b/src/main/java/org/caosdb/server/transaction/Insert.java
@@ -27,7 +27,6 @@ import org.caosdb.server.database.access.Access;
import org.caosdb.server.database.backend.transaction.InsertEntity;
import org.caosdb.server.entity.EntityInterface;
import org.caosdb.server.entity.FileProperties;
-import org.caosdb.server.entity.Version;
import org.caosdb.server.entity.container.InsertContainer;
import org.caosdb.server.entity.container.TransactionContainer;
import org.caosdb.server.permissions.EntityACL;
@@ -104,10 +103,6 @@ public class Insert extends WriteTransaction<InsertContainer> {
public void insert(final TransactionContainer container, final Access access) throws Exception {
if (container.getStatus().ordinal() >= EntityStatus.QUALIFIED.ordinal()) {
execute(new InsertEntity(container), access);
- for (EntityInterface e : container) {
- // TODO move to InsertEntity transaction
- e.setVersion(new Version(e.getVersion().getId(), this.getTimestamp()));
- }
}
}
diff --git a/src/main/java/org/caosdb/server/transaction/UpdateUserTransaction.java b/src/main/java/org/caosdb/server/transaction/UpdateUserTransaction.java
index dc93549a4e5a65d474043f109d136048c37d6146..330ce5907f601b4b4345c26b1d483029f7d049b1 100644
--- a/src/main/java/org/caosdb/server/transaction/UpdateUserTransaction.java
+++ b/src/main/java/org/caosdb/server/transaction/UpdateUserTransaction.java
@@ -78,7 +78,7 @@ public class UpdateUserTransaction extends AccessControlTransaction {
+ this.user.realm
+ "' cannot be updated. Only the users from the realm '"
+ UserSources.getInternalRealm().getName()
- + "' can change their passwords from withing caosdb.");
+ + "' can change their passwords from within caosdb.");
}
SecurityUtils.getSubject()
diff --git a/src/test/docker/Dockerfile b/src/test/docker/Dockerfile
index ecc6b332b2d88e0d587fdb3df0fc13cdfd0c159a..08a2a0884d1f154bb941388075b2bdd915125f99 100644
--- a/src/test/docker/Dockerfile
+++ b/src/test/docker/Dockerfile
@@ -1,5 +1,16 @@
FROM debian:buster
RUN apt-get update && \
- apt-get install \
- git make mariadb-server maven openjdk-11-jdk-headless \
- python3-pip screen libpam0g-dev unzip curl shunit2 -y
+ apt-get install -y \
+ git make mariadb-server maven openjdk-11-jdk-headless \
+ python3-pip screen libpam0g-dev unzip curl shunit2 \
+ python3-sphinx \
+ && \
+ pip3 install javasphinx recommonmark sphinx-rtd-theme
+
+# Alternative, if javasphinx fails because python3-sphinx is too recent:
+# (`_l` not found):
+#
+# git clone git@github.com:simgrid/javasphinx.git
+# cd javasphinx
+# git checkout 659209069603a
+# pip3 install .
diff --git a/src/test/java/org/caosdb/server/query/TestCQL.java b/src/test/java/org/caosdb/server/query/TestCQL.java
index 2264c9f02233c5462339615e142c38e2df4eb033..2f6651cc81a2614b63f1314f35052fe2096758b0 100644
--- a/src/test/java/org/caosdb/server/query/TestCQL.java
+++ b/src/test/java/org/caosdb/server/query/TestCQL.java
@@ -241,6 +241,9 @@ public class TestCQL {
String referenceByLikePattern = "FIND ENTITY WHICH IS REFERENCED BY *name*";
String emptyTextValue = "FIND ENTITY WITH prop=''";
+ String queryMR56 = "FIND ENTITY WITH ((p0 = v0 OR p1=v1) AND p2=v2)";
+
+ String versionedQuery1 = "FIND ANY VERSION OF ENTITY e1";
@Test
public void testQuery1()
@@ -6417,4 +6420,55 @@ public class TestCQL {
final StoredAt storedAt = (StoredAt) sfq.filter;
assertEquals("SAT(/data/in0.foo)", storedAt.toString());
}
+
+ /**
+ * Testing a conjunction which begins with a nested disjunction. The bug was a parsing error which
+ * was caused by a missing option for a disjunction/conjunction, wrapped into parenthesis as first
+ * element of a conjunction/disjunction
+ *
+ * <p>String queryMR56 = "FIND ENTITY WITH ((p0 = v0 OR p1=v1) AND p2=v2)";
+ */
+ @Test
+ public void testMR56() {
+ CQLLexer lexer;
+ lexer = new CQLLexer(CharStreams.fromString(this.queryMR56));
+ final CommonTokenStream tokens = new CommonTokenStream(lexer);
+
+ final CQLParser parser = new CQLParser(tokens);
+ final CqContext sfq = parser.cq();
+
+ System.out.println(sfq.toStringTree(parser));
+
+ // 4 children: FIND, role, WHICHCLAUSE, EOF
+ assertEquals(4, sfq.getChildCount());
+ assertEquals("WITH((p0=v0ORp1=v1)ANDp2=v2)", sfq.getChild(2).getText());
+ assertEquals("ENTITY", sfq.r.toString());
+ assertEquals("Conjunction", sfq.filter.getClass().getSimpleName());
+ final ParseTree whichclause = sfq.getChild(2);
+ final ParseTree conjunction = whichclause.getChild(2);
+ assertEquals("(p0=v0ORp1=v1)ANDp2=v2", conjunction.getText());
+ final ParseTree disjunction = conjunction.getChild(1);
+ assertEquals("p0=v0ORp1=v1", disjunction.getText());
+ final ParseTree pov = conjunction.getChild(4);
+ assertEquals("p2=v2", pov.getText());
+ }
+
+ /** String versionedQuery1 = "FIND ANY VERSION OF ENTITY e1"; */
+ @Test
+ public void testVersionedQuery1() {
+ CQLLexer lexer;
+ lexer = new CQLLexer(CharStreams.fromString(this.versionedQuery1));
+ final CommonTokenStream tokens = new CommonTokenStream(lexer);
+
+ final CQLParser parser = new CQLParser(tokens);
+ final CqContext sfq = parser.cq();
+
+ System.out.println(sfq.toStringTree(parser));
+
+ // 4 children: FIND, version, role, entity, EOF
+ assertEquals(5, sfq.getChildCount());
+ assertEquals(VersionFilter.ANY_VERSION, sfq.v);
+ assertEquals(Query.Role.ENTITY, sfq.r);
+ assertEquals("e1", sfq.e.toString());
+ }
}
diff --git a/src/test/java/org/caosdb/server/scripting/TestServerSideScriptingCaller.java b/src/test/java/org/caosdb/server/scripting/TestServerSideScriptingCaller.java
index 28fdd55e78db1a2142b5eac86327617b8f371089..61be0c5c3a06319dbeb826a79ddf1c6e43a0d672 100644
--- a/src/test/java/org/caosdb/server/scripting/TestServerSideScriptingCaller.java
+++ b/src/test/java/org/caosdb/server/scripting/TestServerSideScriptingCaller.java
@@ -39,6 +39,7 @@ import org.apache.commons.io.FileUtils;
import org.caosdb.CaosDBTestClass;
import org.caosdb.server.CaosDBException;
import org.caosdb.server.CaosDBServer;
+import org.caosdb.server.ServerProperties;
import org.caosdb.server.database.exceptions.TransactionException;
import org.caosdb.server.entity.FileProperties;
import org.caosdb.server.entity.Message;
@@ -68,6 +69,10 @@ public class TestServerSideScriptingCaller extends CaosDBTestClass {
@BeforeClass
public static void setupTestFolder() throws IOException {
+ CaosDBServer.getServerProperties()
+ .setProperty(
+ ServerProperties.KEY_SERVER_SIDE_SCRIPTING_BIN_DIRS, testFolder.getAbsolutePath());
+
FileUtils.forceDeleteOnExit(testFolder);
FileUtils.forceDeleteOnExit(testFile);
FileUtils.forceDeleteOnExit(testExecutable);
@@ -305,7 +310,7 @@ public class TestServerSideScriptingCaller extends CaosDBTestClass {
testExeExit(1);
- assertEquals(1, caller.callScript());
+ assertEquals(1, caller.callScript(cmd[0], cmd, null, emptyEnv));
}
@Test
@@ -319,7 +324,7 @@ public class TestServerSideScriptingCaller extends CaosDBTestClass {
testExeExit(0);
- assertEquals(0, caller.callScript());
+ assertEquals(0, caller.callScript(cmd[0], cmd, null, emptyEnv));
}
@Test
@@ -387,7 +392,7 @@ public class TestServerSideScriptingCaller extends CaosDBTestClass {
testSleep(10);
this.exception.expect(TimeoutException.class);
- caller.callScript();
+ caller.callScript(cmd[0], cmd, null, emptyEnv);
}
@Test
@@ -418,7 +423,8 @@ public class TestServerSideScriptingCaller extends CaosDBTestClass {
throws FileNotFoundException, CaosDBException {}
@Override
- public int callScript() {
+ public int callScript(
+ String cmd, String[] cmdLine, Object authToken, Map<String, String> env) {
return 0;
}
@@ -456,7 +462,8 @@ public class TestServerSideScriptingCaller extends CaosDBTestClass {
}
@Override
- public int callScript() {
+ public int callScript(
+ String cmd, String[] cmdLine, Object authToken, Map<String, String> env) {
return 0;
}
@@ -496,7 +503,9 @@ public class TestServerSideScriptingCaller extends CaosDBTestClass {
public void createTmpHomeDir() throws Exception {}
@Override
- public int callScript() throws IOException {
+ public int callScript(
+ String cmd, String[] cmdLine, Object authToken, Map<String, String> env)
+ throws IOException {
throw new IOException();
}
@@ -532,7 +541,8 @@ public class TestServerSideScriptingCaller extends CaosDBTestClass {
throws FileNotFoundException, CaosDBException {}
@Override
- public int callScript() {
+ public int callScript(
+ String cmd, String[] cmdLine, Object authToken, Map<String, String> env) {
return 0;
}
@@ -562,7 +572,7 @@ public class TestServerSideScriptingCaller extends CaosDBTestClass {
testPrintArgsToStdErr();
- caller.callScript();
+ caller.callScript(cmd[0], cmd, "authToken", emptyEnv);
assertEquals(
"--auth-token=authToken opt1 opt2\n", FileUtils.readFileToString(caller.getStdErrFile()));
@@ -579,7 +589,7 @@ public class TestServerSideScriptingCaller extends CaosDBTestClass {
testPrintArgsToStdOut();
- caller.callScript();
+ caller.callScript(cmd[0], cmd, "authToken", emptyEnv);
assertEquals(
"--auth-token=authToken opt1 opt2\n", FileUtils.readFileToString(caller.getStdOutFile()));
@@ -599,7 +609,7 @@ public class TestServerSideScriptingCaller extends CaosDBTestClass {
preparePrintEnv("TEST");
- caller.callScript();
+ caller.callScript(cmd[0], cmd, null, env);
assertEquals("-testcontent-\n", caller.getStdOut());
caller.cleanup();