Merge branch 'f-update-own-password' into f-permission-rules

.gitignore

@@ -44,3 +44,7 @@ __pycache__
 
 # Documentation binaries
 src/doc/development/api/xml/*.jar
+
+
+# data directories
+CaosDBFileSystem

.gitlab-ci.yml

 #
-# ** header v3.0
 # This file is a part of the CaosDB Project.
 #
 # Copyright (C) 2018 Research Group Biomedical Physics,
 # Max-Planck-Institute for Dynamics and Self-Organization Göttingen
 # Copyright (C) 2019 Henrik tom Wörden
+# Copyright (C) 2021 Timm Fitschen <t.fitschen@indiscale.com>
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as
@@ -19,23 +19,45 @@
 # You should have received a copy of the GNU Affero General Public License
 # along with this program. If not, see <https://www.gnu.org/licenses/>.
 #
-# ** end header
-#
 
 variables:
   DEPLOY_REF: dev
   CI_REGISTRY_IMAGE: $CI_REGISTRY/caosdb/src/caosdb-server/caosdb-server-testenv:latest
+  GIT_SUBMODULE_STRATEGY: normal
+
+  DEPLOY_PIPELINE: https://gitlab.indiscale.com/api/v4/projects/14/trigger/pipeline
+
+  ## FOR DEBUGGING
+  TRIGGERED_BY_REPO: SERVER
+  TRIGGERED_BY_REF: $CI_COMMIT_REF_NAME
+  TRIGGERED_BY_HASH: $CI_COMMIT_SHORT_SHA
 
 image: $CI_REGISTRY_IMAGE
 stages:
+  - info
   - setup
   - test
   - deploy
 
+.env: &env
+  - F_BRANCH="${CI_COMMIT_REF_NAME}"
+
+info:
+  tags: [cached-dind]
+  image: docker:20.10
+  stage: info
+  needs: []
+  script:
+    - *env
+    - echo "Pipeline triggered by $TRIGGERED_BY_REPO@$TRIGGERED_BY_REF ($TRIGGERED_BY_HASH)"
+    - echo "Pipeline will trigger DEPLOY with branch $DEPLOY_REF"
+    - echo "F_BRANCH = $F_BRANCH"
+
+
 # Setup: Build a docker image in which tests for this repository can run
 build-testenv:
   tags: [ cached-dind ]
-  image: docker:19.03
+  image: docker:20.10
   stage: setup
   timeout: 3h
   only:
@@ -61,18 +83,24 @@ test:
     - mvn compile
     - mvn test
 
+
+
 # Deploy: Trigger building of server image and integration tests
 trigger_build:
   tags: [ docker ]
   stage: deploy
   script:
+    - *env
+
+    - echo "Triggering pipeline ${DEPLOY_PIPELINE}@${DEPLOY_REF} with F_BRANCH=${F_BRANCH}"
     - /usr/bin/curl -X POST
       -F token=$CI_JOB_TOKEN
-      -F "variables[F_BRANCH]=$CI_COMMIT_REF_NAME"
       -F "variables[SERVER]=$CI_COMMIT_REF_NAME"
-      -F "variables[TriggerdBy]=SERVER"
-      -F "variables[TriggerdByHash]=$CI_COMMIT_SHORT_SHA"
-      -F ref=$DEPLOY_REF https://gitlab.indiscale.com/api/v4/projects/14/trigger/pipeline
+      -F "variables[F_BRANCH]=$F_BRANCH"
+      -F "variables[TRIGGERED_BY_REPO]=$TRIGGERED_BY_REPO"
+      -F "variables[TRIGGERED_BY_REF]=$TRIGGERED_BY_REF"
+      -F "variables[TRIGGERED_BY_HASH]=$TRIGGERED_BY_HASH"
+      -F ref=$DEPLOY_REF $DEPLOY_PIPELINE
 
 # Build the sphinx documentation and make it ready for deployment by Gitlab Pages
 # Special job for serving a static website. See https://docs.gitlab.com/ee/ci/yaml/README.html#pages
@@ -85,7 +113,7 @@ pages_prepare: &pages_prepare
   script:
     - echo "Deploying..."
     - make doc
-    - cp -r build/doc/html public
+    - rm -r public || true ; cp -r build/doc/html public
   artifacts:
     paths:
       - public

.gitmodules

@@ -2,3 +2,6 @@
 	path = caosdb-webui
 	url = ../caosdb-webui/
 	branch = dev
+[submodule "caosdb-proto"]
+	path = caosdb-proto
+	url = ../caosdb-proto/

CHANGELOG.md

@@ -9,8 +9,98 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Added
 
+* Implementation for the ACM GRPC-API
+* Implementation for the EntityACL GRPC-API
+
+### Changed
+
+### Deprecated
+
+### Removed
+
+### Fixed
+
+* Wrong serialization of date time values in the GRPC-API (resulting in
+  org.caosdb.server.datatime@12347abcd or similar).
+* Missing serialization of file descriptors in the GRPC-API during retrievals.
+* [caosdb-server#131](https://gitlab.com/caosdb/caosdb-server/-/issues/131)
+  Query: AND does not work with sub-properties
+* [caosdb-server#132](https://gitlab.com/caosdb/caosdb-server/-/issues/132)
+  Query: subproperties should not require parentheses
+
+### Security
+
+
+## [v0.7.1] - 2021-12-13
+(Timm Fitschen)
+
+This is an important security update.
+
+### Added
+
+### Changed
+
+### Deprecated
+
+### Removed
+
+### Fixed
+
+### Security
+
+* Update of logging backend log4j after a critical security vulnerability
+  [CVE-2021-44228](https://nvd.nist.gov/vuln/detail/CVE-2021-44228) to v2.15.0.
+* [caosdb-deploy#225](https://gitlab.indiscale.com/caosdb/src/caosdb-deploy/-/issues/225)
+  - Denied Edit permission leads to retrieve permission.
+
+## [v0.6.1] - 2021-11-13 [YANKED]
+(Timm Fitschen)
+
+This version's release was pulled after some problems during the release
+process. It is identical to v0.7.1
+
+
+## [v0.6.0] - 2021-11-17
+(Timm Fitschen)
+
+### Added
+
+* Endpoint for CaosDB GRPC API 0.1 (see https://gitlab.com/caosdb-proto.git for
+  more).
+  Authentication is supported via a Basic scheme, using the well-known
+  "authentication" header.
+  Notable limitations of the current implementation of the API:
+  * It is currently not possible to mix retrievals
+    (caosdb.entity.v1.RetrieveRequest) with any other transaction type - so
+    transaction are either read-only or write-only. The server throws an error
+    if it finds mixed read/write transactions.
+  * It is currently not possible to have more that one query
+    (caosdb.entity.v1.Query) in a single transaction. The server throws an
+    error if it finds more than one query.
+
+
+### Changed
+
+### Deprecated
+
+* Legacy XML/HTTP API (also known as the REST API). The API will not be removed
+  until the web interface (caosdb-webui) and the python client libraries have
+  been updated and freed from any dependencies. However, new clients should not
+  implement this API anymore.
+
+### Removed
+
+### Fixed
+
+### Security
+
+## [v0.5.0] - 2021-10-19
+
+### Added
+
 * An openAPI specification of the XML api
-* New server configuration option `SERVER_BIND_ADDRESS`, which is the address to listen to. See [server.conf](conf/core/server.conf).
+* New server configuration option `SERVER_BIND_ADDRESS`, which is the address to listen to. See
+  [server.conf](conf/core/server.conf).
 
 ### Changed
 
@@ -18,8 +108,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Removed
 
+* `IdOnly` flag (see https://gitlab.indiscale.com/caosdb/src/caosdb-server/-/issues/187).
+  The flags was not working anyways. However, `SELECT id FROM ...` queries are
+  now optimized in the way the `IdOnly` flag was supposed to do.
+
 ### Fixed
 
+* #181 CQL's `UPDATED` filter.
+  (https://gitlab.indiscale.com/caosdb/src/caosdb-server/-/issues/181)
 * #183 No reasonable error when using bad datetime format.
   (https://gitlab.indiscale.com/caosdb/src/caosdb-server/-/issues/183)
 * #127 "nan" as value (list item) in properties with data type "LIST<DOUBLE>"

DEPENDENCIES.md

-* caosdb-mysqlbackend == 4.1.0
-* Java 11
-* Apache Maven >= 3.6.0
-* make >= 4.2.0
-- easy-units >= 0.0.1    https://gitlab.com/timm.fitschen/easy-units
+# Dependencies
+
+## For Building and Running the Server
+
+* `>=caosdb-proto 0.1.0`
+* `>=caosdb-mysqlbackend 5.0.0`
+* `>=Java 11`
+* `>=Apache Maven 3.6.0`
+* `>=Make 4.2`
+* `libpam` (if PAM authentication is required)
+* More dependencies are being pulled and installed automatically by Maven. See the complete list of dependencies in the [pom.xml](pom.xml)
+
+## For Deploying a Web User Interface (optional)
+
+* `>=caosdb-webui 0.4.`
+
+## For Building the Documentation (optional)
+
+* `>=Python 3.8`
+* `>=pip 21.0.0`
+
+## For Server-side Scripting (optional)
+
+* `>=Python 3.8`
+* `>=pip 21.0.0`
+* `openpyxl` (for XLS/ODS export)
+
+## Recommended Packages
+
+* `openssl` (if a custom TLS certificate is required)

FEATURES.md

+# Features
+
+* The CaosDB Server implements a CaosDB GRPC API Endpoint (v0.1.0)
+  Authentication is supported via a Basic scheme, using the well-known
+  "authentication" header.
+  Notable limitations of the current implementation of the API:
+  * It is currently not possible to mix retrievals
+    (caosdb.entity.v1.RetrieveRequest) with any other transaction type - so
+    transaction are either read-only or write-only. The server throws an error
+    if it finds mixed read/write transactions.
+  * It is currently not possible to have more that one query
+    (caosdb.entity.v1.Query) in a single transaction. The server throws an
+    error if it finds more than one query.
+* Legacy XML/HTTP API (Deprecated)
+* Deployment of caosdb-webui (>=v0.4.1)
+* Server-side Scripting API (v0.1)
+* CaosDB Query Language Processor
+* CaosDB FileSystem
+* User Management, Authentication and Authorization
+  * Internal authentication service
+  * Authentication via an external service.

Makefile

@@ -57,7 +57,7 @@ formatting:
 
 # Compile into a standalone jar file
 jar: print-version easy-units
-	mvn package -DskipTests
+	mvn -e package -DskipTests
 	@pushd target ; \
 		ln -s caosdb-server-$(CAOSDB_SERVER_VERSION)-jar-with-dependencies.jar caosdb-server.jar; \
 		popd

README_SETUP.md

@@ -5,26 +5,7 @@ more.
 
 ## Requirements
 
-### CaosDB Packages
-
-* caosdb-webui=0.2.1
-* caosdb-mysqlbackend=3.0
-
-### Third-party Software
-
-* `>=Java 8`
-* `>=Apache Maven 3.0.4`
-* `>=Python 3.4`
-* `>=pip 9.0.1`
-* `>=git 1.9.1`
-* `>=Make 3.81`
-* `>=Screen 4.01`
-* `>=MySQL 5.5` (better `>=5.6`) or `>=MariaDB 10.1`
-* `libpam` (if PAM authentication is required)
-* `unzip`
-* `openpyxl` (for XLS/ODS export)
-* `openssl` (if a custom TLS certificate is required)
-- `easy-units` >= 0.0.1    https://gitlab.com/timm.fitschen/easy-units
+See [DEPENDENCIES.md](DEPENDENCIES.md).
 
 #### Install the requirements on Debian
 
@@ -56,9 +37,14 @@ installed and the pam user tool must be compiled:
 
 - `cd misc/pam_authentication/`
 - `make`
-- If you want, you can run a test now: `./pam_authentication.sh asdf ghjk`
-  should print `[FAILED]` and return with a non-zero exit code.  Unless there is
-  a user `asdf` with password `ghjk` on your system, of course.
+- If you want, you can run a test now: `./pam_authentication.sh asdf` asks for a password for user
+  `asdf`.  If no such user exists or the wrong passowrd is entered, it print `[FAILED]` and return
+  with a non-zero exit code.
+- If you want to run the CaosDB server without root privilege, you need to use
+  the setuid bit for the binary. For example, if the user `caosdb` runs the
+  server process the permissions of `bin/pam_authentication` should be the
+  following:
+  `-rwsr-x--- 1 root caosdb  pam_authentication`
 
 ##### Troubleshooting ####
 If `make` fails with `pam_authentication.c:4:31: fatal error:
@@ -72,12 +58,14 @@ libpam0g-dev`. Then try again.
 After a fresh clone of the repository, this is what you need to setup the
 server:
 
-1. Compile the server with `make compile`. This may take a while and there
+1. Install the `proto` submodule (and submodules for those extensions you want, see above):
+   `git submodule update --init caosdb-proto`
+2. Compile the server with `make compile`. This may take a while and there
    needs to be an internet connection as packages are downloaded to be
    integrated in the java file.
    1. It is recommended to run the unit tests with `make test`. It may take a
       while.
-2. Create an SSL certificate somewhere with a `Java Key Store` file.  For
+3. Create an SSL certificate somewhere with a `Java Key Store` file.  For
    self-signed certificates (not recommended for production use) you can do:
    - `mkdir certificates; cd certificates`
    - `keytool -genkey -keyalg RSA -alias selfsigned -keystore caosdb.jks -validity 375 -keysize 2048 -ext san=dns:localhost`
@@ -91,11 +79,11 @@ server:
    Alternatively, you can create a keystore from certificate files that you already have:
    - `openssl pkcs12 -export -inkey privkey.pem -in fullchain.pem -out all-certs.pkcs12`
    - `keytool -importkeystore -srckeystore all-certs.pkcs12 -srcstoretype PKCS12  -deststoretype pkcs12 -destkeystore caosdb.jks`
-3. Install/configure the MySQL back-end: see the `README_SETUP.md` of the
+4. Install/configure the MySQL back-end: see the `README_SETUP.md` of the
    `caosdb-mysqlbackend` repository
-4. Create an authtoken config (e.g. copy `conf/core/authtoken.example.yaml` to
+5. Create an authtoken config (e.g. copy `conf/core/authtoken.example.yaml` to
    `conf/ext/authtoken.yml` and change it)
-5. Copy `conf/core/server.conf` to `conf/ext/server.conf` and change it
+6. Copy `conf/core/server.conf` to `conf/ext/server.conf` and change it
    appropriately:
     * Setup for MySQL back-end:
       specify the fields `MYSQL_USER_NAME`, `MYSQL_USER_PASSWORD`,
@@ -108,7 +96,7 @@ server:
       `CERTIFICATES_KEY_STORE_PATH`, and `CERTIFICATES_KEY_STORE_PASSWORD`.
       Make sure that the conf file is not readable by other users because the
       certificate passwords are stored in plaintext.
-    - Set the path to the authtoken config (see step 4)
+    * Set the path to the authtoken config (see step 4)
     * Set the file system paths:
       - `FILE_SYSTEM_ROOT`: The root for all the files managed by CaosDB.
       - `DROP_OFF_BOX`: Files can be put here for insertion into CaosDB.
@@ -127,7 +115,7 @@ server:
         shall be accessible by CaosDB
     * Maybe set another `SESSION_TIMEOUT_MS`.
      * See also [CONFIGURATION.rst](src/doc/administration/configuration.rst)
-6. Copy `conf/core/usersources.ini.template` to `conf/ext/usersources.ini`.
+7. Copy `conf/core/usersources.ini.template` to `conf/ext/usersources.ini`.
     * You can skip this if you do not want to use an external authentication. 
       Local users (CaosDB realm) are always available.
     * Define the users/groups who you want to include/exclude.
@@ -142,7 +130,7 @@ server:
       Especially that there are no `properties` (aka `keys`) without a
       `value`. An emtpy value can be represented by `""`. Comments are
       everything from `#` or `;` to the end of the line.
-7. Possibly install the PAM caller in `misc/pam_authentication/` if you have 
+8. Possibly install the PAM caller in `misc/pam_authentication/` if you have
    not do so already. See above.
    
 Done!

RELEASE_GUIDELINES.md

@@ -31,3 +31,5 @@ guidelines of the CaosDB Project
 
 8. Update the version property in [pom.xml](./pom.xml) for the next
    developlement round (with a `-SNAPSHOT` suffix).
+
+9. Add a gitlab release in the respective repository.

caosdb-proto @ 02e56997

+Subproject commit 02e56997f29ee3bfc2c079c9776f66659379c58a

caosdb-webui @ b1034fb5

-Subproject commit c2dc8e9e9e0517ee7fb0d280717211a015906f64
+Subproject commit b1034fb515b00293dd48bdb17a230fa5c723dbdf

conf/core/log4j2-default.properties

@@ -2,7 +2,7 @@
 # https://logging.apache.org/log4j/2.x/ for more information.
 
 name = base_configuration
-status = TRACE
+status = DEBUG
 verbose = true
 
 property.LOG_DIR = log

conf/core/server.conf

@@ -53,6 +53,7 @@ USER_SOURCES_INI_FILE=./conf/ext/usersources.ini
 NEW_USER_DEFAULT_ACTIVITY=INACTIVE
 # If set to true, unauthenticated access to the database is possible with an anonymous user.
 AUTH_OPTIONAL=FALSE
+#AUTH_OPTIONAL=TRUE
 
 # --------------------------------------------------
 # MySQL settings
@@ -77,6 +78,8 @@ MYSQL_SCHEMA_VERSION=v5.0
 # The context root is a prefix which allows running multiple instances of CaosDB using the same
 # hostname and port. Must start with "/".
 CONTEXT_ROOT=
+#CONTEXT_ROOT=/caosdb
+
 # Server bind/host address, which is the address to listen to. Set to blank, or
 # 0.0.0.0 in IPv4, to listen to all. Set to 127.0.0.1 to make it available to
 # localhost only.
@@ -92,6 +95,11 @@ INITIAL_CONNECTIONS=1
 MAX_CONNECTIONS=10
 
 
+# HTTPS port of the grpc end-point
+GRPC_SERVER_PORT_HTTPS=8443
+# HTTP port of the grpc end-point
+GRPC_SERVER_PORT_HTTP=
+
 # --------------------------------------------------
 # HTTPS options
 # --------------------------------------------------
@@ -161,7 +169,8 @@ ADMIN_EMAIL=
 BUGTRACKER_URI=
 
 # If set to true MySQL stores transaction benchmarks for all SQL queries. Used for benchmarking and debugging.
-TRANSACTION_BENCHMARK_ENABLED=true
+TRANSACTION_BENCHMARK_ENABLED=FALSE
+#TRANSACTION_BENCHMARK_ENABLED=TRUE
 # Location of the configuration file for the CaosDB cache.
 CACHE_CONF_LOC=./conf/core/cache.ccf
 # Set this option to true to lobally disable caching. Used for debugging.
@@ -169,6 +178,7 @@ CACHE_DISABLE=false
 
 # The server is allowed to create symlinks to files and folders within this whitelist of directories.
 INSERT_FILES_IN_DIR_ALLOWED_DIRS=
+#INSERT_FILES_IN_DIR_ALLOWED_DIRS=/data/caosdb,/fileserver01/caosdb
 
 # Sudo password of the system.
 # Needed by the drop off box to set file permissions.

pom.xml

@@ -25,12 +25,20 @@
   <modelVersion>4.0.0</modelVersion>
   <groupId>org.caosdb</groupId>
   <artifactId>caosdb-server</artifactId>
-  <version>0.5.0-SNAPSHOT</version>
+  <version>0.8.0-SNAPSHOT</version>
   <packaging>jar</packaging>
   <name>CaosDB Server</name>
+  <scm>
+    <connection>scm:git:https://gitlab.indiscale.com/caosdb/src/caosdb-server.git</connection>
+  </scm>
   <properties>
     <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
     <project.build.testSourceDirectory>src/test/java</project.build.testSourceDirectory>
+    <protobuf.version>3.14.0</protobuf.version>
+    <grpc.version>1.42.1</grpc.version>
+    <netty-tcnative.version>2.0.34.Final</netty-tcnative.version>
+    <restlet.version>2.4.3</restlet.version>
+    <log4j.version>2.15.0</log4j.version>
   </properties>
   <repositories>
     <repository>
@@ -40,8 +48,8 @@
     </repository>
     <repository>
       <id>maven-restlet</id>
-      <name>Public online Restlet repository</name>
-      <url>https://maven.restlet.com</url>
+      <name>Restlet repository</name>
+      <url>https://maven.restlet.talend.com</url>
     </repository>
     <repository>
       <id>local-maven-repo</id>
@@ -67,7 +75,7 @@
     <dependency>
       <groupId>org.apache.shiro</groupId>
       <artifactId>shiro-core</artifactId>
-      <version>1.5.3</version>
+      <version>1.8.0</version>
     </dependency>
     <dependency>
       <groupId>junit</groupId>
@@ -90,12 +98,12 @@
     <dependency>
       <groupId>org.restlet.jse</groupId>
       <artifactId>org.restlet</artifactId>
-      <version>2.3.12</version>
+      <version>${restlet.version}</version>
     </dependency>
     <dependency>
       <groupId>org.restlet.jse</groupId>
       <artifactId>org.restlet.ext.fileupload</artifactId>
-      <version>2.3.12</version>
+      <version>${restlet.version}</version>
     </dependency>
     <dependency>
       <groupId>mysql</groupId>
@@ -125,7 +133,7 @@
     <dependency>
       <groupId>org.restlet.jse</groupId>
       <artifactId>org.restlet.ext.jetty</artifactId>
-      <version>2.3.12</version>
+      <version>${restlet.version}</version>
     </dependency>
     <dependency>
       <groupId>org.apache.commons</groupId>
@@ -160,30 +168,75 @@
     <dependency>
       <groupId>org.apache.logging.log4j</groupId>
       <artifactId>log4j-slf4j-impl</artifactId>
-      <version>2.11.1</version>
+      <version>${log4j.version}</version>
     </dependency>
     <dependency>
       <groupId>org.slf4j</groupId>
       <artifactId>slf4j-api</artifactId>
-      <version>1.7.21</version>
+      <version>1.7.32</version>
     </dependency>
     <dependency>
       <groupId>org.apache.logging.log4j</groupId>
       <artifactId>log4j-api</artifactId>
-      <version>2.11.1</version>
+      <version>${log4j.version}</version>
     </dependency>
     <dependency>
       <groupId>org.apache.logging.log4j</groupId>
       <artifactId>log4j-core</artifactId>
-      <version>2.11.1</version>
+      <version>${log4j.version}</version>
+    </dependency>
+    <dependency>
+      <groupId>io.grpc</groupId>
+      <artifactId>grpc-netty</artifactId>
+      <version>${grpc.version}</version>
+    </dependency>
+    <dependency>
+      <groupId>io.netty</groupId>
+      <artifactId>netty-tcnative</artifactId>
+      <version>${netty-tcnative.version}</version>
+      <classifier>${os.detected.classifier}</classifier>
+      <scope>runtime</scope>
+    </dependency>
+    <dependency>
+      <groupId>io.grpc</groupId>
+      <artifactId>grpc-protobuf</artifactId>
+      <version>${grpc.version}</version>
+    </dependency>
+    <dependency>
+      <groupId>io.grpc</groupId>
+      <artifactId>grpc-stub</artifactId>
+      <version>${grpc.version}</version>
+    </dependency>
+    <dependency>
+      <groupId>javax.annotation</groupId>
+      <artifactId>javax.annotation-api</artifactId>
+      <version>1.3.2</version>
+    </dependency>
+    <dependency>
+      <groupId>com.google.protobuf</groupId>
+      <artifactId>protobuf-java</artifactId>
+      <version>${protobuf.version}</version>
     </dependency>
   </dependencies>
   <build>
+    <resources>
+      <resource>
+        <directory>${basedir}/src/main/</directory>
+        <includes><include>**/*.properties</include></includes>
+      </resource>
+    </resources>
     <sourceDirectory>${basedir}/src/main/java</sourceDirectory>
     <scriptSourceDirectory>${basedir}/src/main/scripts</scriptSourceDirectory>
     <testSourceDirectory>${basedir}/src/test/java</testSourceDirectory>
     <outputDirectory>${basedir}/target/classes</outputDirectory>
     <testOutputDirectory>${basedir}/target/test-classes</testOutputDirectory>
+    <extensions>
+      <extension>
+        <groupId>kr.motd.maven</groupId>
+        <artifactId>os-maven-plugin</artifactId>
+        <version>1.7.0</version>
+      </extension>
+    </extensions>
     <plugins>
       <plugin>
         <groupId>org.apache.maven.plugins</groupId>
@@ -256,6 +309,28 @@
           <forkCount>0.5C</forkCount>
         </configuration>
       </plugin>
+      <plugin>
+        <groupId>org.codehaus.mojo</groupId>
+        <artifactId>buildnumber-maven-plugin</artifactId>
+        <version>1.4</version>
+        <executions>
+          <execution>
+            <phase>generate-resources</phase>
+            <goals>
+              <goal>create-metadata</goal>
+            </goals>
+          </execution>
+        </executions>
+        <configuration>
+          <addOutputDirectoryToResources>true</addOutputDirectoryToResources>
+          <applicationPropertyName>project.name</applicationPropertyName>
+          <revisionPropertyName>project.revision</revisionPropertyName>
+          <versionPropertyName>project.version</versionPropertyName>
+          <timestampPropertyName>build.timestamp</timestampPropertyName>
+          <outputDirectory>${basedir}/target/classes/org/caosdb/server/</outputDirectory>
+          <outputName>build.properties</outputName>
+        </configuration>
+      </plugin>
       <plugin>
         <groupId>org.codehaus.mojo</groupId>
         <artifactId>exec-maven-plugin</artifactId>
@@ -320,7 +395,40 @@
           <excludeArtifactIds>easy-units</excludeArtifactIds>
         </configuration>
       </plugin>
+      <plugin>
+        <groupId>kr.motd.maven</groupId>
+        <artifactId>os-maven-plugin</artifactId>
+        <version>1.7.0</version>
+        <executions>
+          <execution>
+            <phase>initialize</phase>
+            <goals>
+              <goal>detect</goal>
+            </goals>
+          </execution>
+        </executions>
+      </plugin>
+      <!-- code generation protobuf/grpc -->
+      <plugin>
+        <groupId>org.xolstice.maven.plugins</groupId>
+        <artifactId>protobuf-maven-plugin</artifactId>
+        <version>0.6.1</version>
+        <configuration>
+          <protoSourceRoot>${basedir}/caosdb-proto/proto/</protoSourceRoot>
+          <protocArtifact>com.google.protobuf:protoc:${protobuf.version}:exe:${os.detected.classifier}</protocArtifact>
+          <pluginId>grpc-java</pluginId>
+          <pluginArtifact>io.grpc:protoc-gen-grpc-java:${grpc.version}:exe:${os.detected.classifier}</pluginArtifact>
+        </configuration>
+        <executions>
+          <execution>
+            <goals>
+              <goal>compile</goal>
+              <goal>compile-custom</goal>
+            </goals>
+          </execution>
+        </executions>
+      </plugin>
     </plugins>
   </build>
-  <url>bmp.ds.mpg.de</url>
+  <url>caosdb.org</url>
 </project>

src/doc/CHANGELOG.md

+../../CHANGELOG.md
\ No newline at end of file

src/doc/DEPENDENCIES.md

+../../DEPENDENCIES.md
\ No newline at end of file

src/doc/conf.py

@@ -25,9 +25,9 @@ copyright = '2020, IndiScale GmbH'
 author = 'Daniel Hornung'
 
 # The short X.Y version
-version = '0.5'
+version = '0.8.0'
 # The full version, including alpha/beta/rc tags
-release = '0.5.0-SNAPSHOT'
+release = '0.8.0-SNAPSHOT'
 
 
 # -- General configuration ---------------------------------------------------

src/doc/index.rst

@@ -10,9 +10,12 @@ Welcome to caosdb-server's documentation!
 
    Getting started <README_SETUP>
    Concepts <concepts>
+   tutorials
    Query Language <CaosDB-Query-Language>
    administration
    Development <development/devel>
+   Dependencies <DEPENDENCIES>
+   Changelog <CHANGELOG>
    specification/index.rst
    Glossary
    API documentation<_apidoc/packages>

src/doc/tutorials.rst

+Tutorials
+==============
+
+.. toctree::
+   :maxdepth: 1
+   :glob:
+
+   tutorials/*
+
+

src/doc/tutorials/setup_state_model.py

+#!/usr/bin/env python3
+# encoding: utf-8
+#
+# This file is a part of the CaosDB Project.
+#
+# Copyright (C) 2021 Indiscale GmbH <info@indiscale.com>
+# Copyright (C) 2021 Henrik tom Wörden <h.tomwoerden@indiscale.com>
+# Copyright (C) 2021 Timm Fitschen <t.fitschen@indiscale.com>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see <https://www.gnu.org/licenses/>.
+#
+"""
+This is a utility script to setup a publication process in LinkAhead using
+states.
+
+If you start from scratch you should perform the following actions in that
+order:
+
+1. setup_roles
+2. setup_state_data_model
+4. setup_model_publication_cycle
+"""
+from argparse import ArgumentParser, RawDescriptionHelpFormatter
+
+import caosdb as db
+from caosdb.common.administration import generate_password
+
+
+def teardown(args):
+    """fully clears the database"""
+
+    if "yes" != input(
+        "Are you really sure that you want to delete ALL "
+        "ENTITIES in LinkAhead? [yes/No]"
+    ):
+
+        print("Nothing done.")
+
+        return
+    d = db.execute_query("FIND ENTITY WITH ID > 99")
+
+    if len(d) > 0:
+        d.delete(flags={"forceFinalState": "true"})
+
+
+def soft_teardown(args):
+    """ allows to remove state data only """
+    recs = db.execute_query("FIND Entity WITH State")
+
+    for rec in recs:
+        rec.state = None
+    recs.update(flags={"forceFinalState": "true"})
+    db.execute_query("FIND StateModel").delete()
+    db.execute_query("FIND Transition").delete()
+    db.execute_query("FIND State").delete()
+    db.execute_query(
+        "FIND Property WITH name=from or name=to or name=initial or name=final or name=color").delete()
+
+
+def setup_user(args):
+    """Creates a user with given username and adds the given role.
+
+    If the user exists, it is deleted first. A random password is generated
+    and printed in clear text in the console output.
+
+    """
+
+    username, role = args.username, args.role
+    try:
+        db.administration._delete_user(name=username)
+    except Exception:
+        pass
+
+    password = generate_password(10)
+    print("new password for {}:\n{}".format(username, password))
+    db.administration._insert_user(
+        name=username, password=password, status="ACTIVE")
+    db.administration._set_roles(username=username, roles=[role])
+
+
+def remove_user(args):
+    """deletes the given user"""
+    db.administration._delete_user(name=args.username)
+
+
+def setup_role_permissions():
+    """
+    Adds the appropriate permissions to the 'normal' and 'publisher' role.
+
+    The permissions are such that they suit the publication life cycle.
+    """
+    db.administration._set_permissions(
+        role="normal",
+        permission_rules=[
+            db.administration.PermissionRule("Grant", "TRANSACTION:*"),
+            db.administration.PermissionRule(
+                "Grant", "ACM:USER:UPDATE_PASSWORD:?REALM?:?USERNAME?"
+            ),
+            db.administration.PermissionRule("Grant", "STATE:TRANSITION:Edit"),
+            db.administration.PermissionRule("Grant", "UPDATE:PROPERTY:ADD"),
+            db.administration.PermissionRule(
+                "Grant", "UPDATE:PROPERTY:REMOVE"),
+            db.administration.PermissionRule(
+                "Grant", "STATE:TRANSITION:Start Review"),
+            db.administration.PermissionRule(
+                "Grant", "STATE:ASSIGN:Publish Life-cycle"
+            ),
+        ],
+    )
+
+    db.administration._set_permissions(
+        role="publisher",
+        permission_rules=[
+            db.administration.PermissionRule(
+                "Grant", "ACM:USER:UPDATE_PASSWORD:?REALM?:?USERNAME?"
+            ),
+            db.administration.PermissionRule("Grant", "TRANSACTION:*"),
+            db.administration.PermissionRule("Grant", "UPDATE:PROPERTY:ADD"),
+            db.administration.PermissionRule(
+                "Grant", "UPDATE:PROPERTY:REMOVE"),
+            db.administration.PermissionRule("Grant", "STATE:*"),
+        ],
+    )
+
+
+def setup_roles(args):
+    """Creates 'publisher' and 'normla' roles and assigns appropriate
+    permissions
+
+    If those roles exist they are deleted first.
+    """
+
+    for role in ["publisher", "normal"]:
+        try:
+            db.administration._delete_role(name=role)
+        except Exception:
+            print("Could not delete role {}".format(role))
+
+    for role in ["publisher", "normal"]:
+        db.administration._insert_role(name=role, description="")
+
+    setup_role_permissions()
+
+
+def setup_state_data_model(args):
+    """Creates the data model for using states
+
+    RecordTypes: State, StateModel, Transition
+    Properties: from, to, initial, final, color
+    """
+    cont = db.Container().extend(
+        [
+            db.RecordType("State"),
+            db.RecordType("StateModel"),
+            db.RecordType("Transition"),
+            db.Property(name="from", datatype="State"),
+            db.Property(name="to", datatype="State"),
+            db.Property(name="initial", datatype="State"),
+            db.Property(name="final", datatype="State"),
+            db.Property(name="color", datatype=db.TEXT),
+        ]
+    )
+    cont.insert()
+
+
+def setup_model_publication_cycle(args):
+    """Creates States and Transitions for the Publication Life Cycle"""
+    unpublished_acl = db.ACL()
+    unpublished_acl.grant(role="publisher", permission="*")
+    unpublished_acl.grant(role="normal", permission="UPDATE:*")
+    unpublished_acl.grant(role="normal", permission="RETRIEVE:ENTITY")
+    unpublished_acl = db.State.create_state_acl(unpublished_acl)
+
+    unpublished_state = (
+        db.Record(
+            "Unpublished",
+            description="Unpublished entries are only visible to the team "
+            "and may be edited by any team member.",
+        )
+        .add_parent("State")
+        .add_property("color", "#5bc0de")
+    )
+    unpublished_state.acl = unpublished_acl
+    unpublished_state.insert()
+
+    review_acl = db.ACL()
+    review_acl.grant(role="publisher", permission="*")
+    review_acl.grant(role="normal", permission="RETRIEVE:ENTITY")
+
+    review_state = (
+        db.Record(
+            "Under Review",
+            description="Entries under review are not publicly available yet, "
+            "but they can only be edited by the members of the publisher "
+            "group.",
+        )
+        .add_parent("State")
+        .add_property("color", "#FFCC33")
+    )
+    review_state.acl = db.State.create_state_acl(review_acl)
+    review_state.insert()
+
+    published_acl = db.ACL()
+    published_acl.grant(role="guest", permission="RETRIEVE:ENTITY")
+
+    published_state = (
+        db.Record(
+            "Published",
+            description="Published entries are publicly available and "
+            "cannot be edited unless they are unpublished again.",
+        )
+        .add_parent("State")
+        .add_property("color", "#333333")
+    )
+    published_state.acl = db.State.create_state_acl(published_acl)
+    published_state.insert()
+
+    # 1->2
+    (
+        db.Record(
+            "Start Review",
+            description="This transitions denies the permissions to edit an "
+            "entry for anyone but the members of the publisher group. "
+            "However, the entry is not yet publicly available.",
+        )
+        .add_parent("Transition")
+        .add_property("from", "unpublished")
+        .add_property("to", "under review")
+        .add_property("color", "#FFCC33")
+        .insert()
+    )
+
+    # 2->3
+    (
+        db.Record(
+            "Publish",
+            description="Published entries are visible for the public and "
+            "cannot be changed unless they are unpublished again. Only members"
+            " of the publisher group can publish or unpublish entries.",
+        )
+        .add_parent("Transition")
+        .add_property("from", "under review")
+        .add_property("to", "published")
+        .add_property("color", "red")
+        .insert()
+    )
+
+    # 3->1
+    (
+        db.Record(
+            "Unpublish",
+            description="Unpublish this entry to hide it from "
+            "the public. Unpublished entries can be edited by any team "
+            "member.",
+        )
+        .add_parent("Transition")
+        .add_property("from", "published")
+        .add_property("to", "unpublished")
+        .insert()
+    )
+
+    # 2->1
+    (
+        db.Record(
+            "Reject",
+            description="Reject the publishing of this entity.  Afterwards, "
+            "the entity is editable for any team member again.",
+        )
+        .add_parent("Transition")
+        .add_property("from", "under review")
+        .add_property("to", "unpublished")
+        .insert()
+    )
+
+    # 1->1
+    (
+        db.Record(
+            "Edit",
+            description="Edit this entity. The changes are not publicly "
+            "available until this entity will have been reviewed and "
+            "published.",
+        )
+        .add_parent(
+            "Transition",
+        )
+        .add_property("from", "unpublished")
+        .add_property("to", "unpublished")
+        .insert()
+    )
+
+    (
+        db.Record(
+            "Publish Life-cycle",
+            description="The publish life-cycle is a quality assurance tool. "
+            "Database entries can be edited without being publicly available "
+            "until the changes have been reviewed and explicitely published by"
+            " an eligible user.",
+        )
+        .add_parent("StateModel")
+        .add_property(
+            "Transition",
+            datatype=db.LIST("Transition"),
+            value=[
+                "Edit",
+                "Start Review",
+                "Reject",
+                "Publish",
+                "Unpublish",
+            ],
+        )
+        .add_property("initial", "Unpublished")
+        .add_property("final", "Unpublished")
+        .insert()
+    )
+
+
+def parse_args():
+    parser = ArgumentParser(
+        description=__doc__, formatter_class=RawDescriptionHelpFormatter
+    )
+    subparsers = parser.add_subparsers(
+        title="action",
+        metavar="ACTION",
+        description=(
+            "You can perform the following actions. "
+            "Print the detailed help for each command with "
+            "#> setup_state_model ACTION -h"
+        ),
+    )
+
+    subparser = subparsers.add_parser(
+        "setup_state_data_model", help=setup_state_data_model.__doc__
+    )
+    subparser.set_defaults(call=setup_state_data_model)
+
+    subparser = subparsers.add_parser(
+        "setup_model_publication_cycle", help=setup_model_publication_cycle.__doc__
+    )
+    subparser.set_defaults(call=setup_model_publication_cycle)
+
+    subparser = subparsers.add_parser("setup_roles", help=setup_roles.__doc__)
+    subparser.set_defaults(call=setup_roles)
+
+    subparser = subparsers.add_parser("remove_user", help=remove_user.__doc__)
+    subparser.set_defaults(call=remove_user)
+    subparser.add_argument("username")
+
+    subparser = subparsers.add_parser("setup_user", help=setup_user.__doc__)
+    subparser.set_defaults(call=setup_user)
+    subparser.add_argument("username")
+    subparser.add_argument("role")
+
+    subparser = subparsers.add_parser(
+        "teardown", help="Removes ALL ENTITIES from LinkAhead!"
+    )
+    subparser.set_defaults(call=teardown)
+
+    subparser = subparsers.add_parser(
+        "soft_teardown", help=soft_teardown.__doc__
+    )
+    subparser.set_defaults(call=soft_teardown)
+
+    return parser.parse_args()
+
+
+if __name__ == "__main__":
+    args = parse_args()
+    args.call(args)