Merge branch 'dev' into f-fsm

.gitlab-ci.yml

@@ -24,7 +24,7 @@
 
 variables:
   DEPLOY_REF: dev
-  CI_REGISTRY_IMAGE: $CI_REGISTRY/caosdb/caosdb-server/caosdb-server-testenv:latest
+  CI_REGISTRY_IMAGE: $CI_REGISTRY/caosdb/src/caosdb-server/caosdb-server-testenv:latest
 
 image: $CI_REGISTRY_IMAGE
 stages:
@@ -42,14 +42,11 @@ build-testenv:
     - schedules
   script:
     - cd src/test/docker
-    - time docker load < /image-cache/caosdb-server-testenv.tar || true
     - docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY
       # use here general latest or specific branch latest...
     - docker build
       --pull
       -t $CI_REGISTRY_IMAGE .
-    - docker save $CI_REGISTRY_IMAGE > image.tar;
-          mv image.tar /image-cache/caosdb-server-testenv.tar;
     - docker push $CI_REGISTRY_IMAGE
 
 # Test: run unit tests of the server
@@ -70,7 +67,7 @@ trigger_build:
   stage: deploy
   script:
     - /usr/bin/curl -X POST
-      -F token=$DEPLOY_TRIGGER_TOKEN
+      -F token=$CI_JOB_TOKEN
       -F "variables[F_BRANCH]=$CI_COMMIT_REF_NAME"
       -F "variables[SERVER]=$CI_COMMIT_REF_NAME"
       -F "variables[TriggerdBy]=SERVER"
@@ -78,15 +75,17 @@ trigger_build:
       -F ref=$DEPLOY_REF https://gitlab.indiscale.com/api/v4/projects/14/trigger/pipeline
 
 # Build the sphinx documentation and make it ready for deployment by Gitlab Pages
-# documentation:
-#   stage: deploy
-
 # Special job for serving a static website. See https://docs.gitlab.com/ee/ci/yaml/README.html#pages
 pages:
   tags: [ cached-dind ]
   stage: deploy
   only:
-    - dev
+    refs:
+      - /^release-.*$/i
+      - master
+    variables:
+      # run pages only on gitlab.com
+      - $CI_SERVER_HOST == "gitlab.com"
   script:
     - echo "Deploying"
     - make doc

CHANGELOG.md

@@ -16,6 +16,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 * `ETag` property for the query. The `ETag` tags a server state and is being
   updated each time the server state is being updated (i.e. the stored entities
   change). This can be used to debug the query cache and also allows a client
+* `ETag` property for the query. The `ETag` is assigned to the query cache
+  each time the cache is cleared (currently whenever the server state is being
+  updated, i.e. the stored entities change).
+  This can be used to debug the query cache and also allows a client
   to determine whether the server's state has changed between queries.
 * Basic caching for queries. The caching is enabled by default and can be
   controlled by the usual "cache" flag.
@@ -28,6 +32,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Fixed
 
+* #122 - Dead-lock due to error in the DatabaseAccessManager.
 * #120 - Editing entities that were created with a no longer existing user
   leads to a server error.
 * #31 - Queries with keywords in the path (e.g. `... STORED AT 0in.txt`)

README.md

-<!--THIS FILE HAS BEEN GENERATED BY A SCRIPT. PLEASE DON'T CHANGE IT MANUALLY.-->
 
-# Welcome
+# README
 
-This is the **CaosDB Server** repository and a part of the CaosDB project.
+## Welcome
 
-# Setup
+This is the **CaosDB Java Server** repository and a part of the
+CaosDB project.
+
+## Setup
 
 Please read the [README_SETUP.md](README_SETUP.md) for instructions on how to
 setup this code.
 
 
-# Further Reading
+## Further Reading
+
+Please refer to the [official documentation](https://docs.indiscale.com/caosdb-server/) for more information.
+
+## Contributing
+
+Thank you very much to all contributers—[past, present](https://gitlab.com/caosdb/caosdb/-/blob/dev/HUMANS.md), and prospective ones.
 
-Please refer to the [official gitlab repository of the CaosDB
-project](https://gitlab.com/caosdb/caosdb) for more information.
+### Code of Conduct
 
-# License
+By participating, you are expected to uphold our [Code of Conduct](https://gitlab.com/caosdb/caosdb/-/blob/dev/CODE_OF_CONDUCT.md).
 
-Copyright (C) 2018 Research Group Biomedical Physics, Max Planck Institute for
-Dynamics and Self-Organization Göttingen.
+### How to Contribute
+
+* You found a bug, have a question, or want to request a feature? Please
+[create an issue](https://gitlab.com/caosdb/caosdb-server/-/issues).
+* You want to contribute code? Please fork the repository and create a merge
+request in GitLab and choose this repository as target. Make sure to select
+"Allow commits from members who can merge the target branch" under Contribution
+when creating the merge request. This allows our team to work with you on your request.
+- If you have a suggestion for the [documentation](https://docs.indiscale.com/caosdb-server/),
+the preferred way is also a merge request as describe above (the documentation resides in `src/doc`).
+However, you can also create an issue for it.
+- You can also contact us at **info (AT) caosdb.de**.
+
+## License
+
+* Copyright (C) 2018 Research Group Biomedical Physics, Max Planck Institute
+  for Dynamics and Self-Organization Göttingen.
+* Copyright (C) 2020-2021 Indiscale GmbH <info@indiscale.com>
 
 All files in this repository are licensed under a [GNU Affero General Public
 License](LICENCE.md) (version 3 or later).
-

conf/core/server.conf

@@ -75,7 +75,7 @@ MYSQL_SCHEMA_VERSION=v4.0.0
 # Server options
 # --------------------------------------------------
 # The context root is a prefix which allows running multiple instances of CaosDB using the same
-# hostname and port.
+# hostname and port. Must start with "/".
 CONTEXT_ROOT=
 # HTTPS port of this server instance.
 SERVER_PORT_HTTPS=443

src/doc/administration/server_side_scripting.rst

@@ -50,7 +50,7 @@ script invocation from a skeleton directory, located in the server directory, in
 - `readme.md` :: A small text file describing the purpose of the directory.
 
 Users of CaosDB are invited to populate the directory with whatever their
-scripts need.
+scripts need (for example a `.pycaosdb.ini` file).
 
 Invocation
 ------------

src/main/java/org/caosdb/server/database/DatabaseAccessManager.java

@@ -26,6 +26,7 @@
 package org.caosdb.server.database;
 
 import java.util.concurrent.Semaphore;
+import java.util.concurrent.atomic.AtomicInteger;
 import java.util.concurrent.locks.ReentrantLock;
 import org.caosdb.server.database.access.Access;
 import org.caosdb.server.database.access.AccessControlAccess;
@@ -56,7 +57,7 @@ import org.caosdb.server.utils.Releasable;
 class ReadAccessSemaphore extends Semaphore implements Releasable {
 
   private static final long serialVersionUID = 4384921156838881337L;
-  private int acquired = 0; // how many threads have read access
+  private AtomicInteger acquired = new AtomicInteger(0); // how many threads have read access
   Semaphore writersBlock =
       new Semaphore(1, true); // This semaphore is blocked as long as there are any
   // unreleased read permits.
@@ -73,10 +74,9 @@ class ReadAccessSemaphore extends Semaphore implements Releasable {
   @Override
   public void acquire() throws InterruptedException {
     super.acquire(); // Protect the next few lines
-    if (this.acquired == 0) {
+    if (this.acquired.getAndIncrement() == 0) {
       this.writersBlock.acquire();
     }
-    this.acquired++;
     super.release();
   }
 
@@ -87,9 +87,7 @@ class ReadAccessSemaphore extends Semaphore implements Releasable {
    */
   @Override
   public void release() {
-    this.acquired--;
-    if (this.acquired <= 0) { // Last permit: release
-      this.acquired = 0;
+    if (this.acquired.decrementAndGet() == 0) { // Last permit: release
       if (this.writersBlock.availablePermits() <= 0) {
         this.writersBlock.release();
       }

src/test/java/org/caosdb/server/database/DatabaseAccessManagerTest.java

@@ -22,10 +22,91 @@
  */
 package org.caosdb.server.database;
 
+import static org.junit.Assert.assertFalse;
+
+import java.util.LinkedList;
+import java.util.List;
 import org.junit.Assert;
 import org.junit.Test;
 
 public class DatabaseAccessManagerTest {
+  Thread createReadThread(long wait, String name, ReadAccessSemaphore readAccess) {
+    return new Thread(
+        new Runnable() {
+
+          @Override
+          public void run() {
+            try {
+              readAccess.acquire();
+              Thread.sleep(wait);
+            } catch (InterruptedException e) {
+              e.printStackTrace();
+            } finally {
+              readAccess.release();
+            }
+          }
+        },
+        name);
+  }
+
+  Thread createWriteThread(long wait, String name, WriteAccessLock writeAccess) {
+    return new Thread(
+        new Runnable() {
+
+          @Override
+          public void run() {
+            try {
+              writeAccess.reserve();
+              Thread.sleep(wait);
+              writeAccess.lockInterruptibly();
+              Thread.sleep(wait);
+            } catch (InterruptedException e) {
+              e.printStackTrace();
+            } finally {
+              writeAccess.release();
+            }
+          }
+        },
+        name);
+  }
+
+  @Test
+  public void testDeadLock() throws InterruptedException {
+    final ReadAccessSemaphore readAccess = new ReadAccessSemaphore();
+    final WriteAccessLock writeAccess = new WriteAccessLock(readAccess);
+    List<Thread> ts = new LinkedList<>();
+    for (int i = 0; i < 1000; i++) {
+      Thread t1 = createReadThread(1, "Ra" + i, readAccess);
+      Thread t2 = createReadThread(2, "Rb" + i, readAccess);
+      Thread t3 = createReadThread(3, "Rc" + i, readAccess);
+      Thread t5 = createReadThread(5, "Rd" + i, readAccess);
+      Thread t7 = createReadThread(7, "Re" + i, readAccess);
+      Thread t11 = createReadThread(11, "Rf" + i, readAccess);
+      Thread w5 = createWriteThread(2, "W" + i, writeAccess);
+
+      t1.start();
+      t2.start();
+      w5.start();
+      t3.start();
+      t5.start();
+      t7.start();
+      t11.start();
+
+      ts.add(t1);
+      ts.add(t2);
+      ts.add(t3);
+      ts.add(t5);
+      ts.add(t7);
+      ts.add(t11);
+      ts.add(w5);
+    }
+
+    for (Thread t : ts) {
+      t.join(10000);
+      assertFalse(t.isAlive());
+    }
+  }
+
   public static final ReadAccessSemaphore readAccess = new ReadAccessSemaphore();
   public static final WriteAccessLock writeAccess = new WriteAccessLock(readAccess);
 

src/test/java/org/caosdb/server/query/QueryTest.java

@@ -67,7 +67,11 @@ public class QueryTest {
     assertEquals(Query.Role.ENTITY, q.getRole());
   }
 
-  /** Assure that {@link WriteTransaction#commit()} calls {@link Query#clearCache()}. */
+  /**
+   * Assure that {@link WriteTransaction#commit()} calls {@link Query#clearCache()}.
+   *
+   * <p>Since currently the cache shall be cleared whenever there is a commit.
+   */
   @Test
   public void testEtagChangesAfterWrite() {
     String old = Query.getETag();