This MR introduces a minimal viable query caching, basically mitigating the...

This MR introduces a minimal viable query caching, basically mitigating the bad performance effects of running the same query several times with COUNT <Something>, FIND <Something>, Select ... FROM <Something> and pagination.

Tests in caosdb-pyinttest!51

CHANGELOG.md

@@ -9,6 +9,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Added
 
+* Basic caching for queries. The caching is enabled by default and can be
+  controlled by the usual "cache" flag.
 * New version history feature. The "H" container flag retrieves the full
   version history during a transaction (e.g. during Retrievals) and constructs
   a tree of successors and predecessors of the requested entity version.

conf/core/cache.ccf

@@ -36,6 +36,9 @@ jcs.region.BACKEND_SparseEntities.cacheattributes.MaxObjects=1002
 jcs.region.BACKEND_RetrieveVersionHistory
 jcs.region.BACKEND_RetrieveVersionHistory.cacheattributes.MaxObjects=1006
 
+jcs.region.HIGH_LEVEL_QUERY_CACHE
+jcs.region.HIGH_LEVEL_QUERY_CACHE.cacheattributes.MaxObjects=1000
+
 # PAM UserSource Caching: Cached Items expire after 60 seconds if they are not requested (idle) and after 600 seconds max.
 # PAM_UnixUserGroups
 jcs.region.PAM_UnixUserGroups

src/main/java/org/caosdb/server/query/Backreference.java

@@ -330,4 +330,12 @@ public class Backreference implements EntityFilterInterface, QueryInterface {
   public boolean isVersioned() {
     return this.query.isVersioned();
   }
+
+  @Override
+  public String getCacheKey() {
+    StringBuilder sb = new StringBuilder();
+    sb.append(toString());
+    if (this.hasSubProperty()) sb.append(getSubProperty().getCacheKey());
+    return sb.toString();
+  }
 }

src/main/java/org/caosdb/server/query/Conjunction.java

@@ -34,7 +34,8 @@ import org.caosdb.server.database.access.Access;
 import org.caosdb.server.query.Query.QueryException;
 import org.jdom2.Element;
 
-public class Conjunction extends EntityFilterContainer implements QueryInterface {
+public class Conjunction extends EntityFilterContainer
+    implements QueryInterface, EntityFilterInterface {
 
   private String sourceSet = null;
   private String targetSet = null;
@@ -155,4 +156,15 @@ public class Conjunction extends EntityFilterContainer implements QueryInterface
   public boolean isVersioned() {
     return this.query.isVersioned();
   }
+
+  @Override
+  public String getCacheKey() {
+    StringBuilder sb = new StringBuilder();
+    sb.append("Conj(");
+    for (EntityFilterInterface filter : getFilters()) {
+      sb.append(filter.getCacheKey());
+    }
+    sb.append(")");
+    return sb.toString();
+  }
 }

src/main/java/org/caosdb/server/query/Disjunction.java

@@ -34,7 +34,8 @@ import org.caosdb.server.database.access.Access;
 import org.caosdb.server.query.Query.QueryException;
 import org.jdom2.Element;
 
-public class Disjunction extends EntityFilterContainer implements QueryInterface {
+public class Disjunction extends EntityFilterContainer
+    implements QueryInterface, EntityFilterInterface {
 
   private String targetSet = null;
   private int targetSetCount = -1;
@@ -143,4 +144,15 @@ public class Disjunction extends EntityFilterContainer implements QueryInterface
   public boolean isVersioned() {
     return this.query.isVersioned();
   }
+
+  @Override
+  public String getCacheKey() {
+    StringBuilder sb = new StringBuilder();
+    sb.append("Disj(");
+    for (EntityFilterInterface filter : getFilters()) {
+      sb.append(filter.getCacheKey());
+    }
+    sb.append(")");
+    return sb.toString();
+  }
 }

src/main/java/org/caosdb/server/query/EntityFilterContainer.java

@@ -24,7 +24,7 @@ package org.caosdb.server.query;
 
 import java.util.LinkedList;
 
-public abstract class EntityFilterContainer implements EntityFilterInterface {
+public abstract class EntityFilterContainer {
   private final LinkedList<EntityFilterInterface> filters = new LinkedList<EntityFilterInterface>();
 
   public void add(final EntityFilterInterface filter) {

src/main/java/org/caosdb/server/query/EntityFilterInterface.java

@@ -30,4 +30,12 @@ public interface EntityFilterInterface {
   void apply(QueryInterface query) throws QueryException;
 
   public Element toElement();
+
+  /**
+   * Return a string which can serve as a cache key. The string must describe all relevant
+   * parameters of the filter.
+   *
+   * @return a cache key.
+   */
+  public String getCacheKey();
 }

src/main/java/org/caosdb/server/query/IDFilter.java

@@ -125,4 +125,17 @@ public class IDFilter implements EntityFilterInterface {
   public String getAggregate() {
     return this.aggregate;
   }
+
+  @Override
+  public String getCacheKey() {
+    StringBuilder sb = new StringBuilder();
+    sb.append("ID(");
+    if (this.aggregate != null) sb.append(this.aggregate);
+    sb.append(",");
+    if (this.operator != null) sb.append(this.operator);
+    sb.append(",");
+    if (this.value != null) sb.append(this.value);
+    sb.append(")");
+    return sb.toString();
+  }
 }

src/main/java/org/caosdb/server/query/Negation.java

@@ -55,14 +55,6 @@ public class Negation implements EntityFilterInterface, QueryInterface {
       }
       return this.neg;
     }
-
-    @Override
-    public void apply(final QueryInterface query) {}
-
-    @Override
-    public Element toElement() {
-      return null;
-    }
   }
 
   private String targetSet = null;
@@ -175,4 +167,13 @@ public class Negation implements EntityFilterInterface, QueryInterface {
   public boolean isVersioned() {
     return this.query.isVersioned();
   }
+
+  @Override
+  public String getCacheKey() {
+    StringBuilder sb = new StringBuilder();
+    sb.append("NEG(");
+    if (this.filter != null) sb.append(this.filter.getCacheKey());
+    sb.append(")");
+    return sb.toString();
+  }
 }

src/main/java/org/caosdb/server/query/POV.java

@@ -512,4 +512,15 @@ public class POV implements EntityFilterInterface {
   private String measurement(String m) {
     return String.join("", prefix) + m;
   }
+
+  @Override
+  public String getCacheKey() {
+    StringBuilder sb = new StringBuilder();
+    if (this.getAggregate() != null) sb.append(this.aggregate);
+    sb.append(toString());
+    if (this.hasSubProperty()) {
+      sb.append(getSubProperty().getCacheKey());
+    }
+    return sb.toString();
+  }
 }

src/main/java/org/caosdb/server/query/Query.java

@@ -24,6 +24,7 @@ package org.caosdb.server.query;
 
 import static org.caosdb.server.database.DatabaseUtils.bytes2UTF8;
 
+import java.io.Serializable;
 import java.sql.CallableStatement;
 import java.sql.Connection;
 import java.sql.PreparedStatement;
@@ -40,9 +41,11 @@ import java.util.Map;
 import java.util.Map.Entry;
 import org.antlr.v4.runtime.CharStreams;
 import org.antlr.v4.runtime.CommonTokenStream;
+import org.apache.commons.jcs.access.behavior.ICacheAccess;
 import org.apache.shiro.subject.Subject;
 import org.caosdb.server.CaosDBServer;
 import org.caosdb.server.ServerProperties;
+import org.caosdb.server.caching.Cache;
 import org.caosdb.server.database.access.Access;
 import org.caosdb.server.database.backend.implementation.MySQL.ConnectionException;
 import org.caosdb.server.database.backend.implementation.MySQL.MySQLHelper;
@@ -63,6 +66,7 @@ import org.caosdb.server.query.CQLParser.CqContext;
 import org.caosdb.server.query.CQLParsingErrorListener.ParsingError;
 import org.caosdb.server.transaction.TransactionInterface;
 import org.jdom2.Element;
+import org.slf4j.Logger;
 
 public class Query implements QueryInterface, ToElementable, TransactionInterface {
 
@@ -184,6 +188,7 @@ public class Query implements QueryInterface, ToElementable, TransactionInterfac
               ServerProperties.KEY_QUERY_FILTER_ENTITIES_WITHOUT_RETRIEVE_PERMISSIONS)
           .equalsIgnoreCase("FALSE");
 
+  private Logger logger = org.slf4j.LoggerFactory.getLogger(getClass());
   List<IdVersionPair> resultSet = null;
   private final String query;
   private Pattern entity = null;
@@ -200,6 +205,16 @@ public class Query implements QueryInterface, ToElementable, TransactionInterfac
   private final ArrayList<ToElementable> messages = new ArrayList<>();
   private Access access;
   private boolean versioned = false;
+  private static ICacheAccess<String, Serializable> cache =
+      Cache.getCache("HIGH_LEVEL_QUERY_CACHE");
+  /** Cached=true means that the results of this query have actually been pulled from the cache. */
+  private boolean cached = false;
+  /**
+   * Cachable=false means that the results of this query cannot be used for the cache, because the
+   * query evaluation contains complex permission checking which could only be cached on a per-user
+   * basis (maybe in the future).
+   */
+  private boolean cachable = true;
 
   public Type getType() {
     return this.type;
@@ -500,31 +515,95 @@ public class Query implements QueryInterface, ToElementable, TransactionInterfac
     }
   }
 
+  /**
+   * Whether the transaction allows this query instance to use the query cache. This is controlled
+   * by the "cache" flag.
+   *
+   * @see {@link NoCache}
+   * @return true if caching is encouraged.
+   */
+  private boolean useCache() {
+    return getAccess().useCache();
+  }
+
+  /**
+   * Execute the query.
+   *
+   * <p>First try the cache and only then use the back-end.
+   *
+   * @param access
+   * @return
+   * @throws ParsingException
+   */
   public Query execute(final Access access) throws ParsingException {
-    setAccess(access);
     parse();
+    setAccess(access);
+    if (useCache()) {
+      this.resultSet = getCached(getCacheKey());
+    }
 
-    try {
-
-      this.resultSet = getResultSet(executeStrategy(this.versioned), this.versioned);
+    if (this.resultSet == null) {
+      executeNoCache(access);
+      if (this.cachable) {
+        setCache(getCacheKey(), this.resultSet);
+      }
+      this.logger.debug("Uncached query {}", this.query);
+    } else {
+      this.logger.debug("Using cached result for {}", this.query);
+      this.cached = true;
+    }
 
-      filterEntitiesWithoutRetrievePermission(this.resultSet);
+    this.resultSet = filterEntitiesWithoutRetrievePermission(this.resultSet);
 
-      // Fill resulting entities into container
-      if (this.container != null && this.type == Type.FIND) {
-        for (final IdVersionPair p : this.resultSet) {
+    // Fill resulting entities into container
+    if (this.container != null && this.type == Type.FIND) {
+      for (final IdVersionPair p : this.resultSet) {
 
-          final Entity e = new RetrieveEntity(p.id, p.version);
+        final Entity e = new RetrieveEntity(p.id, p.version);
 
-          // if query has select-clause:
-          if (this.selections != null && !this.selections.isEmpty()) {
-            e.addSelections(this.selections);
-          }
-          this.container.add(e);
+        // if query has select-clause:
+        if (this.selections != null && !this.selections.isEmpty()) {
+          e.addSelections(this.selections);
         }
+        this.container.add(e);
       }
-      return this;
+    }
+    return this;
+  }
 
+  /** Remove all cached queries from the cache. */
+  public static void clearCache() {
+    cache.clear();
+  }
+
+  /**
+   * Cache a query result.
+   *
+   * @param key
+   * @param resultSet
+   */
+  private void setCache(String key, List<IdVersionPair> resultSet) {
+    if (resultSet instanceof Serializable) {
+      cache.put(key, (Serializable) resultSet);
+    } else {
+      cache.put(key, new ArrayList<>(resultSet));
+    }
+  }
+
+  /**
+   * Retrieve a result set of entity ids (and the version) from the cache.
+   *
+   * @param key
+   * @return
+   */
+  @SuppressWarnings("unchecked")
+  private List<IdVersionPair> getCached(String key) {
+    return (List<IdVersionPair>) cache.get(key);
+  }
+
+  protected void executeNoCache(Access access) {
+    try {
+      this.resultSet = getResultSet(executeStrategy(this.versioned), this.versioned);
     } finally {
       cleanUp();
     }
@@ -564,31 +643,31 @@ public class Query implements QueryInterface, ToElementable, TransactionInterfac
    * Filter out all entities which may not be retrieved by this user due to a missing RETRIEVE
    * permission. This one is also designed for filtering of intermediate results.
    *
-   * @param query
    * @param resultSet
    * @throws SQLException
    * @throws TransactionException
    */
-  public void filterEntitiesWithoutRetrievePermission(
-      final QueryInterface query, final String resultSet)
+  public void filterEntitiesWithoutRetrievePermission(final String resultSet)
       throws SQLException, TransactionException {
     if (!filterEntitiesWithoutRetrievePermisions) {
       return;
     }
-    try (final Statement stmt = query.getConnection().createStatement()) {
+    cachable = false;
+    try (final Statement stmt = this.getConnection().createStatement()) {
       final ResultSet rs = stmt.executeQuery("SELECT id from `" + resultSet + "`");
       final List<Integer> toBeDeleted = new LinkedList<Integer>();
       while (rs.next()) {
         final long t1 = System.currentTimeMillis();
         final Integer id = rs.getInt("id");
-        if (!execute(new RetrieveSparseEntity(id, null), query.getAccess())
-            .getEntity()
-            .getEntityACL()
-            .isPermitted(query.getUser(), EntityPermission.RETRIEVE_ENTITY)) {
+        if (id > 99
+            && !execute(new RetrieveSparseEntity(id, null), this.getAccess())
+                .getEntity()
+                .getEntityACL()
+                .isPermitted(this.getUser(), EntityPermission.RETRIEVE_ENTITY)) {
           toBeDeleted.add(id);
         }
         final long t2 = System.currentTimeMillis();
-        query.addBenchmark("filterEntitiesWithoutRetrievePermission", t2 - t1);
+        this.addBenchmark("filterEntitiesWithoutRetrievePermission", t2 - t1);
       }
       rs.close();
       for (final Integer id : toBeDeleted) {
@@ -604,25 +683,30 @@ public class Query implements QueryInterface, ToElementable, TransactionInterfac
    *
    * @param entities
    * @throws TransactionException
+   * @return the filtered list.
    */
-  private void filterEntitiesWithoutRetrievePermission(final List<IdVersionPair> entities)
-      throws TransactionException {
+  private List<IdVersionPair> filterEntitiesWithoutRetrievePermission(
+      final List<IdVersionPair> entities) throws TransactionException {
     if (!filterEntitiesWithoutRetrievePermisions) {
-      return;
+      return entities;
     }
+
+    List<IdVersionPair> result = new ArrayList<>();
     final Iterator<IdVersionPair> iterator = entities.iterator();
     while (iterator.hasNext()) {
       final long t1 = System.currentTimeMillis();
       final IdVersionPair next = iterator.next();
-      if (!execute(new RetrieveSparseEntity(next.id, next.version), getAccess())
-          .getEntity()
-          .getEntityACL()
-          .isPermitted(getUser(), EntityPermission.RETRIEVE_ENTITY)) {
-        iterator.remove();
+      if (next.id > 99
+          && execute(new RetrieveSparseEntity(next.id, next.version), getAccess())
+              .getEntity()
+              .getEntityACL()
+              .isPermitted(getUser(), EntityPermission.RETRIEVE_ENTITY)) {
+        result.add(next);
       }
       final long t2 = System.currentTimeMillis();
       addBenchmark("filterEntitiesWithoutRetrievePermission", t2 - t1);
     }
+    return result;
   }
 
   @Override
@@ -679,6 +763,7 @@ public class Query implements QueryInterface, ToElementable, TransactionInterfac
     } else {
       ret.setAttribute("results", "0");
     }
+    ret.setAttribute("cached", Boolean.toString(this.cached));
 
     final Element parseTreeElem = new Element("ParseTree");
     if (this.el.hasErrors()) {
@@ -769,4 +854,19 @@ public class Query implements QueryInterface, ToElementable, TransactionInterfac
   public boolean isVersioned() {
     return this.versioned;
   }
+
+  /**
+   * Return a key for the query cache. The key should describe the query with all the filters but
+   * without the FIND, COUNT and SELECT ... FROM parts.
+   *
+   * @return A Cache key.
+   */
+  String getCacheKey() {
+    StringBuilder sb = new StringBuilder();
+    if (this.versioned) sb.append("versioned");
+    if (this.role != null) sb.append(this.role.toString());
+    if (this.entity != null) sb.append(this.entity.toString());
+    if (this.filter != null) sb.append(this.filter.getCacheKey());
+    return sb.toString();
+  }
 }

src/main/java/org/caosdb/server/query/RoleFilter.java

@@ -148,4 +148,10 @@ public class RoleFilter implements EntityFilterInterface {
   private String getOperator() {
     return this.operator;
   }
+
+  @Override
+  public String getCacheKey() {
+    // unused
+    return null;
+  }
 }

src/main/java/org/caosdb/server/query/StoredAt.java

@@ -201,4 +201,9 @@ public class StoredAt implements EntityFilterInterface {
   public String toString() {
     return "SAT(" + (this.pattern_matching ? this.likeLocation : this.location) + ")";
   }
+
+  @Override
+  public String getCacheKey() {
+    return toString();
+  }
 }

src/main/java/org/caosdb/server/query/SubProperty.java

@@ -87,7 +87,7 @@ public class SubProperty implements QueryInterface, EntityFilterInterface {
 
         this.filter.apply(this);
 
-        getQuery().filterEntitiesWithoutRetrievePermission(this, this.sourceSet);
+        getQuery().filterEntitiesWithoutRetrievePermission(this.sourceSet);
 
         final CallableStatement callFinishSubProperty =
             getConnection().prepareCall("call finishSubProperty(?,?,?,?)");
@@ -167,4 +167,12 @@ public class SubProperty implements QueryInterface, EntityFilterInterface {
   public boolean isVersioned() {
     return this.query.isVersioned();
   }
+
+  @Override
+  public String getCacheKey() {
+    StringBuilder sb = new StringBuilder();
+    sb.append("SUB(");
+    if (this.filter != null) sb.append(this.filter.getCacheKey());
+    return sb.toString();
+  }
 }

src/main/java/org/caosdb/server/query/TransactionFilter.java

@@ -256,4 +256,9 @@ public class TransactionFilter implements EntityFilterInterface {
         + this.transactor
         + ")";
   }
+
+  @Override
+  public String getCacheKey() {
+    return toString();
+  }
 }

src/main/java/org/caosdb/server/transaction/WriteTransaction.java

@@ -25,6 +25,7 @@ package org.caosdb.server.transaction;
 import org.caosdb.server.database.misc.RollBackHandler;
 import org.caosdb.server.entity.FileProperties;
 import org.caosdb.server.entity.container.TransactionContainer;
+import org.caosdb.server.query.Query;
 
 public abstract class WriteTransaction<C extends TransactionContainer> extends Transaction<C> {
 
@@ -42,6 +43,7 @@ public abstract class WriteTransaction<C extends TransactionContainer> extends T
   @Override
   protected void commit() throws Exception {
     getAccess().commit();
+    Query.clearCache();
   }
 
   @Override

src/test/java/org/caosdb/server/query/QueryTest.java

+package org.caosdb.server.query;
+
+import static org.junit.Assert.assertEquals;
+
+import java.io.IOException;
+import org.caosdb.server.CaosDBServer;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+public class QueryTest {
+
+  @BeforeClass
+  public static void initServerProperties() throws IOException {
+    CaosDBServer.initServerProperties();
+  }
+
+  String getCacheKey(String query) {
+    Query q = new Query(query);
+    q.parse();
+    return q.getCacheKey();
+  }
+
+  @Test
+  public void testGetKey() {
+    assertEquals("enamePOV(pname,=,val1)", getCacheKey("FIND ename WITH pname = val1"));
+    assertEquals("enamePOV(pname,=,val1)", getCacheKey("COUNT ename WITH pname = val1"));
+    assertEquals("enamePOV(pname,=,val1)", getCacheKey("SELECT bla FROM ename WITH pname = val1"));
+    assertEquals("enamePOV(pname,null,null)", getCacheKey("SELECT bla FROM ename WITH pname"));
+    assertEquals(
+        "enamemaxPOV(pname,null,null)",
+        getCacheKey("SELECT bla FROM ename WITH THE GREATEST pname"));
+
+    assertEquals(
+        "RECORDenamePOV(pname,=,val1)", getCacheKey("FIND RECORD ename WITH pname = val1"));
+    assertEquals("ENTITYPOV(pname,=,val1)", getCacheKey("COUNT ENTITY WITH pname = val1"));
+    assertEquals(
+        "enameConj(POV(pname,=,val1)POV(ename2,=,val2))",
+        getCacheKey("SELECT bla FROM ename WITH pname = val1 AND ename2 = val2"));
+
+    assertEquals("versionedENTITYID(,>,2)", getCacheKey("FIND ANY VERSION OF ENTITY WITH ID > 2"));
+    assertEquals("ENTITYID(min,,)", getCacheKey("FIND ENTITY WITH THE SMALLEST ID"));
+    assertEquals("ENTITYSAT(asdf/%%)", getCacheKey("FIND ENTITY WHICH IS STORED AT /asdf/*"));
+    assertEquals("ENTITYSAT(asdf/asdf)", getCacheKey("FIND ENTITY WHICH IS STORED AT asdf/asdf"));
+    assertEquals(
+        "enamePOV(ref1,null,null)SUB(POV(pname,>,val1)",
+        getCacheKey("FIND ename WITH ref1 WITH pname > val1 "));
+    assertEquals(
+        "ename@(ref1,null)SUB(POV(pname,>,val1)",
+        getCacheKey("FIND ename WHICH IS REFERENCED BY ref1 WITH pname > val1 "));
+  }
+}