Merge branch 'f-grpc-main' into f-grpc-dev

CHANGELOG.md

@@ -18,8 +18,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Removed
 
+* `IdOnly` flag (see https://gitlab.indiscale.com/caosdb/src/caosdb-server/-/issues/187).
+  The flags was not working anyways. However, `SELECT id FROM ...` queries are
+  now optimized in the way the `IdOnly` flag was supposed to do.
+
 ### Fixed
 
+* #181 CQL's `UPDATED` filter.
+  (https://gitlab.indiscale.com/caosdb/src/caosdb-server/-/issues/181)
 * #183 No reasonable error when using bad datetime format.
   (https://gitlab.indiscale.com/caosdb/src/caosdb-server/-/issues/183)
 * #127 "nan" as value (list item) in properties with data type "LIST<DOUBLE>"

src/main/java/org/caosdb/server/database/backend/transaction/InsertTransactionHistory.java

@@ -26,6 +26,8 @@ import org.caosdb.server.database.BackendTransaction;
 import org.caosdb.server.database.backend.interfaces.InsertTransactionHistoryImpl;
 import org.caosdb.server.database.exceptions.TransactionException;
 import org.caosdb.server.entity.EntityInterface;
+import org.caosdb.server.entity.InsertEntity;
+import org.caosdb.server.entity.UpdateEntity;
 import org.caosdb.server.entity.container.TransactionContainer;
 import org.caosdb.server.utils.EntityStatus;
 
@@ -60,7 +62,8 @@ public class InsertTransactionHistory extends BackendTransaction {
 
     for (final EntityInterface e : this.container) {
       if (e.getEntityStatus() == EntityStatus.DELETED
-          || e.getEntityStatus() == EntityStatus.VALID) {
+          || (e instanceof UpdateEntity && e.getEntityStatus() == EntityStatus.QUALIFIED)
+          || (e instanceof InsertEntity && e.getEntityStatus() == EntityStatus.VALID)) {
 
         t.execute(
             e.getClass().getSimpleName().replace("Entity", ""),

src/main/java/org/caosdb/server/database/backend/transaction/RetrieveFullEntityTransaction.java

@@ -101,6 +101,9 @@ public class RetrieveFullEntityTransaction extends BackendTransaction {
    * @param selections
    */
   public void retrieveFullEntity(EntityInterface e, List<Selection> selections) {
+    if (!needMoreThanId(selections)) {
+      return;
+    }
     execute(new RetrieveSparseEntity(e));
 
     if (e.getEntityStatus() == EntityStatus.VALID) {
@@ -108,8 +111,13 @@ public class RetrieveFullEntityTransaction extends BackendTransaction {
       if (e.getRole() == Role.QueryTemplate) {
         execute(new RetrieveQueryTemplateDefinition(e));
       }
-      execute(new RetrieveParents(e));
-      execute(new RetrieveProperties(e));
+      if (needParents(selections)) {
+        execute(new RetrieveParents(e));
+      }
+
+      if (needProperties(selections)) {
+        execute(new RetrieveProperties(e));
+      }
 
       // recursion! retrieveSubEntities calls retrieveFull sometimes, but with reduced selectors.
       if (selections != null && !selections.isEmpty()) {
@@ -118,6 +126,55 @@ public class RetrieveFullEntityTransaction extends BackendTransaction {
     }
   }
 
+  /**
+   * Return true iff anything else than the id is needed for this retrieval.
+   *
+   * <p>The notorious case, where it is not necessary to retrieve the sparse entity is during
+   * `SELECT id FROM ...` queries.
+   */
+  private boolean needMoreThanId(List<Selection> selections) {
+    if (selections == null || selections.isEmpty()) {
+      return true;
+    } else if (selections.size() == 1 && selections.get(0).isId()) {
+      return false;
+    }
+    return true;
+  }
+
+  /**
+   * Return true iff the properties need to be retrieved.
+   *
+   * <p>It is not necessary during `SELECT parent, name, version, ...` queries where no actual
+   * properties are being selected.
+   */
+  private boolean needProperties(List<Selection> selections) {
+    if (selections == null || selections.isEmpty()) {
+      return true;
+    }
+    for (Selection s : selections) {
+      if (s.isProperty()) {
+        return true;
+      }
+    }
+    return false;
+  }
+  /**
+   * Return true iff the parents need to be retrieved.
+   *
+   * <p>It is not necessary during `SELECT` queries that do not select the parent.
+   */
+  private boolean needParents(List<Selection> selections) {
+    if (selections == null || selections.isEmpty()) {
+      return true;
+    }
+    for (Selection s : selections) {
+      if (s.isParent()) {
+        return true;
+      }
+    }
+    return false;
+  }
+
   /**
    * Recursively resolve the reference values of the list of reference property `p` (but only the
    * selected sub-properties).

src/main/java/org/caosdb/server/jobs/core/RetrieveIdOnlyFlag.java

-/*
- * ** header v3.0
- * This file is a part of the CaosDB Project.
- *
- * Copyright (C) 2018 Research Group Biomedical Physics,
- * Max-Planck-Institute for Dynamics and Self-Organization Göttingen
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as
- * published by the Free Software Foundation, either version 3 of the
- * License, or (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <https://www.gnu.org/licenses/>.
- *
- * ** end header
- */
-package org.caosdb.server.jobs.core;
-
-import org.caosdb.server.entity.EntityInterface;
-import org.caosdb.server.jobs.FlagJob;
-import org.caosdb.server.jobs.JobAnnotation;
-import org.caosdb.server.jobs.TransactionStage;
-import org.caosdb.server.utils.EntityStatus;
-
-@JobAnnotation(flag = "IdOnly", stage = TransactionStage.PRE_TRANSACTION)
-public class RetrieveIdOnlyFlag extends FlagJob {
-
-  @Override
-  protected void job(final String value) {
-    if (value == null || value.equalsIgnoreCase("true")) {
-      for (final EntityInterface e : getContainer()) {
-        // do not retrieve this entity
-        e.setEntityStatus(EntityStatus.IGNORE);
-      }
-    }
-  }
-}

src/main/java/org/caosdb/server/query/Query.java

@@ -32,12 +32,15 @@ import java.sql.SQLException;
 import java.sql.Statement;
 import java.sql.Types;
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.HashMap;
+import java.util.HashSet;
 import java.util.Iterator;
 import java.util.LinkedList;
 import java.util.List;
 import java.util.Map;
 import java.util.Map.Entry;
+import java.util.Set;
 import java.util.UUID;
 import org.antlr.v4.runtime.CharStreams;
 import org.antlr.v4.runtime.CommonTokenStream;
@@ -75,6 +78,16 @@ public class Query implements QueryInterface, ToElementable, TransactionInterfac
     private final String selector;
     private Selection subselection = null;
 
+    /**
+     * These magic non-properties are those which are in the results of a {@link
+     * RetrieveSparseEntity} transaction which is why they do not need to be retrieved with a {@link
+     * RetrieveEntityProperties} transaction.
+     */
+    static final Set<String> MAGIC_NON_PROPERTIES =
+        new HashSet<>(
+            Arrays.asList(
+                new String[] {"value", "parent", "id", "name", "description", "datatype"}));
+
     public Selection setSubSelection(final Selection sub) {
       if (this.subselection != null) {
         throw new UnsupportedOperationException("SubSelection is immutable!");
@@ -111,6 +124,29 @@ public class Query implements QueryInterface, ToElementable, TransactionInterfac
       ret.setAttribute("name", toString());
       return ret;
     }
+
+    /**
+     * Return true iff this selector selects the parent of an entity. If not, the retrieval in
+     * {@link RetrieveFullEntityTransaction} might be optimized by not retrieving the parents at
+     * all.
+     */
+    public boolean isParent() {
+      return this.selector.equalsIgnoreCase("parent");
+    }
+
+    /**
+     * Return true iff this selector selects anything that is most likely a property. If not, the
+     * retrieval in {@link RetrieveFullEntityTransaction} might be optimized by not retrieving the
+     * properties at all.
+     */
+    public boolean isProperty() {
+      return !MAGIC_NON_PROPERTIES.contains(this.selector.toLowerCase());
+    }
+
+    /** Return true iff this selector selects the id of an entity. */
+    public boolean isId() {
+      return this.selector.equalsIgnoreCase("id");
+    }
   }
 
   public enum Role {