Merge branch 'f-enh-paging' into 'dev'

ENH: apply paging such that no unwanted jobs are being triggered

See merge request !81

CHANGELOG.md

@@ -31,8 +31,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 ### Removed ###
 
 ### Fixed ###
-- [#203](https://gitlab.com/caosdb/caosdb-server/-/issues/203)
 
+* Bad performance due to the execution of unnecessary jobs during retrieval.
+  [#189](https://gitlab.com/caosdb/caosdb-server/-/issues/189)
+* Query Language: Parentheses change filter to subproperty filter
+  [#203](https://gitlab.com/caosdb/caosdb-server/-/issues/203)
 * Searching for values in scientific notation
   [#143](https://gitlab.com/caosdb/caosdb-server/-/issues/143)
 * Denying a role permission has no effect

src/main/java/org/caosdb/server/entity/Entity.java

@@ -1004,7 +1004,7 @@ public abstract class Entity extends AbstractObservable implements EntityInterfa
 
   @Override
   public boolean skipJob() {
-    return false;
+    return this.entityStatus == EntityStatus.IGNORE;
   }
 
   @Override

src/main/java/org/caosdb/server/entity/xml/ParentToElementStrategy.java

@@ -26,7 +26,6 @@ package org.caosdb.server.entity.xml;
 
 import org.caosdb.server.entity.EntityInterface;
 import org.caosdb.server.entity.wrapper.Parent;
-import org.caosdb.server.utils.EntityStatus;
 import org.jdom2.Element;
 
 /**
@@ -51,15 +50,4 @@ public class ParentToElementStrategy extends EntityToElementStrategy {
     }
     return element;
   }
-
-  @Override
-  public Element addToElement(
-      final EntityInterface entity,
-      final Element element,
-      final SerializeFieldStrategy setFieldStrategy) {
-    if (entity.getEntityStatus() != EntityStatus.IGNORE) {
-      element.addContent(toElement(entity, setFieldStrategy));
-    }
-    return element;
-  }
 }

src/main/java/org/caosdb/server/jobs/core/ExecuteQuery.java

@@ -31,15 +31,20 @@ import org.caosdb.server.jobs.JobAnnotation;
 import org.caosdb.server.jobs.TransactionStage;
 import org.caosdb.server.query.Query;
 import org.caosdb.server.query.Query.ParsingException;
+import org.caosdb.server.transaction.Retrieve;
+import org.caosdb.server.utils.EntityStatus;
 import org.caosdb.server.utils.ServerMessages;
 
-@JobAnnotation(flag = "query", stage = TransactionStage.INIT)
+@JobAnnotation(flag = "query", stage = TransactionStage.INIT, transaction = Retrieve.class)
 public class ExecuteQuery extends FlagJob {
 
   @Override
   protected void job(final String value) {
 
     if (value != null) {
+      // run paging job first
+      getTransaction().getSchedule().runJob(null, Paging.class);
+
       final Query queryInstance =
           new Query(value, getTransaction().getTransactor(), getContainer());
       getContainer().setQuery(queryInstance);
@@ -53,8 +58,24 @@ public class ExecuteQuery extends FlagJob {
             .addMessage(new Message(MessageType.Info, (MessageCode) null, e.getMessage()));
       }
       getContainer().addMessage(queryInstance);
+
+      int startIndex = 0;
+      int endIndex = getContainer().size();
+
+      if (((Retrieve) getTransaction()).hasPaging()) {
+        Retrieve.Paging paging = ((Retrieve) getTransaction()).getPaging();
+        startIndex = Math.min(getContainer().size(), paging.startIndex);
+        endIndex = Math.min(getContainer().size(), paging.endIndex);
+      }
+
+      int ii = 0;
       for (final EntityInterface entity : getContainer()) {
-        getTransaction().getSchedule().addAll(loadJobs(entity, getTransaction()));
+        if (ii >= startIndex && ii < endIndex) {
+          getTransaction().getSchedule().addAll(loadJobs(entity, getTransaction()));
+        } else {
+          entity.setEntityStatus(EntityStatus.IGNORE);
+        }
+        ii++;
       }
     }
   }

src/main/java/org/caosdb/server/jobs/core/Paging.java

@@ -31,7 +31,7 @@ import org.caosdb.server.jobs.TransactionStage;
 import org.caosdb.server.transaction.Retrieve;
 import org.caosdb.server.utils.EntityStatus;
 
-@JobAnnotation(flag = "P", stage = TransactionStage.PRE_TRANSACTION)
+@JobAnnotation(flag = "P", stage = TransactionStage.INIT, transaction = Retrieve.class)
 public class Paging extends FlagJob {
 
   public static final int DEFAULT_LENGTH = 100;
@@ -42,20 +42,24 @@ public class Paging extends FlagJob {
   protected void job(final String value) {
     if (getTransaction() instanceof Retrieve) {
       if (value != null) {
-        int index1 = DEFAULT_INDEX;
-        int index2 = index1 + DEFAULT_LENGTH;
+        int startIndex = DEFAULT_INDEX;
+        int endIndex = startIndex + DEFAULT_LENGTH;
         final Matcher m = pattern.matcher(value);
         if (m.matches()) {
           if (m.group(1) != null) {
-            index1 = Integer.parseInt(m.group(1));
+            startIndex = Integer.parseInt(m.group(1));
           }
           if (m.group(2) != null) {
-            index2 = index1 + Integer.parseInt(m.group(2));
+            endIndex = startIndex + Integer.parseInt(m.group(2));
           }
         }
+
+        // this info may be used by other jobs
+        ((Retrieve) getTransaction()).setPaging(startIndex, endIndex);
+
         int i = 0;
         for (final EntityInterface e : getContainer()) {
-          if (i >= index2 || i < index1) {
+          if (i >= endIndex || i < startIndex) {
             // do not retrieve this entity
             e.setEntityStatus(EntityStatus.IGNORE);
           }

src/main/java/org/caosdb/server/jobs/core/RetrieveAllJob.java

@@ -27,19 +27,43 @@ import org.caosdb.server.entity.EntityInterface;
 import org.caosdb.server.jobs.FlagJob;
 import org.caosdb.server.jobs.JobAnnotation;
 import org.caosdb.server.jobs.TransactionStage;
+import org.caosdb.server.transaction.Retrieve;
+import org.caosdb.server.utils.EntityStatus;
 
-@JobAnnotation(flag = "all", stage = TransactionStage.INIT)
+@JobAnnotation(flag = "all", stage = TransactionStage.INIT, transaction = Retrieve.class)
 public class RetrieveAllJob extends FlagJob {
 
   @Override
   protected void job(String value) {
     if (getContainer().isEmpty()) {
+      // run paging job first
+      getTransaction().getSchedule().runJob(null, Paging.class);
+
       if (value == null) {
         value = "ENTITY";
       }
+
       execute(new RetrieveAll(getContainer(), value));
+
+      int startIndex = 0;
+      int endIndex = getContainer().size();
+
+      if (((Retrieve) getTransaction()).hasPaging()) {
+        startIndex =
+            Math.min(getContainer().size(), ((Retrieve) getTransaction()).getPaging().startIndex);
+        endIndex =
+            Math.min(getContainer().size(), ((Retrieve) getTransaction()).getPaging().endIndex);
+      }
+
+      // only add jobs for those which are on this page
+      int ii = 0;
       for (EntityInterface entity : getContainer()) {
-        getTransaction().getSchedule().addAll(loadJobs(entity, getTransaction()));
+        if (ii >= startIndex && ii < endIndex) {
+          getTransaction().getSchedule().addAll(loadJobs(entity, getTransaction()));
+        } else {
+          entity.setEntityStatus(EntityStatus.IGNORE);
+        }
+        ii++;
       }
     }
   }

src/main/java/org/caosdb/server/transaction/Retrieve.java

@@ -121,4 +121,28 @@ public class Retrieve extends Transaction<RetrieveContainer> {
   public boolean logHistory() {
     return false;
   }
+
+  public static class Paging {
+    public Paging(int startIndex, int endIndex) {
+      this.startIndex = startIndex;
+      this.endIndex = endIndex;
+    }
+
+    public final int startIndex;
+    public final int endIndex;
+  }
+
+  private Retrieve.Paging paging = null;
+
+  public boolean hasPaging() {
+    return this.paging != null;
+  }
+
+  public void setPaging(int startIndex, int endIndex) {
+    this.paging = new Retrieve.Paging(startIndex, endIndex);
+  }
+
+  public Retrieve.Paging getPaging() {
+    return paging;
+  }
 }

src/main/java/org/caosdb/server/transaction/Transaction.java

@@ -105,6 +105,7 @@ public abstract class Transaction<C extends TransactionContainer> extends Abstra
     this.schedule.addAll(Job.loadPermanentContainerJobs(this));
 
     for (final EntityInterface e : getContainer()) {
+      if (e.skipJob()) continue;
       final List<Job> loadJobs = loadContainerFlags.loadJobs(e, this);
       this.schedule.addAll(loadJobs);