From 8d288196f72c423fc78fe3dd8e9bd4326b4e0d8a Mon Sep 17 00:00:00 2001
From: Timm Fitschen <t.fitschen@indiscale.com>
Date: Wed, 24 May 2023 01:26:00 +0200
Subject: [PATCH] WIP: enhanced paging
---
.../java/org/caosdb/server/entity/Entity.java | 2 +-
.../entity/xml/ParentToElementStrategy.java | 12 ---------
.../caosdb/server/jobs/TransactionStage.java | 1 -
.../caosdb/server/jobs/core/ExecuteQuery.java | 21 ++++++++++-----
.../org/caosdb/server/jobs/core/Paging.java | 21 ++++++++-------
.../server/jobs/core/RetrieveAllJob.java | 26 +++++++++++++++++--
.../java/org/caosdb/server/query/Query.java | 3 +--
.../caosdb/server/transaction/Retrieve.java | 24 +++++++++++++++++
.../server/transaction/Transaction.java | 23 +---------------
9 files changed, 78 insertions(+), 55 deletions(-)
diff --git a/src/main/java/org/caosdb/server/entity/Entity.java b/src/main/java/org/caosdb/server/entity/Entity.java
index 2668e770..48367508 100644
--- a/src/main/java/org/caosdb/server/entity/Entity.java
+++ b/src/main/java/org/caosdb/server/entity/Entity.java
@@ -1004,7 +1004,7 @@ public abstract class Entity extends AbstractObservable implements EntityInterfa
@Override
public boolean skipJob() {
- return false;
+ return this.entityStatus == EntityStatus.IGNORE;
}
@Override
diff --git a/src/main/java/org/caosdb/server/entity/xml/ParentToElementStrategy.java b/src/main/java/org/caosdb/server/entity/xml/ParentToElementStrategy.java
index 4f790682..2556fdb2 100644
--- a/src/main/java/org/caosdb/server/entity/xml/ParentToElementStrategy.java
+++ b/src/main/java/org/caosdb/server/entity/xml/ParentToElementStrategy.java
@@ -26,7 +26,6 @@ package org.caosdb.server.entity.xml;
import org.caosdb.server.entity.EntityInterface;
import org.caosdb.server.entity.wrapper.Parent;
-import org.caosdb.server.utils.EntityStatus;
import org.jdom2.Element;
/**
@@ -51,15 +50,4 @@ public class ParentToElementStrategy extends EntityToElementStrategy {
}
return element;
}
-
- @Override
- public Element addToElement(
- final EntityInterface entity,
- final Element element,
- final SerializeFieldStrategy setFieldStrategy) {
- if (entity.getEntityStatus() != EntityStatus.IGNORE) {
- element.addContent(toElement(entity, setFieldStrategy));
- }
- return element;
- }
}
diff --git a/src/main/java/org/caosdb/server/jobs/TransactionStage.java b/src/main/java/org/caosdb/server/jobs/TransactionStage.java
index c265fc25..9e15b4e5 100644
--- a/src/main/java/org/caosdb/server/jobs/TransactionStage.java
+++ b/src/main/java/org/caosdb/server/jobs/TransactionStage.java
@@ -50,7 +50,6 @@ import org.caosdb.server.utils.UndoHandler;
* @author Timm Fitschen <t.fitschen@indiscale.com>
*/
public enum TransactionStage {
- PRE_INIT,
INIT,
PRE_CHECK,
CHECK,
diff --git a/src/main/java/org/caosdb/server/jobs/core/ExecuteQuery.java b/src/main/java/org/caosdb/server/jobs/core/ExecuteQuery.java
index 017555e0..5e4d02f7 100644
--- a/src/main/java/org/caosdb/server/jobs/core/ExecuteQuery.java
+++ b/src/main/java/org/caosdb/server/jobs/core/ExecuteQuery.java
@@ -35,13 +35,16 @@ import org.caosdb.server.transaction.Retrieve;
import org.caosdb.server.utils.EntityStatus;
import org.caosdb.server.utils.ServerMessages;
-@JobAnnotation(flag = "query", stage = TransactionStage.INIT)
+@JobAnnotation(flag = "query", stage = TransactionStage.INIT, transaction = Retrieve.class)
public class ExecuteQuery extends FlagJob {
@Override
protected void job(final String value) {
if (value != null) {
+ // run paging job first
+ getTransaction().getSchedule().runJob(null, Paging.class);
+
final Query queryInstance =
new Query(value, getTransaction().getTransactor(), getContainer());
getContainer().setQuery(queryInstance);
@@ -55,13 +58,19 @@ public class ExecuteQuery extends FlagJob {
.addMessage(new Message(MessageType.Info, (MessageCode) null, e.getMessage()));
}
getContainer().addMessage(queryInstance);
- int index1 = getTransaction().getPagingIndex1();
- int index2 = getTransaction().getPagingIndex2();
+
+ int index = 0;
+ int length = getContainer().size();
+
+ if (((Retrieve) getTransaction()).hasPaging()) {
+ Retrieve.Paging paging = ((Retrieve) getTransaction()).getPaging();
+ index = Math.min(getContainer().size(), paging.index);
+ length = Math.min(getContainer().size(), paging.length);
+ }
+
int ii = 0;
for (final EntityInterface entity : getContainer()) {
- if (!(getTransaction() instanceof Retrieve)
- || !getTransaction().hasPaging()
- || (ii >= index1 && ii < index2)) {
+ if (ii >= index && ii < length) {
getTransaction().getSchedule().addAll(loadJobs(entity, getTransaction()));
} else {
entity.setEntityStatus(EntityStatus.IGNORE);
diff --git a/src/main/java/org/caosdb/server/jobs/core/Paging.java b/src/main/java/org/caosdb/server/jobs/core/Paging.java
index ce499a5b..d7814b71 100644
--- a/src/main/java/org/caosdb/server/jobs/core/Paging.java
+++ b/src/main/java/org/caosdb/server/jobs/core/Paging.java
@@ -24,12 +24,14 @@ package org.caosdb.server.jobs.core;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
+import org.caosdb.server.entity.EntityInterface;
import org.caosdb.server.jobs.FlagJob;
import org.caosdb.server.jobs.JobAnnotation;
import org.caosdb.server.jobs.TransactionStage;
import org.caosdb.server.transaction.Retrieve;
+import org.caosdb.server.utils.EntityStatus;
-@JobAnnotation(flag = "P", stage = TransactionStage.PRE_INIT)
+@JobAnnotation(flag = "P", stage = TransactionStage.INIT, transaction = Retrieve.class)
public class Paging extends FlagJob {
public static final int DEFAULT_LENGTH = 100;
@@ -40,28 +42,29 @@ public class Paging extends FlagJob {
protected void job(final String value) {
if (getTransaction() instanceof Retrieve) {
if (value != null) {
- int index1 = DEFAULT_INDEX;
- int index2 = index1 + DEFAULT_LENGTH;
+ int index = DEFAULT_INDEX;
+ int length = index + DEFAULT_LENGTH;
final Matcher m = pattern.matcher(value);
if (m.matches()) {
if (m.group(1) != null) {
- index1 = Integer.parseInt(m.group(1));
+ index = Integer.parseInt(m.group(1));
}
if (m.group(2) != null) {
- index2 = index1 + Integer.parseInt(m.group(2));
+ length = index + Integer.parseInt(m.group(2));
}
}
- getTransaction().setPagingIndices(index1, index2);
- /*
+
+ // this info may be used by other jobs
+ ((Retrieve) getTransaction()).setPaging(index, length);
+
int i = 0;
for (final EntityInterface e : getContainer()) {
- if (i >= index2 || i < index1) {
+ if (i >= length || i < index) {
// do not retrieve this entity
e.setEntityStatus(EntityStatus.IGNORE);
}
i++;
}
- */
}
}
}
diff --git a/src/main/java/org/caosdb/server/jobs/core/RetrieveAllJob.java b/src/main/java/org/caosdb/server/jobs/core/RetrieveAllJob.java
index 7cd7791e..fcd1ee7f 100644
--- a/src/main/java/org/caosdb/server/jobs/core/RetrieveAllJob.java
+++ b/src/main/java/org/caosdb/server/jobs/core/RetrieveAllJob.java
@@ -27,19 +27,41 @@ import org.caosdb.server.entity.EntityInterface;
import org.caosdb.server.jobs.FlagJob;
import org.caosdb.server.jobs.JobAnnotation;
import org.caosdb.server.jobs.TransactionStage;
+import org.caosdb.server.transaction.Retrieve;
+import org.caosdb.server.utils.EntityStatus;
-@JobAnnotation(flag = "all", stage = TransactionStage.INIT)
+@JobAnnotation(flag = "all", stage = TransactionStage.INIT, transaction = Retrieve.class)
public class RetrieveAllJob extends FlagJob {
@Override
protected void job(String value) {
if (getContainer().isEmpty()) {
+ // run paging job first
+ getTransaction().getSchedule().runJob(null, Paging.class);
+
if (value == null) {
value = "ENTITY";
}
+
execute(new RetrieveAll(getContainer(), value));
+
+ int index = 0;
+ int length = getContainer().size();
+
+ if (((Retrieve) getTransaction()).hasPaging()) {
+ index = Math.min(getContainer().size(), ((Retrieve) getTransaction()).getPaging().index);
+ length = Math.min(getContainer().size(), ((Retrieve) getTransaction()).getPaging().length);
+ }
+
+ // only add jobs for those which are on this page
+ int ii = 0;
for (EntityInterface entity : getContainer()) {
- getTransaction().getSchedule().addAll(loadJobs(entity, getTransaction()));
+ if (ii >= index && ii < length) {
+ getTransaction().getSchedule().addAll(loadJobs(entity, getTransaction()));
+ } else {
+ entity.setEntityStatus(EntityStatus.IGNORE);
+ }
+ ii++;
}
}
}
diff --git a/src/main/java/org/caosdb/server/query/Query.java b/src/main/java/org/caosdb/server/query/Query.java
index f1abe03e..c4a45863 100644
--- a/src/main/java/org/caosdb/server/query/Query.java
+++ b/src/main/java/org/caosdb/server/query/Query.java
@@ -876,8 +876,7 @@ public class Query implements QueryInterface, ToElementable, TransactionInterfac
/**
* Filter out all entities which may not be retrieved by this user due to a missing RETRIEVE
- * permission. This function is also designed for filtering of intermediate results
- * TODO describe intermediate results.
+ * permission. This function is also designed for filtering of intermediate results.
*
* @param tabname
* @throws SQLException
diff --git a/src/main/java/org/caosdb/server/transaction/Retrieve.java b/src/main/java/org/caosdb/server/transaction/Retrieve.java
index 86b7672c..859a6ad6 100644
--- a/src/main/java/org/caosdb/server/transaction/Retrieve.java
+++ b/src/main/java/org/caosdb/server/transaction/Retrieve.java
@@ -121,4 +121,28 @@ public class Retrieve extends Transaction<RetrieveContainer> {
public boolean logHistory() {
return false;
}
+
+ public static class Paging {
+ public Paging(int index, int length) {
+ this.index = index;
+ this.length = length;
+ }
+
+ public final int index;
+ public final int length;
+ }
+
+ private Retrieve.Paging paging = null;
+
+ public boolean hasPaging() {
+ return this.paging != null;
+ }
+
+ public void setPaging(int index, int length) {
+ this.paging = new Retrieve.Paging(index, length);
+ }
+
+ public Retrieve.Paging getPaging() {
+ return paging;
+ }
}
diff --git a/src/main/java/org/caosdb/server/transaction/Transaction.java b/src/main/java/org/caosdb/server/transaction/Transaction.java
index a9bac31c..5fc06f5e 100644
--- a/src/main/java/org/caosdb/server/transaction/Transaction.java
+++ b/src/main/java/org/caosdb/server/transaction/Transaction.java
@@ -60,9 +60,6 @@ public abstract class Transaction<C extends TransactionContainer> extends Abstra
public static final String CLEAN_UP = "TransactionCleanUp";
private final C container;
- private int pagingIndex1;
- private int pagingIndex2;
- private boolean ThasPaging = false;
private Access access = null;
private final Schedule schedule = new Schedule();
@@ -85,24 +82,6 @@ public abstract class Transaction<C extends TransactionContainer> extends Abstra
return this.container;
}
- public boolean hasPaging() {
- return ThasPaging;
- }
-
- public void setPagingIndices(int index1, int index2) {
- ThasPaging = true;
- pagingIndex1 = index1;
- pagingIndex2 = index2;
- }
-
- public int getPagingIndex1() {
- return pagingIndex1;
- }
-
- public int getPagingIndex2() {
- return pagingIndex2;
- }
-
/**
* Implementation note: Not called in this class, but may be used by subclasses.
*
@@ -126,6 +105,7 @@ public abstract class Transaction<C extends TransactionContainer> extends Abstra
this.schedule.addAll(Job.loadPermanentContainerJobs(this));
for (final EntityInterface e : getContainer()) {
+ if (e.skipJob()) continue;
final List<Job> loadJobs = loadContainerFlags.loadJobs(e, this);
this.schedule.addAll(loadJobs);
@@ -176,7 +156,6 @@ public abstract class Transaction<C extends TransactionContainer> extends Abstra
long t1 = System.currentTimeMillis();
try {
init();
- this.schedule.runJobs(TransactionStage.PRE_INIT);
this.schedule.runJobs(TransactionStage.INIT);
getContainer()
.getTransactionBenchmark()
--
GitLab