Skip to content
Snippets Groups Projects
Commit 31243e69 authored by Timm Fitschen's avatar Timm Fitschen
Browse files

Merge branch 'f-enh-paging' into 'dev'

ENH: apply paging such that no unwanted jobs are being triggered

See merge request !81
parents 35a40981 7028a06b
No related branches found
No related tags found
2 merge requests!96DOC: Added CITATION.cff to the list of files in the release guide where the...,!81ENH: apply paging such that no unwanted jobs are being triggered
Pipeline #37121 failed
...@@ -31,8 +31,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ...@@ -31,8 +31,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
### Removed ### ### Removed ###
### Fixed ### ### Fixed ###
- [#203](https://gitlab.com/caosdb/caosdb-server/-/issues/203)
* Bad performance due to the execution of unnecessary jobs during retrieval.
[#189](https://gitlab.com/caosdb/caosdb-server/-/issues/189)
* Query Language: Parentheses change filter to subproperty filter
[#203](https://gitlab.com/caosdb/caosdb-server/-/issues/203)
* Searching for values in scientific notation * Searching for values in scientific notation
[#143](https://gitlab.com/caosdb/caosdb-server/-/issues/143) [#143](https://gitlab.com/caosdb/caosdb-server/-/issues/143)
* Denying a role permission has no effect * Denying a role permission has no effect
......
...@@ -1004,7 +1004,7 @@ public abstract class Entity extends AbstractObservable implements EntityInterfa ...@@ -1004,7 +1004,7 @@ public abstract class Entity extends AbstractObservable implements EntityInterfa
@Override @Override
public boolean skipJob() { public boolean skipJob() {
return false; return this.entityStatus == EntityStatus.IGNORE;
} }
@Override @Override
......
...@@ -26,7 +26,6 @@ package org.caosdb.server.entity.xml; ...@@ -26,7 +26,6 @@ package org.caosdb.server.entity.xml;
import org.caosdb.server.entity.EntityInterface; import org.caosdb.server.entity.EntityInterface;
import org.caosdb.server.entity.wrapper.Parent; import org.caosdb.server.entity.wrapper.Parent;
import org.caosdb.server.utils.EntityStatus;
import org.jdom2.Element; import org.jdom2.Element;
/** /**
...@@ -51,15 +50,4 @@ public class ParentToElementStrategy extends EntityToElementStrategy { ...@@ -51,15 +50,4 @@ public class ParentToElementStrategy extends EntityToElementStrategy {
} }
return element; return element;
} }
@Override
public Element addToElement(
final EntityInterface entity,
final Element element,
final SerializeFieldStrategy setFieldStrategy) {
if (entity.getEntityStatus() != EntityStatus.IGNORE) {
element.addContent(toElement(entity, setFieldStrategy));
}
return element;
}
} }
...@@ -31,15 +31,20 @@ import org.caosdb.server.jobs.JobAnnotation; ...@@ -31,15 +31,20 @@ import org.caosdb.server.jobs.JobAnnotation;
import org.caosdb.server.jobs.TransactionStage; import org.caosdb.server.jobs.TransactionStage;
import org.caosdb.server.query.Query; import org.caosdb.server.query.Query;
import org.caosdb.server.query.Query.ParsingException; import org.caosdb.server.query.Query.ParsingException;
import org.caosdb.server.transaction.Retrieve;
import org.caosdb.server.utils.EntityStatus;
import org.caosdb.server.utils.ServerMessages; import org.caosdb.server.utils.ServerMessages;
@JobAnnotation(flag = "query", stage = TransactionStage.INIT) @JobAnnotation(flag = "query", stage = TransactionStage.INIT, transaction = Retrieve.class)
public class ExecuteQuery extends FlagJob { public class ExecuteQuery extends FlagJob {
@Override @Override
protected void job(final String value) { protected void job(final String value) {
if (value != null) { if (value != null) {
// run paging job first
getTransaction().getSchedule().runJob(null, Paging.class);
final Query queryInstance = final Query queryInstance =
new Query(value, getTransaction().getTransactor(), getContainer()); new Query(value, getTransaction().getTransactor(), getContainer());
getContainer().setQuery(queryInstance); getContainer().setQuery(queryInstance);
...@@ -53,8 +58,24 @@ public class ExecuteQuery extends FlagJob { ...@@ -53,8 +58,24 @@ public class ExecuteQuery extends FlagJob {
.addMessage(new Message(MessageType.Info, (MessageCode) null, e.getMessage())); .addMessage(new Message(MessageType.Info, (MessageCode) null, e.getMessage()));
} }
getContainer().addMessage(queryInstance); getContainer().addMessage(queryInstance);
int startIndex = 0;
int endIndex = getContainer().size();
if (((Retrieve) getTransaction()).hasPaging()) {
Retrieve.Paging paging = ((Retrieve) getTransaction()).getPaging();
startIndex = Math.min(getContainer().size(), paging.startIndex);
endIndex = Math.min(getContainer().size(), paging.endIndex);
}
int ii = 0;
for (final EntityInterface entity : getContainer()) { for (final EntityInterface entity : getContainer()) {
getTransaction().getSchedule().addAll(loadJobs(entity, getTransaction())); if (ii >= startIndex && ii < endIndex) {
getTransaction().getSchedule().addAll(loadJobs(entity, getTransaction()));
} else {
entity.setEntityStatus(EntityStatus.IGNORE);
}
ii++;
} }
} }
} }
......
...@@ -31,7 +31,7 @@ import org.caosdb.server.jobs.TransactionStage; ...@@ -31,7 +31,7 @@ import org.caosdb.server.jobs.TransactionStage;
import org.caosdb.server.transaction.Retrieve; import org.caosdb.server.transaction.Retrieve;
import org.caosdb.server.utils.EntityStatus; import org.caosdb.server.utils.EntityStatus;
@JobAnnotation(flag = "P", stage = TransactionStage.PRE_TRANSACTION) @JobAnnotation(flag = "P", stage = TransactionStage.INIT, transaction = Retrieve.class)
public class Paging extends FlagJob { public class Paging extends FlagJob {
public static final int DEFAULT_LENGTH = 100; public static final int DEFAULT_LENGTH = 100;
...@@ -42,20 +42,24 @@ public class Paging extends FlagJob { ...@@ -42,20 +42,24 @@ public class Paging extends FlagJob {
protected void job(final String value) { protected void job(final String value) {
if (getTransaction() instanceof Retrieve) { if (getTransaction() instanceof Retrieve) {
if (value != null) { if (value != null) {
int index1 = DEFAULT_INDEX; int startIndex = DEFAULT_INDEX;
int index2 = index1 + DEFAULT_LENGTH; int endIndex = startIndex + DEFAULT_LENGTH;
final Matcher m = pattern.matcher(value); final Matcher m = pattern.matcher(value);
if (m.matches()) { if (m.matches()) {
if (m.group(1) != null) { if (m.group(1) != null) {
index1 = Integer.parseInt(m.group(1)); startIndex = Integer.parseInt(m.group(1));
} }
if (m.group(2) != null) { if (m.group(2) != null) {
index2 = index1 + Integer.parseInt(m.group(2)); endIndex = startIndex + Integer.parseInt(m.group(2));
} }
} }
// this info may be used by other jobs
((Retrieve) getTransaction()).setPaging(startIndex, endIndex);
int i = 0; int i = 0;
for (final EntityInterface e : getContainer()) { for (final EntityInterface e : getContainer()) {
if (i >= index2 || i < index1) { if (i >= endIndex || i < startIndex) {
// do not retrieve this entity // do not retrieve this entity
e.setEntityStatus(EntityStatus.IGNORE); e.setEntityStatus(EntityStatus.IGNORE);
} }
......
...@@ -27,19 +27,43 @@ import org.caosdb.server.entity.EntityInterface; ...@@ -27,19 +27,43 @@ import org.caosdb.server.entity.EntityInterface;
import org.caosdb.server.jobs.FlagJob; import org.caosdb.server.jobs.FlagJob;
import org.caosdb.server.jobs.JobAnnotation; import org.caosdb.server.jobs.JobAnnotation;
import org.caosdb.server.jobs.TransactionStage; import org.caosdb.server.jobs.TransactionStage;
import org.caosdb.server.transaction.Retrieve;
import org.caosdb.server.utils.EntityStatus;
@JobAnnotation(flag = "all", stage = TransactionStage.INIT) @JobAnnotation(flag = "all", stage = TransactionStage.INIT, transaction = Retrieve.class)
public class RetrieveAllJob extends FlagJob { public class RetrieveAllJob extends FlagJob {
@Override @Override
protected void job(String value) { protected void job(String value) {
if (getContainer().isEmpty()) { if (getContainer().isEmpty()) {
// run paging job first
getTransaction().getSchedule().runJob(null, Paging.class);
if (value == null) { if (value == null) {
value = "ENTITY"; value = "ENTITY";
} }
execute(new RetrieveAll(getContainer(), value)); execute(new RetrieveAll(getContainer(), value));
int startIndex = 0;
int endIndex = getContainer().size();
if (((Retrieve) getTransaction()).hasPaging()) {
startIndex =
Math.min(getContainer().size(), ((Retrieve) getTransaction()).getPaging().startIndex);
endIndex =
Math.min(getContainer().size(), ((Retrieve) getTransaction()).getPaging().endIndex);
}
// only add jobs for those which are on this page
int ii = 0;
for (EntityInterface entity : getContainer()) { for (EntityInterface entity : getContainer()) {
getTransaction().getSchedule().addAll(loadJobs(entity, getTransaction())); if (ii >= startIndex && ii < endIndex) {
getTransaction().getSchedule().addAll(loadJobs(entity, getTransaction()));
} else {
entity.setEntityStatus(EntityStatus.IGNORE);
}
ii++;
} }
} }
} }
......
...@@ -121,4 +121,28 @@ public class Retrieve extends Transaction<RetrieveContainer> { ...@@ -121,4 +121,28 @@ public class Retrieve extends Transaction<RetrieveContainer> {
public boolean logHistory() { public boolean logHistory() {
return false; return false;
} }
public static class Paging {
public Paging(int startIndex, int endIndex) {
this.startIndex = startIndex;
this.endIndex = endIndex;
}
public final int startIndex;
public final int endIndex;
}
private Retrieve.Paging paging = null;
public boolean hasPaging() {
return this.paging != null;
}
public void setPaging(int startIndex, int endIndex) {
this.paging = new Retrieve.Paging(startIndex, endIndex);
}
public Retrieve.Paging getPaging() {
return paging;
}
} }
...@@ -105,6 +105,7 @@ public abstract class Transaction<C extends TransactionContainer> extends Abstra ...@@ -105,6 +105,7 @@ public abstract class Transaction<C extends TransactionContainer> extends Abstra
this.schedule.addAll(Job.loadPermanentContainerJobs(this)); this.schedule.addAll(Job.loadPermanentContainerJobs(this));
for (final EntityInterface e : getContainer()) { for (final EntityInterface e : getContainer()) {
if (e.skipJob()) continue;
final List<Job> loadJobs = loadContainerFlags.loadJobs(e, this); final List<Job> loadJobs = loadContainerFlags.loadJobs(e, this);
this.schedule.addAll(loadJobs); this.schedule.addAll(loadJobs);
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment