From 10d51e31073459fb926ae04ce3051effceeb07ef Mon Sep 17 00:00:00 2001
From: Timm Fitschen <t.fitschen@indiscale.com>
Date: Mon, 7 Jun 2021 08:58:36 +0000
Subject: [PATCH] ENH: Add query transaction filter (BEFORE/AFTER)

---
 CHANGELOG.md                                  |  3 ++
 doc/Query.md                                  | 12 ++---
 .../java/org/caosdb/server/query/CQLLexer.g4  | 16 +++++++
 .../java/org/caosdb/server/query/CQLParser.g4 | 30 ++++++++----
 .../server/query/TransactionFilter.java       | 16 +++++--
 .../java/org/caosdb/server/query/TestCQL.java | 48 +++++++++++++++----
 6 files changed, 98 insertions(+), 27 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 64c6a1a3..2974b08d 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -20,6 +20,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
   to determine whether the server's state has changed between queries.
 * Basic caching for queries. The caching is enabled by default and can be
   controlled by the usual "cache" flag.
+* Add `BEFORE`, `AFTER`, `UNTIL`, `SINCE` keywords for query transaction
+  filters.
+
 
 ### Changed
 
diff --git a/doc/Query.md b/doc/Query.md
index 4ff07fba..1d774843 100644
--- a/doc/Query.md
+++ b/doc/Query.md
@@ -226,14 +226,13 @@ The following query returns entities which have a _pname1_ property with any val
 ### TransactionFilter
 
 *Definition*
- sugar:: `HAS BEEN` | `HAVE BEEN` | `HAD BEEN` | `WAS` | `IS` | 
+ sugar:: `HAS BEEN` | `HAVE BEEN` | `HAD BEEN` | `WAS` | `IS`
  negated_sugar:: `HAS NOT BEEN` | `HASN'T BEEN` | `WAS NOT` | `WASN'T` | `IS NOT` | `ISN'T`  | `HAVN'T BEEN` | `HAVE NOT BEEN` | `HADN'T BEEN` | `HAD NOT BEEN`
  by_clause:: `BY (ME | username | SOMEONE ELSE (BUT ME)? | SOMEONE ELSE BUT username)`
- date:: A date string of the form `YYYY-MM-DD`
- datetime:: A datetime string of the form `YYYY-MM-DD hh:mm:ss`
- time_clause:: `ON ($date|$datetime) ` Here is plenty of room for more syntactic sugar, e.g. a `TODAY` keyword, and more funcionality, e.g. ranges.  
+ datetime:: A datetime string of the form `YYYY[-MM[-DD(T| )[hh[:mm[:ss[.nnn][(+|-)zzzz]]]]]]`
+ time_clause:: `[AT|ON|IN|BEFORE|AFTER|UNTIL|SINCE] (datetime) `
 
-`FIND ename WHICH ($sugar|$negated_sugar)? (NOT)? (CREATED|INSERTED|UPDATED|DELETED) (by_clause time_clause?| time_clause by_clause?)`
+`FIND ename WHICH (sugar|negated_sugar)? (NOT)? (CREATED|INSERTED|UPDATED) (by_clause time_clause?| time_clause by_clause?)`
 
 *Examples*
 
@@ -247,8 +246,9 @@ The following query returns entities which have a _pname1_ property with any val
 
 `FIND ename WHICH HAS BEEN CREATED BY erwin`
 
-`FIND ename . CREATED BY erwin ON `
+`FIND ename WHICH HAS BEEN INSERTED SINCE 2021-04`
 
+Note that `SINCE` and `UNTIL` are inclusive, while `BEFORE` and `AFTER` are not.
 
 
 ### File Location
diff --git a/src/main/java/org/caosdb/server/query/CQLLexer.g4 b/src/main/java/org/caosdb/server/query/CQLLexer.g4
index 85061d30..74331e0f 100644
--- a/src/main/java/org/caosdb/server/query/CQLLexer.g4
+++ b/src/main/java/org/caosdb/server/query/CQLLexer.g4
@@ -77,6 +77,22 @@ IN:
 	[Ii][Nn] WHITE_SPACE_f?
 ;
 
+AFTER:
+	[Aa][Ff][Tt][Ee][Rr] WHITE_SPACE_f?
+;
+
+BEFORE:
+    [Bb][Ee][Ff][Oo][Rr][Ee] WHITE_SPACE_f?
+;
+
+UNTIL:
+    [Uu][Nn][Tt][Ii][Ll] WHITE_SPACE_f?
+;
+
+SINCE:
+	[Ss][Ii][Nn][Cc][Ee] WHITE_SPACE_f?
+;
+
 IS_STORED_AT:
 	(IS_f WHITE_SPACE_f?)? [Ss][Tt][Oo][Rr][Ee][Dd] (WHITE_SPACE_f? AT)? WHITE_SPACE_f?
 ;
diff --git a/src/main/java/org/caosdb/server/query/CQLParser.g4 b/src/main/java/org/caosdb/server/query/CQLParser.g4
index 452321c6..6c6aa748 100644
--- a/src/main/java/org/caosdb/server/query/CQLParser.g4
+++ b/src/main/java/org/caosdb/server/query/CQLParser.g4
@@ -153,14 +153,15 @@ idfilter returns [IDFilter filter] locals [String o, String v, String a]
 	)?
 ;
 
-transaction returns [TransactionFilter filter] locals [String type, TransactionFilter.Transactor user, String time]
+transaction returns [TransactionFilter filter] locals [String type, TransactionFilter.Transactor user, String time, String time_op]
 @init{
 	$time = null;
 	$user = null;
 	$type = null;
+	$time_op = null;
 }
 @after{
-	$filter = new TransactionFilter($type,$user,$time);
+	$filter = new TransactionFilter($type,$user,$time,$time_op);
 }
 :
 	(
@@ -169,8 +170,8 @@ transaction returns [TransactionFilter filter] locals [String type, TransactionF
 	)
 	
 	(
-		transactor (transaction_time {$time = $transaction_time.tqp;})? {$user = $transactor.t;}
-		| transaction_time (transactor {$user = $transactor.t;})? {$time = $transaction_time.tqp;}
+		transactor (transaction_time {$time = $transaction_time.tqp; $time_op = $transaction_time.op;})? {$user = $transactor.t;}
+		| transaction_time (transactor {$user = $transactor.t;})? {$time = $transaction_time.tqp; $time_op = $transaction_time.op;}
 	)
 ;
 
@@ -199,12 +200,25 @@ username returns [Query.Pattern ep] locals [int type]
     ( STAR {$type = Query.Pattern.TYPE_LIKE;} | ~(STAR | WHITE_SPACE) )+
 ;
 
-transaction_time returns [String tqp]
+transaction_time returns [String tqp, String op]
+@init {
+     $op = "(";
+}
 :
+	(
+	    AT  {$op = "=";}
+        | (ON | IN)
+        | (
+        	BEFORE {$op = "<";}
+          	| UNTIL {$op = "<=";}
+          	| AFTER {$op = ">";}
+          	| SINCE {$op = ">=";}
+        )
+    )?
     (
-        (ON | IN)
-        (value {$tqp = $value.text;})
-    ) | TODAY {$tqp = TransactionFilter.TODAY;}
+        TODAY {$tqp = TransactionFilter.TODAY;}
+        | value {$tqp = $value.text;}
+    )
 ;
 
 /*
diff --git a/src/main/java/org/caosdb/server/query/TransactionFilter.java b/src/main/java/org/caosdb/server/query/TransactionFilter.java
index fed4a715..2099639b 100644
--- a/src/main/java/org/caosdb/server/query/TransactionFilter.java
+++ b/src/main/java/org/caosdb/server/query/TransactionFilter.java
@@ -29,7 +29,6 @@ import java.sql.Types;
 import org.caosdb.datetime.Date;
 import org.caosdb.datetime.DateTimeFactory2;
 import org.caosdb.datetime.Interval;
-import org.caosdb.datetime.SemiCompleteDateTime;
 import org.caosdb.datetime.UTCDateTime;
 import org.caosdb.server.accessControl.Principal;
 import org.caosdb.server.accessControl.UserSources;
@@ -96,14 +95,20 @@ public class TransactionFilter implements EntityFilterInterface {
     }
   }
 
-  public TransactionFilter(final String type, final Transactor transactor, final String time) {
+  public TransactionFilter(
+      final String type,
+      final Transactor transactor,
+      final String time,
+      final String timeOperator) {
     this.transactor = transactor;
     this.transactionTime = time;
     this.transactionType = type;
+    this.transactionTimeOperator = timeOperator;
   }
 
   private final Transactor transactor;
   private final String transactionTime;
+  private final String transactionTimeOperator;
   private final String transactionType;
 
   @Override
@@ -123,7 +128,7 @@ public class TransactionFilter implements EntityFilterInterface {
 
       } else {
         try {
-          dt = (SemiCompleteDateTime) DateTimeFactory2.valueOf(this.transactionTime);
+          dt = (Interval) DateTimeFactory2.valueOf(this.transactionTime);
         } catch (final ClassCastException e) {
           throw new QueryException("Transaction time must be a SemiCompleteDateTime.");
         } catch (final IllegalArgumentException e) {
@@ -201,14 +206,13 @@ public class TransactionFilter implements EntityFilterInterface {
           } else {
             prepareCall.setNull(10, Types.INTEGER);
           }
-          prepareCall.setString(11, "("); // '(' means 'is in the
           // interval'
         } else {
           prepareCall.setNull(9, Types.BIGINT);
           prepareCall.setNull(10, Types.INTEGER);
-          prepareCall.setNull(11, Types.CHAR);
         }
 
+        prepareCall.setString(11, transactionTimeOperator);
       } else {
         // ilb_sec, ilb_nanos, eub_sec, eub_nanos, operator_t
         prepareCall.setNull(7, Types.BIGINT);
@@ -251,6 +255,8 @@ public class TransactionFilter implements EntityFilterInterface {
     return "TRANS("
         + this.transactionType
         + ","
+        + this.transactionTimeOperator
+        + ","
         + this.transactionTime
         + ","
         + this.transactor
diff --git a/src/test/java/org/caosdb/server/query/TestCQL.java b/src/test/java/org/caosdb/server/query/TestCQL.java
index 2c28d7f5..92e82902 100644
--- a/src/test/java/org/caosdb/server/query/TestCQL.java
+++ b/src/test/java/org/caosdb/server/query/TestCQL.java
@@ -237,6 +237,8 @@ public class TestCQL {
 
   String queryIssue31 = "FIND FILE WHICH IS STORED AT /data/in0.foo";
   String queryIssue116 = "FIND *";
+  String queryIssue132a = "FIND ENTITY WHICH HAS BEEN INSERTED AFTER TODAY";
+  String queryIssue132b = "FIND ENTITY WHICH HAS BEEN CREATED TODAY BY ME";
 
   // File paths ///////////////////////////////////////////////////////////////
   String filepath_verb01 = "/foo/";
@@ -5692,7 +5694,7 @@ public class TestCQL {
     System.out.println(sfq.toStringTree(parser));
 
     assertTrue(sfq.filter instanceof TransactionFilter);
-    assertEquals("TRANS(Insert,null,Transactor(some%,=))", sfq.filter.toString());
+    assertEquals("TRANS(Insert,null,null,Transactor(some%,=))", sfq.filter.toString());
   }
 
   /** String ticket242 = "FIND RECORD WHICH HAS been created by some.user"; */
@@ -5707,7 +5709,7 @@ public class TestCQL {
 
     System.out.println(sfq.toStringTree(parser));
 
-    assertEquals("TRANS(Insert,null,Transactor(some.user,=))", sfq.filter.toString());
+    assertEquals("TRANS(Insert,null,null,Transactor(some.user,=))", sfq.filter.toString());
     assertTrue(sfq.filter instanceof TransactionFilter);
   }
 
@@ -5781,7 +5783,7 @@ public class TestCQL {
     assertEquals("@(null,null)", n.getFilter().toString());
 
     assertTrue(f2.getLast() instanceof TransactionFilter);
-    assertEquals("TRANS(Insert,null,Transactor(null,=))", f2.getLast().toString());
+    assertEquals("TRANS(Insert,null,null,Transactor(null,=))", f2.getLast().toString());
   }
 
   /** String ticket262e = "COUNT FILE WHICH IS NOT REFERENCED AND WAS created by me"; */
@@ -5804,7 +5806,7 @@ public class TestCQL {
     assertEquals("@(null,null)", n.getFilter().toString());
 
     assertTrue(f2.getLast() instanceof TransactionFilter);
-    assertEquals("TRANS(Insert,null,Transactor(null,=))", f2.getLast().toString());
+    assertEquals("TRANS(Insert,null,null,Transactor(null,=))", f2.getLast().toString());
   }
 
   /** String ticket262f = "COUNT FILE WHICH IS NOT REFERENCED BY entity AND WAS created by me"; */
@@ -5827,7 +5829,7 @@ public class TestCQL {
     assertEquals("@(entity,null)", n.getFilter().toString());
 
     assertTrue(f2.getLast() instanceof TransactionFilter);
-    assertEquals("TRANS(Insert,null,Transactor(null,=))", f2.getLast().toString());
+    assertEquals("TRANS(Insert,null,null,Transactor(null,=))", f2.getLast().toString());
   }
 
   /**
@@ -5853,7 +5855,7 @@ public class TestCQL {
     assertEquals("@(entity,null)", n.getFilter().toString());
 
     assertTrue(f2.getLast() instanceof TransactionFilter);
-    assertEquals("TRANS(Insert,null,Transactor(null,=))", f2.getLast().toString());
+    assertEquals("TRANS(Insert,null,null,Transactor(null,=))", f2.getLast().toString());
   }
 
   /** String ticket262h = "COUNT FILE WHICH IS NOT REFERENCED BY entity WHICH WAS created by me"; */
@@ -5876,7 +5878,7 @@ public class TestCQL {
 
     assertNotNull(((Backreference) backref).getSubProperty());
     assertEquals(
-        "TRANS(Insert,null,Transactor(null,=))",
+        "TRANS(Insert,null,null,Transactor(null,=))",
         ((Backreference) backref).getSubProperty().getFilter().toString());
   }
 
@@ -5917,7 +5919,7 @@ public class TestCQL {
 
     assertNotNull(((Backreference) backref).getSubProperty());
     assertEquals(
-        "TRANS(Insert,null,Transactor(null,=))",
+        "TRANS(Insert,null,null,Transactor(null,=))",
         ((Backreference) backref).getSubProperty().getFilter().toString());
   }
 
@@ -6686,4 +6688,34 @@ public class TestCQL {
     assertEquals("POV(pname,=,with)", sfq.filter.toString());
     assertNull(((POV) sfq.filter).getSubProperty());
   }
+
+  @Test
+  /** String queryIssue132a = "FIND ENTITY WHICH HAS BEEN INSERTED AFTER TODAY"; */
+  public void testIssue132a() {
+    CQLLexer lexer;
+    lexer = new CQLLexer(CharStreams.fromString(this.queryIssue132a));
+    final CommonTokenStream tokens = new CommonTokenStream(lexer);
+
+    final CQLParser parser = new CQLParser(tokens);
+    final CqContext sfq = parser.cq();
+
+    System.out.println(sfq.toStringTree(parser));
+
+    assertEquals("TRANS(Insert,>,Today,null)", sfq.filter.toString());
+  }
+
+  @Test
+  /** String queryIssue132b = "FIND ENTITY WHICH HAS BEEN CREATED TODAY BY ME"; */
+  public void testIssue132b() {
+    CQLLexer lexer;
+    lexer = new CQLLexer(CharStreams.fromString(this.queryIssue132b));
+    final CommonTokenStream tokens = new CommonTokenStream(lexer);
+
+    final CQLParser parser = new CQLParser(tokens);
+    final CqContext sfq = parser.cq();
+
+    System.out.println(sfq.toStringTree(parser));
+
+    assertEquals("TRANS(Insert,(,Today,Transactor(null,=))", sfq.filter.toString());
+  }
 }
-- 
GitLab