Improving file query

doc/devel/Development.md

@@ -20,4 +20,5 @@ writing tests.
 ### Running tests with Maven ###
 - Automatic testing can be done with `make test` or, after compilation, `mvn
   test`.
-- Tests of single modules can be started with `mvn test -Dtest=TextClass`
+- Tests of single modules can be started with `mvn test -Dtest=TestClass`
+- Test of a single method `footest`: `mvn test -Dtest=TestClass#footest`

src/main/java/caosdb/server/query/CQLLexer.g4

@@ -349,8 +349,9 @@ ID:
 	[Ii][Dd]
 ;
 
-SLASH:
-	'/'
+// Multiple slashes should be allowed in paths instead of a single slash.
+SLASHES:
+	'/'+
 ;
 
 STAR:

src/main/java/caosdb/server/query/CQLParser.g4

@@ -456,8 +456,8 @@ location returns [String str]
 :
     atom {$str = $atom.str;}
     |
-	SLASH ?
-	((WHICH | WITH)+ SLASH |( TXT | COLON | HYPHEN | NUM | DOT | ESC_STAR | ESC_BS | ESC_REGEXP_END | STAR )+ SLASH ?)* {$str = $text; }
+	SLASHES ?
+	((WHICH | WITH)+ SLASHES |( TXT | COLON | HYPHEN | NUM | DOT | ESC_STAR | ESC_BS | ESC_REGEXP_END | STAR )+ SLASHES ?)* {$str = $text; }
 ;
 
 atom returns [String str]

src/main/java/caosdb/server/query/StoredAt.java

@@ -25,6 +25,8 @@ package caosdb.server.query;
 import static java.sql.Types.VARCHAR;
 
 import caosdb.server.query.Query.QueryException;
+import java.nio.file.Path;
+import java.nio.file.Paths;
 import java.sql.CallableStatement;
 import java.sql.SQLException;
 import java.util.regex.Matcher;
@@ -34,47 +36,73 @@ import org.jdom2.Element;
 public class StoredAt implements EntityFilterInterface {
 
   public static final java.util.regex.Pattern PATTERN =
-      Pattern.compile("((?:(?:\\\\[\\*\\\\])|[^\\*\\\\])+)?(\\*{2,})?(\\*)?");
-  private final String location;
+    // heading and escaped * and \ removed, then
+    // (part of directory name: no \, no *)?(**)?(*)?
+    Pattern.compile("((?:(?:\\\\[\\*\\\\])|[^\\*\\\\])+)?(\\*{2,})?(\\*)?");
+  private String location;
+  private final String likeLocation;
   private boolean pattern_matching = false;
 
   public StoredAt(final String loc) {
-    final String[] dirs = loc.split("/");
-    final StringBuffer sb = new StringBuffer();
-
-    // if (dirs[0].length() == 0) {
-    // sb.append("/");
-    // }
-    for (final String dir : dirs) {
-      final Matcher m = PATTERN.matcher(dir);
-
-      sb.append("/");
-      while (m.find()) {
-        // part of a directory name
-        if (m.group(1) != null) {
-          sb.append(m.group(1).replace("%", "\\%").replace("_", "\\_").replaceAll("\\\\\\*", "*"));
-        }
-
-        // **
-        if (m.group(2) != null) {
-          this.pattern_matching = true;
-          sb.append("%");
-        }
+    Path locPath = Paths.get(loc);
+    this.location =
+        locPath.normalize().toString().replaceFirst("^/", "") + (loc.endsWith("/") ? "/" : "");
+    this.pattern_matching = requiresPatternMatching();
 
-        // *
-        if (m.group(3) != null) {
-          this.pattern_matching = true;
-          sb.append("%%");
-        }
-      }
+    if (this.pattern_matching) {
+      this.likeLocation = convertLikeLocation();
+    } else {
+      this.likeLocation = null;
     }
+  }
 
-    if (loc.endsWith("/")) {
-      sb.append("/");
+  /**
+   * Does some special character escaping for LIKE query
+   *
+   * <p>The following rules should take place in the future (for now, only a subset is implemented
+   * correctly):
+   *
+   * <ul>
+   *   <li>\* -> * (No special meaning in SQL, even numbers of \ are ignored.)
+   *   <li>* -> %% (Wildcard, to be converted to a more complicated regex without / later.
+   *       Exception: at the beginning of a search expression, convert to %.)
+   *   <li>** -> % (Wildcard, also ***, ****, ... -> %)
+   *   <li>_ -> \_ (Prevent special meaning in SQL)
+   *   <li>% -> \% (Prevent special meaning in SQL)
+   *   <li>Heading `/` is removed
+   * </ul>
+   *
+   * @return The converted String.
+   */
+  String convertLikeLocation() {
+    final String evenB = "(^|[^b*])(bb)*";
+    final String oddB = evenB + "b";
+    final String singleEscapedAst = "b\\*";
+    final String findSingleAst =
+        (evenB + "(" + singleEscapedAst + ")?" + "\\*([^*]|$)").replace("b", "\\\\");
+    final String findMultipleAst =
+        (evenB + "(" + singleEscapedAst + ")?" + "\\*{2,}").replace("b", "\\\\");
+    java.util.regex.Pattern singlePat = Pattern.compile(findSingleAst);
+    java.util.regex.Pattern multiplePat = Pattern.compile(findMultipleAst);
+
+    // Simple SQL escape replacements
+    String converted = this.location.replace("%", "\\%").replace("_", "\\_");
+    // * -> %%, has to run mutliple times, because otherwise the separation between patterns would
+    // need to be larger than one character
+    while (singlePat.matcher(converted).find()) {
+      converted = converted.replaceAll(findSingleAst, "$1$2$3%%$4");
     }
+    // ** -> %
+    converted = converted.replaceAll(findMultipleAst, "$1$2$3%");
+    // All remaining asterisks are now guaranteed to have an odd number of backslashes in front
 
-    this.location =
-        sb.toString().replaceFirst("^/", "").replaceFirst("^%{2,}", "%").replaceFirst("^/", "");
+    // So we can now do \* -> *
+    converted = converted.replaceAll(singleEscapedAst.replace("b", "\\\\"), "*");
+
+    // Final exceptions
+    converted = converted.replaceFirst("^%{2,}", "%"); // .replaceFirst("^/", "");
+
+    return converted;
   }
 
   @Override
@@ -99,14 +127,14 @@ public class StoredAt implements EntityFilterInterface {
       if (this.location == null) { // location
         callSAT.setNull(3, VARCHAR);
       } else {
-        callSAT.setString(3, this.location);
+        callSAT.setString(3, (this.pattern_matching ? this.likeLocation : this.location));
       }
 
       callSAT.setString(4, (this.pattern_matching ? "LIKE" : "="));
       callSAT.execute();
 
-      // ... and then cancel out the wrong guys via regexp
-      if (this.pattern_matching && this.location.contains("%%")) {
+      // ... and then cancel out the wrong guys ('*' matching '/') via regexp
+      if (this.pattern_matching && this.likeLocation.contains("%%")) {
 
         // make a MySQL-conform regexp for this.location:
         // 1) escape literal dots (. = \.)
@@ -116,7 +144,7 @@ public class StoredAt implements EntityFilterInterface {
         // 5) replace non-escaped %% with [^/]*
         // 6) replace non-escaped % with .*
         final String nloc =
-            Pattern.quote(this.location.replace(".", "\\."))
+            Pattern.quote(this.likeLocation.replace(".", "\\."))
                 .replaceFirst("..$", "\\$")
                 .replaceFirst("^..", "^")
                 .replaceAll("(?<!\\\\)%%", "[^/]*")
@@ -139,6 +167,20 @@ public class StoredAt implements EntityFilterInterface {
     query.addBenchmark(this.getClass().getSimpleName(), System.currentTimeMillis() - t1);
   }
 
+  /**
+   * Tests if the location String is intended for pattern matching.
+   *
+   * @return True if the location requires pattern matching for a LIKE query, false otherwise.
+   */
+  boolean requiresPatternMatching() {
+    // Looking for *, preceded by no or an even number of backslashes
+    String requiredPatternStr = "(^|[^b])(bb)*\\*";
+    requiredPatternStr = requiredPatternStr.replace("b", "\\\\");
+    java.util.regex.Pattern REQPATTERN = Pattern.compile(requiredPatternStr);
+    final Matcher m = REQPATTERN.matcher(this.location);
+    return m.find();
+  }
+
   @Override
   public Element toElement() {
     final Element ret = new Element("StoredAt");
@@ -146,8 +188,13 @@ public class StoredAt implements EntityFilterInterface {
     return ret;
   }
 
+  /**
+   * For testing purposes, create a string representing the corresponding query.
+   *
+   * @return The String "SAT(<location>)", where "<location>" is the query string.
+   */
   @Override
   public String toString() {
-    return "SAT(" + this.location + ")";
+    return "SAT(" + (this.pattern_matching ? this.likeLocation : this.location) + ")";
   }
 }