From 9b792fe586729c9907bf92ef2887217a8d2a3931 Mon Sep 17 00:00:00 2001
From: Timm Fitschen <t.fitschen@indiscale.com>
Date: Tue, 2 Nov 2021 03:20:03 +0100
Subject: [PATCH 01/38] WIP: handle quotations in select queries
---
src/main/java/org/caosdb/server/query/Query.java | 8 +++++++-
1 file changed, 7 insertions(+), 1 deletion(-)
diff --git a/src/main/java/org/caosdb/server/query/Query.java b/src/main/java/org/caosdb/server/query/Query.java
index 22a513f6..d57189c3 100644
--- a/src/main/java/org/caosdb/server/query/Query.java
+++ b/src/main/java/org/caosdb/server/query/Query.java
@@ -98,7 +98,13 @@ public class Query implements QueryInterface, ToElementable, TransactionInterfac
/** No parsing, just sets the selector string. */
public Selection(final String selector) {
- this.selector = selector.trim();
+ if (selector.trim().startsWith("'") && selector.trim().endsWith("'")) {
+ this.selector = selector.replaceFirst("^\\s*'", "").replaceFirst("'\\s*$", "").trim();
+ } else if (selector.trim().startsWith("\"") && selector.trim().endsWith("\"")) {
+ this.selector = selector.replaceFirst("^\\s*\"", "").replaceFirst("\"\\s*$", "").trim();
+ } else {
+ this.selector = selector.trim();
+ }
}
public String getSelector() {
--
GitLab
From 136d1203bc31fbb05ca6b4dae8e9ab4916fd86b2 Mon Sep 17 00:00:00 2001
From: Daniel <d.hornung@indiscale.com>
Date: Mon, 16 May 2022 09:41:01 +0200
Subject: [PATCH 02/38] DOC: Release guidelines.
---
RELEASE_GUIDELINES.md | 10 +++++++++-
1 file changed, 9 insertions(+), 1 deletion(-)
diff --git a/RELEASE_GUIDELINES.md b/RELEASE_GUIDELINES.md
index b2c78172..fff7dac5 100644
--- a/RELEASE_GUIDELINES.md
+++ b/RELEASE_GUIDELINES.md
@@ -36,4 +36,12 @@ guidelines of the CaosDB Project
* `src/doc/conf.py`
* `CHANGELOG.md`: Re-add the `[Unreleased]` section.
-10. Add a gitlab release in the respective repository.
+10. Add a gitlab release in the respective repository:
+ https://gitlab.indiscale.com/caosdb/src/caosdb-server/-/releases
+
+ Add a description, which can be a copy&paste from the CHANGELOG, possibly prepended by:
+ ```md
+# Changelog
+
+[See full changelog](https://gitlab.indiscale.com/caosdb/src/caosdb-server/-/blob/v0.7.3/CHANGELOG.md)
+ ```
--
GitLab
From 70ac5281f65d03b92e83ffdcaff3477c4d10f773 Mon Sep 17 00:00:00 2001
From: Timm Fitschen <t.fitschen@indiscale.com>
Date: Mon, 13 Jun 2022 13:38:28 +0200
Subject: [PATCH 03/38] DOC: minimal additions to docs of
usersources.ini.template
---
conf/core/usersources.ini.template | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/conf/core/usersources.ini.template b/conf/core/usersources.ini.template
index 2e0fe249..fb328572 100644
--- a/conf/core/usersources.ini.template
+++ b/conf/core/usersources.ini.template
@@ -42,7 +42,8 @@ defaultRealm = PAM
# Options for authentication against Linux' PAM.
[PAM]
class = org.caosdb.server.accessControl.Pam
-# The script which does the actual checking.
+# The script which does the actual checking. It is possible to put custom
+# scripts or the misc/pam_authentication/ldap_authentication.sh script here.
; pam_script = ./misc/pam_authentication/pam_authentication.sh
default_status = ACTIVE
# Only users which fulfill these criteria are accepted.
--
GitLab
From 9c9c2b9f8040bdb45e41d283fdee5379281b50fd Mon Sep 17 00:00:00 2001
From: Daniel <d.hornung@indiscale.com>
Date: Wed, 15 Jun 2022 15:33:13 +0200
Subject: [PATCH 04/38] DOC: Removed historic and unnecessary sentence from
permissions.rst.
Also fixed some links and added a bit of information to README_SETUP.md
---
README_SETUP.md | 4 ++--
src/doc/permissions.rst | 10 ++++------
2 files changed, 6 insertions(+), 8 deletions(-)
diff --git a/README_SETUP.md b/README_SETUP.md
index a264669c..6bbbd711 100644
--- a/README_SETUP.md
+++ b/README_SETUP.md
@@ -84,8 +84,8 @@ server:
`caosdb-mysqlbackend` repository
5. Create an authtoken config (e.g. copy `conf/core/authtoken.example.yaml` to
`conf/ext/authtoken.yml` and change it)
-6. Copy `conf/core/server.conf` to `conf/ext/server.conf` and change it
- appropriately:
+6. If any of the settings in `conf/core/server.conf` do not fit your needs, create
+ `conf/ext/server.conf` and add settings as required:
* Setup for MySQL back-end:
specify the fields `MYSQL_USER_NAME`, `MYSQL_USER_PASSWORD`,
`MYSQL_DATABASE_NAME`, and `MYSQL_HOST`.
diff --git a/src/doc/permissions.rst b/src/doc/permissions.rst
index e61ee49a..c0f30a0e 100644
--- a/src/doc/permissions.rst
+++ b/src/doc/permissions.rst
@@ -5,9 +5,7 @@ CaosDB has a fine grained role based permission system. Each interaction
with the server is governed by the current rules of the user, by default
this is the ``anonymous`` role. The permissions for an action which
involves one or more objects are set either manually or via default
-permissions which can be configured. For more detailed information,
-there is separate
-:doc:``documentation of the permission system<permissions>``.
+permissions which can be configured.
Permissions are needed to perform particular elementary *actions* during
any interaction with the the server. E.g. retrieving an Entity requires
@@ -87,11 +85,11 @@ How to set permissions
various files which use the permission API:
- The `example
- file <https://gitlab.com/caosdb/caosdb-pylib/blob/HEAD/examples/set_permissions.py>`__
+ file <https://gitlab.com/caosdb/caosdb-pylib/-/blob/main/examples/set_permissions.py>`__
- The ```caosdb_admin.py`` utility
- script <https://gitlab.com/caosdb/caosdb-pylib/blob/HEAD/src/caosdb/utils/caosdb_admin.py>`__
+ script <https://gitlab.com/caosdb/caosdb-pylib/-/blob/main/src/caosdb/utils/caosdb_admin.py>`__
- The `integration
- tests <https://gitlab.com/caosdb/caosdb-pyinttest/blob/HEAD/tests/test_permissions.py>`__
+ tests <https://gitlab.com/caosdb/caosdb-pyinttest/-/blob/main/tests/test_permissions.py>`__
also cover quite a bit of the permission API.
- WebUI :: Not implemented (or documented?) yet.
--
GitLab
From a44e0650cf5e1e37b2f0844e15cd2c0154824940 Mon Sep 17 00:00:00 2001
From: Daniel Hornung <d.hornung@indiscale.com>
Date: Mon, 20 Jun 2022 13:05:53 +0200
Subject: [PATCH 05/38] MAINT: Switching CI pipeline triggering to the
"trigger" keyword.
---
.gitlab-ci.yml | 42 ++++++++++++++++++++----------------------
1 file changed, 20 insertions(+), 22 deletions(-)
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index c2de130b..609009d0 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -3,8 +3,10 @@
#
# Copyright (C) 2018 Research Group Biomedical Physics,
# Max-Planck-Institute for Dynamics and Self-Organization Göttingen
+# Copyright (C) 2019-2022 Indiscale GmbH <info@indiscale.com>
# Copyright (C) 2019 Henrik tom Wörden
# Copyright (C) 2021 Timm Fitschen <t.fitschen@indiscale.com>
+# Copyright (C) 2020-2022 Daniel Hornung <d.hornung@indiscale.com>
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as
@@ -25,8 +27,6 @@ variables:
CI_REGISTRY_IMAGE: $CI_REGISTRY/caosdb/src/caosdb-server/caosdb-server-testenv:latest
GIT_SUBMODULE_STRATEGY: normal
- DEPLOY_PIPELINE: https://gitlab.indiscale.com/api/v4/projects/14/trigger/pipeline
-
## FOR DEBUGGING
TRIGGERED_BY_REPO: SERVER
TRIGGERED_BY_REF: $CI_COMMIT_REF_NAME
@@ -39,16 +39,13 @@ stages:
- test
- deploy
-.env: &env
- - F_BRANCH="${CI_COMMIT_REF_NAME}"
-
info:
tags: [cached-dind]
image: docker:20.10
stage: info
needs: []
script:
- - *env
+ - F_BRANCH="${CI_COMMIT_REF_NAME}"
- echo "Pipeline triggered by $TRIGGERED_BY_REPO@$TRIGGERED_BY_REF ($TRIGGERED_BY_HASH)"
- echo "Pipeline will trigger DEPLOY with branch $DEPLOY_REF"
- echo "F_BRANCH = $F_BRANCH"
@@ -83,25 +80,26 @@ test:
- mvn compile
- mvn test
-
-
# Deploy: Trigger building of server image and integration tests
-trigger_build:
- tags: [ docker ]
+trigger_inttest:
stage: deploy
needs: [ test ]
- script:
- - *env
-
- - echo "Triggering pipeline ${DEPLOY_PIPELINE}@${DEPLOY_REF} with F_BRANCH=${F_BRANCH}"
- - /usr/bin/curl -X POST
- -F token=$CI_JOB_TOKEN
- -F "variables[SERVER]=$CI_COMMIT_REF_NAME"
- -F "variables[F_BRANCH]=$F_BRANCH"
- -F "variables[TRIGGERED_BY_REPO]=$TRIGGERED_BY_REPO"
- -F "variables[TRIGGERED_BY_REF]=$TRIGGERED_BY_REF"
- -F "variables[TRIGGERED_BY_HASH]=$TRIGGERED_BY_HASH"
- -F ref=$DEPLOY_REF $DEPLOY_PIPELINE
+ inherit:
+ variables:
+ # List the variables that shall be inherited, which also means they will override any equally
+ # named varibles in child pipelines.
+ - DEPLOY_REF
+ - TRIGGERED_BY_REPO
+ - TRIGGERED_BY_REF
+ - TRIGGERED_BY_HASH
+ variables:
+ # Renaming variables.
+ F_BRANCH: $CI_COMMIT_REF_NAME
+ SERVER: $CI_COMMIT_REF_NAME
+ trigger:
+ project: caosdb/src/caosdb-deploy
+ branch: $DEPLOY_REF
+ strategy: depend
# Build the sphinx documentation and make it ready for deployment by Gitlab Pages
# Special job for serving a static website. See https://docs.gitlab.com/ee/ci/yaml/README.html#pages
--
GitLab
From 06ea0ac153eb6d871d07119eed8c0556118c5277 Mon Sep 17 00:00:00 2001
From: Daniel Hornung <d.hornung@indiscale.com>
Date: Mon, 20 Jun 2022 14:19:17 +0200
Subject: [PATCH 06/38] WIP: Removed unnecessary variable reference.
---
.gitlab-ci.yml | 3 +--
1 file changed, 1 insertion(+), 2 deletions(-)
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 609009d0..ed9b2c48 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -83,12 +83,11 @@ test:
# Deploy: Trigger building of server image and integration tests
trigger_inttest:
stage: deploy
- needs: [ test ]
+ needs: [ ]
inherit:
variables:
# List the variables that shall be inherited, which also means they will override any equally
# named varibles in child pipelines.
- - DEPLOY_REF
- TRIGGERED_BY_REPO
- TRIGGERED_BY_REF
- TRIGGERED_BY_HASH
--
GitLab
From 2896007b1c25f3fcbcc515153bf8583c3ae302da Mon Sep 17 00:00:00 2001
From: Daniel Hornung <d.hornung@indiscale.com>
Date: Mon, 20 Jun 2022 14:29:05 +0200
Subject: [PATCH 07/38] Revert "WIP: Removed unnecessary variable reference."
This reverts commit 06ea0ac153eb6d871d07119eed8c0556118c5277.
---
.gitlab-ci.yml | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index ed9b2c48..609009d0 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -83,11 +83,12 @@ test:
# Deploy: Trigger building of server image and integration tests
trigger_inttest:
stage: deploy
- needs: [ ]
+ needs: [ test ]
inherit:
variables:
# List the variables that shall be inherited, which also means they will override any equally
# named varibles in child pipelines.
+ - DEPLOY_REF
- TRIGGERED_BY_REPO
- TRIGGERED_BY_REF
- TRIGGERED_BY_HASH
--
GitLab
From 81f883a0aa9a296d4cdeebc6bba82e4325d4b017 Mon Sep 17 00:00:00 2001
From: Daniel Hornung <d.hornung@indiscale.com>
Date: Mon, 20 Jun 2022 14:34:11 +0200
Subject: [PATCH 08/38] DOC: Fixed links in documentation.
---
src/doc/roles.md | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/src/doc/roles.md b/src/doc/roles.md
index 2d1d36c4..138c75d3 100644
--- a/src/doc/roles.md
+++ b/src/doc/roles.md
@@ -10,7 +10,7 @@ users may have the same role, and there may be roles without any users.
The user and their roles are always returned by the server in answers to requests
and can thus be interpreted and used by clients. The most important use though
-is [permission](doc:`permissions`) checking in the server: Access and
+is [permission](permissions.rst) checking in the server: Access and
modification of
entities can be controlled via roles, so that users of a given role are allowed
or denied certain actions. Incidentally, the permission to edit the permissions
@@ -32,4 +32,4 @@ There are some special roles, which are automatically assigned to users:
Except for the `anonymous` role, these special roles are not returned by the
server, but can nevertheless be used to define
-[permissions](doc:`permissions`).
+[permissions](permissions.rst).
--
GitLab
From f42f03a6c3879d2b5d252364e31bb2c606ce99ba Mon Sep 17 00:00:00 2001
From: Daniel Hornung <d.hornung@indiscale.com>
Date: Tue, 21 Jun 2022 16:14:13 +0200
Subject: [PATCH 09/38] WIP: Fix #145: large integers in query.
---
.../java/org/caosdb/server/query/POV.java | 25 +++++++++++++++---
.../java/org/caosdb/server/query/TestCQL.java | 26 +++++++++++++++++++
2 files changed, 47 insertions(+), 4 deletions(-)
diff --git a/src/main/java/org/caosdb/server/query/POV.java b/src/main/java/org/caosdb/server/query/POV.java
index b1a45752..0bd04879 100644
--- a/src/main/java/org/caosdb/server/query/POV.java
+++ b/src/main/java/org/caosdb/server/query/POV.java
@@ -125,6 +125,7 @@ public class POV implements EntityFilterInterface {
unitStr = m.group(2);
this.vInt = Integer.parseInt(vIntStr);
} catch (final NumberFormatException e) {
+ System.out.println("Integer exception");
this.vInt = null;
}
@@ -133,7 +134,8 @@ public class POV implements EntityFilterInterface {
this.vDouble = (double) this.vInt;
} else {
try {
- final Pattern dp = Pattern.compile("^(-?[0-9]+(?:\\.[0-9]+))\\s*([^-]*)$");
+ final Pattern dp = Pattern.compile("^(-?[0-9]+(?:(\\.)?[0-9]+))\\s*([^-]*)$");
+ // final Pattern dp = Pattern.compile("^(-?[0-9]+(?:\\.[0-9]+))\\s*([^-]*)$");
final Matcher m = dp.matcher(value);
if (!m.matches()) {
throw new NumberFormatException();
@@ -142,9 +144,9 @@ public class POV implements EntityFilterInterface {
unitStr = m.group(2);
this.vDouble = Double.parseDouble(vDoubleStr);
- if (this.vDouble % 1 == 0) {
- this.vInt = (int) Math.floor(this.vDouble);
- }
+ // if (this.vDouble % 1 == 0) {
+ // this.vInt = (int) Math.floor(this.vDouble);
+ // }
} catch (final NumberFormatException e) {
this.vDouble = null;
}
@@ -505,6 +507,21 @@ public class POV implements EntityFilterInterface {
return ret;
}
+ /** Return the value type as string, for debugging puposes. */
+ public String getValueType() {
+ if (this.vInt != null) { // vInt
+ System.out.println(this.vInt);
+ return "Integer";
+ }
+ if (this.vDouble != null) { // vInt
+ return "Double";
+ }
+ if (this.vDatetime != null) { // vInt
+ return "Datetime";
+ }
+ return "String";
+ }
+
public String getAggregate() {
return this.aggregate;
}
diff --git a/src/test/java/org/caosdb/server/query/TestCQL.java b/src/test/java/org/caosdb/server/query/TestCQL.java
index ff1be776..707462e5 100644
--- a/src/test/java/org/caosdb/server/query/TestCQL.java
+++ b/src/test/java/org/caosdb/server/query/TestCQL.java
@@ -240,6 +240,7 @@ public class TestCQL {
String queryIssue132b = "FIND ENTITY WHICH HAS BEEN CREATED TODAY BY ME";
String queryIssue134 = "SELECT pname FROM ename";
String queryIssue131 = "FIND ENTITY WITH pname = 13 €";
+ String queryIssue145 = "FIND ENTITY WITH pname145 = 100000000000000000000";
// File paths ///////////////////////////////////////////////////////////////
String filepath_verb01 = "/foo/";
@@ -6906,4 +6907,29 @@ public class TestCQL {
assertEquals("POV(pname2,>,30)", pov1.getSubProperty().getFilter().toString());
assertEquals("POV(pname2,<,40)", pov2.getSubProperty().getFilter().toString());
}
+
+ /**
+ * Integer values which are too large for Int32
+ *
+ * <p>String queryIssue145= "FIND ENTITY WITH pname145 = 10000000000";
+ */
+ @Test
+ public void testIssue145() {
+ // must yield a valid value
+ CQLLexer lexer;
+ lexer = new CQLLexer(CharStreams.fromString(this.queryIssue145));
+ final CommonTokenStream tokens = new CommonTokenStream(lexer);
+
+ final CQLParser parser = new CQLParser(tokens);
+ final CqContext sfq = parser.cq();
+
+ System.out.println(sfq.toStringTree(parser));
+ // assertEquals("POV(pname145,=,10000000000)", sfq.filter.toString());
+
+ // assert value
+ POV pov = ((POV) sfq.filter);
+ System.out.println(pov.getValue());
+ // // assertEquals("10000000000", pov.getValue());
+ // assertEquals("Double", pov.getValueType());
+ }
}
--
GitLab
From 149edf1f0cb04a0731072473c1420cf5bee9ef0a Mon Sep 17 00:00:00 2001
From: Daniel Hornung <d.hornung@indiscale.com>
Date: Tue, 21 Jun 2022 17:04:39 +0200
Subject: [PATCH 10/38] WIP: Undoing number pattern in query.
---
src/main/java/org/caosdb/server/query/POV.java | 4 ++--
src/test/java/org/caosdb/server/query/TestCQL.java | 8 ++++----
2 files changed, 6 insertions(+), 6 deletions(-)
diff --git a/src/main/java/org/caosdb/server/query/POV.java b/src/main/java/org/caosdb/server/query/POV.java
index 0bd04879..99aecf7f 100644
--- a/src/main/java/org/caosdb/server/query/POV.java
+++ b/src/main/java/org/caosdb/server/query/POV.java
@@ -134,8 +134,8 @@ public class POV implements EntityFilterInterface {
this.vDouble = (double) this.vInt;
} else {
try {
- final Pattern dp = Pattern.compile("^(-?[0-9]+(?:(\\.)?[0-9]+))\\s*([^-]*)$");
- // final Pattern dp = Pattern.compile("^(-?[0-9]+(?:\\.[0-9]+))\\s*([^-]*)$");
+ // final Pattern dp = Pattern.compile("^(-?[0-9]+(?:(\\.)?[0-9]+))\\s*([^-]*)$");
+ final Pattern dp = Pattern.compile("^(-?[0-9]+(?:\\.[0-9]+))\\s*([^-]*)$");
final Matcher m = dp.matcher(value);
if (!m.matches()) {
throw new NumberFormatException();
diff --git a/src/test/java/org/caosdb/server/query/TestCQL.java b/src/test/java/org/caosdb/server/query/TestCQL.java
index 707462e5..9abed098 100644
--- a/src/test/java/org/caosdb/server/query/TestCQL.java
+++ b/src/test/java/org/caosdb/server/query/TestCQL.java
@@ -240,7 +240,7 @@ public class TestCQL {
String queryIssue132b = "FIND ENTITY WHICH HAS BEEN CREATED TODAY BY ME";
String queryIssue134 = "SELECT pname FROM ename";
String queryIssue131 = "FIND ENTITY WITH pname = 13 €";
- String queryIssue145 = "FIND ENTITY WITH pname145 = 100000000000000000000";
+ String queryIssue145 = "FIND ENTITY WITH pname145 = 10000000000";
// File paths ///////////////////////////////////////////////////////////////
String filepath_verb01 = "/foo/";
@@ -6924,12 +6924,12 @@ public class TestCQL {
final CqContext sfq = parser.cq();
System.out.println(sfq.toStringTree(parser));
- // assertEquals("POV(pname145,=,10000000000)", sfq.filter.toString());
+ assertEquals("POV(pname145,=,10000000000)", sfq.filter.toString());
// assert value
POV pov = ((POV) sfq.filter);
System.out.println(pov.getValue());
- // // assertEquals("10000000000", pov.getValue());
- // assertEquals("Double", pov.getValueType());
+ assertEquals("10000000000", pov.getValue());
+ assertEquals("Double", pov.getValueType());
}
}
--
GitLab
From 9d2c1b6b542a09644bfde9cb503a0b3cdd42c1bf Mon Sep 17 00:00:00 2001
From: Daniel Hornung <d.hornung@indiscale.com>
Date: Tue, 21 Jun 2022 17:07:55 +0200
Subject: [PATCH 11/38] WIP: Fixing number pattern in query for floats without
dot.
---
src/main/java/org/caosdb/server/query/POV.java | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/src/main/java/org/caosdb/server/query/POV.java b/src/main/java/org/caosdb/server/query/POV.java
index 99aecf7f..43b3ca23 100644
--- a/src/main/java/org/caosdb/server/query/POV.java
+++ b/src/main/java/org/caosdb/server/query/POV.java
@@ -134,8 +134,8 @@ public class POV implements EntityFilterInterface {
this.vDouble = (double) this.vInt;
} else {
try {
- // final Pattern dp = Pattern.compile("^(-?[0-9]+(?:(\\.)?[0-9]+))\\s*([^-]*)$");
- final Pattern dp = Pattern.compile("^(-?[0-9]+(?:\\.[0-9]+))\\s*([^-]*)$");
+ final Pattern dp = Pattern.compile("^(-?[0-9]+(?:\\.)?(?:[0-9]+))\\s*([^-]*)$");
+ // final Pattern dp = Pattern.compile("^(-?[0-9]+(?:\\.[0-9]+))\\s*([^-]*)$");
final Matcher m = dp.matcher(value);
if (!m.matches()) {
throw new NumberFormatException();
--
GitLab
From 6ec140d366da840c237910d0e02842ade90bf5a4 Mon Sep 17 00:00:00 2001
From: Daniel Hornung <d.hornung@indiscale.com>
Date: Tue, 21 Jun 2022 17:30:46 +0200
Subject: [PATCH 12/38] ENH: Allow testing for actually stored POV values.
---
src/main/java/org/caosdb/server/query/POV.java | 17 ++++++++++++++++-
.../java/org/caosdb/server/query/TestCQL.java | 1 +
2 files changed, 17 insertions(+), 1 deletion(-)
diff --git a/src/main/java/org/caosdb/server/query/POV.java b/src/main/java/org/caosdb/server/query/POV.java
index 43b3ca23..1c04443f 100644
--- a/src/main/java/org/caosdb/server/query/POV.java
+++ b/src/main/java/org/caosdb/server/query/POV.java
@@ -134,8 +134,8 @@ public class POV implements EntityFilterInterface {
this.vDouble = (double) this.vInt;
} else {
try {
+ // Doubles are allowed without dots, for example when the integer overflows.
final Pattern dp = Pattern.compile("^(-?[0-9]+(?:\\.)?(?:[0-9]+))\\s*([^-]*)$");
- // final Pattern dp = Pattern.compile("^(-?[0-9]+(?:\\.[0-9]+))\\s*([^-]*)$");
final Matcher m = dp.matcher(value);
if (!m.matches()) {
throw new NumberFormatException();
@@ -522,6 +522,21 @@ public class POV implements EntityFilterInterface {
return "String";
}
+ /** Return the Int value, which may be null. */
+ public Integer getVInt() {
+ return this.vInt;
+ }
+
+ /** Return the Double value, which may be null. */
+ public Double getVDouble() {
+ return this.vDouble;
+ }
+
+ /** Return the Datetime value, which may be null. */
+ public DateTimeInterface getVDatetime() {
+ return this.vDatetime;
+ }
+
public String getAggregate() {
return this.aggregate;
}
diff --git a/src/test/java/org/caosdb/server/query/TestCQL.java b/src/test/java/org/caosdb/server/query/TestCQL.java
index 9abed098..a4b08515 100644
--- a/src/test/java/org/caosdb/server/query/TestCQL.java
+++ b/src/test/java/org/caosdb/server/query/TestCQL.java
@@ -6931,5 +6931,6 @@ public class TestCQL {
System.out.println(pov.getValue());
assertEquals("10000000000", pov.getValue());
assertEquals("Double", pov.getValueType());
+ assertEquals(1e10, pov.getVDouble().doubleValue(), 0.0);
}
}
--
GitLab
From 6d747ce940d0fefeb4c8fbde41b59e7db4f1c67b Mon Sep 17 00:00:00 2001
From: Daniel Hornung <d.hornung@indiscale.com>
Date: Tue, 21 Jun 2022 17:32:46 +0200
Subject: [PATCH 13/38] DOC: Changelog notice for #145
---
CHANGELOG.md | 3 +++
1 file changed, 3 insertions(+)
diff --git a/CHANGELOG.md b/CHANGELOG.md
index f0904bed..83f1c3ff 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -17,6 +17,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
### Fixed
+* [caosdb-server#145](https://gitlab.com/caosdb/caosdb-server/-/issues/145) Searching for large numbers results in wrong results if integer values are
+ used.
+
### Security
## [0.7.3] - 2022-05-03
--
GitLab
From 3063dbacb006bfbbee1e2df51f23389cba8b1ec9 Mon Sep 17 00:00:00 2001
From: Daniel Hornung <d.hornung@indiscale.com>
Date: Tue, 21 Jun 2022 17:57:04 +0200
Subject: [PATCH 14/38] MAINT: Removed debugging output and commented code.
---
src/main/java/org/caosdb/server/query/POV.java | 4 ----
1 file changed, 4 deletions(-)
diff --git a/src/main/java/org/caosdb/server/query/POV.java b/src/main/java/org/caosdb/server/query/POV.java
index 1c04443f..c64d522a 100644
--- a/src/main/java/org/caosdb/server/query/POV.java
+++ b/src/main/java/org/caosdb/server/query/POV.java
@@ -125,7 +125,6 @@ public class POV implements EntityFilterInterface {
unitStr = m.group(2);
this.vInt = Integer.parseInt(vIntStr);
} catch (final NumberFormatException e) {
- System.out.println("Integer exception");
this.vInt = null;
}
@@ -144,9 +143,6 @@ public class POV implements EntityFilterInterface {
unitStr = m.group(2);
this.vDouble = Double.parseDouble(vDoubleStr);
- // if (this.vDouble % 1 == 0) {
- // this.vInt = (int) Math.floor(this.vDouble);
- // }
} catch (final NumberFormatException e) {
this.vDouble = null;
}
--
GitLab
From c3ba30cc736d7fe8563a07789aa3f9acfd571a28 Mon Sep 17 00:00:00 2001
From: Daniel Hornung <d.hornung@indiscale.com>
Date: Tue, 21 Jun 2022 18:58:23 +0200
Subject: [PATCH 15/38] WIP: Find regression.
---
src/main/java/org/caosdb/server/query/POV.java | 3 ++-
src/test/java/org/caosdb/server/query/TestCQL.java | 4 ++--
2 files changed, 4 insertions(+), 3 deletions(-)
diff --git a/src/main/java/org/caosdb/server/query/POV.java b/src/main/java/org/caosdb/server/query/POV.java
index c64d522a..1dcc44ec 100644
--- a/src/main/java/org/caosdb/server/query/POV.java
+++ b/src/main/java/org/caosdb/server/query/POV.java
@@ -134,7 +134,8 @@ public class POV implements EntityFilterInterface {
} else {
try {
// Doubles are allowed without dots, for example when the integer overflows.
- final Pattern dp = Pattern.compile("^(-?[0-9]+(?:\\.)?(?:[0-9]+))\\s*([^-]*)$");
+ // final Pattern dp = Pattern.compile("^(-?[0-9]+(?:\\.)?(?:[0-9]+))\\s*([^-]*)$");
+ final Pattern dp = Pattern.compile("^(-?[0-9]+(?:\\.[0-9]+))\\s*([^-]*)$");
final Matcher m = dp.matcher(value);
if (!m.matches()) {
throw new NumberFormatException();
diff --git a/src/test/java/org/caosdb/server/query/TestCQL.java b/src/test/java/org/caosdb/server/query/TestCQL.java
index a4b08515..c600fc32 100644
--- a/src/test/java/org/caosdb/server/query/TestCQL.java
+++ b/src/test/java/org/caosdb/server/query/TestCQL.java
@@ -6930,7 +6930,7 @@ public class TestCQL {
POV pov = ((POV) sfq.filter);
System.out.println(pov.getValue());
assertEquals("10000000000", pov.getValue());
- assertEquals("Double", pov.getValueType());
- assertEquals(1e10, pov.getVDouble().doubleValue(), 0.0);
+ // assertEquals("Double", pov.getValueType());
+ // assertEquals(1e10, pov.getVDouble().doubleValue(), 0.0);
}
}
--
GitLab
From 901b19c7ff5534a721c7bbeeedee9aa763b0d9f6 Mon Sep 17 00:00:00 2001
From: Daniel Hornung <d.hornung@indiscale.com>
Date: Tue, 21 Jun 2022 21:14:16 +0200
Subject: [PATCH 16/38] WIP: Find regression.
---
src/main/java/org/caosdb/server/query/POV.java | 3 +++
1 file changed, 3 insertions(+)
diff --git a/src/main/java/org/caosdb/server/query/POV.java b/src/main/java/org/caosdb/server/query/POV.java
index 1dcc44ec..8632070e 100644
--- a/src/main/java/org/caosdb/server/query/POV.java
+++ b/src/main/java/org/caosdb/server/query/POV.java
@@ -144,6 +144,9 @@ public class POV implements EntityFilterInterface {
unitStr = m.group(2);
this.vDouble = Double.parseDouble(vDoubleStr);
+ if (this.vDouble % 1 == 0) {
+ this.vInt = (int) Math.floor(this.vDouble);
+ }
} catch (final NumberFormatException e) {
this.vDouble = null;
}
--
GitLab
From 39db84a04c1668d8893c3002ee0596bf45f26bf9 Mon Sep 17 00:00:00 2001
From: Daniel Hornung <d.hornung@indiscale.com>
Date: Tue, 21 Jun 2022 22:17:16 +0200
Subject: [PATCH 17/38] WIP: Find regression.
---
.../java/org/caosdb/server/query/POV.java | 58 +++++++++----------
1 file changed, 29 insertions(+), 29 deletions(-)
diff --git a/src/main/java/org/caosdb/server/query/POV.java b/src/main/java/org/caosdb/server/query/POV.java
index 8632070e..b030e0f4 100644
--- a/src/main/java/org/caosdb/server/query/POV.java
+++ b/src/main/java/org/caosdb/server/query/POV.java
@@ -507,35 +507,35 @@ public class POV implements EntityFilterInterface {
return ret;
}
- /** Return the value type as string, for debugging puposes. */
- public String getValueType() {
- if (this.vInt != null) { // vInt
- System.out.println(this.vInt);
- return "Integer";
- }
- if (this.vDouble != null) { // vInt
- return "Double";
- }
- if (this.vDatetime != null) { // vInt
- return "Datetime";
- }
- return "String";
- }
-
- /** Return the Int value, which may be null. */
- public Integer getVInt() {
- return this.vInt;
- }
-
- /** Return the Double value, which may be null. */
- public Double getVDouble() {
- return this.vDouble;
- }
-
- /** Return the Datetime value, which may be null. */
- public DateTimeInterface getVDatetime() {
- return this.vDatetime;
- }
+ /** Return the value type as string, for debugging purposes. */
+ // public String getValueType() {
+ // if (this.vInt != null) { // vInt
+ // System.out.println(this.vInt);
+ // return "Integer";
+ // }
+ // if (this.vDouble != null) { // vInt
+ // return "Double";
+ // }
+ // if (this.vDatetime != null) { // vInt
+ // return "Datetime";
+ // }
+ // return "String";
+ // }
+
+ // /** Return the Int value, which may be null. */
+ // public Integer getVInt() {
+ // return this.vInt;
+ // }
+
+ // /** Return the Double value, which may be null. */
+ // public Double getVDouble() {
+ // return this.vDouble;
+ // }
+
+ // /** Return the Datetime value, which may be null. */
+ // public DateTimeInterface getVDatetime() {
+ // return this.vDatetime;
+ // }
public String getAggregate() {
return this.aggregate;
--
GitLab
From e85cd4189be2b1d4c36f93c445ad3ea235510147 Mon Sep 17 00:00:00 2001
From: Daniel Hornung <d.hornung@indiscale.com>
Date: Wed, 22 Jun 2022 16:40:28 +0200
Subject: [PATCH 18/38] Revert "WIP: Find regression."
---
.../java/org/caosdb/server/query/POV.java | 64 +++++++++----------
.../java/org/caosdb/server/query/TestCQL.java | 4 +-
2 files changed, 32 insertions(+), 36 deletions(-)
diff --git a/src/main/java/org/caosdb/server/query/POV.java b/src/main/java/org/caosdb/server/query/POV.java
index b030e0f4..c64d522a 100644
--- a/src/main/java/org/caosdb/server/query/POV.java
+++ b/src/main/java/org/caosdb/server/query/POV.java
@@ -134,8 +134,7 @@ public class POV implements EntityFilterInterface {
} else {
try {
// Doubles are allowed without dots, for example when the integer overflows.
- // final Pattern dp = Pattern.compile("^(-?[0-9]+(?:\\.)?(?:[0-9]+))\\s*([^-]*)$");
- final Pattern dp = Pattern.compile("^(-?[0-9]+(?:\\.[0-9]+))\\s*([^-]*)$");
+ final Pattern dp = Pattern.compile("^(-?[0-9]+(?:\\.)?(?:[0-9]+))\\s*([^-]*)$");
final Matcher m = dp.matcher(value);
if (!m.matches()) {
throw new NumberFormatException();
@@ -144,9 +143,6 @@ public class POV implements EntityFilterInterface {
unitStr = m.group(2);
this.vDouble = Double.parseDouble(vDoubleStr);
- if (this.vDouble % 1 == 0) {
- this.vInt = (int) Math.floor(this.vDouble);
- }
} catch (final NumberFormatException e) {
this.vDouble = null;
}
@@ -507,35 +503,35 @@ public class POV implements EntityFilterInterface {
return ret;
}
- /** Return the value type as string, for debugging purposes. */
- // public String getValueType() {
- // if (this.vInt != null) { // vInt
- // System.out.println(this.vInt);
- // return "Integer";
- // }
- // if (this.vDouble != null) { // vInt
- // return "Double";
- // }
- // if (this.vDatetime != null) { // vInt
- // return "Datetime";
- // }
- // return "String";
- // }
-
- // /** Return the Int value, which may be null. */
- // public Integer getVInt() {
- // return this.vInt;
- // }
-
- // /** Return the Double value, which may be null. */
- // public Double getVDouble() {
- // return this.vDouble;
- // }
-
- // /** Return the Datetime value, which may be null. */
- // public DateTimeInterface getVDatetime() {
- // return this.vDatetime;
- // }
+ /** Return the value type as string, for debugging puposes. */
+ public String getValueType() {
+ if (this.vInt != null) { // vInt
+ System.out.println(this.vInt);
+ return "Integer";
+ }
+ if (this.vDouble != null) { // vInt
+ return "Double";
+ }
+ if (this.vDatetime != null) { // vInt
+ return "Datetime";
+ }
+ return "String";
+ }
+
+ /** Return the Int value, which may be null. */
+ public Integer getVInt() {
+ return this.vInt;
+ }
+
+ /** Return the Double value, which may be null. */
+ public Double getVDouble() {
+ return this.vDouble;
+ }
+
+ /** Return the Datetime value, which may be null. */
+ public DateTimeInterface getVDatetime() {
+ return this.vDatetime;
+ }
public String getAggregate() {
return this.aggregate;
diff --git a/src/test/java/org/caosdb/server/query/TestCQL.java b/src/test/java/org/caosdb/server/query/TestCQL.java
index c600fc32..a4b08515 100644
--- a/src/test/java/org/caosdb/server/query/TestCQL.java
+++ b/src/test/java/org/caosdb/server/query/TestCQL.java
@@ -6930,7 +6930,7 @@ public class TestCQL {
POV pov = ((POV) sfq.filter);
System.out.println(pov.getValue());
assertEquals("10000000000", pov.getValue());
- // assertEquals("Double", pov.getValueType());
- // assertEquals(1e10, pov.getVDouble().doubleValue(), 0.0);
+ assertEquals("Double", pov.getValueType());
+ assertEquals(1e10, pov.getVDouble().doubleValue(), 0.0);
}
}
--
GitLab
From 92f7dcb441eea4a8322a00dbd6af99fbd8c65ff9 Mon Sep 17 00:00:00 2001
From: Daniel Hornung <d.hornung@indiscale.com>
Date: Wed, 22 Jun 2022 20:04:05 +0200
Subject: [PATCH 19/38] ENH: An additional query test.
---
src/test/java/org/caosdb/server/query/TestCQL.java | 1 +
1 file changed, 1 insertion(+)
diff --git a/src/test/java/org/caosdb/server/query/TestCQL.java b/src/test/java/org/caosdb/server/query/TestCQL.java
index a4b08515..87cfddbb 100644
--- a/src/test/java/org/caosdb/server/query/TestCQL.java
+++ b/src/test/java/org/caosdb/server/query/TestCQL.java
@@ -3782,6 +3782,7 @@ public class TestCQL {
System.out.println(sfq.toStringTree(parser));
assertEquals("POV(pname,=,1.02m)", sfq.filter.toString());
+ assertEquals(1.02, ((POV) sfq.filter).getVDouble(), 0.0);
// 5 children: FIND, entity, WHITE_SPACE, filter, EOF
assertEquals(5, sfq.getChildCount());
--
GitLab
From 81cf4de4debdf9ef00bcd1a2114b921eb84084aa Mon Sep 17 00:00:00 2001
From: Daniel Hornung <d.hornung@indiscale.com>
Date: Thu, 23 Jun 2022 13:05:00 +0200
Subject: [PATCH 20/38] MAINT: Removed debugging output.
---
src/main/java/org/caosdb/server/query/POV.java | 1 -
1 file changed, 1 deletion(-)
diff --git a/src/main/java/org/caosdb/server/query/POV.java b/src/main/java/org/caosdb/server/query/POV.java
index c64d522a..61711b4d 100644
--- a/src/main/java/org/caosdb/server/query/POV.java
+++ b/src/main/java/org/caosdb/server/query/POV.java
@@ -506,7 +506,6 @@ public class POV implements EntityFilterInterface {
/** Return the value type as string, for debugging puposes. */
public String getValueType() {
if (this.vInt != null) { // vInt
- System.out.println(this.vInt);
return "Integer";
}
if (this.vDouble != null) { // vInt
--
GitLab
From 0c329e01a2a30d7cd602c9c9b4d2342d59171a78 Mon Sep 17 00:00:00 2001
From: Daniel Hornung <d.hornung@indiscale.com>
Date: Thu, 23 Jun 2022 13:53:45 +0200
Subject: [PATCH 21/38] MAINT: Remove pure debugging method, add cloning for
encapsulation.
---
.../java/org/caosdb/server/query/POV.java | 29 ++++++++-----------
.../java/org/caosdb/server/query/TestCQL.java | 3 +-
2 files changed, 14 insertions(+), 18 deletions(-)
diff --git a/src/main/java/org/caosdb/server/query/POV.java b/src/main/java/org/caosdb/server/query/POV.java
index 61711b4d..62ad1aa7 100644
--- a/src/main/java/org/caosdb/server/query/POV.java
+++ b/src/main/java/org/caosdb/server/query/POV.java
@@ -503,33 +503,28 @@ public class POV implements EntityFilterInterface {
return ret;
}
- /** Return the value type as string, for debugging puposes. */
- public String getValueType() {
- if (this.vInt != null) { // vInt
- return "Integer";
- }
- if (this.vDouble != null) { // vInt
- return "Double";
- }
- if (this.vDatetime != null) { // vInt
- return "Datetime";
- }
- return "String";
- }
-
/** Return the Int value, which may be null. */
public Integer getVInt() {
- return this.vInt;
+ if (this.vInt != null){
+ return vInt.clone();
+ }
+ return null;
}
/** Return the Double value, which may be null. */
public Double getVDouble() {
- return this.vDouble;
+ if (this.vDouble != null) {
+ return this.vDouble.clone();
+ }
+ return null;
}
/** Return the Datetime value, which may be null. */
public DateTimeInterface getVDatetime() {
- return this.vDatetime;
+ if (this.vDatetime != null) {
+ return DateTimeFactory2.valueOf(this.vDatetime.toDateTimeString());
+ }
+ return null;
}
public String getAggregate() {
diff --git a/src/test/java/org/caosdb/server/query/TestCQL.java b/src/test/java/org/caosdb/server/query/TestCQL.java
index 87cfddbb..3a151fc3 100644
--- a/src/test/java/org/caosdb/server/query/TestCQL.java
+++ b/src/test/java/org/caosdb/server/query/TestCQL.java
@@ -6931,7 +6931,8 @@ public class TestCQL {
POV pov = ((POV) sfq.filter);
System.out.println(pov.getValue());
assertEquals("10000000000", pov.getValue());
- assertEquals("Double", pov.getValueType());
+ assertNotNull(pov.getVDouble());
+ assertNull(pov.getVInt());
assertEquals(1e10, pov.getVDouble().doubleValue(), 0.0);
}
}
--
GitLab
From 37b188010bc58ade5740241c04dc4f1e29610e2c Mon Sep 17 00:00:00 2001
From: Daniel Hornung <d.hornung@indiscale.com>
Date: Thu, 23 Jun 2022 14:14:20 +0200
Subject: [PATCH 22/38] ENH: Always apply POV to ints and doubles alike.
---
.../java/org/caosdb/server/query/POV.java | 28 ++++++++++---------
1 file changed, 15 insertions(+), 13 deletions(-)
diff --git a/src/main/java/org/caosdb/server/query/POV.java b/src/main/java/org/caosdb/server/query/POV.java
index 62ad1aa7..ae1ac705 100644
--- a/src/main/java/org/caosdb/server/query/POV.java
+++ b/src/main/java/org/caosdb/server/query/POV.java
@@ -128,10 +128,8 @@ public class POV implements EntityFilterInterface {
this.vInt = null;
}
- // try and parse as double
- if (this.vInt != null) {
- this.vDouble = (double) this.vInt;
- } else {
+ // Try and parse as double, if integer parsing was unsuccessful.
+ if (this.vInt == null) {
try {
// Doubles are allowed without dots, for example when the integer overflows.
final Pattern dp = Pattern.compile("^(-?[0-9]+(?:\\.)?(?:[0-9]+))\\s*([^-]*)$");
@@ -217,6 +215,11 @@ public class POV implements EntityFilterInterface {
"Versioned queries are not supported for subqueries yet. Please file a feature request.");
}
final long t1 = System.currentTimeMillis();
+ // Add type-converted substitutes for ints/doubles.
+ final Integer vIntSubst = (this.vDouble != null && this.vDouble % 1 == 0)
+ ? (int) Math.rint(this.vDouble) : null;
+ final Double vDoubleSubst = (this.vInt != null) ? (double) this.vInt : null;
+
try {
this.connection = query.getConnection();
this.targetSet = query.getTargetSet();
@@ -270,15 +273,14 @@ public class POV implements EntityFilterInterface {
} else {
callPOV.setNull(6, VARCHAR);
}
- if (this.vInt != null) { // vInt
- callPOV.setInt(7, this.vInt);
- } else {
- callPOV.setNull(7, INTEGER);
- }
- if (this.vDouble != null) { // vDouble
- callPOV.setDouble(8, this.vDouble);
- } else {
- callPOV.setNull(8, DOUBLE);
+ if (this.vInt != null || this.vDouble != null) { // Some numeric
+ if (this.vInt != null) {
+ callPOV.setInt(7, this.vInt);
+ callPOV.setDouble(8, vDoubleSubst);
+ } else {
+ callPOV.setInt(7, vIntSubst);
+ callPOV.setDouble(8, this.vDouble);
+ }
}
if (this.unit != null) {
final long unitSig = this.unit.getSignature();
--
GitLab
From a47a3aad802a6bbbb73e52b817234a0297ba3d89 Mon Sep 17 00:00:00 2001
From: Daniel Hornung <d.hornung@indiscale.com>
Date: Thu, 23 Jun 2022 14:54:11 +0200
Subject: [PATCH 23/38] FIX: Fixed compilation errors.
---
src/main/java/org/caosdb/server/query/POV.java | 16 ++++++----------
1 file changed, 6 insertions(+), 10 deletions(-)
diff --git a/src/main/java/org/caosdb/server/query/POV.java b/src/main/java/org/caosdb/server/query/POV.java
index ae1ac705..3b26fecd 100644
--- a/src/main/java/org/caosdb/server/query/POV.java
+++ b/src/main/java/org/caosdb/server/query/POV.java
@@ -23,7 +23,6 @@
package org.caosdb.server.query;
import static java.sql.Types.DOUBLE;
-import static java.sql.Types.INTEGER;
import static java.sql.Types.VARCHAR;
import static org.caosdb.server.database.DatabaseUtils.bytes2UTF8;
@@ -216,8 +215,8 @@ public class POV implements EntityFilterInterface {
}
final long t1 = System.currentTimeMillis();
// Add type-converted substitutes for ints/doubles.
- final Integer vIntSubst = (this.vDouble != null && this.vDouble % 1 == 0)
- ? (int) Math.rint(this.vDouble) : null;
+ final Integer vIntSubst =
+ (this.vDouble != null && this.vDouble % 1 == 0) ? (int) Math.rint(this.vDouble) : null;
final Double vDoubleSubst = (this.vInt != null) ? (double) this.vInt : null;
try {
@@ -507,8 +506,8 @@ public class POV implements EntityFilterInterface {
/** Return the Int value, which may be null. */
public Integer getVInt() {
- if (this.vInt != null){
- return vInt.clone();
+ if (this.vInt != null) {
+ return Integer.valueOf(vInt);
}
return null;
}
@@ -516,17 +515,14 @@ public class POV implements EntityFilterInterface {
/** Return the Double value, which may be null. */
public Double getVDouble() {
if (this.vDouble != null) {
- return this.vDouble.clone();
+ return Double.valueOf(vDouble);
}
return null;
}
/** Return the Datetime value, which may be null. */
public DateTimeInterface getVDatetime() {
- if (this.vDatetime != null) {
- return DateTimeFactory2.valueOf(this.vDatetime.toDateTimeString());
- }
- return null;
+ return this.vDatetime;
}
public String getAggregate() {
--
GitLab
From 5c38b39ee2275a395f90ac00890dcc9edf201512 Mon Sep 17 00:00:00 2001
From: Daniel Hornung <d.hornung@indiscale.com>
Date: Thu, 23 Jun 2022 16:14:47 +0200
Subject: [PATCH 24/38] FIX: Fixed SQL errors.
---
src/main/java/org/caosdb/server/query/POV.java | 4 ++++
1 file changed, 4 insertions(+)
diff --git a/src/main/java/org/caosdb/server/query/POV.java b/src/main/java/org/caosdb/server/query/POV.java
index 3b26fecd..0fb67838 100644
--- a/src/main/java/org/caosdb/server/query/POV.java
+++ b/src/main/java/org/caosdb/server/query/POV.java
@@ -23,6 +23,7 @@
package org.caosdb.server.query;
import static java.sql.Types.DOUBLE;
+import static java.sql.Types.INTEGER;
import static java.sql.Types.VARCHAR;
import static org.caosdb.server.database.DatabaseUtils.bytes2UTF8;
@@ -280,6 +281,9 @@ public class POV implements EntityFilterInterface {
callPOV.setInt(7, vIntSubst);
callPOV.setDouble(8, this.vDouble);
}
+ } else {
+ callPOV.setNull(7, INTEGER);
+ callPOV.setNull(8, DOUBLE);
}
if (this.unit != null) {
final long unitSig = this.unit.getSignature();
--
GitLab
From 268b0792b6094ca80acf7072e362e90e3814d257 Mon Sep 17 00:00:00 2001
From: Daniel Hornung <d.hornung@indiscale.com>
Date: Fri, 24 Jun 2022 10:20:29 +0200
Subject: [PATCH 25/38] FIX: Fixed a NullPointerException
---
src/main/java/org/caosdb/server/query/POV.java | 6 +++++-
1 file changed, 5 insertions(+), 1 deletion(-)
diff --git a/src/main/java/org/caosdb/server/query/POV.java b/src/main/java/org/caosdb/server/query/POV.java
index 0fb67838..95c50420 100644
--- a/src/main/java/org/caosdb/server/query/POV.java
+++ b/src/main/java/org/caosdb/server/query/POV.java
@@ -278,7 +278,11 @@ public class POV implements EntityFilterInterface {
callPOV.setInt(7, this.vInt);
callPOV.setDouble(8, vDoubleSubst);
} else {
- callPOV.setInt(7, vIntSubst);
+ if (vIntSubst == null) {
+ callPOV.setNull(7, INTEGER);
+ } else {
+ callPOV.setInt(7, vIntSubst);
+ }
callPOV.setDouble(8, this.vDouble);
}
} else {
--
GitLab
From c7b2ab0e42471cfea1cfc617e4a05ff920ab0c03 Mon Sep 17 00:00:00 2001
From: Daniel Hornung <d.hornung@indiscale.com>
Date: Fri, 24 Jun 2022 11:58:30 +0200
Subject: [PATCH 26/38] FIX: POV.java: Fixed unit-induced value conversion.
---
src/main/java/org/caosdb/server/query/POV.java | 18 ++++++++++++------
1 file changed, 12 insertions(+), 6 deletions(-)
diff --git a/src/main/java/org/caosdb/server/query/POV.java b/src/main/java/org/caosdb/server/query/POV.java
index 95c50420..89ea9ad4 100644
--- a/src/main/java/org/caosdb/server/query/POV.java
+++ b/src/main/java/org/caosdb/server/query/POV.java
@@ -63,6 +63,7 @@ public class POV implements EntityFilterInterface {
private DateTimeInterface vDatetime;
private final String aggregate;
private String targetSet = null;
+ private String unitStr = null;
private Unit unit = null;
private Long stdUnitSig = null;
private Double vDoubleConvertedToStdUnit = null;
@@ -112,7 +113,6 @@ public class POV implements EntityFilterInterface {
// parse value to int/double/datetime
if (this.value != null) {
- String unitStr = null;
// try and parse as integer
try {
@@ -122,7 +122,7 @@ public class POV implements EntityFilterInterface {
throw new NumberFormatException();
}
final String vIntStr = m.group(1);
- unitStr = m.group(2);
+ this.unitStr = m.group(2);
this.vInt = Integer.parseInt(vIntStr);
} catch (final NumberFormatException e) {
this.vInt = null;
@@ -138,7 +138,7 @@ public class POV implements EntityFilterInterface {
throw new NumberFormatException();
}
final String vDoubleStr = m.group(1);
- unitStr = m.group(2);
+ this.unitStr = m.group(2);
this.vDouble = Double.parseDouble(vDoubleStr);
} catch (final NumberFormatException e) {
@@ -146,16 +146,17 @@ public class POV implements EntityFilterInterface {
}
}
- if (this.vDouble != null && unitStr != null && unitStr.length() > 0) {
+ if ((this.vDouble != null || this.vInt != null)
+ && this.unitStr != null
+ && this.unitStr.length() > 0) {
try {
- this.unit = getUnit(unitStr);
+ this.unit = getUnit(this.unitStr);
} catch (final ParserException e) {
e.printStackTrace();
throw new UnsupportedOperationException("Could not parse the unit.");
}
this.stdUnitSig = this.unit.normalize().getSignature();
- this.vDoubleConvertedToStdUnit = this.unit.convert(this.vDouble);
}
// try and parse as datetime
try {
@@ -285,6 +286,11 @@ public class POV implements EntityFilterInterface {
}
callPOV.setDouble(8, this.vDouble);
}
+ // finally: do unit conversion
+ if (this.unitStr != null && this.unitStr.length() > 0) {
+ this.vDoubleConvertedToStdUnit =
+ this.unit.convert(this.vDouble != null ? this.vDouble : vDoubleSubst);
+ }
} else {
callPOV.setNull(7, INTEGER);
callPOV.setNull(8, DOUBLE);
--
GitLab
From d7fa78c01838fdf9bac30cde5c6b5f1d5c7cd0a9 Mon Sep 17 00:00:00 2001
From: Timm Fitschen <t.fitschen@indiscale.com>
Date: Wed, 29 Jun 2022 21:53:40 +0200
Subject: [PATCH 27/38] Make password and user name requirements configurable
---
conf/core/server.conf | 12 ++++++++++++
.../org/caosdb/server/ServerProperties.java | 8 ++++++--
.../caosdb/server/resource/UserResource.java | 10 ++++------
.../transaction/InsertUserTransaction.java | 15 ++++++---------
.../org/caosdb/server/utils/ServerMessages.java | 13 +++++++------
.../java/org/caosdb/server/utils/Utils.java | 17 ++++++++---------
6 files changed, 43 insertions(+), 32 deletions(-)
diff --git a/conf/core/server.conf b/conf/core/server.conf
index 246be9aa..5cad243f 100644
--- a/conf/core/server.conf
+++ b/conf/core/server.conf
@@ -199,6 +199,18 @@ CHECK_ENTITY_ACL_ROLES_MODE=MUST
# part of any Entity ACL.
GLOBAL_ENTITY_PERMISSIONS_FILE=./conf/core/global_entity_permissions.xml
+# --------------------------------------------------
+# User Account Settings
+# --------------------------------------------------
+
+# Requirements for user names. The default is POSIX compliant, see
+# https://pubs.opengroup.org/onlinepubs/000095399/basedefs/xbd_chap03.html#tag_03_426
+USER_NAME_VALID_REGEX=^[\\w\\.][\\w\\.-]*{1,32}$
+USER_NAME_INVALID_MESSAGE=User names must have a length from 1 to 32 characters. They must contain only latin letters a-z (upper case or lower case), number 0-9, dots (.), underscores (_), or hyphens (-). They must no start with a hyphen.
+
+PASSWORD_VALID_REGEX=^((?=.*\\d)(?=.*[a-z])(?=.*[A-Z])(?=.*[\\p{Punct}]).{8,128})$
+PASSWORD_INVALID_MESSAGE=blablabla
+
# --------------------------------------------------
# Extensions
# --------------------------------------------------
diff --git a/src/main/java/org/caosdb/server/ServerProperties.java b/src/main/java/org/caosdb/server/ServerProperties.java
index 34899f75..aa8fc43e 100644
--- a/src/main/java/org/caosdb/server/ServerProperties.java
+++ b/src/main/java/org/caosdb/server/ServerProperties.java
@@ -108,8 +108,6 @@ public class ServerProperties extends Properties {
public static final String KEY_NEW_USER_DEFAULT_ACTIVITY = "NEW_USER_DEFAULT_ACTIVITY";
- public static final String KEY_PASSWORD_STRENGTH_REGEX = "PASSWORD_STRENGTH_REGEX";
-
public static final String KEY_QUERY_FILTER_ENTITIES_WITHOUT_RETRIEVE_PERMISSIONS =
"QUERY_FILTER_ENTITIES_WITHOUT_RETRIEVE_PERMISSIONS";
@@ -141,6 +139,12 @@ public class ServerProperties extends Properties {
public static final String KEY_PROJECT_REVISTION = "project.revision";
public static final String KEY_BUILD_TIMESTAMP = "build.timestamp";
+ public static final String KEY_USER_NAME_VALID_REGEX = "USER_NAME_VALID_REGEX";
+ public static final String KEY_USER_NAME_INVALID_MESSAGE = "USER_NAME_INVALID_MESSAGE";
+
+ public static final String KEY_PASSWORD_STRENGTH_REGEX = "PASSWORD_VALID_REGEX";
+ public static final String KEY_PASSWORD_WEAK_MESSAGE = "PASSWORD_INVALID_MESSAGE";
+
/**
* Read the config files and initialize the server properties.
*
diff --git a/src/main/java/org/caosdb/server/resource/UserResource.java b/src/main/java/org/caosdb/server/resource/UserResource.java
index 2c463e0e..de07331e 100644
--- a/src/main/java/org/caosdb/server/resource/UserResource.java
+++ b/src/main/java/org/caosdb/server/resource/UserResource.java
@@ -128,10 +128,9 @@ public class UserResource extends AbstractCaosDBServerResource {
return error(m, Status.CLIENT_ERROR_NOT_FOUND);
} else if (m == ServerMessages.ENTITY_DOES_NOT_EXIST) {
return error(m, Status.CLIENT_ERROR_CONFLICT);
- } else if (m == ServerMessages.PASSWORD_TOO_WEAK) {
- return error(m, Status.CLIENT_ERROR_UNPROCESSABLE_ENTITY);
+ } else {
+ return error(m, Status.CLIENT_ERROR_BAD_REQUEST);
}
- throw m;
} catch (final NumberFormatException e) {
return error(ServerMessages.CANNOT_PARSE_INT_VALUE, Status.CLIENT_ERROR_BAD_REQUEST);
}
@@ -163,10 +162,9 @@ public class UserResource extends AbstractCaosDBServerResource {
} catch (final Message m) {
if (m == ServerMessages.ACCOUNT_NAME_NOT_UNIQUE) {
return error(m, Status.CLIENT_ERROR_CONFLICT);
- } else if (m == ServerMessages.PASSWORD_TOO_WEAK) {
- return error(m, Status.CLIENT_ERROR_UNPROCESSABLE_ENTITY);
+ } else {
+ return error(m, Status.CLIENT_ERROR_BAD_REQUEST);
}
- throw m;
}
final Document doc = new Document();
diff --git a/src/main/java/org/caosdb/server/transaction/InsertUserTransaction.java b/src/main/java/org/caosdb/server/transaction/InsertUserTransaction.java
index 09d59e8d..8abc2ede 100644
--- a/src/main/java/org/caosdb/server/transaction/InsertUserTransaction.java
+++ b/src/main/java/org/caosdb/server/transaction/InsertUserTransaction.java
@@ -24,6 +24,8 @@
package org.caosdb.server.transaction;
import org.apache.shiro.SecurityUtils;
+import org.caosdb.server.CaosDBServer;
+import org.caosdb.server.ServerProperties;
import org.caosdb.server.accessControl.ACMPermissions;
import org.caosdb.server.accessControl.Principal;
import org.caosdb.server.accessControl.UserSources;
@@ -86,18 +88,13 @@ public class InsertUserTransaction extends AccessControlTransaction {
}
/*
- * Names should have at least a length of 1, a maximum length of 32 and match
- * ^[a-zA-Z_][a-zA-Z0-9_-]*$.
+ * Check requirements for user names (length, character set). Default config is POSIX compliant.
*/
private void checkUserName(String name) throws Message {
- // Make this configurable?
- final boolean length = name.length() >= 1 && name.length() <= 32;
- final boolean match =
- name.matches("^[\\p{Lower}\\p{Upper}_][\\p{Lower}\\p{Upper}\\p{Digit}_-]*$");
-
- if (!(length && match)) {
+ String regex = CaosDBServer.getServerProperty(ServerProperties.KEY_USER_NAME_VALID_REGEX);
+ if (!name.matches(regex)) {
throw ServerMessages.INVALID_USER_NAME(
- "User names must have a length from 1 to 32 characters, begin with a latin letter a-z (upper case or lower case) or an underscore (_), and all other characters must be latin letters, arabic numbers, hyphens (-) or undescores (_).");
+ CaosDBServer.getServerProperty(ServerProperties.KEY_USER_NAME_INVALID_MESSAGE));
}
if (UserSources.isUserExisting(new Principal(this.user.realm, this.user.name))) {
diff --git a/src/main/java/org/caosdb/server/utils/ServerMessages.java b/src/main/java/org/caosdb/server/utils/ServerMessages.java
index 151f55fa..0ec31162 100644
--- a/src/main/java/org/caosdb/server/utils/ServerMessages.java
+++ b/src/main/java/org/caosdb/server/utils/ServerMessages.java
@@ -337,7 +337,7 @@ public class ServerMessages {
new Message(
MessageType.Error,
MessageCode.MESSAGE_CODE_UNKNOWN,
- "This user name is yet in use. Please choose another one.");
+ "This user name is already in use. Please choose another one.");
public static final Message ACCOUNT_HAS_BEEN_DELETED =
new Message(
@@ -399,11 +399,12 @@ public class ServerMessages {
MessageCode.MESSAGE_CODE_UNKNOWN,
"This email address is not RFC822 compliant.");
- public static final Message PASSWORD_TOO_WEAK =
- new Message(
- MessageType.Error,
- MessageCode.MESSAGE_CODE_UNKNOWN,
- "This password is too weak. It should be longer than 8 characters and sufficiently random.");
+ public static final Message PASSWORD_TOO_WEAK(String policy) {
+ return new Message(
+ MessageType.Error,
+ MessageCode.MESSAGE_CODE_UNKNOWN,
+ "This password does not match the current policies for passwords: " + policy);
+ }
public static final Message AFFILIATION_ERROR =
new Message(
diff --git a/src/main/java/org/caosdb/server/utils/Utils.java b/src/main/java/org/caosdb/server/utils/Utils.java
index b21ba0f5..07a7e961 100644
--- a/src/main/java/org/caosdb/server/utils/Utils.java
+++ b/src/main/java/org/caosdb/server/utils/Utils.java
@@ -35,6 +35,8 @@ import java.text.DecimalFormat;
import java.util.Scanner;
import java.util.regex.Pattern;
import org.apache.commons.codec.binary.Base32;
+import org.caosdb.server.CaosDBServer;
+import org.caosdb.server.ServerProperties;
import org.caosdb.server.entity.Message;
import org.jdom2.Document;
import org.jdom2.Element;
@@ -304,18 +306,15 @@ public class Utils {
* @param password The password to be checked.
*/
public static void checkPasswordStrength(final String password) throws Message {
- final boolean length = password.length() >= 8;
- final boolean uppercase = password.matches(".*\\p{Upper}.*");
- final boolean lowercase = password.matches(".*\\p{Lower}.*");
- final boolean number = password.matches(".*\\p{Digit}.*");
- final boolean punct = password.matches(".*\\p{Punct}.*");
+ String regex = CaosDBServer.getServerProperty(ServerProperties.KEY_PASSWORD_STRENGTH_REGEX);
- if (!(length && uppercase && lowercase && number && punct)) {
- throw ServerMessages.PASSWORD_TOO_WEAK;
+ if (password.equals("correcthorsebatterystaple")) {
+ throw ServerMessages.PASSWORD_TOO_WEAK("PWNED!");
}
- if (password.equals("correcthorsebatterystaple")) {
- throw ServerMessages.PASSWORD_TOO_WEAK;
+ if (!password.matches(regex)) {
+ throw ServerMessages.PASSWORD_TOO_WEAK(
+ CaosDBServer.getServerProperty(ServerProperties.KEY_PASSWORD_WEAK_MESSAGE));
}
}
}
--
GitLab
From e70bbafcd58d7f163a2bd3c95071a5a9e678be3a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Henrik=20tom=20W=C3=B6rden?= <henrik@trineo.org>
Date: Thu, 30 Jun 2022 10:05:50 +0200
Subject: [PATCH 28/38] DOC: add hint for benchmark tests
---
doc/devel/Benchmarking.md | 5 ++++-
1 file changed, 4 insertions(+), 1 deletion(-)
diff --git a/doc/devel/Benchmarking.md b/doc/devel/Benchmarking.md
index 5fc3f75f..8a3eff2a 100644
--- a/doc/devel/Benchmarking.md
+++ b/doc/devel/Benchmarking.md
@@ -59,7 +59,10 @@ enabled on start up.
This script expects the MariaDB server to be accessible on 127.0.0.1 with the default caosdb user
and password (caosdb;random1234).
-
+You might consider to increase `performance_schema_events_transactions_history_long_size`.
+```
+performance_schema_events_transactions_history_long_size=1000000
+```
The performance schema must be enabled (see below).
### MariaDB General Query Log ###
--
GitLab
From 1d69714951cef7c086563e11b79c515e6437499f Mon Sep 17 00:00:00 2001
From: Timm Fitschen <t.fitschen@indiscale.com>
Date: Thu, 30 Jun 2022 17:24:15 +0200
Subject: [PATCH 29/38] MAINT: nicer error messages for user name/password
invalid
---
conf/core/server.conf | 2 +-
src/main/java/org/caosdb/server/utils/ServerMessages.java | 4 ++--
2 files changed, 3 insertions(+), 3 deletions(-)
diff --git a/conf/core/server.conf b/conf/core/server.conf
index 5cad243f..57744250 100644
--- a/conf/core/server.conf
+++ b/conf/core/server.conf
@@ -209,7 +209,7 @@ USER_NAME_VALID_REGEX=^[\\w\\.][\\w\\.-]*{1,32}$
USER_NAME_INVALID_MESSAGE=User names must have a length from 1 to 32 characters. They must contain only latin letters a-z (upper case or lower case), number 0-9, dots (.), underscores (_), or hyphens (-). They must no start with a hyphen.
PASSWORD_VALID_REGEX=^((?=.*\\d)(?=.*[a-z])(?=.*[A-Z])(?=.*[\\p{Punct}]).{8,128})$
-PASSWORD_INVALID_MESSAGE=blablabla
+PASSWORD_INVALID_MESSAGE=Passwords must have a length from 8 to 128 characters. THe must contain at least one [A-Z], one [a-z], one [0-9] and a special character e.g. [!§$%&/()=?.;,:#+*+~].
# --------------------------------------------------
# Extensions
diff --git a/src/main/java/org/caosdb/server/utils/ServerMessages.java b/src/main/java/org/caosdb/server/utils/ServerMessages.java
index 0ec31162..835087ce 100644
--- a/src/main/java/org/caosdb/server/utils/ServerMessages.java
+++ b/src/main/java/org/caosdb/server/utils/ServerMessages.java
@@ -403,7 +403,7 @@ public class ServerMessages {
return new Message(
MessageType.Error,
MessageCode.MESSAGE_CODE_UNKNOWN,
- "This password does not match the current policies for passwords: " + policy);
+ "The password does not comply with the current policies for passwords: " + policy);
}
public static final Message AFFILIATION_ERROR =
@@ -610,7 +610,7 @@ public class ServerMessages {
return new Message(
MessageType.Error,
MessageCode.MESSAGE_CODE_UNKNOWN,
- "The user name does not comply with the policies for user names: " + policy);
+ "The user name does not comply with the current policies for user names: " + policy);
}
public static final Message CANNOT_DELETE_YOURSELF() {
--
GitLab
From 71f839e62dfbf2467cb773cebf00a606f499bf8c Mon Sep 17 00:00:00 2001
From: Timm Fitschen <t.fitschen@indiscale.com>
Date: Thu, 30 Jun 2022 21:22:55 +0200
Subject: [PATCH 30/38] BUG: Fix for
https://gitlab.com/caosdb/caosdb-server/-/issues/148
---
src/main/java/org/caosdb/server/utils/FileUtils.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/src/main/java/org/caosdb/server/utils/FileUtils.java b/src/main/java/org/caosdb/server/utils/FileUtils.java
index 70ba0030..eb4270f8 100644
--- a/src/main/java/org/caosdb/server/utils/FileUtils.java
+++ b/src/main/java/org/caosdb/server/utils/FileUtils.java
@@ -525,7 +525,7 @@ public class FileUtils {
}
private static void callPosixUnlink(File file) throws IOException, InterruptedException {
- final Process cmd = Runtime.getRuntime().exec("unlink " + file.getAbsolutePath());
+ final Process cmd = Runtime.getRuntime().exec(new String[] {"unlink", file.getAbsolutePath()});
if (cmd.waitFor() != 0) {
throw new CaosDBException("could not unlink " + file.getAbsolutePath());
}
--
GitLab
From 4ff44a996a3288c3ba0d1200738c9827b0709edf Mon Sep 17 00:00:00 2001
From: Timm Fitschen <t.fitschen@indiscale.com>
Date: Fri, 1 Jul 2022 09:16:35 +0000
Subject: [PATCH 31/38] DOC: Update CHANGELOG.md
---
CHANGELOG.md | 5 ++++-
1 file changed, 4 insertions(+), 1 deletion(-)
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 83f1c3ff..9d20d9fd 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -17,7 +17,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
### Fixed
-* [caosdb-server#145](https://gitlab.com/caosdb/caosdb-server/-/issues/145) Searching for large numbers results in wrong results if integer values are
+* [caosdb-server#148](https://gitlab.com/caosdb/caosdb-server/-/issues/148)
+ Cannot delete file entities containing unescaped space characters
+* [caosdb-server#145](https://gitlab.com/caosdb/caosdb-server/-/issues/145)
+ Searching for large numbers results in wrong results if integer values are
used.
### Security
--
GitLab
From 117cc4bc7fe630128c1e987d0f248e73d5cf38a3 Mon Sep 17 00:00:00 2001
From: Timm Fitschen <t.fitschen@indiscale.com>
Date: Tue, 5 Jul 2022 10:08:23 +0200
Subject: [PATCH 32/38] DOC: update CHANGELOG
---
CHANGELOG.md | 4 ++++
1 file changed, 4 insertions(+)
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 9d20d9fd..0475a4e9 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -9,8 +9,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
### Added
+* Configurable requirements for user names and passwords. The default is the old hard-coded configuration.
+
### Changed
+* Minimal changes to the error messages for invalid user names and passwords.
+
### Deprecated
### Removed
--
GitLab
From a1f3c3349ff63c500f76cc4f63e9972a16e477d9 Mon Sep 17 00:00:00 2001
From: Timm Fitschen <t.fitschen@indiscale.com>
Date: Tue, 5 Jul 2022 10:11:40 +0200
Subject: [PATCH 33/38] DOC: update CHANGELOG
---
CHANGELOG.md | 2 ++
1 file changed, 2 insertions(+)
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 0475a4e9..f8a6a7dc 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -41,6 +41,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
### Fixed
+* [caosdb-server#142](https://gitlab.com/caosdb/caosdb-server/-/issues/142)
+ Can't create users with dots in their user names
* `ldap_authentication.sh <username>` failed on every attempt when used in
combination with OpenLDAP with default configuration.
* `ldap_authentication.sh` allowed empty and even wrong passwords when used in
--
GitLab
From cd6c702b79fc3e145453ad557c9800c8c3356d07 Mon Sep 17 00:00:00 2001
From: Timm Fitschen <t.fitschen@indiscale.com>
Date: Thu, 7 Jul 2022 22:20:58 +0200
Subject: [PATCH 34/38] TST: unit test for caosdb-server#130
---
.../java/org/caosdb/server/query/CQLLexer.g4 | 45 +++++++++++++++++++
.../java/org/caosdb/server/query/CQLParser.g4 | 19 ++++++--
.../java/org/caosdb/server/query/Query.java | 8 +---
.../java/org/caosdb/server/query/TestCQL.java | 38 ++++++++++++++++
4 files changed, 100 insertions(+), 10 deletions(-)
diff --git a/src/main/java/org/caosdb/server/query/CQLLexer.g4 b/src/main/java/org/caosdb/server/query/CQLLexer.g4
index a091027c..8f58cb2c 100644
--- a/src/main/java/org/caosdb/server/query/CQLLexer.g4
+++ b/src/main/java/org/caosdb/server/query/CQLLexer.g4
@@ -498,18 +498,63 @@ mode DOUBLE_QUOTE_MODE;
;
+mode SELECT_DOUBLE_QUOTED;
+
+ SELECT_DOUBLE_QUOTE_ESCAPED:
+ ESC_MARKER
+ '"'
+ ;
+
+ SELECT_DOUBLE_QUOTE_END:
+ '"' {setText("");} -> mode(SELECT_MODE)
+ ;
+
+ SELECT_DOUBLE_QUOTE_TXT:
+ .
+ ;
+
+mode SELECT_SINGLE_QUOTED;
+
+ SELECT_SINGLE_QUOTE_ESCAPED:
+ ESC_MARKER
+ '\''
+ ;
+
+ SELECT_SINGLE_QUOTE_END:
+ '\'' {setText("");} -> mode(SELECT_MODE)
+ ;
+
+ SELECT_SINGLE_QUOTE_TXT:
+ .
+ ;
+
mode SELECT_MODE;
FROM:
[Ff][Rr][Oo][Mm]([ \t\n\r])* -> mode(DEFAULT_MODE)
;
+ SELECT_ESCAPED:
+ ESC_MARKER
+ ( '"' | '\\' | '\'' | ',' | '.' ) {setText(getText().substring(1));}
+ ;
+
SELECT_DOT:
'.'
+ WHITE_SPACE_f?
+ ;
+
+ SELECT_DOUBLE_QUOTE:
+ '"' {setText("");} -> mode(SELECT_DOUBLE_QUOTED)
+ ;
+
+ SELECT_SINGLE_QUOTE:
+ '\'' {setText("");} -> mode(SELECT_SINGLE_QUOTED)
;
SELECT_COMMA:
','
+ WHITE_SPACE_f?
;
SELECTOR_TXT:
diff --git a/src/main/java/org/caosdb/server/query/CQLParser.g4 b/src/main/java/org/caosdb/server/query/CQLParser.g4
index f8c2a3b5..c4a68e5f 100644
--- a/src/main/java/org/caosdb/server/query/CQLParser.g4
+++ b/src/main/java/org/caosdb/server/query/CQLParser.g4
@@ -74,11 +74,24 @@ prop_sel returns [List<Query.Selection> s]
;
prop_subsel returns [Query.Selection sub]:
- selector_txt {$sub = new Query.Selection($selector_txt.text);}(SELECT_DOT s=prop_subsel {$sub.setSubSelection($s.sub);})?
+ selector_txt {$sub = new Query.Selection($selector_txt.text);}
+ (
+ SELECT_DOT s=prop_subsel {$sub.setSubSelection($s.sub);}
+ )?
;
selector_txt:
- SELECTOR_TXT+
+ (
+ SELECT_DOUBLE_QUOTE
+ ( SELECT_DOUBLE_QUOTE_TXT | SELECT_DOUBLE_QUOTE_ESCAPED )*
+ SELECT_DOUBLE_QUOTE_END
+ ) | (
+ SELECT_SINGLE_QUOTE
+ ( SELECT_SINGLE_QUOTE_TXT | SELECT_SINGLE_QUOTE_ESCAPED )*
+ SELECT_SINGLE_QUOTE_END
+ )
+ |
+ ( SELECTOR_TXT | SELECT_ESCAPED )+
;
role returns [Query.Role r]:
@@ -292,7 +305,7 @@ subproperty returns [SubProperty subp]
$subp = null;
}
:
- subproperty_filter {$subp = new SubProperty($subproperty_filter.filter);}
+ subproperty_filter {$subp = new SubProperty($subproperty_filter.filter);}
;
subproperty_filter returns [EntityFilterInterface filter]
diff --git a/src/main/java/org/caosdb/server/query/Query.java b/src/main/java/org/caosdb/server/query/Query.java
index 39152435..07a165a3 100644
--- a/src/main/java/org/caosdb/server/query/Query.java
+++ b/src/main/java/org/caosdb/server/query/Query.java
@@ -100,13 +100,7 @@ public class Query implements QueryInterface, ToElementable, TransactionInterfac
/** No parsing, just sets the selector string. */
public Selection(final String selector) {
- if (selector.trim().startsWith("'") && selector.trim().endsWith("'")) {
- this.selector = selector.replaceFirst("^\\s*'", "").replaceFirst("'\\s*$", "").trim();
- } else if (selector.trim().startsWith("\"") && selector.trim().endsWith("\"")) {
- this.selector = selector.replaceFirst("^\\s*\"", "").replaceFirst("\"\\s*$", "").trim();
- } else {
- this.selector = selector.trim();
- }
+ this.selector = selector.trim();
}
public String getSelector() {
diff --git a/src/test/java/org/caosdb/server/query/TestCQL.java b/src/test/java/org/caosdb/server/query/TestCQL.java
index 3a151fc3..58de2f31 100644
--- a/src/test/java/org/caosdb/server/query/TestCQL.java
+++ b/src/test/java/org/caosdb/server/query/TestCQL.java
@@ -273,6 +273,10 @@ public class TestCQL {
String issue131e = "FIND ename WITH (pname1.pname2 > 30) AND (pname1.pname2 < 40)";
String issue131f = "FIND ename WITH (pname1.pname2 > 30) AND pname1.pname2 < 40";
+ // https://gitlab.com/caosdb/caosdb-server/-/issues/130
+ String issue130 =
+ "SELECT 'name with spaces.and dot', 'name with spaces'.name, name with spaces.name, name with\\,comma and\\.dot and \\'single_quote.sub FROM ENTITY";
+
@Test
public void testQuery1()
throws InterruptedException, SQLException, ConnectionException, QueryException {
@@ -6935,4 +6939,38 @@ public class TestCQL {
assertNull(pov.getVInt());
assertEquals(1e10, pov.getVDouble().doubleValue(), 0.0);
}
+
+ /**
+ * Spaces and escaped dots in selects
+ *
+ * <p>String issue130 = "SELECT 'name with spaces.and dot', 'name with spaces'.name, name with
+ * spaces.name FROM ENTITY";
+ */
+ @Test
+ public void testIssue130() {
+ CQLLexer lexer;
+ lexer = new CQLLexer(CharStreams.fromString(this.issue130));
+ final CommonTokenStream tokens = new CommonTokenStream(lexer);
+
+ final CQLParser parser = new CQLParser(tokens);
+ final CqContext sfq = parser.cq();
+
+ System.out.println(sfq.toStringTree(parser));
+
+ assertNotNull(sfq.s);
+ assertFalse(sfq.s.isEmpty());
+ assertEquals(4, sfq.s.size());
+ assertEquals("name with spaces.and dot", sfq.s.get(0).toString());
+ assertEquals("name with spaces.and dot", sfq.s.get(0).getSelector());
+ assertNull(sfq.s.get(0).getSubselection());
+ assertEquals("name with spaces.name", sfq.s.get(1).toString());
+ assertEquals("name with spaces", sfq.s.get(1).getSelector());
+ assertEquals("name", sfq.s.get(1).getSubselection().toString());
+ assertEquals("name with spaces.name", sfq.s.get(2).toString());
+ assertEquals("name with spaces", sfq.s.get(2).getSelector());
+ assertEquals("name", sfq.s.get(2).getSubselection().toString());
+ assertEquals("name with,comma and.dot and 'single_quote.sub", sfq.s.get(3).toString());
+ assertEquals("name with,comma and.dot and 'single_quote", sfq.s.get(3).getSelector());
+ assertEquals("sub", sfq.s.get(3).getSubselection().toString());
+ }
}
--
GitLab
From 857311e038580a04d9deb64f271e0b58250ee104 Mon Sep 17 00:00:00 2001
From: Timm Fitschen <t.fitschen@indiscale.com>
Date: Thu, 7 Jul 2022 22:53:15 +0200
Subject: [PATCH 35/38] TST: more unit tests for caosdb-server#130
---
.../java/org/caosdb/server/query/CQLLexer.g4 | 4 +--
.../java/org/caosdb/server/query/TestCQL.java | 34 ++++++++++++++-----
2 files changed, 27 insertions(+), 11 deletions(-)
diff --git a/src/main/java/org/caosdb/server/query/CQLLexer.g4 b/src/main/java/org/caosdb/server/query/CQLLexer.g4
index 8f58cb2c..99c9879d 100644
--- a/src/main/java/org/caosdb/server/query/CQLLexer.g4
+++ b/src/main/java/org/caosdb/server/query/CQLLexer.g4
@@ -506,7 +506,7 @@ mode SELECT_DOUBLE_QUOTED;
;
SELECT_DOUBLE_QUOTE_END:
- '"' {setText("");} -> mode(SELECT_MODE)
+ '"' WHITE_SPACE_f? {setText("");} -> mode(SELECT_MODE)
;
SELECT_DOUBLE_QUOTE_TXT:
@@ -521,7 +521,7 @@ mode SELECT_SINGLE_QUOTED;
;
SELECT_SINGLE_QUOTE_END:
- '\'' {setText("");} -> mode(SELECT_MODE)
+ '\'' WHITE_SPACE_f? {setText("");} -> mode(SELECT_MODE)
;
SELECT_SINGLE_QUOTE_TXT:
diff --git a/src/test/java/org/caosdb/server/query/TestCQL.java b/src/test/java/org/caosdb/server/query/TestCQL.java
index 58de2f31..3217d422 100644
--- a/src/test/java/org/caosdb/server/query/TestCQL.java
+++ b/src/test/java/org/caosdb/server/query/TestCQL.java
@@ -274,8 +274,9 @@ public class TestCQL {
String issue131f = "FIND ename WITH (pname1.pname2 > 30) AND pname1.pname2 < 40";
// https://gitlab.com/caosdb/caosdb-server/-/issues/130
- String issue130 =
+ String issue130a =
"SELECT 'name with spaces.and dot', 'name with spaces'.name, name with spaces.name, name with\\,comma and\\.dot and \\'single_quote.sub FROM ENTITY";
+ String issue130b = "SELECT 'Wrapper' FROM RECORD TestRT";
@Test
public void testQuery1()
@@ -6940,16 +6941,11 @@ public class TestCQL {
assertEquals(1e10, pov.getVDouble().doubleValue(), 0.0);
}
- /**
- * Spaces and escaped dots in selects
- *
- * <p>String issue130 = "SELECT 'name with spaces.and dot', 'name with spaces'.name, name with
- * spaces.name FROM ENTITY";
- */
+ /** Spaces and escaped dots in selects */
@Test
- public void testIssue130() {
+ public void testIssue130a() {
CQLLexer lexer;
- lexer = new CQLLexer(CharStreams.fromString(this.issue130));
+ lexer = new CQLLexer(CharStreams.fromString(this.issue130a));
final CommonTokenStream tokens = new CommonTokenStream(lexer);
final CQLParser parser = new CQLParser(tokens);
@@ -6973,4 +6969,24 @@ public class TestCQL {
assertEquals("name with,comma and.dot and 'single_quote", sfq.s.get(3).getSelector());
assertEquals("sub", sfq.s.get(3).getSubselection().toString());
}
+
+ /** Single quotes around the selector */
+ @Test
+ public void testIssue130b() {
+ CQLLexer lexer;
+ lexer = new CQLLexer(CharStreams.fromString(this.issue130b));
+ final CommonTokenStream tokens = new CommonTokenStream(lexer);
+
+ final CQLParser parser = new CQLParser(tokens);
+ final CqContext sfq = parser.cq();
+
+ System.out.println(sfq.toStringTree(parser));
+
+ assertNotNull(sfq.s);
+ assertFalse(sfq.s.isEmpty());
+ assertEquals(1, sfq.s.size());
+ assertEquals("Wrapper", sfq.s.get(0).toString());
+ assertEquals("Wrapper", sfq.s.get(0).getSelector());
+ assertNull(sfq.s.get(0).getSubselection());
+ }
}
--
GitLab
From 59158f519762fcac22cdabe45a933c41af49237f Mon Sep 17 00:00:00 2001
From: Timm Fitschen <t.fitschen@indiscale.com>
Date: Fri, 8 Jul 2022 00:08:48 +0200
Subject: [PATCH 36/38] DOC: update CHANGELOG
---
CHANGELOG.md | 2 ++
1 file changed, 2 insertions(+)
diff --git a/CHANGELOG.md b/CHANGELOG.md
index f8a6a7dc..249f8186 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -21,6 +21,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
### Fixed
+* [caosdb-server#130](https://gitlab.com/caosdb/caosdb-server/-/issues/130)
+ CQL: Spaces are not recognised in sub-property queries.
* [caosdb-server#148](https://gitlab.com/caosdb/caosdb-server/-/issues/148)
Cannot delete file entities containing unescaped space characters
* [caosdb-server#145](https://gitlab.com/caosdb/caosdb-server/-/issues/145)
--
GitLab
From 628512427dd3bb3bacd592daafa424cf5aac4486 Mon Sep 17 00:00:00 2001
From: Timm Fitschen <t.fitschen@indiscale.com>
Date: Tue, 12 Jul 2022 11:10:46 +0200
Subject: [PATCH 37/38] REL: prepare release 0.8.0
---
CHANGELOG.md | 3 ++-
DEPENDENCIES.md | 2 +-
FEATURES.md | 2 +-
caosdb-webui | 2 +-
pom.xml | 2 +-
src/doc/conf.py | 2 +-
6 files changed, 7 insertions(+), 6 deletions(-)
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 249f8186..5b655206 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -5,7 +5,8 @@ All notable changes to this project will be documented in this file.
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
-## [Unreleased]
+## [0.8.0] - 2022-07-12
+(Timm Fitschen)
### Added
diff --git a/DEPENDENCIES.md b/DEPENDENCIES.md
index ba9de363..0f380b92 100644
--- a/DEPENDENCIES.md
+++ b/DEPENDENCIES.md
@@ -12,7 +12,7 @@
## For Deploying a Web User Interface (optional)
-* `>=caosdb-webui 0.6.0`
+* `>=caosdb-webui 0.8.0`
## For Building the Documentation (optional)
diff --git a/FEATURES.md b/FEATURES.md
index 9cad165e..485172f0 100644
--- a/FEATURES.md
+++ b/FEATURES.md
@@ -12,7 +12,7 @@
(caosdb.entity.v1.Query) in a single transaction. The server throws an
error if it finds more than one query.
* Legacy XML/HTTP API (Deprecated)
-* Deployment of caosdb-webui (>=v0.4.1)
+* Deployment of caosdb-webui (>=0.8.0)
* Server-side Scripting API (v0.1)
* CaosDB Query Language Processor
* CaosDB FileSystem
diff --git a/caosdb-webui b/caosdb-webui
index 86dc30e3..e719e10b 160000
--- a/caosdb-webui
+++ b/caosdb-webui
@@ -1 +1 @@
-Subproject commit 86dc30e3526f2eab97de5dcd53c8eaa12c0e42e6
+Subproject commit e719e10b90594a158e704863611b7ab9b0440f80
diff --git a/pom.xml b/pom.xml
index 733a5b4c..8152f45b 100644
--- a/pom.xml
+++ b/pom.xml
@@ -25,7 +25,7 @@
<modelVersion>4.0.0</modelVersion>
<groupId>org.caosdb</groupId>
<artifactId>caosdb-server</artifactId>
- <version>0.8.0-SNAPSHOT</version>
+ <version>0.8.0</version>
<packaging>jar</packaging>
<name>CaosDB Server</name>
<scm>
diff --git a/src/doc/conf.py b/src/doc/conf.py
index ba98f7e5..181ff4fb 100644
--- a/src/doc/conf.py
+++ b/src/doc/conf.py
@@ -27,7 +27,7 @@ author = 'Daniel Hornung'
# The short X.Y version
version = '0.8.0'
# The full version, including alpha/beta/rc tags
-release = '0.8.0-SNAPSHOT'
+release = '0.8.0'
# -- General configuration ---------------------------------------------------
--
GitLab
From 63c6d7eb157ca8138eb22ad95585f14e44a45110 Mon Sep 17 00:00:00 2001
From: Timm Fitschen <t.fitschen@indiscale.com>
Date: Thu, 14 Jul 2022 10:02:19 +0200
Subject: [PATCH 38/38] PIPELINE
---
.gitlab-ci.yml | 12 ++++++++++++
1 file changed, 12 insertions(+)
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 609009d0..0359e740 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -49,6 +49,12 @@ info:
- echo "Pipeline triggered by $TRIGGERED_BY_REPO@$TRIGGERED_BY_REF ($TRIGGERED_BY_HASH)"
- echo "Pipeline will trigger DEPLOY with branch $DEPLOY_REF"
- echo "F_BRANCH = $F_BRANCH"
+ - echo "WEBUI = $WEBUI"
+ - echo "PYLIB = $PYLIB"
+ - echo "CPPLIB = $CPPLIB"
+ - echo "PYINT = $PYINT"
+ - echo "CPPINT = $CPPINT"
+ - echo "MYSQLBACKEND = $MYSQLBACKEND"
# Setup: Build a docker image in which tests for this repository can run
@@ -96,6 +102,12 @@ trigger_inttest:
# Renaming variables.
F_BRANCH: $CI_COMMIT_REF_NAME
SERVER: $CI_COMMIT_REF_NAME
+ WEBUI: $WEBUI
+ PYLIB: $PYLIB
+ CPPLIB: $CPPLIB
+ PYINT: $PYINT
+ CPPINT: $CPPINT
+ MYSQLBACKEND: $MYSQLBACKEND
trigger:
project: caosdb/src/caosdb-deploy
branch: $DEPLOY_REF
--
GitLab