diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index c38f56968aed1f151139f7f15990ec993d94057e..885b791421f5ddef4014692409e9358d7e6c507d 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -39,6 +39,10 @@ variables: CPPINT: "" MYSQLBACKEND: "" +workflow: + rules: + - if: $CI_PIPELINE_SOURCE != "merge_request_event" && $CI_COMMIT_REF_NAME != $CI_COMMIT_TAG + image: $CI_REGISTRY_IMAGE stages: - info @@ -70,8 +74,11 @@ build-testenv: image: docker:20.10 stage: setup timeout: 3h - only: - - schedules + rules: + - if: $CI_PIPELINE_SOURCE == "schedule" + - if: $CI_PIPELINE_SOURCE != "schedule" + changes: + - src/test/docker/Dockerfile script: - cd src/test/docker - docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY @@ -136,9 +143,8 @@ trigger_inttest: pages_prepare: &pages_prepare tags: [ cached-dind ] stage: deploy - only: - refs: - - /^release-.*$/i + rules: + - if: $CI_COMMIT_REF_NAME =~ /^release-.*$/i script: - echo "Deploying..." - make doc @@ -148,6 +154,5 @@ pages_prepare: &pages_prepare - public pages: <<: *pages_prepare - only: - refs: - - main + rules: + - if: $CI_COMMIT_REF_NAME == 'main' diff --git a/.gitlab/issue_templates/Default.md b/.gitlab/issue_templates/Default.md new file mode 100644 index 0000000000000000000000000000000000000000..aa1a65aca363b87aff50280e1a86824009d2098b --- /dev/null +++ b/.gitlab/issue_templates/Default.md @@ -0,0 +1,28 @@ +## Summary + +*Please give a short summary of what the issue is.* + +## Expected Behavior + +*What did you expect how the software should behave?* + +## Actual Behavior + +*What did the software actually do?* + +## Steps to Reproduce the Problem + +*Please describe, step by step, how others can reproduce the problem. Please try these steps for yourself on a clean system.* + +1. +2. +3. + +## Specifications + +- Version: *Which version of this software?* +- Platform: *Which operating system, which other relevant software versions?* + +## Possible fixes + +*Do you have ideas how the issue can be resolved?* diff --git a/.gitlab/merge_request_templates/Default.md b/.gitlab/merge_request_templates/Default.md new file mode 100644 index 0000000000000000000000000000000000000000..35c6d01c5904289b77fc7f1de9419ef91a1510e9 --- /dev/null +++ b/.gitlab/merge_request_templates/Default.md @@ -0,0 +1,54 @@ +# Summary + +*Insert a meaningful description for this merge request here: What is the new/changed behavior? +Which bug has been fixed? Are there related issues?* + + +# Focus + +*Point the reviewer to the core of the code change. Where should they start reading? What should +they focus on (e.g. security, performance, maintainability, user-friendliness, compliance with the +specs, finding more corner cases, concrete questions)?* + + +# Test Environment + +*How to set up a test environment for manual testing?* + + +# Check List for the Author + +Please, prepare your MR for a review. Be sure to write a summary and a focus and create gitlab +comments for the reviewer. They should guide the reviewer through the changes, explain your changes +and also point out open questions. For further good practices have a look at [our review +guidelines](https://gitlab.com/caosdb/caosdb/-/blob/dev/REVIEW_GUIDELINES.md) + +- [ ] All automated tests pass +- [ ] Reference related issues +- [ ] Up-to-date CHANGELOG.md (or not necessary) +- [ ] Up-to-date JSON schema (or not necessary) +- [ ] Appropriate user and developer documentation (or not necessary) + - How do I use the software? Assume "stupid" users. + - How do I develop or debug the software? Assume novice developers. +- [ ] Annotations in code (Gitlab comments) + - Intent of new code + - Problems with old code + - Why this implementation? + + +# Check List for the Reviewer + +- [ ] I understand the intent of this MR +- [ ] All automated tests pass +- [ ] Up-to-date CHANGELOG.md (or not necessary) +- [ ] Appropriate user and developer documentation (or not necessary) +- [ ] The test environment setup works and the intended behavior is reproducible in the test + environment +- [ ] In-code documentation and comments are up-to-date. +- [ ] Check: Are there specifications? Are they satisfied? + +For further good practices have a look at [our review guidelines](https://gitlab.com/caosdb/caosdb/-/blob/dev/REVIEW_GUIDELINES.md). + + +/assign me +/target_branch dev diff --git a/CHANGELOG.md b/CHANGELOG.md index e6f200e107cdee3aa2382c99166b921650a8eb5f..0c5decad3d8be81a49dccdd4a748dc667994c587 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,23 +5,59 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). -## [Unreleased] +## [Unreleased] ## -### Added +### Added ### -* SELECT query support for the GRPC API +### Changed ### -### Changed +* The default behavior of the query `FIND SomeName [...]` (as well as COUNT and SELECT) is being + made configurable and changes: + * `FIND SomeName` will be interpreted as `FIND <FIND_QUERY_DEFAULT_ROLE> + SomeName` from now on where `FIND_QUERY_DEFAULT_ROLE` is a newly introduced + server property. + * The new `FIND_QUERY_DEFAULT_ROLE` server property defaults to `RECORD` + which is why the behavior of the server api has a **breaking change**. + * The semantics of `FIND *` are affected as well. `FIND *` is equivalent to + `FIND <FIND_QUERY_DEFAULT_ROLE>`. + * Of course, administrators can choose to retain the old behavior by setting + `FIND_QUERY_DEFAULT_ROLE=ENTITY`. +* CQL now treats `WITH` and `WITH A` equivalently. Issue: [#192](https://gitlab.com/caosdb/caosdb-server/-/issues/192) +* The InsertFilesInDir FlagJob now creates File entities without a name. The previous behavior + caused severe performance problems for very large numbers of files. Issue: [#197](https://gitlab.com/caosdb/caosdb-server/-/issues/197) -### Deprecated +### Deprecated ### -### Removed +### Removed ### -### Fixed +### Fixed ### -### Security +* Denying a role permission has no effect + [#196](https://gitlab.com/caosdb/caosdb-server/-/issues/196). See security + notes below. +* Missing RecordType leads to unexpected server error + [#166](https://gitlab.com/caosdb/caosdb-server/-/issues/166) -### Documentation +### Security ### + +* Fixed [#196](https://gitlab.com/caosdb/caosdb-server/-/issues/196). This was + an error in the authorization procedure which allowed unprivileged users + execute insert, update or delete transactions on entities. However, the + unprivileged users would also need the correct entity permissions to do that. + + Without backup, this means possible data loss. Also there was the possibility + to spam the database by creating unwanted entities. + +### Documentation ### + +- Nested queries. +- Global entity permissions. + +## [0.9.0] - 2023-01-19 + +### Added + +* SELECT query support for the GRPC API ## [0.8.1] - 2022-11-07 (Timm Fitschen) diff --git a/CITATION.cff b/CITATION.cff new file mode 100644 index 0000000000000000000000000000000000000000..d6e80aaa203bf09b085514ff25aa5927b7965f32 --- /dev/null +++ b/CITATION.cff @@ -0,0 +1,28 @@ +cff-version: 1.2.0 +message: "If you use this software, please cite it as below." +authors: + - family-names: Fitschen + given-names: Timm + orcid: https://orcid.org/0000-0002-4022-432X + - family-names: Schlemmer + given-names: Alexander + orcid: https://orcid.org/0000-0003-4124-9649 + - family-names: Hornung + given-names: Daniel + orcid: https://orcid.org/0000-0002-7846-6375 + - family-names: tom Wörden + given-names: Henrik + orcid: https://orcid.org/0000-0002-5549-578X + - family-names: Spreckelsen + given-names: Florian + orcid: https://orcid.org/0000-0002-6856-2910 + - family-names: Parlitz + given-names: Ulrich + orcid: https://orcid.org/0000-0003-3058-1435 + - family-names: Luther + given-names: Stefan + orcid: https://orcid.org/0000-0001-7214-8125 +title: "CaosDB - Server" +version: 0.8.1 +doi: 10.3390/data4020083 +date-released: 2022-11-07 diff --git a/DEPENDENCIES.md b/DEPENDENCIES.md index 0f380b923c39129a7f29e380436857a3e9c6d8f6..fbcb186e45ff5bff20ca7ff20fa321f547048527 100644 --- a/DEPENDENCIES.md +++ b/DEPENDENCIES.md @@ -2,11 +2,12 @@ ## For Building and Running the Server -* `>=caosdb-proto 0.2.0` +* `>=caosdb-proto 0.3.0` * `>=caosdb-mysqlbackend 5.0.0` * `>=Java 11` * `>=Apache Maven 3.6.0` * `>=Make 4.2` +* `>=gcc 8` (if PAM authentication is required) * `libpam` (if PAM authentication is required) * More dependencies are being pulled and installed automatically by Maven. See the complete list of dependencies in the [pom.xml](pom.xml) diff --git a/FEATURES.md b/FEATURES.md index 485172f0dd8c930e36a7c216d0d8a9dd7e8086f3..2c046daee0518eb12e4de807a7ffe165c74d8664 100644 --- a/FEATURES.md +++ b/FEATURES.md @@ -1,6 +1,6 @@ # Features -* The CaosDB Server implements a CaosDB GRPC API Endpoint (v0.2.0) +* The CaosDB Server implements a CaosDB GRPC API Endpoint (v0.3.0) Authentication is supported via a Basic scheme, using the well-known "authentication" header. Notable limitations of the current implementation of the API: diff --git a/README_SETUP.md b/README_SETUP.md index ad4dc1bfd087d235aa903dee9183b78bc6ff3094..acb3a79ad256fbba6a0d0a69887bb7c7ba7b28b9 100644 --- a/README_SETUP.md +++ b/README_SETUP.md @@ -15,12 +15,18 @@ See [DEPENDENCIES.md](DEPENDENCIES.md). On Debian, the required packages can be installed with: - apt-get install git make mariadb-server maven openjdk-11-jdk-headless \ - python3-pip screen libpam0g-dev unzip + apt-get install make mariadb-server maven openjdk-11-jdk-headless \ + python3-pip libpam0g-dev unzip Note that installing MariaDB will uninstall existing MySQL packages and vice versa. +#### Install the requirements on Fedora + +On Fedora, the required packages can be installed with: + + sudo dnf install make pam-devel mariadb-server mariadb python3 java-17-openjdk-headless unzip gcc + ### System * `>=Linux 4.0.0`, `x86_64`, e.g. Ubuntu 18.04 @@ -160,20 +166,47 @@ sources (if you called `make run` previously). ## Setup Eclipse -1. Open Eclipse (recommended version: Oxygen.1a Release (4.7.1a)) -2. `File > New > Java Project`: Choose a project name and specify the location - of this repo. The JRE and Project layout should be configured automatically. - Now, the project should initially have two source-folders: `./src/main/java` - and `./src/test/java`. After a build, another one, - `./target/generated-sources/antlr4` should be generated. If there are more - than these three source-folders, reconfigure the projects source folders - appropriately with `Project > Properties > Java Build Path > Source`. -3. In the `Package Explorer` view, right-click on the project and `Configure > - Convert to Maven Project`. -4. In the `Package Explorer` view, right-click on the project and `Maven > - Update Project`. -5. Usually a build of the project is started automatically. Otherwise `Project > - Build Project`. +1. Open Eclipse (tested with 2022-R12) +2. File > Import > Maven > Existing Maven Projects: Specify location. +3. You will most likely encounter "Plugin execution not covered by lifecycle + configuration: ..." errors. Adapt the file + `<eclipse-workspace>/.metadata/.plugins/org.eclipse.m2e.core/lifecycle-mapping-metadata.xml`. + + Example: + + ```xml + <?xml version="1.0" encoding="UTF-8"?> + <lifecycleMappingMetadata> + <pluginExecutions> + <pluginExecution> + <pluginExecutionFilter> + <groupId>com.coveo</groupId> + <artifactId>fmt-maven-plugin</artifactId> + <versionRange>2.5.1</versionRange> + <goals> + <goal>format</goal> + </goals> + </pluginExecutionFilter> + <action> + <ignore /> + </action> + </pluginExecution> + <pluginExecution> + <pluginExecutionFilter> + <groupId>org.codehaus.mojo</groupId> + <artifactId>buildnumber-maven-plugin</artifactId> + <versionRange>1.4</versionRange> + <goals> + <goal>create-metadata</goal> + </goals> + </pluginExecutionFilter> + <action> + <ignore /> + </action> + </pluginExecution> + </pluginExecutions> + </lifecycleMappingMetadata> + ``` Done! @@ -211,7 +244,7 @@ Stand-alone documentation is built using Sphinx: `make doc` (`l_` not found): ```sh -git clone git@github.com:simgrid/javasphinx.git +git clone https://github.com/simgrid/javasphinx.git cd javasphinx git checkout 659209069603a pip3 install . diff --git a/RELEASE_GUIDELINES.md b/RELEASE_GUIDELINES.md index 58484d51565a87d98929a37e842c7040fc08cabf..1b5ae48906d4e8328064ecaf7e7c4f7a4a2bd20f 100644 --- a/RELEASE_GUIDELINES.md +++ b/RELEASE_GUIDELINES.md @@ -22,6 +22,7 @@ guidelines of the CaosDB Project * [pom.xml](./pom.xml) (probably this means to remove the `-SNAPSHOT`) * `src/doc/conf.py` * `CHANGELOG.md` + * `CITATION.cff` (update version and date) 5. Merge the release branch into the main branch. diff --git a/caosdb-proto b/caosdb-proto index bbab921f9aff5ac1945b299973e2b2acac75128a..c6405e538c179d2a8af952f85d9e9dc51fbadb92 160000 --- a/caosdb-proto +++ b/caosdb-proto @@ -1 +1 @@ -Subproject commit bbab921f9aff5ac1945b299973e2b2acac75128a +Subproject commit c6405e538c179d2a8af952f85d9e9dc51fbadb92 diff --git a/caosdb-webui b/caosdb-webui index bc199886b78a5dd39eb3bccd3a1451c8ab1e7552..421b2dce5199a5a2c96bcc638543c1fd51d48870 160000 --- a/caosdb-webui +++ b/caosdb-webui @@ -1 +1 @@ -Subproject commit bc199886b78a5dd39eb3bccd3a1451c8ab1e7552 +Subproject commit 421b2dce5199a5a2c96bcc638543c1fd51d48870 diff --git a/conf/core/server.conf b/conf/core/server.conf index ed9e1f1e16ade9a5c69ee40bf76502d799e0a50c..864460395dfe92c3078aab8c331be1c4e604916a 100644 --- a/conf/core/server.conf +++ b/conf/core/server.conf @@ -64,7 +64,7 @@ MYSQL_DATABASE_NAME=caosdb # User name for connecting to mysql MYSQL_USER_NAME=caosdb # Password for the user -MYSQL_USER_PASSWORD=caosdb +MYSQL_USER_PASSWORD=random1234 # Schema of mysql procedures and tables which is required by this CaosDB instance MYSQL_SCHEMA_VERSION=v6.0-SNAPSHOT @@ -204,6 +204,15 @@ USER_NAME_INVALID_MESSAGE=User names must have a length from 1 to 32 characters. PASSWORD_VALID_REGEX=^((?=.*\\d)(?=.*[a-z])(?=.*[A-Z])(?=.*[\\p{Punct}]).{8,128})$ PASSWORD_INVALID_MESSAGE=Passwords must have a length from 8 to 128 characters. THe must contain at least one [A-Z], one [a-z], one [0-9] and a special character e.g. [!§$%&/()=?.;,:#+*+~]. +# -------------------------------------------------- +# Query Settings +# -------------------------------------------------- + +# FIND blablabla is short for FIND $FIND_QUERY_DEFAULT_ROLE blablabla. Should be +# either ENTITY or RECORD. RECORDTYPE, FILE, and PROPERTY will work as well but +# are rather unexpected for human users. +FIND_QUERY_DEFAULT_ROLE=RECORD + # -------------------------------------------------- # Extensions # -------------------------------------------------- @@ -213,4 +222,4 @@ ENTITY_VERSIONING_ENABLED=true # Enabling the state machine extension -# EXT_STATE_ENTITY=ENABLE +# EXT_STATE_ENTITY=ENABLE \ No newline at end of file diff --git a/doc/Query.md b/doc/Query.md index 1d7748438cd7f1a4a84f0ab66215564cbc5ad36d..1a875bf3008be0ef2b078ccae2b713d533acfaf4 100644 --- a/doc/Query.md +++ b/doc/Query.md @@ -219,6 +219,12 @@ The following query returns entities which have a _pname1_ property with any val `FIND ename WITH pname1` +`FIND ename WITH A pname1` + +`FIND ename WITH A PROPERTY pname1` + +`FIND ename WITH PROPERTY pname1` + `FIND ename . pname1` `FIND ename.pname1` @@ -338,7 +344,7 @@ Any result set can be filtered by logically combining POV filters or back refere * NOT:: The logical negation. Equivalent expressions: `NOT, DOESN'T HAVE A PROPERTY, DOES NOT HAVE A PROPERTY, DOESN'T HAVE A, DOES NOT HAVE A, DOES NOT, DOESN'T, IS NOT, ISN'T, !` * OR:: The logical _or_. Equivalent expressions: `OR, |` * RECORD,RECORDTYPE,FILE,PROPERTY:: Role expression for restricting the result set to a specific role. -* WHICH:: The marker for the beginning of the filters. Equivalent expressions: `WHICH, WHICH HAS A, WHICH HAS A PROPERTY, WHERE, WITH, .` +* WHICH:: The marker for the beginning of the filters. Equivalent expressions: `WHICH, WHICH HAS A, WHICH HAS A PROPERTY, WHERE, WITH (A), .` * REFERENCE:: This one is tricky: `REFERENCE TO` expresses a the state of _having_ a reference property. `REFERENCED BY` expresses the state of _being_ referenced by another entity. * COUNT:: `COUNT` works like `FIND` but doesn't return the entities. diff --git a/pom.xml b/pom.xml index c76d397b8fc1e655f1cf0fbf675e3b549eea876a..b251820a27733901e47012ecc9b9a4fb682b0229 100644 --- a/pom.xml +++ b/pom.xml @@ -25,7 +25,7 @@ <modelVersion>4.0.0</modelVersion> <groupId>org.caosdb</groupId> <artifactId>caosdb-server</artifactId> - <version>0.9.0-SNAPSHOT</version> + <version>0.9.1-SNAPSHOT</version> <packaging>jar</packaging> <name>CaosDB Server</name> <scm> diff --git a/src/doc/CaosDB-Query-Language.md b/src/doc/CaosDB-Query-Language.md index ffa889c6b8acd0cd635049ed2aaceb748d56f2a9..a34191d051fec9907a6cbb47f11f97d60ac972ac 100644 --- a/src/doc/CaosDB-Query-Language.md +++ b/src/doc/CaosDB-Query-Language.md @@ -1,49 +1,66 @@ -# CaosDB Query Language -**WIP This is going to be the specification. CQL tutorials are in the webui** +# CaosDB Query Language Examples -## Example queries +See syntax specification in [CaosDB Query Language Syntax](query-syntax). -### Simple FIND Query -The following query will return any entity which has the name _ename_ and all its children. -`FIND ename` +## Simple FIND Query -The following queries are equivalent and will return any entity which has the name _ename_ and all its children, but only if they are genuin records. Of course, the returned set of entities (henceforth referred to as _resultset_) can also be restricted to recordtypes, properties and files. +The following query will return any record which has the name _somename_ and all +record children of any entity with that name. -`FIND RECORD ename` +`FIND somename` -`FIND RECORDS ename` +On server in the default configuration, the following queries are equivalent to this one. -Wildcards use `*` for any characters or none at all. Wildcards for single characters (like the '_' wildcard from mysql) are not implemented yet. +`FIND RECORD somename` -`FIND RECORD en*` returns any entity which has a name beginning with _en_. +`FIND RECORDS somename` -Regular expressions must be surrounded by _<<_ and '>>': +Of course, the returned set of entities (henceforth referred to as _resultset_) can also be +restricted to RecordTypes (`FIND RECORDTYPE ...`), Properties (`FIND PROPERTY ...`) and Files (`FIND +FILE ...`). -`FIND RECORD <<e[aemn]{2,5}>>` +You can include all entities (Records, RecordTypes, Properties, ...) into the results by using the +`ENTITY` keyword: -`FIND RECORD <<[cC]am_[0-9]*>>` +`FIND ENTITY somename` -*TODO* (Timm): -Describe escape sequences like `\\ `, `\*`, `\<<` and `\>>`. +Wildcards use `*` for any characters or none at all. Wildcards for single characters (like the `_` wildcard from mysql) are not implemented yet. -Currently, wildcards and regular expressions are only available for the _simple-find-part_ of the query, i. e. no wildcards/regexps for filters. +`FIND en*` returns any record which has a name beginning with _en_. -### Simple COUNT Query +Regular expressions must be surrounded by _<<_ and _>>_: -This query counts entities which have certain properties. +`FIND <<e[aemn]{2,5}>>` -`COUNT ename` -will return the number of entities which have the name _ename_ and all their children. +`FIND <<[cC]amera_[0-9]*>>` -The syntax of the COUNT queries is equivalent to the FIND queries in any respect (this also applies to wildcards and regular expressions) but one: The prefix is to be `COUNT` instead of `FIND`. +*TODO*: +Describe escape sequences like `\\\\ `, `\*`, `\<<` and `\>>`. -Unlike the FIND queries, the COUNT queries do not return any entities. The result of the query is the number of entities which _would be_ returned if the query was a FIND query. +Currently, wildcards and regular expressions are only available for the _simple-find-part_ of the +query, i.e. not for property-operator-value filters (see below). + +## Simple COUNT Query + +COUNT queries count entities which have certain properties. + +`COUNT ... rname ...` + +will return the number of records which have the name _rname_ and all record +children of any entity with that name. + +The syntax of the COUNT queries is equivalent to the FIND queries in any +respect (this also applies to wildcards and regular expressions) but one: The +prefix is to be `COUNT` instead of `FIND`. + +Unlike the FIND queries, the COUNT queries do not return any entities. The result of the query is +the number of entities which _would be_ returned if the query was a FIND query. ## Filters ### POV - Property-Operator-Value -The following queries are equivalent and will restrict the result set to entities which have a property named _pname1_ that has a value _val1_. +The following queries are equivalent and will restrict the result set to records which have a property named _pname1_ that has a value _val1_. `FIND ename.pname1=val1` @@ -53,31 +70,33 @@ The following queries are equivalent and will restrict the result set to entitie `FIND ename WHICH HAS A pname1=val1` -Again, the resultset can be restricted to records: +Again, the resultset can be restricted to any other entity role as well: -`FIND RECORD ename WHICH HAS A pname1=val1` +`FIND RECORDTYPE ename WHICH HAS A pname1=val1` _currently known operators:_ `=, !=, <=, <, >=, >` (and cf. next paragraphes!) #### Special Operator: LIKE -The _LIKE_ can be used with wildcards. The `*` is a wildcard for any (possibly empty) sequence of characters. Examples: +The _LIKE_ can be used with wildcards. The `*` is a wildcard for any (possibly empty) sequence of characters. + +Examples: -`FIND RECORD ename WHICH HAS A pname1 LIKE va*` +`FIND ename WHICH HAS A pname1 LIKE va*` -`FIND RECORD ename WHICH HAS A pname1 LIKE va*1` +`FIND ename WHICH HAS A pname1 LIKE va*1` -`FIND RECORD ename WHICH HAS A pname1 LIKE *al1` +`FIND ename WHICH HAS A pname1 LIKE *al1` -_Note:_ The _LIKE_ operator is will only produce expectable results with text properties. +_Note:_ The _LIKE_ operator will only produce expectable results with text properties. #### Special Case: References -In general a reference can be addressed just like a POV filter. So +In general a reference can be addressed just like a POV filter. So -`FIND ename1.pname1=ename2` +`FIND ename1 WITH pname1=ename2` -will also return any entity named _ename1_ which references the entity with name or id _ename2_ via a reference property named _pname1_. However, it will also return any entity with a text property of that name with the string value _ename2_. In order to restrict the result set to reference properties one may make use of special reference operators: +will also return any record named _ename1_ which references the entity with name or id _ename2_ via a reference property named _pname1_. However, it will also return any entity with a text property of that name with the string value _ename2_. In order to restrict the result set to reference properties one may make use of special reference operators: _reference operators:_ `->, REFERENCES, REFERENCE TO` @@ -88,12 +107,12 @@ The query looks like this: `FIND ename1 WHICH HAS A pname1->ename2` -#### Time Special Case: DateTime +#### Special Case: DateTime _DateTime operators:_ `=, !=, <, >, IN, NOT IN` -##### `d1=d2`: Equivalence relation. -* ''True'' iff d1 and d2 are equal in every respect (same DateTime flavor, same fields are defined/undefined and all defined fields are equal respectively). +##### `d1=d2`: Equivalence relation. +* ''True'' iff d1 and d2 are equal in every respect (same DateTime flavor, same fields are defined/undefined and all defined fields are equal respectively). * ''False'' iff they have the same DateTime flavor but have different fields defined or fields with differing values. * ''Undefined'' otherwise. @@ -103,11 +122,11 @@ Examples: * `2015-04-03T00:00:00.0=2015-04-03T00:00:00.0` is true. * `2015-04-03T00:00:00=2015-04-03T00:00:00` is true. * `2015-04=2015-05` is false. -* `2015-04=2015-04` is true. +* `2015-04=2015-04` is true. -##### `d1!=d2`: Intransitive, symmetric relation. -* ''True'' iff `d1=d2` is false. -* ''False'' iff `d1=d2` is true. +##### `d1!=d2`: Intransitive, symmetric relation. +* ''True'' iff `d1=d2` is false. +* ''False'' iff `d1=d2` is true. * ''Undefined'' otherwise. Examples: @@ -116,15 +135,15 @@ Examples: * `2015-04-03T00:00:00.0!=2015-04-03T00:00:00.0` is false. * `2015-04-03T00:00:00!=2015-04-03T00:00:00` is false. * `2015-04!=2015-05` is true. -* `2015-04!=2015-04` is false. +* `2015-04!=2015-04` is false ##### `d1>d2`: Transitive, non-symmetric relation. Semantics depend on the flavors of d1 and d2. If both are... -###### [UTCDateTime](Datatype#datetime) +###### [UTCDateTime](specification/Datatype.html#datetime) * ''True'' iff the time of d1 is after the the time of d2 according to [UTC](https://en.wikipedia.org/wiki/Coordinated_Universal_Time). * ''False'' otherwise. -###### [SemiCompleteDateTime](Datatype#datetime) +###### [SemiCompleteDateTime](specification/Datatype.html#datetime) * ''True'' iff `d1.ILB>d2.EUB` is true or `d1.ILB=d2.EUB` is true. * ''False'' iff `d1.EUB<d2.ILB}} is true or {{{d1.EUB=d2.ILB` is true. * ''Undefined'' otherwise. @@ -142,11 +161,11 @@ Examples: ##### `d1<d2`: Transitive, non-symmetric relation. Semantics depend on the flavors of d1 and d2. If both are... -###### [UTCDateTime](Datatype#datetime) +###### [UTCDateTime](specification/Datatype.html#datetime) * ''True'' iff the time of d1 is before the the time of d2 according to [UTC](https://en.wikipedia.org/wiki/Coordinated_Universal_Time) * ''False'' otherwise. -###### [SemiCompleteDateTime](Datatype#datetime) +###### [SemiCompleteDateTime](specification/Datatype.html#datetime) * ''True'' iff `d1.EUB<d2.ILB` is true or `d1.EUB=d2.ILB` is true. * ''False'' iff `d1.ILB>d2.EUB}} is true or {{{d1.ILB=d2.EUB` is true. * ''Undefined'' otherwise. @@ -164,7 +183,7 @@ Examples: ##### `d1 IN d2`: Transitive, non-symmetric relation. Semantics depend on the flavors of d1 and d2. If both are... -###### [SemiCompleteDateTime](Datatype#datetime) +###### [SemiCompleteDateTime](specification/Datatype.html#datetime) * ''True'' iff (`d1.ILB>d2.ILB` is true or `d1.ILB=d2.ILB` is true) and (`d1.EUB<d2.EUB` is true or `d1.EUB=d2.EUB` is true). * ''False'' otherwise. @@ -175,9 +194,9 @@ Examples: * `2015 IN 2015-01-01` is false. * `2015-01-01 IN 2015-01-01T20:15:30` is false. -##### `d1 NOT IN d2`: Non-symmetric relation. +##### `d1 NOT IN d2`: Transitive, non-symmetric relation. Semantics depend on the flavors of d1 and d2. If both are... -###### [SemiCompleteDateTime](Datatype#datetime) +###### [SemiCompleteDateTime](specification/Datatype.html#datetime) * ''True'' iff `d1.ILB IN d2.ILB` is false. * ''False'' otherwise. @@ -193,7 +212,7 @@ These semantics follow a three-valued logic with ''true'', ''false'' and ''undef #### Omitting the Property or the Value -One doesn't have to specify the property or the value at all. The following query filters the result set for entities which have any property with a value greater than _val1_. +One doesn't have to specify the property or the value at all. The following query filters the result set for records which have any property with a value greater than _val1_. `FIND ename WHICH HAS A PROPERTY > val1` @@ -213,7 +232,7 @@ And for references... `FIND ename1.->ename2` -The following query returns entities which have a _pname1_ property with any value. +The following query returns records which have a _pname1_ property with any value. `FIND ename WHICH HAS A PROPERTY pname1` @@ -227,22 +246,17 @@ The following query returns entities which have a _pname1_ property with any val ### TransactionFilter +`FIND ename WHICH (sugar|negated_sugar)? (NOT)? (CREATED|INSERTED|UPDATED) (by_clause time_clause?| time_clause by_clause?)` + *Definition* - sugar:: `HAS BEEN` | `HAVE BEEN` | `HAD BEEN` | `WAS` | `IS` | + sugar:: `HAS BEEN` | `HAVE BEEN` | `HAD BEEN` | `WAS` | `IS` - negated_sugar:: `HAS NOT BEEN` | `HASN'T BEEN` | `WAS NOT` | `WASN'T` | `IS NOT` | `ISN'T` | `HAVN'T BEEN` | -`HAVE NOT BEEN` | `HADN'T BEEN` | `HAD NOT BEEN` + negated_sugar:: `HAS NOT BEEN` | `HASN'T BEEN` | `WAS NOT` | `WASN'T` | `IS NOT` | `ISN'T` | `HAVN'T BEEN` | `HAVE NOT BEEN` | `HADN'T BEEN` | `HAD NOT BEEN` by_clause:: `BY (ME | username | SOMEONE ELSE (BUT ME)? | SOMEONE ELSE BUT username)` - - date:: A date string of the form `YYYY-MM-DD` - - datetime:: A datetime string of the form `YYYY-MM-DD hh:mm:ss` - - time_clause:: `ON ($date|$datetime) ` Here is plenty of room for more syntactic sugar, e.g. a `TODAY` keyword, and more funcionality, e.g. ranges. - -`FIND ename WHICH ($sugar|$negated_sugar)? (NOT)? (CREATED|INSERTED|UPDATED|DELETED) (by_clause time_clause?| time_clause by_clause?)` + datetime:: A datetime string of the form `YYYY[-MM[-DD(T| )[hh[:mm[:ss[.nnn][(+|-)zzzz]]]]]]` or `TODAY`. + time_clause:: `[AT|ON|IN|BEFORE|AFTER|UNTIL|SINCE] (datetime) ` *Examples* @@ -252,12 +266,13 @@ The following query returns entities which have a _pname1_ property with any val `FIND ename WHICH HAS BEEN CREATED BY erwin ON 2014-12-24` -`FIND ename WHICH HAS BEEN CREATED BY SOMEONE ELSE BUT erwin ON 2014-12-24` +`FIND ename WHICH HAS BEEN UPDATED BY SOMEONE ELSE BUT erwin ON 2014-12-24` -`FIND ename WHICH HAS BEEN CREATED BY erwin` +`FIND ename WHICH HAS BEEN INSERTED BY erwin` -`FIND ename . CREATED BY erwin ON ` +`FIND ename WHICH HAS BEEN INSERTED SINCE 2021-04` +Note that `SINCE` and `UNTIL` are inclusive, while `BEFORE` and `AFTER` are not. ### File Location @@ -297,7 +312,7 @@ Find any file in a directory which begins with `2016-02`: ### Back References -The back reference filters for entities that are referenced by another entity. The following query returns entities of the type _ename1_ which are referenced by _ename2_ entities via the reference property _pname1_. +The back reference filters for entities that are referenced by another entity. The following query returns records of the type _ename1_ which are referenced by _ename2_ entities via the reference property _pname1_. * `FIND ename1 WHICH IS REFERENCED BY ename2 AS A pname1` * `FIND ename1 WITH @ ename2 / pname1` @@ -310,6 +325,13 @@ One may omit the property specification: * `FIND ename1 WITH @ ename2` * `FIND ename1 . @ ename2` +### Nested queries, or filtering by sub-properties ### + +Nested queries can easily be searched by simply concatenating `WHICH` or `WITH` expressions: + +* `FIND ename WHICH HAS A pname WHICH HAS A subpname=val` +* For example: `FIND AN experiment WHICH HAS A camera WHICH HAS A 'serial number'= 1234567890` + ### Combining Filters with Propositional Logic Any result set can be filtered by logically combining POV filters or back reference filters: @@ -356,7 +378,7 @@ Any result set can be filtered by logically combining POV filters or back refere In contrast to `FIND` queries, which always return the complete entity, there are `SELECT` queries which only return the entity with only those properties which are specified in the query. The syntax is very similar to `FIND` queries - just replace the `FIND` by `SELECT <comma separated list of selectors> FROM`: -`SELECT p1, p2, p3 FROM Record ename` +`SELECT p1, p2, p3 FROM ename` However, the `SELECT` query can also return properties of referenced entities and thereby are a means of joining entities together and return a custom view or projection: @@ -391,14 +413,22 @@ Since Caosdb 0.2 entities are optionally version controlled. The query language * The `ANY VERSION OF` modifier currently the only expression for taking the versioning into account when using the query language. * Subproperties are not supported yet, e.g. `FIND ANY VERSION OF ENTITY WHICH IS REFERENCED BY ename WITH ...`. This applies to all cases where you specify properties of *referenced* entities or *referencing* entities. -### Future +## Configuration -* Add `(LATEST|LAST|OLDEST|NEWEST|FIRST) VERSION OF` modifiers. -* Add `(ANY|LATEST|LAST|OLDEST|NEWEST|FIRST) VERSION (BEFORE|AFTER) (<timestamp>|<transaction id>|<entity@version>) OF` modifier. -* Add support for subproperties, e.g. `FIND ANY VERSION OF ENTITY WHICH IS REFERENCED BY ename WITH ...`. +In CaosDB Server implementations before version 0.9.0, the `FIND ename` query +would return any entity with that name and all children, regardless of the +entity's role. Basically, `FIND ename` *was* equivalent to `FIND ENTITY ename`. +Since 0.9.0 the default behavior has changed and now `FIND ename` is equivalent +to `FIND RECORD ename`. This default is, however, configurable via the +`FIND_QUERY_DEFAULT_ROLE` server property. See [Server Configuration](./administration/configuration.rst). ## Future - * *Sub Queries* (or *Sub Properties*): `FIND ename WHICH HAS A pname WHICH HAS A subpname=val`. This is like: `FIND AN experiment WHICH HAS A camera WHICH HAS A 'serial number'= 1234567890` - * *More Logic*, especially `ANY`, `ALL`, `NONE`, and `SUCH THAT` key words (and equivalents) for logical quantisation: `FIND ename1 SUCH THAT ALL ename2 WHICH HAVE A REFERENCE TO ename1 HAVE A pname=val`. This is like `FIND experiment SUCH THAT ALL person WHICH ARE REFERENCED BY THIS experiment AS conductor HAVE AN 'academic title'=professor.` - +* Additional versioning queries: + * Add `(LATEST|LAST|OLDEST|NEWEST|FIRST) VERSION OF` modifiers. + * Add `(ANY|LATEST|LAST|OLDEST|NEWEST|FIRST) VERSION (BEFORE|AFTER) (<timestamp>|<transaction + id>|<entity@version>) OF` modifier. + * Add support for subproperties, e.g. `FIND ANY VERSION OF ENTITY WHICH IS REFERENCED BY ename + WITH ...`. + * Find deleted entities: `FIND ename WHICH WAS DELETED (BY ME | ON 2014-12-24)` +* *More Logic*, especially `ANY`, `ALL`, `NONE`, and `SUCH THAT` key words (and equivalents) for logical quantisation: `FIND ename1 SUCH THAT ALL ename2 WHICH HAVE A REFERENCE TO ename1 HAVE A pname=val`. This is like `FIND experiment SUCH THAT ALL person WHICH ARE REFERENCED BY THIS experiment AS conductor HAVE AN 'academic title'=professor.` diff --git a/src/doc/conf.py b/src/doc/conf.py index 02576662d8d15acec8d5cacef2126e6b0102339b..700a009370e0925a209efe66da04f18324beea69 100644 --- a/src/doc/conf.py +++ b/src/doc/conf.py @@ -17,6 +17,7 @@ # sys.path.insert(0, os.path.abspath('../caosdb')) import sphinx_rtd_theme +from os.path import dirname, abspath # -- Project information ----------------------------------------------------- @@ -25,9 +26,9 @@ copyright = '2022, IndiScale GmbH' author = 'Daniel Hornung, Timm Fitschen' # The short X.Y version -version = '0.9.0' +version = '0.9.1' # The full version, including alpha/beta/rc tags -release = '0.9.0-SNAPSHOT' +release = '0.9.1-SNAPSHOT' # -- General configuration --------------------------------------------------- @@ -48,6 +49,7 @@ extensions = [ "sphinx.ext.autosectionlabel", # Allow reference sections using its title "sphinx_rtd_theme", "sphinxcontrib.plantuml", # PlantUML diagrams + "sphinx_a4doc", # antl4 ] # Add any paths that contain templates here, relative to this directory. @@ -223,3 +225,7 @@ autodoc_default_options = { # -- Options for autosectionlabel -------------------------------------------- autosectionlabel_prefix_document = True + +# -- Options for sphinx_a4doc ------------------------------------------------ + +a4_base_path = abspath(dirname(__file__) + '/../main/java/org/caosdb/server/query') diff --git a/src/doc/index.rst b/src/doc/index.rst index b5ce9f3235277b613b357dca5dfc3334d80aeb3f..1a5e4134ef7f934ff5018cc8847603f1165ab16e 100644 --- a/src/doc/index.rst +++ b/src/doc/index.rst @@ -18,7 +18,7 @@ Welcome to caosdb-server's documentation! Changelog <CHANGELOG> specification/index.rst Glossary - API documentation<_apidoc/packages> + Server Internals<_apidoc/packages> Welcome to the CaosDB, the flexible semantic data management toolkit! diff --git a/src/doc/permissions.rst b/src/doc/permissions.rst index 6b07f48fa469e81069b2f01559957fb76ef27f5a..3d9c5ed7c26349ca0a20dec4e89b8889d4e10bf1 100644 --- a/src/doc/permissions.rst +++ b/src/doc/permissions.rst @@ -124,8 +124,8 @@ How to set permissions ---------------------- There are multiple ways to set role and entity permissions. The most -common and best tested way currently is to set global default entity permissions -in the ``global_entity_permissions.xml`` config file, and role-based role +common and best tested way currently is to set global default *entity* permissions +in the ``global_entity_permissions.xml`` config file, and role-based *role* permissions with the ``caosdb_admin.py`` `utility script <https://gitlab.com/caosdb/caosdb-pylib/-/blob/main/src/caosdb/utils/caosdb_admin.py>`__ of CaosDB's Python library which is also used to `manage users and @@ -138,8 +138,8 @@ find a more detailed description of the possible ways of setting permissions. you can set the default permissions that every entity on the server has. The global default permissions can **only** be set in this file; all other ways below can only change the permissions of individual entities. Note that you - can add more rules but you can never remove rules set in the - ``global_entity_permissions.xml``. Thus, it might not be possible to overrule + can add more rules in the ``global_entity_permissions.xml``, but you can not remove rules by + writing to this file. Thus, it might not be possible to overrule permissions defined here (see :ref:`Permission calculation<Calculation>`). Note also that, as the name suggests, only :ref:`entity permissions<entity-permissions>` can be set this way. The diff --git a/src/doc/query-syntax.rst b/src/doc/query-syntax.rst new file mode 100644 index 0000000000000000000000000000000000000000..00df440fb9cf2a175e371c3038b164643d89a84a --- /dev/null +++ b/src/doc/query-syntax.rst @@ -0,0 +1,15 @@ +CaosDB Query Language Syntax +============================ + +This is the documentation of the CaosDB Query Language Syntax. The +authoritative specification of the syntax is the ANTLR4 grammar you can find in +the source code: +`CQLParser.g4 <https://gitlab.com/caosdb/caosdb-server/-/blob/main/src/main/java/org/caosdb/server/query/CQLParser.g4>`__ +and +`CQLLexer.g4 <https://https://gitlab.com/caosdb/caosdb-server/-/blob/main/src/main/java/org/caosdb/server/query/CQLLexer.g4>`__ + +See examples in :doc:`Query Language<CaosDB-Query-Language>`. + +.. a4:autogrammar:: CQLParser.g4 + +.. a4:autogrammar:: CQLLexer.g4 diff --git a/src/doc/roles.md b/src/doc/roles.md index 138c75d37df7b883d1e494a438283f1a610f07c3..838b3d9ec92626676c2ca16535496620bedd5842 100644 --- a/src/doc/roles.md +++ b/src/doc/roles.md @@ -10,7 +10,7 @@ users may have the same role, and there may be roles without any users. The user and their roles are always returned by the server in answers to requests and can thus be interpreted and used by clients. The most important use though -is [permission](permissions.rst) checking in the server: Access and +is [permission](permissions) checking in the server: Access and modification of entities can be controlled via roles, so that users of a given role are allowed or denied certain actions. Incidentally, the permission to edit the permissions @@ -32,4 +32,4 @@ There are some special roles, which are automatically assigned to users: Except for the `anonymous` role, these special roles are not returned by the server, but can nevertheless be used to define -[permissions](permissions.rst). +[permissions](permissions.html#permissions). diff --git a/src/doc/specification/AbstractProperty.md b/src/doc/specification/AbstractProperty.md index dee88e665c1609224bdab185ea147b91c49dc8f3..3a2fe7583480c7467bccb0a2bf94850ea30d7be4 100644 --- a/src/doc/specification/AbstractProperty.md +++ b/src/doc/specification/AbstractProperty.md @@ -132,14 +132,13 @@ Any xml representation of an `AbstractProperty` that is to be posted to the Caos #### Get all *Request:* - GET http://localhost:8122/mpidsserver/AbstractProperty/ - GET http://localhost:8122/mpidsserver/AbstractProperty + GET http://localhost:8122/server/AbstractProperty/ + GET http://localhost:8122/server/AbstractProperty *Response:* <?xml version="1.0" encoding="UTF-8"?> <Response /> -[View Ticket #2](http://dev.bmp.ds.mpg.de/heartdb/ticket/2) #### Erroneous Requests ##### Non-existing *Request:* diff --git a/src/doc/specification/Datatype.md b/src/doc/specification/Datatype.md index ffc9794ea36cfe748e8c11cee2bcd32adc8ebe0a..6a169042dce2be2e6dc939d0935f3336de264308 100644 --- a/src/doc/specification/Datatype.md +++ b/src/doc/specification/Datatype.md @@ -43,7 +43,7 @@ ---- ## DATETIME -The DateTime data type exists in (currently) three flavors which are dynamically chosen during parsing on the the serverside. The flavors have different ranges, support of time zones and intended use cases. Only the first two flavors are actually implemented for storage and queries. The third one is implemented for queries exclusively. +The DateTime data type exists in (currently) three flavors which are dynamically chosen during parsing on the serverside. The flavors have different ranges, support of time zones and intended use cases. Only the first two flavors are actually implemented for storage and queries. The third one is implemented for queries exclusively. ### UTCDateTime * Description: This DATETIME flavor stores values which represent a single point of time according to [UTC](https://en.wikipedia.org/wiki/Coordinated_Universal_Time) with the format specified by [ISO 8601](https://en.wikipedia.org/wiki/ISO_8601) (Combined date and time). It does support [UTC Leap Seconds](https://en.wikipedia.org/wiki/Leap_second) and time zones. @@ -55,7 +55,7 @@ The DateTime data type exists in (currently) three flavors which are dynamically * It is allowed to ommit the nanosecond part of a UTCDateTime (`2016-01-01T13:23:00CEST`). This indicates a precision of seconds for a UTCDateTime value. ### Date - Description:: This DATETIME flavor stores values which represent a single date, month or year according to the [gregorian calendar](https://en.wikipedia.org/wiki/Gregorian_Calendar). A month/year is conceived as a single date with the presion of a month/year. This concept is useful if you try to understand the query semantics which are explained [elsewhere](./QueryLanguage#POVDateTime). + Description:: This DATETIME flavor stores values which represent a single date, month or year according to the [gregorian calendar](https://en.wikipedia.org/wiki/Gregorian_Calendar). A month/year is conceived as a single date with the presion of a month/year. This concept is useful if you try to understand the query semantics which are explained [elsewhere](../CaosDB-Query-Language.html#pov-property-operator-value). Format:: `Y[YYY][-MM[-dd]]` (where square brackets mean that the expression is optional). Range:: Any valid date according to the gregorian calendar from `-9999-01-01` to `9999-12-31` (and respective dates with lower precision. E.g. the year `-9999`). There is no year `0`. * Note: Date is a specialization of [#SemiCompleteDateTime]. @@ -73,7 +73,7 @@ Please file a new feature request as soon as you need them. ---- ## REFERENCE -* Description: REFERENCE values store the [Valid ID](./Glossary#valid-id) of an existing entity. The are useful to establish links between two entities. +* Description: REFERENCE values store the [Valid ID](../Glossary#valid-id) of an existing entity. The are useful to establish links between two entities. * Accepted Values: Any [Valid ID](./Glossary#valid-id) or [Valid Unique Existing Name](./Glossary#valid-unique-existing-name) or [Valid Unique Temporary ID](./Glossary#valid-unique-temporary-id) or [Valid Unique Prospective Name](./Glossary#valid-unique-prospective-pame). * Note: * After beeing processed successfully by the server the REFERENCE value is normalized to a [Valid ID](./Glossary#valid-id). I.e. it is guaranteed that a REFERENCE value of a valid property is a positive integer. diff --git a/src/doc/specification/RecordType.rst b/src/doc/specification/RecordType.rst index 0c196fea86e74f1014e34c0bd1e5b096acd1a5d9..53e8e1613dddeea901e1b8404f186d67ebc2be23 100644 --- a/src/doc/specification/RecordType.rst +++ b/src/doc/specification/RecordType.rst @@ -190,8 +190,6 @@ Get all <?xml version="1.0" encoding="UTF-8"?> <Response /> -[View Ticket #2](http://caosdb.example.com/caosdb/ticket/2) - POST Requests ~~~~~~~~~~~~~ diff --git a/src/main/java/org/caosdb/server/ServerProperties.java b/src/main/java/org/caosdb/server/ServerProperties.java index 03ffadcc4f48d9d8b1f941de102c6d6c03abd6f8..3027940e5c8926d5c2b6bd64430e52b42f8685e0 100644 --- a/src/main/java/org/caosdb/server/ServerProperties.java +++ b/src/main/java/org/caosdb/server/ServerProperties.java @@ -31,6 +31,7 @@ import java.util.Arrays; import java.util.Collections; import java.util.Comparator; import java.util.Properties; +import org.caosdb.server.query.Query; import org.caosdb.server.utils.AbstractObservable; import org.caosdb.server.utils.Observable; import org.caosdb.server.utils.Observer; @@ -101,6 +102,7 @@ public class ServerProperties extends Properties implements Observable { public static final String KEY_QUERY_FILTER_ENTITIES_WITHOUT_RETRIEVE_PERMISSIONS = "QUERY_FILTER_ENTITIES_WITHOUT_RETRIEVE_PERMISSIONS"; + public static final String KEY_FIND_QUERY_DEFAULT_ROLE = "FIND_QUERY_DEFAULT_ROLE"; public static final String KEY_SERVER_NAME = "SERVER_NAME"; @@ -186,9 +188,25 @@ public class ServerProperties extends Properties implements Observable { logger.info(name + "=" + val); } } + + sanityCheck(serverProperties); + return serverProperties; } + private static void sanityCheck(ServerProperties serverProperties) { + try { + Query.Role.valueOf(serverProperties.getProperty(KEY_FIND_QUERY_DEFAULT_ROLE)); + } catch (IllegalArgumentException e) { + logger.error( + "Unsupported value for server property " + + KEY_FIND_QUERY_DEFAULT_ROLE + + ": " + + serverProperties.getProperty(KEY_FIND_QUERY_DEFAULT_ROLE)); + System.exit(1); + } + } + private static void loadConfigFile(final Properties serverProperties, final File confFile) throws IOException { if (confFile.exists() && confFile.isFile()) { diff --git a/src/main/java/org/caosdb/server/jobs/Job.java b/src/main/java/org/caosdb/server/jobs/Job.java index 0e9d7ef79377884853caa2198c84d428aed38e01..9da74edd0db86f6d81a8636020afcdaf3faa9e09 100644 --- a/src/main/java/org/caosdb/server/jobs/Job.java +++ b/src/main/java/org/caosdb/server/jobs/Job.java @@ -555,7 +555,7 @@ public abstract class Job { * * @param entity the entity to be resolved. * @return the resolved entity. - * @throws EntityWasNotUniqueException if the resolution failed due to ambuiguity of the name. + * @throws EntityWasNotUniqueException if the resolution failed due to ambiguity of the name. */ protected EntityInterface resolve(final EntityInterface entity) throws EntityWasNotUniqueException { diff --git a/src/main/java/org/caosdb/server/jobs/core/AccessControl.java b/src/main/java/org/caosdb/server/jobs/core/AccessControl.java index 4217c55e1153dafc302630823873fc75640ca9be..ede7658580811671f19f955f36faa59db4df922f 100644 --- a/src/main/java/org/caosdb/server/jobs/core/AccessControl.java +++ b/src/main/java/org/caosdb/server/jobs/core/AccessControl.java @@ -1,9 +1,10 @@ /* - * ** header v3.0 * This file is a part of the CaosDB Project. * * Copyright (C) 2018 Research Group Biomedical Physics, * Max-Planck-Institute for Dynamics and Self-Organization Göttingen + * Copyright (C) 2023 Timm Fitschen <t.fitschen@indiscale.com> + * Copyright (C) 2023 IndiScale <info@indiscale.com> * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU Affero General Public License as @@ -17,17 +18,17 @@ * * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see <https://www.gnu.org/licenses/>. - * - * ** end header */ package org.caosdb.server.jobs.core; import org.apache.shiro.SecurityUtils; import org.apache.shiro.subject.Subject; import org.caosdb.server.accessControl.ACMPermissions; -import org.caosdb.server.database.backend.transaction.RetrieveSparseEntity; +import org.caosdb.server.entity.DeleteEntity; import org.caosdb.server.entity.EntityInterface; -import org.caosdb.server.entity.wrapper.Parent; +import org.caosdb.server.entity.InsertEntity; +import org.caosdb.server.entity.Role; +import org.caosdb.server.entity.UpdateEntity; import org.caosdb.server.jobs.ContainerJob; import org.caosdb.server.jobs.JobAnnotation; import org.caosdb.server.jobs.TransactionStage; @@ -35,6 +36,14 @@ import org.caosdb.server.transaction.Retrieve; import org.caosdb.server.utils.EntityStatus; import org.caosdb.server.utils.ServerMessages; +/** + * Checks the TRANSACTION:* permissions before a transaction begins. + * + * <p>Users need TRANSACTION:INSERT:?ENTITY_ROLE? permission to insert an entity of the particular + * entity role. Likewise, they need the TRANSACTION:UPDATE or TRANSACTION:DELETE permissions. + * + * @author Timm Fitschen <f.fitschen@indiscale.com> + */ @JobAnnotation(stage = TransactionStage.INIT) public class AccessControl extends ContainerJob { @@ -46,12 +55,15 @@ public class AccessControl extends ContainerJob { super(permission, description); } - public final String toString(String entityRole) { - return toString().replace(ENTITY_ROLE_PARAMETER, entityRole); + public final String toString(Role entityRole) { + String roleString = entityRole == null ? "" : entityRole.toString(); + return toString().replace(ENTITY_ROLE_PARAMETER, roleString); } - public final String toString(String transaction, String entityRole) { - return "TRANSACTION:" + transaction + (entityRole != null ? (":" + entityRole) : ""); + public final String toString(String transaction, Role entityRole) { + return "TRANSACTION:" + + transaction + + (entityRole != null ? (":" + entityRole.toString()) : ""); } public static String init() { @@ -80,38 +92,26 @@ public class AccessControl extends ContainerJob { protected void run() { final Subject subject = SecurityUtils.getSubject(); - // subject has complete permissions for this kind of transaction - if (subject.isPermitted( - TRANSACTION_PERMISSIONS.toString(getTransaction().getClass().getSimpleName(), null))) { - return; - } - if (getTransaction() instanceof Retrieve) { return; } for (final EntityInterface e : getContainer()) { - // per role permission - if (subject.isPermitted( - TRANSACTION_PERMISSIONS.toString( - getTransaction().getClass().getSimpleName(), e.getRole().toString()))) { - continue; - } - - // special annotations permission - if (e.hasParents() && e.getParents().size() == 1) { - final Parent par1 = e.getParents().get(0); - if (par1.hasId() && !par1.getId().isTemporary()) { - execute(new RetrieveSparseEntity(par1)); + if (e instanceof InsertEntity) { + if (subject.isPermitted(INSERT.toString(e.getRole()))) { + continue; } - if (par1.hasName() - && par1.getName().equals("CommentAnnotation") - && subject.isPermitted( - getTransaction().getClass().getSimpleName() + ":CommentAnnotation")) { + } else if (e instanceof DeleteEntity) { + if (subject.isPermitted(DELETE.toString(e.getRole()))) { + continue; + } + } else if (e instanceof UpdateEntity) { + if (subject.isPermitted(UPDATE.toString(e.getRole()))) { continue; } } + e.setEntityStatus(EntityStatus.UNQUALIFIED); e.addMessage(ServerMessages.AUTHORIZATION_ERROR); } diff --git a/src/main/java/org/caosdb/server/jobs/core/Inheritance.java b/src/main/java/org/caosdb/server/jobs/core/Inheritance.java index 33c7772811d36a5b9430556fc8926e88e2cd302c..a1bd800c7ca40b28d7b9ff27dd8f828ed112891f 100644 --- a/src/main/java/org/caosdb/server/jobs/core/Inheritance.java +++ b/src/main/java/org/caosdb/server/jobs/core/Inheritance.java @@ -119,7 +119,7 @@ public class Inheritance extends EntityJob { } // implement properties - if (getEntity().hasProperties()) { + if (getEntity().getEntityStatus() == EntityStatus.QUALIFIED && getEntity().hasProperties()) { propertyLoop: for (final Property property : getEntity().getProperties()) { final ArrayList<Property> transfer = new ArrayList<>(); diff --git a/src/main/java/org/caosdb/server/query/CQLLexer.g4 b/src/main/java/org/caosdb/server/query/CQLLexer.g4 index aa4a080e2fc2c858e2dc7da846a1bbfb593dd93d..7a2fda860485fc366d4b7aa8d3381985f806387f 100644 --- a/src/main/java/org/caosdb/server/query/CQLLexer.g4 +++ b/src/main/java/org/caosdb/server/query/CQLLexer.g4 @@ -21,93 +21,115 @@ */ lexer grammar CQLLexer; +/** */ AS_A: [Aa][Ss] (WHITE_SPACE_f? A)? WHITE_SPACE_f? ; +/** */ IS_REFERENCED: (IS_f WHITE_SPACE_f?)? [Rr][Ee][Ff][Ee][Rr][Ee][Nn][Cc][Ee][Dd] WHITE_SPACE_f? ; +/** */ BY: [Bb][Yy] WHITE_SPACE_f? ; +/** */ fragment OF_f: [Oo][Ff] ; +/** */ fragment ANY_f: [Aa][Nn][Yy] ; +/** */ fragment VERSION_f: [Vv][Ee][Rr][Ss][Ii][Oo][Nn] ; +/** */ ANY_VERSION_OF: (ANY_f WHITE_SPACE_f VERSION_f WHITE_SPACE_f OF_f) WHITE_SPACE_f? ; +/** */ SELECT: [Ss][Ee][Ll][Ee][Cc][Tt] WHITE_SPACE_f? -> pushMode(SELECT_MODE) ; +/** */ INSERTED: [Ii][Nn][Ss][Ee][Rr][Tt][Ee][Dd] WHITE_SPACE_f? ; +/** */ CREATED: [Cc][Rr][Ee][Aa][Tt][Ee][Dd] WHITE_SPACE_f? ; +/** */ UPDATED: [Uu][Pp][Dd][Aa][Tt][Ee][Dd] WHITE_SPACE_f? ; +/** */ ON: [Oo][Nn] WHITE_SPACE_f? ; +/** */ IN: [Ii][Nn] WHITE_SPACE_f? ; +/** */ AFTER: [Aa][Ff][Tt][Ee][Rr] WHITE_SPACE_f? ; +/** */ BEFORE: [Bb][Ee][Ff][Oo][Rr][Ee] WHITE_SPACE_f? ; +/** */ UNTIL: [Uu][Nn][Tt][Ii][Ll] WHITE_SPACE_f? ; +/** */ SINCE: [Ss][Ii][Nn][Cc][Ee] WHITE_SPACE_f? ; +/** */ IS_STORED_AT: (IS_f WHITE_SPACE_f?)? [Ss][Tt][Oo][Rr][Ee][Dd] (WHITE_SPACE_f? AT)? WHITE_SPACE_f? ; +/** */ AT: [Aa][Tt] WHITE_SPACE_f? ; +/** */ FIND: [Ff][Ii][Nn][Dd] WHITE_SPACE_f? ; +/** */ COUNT: [Cc][Oo][Uu][Nn][Tt] WHITE_SPACE_f? ; +/** */ AND: ( ( @@ -117,6 +139,7 @@ AND: ) WHITE_SPACE_f? ; +/** */ OR: ( ( @@ -126,22 +149,27 @@ OR: ) WHITE_SPACE_f? ; +/** */ LPAREN: '(' WHITE_SPACE_f? ; +/** */ RPAREN: ')' WHITE_SPACE_f? ; +/** */ SINGLE_QUOTE_START: '\'' -> pushMode(SINGLE_QUOTE_MODE) ; +/** */ DOUBLE_QUOTE_START: '"' -> pushMode(DOUBLE_QUOTE_MODE) ; +/** */ OPERATOR: '=' | '<' @@ -153,115 +181,138 @@ OPERATOR: | [Rr][Ee][Ff][Ee][Rr][Ee][Nn][Cc][Ee]([Ss]|WHITE_SPACE_f? [Tt][Oo]) (WHITE_SPACE_f? A {_input.LA(1) == ' '}?)? {setText("->");} ; +/** */ LIKE: [Ll][Ii][Kk][Ee] WHITE_SPACE_f? ; +/** */ IS_NULL: IS_f WHITE_SPACE_f NULL_f WHITE_SPACE_f? ; +/** */ IS_NOT_NULL: IS_f WHITE_SPACE_f NOT_f WHITE_SPACE_f NULL_f WHITE_SPACE_f? ; +/** */ fragment NULL_f: [Nn][Uu][Ll][Ll] ; +/** */ fragment DOES_f: [Dd][Oo][Ee][Ss] ; +/** */ fragment NOT_f: [Nn][Oo][Tt] ; +/** */ fragment DOESNT_f: DOES_f WHITE_SPACE_f? NOT_f | DOES_f [Nn] SINGLE_QUOTE [Tt] ; +/** */ fragment ISNT_f: IS_f [Nn] SINGLE_QUOTE [Tt] ; +/** */ fragment WERE_f: [Ww][Ee][Rr][Ee] ; +/** */ fragment WERENT_f: WERE_f [Nn] SINGLE_QUOTE [Tt] ; +/** */ fragment HAVENT_f: HAVE_f [Nn] SINGLE_QUOTE [Tt] ; +/** */ fragment HADNT_f: HAD_f [Nn] SINGLE_QUOTE [Tt] ; +/** */ fragment HAD_f: [Hh][Aa][Dd] ; +/** */ fragment HAVE_f: [Hh][Aa][Vv][Ee] ; +/** */ fragment HAS_f: [Hh][Aa][Ss] ; +/** */ fragment HASNT_f: HAS_f [Nn] SINGLE_QUOTE [Tt] ; +/** */ fragment BEEN_f: [Bb][Ee][Ee][Nn] ; +/** */ fragment HAVE_A_f: HAVE_f (WHITE_SPACE? A)? ; +/** */ fragment DO_f: [Dd][Oo] ; +/** */ fragment DONT_f: DO_f NOT_f | DO_f [Nn] SINGLE_QUOTE [Tt] ; +/** */ fragment WAS_f: [Ww][Aa][Ss] ; +/** */ fragment WASNT_f: WAS_f [Nn] SINGLE_QUOTE [Tt] ; +/** */ NEGATION: ( '!' @@ -277,46 +328,57 @@ NEGATION: ) WHITE_SPACE_f? ; -WITH: - [Ww][Ii][Tt][Hh] WHITE_SPACE_f? +/** */ +WITH_A: + [Ww][Ii][Tt][Hh] (WHITE_SPACE_f? A)? WHITE_SPACE_f? ; +/** */ THE: [Tt][Hh][Ee] WHITE_SPACE_f? ; +/** */ GREATEST: [Gg][Rr][Ee][Aa][Tt][Ee][Ss][Tt] WHITE_SPACE_f? ; +/** */ SMALLEST: [Ss][Mm][Aa][Ll][Ll][Ee][Ss][Tt] WHITE_SPACE_f? ; +/** */ A: [Aa][Nn]? WHITE_SPACE_f? ; +/** */ ME: [Mm][Ee] WHITE_SPACE_f? ; +/** */ SOMEONE: [Ss][Oo][Mm][Ee][Oo][Nn][Ee] WHITE_SPACE_f? ; +/** */ ELSE: [Ee][Ll][Ss][Ee] WHITE_SPACE_f? ; +/** */ WHERE: [Ww][Hh][Ee][Rr][Ee] WHITE_SPACE_f? ; +/** */ WHICH: [Ww][Hh][Ii][Cc][Hh] WHITE_SPACE_f? ; +/** */ HAS_A: ( (HAS_f | HAD_f | HAVE_f | WERE_f | WAS_f | IS_f) @@ -327,244 +389,303 @@ HAS_A: ) WHITE_SPACE_f? ; +/** */ PROPERTY: [Pp][Rr][Oo][Pp][Ee][Rr][Tt]([Yy]|[Ii][Ee][Ss]) WHITE_SPACE_f? ; +/** */ RECORDTYPE: [Rr][Ee][Cc][Oo][Rr][Dd][Tt][Yy][Pp][Ee]([Ss])? WHITE_SPACE_f? ; +/** */ RECORD: [Rr][Ee][Cc][Oo][Rr][Dd]([Ss])? WHITE_SPACE_f? ; +/** */ FILE: [Ff][Ii][Ll][Ee]([Ss])? WHITE_SPACE_f? ; +/** */ DIRECTORY: - [Dd][Ii][Rr][Ee][Cc][Tt][Oo][Rr]([Yy]|[Ii][Ee][Ss]) WHITE_SPACE_f? + [Dd][Ii][Rr][Ee][Cc][Tt][Oo][Rr]([Yy]|[Ii][Ee][Ss]) WHITE_SPACE_f? ; +/** */ LINK: [Ll][Ii][Nn][Kk]([Ss])? WHITE_SPACE_f? ; +/** */ ENTITY: [Ee][Nn][Tt][Ii][Tt]([Yy]|[Ii][Ee][Ss]) WHITE_SPACE_f? ; +/** */ QUERYTEMPLATE: [Qq][Uu][Ee][Rr][yY][Tt][Ee][Mm][Pp][Ll][Aa][Tt][Ee] WHITE_SPACE_f? ; +/** */ fragment IS_f: [Ii][Ss] ; +/** */ fragment WHITE_SPACE_f: [ \t\n\r]+ ; +/** */ WHITE_SPACE: [ \t\n\r]+ ; +/** */ fragment DOUBLE_QUOTE: '"' ; +/** */ fragment SINGLE_QUOTE: '\'' ; +/** */ REGEXP_MARKER: '#' ; +/** */ REGEXP_BEGIN: '<<' ; +/** */ REGEXP_END: '>>' ; +/** */ ID: [Ii][Dd] WHITE_SPACE_f? ; +/** */ SLASH: '/' ; +/** */ STAR: '*' ; +/** */ DOT: '.' ; +/** */ QMARK: '?' WHITE_SPACE_f? ; +/** */ BUT: [Bb][Uu][Tt] WHITE_SPACE_f? ; +/** */ ESC_REGEXP_END: ESC_MARKER '>>' WHITE_SPACE_f? ; +/** */ ESC_STAR: ESC_MARKER '*' WHITE_SPACE_f? ; +/** */ ESC_BS: ESC_MARKER '\\' WHITE_SPACE_f? ; +/** */ fragment ESC_MARKER: '\\' ; +/** */ TODAY: [Tt][Oo][Dd][Aa][Yy] WHITE_SPACE_f? ; +/** */ HYPHEN: '-' ; +/** */ COLON: ':' ; +/** */ NUM: NUM_f | DOT NUM_f | NUM_f DOT NUM_f ; +/** */ NUM_f: ('0'..'9')+ ; +/** */ TXT: ('a'..'z' | 'A'..'Z' | '0'..'9' | '_' | '-' {_input.LA(1) != '>'}? | '+' | '&' | ';' | ',' | '$' | ':' | '%' | '^' | '~' {_input.LA(1) != '='}? | '`' | '´' | 'ö' | 'ä' | 'ß' | 'ü' | 'Ö' | 'Ä' | 'Ü' | '@' | '[' | ']' | '{' | '}' )+ ; +/** */ UNKNOWN_CHAR: . ; +/** */ mode SINGLE_QUOTE_MODE; + /** */ SINGLE_QUOTE_ESCAPED_CHAR: ESC_MARKER ( '\'' | '\\' | '*' ) ; + /** */ SINGLE_QUOTE_END: '\'' -> mode(DEFAULT_MODE) ; + /** */ SINGLE_QUOTE_STAR: '*' ; + /** */ SINGLE_QUOTE_ANY_CHAR: ~('\''|'\\'|'*')+ ; +/** */ mode DOUBLE_QUOTE_MODE; + /** */ DOUBLE_QUOTE_ESCAPED_CHAR: ESC_MARKER ( '"' | '\\' | '*' ) ; + /** */ DOUBLE_QUOTE_END: '"' -> mode(DEFAULT_MODE) ; + /** */ DOUBLE_QUOTE_STAR: '*' ; + /** */ DOUBLE_QUOTE_ANY_CHAR: ~('"'|'\\'|'*')+ ; +/** */ mode SELECT_DOUBLE_QUOTED; + /** */ SELECT_DOUBLE_QUOTE_ESCAPED: ESC_MARKER '"' ; + /** */ SELECT_DOUBLE_QUOTE_END: '"' WHITE_SPACE_f? {setText("");} -> mode(SELECT_MODE) ; + /** */ SELECT_DOUBLE_QUOTE_TXT: . ; +/** */ mode SELECT_SINGLE_QUOTED; + /** */ SELECT_SINGLE_QUOTE_ESCAPED: ESC_MARKER '\'' ; + /** */ SELECT_SINGLE_QUOTE_END: '\'' WHITE_SPACE_f? {setText("");} -> mode(SELECT_MODE) ; + /** */ SELECT_SINGLE_QUOTE_TXT: . ; +/** */ mode SELECT_MODE; + /** */ FROM: [Ff][Rr][Oo][Mm]([ \t\n\r])* -> mode(DEFAULT_MODE) ; + /** */ SELECT_ESCAPED: ESC_MARKER ( '"' | '\\' | '\'' | ',' | '.' ) {setText(getText().substring(1));} ; + /** */ SELECT_DOT: '.' WHITE_SPACE_f? ; + /** */ SELECT_DOUBLE_QUOTE: '"' {setText("");} -> mode(SELECT_DOUBLE_QUOTED) ; + /** */ SELECT_SINGLE_QUOTE: '\'' {setText("");} -> mode(SELECT_SINGLE_QUOTED) ; + /** */ SELECT_COMMA: ',' WHITE_SPACE_f? ; + /** */ SELECTOR_TXT: . ; diff --git a/src/main/java/org/caosdb/server/query/CQLParser.g4 b/src/main/java/org/caosdb/server/query/CQLParser.g4 index 2bb92a65e9597d9038acb175db3dfdfa8afa23fd..db84e8a2a86064569b43ce239d02b6b31a29f45b 100644 --- a/src/main/java/org/caosdb/server/query/CQLParser.g4 +++ b/src/main/java/org/caosdb/server/query/CQLParser.g4 @@ -29,6 +29,9 @@ options { tokenVocab = CQLLexer; } import java.util.List; } +/** + * This is the root of the CQL grammar. + */ cq returns [Query.Type t, List<Query.Selection> s, Query.Pattern e, Query.Role r, EntityFilterInterface filter, VersionFilter v] @init{ $s = null; @@ -56,6 +59,9 @@ cq returns [Query.Type t, List<Query.Selection> s, Query.Pattern e, Query.Role r EOF ; +/** + * For versioned queries. + */ version returns [VersionFilter v] @init{ $v = null; @@ -64,6 +70,9 @@ version returns [VersionFilter v] ANY_VERSION_OF {$v = VersionFilter.ANY_VERSION;} ; +/** + * The comma-separated list of selected (sub)properties of a SELECT query. + */ prop_sel returns [List<Query.Selection> s] @init{ $s = new LinkedList<Query.Selection>(); @@ -73,6 +82,9 @@ prop_sel returns [List<Query.Selection> s] (SELECT_COMMA prop_subsel {$s.add($prop_subsel.sub);})* ; +/** + * The (sub-)selections (e.g. geolocation.longitude) of a SELECT query. + */ prop_subsel returns [Query.Selection sub]: selector_txt {$sub = new Query.Selection($selector_txt.text);} ( @@ -80,6 +92,9 @@ prop_subsel returns [Query.Selection sub]: )? ; +/** + * Rule for the allowed characters of a prop_subsel. + */ selector_txt: ( SELECT_DOUBLE_QUOTE @@ -94,6 +109,9 @@ selector_txt: ( SELECTOR_TXT | SELECT_ESCAPED )+ ; +/** + * The entity role. + */ role returns [Query.Role r]: RECORDTYPE {$r = Query.Role.RECORDTYPE;} | RECORD {$r = Query.Role.RECORD;} @@ -105,6 +123,9 @@ role returns [Query.Role r]: | ENTITY {$r = Query.Role.ENTITY;} ; +/** + * The filters of a FIND, SELECT, or COUNT query. + */ entity_filter returns [EntityFilterInterface filter] @init{ $filter = null; @@ -128,14 +149,20 @@ entity_filter returns [EntityFilterInterface filter] )? ; +/** + * WHICH keyword and syntactic sugar. + */ which_exp: WHICH (HAS_A (PROPERTY)?)? | HAS_A (PROPERTY)? - | WITH (A (PROPERTY)?)? + | WITH_A (PROPERTY)? | WHERE | DOT WHITE_SPACE? ; +/** + * Collection of different filters. + */ filter_expression returns [EntityFilterInterface efi] : backreference (subproperty {((Backreference) $backreference.ref).setSubProperty($subproperty.subp);})? {$efi = $backreference.ref;} @@ -147,6 +174,9 @@ filter_expression returns [EntityFilterInterface efi] | negation {$efi=$negation.n;} ; +/** + * ID Filter (e.g. id > 10123). + */ idfilter returns [IDFilter filter] locals [String o, String v, String a] @init{ $a = null; @@ -166,6 +196,9 @@ idfilter returns [IDFilter filter] locals [String o, String v, String a] )? ; +/** + * Transaction filter (e.g INSERTED BY ME). + */ transaction returns [TransactionFilter filter] locals [String type, TransactionFilter.Transactor user, String time, String time_op] @init{ $time = null; @@ -188,6 +221,9 @@ transaction returns [TransactionFilter filter] locals [String type, TransactionF ) ; +/** + * The transactor (e.g. "user1", or "ME"). + */ transactor returns [TransactionFilter.Transactor t] : BY @@ -202,6 +238,9 @@ transactor returns [TransactionFilter.Transactor t] ) ; +/** + * A user name or a user name pattern. + */ username returns [Query.Pattern ep] locals [int type] @init{ $type = Query.Pattern.TYPE_NORMAL; @@ -213,6 +252,9 @@ username returns [Query.Pattern ep] locals [int type] ( STAR {$type = Query.Pattern.TYPE_LIKE;} | ~(STAR | WHITE_SPACE) )+ ; +/** + * Time or timeframe of a transaction (for the transaction filter). + */ transaction_time returns [String tqp, String op] @init { $op = "("; @@ -234,9 +276,11 @@ transaction_time returns [String tqp, String op] ) ; -/* -* not fully compliant with iso 8601 (TODO) -*/ +/** + * A date time or a fragment of a date time. + * + * Not fully compliant with iso 8601 (TODO). + */ datetime : NUM // year @@ -261,6 +305,10 @@ datetime )? ; +/** + * The property-operator-value filter (e.g. temperator > 240°C) and + * (optionally) subfilters. + */ pov returns [POV filter] locals [Query.Pattern p, String o, String v, String a] @init{ $p = null; @@ -302,6 +350,9 @@ pov returns [POV filter] locals [Query.Pattern p, String o, String v, String a] ; +/** + * Wrapper for a filter on referenced entities. + */ subproperty returns [SubProperty subp] @init{ $subp = null; @@ -310,6 +361,9 @@ subproperty returns [SubProperty subp] subproperty_filter {$subp = new SubProperty($subproperty_filter.filter);} ; +/** + * The actual filter on referenced entities. + */ subproperty_filter returns [EntityFilterInterface filter] @init{ $filter = null; @@ -331,7 +385,9 @@ subproperty_filter returns [EntityFilterInterface filter] )? ; - +/** + * The backreference filter (e.g. REFERENCED BY ...). + */ backreference returns [Backreference ref] locals [Query.Pattern e, Query.Pattern p] @init{ $e = null; @@ -351,6 +407,9 @@ backreference returns [Backreference ref] locals [Query.Pattern e, Query.Pattern WHITE_SPACE? ; +/** + * Stored-at filter for the path of files. + */ storedat returns [StoredAt filter] locals [String loc] @init{ $loc = null; @@ -364,6 +423,9 @@ storedat returns [StoredAt filter] locals [String loc] WHITE_SPACE? ; +/** + * Combine other filters with a logical AND. + */ conjunction returns [Conjunction c] @init{ $c = new Conjunction(); @@ -401,6 +463,9 @@ conjunction returns [Conjunction c] )+ ; +/** + * Combine other filter with a logical OR. + */ disjunction returns [Disjunction d] @init{ $d = new Disjunction(); @@ -437,6 +502,9 @@ disjunction returns [Disjunction d] )+ ; +/** + * Negation of another filter. + */ negation returns [Negation n] @init{ } @@ -456,6 +524,9 @@ negation returns [Negation n] ) ; +/** + * An entity's full name, id, or a pattern for a name. + */ entity returns [Query.Pattern ep] : regexp_pattern {$ep = $regexp_pattern.ep;} @@ -466,6 +537,10 @@ entity returns [Query.Pattern ep] | ( ~(ENTITY |WHITE_SPACE | DOT) )+ {$ep = new Query.Pattern((String) $text.trim(), Query.Pattern.TYPE_NORMAL);} ; + +/** + * A regexp pattern. + */ regexp_pattern returns [Query.Pattern ep] locals [StringBuffer sb] @init{ $sb = new StringBuffer(); @@ -476,6 +551,9 @@ regexp_pattern returns [Query.Pattern ep] locals [StringBuffer sb] REGEXP_END {$ep = new Query.Pattern((String) $sb.toString(), Query.Pattern.TYPE_REGEXP);} ; +/** + * A like pattern (e.g. "*oxide"). + */ like_pattern returns [Query.Pattern ep] locals [StringBuffer sb] @init{ $sb = new StringBuffer(); @@ -487,6 +565,9 @@ like_pattern returns [Query.Pattern ep] locals [StringBuffer sb] {$ep = new Query.Pattern((String) $text, Query.Pattern.TYPE_LIKE);} ; +/** + * A Property's name, id or a pattern of a name. + */ property returns [Query.Pattern pp, String agg]locals [StringBuffer sb] @init{ $sb = new StringBuffer(); @@ -504,6 +585,9 @@ property returns [Query.Pattern pp, String agg]locals [StringBuffer sb] WHITE_SPACE? ; +/** + * Expression for minumum or maximum. + */ minmax returns [String agg] : (THE?? ( @@ -512,6 +596,9 @@ minmax returns [String agg] )) ; +/** + * A property value. + */ value returns [String str] : number_with_unit {$str = $number_with_unit.text;} @@ -520,6 +607,9 @@ value returns [String str] WHITE_SPACE? ; +/** + * A number with a unit (e.g. 20m). + */ number_with_unit : HYPHEN?? @@ -527,14 +617,20 @@ number_with_unit (WHITE_SPACE?? unit)? ; +/** + * A unit from a physical system of units or any other kind of system. + */ unit : - (~(WHITE_SPACE | WHICH | HAS_A | WITH | WHERE | DOT | AND | OR | RPAREN )) + (~(WHITE_SPACE | WHICH | HAS_A | WITH_A | WHERE | DOT | AND | OR | RPAREN )) (~(WHITE_SPACE))* | NUM SLASH (~(WHITE_SPACE))+ ; +/** + * A files path. + */ location returns [String str] : atom {$str = $atom.ep.str;} @@ -542,6 +638,9 @@ location returns [String str] (~WHITE_SPACE)+ {$str = $text; } ; +/** + * An atomic string or pattern. + */ atom returns [Query.Pattern ep] : double_quoted {$ep = $double_quoted.ep;} @@ -549,6 +648,9 @@ atom returns [Query.Pattern ep] | (~(WHITE_SPACE | DOT | RPAREN | LPAREN ))+ {$ep = new Query.Pattern($text, Query.Pattern.TYPE_NORMAL);} ; +/** + * A single-quoted string or pattern. + */ single_quoted returns [Query.Pattern ep] locals [StringBuffer sb, int patternType] @init{ $sb = new StringBuffer(); @@ -569,6 +671,9 @@ single_quoted returns [Query.Pattern ep] locals [StringBuffer sb, int patternTyp SINGLE_QUOTE_END ; +/** + * A double-quoted string or pattern. + */ double_quoted returns [Query.Pattern ep] locals [StringBuffer sb, int patternType] @init{ $sb = new StringBuffer(); diff --git a/src/main/java/org/caosdb/server/query/Query.java b/src/main/java/org/caosdb/server/query/Query.java index 5c07b9dd0087cbd970af333e68833a2c0f5d6403..64019d3bd3f56b8e2fa83e72adbfc966b2b596ad 100644 --- a/src/main/java/org/caosdb/server/query/Query.java +++ b/src/main/java/org/caosdb/server/query/Query.java @@ -55,6 +55,7 @@ import org.caosdb.server.caching.Cache; import org.caosdb.server.database.access.Access; import org.caosdb.server.database.backend.implementation.MySQL.ConnectionException; import org.caosdb.server.database.backend.implementation.MySQL.MySQLHelper; +import org.caosdb.server.database.backend.transaction.RetrieveFullEntityTransaction; import org.caosdb.server.database.backend.transaction.RetrieveSparseEntity; import org.caosdb.server.database.misc.DBHelper; import org.caosdb.server.database.misc.TransactionBenchmark; @@ -598,14 +599,27 @@ public class Query implements QueryInterface, ToElementable, TransactionInterfac * </ol> */ public void optimize() { - // basic optimization + // "FIND Person" is interpreted as "FIND RECORD Person" + if (this.role == null) { + try { + this.role = + Role.valueOf( + CaosDBServer.getServerProperty(ServerProperties.KEY_FIND_QUERY_DEFAULT_ROLE)); + } catch (IllegalArgumentException e) { + addError( + "Unsupportet value of the server property " + + ServerProperties.KEY_FIND_QUERY_DEFAULT_ROLE + + ": " + + CaosDBServer.getServerProperty(ServerProperties.KEY_FIND_QUERY_DEFAULT_ROLE)); + throw new UnsupportedOperationException(e); + } + } + + // "FIND *" is interpreted as "FIND RECORD", "FIND <ROLE> *" as "FIND <ROLE>" if (this.entity != null && this.entity.type == Pattern.TYPE_LIKE && this.entity.str.equals("*")) { this.entity = null; - if (this.role == null) { - this.role = Role.ENTITY; - } } } @@ -832,6 +846,10 @@ public class Query implements QueryInterface, ToElementable, TransactionInterfac this.messages.add(new Message(MessageType.Warning, MessageCode.MESSAGE_CODE_UNKNOWN, w)); } + private void addError(final String e) { + this.messages.add(new Message(MessageType.Error, MessageCode.MESSAGE_CODE_UNKNOWN, e)); + } + private void cleanUp() { if (getConnection() != null) { ResultSet rs = null; diff --git a/src/test/docker/Dockerfile b/src/test/docker/Dockerfile index f3c8b98fdab34400cccfc11f0c9211eeca37f845..19cf9dcd5cae9431ae0dc265870acbdfb1b3e650 100644 --- a/src/test/docker/Dockerfile +++ b/src/test/docker/Dockerfile @@ -3,10 +3,13 @@ RUN apt-get update && \ apt-get install -y \ git make mariadb-server maven openjdk-11-jdk-headless \ plantuml \ + libtiff5-dev libjpeg-dev libopenjp2-7-dev zlib1g-dev \ + libfreetype6-dev liblcms2-dev libwebp-dev tcl8.6-dev tk8.6-dev python3-tk \ + libharfbuzz-dev libfribidi-dev libxcb1-dev \ python3-pip screen libpam0g-dev unzip curl shunit2 \ - python3-sphinx \ - && \ - pip3 install javasphinx recommonmark sphinx-rtd-theme sphinxcontrib-plantuml + python3-sphinx + +RUN pip3 install javasphinx recommonmark sphinx-rtd-theme sphinxcontrib-plantuml sphinx-a4doc # Alternative, if javasphinx fails because python3-sphinx is too recent: # (`_l` not found): diff --git a/src/test/java/org/caosdb/server/query/QueryTest.java b/src/test/java/org/caosdb/server/query/QueryTest.java index 1827ceaf576bfba79059d557fa1422839cad8d28..986122ca46307173c6c9f80b2085db3f1776904a 100644 --- a/src/test/java/org/caosdb/server/query/QueryTest.java +++ b/src/test/java/org/caosdb/server/query/QueryTest.java @@ -61,20 +61,23 @@ public class QueryTest { @Test public void testGetKey() { - assertEquals("E_enameF_POV(pname,=,val1)", getCacheKey("FIND ename WITH pname = val1")); - assertEquals("E_enameF_POV(pname,=,val1)", getCacheKey("COUNT ename WITH pname = val1")); + assertEquals("R_RECORDE_enameF_POV(pname,=,val1)", getCacheKey("FIND ename WITH pname = val1")); assertEquals( - "E_enameF_POV(pname,=,val1)", getCacheKey("SELECT bla FROM ename WITH pname = val1")); - assertEquals("E_enameF_POV(pname,null,null)", getCacheKey("SELECT bla FROM ename WITH pname")); + "R_RECORDE_enameF_POV(pname,=,val1)", getCacheKey("COUNT ename WITH pname = val1")); assertEquals( - "E_enameF_maxPOV(pname,null,null)", + "R_RECORDE_enameF_POV(pname,=,val1)", + getCacheKey("SELECT bla FROM ename WITH pname = val1")); + assertEquals( + "R_RECORDE_enameF_POV(pname,null,null)", getCacheKey("SELECT bla FROM ename WITH pname")); + assertEquals( + "R_RECORDE_enameF_maxPOV(pname,null,null)", getCacheKey("SELECT bla FROM ename WITH THE GREATEST pname")); assertEquals( "R_RECORDE_enameF_POV(pname,=,val1)", getCacheKey("FIND RECORD ename WITH pname = val1")); assertEquals("R_ENTITYF_POV(pname,=,val1)", getCacheKey("COUNT ENTITY WITH pname = val1")); assertEquals( - "E_enameF_Conj(POV(pname,=,val1)POV(ename2,=,val2))", + "R_RECORDE_enameF_Conj(POV(pname,=,val1)POV(ename2,=,val2))", getCacheKey("SELECT bla FROM ename WITH pname = val1 AND ename2 = val2")); assertEquals("V_R_ENTITYF_ID(,>,2)", getCacheKey("FIND ANY VERSION OF ENTITY WITH ID > 2")); @@ -83,14 +86,14 @@ public class QueryTest { assertEquals( "R_ENTITYF_SAT(asdf/asdf)", getCacheKey("FIND ENTITY WHICH IS STORED AT asdf/asdf")); assertEquals( - "E_enameF_POV(ref1,null,null)SUB(POV(pname,>,val1)", + "R_RECORDE_enameF_POV(ref1,null,null)SUB(POV(pname,>,val1)", getCacheKey("FIND ename WITH ref1 WITH pname > val1 ")); assertEquals( - "E_enameF_@(ref1,null)SUB(POV(pname,>,val1)", + "R_RECORDE_enameF_@(ref1,null)SUB(POV(pname,>,val1)", getCacheKey("FIND ename WHICH IS REFERENCED BY ref1 WITH pname > val1 ")); assertEquals( - "U_anonymous@anonymousE_enameF_POV(pname,=,val1)", + "U_anonymous@anonymousR_RECORDE_enameF_POV(pname,=,val1)", getCacheKeyWithUser("FIND ename WITH pname = val1")); } @@ -103,7 +106,7 @@ public class QueryTest { q.optimize(); assertNull(q.getEntity()); - assertEquals(Query.Role.ENTITY, q.getRole()); + assertEquals(Query.Role.RECORD, q.getRole()); } /**