diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index cf8544c1782b0d775cb6cec5694167219ec47e9b..8587b4072e1d7485406cd432aff9034e6c2e6b84 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -23,6 +23,7 @@ # variables: + DEPLOY_REF: dev CI_REGISTRY_IMAGE: $CI_REGISTRY/caosdb/caosdb-server/caosdb-server-testenv:latest image: $CI_REGISTRY_IMAGE @@ -69,7 +70,8 @@ trigger_build: script: - /usr/bin/curl -X POST -F token=$DEPLOY_TRIGGER_TOKEN + -F "variables[F_BRANCH]=$CI_COMMIT_REF_NAME" -F "variables[SERVER]=$CI_COMMIT_REF_NAME" -F "variables[TriggerdBy]=SERVER" -F "variables[TriggerdByHash]=$CI_COMMIT_SHORT_SHA" - -F ref=dev https://gitlab.indiscale.com/api/v4/projects/14/trigger/pipeline + -F ref=$DEPLOY_REF https://gitlab.indiscale.com/api/v4/projects/14/trigger/pipeline diff --git a/CHANGELOG.md b/CHANGELOG.md index 8ac953fc1ed36b8396cac6481cbd40fd5b3bd199..88ac0b5297a401ebcc3edda414cce13d3142d3fd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -36,10 +36,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed +* #51 - name queries (e.g. `FIND ENTITY WITH name = ...`) - #27 - star matches slashes (e.g. for `FIND ... STORED AT /*.dat`). - #30 - file path cannot be in quotes -- #46 - Server-side scripting failed as an unprivileged user because the was no - writable home directory. +- #46 - Server-side scripting failed as an unprivileged user because there was + no writable home directory. - NaN Double Values (see #41) - #14 - Handle files on file system without File entity: Those entries are returned without ID but with a notice now. @@ -49,8 +50,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Security (in case of vulnerabilities) - TLS is by default restricted to v1.2 and v1.3 now. -- #11 - pam_authentication leaks the password to unprivileged processes on the - same machine. +- #11 - PAM and LDAP authentication no longer leak the password to unprivileged + processes on the same machine. - #68 - Shadow sensitive information when logging for debugging purposes. diff --git a/conf/core/server.conf b/conf/core/server.conf index 66d59c276eba895c7f7885292dc004634c7d20f3..358f6b5c14106b87d7a676e2262474772ae98512 100644 --- a/conf/core/server.conf +++ b/conf/core/server.conf @@ -67,7 +67,7 @@ MYSQL_USER_NAME=caosdb # Password for the user MYSQL_USER_PASSWORD=caosdb # Schema of mysql procedures and tables which is required by this CaosDB instance -MYSQL_SCHEMA_VERSION=v2.1.2 +MYSQL_SCHEMA_VERSION=v3.0.0-rc1 # -------------------------------------------------- diff --git a/misc/pam_authentication/ldap.conf b/misc/pam_authentication/ldap.conf index aca5b0491bc937997f7e70efae09d92c55564e67..117d1074915e3cacd9a97b83b5a2e83e3c50d451 100644 --- a/misc/pam_authentication/ldap.conf +++ b/misc/pam_authentication/ldap.conf @@ -2,6 +2,6 @@ # Set the ldap server here. This is also used to generate a fully qualified -# user name. +# user name: <USER>@$LDAP_SERVER # LDAP_SERVER="example.com" diff --git a/misc/pam_authentication/ldap_authentication.sh b/misc/pam_authentication/ldap_authentication.sh index 1e3ee2e8da5cc4aed55c55359ae1086e6b7de129..f887bf99f47c827fd712d2189a5ca89ec2981e6c 100755 --- a/misc/pam_authentication/ldap_authentication.sh +++ b/misc/pam_authentication/ldap_authentication.sh @@ -21,10 +21,10 @@ # # ** end header -# Try to authenticate a user ($1) with a password ($2) via LDAP +# Try to authenticate a user ($1) via LDAP, either via stdin or a password file ($2, if given). -[[ "$#" == "2" ]] || { - echo "call this script with two arguments: user and password" +[[ "$#" == "1" || "$#" == "2" ]] || { + echo "Call this script as: $0 <user> [<password file>]" exit 1 } @@ -33,11 +33,17 @@ exe_dir=$(dirname $0) . "$exe_dir/"ldap.conf +# If the second argument is empty or "-", take password from stdin, else use the argument as a file. testpw() { username="${1}@${LDAP_SERVER}" - pw="$2" + pwfile="$2" + pwargs=("-w" "$pwfile") + if [[ $pwfile == "-" ]] ; then + pwargs=("-W") + fi - if timeout 5s ldapwhoami -x -H "ldap://$LDAP_SERVER" -D "$username" -w "$pw"; then + export LDAPTLS_REQCERT=ALLOW + if timeout 5s ldapwhoami -x -H "ldaps://$LDAP_SERVER" -D "$username" "${pwargs[@]}"; then return 0 else ret_code="$?" @@ -48,7 +54,7 @@ testpw() { return "$ret_code" fi - ldapwhoami -x -H "ldap://$LDAP_SERVER" -D "$username" -w "$pw" + ldapwhoami -x -H "ldaps://$LDAP_SERVER" -D "$username" "${pwargs[@]}" } diff --git a/pom.xml b/pom.xml index be9279b6e1e927339d1c044b4cbc4313f992fba5..c8f79b3dcc43759cc941244283d13b248d811bb8 100644 --- a/pom.xml +++ b/pom.xml @@ -115,7 +115,7 @@ <dependency> <groupId>org.antlr</groupId> <artifactId>antlr4</artifactId> - <version>4.7.2</version> + <version>4.8-1</version> </dependency> <dependency> <groupId>org.restlet.jse</groupId> @@ -212,7 +212,7 @@ <plugin> <groupId>org.antlr</groupId> <artifactId>antlr4-maven-plugin</artifactId> - <version>4.7.2</version> + <version>4.8-1</version> <configuration> <sourceDirectory>${basedir}/src/main/java</sourceDirectory> </configuration> diff --git a/src/main/java/caosdb/server/database/backend/implementation/MySQL/MySQLGetAllNames.java b/src/main/java/caosdb/server/database/backend/implementation/MySQL/MySQLGetAllNames.java index d7c641cadebcde93b97187aa7a0ea2ded72d4678..060cf9b8a77ad15e430b34f06e2b17737dd27cc2 100644 --- a/src/main/java/caosdb/server/database/backend/implementation/MySQL/MySQLGetAllNames.java +++ b/src/main/java/caosdb/server/database/backend/implementation/MySQL/MySQLGetAllNames.java @@ -18,8 +18,12 @@ public class MySQLGetAllNames extends MySQLTransaction implements GetAllNamesImp super(access); } + /** Retrieve tuples (EntityName, EntityRole, ACL) for all entities which have a name. */ public static final String STMT_GET_ALL_NAMES = - "Select e.name as EntityName, e.role as EntityRole, a.acl as ACL FROM entities as e JOIN entity_acl as a ON (a.id=e.acl) WHERE e.name IS NOT NULL and e.role!='ROLE'"; + "SELECT d.value as EntityName, e.role AS EntityRole, a.acl AS ACL " + + "FROM name_data AS d JOIN entities AS e JOIN entity_acl AS a " + + "ON (d.domain_id = 0 AND d.property_id = 20 AND d.entity_id = e.id AND a.id = e.acl) " + + "WHERE e.role != 'ROLE' AND e.role != 'DATATYPE'"; @Override public List<SparseEntity> execute() { diff --git a/src/main/java/caosdb/server/database/backend/implementation/MySQL/MySQLGetIDByName.java b/src/main/java/caosdb/server/database/backend/implementation/MySQL/MySQLGetIDByName.java index ff6d2b53d4e142cd634848d1c2edbccbe30892ad..e9797640d98b6e9f2afb2b95ff3ea5103b623070 100644 --- a/src/main/java/caosdb/server/database/backend/implementation/MySQL/MySQLGetIDByName.java +++ b/src/main/java/caosdb/server/database/backend/implementation/MySQL/MySQLGetIDByName.java @@ -36,9 +36,19 @@ public class MySQLGetIDByName extends MySQLTransaction implements GetIDByNameImp super(access); } - public static final String STMT_GET_ID_BY_NAME = "Select id from entities where name=?"; - public static final String STMT_AND_ROLE = " AND role=?"; - public static final String STMT_NOT_ROLE = " AND role!='ROLE'"; + /** + * Resolves the (primary) name of an entity to an id. This query is not necessarily unique. + * Therefore {@link #STMT_AND_ROLE}, {@link #STMT_NOT_ROLE}, and {@link #STMT_LIMIT} can as + * additional conditions. + */ + public static final String STMT_GET_ID_BY_NAME = + "Select n.entity_id AS id " + + "FROM name_data AS n JOIN entities AS e " + + "ON (n.domain_id=0 AND n.property_id=20 AND e.id = n.entity_id)" + + "WHERE n.value=?"; + + public static final String STMT_AND_ROLE = " AND e.role=?"; + public static final String STMT_NOT_ROLE = " AND e.role!='ROLE'"; public static final String STMT_LIMIT = " LIMIT "; @Override @@ -55,17 +65,13 @@ public class MySQLGetIDByName extends MySQLTransaction implements GetIDByNameImp if (role != null) { stmt.setString(2, role); } - ResultSet rs = null; - try { - rs = stmt.executeQuery(); + try (ResultSet rs = stmt.executeQuery()) { final ArrayList<Integer> ret = new ArrayList<Integer>(); while (rs.next()) { ret.add(rs.getInt("id")); } return ret; - } finally { - rs.close(); } } catch (final Exception e) { throw new TransactionException(e); diff --git a/src/main/java/caosdb/server/database/backend/implementation/MySQL/MySQLInsertEntityDatatype.java b/src/main/java/caosdb/server/database/backend/implementation/MySQL/MySQLInsertEntityDatatype.java index a2682e80b1997b53496e3ac2f8e6147726fb77c9..47671864570c1a041b8b164a0f045fbf70b47987 100644 --- a/src/main/java/caosdb/server/database/backend/implementation/MySQL/MySQLInsertEntityDatatype.java +++ b/src/main/java/caosdb/server/database/backend/implementation/MySQL/MySQLInsertEntityDatatype.java @@ -15,10 +15,22 @@ public class MySQLInsertEntityDatatype extends MySQLTransaction super(access); } + /** + * Inserts atomic data types of properties into the data_type table. Has two parameters, the + * property_id and the data type name. + */ public static final String STMT_INSERT_ENTITY_DATATYPE = - "INSERT INTO data_type (domain_id, entity_id, property_id, datatype) SELECT 0, 0, ?, ( SELECT id from entities where name = ? LIMIT 1);"; + "INSERT INTO data_type (domain_id, entity_id, property_id, datatype) " + + "SELECT 0, 0, ?, " + + "( SELECT entity_id FROM name_data WHERE domain_id = 0 AND property_id = 20 AND value = ? LIMIT 1);"; + + /** + * Inserts collection data types of properties into the data_type table. Has two parameters, the + * property_id and the type of collection (e.g. 'LIST'). + */ public static final String STMT_INSERT_ENTITY_COLLECTION = - "INSERT INTO collection_type (domain_id, entity_id, property_id, collection) SELECT 0, 0, ?, ?;"; + "INSERT INTO collection_type (domain_id, entity_id, property_id, collection) " + + "SELECT 0, 0, ?, ?;"; @Override public void execute(final SparseEntity entity) { diff --git a/src/main/java/caosdb/server/database/backend/implementation/MySQL/MySQLRetrieveAll.java b/src/main/java/caosdb/server/database/backend/implementation/MySQL/MySQLRetrieveAll.java index b93615ea529291b967ef1fc22206b8d953e408d0..7ae921315f55e94994a2868d5b0a9b8d8406542b 100644 --- a/src/main/java/caosdb/server/database/backend/implementation/MySQL/MySQLRetrieveAll.java +++ b/src/main/java/caosdb/server/database/backend/implementation/MySQL/MySQLRetrieveAll.java @@ -38,17 +38,17 @@ public class MySQLRetrieveAll extends MySQLTransaction implements RetrieveAllImp super(access); } - public static final String STMT_GET_ALL_HEAD = "Select id from entities where "; + public static final String STMT_GET_ALL_HEAD = "Select id from entities where id > 99"; public static final String STMT_ENTITY_WHERE_CLAUSE = - " ( role=? OR role='" + " AND ( role=? OR role='" + Role.RecordType + "' OR role='" + Role.Property + "' OR role='" + Role.File + "'" - + " ) AND ( NOT name=role OR name IS NULL)"; - public static final String STMT_OTHER_ROLES = " role=? AND ( NOT name=role OR name IS NULL)"; + + " )"; + public static final String STMT_OTHER_ROLES = " AND role=?"; @Override public List<Integer> execute(final String role) throws TransactionException { diff --git a/src/main/java/caosdb/server/database/backend/implementation/MySQL/MySQLRetrieveDatatypes.java b/src/main/java/caosdb/server/database/backend/implementation/MySQL/MySQLRetrieveDatatypes.java index 1c7609cc5ddf4eaff9bf2c742b43dd2bd27b0337..a8e12672cf21592c23d8cea0aa22629b4b68855c 100644 --- a/src/main/java/caosdb/server/database/backend/implementation/MySQL/MySQLRetrieveDatatypes.java +++ b/src/main/java/caosdb/server/database/backend/implementation/MySQL/MySQLRetrieveDatatypes.java @@ -38,8 +38,19 @@ public class MySQLRetrieveDatatypes extends MySQLTransaction implements Retrieve super(access); } + /** + * Retrieve (ParentID, ParentName, ParentDescription, ParentRole, ACL) tuple which actually + * contains the ID, name, description, role and ACL of the datatype. The misleading names should + * be fixed sometimes (TODO) but this also requires to adjust the code below, which uses {@link + * DatabaseUtils#parseParentResultSet(ResultSet)}. + */ private static final String STMT_GET_DATATYPE = - "select id AS ParentID, name AS ParentName, description as ParentDescription, role as ParentRole, (SELECT acl FROM entity_acl as a WHERE a.id=e.acl) as ACL from entities as e where e.role='DATATYPE'"; + "SELECT id AS ParentID, " + + "(SELECT value FROM name_data WHERE domain_id = 0 AND entity_ID = e.id AND property_id = 20) AS ParentName, " + + "description AS ParentDescription, " + + "role AS ParentRole, " + + "(SELECT acl FROM entity_acl AS a WHERE a.id=e.acl) as ACL " + + "FROM entities AS e WHERE e.role='DATATYPE'"; @Override public ArrayList<VerySparseEntity> execute() throws TransactionException { diff --git a/src/main/java/caosdb/server/database/backend/implementation/MySQL/MySQLRetrieveSparseEntity.java b/src/main/java/caosdb/server/database/backend/implementation/MySQL/MySQLRetrieveSparseEntity.java index 749b0b91455c9f429821bbabce3aa99375a417cd..2aff2aaedda70672eb676597a2e3ba7a1db14352 100644 --- a/src/main/java/caosdb/server/database/backend/implementation/MySQL/MySQLRetrieveSparseEntity.java +++ b/src/main/java/caosdb/server/database/backend/implementation/MySQL/MySQLRetrieveSparseEntity.java @@ -51,13 +51,10 @@ public class MySQLRetrieveSparseEntity extends MySQLTransaction final PreparedStatement preparedStatement = prepareStatement(stmtStr); preparedStatement.setInt(1, id); - final ResultSet rs = preparedStatement.executeQuery(); - try { + try (final ResultSet rs = preparedStatement.executeQuery()) { if (rs.next()) { return DatabaseUtils.parseEntityResultSet(rs); } - } finally { - rs.close(); } } catch (final SQLException e) { throw new TransactionException(e); diff --git a/src/test/java/caosdb/server/query/TestCQL.java b/src/test/java/caosdb/server/query/TestCQL.java index 578cb5c9e918c4a81ced10a3c65248356f762fe5..402cddd2be631291d772d494804cf21bc9dbc6b7 100644 --- a/src/test/java/caosdb/server/query/TestCQL.java +++ b/src/test/java/caosdb/server/query/TestCQL.java @@ -219,6 +219,7 @@ public class TestCQL { String query56b = "FIND RECORD WHICH REFERENCES AN ename2"; String query56c = "FIND RECORD WHICH REFERENCES atom"; String query56d = "FIND RECORD WHICH REFERENCES A tom"; + String queryIssue31 = "FIND FILE WHICH IS STORED AT /data/in0.foo"; // File paths /////////////////////////////////////////////////////////////// String filepath_verb01 = "/foo/"; @@ -6390,4 +6391,30 @@ public class TestCQL { EntityFilterInterface pov = sfq.filter; assertEquals("POV(prop,=,)", pov.toString()); } + + /** String queryIssue31 = "FIND FILE WHICH IS STORED AT /data/in0.foo"; */ + // FIXME Remove "expected" annotation. + @Test(expected = AssertionError.class) + public void testIssue31() { + CQLLexer lexer; + lexer = new CQLLexer(CharStreams.fromString(this.queryIssue31)); + final CommonTokenStream tokens = new CommonTokenStream(lexer); + + final CQLParser parser = new CQLParser(tokens); + final CqContext sfq = parser.cq(); + + System.out.println(sfq.toStringTree(parser)); + + // 4 children: FIND, role, WHICHCLAUSE, EOF + assertEquals(4, sfq.getChildCount()); + assertEquals("WHICHIS STORED AT/data/in0.foo", sfq.getChild(2).getText()); + assertEquals("FILE", sfq.r.toString()); + assertNull(sfq.e); + assertEquals("StoredAt", sfq.filter.getClass().getSimpleName()); + final ParseTree whichclause = sfq.getChild(2); + final ParseTree transactionFilter = whichclause.getChild(1).getChild(0); + assertEquals("/data/in0.foo", transactionFilter.getChild(1).getText()); + final StoredAt storedAt = (StoredAt) sfq.filter; + assertEquals("SAT(/data/in0.foo)", storedAt.toString()); + } }