diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 4709a64daf23b5628ae69bfd8f4877cb37abbe74..3878ddb5d95b9319fc6c44481efe9a63d284da41 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -62,21 +62,6 @@ unittests-mariadb: script: - make pipeline-test SQL_HOST=mariadb -# Run the unit tests with MySQL 8 -unittests-mysql-8: - tags: [ docker ] - stage: test - # Should not stop the pipeline from continuing. - allow_failure: true - services: - - name: mysql:8.0.32 - command: ["--default-authentication-plugin=mysql_native_password"] - - script: - - sed "s/NO_AUTO_CREATE_USER,//" -i tests/example.dump.sql - - rm tests/test_autotap.sql - - make pipeline-test SQL_HOST=mysql - # Run the unit tests with MySQL 5 unittests-mysql-5: tags: [ docker ] diff --git a/CHANGELOG.md b/CHANGELOG.md index c7746bc3e81fea0137cf388e69da8e6770eb9299..2afc8718c347b64e65696d53449d0c8e664758a0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,12 +5,32 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [Unreleased, upcomming 6.1.0] ## + +### Added ### + +### Changed ### + +### Deprecated ### + +* MySQL Support. Last version which is known to work well with LinkAhead is MySQL 5.7.36 + +### Removed ### + +### Fixed ### + +* Fix broken `initEntity` procedure (and change the first parameters type from + INT UNSIGNED to VARCHAR(255) which is non-breaking). + +### Security ### + ## [6.0.1 2023-10-18] ## ### Fixed ### * Old passwords table is removed from dumps where it still exists. ## [6.0.0 2023-10-17] ## +>>>>>>> dev This is a major update. Switching from integer ids for internal and external use to string ids for the external use while keeping the old integer ids for internal use. diff --git a/Makefile b/Makefile index 544c0b6abc7627bb7a78f5312ee05d70b2d4d5ac..efe8b68e57427b057f6b9db4a6b64fb73a1ec2f3 100644 --- a/Makefile +++ b/Makefile @@ -88,4 +88,5 @@ pipeline-test: sleep 10 make install ./utils/make_db restore_db tests/example.dump.sql + make upgrade ./tests/test_utils.sh diff --git a/README_SETUP.md b/README_SETUP.md index 1b88d56636ccbbbfa3ed616ff323d8e9fdf4835c..95f975d031ded904b2656230f5ec72d98447a7b6 100644 --- a/README_SETUP.md +++ b/README_SETUP.md @@ -1,9 +1,9 @@ # Setup of the CaosDB SQL back end ## Dependencies -* `MariaDB Client 10.1`, `MySQL Client 5.5`, or later versions. In the case of - MySQL, version 5.6 is recommended. -- make + +* `MariaDB Client 10.1` or later, `MySQL Client >=5.5, <=5.7.36`. +* make ## Create the configuration * Create an empty `.config` file. For the default values and the meaning of diff --git a/patches/patch20231018-6.1.0/patch.sh b/patches/patch20231018-6.1.0/patch.sh new file mode 100755 index 0000000000000000000000000000000000000000..e91f487216ce254fb3b1a78a911f38fcc263d8c7 --- /dev/null +++ b/patches/patch20231018-6.1.0/patch.sh @@ -0,0 +1,42 @@ +#!/bin/bash +# +# This file is a part of the CaosDB Project. +# +# Copyright (C) 2023 IndiScale GmbH <info@indiscale.com> +# Copyright (C) 2023 Timm Fitschen <t.fitschen@indiscale.com> +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see <https://www.gnu.org/licenses/>. +# + +# Update mysql schema to version v6.1.0 +# No migration of the schema, updating procedures only + +NEW_VERSION="v6.1.0" +OLD_VERSION="v6.0.0-SNAPSHOT-EXTIDS" + +if [ -z "$UTILSPATH" ]; then + UTILSPATH="../utils" +fi + +. $UTILSPATH/patch_header.sh $* + +check_version $OLD_VERSION + +# Drop insertUser procedure +mysql_execute "DROP PROCEDURE IF EXISTS insertUser;" + +update_version $NEW_VERSION + +success + diff --git a/procedures/insertUser.sql b/procedures/insertUser.sql deleted file mode 100644 index 4f2f01eea78fd5190c7ab1fbf106fa435b0da2ef..0000000000000000000000000000000000000000 --- a/procedures/insertUser.sql +++ /dev/null @@ -1,43 +0,0 @@ -/* - * ** header v3.0 - * This file is a part of the CaosDB Project. - * - * Copyright (C) 2018 Research Group Biomedical Physics, - * Max-Planck-Institute for Dynamics and Self-Organization Göttingen - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of the - * License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see <https://www.gnu.org/licenses/>. - * - * ** end header - */ - -Drop Procedure if exists db_5_0.insertUser; -Delimiter // -Create Procedure db_5_0.insertUser(in Name VARCHAR(255), in Password VARCHAR(255)) - -BEGIN - - -INSERT INTO entities (name, role, acl) VALUES (Name, 'USER', 0); - -SET @LAST_UserID = LAST_INSERT_ID(); - -INSERT INTO passwords VALUES (@LAST_UserID, Password); - -Select @LAST_UserID as UserID; - -END; -// - - -delimiter ; diff --git a/procedures/query/initEntity.sql b/procedures/query/initEntity.sql index 65d03b818f68991acabd82890c1730572ecc822c..d63c7773867cb9e5c29bdc4f95cbe866c3b426c4 100644 --- a/procedures/query/initEntity.sql +++ b/procedures/query/initEntity.sql @@ -30,7 +30,7 @@ DELIMITER // * If `versioned` is `TRUE`, also add archived entities (for example if the name was changed in the * past). */ -CREATE PROCEDURE db_5_0.initEntity(in eid INT UNSIGNED, in ename VARCHAR(255), +CREATE PROCEDURE db_5_0.initEntity(in eid VARCHAR(255), in ename VARCHAR(255), in enameLike VARCHAR(255), in enameRegexp VARCHAR(255), in resultset VARCHAR(255), in versioned BOOLEAN) initEntityLabel: BEGIN @@ -78,9 +78,9 @@ initEntityLabel: BEGIN 'INSERT IGNORE INTO `', resultset, IF(versioned, - '` (id, _iversion) SELECT id, _get_head_iversion(id) ', - '` (id) SELECT id '), - 'FROM entities WHERE id=',eid,';'); + '` (id, _iversion) SELECT eids.internal_id, _get_head_iversion(eids.internal_id) ', + '` (id) SELECT eids.internal_id '), + 'FROM entity_ids AS eids WHERE eids.id=',eid,';'); PREPARE initEntityStmt FROM @initEntityStmtStr; EXECUTE initEntityStmt; DEALLOCATE PREPARE initEntityStmt; diff --git a/tests/example.dump.sql b/tests/example.dump.sql index cad5a03967a807005a217733c557a284408f304c..dc29bb5ea61dfc09e2374da080a03b21227440f4 100644 --- a/tests/example.dump.sql +++ b/tests/example.dump.sql @@ -563,31 +563,6 @@ INSERT INTO `passwd` VALUES ('guest','3fe82ca86837d4f9ff765d2d4265ded96843314936 /*!40000 ALTER TABLE `passwd` ENABLE KEYS */; UNLOCK TABLES; --- --- Table structure for table `passwords` --- - -DROP TABLE IF EXISTS `passwords`; -/*!40101 SET @saved_cs_client = @@character_set_client */; -/*!40101 SET character_set_client = utf8 */; -CREATE TABLE `passwords` ( - `entity_id` int(10) unsigned NOT NULL COMMENT 'User ID.', - `password` varchar(255) COLLATE utf8_unicode_ci NOT NULL COMMENT 'Password.', - PRIMARY KEY (`entity_id`), - CONSTRAINT `use_entity_id_entity` FOREIGN KEY (`entity_id`) REFERENCES `entities` (`id`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci; -/*!40101 SET character_set_client = @saved_cs_client */; - --- --- Dumping data for table `passwords` --- - -LOCK TABLES `passwords` WRITE; -/*!40000 ALTER TABLE `passwords` DISABLE KEYS */; -INSERT INTO `passwords` VALUES (98,'37d7bd8a833261b4e4653644ee0a065f522b92b3738ca9ae2cb43a83844bf352c4a59c386a44965997a508c61988c9484c093775027425091d6d3d435c3c0e0c'),(99,'37d7bd8a833261b4e4653644ee0a065f522b92b3738ca9ae2cb43a83844bf352c4a59c386a44965997a508c61988c9484c093775027425091d6d3d435c3c0e0c'); -/*!40000 ALTER TABLE `passwords` ENABLE KEYS */; -UNLOCK TABLES; - -- -- Table structure for table `permissions` -- @@ -668,7 +643,7 @@ CREATE TABLE `reference_data` ( LOCK TABLES `reference_data` WRITE; /*!40000 ALTER TABLE `reference_data` DISABLE KEYS */; -INSERT INTO `reference_data` VALUES (0,243,233,119,'REPLACEMENT',6),(0,243,119,240,'FIX',0),(0,243,232,120,'REPLACEMENT',7),(0,243,120,241,'FIX',0),(0,243,234,121,'REPLACEMENT',8),(0,243,121,242,'FIX',0),(0,265,255,230,'FIX',3),(0,265,256,243,'FIX',4),(0,265,257,119,'REPLACEMENT',6),(0,265,119,262,'FIX',0),(0,265,260,120,'REPLACEMENT',7),(0,265,120,263,'FIX',0),(0,265,259,121,'REPLACEMENT',8),(0,265,121,264,'FIX',0),(0,270,257,119,'REPLACEMENT',1),(0,270,260,120,'REPLACEMENT',2),(0,271,257,119,'REPLACEMENT',1),(0,271,119,268,'FIX',0),(0,271,260,120,'REPLACEMENT',2),(0,271,120,269,'FIX',0),(0,271,267,266,'FIX',3),(0,273,222,271,'FIX',1),(0,277,222,230,'FIX',1),(0,279,222,243,'FIX',1),(0,278,222,265,'FIX',1),(0,276,222,99,'FIX',1),(0,230,221,119,'REPLACEMENT',3),(0,230,119,226,'FIX',0),(0,230,220,120,'REPLACEMENT',4),(0,230,120,227,'FIX',0),(0,230,218,121,'REPLACEMENT',5),(0,230,121,228,'FIX',0),(0,230,219,122,'REPLACEMENT',6),(0,230,122,229,'FIX',0); +INSERT INTO `reference_data` VALUES (0,243,233,119,'REPLACEMENT',6),(0,243,119,240,'FIX',0),(0,243,232,120,'REPLACEMENT',7),(0,243,120,241,'FIX',0),(0,243,234,121,'REPLACEMENT',8),(0,243,121,242,'FIX',0),(0,265,255,230,'FIX',3),(0,265,256,243,'FIX',4),(0,265,257,119,'REPLACEMENT',6),(0,265,119,262,'FIX',0),(0,265,260,120,'REPLACEMENT',7),(0,265,120,263,'FIX',0),(0,265,259,121,'REPLACEMENT',8),(0,265,121,264,'FIX',0),(0,270,257,119,'REPLACEMENT',1),(0,270,260,120,'REPLACEMENT',2),(0,271,257,119,'REPLACEMENT',1),(0,271,119,268,'FIX',0),(0,271,260,120,'REPLACEMENT',2),(0,271,120,269,'FIX',0),(0,271,267,266,'FIX',3),(0,273,222,271,'FIX',1),(0,277,222,230,'FIX',1),(0,279,222,243,'FIX',1),(0,278,222,265,'FIX',1),(0,230,221,119,'REPLACEMENT',3),(0,230,119,226,'FIX',0),(0,230,220,120,'REPLACEMENT',4),(0,230,120,227,'FIX',0),(0,230,218,121,'REPLACEMENT',5),(0,230,121,228,'FIX',0),(0,230,219,122,'REPLACEMENT',6),(0,230,122,229,'FIX',0); /*!40000 ALTER TABLE `reference_data` ENABLE KEYS */; UNLOCK TABLES; @@ -2783,34 +2758,6 @@ BEGIN INSERT IGNORE INTO units_lin_con (signature_from, signature_to, a, b_dividend, b_divisor, c) VALUES (signature_from, signature_to, a, b_dividend, b_divisor, c); -END ;; -DELIMITER ; -/*!50003 SET sql_mode = @saved_sql_mode */ ; -/*!50003 SET character_set_client = @saved_cs_client */ ; -/*!50003 SET character_set_results = @saved_cs_results */ ; -/*!50003 SET collation_connection = @saved_col_connection */ ; -/*!50003 DROP PROCEDURE IF EXISTS `insertUser` */; -/*!50003 SET @saved_cs_client = @@character_set_client */ ; -/*!50003 SET @saved_cs_results = @@character_set_results */ ; -/*!50003 SET @saved_col_connection = @@collation_connection */ ; -/*!50003 SET character_set_client = utf8mb4 */ ; -/*!50003 SET character_set_results = utf8mb4 */ ; -/*!50003 SET collation_connection = utf8mb4_general_ci */ ; -/*!50003 SET @saved_sql_mode = @@sql_mode */ ; -/*!50003 SET sql_mode = 'STRICT_TRANS_TABLES,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION' */ ; -DELIMITER ;; -CREATE DEFINER=`caosdb`@`%` PROCEDURE `insertUser`(in Name VARCHAR(255), in Password VARCHAR(255)) -BEGIN - - -INSERT INTO entities (name, role, acl) VALUES (Name, 'USER', 0); - -SET @LAST_UserID = LAST_INSERT_ID(); - -INSERT INTO passwords VALUES (@LAST_UserID, Password); - -Select @LAST_UserID as UserID; - END ;; DELIMITER ; /*!50003 SET sql_mode = @saved_sql_mode */ ; @@ -3168,29 +3115,6 @@ BEGIN SELECT collection as collection_override, NULL as name_override, NULL as desc_override, NULL as type_override, entity_id, property_id from collection_type where domain_id=DomainID and entity_id=EntityID; -END ;; -DELIMITER ; -/*!50003 SET sql_mode = @saved_sql_mode */ ; -/*!50003 SET character_set_client = @saved_cs_client */ ; -/*!50003 SET character_set_results = @saved_cs_results */ ; -/*!50003 SET collation_connection = @saved_col_connection */ ; -/*!50003 DROP PROCEDURE IF EXISTS `setPassword` */; -/*!50003 SET @saved_cs_client = @@character_set_client */ ; -/*!50003 SET @saved_cs_results = @@character_set_results */ ; -/*!50003 SET @saved_col_connection = @@collation_connection */ ; -/*!50003 SET character_set_client = utf8mb4 */ ; -/*!50003 SET character_set_results = utf8mb4 */ ; -/*!50003 SET collation_connection = utf8mb4_general_ci */ ; -/*!50003 SET @saved_sql_mode = @@sql_mode */ ; -/*!50003 SET sql_mode = 'STRICT_TRANS_TABLES,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION' */ ; -DELIMITER ;; -CREATE DEFINER=`caosdb`@`%` PROCEDURE `setPassword`(in EntityID INT UNSIGNED, in NewPassword VARCHAR(255)) -BEGIN - - - DELETE FROM passwords where entity_id=EntityID; - INSERT INTO passwords (entity_id, password) VALUES (EntityID, NewPassword); - END ;; DELIMITER ; /*!50003 SET sql_mode = @saved_sql_mode */ ; diff --git a/tests/test_autotap.sql b/tests/test_autotap.sql index 1ae650e346e3cd79e5396af177c6c14ffb875ea1..6109dbe7d39dd62f21232e823fc639d71c5c8a6f 100644 --- a/tests/test_autotap.sql +++ b/tests/test_autotap.sql @@ -2572,13 +2572,6 @@ SELECT tap.procedure_is_deterministic('_caosdb_schema_unit_tests','insertLinCon' SELECT tap.procedure_security_type_is('_caosdb_schema_unit_tests','insertLinCon','DEFINER',''); SELECT tap.procedure_sql_data_access_is('_caosdb_schema_unit_tests','insertLinCon','CONTAINS SQL',''); --- PROCEDURES _caosdb_schema_unit_tests.insertUser - -SELECT tap.has_procedure('_caosdb_schema_unit_tests','insertUser',''); -SELECT tap.procedure_is_deterministic('_caosdb_schema_unit_tests','insertUser','NO',''); -SELECT tap.procedure_security_type_is('_caosdb_schema_unit_tests','insertUser','DEFINER',''); -SELECT tap.procedure_sql_data_access_is('_caosdb_schema_unit_tests','insertUser','CONTAINS SQL',''); - -- PROCEDURES _caosdb_schema_unit_tests.intersectTable SELECT tap.has_procedure('_caosdb_schema_unit_tests','intersectTable',''); diff --git a/utils/make_db b/utils/make_db index 071de0a3270fd0e3480cb54bbc692635983bd287..00a9e82724333f3b46a535e20b367c0604d758f0 100755 --- a/utils/make_db +++ b/utils/make_db @@ -132,8 +132,21 @@ name in your .config file " sed "s/db_5_0/$DATABASE_NAME/g" "$INSTALL_SQL_FILE" | $MYSQL_CMD $(get_mysql_args_nodb) } +function sanity_check_dump() { + SQL_FILE="$1" + DETECTED_VERSION="$(grep -A 5 -i 'CREATE.*FUNCTION.*CaosDBVersion' "$SQL_FILE" | grep -o -E -e "v[0-9][^'\"]*")" + echo "[INFO] Detected schema version of the dump: $DETECTED_VERSION" + AFFECTED_VERSION_REGEX='v([3-9]|2[0-9]|2\.1|2\.0\.[23][0-9]|2\.0\.1[6-9])' + passwords_table_should_be_gone=$(echo "$DETECTED_VERSION" | grep -E -e "$AFFECTED_VERSION_REGEX" -c || true) + if [ "$passwords_table_should_be_gone" -gt "0" ] && grep 'CREATE TABLE `passwords`' -c "$SQL_FILE" > /dev/null ; then + echo "[ERROR] Your dump is broken. The 'passwords' table should be gone: $SQL_FILE" + exit 1 + fi +} + # Inserts the dump (arg 1) into the database function restore_db() { + sanity_check_dump $1 SQL_FILE="$1" $MYSQL_CMD $(get_mysql_args) < "$SQL_FILE" cat <<EOF @@ -256,5 +269,6 @@ case $1 in "test-connection") test-connection ;; "install_db") install_db ;; "restore_db") restore_db $2 ;; + "sanity_check_dump") sanity_check_dump $2 ;; *) echo "Unknown action: $1"; exit 32 esac