diff --git a/.docker/docker-compose.yml b/.docker/docker-compose.yml index 1a63292b080f182821935cb5b20b7500bfcf8f63..8c6555eb457a992f69625c55bfced34d9b1c51fd 100644 --- a/.docker/docker-compose.yml +++ b/.docker/docker-compose.yml @@ -1,7 +1,7 @@ version: '3.6' services: sqldb: - image: mariadb:10.5 + image: mariadb:11.4 volumes: - type: volume source: "caosdb-sqldata" diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 310aad61e9ec2a7f69d9fafbf816e91ce81c02da..561f671767b8d49ad33653f6f6a9ea82142628ca 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -70,24 +70,24 @@ test-dump-update: script: - make test-dump-update -# Run the unit tests with MySQL 5 -unittests-mysql-5: - tags: [ docker ] - stage: test - # Should not stop the pipeline from continuing. - allow_failure: true - services: - - name: mysql:5.7.36 - command: ["--default-authentication-plugin=mysql_native_password"] - - script: - # remove some lines from autotap because the checks of column default - # values don't work with mysql-5 - - sed -i "/col_default_is.*NULL/d" tests/test_autotap.sql - - sed -i "/col_default_is.*INACTIVE/d" tests/test_autotap.sql - - sed -i "/col_default_is.*SHA/d" tests/test_autotap.sql - - sed -i "s/utf8mb3/utf8/" tests/test_autotap.sql - - make pipeline-test SQL_HOST=mysql +# # Run the unit tests with MySQL 5 +# unittests-mysql-5: +# tags: [ docker ] +# stage: test +# # Should not stop the pipeline from continuing. +# allow_failure: true +# services: +# - name: mysql:5.7.36 +# command: ["--default-authentication-plugin=mysql_native_password"] +# +# script: +# # remove some lines from autotap because the checks of column default +# # values don't work with mysql-5 +# - sed -i "/col_default_is.*NULL/d" tests/test_autotap.sql +# - sed -i "/col_default_is.*INACTIVE/d" tests/test_autotap.sql +# - sed -i "/col_default_is.*SHA/d" tests/test_autotap.sql +# - sed -i "s/utf8mb3/utf8/" tests/test_autotap.sql +# - make pipeline-test SQL_HOST=mysql ######## Deploy ######## diff --git a/CHANGELOG.md b/CHANGELOG.md index 5ea239a0d25aca2d3c18f4520fa68a53a5e2512f..e102be5b7c21d102783e38b20f64a7473baa3c87 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,15 +9,25 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added +- New version requirement: MariaDB 11.2 or later. + ### Changed ### Deprecated ### Removed +- MySQL support. See the deprecation in the 7.0.0 release. + ### Fixed -- [SQL dump migration can now be applied twice](https://gitlab.indiscale.com/caosdb/src/caosdb-mysqlbackend/-/issues/60): Fixed an error in the regex. +- [SQL dump migration can now be applied + twice](https://gitlab.indiscale.com/caosdb/src/caosdb-mysqlbackend/-/issues/60): + Fixed an error in the regex. +- [linkahead-server#264](https://gitlab.com/linkahead/linkahead-server/-/issues/264) + Queries ignore accents and umlauts: Changed to more appropriate + collation and character sets. This requires MariaDB 11.2 or later, + see above. ### Security diff --git a/Makefile b/Makefile index 0204e012088c322ade92e529ebabe1b27fe3a469..9bd45f71d56435c68eb953158d0af6d7a8ec8bca 100644 --- a/Makefile +++ b/Makefile @@ -62,10 +62,10 @@ test-dump-update: # Run tests with a database which is started in a MariaDB Docker container .PHONY: test-docker test-docker: - @docker kill caosdb-mysqlserver-test || true - @docker container rm -v caosdb-mysqlserver-test || true - @docker run --name caosdb-mysqlserver-test -p "3306:3306" \ - -e MYSQL_ROOT_PASSWORD="pass-for-test" -d mariadb:10.11 + @docker kill linkahead-mysqlserver-test || true + @docker container rm -v linkahead-mysqlserver-test || true + @docker run --name linkahead-mysqlserver-test -p "3306:3306" \ + -e MYSQL_ROOT_PASSWORD="pass-for-test" -d mariadb:11.4 @sleep 10 MAINPATH=$(realpath tests/docker_env) utils/make_db test --fresh make test-docker-stop @@ -73,8 +73,8 @@ test-docker: # if automatic stopping failed .PHONY: test-docker-stop test-docker-stop: - docker kill caosdb-mysqlserver-test - docker container rm -v caosdb-mysqlserver-test + docker kill linkahead-mysqlserver-test + docker container rm -v linkahead-mysqlserver-test # Compile the standalone documentation .PHONY: doc diff --git a/README.md b/README.md index 7094464dae407a9eff280f2312a7b582f5f29a2b..6b3d3a7c412dfbdb6977b4cb8e7e70266fa2bf37 100644 --- a/README.md +++ b/README.md @@ -3,7 +3,7 @@ ## Welcome -This is the **LinkAhead MySQL Backend** repository and a part of the +This is the **LinkAhead MariaDB Backend** repository and a part of the LinkAhead project. ## Setup @@ -43,7 +43,7 @@ Conduct](https://gitlab.com/linkahead/linkahead/-/blob/main/CODE_OF_CONDUCT.md). * Copyright (C) 2018 Research Group Biomedical Physics, Max Planck Institute for Dynamics and Self-Organization Göttingen. -* Copyright (C) 2020-2021 Indiscale GmbH <info@indiscale.com> +* Copyright (C) 2020-2025 Indiscale GmbH <info@indiscale.com> All files in this repository are licensed under a [GNU Affero General Public License](LICENCE.md) (version 3 or later). diff --git a/README_SETUP.md b/README_SETUP.md index 94a2e9d121631e0189c0b82557b9280e4621e8aa..80340225c252904e0cd753b2fea88f739b343e6c 100644 --- a/README_SETUP.md +++ b/README_SETUP.md @@ -1,8 +1,8 @@ -# Setup of the CaosDB SQL back end +# Setup of the LinkAhead MariaDB backend ## Dependencies -* `MariaDB Client 10.1` or later, `MySQL Client >=5.5, <=5.7.36`. +* `MariaDB Client 11.2` or later. * make ## Create the configuration @@ -11,7 +11,7 @@ parameter that you want to change, add a corresponding line in your `.config` file. You probably want to change the passwords. As the passwords are stored unencrypted in the `.config` file, make sure nobody else can read it. - * If there is no `mysql-config-editor` (`MySQL 5.5`. and `MariaDB`) then the + * If there is no `mysql-config-editor` program, then the `MYSQL_USER_PASSWORD` must be provided, that is the password of the `MYSQL_USER`. * If you are using MariaDB and the `root` database user uses pam @@ -99,20 +99,22 @@ After the first start, you need to install the database: `make install`. Then, you can connect to it with `mariadb --protocol=TCP -u caosdb -prandom1234 caosdb`. +### Developing tests ### -### Troubleshooting - -#### MySQL has failing tests - -Our test suite is developed with MariaDB. That is why some tests, those which -check constraints based on auto-generated names, fail because MySQL generates -other names. +To use the mytap framework: -Also some test fail with the message "Expected: NULL / Was: NULL" which is -obviously what was expected. +```console +$ cd libs; unzip mytap-1.0.zip; cd mytap-1.0 +# Use the necessary mariadb options for connecting to the server in the following commands +$ mariadb ... < mytap.sql # Set up the "tap" database and functions. +$ mariadb ... < scripts/autotap.sql # Insert the "autotap" function. +# Create autotap file (for comparison with current state) +$ mariadb ... --raw -B --skip-column-names -e "call tap.autotap('_caosdb_schema_unit_tests')" > test_autotap.current.sql +# Run the autotap tests (may fail unfortunately with current MariaDB): +$ mariadb ... < test_autotap.current.sql +``` -Please look carefully through the tests. If something more serious than that -comes up, please report a bug. +### Troubleshooting #### Failure to restore a database dump created with an older MariaDB version #### @@ -121,6 +123,11 @@ incompatibilities, the necessary steps to migrate the dump are probably described there and scripts for the migrations are provided in the same directory. +#### MySQL has failing tests + +*Note: Since we switched from MySQL to MariaDB, the two DBMS projects have diverged quite far. As a +result, we no longer support MySQL. + ## Documentation # Build documentation in `build/` with `make doc`. diff --git a/RELEASE_GUIDELINES.md b/RELEASE_GUIDELINES.md index c1d73359114379b74baf19e2bdf35e74be11077f..41c47136c1ab7160578950a384ce9c6ab2182b5b 100644 --- a/RELEASE_GUIDELINES.md +++ b/RELEASE_GUIDELINES.md @@ -1,4 +1,4 @@ -# Release Guidelines for the CaosDB MySQL Backend +# Release Guidelines for the LinkAhead MariaDB backend This document specifies release guidelines in addition to the general release guidelines of the CaosDB Project diff --git a/doc/Maintenance.rst b/doc/Maintenance.rst index 39b2ad35996cd14acad3178f4eada24b1e2c1d21..b8bedee1146009a90a281398fe62d49e1da0351b 100644 --- a/doc/Maintenance.rst +++ b/doc/Maintenance.rst @@ -1,22 +1,19 @@ - Maintenance =========== - Creating a Backup ----------------- - You can use the Python script ``utils/backup.py`` to create a backup (SQL dump) of the SQL-Backend:: ./utils/backup.py -d folder/where/the/backup/is/created -You can do this while CaosDB is online. +You can do this while LinkAhead is online. Restoring a Backup ------------------ -CaosDB should be offline for restoring. +LinkAhead should be offline for restoring. You can use the Bash script ``utils/make_db`` to restore a backup (SQL dump) of the SQL-Backend:: diff --git a/patches/patch20170316-2.0.27/patch.sh b/patches/patch20170316-2.0.27/patch.sh index ecb892e15bd62b673b3447ee7e0d12380b13118b..7a2d0a60f8228628f4c696d4d7a2819d0150bcc8 100755 --- a/patches/patch20170316-2.0.27/patch.sh +++ b/patches/patch20170316-2.0.27/patch.sh @@ -35,7 +35,12 @@ fi check_version $OLD_VERSION -mysql_execute 'DELETE FROM `entities` WHERE id=6; ALTER TABLE `entities` MODIFY COLUMN `role` ENUM("RECORDTYPE","RECORD","FILE","DOMAIN","PROPERTY","DATATYPE","ROLE","QUERYTEMPLATE") NOT NULL; INSERT INTO entities (id, name, description, role, acl) VALUES (8,"QUERYTEMPLATE","The QueryTemplate role.","ROLE",0); UPDATE `entities` SET role="ROLE" WHERE id<100 and name=role; CREATE TABLE `query_template_def` (id INT UNSIGNED PRIMARY KEY, definition MEDIUMTEXT NOT NULL, CONSTRAINT `query_template_def_ibfk_1` FOREIGN KEY (`id`) REFERENCES `entities` (`id`));' +mysql_execute ' +DELETE FROM `entities` WHERE id=6; +ALTER TABLE `entities` MODIFY COLUMN `role` ENUM("RECORDTYPE","RECORD","FILE","DOMAIN","PROPERTY","DATATYPE","ROLE","QUERYTEMPLATE") NOT NULL; +INSERT INTO entities (id, name, description, role, acl) VALUES (8,"QUERYTEMPLATE","The QueryTemplate role.","ROLE",0); +UPDATE `entities` SET role="ROLE" WHERE id<100 and name=role; +CREATE TABLE `query_template_def` (id INT UNSIGNED PRIMARY KEY, definition MEDIUMTEXT NOT NULL, CONSTRAINT `query_template_def_ibfk_1` FOREIGN KEY (`id`) REFERENCES `entities` (`id`));' update_version $NEW_VERSION diff --git a/patches/patch20250130-8.1.0/patch.sh b/patches/patch20250130-8.1.0/patch.sh new file mode 100755 index 0000000000000000000000000000000000000000..5a092150f217c44d100a15735523ef74ea613a8b --- /dev/null +++ b/patches/patch20250130-8.1.0/patch.sh @@ -0,0 +1,99 @@ +#!/bin/bash +# +# This file is a part of the Linkahead Project. +# +# Copyright (C) 2025 IndiScale GmbH <info@indiscale.com> +# Copyright (C) 2025 Daniel Hornung <d.hornung@indiscale.com> +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see <https://www.gnu.org/licenses/>. +# + +# Update mysql schema to version v8.1.0 +# +# Set the collation so that accents on characters become relevant. + +OLD_VERSION="v8.0.0" +NEW_VERSION="v8.1.0" + +if [ -z "$UTILSPATH" ]; then + UTILSPATH="../utils" +fi + +. $UTILSPATH/patch_header.sh $* + +check_version $OLD_VERSION + +# Update charsets and collations. + +# Change database default. +mysql_execute "ALTER DATABASE \`$DATABASE_NAME\` COLLATE = 'uca1400_as_ci';" + +# Some columns should be changed manually to prevent text length changes. +mysql_execute "ALTER TABLE desc_overrides MODIFY description TEXT CHARACTER SET utf8mb4;" +mysql_execute "ALTER TABLE entities MODIFY description TEXT CHARACTER SET utf8mb4;" +mysql_execute "ALTER TABLE permissions MODIFY permissions MEDIUMTEXT CHARACTER SET utf8mb4 NOT NULL;" +mysql_execute "ALTER TABLE query_template_def MODIFY definition MEDIUMTEXT CHARACTER SET utf8mb4 NOT NULL;" +mysql_execute "ALTER TABLE roles MODIFY description MEDIUMTEXT CHARACTER SET utf8mb4;" +mysql_execute "ALTER TABLE text_data MODIFY value TEXT CHARACTER SET utf8mb4 NOT NULL;" + +# Remove some constraints first. +mysql_execute 'ALTER TABLE user_info DROP FOREIGN KEY `subjects_ibfk_2`;' + +# Get all tables, drop first line, take first column +tables=$(mysql_execute "SHOW TABLE status where Collation='utf8mb3_unicode_ci';" -B | tail +2 | awk '{print $1}') +for table in $tables; do + mysql_execute "ALTER TABLE $table CONVERT TO CHARACTER SET utf8mb4 COLLATE uca1400_as_ci;" +done + +# Add constraints again. +mysql_execute 'ALTER TABLE user_info MODIFY COLUMN `entity` VARCHAR(255) COLLATE utf8mb4_bin DEFAULT NULL;' +mysql_execute 'ALTER TABLE user_info ADD CONSTRAINT `subjects_ibfk_2` FOREIGN KEY (`entity`) REFERENCES `entity_ids` (`id`);' + +# Set custom collations for columns +mysql_execute "ALTER TABLE transaction_log MODIFY entity_id VARCHAR(255) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL;" + + +### Finished ### + +update_version $NEW_VERSION + +success + + +##### Useful commands ##### + +# cd git/caosdb-mysqlbackend/ +# UTILSPATH=utils +# . utils/patch_header.sh + + +# show table status where Collation='utf8mb3_unicode_ci'; + +# name_data +# text_data + +# show table status like 'name_data'; +# select * from name_data where value like 'Yield%'; +# update name_data SET value='Yield-stréss' where entity_id=354; +# update name_data SET value='Yield-strés😋s' where entity_id=354; +# select * from name_data where value='Yield-stress'; + +# # Unicode 14.0.0 +# ALTER TABLE name_data CONVERT TO CHARACTER SET utf8mb4 COLLATE uca1400_as_ci; + +# FULL_COLLATION_NAME: utf8mb4_uca1400_as_ci; +# COLLATION_NAME: uca1400_as_ci; + +# mariadb -h sqldb -u caosdb -prandom1234 --default-character-set=utf8mb4 caosdb + diff --git a/tests/test_autotap.sql b/tests/test_autotap.sql index 168758c1482d21345e025f3d3ca2e33a9e0c607c..6d62a0ced953fddc7afe1ccc06d45ef339e8f47e 100644 --- a/tests/test_autotap.sql +++ b/tests/test_autotap.sql @@ -38,7 +38,11 @@ SELECT tap.has_schema('_caosdb_schema_unit_tests',''); -- *************************************************************** SELECT tap.has_table('_caosdb_schema_unit_tests','archive_isa',''); -SELECT tap.table_collation_is('_caosdb_schema_unit_tests','archive_isa','utf8mb3_unicode_ci',''); +-- ******* +-- Collation testing of tables fails for uca1400 collations, see upstream issue: +-- https://github.com/hepabolu/mytap/issues/58 +-- ******* +-- SELECT tap.table_collation_is('_caosdb_schema_unit_tests','archive_isa','utf8mb4_uca1400_as_ci',''); SELECT tap.table_engine_is('_caosdb_schema_unit_tests','archive_isa','InnoDB',''); -- COLUMNS @@ -116,7 +120,7 @@ SELECT tap.fk_on_update('_caosdb_schema_unit_tests','archive_isa','archive_isa_i -- *************************************************************** SELECT tap.has_table('_caosdb_schema_unit_tests','collection_type',''); -SELECT tap.table_collation_is('_caosdb_schema_unit_tests','collection_type','utf8mb3_unicode_ci',''); +-- SELECT tap.table_collation_is('_caosdb_schema_unit_tests','collection_type','utf8mb4_uca1400_as_ci',''); SELECT tap.table_engine_is('_caosdb_schema_unit_tests','collection_type','InnoDB',''); -- COLUMNS @@ -155,8 +159,8 @@ SELECT tap.has_column('_caosdb_schema_unit_tests','collection_type','collection' SELECT tap.col_column_type_is('_caosdb_schema_unit_tests','collection_type','collection','varchar(255)',''); SELECT tap.col_extra_is('_caosdb_schema_unit_tests','collection_type','collection','',''); SELECT tap.col_default_is('_caosdb_schema_unit_tests','collection_type','collection',NULL,''); -SELECT tap.col_charset_is('_caosdb_schema_unit_tests','collection_type','collection','utf8mb3',''); -SELECT tap.col_collation_is('_caosdb_schema_unit_tests','collection_type','collection','utf8mb3_unicode_ci',''); +SELECT tap.col_charset_is('_caosdb_schema_unit_tests','collection_type','collection','utf8mb4',''); +SELECT tap.col_collation_is('_caosdb_schema_unit_tests','collection_type','collection','utf8mb4_uca1400_as_ci',''); -- INDEXES SELECT tap.indexes_are('_caosdb_schema_unit_tests','collection_type','`domain_id`,`entity_id`,`property_id`',''); @@ -208,7 +212,7 @@ SELECT tap.fk_on_update('_caosdb_schema_unit_tests','collection_type','collectio -- *************************************************************** SELECT tap.has_table('_caosdb_schema_unit_tests','data_type',''); -SELECT tap.table_collation_is('_caosdb_schema_unit_tests','data_type','utf8mb3_unicode_ci',''); +-- SELECT tap.table_collation_is('_caosdb_schema_unit_tests','data_type','utf8mb4_uca1400_as_ci',''); SELECT tap.table_engine_is('_caosdb_schema_unit_tests','data_type','InnoDB',''); -- COLUMNS @@ -300,7 +304,7 @@ SELECT tap.constraint_type_is('_caosdb_schema_unit_tests','data_type','datatype_ -- *************************************************************** SELECT tap.has_table('_caosdb_schema_unit_tests','date_data',''); -SELECT tap.table_collation_is('_caosdb_schema_unit_tests','date_data','utf8mb3_unicode_ci',''); +-- SELECT tap.table_collation_is('_caosdb_schema_unit_tests','date_data','utf8mb4_uca1400_as_ci',''); SELECT tap.table_engine_is('_caosdb_schema_unit_tests','date_data','InnoDB',''); -- COLUMNS @@ -348,8 +352,8 @@ SELECT tap.has_column('_caosdb_schema_unit_tests','date_data','status',''); SELECT tap.col_column_type_is('_caosdb_schema_unit_tests','date_data','status','enum(\'OBLIGATORY\',\'RECOMMENDED\',\'SUGGESTED\',\'FIX\')',''); SELECT tap.col_extra_is('_caosdb_schema_unit_tests','date_data','status','',''); SELECT tap.col_default_is('_caosdb_schema_unit_tests','date_data','status','NULL',''); -SELECT tap.col_charset_is('_caosdb_schema_unit_tests','date_data','status','utf8mb3',''); -SELECT tap.col_collation_is('_caosdb_schema_unit_tests','date_data','status','utf8mb3_unicode_ci',''); +SELECT tap.col_charset_is('_caosdb_schema_unit_tests','date_data','status','utf8mb4',''); +SELECT tap.col_collation_is('_caosdb_schema_unit_tests','date_data','status','utf8mb4_uca1400_as_ci',''); -- COLUMN date_data.pidx @@ -398,7 +402,7 @@ SELECT tap.fk_on_update('_caosdb_schema_unit_tests','date_data','date_ov_forkey_ -- *************************************************************** SELECT tap.has_table('_caosdb_schema_unit_tests','datetime_data',''); -SELECT tap.table_collation_is('_caosdb_schema_unit_tests','datetime_data','utf8mb3_unicode_ci',''); +-- SELECT tap.table_collation_is('_caosdb_schema_unit_tests','datetime_data','utf8mb4_uca1400_as_ci',''); SELECT tap.table_engine_is('_caosdb_schema_unit_tests','datetime_data','InnoDB',''); -- COLUMNS @@ -437,8 +441,8 @@ SELECT tap.has_column('_caosdb_schema_unit_tests','datetime_data','status',''); SELECT tap.col_column_type_is('_caosdb_schema_unit_tests','datetime_data','status','enum(\'OBLIGATORY\',\'RECOMMENDED\',\'SUGGESTED\',\'FIX\',\'REPLACEMENT\')',''); SELECT tap.col_extra_is('_caosdb_schema_unit_tests','datetime_data','status','',''); SELECT tap.col_default_is('_caosdb_schema_unit_tests','datetime_data','status',NULL,''); -SELECT tap.col_charset_is('_caosdb_schema_unit_tests','datetime_data','status','utf8mb3',''); -SELECT tap.col_collation_is('_caosdb_schema_unit_tests','datetime_data','status','utf8mb3_unicode_ci',''); +SELECT tap.col_charset_is('_caosdb_schema_unit_tests','datetime_data','status','utf8mb4',''); +SELECT tap.col_collation_is('_caosdb_schema_unit_tests','datetime_data','status','utf8mb4_uca1400_as_ci',''); -- COLUMN datetime_data.pidx @@ -505,7 +509,7 @@ SELECT tap.fk_on_update('_caosdb_schema_unit_tests','datetime_data','dat_propert -- *************************************************************** SELECT tap.has_table('_caosdb_schema_unit_tests','desc_overrides',''); -SELECT tap.table_collation_is('_caosdb_schema_unit_tests','desc_overrides','utf8mb3_unicode_ci',''); +-- SELECT tap.table_collation_is('_caosdb_schema_unit_tests','desc_overrides','utf8mb4_uca1400_as_ci',''); SELECT tap.table_engine_is('_caosdb_schema_unit_tests','desc_overrides','InnoDB',''); -- COLUMNS @@ -544,8 +548,8 @@ SELECT tap.has_column('_caosdb_schema_unit_tests','desc_overrides','description' SELECT tap.col_column_type_is('_caosdb_schema_unit_tests','desc_overrides','description','text',''); SELECT tap.col_extra_is('_caosdb_schema_unit_tests','desc_overrides','description','',''); SELECT tap.col_default_is('_caosdb_schema_unit_tests','desc_overrides','description','NULL',''); -SELECT tap.col_charset_is('_caosdb_schema_unit_tests','desc_overrides','description','utf8mb3',''); -SELECT tap.col_collation_is('_caosdb_schema_unit_tests','desc_overrides','description','utf8mb3_unicode_ci',''); +SELECT tap.col_charset_is('_caosdb_schema_unit_tests','desc_overrides','description','utf8mb4',''); +SELECT tap.col_collation_is('_caosdb_schema_unit_tests','desc_overrides','description','utf8mb4_uca1400_as_ci',''); -- INDEXES SELECT tap.indexes_are('_caosdb_schema_unit_tests','desc_overrides','`desc_ov_dom_ent_idx`',''); @@ -590,7 +594,7 @@ SELECT tap.constraint_type_is('_caosdb_schema_unit_tests','desc_overrides','desc -- *************************************************************** SELECT tap.has_table('_caosdb_schema_unit_tests','double_data',''); -SELECT tap.table_collation_is('_caosdb_schema_unit_tests','double_data','utf8mb3_unicode_ci',''); +-- SELECT tap.table_collation_is('_caosdb_schema_unit_tests','double_data','utf8mb4_uca1400_as_ci',''); SELECT tap.table_engine_is('_caosdb_schema_unit_tests','double_data','InnoDB',''); -- COLUMNS @@ -638,8 +642,8 @@ SELECT tap.has_column('_caosdb_schema_unit_tests','double_data','status',''); SELECT tap.col_column_type_is('_caosdb_schema_unit_tests','double_data','status','enum(\'OBLIGATORY\',\'RECOMMENDED\',\'SUGGESTED\',\'FIX\',\'REPLACEMENT\')',''); SELECT tap.col_extra_is('_caosdb_schema_unit_tests','double_data','status','',''); SELECT tap.col_default_is('_caosdb_schema_unit_tests','double_data','status',NULL,''); -SELECT tap.col_charset_is('_caosdb_schema_unit_tests','double_data','status','utf8mb3',''); -SELECT tap.col_collation_is('_caosdb_schema_unit_tests','double_data','status','utf8mb3_unicode_ci',''); +SELECT tap.col_charset_is('_caosdb_schema_unit_tests','double_data','status','utf8mb4',''); +SELECT tap.col_collation_is('_caosdb_schema_unit_tests','double_data','status','utf8mb4_uca1400_as_ci',''); -- COLUMN double_data.pidx @@ -697,7 +701,7 @@ SELECT tap.fk_on_update('_caosdb_schema_unit_tests','double_data','dou_property_ -- *************************************************************** SELECT tap.has_table('_caosdb_schema_unit_tests','entities',''); -SELECT tap.table_collation_is('_caosdb_schema_unit_tests','entities','utf8mb3_unicode_ci',''); +-- SELECT tap.table_collation_is('_caosdb_schema_unit_tests','entities','utf8mb4_uca1400_as_ci',''); SELECT tap.table_engine_is('_caosdb_schema_unit_tests','entities','InnoDB',''); -- COLUMNS @@ -718,8 +722,8 @@ SELECT tap.has_column('_caosdb_schema_unit_tests','entities','description',''); SELECT tap.col_column_type_is('_caosdb_schema_unit_tests','entities','description','text',''); SELECT tap.col_extra_is('_caosdb_schema_unit_tests','entities','description','',''); SELECT tap.col_default_is('_caosdb_schema_unit_tests','entities','description','NULL',''); -SELECT tap.col_charset_is('_caosdb_schema_unit_tests','entities','description','utf8mb3',''); -SELECT tap.col_collation_is('_caosdb_schema_unit_tests','entities','description','utf8mb3_unicode_ci',''); +SELECT tap.col_charset_is('_caosdb_schema_unit_tests','entities','description','utf8mb4',''); +SELECT tap.col_collation_is('_caosdb_schema_unit_tests','entities','description','utf8mb4_uca1400_as_ci',''); -- COLUMN entities.role @@ -727,8 +731,8 @@ SELECT tap.has_column('_caosdb_schema_unit_tests','entities','role',''); SELECT tap.col_column_type_is('_caosdb_schema_unit_tests','entities','role','enum(\'RECORDTYPE\',\'RECORD\',\'FILE\',\'_REPLACEMENT\',\'PROPERTY\',\'DATATYPE\',\'ROLE\',\'QUERYTEMPLATE\')',''); SELECT tap.col_extra_is('_caosdb_schema_unit_tests','entities','role','',''); SELECT tap.col_default_is('_caosdb_schema_unit_tests','entities','role',NULL,''); -SELECT tap.col_charset_is('_caosdb_schema_unit_tests','entities','role','utf8mb3',''); -SELECT tap.col_collation_is('_caosdb_schema_unit_tests','entities','role','utf8mb3_unicode_ci',''); +SELECT tap.col_charset_is('_caosdb_schema_unit_tests','entities','role','utf8mb4',''); +SELECT tap.col_collation_is('_caosdb_schema_unit_tests','entities','role','utf8mb4_uca1400_as_ci',''); -- COLUMN entities.acl @@ -760,7 +764,7 @@ SELECT tap.col_is_pk('_caosdb_schema_unit_tests','entities','`id`',''); -- *************************************************************** SELECT tap.has_table('_caosdb_schema_unit_tests','entity_acl',''); -SELECT tap.table_collation_is('_caosdb_schema_unit_tests','entity_acl','utf8mb3_unicode_ci',''); +-- SELECT tap.table_collation_is('_caosdb_schema_unit_tests','entity_acl','utf8mb4_uca1400_as_ci',''); SELECT tap.table_engine_is('_caosdb_schema_unit_tests','entity_acl','InnoDB',''); -- COLUMNS @@ -807,7 +811,7 @@ SELECT tap.col_is_pk('_caosdb_schema_unit_tests','entity_acl','`id`',''); -- *************************************************************** SELECT tap.has_table('_caosdb_schema_unit_tests','enum_data',''); -SELECT tap.table_collation_is('_caosdb_schema_unit_tests','enum_data','utf8mb3_unicode_ci',''); +-- SELECT tap.table_collation_is('_caosdb_schema_unit_tests','enum_data','utf8mb4_uca1400_as_ci',''); SELECT tap.table_engine_is('_caosdb_schema_unit_tests','enum_data','InnoDB',''); -- COLUMNS @@ -855,8 +859,8 @@ SELECT tap.has_column('_caosdb_schema_unit_tests','enum_data','status',''); SELECT tap.col_column_type_is('_caosdb_schema_unit_tests','enum_data','status','enum(\'OBLIGATORY\',\'RECOMMENDED\',\'SUGGESTED\',\'FIX\')',''); SELECT tap.col_extra_is('_caosdb_schema_unit_tests','enum_data','status','',''); SELECT tap.col_default_is('_caosdb_schema_unit_tests','enum_data','status','NULL',''); -SELECT tap.col_charset_is('_caosdb_schema_unit_tests','enum_data','status','utf8mb3',''); -SELECT tap.col_collation_is('_caosdb_schema_unit_tests','enum_data','status','utf8mb3_unicode_ci',''); +SELECT tap.col_charset_is('_caosdb_schema_unit_tests','enum_data','status','utf8mb4',''); +SELECT tap.col_collation_is('_caosdb_schema_unit_tests','enum_data','status','utf8mb4_uca1400_as_ci',''); -- COLUMN enum_data.pidx @@ -905,7 +909,7 @@ SELECT tap.fk_on_update('_caosdb_schema_unit_tests','enum_data','enum_ov_forkey_ -- *************************************************************** SELECT tap.has_table('_caosdb_schema_unit_tests','files',''); -SELECT tap.table_collation_is('_caosdb_schema_unit_tests','files','utf8mb3_unicode_ci',''); +-- SELECT tap.table_collation_is('_caosdb_schema_unit_tests','files','utf8mb4_uca1400_as_ci',''); SELECT tap.table_engine_is('_caosdb_schema_unit_tests','files','InnoDB',''); -- COLUMNS @@ -926,8 +930,8 @@ SELECT tap.has_column('_caosdb_schema_unit_tests','files','path',''); SELECT tap.col_column_type_is('_caosdb_schema_unit_tests','files','path','varchar(255)',''); SELECT tap.col_extra_is('_caosdb_schema_unit_tests','files','path','',''); SELECT tap.col_default_is('_caosdb_schema_unit_tests','files','path',NULL,''); -SELECT tap.col_charset_is('_caosdb_schema_unit_tests','files','path','utf8mb3',''); -SELECT tap.col_collation_is('_caosdb_schema_unit_tests','files','path','utf8mb3_unicode_ci',''); +SELECT tap.col_charset_is('_caosdb_schema_unit_tests','files','path','utf8mb4',''); +SELECT tap.col_collation_is('_caosdb_schema_unit_tests','files','path','utf8mb4_uca1400_as_ci',''); -- COLUMN files.size @@ -977,7 +981,7 @@ SELECT tap.col_is_pk('_caosdb_schema_unit_tests','files','`file_id`',''); -- *************************************************************** SELECT tap.has_table('_caosdb_schema_unit_tests','integer_data',''); -SELECT tap.table_collation_is('_caosdb_schema_unit_tests','integer_data','utf8mb3_unicode_ci',''); +-- SELECT tap.table_collation_is('_caosdb_schema_unit_tests','integer_data','utf8mb4_uca1400_as_ci',''); SELECT tap.table_engine_is('_caosdb_schema_unit_tests','integer_data','InnoDB',''); -- COLUMNS @@ -1025,8 +1029,8 @@ SELECT tap.has_column('_caosdb_schema_unit_tests','integer_data','status',''); SELECT tap.col_column_type_is('_caosdb_schema_unit_tests','integer_data','status','enum(\'OBLIGATORY\',\'RECOMMENDED\',\'SUGGESTED\',\'FIX\',\'REPLACEMENT\')',''); SELECT tap.col_extra_is('_caosdb_schema_unit_tests','integer_data','status','',''); SELECT tap.col_default_is('_caosdb_schema_unit_tests','integer_data','status',NULL,''); -SELECT tap.col_charset_is('_caosdb_schema_unit_tests','integer_data','status','utf8mb3',''); -SELECT tap.col_collation_is('_caosdb_schema_unit_tests','integer_data','status','utf8mb3_unicode_ci',''); +SELECT tap.col_charset_is('_caosdb_schema_unit_tests','integer_data','status','utf8mb4',''); +SELECT tap.col_collation_is('_caosdb_schema_unit_tests','integer_data','status','utf8mb4_uca1400_as_ci',''); -- COLUMN integer_data.pidx @@ -1084,7 +1088,7 @@ SELECT tap.fk_on_update('_caosdb_schema_unit_tests','integer_data','int_property -- *************************************************************** SELECT tap.has_table('_caosdb_schema_unit_tests','isa_cache',''); -SELECT tap.table_collation_is('_caosdb_schema_unit_tests','isa_cache','utf8mb3_unicode_ci',''); +-- SELECT tap.table_collation_is('_caosdb_schema_unit_tests','isa_cache','utf8mb4_uca1400_as_ci',''); SELECT tap.table_engine_is('_caosdb_schema_unit_tests','isa_cache','InnoDB',''); -- COLUMNS @@ -1114,8 +1118,8 @@ SELECT tap.has_column('_caosdb_schema_unit_tests','isa_cache','rpath',''); SELECT tap.col_column_type_is('_caosdb_schema_unit_tests','isa_cache','rpath','varchar(255)',''); SELECT tap.col_extra_is('_caosdb_schema_unit_tests','isa_cache','rpath','',''); SELECT tap.col_default_is('_caosdb_schema_unit_tests','isa_cache','rpath',NULL,''); -SELECT tap.col_charset_is('_caosdb_schema_unit_tests','isa_cache','rpath','utf8mb3',''); -SELECT tap.col_collation_is('_caosdb_schema_unit_tests','isa_cache','rpath','utf8mb3_unicode_ci',''); +SELECT tap.col_charset_is('_caosdb_schema_unit_tests','isa_cache','rpath','utf8mb4',''); +SELECT tap.col_collation_is('_caosdb_schema_unit_tests','isa_cache','rpath','utf8mb4_uca1400_as_ci',''); -- CONSTRAINTS SELECT tap.constraints_are('_caosdb_schema_unit_tests','isa_cache','`PRIMARY`,`isa_cache_child_entity`,`isa_cache_parent_entity`',''); @@ -1145,7 +1149,7 @@ SELECT tap.col_is_pk('_caosdb_schema_unit_tests','isa_cache','`child`,`parent`,` -- *************************************************************** SELECT tap.has_table('_caosdb_schema_unit_tests','name_data',''); -SELECT tap.table_collation_is('_caosdb_schema_unit_tests','name_data','utf8mb3_unicode_ci',''); +-- SELECT tap.table_collation_is('_caosdb_schema_unit_tests','name_data','utf8mb4_uca1400_as_ci',''); SELECT tap.table_engine_is('_caosdb_schema_unit_tests','name_data','InnoDB',''); -- COLUMNS @@ -1184,8 +1188,8 @@ SELECT tap.has_column('_caosdb_schema_unit_tests','name_data','value',''); SELECT tap.col_column_type_is('_caosdb_schema_unit_tests','name_data','value','varchar(255)',''); SELECT tap.col_extra_is('_caosdb_schema_unit_tests','name_data','value','',''); SELECT tap.col_default_is('_caosdb_schema_unit_tests','name_data','value',NULL,''); -SELECT tap.col_charset_is('_caosdb_schema_unit_tests','name_data','value','utf8mb3',''); -SELECT tap.col_collation_is('_caosdb_schema_unit_tests','name_data','value','utf8mb3_unicode_ci',''); +SELECT tap.col_charset_is('_caosdb_schema_unit_tests','name_data','value','utf8mb4',''); +SELECT tap.col_collation_is('_caosdb_schema_unit_tests','name_data','value','utf8mb4_uca1400_as_ci',''); -- COLUMN name_data.status @@ -1193,8 +1197,8 @@ SELECT tap.has_column('_caosdb_schema_unit_tests','name_data','status',''); SELECT tap.col_column_type_is('_caosdb_schema_unit_tests','name_data','status','enum(\'OBLIGATORY\',\'RECOMMENDED\',\'SUGGESTED\',\'FIX\',\'REPLACEMENT\')',''); SELECT tap.col_extra_is('_caosdb_schema_unit_tests','name_data','status','',''); SELECT tap.col_default_is('_caosdb_schema_unit_tests','name_data','status',NULL,''); -SELECT tap.col_charset_is('_caosdb_schema_unit_tests','name_data','status','utf8mb3',''); -SELECT tap.col_collation_is('_caosdb_schema_unit_tests','name_data','status','utf8mb3_unicode_ci',''); +SELECT tap.col_charset_is('_caosdb_schema_unit_tests','name_data','status','utf8mb4',''); +SELECT tap.col_collation_is('_caosdb_schema_unit_tests','name_data','status','utf8mb4_uca1400_as_ci',''); -- COLUMN name_data.pidx @@ -1266,7 +1270,7 @@ SELECT tap.fk_on_update('_caosdb_schema_unit_tests','name_data','name_data_prope -- *************************************************************** SELECT tap.has_table('_caosdb_schema_unit_tests','name_overrides',''); -SELECT tap.table_collation_is('_caosdb_schema_unit_tests','name_overrides','utf8mb3_unicode_ci',''); +-- SELECT tap.table_collation_is('_caosdb_schema_unit_tests','name_overrides','utf8mb4_uca1400_as_ci',''); SELECT tap.table_engine_is('_caosdb_schema_unit_tests','name_overrides','InnoDB',''); -- COLUMNS @@ -1305,8 +1309,8 @@ SELECT tap.has_column('_caosdb_schema_unit_tests','name_overrides','name',''); SELECT tap.col_column_type_is('_caosdb_schema_unit_tests','name_overrides','name','varchar(255)',''); SELECT tap.col_extra_is('_caosdb_schema_unit_tests','name_overrides','name','',''); SELECT tap.col_default_is('_caosdb_schema_unit_tests','name_overrides','name','NULL',''); -SELECT tap.col_charset_is('_caosdb_schema_unit_tests','name_overrides','name','utf8mb3',''); -SELECT tap.col_collation_is('_caosdb_schema_unit_tests','name_overrides','name','utf8mb3_unicode_ci',''); +SELECT tap.col_charset_is('_caosdb_schema_unit_tests','name_overrides','name','utf8mb4',''); +SELECT tap.col_collation_is('_caosdb_schema_unit_tests','name_overrides','name','utf8mb4_uca1400_as_ci',''); -- INDEXES SELECT tap.indexes_are('_caosdb_schema_unit_tests','name_overrides','`name_ov_dom_ent_idx`',''); @@ -1351,7 +1355,7 @@ SELECT tap.constraint_type_is('_caosdb_schema_unit_tests','name_overrides','name -- *************************************************************** SELECT tap.has_table('_caosdb_schema_unit_tests','null_data',''); -SELECT tap.table_collation_is('_caosdb_schema_unit_tests','null_data','utf8mb3_unicode_ci',''); +-- SELECT tap.table_collation_is('_caosdb_schema_unit_tests','null_data','utf8mb4_uca1400_as_ci',''); SELECT tap.table_engine_is('_caosdb_schema_unit_tests','null_data','InnoDB',''); -- COLUMNS @@ -1390,8 +1394,8 @@ SELECT tap.has_column('_caosdb_schema_unit_tests','null_data','status',''); SELECT tap.col_column_type_is('_caosdb_schema_unit_tests','null_data','status','enum(\'OBLIGATORY\',\'RECOMMENDED\',\'SUGGESTED\',\'FIX\')',''); SELECT tap.col_extra_is('_caosdb_schema_unit_tests','null_data','status','',''); SELECT tap.col_default_is('_caosdb_schema_unit_tests','null_data','status','NULL',''); -SELECT tap.col_charset_is('_caosdb_schema_unit_tests','null_data','status','utf8mb3',''); -SELECT tap.col_collation_is('_caosdb_schema_unit_tests','null_data','status','utf8mb3_unicode_ci',''); +SELECT tap.col_charset_is('_caosdb_schema_unit_tests','null_data','status','utf8mb4',''); +SELECT tap.col_collation_is('_caosdb_schema_unit_tests','null_data','status','utf8mb4_uca1400_as_ci',''); -- COLUMN null_data.pidx @@ -1440,7 +1444,7 @@ SELECT tap.fk_on_update('_caosdb_schema_unit_tests','null_data','null_forkey_pro -- *************************************************************** SELECT tap.has_table('_caosdb_schema_unit_tests','passwd',''); -SELECT tap.table_collation_is('_caosdb_schema_unit_tests','passwd','utf8mb3_unicode_ci',''); +-- SELECT tap.table_collation_is('_caosdb_schema_unit_tests','passwd','utf8mb4_uca1400_as_ci',''); SELECT tap.table_engine_is('_caosdb_schema_unit_tests','passwd','InnoDB',''); -- COLUMNS @@ -1470,8 +1474,8 @@ SELECT tap.has_column('_caosdb_schema_unit_tests','passwd','alg',''); SELECT tap.col_column_type_is('_caosdb_schema_unit_tests','passwd','alg','varchar(255)',''); SELECT tap.col_extra_is('_caosdb_schema_unit_tests','passwd','alg','',''); SELECT tap.col_default_is('_caosdb_schema_unit_tests','passwd','alg','\'SHA-512\'',''); -SELECT tap.col_charset_is('_caosdb_schema_unit_tests','passwd','alg','utf8mb3',''); -SELECT tap.col_collation_is('_caosdb_schema_unit_tests','passwd','alg','utf8mb3_unicode_ci',''); +SELECT tap.col_charset_is('_caosdb_schema_unit_tests','passwd','alg','utf8mb4',''); +SELECT tap.col_collation_is('_caosdb_schema_unit_tests','passwd','alg','utf8mb4_uca1400_as_ci',''); -- COLUMN passwd.it @@ -1505,7 +1509,7 @@ SELECT tap.col_is_pk('_caosdb_schema_unit_tests','passwd','`principal`',''); -- *************************************************************** SELECT tap.has_table('_caosdb_schema_unit_tests','permissions',''); -SELECT tap.table_collation_is('_caosdb_schema_unit_tests','permissions','utf8mb3_unicode_ci',''); +-- SELECT tap.table_collation_is('_caosdb_schema_unit_tests','permissions','utf8mb4_uca1400_as_ci',''); SELECT tap.table_engine_is('_caosdb_schema_unit_tests','permissions','InnoDB',''); -- COLUMNS @@ -1526,8 +1530,8 @@ SELECT tap.has_column('_caosdb_schema_unit_tests','permissions','permissions','' SELECT tap.col_column_type_is('_caosdb_schema_unit_tests','permissions','permissions','mediumtext',''); SELECT tap.col_extra_is('_caosdb_schema_unit_tests','permissions','permissions','',''); SELECT tap.col_default_is('_caosdb_schema_unit_tests','permissions','permissions',NULL,''); -SELECT tap.col_charset_is('_caosdb_schema_unit_tests','permissions','permissions','utf8mb3',''); -SELECT tap.col_collation_is('_caosdb_schema_unit_tests','permissions','permissions','utf8mb3_unicode_ci',''); +SELECT tap.col_charset_is('_caosdb_schema_unit_tests','permissions','permissions','utf8mb4',''); +SELECT tap.col_collation_is('_caosdb_schema_unit_tests','permissions','permissions','utf8mb4_uca1400_as_ci',''); -- CONSTRAINTS SELECT tap.constraints_are('_caosdb_schema_unit_tests','permissions','`PRIMARY`,`perm_name_roles`',''); @@ -1550,7 +1554,7 @@ SELECT tap.col_is_pk('_caosdb_schema_unit_tests','permissions','`role`',''); -- *************************************************************** SELECT tap.has_table('_caosdb_schema_unit_tests','query_template_def',''); -SELECT tap.table_collation_is('_caosdb_schema_unit_tests','query_template_def','utf8mb3_unicode_ci',''); +-- SELECT tap.table_collation_is('_caosdb_schema_unit_tests','query_template_def','utf8mb4_uca1400_as_ci',''); SELECT tap.table_engine_is('_caosdb_schema_unit_tests','query_template_def','InnoDB',''); -- COLUMNS @@ -1571,8 +1575,8 @@ SELECT tap.has_column('_caosdb_schema_unit_tests','query_template_def','definiti SELECT tap.col_column_type_is('_caosdb_schema_unit_tests','query_template_def','definition','mediumtext',''); SELECT tap.col_extra_is('_caosdb_schema_unit_tests','query_template_def','definition','',''); SELECT tap.col_default_is('_caosdb_schema_unit_tests','query_template_def','definition',NULL,''); -SELECT tap.col_charset_is('_caosdb_schema_unit_tests','query_template_def','definition','utf8mb3',''); -SELECT tap.col_collation_is('_caosdb_schema_unit_tests','query_template_def','definition','utf8mb3_unicode_ci',''); +SELECT tap.col_charset_is('_caosdb_schema_unit_tests','query_template_def','definition','utf8mb4',''); +SELECT tap.col_collation_is('_caosdb_schema_unit_tests','query_template_def','definition','utf8mb4_uca1400_as_ci',''); -- CONSTRAINTS SELECT tap.constraints_are('_caosdb_schema_unit_tests','query_template_def','`PRIMARY`,`query_template_def_ibfk_1`',''); @@ -1595,7 +1599,7 @@ SELECT tap.fk_on_update('_caosdb_schema_unit_tests','query_template_def','query_ -- *************************************************************** SELECT tap.has_table('_caosdb_schema_unit_tests','reference_data',''); -SELECT tap.table_collation_is('_caosdb_schema_unit_tests','reference_data','utf8mb3_unicode_ci',''); +-- SELECT tap.table_collation_is('_caosdb_schema_unit_tests','reference_data','utf8mb4_uca1400_as_ci',''); SELECT tap.table_engine_is('_caosdb_schema_unit_tests','reference_data','InnoDB',''); -- COLUMNS @@ -1643,8 +1647,8 @@ SELECT tap.has_column('_caosdb_schema_unit_tests','reference_data','status',''); SELECT tap.col_column_type_is('_caosdb_schema_unit_tests','reference_data','status','enum(\'OBLIGATORY\',\'RECOMMENDED\',\'SUGGESTED\',\'FIX\',\'REPLACEMENT\')',''); SELECT tap.col_extra_is('_caosdb_schema_unit_tests','reference_data','status','',''); SELECT tap.col_default_is('_caosdb_schema_unit_tests','reference_data','status',NULL,''); -SELECT tap.col_charset_is('_caosdb_schema_unit_tests','reference_data','status','utf8mb3',''); -SELECT tap.col_collation_is('_caosdb_schema_unit_tests','reference_data','status','utf8mb3_unicode_ci',''); +SELECT tap.col_charset_is('_caosdb_schema_unit_tests','reference_data','status','utf8mb4',''); +SELECT tap.col_collation_is('_caosdb_schema_unit_tests','reference_data','status','utf8mb4_uca1400_as_ci',''); -- COLUMN reference_data.pidx @@ -1722,7 +1726,7 @@ SELECT tap.fk_on_update('_caosdb_schema_unit_tests','reference_data','reference_ -- *************************************************************** SELECT tap.has_table('_caosdb_schema_unit_tests','roles',''); -SELECT tap.table_collation_is('_caosdb_schema_unit_tests','roles','utf8mb3_unicode_ci',''); +-- SELECT tap.table_collation_is('_caosdb_schema_unit_tests','roles','utf8mb4_uca1400_as_ci',''); SELECT tap.table_engine_is('_caosdb_schema_unit_tests','roles','InnoDB',''); -- COLUMNS @@ -1743,8 +1747,8 @@ SELECT tap.has_column('_caosdb_schema_unit_tests','roles','description',''); SELECT tap.col_column_type_is('_caosdb_schema_unit_tests','roles','description','mediumtext',''); SELECT tap.col_extra_is('_caosdb_schema_unit_tests','roles','description','',''); SELECT tap.col_default_is('_caosdb_schema_unit_tests','roles','description','NULL',''); -SELECT tap.col_charset_is('_caosdb_schema_unit_tests','roles','description','utf8mb3',''); -SELECT tap.col_collation_is('_caosdb_schema_unit_tests','roles','description','utf8mb3_unicode_ci',''); +SELECT tap.col_charset_is('_caosdb_schema_unit_tests','roles','description','utf8mb4',''); +SELECT tap.col_collation_is('_caosdb_schema_unit_tests','roles','description','utf8mb4_uca1400_as_ci',''); -- CONSTRAINTS SELECT tap.constraints_are('_caosdb_schema_unit_tests','roles','`PRIMARY`',''); @@ -1760,7 +1764,7 @@ SELECT tap.col_is_pk('_caosdb_schema_unit_tests','roles','`name`',''); -- *************************************************************** SELECT tap.has_table('_caosdb_schema_unit_tests','stats',''); -SELECT tap.table_collation_is('_caosdb_schema_unit_tests','stats','utf8mb3_unicode_ci',''); +-- SELECT tap.table_collation_is('_caosdb_schema_unit_tests','stats','utf8mb4_uca1400_as_ci',''); SELECT tap.table_engine_is('_caosdb_schema_unit_tests','stats','InnoDB',''); -- COLUMNS @@ -1772,8 +1776,8 @@ SELECT tap.has_column('_caosdb_schema_unit_tests','stats','name',''); SELECT tap.col_column_type_is('_caosdb_schema_unit_tests','stats','name','varchar(255)',''); SELECT tap.col_extra_is('_caosdb_schema_unit_tests','stats','name','',''); SELECT tap.col_default_is('_caosdb_schema_unit_tests','stats','name',NULL,''); -SELECT tap.col_charset_is('_caosdb_schema_unit_tests','stats','name','utf8mb3',''); -SELECT tap.col_collation_is('_caosdb_schema_unit_tests','stats','name','utf8mb3_unicode_ci',''); +SELECT tap.col_charset_is('_caosdb_schema_unit_tests','stats','name','utf8mb4',''); +SELECT tap.col_collation_is('_caosdb_schema_unit_tests','stats','name','utf8mb4_uca1400_as_ci',''); -- COLUMN stats.value @@ -1798,7 +1802,7 @@ SELECT tap.col_is_pk('_caosdb_schema_unit_tests','stats','`name`',''); -- *************************************************************** SELECT tap.has_table('_caosdb_schema_unit_tests','text_data',''); -SELECT tap.table_collation_is('_caosdb_schema_unit_tests','text_data','utf8mb3_unicode_ci',''); +-- SELECT tap.table_collation_is('_caosdb_schema_unit_tests','text_data','utf8mb4_uca1400_as_ci',''); SELECT tap.table_engine_is('_caosdb_schema_unit_tests','text_data','InnoDB',''); -- COLUMNS @@ -1837,8 +1841,8 @@ SELECT tap.has_column('_caosdb_schema_unit_tests','text_data','value',''); SELECT tap.col_column_type_is('_caosdb_schema_unit_tests','text_data','value','text',''); SELECT tap.col_extra_is('_caosdb_schema_unit_tests','text_data','value','',''); SELECT tap.col_default_is('_caosdb_schema_unit_tests','text_data','value',NULL,''); -SELECT tap.col_charset_is('_caosdb_schema_unit_tests','text_data','value','utf8mb3',''); -SELECT tap.col_collation_is('_caosdb_schema_unit_tests','text_data','value','utf8mb3_unicode_ci',''); +SELECT tap.col_charset_is('_caosdb_schema_unit_tests','text_data','value','utf8mb4',''); +SELECT tap.col_collation_is('_caosdb_schema_unit_tests','text_data','value','utf8mb4_uca1400_as_ci',''); -- COLUMN text_data.status @@ -1846,8 +1850,8 @@ SELECT tap.has_column('_caosdb_schema_unit_tests','text_data','status',''); SELECT tap.col_column_type_is('_caosdb_schema_unit_tests','text_data','status','enum(\'OBLIGATORY\',\'RECOMMENDED\',\'SUGGESTED\',\'FIX\',\'REPLACEMENT\')',''); SELECT tap.col_extra_is('_caosdb_schema_unit_tests','text_data','status','',''); SELECT tap.col_default_is('_caosdb_schema_unit_tests','text_data','status',NULL,''); -SELECT tap.col_charset_is('_caosdb_schema_unit_tests','text_data','status','utf8mb3',''); -SELECT tap.col_collation_is('_caosdb_schema_unit_tests','text_data','status','utf8mb3_unicode_ci',''); +SELECT tap.col_charset_is('_caosdb_schema_unit_tests','text_data','status','utf8mb4',''); +SELECT tap.col_collation_is('_caosdb_schema_unit_tests','text_data','status','utf8mb4_uca1400_as_ci',''); -- COLUMN text_data.pidx @@ -1896,7 +1900,7 @@ SELECT tap.fk_on_update('_caosdb_schema_unit_tests','text_data','str_property_id -- *************************************************************** SELECT tap.has_table('_caosdb_schema_unit_tests','transaction_log',''); -SELECT tap.table_collation_is('_caosdb_schema_unit_tests','transaction_log','utf8mb3_unicode_ci',''); +-- SELECT tap.table_collation_is('_caosdb_schema_unit_tests','transaction_log','utf8mb4_uca1400_as_ci',''); SELECT tap.table_engine_is('_caosdb_schema_unit_tests','transaction_log','InnoDB',''); -- COLUMNS @@ -1908,8 +1912,8 @@ SELECT tap.has_column('_caosdb_schema_unit_tests','transaction_log','transaction SELECT tap.col_column_type_is('_caosdb_schema_unit_tests','transaction_log','transaction','varchar(255)',''); SELECT tap.col_extra_is('_caosdb_schema_unit_tests','transaction_log','transaction','',''); SELECT tap.col_default_is('_caosdb_schema_unit_tests','transaction_log','transaction',NULL,''); -SELECT tap.col_charset_is('_caosdb_schema_unit_tests','transaction_log','transaction','utf8mb3',''); -SELECT tap.col_collation_is('_caosdb_schema_unit_tests','transaction_log','transaction','utf8mb3_unicode_ci',''); +SELECT tap.col_charset_is('_caosdb_schema_unit_tests','transaction_log','transaction','utf8mb4',''); +SELECT tap.col_collation_is('_caosdb_schema_unit_tests','transaction_log','transaction','utf8mb4_uca1400_as_ci',''); -- COLUMN transaction_log.entity_id @@ -1970,7 +1974,7 @@ SELECT tap.is_indexed('_caosdb_schema_unit_tests','transaction_log','`entity_id` -- *************************************************************** SELECT tap.has_table('_caosdb_schema_unit_tests','units_lin_con',''); -SELECT tap.table_collation_is('_caosdb_schema_unit_tests','units_lin_con','utf8mb3_unicode_ci',''); +-- SELECT tap.table_collation_is('_caosdb_schema_unit_tests','units_lin_con','utf8mb4_uca1400_as_ci',''); SELECT tap.table_engine_is('_caosdb_schema_unit_tests','units_lin_con','InnoDB',''); -- COLUMNS @@ -2044,7 +2048,7 @@ SELECT tap.col_is_pk('_caosdb_schema_unit_tests','units_lin_con','`signature_fro -- *************************************************************** SELECT tap.has_table('_caosdb_schema_unit_tests','user_info',''); -SELECT tap.table_collation_is('_caosdb_schema_unit_tests','user_info','utf8mb3_unicode_ci',''); +-- SELECT tap.table_collation_is('_caosdb_schema_unit_tests','user_info','utf8mb4_uca1400_as_ci',''); SELECT tap.table_engine_is('_caosdb_schema_unit_tests','user_info','InnoDB',''); -- COLUMNS @@ -2083,8 +2087,8 @@ SELECT tap.has_column('_caosdb_schema_unit_tests','user_info','status',''); SELECT tap.col_column_type_is('_caosdb_schema_unit_tests','user_info','status','enum(\'ACTIVE\',\'INACTIVE\')',''); SELECT tap.col_extra_is('_caosdb_schema_unit_tests','user_info','status','',''); SELECT tap.col_default_is('_caosdb_schema_unit_tests','user_info','status','\'INACTIVE\'',''); -SELECT tap.col_charset_is('_caosdb_schema_unit_tests','user_info','status','utf8mb3',''); -SELECT tap.col_collation_is('_caosdb_schema_unit_tests','user_info','status','utf8mb3_unicode_ci',''); +SELECT tap.col_charset_is('_caosdb_schema_unit_tests','user_info','status','utf8mb4',''); +SELECT tap.col_collation_is('_caosdb_schema_unit_tests','user_info','status','utf8mb4_uca1400_as_ci',''); -- COLUMN user_info.entity @@ -2125,7 +2129,7 @@ SELECT tap.fk_on_update('_caosdb_schema_unit_tests','user_info','subjects_ibfk_2 -- *************************************************************** SELECT tap.has_table('_caosdb_schema_unit_tests','user_roles',''); -SELECT tap.table_collation_is('_caosdb_schema_unit_tests','user_roles','utf8mb3_unicode_ci',''); +-- SELECT tap.table_collation_is('_caosdb_schema_unit_tests','user_roles','utf8mb4_uca1400_as_ci',''); SELECT tap.table_engine_is('_caosdb_schema_unit_tests','user_roles','InnoDB',''); -- COLUMNS @@ -2179,7 +2183,7 @@ SELECT tap.fk_on_update('_caosdb_schema_unit_tests','user_roles','user_roles_ibf -- *************************************************************** SELECT tap.has_table('_caosdb_schema_unit_tests','entity_version',''); -SELECT tap.table_collation_is('_caosdb_schema_unit_tests','entity_version','utf8mb3_unicode_ci',''); +-- SELECT tap.table_collation_is('_caosdb_schema_unit_tests','entity_version','utf8mb4_uca1400_as_ci',''); SELECT tap.table_engine_is('_caosdb_schema_unit_tests','entity_version','InnoDB',''); -- COLUMNS diff --git a/utils/helpers.sh b/utils/helpers.sh index f335dfa179e26e0237149572168884b554ee5f91..ff4b189bffeabf467e0725de9a4b62fe72cd6bf3 100644 --- a/utils/helpers.sh +++ b/utils/helpers.sh @@ -34,7 +34,7 @@ function mysql_execute { #TODO is it meaningful that here always the database user is used??? set -e - $MYSQL_CMD $(get_db_args) -e "$1" + $MYSQL_CMD $(get_db_args) -e "$@" ret=${PIPESTATUS[0]} if [ "$ret" -ne 0 ]; then failure "MYSQL ERROR" diff --git a/utils/patch_header.sh b/utils/patch_header.sh index 33fbff879b3166596e153685105873adeae8405b..c7d25ae3f21ccbba30658182572177847d5610a7 100644 --- a/utils/patch_header.sh +++ b/utils/patch_header.sh @@ -1,12 +1,10 @@ -# -# ** header v3.0 -# This file is a part of the CaosDB Project. +# This file is a part of the LinkAhead Project. # # Copyright (C) 2018 Research Group Biomedical Physics, # Max-Planck-Institute for Dynamics and Self-Organization Göttingen -# Copyright (C) 2019, 2020 Daniel Hornung <d.hornung@indiscale.com> +# Copyright (C) 2019, 2020, 2025 Daniel Hornung <d.hornung@indiscale.com> # Copyright (C) 2020 Henrik tom Wörden <h.tomwoerden@indiscale.com> -# Copyright (C) 2020 IndiScale <info@indiscale.com> +# Copyright (C) 2020, 2025 IndiScale <info@indiscale.com> # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License as @@ -21,12 +19,20 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see <https://www.gnu.org/licenses/>. # -# ** end header -# ############################################################################### # header for patch scripts # ############################################################################### +# +# This file defines the following functions (see below for more detailed descriptions): +# +# - check_version_or_exit: Check if the stored version is as expected. +# - update_version: Set the stored version. +# - dump_table: Store table content into a file. +# - redo_table: Apply the table content stored by dump_table. +# +# This file also sets the `errexit` option (`set -e`). +# set -e @@ -63,7 +69,7 @@ configuration make file. Notable examples are: - DATABASE_USER_PW EOF - ) + ) function _print_help() { echo -e "$USAGE" @@ -115,21 +121,51 @@ done if [ -n "$PATCH" ]; then echo -ne "applying patch $PATCH to $DATABASE_NAME ... " fi -# @param $1: db version string, e.g. v2.0.0 -# @return: 0 on success, 1 on failure -function check_version { + +# usage: check_version_or_exit version +# +# Check if the version string is as expected. Otherwise, `exit 0`. +# +# Arguments +# --------- +# +# version: str +# The expected database version string, e.g. 'v2.0.0' +# +# Returns +# ------- +# +# 0 on success +function check_version_or_exit { local version=$($MYSQL_CMD $(get_db_args) -B -e "Select CaosDBVersion();") - if [[ "$(echo $version | sed 's/^CaosDBVersion()\s//')" = "$1" ]]; then + local version="$(echo $version | sed 's/^CaosDBVersion()\s//')" + if [[ "$version" = "$1" ]]; then return 0 fi uptodate } -# @param $1: new version string +# Deprecated name, call "check_version_or_exit" directly instead. +function check_version { + check_version_or_exit "$1" +} + +# usage: update_version version +# +# Set the version string. +# +# Arguments +# --------- +# +# version: str +# The new database version string, e.g. 'v2.0.0' function update_version { mysql_execute "DROP FUNCTION IF EXISTS CaosDBVersion; CREATE FUNCTION CaosDBVersion() RETURNS VARCHAR(255) DETERMINISTIC RETURN '$1';" } +# usage: dump_table table +# +# Dump the table given in the argument into file "caosdb.<table>.<OLD_VERSION>.dump.sql" function dump_table { if [[ -z $MYSQLDUMP_CMD ]]; then echo "Cannot find mysqldump program!" >&2 @@ -139,6 +175,9 @@ function dump_table { > ${DATABASE_NAME}.${1}.${OLD_VERSION}.dump.sql } +# usage: redo_table table +# +# Apply the dump from `dump_table` to the database. Takes the same argument. function redo_table { $MYSQL_CMD $(get_db_args) < ${DATABASE_NAME}.${1}.${OLD_VERSION}.dump.sql } diff --git a/utils/sanity_check.sql b/utils/sanity_check.sql index c29e7c8a82f238f5836e519159fa48d63933a609..5e3d580cc386feb44b5e97269ac384c050d6cf84 100644 --- a/utils/sanity_check.sql +++ b/utils/sanity_check.sql @@ -55,9 +55,29 @@ sanityCheckBody: BEGIN ("user_roles") ; -SELECT COUNT(WRONG) INTO @count_wrong FROM ( SELECT l.table_name AS MATCHED, r.table_name AS WRONG FROM expected_tables AS l RIGHT OUTER JOIN information_schema.tables AS r ON (r.table_name COLLATE utf8_unicode_ci = l.table_name) WHERE r.table_schema = database()) AS temp WHERE temp.MATCHED IS NULL; +SELECT COUNT(WRONG) INTO @count_wrong FROM ( + SELECT l.table_name AS MATCHED, + r.table_name AS WRONG + FROM + expected_tables AS l + RIGHT OUTER JOIN + information_schema.tables AS r + ON (r.table_name COLLATE utf8_unicode_ci = l.table_name) + WHERE r.table_schema = database() AND r.table_type != 'TEMPORARY' + ) + AS temp WHERE temp.MATCHED IS NULL; -SELECT COUNT(MISSING) INTO @count_missing FROM ( SELECT l.table_name AS MISSING, r.table_name AS MATCHED FROM expected_tables AS l LEFT OUTER JOIN information_schema.tables AS r ON (r.table_name COLLATE utf8_unicode_ci = l.table_name) WHERE r.table_schema = database() OR r.table_schema IS NULL) AS temp WHERE temp.MATCHED IS NULL; +SELECT COUNT(MISSING) INTO @count_missing FROM ( + SELECT l.table_name AS MISSING, + r.table_name AS MATCHED + FROM + expected_tables AS l + LEFT OUTER JOIN + information_schema.tables AS r + ON (r.table_name COLLATE utf8_unicode_ci = l.table_name) + WHERE r.table_schema = database() OR r.table_schema IS NULL + ) + AS temp WHERE temp.MATCHED IS NULL; IF @count_missing = 0 AND @count_wrong = 0 THEN LEAVE sanityCheckBody; @@ -66,12 +86,28 @@ END IF; SELECT "--------------"; SELECT @count_missing AS "Number of missing tables"; -SELECT MISSING AS "Missing tables" FROM ( SELECT l.table_name AS MISSING, r.table_name AS MATCHED FROM expected_tables AS l LEFT OUTER JOIN information_schema.tables AS r ON (r.table_name COLLATE utf8_unicode_ci = l.table_name) WHERE r.table_schema = database() OR r.table_schema IS NULL) AS temp WHERE temp.MATCHED IS NULL; +SELECT MISSING AS "Missing tables" FROM ( + SELECT l.table_name AS MISSING, r.table_name AS MATCHED FROM + expected_tables AS l + LEFT OUTER JOIN + information_schema.tables AS r + ON (r.table_name COLLATE utf8_unicode_ci = l.table_name) + WHERE r.table_schema = database() OR r.table_schema IS NULL + ) + AS temp WHERE temp.MATCHED IS NULL; SELECT "--------------"; SELECT @count_wrong AS "Number of tables which should not exist"; -SELECT WRONG AS "Tables which should not exist" FROM ( SELECT l.table_name AS MATCHED, r.table_name AS WRONG FROM expected_tables AS l RIGHT OUTER JOIN information_schema.tables AS r ON (r.table_name COLLATE utf8_unicode_ci = l.table_name) WHERE r.table_schema = database()) AS temp WHERE temp.MATCHED IS NULL; +SELECT WRONG AS "Tables which should not exist" FROM ( + SELECT expec.table_name AS MATCHED, info.table_name AS WRONG FROM + expected_tables AS expec + RIGHT OUTER JOIN + information_schema.tables AS info + ON (info.table_name COLLATE utf8_unicode_ci = expec.table_name) + WHERE info.table_schema = database() and info.table_type != 'TEMPORARY' + ) + AS temp WHERE temp.MATCHED IS NULL; SELECT "--------------"; SELECT "ERROR" from sanity_check_failed;