Merge branch 'f-fix-accent-sensitivity' into 'dev'

Fix accent sensitivity

See merge request !33

.docker/docker-compose.yml

 version: '3.6'
 services:
   sqldb:
-    image: mariadb:10.5
+    image: mariadb:11.4
     volumes:
       - type: volume
         source: "caosdb-sqldata"

.gitlab-ci.yml

@@ -70,24 +70,24 @@ test-dump-update:
   script:
     - make test-dump-update
 
-# Run the unit tests with MySQL 5
-unittests-mysql-5:
-  tags: [ docker ]
-  stage: test
-  # Should not stop the pipeline from continuing.
-  allow_failure: true
-  services:
-    - name: mysql:5.7.36
-      command: ["--default-authentication-plugin=mysql_native_password"]
-
-  script:
-    # remove some lines from autotap because the checks of column default
-    # values don't work with mysql-5
-    - sed -i "/col_default_is.*NULL/d" tests/test_autotap.sql
-    - sed -i "/col_default_is.*INACTIVE/d" tests/test_autotap.sql
-    - sed -i "/col_default_is.*SHA/d" tests/test_autotap.sql
-    - sed -i "s/utf8mb3/utf8/" tests/test_autotap.sql
-    - make pipeline-test SQL_HOST=mysql
+# # Run the unit tests with MySQL 5
+# unittests-mysql-5:
+#   tags: [ docker ]
+#   stage: test
+#   # Should not stop the pipeline from continuing.
+#   allow_failure: true
+#   services:
+#     - name: mysql:5.7.36
+#       command: ["--default-authentication-plugin=mysql_native_password"]
+#
+#   script:
+#     # remove some lines from autotap because the checks of column default
+#     # values don't work with mysql-5
+#     - sed -i "/col_default_is.*NULL/d" tests/test_autotap.sql
+#     - sed -i "/col_default_is.*INACTIVE/d" tests/test_autotap.sql
+#     - sed -i "/col_default_is.*SHA/d" tests/test_autotap.sql
+#     - sed -i "s/utf8mb3/utf8/" tests/test_autotap.sql
+#     - make pipeline-test SQL_HOST=mysql
 
 ######## Deploy ########
 

CHANGELOG.md

@@ -9,15 +9,25 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Added
 
+- New version requirement: MariaDB 11.2 or later.
+
 ### Changed
 
 ### Deprecated
 
 ### Removed
 
+- MySQL support.  See the deprecation in the 7.0.0 release.
+
 ### Fixed
 
-- [SQL dump migration can now be applied twice](https://gitlab.indiscale.com/caosdb/src/caosdb-mysqlbackend/-/issues/60): Fixed an error in the regex.
+- [SQL dump migration can now be applied
+  twice](https://gitlab.indiscale.com/caosdb/src/caosdb-mysqlbackend/-/issues/60):
+  Fixed an error in the regex.
+- [linkahead-server#264](https://gitlab.com/linkahead/linkahead-server/-/issues/264)
+  Queries ignore accents and umlauts: Changed to more appropriate
+  collation and character sets. This requires MariaDB 11.2 or later,
+  see above.
 
 ### Security
 

Makefile

@@ -62,10 +62,10 @@ test-dump-update:
 # Run tests with a database which is started in a MariaDB Docker container
 .PHONY: test-docker
 test-docker:
-	@docker kill caosdb-mysqlserver-test || true
-	@docker container rm -v caosdb-mysqlserver-test || true
-	@docker run --name caosdb-mysqlserver-test -p "3306:3306" \
-		-e MYSQL_ROOT_PASSWORD="pass-for-test" -d mariadb:10.11
+	@docker kill linkahead-mysqlserver-test || true
+	@docker container rm -v linkahead-mysqlserver-test || true
+	@docker run --name linkahead-mysqlserver-test -p "3306:3306" \
+		-e MYSQL_ROOT_PASSWORD="pass-for-test" -d mariadb:11.4
 	@sleep 10
 	MAINPATH=$(realpath tests/docker_env) utils/make_db test --fresh
 	make test-docker-stop
@@ -73,8 +73,8 @@ test-docker:
 # if automatic stopping failed
 .PHONY: test-docker-stop
 test-docker-stop:
-	docker kill caosdb-mysqlserver-test
-	docker container rm -v caosdb-mysqlserver-test
+	docker kill linkahead-mysqlserver-test
+	docker container rm -v linkahead-mysqlserver-test
 
 # Compile the standalone documentation
 .PHONY: doc

README.md

@@ -3,7 +3,7 @@
 
 ## Welcome
 
-This is the **LinkAhead MySQL Backend** repository and a part of the
+This is the **LinkAhead MariaDB Backend** repository and a part of the
 LinkAhead project.
 
 ## Setup
@@ -43,7 +43,7 @@ Conduct](https://gitlab.com/linkahead/linkahead/-/blob/main/CODE_OF_CONDUCT.md).
 
 * Copyright (C) 2018 Research Group Biomedical Physics, Max Planck Institute
   for Dynamics and Self-Organization Göttingen.
-* Copyright (C) 2020-2021 Indiscale GmbH <info@indiscale.com>
+* Copyright (C) 2020-2025 Indiscale GmbH <info@indiscale.com>
 
 All files in this repository are licensed under a [GNU Affero General Public
 License](LICENCE.md) (version 3 or later).

README_SETUP.md

-# Setup of the CaosDB SQL back end
+# Setup of the LinkAhead MariaDB backend
 
 ## Dependencies
 
-* `MariaDB Client 10.1` or later, `MySQL Client >=5.5, <=5.7.36`.
+* `MariaDB Client 11.2` or later.
 * make
 
 ## Create the configuration
@@ -11,7 +11,7 @@
   parameter that you want to change, add a corresponding line in your `.config`
   file. You probably want to change the passwords.  As the passwords are stored
   unencrypted in the `.config` file, make sure nobody else can read it.
-    * If there is no `mysql-config-editor` (`MySQL 5.5`. and `MariaDB`) then the
+    * If there is no `mysql-config-editor` program, then the
       `MYSQL_USER_PASSWORD` must be provided, that is the password of the
       `MYSQL_USER`.
     * If you are using MariaDB and the `root` database user uses pam
@@ -99,20 +99,22 @@ After the first start, you need to install the database: `make install`.
 Then, you can connect to it with `mariadb --protocol=TCP -u caosdb -prandom1234
 caosdb`.
 
+### Developing tests ###
 
-### Troubleshooting
-
-#### MySQL has failing tests
-
-Our test suite is developed with MariaDB. That is why some tests, those which
-check constraints based on auto-generated names, fail because MySQL generates
-other names.
+To use the mytap framework:
 
-Also some test fail with the message "Expected: NULL / Was: NULL" which is
-obviously what was expected.
+```console
+$ cd libs; unzip mytap-1.0.zip; cd mytap-1.0
+# Use the necessary mariadb options for connecting to the server in the following commands
+$ mariadb ... < mytap.sql  # Set up the "tap" database and functions.
+$ mariadb ... < scripts/autotap.sql  # Insert the "autotap" function.
+# Create autotap file (for comparison with current state)
+$ mariadb ... --raw -B --skip-column-names -e "call tap.autotap('_caosdb_schema_unit_tests')" > test_autotap.current.sql 
+# Run the autotap tests (may fail unfortunately with current MariaDB):
+$ mariadb ... < test_autotap.current.sql
+```
 
-Please look carefully through the tests. If something more serious than that
-comes up, please report a bug.
+### Troubleshooting
 
 #### Failure to restore a database dump created with an older MariaDB version ####
 
@@ -121,6 +123,11 @@ incompatibilities, the necessary steps to migrate the dump are
 probably described there and scripts for the migrations are provided
 in the same directory.
 
+#### MySQL has failing tests
+
+*Note: Since we switched from MySQL to MariaDB, the two DBMS projects have diverged quite far.  As a
+result, we no longer support MySQL.
+
 ## Documentation #
 
 Build documentation in `build/` with `make doc`.

RELEASE_GUIDELINES.md

-# Release Guidelines for the CaosDB MySQL Backend
+# Release Guidelines for the LinkAhead MariaDB backend
 
 This document specifies release guidelines in addition to the general release
 guidelines of the CaosDB Project

doc/Maintenance.rst

-
 Maintenance
 ===========
 
-
 Creating a Backup
 -----------------
 
-
 You can use the Python script ``utils/backup.py`` to create a backup (SQL dump) 
 of the SQL-Backend::
 
     ./utils/backup.py -d folder/where/the/backup/is/created
 
-You can do this while CaosDB is online.
+You can do this while LinkAhead is online.
 
 Restoring a Backup
 ------------------
-CaosDB should be offline for restoring. 
+LinkAhead should be offline for restoring. 
 
 You can use the Bash script ``utils/make_db`` to restore a backup (SQL dump) 
 of the SQL-Backend::

patches/patch20170316-2.0.27/patch.sh

@@ -35,7 +35,12 @@ fi
 
 check_version $OLD_VERSION
 
-mysql_execute 'DELETE FROM `entities` WHERE id=6; ALTER TABLE `entities` MODIFY COLUMN `role` ENUM("RECORDTYPE","RECORD","FILE","DOMAIN","PROPERTY","DATATYPE","ROLE","QUERYTEMPLATE") NOT NULL; INSERT INTO entities (id, name, description, role, acl) VALUES (8,"QUERYTEMPLATE","The QueryTemplate role.","ROLE",0); UPDATE `entities` SET role="ROLE" WHERE id<100 and name=role; CREATE TABLE `query_template_def` (id INT UNSIGNED PRIMARY KEY, definition MEDIUMTEXT NOT NULL, CONSTRAINT `query_template_def_ibfk_1` FOREIGN KEY (`id`) REFERENCES `entities` (`id`));'
+mysql_execute '
+DELETE FROM `entities` WHERE id=6;
+ALTER TABLE `entities` MODIFY COLUMN `role` ENUM("RECORDTYPE","RECORD","FILE","DOMAIN","PROPERTY","DATATYPE","ROLE","QUERYTEMPLATE") NOT NULL;
+INSERT INTO entities (id, name, description, role, acl) VALUES (8,"QUERYTEMPLATE","The QueryTemplate role.","ROLE",0);
+UPDATE `entities` SET role="ROLE" WHERE id<100 and name=role;
+CREATE TABLE `query_template_def` (id INT UNSIGNED PRIMARY KEY, definition MEDIUMTEXT NOT NULL, CONSTRAINT `query_template_def_ibfk_1` FOREIGN KEY (`id`) REFERENCES `entities` (`id`));'
 
 update_version $NEW_VERSION
 

patches/patch20250130-8.1.0/patch.sh

+#!/bin/bash
+#
+# This file is a part of the Linkahead Project.
+#
+# Copyright (C) 2025 IndiScale GmbH <info@indiscale.com>
+# Copyright (C) 2025 Daniel Hornung <d.hornung@indiscale.com>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see <https://www.gnu.org/licenses/>.
+#
+
+# Update mysql schema to version v8.1.0
+#
+# Set the collation so that accents on characters become relevant.
+
+OLD_VERSION="v8.0.0"
+NEW_VERSION="v8.1.0"
+
+if [ -z "$UTILSPATH" ]; then
+  UTILSPATH="../utils"
+fi
+
+. $UTILSPATH/patch_header.sh $*
+
+check_version $OLD_VERSION
+
+# Update charsets and collations.
+
+# Change database default.
+mysql_execute "ALTER DATABASE \`$DATABASE_NAME\` COLLATE = 'uca1400_as_ci';"
+
+# Some columns should be changed manually to prevent text length changes.
+mysql_execute "ALTER TABLE desc_overrides MODIFY description TEXT CHARACTER SET utf8mb4;"
+mysql_execute "ALTER TABLE entities MODIFY description TEXT CHARACTER SET utf8mb4;"
+mysql_execute "ALTER TABLE permissions MODIFY permissions MEDIUMTEXT CHARACTER SET utf8mb4 NOT NULL;"
+mysql_execute "ALTER TABLE query_template_def MODIFY definition MEDIUMTEXT CHARACTER SET utf8mb4 NOT NULL;"
+mysql_execute "ALTER TABLE roles MODIFY description MEDIUMTEXT CHARACTER SET utf8mb4;"
+mysql_execute "ALTER TABLE text_data MODIFY value TEXT CHARACTER SET utf8mb4 NOT NULL;"
+
+# Remove some constraints first.
+mysql_execute 'ALTER TABLE user_info DROP FOREIGN KEY `subjects_ibfk_2`;'
+
+# Get all tables, drop first line, take first column
+tables=$(mysql_execute "SHOW TABLE status where Collation='utf8mb3_unicode_ci';" -B | tail +2 | awk '{print $1}')
+for table in $tables; do
+    mysql_execute "ALTER TABLE $table CONVERT TO CHARACTER SET utf8mb4 COLLATE uca1400_as_ci;"
+done
+
+# Add constraints again.
+mysql_execute 'ALTER TABLE user_info MODIFY COLUMN `entity` VARCHAR(255) COLLATE utf8mb4_bin DEFAULT NULL;'
+mysql_execute 'ALTER TABLE user_info ADD CONSTRAINT `subjects_ibfk_2` FOREIGN KEY (`entity`) REFERENCES `entity_ids` (`id`);'
+
+# Set custom collations for columns
+mysql_execute "ALTER TABLE transaction_log MODIFY entity_id VARCHAR(255) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL;"
+
+
+### Finished ###
+
+update_version $NEW_VERSION
+
+success
+
+
+##### Useful commands #####
+
+# cd git/caosdb-mysqlbackend/
+# UTILSPATH=utils
+# . utils/patch_header.sh
+
+
+# show table status where Collation='utf8mb3_unicode_ci';
+
+# name_data
+# text_data
+
+# show table status like 'name_data';
+# select * from name_data where value like 'Yield%';
+# update name_data SET value='Yield-stréss' where entity_id=354;
+# update name_data SET value='Yield-strés😋s' where entity_id=354;
+# select * from name_data where value='Yield-stress';
+
+# # Unicode 14.0.0
+# ALTER TABLE name_data CONVERT TO CHARACTER SET utf8mb4 COLLATE uca1400_as_ci;
+
+# FULL_COLLATION_NAME: utf8mb4_uca1400_as_ci;
+# COLLATION_NAME: uca1400_as_ci;
+
+# mariadb -h sqldb -u caosdb -prandom1234 --default-character-set=utf8mb4 caosdb
+

utils/helpers.sh

@@ -34,7 +34,7 @@
 function mysql_execute {
 	  #TODO is it meaningful that here always the database user is used???
 	  set -e
-	  $MYSQL_CMD $(get_db_args) -e "$1"
+	  $MYSQL_CMD $(get_db_args) -e "$@"
 	  ret=${PIPESTATUS[0]}
 	  if [ "$ret" -ne 0 ]; then
 		    failure "MYSQL ERROR"

utils/patch_header.sh

-#
-# ** header v3.0
-# This file is a part of the CaosDB Project.
+# This file is a part of the LinkAhead Project.
 #
 # Copyright (C) 2018 Research Group Biomedical Physics,
 # Max-Planck-Institute for Dynamics and Self-Organization Göttingen
-# Copyright (C) 2019, 2020 Daniel Hornung <d.hornung@indiscale.com>
+# Copyright (C) 2019, 2020, 2025 Daniel Hornung <d.hornung@indiscale.com>
 # Copyright (C) 2020 Henrik tom Wörden <h.tomwoerden@indiscale.com>
-# Copyright (C) 2020 IndiScale <info@indiscale.com>
+# Copyright (C) 2020, 2025 IndiScale <info@indiscale.com>
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as
@@ -21,12 +19,20 @@
 # You should have received a copy of the GNU Affero General Public License
 # along with this program. If not, see <https://www.gnu.org/licenses/>.
 #
-# ** end header
-#
 
 ###############################################################################
 #                          header for patch scripts                           #
 ###############################################################################
+#
+# This file defines the following functions (see below for more detailed descriptions):
+#
+# - check_version_or_exit: Check if the stored version is as expected.
+# - update_version: Set the stored version.
+# - dump_table: Store table content into a file.
+# - redo_table: Apply the table content stored by dump_table.
+#
+# This file also sets the `errexit` option (`set -e`).
+#
 
 set -e
 
@@ -115,21 +121,51 @@ done
 if [ -n "$PATCH" ]; then
     echo -ne "applying patch $PATCH to $DATABASE_NAME ... "
 fi
-# @param $1: db version string, e.g. v2.0.0
-# @return: 0 on success, 1 on failure
-function check_version {
+
+# usage: check_version_or_exit version
+#
+# Check if the version string is as expected.  Otherwise, `exit 0`.
+#
+# Arguments
+# ---------
+#
+# version: str
+#   The expected database version string, e.g. 'v2.0.0'
+#
+# Returns
+# -------
+#
+# 0 on success
+function check_version_or_exit {
     local version=$($MYSQL_CMD $(get_db_args) -B -e "Select CaosDBVersion();")
-	  if [[ "$(echo $version | sed 's/^CaosDBVersion()\s//')" = "$1" ]]; then
+    local version="$(echo $version | sed 's/^CaosDBVersion()\s//')"
+	  if [[ "$version" = "$1" ]]; then
         return 0
 	  fi
     uptodate
 }
 
-# @param $1: new version string
+# Deprecated name, call "check_version_or_exit" directly instead.
+function check_version {
+    check_version_or_exit "$1"
+}
+
+# usage: update_version version
+#
+# Set the version string.
+#
+# Arguments
+# ---------
+#
+# version: str
+#   The new database version string, e.g. 'v2.0.0'
 function update_version {
     mysql_execute "DROP FUNCTION IF EXISTS CaosDBVersion; CREATE FUNCTION CaosDBVersion() RETURNS VARCHAR(255) DETERMINISTIC RETURN '$1';"
 }
 
+# usage: dump_table table
+#
+# Dump the table given in the argument into file "caosdb.<table>.<OLD_VERSION>.dump.sql"
 function dump_table {
     if [[ -z $MYSQLDUMP_CMD ]]; then
         echo "Cannot find mysqldump program!" >&2
@@ -139,6 +175,9 @@ function dump_table {
                     > ${DATABASE_NAME}.${1}.${OLD_VERSION}.dump.sql
 }
 
+# usage: redo_table table
+#
+# Apply the dump from `dump_table` to the database.  Takes the same argument.
 function redo_table {
          $MYSQL_CMD $(get_db_args) < ${DATABASE_NAME}.${1}.${OLD_VERSION}.dump.sql
 }

utils/sanity_check.sql

@@ -55,9 +55,29 @@ sanityCheckBody: BEGIN
 ("user_roles")
 ;
 
-SELECT COUNT(WRONG) INTO @count_wrong FROM ( SELECT l.table_name AS MATCHED, r.table_name AS WRONG FROM expected_tables AS l RIGHT OUTER JOIN information_schema.tables AS r ON (r.table_name COLLATE utf8_unicode_ci = l.table_name) WHERE r.table_schema = database()) AS temp WHERE temp.MATCHED IS NULL;
+SELECT COUNT(WRONG) INTO @count_wrong FROM (
+  SELECT l.table_name AS MATCHED,
+         r.table_name AS WRONG
+    FROM
+        expected_tables AS l
+      RIGHT OUTER JOIN
+        information_schema.tables AS r
+      ON (r.table_name COLLATE utf8_unicode_ci = l.table_name)
+      WHERE r.table_schema = database() AND r.table_type != 'TEMPORARY'
+  )
+  AS temp WHERE temp.MATCHED IS NULL;
 
-SELECT COUNT(MISSING) INTO @count_missing FROM ( SELECT l.table_name AS MISSING, r.table_name AS MATCHED FROM expected_tables AS l LEFT OUTER JOIN information_schema.tables AS r ON (r.table_name COLLATE utf8_unicode_ci = l.table_name) WHERE r.table_schema = database() OR r.table_schema IS NULL) AS temp WHERE temp.MATCHED IS NULL;
+SELECT COUNT(MISSING) INTO @count_missing FROM (
+  SELECT l.table_name AS MISSING,
+         r.table_name AS MATCHED
+    FROM
+        expected_tables AS l
+      LEFT OUTER JOIN
+        information_schema.tables AS r
+        ON (r.table_name COLLATE utf8_unicode_ci = l.table_name)
+        WHERE r.table_schema = database() OR r.table_schema IS NULL
+  )
+  AS temp WHERE temp.MATCHED IS NULL;
 
 IF @count_missing = 0 AND @count_wrong = 0 THEN
     LEAVE sanityCheckBody;
@@ -66,12 +86,28 @@ END IF;
 SELECT "--------------";
 SELECT @count_missing AS "Number of missing tables";
 
-SELECT MISSING AS "Missing tables" FROM ( SELECT l.table_name AS MISSING, r.table_name AS MATCHED FROM expected_tables AS l LEFT OUTER JOIN information_schema.tables AS r ON (r.table_name COLLATE utf8_unicode_ci = l.table_name) WHERE r.table_schema = database() OR r.table_schema IS NULL) AS temp WHERE temp.MATCHED IS NULL;
+SELECT MISSING AS "Missing tables" FROM (
+  SELECT l.table_name AS MISSING, r.table_name AS MATCHED FROM
+      expected_tables AS l
+    LEFT OUTER JOIN
+      information_schema.tables AS r
+    ON (r.table_name COLLATE utf8_unicode_ci = l.table_name)
+    WHERE r.table_schema = database() OR r.table_schema IS NULL
+  )
+  AS temp WHERE temp.MATCHED IS NULL;
 
 SELECT "--------------";
 SELECT @count_wrong AS "Number of tables which should not exist";
 
-SELECT WRONG AS "Tables which should not exist" FROM ( SELECT l.table_name AS MATCHED, r.table_name AS WRONG FROM expected_tables AS l RIGHT OUTER JOIN information_schema.tables AS r ON (r.table_name COLLATE utf8_unicode_ci = l.table_name) WHERE r.table_schema = database()) AS temp WHERE temp.MATCHED IS NULL;
+SELECT WRONG AS "Tables which should not exist" FROM (
+  SELECT expec.table_name AS MATCHED, info.table_name AS WRONG FROM
+      expected_tables AS expec
+    RIGHT OUTER JOIN
+      information_schema.tables AS info
+    ON (info.table_name COLLATE utf8_unicode_ci = expec.table_name)
+    WHERE info.table_schema = database() and info.table_type != 'TEMPORARY'
+  )
+  AS temp WHERE temp.MATCHED IS NULL;
 
 SELECT "--------------";
 SELECT "ERROR" from sanity_check_failed;