Merge branch 'dev' into f-real-id

patches/applyPatches.sh

@@ -47,4 +47,5 @@ do
 done
 
 cd ../
+
 $UTILSPATH/update_sql_procedures.sh

patches/patch20170316-2.0.27/patch.sh

@@ -35,7 +35,12 @@ fi
 
 check_version $OLD_VERSION
 
-mysql_execute 'DELETE FROM `entities` WHERE id=6; ALTER TABLE `entities` MODIFY COLUMN `role` ENUM("RECORDTYPE","RECORD","FILE","DOMAIN","PROPERTY","DATATYPE","ROLE","QUERYTEMPLATE") NOT NULL; INSERT INTO entities (id, name, description, role, acl) VALUES (8,"QUERYTEMPLATE","The QueryTemplate role.","ROLE",0); UPDATE `entities` SET role="ROLE" WHERE id<100 and name=role; CREATE TABLE `query_template_def` (id INT UNSIGNED PRIMARY KEY, definition MEDIUMTEXT NOT NULL, CONSTRAINT `query_template_def_ibfk_1` FOREIGN KEY (`id`) REFERENCES `entities` (`id`));'
+mysql_execute '
+DELETE FROM `entities` WHERE id=6;
+ALTER TABLE `entities` MODIFY COLUMN `role` ENUM("RECORDTYPE","RECORD","FILE","DOMAIN","PROPERTY","DATATYPE","ROLE","QUERYTEMPLATE") NOT NULL;
+INSERT INTO entities (id, name, description, role, acl) VALUES (8,"QUERYTEMPLATE","The QueryTemplate role.","ROLE",0);
+UPDATE `entities` SET role="ROLE" WHERE id<100 and name=role;
+CREATE TABLE `query_template_def` (id INT UNSIGNED PRIMARY KEY, definition MEDIUMTEXT NOT NULL, CONSTRAINT `query_template_def_ibfk_1` FOREIGN KEY (`id`) REFERENCES `entities` (`id`));'
 
 update_version $NEW_VERSION
 

patches/patch20241024-8.0.0/patch.sh

+#!/bin/bash
+#
+# This file is a part of the LinkAhead Project.
+#
+# Copyright (C) 2024 IndiScale GmbH <info@indiscale.com>
+# Copyright (C) 2024 Timm Fitschen <t.fitschen@indiscale.com>
+# Copyright (C) 2024 Florian Spreckelsen <f.spreckelsen@indiscale.com>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see <https://www.gnu.org/licenses/>.
+#
+
+# Update mysql schema to version v8.0.0
+#
+# The patch is empty because the change has to be applied manually
+# from ../../dump_updates/2024-10-02.dump_fix_mariadb_10_6.sh if necessary.
+
+NEW_VERSION="v8.0.0"
+OLD_VERSION="v7.0.2"
+
+if [ -z "$UTILSPATH" ]; then
+ UTILSPATH="../utils"
+fi
+
+. $UTILSPATH/patch_header.sh $*
+
+check_version $OLD_VERSION
+
+update_version $NEW_VERSION
+
+success

patches/patch20250130-8.1.0/patch.sh

+#!/bin/bash
+#
+# This file is a part of the Linkahead Project.
+#
+# Copyright (C) 2025 IndiScale GmbH <info@indiscale.com>
+# Copyright (C) 2025 Daniel Hornung <d.hornung@indiscale.com>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see <https://www.gnu.org/licenses/>.
+#
+
+# Update mysql schema to version v8.1.0
+#
+# Set the collation so that accents on characters become relevant.
+
+OLD_VERSION="v8.0.0"
+NEW_VERSION="v8.1.0"
+
+if [ -z "$UTILSPATH" ]; then
+  UTILSPATH="../utils"
+fi
+
+. $UTILSPATH/patch_header.sh $*
+
+check_version $OLD_VERSION
+
+# Update charsets and collations.
+
+# Change database default.
+mysql_execute "ALTER DATABASE \`$DATABASE_NAME\` COLLATE = 'uca1400_as_ci';"
+
+# Some columns should be changed manually to prevent text length changes.
+mysql_execute "ALTER TABLE desc_overrides MODIFY description TEXT CHARACTER SET utf8mb4;"
+mysql_execute "ALTER TABLE entities MODIFY description TEXT CHARACTER SET utf8mb4;"
+mysql_execute "ALTER TABLE permissions MODIFY permissions MEDIUMTEXT CHARACTER SET utf8mb4 NOT NULL;"
+mysql_execute "ALTER TABLE query_template_def MODIFY definition MEDIUMTEXT CHARACTER SET utf8mb4 NOT NULL;"
+mysql_execute "ALTER TABLE roles MODIFY description MEDIUMTEXT CHARACTER SET utf8mb4;"
+mysql_execute "ALTER TABLE text_data MODIFY value TEXT CHARACTER SET utf8mb4 NOT NULL;"
+
+# Remove some constraints first.
+mysql_execute 'ALTER TABLE user_info DROP FOREIGN KEY `subjects_ibfk_2`;'
+
+# Get all tables, drop first line, take first column
+tables=$(mysql_execute "SHOW TABLE status where Collation='utf8mb3_unicode_ci';" -B | tail +2 | awk '{print $1}')
+for table in $tables; do
+    mysql_execute "ALTER TABLE $table CONVERT TO CHARACTER SET utf8mb4 COLLATE uca1400_as_ci;"
+done
+
+# Add constraints again.
+mysql_execute 'ALTER TABLE user_info MODIFY COLUMN `entity` VARCHAR(255) COLLATE utf8mb4_bin DEFAULT NULL;'
+mysql_execute 'ALTER TABLE user_info ADD CONSTRAINT `subjects_ibfk_2` FOREIGN KEY (`entity`) REFERENCES `entity_ids` (`id`);'
+
+# Set custom collations for columns
+mysql_execute "ALTER TABLE transaction_log MODIFY entity_id VARCHAR(255) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL;"
+
+
+### Finished ###
+
+update_version $NEW_VERSION
+
+success
+
+
+##### Useful commands #####
+
+# cd git/caosdb-mysqlbackend/
+# UTILSPATH=utils
+# . utils/patch_header.sh
+
+
+# show table status where Collation='utf8mb3_unicode_ci';
+
+# name_data
+# text_data
+
+# show table status like 'name_data';
+# select * from name_data where value like 'Yield%';
+# update name_data SET value='Yield-stréss' where entity_id=354;
+# update name_data SET value='Yield-strés😋s' where entity_id=354;
+# select * from name_data where value='Yield-stress';
+
+# # Unicode 14.0.0
+# ALTER TABLE name_data CONVERT TO CHARACTER SET utf8mb4 COLLATE uca1400_as_ci;
+
+# FULL_COLLATION_NAME: utf8mb4_uca1400_as_ci;
+# COLLATION_NAME: uca1400_as_ci;
+
+# mariadb -h sqldb -u caosdb -prandom1234 --default-character-set=utf8mb4 caosdb
+

procedures/entityVersioning.sql

 /*
- * This file is a part of the CaosDB Project.
+ * This file is a part of the LinkAhead Project.
  *
- * Copyright (C) 2020,2023 IndiScale GmbH <info@indiscale.com>
+ * Copyright (C) 2020-2024 IndiScale GmbH <info@indiscale.com>
  * Copyright (C) 2020,2023 Timm Fitschen <t.fitschen@indiscale.com>
- *
+ * Copyright (C) 2024 Daniel Hornung <d.hornung@indiscale.com>
  * This program is free software: you can redistribute it and/or modify
  * it under the terms of the GNU Affero General Public License as
  * published by the Free Software Foundation, either version 3 of the
@@ -283,7 +283,7 @@ DROP FUNCTION IF EXISTS db_5_0.get_head_relative //
  * ----------
  * EntityID : VARCHAR(255)
  *   The entity id.
- * Offset : INT UNSIGNED
+ * HeadOffset : INT UNSIGNED
  *   Distance in the sequence of primary parents of the entity. E.g. `0` is the
  *   HEAD itself. `1` is the primary parent of the HEAD. `2` is the primary
  *   parent of the primary parent of the HEAD, and so on.
@@ -294,7 +294,7 @@ DROP FUNCTION IF EXISTS db_5_0.get_head_relative //
  */
 CREATE FUNCTION db_5_0.get_head_relative(
     EntityID VARCHAR(255),
-    Offset INT UNSIGNED)
+    HeadOffset INT UNSIGNED)
 RETURNS VARBINARY(255)
 READS SQL DATA
 BEGIN
@@ -311,7 +311,7 @@ BEGIN
             FROM entity_version AS e
             WHERE e.entity_id = InternalEntityID
             ORDER BY e._iversion DESC
-            LIMIT 1 OFFSET Offset
+            LIMIT 1 OFFSET HeadOffset
         );
 END;
 //

procedures/query/initEntity.sql

@@ -80,9 +80,10 @@ initEntityLabel: BEGIN
             IF(versioned,
                 '` (id, _iversion) SELECT eids.internal_id, _get_head_iversion(eids.internal_id) ',
                 '` (id) SELECT eids.internal_id '),
-            'FROM entity_ids AS eids WHERE eids.id=',eid,';');
+            'FROM entity_ids AS eids WHERE eids.id=?;');
+        SET @query_param = eid;
         PREPARE initEntityStmt FROM @initEntityStmtStr;
-        EXECUTE initEntityStmt;
+        EXECUTE initEntityStmt USING @query_param;
         DEALLOCATE PREPARE initEntityStmt;
     END IF;
 

utils/helpers.sh

-# This file is a part of the CaosDB Project.
+# This file is a part of the LinkAhead Project.
 #
 # Copyright (C) 2018 Research Group Biomedical Physics,
 # Max-Planck-Institute for Dynamics and Self-Organization Göttingen
-# Copyright (C) 2019, 2020 Daniel Hornung (d.hornung@indiscale.com)
+# Copyright (C) 2019, 2020, 2024 Daniel Hornung (d.hornung@indiscale.com)
 # Copyright (C) 2020 Henrik tom Wörden
-# Copyright (C) 2020 IndiScale GmbH <info@indiscale.com>
+# Copyright (C) 2020, 2024 IndiScale GmbH <info@indiscale.com>
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as
@@ -34,7 +34,7 @@
 function mysql_execute {
 	  #TODO is it meaningful that here always the database user is used???
 	  set -e
-	  $MYSQL_CMD $(get_db_args) -e "$1"
+	  $MYSQL_CMD $(get_db_args) -e "$@"
 	  ret=${PIPESTATUS[0]}
 	  if [ "$ret" -ne 0 ]; then
 		    failure "MYSQL ERROR"
@@ -52,10 +52,16 @@ function mysql_execute_file {
 	fi
 }
 
+# Return the arguments from `get_db_args_nodb` and additionally the database name as
+# `--database=mydatabasename`.
 function get_db_args {
 	echo "$(get_db_args_nodb) --database=$DATABASE_NAME"
 }
 
+# Return arguments for user, password, host, port and additional options in MYSQL_OPTS.
+#
+# For example, the output may be:
+# --user=myuser --password=mypassword --host=example.com --port=1234 --other --option
 function get_db_args_nodb {
 	if [ -n "$MYSQL_USER" ]; then
 		mysql_con_arguments="--user=$MYSQL_USER"

utils/load_settings.sh

-# This file is a part of the CaosDB Project.
+# This file is a part of the Linkahead Project.
 #
 # Copyright (C) 2018 Research Group Biomedical Physics,
 #   Max-Planck-Institute for Dynamics and Self-Organization Göttingen
-# Copyright (C) 2019, 2020 Daniel Hornung (d.hornung@indiscale.com)
+# Copyright (C) 2019, 2020, 2024 Daniel Hornung <d.hornung@indiscale.com>
 # Copyright (C) 2020 Henrik tom Wörden
-# Copyright (C) 2020 IndiScale GmbH <info@indiscale.com>
+# Copyright (C) 2020, 2024 IndiScale GmbH <info@indiscale.com>
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as
@@ -62,10 +62,12 @@ export MYSQL_CMD
 export MYSQLDUMP_CMD
 export MYSQLADMIN_CMD
 export MYSQL_CONFIG_EDITOR_CMD
+
 export MYSQL_HOST
 export MYSQL_PORT
 export MYSQL_USER
 export MYSQL_OPTS
+
 export DATABASE_NAME
 export DATABASE_USER
 export DATABASE_USER_PW

utils/make_db

 #!/bin/bash
 
-# ** header v3.0
-# This file is a part of the CaosDB Project.
+# This file is a part of the LinkAhead Project.
 #
-# Copyright (C) 2021 Indiscale GmbH <info@indiscale.com>
-# Copyright (C) 2019, 2020, 2021 Daniel Hornung <d.hornung@indiscale.com>
+# Copyright (C) 2021, 2024 Indiscale GmbH <info@indiscale.com>
+# Copyright (C) 2019, 2020, 2021, 2024 Daniel Hornung <d.hornung@indiscale.com>
 # Copyright (C) 2020 Timm Fitschen <t.fitschen@indiscale.com>
 # Copyright (C) 2020 Henrik tom Wörden <h.tomwoerden@indiscale.com>
 # Copyright (C) 2020 IndiScale <info@indiscale.com>
@@ -22,7 +21,6 @@
 # You should have received a copy of the GNU Affero General Public License
 # along with this program. If not, see <https://www.gnu.org/licenses/>.
 #
-# ** end header
 
 # Although some sanity checks are performed, this script still allows lots of SQL injection
 # possibilities.

utils/patch_header.sh

-#
-# ** header v3.0
-# This file is a part of the CaosDB Project.
+# This file is a part of the LinkAhead Project.
 #
 # Copyright (C) 2018 Research Group Biomedical Physics,
 # Max-Planck-Institute for Dynamics and Self-Organization Göttingen
-# Copyright (C) 2019, 2020 Daniel Hornung <d.hornung@indiscale.com>
+# Copyright (C) 2019, 2020, 2025 Daniel Hornung <d.hornung@indiscale.com>
 # Copyright (C) 2020 Henrik tom Wörden <h.tomwoerden@indiscale.com>
-# Copyright (C) 2020 IndiScale <info@indiscale.com>
+# Copyright (C) 2020, 2025 IndiScale <info@indiscale.com>
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as
@@ -21,12 +19,20 @@
 # You should have received a copy of the GNU Affero General Public License
 # along with this program. If not, see <https://www.gnu.org/licenses/>.
 #
-# ** end header
-#
 
 ###############################################################################
 #                          header for patch scripts                           #
 ###############################################################################
+#
+# This file defines the following functions (see below for more detailed descriptions):
+#
+# - check_version_or_exit: Check if the stored version is as expected.
+# - update_version: Set the stored version.
+# - dump_table: Store table content into a file.
+# - redo_table: Apply the table content stored by dump_table.
+#
+# This file also sets the `errexit` option (`set -e`).
+#
 
 set -e
 
@@ -63,7 +69,7 @@ configuration make file.  Notable examples are:
 - DATABASE_USER_PW
 
 EOF
-              )
+           )
 
 function _print_help() {
   echo -e "$USAGE"
@@ -115,21 +121,51 @@ done
 if [ -n "$PATCH" ]; then
     echo -ne "applying patch $PATCH to $DATABASE_NAME ... "
 fi
-# @param $1: db version string, e.g. v2.0.0
-# @return: 0 on success, 1 on failure
-function check_version {
+
+# usage: check_version_or_exit version
+#
+# Check if the version string is as expected.  Otherwise, `exit 0`.
+#
+# Arguments
+# ---------
+#
+# version: str
+#   The expected database version string, e.g. 'v2.0.0'
+#
+# Returns
+# -------
+#
+# 0 on success
+function check_version_or_exit {
     local version=$($MYSQL_CMD $(get_db_args) -B -e "Select CaosDBVersion();")
-	  if [[ "$(echo $version | sed 's/^CaosDBVersion()\s//')" = "$1" ]]; then
+    local version="$(echo $version | sed 's/^CaosDBVersion()\s//')"
+	  if [[ "$version" = "$1" ]]; then
         return 0
 	  fi
     uptodate
 }
 
-# @param $1: new version string
+# Deprecated name, call "check_version_or_exit" directly instead.
+function check_version {
+    check_version_or_exit "$1"
+}
+
+# usage: update_version version
+#
+# Set the version string.
+#
+# Arguments
+# ---------
+#
+# version: str
+#   The new database version string, e.g. 'v2.0.0'
 function update_version {
     mysql_execute "DROP FUNCTION IF EXISTS CaosDBVersion; CREATE FUNCTION CaosDBVersion() RETURNS VARCHAR(255) DETERMINISTIC RETURN '$1';"
 }
 
+# usage: dump_table table
+#
+# Dump the table given in the argument into file "caosdb.<table>.<OLD_VERSION>.dump.sql"
 function dump_table {
     if [[ -z $MYSQLDUMP_CMD ]]; then
         echo "Cannot find mysqldump program!" >&2
@@ -139,6 +175,9 @@ function dump_table {
                     > ${DATABASE_NAME}.${1}.${OLD_VERSION}.dump.sql
 }
 
+# usage: redo_table table
+#
+# Apply the dump from `dump_table` to the database.  Takes the same argument.
 function redo_table {
          $MYSQL_CMD $(get_db_args) < ${DATABASE_NAME}.${1}.${OLD_VERSION}.dump.sql
 }

utils/sanity_check.sql

@@ -55,9 +55,29 @@ sanityCheckBody: BEGIN
 ("user_roles")
 ;
 
-SELECT COUNT(WRONG) INTO @count_wrong FROM ( SELECT l.table_name AS MATCHED, r.table_name AS WRONG FROM expected_tables AS l RIGHT OUTER JOIN information_schema.tables AS r ON (r.table_name COLLATE utf8_unicode_ci = l.table_name) WHERE r.table_schema = database()) AS temp WHERE temp.MATCHED IS NULL;
+SELECT COUNT(WRONG) INTO @count_wrong FROM (
+  SELECT l.table_name AS MATCHED,
+         r.table_name AS WRONG
+    FROM
+        expected_tables AS l
+      RIGHT OUTER JOIN
+        information_schema.tables AS r
+      ON (r.table_name COLLATE utf8_unicode_ci = l.table_name)
+      WHERE r.table_schema = database() AND r.table_type != 'TEMPORARY'
+  )
+  AS temp WHERE temp.MATCHED IS NULL;
 
-SELECT COUNT(MISSING) INTO @count_missing FROM ( SELECT l.table_name AS MISSING, r.table_name AS MATCHED FROM expected_tables AS l LEFT OUTER JOIN information_schema.tables AS r ON (r.table_name COLLATE utf8_unicode_ci = l.table_name) WHERE r.table_schema = database() OR r.table_schema IS NULL) AS temp WHERE temp.MATCHED IS NULL;
+SELECT COUNT(MISSING) INTO @count_missing FROM (
+  SELECT l.table_name AS MISSING,
+         r.table_name AS MATCHED
+    FROM
+        expected_tables AS l
+      LEFT OUTER JOIN
+        information_schema.tables AS r
+        ON (r.table_name COLLATE utf8_unicode_ci = l.table_name)
+        WHERE r.table_schema = database() OR r.table_schema IS NULL
+  )
+  AS temp WHERE temp.MATCHED IS NULL;
 
 IF @count_missing = 0 AND @count_wrong = 0 THEN
     LEAVE sanityCheckBody;
@@ -66,12 +86,28 @@ END IF;
 SELECT "--------------";
 SELECT @count_missing AS "Number of missing tables";
 
-SELECT MISSING AS "Missing tables" FROM ( SELECT l.table_name AS MISSING, r.table_name AS MATCHED FROM expected_tables AS l LEFT OUTER JOIN information_schema.tables AS r ON (r.table_name COLLATE utf8_unicode_ci = l.table_name) WHERE r.table_schema = database() OR r.table_schema IS NULL) AS temp WHERE temp.MATCHED IS NULL;
+SELECT MISSING AS "Missing tables" FROM (
+  SELECT l.table_name AS MISSING, r.table_name AS MATCHED FROM
+      expected_tables AS l
+    LEFT OUTER JOIN
+      information_schema.tables AS r
+    ON (r.table_name COLLATE utf8_unicode_ci = l.table_name)
+    WHERE r.table_schema = database() OR r.table_schema IS NULL
+  )
+  AS temp WHERE temp.MATCHED IS NULL;
 
 SELECT "--------------";
 SELECT @count_wrong AS "Number of tables which should not exist";
 
-SELECT WRONG AS "Tables which should not exist" FROM ( SELECT l.table_name AS MATCHED, r.table_name AS WRONG FROM expected_tables AS l RIGHT OUTER JOIN information_schema.tables AS r ON (r.table_name COLLATE utf8_unicode_ci = l.table_name) WHERE r.table_schema = database()) AS temp WHERE temp.MATCHED IS NULL;
+SELECT WRONG AS "Tables which should not exist" FROM (
+  SELECT expec.table_name AS MATCHED, info.table_name AS WRONG FROM
+      expected_tables AS expec
+    RIGHT OUTER JOIN
+      information_schema.tables AS info
+    ON (info.table_name COLLATE utf8_unicode_ci = expec.table_name)
+    WHERE info.table_schema = database() and info.table_type != 'TEMPORARY'
+  )
+  AS temp WHERE temp.MATCHED IS NULL;
 
 SELECT "--------------";
 SELECT "ERROR" from sanity_check_failed;

utils/update_sql_procedures.sh

 #!/bin/bash
 #
-# ** header v3.0
-# This file is a part of the CaosDB Project.
+# This file is a part of the LinkAhead Project.
 #
 # Copyright (C) 2018 Research Group Biomedical Physics,
 #   Max-Planck-Institute for Dynamics and Self-Organization Göttingen
-# Copyright (C) 2020 Daniel Hornung <d.hornung@indiscale.com>
+# Copyright (C) 2020, 2024 Daniel Hornung <d.hornung@indiscale.com>
 # Copyright (C) 2020 Henrik tom Wörden <h.tomwoerden@indiscale.com>
-# Copyright (C) 2020 IndiScale <info@indiscale.com>
+# Copyright (C) 2020, 2024 IndiScale <info@indiscale.com>
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as
@@ -22,8 +21,6 @@
 # You should have received a copy of the GNU Affero General Public License
 # along with this program. If not, see <https://www.gnu.org/licenses/>.
 #
-# ** end header
-#
 
 # Updates all SQL procedures
 
@@ -37,7 +34,7 @@ fi
 source $UTILSPATH/load_settings.sh
 source $UTILSPATH/helpers.sh
 
-echo -n "updating procedures ... "
+echo "Updating procedures ... "
 temp_proc_sql=$(mktemp --suffix=.sql)
 sed -e "s/db_5_0/$DATABASE_NAME/g" procedures/*.sql procedures/query/*.sql \
     > "$temp_proc_sql"