Merge branch 'f-mariadb-10.11' into 'dev'

Mariadb 10.11: New reserved keyword

See merge request !30

.docker/Dockerfile

-FROM debian:buster
+FROM debian:bookworm
 RUN apt-get update \
     && \
     apt-get install -y \
@@ -11,4 +11,8 @@ RUN apt-get update \
 		python3-sqlparse \
 		python3-sphinx \
 		doxygen
-RUN pip3 install breathe sphinx-rtd-theme recommonmark
+RUN pip3 install --break-system-packages \
+  breathe \
+  sphinx-rtd-theme \
+  pytest \
+  recommonmark

.gitlab-ci.yml

 #
-# This file is a part of the CaosDB Project.
+# This file is a part of the LinkAhead Project.
 #
 # Copyright (C) 2018 Research Group Biomedical Physics,
 # Max-Planck-Institute for Dynamics and Self-Organization Göttingen
+# Copyright (C) 2024 Indiscale GmbH <info@indiscale.com>
 # Copyright (C) 2019 Henrik tom Wörden
+# Copyright (C) 2024 Daniel Hornung <d.hornung@indiscale.com>
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as
@@ -57,11 +59,17 @@ unittests-mariadb:
   tags: [ docker ]
   stage: test
   services:
-    - mariadb:10.4
-
+    - mariadb:10.11
   script:
     - make pipeline-test SQL_HOST=mariadb
 
+# Run the dump update tests
+test-dump-update:
+  tags: [ docker ]
+  stage: test
+  script:
+    - make test-dump-update
+
 # Run the unit tests with MySQL 5
 unittests-mysql-5:
   tags: [ docker ]
@@ -78,6 +86,7 @@ unittests-mysql-5:
     - sed -i "/col_default_is.*NULL/d" tests/test_autotap.sql
     - sed -i "/col_default_is.*INACTIVE/d" tests/test_autotap.sql
     - sed -i "/col_default_is.*SHA/d" tests/test_autotap.sql
+    - sed -i "s/utf8mb3/utf8/" tests/test_autotap.sql
     - make pipeline-test SQL_HOST=mysql
 
 ######## Deploy ########

CHANGELOG.md

@@ -9,6 +9,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Added ###
 
+- Script to update outdated database dumps.
+
 ### Changed ###
 
 ### Deprecated ###
@@ -17,6 +19,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Fixed ###
 
+- `OFFSET` is a reserved keyword in MariaDB 10.6, so a parameter name had to be changed.
+
 ### Security ###
 
 ## [7.0.2] - 2023-12-12 ##

Makefile

+# This file is a part of the Linkahead Project.
 #
-# ** header v3.0
-# This file is a part of the CaosDB Project.
-#
+# Copyright (C) 2024 Daniel Hornung <d.hornung@indiscale.com>
+# Copyright (C) 2024 IndiScale GmbH <info@indiscale.com>
 # Copyright (C) 2018 Research Group Biomedical Physics,
 # Max-Planck-Institute for Dynamics and Self-Organization Göttingen
 #
@@ -18,8 +18,7 @@
 # You should have received a copy of the GNU Affero General Public License
 # along with this program. If not, see <https://www.gnu.org/licenses/>.
 #
-# ** end header
-#
+
 SHELL=/bin/bash
 
 INSTALL_SQL_FILE=db_5_0.sql
@@ -56,23 +55,26 @@ drop-%:
 test:
 	./utils/make_db test --fresh
 
-# Run tests with a database which is started in a Docker container
+.PHONY: test-dump-update
+test-dump-update:
+	pytest dump_updates
+
+# Run tests with a database which is started in a MariaDB Docker container
 .PHONY: test-docker
 test-docker:
 	@docker kill caosdb-mysqlserver-test || true
-	@docker container rm caosdb-mysqlserver-test || true
+	@docker container rm -v caosdb-mysqlserver-test || true
 	@docker run --name caosdb-mysqlserver-test -p "3306:3306" \
-		-e MYSQL_ROOT_PASSWORD="pass-for-test" -d mariadb
+		-e MYSQL_ROOT_PASSWORD="pass-for-test" -d mariadb:10.11
 	@sleep 10
 	MAINPATH=$(realpath tests/docker_env) utils/make_db test --fresh
-	@docker kill caosdb-mysqlserver-test
-	@docker container rm caosdb-mysqlserver-test
+	make test-docker-stop
 
 # if automatic stopping failed
 .PHONY: test-docker-stop
 test-docker-stop:
 	docker kill caosdb-mysqlserver-test
-	docker container rm caosdb-mysqlserver-test
+	docker container rm -v caosdb-mysqlserver-test
 
 # Compile the standalone documentation
 .PHONY: doc

README_SETUP.md

@@ -114,6 +114,13 @@ obviously what was expected.
 Please look carefully through the tests. If something more serious than that
 comes up, please report a bug.
 
+#### Failure to restore a database dump created with an older MariaDB version ####
+
+Have a look into the `dump_updates/README.md`. In cases of version
+incompatibilities, the necessary steps to migrate the dump are
+probably described there and scripts for the migrations are provided
+in the same directory.
+
 ## Documentation #
 
 Build documentation in `build/` with `make doc`.

config.defaults

+# This file is a part of the LinkAhead Project.
 #
-# ** header v2.0
-# This file is a part of the CaosDB Project.
-#
+# Copyright (C) 2024 Daniel Hornung <d.hornung@indiscale.com>
+# Copyright (C) 2024 IndiScale GmbH <info@indiscale.com>
 # Copyright (C) 2018 Research Group Biomedical Physics,
 # Max-Planck-Institute for Dynamics and Self-Organization Göttingen
 #
@@ -15,11 +15,9 @@
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
-# You should have received a copy of the GNU General Public License
+# You should have received a copy of the GNU Affero General Public License
 # along with this program. If not, see <https://www.gnu.org/licenses/>.
 #
-# ** end header
-#
 
 # # Commands
 # The MySQL client program.

dump_updates/2024-10-02.dump_fix_mariadb_10_6.sh

+#!/usr/bin/env bash
+
+# This file is a part of the LinkAhead Project.
+#
+# Copyright (C) 2024 IndiScale GmbH <info@indiscale.com>
+# Copyright (C) 2024 Daniel Hornung <d.hornung@indiscale.com>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+# About this script
+# =================
+#
+# Fix the use of "offset" as a procedure parameter name.  "OFFSET" became a reserved keyword in
+# MariaDB 10.6.
+#
+# This script takes an SQL dump from stdin and prints the fixed SQL to stdout.
+#
+# Usage
+# -----
+#
+# 2024-10-02.dump_fix_mariadb_10_6.sh < yourdump.sql > yourdump.fixed.sql
+
+set -euo pipefail
+IFS=$'\n\t'
+
+script='
+s/Offset INT UNSIGNED) RETURNS varbinary(255)/HeadOffset INT UNSIGNED) RETURNS varbinary(255)/
+s/LIMIT 1 OFFSET Offset/LIMIT 1 OFFSET HeadOffset/
+'
+
+sed -e "$script"
+
+unset script

dump_updates/README.md

+# SQL dump updates #
+
+## Dump ##
+
+This directory contains scripts to update database dumps to newer versions of the MariaDB server.
+
+## Background ##
+
+In some cases, restoring the database content from existing SQL dumps may not be possible in a
+straightforward manner.  For those cases, this directory contains scripts to help with known issues.
+
+Examples for problems include:
+
+- New reserved keywords in MariaDB, that were previously used as identifiers in code.
+
+# Known issues and their fix #
+
+- SQL syntax error near `Offset INT UNSIGNED`: If a dump was made before MariaDB 10.6 (LinkAhead <
+  0.15) and with the SQL backend before 7.0.3, there was a pramater named `Offset`.  With MariaDB
+  10.6 however, `OFFSET` became a reserved keyword.  This can be fixed by running  
+  `2024-10-02.dump_fix_mariadb_10_6.sh < yourdump.sql > yourdump.fixed.sql`

dump_updates/test/test_all.py

+#!/usr/bin/env python3
+
+# This file is a part of the LinkAhead Project.
+#
+# Copyright (C) 2024 IndiScale GmbH <www.indiscale.com>
+# Copyright (C) 2024 Daniel Hornung <d.hornung@indiscale.com>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+"""Testing the dump update scripts
+"""
+
+import filecmp
+from pathlib import Path
+from subprocess import run
+from tempfile import NamedTemporaryFile
+
+
+def get_basedir() -> str:
+    """Return the assumped base dir for the dump updates.
+    """
+    path = Path(__file__).parents[1]
+    return str(path)
+
+
+def get_test_data(basename: str) -> list[tuple[str, str]]:
+    """Return a list of [input, expectedoutput] tuples.
+
+    The output may be an empty string if no corresponding file can be found.
+    """
+    basedir = get_basedir()
+    datadir = Path(basedir) / "test" / "test_data"
+    results = []
+    for input_path in datadir.glob(f"{basename}.example*[0-9].sql"):
+        expected_path = datadir / f"{input_path.name[:-4]}.expected.sql"
+        if expected_path.exists():
+            expected = str(expected_path)
+        else:
+            expected = ""
+        results.append((str(input_path), expected))
+    return results
+
+
+def test_2024_10_02(tmpdir):
+    """``Offset`` became a reserved keyword in MariaDB 10.6.
+    """
+    script = "2024-10-02.dump_fix_mariadb_10_6.sh"
+    script_fullname = str(Path(get_basedir()) / script)
+    test_data = get_test_data(script[:-3])
+    for infile, expectedfile in test_data:
+        with (NamedTemporaryFile(dir=tmpdir, suffix=".sql", delete=True) as output,
+              open(infile, mode="rb") as infile_stream
+              ):
+            run([script_fullname],
+                stdin=infile_stream,
+                stdout=output,
+                check=True
+                )
+            assert filecmp.cmp(output.name, expectedfile), "Output does not match expected output."

dump_updates/test/test_data/2024-10-02.dump_fix_mariadb_10_6.example1.expected.sql

+/* Just a short snippt with the problem */
+
+
+/*!50003 SET @saved_col_connection = @@collation_connection */ ;
+/*!50003 SET character_set_client  = utf8 */ ;
+/*!50003 SET character_set_results = utf8 */ ;
+/*!50003 SET collation_connection  = utf8_general_ci */ ;
+DELIMITER ;;
+CREATE DEFINER=`root`@`%` FUNCTION `get_head_relative`(EntityID VARCHAR(255),
+    HeadOffset INT UNSIGNED) RETURNS varbinary(255)
+    READS SQL DATA
+BEGIN
+    DECLARE InternalEntityID INT UNSIGNED DEFAULT NULL;
+
+    SELECT internal_id INTO InternalEntityID FROM entity_ids WHERE id = EntityID;
+
+    
+    
+    
+    
+    RETURN (
+        SELECT e.version
+            FROM entity_version AS e
+            WHERE e.entity_id = InternalEntityID
+            ORDER BY e._iversion DESC
+            LIMIT 1 OFFSET HeadOffset
+        );
+END ;;
+DELIMITER ;
+/*!50003 SET sql_mode              = @saved_sql_mode */ ;
+/*!50003 SET character_set_client  = @saved_cs_client */ ;
+/*!50003 SET character_set_results = @saved_cs_results */ ;
+/*!50003 SET collation_connection  = @saved_col_connection */ ;
+/*!50003 SET @saved_sql_mode       = @@sql_mode */ ;
+/*!50003 SET sql_mode              = 'STRICT_TRANS_TABLES,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION' */ ;
+/*!50003 D */

dump_updates/test/test_data/2024-10-02.dump_fix_mariadb_10_6.example1.sql

+/* Just a short snippt with the problem */
+
+
+/*!50003 SET @saved_col_connection = @@collation_connection */ ;
+/*!50003 SET character_set_client  = utf8 */ ;
+/*!50003 SET character_set_results = utf8 */ ;
+/*!50003 SET collation_connection  = utf8_general_ci */ ;
+DELIMITER ;;
+CREATE DEFINER=`root`@`%` FUNCTION `get_head_relative`(EntityID VARCHAR(255),
+    Offset INT UNSIGNED) RETURNS varbinary(255)
+    READS SQL DATA
+BEGIN
+    DECLARE InternalEntityID INT UNSIGNED DEFAULT NULL;
+
+    SELECT internal_id INTO InternalEntityID FROM entity_ids WHERE id = EntityID;
+
+    
+    
+    
+    
+    RETURN (
+        SELECT e.version
+            FROM entity_version AS e
+            WHERE e.entity_id = InternalEntityID
+            ORDER BY e._iversion DESC
+            LIMIT 1 OFFSET Offset
+        );
+END ;;
+DELIMITER ;
+/*!50003 SET sql_mode              = @saved_sql_mode */ ;
+/*!50003 SET character_set_client  = @saved_cs_client */ ;
+/*!50003 SET character_set_results = @saved_cs_results */ ;
+/*!50003 SET collation_connection  = @saved_col_connection */ ;
+/*!50003 SET @saved_sql_mode       = @@sql_mode */ ;
+/*!50003 SET sql_mode              = 'STRICT_TRANS_TABLES,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION' */ ;
+/*!50003 D */

patches/applyPatches.sh

@@ -47,4 +47,5 @@ do
 done
 
 cd ../
+
 $UTILSPATH/update_sql_procedures.sh

procedures/entityVersioning.sql

 /*
- * This file is a part of the CaosDB Project.
+ * This file is a part of the LinkAhead Project.
  *
- * Copyright (C) 2020,2023 IndiScale GmbH <info@indiscale.com>
+ * Copyright (C) 2020-2024 IndiScale GmbH <info@indiscale.com>
  * Copyright (C) 2020,2023 Timm Fitschen <t.fitschen@indiscale.com>
- *
+ * Copyright (C) 2024 Daniel Hornung <d.hornung@indiscale.com>
  * This program is free software: you can redistribute it and/or modify
  * it under the terms of the GNU Affero General Public License as
  * published by the Free Software Foundation, either version 3 of the
@@ -283,7 +283,7 @@ DROP FUNCTION IF EXISTS db_5_0.get_head_relative //
  * ----------
  * EntityID : VARCHAR(255)
  *   The entity id.
- * Offset : INT UNSIGNED
+ * HeadOffset : INT UNSIGNED
  *   Distance in the sequence of primary parents of the entity. E.g. `0` is the
  *   HEAD itself. `1` is the primary parent of the HEAD. `2` is the primary
  *   parent of the primary parent of the HEAD, and so on.
@@ -294,7 +294,7 @@ DROP FUNCTION IF EXISTS db_5_0.get_head_relative //
  */
 CREATE FUNCTION db_5_0.get_head_relative(
     EntityID VARCHAR(255),
-    Offset INT UNSIGNED)
+    HeadOffset INT UNSIGNED)
 RETURNS VARBINARY(255)
 READS SQL DATA
 BEGIN
@@ -311,7 +311,7 @@ BEGIN
             FROM entity_version AS e
             WHERE e.entity_id = InternalEntityID
             ORDER BY e._iversion DESC
-            LIMIT 1 OFFSET Offset
+            LIMIT 1 OFFSET HeadOffset
         );
 END;
 //

utils/helpers.sh

-# This file is a part of the CaosDB Project.
+# This file is a part of the LinkAhead Project.
 #
 # Copyright (C) 2018 Research Group Biomedical Physics,
 # Max-Planck-Institute for Dynamics and Self-Organization Göttingen
-# Copyright (C) 2019, 2020 Daniel Hornung (d.hornung@indiscale.com)
+# Copyright (C) 2019, 2020, 2024 Daniel Hornung (d.hornung@indiscale.com)
 # Copyright (C) 2020 Henrik tom Wörden
-# Copyright (C) 2020 IndiScale GmbH <info@indiscale.com>
+# Copyright (C) 2020, 2024 IndiScale GmbH <info@indiscale.com>
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as
@@ -52,10 +52,16 @@ function mysql_execute_file {
 	fi
 }
 
+# Return the arguments from `get_db_args_nodb` and additionally the database name as
+# `--database=mydatabasename`.
 function get_db_args {
 	echo "$(get_db_args_nodb) --database=$DATABASE_NAME"
 }
 
+# Return arguments for user, password, host, port and additional options in MYSQL_OPTS.
+#
+# For example, the output may be:
+# --user=myuser --password=mypassword --host=example.com --port=1234 --other --option
 function get_db_args_nodb {
 	if [ -n "$MYSQL_USER" ]; then
 		mysql_con_arguments="--user=$MYSQL_USER"

utils/load_settings.sh

-# This file is a part of the CaosDB Project.
+# This file is a part of the Linkahead Project.
 #
 # Copyright (C) 2018 Research Group Biomedical Physics,
 #   Max-Planck-Institute for Dynamics and Self-Organization Göttingen
-# Copyright (C) 2019, 2020 Daniel Hornung (d.hornung@indiscale.com)
+# Copyright (C) 2019, 2020, 2024 Daniel Hornung <d.hornung@indiscale.com>
 # Copyright (C) 2020 Henrik tom Wörden
-# Copyright (C) 2020 IndiScale GmbH <info@indiscale.com>
+# Copyright (C) 2020, 2024 IndiScale GmbH <info@indiscale.com>
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as
@@ -62,10 +62,12 @@ export MYSQL_CMD
 export MYSQLDUMP_CMD
 export MYSQLADMIN_CMD
 export MYSQL_CONFIG_EDITOR_CMD
+
 export MYSQL_HOST
 export MYSQL_PORT
 export MYSQL_USER
 export MYSQL_OPTS
+
 export DATABASE_NAME
 export DATABASE_USER
 export DATABASE_USER_PW

utils/make_db

 #!/bin/bash
 
-# ** header v3.0
-# This file is a part of the CaosDB Project.
+# This file is a part of the LinkAhead Project.
 #
-# Copyright (C) 2021 Indiscale GmbH <info@indiscale.com>
-# Copyright (C) 2019, 2020, 2021 Daniel Hornung <d.hornung@indiscale.com>
+# Copyright (C) 2021, 2024 Indiscale GmbH <info@indiscale.com>
+# Copyright (C) 2019, 2020, 2021, 2024 Daniel Hornung <d.hornung@indiscale.com>
 # Copyright (C) 2020 Timm Fitschen <t.fitschen@indiscale.com>
 # Copyright (C) 2020 Henrik tom Wörden <h.tomwoerden@indiscale.com>
 # Copyright (C) 2020 IndiScale <info@indiscale.com>
@@ -22,7 +21,6 @@
 # You should have received a copy of the GNU Affero General Public License
 # along with this program. If not, see <https://www.gnu.org/licenses/>.
 #
-# ** end header
 
 # Although some sanity checks are performed, this script still allows lots of SQL injection
 # possibilities.

utils/update_sql_procedures.sh

 #!/bin/bash
 #
-# ** header v3.0
-# This file is a part of the CaosDB Project.
+# This file is a part of the LinkAhead Project.
 #
 # Copyright (C) 2018 Research Group Biomedical Physics,
 #   Max-Planck-Institute for Dynamics and Self-Organization Göttingen
-# Copyright (C) 2020 Daniel Hornung <d.hornung@indiscale.com>
+# Copyright (C) 2020, 2024 Daniel Hornung <d.hornung@indiscale.com>
 # Copyright (C) 2020 Henrik tom Wörden <h.tomwoerden@indiscale.com>
-# Copyright (C) 2020 IndiScale <info@indiscale.com>
+# Copyright (C) 2020, 2024 IndiScale <info@indiscale.com>
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as
@@ -22,8 +21,6 @@
 # You should have received a copy of the GNU Affero General Public License
 # along with this program. If not, see <https://www.gnu.org/licenses/>.
 #
-# ** end header
-#
 
 # Updates all SQL procedures
 
@@ -37,7 +34,7 @@ fi
 source $UTILSPATH/load_settings.sh
 source $UTILSPATH/helpers.sh
 
-echo -n "updating procedures ... "
+echo "Updating procedures ... "
 temp_proc_sql=$(mktemp --suffix=.sql)
 sed -e "s/db_5_0/$DATABASE_NAME/g" procedures/*.sql procedures/query/*.sql \
     > "$temp_proc_sql"