Merge branch 'release-8.0.0' into 'main'

ENH: Update script for outdated dumps.

See merge request !31

.docker/Dockerfile

-FROM debian:buster
+FROM debian:bookworm
 RUN apt-get update \
     && \
     apt-get install -y \
@@ -11,4 +11,8 @@ RUN apt-get update \
 		python3-sqlparse \
 		python3-sphinx \
 		doxygen
-RUN pip3 install breathe sphinx-rtd-theme recommonmark
+RUN pip3 install --break-system-packages \
+  breathe \
+  sphinx-rtd-theme \
+  pytest \
+  recommonmark

.gitignore

@@ -14,3 +14,4 @@ libs/*
 _doxygen/
 _generated/
 /build/
+*~

.gitlab-ci.yml

 #
-# This file is a part of the CaosDB Project.
+# This file is a part of the LinkAhead Project.
 #
 # Copyright (C) 2018 Research Group Biomedical Physics,
 # Max-Planck-Institute for Dynamics and Self-Organization Göttingen
+# Copyright (C) 2024 Indiscale GmbH <info@indiscale.com>
 # Copyright (C) 2019 Henrik tom Wörden
+# Copyright (C) 2024 Daniel Hornung <d.hornung@indiscale.com>
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as
@@ -57,11 +59,17 @@ unittests-mariadb:
   tags: [ docker ]
   stage: test
   services:
-    - mariadb:10.4
-
+    - mariadb:10.11
   script:
     - make pipeline-test SQL_HOST=mariadb
 
+# Run the dump update tests
+test-dump-update:
+  tags: [ docker ]
+  stage: test
+  script:
+    - make test-dump-update
+
 # Run the unit tests with MySQL 5
 unittests-mysql-5:
   tags: [ docker ]
@@ -78,6 +86,7 @@ unittests-mysql-5:
     - sed -i "/col_default_is.*NULL/d" tests/test_autotap.sql
     - sed -i "/col_default_is.*INACTIVE/d" tests/test_autotap.sql
     - sed -i "/col_default_is.*SHA/d" tests/test_autotap.sql
+    - sed -i "s/utf8mb3/utf8/" tests/test_autotap.sql
     - make pipeline-test SQL_HOST=mysql
 
 ######## Deploy ########

CHANGELOG.md

@@ -5,6 +5,21 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [8.0.0] - 2024-10-24 ##
+
+### Added ###
+
+- Script to update outdated database dumps.
+
+### Changed ###
+
+- `OFFSET` is a reserved keyword in MariaDB 10.6, so the former
+  `OFFSET` parameter name in the `get_head_relative` function was
+  changed to HeadOffset. See
+  [`update_dumps/README.md`](update_dumps/README.md) for information
+  on how to update old SQL dumps to be compatible with this renaming
+  and MariaDB >= 10.6.
+
 ## [7.0.2] - 2023-12-12 ##
 (Timm Fitschen)
 

CITATION.cff

+cff-version: 1.2.0
+message: "If you use this software, please cite it as below."
+authors:
+  - family-names: Fitschen
+    given-names: Timm
+    orcid: https://orcid.org/0000-0002-4022-432X
+  - family-names: Schlemmer
+    given-names: Alexander
+    orcid: https://orcid.org/0000-0003-4124-9649
+  - family-names: Hornung
+    given-names: Daniel
+    orcid: https://orcid.org/0000-0002-7846-6375
+  - family-names: tom Wörden
+    given-names: Henrik
+    orcid: https://orcid.org/0000-0002-5549-578X
+  - family-names: Spreckelsen
+    given-names: Florian
+    orcid: https://orcid.org/0000-0002-6856-2910
+  - family-names: Parlitz
+    given-names: Ulrich
+    orcid: https://orcid.org/0000-0003-3058-1435
+  - family-names: Luther
+    given-names: Stefan
+    orcid: https://orcid.org/0000-0001-7214-8125
+title: "LinkAhead - MariaDB Backend"
+version: 8.0.0
+doi: 10.3390/data4020083
+date-released: 2024-10-10

Makefile

+# This file is a part of the Linkahead Project.
 #
-# ** header v3.0
-# This file is a part of the CaosDB Project.
-#
+# Copyright (C) 2024 Daniel Hornung <d.hornung@indiscale.com>
+# Copyright (C) 2024 IndiScale GmbH <info@indiscale.com>
 # Copyright (C) 2018 Research Group Biomedical Physics,
 # Max-Planck-Institute for Dynamics and Self-Organization Göttingen
 #
@@ -18,8 +18,7 @@
 # You should have received a copy of the GNU Affero General Public License
 # along with this program. If not, see <https://www.gnu.org/licenses/>.
 #
-# ** end header
-#
+
 SHELL=/bin/bash
 
 INSTALL_SQL_FILE=db_5_0.sql
@@ -56,23 +55,26 @@ drop-%:
 test:
 	./utils/make_db test --fresh
 
-# Run tests with a database which is started in a Docker container
+.PHONY: test-dump-update
+test-dump-update:
+	pytest dump_updates
+
+# Run tests with a database which is started in a MariaDB Docker container
 .PHONY: test-docker
 test-docker:
 	@docker kill caosdb-mysqlserver-test || true
-	@docker container rm caosdb-mysqlserver-test || true
+	@docker container rm -v caosdb-mysqlserver-test || true
 	@docker run --name caosdb-mysqlserver-test -p "3306:3306" \
-		-e MYSQL_ROOT_PASSWORD="pass-for-test" -d mariadb
+		-e MYSQL_ROOT_PASSWORD="pass-for-test" -d mariadb:10.11
 	@sleep 10
 	MAINPATH=$(realpath tests/docker_env) utils/make_db test --fresh
-	@docker kill caosdb-mysqlserver-test
-	@docker container rm caosdb-mysqlserver-test
+	make test-docker-stop
 
 # if automatic stopping failed
 .PHONY: test-docker-stop
 test-docker-stop:
 	docker kill caosdb-mysqlserver-test
-	docker container rm caosdb-mysqlserver-test
+	docker container rm -v caosdb-mysqlserver-test
 
 # Compile the standalone documentation
 .PHONY: doc

README_SETUP.md

@@ -114,6 +114,13 @@ obviously what was expected.
 Please look carefully through the tests. If something more serious than that
 comes up, please report a bug.
 
+#### Failure to restore a database dump created with an older MariaDB version ####
+
+Have a look into the `dump_updates/README.md`. In cases of version
+incompatibilities, the necessary steps to migrate the dump are
+probably described there and scripts for the migrations are provided
+in the same directory.
+
 ## Documentation #
 
 Build documentation in `build/` with `make doc`.

RELEASE_GUIDELINES.md

@@ -6,7 +6,8 @@ guidelines of the CaosDB Project
 
 ## General Prerequisites
 
-* All tests are passing.
+* All tests are passing. (Note that pipelines may fail if the major
+  version increased. Rely on manual tests in that case.)
 * (FEATURES.md is up-to-date and a public API is being declared in that document.)
 * CHANGELOG.md is up-to-date.
 * (DEPENDENCIES.md is up-to-date.)
@@ -22,7 +23,7 @@ guidelines of the CaosDB Project
 3. Assure that the latest patch actually updates the CaosDBVersion() procedure
    and that the version matches the version you are about to release.
 
-4. Update version in `doc/conf.py`.
+4. Update version in `doc/conf.py` and in `CITATION.cff`.
 
 5. Merge the release branch into the main branch.
 
@@ -36,3 +37,6 @@ guidelines of the CaosDB Project
    section to the CHANGELOG.
 
 10. Create a release at the [gitlab repo](https://gitlab.com/linkahead/linkahead-mariadbbackend/-/releases)
+
+11. If necessary, i.e., if the patch or major versions were increased
+	with the release, do a LinkAhead server release.

config.defaults

+# This file is a part of the LinkAhead Project.
 #
-# ** header v2.0
-# This file is a part of the CaosDB Project.
-#
+# Copyright (C) 2024 Daniel Hornung <d.hornung@indiscale.com>
+# Copyright (C) 2024 IndiScale GmbH <info@indiscale.com>
 # Copyright (C) 2018 Research Group Biomedical Physics,
 # Max-Planck-Institute for Dynamics and Self-Organization Göttingen
 #
@@ -15,11 +15,9 @@
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
-# You should have received a copy of the GNU General Public License
+# You should have received a copy of the GNU Affero General Public License
 # along with this program. If not, see <https://www.gnu.org/licenses/>.
 #
-# ** end header
-#
 
 # # Commands
 # The MySQL client program.

doc/conf.py

@@ -22,13 +22,13 @@ import sphinx_rtd_theme
 # -- Project information -----------------------------------------------------
 
 project = 'caosdb-mysqlbackend'
-copyright = '2021, IndiScale GmbH'
+copyright = '2021 - 2024, IndiScale GmbH'
 author = 'Daniel Hornung'
 
 # The short X.Y version
-version = '7.0.2'
+version = '8.0.0'
 # The full version, including alpha/beta/rc tags
-release = '7.0.2'
+release = '8.0.0'
 
 
 # -- General configuration ---------------------------------------------------

doc/index.rst

@@ -11,6 +11,8 @@ Welcome to documentation of CaosDB's MySQL Backend!
    Concepts <concepts>
    Maintenance 
    API documentation <functions>
+   Related Projects <related_projects/index>
+   Back to Overview <https://docs.indiscale.com/>
 
 This documentation helps you to :doc:`get started<README_SETUP>`, explains the most important
 :doc:`concepts<concepts>` .

doc/related_projects/index.rst

+Related Projects
+++++++++++++++++
+
+.. toctree::
+   :maxdepth: 2
+   :caption: Contents:
+   :hidden:
+
+.. container:: projects
+
+   For in-depth documentation for users, administrators  and developers, you may want to visit the subproject-specific documentation pages for:
+
+   :`Server <https://docs.indiscale.com/caosdb-server>`_: The Java part of the LinkAhead server.
+
+   :`WebUI <https://docs.indiscale.com/caosdb-webui>`_: The default web frontend for the LinkAhead server.
+
+   :`PyLinkAhead <https://docs.indiscale.com/caosdb-pylib>`_: The LinkAhead Python library.
+
+   :`Advanced user tools <https://docs.indiscale.com/caosdb-advanced-user-tools>`_: The advanced Python tools for LinkAhead.
+
+   :`LinkAhead Crawler <https://docs.indiscale.com/caosdb-crawler/>`_: The crawler is the main tool for automatic data integration in LinkAhead.
+
+   :`LinkAhead <https://docs.indiscale.com/caosdb-deploy>`_: Your all inclusive LinkAhead software package.
+
+   :`Back to Overview <https://docs.indiscale.com/>`_: LinkAhead Documentation.

dump_updates/2024-10-02.dump_fix_mariadb_10_6.sh

+#!/usr/bin/env bash
+
+# This file is a part of the LinkAhead Project.
+#
+# Copyright (C) 2024 IndiScale GmbH <info@indiscale.com>
+# Copyright (C) 2024 Daniel Hornung <d.hornung@indiscale.com>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+# About this script
+# =================
+#
+# Fix the use of "offset" as a procedure parameter name.  "OFFSET" became a reserved keyword in
+# MariaDB 10.6.
+#
+# This script takes an SQL dump from stdin and prints the fixed SQL to stdout.
+#
+# Usage
+# -----
+#
+# 2024-10-02.dump_fix_mariadb_10_6.sh < yourdump.sql > yourdump.fixed.sql
+
+set -euo pipefail
+IFS=$'\n\t'
+
+script='
+s/Offset INT UNSIGNED) RETURNS varbinary(255)/HeadOffset INT UNSIGNED) RETURNS varbinary(255)/
+s/LIMIT 1 OFFSET Offset/LIMIT 1 OFFSET HeadOffset/
+'
+
+sed -e "$script"
+
+unset script

dump_updates/README.md

+# SQL dump updates #
+
+## Dump ##
+
+This directory contains scripts to update database dumps to newer versions of the MariaDB server.
+
+## Background ##
+
+In some cases, restoring the database content from existing SQL dumps may not be possible in a
+straightforward manner.  For those cases, this directory contains scripts to help with known issues.
+
+Examples for problems include:
+
+- New reserved keywords in MariaDB, that were previously used as identifiers in code.
+
+# Known issues and their fix #
+
+- SQL syntax error near `Offset INT UNSIGNED`: If a dump was made before MariaDB 10.6 (LinkAhead <
+  0.15) and with the SQL backend before 7.0.3, there was a pramater named `Offset`.  With MariaDB
+  10.6 however, `OFFSET` became a reserved keyword.  This can be fixed by running  
+  `2024-10-02.dump_fix_mariadb_10_6.sh < yourdump.sql > yourdump.fixed.sql`

dump_updates/test/test_all.py

+#!/usr/bin/env python3
+
+# This file is a part of the LinkAhead Project.
+#
+# Copyright (C) 2024 IndiScale GmbH <www.indiscale.com>
+# Copyright (C) 2024 Daniel Hornung <d.hornung@indiscale.com>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+"""Testing the dump update scripts
+"""
+
+import filecmp
+from pathlib import Path
+from subprocess import run
+from tempfile import NamedTemporaryFile
+
+
+def get_basedir() -> str:
+    """Return the assumped base dir for the dump updates.
+    """
+    path = Path(__file__).parents[1]
+    return str(path)
+
+
+def get_test_data(basename: str) -> list[tuple[str, str]]:
+    """Return a list of [input, expectedoutput] tuples.
+
+    The output may be an empty string if no corresponding file can be found.
+    """
+    basedir = get_basedir()
+    datadir = Path(basedir) / "test" / "test_data"
+    results = []
+    for input_path in datadir.glob(f"{basename}.example*[0-9].sql"):
+        expected_path = datadir / f"{input_path.name[:-4]}.expected.sql"
+        if expected_path.exists():
+            expected = str(expected_path)
+        else:
+            expected = ""
+        results.append((str(input_path), expected))
+    return results
+
+
+def test_2024_10_02(tmpdir):
+    """``Offset`` became a reserved keyword in MariaDB 10.6.
+    """
+    script = "2024-10-02.dump_fix_mariadb_10_6.sh"
+    script_fullname = str(Path(get_basedir()) / script)
+    test_data = get_test_data(script[:-3])
+    for infile, expectedfile in test_data:
+        with (NamedTemporaryFile(dir=tmpdir, suffix=".sql", delete=True) as output,
+              open(infile, mode="rb") as infile_stream
+              ):
+            run([script_fullname],
+                stdin=infile_stream,
+                stdout=output,
+                check=True
+                )
+            assert filecmp.cmp(output.name, expectedfile), "Output does not match expected output."

dump_updates/test/test_data/2024-10-02.dump_fix_mariadb_10_6.example1.expected.sql

+/* Just a short snippt with the problem */
+
+
+/*!50003 SET @saved_col_connection = @@collation_connection */ ;
+/*!50003 SET character_set_client  = utf8 */ ;
+/*!50003 SET character_set_results = utf8 */ ;
+/*!50003 SET collation_connection  = utf8_general_ci */ ;
+DELIMITER ;;
+CREATE DEFINER=`root`@`%` FUNCTION `get_head_relative`(EntityID VARCHAR(255),
+    HeadOffset INT UNSIGNED) RETURNS varbinary(255)
+    READS SQL DATA
+BEGIN
+    DECLARE InternalEntityID INT UNSIGNED DEFAULT NULL;
+
+    SELECT internal_id INTO InternalEntityID FROM entity_ids WHERE id = EntityID;
+
+    
+    
+    
+    
+    RETURN (
+        SELECT e.version
+            FROM entity_version AS e
+            WHERE e.entity_id = InternalEntityID
+            ORDER BY e._iversion DESC
+            LIMIT 1 OFFSET HeadOffset
+        );
+END ;;
+DELIMITER ;
+/*!50003 SET sql_mode              = @saved_sql_mode */ ;
+/*!50003 SET character_set_client  = @saved_cs_client */ ;
+/*!50003 SET character_set_results = @saved_cs_results */ ;
+/*!50003 SET collation_connection  = @saved_col_connection */ ;
+/*!50003 SET @saved_sql_mode       = @@sql_mode */ ;
+/*!50003 SET sql_mode              = 'STRICT_TRANS_TABLES,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION' */ ;
+/*!50003 D */

dump_updates/test/test_data/2024-10-02.dump_fix_mariadb_10_6.example1.sql

+/* Just a short snippt with the problem */
+
+
+/*!50003 SET @saved_col_connection = @@collation_connection */ ;
+/*!50003 SET character_set_client  = utf8 */ ;
+/*!50003 SET character_set_results = utf8 */ ;
+/*!50003 SET collation_connection  = utf8_general_ci */ ;
+DELIMITER ;;
+CREATE DEFINER=`root`@`%` FUNCTION `get_head_relative`(EntityID VARCHAR(255),
+    Offset INT UNSIGNED) RETURNS varbinary(255)
+    READS SQL DATA
+BEGIN
+    DECLARE InternalEntityID INT UNSIGNED DEFAULT NULL;
+
+    SELECT internal_id INTO InternalEntityID FROM entity_ids WHERE id = EntityID;
+
+    
+    
+    
+    
+    RETURN (
+        SELECT e.version
+            FROM entity_version AS e
+            WHERE e.entity_id = InternalEntityID
+            ORDER BY e._iversion DESC
+            LIMIT 1 OFFSET Offset
+        );
+END ;;
+DELIMITER ;
+/*!50003 SET sql_mode              = @saved_sql_mode */ ;
+/*!50003 SET character_set_client  = @saved_cs_client */ ;
+/*!50003 SET character_set_results = @saved_cs_results */ ;
+/*!50003 SET collation_connection  = @saved_col_connection */ ;
+/*!50003 SET @saved_sql_mode       = @@sql_mode */ ;
+/*!50003 SET sql_mode              = 'STRICT_TRANS_TABLES,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION' */ ;
+/*!50003 D */

patches/applyPatches.sh

@@ -47,4 +47,5 @@ do
 done
 
 cd ../
+
 $UTILSPATH/update_sql_procedures.sh

patches/patch20241024-8.0.0/patch.sh

+#!/bin/bash
+#
+# This file is a part of the LinkAhead Project.
+#
+# Copyright (C) 2024 IndiScale GmbH <info@indiscale.com>
+# Copyright (C) 2024 Timm Fitschen <t.fitschen@indiscale.com>
+# Copyright (C) 2024 Florian Spreckelsen <f.spreckelsen@indiscale.com>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see <https://www.gnu.org/licenses/>.
+#
+
+# Update mysql schema to version v8.0.0
+#
+# The patch is empty because the change has to be applied manually
+# from ../../dump_updates/2024-10-02.dump_fix_mariadb_10_6.sh if necessary.
+
+NEW_VERSION="v8.0.0"
+OLD_VERSION="v7.0.2"
+
+if [ -z "$UTILSPATH" ]; then
+ UTILSPATH="../utils"
+fi
+
+. $UTILSPATH/patch_header.sh $*
+
+check_version $OLD_VERSION
+
+update_version $NEW_VERSION
+
+success

procedures/entityVersioning.sql

 /*
- * This file is a part of the CaosDB Project.
+ * This file is a part of the LinkAhead Project.
  *
- * Copyright (C) 2020,2023 IndiScale GmbH <info@indiscale.com>
+ * Copyright (C) 2020-2024 IndiScale GmbH <info@indiscale.com>
  * Copyright (C) 2020,2023 Timm Fitschen <t.fitschen@indiscale.com>
- *
+ * Copyright (C) 2024 Daniel Hornung <d.hornung@indiscale.com>
  * This program is free software: you can redistribute it and/or modify
  * it under the terms of the GNU Affero General Public License as
  * published by the Free Software Foundation, either version 3 of the
@@ -283,7 +283,7 @@ DROP FUNCTION IF EXISTS db_5_0.get_head_relative //
  * ----------
  * EntityID : VARCHAR(255)
  *   The entity id.
- * Offset : INT UNSIGNED
+ * HeadOffset : INT UNSIGNED
  *   Distance in the sequence of primary parents of the entity. E.g. `0` is the
  *   HEAD itself. `1` is the primary parent of the HEAD. `2` is the primary
  *   parent of the primary parent of the HEAD, and so on.
@@ -294,7 +294,7 @@ DROP FUNCTION IF EXISTS db_5_0.get_head_relative //
  */
 CREATE FUNCTION db_5_0.get_head_relative(
     EntityID VARCHAR(255),
-    Offset INT UNSIGNED)
+    HeadOffset INT UNSIGNED)
 RETURNS VARBINARY(255)
 READS SQL DATA
 BEGIN
@@ -311,7 +311,7 @@ BEGIN
             FROM entity_version AS e
             WHERE e.entity_id = InternalEntityID
             ORDER BY e._iversion DESC
-            LIMIT 1 OFFSET Offset
+            LIMIT 1 OFFSET HeadOffset
         );
 END;
 //