Skip to content
Snippets Groups Projects
Verified Commit 0801b110 authored by Daniel Hornung's avatar Daniel Hornung
Browse files

ENH: Update script for outdated dumps.

Also updated the MariaDB server image for test pipeline.
parent 2cd39d14
Branches
Tags
2 merge requests!31ENH: Update script for outdated dumps.,!30Mariadb 10.11: New reserved keyword
Pipeline #55923 failed
...@@ -8,8 +8,11 @@ RUN apt-get update \ ...@@ -8,8 +8,11 @@ RUN apt-get update \
unzip \ unzip \
python3-lxml \ python3-lxml \
python3-pip \ python3-pip \
python3-pytest \
python3-sqlparse \ python3-sqlparse \
python3-sphinx \ python3-sphinx \
doxygen doxygen
RUN pip3 install breathe sphinx-rtd-theme recommonmark RUN pip3 install --break-system-packages \
breathe \
sphinx-rtd-theme \
pytest \
recommonmark
...@@ -59,8 +59,7 @@ unittests-mariadb: ...@@ -59,8 +59,7 @@ unittests-mariadb:
tags: [ docker ] tags: [ docker ]
stage: test stage: test
services: services:
- mariadb:10.4 - mariadb:10.11
script: script:
- make pipeline-test SQL_HOST=mariadb - make pipeline-test SQL_HOST=mariadb
......
#!/usr/bin/env bash
# This file is a part of the LinkAhead Project.
#
# Copyright (C) 2024 IndiScale GmbH <info@indiscale.com>
# Copyright (C) 2024 Daniel Hornung <d.hornung@indiscale.com>
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as
# published by the Free Software Foundation, either version 3 of the
# License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <https://www.gnu.org/licenses/>.
# About this script
# =================
#
# Fix the use of "offset" as a procedure parameter name. "OFFSET" became a reserved keyword in
# MariaDB 10.6.
#
# This script takes an SQL dump from stdin and prints the fixed SQL to stdout.
#
# Usage
# -----
#
# 2024-10-02.dump_fix_mariadb_10_6.sh < yourdump.sql > yourdump.fixed.sql
set -euo pipefail
IFS=$'\n\t'
script='
s/Offset INT UNSIGNED) RETURNS varbinary(255)/HeadOffset INT UNSIGNED) RETURNS varbinary(255)/
s/LIMIT 1 OFFSET Offset/LIMIT 1 OFFSET HeadOffset/
'
sed -e "$script"
unset script
# SQL dump updates #
## Dump ##
This directory contains scripts to update database dumps to newer versions of the MariaDB server.
## Background ##
In some cases, restoring the database content from existing SQL dumps may not be possible in a
straightforward manner. For those cases, this directory contains scripts to help with known issues.
Examples for problems include:
- New reserved keywords in MariaDB, that were previously used as identifiers in code.
# Known issues and their fix #
- SQL syntax error near `Offset INT UNSIGNED`: If a dump was made before MariaDB 10.6 (LinkAhead <
0.15) and with the SQL backend before 7.0.3, there was a pramater named `Offset`. With MariaDB
10.6 however, `OFFSET` became a reserved keyword. This can be fixed by running
`2024-10-02.dump_fix_mariadb_10_6.sh < yourdump.sql > yourdump.fixed.sql`
#!/usr/bin/env python3
# This file is a part of the LinkAhead Project.
#
# Copyright (C) 2024 IndiScale GmbH <www.indiscale.com>
# Copyright (C) 2024 Daniel Hornung <d.hornung@indiscale.com>
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as
# published by the Free Software Foundation, either version 3 of the
# License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <https://www.gnu.org/licenses/>.
"""Testing the dump update scripts
"""
import filecmp
from pathlib import Path
from subprocess import run
from tempfile import NamedTemporaryFile
def get_basedir() -> str:
"""Return the assumped base dir for the dump updates.
"""
path = Path(__file__).parents[1]
return str(path)
def get_test_data(basename: str) -> list[tuple[str, str]]:
"""Return a list of [input, expectedoutput] tuples.
The output may be an empty string if no corresponding file can be found.
"""
basedir = get_basedir()
datadir = Path(basedir) / "test" / "test_data"
results = []
for input_path in datadir.glob(f"{basename}.example*[0-9].sql"):
expected_path = datadir / f"{input_path.name[:-4]}.expected.sql"
if expected_path.exists():
expected = str(expected_path)
else:
expected = ""
results.append((str(input_path), expected))
return results
def test_2024_10_02(tmpdir):
"""``Offset`` became a reserved keyword in MariaDB 10.6.
"""
script = "2024-10-02.dump_fix_mariadb_10_6.sh"
script_fullname = str(Path(get_basedir()) / script)
test_data = get_test_data(script[:-3])
for infile, expectedfile in test_data:
with (NamedTemporaryFile(dir=tmpdir, suffix=".sql", delete=True) as output,
open(infile, mode="rb") as infile_stream
):
run([script_fullname],
stdin=infile_stream,
stdout=output,
check=True
)
assert filecmp.cmp(output.name, expectedfile), "Output does not match expected output."
/* Just a short snippt with the problem */
/*!50003 SET @saved_col_connection = @@collation_connection */ ;
/*!50003 SET character_set_client = utf8 */ ;
/*!50003 SET character_set_results = utf8 */ ;
/*!50003 SET collation_connection = utf8_general_ci */ ;
DELIMITER ;;
CREATE DEFINER=`root`@`%` FUNCTION `get_head_relative`(EntityID VARCHAR(255),
HeadOffset INT UNSIGNED) RETURNS varbinary(255)
READS SQL DATA
BEGIN
DECLARE InternalEntityID INT UNSIGNED DEFAULT NULL;
SELECT internal_id INTO InternalEntityID FROM entity_ids WHERE id = EntityID;
RETURN (
SELECT e.version
FROM entity_version AS e
WHERE e.entity_id = InternalEntityID
ORDER BY e._iversion DESC
LIMIT 1 OFFSET HeadOffset
);
END ;;
DELIMITER ;
/*!50003 SET sql_mode = @saved_sql_mode */ ;
/*!50003 SET character_set_client = @saved_cs_client */ ;
/*!50003 SET character_set_results = @saved_cs_results */ ;
/*!50003 SET collation_connection = @saved_col_connection */ ;
/*!50003 SET @saved_sql_mode = @@sql_mode */ ;
/*!50003 SET sql_mode = 'STRICT_TRANS_TABLES,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION' */ ;
/*!50003 D */
/* Just a short snippt with the problem */
/*!50003 SET @saved_col_connection = @@collation_connection */ ;
/*!50003 SET character_set_client = utf8 */ ;
/*!50003 SET character_set_results = utf8 */ ;
/*!50003 SET collation_connection = utf8_general_ci */ ;
DELIMITER ;;
CREATE DEFINER=`root`@`%` FUNCTION `get_head_relative`(EntityID VARCHAR(255),
Offset INT UNSIGNED) RETURNS varbinary(255)
READS SQL DATA
BEGIN
DECLARE InternalEntityID INT UNSIGNED DEFAULT NULL;
SELECT internal_id INTO InternalEntityID FROM entity_ids WHERE id = EntityID;
RETURN (
SELECT e.version
FROM entity_version AS e
WHERE e.entity_id = InternalEntityID
ORDER BY e._iversion DESC
LIMIT 1 OFFSET Offset
);
END ;;
DELIMITER ;
/*!50003 SET sql_mode = @saved_sql_mode */ ;
/*!50003 SET character_set_client = @saved_cs_client */ ;
/*!50003 SET character_set_results = @saved_cs_results */ ;
/*!50003 SET collation_connection = @saved_col_connection */ ;
/*!50003 SET @saved_sql_mode = @@sql_mode */ ;
/*!50003 SET sql_mode = 'STRICT_TRANS_TABLES,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION' */ ;
/*!50003 D */
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment