Merge branch 'main' into dev

CHANGELOG.md

@@ -5,7 +5,6 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
-
 ## [Unreleased] ##
 
 ### Added ###
@@ -20,6 +19,13 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Security ###
 
+## [7.0.1] - 2023-11-01
+(Timm Fitschen)
+
+### Added ###
+
+* Sanity checks during `make install` and `make upgrade`.
+
 ## [7.0.0] - 2023-10-25 ##
 (Timm Fitschen)
 

Makefile

@@ -34,6 +34,7 @@ test-connection:
 .PHONY: upgrade
 upgrade:
 	@cd patches; ./applyPatches.sh --env=../.config
+	@./utils/make_db sanity_check
 
 .PHONY: install
 install: _install _grant upgrade

doc/conf.py

@@ -26,9 +26,9 @@ copyright = '2021, IndiScale GmbH'
 author = 'Daniel Hornung'
 
 # The short X.Y version
-version = '7.0'
+version = '7.0.1'
 # The full version, including alpha/beta/rc tags
-release = '7.0.0'
+release = '7.0.1'
 
 
 # -- General configuration ---------------------------------------------------

patches/patch20231101-7.0.1/patch.sh

+#!/bin/bash
+#
+# This file is a part of the CaosDB Project.
+#
+# Copyright (C) 2023 IndiScale GmbH <info@indiscale.com>
+# Copyright (C) 2023 Timm Fitschen <t.fitschen@indiscale.com>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see <https://www.gnu.org/licenses/>.
+#
+
+# Update mysql schema to version v7.0.1
+# Removing remnant tables. This patch version also adds new sanity checks
+# during installation/upgrade
+
+NEW_VERSION="v7.0.1"
+OLD_VERSION="v7.0.0"
+
+if [ -z "$UTILSPATH" ]; then
+ UTILSPATH="../utils"
+fi
+
+. $UTILSPATH/patch_header.sh $*
+
+check_version $OLD_VERSION
+
+mysql_execute "DROP TABLE IF EXISTS groups";
+mysql_execute "DROP TABLE IF EXISTS passwords";
+mysql_execute "DROP TABLE IF EXISTS logging";
+mysql_execute "DROP TABLE IF EXISTS isa";
+
+update_version $NEW_VERSION
+
+success

utils/make_db

@@ -132,21 +132,23 @@ name in your .config file "
     sed "s/db_5_0/$DATABASE_NAME/g" "$INSTALL_SQL_FILE" | $MYSQL_CMD $(get_mysql_args_nodb)
 }
 
-function sanity_check_dump() {
-    SQL_FILE="$1"
-    DETECTED_VERSION="$(grep -A 5 -i 'CREATE.*FUNCTION.*CaosDBVersion' "$SQL_FILE" | grep -o -E -e "v[0-9][^'\"]*")"
-    echo "[INFO] Detected schema version of the dump: $DETECTED_VERSION"
-    AFFECTED_VERSION_REGEX='v([3-9]|2[0-9]|2\.1|2\.0\.[23][0-9]|2\.0\.1[6-9])'
-    passwords_table_should_be_gone=$(echo "$DETECTED_VERSION" | grep -E -e "$AFFECTED_VERSION_REGEX" -c || true)
-    if [ "$passwords_table_should_be_gone" -gt "0" ] && grep 'CREATE TABLE `passwords`' -c "$SQL_FILE" > /dev/null ; then
-        echo "[ERROR] Your dump is broken. The 'passwords' table should be gone: $SQL_FILE"
+function sanity_check() {
+    trap "$(shopt -po errexit)" RETURN
+    set +e
+    echo -n "running sanity checks ..."
+    msg="$($MYSQL_CMD $(get_mysql_args) < "utils/sanity_check.sql" 2>/dev/null)"
+    code="$?"
+    if [ "$code" -eq "0" ] ; then
+        echo " [OK]"
+    else
+        echo " [FAILED]"
+        echo "$msg"
         exit 1
     fi
 }
 
 # Inserts the dump (arg 1) into the database
 function restore_db() {
-    sanity_check_dump $1
     SQL_FILE="$1"
     $MYSQL_CMD $(get_mysql_args) < "$SQL_FILE"
     cat <<EOF
@@ -269,6 +271,6 @@ case $1 in
     "test-connection") test-connection ;;
     "install_db") install_db ;;
     "restore_db") restore_db $2 ;;
-    "sanity_check_dump") sanity_check_dump $2 ;;
+    "sanity_check") sanity_check ;;
     *) echo "Unknown action: $1"; exit 32
 esac

utils/sanity_check.sql

+
+
+DROP PROCEDURE IF EXISTS sanity_check;
+delimiter //
+CREATE PROCEDURE sanity_check()
+sanityCheckBody: BEGIN
+    CREATE TEMPORARY TABLE expected_tables (table_name VARCHAR(255));
+    INSERT INTO expected_tables (table_name) VALUES
+("archive_collection_type"),
+("archive_data_type"),
+("archive_date_data"),
+("archive_datetime_data"),
+("archive_desc_overrides"),
+("archive_double_data"),
+("archive_entities"),
+("archive_enum_data"),
+("archive_files"),
+("archive_integer_data"),
+("archive_isa"),
+("archive_name_data"),
+("archive_name_overrides"),
+("archive_null_data"),
+("archive_query_template_def"),
+("archive_reference_data"),
+("archive_text_data"),
+("collection_type"),
+("data_type"),
+("date_data"),
+("datetime_data"),
+("desc_overrides"),
+("double_data"),
+("entities"),
+("entity_ids"),
+("entity_acl"),
+("entity_version"),
+("enum_data"),
+("feature_config"),
+("files"),
+("integer_data"),
+("isa_cache"),
+("name_data"),
+("name_overrides"),
+("null_data"),
+("passwd"),
+("permissions"),
+("query_template_def"),
+("reference_data"),
+("roles"),
+("stats"),
+("text_data"),
+("transaction_log"),
+("transactions"),
+("units_lin_con"),
+("user_info"),
+("user_roles")
+;
+
+SELECT COUNT(WRONG) INTO @count_wrong FROM ( SELECT l.table_name AS MATCHED, r.table_name AS WRONG FROM expected_tables AS l RIGHT OUTER JOIN information_schema.tables AS r ON (r.table_name COLLATE utf8_unicode_ci = l.table_name) WHERE r.table_schema = database()) AS temp WHERE temp.MATCHED IS NULL;
+
+SELECT COUNT(MISSING) INTO @count_missing FROM ( SELECT l.table_name AS MISSING, r.table_name AS MATCHED FROM expected_tables AS l LEFT OUTER JOIN information_schema.tables AS r ON (r.table_name COLLATE utf8_unicode_ci = l.table_name) WHERE r.table_schema = database() OR r.table_schema IS NULL) AS temp WHERE temp.MATCHED IS NULL;
+
+IF @count_missing = 0 AND @count_wrong = 0 THEN
+    LEAVE sanityCheckBody;
+END IF;
+
+SELECT "--------------";
+SELECT @count_missing AS "Number of missing tables";
+
+SELECT MISSING AS "Missing tables" FROM ( SELECT l.table_name AS MISSING, r.table_name AS MATCHED FROM expected_tables AS l LEFT OUTER JOIN information_schema.tables AS r ON (r.table_name COLLATE utf8_unicode_ci = l.table_name) WHERE r.table_schema = database() OR r.table_schema IS NULL) AS temp WHERE temp.MATCHED IS NULL;
+
+SELECT "--------------";
+SELECT @count_wrong AS "Number of tables which should not exist";
+
+SELECT WRONG AS "Tables which should not exist" FROM ( SELECT l.table_name AS MATCHED, r.table_name AS WRONG FROM expected_tables AS l RIGHT OUTER JOIN information_schema.tables AS r ON (r.table_name COLLATE utf8_unicode_ci = l.table_name) WHERE r.table_schema = database()) AS temp WHERE temp.MATCHED IS NULL;
+
+SELECT "--------------";
+SELECT "ERROR" from sanity_check_failed;
+
+END;
+//
+delimiter ;
+
+CALL sanity_check();
+
+DROP PROCEDURE sanity_check;