Skip to content
Snippets Groups Projects
Commit bac0a91e authored by Henrik tom Wörden's avatar Henrik tom Wörden
Browse files

Merge branch 'f-clean-scripts' into 's-clean-up-stage'

Disentagle mysql scripts

See merge request caosdb/caosdb-mysqlbackend!6
parents 079cf7b8 f5c84576
Branches
Tags
No related merge requests found
......@@ -51,8 +51,8 @@ unittests:
- echo 'DATABASE_USER_HOST_LIST="%,"' >> .config
- echo "MYSQL_USER_PASSWORD=$MYSQL_ROOT_PASSWORD" >> .config
- echo "MYSQL_HOST=mariadb" >> .config
- ./make_db test-connection
- ./make_db test --fresh
- ./utils/make_db test-connection
- ./utils/make_db test --fresh
# Build a docker image in which tests for this repository can run
build-testenv:
......
......@@ -23,13 +23,13 @@
# # Commands
# The MySQL client program.
MYSQL_CMD=$(command -v mysql)
MYSQL_CMD=$(command -v mysql || echo "mysql not found!" >&2)
# The mysqladmin program which comes with the MySQL client.
MYSQLADMIN_CMD=$(command -v mysqladmin)
MYSQLADMIN_CMD=$(command -v mysqladmin || echo "mysqladmin not found!" >&2)
# The mysqldump program which comes with the MySQL client.
MYSQLDUMP_CMD=$(command -v mysqldump)
MYSQLDUMP_CMD=$(command -v mysqldump || echo "mysqldump not found!" >&2)
# The mysql_config_editor program which is used to store the credentials.
MYSQL_CONFIG_EDITOR_CMD=$(command -v mysql_config_editor)
MYSQL_CONFIG_EDITOR_CMD=$(command -v mysql_config_editor || true)
# # MySQL Connection
# The host of the MySQL server.
......@@ -55,5 +55,3 @@ DATABASE_USER_PW=random1234
# wildcard for all hosts, so `192.168.0.%` permits logins from the local
# network.
DATABASE_USER_HOST_LIST=localhost,
......@@ -22,19 +22,11 @@
#
SHELL=/bin/bash
include .config
INSTALL_SQL_FILE=db_2_0.sql
ifdef LOGIN_PATH
M_MYSQL_CONNECTION=--login-path=$(M_LOGIN_PATH)
else
MYSQL_CONNECTION=--host="$(MYSQL_HOST)" --port="$(MYSQL_PORT)" \
--user="$(MYSQL_USER)" --password="$(MYSQL_USER_PASSWORD)"
endif
.PHONY: test-connection
test-connection:
./make_db test-connection
./utils/make_db test-connection
.PHONY: upgrade
upgrade:
......@@ -45,17 +37,17 @@ install: _install _grant upgrade
.PHONY: _install
_install:
./make_db install_db
./utils/make_db install_db
.PHONY: _grant
_grant:
./make_db grant
./utils/make_db grant
# Drop the user and a given database
.PHONY: drop-%
drop-%:
./make_db drop $(patsubst drop-%,%,$@)
./utils/make_db drop $(patsubst drop-%,%,$@)
.PHONY: test
test:
./make_db test --fresh
./utils/make_db test --fresh
......@@ -6,6 +6,7 @@
# Copyright (C) 2018 Research Group Biomedical Physics,
# Max-Planck-Institute for Dynamics and Self-Organization Göttingen
# Copyright (C) 2020 Timm Fitschen <t.fitschen@indiscale.com>
# Copyright (C) 2020 Daniel Hornung <d.hornung@indiscale.com>
# Copyright (C) 2020 IndiScale <info@indiscale.com>
#
# This program is free software: you can redistribute it and/or modify
......@@ -24,16 +25,26 @@
# ** end header
#
#apply all available patches.
###############################################################################
# Apply all available patches #
###############################################################################
set -e
if [ -z "$UTILSPATH" ]; then
UTILSPATH=$(realpath "$(dirname $0)/../utils")
export UTILSPATH
fi
source $UTILSPATH/load_settings.sh
source $UTILSPATH/helpers.sh
PATCHES="./patch*/patch.sh"
export UTILSPATH="./utils"
for p in $PATCHES
do
$p "$@" --patch=$p
done
source ./utils/patch_header.sh
cd ../
./update_sql_procedures.sh
$UTILSPATH/update_sql_procedures.sh
......@@ -35,7 +35,7 @@ fi
check_version $OLD_VERSION
function addIdx {
$CMD_MYSQL $MYSQL_CONNECTION -D $DATABASE -e "ALTER TABLE $1 ADD COLUMN pidx TINYINT UNSIGNED NOT NULL DEFAULT 0;"
$MYSQL_CMD $(get_db_args) -e "ALTER TABLE $1 ADD COLUMN pidx TINYINT UNSIGNED NOT NULL DEFAULT 0;"
}
addIdx "reference_data"
......
......@@ -5,6 +5,8 @@
#
# Copyright (C) 2018 Research Group Biomedical Physics,
# Max-Planck-Institute for Dynamics and Self-Organization Göttingen
# Copyright (C) 2020 Daniel Hornung <d.hornung@indiscale.com>
# Copyright (C) 2020 IndiScale <info@indiscale.com>
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as
......@@ -24,6 +26,10 @@
# Dump a database with all procedures, permissions, structure and data
if [ -z "$UTILSPATH" ]; then
UTILSPATH="$(realpath $(dirname $0))"
export UTILSPATH
fi
# The directory which the dump is to be stored to. Do not change it here. Use
# the --backupdir=./my/dir/ option or an environment variable instead.
BACKUPDIR="${BACKUPDIR:-../backup}"
......@@ -58,12 +64,11 @@ backup:
EOF
)
# Config and command line parameter loading ###################################
. ../.config
# Load settings from .config and defaults #####################################
. $UTILSPATH/load_settings.sh
# load useful stuff - parses the commandline parameters and so on...
. ../patches/utils/patch_header.sh
# load useful functions #######################################################
. $UTILSPATH/helpers.sh
function backup() {
NARG_NAME="$1"
......@@ -75,7 +80,7 @@ function backup() {
failure "dumpfile already exists."
fi
echo "Dumping database $NARG_NAME to $NARG_FILE ... "
$CMD_MYSQL_DUMP $MYSQL_CONNECTION_NO_DB $* --opt --default-character-set=utf8 --routines \
$MYSQLDUMP_CMD $(get_mysql_args_nodb) $* --opt --default-character-set=utf8 --routines \
"$NARG_NAME" > "$NARG_FILE"
if [[ "$YAML" == "true" ]] ; then
......
# This file is a part of the CaosDB Project.
#
# Copyright (C) 2018 Research Group Biomedical Physics,
# Max-Planck-Institute for Dynamics and Self-Organization Göttingen
# Copyright (C) 2019, 2020 Daniel Hornung (d.hornung@indiscale.com)
# Copyright (C) 2020 Henrik tom Wörden
# Copyright (C) 2020 IndiScale GmbH <info@indiscale.com>
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as
# published by the Free Software Foundation, either version 3 of the
# License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <https://www.gnu.org/licenses/>.
###############################################################################
# Helper function definitions #
###############################################################################
# When sourcing this file, the following variables must be set:
#
# MYSQL_CMD : source $UTILSPATH/load_settings.sh
function mysql_execute {
#TODO is it meaningful that here always the database user is used???
set -e
$MYSQL_CMD $(get_db_args) -e "$1"
ret=${PIPESTATUS[0]}
if [ "$ret" -ne 0 ]; then
failure "MYSQL ERROR"
fi
}
function get_mysql_args {
echo "$(get_mysql_args_nodb) --database=$DATABASE_NAME"
}
function get_db_args {
echo "$(get_db_args_nodb) --database=$DATABASE_NAME"
}
function get_db_args_nodb {
if [ "$DATABASE_USER" ]; then
mysql_con_arguments="--user=$DATABASE_USER"
fi
if [ "$DATABASE_USER_PW" ]; then
mysql_con_arguments="$mysql_con_arguments --password=$DATABASE_USER_PW"
fi
if [[ "$MYSQL_HOST" && ( "$MYSQL_HOST" != "localhost" ) ]]; then
mysql_con_arguments="$mysql_con_arguments --host=$MYSQL_HOST"
if [ "$MYSQL_PORT" ]; then
mysql_con_arguments="$mysql_con_arguments --port=$MYSQL_PORT"
fi
fi
echo $mysql_con_arguments
}
function get_mysql_args_nodb {
if [ "$LOGIN_PATH" ]; then
mysql_con_arguments="--login-path=$LOGIN_PATH"
else
if [ "$MYSQL_USER" ]; then
mysql_con_arguments="--user=$MYSQL_USER"
fi
if [ "$MYSQL_USER_PASSWORD" ]; then
mysql_con_arguments="$mysql_con_arguments --password=$MYSQL_USER_PASSWORD"
fi
if [[ "$MYSQL_HOST" && ( "$MYSQL_HOST" != "localhost" ) ]]; then
mysql_con_arguments="$mysql_con_arguments --host=$MYSQL_HOST"
if [ "$MYSQL_PORT" ]; then
mysql_con_arguments="$mysql_con_arguments --port=$MYSQL_PORT"
fi
fi
fi
echo $mysql_con_arguments
}
function success {
echo "[OK]"
exit 0
}
function failure {
echo "[FAILED] $*"
exit 1
}
function uptodate {
echo "[UPTODATE]"
exit 0
}
# This file is a part of the CaosDB Project.
#
# Copyright (C) 2018 Research Group Biomedical Physics,
# Max-Planck-Institute for Dynamics and Self-Organization Göttingen
# Copyright (C) 2019, 2020 Daniel Hornung (d.hornung@indiscale.com)
# Copyright (C) 2020 Henrik tom Wörden
# Copyright (C) 2020 IndiScale GmbH <info@indiscale.com>
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as
# published by the Free Software Foundation, either version 3 of the
# License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <https://www.gnu.org/licenses/>.
# Load setting from different files, in this order:
# - config.defaults (in the current directory)
# - .config (in the current directory)
# - The file given in ENV_FILE.
if [ -z "$MAINPATH" ]; then
if [ -z "$UTILSPATH" ] ; then
echo "At least one of MAINPATH and UTILSPATH must be defined in load_settings.sh" >&2
exit 1
fi
MAINPATH="$(dirname $UTILSPATH)"
export MAINPATH
fi
# Make a backup of the important environment variables.
my_env=$(export -p | grep -E '(MYSQL|DATABASE)')
source "$MAINPATH/config.defaults"
if [[ -r "$MAINPATH/.config" ]] ; then
source "$MAINPATH/.config"
fi
if [[ -n "$ENV_FILE" ]] ; then
source "$ENV_FILE"
fi
if [[ -z "$MYSQL_CMD" ]] ; then
echo "'mysql' executable not defined!"
exit 1
fi
# Restore backup.
eval "$my_env"
export MYSQL_CMD
export MYSQLDUMP_CMD
export MYSQLADMIN_CMD
export MYSQL_CONFIG_EDITOR_CMD
export MYSQL_HOST
export MYSQL_PORT
export MYSQL_USER
export DATABASE_NAME
export DATABASE_USER
export DATABASE_USER_PW
export DATABASE_USER_HOST_LIST
......@@ -3,8 +3,9 @@
# ** header v3.0
# This file is a part of the CaosDB Project.
#
# Copyright (C) 2019 Daniel Hornung, Göttingen
# Copyright (C) 2019, 2020 Daniel Hornung <d.hornung@indiscale.com>
# Copyright (C) 2020 Timm Fitschen <t.fitschen@indiscale.com>
# Copyright (C) 2020 Henrik tom Wörden <h.tomwoerden@indiscale.com>
# Copyright (C) 2020 IndiScale <info@indiscale.com>
#
# This program is free software: you can redistribute it and/or modify
......@@ -22,13 +23,17 @@
#
# ** end header
# To fail fast, but beware https://mywiki.wooledge.org/BashFAQ/105
set -e
function fail() {
echo "Some error occured, exiting."
exit 1
}
INSTALL_SQL_FILE="db_2_0.sql"
if [ -z "$UTILSPATH" ]; then
UTILSPATH="$(realpath $(dirname $0))"
export UTILSPATH
fi
source $UTILSPATH/load_settings.sh
source $UTILSPATH/helpers.sh
UNITTEST_DATABASE=${UNITTEST_DATABASE-_caosdb_schema_unit_tests}
......@@ -39,7 +44,7 @@ function runtests() {
_install_unit_test_database $@
_execute_tests || ( echo "[FAILURE]" && exit 1 )
_execute_tests || ( failure )
echo "[PASS]"
}
......@@ -51,7 +56,7 @@ function _execute_tests () {
for tfile in $TESTS ; do
echo "Running $tfile"
echo "----- $tfile -----" >> .TEST_RESULTS
cat $tfile | $SQL --disable-pager --batch --raw --skip-column-names --unbuffered >> .TEST_RESULTS
cat $tfile | $MYSQL_CMD $(get_db_args_nodb) --disable-pager --batch --raw --skip-column-names --unbuffered >> .TEST_RESULTS
done;
popd
......@@ -72,9 +77,9 @@ function _install_unit_test_database () {
fi
fi
sed "s/db_2_0/$DATABASE_NAME/g" "$INSTALL_SQL_FILE" | $SQL
sed "s/db_2_0/$DATABASE_NAME/g" "$INSTALL_SQL_FILE" | $MYSQL_CMD $(get_mysql_args_nodb)
# crate test user
# create test user
grant
# update to latest
......@@ -92,15 +97,15 @@ function _setup_mytap() {
echo MyTAB framework is already installed [OK]
return 0
fi
echo -n "Installing MyTAB framework ... "
echo -n "Installing MyTAP framework ... "
pushd libs > /dev/null
unzip -u mytap*.zip > /dev/null
pushd mytap*/ > /dev/null
$SQL < mytap.sql > /dev/null || exit 1
$MYSQL_CMD $(get_mysql_args_nodb) < mytap.sql > /dev/null || exit 1
popd > /dev/null
rm -r mytap*/
popd > /dev/null
echo [DONE]
echo "[DONE]"
}
function install_db() {
......@@ -112,17 +117,17 @@ Call 'make drop-$DATABASE_NAME' to delete that database or reconfigure with
exit 0
fi
sed "s/db_2_0/$DATABASE_NAME/g" "$INSTALL_SQL_FILE" | $SQL
sed "s/db_2_0/$DATABASE_NAME/g" "$INSTALL_SQL_FILE" | $MYSQL_CMD $(get_mysql_args_nodb)
}
# Inserts the dump (arg 1) into the database
function restore_db() {
SQL_FILE="$1"
$SQL -D "$DATABASE_NAME" < "$SQL_FILE"
$MYSQL_CMD $(get_mysql_args) < "$SQL_FILE"
}
function test-connection() {
$SQL -e "select 0;"
$MYSQL_CMD $(get_mysql_args_nodb) -e "select 0;"
}
# Creates a user and grants it sufficient rights.
......@@ -146,7 +151,7 @@ function grant() {
if [[ "$1" == "--strict" ]] ; then
for host in ${DATABASE_USER_HOST_LIST//,/ } ; do
CMD="SELECT COUNT(*) FROM mysql.user WHERE user='${DATABASE_USER}' AND host='${host}';"
[[ $($SQL -s -N -e "$CMD") == 0 ]] || {
[[ $($MYSQL_CMD $(get_mysql_args_nodb) -s -N -e "$CMD") == 0 ]] || {
echo "The user '${DATABASE_USER}@${host}' is already in the database."
echo "Please use another user or delete it, e.g. with"
echo "'mysql -u ${MYSQL_USER} -p -e \"DROP USER ${DATABASE_USER}@${host};\"'"
......@@ -157,7 +162,7 @@ function grant() {
for host in ${DATABASE_USER_HOST_LIST//,/ } ; do
echo "Granting admin privileges to '$DATABASE_USER'@'$host'"
$SQL <<EOF
$MYSQL_CMD $(get_mysql_args_nodb) <<EOF
CREATE USER IF NOT EXISTS
'$DATABASE_USER'@'$host' identified by '$DATABASE_USER_PW';
GRANT USAGE ON *.* TO '$DATABASE_USER'@'$host';
......@@ -172,31 +177,19 @@ EOF
function drop() {
DROPDB="$1"
for host in ${DATABASE_USER_HOST_LIST//,/ } ; do
$SQL -e "DROP USER '${DATABASE_USER}'@'${host}';" || true
$MYSQL_CMD $(get_mysql_args_nodb) -e "DROP USER '${DATABASE_USER}'@'${host}';" || true
done
"$MYSQLADMIN_CMD" $MYSQL_CONNECTION -f drop "$DROPDB"
"$MYSQLADMIN_CMD" $(get_mysql_args_nodb) -f drop "$DROPDB"
}
# Returns 0 or non-zero, depending on whether the database exists already.
# Optional parameter: [DATABASE_NAME], else $DATABASE_NAME is used.
function _db_exists() {
$SQL -D "${1-${DATABASE_NAME}}" -e "show tables;" > /dev/null 2>&1 \
$MYSQL_CMD $(get_mysql_args_nodb) -D "${1-${DATABASE_NAME}}" -e "show tables;" > /dev/null 2>&1 \
&& return 0 || return 1
}
source .config || true
INSTALL_SQL_FILE="db_2_0.sql"
if [ -z $LOGIN_PATH ] ; then
MYSQL_CONNECTION="--host=${MYSQL_HOST} --port=${MYSQL_PORT}
--user=${MYSQL_USER}
--password=${MYSQL_USER_PASSWORD}"
else
MYSQL_CONNECTION="--login-path=$LOGIN_PATH"
fi
SQL="$MYSQL_CMD $MYSQL_CONNECTION"
case $1 in
"drop") drop $2 ;;
......
......@@ -4,7 +4,9 @@
#
# Copyright (C) 2018 Research Group Biomedical Physics,
# Max-Planck-Institute for Dynamics and Self-Organization Göttingen
# Copyright 2019 Daniel Hornung
# Copyright (C) 2019, 2020 Daniel Hornung <d.hornung@indiscale.com>
# Copyright (C) 2020 Henrik tom Wörden <h.tomwoerden@indiscale.com>
# Copyright (C) 2020 IndiScale <info@indiscale.com>
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as
......@@ -21,9 +23,17 @@
#
# ** end header
#
#header for patch scripts
CMD_MYSQL=mysql
CMD_MYSQL_DUMP=mysqldump
###############################################################################
# header for patch scripts #
###############################################################################
set -e
# load useful functions #######################################################
source $UTILSPATH/load_settings.sh
source $UTILSPATH/helpers.sh
USAGE="$1 [ --env=ENV_FILE ] [ --patch=PATCH ] [ --backupdir=BACKUPDIR ]\n\n"
CMD_OPTIONS=$(cat <<EOF
options:
......@@ -54,7 +64,6 @@ configuration make file. Notable examples are:
EOF
)
set -e
function _print_help() {
echo -e "$USAGE"
......@@ -101,65 +110,14 @@ while test $# -gt 0; do
esac
done
source "$ENV_FILE" || true
if [[ -z "$DATABASE_NAME" && -z "$MYSQL_CONNECTION" ]]
then
_print_help "Please specify the database."
exit 1
fi
if [ "$LOGIN_PATH" ]; then
MYSQL_CONNECTION="--login-path=$LOGIN_PATH"
MYSQL_CONNECTION_NO_DB="$MYSQL_CONNECTION"
MYSQL_CONNECTION="$MYSQL_CONNECTION --database=$DATABASE_NAME"
elif [[ -z "$MYSQL_CONNECTION" ]]; then
MYSQL_CONNECTION=""
if [ "$DATABASE_USER" ]
then
MYSQL_CONNECTION="--user=$DATABASE_USER"
fi
if [ "$DATABASE_USER_PW" ]
then
MYSQL_CONNECTION="$MYSQL_CONNECTION --password=$DATABASE_USER_PW"
fi
if [[ "$MYSQL_HOST" && ( "$MYSQL_HOST" != "localhost" ) ]]; then
MYSQL_CONNECTION="$MYSQL_CONNECTION --host=$MYSQL_HOST"
if [ "$MYSQL_PORT" ]; then
MYSQL_CONNECTION="$MYSQL_CONNECTION --port=$MYSQL_PORT"
fi
fi
# This option should come last, so we also have one version without the database
MYSQL_CONNECTION_NO_DB="$MYSQL_CONNECTION"
MYSQL_CONNECTION="$MYSQL_CONNECTION --database=$DATABASE_NAME"
fi
export MYSQL_CONNECTION
export MYSQL_CONNECTION_NO_DB
export DATABASE_NAME
if [ -n "$PATCH" ]; then
echo -ne "applying patch $PATCH to $DATABASE_NAME ... "
fi
function success {
echo "[OK]"
exit 0
}
function failure {
echo "[FAILED] $*"
exit 1
}
function uptodate {
echo "[UPTODATE]"
exit 0
}
# @param $1: db version string, e.g. v2.0.0
# @return: 0 on success, 1 on failure
function check_version {
local version=$($CMD_MYSQL $MYSQL_CONNECTION -B -e "Select CaosDBVersion();")
local version=$($MYSQL_CMD $(get_db_args) -B -e "Select CaosDBVersion();")
if [[ "$(echo $version | sed 's/^CaosDBVersion()\s//')" = "$1" ]]; then
return 0
fi
......@@ -172,22 +130,12 @@ function update_version {
}
function dump_table {
$CMD_MYSQL_DUMP $MYSQL_CONNECTION_NO_DB $DATABASE_NAME $1 \
$MYSQLDUMP_CMD $(get_db_args_nodb) $DATABASE_NAME $1 \
> ${DATABASE_NAME}.${1}.${OLD_VERSION}.dump.sql
}
function mysql_execute {
set +e
$CMD_MYSQL $MYSQL_CONNECTION -e "$1"
ret=${PIPESTATUS[0]}
if [ "$ret" -ne 0 ]; then
failure "MYSQL ERROR"
fi
set -e
}
function redo_table {
$CMD_MYSQL $MYSQL_CONNECTION < ${DATABASE_NAME}.${1}.${OLD_VERSION}.dump.sql
$MYSQL_CMD $(get_db_args) < ${DATABASE_NAME}.${1}.${OLD_VERSION}.dump.sql
}
......@@ -5,6 +5,9 @@
#
# Copyright (C) 2018 Research Group Biomedical Physics,
# Max-Planck-Institute for Dynamics and Self-Organization Göttingen
# Copyright (C) 2020 Daniel Hornung <d.hornung@indiscale.com>
# Copyright (C) 2020 Henrik tom Wörden <h.tomwoerden@indiscale.com>
# Copyright (C) 2020 IndiScale <info@indiscale.com>
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as
......@@ -24,16 +27,21 @@
# Updates all SQL procedures
source patches/utils/patch_header.sh
set -e
if [ -z "$UTILSPATH" ]; then
UTILSPATH=$(realpath "$(dirname $0)")
export UTILSPATH
fi
source $UTILSPATH/load_settings.sh
source $UTILSPATH/helpers.sh
echo -n "updating rules ... "
set -e
mysql_execute "$(sed s/db_2_0/$DATABASE_NAME/g rules.sql)"
echo "[OK]"
echo "OK"
echo -n "updating procedures ... "
mysql_execute "$(sed s/db_2_0/$DATABASE_NAME/g procedures/*.sql procedures/query/*.sql)"
success
echo "OK"
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment