diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 076a97f06ab2eb06f01f48b855f4367c9f07408f..94567fb2a0c8c78dabd895ba2277fad69106e65a 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -51,8 +51,8 @@ unittests: - echo 'DATABASE_USER_HOST_LIST="%,"' >> .config - echo "MYSQL_USER_PASSWORD=$MYSQL_ROOT_PASSWORD" >> .config - echo "MYSQL_HOST=mariadb" >> .config - - ./make_db test-connection - - ./make_db test --fresh + - ./utils/make_db test-connection + - ./utils/make_db test --fresh # Build a docker image in which tests for this repository can run build-testenv: diff --git a/config.defaults b/config.defaults index c9101986c5d0099b0031f68665e05e54d0472ac2..bd669cb2cc724a74e3dd81fc708a96dfdd6a4ebe 100644 --- a/config.defaults +++ b/config.defaults @@ -55,5 +55,3 @@ DATABASE_USER_PW=random1234 # wildcard for all hosts, so `192.168.0.%` permits logins from the local # network. DATABASE_USER_HOST_LIST=localhost, - - diff --git a/makefile b/makefile index f08e739d464c274f6c23138dc73a8383895d90ca..0c9ddabafadaacf14a122ea1c0df9463558219ac 100644 --- a/makefile +++ b/makefile @@ -22,19 +22,11 @@ # SHELL=/bin/bash -include .config - INSTALL_SQL_FILE=db_2_0.sql -ifdef LOGIN_PATH - M_MYSQL_CONNECTION=--login-path=$(M_LOGIN_PATH) -else - MYSQL_CONNECTION=--host="$(MYSQL_HOST)" --port="$(MYSQL_PORT)" \ - --user="$(MYSQL_USER)" --password="$(MYSQL_USER_PASSWORD)" -endif .PHONY: test-connection test-connection: - ./make_db test-connection + ./utils/make_db test-connection .PHONY: upgrade upgrade: @@ -45,17 +37,17 @@ install: _install _grant upgrade .PHONY: _install _install: - ./make_db install_db + ./utils/make_db install_db .PHONY: _grant _grant: - ./make_db grant + ./utils/make_db grant # Drop the user and a given database .PHONY: drop-% drop-%: - ./make_db drop $(patsubst drop-%,%,$@) + ./utils/make_db drop $(patsubst drop-%,%,$@) .PHONY: test test: - ./make_db test --fresh + ./utils/make_db test --fresh diff --git a/patches/applyPatches.sh b/patches/applyPatches.sh index bec377c249e18ce0272fb4ce0fed3fde522b65b6..56cbac7e05a1528ee041540ed84cc7faf7b3a5d6 100755 --- a/patches/applyPatches.sh +++ b/patches/applyPatches.sh @@ -27,13 +27,11 @@ #apply all available patches. set -e PATCHES="./patch*/patch.sh" -export UTILSPATH="./utils" for p in $PATCHES do $p "$@" --patch=$p done -source ./utils/patch_header.sh cd ../ -./update_sql_procedures.sh +$UTILSPATH/update_sql_procedures.sh diff --git a/patches/patch20151022-0/patch.sh b/patches/patch20151022-0/patch.sh index 0f529792f8078c51681972e8aec6852d7621fb2d..f361fb4dac4470e7c2060556a69e01911f94cf60 100755 --- a/patches/patch20151022-0/patch.sh +++ b/patches/patch20151022-0/patch.sh @@ -35,7 +35,7 @@ fi check_version $OLD_VERSION function addIdx { - $CMD_MYSQL $MYSQL_CONNECTION -D $DATABASE -e "ALTER TABLE $1 ADD COLUMN pidx TINYINT UNSIGNED NOT NULL DEFAULT 0;" + $MYSQL_CMD $(get_db_args) -e "ALTER TABLE $1 ADD COLUMN pidx TINYINT UNSIGNED NOT NULL DEFAULT 0;" } addIdx "reference_data" diff --git a/utils/backup.sh b/utils/backup.sh index ff671437f42832c8c66ba250f777c4aa8d3f0c88..752c4e41e0ff1d254cd6f19c8f7a4738e8dac534 100755 --- a/utils/backup.sh +++ b/utils/backup.sh @@ -24,6 +24,11 @@ # Dump a database with all procedures, permissions, structure and data + +if [ -z "$UTILSPATH" ]; then + UTILSPATH="$(realpath $(dirname $0))" + export UTILSPATH +fi # The directory which the dump is to be stored to. Do not change it here. Use # the --backupdir=./my/dir/ option or an environment variable instead. BACKUPDIR="${BACKUPDIR:-../backup}" @@ -58,12 +63,11 @@ backup: EOF ) -# Config and command line parameter loading ################################### - -. ../.config +# Load settings from .config and defaults ##################################### +. $UTILSPATH/load_settings.sh -# load useful stuff - parses the commandline parameters and so on... -. ../patches/utils/patch_header.sh +# load useful functions ####################################################### +. $UTILSPATH/helpers.sh function backup() { NARG_NAME="$1" @@ -75,7 +79,7 @@ function backup() { failure "dumpfile already exists." fi echo "Dumping database $NARG_NAME to $NARG_FILE ... " - $CMD_MYSQL_DUMP $MYSQL_CONNECTION_NO_DB $* --opt --default-character-set=utf8 --routines \ + $MYSQLDUMP_CMD $(get_mysql_args_nodb) $* --opt --default-character-set=utf8 --routines \ "$NARG_NAME" > "$NARG_FILE" if [[ "$YAML" == "true" ]] ; then diff --git a/utils/helpers.sh b/utils/helpers.sh new file mode 100644 index 0000000000000000000000000000000000000000..b1f6d19c2270f8b3207a88f36151f6633318cfc5 --- /dev/null +++ b/utils/helpers.sh @@ -0,0 +1,90 @@ +# This file is a part of the CaosDB Project. +# +# Copyright (C) 2018 Research Group Biomedical Physics, +# Max-Planck-Institute for Dynamics and Self-Organization Göttingen +# Copyright 2019 Daniel Hornung +# Copyright 2020 Henrik tom Wörden, IndiScale GmbH +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see <https://www.gnu.org/licenses/>. + + +function mysql_execute { + #TODO is it meaningful that here always the database user is used??? + set +e + $MYSQL_CMD $(get_db_args) -e "$1" + ret=${PIPESTATUS[0]} + if [ "$ret" -ne 0 ]; then + failure "MYSQL ERROR" + fi + set -e +} + +function get_mysql_args { + echo "$(get_mysql_args_nodb) --database=$DATABASE_NAME" +} + +function get_db_args { + echo "$(get_db_args_nodb) --database=$DATABASE_NAME" +} + +function get_db_args_nodb { + if [ "$DATABASE_USER" ]; then + mysql_con_arguments="--user=$DATABASE_USER" + fi + if [ "$DATABASE_USER_PW" ]; then + mysql_con_arguments="$mysql_con_arguments --password=$DATABASE_USER_PW" + fi + if [[ "$MYSQL_HOST" && ( "$MYSQL_HOST" != "localhost" ) ]]; then + mysql_con_arguments="$mysql_con_arguments --host=$MYSQL_HOST" + if [ "$MYSQL_PORT" ]; then + mysql_con_arguments="$mysql_con_arguments --port=$MYSQL_PORT" + fi + fi + echo $mysql_con_arguments +} + +function get_mysql_args_nodb { + if [ "$LOGIN_PATH" ]; then + mysql_con_arguments="--login-path=$LOGIN_PATH" + else + if [ "$MYSQL_USER" ]; then + mysql_con_arguments="--user=$MYSQL_USER" + fi + if [ "$MYSQL_USER_PASSWORD" ]; then + mysql_con_arguments="$mysql_con_arguments --password=$MYSQL_USER_PASSWORD" + fi + if [[ "$MYSQL_HOST" && ( "$MYSQL_HOST" != "localhost" ) ]]; then + mysql_con_arguments="$mysql_con_arguments --host=$MYSQL_HOST" + if [ "$MYSQL_PORT" ]; then + mysql_con_arguments="$mysql_con_arguments --port=$MYSQL_PORT" + fi + fi + fi + echo $mysql_con_arguments +} + +function success { + echo "[OK]" + exit 0 +} + +function failure { + echo "[FAILED] $*" + exit 1 +} + +function uptodate { + echo "[UPTODATE]" + exit 0 +} diff --git a/utils/load_settings.sh b/utils/load_settings.sh new file mode 100644 index 0000000000000000000000000000000000000000..a001b98b4e9188ac89f8200069dad3a34e2ea354 --- /dev/null +++ b/utils/load_settings.sh @@ -0,0 +1,35 @@ +# This file is a part of the CaosDB Project. +# +# Copyright (C) 2018 Research Group Biomedical Physics, +# Max-Planck-Institute for Dynamics and Self-Organization Göttingen +# Copyright 2019 Daniel Hornung +# Copyright 2020 Henrik tom Wörden, IndiScale GmbH +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see <https://www.gnu.org/licenses/>. + +[[ -n "config.defaults" ]] && source "config.defaults" +[[ -n ".config" ]] && source ".config" +[[ -n "$ENV_FILE" ]] && source "$ENV_FILE" + +export MYSQL_CMD +export MYSQLDUMP_CMD +export MYSQLADMIN_CMD +export MYSQL_CONFIG_EDITOR_CMD +export MYSQL_HOST +export MYSQL_PORT +export MYSQL_USER +export DATABASE_NAME +export DATABASE_USER +export DATABASE_USER_PW +export DATABASE_USER_HOST_LIST diff --git a/make_db b/utils/make_db similarity index 83% rename from make_db rename to utils/make_db index 07ca5e368142f143a3624b51e91da7532d2c8346..bd175b6565e5eda26ff68f591ab2b85855b849b2 100755 --- a/make_db +++ b/utils/make_db @@ -22,7 +22,16 @@ # # ** end header -# To fail fast, but beware https://mywiki.wooledge.org/BashFAQ/105 +INSTALL_SQL_FILE="db_2_0.sql" + +if [ -z "$UTILSPATH" ]; then + UTILSPATH="$(realpath $(dirname $0))" + export UTILSPATH +fi + +source $UTILSPATH/load_settings.sh +source $UTILSPATH/helpers.sh + set -e function fail() { @@ -51,7 +60,7 @@ function _execute_tests () { for tfile in $TESTS ; do echo "Running $tfile" echo "----- $tfile -----" >> .TEST_RESULTS - cat $tfile | $SQL --disable-pager --batch --raw --skip-column-names --unbuffered >> .TEST_RESULTS + cat $tfile | $MYSQL_CMD $(get_db_args_nodb) --disable-pager --batch --raw --skip-column-names --unbuffered >> .TEST_RESULTS done; popd @@ -72,9 +81,9 @@ function _install_unit_test_database () { fi fi - sed "s/db_2_0/$DATABASE_NAME/g" "$INSTALL_SQL_FILE" | $SQL + sed "s/db_2_0/$DATABASE_NAME/g" "$INSTALL_SQL_FILE" | $MYSQL_CMD $(get_mysql_args_nodb) - # crate test user + # create test user grant # update to latest @@ -96,7 +105,7 @@ function _setup_mytap() { pushd libs > /dev/null unzip -u mytap*.zip > /dev/null pushd mytap*/ > /dev/null - $SQL < mytap.sql > /dev/null || exit 1 + $MYSQL_CMD $(get_mysql_args_nodb) < mytap.sql > /dev/null || exit 1 popd > /dev/null rm -r mytap*/ popd > /dev/null @@ -112,17 +121,17 @@ Call 'make drop-$DATABASE_NAME' to delete that database or reconfigure with exit 0 fi - sed "s/db_2_0/$DATABASE_NAME/g" "$INSTALL_SQL_FILE" | $SQL + sed "s/db_2_0/$DATABASE_NAME/g" "$INSTALL_SQL_FILE" | $MYSQL_CMD $(get_mysql_args_nodb) } # Inserts the dump (arg 1) into the database function restore_db() { SQL_FILE="$1" - $SQL -D "$DATABASE_NAME" < "$SQL_FILE" + $MYSQL_CMD $(get_mysql_args) < "$SQL_FILE" } function test-connection() { - $SQL -e "select 0;" + $MYSQL_CMD $(get_mysql_args_nodb) -e "select 0;" } # Creates a user and grants it sufficient rights. @@ -146,7 +155,7 @@ function grant() { if [[ "$1" == "--strict" ]] ; then for host in ${DATABASE_USER_HOST_LIST//,/ } ; do CMD="SELECT COUNT(*) FROM mysql.user WHERE user='${DATABASE_USER}' AND host='${host}';" - [[ $($SQL -s -N -e "$CMD") == 0 ]] || { + [[ $($MYSQL_CMD $(get_mysql_args_nodb) -s -N -e "$CMD") == 0 ]] || { echo "The user '${DATABASE_USER}@${host}' is already in the database." echo "Please use another user or delete it, e.g. with" echo "'mysql -u ${MYSQL_USER} -p -e \"DROP USER ${DATABASE_USER}@${host};\"'" @@ -157,7 +166,7 @@ function grant() { for host in ${DATABASE_USER_HOST_LIST//,/ } ; do echo "Granting admin privileges to '$DATABASE_USER'@'$host'" - $SQL <<EOF + $MYSQL_CMD $(get_mysql_args_nodb) <<EOF CREATE USER IF NOT EXISTS '$DATABASE_USER'@'$host' identified by '$DATABASE_USER_PW'; GRANT USAGE ON *.* TO '$DATABASE_USER'@'$host'; @@ -172,31 +181,19 @@ EOF function drop() { DROPDB="$1" for host in ${DATABASE_USER_HOST_LIST//,/ } ; do - $SQL -e "DROP USER '${DATABASE_USER}'@'${host}';" || true + $MYSQL_CMD $(get_mysql_args_nodb) -e "DROP USER '${DATABASE_USER}'@'${host}';" || true done - "$MYSQLADMIN_CMD" $MYSQL_CONNECTION -f drop "$DROPDB" + "$MYSQLADMIN_CMD" $(get_mysql_args_nodb) -f drop "$DROPDB" } # Returns 0 or non-zero, depending on whether the database exists already. # Optional parameter: [DATABASE_NAME], else $DATABASE_NAME is used. function _db_exists() { - $SQL -D "${1-${DATABASE_NAME}}" -e "show tables;" > /dev/null 2>&1 \ + $MYSQL_CMD $(get_mysql_args_nodb) -D "${1-${DATABASE_NAME}}" -e "show tables;" > /dev/null 2>&1 \ && return 0 || return 1 } -source .config || true - -INSTALL_SQL_FILE="db_2_0.sql" -if [ -z $LOGIN_PATH ] ; then - MYSQL_CONNECTION="--host=${MYSQL_HOST} --port=${MYSQL_PORT} - --user=${MYSQL_USER} - --password=${MYSQL_USER_PASSWORD}" -else - MYSQL_CONNECTION="--login-path=$LOGIN_PATH" -fi - -SQL="$MYSQL_CMD $MYSQL_CONNECTION" case $1 in "drop") drop $2 ;; diff --git a/patches/utils/patch_header.sh b/utils/patch_header.sh similarity index 64% rename from patches/utils/patch_header.sh rename to utils/patch_header.sh index fec3997fc96fcd710922043db8dfacc1e2886b5e..7f85505234b2b5dd1232d2f7614b73908e70efc2 100644 --- a/patches/utils/patch_header.sh +++ b/utils/patch_header.sh @@ -22,8 +22,10 @@ # ** end header # #header for patch scripts -CMD_MYSQL=mysql -CMD_MYSQL_DUMP=mysqldump + +# load useful functions ####################################################### +. $UTILSPATH/helpers.sh + USAGE="$1 [ --env=ENV_FILE ] [ --patch=PATCH ] [ --backupdir=BACKUPDIR ]\n\n" CMD_OPTIONS=$(cat <<EOF options: @@ -101,65 +103,14 @@ while test $# -gt 0; do esac done -source "$ENV_FILE" || true - -if [[ -z "$DATABASE_NAME" && -z "$MYSQL_CONNECTION" ]] -then - _print_help "Please specify the database." - exit 1 -fi - -if [ "$LOGIN_PATH" ]; then - MYSQL_CONNECTION="--login-path=$LOGIN_PATH" - MYSQL_CONNECTION_NO_DB="$MYSQL_CONNECTION" - MYSQL_CONNECTION="$MYSQL_CONNECTION --database=$DATABASE_NAME" - -elif [[ -z "$MYSQL_CONNECTION" ]]; then - MYSQL_CONNECTION="" - if [ "$DATABASE_USER" ] - then - MYSQL_CONNECTION="--user=$DATABASE_USER" - fi - if [ "$DATABASE_USER_PW" ] - then - MYSQL_CONNECTION="$MYSQL_CONNECTION --password=$DATABASE_USER_PW" - fi - if [[ "$MYSQL_HOST" && ( "$MYSQL_HOST" != "localhost" ) ]]; then - MYSQL_CONNECTION="$MYSQL_CONNECTION --host=$MYSQL_HOST" - if [ "$MYSQL_PORT" ]; then - MYSQL_CONNECTION="$MYSQL_CONNECTION --port=$MYSQL_PORT" - fi - fi - # This option should come last, so we also have one version without the database - MYSQL_CONNECTION_NO_DB="$MYSQL_CONNECTION" - MYSQL_CONNECTION="$MYSQL_CONNECTION --database=$DATABASE_NAME" -fi -export MYSQL_CONNECTION -export MYSQL_CONNECTION_NO_DB -export DATABASE_NAME if [ -n "$PATCH" ]; then echo -ne "applying patch $PATCH to $DATABASE_NAME ... " fi - -function success { - echo "[OK]" - exit 0 -} - -function failure { - echo "[FAILED] $*" - exit 1 -} - -function uptodate { - echo "[UPTODATE]" - exit 0 -} # @param $1: db version string, e.g. v2.0.0 # @return: 0 on success, 1 on failure function check_version { - local version=$($CMD_MYSQL $MYSQL_CONNECTION -B -e "Select CaosDBVersion();") + local version=$($MYSQL_CMD $(get_db_args) -B -e "Select CaosDBVersion();") if [[ "$(echo $version | sed 's/^CaosDBVersion()\s//')" = "$1" ]]; then return 0 fi @@ -172,22 +123,12 @@ function update_version { } function dump_table { - $CMD_MYSQL_DUMP $MYSQL_CONNECTION_NO_DB $DATABASE_NAME $1 \ + $MYSQLDUMP_CMD $(get_db_args_nodb) $DATABASE_NAME $1 \ > ${DATABASE_NAME}.${1}.${OLD_VERSION}.dump.sql } - -function mysql_execute { - set +e - $CMD_MYSQL $MYSQL_CONNECTION -e "$1" - ret=${PIPESTATUS[0]} - if [ "$ret" -ne 0 ]; then - failure "MYSQL ERROR" - fi - set -e -} - function redo_table { - $CMD_MYSQL $MYSQL_CONNECTION < ${DATABASE_NAME}.${1}.${OLD_VERSION}.dump.sql + $MYSQL_CMD $(get_db_args) < ${DATABASE_NAME}.${1}.${OLD_VERSION}.dump.sql } + diff --git a/update_sql_procedures.sh b/utils/update_sql_procedures.sh similarity index 95% rename from update_sql_procedures.sh rename to utils/update_sql_procedures.sh index 7b0b457f07456663645c383703e8c1c7ec962291..73f6bacc86c80bc5b1be83e2c9c762777f99e878 100755 --- a/update_sql_procedures.sh +++ b/utils/update_sql_procedures.sh @@ -24,16 +24,16 @@ # Updates all SQL procedures -source patches/utils/patch_header.sh +source $UTILSPATH/helpers.sh echo -n "updating rules ... " set -e mysql_execute "$(sed s/db_2_0/$DATABASE_NAME/g rules.sql)" -echo "[OK]" +echo "OK" echo -n "updating procedures ... " mysql_execute "$(sed s/db_2_0/$DATABASE_NAME/g procedures/*.sql procedures/query/*.sql)" -success +echo "OK"