Merge branch 'tmp-stage' into f-new-tools

.gitlab-ci.yml

@@ -51,8 +51,8 @@ unittests:
     - echo 'DATABASE_USER_HOST_LIST="%,"' >> .config
     - echo "MYSQL_USER_PASSWORD=$MYSQL_ROOT_PASSWORD" >> .config
     - echo "MYSQL_HOST=mariadb" >> .config
-    - ./make_db test-connection
-    - ./make_db test --fresh
+    - ./utils/make_db test-connection
+    - ./utils/make_db test --fresh
 
 # Build a docker image in which tests for this repository can run
 build-testenv:

README_SETUP.md

@@ -3,6 +3,7 @@
 ## Dependencies
 * `MariaDB Client 10.1`, `MySQL Client 5.5`, or later versions.  In the case of
   MySQL, version 5.6 is recommended.
+- make
 
 ## Create the configuration
 * Run `./configure`. For the default values and the meaning of these default

config.defaults

@@ -55,5 +55,3 @@ DATABASE_USER_PW=random1234
 # wildcard for all hosts, so `192.168.0.%` permits logins from the local
 # network.
 DATABASE_USER_HOST_LIST=localhost,
-
-

makefile

@@ -22,19 +22,11 @@
 #
 SHELL=/bin/bash
 
-include .config
-
 INSTALL_SQL_FILE=db_2_0.sql
-ifdef LOGIN_PATH
-	M_MYSQL_CONNECTION=--login-path=$(M_LOGIN_PATH)
-else
-	MYSQL_CONNECTION=--host="$(MYSQL_HOST)" --port="$(MYSQL_PORT)"	\
-		--user="$(MYSQL_USER)" --password="$(MYSQL_USER_PASSWORD)"
-endif
 
 .PHONY: test-connection
 test-connection:
-	./make_db test-connection
+	./utils/make_db test-connection
 
 .PHONY: upgrade
 upgrade:
@@ -45,17 +37,17 @@ install: _install _grant upgrade
 
 .PHONY: _install
 _install:
-	./make_db install_db
+	./utils/make_db install_db
 
 .PHONY: _grant
 _grant:
-	./make_db grant
+	./utils/make_db grant
 
 # Drop the user and a given database
 .PHONY: drop-%
 drop-%:
-	./make_db drop $(patsubst drop-%,%,$@)
+	./utils/make_db drop $(patsubst drop-%,%,$@)
 
 .PHONY: test
 test:
-	./make_db test --fresh
+	./utils/make_db test --fresh

patches/applyPatches.sh

@@ -27,13 +27,11 @@
 #apply all available patches.
 set -e
 PATCHES="./patch*/patch.sh"
-export UTILSPATH="./utils"
 
 for p in $PATCHES
 do
     $p "$@" --patch=$p
 done
 
-source ./utils/patch_header.sh
 cd ../
-./update_sql_procedures.sh
+$UTILSPATH/update_sql_procedures.sh

patches/patch20151022-0/patch.sh

@@ -35,7 +35,7 @@ fi
 check_version $OLD_VERSION
 
 function addIdx {
-    $CMD_MYSQL $MYSQL_CONNECTION -D $DATABASE -e "ALTER TABLE $1 ADD COLUMN pidx TINYINT UNSIGNED NOT NULL DEFAULT 0;"
+    $MYSQL_CMD $(get_db_args) -e "ALTER TABLE $1 ADD COLUMN pidx TINYINT UNSIGNED NOT NULL DEFAULT 0;"
 }
 
 addIdx "reference_data"

utils/backup.sh

@@ -24,18 +24,22 @@
 
 # Dump a database with all procedures, permissions, structure and data
 
+
+if [ -z "$UTILSPATH" ]; then
+	UTILSPATH="$(realpath $(dirname $0))"
+	export UTILSPATH
+fi
 # The directory which the dump is to be stored to. Do not change it here. Use
 # the --backupdir=./my/dir/ option or an environment variable instead.
 BACKUPDIR="${BACKUPDIR:-../backup}"
 PRINT_HELP="--backupdir=BACKUPDIR\n\tThe directory which the dump is to be stored to. (Defaults to ./backup,\n\tcan also be set as an environment variable.)\n"
 
 
-# Config and command line parameter loading ###################################
-
-. ../.config
+# Load settings from .config and defaults #####################################
+. $UTILSPATH/load_settings.sh
 
-# load useful stuff - parses the commandline parameters and so on...
-. ../patches/utils/patch_header.sh
+# load useful functions #######################################################
+. $UTILSPATH/helpers.sh
 
 function backup() {
     NARG_NAME="$1"
@@ -47,7 +51,7 @@ function backup() {
 		    failure "dumpfile already exists."
 	  fi
     echo "Dumping database $NARG_NAME to $NARG_FILE ... "
-	  $CMD_MYSQL_DUMP $MYSQL_CONNECTION_NO_DB $* --opt --default-character-set=utf8 --routines \
+	  $MYSQLDUMP_CMD $(get_mysql_args_nodb) $* --opt --default-character-set=utf8 --routines \
                     "$NARG_NAME" > "$NARG_FILE"
 
 	  success

utils/helpers.sh

+# This file is a part of the CaosDB Project.
+#
+# Copyright (C) 2018 Research Group Biomedical Physics,
+# Max-Planck-Institute for Dynamics and Self-Organization Göttingen
+# Copyright 2019 Daniel Hornung
+# Copyright 2020 Henrik tom Wörden, IndiScale GmbH
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+
+function mysql_execute {
+	#TODO is it meaningful that here always the database user is used???
+	set +e
+	$MYSQL_CMD $(get_db_args) -e "$1"
+	ret=${PIPESTATUS[0]}
+	if [ "$ret" -ne 0 ]; then
+		failure "MYSQL ERROR"
+	fi
+	set -e
+}
+
+function get_mysql_args {
+	echo "$(get_mysql_args_nodb) --database=$DATABASE_NAME"
+}
+
+function get_db_args {
+	echo "$(get_db_args_nodb) --database=$DATABASE_NAME"
+}
+
+function get_db_args_nodb {
+	if [ "$DATABASE_USER" ]; then
+		mysql_con_arguments="--user=$DATABASE_USER"
+	fi
+	if [ "$DATABASE_USER_PW" ]; then
+		mysql_con_arguments="$mysql_con_arguments --password=$DATABASE_USER_PW"
+	fi
+	if [[ "$MYSQL_HOST" && ( "$MYSQL_HOST" != "localhost" ) ]]; then
+		mysql_con_arguments="$mysql_con_arguments --host=$MYSQL_HOST"
+		if [ "$MYSQL_PORT" ]; then
+			mysql_con_arguments="$mysql_con_arguments --port=$MYSQL_PORT"
+		fi
+	fi
+	echo $mysql_con_arguments
+}
+
+function get_mysql_args_nodb {
+	if [ "$LOGIN_PATH" ]; then
+		mysql_con_arguments="--login-path=$LOGIN_PATH"
+	else
+		if [ "$MYSQL_USER" ]; then
+			mysql_con_arguments="--user=$MYSQL_USER"
+		fi
+		if [ "$MYSQL_USER_PASSWORD" ]; then
+			mysql_con_arguments="$mysql_con_arguments --password=$MYSQL_USER_PASSWORD"
+		fi
+		if [[ "$MYSQL_HOST" && ( "$MYSQL_HOST" != "localhost" ) ]]; then
+			mysql_con_arguments="$mysql_con_arguments --host=$MYSQL_HOST"
+			if [ "$MYSQL_PORT" ]; then
+				mysql_con_arguments="$mysql_con_arguments --port=$MYSQL_PORT"
+			fi
+		fi
+	fi
+	echo $mysql_con_arguments
+}
+
+function success {
+    echo "[OK]"
+    exit 0
+}
+
+function failure {
+    echo "[FAILED] $*"
+    exit 1
+}
+
+function uptodate {
+    echo "[UPTODATE]"
+    exit 0
+}

utils/load_settings.sh

+# This file is a part of the CaosDB Project.
+#
+# Copyright (C) 2018 Research Group Biomedical Physics,
+# Max-Planck-Institute for Dynamics and Self-Organization Göttingen
+# Copyright 2019 Daniel Hornung
+# Copyright 2020 Henrik tom Wörden, IndiScale GmbH
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+[[ -n "config.defaults" ]] && source "config.defaults"
+[[ -n ".config" ]] && source ".config"
+[[ -n "$ENV_FILE" ]] && source "$ENV_FILE"
+
+export MYSQL_CMD
+export MYSQLDUMP_CMD
+export MYSQLADMIN_CMD
+export MYSQL_CONFIG_EDITOR_CMD
+export MYSQL_HOST
+export MYSQL_PORT
+export MYSQL_USER
+export DATABASE_NAME
+export DATABASE_USER
+export DATABASE_USER_PW
+export DATABASE_USER_HOST_LIST

make_db → utils/make_db

@@ -22,7 +22,16 @@
 #
 # ** end header
 
-# To fail fast, but beware https://mywiki.wooledge.org/BashFAQ/105
+INSTALL_SQL_FILE="db_2_0.sql"
+
+if [ -z "$UTILSPATH" ]; then
+	UTILSPATH="$(realpath $(dirname $0))"
+	export UTILSPATH
+fi
+
+source $UTILSPATH/load_settings.sh
+source $UTILSPATH/helpers.sh
+
 set -e
 
 function fail() {
@@ -51,7 +60,7 @@ function _execute_tests () {
     for tfile in $TESTS ; do
         echo "Running $tfile"
         echo "----- $tfile -----" >> .TEST_RESULTS
-        cat $tfile | $SQL --disable-pager --batch --raw --skip-column-names --unbuffered >> .TEST_RESULTS
+        cat $tfile | $MYSQL_CMD $(get_db_args_nodb) --disable-pager --batch --raw --skip-column-names --unbuffered >> .TEST_RESULTS
     done;
 
     popd
@@ -72,9 +81,9 @@ function _install_unit_test_database () {
         fi
     fi
 
-    sed "s/db_2_0/$DATABASE_NAME/g" "$INSTALL_SQL_FILE" | $SQL
+    sed "s/db_2_0/$DATABASE_NAME/g" "$INSTALL_SQL_FILE" | $MYSQL_CMD $(get_mysql_args_nodb)
 
-    # crate test user
+    # create test user
     grant
 
     # update to latest
@@ -96,32 +105,13 @@ function _setup_mytap() {
     pushd libs > /dev/null
     unzip -u mytap*.zip > /dev/null
     pushd mytap*/ > /dev/null
-    $SQL < mytap.sql > /dev/null || exit 1
+    $MYSQL_CMD $(get_mysql_args_nodb) < mytap.sql > /dev/null || exit 1
     popd > /dev/null
     rm -r mytap*/
     popd > /dev/null
     echo [DONE]
 }
 
-function setup_os() {
-  # - mariadb-client :: For SQL server configuration.
-  PACKAGES="git
-  make
-  mariadb-client
-  maven
-  openjdk-8-jdk-headless
-  python3-pip
-  screen
-  unzip
-"
-  # - vim :: For debugging
-  PACKAGES+="vim
-"
-  apt-get update
-  apt-get dist-upgrade -y
-  apt-get install -y $PACKAGES
-}
-
 function install_db() {
     if _db_exists ; then
         echo -e "\n
@@ -131,17 +121,17 @@ Call 'make drop-$DATABASE_NAME' to delete that database or reconfigure with
         exit 0
     fi
 
-	  sed "s/db_2_0/$DATABASE_NAME/g" "$INSTALL_SQL_FILE" | $SQL
+	  sed "s/db_2_0/$DATABASE_NAME/g" "$INSTALL_SQL_FILE" | $MYSQL_CMD $(get_mysql_args_nodb)
 }
 
 # Inserts the dump (arg 1) into the database
 function restore_db() {
     SQL_FILE="$1"
-    $SQL -D "$DATABASE_NAME" < "$SQL_FILE"
+    $MYSQL_CMD $(get_mysql_args) < "$SQL_FILE"
 }
 
 function test-connection() {
-    $SQL -e "select 0;"
+    $MYSQL_CMD $(get_mysql_args_nodb) -e "select 0;"
 }
 
 # Creates a user and grants it sufficient rights.
@@ -165,7 +155,7 @@ function grant() {
     if [[ "$1" == "--strict" ]] ; then
         for host in ${DATABASE_USER_HOST_LIST//,/ } ; do
             CMD="SELECT COUNT(*) FROM mysql.user WHERE user='${DATABASE_USER}' AND host='${host}';"
-		        [[ $($SQL -s -N -e "$CMD") == 0 ]] || {
+		        [[ $($MYSQL_CMD $(get_mysql_args_nodb) -s -N -e "$CMD") == 0 ]] || {
                 echo "The user '${DATABASE_USER}@${host}' is already in the database."
                 echo "Please use another user or delete it, e.g. with"
                 echo "'mysql -u ${MYSQL_USER} -p -e \"DROP USER ${DATABASE_USER}@${host};\"'"
@@ -176,7 +166,7 @@ function grant() {
 
     for host in ${DATABASE_USER_HOST_LIST//,/ } ; do
         echo "Granting admin privileges to '$DATABASE_USER'@'$host'"
-        $SQL <<EOF
+        $MYSQL_CMD $(get_mysql_args_nodb) <<EOF
 CREATE USER IF NOT EXISTS
     '$DATABASE_USER'@'$host' identified by '$DATABASE_USER_PW';
 GRANT USAGE ON *.* TO '$DATABASE_USER'@'$host';
@@ -191,31 +181,19 @@ EOF
 function drop() {
     DROPDB="$1"
     for host in ${DATABASE_USER_HOST_LIST//,/ } ; do
-        $SQL -e "DROP USER '${DATABASE_USER}'@'${host}';" || true
+        $MYSQL_CMD $(get_mysql_args_nodb) -e "DROP USER '${DATABASE_USER}'@'${host}';" || true
     done
-    "$MYSQLADMIN_CMD" $MYSQL_CONNECTION -f drop "$DROPDB"
+    "$MYSQLADMIN_CMD" $(get_mysql_args_nodb) -f drop "$DROPDB"
 }
 
 
 # Returns 0 or non-zero, depending on whether the database exists already.
 # Optional parameter: [DATABASE_NAME], else $DATABASE_NAME is used.
 function _db_exists() {
-    $SQL -D "${1-${DATABASE_NAME}}" -e "show tables;" > /dev/null 2>&1 \
+    $MYSQL_CMD $(get_mysql_args_nodb) -D "${1-${DATABASE_NAME}}" -e "show tables;" > /dev/null 2>&1 \
         && return 0 || return 1
 }
 
-source .config || true
-
-INSTALL_SQL_FILE="db_2_0.sql"
-if [ -z $LOGIN_PATH ] ; then
-    MYSQL_CONNECTION="--host=${MYSQL_HOST} --port=${MYSQL_PORT}
- --user=${MYSQL_USER}
- --password=${MYSQL_USER_PASSWORD}"
-else
-    MYSQL_CONNECTION="--login-path=$LOGIN_PATH"
-fi
-
-SQL="$MYSQL_CMD $MYSQL_CONNECTION"
 
 case $1 in
   "drop") drop $2 ;;

patches/utils/patch_header.sh → utils/patch_header.sh

@@ -22,8 +22,10 @@
 # ** end header
 #
 #header for patch scripts
-CMD_MYSQL=mysql
-CMD_MYSQL_DUMP=mysqldump
+
+# load useful functions #######################################################
+. $UTILSPATH/helpers.sh
+
 USAGE="$1 [ --env=ENV_FILE ] [ --patch=PATCH ] [ --backupdir=BACKUPDIR ]\n\n"
 CMD_OPTIONS=$(cat <<EOF
 options:
@@ -101,65 +103,14 @@ while test $# -gt 0; do
  esac
 done
 
-source "$ENV_FILE" || true
-
-if [[ -z "$DATABASE_NAME" && -z "$MYSQL_CONNECTION" ]]
-then
- _print_help "Please specify the database."
- exit 1
-fi
-
-if [ "$LOGIN_PATH" ]; then
-  MYSQL_CONNECTION="--login-path=$LOGIN_PATH"
-  MYSQL_CONNECTION_NO_DB="$MYSQL_CONNECTION"
-  MYSQL_CONNECTION="$MYSQL_CONNECTION --database=$DATABASE_NAME"
-
-elif [[ -z "$MYSQL_CONNECTION" ]]; then
-  MYSQL_CONNECTION=""
-  if [ "$DATABASE_USER" ]
-  then
-    MYSQL_CONNECTION="--user=$DATABASE_USER"
-  fi
-  if [ "$DATABASE_USER_PW" ]
-  then
-    MYSQL_CONNECTION="$MYSQL_CONNECTION --password=$DATABASE_USER_PW"
-  fi
-  if [[ "$MYSQL_HOST" && ( "$MYSQL_HOST" != "localhost" ) ]]; then
-      MYSQL_CONNECTION="$MYSQL_CONNECTION --host=$MYSQL_HOST"
-      if [ "$MYSQL_PORT" ]; then
-          MYSQL_CONNECTION="$MYSQL_CONNECTION --port=$MYSQL_PORT"
-      fi
-  fi
-  # This option should come last, so we also have one version without the database
-  MYSQL_CONNECTION_NO_DB="$MYSQL_CONNECTION"
-  MYSQL_CONNECTION="$MYSQL_CONNECTION --database=$DATABASE_NAME"
-fi
-export MYSQL_CONNECTION
-export MYSQL_CONNECTION_NO_DB
-export DATABASE_NAME
 
 if [ -n "$PATCH" ]; then
     echo -ne "applying patch $PATCH to $DATABASE_NAME ... "
 fi
-
-function success {
-    echo "[OK]"
-    exit 0
-}
-
-function failure {
-    echo "[FAILED] $*"
-    exit 1
-}
-
-function uptodate {
-    echo "[UPTODATE]"
-    exit 0
-}
 # @param $1: db version string, e.g. v2.0.0
 # @return: 0 on success, 1 on failure
 function check_version {
-    local version=$($CMD_MYSQL $MYSQL_CONNECTION -B -e "Select CaosDBVersion();")
+    local version=$($MYSQL_CMD $(get_db_args) -B -e "Select CaosDBVersion();")
 	  if [[ "$(echo $version | sed 's/^CaosDBVersion()\s//')" = "$1" ]]; then
         return 0
 	  fi
@@ -172,22 +123,12 @@ function update_version {
 }
 
 function dump_table {
-    $CMD_MYSQL_DUMP $MYSQL_CONNECTION_NO_DB $DATABASE_NAME $1 \
+    $MYSQLDUMP_CMD $(get_db_args_nodb) $DATABASE_NAME $1 \
                     > ${DATABASE_NAME}.${1}.${OLD_VERSION}.dump.sql
 }
 
-
-function mysql_execute {
-	set +e
-	$CMD_MYSQL $MYSQL_CONNECTION -e "$1"
-	ret=${PIPESTATUS[0]}
-	if [ "$ret" -ne 0 ]; then
-		failure "MYSQL ERROR"
-	fi
-	set -e
-}
-
 function redo_table {
-	  $CMD_MYSQL $MYSQL_CONNECTION < ${DATABASE_NAME}.${1}.${OLD_VERSION}.dump.sql
+         $MYSQL_CMD $(get_db_args) < ${DATABASE_NAME}.${1}.${OLD_VERSION}.dump.sql
 }
 
+

update_sql_procedures.sh → utils/update_sql_procedures.sh

@@ -24,16 +24,16 @@
 
 # Updates all SQL procedures
 
-source patches/utils/patch_header.sh
+source $UTILSPATH/helpers.sh
 
 echo -n "updating rules ... "
 set -e
 
 mysql_execute "$(sed s/db_2_0/$DATABASE_NAME/g rules.sql)"
-echo "[OK]"
+echo "OK"
 
 echo -n "updating procedures ... "
 mysql_execute "$(sed s/db_2_0/$DATABASE_NAME/g procedures/*.sql procedures/query/*.sql)" 
-success
+echo "OK"