WIP: Merge branch 'f-docker' into merge_docker_sss

README_SETUP.md

-# Setup the CaosDB MySQL Back-end
+# Setup of the CaosDB SQL back end
 
 ## Dependencies
-
-* `MariaDB Client 10.1`, `MySQL Client 5.5`, or later versions. MySQL 5.6 is recommended.
-## Configuration
-
-* Run `./configure`. For the default values and the meaning of these default values see [config.defaults](config.defaults). The `configure` script also promts for a (new) user password. Remember that one, since it will be the password of the mysql user which is used by the caosdb server. 
-* If there is no `mysql-config-editor` (`MySQL 5.5`. and `MariaDB`) then the user is promted for the `MYSQL_USER_PASSWORD`, that is the password of the `MYSQL_USER`. The plain password is stored in the `.config` file. So make sure nobody else can read it.
-* If you are using MariaDB and your `root` user uses pam authentication, no pasword string is required. When promted for the `MYSQL_USER_PASSWORD` just leave it blank and hit enter. But you need be login as root for the installation and upgrade process.
-
-## Install
-
-* Run `make install`. If a there is a database with the name you have choosen during the configuration, you need to reconfigure or delete the database first.
-
-## Upgrade
-
-* Run `make upgrade`. Upgrades the database to the last version.
+* `MariaDB Client 10.1`, `MySQL Client 5.5`, or later versions.  In the case of
+  MySQL, version 5.6 is recommended.
+
+## Create the configuration
+* Run `./configure`. For the default values and the meaning of these default
+  values see [config.defaults](config.defaults). The `configure` script also
+  prompts for a (new) user password. Remember that one, since it will be the
+  password of the mysql user which is used by the caosdb server.  Upon succesful
+  completion, the `configure` script writes the configuration to `.config`.
+    * If there is no `mysql-config-editor` (`MySQL 5.5`. and `MariaDB`) then the
+      user is prompted for the `MYSQL_USER_PASSWORD`, that is the password of
+      the `MYSQL_USER`. The plain password is stored in the `.config` file. So
+      make sure nobody else can read it.
+    * If you are using MariaDB and the `root` database user uses pam authentication,
+      no pasword string is required. When prompted for the `MYSQL_USER_PASSWORD`
+      just leave it blank and hit enter. But you need be login as root for the
+      installation and upgrade process.
+
+## Setup the SQL database for CaosDB
+
+* Run `make install`. If a there is a database with the name you have choosen
+  during the configuration, you need to reconfigure or delete the database
+  first.
+
+## Upgrade the SQL database
+
+* Run `make upgrade`. This upgrades the database to the last version.
 
 ## Drop Database
 
-* If you want to delete your database, run `make drop-$DATABASE_NAME`. **Use this with caution!!!** If you did not backup your database, **everything will be lost afterwards.** And no, there is no addional promt "Are you sure, that you want to do this?". If you do this, you *actually* do this.
+* If you want to delete your database, run `make drop-$DATABASE_NAME`. **Use
+  this with caution!!!** If you did not backup your database, **everything will
+  be lost afterwards.** And no, there is *no* additional prompt to ask if you
+  are sure.  If you `make drop-...`, you *actually* delete the database.

config.defaults

@@ -28,12 +28,22 @@ MYSQLDUMP_CMD=$(command -v mysqldump) # The mysqldump program which comes with t
 MYSQL_CONFIG_EDITOR_CMD=$(command -v mysql_config_editor) # The mysql_config_editor program which is used to store the credentials.
 
 # MySQL Connection
-MYSQL_HOST=localhost #The host of the MySQL server.
-MYSQL_USER=root # The user for the installation. Note: This is not the user which will then be used by the CaosDB Server.
+MYSQL_HOST=localhost # The host of the MySQL server.
+MYSQL_PORT=3306 # The port number of the MySQL server.
+MYSQL_USER=root # The user for the installation. Note: This is not the user
+                # which will then be used by the CaosDB Server.
 
 # DATABASE
-DATABASE_NAME=caosdb #The name of the database.
-DATABASE_USER=caosdb_user #The user which is used by the CaosDB Server for the connection.
-DATABASE_USER_HOST_LIST=localhost, # A comma-separated list of hosts from which MySQL will accept logins. This option follows the MySQL-style for host-names and ip addresses. E.g. `%` is a wildcard for all hosts, `192.168.0.%` permits logins from the local network etc.
+DATABASE_NAME=caosdb # The name of the SQL database.
+DATABASE_USER=caosdb # The user which is used by the CaosDB Server for accessing
+                     # the database.     
+DATABASE_USER_PW=random1234 # The password for the database user.  Leave empty
+                            # if you want to be prompted at make time.
+
+# A comma-separated list of hosts from which MySQL will accept logins. This
+# option follows the MySQL-style for host-names and ip addresses. E.g. `%` is a
+# wildcard for all hosts, so `192.168.0.%` permits logins from the local
+# network.
+DATABASE_USER_HOST_LIST=localhost,
 
 

make_db

+#!/bin/bash
+
+# ** header v3.0
+# This file is a part of the CaosDB Project.
+#
+# Copyright (C) 2019 Daniel Hornung, Göttingen
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see <https://www.gnu.org/licenses/>.
+#
+# ** end header
+
+# To fail fast, but beware https://mywiki.wooledge.org/BashFAQ/105
+set -e
+
+function fail() {
+  echo "Some error occured, exiting."
+  exit 1
+}
+
+function setup_os() {
+  # - mariadb-client :: For SQL server configuration.
+  PACKAGES="git
+  make
+  mariadb-client
+  maven
+  openjdk-8-jdk-headless
+  python3-pip
+  screen
+  unzip
+"
+  # - vim :: For debugging
+  PACKAGES+="vim
+"
+  apt-get update
+  apt-get dist-upgrade -y
+  apt-get install -y $PACKAGES
+}
+
+function install_db() {
+    if _db_exists ; then
+        echo -e "\n
+A database with with the name '$DATABASE_NAME' exists already.
+Call 'make drop-$DATABASE_NAME' to delete that database or reconfigure with
+'./configure'."
+        exit 0
+    fi
+
+	  sed "s/db_2_0/$DATABASE_NAME/g" "$INSTALL_SQL_FILE" | $SQL
+}
+
+function test-connection() {
+  	$SQL -e "select 0;"
+}
+
+# Creates a user and grants it sufficient rights.
+# If called with --strict, the user is not allowed to exist previously.
+function grant() {
+    if [ -z "$DATABASE_USER_PW" ] ; then
+        while true; do
+            read -p "Please enter the password for ${DATABASE_USER}: " \
+                 -s password
+            echo                
+            read -p "Please repeat the password: " -s password2
+            echo
+            [[ "$password" == "$password2" ]] && {
+                DATABASE_USER_PW="$password"
+                break
+            }
+            echo -e "\nThe passwords didn't match. Try again."
+        done
+    fi
+
+    if [[ "$1" == "--strict" ]] ; then
+        for host in ${DATABASE_USER_HOST_LIST//,/ } ; do
+            CMD="SELECT COUNT(*) FROM mysql.user WHERE user='${DATABASE_USER}' AND host='${host}';"
+		        [[ $SQL -s -N -e "$CMD" == 0 ]] || {
+                echo "The user '${DATABASE_USER}@${host}' is already in the database."
+                echo "Please use another user or delete it, e.g. with"
+                echo "'mysql -u ${MYSQL_USER} -p -e \"DROP USER ${DATABASE_USER}@${host};\"'"
+                exit 1
+            }
+        done
+    fi
+
+    for host in ${DATABASE_USER_HOST_LIST//,/ } ; do
+        $SQL <<EOF
+CREATE USER IF NOT EXISTS
+    '$DATABASE_USER'@'$host' identified by '$DATABASE_USER_PW';
+GRANT USAGE ON *.* TO '$DATABASE_USER'@'$host';
+GRANT ALL PRIVILEGES ON *.* TO '$DATABASE_USER'@'$host' WITH GRANT OPTION;
+GRANT EXECUTE ON *.* TO '$DATABASE_USER'@'$host';
+EOF
+    done
+}
+
+# Drops the caosdb user plus a given database from all listed database hosts.
+# The first argument to this function is the database that shal be dropped.
+function drop() {
+    DROPDB="$1"
+    for host in ${DATABASE_USER_HOST_LIST//,/ } ; do
+        $SQL -e "DROP USER '${DATABASE_USER}'@'${host}';" || true
+    done
+    "$MYSQLADMIN_CMD" $MYSQL_CONNECTION -f drop "$DROPDB"
+}
+
+# Returns 0 or non-zero, depending on whether the database exists already.
+function _db_exists() {
+    $SQL -D "$DATABASE_NAME" -e "show tables;" > /dev/null 2>&1 \
+        && return 0 || return 1
+}
+
+. .config
+
+INSTALL_SQL_FILE="db_2_0.sql"
+if [ -z $LOGIN_PATH ] ; then
+    MYSQL_CONNECTION="--host=${MYSQL_HOST} --port=${MYSQL_PORT}
+ --user=${MYSQL_USER}
+ --password=${MYSQL_USER_PASSWORD}"
+else
+    MYSQL_CONNECTION="--login-path=$LOGIN_PATH"
+fi
+
+SQL="$MYSQL_CMD $MYSQL_CONNECTION"
+
+case $1 in
+  "drop") drop $2 ;;
+  "grant") grant $2;;
+  "test-connection") test-connection ;;
+  "install_db") install_db ;;
+  # "prep_sql") prepare_sql ;;
+  *) echo "Unknown action: $1"
+esac

makefile

@@ -23,66 +23,35 @@
 SHELL=/bin/bash
 
 include .config
-M_MYSQL_USER=$(shell echo $(MYSQL_USER))
-M_MYSQL_USER_PASSWORD=$(shell echo $(MYSQL_USER_PASSWORD))
-M_LOGIN_PATH=$(shell echo $(LOGIN_PATH))
-M_DATABASE_NAME=$(shell echo $(DATABASE_NAME))
-M_DATABASE_USER_HOST_LIST=$(shell echo $(DATABASE_USER_HOST_LIST))
-M_DATABASE_USER=$(shell echo $(DATABASE_USER))
-M_MYSQL_CMD=$(shell echo $(MYSQL_CMD))
-M_MYSQLDUMP_CMD=$(shell echo $(MYSQLDUMP_CMD))
-M_MYSQLADMIN_CMD=$(shell echo $(MYSQLADMIN_CMD))
 
 INSTALL_SQL_FILE=db_2_0.sql
 ifdef LOGIN_PATH
 	M_MYSQL_CONNECTION=--login-path=$(M_LOGIN_PATH)
 else
-	M_MYSQL_CONNECTION=--user="$(M_MYSQL_USER)" --password="$(M_MYSQL_USER_PASSWORD)"
+	MYSQL_CONNECTION=--host="$(MYSQL_HOST)" --port="$(MYSQL_PORT)"	\
+		--user="$(MYSQL_USER)" --password="$(MYSQL_USER_PASSWORD)"
 endif
 
 .PHONY: test-connection
 test-connection:
-	$(M_MYSQL_CMD) $(M_MYSQL_CONNECTION) -e "select 0;"
+	./make_db test-connection
 
 .PHONY: upgrade
 upgrade:
-	@cd patches; ./applyPatches.sh $(M_MYSQL_CONNECTION) --database=$(M_DATABASE_NAME)
+	@cd patches; ./applyPatches.sh --env=../.config
 
 .PHONY: install
 install: _install _grant upgrade
 
 .PHONY: _install
 _install:
-	@if $(MAKE) _exists > /dev/null 2>&1; then \
-		printf "\n\nA database with with the name \"$(M_DATABASE_NAME)\" does already exist.\nCall 'make drop-$(M_DATABASE_NAME)' to delete that database or reconfigure with './configure'.\n"; \
-		exit 1; \
-	else sed 's/db_2_0/$(M_DATABASE_NAME)/g' $(INSTALL_SQL_FILE) | $(M_MYSQL_CMD) $(M_MYSQL_CONNECTION); fi
+	./make_db install_db
 
-comma:=,
 .PHONY: _grant
 _grant:
-	@for host in $(subst $(comma), ,$(M_DATABASE_USER_HOST_LIST)); do \
-		( $(M_MYSQL_CMD) $(M_MYSQL_CONNECTION) -e "SELECT COUNT(*) FROM mysql.user WHERE user='$(M_DATABASE_USER)' AND host=\"$$host\";" | grep 0 ) || ( echo "The user '$(M_DATABASE_USER)@$$host' is already in the database. Please use another user or delete her. E.g. with 'mysql -u $(M_MYSQL_USER) -p -e \"DROP USER $(M_DATABASE_USER)@$$host;\"'" ; exit 1 ) ; \
-	done
-	@while true; do \
-		printf "\n Please enter the password for $(M_DATABASE_USER): "; read -s password; \
-		printf "\nPlease repeat the password for $(M_DATABASE_USER): "; read -s password2; \
-		if [ "$$password" != "$$password2" ]; then printf "\n\nThe passwords didn't match. Try again." ; else break ; fi; \
-	done ; \
-	echo "" ; \
-	for host in $(subst $(comma), ,$(M_DATABASE_USER_HOST_LIST)); do \
-		$(M_MYSQL_CMD) $(M_MYSQL_CONNECTION) -e "CREATE USER IF NOT EXISTS '$(M_DATABASE_USER)'@'$$host' identified by '$$password';" ; \
-		$(M_MYSQL_CMD) $(M_MYSQL_CONNECTION) -e "GRANT USAGE ON *.* TO '$(M_DATABASE_USER)'@'$$host';"; \
-		$(M_MYSQL_CMD) $(M_MYSQL_CONNECTION) -e "GRANT ALL PRIVILEGES ON *.* TO '$(M_DATABASE_USER)'@'$$host' WITH GRANT OPTION;"; \
-		$(M_MYSQL_CMD) $(M_MYSQL_CONNECTION) -e "GRANT EXECUTE ON *.* TO '$(M_DATABASE_USER)'@'$$host';"; \
-	done
+	./make_db grant
 
+# Drop the user and a given database
 .PHONY: drop-%
 drop-%:
-	@$(M_MYSQLADMIN_CMD) $(M_MYSQL_CONNECTION) -f drop $(patsubst drop-%,%,$@)
-
-.PHONY: _exists
-_exists:
-	@if $(M_MYSQL_CMD) $(M_MYSQL_CONNECTION) -D "$(M_DATABASE_NAME)" -e "show tables;" > /dev/null 2>&1 ; then \
-		echo "$(M_DATABASE_NAME) does exist." ; exit 0 ; \
-	else echo "$(M_DATABASE_NAME) does not exist." ; exit 1; fi
+	./make_db drop $(patsubst drop-%,%,$@)

patches/applyPatches.sh

@@ -30,8 +30,8 @@ export UTILSPATH="./utils"
 
 for p in $PATCHES
 do
-    $p $MYSQL_CONNECTION --database=$DATABASE_NAME --patch=$p
+    $p "$@" --patch=$p
 done
 
 cd ../
-./update_sql_procedures.sh $MYSQL_CONNECTION --database=$DATABASE_NAME
+./update_sql_procedures.sh

patches/utils/patch_header.sh

@@ -4,6 +4,7 @@
 #
 # Copyright (C) 2018 Research Group Biomedical Physics,
 # Max-Planck-Institute for Dynamics and Self-Organization Göttingen
+# Copyright 2019 Daniel Hornung
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as
@@ -21,121 +22,110 @@
 # ** end header
 #
 #header for patch scripts
-if test -f ../.config ; then
-    . ../.config
-fi
-if test -f .config ; then
-    . .config
-fi
-CMD_OPTIONS="options:\n\n-h, --help\n\tShow brief help.\n-l LOGIN_PATH, --login-path=LOGIN_PATH\n\tA login-path for the mysql connection (see 'man mysql' and 'man mysql-config-editor').\n-u USER, --user=USER\n\tA mysql user.\n-p PASSWORD, --password=PASSWORD\n\tThe password for the mysql user.\n"
+CMD_MYSQL=mysql
+CMD_MYSQL_DUMP=mysqldump
+USAGE="$1 [ --env=ENV_FILE ] [ --patch=PATCH ] [ --backupdir=BACKUPDIR ]\n\n"
+CMD_OPTIONS=cat <<EOF
+options:
+
+-h, --help
+    Show brief help.
+--env=ENV_FILE
+    A file where variables are stored in the format of shell environment
+    variables.  Content of this file overrides environment variables already
+    present.
+--patch=PATCH
+    *TODO* Please document this option.
+--backupdir=BACKUPDIR
+    *TODO* Please document this option.
+
+This script basically uses the same environment variables as the server
+configuration make file.  Notable examples are:
+
+- LOGIN_PATH
+- MYSQL_HOST
+- MYSQL_PORT
+- DATABASE_NAME
+- DATABASE_USER
+- DATABASE_USER_PW
+
+EOF
 set -e
 
 function _print_help() {
-  printf -- "$USAGE"
-  printf -- "$CMD_OPTIONS"
+  echo -e "$USAGE"
+  echo -e "$CMD_OPTIONS"
   if [ -n "$PRINT_HELP" ]; then
-    printf -- "$PRINT_HELP"
+    echo -e "$PRINT_HELP"
   fi
   if [ -n "$1" ]; then
-    printf -- "$1"
+    echo -e "$1"
   fi
 }
+
 while test $# -gt 0; do
  case "$1" in
  -h|--help)
   _print_help
   exit 0
   ;;
- -p)
-  shift
-  if test $# -gt 0; then
-   MYSQL_USER_PASSWORD=$1
-  else
-   echo "no database specified"
-   exit 1
-  fi
-  shift
-  ;;
- --password*)
-  MYSQL_USER_PASSWORD=`echo $1 | sed -e 's/^[^=]*=//g'`
-  shift
-  ;;
- -u)
-  shift
-  if test $# -gt 0; then
-   MYSQL_USER=$1
-  else
-   echo "no database specified"
-   exit 1
-  fi
-  shift
-  ;;
- --user*)
-  MYSQL_USER=`echo $1 | sed -e 's/^[^=]*=//g'`
-  shift
-  ;;
- -l)
-  shift
-  if test $# -gt 0; then
-   LOGIN_PATH=$1
-  else
-   echo "no database specified"
-   exit 1
-  fi
-  shift
-  ;;
- --login-path*)
-  LOGIN_PATH=`echo $1 | sed -e 's/^[^=]*=//g'`
-  shift
-  ;;
- -d)
-  shift
-  if test $# -gt 0; then
-   DATABASE_NAME=$1
-  else
-   echo "no database specified"
-   exit 1
-  fi
-  shift
-  ;;
- --database*)
-  DATABASE_NAME=`echo $1 | sed -e 's/^[^=]*=//g'`
+ --env*)
+  ENV_FILE="${1#--*=}"
   shift
   ;;
  --patch*)
-  PATCH=`echo $1 | sed -e 's/^[^=]*=//g'`
+  PATCH="${1#--*=}"
   shift
   ;;
  --backupdir*)
-  BACKUPDIR=`echo $1 | sed -e 's/^[^=]*=//g'`
+  BACKUPDIR="${1#--*=}"
+  echo "Warning: BACKUPDIR is not used currently."
   shift
   ;;
  *)
+  echo "Unknown option $1"
+  exit 1
   shift
   ;;
  esac
 done
 
-if [ -z "$DATABASE_NAME" ]
+[[ -n "$ENV_FILE" ]] && source "$ENV_FILE"
+
+if [[ -z "$DATABASE_NAME" && -z "$MYSQL_CONNECTION" ]]
 then
  _print_help "Please specify the database."
  exit 1
 fi
 
-if [ "$LOGIN_PATH" ]
-then
+if [ "$LOGIN_PATH" ]; then
   MYSQL_CONNECTION="--login-path=$LOGIN_PATH"
-else
+  MYSQL_CONNECTION_NO_DB="$MYSQL_CONNECTION"
+  MYSQL_CONNECTION="$MYSQL_CONNECTION --database=$DATABASE_NAME"
+
+elif [[ -z "$MYSQL_CONNECTION" ]]; then
   MYSQL_CONNECTION=""
-  if [ "$MYSQL_USER" ]
+  if [ "$DATABASE_USER" ]
   then
-    MYSQL_CONNECTION="--user=$MYSQL_USER"
+    MYSQL_CONNECTION="--user=$DATABASE_USER"
   fi
-  if [ "$MYSQL_USER_PASSWORD" ]
+  if [ "$DATABASE_USER_PW" ]
   then
-    MYSQL_CONNECTION="$MYSQL_CONNECTION --password=$MYSQL_USER_PASSWORD"
+    MYSQL_CONNECTION="$MYSQL_CONNECTION --password=$DATABASE_USER_PW"
   fi
+  if [[ "$MYSQL_HOST" && ( "$MYSQL_HOST" != "localhost" ) ]]; then
+      MYSQL_CONNECTION="$MYSQL_CONNECTION --host=$MYSQL_HOST"
+      if [ "$MYSQL_PORT" ]; then
+          MYSQL_CONNECTION="$MYSQL_CONNECTION --port=$MYSQL_PORT"
+      fi
+  fi
+  # This option should come last, so we also have one version without the database
+  MYSQL_CONNECTION_NO_DB="$MYSQL_CONNECTION"
+  MYSQL_CONNECTION="$MYSQL_CONNECTION --database=$DATABASE_NAME"
 fi
+export MYSQL_CONNECTION
+export MYSQL_CONNECTION_NO_DB
+export DATABASE_NAME
 
 if [ -n "$PATCH" ]; then
     echo -ne "applying patch $PATCH to $DATABASE_NAME ... "
@@ -158,10 +148,10 @@ function uptodate {
 # @param $1: db version string, e.g. v2.0.0
 # @return: 0 on success, 1 on failure
 function check_version {
-    local version=$($MYSQL_CMD $MYSQL_CONNECTION -D $DATABASE_NAME -B -e "Select CaosDBVersion();")
-	if [[ "$(echo $version | sed 's/^CaosDBVersion()\s//')" = "$1" ]]; then
+    local version=$($CMD_MYSQL $MYSQL_CONNECTION -B -e "Select CaosDBVersion();")
+	  if [[ "$(echo $version | sed 's/^CaosDBVersion()\s//')" = "$1" ]]; then
         return 0
-	fi
+	  fi
     uptodate
 }
 
@@ -171,13 +161,14 @@ function update_version {
 }
 
 function dump_table {
-    $MYSQLDUMP_CMD $MYSQL_CONNECTION $DATABASE_NAME $1 > ${DATABASE_NAME}.${1}.${OLD_VERSION}.dump.sql
+    $CMD_MYSQL_DUMP $MYSQL_CONNECTION_NO_DB $DATABASE_NAME $1 \
+                    > ${DATABASE_NAME}.${1}.${OLD_VERSION}.dump.sql
 }
 
 
 function mysql_execute {
 	set +e
-	$MYSQL_CMD $MYSQL_CONNECTION -D $DATABASE_NAME -e "$1"
+	$CMD_MYSQL $MYSQL_CONNECTION -e "$1"
 	ret=${PIPESTATUS[0]}
 	if [ "$ret" -ne 0 ]; then
 		failure "MYSQL ERROR"
@@ -186,6 +177,6 @@ function mysql_execute {
 }
 
 function redo_table {
-	$MYSQL_CMD $MYSQL_CONNECTION -D $DATABASE_NAME < ${DATABASE_NAME}.${1}.${OLD_VERSION}.dump.sql
+	  $CMD_MYSQL $MYSQL_CONNECTION < ${DATABASE_NAME}.${1}.${OLD_VERSION}.dump.sql
 }