diff --git a/README_SETUP.md b/README_SETUP.md index f5bcc2d063f5d34c9ef84f5eeca0a3fc96266681..e409edaa9612340490f8cf8642201a5e1812d867 100644 --- a/README_SETUP.md +++ b/README_SETUP.md @@ -1,22 +1,37 @@ -# Setup the CaosDB MySQL Back-end +# Setup of the CaosDB SQL back end ## Dependencies - -* `MariaDB Client 10.1`, `MySQL Client 5.5`, or later versions. MySQL 5.6 is recommended. -## Configuration - -* Run `./configure`. For the default values and the meaning of these default values see [config.defaults](config.defaults). The `configure` script also promts for a (new) user password. Remember that one, since it will be the password of the mysql user which is used by the caosdb server. -* If there is no `mysql-config-editor` (`MySQL 5.5`. and `MariaDB`) then the user is promted for the `MYSQL_USER_PASSWORD`, that is the password of the `MYSQL_USER`. The plain password is stored in the `.config` file. So make sure nobody else can read it. -* If you are using MariaDB and your `root` user uses pam authentication, no pasword string is required. When promted for the `MYSQL_USER_PASSWORD` just leave it blank and hit enter. But you need be login as root for the installation and upgrade process. - -## Install - -* Run `make install`. If a there is a database with the name you have choosen during the configuration, you need to reconfigure or delete the database first. - -## Upgrade - -* Run `make upgrade`. Upgrades the database to the last version. +* `MariaDB Client 10.1`, `MySQL Client 5.5`, or later versions. In the case of + MySQL, version 5.6 is recommended. + +## Create the configuration +* Run `./configure`. For the default values and the meaning of these default + values see [config.defaults](config.defaults). The `configure` script also + prompts for a (new) user password. Remember that one, since it will be the + password of the mysql user which is used by the caosdb server. Upon succesful + completion, the `configure` script writes the configuration to `.config`. + * If there is no `mysql-config-editor` (`MySQL 5.5`. and `MariaDB`) then the + user is prompted for the `MYSQL_USER_PASSWORD`, that is the password of + the `MYSQL_USER`. The plain password is stored in the `.config` file. So + make sure nobody else can read it. + * If you are using MariaDB and the `root` database user uses pam authentication, + no pasword string is required. When prompted for the `MYSQL_USER_PASSWORD` + just leave it blank and hit enter. But you need be login as root for the + installation and upgrade process. + +## Setup the SQL database for CaosDB + +* Run `make install`. If a there is a database with the name you have choosen + during the configuration, you need to reconfigure or delete the database + first. + +## Upgrade the SQL database + +* Run `make upgrade`. This upgrades the database to the last version. ## Drop Database -* If you want to delete your database, run `make drop-$DATABASE_NAME`. **Use this with caution!!!** If you did not backup your database, **everything will be lost afterwards.** And no, there is no addional promt "Are you sure, that you want to do this?". If you do this, you *actually* do this. +* If you want to delete your database, run `make drop-$DATABASE_NAME`. **Use + this with caution!!!** If you did not backup your database, **everything will + be lost afterwards.** And no, there is *no* additional prompt to ask if you + are sure. If you `make drop-...`, you *actually* delete the database. diff --git a/config.defaults b/config.defaults index 088143f8b2e87365f3560964a0619daf44c579e9..625d47d57f32a74d1bb42039efbeb4102da7e3c7 100644 --- a/config.defaults +++ b/config.defaults @@ -28,12 +28,22 @@ MYSQLDUMP_CMD=$(command -v mysqldump) # The mysqldump program which comes with t MYSQL_CONFIG_EDITOR_CMD=$(command -v mysql_config_editor) # The mysql_config_editor program which is used to store the credentials. # MySQL Connection -MYSQL_HOST=localhost #The host of the MySQL server. -MYSQL_USER=root # The user for the installation. Note: This is not the user which will then be used by the CaosDB Server. +MYSQL_HOST=localhost # The host of the MySQL server. +MYSQL_PORT=3306 # The port number of the MySQL server. +MYSQL_USER=root # The user for the installation. Note: This is not the user + # which will then be used by the CaosDB Server. # DATABASE -DATABASE_NAME=caosdb #The name of the database. -DATABASE_USER=caosdb_user #The user which is used by the CaosDB Server for the connection. -DATABASE_USER_HOST_LIST=localhost, # A comma-separated list of hosts from which MySQL will accept logins. This option follows the MySQL-style for host-names and ip addresses. E.g. `%` is a wildcard for all hosts, `192.168.0.%` permits logins from the local network etc. +DATABASE_NAME=caosdb # The name of the SQL database. +DATABASE_USER=caosdb # The user which is used by the CaosDB Server for accessing + # the database. +DATABASE_USER_PW=random1234 # The password for the database user. Leave empty + # if you want to be prompted at make time. + +# A comma-separated list of hosts from which MySQL will accept logins. This +# option follows the MySQL-style for host-names and ip addresses. E.g. `%` is a +# wildcard for all hosts, so `192.168.0.%` permits logins from the local +# network. +DATABASE_USER_HOST_LIST=localhost, diff --git a/make_db b/make_db new file mode 100755 index 0000000000000000000000000000000000000000..41bf13832d4e58a7127630f52a3f460a4056eabf --- /dev/null +++ b/make_db @@ -0,0 +1,143 @@ +#!/bin/bash + +# ** header v3.0 +# This file is a part of the CaosDB Project. +# +# Copyright (C) 2019 Daniel Hornung, Göttingen +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see <https://www.gnu.org/licenses/>. +# +# ** end header + +# To fail fast, but beware https://mywiki.wooledge.org/BashFAQ/105 +set -e + +function fail() { + echo "Some error occured, exiting." + exit 1 +} + +function setup_os() { + # - mariadb-client :: For SQL server configuration. + PACKAGES="git + make + mariadb-client + maven + openjdk-8-jdk-headless + python3-pip + screen + unzip +" + # - vim :: For debugging + PACKAGES+="vim +" + apt-get update + apt-get dist-upgrade -y + apt-get install -y $PACKAGES +} + +function install_db() { + if _db_exists ; then + echo -e "\n +A database with with the name '$DATABASE_NAME' exists already. +Call 'make drop-$DATABASE_NAME' to delete that database or reconfigure with +'./configure'." + exit 0 + fi + + sed "s/db_2_0/$DATABASE_NAME/g" "$INSTALL_SQL_FILE" | $SQL +} + +function test-connection() { + $SQL -e "select 0;" +} + +# Creates a user and grants it sufficient rights. +# If called with --strict, the user is not allowed to exist previously. +function grant() { + if [ -z "$DATABASE_USER_PW" ] ; then + while true; do + read -p "Please enter the password for ${DATABASE_USER}: " \ + -s password + echo + read -p "Please repeat the password: " -s password2 + echo + [[ "$password" == "$password2" ]] && { + DATABASE_USER_PW="$password" + break + } + echo -e "\nThe passwords didn't match. Try again." + done + fi + + if [[ "$1" == "--strict" ]] ; then + for host in ${DATABASE_USER_HOST_LIST//,/ } ; do + CMD="SELECT COUNT(*) FROM mysql.user WHERE user='${DATABASE_USER}' AND host='${host}';" + [[ $SQL -s -N -e "$CMD" == 0 ]] || { + echo "The user '${DATABASE_USER}@${host}' is already in the database." + echo "Please use another user or delete it, e.g. with" + echo "'mysql -u ${MYSQL_USER} -p -e \"DROP USER ${DATABASE_USER}@${host};\"'" + exit 1 + } + done + fi + + for host in ${DATABASE_USER_HOST_LIST//,/ } ; do + $SQL <<EOF +CREATE USER IF NOT EXISTS + '$DATABASE_USER'@'$host' identified by '$DATABASE_USER_PW'; +GRANT USAGE ON *.* TO '$DATABASE_USER'@'$host'; +GRANT ALL PRIVILEGES ON *.* TO '$DATABASE_USER'@'$host' WITH GRANT OPTION; +GRANT EXECUTE ON *.* TO '$DATABASE_USER'@'$host'; +EOF + done +} + +# Drops the caosdb user plus a given database from all listed database hosts. +# The first argument to this function is the database that shal be dropped. +function drop() { + DROPDB="$1" + for host in ${DATABASE_USER_HOST_LIST//,/ } ; do + $SQL -e "DROP USER '${DATABASE_USER}'@'${host}';" || true + done + "$MYSQLADMIN_CMD" $MYSQL_CONNECTION -f drop "$DROPDB" +} + +# Returns 0 or non-zero, depending on whether the database exists already. +function _db_exists() { + $SQL -D "$DATABASE_NAME" -e "show tables;" > /dev/null 2>&1 \ + && return 0 || return 1 +} + +. .config + +INSTALL_SQL_FILE="db_2_0.sql" +if [ -z $LOGIN_PATH ] ; then + MYSQL_CONNECTION="--host=${MYSQL_HOST} --port=${MYSQL_PORT} + --user=${MYSQL_USER} + --password=${MYSQL_USER_PASSWORD}" +else + MYSQL_CONNECTION="--login-path=$LOGIN_PATH" +fi + +SQL="$MYSQL_CMD $MYSQL_CONNECTION" + +case $1 in + "drop") drop $2 ;; + "grant") grant $2;; + "test-connection") test-connection ;; + "install_db") install_db ;; + # "prep_sql") prepare_sql ;; + *) echo "Unknown action: $1" +esac diff --git a/makefile b/makefile index a505f77b5bfe597a040d5a210fcf0b4a02369177..98d20a6cff558391e344a81c3cac9bea52c741f7 100644 --- a/makefile +++ b/makefile @@ -23,66 +23,35 @@ SHELL=/bin/bash include .config -M_MYSQL_USER=$(shell echo $(MYSQL_USER)) -M_MYSQL_USER_PASSWORD=$(shell echo $(MYSQL_USER_PASSWORD)) -M_LOGIN_PATH=$(shell echo $(LOGIN_PATH)) -M_DATABASE_NAME=$(shell echo $(DATABASE_NAME)) -M_DATABASE_USER_HOST_LIST=$(shell echo $(DATABASE_USER_HOST_LIST)) -M_DATABASE_USER=$(shell echo $(DATABASE_USER)) -M_MYSQL_CMD=$(shell echo $(MYSQL_CMD)) -M_MYSQLDUMP_CMD=$(shell echo $(MYSQLDUMP_CMD)) -M_MYSQLADMIN_CMD=$(shell echo $(MYSQLADMIN_CMD)) INSTALL_SQL_FILE=db_2_0.sql ifdef LOGIN_PATH M_MYSQL_CONNECTION=--login-path=$(M_LOGIN_PATH) else - M_MYSQL_CONNECTION=--user="$(M_MYSQL_USER)" --password="$(M_MYSQL_USER_PASSWORD)" + MYSQL_CONNECTION=--host="$(MYSQL_HOST)" --port="$(MYSQL_PORT)" \ + --user="$(MYSQL_USER)" --password="$(MYSQL_USER_PASSWORD)" endif .PHONY: test-connection test-connection: - $(M_MYSQL_CMD) $(M_MYSQL_CONNECTION) -e "select 0;" + ./make_db test-connection .PHONY: upgrade upgrade: - @cd patches; ./applyPatches.sh $(M_MYSQL_CONNECTION) --database=$(M_DATABASE_NAME) + @cd patches; ./applyPatches.sh --env=../.config .PHONY: install install: _install _grant upgrade .PHONY: _install _install: - @if $(MAKE) _exists > /dev/null 2>&1; then \ - printf "\n\nA database with with the name \"$(M_DATABASE_NAME)\" does already exist.\nCall 'make drop-$(M_DATABASE_NAME)' to delete that database or reconfigure with './configure'.\n"; \ - exit 1; \ - else sed 's/db_2_0/$(M_DATABASE_NAME)/g' $(INSTALL_SQL_FILE) | $(M_MYSQL_CMD) $(M_MYSQL_CONNECTION); fi + ./make_db install_db -comma:=, .PHONY: _grant _grant: - @for host in $(subst $(comma), ,$(M_DATABASE_USER_HOST_LIST)); do \ - ( $(M_MYSQL_CMD) $(M_MYSQL_CONNECTION) -e "SELECT COUNT(*) FROM mysql.user WHERE user='$(M_DATABASE_USER)' AND host=\"$$host\";" | grep 0 ) || ( echo "The user '$(M_DATABASE_USER)@$$host' is already in the database. Please use another user or delete her. E.g. with 'mysql -u $(M_MYSQL_USER) -p -e \"DROP USER $(M_DATABASE_USER)@$$host;\"'" ; exit 1 ) ; \ - done - @while true; do \ - printf "\n Please enter the password for $(M_DATABASE_USER): "; read -s password; \ - printf "\nPlease repeat the password for $(M_DATABASE_USER): "; read -s password2; \ - if [ "$$password" != "$$password2" ]; then printf "\n\nThe passwords didn't match. Try again." ; else break ; fi; \ - done ; \ - echo "" ; \ - for host in $(subst $(comma), ,$(M_DATABASE_USER_HOST_LIST)); do \ - $(M_MYSQL_CMD) $(M_MYSQL_CONNECTION) -e "CREATE USER IF NOT EXISTS '$(M_DATABASE_USER)'@'$$host' identified by '$$password';" ; \ - $(M_MYSQL_CMD) $(M_MYSQL_CONNECTION) -e "GRANT USAGE ON *.* TO '$(M_DATABASE_USER)'@'$$host';"; \ - $(M_MYSQL_CMD) $(M_MYSQL_CONNECTION) -e "GRANT ALL PRIVILEGES ON *.* TO '$(M_DATABASE_USER)'@'$$host' WITH GRANT OPTION;"; \ - $(M_MYSQL_CMD) $(M_MYSQL_CONNECTION) -e "GRANT EXECUTE ON *.* TO '$(M_DATABASE_USER)'@'$$host';"; \ - done + ./make_db grant +# Drop the user and a given database .PHONY: drop-% drop-%: - @$(M_MYSQLADMIN_CMD) $(M_MYSQL_CONNECTION) -f drop $(patsubst drop-%,%,$@) - -.PHONY: _exists -_exists: - @if $(M_MYSQL_CMD) $(M_MYSQL_CONNECTION) -D "$(M_DATABASE_NAME)" -e "show tables;" > /dev/null 2>&1 ; then \ - echo "$(M_DATABASE_NAME) does exist." ; exit 0 ; \ - else echo "$(M_DATABASE_NAME) does not exist." ; exit 1; fi + ./make_db drop $(patsubst drop-%,%,$@) diff --git a/patches/applyPatches.sh b/patches/applyPatches.sh index a9928c3772c84ace90871b0a5a3395cbcbd4ffa0..76db457f0c5551351b4da4915002ac452d4eb2a4 100755 --- a/patches/applyPatches.sh +++ b/patches/applyPatches.sh @@ -30,8 +30,8 @@ export UTILSPATH="./utils" for p in $PATCHES do - $p $MYSQL_CONNECTION --database=$DATABASE_NAME --patch=$p + $p "$@" --patch=$p done cd ../ -./update_sql_procedures.sh $MYSQL_CONNECTION --database=$DATABASE_NAME +./update_sql_procedures.sh diff --git a/patches/utils/patch_header.sh b/patches/utils/patch_header.sh index 316f5b2b00087702ac5e22cacc7f11252dfc3fb9..7cb477b93f635679f8fdf529623fa36663e49eaa 100644 --- a/patches/utils/patch_header.sh +++ b/patches/utils/patch_header.sh @@ -4,6 +4,7 @@ # # Copyright (C) 2018 Research Group Biomedical Physics, # Max-Planck-Institute for Dynamics and Self-Organization Göttingen +# Copyright 2019 Daniel Hornung # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License as @@ -21,121 +22,110 @@ # ** end header # #header for patch scripts -if test -f ../.config ; then - . ../.config -fi -if test -f .config ; then - . .config -fi -CMD_OPTIONS="options:\n\n-h, --help\n\tShow brief help.\n-l LOGIN_PATH, --login-path=LOGIN_PATH\n\tA login-path for the mysql connection (see 'man mysql' and 'man mysql-config-editor').\n-u USER, --user=USER\n\tA mysql user.\n-p PASSWORD, --password=PASSWORD\n\tThe password for the mysql user.\n" +CMD_MYSQL=mysql +CMD_MYSQL_DUMP=mysqldump +USAGE="$1 [ --env=ENV_FILE ] [ --patch=PATCH ] [ --backupdir=BACKUPDIR ]\n\n" +CMD_OPTIONS=cat <<EOF +options: + +-h, --help + Show brief help. +--env=ENV_FILE + A file where variables are stored in the format of shell environment + variables. Content of this file overrides environment variables already + present. +--patch=PATCH + *TODO* Please document this option. +--backupdir=BACKUPDIR + *TODO* Please document this option. + +This script basically uses the same environment variables as the server +configuration make file. Notable examples are: + +- LOGIN_PATH +- MYSQL_HOST +- MYSQL_PORT +- DATABASE_NAME +- DATABASE_USER +- DATABASE_USER_PW + +EOF set -e function _print_help() { - printf -- "$USAGE" - printf -- "$CMD_OPTIONS" + echo -e "$USAGE" + echo -e "$CMD_OPTIONS" if [ -n "$PRINT_HELP" ]; then - printf -- "$PRINT_HELP" + echo -e "$PRINT_HELP" fi if [ -n "$1" ]; then - printf -- "$1" + echo -e "$1" fi } + while test $# -gt 0; do case "$1" in -h|--help) _print_help exit 0 ;; - -p) - shift - if test $# -gt 0; then - MYSQL_USER_PASSWORD=$1 - else - echo "no database specified" - exit 1 - fi - shift - ;; - --password*) - MYSQL_USER_PASSWORD=`echo $1 | sed -e 's/^[^=]*=//g'` - shift - ;; - -u) - shift - if test $# -gt 0; then - MYSQL_USER=$1 - else - echo "no database specified" - exit 1 - fi - shift - ;; - --user*) - MYSQL_USER=`echo $1 | sed -e 's/^[^=]*=//g'` - shift - ;; - -l) - shift - if test $# -gt 0; then - LOGIN_PATH=$1 - else - echo "no database specified" - exit 1 - fi - shift - ;; - --login-path*) - LOGIN_PATH=`echo $1 | sed -e 's/^[^=]*=//g'` - shift - ;; - -d) - shift - if test $# -gt 0; then - DATABASE_NAME=$1 - else - echo "no database specified" - exit 1 - fi - shift - ;; - --database*) - DATABASE_NAME=`echo $1 | sed -e 's/^[^=]*=//g'` + --env*) + ENV_FILE="${1#--*=}" shift ;; --patch*) - PATCH=`echo $1 | sed -e 's/^[^=]*=//g'` + PATCH="${1#--*=}" shift ;; --backupdir*) - BACKUPDIR=`echo $1 | sed -e 's/^[^=]*=//g'` + BACKUPDIR="${1#--*=}" + echo "Warning: BACKUPDIR is not used currently." shift ;; *) + echo "Unknown option $1" + exit 1 shift ;; esac done -if [ -z "$DATABASE_NAME" ] +[[ -n "$ENV_FILE" ]] && source "$ENV_FILE" + +if [[ -z "$DATABASE_NAME" && -z "$MYSQL_CONNECTION" ]] then _print_help "Please specify the database." exit 1 fi -if [ "$LOGIN_PATH" ] -then +if [ "$LOGIN_PATH" ]; then MYSQL_CONNECTION="--login-path=$LOGIN_PATH" -else + MYSQL_CONNECTION_NO_DB="$MYSQL_CONNECTION" + MYSQL_CONNECTION="$MYSQL_CONNECTION --database=$DATABASE_NAME" + +elif [[ -z "$MYSQL_CONNECTION" ]]; then MYSQL_CONNECTION="" - if [ "$MYSQL_USER" ] + if [ "$DATABASE_USER" ] then - MYSQL_CONNECTION="--user=$MYSQL_USER" + MYSQL_CONNECTION="--user=$DATABASE_USER" fi - if [ "$MYSQL_USER_PASSWORD" ] + if [ "$DATABASE_USER_PW" ] then - MYSQL_CONNECTION="$MYSQL_CONNECTION --password=$MYSQL_USER_PASSWORD" + MYSQL_CONNECTION="$MYSQL_CONNECTION --password=$DATABASE_USER_PW" fi + if [[ "$MYSQL_HOST" && ( "$MYSQL_HOST" != "localhost" ) ]]; then + MYSQL_CONNECTION="$MYSQL_CONNECTION --host=$MYSQL_HOST" + if [ "$MYSQL_PORT" ]; then + MYSQL_CONNECTION="$MYSQL_CONNECTION --port=$MYSQL_PORT" + fi + fi + # This option should come last, so we also have one version without the database + MYSQL_CONNECTION_NO_DB="$MYSQL_CONNECTION" + MYSQL_CONNECTION="$MYSQL_CONNECTION --database=$DATABASE_NAME" fi +export MYSQL_CONNECTION +export MYSQL_CONNECTION_NO_DB +export DATABASE_NAME if [ -n "$PATCH" ]; then echo -ne "applying patch $PATCH to $DATABASE_NAME ... " @@ -158,10 +148,10 @@ function uptodate { # @param $1: db version string, e.g. v2.0.0 # @return: 0 on success, 1 on failure function check_version { - local version=$($MYSQL_CMD $MYSQL_CONNECTION -D $DATABASE_NAME -B -e "Select CaosDBVersion();") - if [[ "$(echo $version | sed 's/^CaosDBVersion()\s//')" = "$1" ]]; then + local version=$($CMD_MYSQL $MYSQL_CONNECTION -B -e "Select CaosDBVersion();") + if [[ "$(echo $version | sed 's/^CaosDBVersion()\s//')" = "$1" ]]; then return 0 - fi + fi uptodate } @@ -171,13 +161,14 @@ function update_version { } function dump_table { - $MYSQLDUMP_CMD $MYSQL_CONNECTION $DATABASE_NAME $1 > ${DATABASE_NAME}.${1}.${OLD_VERSION}.dump.sql + $CMD_MYSQL_DUMP $MYSQL_CONNECTION_NO_DB $DATABASE_NAME $1 \ + > ${DATABASE_NAME}.${1}.${OLD_VERSION}.dump.sql } function mysql_execute { set +e - $MYSQL_CMD $MYSQL_CONNECTION -D $DATABASE_NAME -e "$1" + $CMD_MYSQL $MYSQL_CONNECTION -e "$1" ret=${PIPESTATUS[0]} if [ "$ret" -ne 0 ]; then failure "MYSQL ERROR" @@ -186,6 +177,6 @@ function mysql_execute { } function redo_table { - $MYSQL_CMD $MYSQL_CONNECTION -D $DATABASE_NAME < ${DATABASE_NAME}.${1}.${OLD_VERSION}.dump.sql + $CMD_MYSQL $MYSQL_CONNECTION < ${DATABASE_NAME}.${1}.${OLD_VERSION}.dump.sql }