diff --git a/config.defaults b/config.defaults index bf6c941b3251f24e0a0aaf1e7667d11e1ef88638..f12278dcf0f5ce772c4cb77e54b149d3f6880921 100644 --- a/config.defaults +++ b/config.defaults @@ -39,6 +39,7 @@ MYSQL_PORT=3306 # The user for the installation. Note: This is not the user # which will then be used by the CaosDB Server. MYSQL_USER=root +MYSQL_USER_PASSWORD=caosdb1234 # # DATABASE # The name of the SQL database. diff --git a/configure b/configure index 9c6e7544472870152afc3991b4a307dfc3b03420..fa76621e5d042c487ef003aee9b902e35d72ed04 100755 --- a/configure +++ b/configure @@ -22,14 +22,21 @@ # ** end header # +echo "CURRENTLY NOT SUPPORTET" +exit 1 + +if [ -z "$UTILSPATH" ]; then + UTILSPATH="$(realpath $(dirname $0)/utils)" + export UTILSPATH +fi # defaults source config.defaults OPTIONS=$(sed -r '/^(#|\s*$)/d' config.defaults | sed -r 's/=.*$//g') # override with old config -if [ -e .config ]; then - source .config +if [ -e $MAINPATH/.config ]; then + source $MAINPATH/.config fi # DEBUG - print all options: @@ -77,12 +84,12 @@ fi # DEBUG - print all options: # for opt in $OPTIONS; do echo $opt = ${!opt}; done -# write to .config -rm -f .config +# write to $MAINPATH/.config +rm -f $MAINPATH/.config for opt in $OPTIONS; do esc_hash=${!opt//#/\\\#} esc_dollar=${esc_hash//\$/'$$'} esc_dq=${esc_dollar//\"/\\\"} esc_sq=${esc_dq//\'/\\\'} - echo "$opt=\"$esc_sq\"" >> .config + echo "$opt=\"$esc_sq\"" >> $MAINPATH/.config done diff --git a/utils/backup.py b/utils/backup.py new file mode 100755 index 0000000000000000000000000000000000000000..fe0ea8112ee270c880e9ae3101666519f23e7389 --- /dev/null +++ b/utils/backup.py @@ -0,0 +1,71 @@ +#!/usr/bin/env python3 +# ** header v3.0 +# This file is a part of the CaosDB Project. +# +# Copyright (C) 2020 Daniel Hornung (d.hornung@indiscale.com) +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see <https://www.gnu.org/licenses/>. +# +# ** end header + +"""A thin wrapper around the sql backup bash script. + +It provides a meaningful default directory and a nice help message +""" + +import argparse +import os +import subprocess +import tempfile + + +def create_dump(directory=None): + """Create the SQL dump + +Parameters : +------------------------------ +dictionary : str + If not None, the place to write the SQL dump into. + """ + + utils_path = os.path.dirname(__file__) + command = [os.path.join(utils_path, "backup.sh")] + env = os.environ.copy() + + if not directory: + os.makedirs("/tmp/caosdb/tmpfiles/backup/", exist_ok=True) + directory = tempfile.mkdtemp(dir="/tmp/caosdb/tmpfiles/backup/", + prefix="backup.") + env["BACKUPDIR"] = directory + subprocess.run(command, cwd=os.path.dirname(__file__), env=env) + + +def parse_args(): + """Parse the command line arguments.""" + parser = argparse.ArgumentParser( + description=__doc__, + prog="mysql backup", + formatter_class=argparse.RawDescriptionHelpFormatter + ) + parser.add_argument( + '-d', "--directory", + help='Directory to which the sql dump will be saved.') + + return parser.parse_args() + + +if __name__ == "__main__": + """The main function.""" + args = parse_args() + create_dump(args.directory) diff --git a/utils/backup.sh b/utils/backup.sh index 56ca60907bd7a4c4c73db7815efe168374bc37b6..4fc133c6eedd6a222f50eaaef78c874e4102704b 100755 --- a/utils/backup.sh +++ b/utils/backup.sh @@ -30,40 +30,11 @@ if [ -z "$UTILSPATH" ]; then UTILSPATH="$(realpath $(dirname $0))" export UTILSPATH fi + # The directory which the dump is to be stored to. Do not change it here. Use -# the --backupdir=./my/dir/ option or an environment variable instead. +# the the environment variable instead. BACKUPDIR="${BACKUPDIR:-../backup}" -# Iff this (environment) variable is "true" as lower-case, then this script also -# outputs a machine readable status report. The variable can also be set by -# giving the `--yaml` command line option. -YAML="${YAML:-false}" - -PRINT_HELP="--backupdir=BACKUPDIR\n\tThe directory which the dump is to be stored to. (Defaults to ./backup,\n\tcan also be set as an environment variable.)\n" - -# YAML string templates ####################################################### - -YAML_FAIL=$(cat << "EOF" -#### YAML #### -error: - code: $error_code - message: $error_message -#### YAML END #### -EOF - ) - -YAML_SUCCESS=$(cat << "EOF" -#### YAML #### -error: - code: 0 - message: Backup successfully created -backup: - location: $file_location - date: $date -#### YAML END #### -EOF - ) - # Load settings from .config and defaults ##################################### . $UTILSPATH/load_settings.sh @@ -71,74 +42,24 @@ EOF . $UTILSPATH/helpers.sh function backup() { - NARG_NAME="$1" - NARG_FILE="$2" + database="$1" + backupdir="$2" - shift 2 - # parameters: connection, database, outfile - if [ -e "$NARG_FILE" ]; then + # Assert backup dir + mkdir -p "$backupdir" + + datastring=$(date -u --rfc-3339=ns | sed 's/ /T/g') + backupfile=${BACKUPDIR}/${DATABASE_NAME}.${datastring}.dump.sql + + if [ -e "$backupfile" ]; then failure "dumpfile already exists." fi - echo "Dumping database $NARG_NAME to $NARG_FILE ... " - $MYSQLDUMP_CMD $(get_mysql_args_nodb) $* --opt --default-character-set=utf8 --routines \ - "$NARG_NAME" > "$NARG_FILE" - if [[ "$YAML" == "true" ]] ; then - yaml_success "$NARG_FILE" "$DATE" - fi - success -} + echo "Dumping database $database to $backupfile ... " + $MYSQLDUMP_CMD $(get_mysql_args_nodb) --opt --default-character-set=utf8\ + --routines "$database" > "$backupfile" -# Print a YAML failure message, if YAML is true. -# -# Arguments: -# - $1 Error code, integer > 0 -# - $2 Error message -function yaml_fail() { - err_code="$1" - err_msg="$2" - local yaml="${YAML_FAIL/\$err_code/$err_code}" - yaml="${yaml/\$err_msg/$err_msg}" - echo "$yaml" -} - -# Print a YAML success message, if YAML is true. -# -# Arguments: -# - $1 The file location of the backup file. -# - $2 The current date & time -function yaml_success() { - file_location="$1" - date="$2" - local yaml="${YAML_SUCCESS/\$file_location/$file_location}" - yaml="${yaml/\$date/$date}" - echo "$yaml" + success } -## test if dump file exists -#touch dumpfile.tmp -#backup 0 1 2 "dumpfile.tmp" -#rm dumpfile.tmp - -# Assert backup dir -mkdir -p "$BACKUPDIR" - -DATE=$(date -u --rfc-3339=ns | sed 's/ /T/g') -BACKUPFILE=${BACKUPDIR}/${DATABASE_NAME}.${DATE}.dump.sql - -# echo "date: $date" >&2 -# echo "file_location: $file_location" >&2 - -#YAML_SUCCESS=$(cat <<"EOF" -read -r -d '' YAML_SUCCESS <<"EOF" || true -#### YAML #### -error: - code: 0 - message: Backup successfully created -backup: - location: $file_location - date: $date -#### YAML END #### -EOF - -backup $DATABASE_NAME $BACKUPFILE +backup $DATABASE_NAME $BACKUPDIR diff --git a/utils/helpers.sh b/utils/helpers.sh index 37914402380f5b6ef9a6d192d5021e8a8774db43..cec709d4f8705357110ab308c3c9658293f0c494 100644 --- a/utils/helpers.sh +++ b/utils/helpers.sh @@ -1,7 +1,7 @@ # This file is a part of the CaosDB Project. # # Copyright (C) 2018 Research Group Biomedical Physics, -# Max-Planck-Institute for Dynamics and Self-Organization Göttingen +# Max-Planck-Institute for Dynamics and Self-Organization Göttingen # Copyright (C) 2019, 2020 Daniel Hornung (d.hornung@indiscale.com) # Copyright (C) 2020 Henrik tom Wörden # Copyright (C) 2020 IndiScale GmbH <info@indiscale.com> diff --git a/utils/load_settings.sh b/utils/load_settings.sh index 73d63364dd582dbc514ac3c2d95a37346b5b0eed..39cc7b238a41e7940fa5576ca400961191428b06 100644 --- a/utils/load_settings.sh +++ b/utils/load_settings.sh @@ -34,7 +34,7 @@ if [ -z "$MAINPATH" ]; then fi # Make a backup of the important environment variables. -my_env=$(export -p | grep -E '(MYSQL|DATABASE)') +my_env=$(export -p | grep -E '(MYSQL|DATABASE)' || true) source "$MAINPATH/config.defaults" diff --git a/utils/log.py b/utils/log.py new file mode 100755 index 0000000000000000000000000000000000000000..cfc9f0d8684a975c1edc7f753561c2b365239e01 --- /dev/null +++ b/utils/log.py @@ -0,0 +1,126 @@ +#!/usr/bin/env python3 + +# ** header v3.0 +# This file is a part of the CaosDB Project. +# +# Copyright (C) 2019 Daniel Hornung, IndiScale GmbH +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see <https://www.gnu.org/licenses/>. +# +# ** end header + +"""Remote module for SQL logging + +Basically a wrapper with nice arguments around the bash script log.sh +""" + +import argparse +import os +import subprocess +import sys + +UTILS_PATH = os.path.abspath(os.path.dirname(__file__)) +COMMAND = [os.path.join(UTILS_PATH, "log.sh")] + + +def _store_log(): + """Retrieves the log table and returns it as a TSV string. + + Returns + ------- + out : str + The string representing the logging information, as TSV. + """ + result = subprocess.run(COMMAND+["get"], encoding="utf8", + stdout=subprocess.PIPE, stderr=subprocess.PIPE) + # capture_output=True) ## Only works since 3.7 + + if result.returncode > 0: + print("Error during SQL command, while retrieving SQL logs:", file=sys.stderr) + print(result.stderr, file=sys.stderr) + print(result.stdout) + + +def _logging(turn_on): + """Turn SQL logging off/on. + +This function tells the SQL database to turn logging off or on (depending on the +arguments). It returns nothing. + +Parameters +---------- +turn_on : bool + Whether to turn logging on or off. + """ + result = subprocess.run(COMMAND+["start" if turn_on else "stop"], + encoding="utf8", + stdout=subprocess.PIPE, stderr=subprocess.PIPE) + # capture_output=True) ## Only works since 3.7 + + if result.returncode > 0: + print("Error during SQL command, while setting SQL logging to '{on_off}':".format( + on_off=on_off), file=sys.stderr) + print(result.stderr, file=sys.stderr) + print(result.stdout) + + # Return nothing if no error. + + +def start_logging(args): + _logging(True) + + +def stop_logging(args): + _logging(False) + + +def store(args): + _store_log() + + +def parse_args(): + """Parse the command line arguments.""" + parser = argparse.ArgumentParser( + description=__doc__+""" +To view additional information about subcommands, execute: + linkahead <subcmd> -h""", + prog="mysqllog", + formatter_class=argparse.RawDescriptionHelpFormatter + ) + subparsers = parser.add_subparsers( + help='The action to take.', dest="action", required=True) + + # mysql logging + start_parser = subparsers.add_parser( + 'on', + help='Turns SQL logging on.') + start_parser.set_defaults(func=start_logging) + + stop_parser = subparsers.add_parser( + 'off', + help='Turns SQL logging off.') + stop_parser.set_defaults(func=stop_logging) + + stop_parser = subparsers.add_parser( + 'store', + help='Prints SQL logs.') + stop_parser.set_defaults(func=store) + + return parser.parse_args() + + +if __name__ == "__main__": + """The main function.""" + args = parse_args() + args.func(args) diff --git a/utils/log.sh b/utils/log.sh new file mode 100755 index 0000000000000000000000000000000000000000..ad8d508bd4a474d89f0c41f350ad770f9b4cf89b --- /dev/null +++ b/utils/log.sh @@ -0,0 +1,57 @@ +#!/bin/bash +# +# ** header v3.0 +# This file is a part of the CaosDB Project. +# +# Copyright (C) 2018 Research Group Biomedical Physics, +# Max-Planck-Institute for Dynamics and Self-Organization Göttingen +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see <https://www.gnu.org/licenses/>. +# +# ** end header +# + +#start and stop logging; get current logs + +if [ -z "$UTILSPATH" ]; then + UTILSPATH="$(realpath $(dirname $0))" + export UTILSPATH + MAINPATH="$(dirname $UTILSPATH)" + export MAINPATH +fi + +# Load settings from .config and defaults ##################################### +. $UTILSPATH/load_settings.sh + +# load useful functions ####################################################### +. $UTILSPATH/helpers.sh + +function get_logs { + echo 'select event_time, argument from mysql.general_log where command_type="Query";' | \ + $MYSQL_CMD $(get_db_args_nodb) --batch +} + +function set_logging { + echo "Setting logging $1..." + echo "SET GLOBAL log_output = 'TABLE'; SET GLOBAL general_log = '$1';" | \ + $MYSQL_CMD $(get_db_args_nodb) --batch + success +} + +case $1 in + "start") set_logging "on" ;; + "stop") set_logging "off" ;; + "get") get_logs ;; + *) echo "Unknown argument: $1"; exit 1 +esac diff --git a/utils/make_db b/utils/make_db index 6af1a36339fe6ddd9d06858f5ca19913d1b9c73a..3ad93111287a4b87303f5c7b319f7da4afd5e282 100755 --- a/utils/make_db +++ b/utils/make_db @@ -82,11 +82,11 @@ function _install_unit_test_database () { # create test user grant - # update to latest - cp .config .test_config echo "DATABASE_NAME=\"$UNITTEST_DATABASE\"" >> .test_config pushd patches > /dev/null - ./applyPatches.sh --env=../.test_config + export ENV_FILE=$(realpath ../.test_config) + ./applyPatches.sh + unset ENV_FILE popd > /dev/null rm .test_config }