Merge branch 'f-new-tools' into 's-clean-up-stage'

F new tools

See merge request caosdb/caosdb-mysqlbackend!9

config.defaults

@@ -39,6 +39,7 @@ MYSQL_PORT=3306
 # The user for the installation. Note: This is not the user
 # which will then be used by the CaosDB Server.
 MYSQL_USER=root
+MYSQL_USER_PASSWORD=caosdb1234
 
 # # DATABASE
 # The name of the SQL database.

configure

@@ -22,14 +22,21 @@
 # ** end header
 #
 
+echo "CURRENTLY NOT SUPPORTET"
+exit 1
+
+if [ -z "$UTILSPATH" ]; then
+    UTILSPATH="$(realpath $(dirname $0)/utils)"
+    export UTILSPATH
+fi
 
 # defaults
 source config.defaults
 OPTIONS=$(sed -r '/^(#|\s*$)/d' config.defaults | sed -r 's/=.*$//g')
 
 # override with old config
-if [ -e .config ]; then
-    source .config
+if [ -e $MAINPATH/.config ]; then
+    source $MAINPATH/.config
 fi
 
 # DEBUG - print all options:
@@ -77,12 +84,12 @@ fi
 # DEBUG - print all options:
 # for opt in $OPTIONS; do echo $opt = ${!opt}; done
 
-# write to .config
-rm -f .config
+# write to $MAINPATH/.config
+rm -f $MAINPATH/.config
 for opt in $OPTIONS; do
     esc_hash=${!opt//#/\\\#}
     esc_dollar=${esc_hash//\$/'$$'}
     esc_dq=${esc_dollar//\"/\\\"}
     esc_sq=${esc_dq//\'/\\\'}
-    echo "$opt=\"$esc_sq\"" >> .config
+    echo "$opt=\"$esc_sq\"" >> $MAINPATH/.config
 done

utils/backup.py

+#!/usr/bin/env python3
+# ** header v3.0
+# This file is a part of the CaosDB Project.
+#
+# Copyright (C) 2020 Daniel Hornung (d.hornung@indiscale.com)
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see <https://www.gnu.org/licenses/>.
+#
+# ** end header
+
+"""A thin wrapper around the sql backup bash script.
+
+It provides a meaningful default directory and a nice help message
+"""
+
+import argparse
+import os
+import subprocess
+import tempfile
+
+
+def create_dump(directory=None):
+    """Create the SQL dump
+
+Parameters :
+------------------------------
+dictionary : str
+    If not None, the place to write the SQL dump into.
+    """
+
+    utils_path = os.path.dirname(__file__)
+    command = [os.path.join(utils_path, "backup.sh")]
+    env = os.environ.copy()
+
+    if not directory:
+        os.makedirs("/tmp/caosdb/tmpfiles/backup/", exist_ok=True)
+        directory = tempfile.mkdtemp(dir="/tmp/caosdb/tmpfiles/backup/",
+                                     prefix="backup.")
+    env["BACKUPDIR"] = directory
+    subprocess.run(command, cwd=os.path.dirname(__file__), env=env)
+
+
+def parse_args():
+    """Parse the command line arguments."""
+    parser = argparse.ArgumentParser(
+        description=__doc__,
+        prog="mysql backup",
+        formatter_class=argparse.RawDescriptionHelpFormatter
+    )
+    parser.add_argument(
+        '-d', "--directory",
+        help='Directory to which the sql dump will be saved.')
+
+    return parser.parse_args()
+
+
+if __name__ == "__main__":
+    """The main function."""
+    args = parse_args()
+    create_dump(args.directory)

utils/backup.sh

@@ -30,40 +30,11 @@ if [ -z "$UTILSPATH" ]; then
     UTILSPATH="$(realpath $(dirname $0))"
     export UTILSPATH
 fi
+
 # The directory which the dump is to be stored to. Do not change it here. Use
-# the --backupdir=./my/dir/ option or an environment variable instead.
+# the the environment variable instead.
 BACKUPDIR="${BACKUPDIR:-../backup}"
 
-# Iff this (environment) variable is "true" as lower-case, then this script also
-# outputs a machine readable status report.  The variable can also be set by
-# giving the `--yaml` command line option.
-YAML="${YAML:-false}"
-
-PRINT_HELP="--backupdir=BACKUPDIR\n\tThe directory which the dump is to be stored to. (Defaults to ./backup,\n\tcan also be set as an environment variable.)\n"
-
-# YAML string templates #######################################################
-
-YAML_FAIL=$(cat << "EOF"
-#### YAML ####
-error:
-  code: $error_code
-  message: $error_message
-#### YAML END ####
-EOF
-        )
-
-YAML_SUCCESS=$(cat << "EOF"
-#### YAML ####
-error:
-  code: 0
-  message: Backup successfully created
-backup:
-  location: $file_location
-  date: $date
-#### YAML END ####
-EOF
-        )
-
 # Load settings from .config and defaults #####################################
 . $UTILSPATH/load_settings.sh
 
@@ -71,74 +42,24 @@ EOF
 . $UTILSPATH/helpers.sh
 
 function backup() {
-    NARG_NAME="$1"
-    NARG_FILE="$2"
+    database="$1"
+    backupdir="$2"
 
-    shift 2
-    # parameters: connection, database, outfile
-    if [ -e "$NARG_FILE" ]; then
+    # Assert backup dir
+    mkdir -p "$backupdir"
+
+    datastring=$(date -u --rfc-3339=ns | sed 's/ /T/g')
+    backupfile=${BACKUPDIR}/${DATABASE_NAME}.${datastring}.dump.sql
+
+    if [ -e "$backupfile" ]; then
         failure "dumpfile already exists."
     fi
-    echo "Dumping database $NARG_NAME to $NARG_FILE ... "
-  $MYSQLDUMP_CMD $(get_mysql_args_nodb) $* --opt --default-character-set=utf8 --routines \
-                    "$NARG_NAME" > "$NARG_FILE"
 
-    if [[ "$YAML" == "true" ]] ; then
-        yaml_success "$NARG_FILE" "$DATE"
-    fi
-    success
-}
+    echo "Dumping database $database to $backupfile ... "
+    $MYSQLDUMP_CMD $(get_mysql_args_nodb) --opt --default-character-set=utf8\
+          --routines  "$database" > "$backupfile"
 
-# Print a YAML failure message, if YAML is true.
-#
-# Arguments:
-# - $1  Error code, integer > 0
-# - $2  Error message
-function yaml_fail() {
-    err_code="$1"
-    err_msg="$2"
-    local yaml="${YAML_FAIL/\$err_code/$err_code}"
-    yaml="${yaml/\$err_msg/$err_msg}"
-    echo "$yaml"
-}
-
-# Print a YAML success message, if YAML is true.
-#
-# Arguments:
-# - $1  The file location of the backup file.
-# - $2  The current date & time
-function yaml_success() {
-    file_location="$1"
-    date="$2"
-    local yaml="${YAML_SUCCESS/\$file_location/$file_location}"
-    yaml="${yaml/\$date/$date}"
-    echo "$yaml"
+    success
 }
 
-## test if dump file exists
-#touch dumpfile.tmp
-#backup 0 1 2 "dumpfile.tmp"
-#rm dumpfile.tmp
-
-# Assert backup dir
-mkdir -p "$BACKUPDIR"
-
-DATE=$(date -u --rfc-3339=ns | sed 's/ /T/g')
-BACKUPFILE=${BACKUPDIR}/${DATABASE_NAME}.${DATE}.dump.sql
-
-# echo "date: $date" >&2
-# echo "file_location: $file_location" >&2
-
-#YAML_SUCCESS=$(cat <<"EOF"
-read -r -d '' YAML_SUCCESS <<"EOF" || true
-#### YAML ####
-error:
-  code: 0
-  message: Backup successfully created
-backup:
-  location: $file_location
-  date: $date
-#### YAML END ####
-EOF
-
-backup $DATABASE_NAME $BACKUPFILE
+backup $DATABASE_NAME $BACKUPDIR

utils/helpers.sh

 # This file is a part of the CaosDB Project.
 #
 # Copyright (C) 2018 Research Group Biomedical Physics,
-#   Max-Planck-Institute for Dynamics and Self-Organization Göttingen
+# Max-Planck-Institute for Dynamics and Self-Organization Göttingen
 # Copyright (C) 2019, 2020 Daniel Hornung (d.hornung@indiscale.com)
 # Copyright (C) 2020 Henrik tom Wörden
 # Copyright (C) 2020 IndiScale GmbH <info@indiscale.com>

utils/load_settings.sh

@@ -34,7 +34,7 @@ if [ -z "$MAINPATH" ]; then
 fi
 
 # Make a backup of the important environment variables.
-my_env=$(export -p | grep -E '(MYSQL|DATABASE)')
+my_env=$(export -p | grep -E '(MYSQL|DATABASE)' || true)
 
 source "$MAINPATH/config.defaults"
 

utils/log.py

+#!/usr/bin/env python3
+
+# ** header v3.0
+# This file is a part of the CaosDB Project.
+#
+# Copyright (C) 2019 Daniel Hornung, IndiScale GmbH
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see <https://www.gnu.org/licenses/>.
+#
+# ** end header
+
+"""Remote module for SQL logging
+
+Basically a wrapper with nice arguments around the bash script log.sh
+"""
+
+import argparse
+import os
+import subprocess
+import sys
+
+UTILS_PATH = os.path.abspath(os.path.dirname(__file__))
+COMMAND = [os.path.join(UTILS_PATH, "log.sh")]
+
+
+def _store_log():
+    """Retrieves the log table and returns it as a TSV string.
+
+    Returns
+    -------
+    out : str
+      The string representing the logging information, as TSV.
+    """
+    result = subprocess.run(COMMAND+["get"], encoding="utf8",
+                            stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+    # capture_output=True)  ## Only works since 3.7
+
+    if result.returncode > 0:
+        print("Error during SQL command, while retrieving SQL logs:", file=sys.stderr)
+    print(result.stderr, file=sys.stderr)
+    print(result.stdout)
+
+
+def _logging(turn_on):
+    """Turn SQL logging off/on.
+
+This function tells the SQL database to turn logging off or on (depending on the
+arguments).  It returns nothing.
+
+Parameters
+----------
+turn_on : bool
+  Whether to turn logging on or off.
+    """
+    result = subprocess.run(COMMAND+["start" if turn_on else "stop"],
+                            encoding="utf8",
+                            stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+    # capture_output=True)  ## Only works since 3.7
+
+    if result.returncode > 0:
+        print("Error during SQL command, while setting SQL logging to '{on_off}':".format(
+            on_off=on_off), file=sys.stderr)
+    print(result.stderr, file=sys.stderr)
+    print(result.stdout)
+
+    # Return nothing if no error.
+
+
+def start_logging(args):
+    _logging(True)
+
+
+def stop_logging(args):
+    _logging(False)
+
+
+def store(args):
+    _store_log()
+
+
+def parse_args():
+    """Parse the command line arguments."""
+    parser = argparse.ArgumentParser(
+        description=__doc__+"""
+To view additional information about subcommands, execute:
+    linkahead <subcmd> -h""",
+        prog="mysqllog",
+        formatter_class=argparse.RawDescriptionHelpFormatter
+    )
+    subparsers = parser.add_subparsers(
+        help='The action to take.', dest="action", required=True)
+
+    # mysql logging
+    start_parser = subparsers.add_parser(
+        'on',
+        help='Turns SQL logging on.')
+    start_parser.set_defaults(func=start_logging)
+
+    stop_parser = subparsers.add_parser(
+        'off',
+        help='Turns SQL logging off.')
+    stop_parser.set_defaults(func=stop_logging)
+
+    stop_parser = subparsers.add_parser(
+        'store',
+        help='Prints SQL logs.')
+    stop_parser.set_defaults(func=store)
+
+    return parser.parse_args()
+
+
+if __name__ == "__main__":
+    """The main function."""
+    args = parse_args()
+    args.func(args)

utils/log.sh

+#!/bin/bash
+#
+# ** header v3.0
+# This file is a part of the CaosDB Project.
+#
+# Copyright (C) 2018 Research Group Biomedical Physics,
+# Max-Planck-Institute for Dynamics and Self-Organization Göttingen
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see <https://www.gnu.org/licenses/>.
+#
+# ** end header
+#
+
+#start and stop logging; get current logs
+
+if [ -z "$UTILSPATH" ]; then
+	UTILSPATH="$(realpath $(dirname $0))"
+	export UTILSPATH
+	MAINPATH="$(dirname $UTILSPATH)"
+	export MAINPATH
+fi
+
+# Load settings from .config and defaults #####################################
+. $UTILSPATH/load_settings.sh
+
+# load useful functions #######################################################
+. $UTILSPATH/helpers.sh
+
+function get_logs {
+    echo 'select event_time, argument from mysql.general_log where command_type="Query";' | \
+		$MYSQL_CMD $(get_db_args_nodb) --batch
+}
+
+function set_logging {
+    echo "Setting logging $1..."
+    echo "SET GLOBAL log_output = 'TABLE'; SET GLOBAL general_log = '$1';" | \
+        $MYSQL_CMD $(get_db_args_nodb) --batch
+	success
+}
+
+case $1 in
+    "start") set_logging "on" ;;
+    "stop") set_logging "off" ;;
+    "get") get_logs ;;
+    *) echo "Unknown argument: $1"; exit 1
+esac

utils/make_db

@@ -82,11 +82,11 @@ function _install_unit_test_database () {
     # create test user
     grant
 
-    # update to latest
-    cp .config .test_config
     echo "DATABASE_NAME=\"$UNITTEST_DATABASE\"" >> .test_config
     pushd patches > /dev/null
-    ./applyPatches.sh --env=../.test_config
+    export ENV_FILE=$(realpath ../.test_config)
+    ./applyPatches.sh
+    unset ENV_FILE
     popd > /dev/null
     rm .test_config
 }