Merge branch 'f-unittests' into 'dev'

f-unittests -> dev

See merge request caosdb/caosdb-mysqlbackend!4

.docker/Dockerfile

-FROM debian:stretch
+FROM debian:buster
 RUN apt-get update && \
-	apt-get install  curl -y
+	apt-get install curl mariadb-client make unzip -y

.gitignore

@@ -2,6 +2,8 @@
 .*
 !/.gitignore
 !/.gitlab-ci.yml
+!libs/*.zip
 
 # dumps
 *.dump.sql
+libs/*

.gitlab-ci.yml

@@ -19,14 +19,16 @@
 # along with this program. If not, see <https://www.gnu.org/licenses/>.
 #
 
+services:
+  - mariadb:10.2
 variables:
   CI_REGISTRY_IMAGE: $CI_REGISTRY/caosdb-mysqlbackend-testenv:latest
-  # When using dind, it's wise to use the overlayfs driver for
-  # improved performance.
+  MYSQL_ROOT_PASSWORD: caosdb1234
 
 image: $CI_REGISTRY_IMAGE
 stages:
   - setup
+  - test
   - deploy
 
 # Trigger building of server image and integration tests
@@ -41,6 +43,17 @@ trigger_build:
       -F "variables[TriggerdByHash]=$CI_COMMIT_SHORT_SHA"
       -F ref=dev https://gitlab.indiscale.com/api/v4/projects/14/trigger/pipeline
 
+unittests:
+  tags: [ docker ]
+  stage: test
+  script:
+    - cp config.defaults .config
+    - echo 'DATABASE_USER_HOST_LIST="%,"' >> .config
+    - echo "MYSQL_USER_PASSWORD=$MYSQL_ROOT_PASSWORD" >> .config
+    - echo "MYSQL_HOST=mariadb" >> .config
+    - ./make_db test-connection
+    - ./make_db test --fresh
+
 # Build a docker image in which tests for this repository can run
 build-testenv:
   tags: [ cached-dind ]
@@ -51,7 +64,7 @@ build-testenv:
     - docker login -u indiscale -p $CI_REGISTRY_PASSWORD $CI_REGISTRY
       # use here general latest or specific branch latest...
     - docker pull $CI_REGISTRY_IMAGE || true
-    - docker build 
+    - docker build
       --pull
       --cache-from $CI_REGISTRY_IMAGE 
       -t $CI_REGISTRY_IMAGE .

README_SETUP.md

@@ -35,3 +35,9 @@
   this with caution!!!** If you did not backup your database, **everything will
   be lost afterwards.** And no, there is *no* additional prompt to ask if you
   are sure.  If you `make drop-...`, you *actually* delete the database.
+
+## Unit tests
+
+* We use [MyTAP-1.0](https://hepabolu.github.io/mytap/) for unit tests.
+* Tests are in `tests/test_*.sql`.
+* Run `make test`.

make_db

@@ -4,6 +4,8 @@
 # This file is a part of the CaosDB Project.
 #
 # Copyright (C) 2019 Daniel Hornung, Göttingen
+# Copyright (C) 2020 Timm Fitschen <t.fitschen@indiscale.com>
+# Copyright (C) 2020 IndiScale <info@indiscale.com>
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as
@@ -28,6 +30,79 @@ function fail() {
   exit 1
 }
 
+UNITTEST_DATABASE=${UNITTEST_DATABASE-_caosdb_schema_unit_tests}
+
+# optional parameter: [--fresh] for installing a fresh data base. Otherwise an existing one would be reused.
+function runtests() {
+    DATABASE_NAME=$UNITTEST_DATABASE
+    _setup_mytap
+
+    _install_unit_test_database $@
+
+    _execute_tests || ( echo "[FAILURE]" && exit 1 )
+    echo "[PASS]"
+}
+
+function _execute_tests () {
+    pushd tests
+    TESTS="./test*.sql"
+
+    rm -f .TEST_RESULTS
+    for tfile in $TESTS ; do
+        echo "Running $tfile"
+        echo "----- $tfile -----" >> .TEST_RESULTS
+        cat $tfile | $SQL --disable-pager --batch --raw --skip-column-names --unbuffered >> .TEST_RESULTS
+    done;
+
+    popd
+    cat tests/.TEST_RESULTS
+    grep -c -i "failed" tests/.TEST_RESULTS > /dev/null && return 1
+    return 0
+}
+
+# install/reset database for unit tests.
+# optional parameter: [--fresh] for installing a fresh data base. Otherwise an existing one would be reused.
+function _install_unit_test_database () {
+    DATABASE_NAME=$UNITTEST_DATABASE
+    if _db_exists "$DATABASE_NAME"; then
+        if [[ "$1" == "--fresh" ]] ; then
+            drop "$DATABASE_NAME" ;
+        else
+            return 0;
+        fi
+    fi
+
+    sed "s/db_2_0/$DATABASE_NAME/g" "$INSTALL_SQL_FILE" | $SQL
+
+    # crate test user
+    grant
+
+    # update to latest
+    cp .config .test_config
+    echo "DATABASE_NAME=\"$UNITTEST_DATABASE\"" >> .test_config
+    pushd patches > /dev/null
+    ./applyPatches.sh --env=../.test_config
+    popd > /dev/null
+    rm .test_config
+}
+
+# install test framework MyTAP if not installed
+function _setup_mytap() {
+    if _db_exists "tap" ; then
+        echo MyTAB framework is already installed [OK]
+        return 0
+    fi
+    echo -n "Installing MyTAB framework ... "
+    pushd libs > /dev/null
+    unzip -u mytap*.zip > /dev/null
+    pushd mytap*/ > /dev/null
+    $SQL < mytap.sql > /dev/null || exit 1
+    popd > /dev/null
+    rm -r mytap*/
+    popd > /dev/null
+    echo [DONE]
+}
+
 function setup_os() {
   # - mariadb-client :: For SQL server configuration.
   PACKAGES="git
@@ -66,7 +141,7 @@ function restore_db() {
 }
 
 function test-connection() {
-  	$SQL -e "select 0;"
+    $SQL -e "select 0;"
 }
 
 # Creates a user and grants it sufficient rights.
@@ -90,7 +165,7 @@ function grant() {
     if [[ "$1" == "--strict" ]] ; then
         for host in ${DATABASE_USER_HOST_LIST//,/ } ; do
             CMD="SELECT COUNT(*) FROM mysql.user WHERE user='${DATABASE_USER}' AND host='${host}';"
-		        [[ $(SQL -s -N -e "$CMD") == 0 ]] || {
+		        [[ $($SQL -s -N -e "$CMD") == 0 ]] || {
                 echo "The user '${DATABASE_USER}@${host}' is already in the database."
                 echo "Please use another user or delete it, e.g. with"
                 echo "'mysql -u ${MYSQL_USER} -p -e \"DROP USER ${DATABASE_USER}@${host};\"'"
@@ -100,6 +175,7 @@ function grant() {
     fi
 
     for host in ${DATABASE_USER_HOST_LIST//,/ } ; do
+        echo "Granting admin privileges to '$DATABASE_USER'@'$host'"
         $SQL <<EOF
 CREATE USER IF NOT EXISTS
     '$DATABASE_USER'@'$host' identified by '$DATABASE_USER_PW';
@@ -120,13 +196,15 @@ function drop() {
     "$MYSQLADMIN_CMD" $MYSQL_CONNECTION -f drop "$DROPDB"
 }
 
+
 # Returns 0 or non-zero, depending on whether the database exists already.
+# Optional parameter: [DATABASE_NAME]
 function _db_exists() {
-    $SQL -D "$DATABASE_NAME" -e "show tables;" > /dev/null 2>&1 \
+    $SQL -D "${1-${DATABASE_NAME}}" -e "show tables;" > /dev/null 2>&1 \
         && return 0 || return 1
 }
 
-. .config
+source .config || true
 
 INSTALL_SQL_FILE="db_2_0.sql"
 if [ -z $LOGIN_PATH ] ; then
@@ -142,6 +220,7 @@ SQL="$MYSQL_CMD $MYSQL_CONNECTION"
 case $1 in
   "drop") drop $2 ;;
   "grant") grant $2 ;;
+  "test") shift ; runtests $@ ;;
   "test-connection") test-connection ;;
   "install_db") install_db ;;
   "restore_db") restore_db $2 ;;

makefile

@@ -55,3 +55,7 @@ _grant:
 .PHONY: drop-%
 drop-%:
 	./make_db drop $(patsubst drop-%,%,$@)
+
+.PHONY: test
+test:
+	./make_db test --fresh

patches/applyPatches.sh

@@ -24,7 +24,6 @@
 
 #apply all available patches.
 set -e
-. ./utils/patch_header.sh
 PATCHES="./patch*/patch.sh"
 export UTILSPATH="./utils"
 
@@ -33,5 +32,6 @@ do
     $p "$@" --patch=$p
 done
 
+source ./utils/patch_header.sh
 cd ../
 ./update_sql_procedures.sh

patches/utils/patch_header.sh

@@ -101,7 +101,7 @@ while test $# -gt 0; do
  esac
 done
 
-[[ -n "$ENV_FILE" ]] && source "$ENV_FILE"
+source "$ENV_FILE" || true
 
 if [[ -z "$DATABASE_NAME" && -z "$MYSQL_CONNECTION" ]]
 then