Merge branch 'dev' into f-new-tools

.docker/Dockerfile

-FROM debian:stretch
+FROM debian:buster
 RUN apt-get update && \
-	apt-get install  curl -y
+	apt-get install curl mariadb-client make unzip -y

.gitignore

@@ -2,6 +2,8 @@
 .*
 !/.gitignore
 !/.gitlab-ci.yml
+!libs/*.zip
 
 # dumps
 *.dump.sql
+libs/*

.gitlab-ci.yml

@@ -19,14 +19,16 @@
 # along with this program. If not, see <https://www.gnu.org/licenses/>.
 #
 
+services:
+  - mariadb:10.2
 variables:
-  CI_REGISTRY_IMAGE: $CI_REGISTRY/caosdb-mysqlbackend-testenv:latest
-  # When using dind, it's wise to use the overlayfs driver for
-  # improved performance.
+  CI_REGISTRY_IMAGE: $CI_REGISTRY/caosdb/caosdb-mysqlbackend/testenv:latest
+  MYSQL_ROOT_PASSWORD: caosdb1234
 
 image: $CI_REGISTRY_IMAGE
 stages:
   - setup
+  - test
   - deploy
 
 # Trigger building of server image and integration tests
@@ -41,6 +43,17 @@ trigger_build:
       -F "variables[TriggerdByHash]=$CI_COMMIT_SHORT_SHA"
       -F ref=dev https://gitlab.indiscale.com/api/v4/projects/14/trigger/pipeline
 
+unittests:
+  tags: [ docker ]
+  stage: test
+  script:
+    - cp config.defaults .config
+    - echo 'DATABASE_USER_HOST_LIST="%,"' >> .config
+    - echo "MYSQL_USER_PASSWORD=$MYSQL_ROOT_PASSWORD" >> .config
+    - echo "MYSQL_HOST=mariadb" >> .config
+    - ./make_db test-connection
+    - ./make_db test --fresh
+
 # Build a docker image in which tests for this repository can run
 build-testenv:
   tags: [ cached-dind ]
@@ -48,7 +61,7 @@ build-testenv:
   stage: setup
   script: 
     - cd .docker
-    - docker login -u indiscale -p $CI_REGISTRY_PASSWORD $CI_REGISTRY
+    - docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY
       # use here general latest or specific branch latest...
     - docker pull $CI_REGISTRY_IMAGE || true
     - docker build

README_SETUP.md

@@ -35,3 +35,9 @@
   this with caution!!!** If you did not backup your database, **everything will
   be lost afterwards.** And no, there is *no* additional prompt to ask if you
   are sure.  If you `make drop-...`, you *actually* delete the database.
+
+## Unit tests
+
+* We use [MyTAP-1.0](https://hepabolu.github.io/mytap/) for unit tests.
+* Tests are in `tests/test_*.sql`.
+* Run `make test`.

RELEASE_GUIDELINES.md

+# Release Guidelines for the CaosDB MySQL Backend
+
+This document specifies release guidelines in addition to the generel release
+guidelines of the CaosDB Project
+([RELEASE_GUIDELINES.md](https://gitlab.com/caosdb/caosdb/blob/dev/RELEASE_GUIDELINES.md))
+
+## General Prerequisites
+
+* All tests are passing.
+* FEATURES.md is up-to-date and a public API is being declared in that document.
+* CHANGELOG.md is up-to-date.
+* DEPENDENCIES.md is up-to-date.
+
+## Steps
+
+1. Create a release branch from the dev branch. This prevents further changes
+   to the code base and a never ending release process. Naming: `release-<VERSION>`
+
+2. Check all general prerequisites.
+
+4. Merge the release branch into the master branch.
+
+5. Tag the latest commit of the master branch with `v<VERSION>`.
+
+6. Delete the release branch.
+
+7. Merge the master branch back into the dev branch.

VERSIONING.md

-# Versioning
-
-Starting with version `TODO: the next version`, the versions of the CaosDB MySQL Back-end must follow the principles of [Semantic Versioning 2.0.0](https://semver.org). In summary:
-
-> Given a version number MAJOR.MINOR.PATCH, increment the:
->
-> 1. MAJOR version when you make incompatible API changes,
-> 2. MINOR version when you add functionality in a backwards-compatible manner, and
-> 3. PATCH version when you make backwards-compatible bug fixes.
->
-> Additional labels for pre-release and build metadata are available as extensions to the MAJOR.MINOR.PATCH format.
-
-
-## §1 API Changes.
-
-1. All tables, stored procedures and stored functions which names do not begin with the character `_` belong to the api and are not implementation specific.
-2. Specifically, that means for all components of the API:
-    1. Changes to the design of existing tables or their colums (name, datatype, etc.) MUST always result in anew MAJOR version. The only exception from this rule are changes to the description of a table or column.
-    2. Deleting tables is an incompatible API change and results in a new MAJOR version.
-    3. Changes to the stored procedures and functions, which change the signature (i.e the list of parameters) is an incompatible change, and results in a new MAJOR version.
-    4. Adding new stored functions, procedures or tables results in a new MINOR version.
-    5. Changing just a bit of code inside a stored procedure or function results in a new PATCH version.
-
-## §2 Other Changes.
-
-1. Changes to any non-API tables, functions and procedures results in a new PATCH version.

doc/table_structure.md

+This document describes the SQL tables used by CaosDB and how they represent the
+internal data structure.
+
+# Tables #
+
+## entities ##
+All entities (RecordTypes, Records, Properties, ...) have an entry here.  The
+columns are:
+- `id` :: The (unique) ID of this entity.
+- `name` :: Name of the entity.
+- `description` :: A description of the entity.
+- `role` :: The role of the entity (e.g. data type, RecordType, Property,
+  Record, ...).
+- `acl` :: Access control ID.
+
+## isa ##
+
+Inheritance as in "A is a B" is stored in this table.  The columns:
+
+- `child` :: The child entity.
+- `parent` :: The parent entity.
+- `type` :: The inheritance type (either `INHERITANCE` or `SUBTYPING` (what's
+  "subtyping"?)).
+
+**TODO** Why is this table empty on the default example?
+
+## isa_cache ##
+
+Here, information about nested inheritance is cached (upon creation?).
+
+- `child` :: The child entity.
+- `parent` :: The parent entity.
+- `rpath` :: Possible intermediate steps on a path from `child` to `parent`.
+  See the *Inheritance* section for more details.
+
+## foo_data ##
+Property data is stored here, in tables named after the data type:
+- date_data
+- datetime_data
+- double_data
+- integer_data
+- text_data
+- enum_data
+- name_data
+- null_data
+- reference_data
+
+These tables share (generally) the same set of columns:
+- `domain_id` :: The domain of an item, see the *multipurpose subdomains*
+  section.  For "normal" data, this is 0.
+- `entity_id` :: The entity to which this property belongs.  In most cases, this
+  is a "normal" entity like a `Record` or `RecordType`.
+- `property_id` :: In most cases, the property to which the value belongs.
+- `value` :: The value of the property.
+- `status` :: The importance of the property (?).
+- `pidx` :: The property index, becomes nonzero when a property occurs more than
+  once in an entity.
+
+## data_type ##
+The type of properties is stored here.  The columns are:
+- `domain_id` :: Property identitification, same as for plain data?
+- `entity_id` :: Property identitification, same as for plain data?
+- `property_id` :: Property identitification, same as for plain data?
+- `datatype` :: The data type of the property, a reference to an entity.
+
+## files ##
+
+- `file_id` :: ID.
+- `path` :: (Relative) path to the file.
+- `size` :: Size in bytes.
+- `hash` :: Hash of the file contents, as binary. **TODO** Which algorithm?
+- `checked_timestamp` :: Timestamp when last checked? 
+
+## TODO ##
+
+- entity_acl
+- groups
+- logging
+- desc_overrides
+- name_overrides
+- passwd
+- passwords
+- permissions
+- query_template_def :: User-defined query templates?
+- roles :: possible user roles
+- rules :: SQL action/verification hooks?
+- stats :: benchmarking?
+- transaction_log :: benchmarking?
+- units_lin_con :: fraction and unit conversion?
+- user_info :: Metadata for (local?) user accounts
+- user_roles :: User permissions?
+
+# Inheritance caching #
+
+The `isa_cache` table does not only store direct inheritances, but also nested
+parentships.  This is done by adding intermediate inheritance steps in the
+`rpath` column:
+
+- For *direct inheritance*, the `rpath` column contains the child itself.
+- For all other *possible paths* between the child and parent, it contains the
+  entities between, separated by the `>` character.
+  
+## Example ##
+
+Consider this inheritance structure and the resulting table:
+
+```
+   0    Read this from top to bottom: "1 is a 0",
+  / \   "2 is a 0", and so on.
+ 1   2
+  \ /
+   3
+   |
+   4
+   |
+   5
+```
+
+|-------|--------|-------|
+| child | parent | rpath |
+|-------|--------|-------|
+| 1     | 0      | 1     |
+| 2     | 0      | 2     |
+| 3     | 0      | 1     |
+| 3     | 0      | 2     |
+| 3     | 1      | 3     |
+| 3     | 2      | 3     |
+| 4     | 0      | 3>1   |
+| 4     | 0      | 3>2   |
+| 4     | 1      | 3     |
+| 4     | 2      | 3     |
+| 4     | 3      | 4     |
+| 5     | 0      | 4>3>1 |
+| 5     | 0      | 4>3>2 |
+| 5     | 1      | 4>3   |
+| 5     | 2      | 4>3   |
+| 5     | 3      | 4     |
+| 5     | 4      | 5     |
+|-------|--------|-------|
+
+
+# Multipurpose subdomains #
+Multipurpose subdomains are generically used to work with composite
+properties. Currently implemented examples are:
+- Properties with units (if multiple Properties exist)
+- Lists
+- Name overrides
+
+## Example ##
+Let's have a look at this *Record* (simplified XML):
+
+```xml
+<R1>
+    <P1 name="Comment">Hello World</P1>
+    <P2 name="voltage" unit="V">
+        23
+    </P2>
+    <P3 comment="list of something">
+        V1, V2, V3, V4, ...
+    </P3>
+</R1>
+```
+

make_db

@@ -4,6 +4,8 @@
 # This file is a part of the CaosDB Project.
 #
 # Copyright (C) 2019 Daniel Hornung, Göttingen
+# Copyright (C) 2020 Timm Fitschen <t.fitschen@indiscale.com>
+# Copyright (C) 2020 IndiScale <info@indiscale.com>
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as
@@ -28,6 +30,79 @@ function fail() {
   exit 1
 }
 
+UNITTEST_DATABASE=${UNITTEST_DATABASE-_caosdb_schema_unit_tests}
+
+# optional parameter: [--fresh] for installing a fresh data base. Otherwise an existing one would be reused.
+function runtests() {
+    DATABASE_NAME=$UNITTEST_DATABASE
+    _setup_mytap
+
+    _install_unit_test_database $@
+
+    _execute_tests || ( echo "[FAILURE]" && exit 1 )
+    echo "[PASS]"
+}
+
+function _execute_tests () {
+    pushd tests
+    TESTS="./test*.sql"
+
+    rm -f .TEST_RESULTS
+    for tfile in $TESTS ; do
+        echo "Running $tfile"
+        echo "----- $tfile -----" >> .TEST_RESULTS
+        cat $tfile | $SQL --disable-pager --batch --raw --skip-column-names --unbuffered >> .TEST_RESULTS
+    done;
+
+    popd
+    cat tests/.TEST_RESULTS
+    grep -c -i "failed" tests/.TEST_RESULTS > /dev/null && return 1
+    return 0
+}
+
+# install/reset database for unit tests.
+# optional parameter: [--fresh] for installing a fresh data base. Otherwise an existing one would be reused.
+function _install_unit_test_database () {
+    DATABASE_NAME=$UNITTEST_DATABASE
+    if _db_exists "$DATABASE_NAME"; then
+        if [[ "$1" == "--fresh" ]] ; then
+            drop "$DATABASE_NAME" ;
+        else
+            return 0;
+        fi
+    fi
+
+    sed "s/db_2_0/$DATABASE_NAME/g" "$INSTALL_SQL_FILE" | $SQL
+
+    # crate test user
+    grant
+
+    # update to latest
+    cp .config .test_config
+    echo "DATABASE_NAME=\"$UNITTEST_DATABASE\"" >> .test_config
+    pushd patches > /dev/null
+    ./applyPatches.sh --env=../.test_config
+    popd > /dev/null
+    rm .test_config
+}
+
+# install test framework MyTAP if not installed
+function _setup_mytap() {
+    if _db_exists "tap" ; then
+        echo MyTAB framework is already installed [OK]
+        return 0
+    fi
+    echo -n "Installing MyTAB framework ... "
+    pushd libs > /dev/null
+    unzip -u mytap*.zip > /dev/null
+    pushd mytap*/ > /dev/null
+    $SQL < mytap.sql > /dev/null || exit 1
+    popd > /dev/null
+    rm -r mytap*/
+    popd > /dev/null
+    echo [DONE]
+}
+
 function setup_os() {
   # - mariadb-client :: For SQL server configuration.
   PACKAGES="git
@@ -90,7 +165,7 @@ function grant() {
     if [[ "$1" == "--strict" ]] ; then
         for host in ${DATABASE_USER_HOST_LIST//,/ } ; do
             CMD="SELECT COUNT(*) FROM mysql.user WHERE user='${DATABASE_USER}' AND host='${host}';"
-		        [[ $(SQL -s -N -e "$CMD") == 0 ]] || {
+		        [[ $($SQL -s -N -e "$CMD") == 0 ]] || {
                 echo "The user '${DATABASE_USER}@${host}' is already in the database."
                 echo "Please use another user or delete it, e.g. with"
                 echo "'mysql -u ${MYSQL_USER} -p -e \"DROP USER ${DATABASE_USER}@${host};\"'"
@@ -100,6 +175,7 @@ function grant() {
     fi
 
     for host in ${DATABASE_USER_HOST_LIST//,/ } ; do
+        echo "Granting admin privileges to '$DATABASE_USER'@'$host'"
         $SQL <<EOF
 CREATE USER IF NOT EXISTS
     '$DATABASE_USER'@'$host' identified by '$DATABASE_USER_PW';
@@ -120,13 +196,15 @@ function drop() {
     "$MYSQLADMIN_CMD" $MYSQL_CONNECTION -f drop "$DROPDB"
 }
 
+
 # Returns 0 or non-zero, depending on whether the database exists already.
+# Optional parameter: [DATABASE_NAME], else $DATABASE_NAME is used.
 function _db_exists() {
-    $SQL -D "$DATABASE_NAME" -e "show tables;" > /dev/null 2>&1 \
+    $SQL -D "${1-${DATABASE_NAME}}" -e "show tables;" > /dev/null 2>&1 \
         && return 0 || return 1
 }
 
-. .config
+source .config || true
 
 INSTALL_SQL_FILE="db_2_0.sql"
 if [ -z $LOGIN_PATH ] ; then
@@ -142,6 +220,7 @@ SQL="$MYSQL_CMD $MYSQL_CONNECTION"
 case $1 in
   "drop") drop $2 ;;
   "grant") grant $2 ;;
+  "test") shift ; runtests $@ ;;
   "test-connection") test-connection ;;
   "install_db") install_db ;;
   "restore_db") restore_db $2 ;;

makefile

@@ -55,3 +55,7 @@ _grant:
 .PHONY: drop-%
 drop-%:
 	./make_db drop $(patsubst drop-%,%,$@)
+
+.PHONY: test
+test:
+	./make_db test --fresh

patches/applyPatches.sh

@@ -5,6 +5,8 @@
 #
 # Copyright (C) 2018 Research Group Biomedical Physics,
 # Max-Planck-Institute for Dynamics and Self-Organization Göttingen
+# Copyright (C) 2020 Timm Fitschen <t.fitschen@indiscale.com>
+# Copyright (C) 2020 IndiScale <info@indiscale.com>
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as
@@ -24,7 +26,6 @@
 
 #apply all available patches.
 set -e
-. ./utils/patch_header.sh
 PATCHES="./patch*/patch.sh"
 export UTILSPATH="./utils"
 
@@ -33,5 +34,6 @@ do
     $p "$@" --patch=$p
 done
 
+source ./utils/patch_header.sh
 cd ../
 ./update_sql_procedures.sh

patches/utils/patch_header.sh

@@ -25,7 +25,7 @@
 CMD_MYSQL=mysql
 CMD_MYSQL_DUMP=mysqldump
 USAGE="$1 [ --env=ENV_FILE ] [ --patch=PATCH ] [ --backupdir=BACKUPDIR ]\n\n"
-CMD_OPTIONS=cat <<EOF
+CMD_OPTIONS=$(cat <<EOF
 options:
 
 -h, --help
@@ -37,7 +37,10 @@ options:
 --patch=PATCH
     *TODO* Please document this option.
 --backupdir=BACKUPDIR
-    *TODO* Please document this option.
+    The directory where backup files are stored to.
+--yaml[=True|False]
+    If given (and the optional value is not False), additional machine-readable
+    YAML output is emitted.  Only implemented for some commands yet.
 
 This script basically uses the same environment variables as the server
 configuration make file.  Notable examples are:
@@ -50,6 +53,7 @@ configuration make file.  Notable examples are:
 - DATABASE_USER_PW
 
 EOF
+              )
 set -e
 
 function _print_help() {
@@ -79,7 +83,14 @@ while test $# -gt 0; do
   ;;
  --backupdir*)
   BACKUPDIR="${1#--*=}"
-  echo "Warning: BACKUPDIR is not used currently."
+  shift
+  ;;
+ --yaml*)
+     if [[ "$1" == "--yaml" ]] ; then
+         YAML="True"
+     else
+         YAML="${1#--*=}"
+     fi
   shift
   ;;
  *)
@@ -90,7 +101,7 @@ while test $# -gt 0; do
  esac
 done
 
-[[ -n "$ENV_FILE" ]] && source "$ENV_FILE"
+source "$ENV_FILE" || true
 
 if [[ -z "$DATABASE_NAME" && -z "$MYSQL_CONNECTION" ]]
 then

backup.sh → utils/backup.sh

@@ -22,45 +22,47 @@
 # ** end header
 #
 
-#dump a database with all procedures, permissions, structure and data
+# Dump a database with all procedures, permissions, structure and data
 
-BACKUPDIR=./backup #The directory which the dump is to be stored to. Do not change it here. Use the --backupdir=./my/dir/ option!
-PRINT_HELP="--backupdir=BACKUPDIR\n\tThe directory which the dump is to be stored to. (Defaults to ./backup)\n"
+# The directory which the dump is to be stored to. Do not change it here. Use
+# the --backupdir=./my/dir/ option or an environment variable instead.
+BACKUPDIR="${BACKUPDIR:-../backup}"
+PRINT_HELP="--backupdir=BACKUPDIR\n\tThe directory which the dump is to be stored to. (Defaults to ./backup,\n\tcan also be set as an environment variable.)\n"
 
-. .config
-# load useful stuff - scans the parameters and so on...
-. patches/utils/patch_header.sh
 
-#create backup dir if not exists
-[ -d $BACKUPDIR ] || mkdir $BACKUPDIR
+# Config and command line parameter loading ###################################
 
+. ../.config
 
+# load useful stuff - parses the commandline parameters and so on...
+. ../patches/utils/patch_header.sh
+
+function backup() {
+    NARG_NAME="$1"
+    NARG_FILE="$2"
 
-function backup {
-    NARG_NAME=$1
-    NARG_FILE=$2
     shift 2
 	  # parameters: connection, database, outfile
 	  if [ -e "$NARG_FILE" ]; then
 		    failure "dumpfile already exists."
 	  fi
     echo "Dumping database $NARG_NAME to $NARG_FILE ... "
-	$CMD_MYSQL_DUMP $MYSQL_CONNECTION_NO_DB $* --opt --default-character-set=utf8 --routines $NARG_NAME > $NARG_FILE
+	  $CMD_MYSQL_DUMP $MYSQL_CONNECTION_NO_DB $* --opt --default-character-set=utf8 --routines \
+                    "$NARG_NAME" > "$NARG_FILE"
+
 	  success
 }
 
 
 
-
-
-##test dump file exists
+## test if dump file exists
 #touch dumpfile.tmp
 #backup 0 1 2 "dumpfile.tmp"
 #rm dumpfile.tmp
 
+# Assert backup dir
+mkdir -p "$BACKUPDIR"
 
 DATE=$(date -u --rfc-3339=ns | sed 's/ /T/g')
 BACKUPFILE=${BACKUPDIR}/${DATABASE_NAME}.${DATE}.dump.sql
 backup $DATABASE_NAME $BACKUPFILE
-
-