Merge branch 'f-mysql-test' into 'dev'

CI pipeline for mysql unit tests

See merge request !4

.gitlab-ci.yml

@@ -19,8 +19,6 @@
 # along with this program. If not, see <https://www.gnu.org/licenses/>.
 #
 
-services:
-  - mariadb:10.4
 variables:
   DEPLOY_REF: dev
   CI_REGISTRY_IMAGE: $CI_REGISTRY/caosdb/src/caosdb-mysqlbackend/testenv:latest
@@ -32,31 +30,7 @@ stages:
   - test
   - deploy
 
-# Trigger building of server image and integration tests
-trigger_build:
-  tags: [ docker ]
-  stage: deploy
-  script:
-    - /usr/bin/curl -X POST
-      -F token=$CI_JOB_TOKEN
-      -F "variables[F_BRANCH]=$CI_COMMIT_REF_NAME"
-      -F "variables[MYSQLBACKEND]=$CI_COMMIT_REF_NAME"
-      -F "variables[TriggerdBy]=MYSQLBACKEND"
-      -F "variables[TriggerdByHash]=$CI_COMMIT_SHORT_SHA"
-      -F ref=$DEPLOY_REF https://gitlab.indiscale.com/api/v4/projects/14/trigger/pipeline
-
-unittests:
-  tags: [ docker ]
-  stage: test
-  script:
-    - cp config.defaults .config
-    - echo 'DATABASE_USER_HOST_LIST="%,"' >> .config
-    - echo "MYSQL_USER_PASSWORD=$MYSQL_ROOT_PASSWORD" >> .config
-    - echo "MYSQL_HOST=mariadb" >> .config
-    - sleep 10
-    - make install
-    - ./utils/make_db restore_db tests/example.dump.sql
-    - ./tests/test_utils.sh
+######## Setup ########
 
 # Build a docker image in which tests for this repository can run
 build-testenv:
@@ -69,12 +43,70 @@ build-testenv:
   script:
     - cd .docker
     - docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY
-      # use here general latest or specific branch latest...
+    # use here general latest or specific branch latest...
     - docker build
       --pull
       -t $CI_REGISTRY_IMAGE .
     - docker push $CI_REGISTRY_IMAGE
 
+
+######## Test ########
+
+# Run the unit tests with MariaDB
+unittests-mariadb:
+  tags: [ docker ]
+  stage: test
+  services:
+    - mariadb:10.4
+
+  script:
+    - make pipeline-test SQL_HOST=mariadb
+
+# Run the unit tests with MySQL 8
+unittests-mysql-8:
+  tags: [ docker ]
+  stage: test
+  # Should not stop the pipeline from continuing.
+  allow_failure: true
+  services:
+    - name: mysql:8.0
+      command: ["--default-authentication-plugin=mysql_native_password"]
+
+  script:
+    - make pipeline-test SQL_HOST=mysql
+
+# Run the unit tests with MySQL 5
+unittests-mysql-5:
+  tags: [ docker ]
+  stage: test
+  # Should not stop the pipeline from continuing.
+  allow_failure: true
+  services:
+    - name: mysql:5.7
+      command: ["--default-authentication-plugin=mysql_native_password"]
+
+  script:
+    # remove autotap because the checks of (autogenerated) names of constraints
+    # and the checks of column default values don't work with mysql
+    - rm tests/test_autotap.sql
+    - make pipeline-test SQL_HOST=mysql
+
+######## Deploy ########
+
+# Trigger building of server image and integration tests
+trigger_build:
+  tags: [ docker ]
+  stage: deploy
+  script:
+    - /usr/bin/curl -X POST
+      -F token=$CI_JOB_TOKEN
+      -F "variables[F_BRANCH]=$CI_COMMIT_REF_NAME"
+      -F "variables[MYSQLBACKEND]=$CI_COMMIT_REF_NAME"
+      -F "variables[TriggerdBy]=MYSQLBACKEND"
+      -F "variables[TriggerdByHash]=$CI_COMMIT_SHORT_SHA"
+      -F ref=$DEPLOY_REF https://gitlab.indiscale.com/api/v4/projects/14/trigger/pipeline
+
+
 # Build the sphinx documentation and make it ready for deployment by Gitlab Pages
 # Special job for serving a static website. See https://docs.gitlab.com/ee/ci/yaml/README.html#pages
 pages_prepare: &pages_prepare
@@ -95,4 +127,3 @@ pages:
   only:
     refs:
       - main
-

CHANGELOG.md

@@ -9,6 +9,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Added ###
 
+* #33 CI pipeline for MySQL (was only MariaDB before).
+
 ### Changed ###
 
 ### Deprecated ###

Makefile

@@ -24,6 +24,9 @@ SHELL=/bin/bash
 
 INSTALL_SQL_FILE=db_2_0.sql
 
+# The hostname, used by testing in the CI pipeline
+SQL_HOST ?= mysql
+
 .PHONY: test-connection
 test-connection:
 	./utils/make_db test-connection
@@ -74,3 +77,15 @@ test-docker-stop:
 .PHONY: doc
 doc:
 	$(MAKE) -C doc html
+
+# Run tests in a Gitlab pipeline
+.PHONY: pipeline-test
+pipeline-test:
+	cp config.defaults .config
+	echo 'DATABASE_USER_HOST_LIST="%,"' >> .config
+	echo "MYSQL_USER_PASSWORD=$(MYSQL_ROOT_PASSWORD)" >> .config
+	echo "MYSQL_HOST=$(SQL_HOST)" >> .config
+	sleep 10
+	make install
+	./utils/make_db restore_db tests/example.dump.sql
+	./tests/test_utils.sh

README_SETUP.md

@@ -82,6 +82,20 @@ with the then current version of the stored entities.
     * Alternatively, to run the tests in a containerized MariaDB instance, run `make test-docker`,
       followed by `make test-docker-stop`.
 
+### Troubleshooting
+
+#### MySQL has failing tests
+
+Our test suite is developed with MariaDB. That is why some tests, those which
+check constraints based on auto-generated names, fail because MySQL generates
+other names.
+
+Also some test fail with the message "Expected: NULL / Was: NULL" which is
+obviously what was expected.
+
+Please look carefully through the tests. If something more serious than that
+comes up, please report a bug.
+
 ## Documentation #
 
 Build documentation in `build/` with `make doc`.

patches/patch20200710-3.0.0-rc2/versioning.sql

@@ -315,6 +315,6 @@ CREATE TABLE archive_entities (
     FOREIGN KEY (`acl`) REFERENCES `entity_acl` (`id`)
 ) ENGINE=InnoDB;
 
-ALTER IGNORE TABLE collection_type ADD UNIQUE KEY `collection_type-d-e-p` (`domain_id`, `entity_id`, `property_id`);
+ALTER TABLE collection_type ADD UNIQUE KEY `collection_type-d-e-p` (`domain_id`, `entity_id`, `property_id`);
 
 INSERT INTO feature_config (_key, _value) VALUES ("ENTITY_VERSIONING", "ENABLED");

utils/load_settings.sh

@@ -34,7 +34,12 @@ if [ -z "$MAINPATH" ]; then
 fi
 
 # Make a backup of the important environment variables.
-my_env=$(export -p | grep -E '(MYSQL|DATABASE)' || true)
+# Take care: Assignments may be more than one line!
+_expr="(MYSQL|DATABASE)"
+my_env=$(export -p | awk '/^declare -x '"$_expr"'/                   {PRINT=1} \
+                          /^declare -x/ && !/^declare -x '"$_expr"'/ {PRINT=0} \
+                          PRINT                                      {print}' \
+             || true)
 
 source "$MAINPATH/config.defaults"