Skip to content
Snippets Groups Projects

Compare revisions

Changes are shown as if the source revision was being merged into the target revision. Learn more about comparing revisions.

Source

Select target project
No results found
Select Git revision

Target

Select target project
  • caosdb/src/caosdb-mysqlbackend
1 result
Select Git revision
Show changes
Commits on Source (112)
12 additional commits have been omitted to prevent performance issues.
Hide whitespace changes
Inline Side-by-side
Showing
with 618 additions and 143 deletions
.docker/Dockerfile
View file @ 5a59dc3a
FROM debian:buster
FROM debian:bookworm
RUN apt-get update \
&& \
apt-get install -y \
......@@ -11,4 +11,8 @@ RUN apt-get update \
python3-sqlparse \
python3-sphinx \
doxygen
RUN pip3 install breathe sphinx-rtd-theme recommonmark
RUN pip3 install --break-system-packages \
breathe \
sphinx-rtd-theme \
pytest \
recommonmark
.docker/docker-compose.yml 0 → 100644
View file @ 5a59dc3a
version: '3.6'
services:
sqldb:
image: mariadb:11.4
volumes:
- type: volume
source: "caosdb-sqldata"
target: /var/lib/mysql
environment:
MYSQL_ROOT_PASSWORD: caosdb1234
networks:
# available on port 3306, host name 'sqldb'
- caosnet
ports:
- 3306:3306
# A well-defined network for caosdb
volumes:
caosdb-sqldata:
networks:
caosnet:
driver: bridge
.gitignore
View file @ 5a59dc3a
......@@ -13,4 +13,5 @@ libs/*
# Generated files
_doxygen/
_generated/
/build/
\ No newline at end of file
/build/
*~
.gitlab-ci.yml
View file @ 5a59dc3a
#
# This file is a part of the CaosDB Project.
# This file is a part of the LinkAhead Project.
#
# Copyright (C) 2018 Research Group Biomedical Physics,
# Max-Planck-Institute for Dynamics and Self-Organization Göttingen
# Copyright (C) 2024 Indiscale GmbH <info@indiscale.com>
# Copyright (C) 2019 Henrik tom Wörden
# Copyright (C) 2024 Daniel Hornung <d.hornung@indiscale.com>
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as
......@@ -57,43 +59,35 @@ unittests-mariadb:
tags: [ docker ]
stage: test
services:
- mariadb:10.4
- mariadb:10.11
script:
- make pipeline-test SQL_HOST=mariadb
# Run the unit tests with MySQL 8
unittests-mysql-8:
tags: [ docker ]
stage: test
# Should not stop the pipeline from continuing.
allow_failure: true
services:
- name: mysql:8.0
command: ["--default-authentication-plugin=mysql_native_password"]
script:
- sed "s/NO_AUTO_CREATE_USER,//" -i tests/example.dump.sql
- rm tests/test_autotap.sql
- make pipeline-test SQL_HOST=mysql
# Run the unit tests with MySQL 5
unittests-mysql-5:
# Run the dump update tests
test-dump-update:
tags: [ docker ]
stage: test
# Should not stop the pipeline from continuing.
allow_failure: true
services:
- name: mysql:5.7
command: ["--default-authentication-plugin=mysql_native_password"]
script:
# remove some lines from autotap because the checks of column default
# values don't work with mysql-5
- sed -i "/col_default_is.*NULL/d" tests/test_autotap.sql
- sed -i "/col_default_is.*INACTIVE/d" tests/test_autotap.sql
- sed -i "/col_default_is.*SHA/d" tests/test_autotap.sql
- make pipeline-test SQL_HOST=mysql
- make test-dump-update
# # Run the unit tests with MySQL 5
# unittests-mysql-5:
# tags: [ docker ]
# stage: test
# # Should not stop the pipeline from continuing.
# allow_failure: true
# services:
# - name: mysql:5.7.36
# command: ["--default-authentication-plugin=mysql_native_password"]
#
# script:
# # remove some lines from autotap because the checks of column default
# # values don't work with mysql-5
# - sed -i "/col_default_is.*NULL/d" tests/test_autotap.sql
# - sed -i "/col_default_is.*INACTIVE/d" tests/test_autotap.sql
# - sed -i "/col_default_is.*SHA/d" tests/test_autotap.sql
# - sed -i "s/utf8mb3/utf8/" tests/test_autotap.sql
# - make pipeline-test SQL_HOST=mysql
######## Deploy ########
......
CHANGELOG.md
View file @ 5a59dc3a
......@@ -5,30 +5,165 @@ All notable changes to this project will be documented in this file.
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
## [Unreleased] ##
## [Unreleased]
### Added
- New version requirement: MariaDB 11.2 or later.
### Changed
### Deprecated
### Removed
- MySQL support. See the deprecation in the 7.0.0 release.
### Fixed
- [SQL dump migration can now be applied
twice](https://gitlab.indiscale.com/caosdb/src/caosdb-mysqlbackend/-/issues/60):
Fixed an error in the regex.
- [linkahead-server#264](https://gitlab.com/linkahead/linkahead-server/-/issues/264)
Queries ignore accents and umlauts: Changed to more appropriate
collation and character sets. This requires MariaDB 11.2 or later,
see above.
### Security
## [8.0.0] - 2024-10-24 ##
### Added ###
- Script to update outdated database dumps.
### Changed ###
### Deprecated ###
- `OFFSET` is a reserved keyword in MariaDB 10.6, so the former
`OFFSET` parameter name in the `get_head_relative` function was
changed to HeadOffset. See
[`update_dumps/README.md`](update_dumps/README.md) for information
on how to update old SQL dumps to be compatible with this renaming
and MariaDB >= 10.6.
### Removed ###
## [7.0.2] - 2023-12-12 ##
(Timm Fitschen)
### Fixed ###
### Security ###
* make install: Could not connect to MariaDB in local docker
container
[linkahead-mariadbbackend](https://gitlab.com/linkahead/linkahead-mariadbbackend/-/issues/33)
* Wrong ids in data_type table due to a bug in the `linkahead-server < v0.11.0`
which has been fixed but left broken data. This resulted in server errors
[linkahead-mariadbbackend#34](https://gitlab.com/linkahead/linkahead-mariadbbackend/-/issues/34)
## [5.0.0] - 2021-10-28 ##
## [7.0.1] - 2023-11-01
(Timm Fitschen)
### Added ###
* #33 CI pipeline for MySQL (was only MariaDB before).
* Sanity checks during `make install` and `make upgrade`.
## [7.0.0] - 2023-10-25 ##
(Timm Fitschen)
### Changed ###
* Change signature of procedure `insertEntityDataType`: expect the data type ID
instead of the name.
* Change result set of procedure `retrieveEntity`: Rename column `Datatype` to
`DatatypeName` and add column `DatatypeID`.
* Change result set of procedure `retrieveOverrides`: Rename column
`type_override` to `type_name_override` and add `type_id_override`.
* Change signature of procedure `updateEntity`: expect the data type ID instead
to the name.
* Internal ids, when returned by `retrieveEntityProperties`,
`registerReplacementIds`, or `retrieveOverrides`, are prefixed by a special
character which serves as a ephemeral marker. Currently this is `$` but this
may change in the future if there is a more suitable candidate.
### Deprecated ###
* MySQL Support. Last version which is known to work well with LinkAhead is MySQL 5.7.36
### Fixed ###
* Unknown Server Error when inserting an Entity.
[linkahead-mariadbbackend#48](https://gitlab.indiscale.com/caosdb/src/caosdb-mysqlbackend/-/issues/48).
* Fix broken `initEntity` procedure (and change the first parameters type from
INT UNSIGNED to VARCHAR(255) which is non-breaking).
## [6.0.1 2023-10-18] ##
### Fixed ###
* Old passwords table is removed from dumps where it still exists.
## [6.0.0 2023-10-17] ##
This is a major update. Switching from integer ids for internal and external use to string ids for the external use while keeping the old integer ids for internal use.
### Added ###
* Table `entity_ids`
* Procedure `getIdByName`
* Procedure `insertEntityDataType` and `insertEntityCollection`
* Procedure `setFileProperties`.
### Changed ###
* Rename PROCEDURE `registerSubdomain` to `registerReplacementIds`.
* Change column `entities.role` datatype: replace 'DOMAIN' enum value with `_REPLACEMENT`.
* Change column `transactions_log.entity_id` to VARCHAR(255)
* Change column `user_info.entity` to VARCHAR(255)
* Change signature of procedure `applyIDFilter` - `EntityID VARCHAR(255)`
* Change signature of procedure `initBackreference` - `EntityID VARCHAR(255)` and `PropertyID VARCHAR(255)`
* Change signature of procedure `initPOVPropertiesTable` - `PropertyID VARCHAR(255)`
* Change signature of procedure `initPOVRefidsTable` - `PropertyID VARCHAR(255)`
* Change signature of procedure `initSubEntity` - `EntityID VARCHAR(255)`
* Change signature of procedure `deleteEntity` - `EntityID VARCHAR(255)`
* Change signature of procedure `deleteEntityProperties` - `EntityID VARCHAR(255)`
* Change signature of procedure `get_primary_parent_version` - `EntityID VARCHAR(255)`
* Change signature of procedure `get_version_timestamp` - `EntityID VARCHAR(255)`
* Change signature of procedure `get_head_version` - `EntityID VARCHAR(255)`
* Change signature of procedure `get_head_relative` - `EntityID VARCHAR(255)`
* Change signature of procedure `get_version_history` - `EntityID VARCHAR(255)`
* Change signature of procedure `retrieveQueryTemplateDef` - `EntityID VARCHAR(255)`
* Change signature of procedure `getDependentEntities` - `EntityID VARCHAR(255)`
* Change signature of procedure `insertEntity` - Add parameter `EntityID VARCHAR(255)`
* Change signature of procedure `insertEntityProperty` - `EntityID VARCHAR(255)` and `PropertyID VARCHAR(255)`, `DomainID VARCHAR(255)`, `DatatypeOverride VARCHAR(255)`
* Change signature of procedure `insertIsa` - `ChildID VARCHAR(255)`, `ParentID VARCHAR(255)`.
* Change signature of procedure `isSubtype` - `ChildID VARCHAR(255)`, `ParentID VARCHAR(255)`.
* Change signature of procedure `retrieveEntity` - `EntityID VARCHAR(255)`
* Change signature of procedure `retrieveOverrides` - `EntityID VARCHAR(255)`, `DomainID VARCHAR(255)`
* Change signature of procedure `retrieveEntityParents` - `EntityID VARCHAR(255)`
* Change signature of procedure `retrieveEntityProperties` - `EntityID VARCHAR(255)`, `DomainID VARCHAR(255)`
* Change signature of procedure `updateEntity` - `EntityID VARCHAR(255)`
### Removed ###
* Deactivate procedure `delete_all_entity_versions`. This might be used in the
future, that is why we keep the code in a comment.
* Drop procedure `retrieveSubEntity`
* Drop procedure `retrieveDatatype`
* Drop procedure `retrieveGroup`
* Drop procedure `getInfo`
* Drop procedure `getRole`
* Drop procedure `setPassword`
* Drop procedure `initAutoIncrement`
* Delete special entity 50 (SQLite Datatype)
* Delete special entity 99 (Work-around for the auto-increment)
### Fixed ###
* Change signature of procedure `getFileIdByPath` - `FilePath TEXT` - this length is needed for the path column of `file_entities`.
## [5.0.0] - 2021-10-28 ##
### Added ###
* #33 CI pipeline for MySQL (was only MariaDB before).
### Removed ###
* `getRules` procedure and the `rules` table. The jobs rules are being
......@@ -41,8 +176,6 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
* #32 Removed unused `groups` table from installation routines which prevented
the installation with MySQL 8.
### Security ###
## [4.1.0] - 2021-06-11 ##
### Added ###
......@@ -63,31 +196,29 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
* `get_version_history` returns two additional columns, `child_username` and
`child_realm`.
* Added a `versioned` flag to the following procedures:
* `applyBackReference`
* `applyIDFilter`
* `applyPOV`
* `applyRefPOV`
* `makeStmt`
* `calcComplementUnion`
* `calcDifference`
* `calcIntersection`
* `finishSubProperty`
* `getChildren`
* `initEmptyTargetSet`
* `initDisjunctionFilter`
* `initEntity`
* `initQuery`
* `createTmpTable`
* `applyBackReference`
* `applyIDFilter`
* `applyPOV`
* `applyRefPOV`
* `makeStmt`
* `calcComplementUnion`
* `calcDifference`
* `calcIntersection`
* `finishSubProperty`
* `getChildren`
* `initEmptyTargetSet`
* `initDisjunctionFilter`
* `initEntity`
* `initQuery`
* `createTmpTable`
* Added a `direct` column to `archive_isa` table
### Deprecated ###
### Removed ###
* unused procedures:
* `initNegationFilter`
* `initConjunctionFilter`
* `finishNegationFilter`
* `initNegationFilter`
* `initConjunctionFilter`
* `finishNegationFilter`
### Fixed ###
......@@ -107,8 +238,6 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
version resultet in an error. See corresponding test in `caosdb-pyinttest`
`tests/test_versioning.py::test_datatype_without_name`
### Security ###
## [3.0.0] - 2020-09-01 ##
### Added ###
......@@ -126,20 +255,20 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
with a lot of additions to the API. E.g.
* New `entity_version`.
* All `*_data` tables have a new twin, the `archive_*_data` table, where all
old versions of entities are stored. The `*_data` tables only contain the
data of the latest version.
old versions of entities are stored. The `*_data` tables only contain the
data of the latest version.
* Additional `archive_isa` for the history entities' parents.
* Additional `_iversion` column for the `reference_data` table for storing
references to particular versions of an entity.
references to particular versions of an entity.
* New `setFileProperties` and `retrieveQueryTemplateDef` procedures which reduce server code and let the
backend decide which tables to use. Also, this is necessary for the
versioning, because these procedures behave differently depending on the
ENTITY_VERSIONING feature being enabled or disabled.
backend decide which tables to use. Also, this is necessary for the
versioning, because these procedures behave differently depending on the
ENTITY_VERSIONING feature being enabled or disabled.
* Several functions and procedures for the interaction with the
`entity_version` table and the `transactions` table. E.g.
`insert_single_child_version`, `delete_all_entity_versions`,
`get_iversion`, `get_primary_parent_version`, `get_version_timestamp`,
`get_head_version`, `get_head_relative`, `get_version_history`.
`entity_version` table and the `transactions` table. E.g.
`insert_single_child_version`, `delete_all_entity_versions`,
`get_iversion`, `get_primary_parent_version`, `get_version_timestamp`,
`get_head_version`, `get_head_relative`, `get_version_history`.
The versions are tracked internally by the `_iversion` field which is an
integer and which should not be used outside of the backend.
* New makefile targets for testing with MariaDB instance from Docker image: call
......@@ -162,5 +291,3 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
* Bug in `updateEntity.sql` (when updating the primary name without a prior call
to `deleteEntityProperties`). Same thing for `deleteEntity`.
* #21 Bug which prevented deletion of deeply inheriting entities, if versioning was enabled.
### Security ###
CITATION.cff 0 → 100644
View file @ 5a59dc3a
cff-version: 1.2.0
message: "If you use this software, please cite it as below."
authors:
- family-names: Fitschen
given-names: Timm
orcid: https://orcid.org/0000-0002-4022-432X
- family-names: Schlemmer
given-names: Alexander
orcid: https://orcid.org/0000-0003-4124-9649
- family-names: Hornung
given-names: Daniel
orcid: https://orcid.org/0000-0002-7846-6375
- family-names: tom Wörden
given-names: Henrik
orcid: https://orcid.org/0000-0002-5549-578X
- family-names: Spreckelsen
given-names: Florian
orcid: https://orcid.org/0000-0002-6856-2910
- family-names: Parlitz
given-names: Ulrich
orcid: https://orcid.org/0000-0003-3058-1435
- family-names: Luther
given-names: Stefan
orcid: https://orcid.org/0000-0001-7214-8125
title: "LinkAhead - MariaDB Backend"
version: 8.0.0
doi: 10.3390/data4020083
date-released: 2024-10-10
Makefile
View file @ 5a59dc3a
# This file is a part of the Linkahead Project.
#
# ** header v3.0
# This file is a part of the CaosDB Project.
#
# Copyright (C) 2024 Daniel Hornung <d.hornung@indiscale.com>
# Copyright (C) 2024 IndiScale GmbH <info@indiscale.com>
# Copyright (C) 2018 Research Group Biomedical Physics,
# Max-Planck-Institute for Dynamics and Self-Organization Göttingen
#
......@@ -18,8 +18,7 @@
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <https://www.gnu.org/licenses/>.
#
# ** end header
#
SHELL=/bin/bash
INSTALL_SQL_FILE=db_5_0.sql
......@@ -34,6 +33,7 @@ test-connection:
.PHONY: upgrade
upgrade:
@cd patches; ./applyPatches.sh --env=../.config
@./utils/make_db sanity_check
.PHONY: install
install: _install _grant upgrade
......@@ -55,23 +55,26 @@ drop-%:
test:
./utils/make_db test --fresh
# Run tests with a database which is started in a Docker container
.PHONY: test-dump-update
test-dump-update:
pytest dump_updates
# Run tests with a database which is started in a MariaDB Docker container
.PHONY: test-docker
test-docker:
@docker kill caosdb-mysqlserver-test || true
@docker container rm caosdb-mysqlserver-test || true
@docker run --name caosdb-mysqlserver-test -p "3306:3306" \
-e MYSQL_ROOT_PASSWORD="pass-for-test" -d mariadb
@docker kill linkahead-mysqlserver-test || true
@docker container rm -v linkahead-mysqlserver-test || true
@docker run --name linkahead-mysqlserver-test -p "3306:3306" \
-e MYSQL_ROOT_PASSWORD="pass-for-test" -d mariadb:11.4
@sleep 10
MAINPATH=$(realpath tests/docker_env) utils/make_db test --fresh
@docker kill caosdb-mysqlserver-test
@docker container rm caosdb-mysqlserver-test
make test-docker-stop
# if automatic stopping failed
.PHONY: test-docker-stop
test-docker-stop:
docker kill caosdb-mysqlserver-test
docker container rm caosdb-mysqlserver-test
docker kill linkahead-mysqlserver-test
docker container rm -v linkahead-mysqlserver-test
# Compile the standalone documentation
.PHONY: doc
......@@ -85,7 +88,9 @@ pipeline-test:
echo 'DATABASE_USER_HOST_LIST="%,"' >> .config
echo "MYSQL_USER_PASSWORD=$(MYSQL_ROOT_PASSWORD)" >> .config
echo "MYSQL_HOST=$(SQL_HOST)" >> .config
echo "MYSQL_OPTS=--protocol=TCP" >> .config
sleep 10
make install
./utils/make_db restore_db tests/example.dump.sql
make upgrade
./tests/test_utils.sh
README.md
View file @ 5a59dc3a
......@@ -3,8 +3,8 @@
## Welcome
This is the **CaosDB MySQL Backend** repository and a part of the
CaosDB project.
This is the **LinkAhead MariaDB Backend** repository and a part of the
LinkAhead project.
## Setup
......@@ -18,30 +18,32 @@ Please refer to the [official documentation](https://docs.indiscale.com/caosdb-m
## Contributing
Thank you very much to all contributers—[past, present](https://gitlab.com/caosdb/caosdb/-/blob/dev/HUMANS.md), and prospective ones.
Thank you very much to all contributers—[past,
present](https://gitlab.com/linkahead/linkahead/-/blob/main/HUMANS.md), and prospective
ones.
### Code of Conduct
By participating, you are expected to uphold our [Code of Conduct](https://gitlab.com/caosdb/caosdb/-/blob/dev/CODE_OF_CONDUCT.md).
By participating, you are expected to uphold our [Code of
Conduct](https://gitlab.com/linkahead/linkahead/-/blob/main/CODE_OF_CONDUCT.md).
### How to Contribute
* You found a bug, have a question, or want to request a feature? Please
[create an issue](https://gitlab.com/caosdb/caosdb-mysqlbackend/-/issues).
* You want to contribute code? Please fork the repository and create a merge
request in GitLab and choose this repository as target. Make sure to select
"Allow commits from members who can merge the target branch" under Contribution
when creating the merge request. This allows our team to work with you on your request.
- If you have a suggestion for the [documentation](https://docs.indiscale.com/caosdb-mysqlbackend/),
the preferred way is also a merge request as describe above (the documentation resides in `src/doc`).
However, you can also create an issue for it.
- You can also contact us at **info (AT) caosdb.org**.
* You found a bug, have a question, or want to request a feature? Please
[create an issue](https://gitlab.com/linkahead/linkahead-mariadbbackend/-/issues).
* You want to contribute code?
* **Forking:** Please fork the repository and create a merge request in GitLab and choose this repository as
target. Make sure to select "Allow commits from members who can merge the target branch" under
Contribution when creating the merge request. This allows our team to work with you on your
request.
* You can also join the LinkAhead community on
[#linkahead:matrix.org](https://matrix.to/#/!unwwlTfOznjEnMMXxf:matrix.org).
## License
* Copyright (C) 2018 Research Group Biomedical Physics, Max Planck Institute
for Dynamics and Self-Organization Göttingen.
* Copyright (C) 2020-2021 Indiscale GmbH <info@indiscale.com>
* Copyright (C) 2020-2025 Indiscale GmbH <info@indiscale.com>
All files in this repository are licensed under a [GNU Affero General Public
License](LICENCE.md) (version 3 or later).
README_SETUP.md
View file @ 5a59dc3a
# Setup of the CaosDB SQL back end
# Setup of the LinkAhead MariaDB backend
## Dependencies
* `MariaDB Client 10.1`, `MySQL Client 5.5`, or later versions. In the case of
MySQL, version 5.6 is recommended.
- make
* `MariaDB Client 11.2` or later.
* make
## Create the configuration
* Create an empty `.config` file. For the default values and the meaning of
......@@ -11,7 +11,7 @@
parameter that you want to change, add a corresponding line in your `.config`
file. You probably want to change the passwords. As the passwords are stored
unencrypted in the `.config` file, make sure nobody else can read it.
* If there is no `mysql-config-editor` (`MySQL 5.5`. and `MariaDB`) then the
* If there is no `mysql-config-editor` program, then the
`MYSQL_USER_PASSWORD` must be provided, that is the password of the
`MYSQL_USER`.
* If you are using MariaDB and the `root` database user uses pam
......@@ -21,7 +21,7 @@
## Setup the SQL database for CaosDB
* Run `make install`. If a there is a database with the name you have choosen
* Run `make install`. If a there is a database with the name you have chosen
during the configuration, you need to reconfigure or delete the database
first.
* *Required database privileges:*
......@@ -87,19 +87,46 @@ with the then current version of the stored entities.
* Alternatively, to run the tests in a containerized MariaDB instance, run `make test-docker`,
followed by `make test-docker-stop`.
### Running in a Docker Container
You can use `.docker/docker-compose.yml` to start a docker container
(`docker-compose -f .docker/docker-compose.yml up -d`) that runs mariadb. You
need appropriate settings in `.config`:
* `MYSQL_OPTS="--protocol=TCP"` and
* `DATABASE_USER_HOST_LIST=%,`
After the first start, you need to install the database: `make install`.
Then, you can connect to it with `mariadb --protocol=TCP -u caosdb -prandom1234
caosdb`.
### Developing tests ###
To use the mytap framework:
```console
$ cd libs; unzip mytap-1.0.zip; cd mytap-1.0
# Use the necessary mariadb options for connecting to the server in the following commands
$ mariadb ... < mytap.sql # Set up the "tap" database and functions.
$ mariadb ... < scripts/autotap.sql # Insert the "autotap" function.
# Create autotap file (for comparison with current state)
$ mariadb ... --raw -B --skip-column-names -e "call tap.autotap('_caosdb_schema_unit_tests')" > test_autotap.current.sql
# Run the autotap tests (may fail unfortunately with current MariaDB):
$ mariadb ... < test_autotap.current.sql
```
### Troubleshooting
#### MySQL has failing tests
#### Failure to restore a database dump created with an older MariaDB version ####
Our test suite is developed with MariaDB. That is why some tests, those which
check constraints based on auto-generated names, fail because MySQL generates
other names.
Have a look into the `dump_updates/README.md`. In cases of version
incompatibilities, the necessary steps to migrate the dump are
probably described there and scripts for the migrations are provided
in the same directory.
Also some test fail with the message "Expected: NULL / Was: NULL" which is
obviously what was expected.
#### MySQL has failing tests
Please look carefully through the tests. If something more serious than that
comes up, please report a bug.
*Note: Since we switched from MySQL to MariaDB, the two DBMS projects have diverged quite far. As a
result, we no longer support MySQL.
## Documentation #
......
RELEASE_GUIDELINES.md
View file @ 5a59dc3a
# Release Guidelines for the CaosDB MySQL Backend
# Release Guidelines for the LinkAhead MariaDB backend
This document specifies release guidelines in addition to the general release
guidelines of the CaosDB Project
([RELEASE_GUIDELINES.md](https://gitlab.com/caosdb/caosdb/blob/dev/RELEASE_GUIDELINES.md))
([RELEASE_GUIDELINES.md](https://gitlab.com/caosdb/caosdb-meta/-/blob/dev/RELEASE_GUIDELINES.md))
## General Prerequisites
* All tests are passing.
* All tests are passing. (Note that pipelines may fail if the major
version increased. Rely on manual tests in that case.)
* (FEATURES.md is up-to-date and a public API is being declared in that document.)
* CHANGELOG.md is up-to-date.
* (DEPENDENCIES.md is up-to-date.)
......@@ -19,10 +20,23 @@ guidelines of the CaosDB Project
2. Check all general prerequisites.
4. Merge the release branch into the main branch.
3. Assure that the latest patch actually updates the CaosDBVersion() procedure
and that the version matches the version you are about to release.
5. Tag the latest commit of the main branch with `v<VERSION>`.
4. Update version in `doc/conf.py` and in `CITATION.cff`.
6. Delete the release branch.
5. Merge the release branch into the main branch.
7. Merge the main branch back into the dev branch.
6. Wait for the main branch pipeline to pass.
7. Tag the latest commit of the main branch with `v<VERSION>`.
8. Delete the release branch.
9. Merge the main branch back into the dev branch and add the `[UNRELEASED]`
section to the CHANGELOG.
10. Create a release at the [gitlab repo](https://gitlab.com/linkahead/linkahead-mariadbbackend/-/releases)
11. If necessary, i.e., if the patch or major versions were increased
with the release, do a LinkAhead server release.
config.defaults
View file @ 5a59dc3a
# This file is a part of the LinkAhead Project.
#
# ** header v2.0
# This file is a part of the CaosDB Project.
#
# Copyright (C) 2024 Daniel Hornung <d.hornung@indiscale.com>
# Copyright (C) 2024 IndiScale GmbH <info@indiscale.com>
# Copyright (C) 2018 Research Group Biomedical Physics,
# Max-Planck-Institute for Dynamics and Self-Organization Göttingen
#
......@@ -15,11 +15,9 @@
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <https://www.gnu.org/licenses/>.
#
# ** end header
#
# # Commands
# The MySQL client program.
......@@ -40,6 +38,12 @@ MYSQL_PORT=3306
# which will then be used by the CaosDB Server.
MYSQL_USER=root
MYSQL_USER_PASSWORD=caosdb1234
# Additional options for the connection
# e.g. if you want to connect a dockerized MariaDB at localhost you need to put
# "--protocol=TCP" here because when host=localhost the connection defaults to
# SOCKET.
# MYSQL_OPTS='--protocol=TCP'
MYSQL_OPTS=
# # DATABASE
# The name of the SQL database.
......
doc/Maintenance.rst
View file @ 5a59dc3a
Maintenance
===========
Creating a Backup
-----------------
You can use the Python script ``utils/backup.py`` to create a backup (SQL dump)
of the SQL-Backend::
./utils/backup.py -d folder/where/the/backup/is/created
You can do this while CaosDB is online.
You can do this while LinkAhead is online.
Restoring a Backup
------------------
CaosDB should be offline for restoring.
LinkAhead should be offline for restoring.
You can use the Bash script ``utils/make_db`` to restore a backup (SQL dump)
of the SQL-Backend::
......
doc/conf.py
View file @ 5a59dc3a
......@@ -22,13 +22,13 @@ import sphinx_rtd_theme
# -- Project information -----------------------------------------------------
project = 'caosdb-mysqlbackend'
copyright = '2021, IndiScale GmbH'
copyright = '2021 - 2024, IndiScale GmbH'
author = 'Daniel Hornung'
# The short X.Y version
version = '5.1'
version = '8.0.1'
# The full version, including alpha/beta/rc tags
release = '5.1.0-SNAPSHOT'
release = '8.0.1-SNAPSHOT'
# -- General configuration ---------------------------------------------------
......@@ -46,7 +46,7 @@ extensions = [
'sphinx.ext.todo',
"sphinx.ext.autodoc",
"recommonmark", # For markdown files.
# 'autoapi.extension',
# 'autoapi.extension',
# 'sphinx.ext.intersphinx',
# 'sphinx.ext.napoleon', # For Google style docstrings
"sphinx_rtd_theme",
......
doc/index.rst
View file @ 5a59dc3a
......@@ -10,7 +10,9 @@ Welcome to documentation of CaosDB's MySQL Backend!
Getting started <README_SETUP>
Concepts <concepts>
Maintenance
API documentation<functions>
API documentation <functions>
Related Projects <related_projects/index>
Back to Overview <https://docs.indiscale.com/>
This documentation helps you to :doc:`get started<README_SETUP>`, explains the most important
:doc:`concepts<concepts>` .
......
doc/related_projects/index.rst 0 → 100644
View file @ 5a59dc3a
Related Projects
++++++++++++++++
.. toctree::
:maxdepth: 2
:caption: Contents:
:hidden:
.. container:: projects
For in-depth documentation for users, administrators and developers, you may want to visit the subproject-specific documentation pages for:
:`Server <https://docs.indiscale.com/caosdb-server>`_: The Java part of the LinkAhead server.
:`WebUI <https://docs.indiscale.com/caosdb-webui>`_: The default web frontend for the LinkAhead server.
:`PyLinkAhead <https://docs.indiscale.com/caosdb-pylib>`_: The LinkAhead Python library.
:`Advanced user tools <https://docs.indiscale.com/caosdb-advanced-user-tools>`_: The advanced Python tools for LinkAhead.
:`LinkAhead Crawler <https://docs.indiscale.com/caosdb-crawler/>`_: The crawler is the main tool for automatic data integration in LinkAhead.
:`LinkAhead <https://docs.indiscale.com/caosdb-deploy>`_: Your all inclusive LinkAhead software package.
:`Back to Overview <https://docs.indiscale.com/>`_: LinkAhead Documentation.
dump_updates/2024-10-02.dump_fix_mariadb_10_6.sh 0 → 100755
View file @ 5a59dc3a
#!/usr/bin/env bash
# This file is a part of the LinkAhead Project.
#
# Copyright (C) 2024 IndiScale GmbH <info@indiscale.com>
# Copyright (C) 2024 Daniel Hornung <d.hornung@indiscale.com>
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as
# published by the Free Software Foundation, either version 3 of the
# License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <https://www.gnu.org/licenses/>.
# About this script
# =================
#
# Fix the use of "offset" as a procedure parameter name. "OFFSET" became a reserved keyword in
# MariaDB 10.6.
#
# This script takes an SQL dump from stdin and prints the fixed SQL to stdout.
#
# Usage
# -----
#
# 2024-10-02.dump_fix_mariadb_10_6.sh < yourdump.sql > yourdump.fixed.sql
set -euo pipefail
IFS=$'\n\t'
script='
/^[[:blank:]]+Offset INT UNSIGNED/s/Offset INT UNSIGNED\) RETURNS varbinary\(255\)/HeadOffset INT UNSIGNED\) RETURNS varbinary(255)/
s/LIMIT 1 OFFSET Offset/LIMIT 1 OFFSET HeadOffset/
'
sed -E -e "$script"
unset script
dump_updates/README.md 0 → 100644
View file @ 5a59dc3a
# SQL dump updates #
## Dump ##
This directory contains scripts to update database dumps to newer versions of the MariaDB server.
## Background ##
In some cases, restoring the database content from existing SQL dumps may not be possible in a
straightforward manner. For those cases, this directory contains scripts to help with known issues.
Examples for problems include:
- New reserved keywords in MariaDB, that were previously used as identifiers in code.
# Known issues and their fix #
- SQL syntax error near `Offset INT UNSIGNED`: If a dump was made before MariaDB 10.6 (LinkAhead <
0.15) and with the SQL backend before 7.0.3, there was a pramater named `Offset`. With MariaDB
10.6 however, `OFFSET` became a reserved keyword. This can be fixed by running
`2024-10-02.dump_fix_mariadb_10_6.sh < yourdump.sql > yourdump.fixed.sql`
dump_updates/test/test_all.py 0 → 100644
View file @ 5a59dc3a
#!/usr/bin/env python3
# This file is a part of the LinkAhead Project.
#
# Copyright (C) 2024 IndiScale GmbH <www.indiscale.com>
# Copyright (C) 2024 Daniel Hornung <d.hornung@indiscale.com>
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as
# published by the Free Software Foundation, either version 3 of the
# License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <https://www.gnu.org/licenses/>.
"""Testing the dump update scripts
As a general rule, dump updaters should be idempotent, so tests should run the update again after
the first checks are successlful.
TODO: Reduce the boilerplate for the tests.
"""
import filecmp
from pathlib import Path
from subprocess import run
from tempfile import NamedTemporaryFile
def get_basedir() -> str:
"""Return the assumped base dir for the dump updates.
"""
path = Path(__file__).parents[1]
return str(path)
def get_test_data(basename: str) -> list[tuple[str, str]]:
"""Return a list of [input, expectedoutput] tuples.
The output may be an empty string if no corresponding file can be found.
"""
basedir = get_basedir()
datadir = Path(basedir) / "test" / "test_data"
results = []
for input_path in datadir.glob(f"{basename}.example*[0-9].sql"):
expected_path = datadir / f"{input_path.name[:-4]}.expected.sql"
if expected_path.exists():
expected = str(expected_path)
else:
expected = ""
results.append((str(input_path), expected))
return results
def test_2024_10_02(tmpdir):
"""``Offset`` became a reserved keyword in MariaDB 10.6.
"""
script = "2024-10-02.dump_fix_mariadb_10_6.sh"
script_fullname = str(Path(get_basedir()) / script)
test_data = get_test_data(script[:-3])
for infile, expectedfile in test_data:
with (NamedTemporaryFile(dir=tmpdir, suffix=".sql", delete=True) as output,
open(infile, mode="rb") as infile_stream
):
run([script_fullname],
stdin=infile_stream,
stdout=output,
check=True
)
assert filecmp.cmp(output.name, expectedfile), "Output does not match expected output."
with (NamedTemporaryFile(dir=tmpdir, suffix=".sql", delete=True) as output2,
open(output.name, mode="rb") as infile_stream
):
run([script_fullname],
stdin=infile_stream,
stdout=output2,
check=True
)
assert filecmp.cmp(output2.name, expectedfile), (
"Run 2: Output does not match expected output.")
dump_updates/test/test_data/2024-10-02.dump_fix_mariadb_10_6.example1.expected.sql 0 → 100644
View file @ 5a59dc3a
/* Just a short snippt with the problem */
/*!50003 SET @saved_col_connection = @@collation_connection */ ;
/*!50003 SET character_set_client = utf8 */ ;
/*!50003 SET character_set_results = utf8 */ ;
/*!50003 SET collation_connection = utf8_general_ci */ ;
DELIMITER ;;
CREATE DEFINER=`root`@`%` FUNCTION `get_head_relative`(EntityID VARCHAR(255),
HeadOffset INT UNSIGNED) RETURNS varbinary(255)
READS SQL DATA
BEGIN
DECLARE InternalEntityID INT UNSIGNED DEFAULT NULL;
SELECT internal_id INTO InternalEntityID FROM entity_ids WHERE id = EntityID;
RETURN (
SELECT e.version
FROM entity_version AS e
WHERE e.entity_id = InternalEntityID
ORDER BY e._iversion DESC
LIMIT 1 OFFSET HeadOffset
);
END ;;
DELIMITER ;
/*!50003 SET sql_mode = @saved_sql_mode */ ;
/*!50003 SET character_set_client = @saved_cs_client */ ;
/*!50003 SET character_set_results = @saved_cs_results */ ;
/*!50003 SET collation_connection = @saved_col_connection */ ;
/*!50003 SET @saved_sql_mode = @@sql_mode */ ;
/*!50003 SET sql_mode = 'STRICT_TRANS_TABLES,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION' */ ;
/*!50003 D */
dump_updates/test/test_data/2024-10-02.dump_fix_mariadb_10_6.example1.sql 0 → 100644
View file @ 5a59dc3a
/* Just a short snippt with the problem */
/*!50003 SET @saved_col_connection = @@collation_connection */ ;
/*!50003 SET character_set_client = utf8 */ ;
/*!50003 SET character_set_results = utf8 */ ;
/*!50003 SET collation_connection = utf8_general_ci */ ;
DELIMITER ;;
CREATE DEFINER=`root`@`%` FUNCTION `get_head_relative`(EntityID VARCHAR(255),
Offset INT UNSIGNED) RETURNS varbinary(255)
READS SQL DATA
BEGIN
DECLARE InternalEntityID INT UNSIGNED DEFAULT NULL;
SELECT internal_id INTO InternalEntityID FROM entity_ids WHERE id = EntityID;
RETURN (
SELECT e.version
FROM entity_version AS e
WHERE e.entity_id = InternalEntityID
ORDER BY e._iversion DESC
LIMIT 1 OFFSET Offset
);
END ;;
DELIMITER ;
/*!50003 SET sql_mode = @saved_sql_mode */ ;
/*!50003 SET character_set_client = @saved_cs_client */ ;
/*!50003 SET character_set_results = @saved_cs_results */ ;
/*!50003 SET collation_connection = @saved_col_connection */ ;
/*!50003 SET @saved_sql_mode = @@sql_mode */ ;
/*!50003 SET sql_mode = 'STRICT_TRANS_TABLES,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION' */ ;
/*!50003 D */