Merge branch 'dev' into f-real-id

.docker/docker-compose.yml

+version: '3.6'
+services:
+  sqldb:
+    image: mariadb:10.5
+    volumes:
+      - type: volume
+        source: "caosdb-sqldata"
+        target: /var/lib/mysql
+    environment:
+      MYSQL_ROOT_PASSWORD: caosdb1234
+    networks:
+      # available on port 3306, host name 'sqldb'
+      - caosnet
+    ports:
+      - 3306:3306
+
+
+# A well-defined network for caosdb
+volumes:
+    caosdb-sqldata:
+networks:
+  caosnet:
+    driver: bridge

.gitlab-ci.yml

@@ -62,21 +62,6 @@ unittests-mariadb:
   script:
     - make pipeline-test SQL_HOST=mariadb
 
-# Run the unit tests with MySQL 8
-unittests-mysql-8:
-  tags: [ docker ]
-  stage: test
-  # Should not stop the pipeline from continuing.
-  allow_failure: true
-  services:
-    - name: mysql:8.0
-      command: ["--default-authentication-plugin=mysql_native_password"]
-
-  script:
-    - sed "s/NO_AUTO_CREATE_USER,//" -i tests/example.dump.sql
-    - rm tests/test_autotap.sql
-    - make pipeline-test SQL_HOST=mysql
-
 # Run the unit tests with MySQL 5
 unittests-mysql-5:
   tags: [ docker ]
@@ -84,7 +69,7 @@ unittests-mysql-5:
   # Should not stop the pipeline from continuing.
   allow_failure: true
   services:
-    - name: mysql:5.7
+    - name: mysql:5.7.36
       command: ["--default-authentication-plugin=mysql_native_password"]
 
   script:

.gitlab/issue_templates/Default.md

+## Summary
+
+*Please give a short summary of what the issue is.*
+
+## Expected Behavior
+
+*What did you expect how the software should behave?*
+
+## Actual Behavior
+
+*What did the software actually do?*
+
+## Steps to Reproduce the Problem
+
+*Please describe, step by step, how others can reproduce the problem.  Please try these steps for yourself on a clean system.*
+
+1.
+2.
+3.
+
+## Specifications
+
+- Version: *Which version of this software?*
+- Platform: *Which operating system, which other relevant software versions?*
+
+## Possible fixes
+
+*Do you have ideas how the issue can be resolved?*

.gitlab/merge_request_templates/Default.md

+# Summary
+
+*Insert a meaningful description for this merge request here:  What is the new/changed behavior?
+Which bug has been fixed? Are there related issues?*
+
+
+# Focus
+
+*Point the reviewer to the core of the code change. Where should they start reading? What should
+they focus on (e.g. security, performance, maintainability, user-friendliness, compliance with the
+specs, finding more corner cases, concrete questions)?*
+
+
+# Test Environment
+
+*How to set up a test environment for manual testing?*
+
+
+# Check List for the Author
+
+Please, prepare your MR for a review. Be sure to write a summary and a focus and create gitlab
+comments for the reviewer. They should guide the reviewer through the changes, explain your changes
+and also point out open questions. For further good practices have a look at [our review
+guidelines](https://gitlab.com/caosdb/caosdb/-/blob/dev/REVIEW_GUIDELINES.md)
+
+- [ ] All automated tests pass
+- [ ] Reference related issues
+- [ ] Up-to-date CHANGELOG.md (or not necessary)
+- [ ] Up-to-date JSON schema (or not necessary)
+- [ ] Appropriate user and developer documentation (or not necessary)
+  - Update / write published documentation (`make doc`).
+  - How do I use the software?  Assume "stupid" users.
+  - How do I develop or debug the software?  Assume novice developers.
+- [ ] Annotations in code (Gitlab comments)
+  - Intent of new code
+  - Problems with old code
+  - Why this implementation?
+
+
+# Check List for the Reviewer
+
+- [ ] I understand the intent of this MR
+- [ ] All automated tests pass
+- [ ] Up-to-date CHANGELOG.md (or not necessary)
+- [ ] Appropriate user and developer documentation (or not necessary), also in published
+      documentation.
+- [ ] The test environment setup works and the intended behavior is reproducible in the test
+  environment
+- [ ] In-code documentation and comments are up-to-date.
+- [ ] Check: Are there specifications? Are they satisfied?
+
+For further good practices have a look at [our review guidelines](https://gitlab.com/caosdb/caosdb/-/blob/dev/REVIEW_GUIDELINES.md).
+
+
+/assign me
+/target_branch dev

CHANGELOG.md

@@ -19,16 +19,124 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Security ###
 
-## [5.0.0] - 2021-10-28 ##
+## [7.0.2] - 2023-12-12 ##
+(Timm Fitschen)
+
+### Fixed ###
+
+* make install: Could not connect to MariaDB in local docker
+  container
+  [linkahead-mariadbbackend](https://gitlab.com/linkahead/linkahead-mariadbbackend/-/issues/33)
+* Wrong ids in data_type table due to a bug in the `linkahead-server < v0.11.0`
+  which has been fixed but left broken data. This resulted in server errors
+  [linkahead-mariadbbackend#34](https://gitlab.com/linkahead/linkahead-mariadbbackend/-/issues/34)
+
+## [7.0.1] - 2023-11-01
+(Timm Fitschen)
 
 ### Added ###
 
-* #33 CI pipeline for MySQL (was only MariaDB before).
+* Sanity checks during `make install` and `make upgrade`.
+
+## [7.0.0] - 2023-10-25 ##
+(Timm Fitschen)
 
 ### Changed ###
 
+* Change signature of procedure `insertEntityDataType`: expect the data type ID
+  instead of the name.
+* Change result set of procedure `retrieveEntity`: Rename column `Datatype` to
+  `DatatypeName` and add column  `DatatypeID`.
+* Change result set of procedure `retrieveOverrides`: Rename column
+  `type_override` to `type_name_override` and add `type_id_override`.
+* Change signature of procedure `updateEntity`: expect the data type ID instead
+  to the name.
+* Internal ids, when returned by `retrieveEntityProperties`,
+  `registerReplacementIds`, or `retrieveOverrides`, are prefixed by a special
+  character which serves as a ephemeral marker. Currently this is `$` but this
+  may change in the future if there is a more suitable candidate.
+
 ### Deprecated ###
 
+* MySQL Support. Last version which is known to work well with LinkAhead is MySQL 5.7.36
+
+### Fixed ###
+
+* Unknown Server Error when inserting an Entity.
+  [linkahead-mariadbbackend#48](https://gitlab.indiscale.com/caosdb/src/caosdb-mysqlbackend/-/issues/48).
+* Fix broken `initEntity` procedure (and change the first parameters type from
+  INT UNSIGNED to VARCHAR(255) which is non-breaking).
+
+## [6.0.1 2023-10-18] ##
+
+### Fixed ###
+* Old passwords table is removed from dumps where it still exists.
+
+## [6.0.0 2023-10-17] ##
+
+This is a major update. Switching from integer ids for internal and external use to string ids for the external use while keeping the old integer ids for internal use.
+
+### Added ###
+
+* Table `entity_ids`
+* Procedure `getIdByName`
+* Procedure `insertEntityDataType` and `insertEntityCollection`
+* Procedure `setFileProperties`.
+
+### Changed ###
+
+* Rename PROCEDURE `registerSubdomain` to `registerReplacementIds`.
+* Change column `entities.role` datatype: replace 'DOMAIN' enum value with `_REPLACEMENT`.
+* Change column `transactions_log.entity_id` to VARCHAR(255)
+* Change column `user_info.entity` to VARCHAR(255)
+* Change signature of procedure `applyIDFilter` - `EntityID VARCHAR(255)`
+* Change signature of procedure `initBackreference` - `EntityID VARCHAR(255)` and `PropertyID VARCHAR(255)`
+* Change signature of procedure `initPOVPropertiesTable` - `PropertyID VARCHAR(255)`
+* Change signature of procedure `initPOVRefidsTable` - `PropertyID VARCHAR(255)`
+* Change signature of procedure `initSubEntity` - `EntityID VARCHAR(255)`
+* Change signature of procedure `deleteEntity` - `EntityID VARCHAR(255)`
+* Change signature of procedure `deleteEntityProperties` - `EntityID VARCHAR(255)`
+* Change signature of procedure `get_primary_parent_version` - `EntityID VARCHAR(255)`
+* Change signature of procedure `get_version_timestamp` - `EntityID VARCHAR(255)`
+* Change signature of procedure `get_head_version` - `EntityID VARCHAR(255)`
+* Change signature of procedure `get_head_relative` - `EntityID VARCHAR(255)`
+* Change signature of procedure `get_version_history` - `EntityID VARCHAR(255)`
+* Change signature of procedure `retrieveQueryTemplateDef` - `EntityID VARCHAR(255)`
+* Change signature of procedure `getDependentEntities` - `EntityID VARCHAR(255)`
+* Change signature of procedure `insertEntity` - Add parameter `EntityID VARCHAR(255)`
+* Change signature of procedure `insertEntityProperty` - `EntityID VARCHAR(255)` and `PropertyID VARCHAR(255)`, `DomainID VARCHAR(255)`, `DatatypeOverride VARCHAR(255)`
+* Change signature of procedure `insertIsa` - `ChildID VARCHAR(255)`, `ParentID VARCHAR(255)`.
+* Change signature of procedure `isSubtype` - `ChildID VARCHAR(255)`, `ParentID VARCHAR(255)`.
+* Change signature of procedure `retrieveEntity` - `EntityID VARCHAR(255)`
+* Change signature of procedure `retrieveOverrides` - `EntityID VARCHAR(255)`, `DomainID VARCHAR(255)`
+* Change signature of procedure `retrieveEntityParents` - `EntityID VARCHAR(255)`
+* Change signature of procedure `retrieveEntityProperties` - `EntityID VARCHAR(255)`, `DomainID VARCHAR(255)`
+* Change signature of procedure `updateEntity` - `EntityID VARCHAR(255)`
+
+### Removed ###
+
+* Deactivate procedure `delete_all_entity_versions`. This might be used in the
+  future, that is why we keep the code in a comment.
+* Drop procedure `retrieveSubEntity`
+* Drop procedure `retrieveDatatype`
+* Drop procedure `retrieveGroup`
+* Drop procedure `getInfo`
+* Drop procedure `getRole`
+* Drop procedure `setPassword`
+* Drop procedure `initAutoIncrement`
+* Delete special entity 50 (SQLite Datatype)
+* Delete special entity 99 (Work-around for the auto-increment)
+
+### Fixed ###
+
+* Change signature of procedure `getFileIdByPath` - `FilePath TEXT` - this length is needed for the path column of `file_entities`.
+
+## [5.0.0] - 2021-10-28 ##
+
+### Added ###
+
+* #33 CI pipeline for MySQL (was only MariaDB before).
+
 ### Removed ###
 
 * `getRules` procedure and the `rules` table. The jobs rules are being
@@ -41,8 +149,6 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 * #32 Removed unused `groups` table from installation routines which prevented
   the installation with MySQL 8.
 
-### Security ###
-
 ## [4.1.0] - 2021-06-11 ##
 
 ### Added ###
@@ -80,8 +186,6 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
     * `createTmpTable`
 * Added a `direct` column to `archive_isa` table
 
-### Deprecated ###
-
 ### Removed ###
 
 * unused procedures:
@@ -107,8 +211,6 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
   version resultet in an error. See corresponding test in `caosdb-pyinttest`
   `tests/test_versioning.py::test_datatype_without_name`
 
-### Security ###
-
 ## [3.0.0] - 2020-09-01 ##
 
 ### Added ###
@@ -162,5 +264,3 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 * Bug in `updateEntity.sql` (when updating the primary name without a prior call
   to `deleteEntityProperties`). Same thing for `deleteEntity`.
 * #21 Bug which prevented deletion of deeply inheriting entities, if versioning was enabled.
-
-### Security ###

Makefile

@@ -34,6 +34,7 @@ test-connection:
 .PHONY: upgrade
 upgrade:
 	@cd patches; ./applyPatches.sh --env=../.config
+	@./utils/make_db sanity_check
 
 .PHONY: install
 install: _install _grant upgrade
@@ -85,7 +86,9 @@ pipeline-test:
 	echo 'DATABASE_USER_HOST_LIST="%,"' >> .config
 	echo "MYSQL_USER_PASSWORD=$(MYSQL_ROOT_PASSWORD)" >> .config
 	echo "MYSQL_HOST=$(SQL_HOST)" >> .config
+	echo "MYSQL_OPTS=--protocol=TCP" >> .config
 	sleep 10
 	make install
 	./utils/make_db restore_db tests/example.dump.sql
+	make upgrade
 	./tests/test_utils.sh

README.md

@@ -3,8 +3,8 @@
 
 ## Welcome
 
-This is the **CaosDB MySQL Backend** repository and a part of the
-CaosDB project.
+This is the **LinkAhead MySQL Backend** repository and a part of the
+LinkAhead project.
 
 ## Setup
 
@@ -18,24 +18,26 @@ Please refer to the [official documentation](https://docs.indiscale.com/caosdb-m
 
 ## Contributing
 
-Thank you very much to all contributers—[past, present](https://gitlab.com/caosdb/caosdb/-/blob/dev/HUMANS.md), and prospective ones.
+Thank you very much to all contributers—[past,
+present](https://gitlab.com/linkahead/linkahead/-/blob/main/HUMANS.md), and prospective
+ones.
 
 ### Code of Conduct
 
-By participating, you are expected to uphold our [Code of Conduct](https://gitlab.com/caosdb/caosdb/-/blob/dev/CODE_OF_CONDUCT.md).
+By participating, you are expected to uphold our [Code of
+Conduct](https://gitlab.com/linkahead/linkahead/-/blob/main/CODE_OF_CONDUCT.md).
 
 ### How to Contribute
 
 * You found a bug, have a question, or want to request a feature? Please 
-[create an issue](https://gitlab.com/caosdb/caosdb-mysqlbackend/-/issues).
-* You want to contribute code? Please fork the repository and create a merge
-request in GitLab and choose this repository as target. Make sure to select
-"Allow commits from members who can merge the target branch" under Contribution
-when creating the merge request. This allows our team to work with you on your request.
-- If you have a suggestion for the [documentation](https://docs.indiscale.com/caosdb-mysqlbackend/),
-the preferred way is also a merge request as describe above (the documentation resides in `src/doc`).
-However, you can also create an issue for it.
-- You can also contact us at **info (AT) caosdb.de**.
+[create an issue](https://gitlab.com/linkahead/linkahead-mariadbbackend/-/issues).
+* You want to contribute code?
+    * **Forking:** Please fork the repository and create a merge request in GitLab and choose this repository as
+      target. Make sure to select "Allow commits from members who can merge the target branch" under
+      Contribution when creating the merge request. This allows our team to work with you on your
+      request.
+* You can also  join the LinkAhead community on
+  [#linkahead:matrix.org](https://matrix.to/#/!unwwlTfOznjEnMMXxf:matrix.org).
 
 ## License
 

README_SETUP.md

 # Setup of the CaosDB SQL back end
 
 ## Dependencies
-* `MariaDB Client 10.1`, `MySQL Client 5.5`, or later versions.  In the case of
-  MySQL, version 5.6 is recommended.
-- make
+
+* `MariaDB Client 10.1` or later, `MySQL Client >=5.5, <=5.7.36`.
+* make
 
 ## Create the configuration
 * Create an empty `.config` file. For the default values and the meaning of
@@ -21,9 +21,14 @@
 
 ## Setup the SQL database for CaosDB
 
-* Run `make install`. If a there is a database with the name you have choosen
+* Run `make install`. If a there is a database with the name you have chosen
   during the configuration, you need to reconfigure or delete the database
   first.
+  * *Required database privileges:*
+    * If the user does not exist yet, you need the [appropriate global privileges](https://mariadb.com/kb/en/grant/#global-privileges), for example
+      `CREATE USER` and the privileges to grant that user all global privileges.  The required
+      privileges may be reduced in the future.
+    * For normal usage, [database privileges](https://mariadb.com/kb/en/grant/#database-privileges) are required.
 
 ## Upgrade the SQL database
 
@@ -82,6 +87,19 @@ with the then current version of the stored entities.
     * Alternatively, to run the tests in a containerized MariaDB instance, run `make test-docker`,
       followed by `make test-docker-stop`.
 
+
+### Running in a Docker Container
+
+You can use `.docker/docker-compose.yml` to start a docker container
+(`docker-compose -f .docker/docker-compose.yml up -d`) that runs mariadb. You
+need appropriate settings in `.config`:
+* `MYSQL_OPTS="--protocol=TCP"` and
+* `DATABASE_USER_HOST_LIST=%,`
+After the first start, you need to install the database: `make install`.
+Then, you can connect to it with `mariadb --protocol=TCP -u caosdb -prandom1234
+caosdb`.
+
+
 ### Troubleshooting
 
 #### MySQL has failing tests

RELEASE_GUIDELINES.md

@@ -2,7 +2,7 @@
 
 This document specifies release guidelines in addition to the general release
 guidelines of the CaosDB Project
-([RELEASE_GUIDELINES.md](https://gitlab.com/caosdb/caosdb/blob/dev/RELEASE_GUIDELINES.md))
+([RELEASE_GUIDELINES.md](https://gitlab.com/caosdb/caosdb-meta/-/blob/dev/RELEASE_GUIDELINES.md))
 
 ## General Prerequisites
 
@@ -19,10 +19,20 @@ guidelines of the CaosDB Project
 
 2. Check all general prerequisites.
 
-4. Merge the release branch into the main branch.
+3. Assure that the latest patch actually updates the CaosDBVersion() procedure
+   and that the version matches the version you are about to release.
 
-5. Tag the latest commit of the main branch with `v<VERSION>`.
+4. Update version in `doc/conf.py`.
 
-6. Delete the release branch.
+5. Merge the release branch into the main branch.
 
-7. Merge the main branch back into the dev branch.
+6. Wait for the main branch pipeline to pass.
+
+7. Tag the latest commit of the main branch with `v<VERSION>`.
+
+8. Delete the release branch.
+
+9. Merge the main branch back into the dev branch and add the `[UNRELEASED]`
+   section to the CHANGELOG.
+
+10. Create a release at the [gitlab repo](https://gitlab.com/linkahead/linkahead-mariadbbackend/-/releases)

config.defaults

@@ -40,6 +40,12 @@ MYSQL_PORT=3306
 # which will then be used by the CaosDB Server.
 MYSQL_USER=root
 MYSQL_USER_PASSWORD=caosdb1234
+# Additional options for the connection
+# e.g. if you want to connect a dockerized MariaDB at localhost you need to put
+# "--protocol=TCP" here because when host=localhost the connection defaults to
+# SOCKET.
+# MYSQL_OPTS='--protocol=TCP'
+MYSQL_OPTS=
 
 # # DATABASE
 # The name of the SQL database.

doc/conf.py

@@ -26,9 +26,9 @@ copyright = '2021, IndiScale GmbH'
 author = 'Daniel Hornung'
 
 # The short X.Y version
-version = '5.1'
+version = '7.0.2'
 # The full version, including alpha/beta/rc tags
-release = '5.1.0-SNAPSHOT'
+release = '7.0.2'
 
 
 # -- General configuration ---------------------------------------------------

patches/patch20221122-6.0-SNAPSHOT/create_entity_ids_table.sql

+/*
+ * This file is a part of the CaosDB Project.
+ *
+ * Copyright (C) 2022-2023 IndiScale GmbH <info@indiscale.com>
+ * Copyright (C) 2022-2023 Timm Fitschen <t.fitschen@indiscale.com>
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <https://www.gnu.org/licenses/>.
+ */
+
+-- a little bit of house keeping
+DROP PROCEDURE IF EXISTS retrieveSubEntity;
+DROP PROCEDURE IF EXISTS retrieveDatatype;
+DROP PROCEDURE IF EXISTS retrieveGroup;
+DROP PROCEDURE IF EXISTS getInfo;
+DROP PROCEDURE IF EXISTS getRole;
+DROP PROCEDURE IF EXISTS setPassword;
+DROP PROCEDURE IF EXISTS initAutoIncrement;
+DELETE FROM name_data WHERE entity_id=50;
+DELETE FROM entities WHERE id=50;
+DELETE FROM entities WHERE id=99;
+
+-- this simply is the more appropriate name
+ALTER TABLE entities MODIFY COLUMN
+  `role` enum('RECORDTYPE','RECORD','FILE','DOMAIN','PROPERTY','DATATYPE','ROLE','QUERYTEMPLATE', '_REPLACEMENT') COLLATE utf8_unicode_ci NOT NULL;
+ALTER TABLE archive_entities MODIFY COLUMN
+  `role` enum('RECORDTYPE','RECORD','FILE','DOMAIN','PROPERTY','DATATYPE','ROLE','QUERYTEMPLATE', '_REPLACEMENT') COLLATE utf8_unicode_ci NOT NULL;
+UPDATE entities SET role = "_REPLACEMENT" WHERE role="DOMAIN";
+UPDATE archive_entities SET role = "_REPLACEMENT" WHERE role="DOMAIN";
+ALTER TABLE entities MODIFY COLUMN
+  `role` enum('RECORDTYPE','RECORD','FILE','_REPLACEMENT','PROPERTY','DATATYPE','ROLE','QUERYTEMPLATE') COLLATE utf8_unicode_ci NOT NULL;
+ALTER TABLE archive_entities MODIFY COLUMN
+  `role` enum('RECORDTYPE','RECORD','FILE','_REPLACEMENT','PROPERTY','DATATYPE','ROLE','QUERYTEMPLATE') COLLATE utf8_unicode_ci NOT NULL;
+
+
+
+-- new entity_ids table
+DROP TABLE IF EXISTS `entity_ids`;
+CREATE TABLE `entity_ids` (
+    `id` VARCHAR(255) NOT NULL,
+    `internal_id` int(10) unsigned NOT NULL COMMENT 'Internal ID of an entity. This id is used internally in the *_data tables and elsewhere. This ID is never exposed via the CaosDB API.',
+    PRIMARY KEY `entity_ids_pk` (`id`),
+    CONSTRAINT `entity_ids_internal_id` FOREIGN KEY (`internal_id`) REFERENCES `entities` (`id`)
+) ENGINE=InnoDB COLLATE utf8mb4_bin;
+
+-- fill all existing entities into the new entity_ids table.
+INSERT INTO entity_ids (id, internal_id) SELECT id, id FROM entities WHERE id>0 AND role!="_REPLACEMENT";
+
+ALTER TABLE transaction_log MODIFY COLUMN `entity_id` VARCHAR(255) COLLATE utf8mb4_bin NOT NULL;
+
+ALTER TABLE user_info DROP FOREIGN KEY `subjects_ibfk_1`;
+ALTER TABle user_info MODIFY COLUMN `entity` VARCHAR(255) COLLATE utf8mb4_bin DEFAULT NULL;
+ALTER TABLE user_info ADD CONSTRAINT `subjects_ibfk_2` FOREIGN KEY (`entity`) REFERENCES `entity_ids` (`id`);

patches/patch20221122-6.0-SNAPSHOT/patch.sh

+#!/bin/bash
+#
+# This file is a part of the CaosDB Project.
+#
+# Copyright (C) 2023 IndiScale GmbH <info@indiscale.com>
+# Copyright (C) 2023 Timm Fitschen <t.fitschen@indiscale.com>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see <https://www.gnu.org/licenses/>.
+#
+
+# Update mysql schema to version v5.0.0
+# Drop the 'rules' table.
+
+NEW_VERSION="v6.0.0-SNAPSHOT-EXTIDS"
+OLD_VERSION="v5.0.0"
+
+DROP_TABLE_PASSWORDS='DROP TABLE IF EXISTS passwords;'
+
+if [ -z "$UTILSPATH" ]; then
+ UTILSPATH="../utils"
+fi
+
+. $UTILSPATH/patch_header.sh $*
+
+check_version $OLD_VERSION
+
+mysql_execute "$DROP_TABLE_PASSWORDS"
+mysql_execute_file $PATCH_DIR/create_entity_ids_table.sql
+
+update_version $NEW_VERSION
+
+
+success
+

patches/patch20231018-6.1.0/patch.sh

+#!/bin/bash
+#
+# This file is a part of the CaosDB Project.
+#
+# Copyright (C) 2023 IndiScale GmbH <info@indiscale.com>
+# Copyright (C) 2023 Timm Fitschen <t.fitschen@indiscale.com>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see <https://www.gnu.org/licenses/>.
+#
+
+# Update mysql schema to version v6.1.0
+# No migration of the schema, updating procedures only
+
+NEW_VERSION="v6.1.0"
+OLD_VERSION="v6.0.0-SNAPSHOT-EXTIDS"
+
+if [ -z "$UTILSPATH" ]; then
+ UTILSPATH="../utils"
+fi
+
+. $UTILSPATH/patch_header.sh $*
+
+check_version $OLD_VERSION
+
+# Drop insertUser procedure
+mysql_execute "DROP PROCEDURE IF EXISTS insertUser;"
+
+update_version $NEW_VERSION
+
+success
+

patches/patch20231025-7.0.0/patch.sh

+#!/bin/bash
+#
+# This file is a part of the CaosDB Project.
+#
+# Copyright (C) 2023 IndiScale GmbH <info@indiscale.com>
+# Copyright (C) 2023 Timm Fitschen <t.fitschen@indiscale.com>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see <https://www.gnu.org/licenses/>.
+#
+
+# Update mysql schema to version v7.0.0
+# No migration of the schema, updating procedures only
+
+NEW_VERSION="v7.0.0"
+OLD_VERSION="v6.1.0"
+
+if [ -z "$UTILSPATH" ]; then
+ UTILSPATH="../utils"
+fi
+
+. $UTILSPATH/patch_header.sh $*
+
+check_version $OLD_VERSION
+
+update_version $NEW_VERSION
+
+success
+

patches/patch20231101-7.0.1/patch.sh

+#!/bin/bash
+#
+# This file is a part of the CaosDB Project.
+#
+# Copyright (C) 2023 IndiScale GmbH <info@indiscale.com>
+# Copyright (C) 2023 Timm Fitschen <t.fitschen@indiscale.com>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see <https://www.gnu.org/licenses/>.
+#
+
+# Update mysql schema to version v7.0.1
+# Removing remnant tables. This patch version also adds new sanity checks
+# during installation/upgrade
+
+NEW_VERSION="v7.0.1"
+OLD_VERSION="v7.0.0"
+
+if [ -z "$UTILSPATH" ]; then
+ UTILSPATH="../utils"
+fi
+
+. $UTILSPATH/patch_header.sh $*
+
+check_version $OLD_VERSION
+
+mysql_execute "DROP TABLE IF EXISTS groups";
+mysql_execute "DROP TABLE IF EXISTS passwords";
+mysql_execute "DROP TABLE IF EXISTS logging";
+mysql_execute "DROP TABLE IF EXISTS isa";
+
+update_version $NEW_VERSION
+
+success

patches/patch20231211-7.0.2/patch.sh

+#!/bin/bash
+#
+# This file is a part of the CaosDB Project.
+#
+# Copyright (C) 2023 IndiScale GmbH <info@indiscale.com>
+# Copyright (C) 2023 Timm Fitschen <t.fitschen@indiscale.com>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see <https://www.gnu.org/licenses/>.
+#
+
+# Update mysql schema to version v7.0.2
+# Patch the broken data_type table (fix id for file references as per
+# https://gitlab.com/linkahead/linkahead-server/-/issues/246)
+
+NEW_VERSION="v7.0.2"
+OLD_VERSION="v7.0.1"
+
+if [ -z "$UTILSPATH" ]; then
+ UTILSPATH="../utils"
+fi
+
+. $UTILSPATH/patch_header.sh $*
+
+check_version $OLD_VERSION
+
+mysql_execute "UPDATE data_type SET datatype=17 WHERE datatype=3"
+
+update_version $NEW_VERSION
+
+success

procedures/deleteEntity.sql

 /*
- * ** header v3.0
  * This file is a part of the CaosDB Project.
  *
  * Copyright (C) 2018 Research Group Biomedical Physics,
  * Max-Planck-Institute for Dynamics and Self-Organization Göttingen
- * Copyright (C) 2020 IndiScale GmbH <info@indiscale.com>
+ * Copyright (C) 2020-2023 IndiScale GmbH <info@indiscale.com>
+ * Copyright (C) 2023 Timm Fitschen <t.fitschen@indiscale.com>
  *
  * This program is free software: you can redistribute it and/or modify
  * it under the terms of the GNU Affero General Public License as
@@ -18,8 +18,6 @@
  *
  * You should have received a copy of the GNU Affero General Public License
  * along with this program. If not, see <https://www.gnu.org/licenses/>.
- *
- * ** end header
  */
 
 /* Delete a (sparse) Entity from the database.
@@ -31,37 +29,43 @@ have been deleted before.  This can be done for example with the
 Parameters
 ==========
 
-EntityID : UNSIGNED
+EntityID : VARCHAR(255)
     The ID of the Entity.
 */
 
 DROP PROCEDURE IF EXISTS db_5_0.deleteEntity;
 delimiter //
 
-CREATE PROCEDURE db_5_0.deleteEntity(in EntityID INT UNSIGNED)
+CREATE PROCEDURE db_5_0.deleteEntity(in EntityID VARCHAR(255))
 BEGIN
+    DECLARE InternalEntityID INT UNSIGNED DEFAULT NULL;
+
+    SELECT internal_id INTO InternalEntityID from entity_ids WHERE id = EntityID;
 
     -- detele file properties
-    DELETE FROM files where file_id=EntityID;
+    DELETE FROM files where file_id=InternalEntityID;
 
     -- delete datatype stuff
     DELETE FROM data_type
         WHERE ( domain_id = 0
             AND entity_id = 0
-            AND property_id = EntityID )
-        OR datatype = EntityID;
+            AND property_id = InternalEntityID )
+        OR datatype = InternalEntityID;
     DELETE FROM collection_type
         WHERE domain_id = 0
         AND entity_id = 0
-        AND property_id = EntityID;
+        AND property_id = InternalEntityID;
 
     -- delete primary name (in case this is called without a prior call to deleteEntityProperties)
     DELETE FROM name_data
         WHERE domain_id = 0
-        AND entity_id = EntityID
+        AND entity_id = InternalEntityID
         AND property_id = 20;
 
-    DELETE FROM entities where id=EntityID;
+    DELETE FROM entity_ids
+        WHERE internal_id = InternalEntityID;
+
+    DELETE FROM entities where id=InternalEntityID;
 
     -- clean up unused acl
     DELETE FROM entity_acl

procedures/deleteEntityProperties.sql

 /*
- * ** header v3.0
  * This file is a part of the CaosDB Project.
  *
  * Copyright (C) 2018 Research Group Biomedical Physics,
  * Max-Planck-Institute for Dynamics and Self-Organization Göttingen
- * Copyright (C) 2020 IndiScale GmbH <info@indiscale.com>
- * Copyright (C) 2020 Timm Fitschen <t.fitschen@indiscale.com>
+ * Copyright (C) 2020,2023 IndiScale GmbH <info@indiscale.com>
+ * Copyright (C) 2020,2023 Timm Fitschen <t.fitschen@indiscale.com>
  *
  * This program is free software: you can redistribute it and/or modify
  * it under the terms of the GNU Affero General Public License as
@@ -19,23 +18,34 @@
  *
  * You should have received a copy of the GNU Affero General Public License
  * along with this program. If not, see <https://www.gnu.org/licenses/>.
- *
- * ** end header
  */
 
 DROP PROCEDURE IF EXISTS db_5_0.deleteEntityProperties;
 delimiter //
 
-CREATE PROCEDURE db_5_0.deleteEntityProperties(in EntityID INT UNSIGNED)
+/*
+ * Delete all properties of an entity (i.e. remove them from the *_data tables
+ * and the isa_cache table).
+ *
+ * Parameters
+ * ==========
+ *
+ * EntityID : VARCHAR(255)
+ *    The entity's id.
+ */
+CREATE PROCEDURE db_5_0.deleteEntityProperties(in EntityID VARCHAR(255))
 BEGIN
     DECLARE IVersion INT UNSIGNED DEFAULT NULL;
+    DECLARE InternalEntityID INT UNSIGNED DEFAULT NULL;
+
+    SELECT internal_id INTO InternalEntityID from entity_ids WHERE id = EntityID;
 
-    CALL deleteIsa(EntityID);
+    CALL deleteIsa(InternalEntityID);
 
     IF is_feature_config("ENTITY_VERSIONING", "ENABLED") THEN
         SELECT max(e._iversion) INTO IVersion -- What's the latest version?
             FROM entity_version AS e
-            WHERE e.entity_id = EntityID;
+            WHERE e.entity_id = InternalEntityID;
 
         -- Copy the rows from *_data to archive_*_data ---------------------
         INSERT INTO archive_reference_data (domain_id, entity_id,
@@ -43,142 +53,142 @@ BEGIN
             SELECT domain_id, entity_id, property_id, value, value_iversion,
                 status, pidx, IVersion AS _iversion
             FROM reference_data
-            WHERE (domain_id = 0 AND entity_id = EntityID)
-            OR domain_id = EntityID;
+            WHERE (domain_id = 0 AND entity_id = InternalEntityID)
+            OR domain_id = InternalEntityID;
 
         INSERT INTO archive_null_data (domain_id, entity_id,
                 property_id, status, pidx, _iversion)
             SELECT domain_id, entity_id, property_id, status,
                 pidx, IVersion AS _iversion
             FROM null_data
-            WHERE (domain_id = 0 AND entity_id = EntityID)
-            OR domain_id = EntityID;
+            WHERE (domain_id = 0 AND entity_id = InternalEntityID)
+            OR domain_id = InternalEntityID;
 
         INSERT INTO archive_text_data (domain_id, entity_id,
                 property_id, value, status, pidx, _iversion)
             SELECT domain_id, entity_id, property_id, value, status,
                 pidx, IVersion AS _iversion
             FROM text_data
-            WHERE (domain_id = 0 AND entity_id = EntityID)
-            OR domain_id = EntityID;
+            WHERE (domain_id = 0 AND entity_id = InternalEntityID)
+            OR domain_id = InternalEntityID;
 
         INSERT INTO archive_name_data (domain_id, entity_id,
                 property_id, value, status, pidx, _iversion)
             SELECT domain_id, entity_id, property_id, value, status,
                 pidx, IVersion AS _iversion
             FROM name_data
-            WHERE (domain_id = 0 AND entity_id = EntityID)
-            OR domain_id = EntityID;
+            WHERE (domain_id = 0 AND entity_id = InternalEntityID)
+            OR domain_id = InternalEntityID;
 
         INSERT INTO archive_enum_data (domain_id, entity_id,
                 property_id, value, status, pidx, _iversion)
             SELECT domain_id, entity_id, property_id, value, status,
                 pidx, IVersion AS _iversion
             FROM enum_data
-            WHERE (domain_id = 0 AND entity_id = EntityID)
-            OR domain_id = EntityID;
+            WHERE (domain_id = 0 AND entity_id = InternalEntityID)
+            OR domain_id = InternalEntityID;
 
         INSERT INTO archive_integer_data (domain_id, entity_id,
                 property_id, value, status, pidx, _iversion, unit_sig)
             SELECT domain_id, entity_id, property_id, value, status,
                 pidx, IVersion AS _iversion, unit_sig
             FROM integer_data
-            WHERE (domain_id = 0 AND entity_id = EntityID)
-            OR domain_id = EntityID;
+            WHERE (domain_id = 0 AND entity_id = InternalEntityID)
+            OR domain_id = InternalEntityID;
 
         INSERT INTO archive_double_data (domain_id, entity_id,
                 property_id, value, status, pidx, _iversion, unit_sig)
             SELECT domain_id, entity_id, property_id, value, status,
                 pidx, IVersion AS _iversion, unit_sig
             FROM double_data
-            WHERE (domain_id = 0 AND entity_id = EntityID)
-            OR domain_id = EntityID;
+            WHERE (domain_id = 0 AND entity_id = InternalEntityID)
+            OR domain_id = InternalEntityID;
 
         INSERT INTO archive_datetime_data (domain_id, entity_id,
                 property_id, value, value_ns, status, pidx, _iversion)
             SELECT domain_id, entity_id, property_id, value, value_ns,
                 status, pidx, IVersion AS _iversion
             FROM datetime_data
-            WHERE (domain_id = 0 AND entity_id = EntityID)
-            OR domain_id = EntityID;
+            WHERE (domain_id = 0 AND entity_id = InternalEntityID)
+            OR domain_id = InternalEntityID;
 
         INSERT INTO archive_date_data (domain_id, entity_id,
                 property_id, value, status, pidx, _iversion)
             SELECT domain_id, entity_id, property_id, value, status,
                 pidx, IVersion AS _iversion
             FROM date_data
-            WHERE (domain_id = 0 AND entity_id = EntityID)
-            OR domain_id = EntityID;
+            WHERE (domain_id = 0 AND entity_id = InternalEntityID)
+            OR domain_id = InternalEntityID;
 
         INSERT INTO archive_name_overrides (domain_id, entity_id,
                 property_id, name, _iversion)
             SELECT domain_id, entity_id, property_id, name,
                 IVersion AS _iversion
             FROM name_overrides
-            WHERE (domain_id = 0 AND entity_id = EntityID)
-            OR domain_id = EntityID;
+            WHERE (domain_id = 0 AND entity_id = InternalEntityID)
+            OR domain_id = InternalEntityID;
 
         INSERT INTO archive_desc_overrides (domain_id, entity_id,
                 property_id, description, _iversion)
             SELECT domain_id, entity_id, property_id, description,
                 IVersion AS _iversion
             FROM desc_overrides
-            WHERE (domain_id = 0 AND entity_id = EntityID)
-            OR domain_id = EntityID;
+            WHERE (domain_id = 0 AND entity_id = InternalEntityID)
+            OR domain_id = InternalEntityID;
 
         INSERT INTO archive_data_type (domain_id, entity_id,
                 property_id, datatype, _iversion)
             SELECT domain_id, entity_id, property_id, datatype,
                 IVersion AS _iversion
             FROM data_type
-            WHERE (domain_id = 0 AND entity_id = EntityID)
-            OR domain_id = EntityID;
+            WHERE (domain_id = 0 AND entity_id = InternalEntityID)
+            OR domain_id = InternalEntityID;
 
         INSERT INTO archive_collection_type (domain_id, entity_id,
                 property_id, collection, _iversion)
             SELECT domain_id, entity_id, property_id, collection,
                 IVersion AS _iversion
             FROM collection_type
-            WHERE (domain_id = 0 AND entity_id = EntityID)
-            OR domain_id = EntityID;
+            WHERE (domain_id = 0 AND entity_id = InternalEntityID)
+            OR domain_id = InternalEntityID;
 
         INSERT INTO archive_query_template_def (id, definition, _iversion)
             SELECT id, definition, IVersion AS _iversion
             FROM query_template_def
-            WHERE id = EntityID;
+            WHERE id = InternalEntityID;
 
     END IF;
 
     DELETE FROM reference_data
-    where (domain_id=0 AND entity_id=EntityID) OR domain_id=EntityID;
+    where (domain_id=0 AND entity_id=InternalEntityID) OR domain_id=InternalEntityID;
     DELETE FROM null_data
-    where (domain_id=0 AND entity_id=EntityID) OR domain_id=EntityID;
+    where (domain_id=0 AND entity_id=InternalEntityID) OR domain_id=InternalEntityID;
     DELETE FROM text_data
-    where (domain_id=0 AND entity_id=EntityID) OR domain_id=EntityID;
+    where (domain_id=0 AND entity_id=InternalEntityID) OR domain_id=InternalEntityID;
     DELETE FROM name_data
-    where (domain_id=0 AND entity_id=EntityID) OR domain_id=EntityID;
+    where (domain_id=0 AND entity_id=InternalEntityID) OR domain_id=InternalEntityID;
     DELETE FROM enum_data
-    where (domain_id=0 AND entity_id=EntityID) OR domain_id=EntityID;
+    where (domain_id=0 AND entity_id=InternalEntityID) OR domain_id=InternalEntityID;
     DELETE FROM integer_data
-    where (domain_id=0 AND entity_id=EntityID) OR domain_id=EntityID;
+    where (domain_id=0 AND entity_id=InternalEntityID) OR domain_id=InternalEntityID;
     DELETE FROM double_data
-    where (domain_id=0 AND entity_id=EntityID) OR domain_id=EntityID;
+    where (domain_id=0 AND entity_id=InternalEntityID) OR domain_id=InternalEntityID;
     DELETE FROM datetime_data
-    where (domain_id=0 AND entity_id=EntityID) OR domain_id=EntityID;
+    where (domain_id=0 AND entity_id=InternalEntityID) OR domain_id=InternalEntityID;
     DELETE FROM date_data
-    where (domain_id=0 AND entity_id=EntityID) OR domain_id=EntityID;
+    where (domain_id=0 AND entity_id=InternalEntityID) OR domain_id=InternalEntityID;
 
     DELETE FROM name_overrides
-    WHERE (domain_id=0 AND entity_id=EntityID) OR domain_id=EntityID;
+    WHERE (domain_id=0 AND entity_id=InternalEntityID) OR domain_id=InternalEntityID;
     DELETE FROM desc_overrides
-    WHERE (domain_id=0 AND entity_id=EntityID) OR domain_id=EntityID;
+    WHERE (domain_id=0 AND entity_id=InternalEntityID) OR domain_id=InternalEntityID;
 
     DELETE FROM data_type
-    WHERE (domain_id=0 AND entity_id=EntityID) OR domain_id=EntityID;
+    WHERE (domain_id=0 AND entity_id=InternalEntityID) OR domain_id=InternalEntityID;
     DELETE FROM collection_type
-    WHERE (domain_id=0 AND entity_id=EntityID) OR domain_id=EntityID;
+    WHERE (domain_id=0 AND entity_id=InternalEntityID) OR domain_id=InternalEntityID;
 
-    DELETE FROM query_template_def WHERE id=EntityID;
+    DELETE FROM query_template_def WHERE id=InternalEntityID;
 
 END;
 //