diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 0852bbaa5e63ed6ceae1fc8ddad79db80d9fc16a..d3331a3b24145747836c0898ff852a8ade438bb4 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -76,20 +76,21 @@ trigger_build: # Build the sphinx documentation and make it ready for deployment by Gitlab Pages # Special job for serving a static website. See https://docs.gitlab.com/ee/ci/yaml/README.html#pages -pages: +pages_prepare: &pages_prepare tags: [ cached-dind ] stage: deploy only: refs: - /^release-.*$/i - - master - variables: - # run pages only on gitlab.com - - $CI_SERVER_HOST == "gitlab.com" script: - - echo "Deploying" + - echo "Deploying..." - make doc - cp -r build/doc/html public artifacts: paths: - public +pages: + <<: *pages_prepare + only: + refs: + - main diff --git a/CHANGELOG.md b/CHANGELOG.md index 7996c53a80d5a7a363baa5bcddbec7e4bf0a6440..df5b85f849385868b6eb8e1d7daa605bc1b4e8d8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,9 +20,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 to determine whether the server's state has changed between queries. * Basic caching for queries. The caching is enabled by default and can be controlled by the usual "cache" flag. +* Documentation for the overall server structure. * Add `BEFORE`, `AFTER`, `UNTIL`, `SINCE` keywords for query transaction - filters. - ### Changed diff --git a/README_SETUP.md b/README_SETUP.md index 60fb212a62f99fdf7b9fe97b1895c39336359e23..37d3e4786ee17db6792c084d21132600be87e1ed 100644 --- a/README_SETUP.md +++ b/README_SETUP.md @@ -197,7 +197,11 @@ Stand-alone documentation is built using Sphinx: `make doc` ### Requirements ## +- plantuml +- recommonmark - sphinx +- sphinx-rtd-theme +- sphinxcontrib-plantuml - javasphinx :: `pip3 install --user javasphinx` - Alternative, if javasphinx fails because python3-sphinx is too recent: (`l_` not found): diff --git a/src/doc/CaosDB-Query-Language.md b/src/doc/CaosDB-Query-Language.md index 07fbdbc310b8d22de4642f041918710e6e707488..ffa889c6b8acd0cd635049ed2aaceb748d56f2a9 100644 --- a/src/doc/CaosDB-Query-Language.md +++ b/src/doc/CaosDB-Query-Language.md @@ -121,7 +121,7 @@ Examples: ##### `d1>d2`: Transitive, non-symmetric relation. Semantics depend on the flavors of d1 and d2. If both are... ###### [UTCDateTime](Datatype#datetime) -* ''True'' iff the time of d1 is after the the time of d2 according to [https://en.wikipedia.org/wiki/Coordinated_Universal_Time](UTC) +* ''True'' iff the time of d1 is after the the time of d2 according to [UTC](https://en.wikipedia.org/wiki/Coordinated_Universal_Time). * ''False'' otherwise. ###### [SemiCompleteDateTime](Datatype#datetime) diff --git a/src/doc/Permissions.rst b/src/doc/Permissions.rst index a5fae0bbfaf8d021691f7f8af93c3db9b5995497..2a1278ec450a9bf38ee03546b7301f2aabd76151 100644 --- a/src/doc/Permissions.rst +++ b/src/doc/Permissions.rst @@ -1,4 +1,5 @@ -#Permissions +Permissions +=========== CaosDB has a fine grained role based permission system. Each interaction with the server is governed by the current rules of the user, by default diff --git a/src/doc/administration/configuration.rst b/src/doc/administration/configuration.rst index bbafcf22bd8576e141cb9d7e8388212ccf24d934..196acb34ee9afbe6b35d530b058fac6c275299aa 100644 --- a/src/doc/administration/configuration.rst +++ b/src/doc/administration/configuration.rst @@ -19,18 +19,21 @@ In each of these directories, the server looks for the following files: ``server.conf`` General server configuration options. The possible configuration options are documented inside - the `default file <https://gitlab.com/caosdb/caosdb-server/-/blob/dev/conf/core/server.conf>`_. + the `default file + <https://gitlab.indiscale.com/caosdb/src/caosdb-server/-/blob/dev/conf/core/server.conf>`__. ``global_entity_permissions.xml`` - :ref:`Permissions <concepts:Permissions>` which are automatically set, based on user roles. - See the `default file <https://gitlab.com/caosdb/caosdb-server/-/blob/dev/conf/core/global_entity_permissions.xml>`_. + :doc:`Permissions<../Permissions>` which are automatically set, based on user roles. See the + `default file + <https://gitlab.indiscale.com/caosdb/src/caosdb-server/-/blob/dev/conf/core/global_entity_permissions.xml>`__. ``usersources.ini`` This file defines possible sources which are checked when a user tries to authenticate. Each defined source has a special section, the possible options are defined separately for each user source. At the moment the best place to look for this specific documentation is at the API documentation of :java:type:`UserSource` and its implementing classes. The provided `template - file <https://gitlab.com/caosdb/caosdb-server/-/blob/dev/conf/core/usersources.ini.template>`_ + file + <https://gitlab.indiscale.com/caosdb/src/caosdb-server/-/blob/dev/conf/core/usersources.ini.template>`__ also has some information. The general concept about authentication realms is described in :java:type:`UserSources`. @@ -38,13 +41,13 @@ In each of these directories, the server looks for the following files: Configuration for dispensed authentication tokens, which can be used to authenticate to CaosDB without the need of a user/password combination. Possible use cases are server-side scripts or initial setup after the server start. There is more documentation inside the `template file - <https://gitlab.com/caosdb/caosdb-server/-/blob/dev/conf/core/authtoken.example.yaml>`_. + <https://gitlab.indiscale.com/caosdb/src/caosdb-server/-/blob/dev/conf/core/authtoken.example.yaml>`__. ``cache.ccf`` Configuration for the Java Caching System (JCS) which can be used by the server. More documentation is `upstream <http://commons.apache.org/proper/commons-jcs/getting_started/intro.html>`_ and inside `the file - <https://gitlab.com/caosdb/caosdb-server/-/blob/dev/conf/core/cache.ccf>`_. + <https://gitlab.indiscale.com/caosdb/src/caosdb-server/-/blob/dev/conf/core/cache.ccf>`_. ``log4j2-default.properties``, ``log4j2-debug.properties`` Configuration for logging, following the standard described by the `log4j library diff --git a/src/doc/administration/maintenance.rst b/src/doc/administration/maintenance.rst index 67d8475bf469957416a6f42e495db07b1533f151..5b9faeb1c5da3e68def79acae2d6901efa2e78de 100644 --- a/src/doc/administration/maintenance.rst +++ b/src/doc/administration/maintenance.rst @@ -21,12 +21,17 @@ The command could look like:: tar czvf /path/to/new/backup /path/to/caosdb/filesystem.tar.gz -You can also save the content of CaosDB using XML. This is **not recommended** since it is less reliable than a real SQL backup. However there may be cases in which an XML backup is desirable, e.g., when transferring entities between two different CaosDB instances. -Collect the entities that you want to export in a :any:`caosdb-pylib:caosdb.common.models.Container`, here -named ``cont``. Then you can export the XML with:: +You can also save the content of CaosDB using XML. This is **not recommended** since it produces +less reproducible results than a plain SQL backup. However there may be cases in which an XML backup +is necessary, e.g., when transferring entities between two different CaosDB instances. + +Collect the entities that you want to export in a +:any:`Container<caosdb-pylib:caosdb.common.models.Container>`, named ``cont`` here. Then you can +export the XML with:: from caosadvancedtools.export_related import invert_ids from lxml import etree + invert_ids(cont) xml = etree.tounicode(cont.to_xml( local_serialization=True), pretty_print=True) @@ -38,11 +43,11 @@ named ``cont``. Then you can export the XML with:: Restoring a Backup ------------------ -.. note : +.. warning:: CaosDB should be offline before restoring data. If you want to restore the internal file system, simply replace it. E.g. if your -Backup is a tarball:: +backup is a tarball:: tar xvf /path/to/caosroot.tar.gz @@ -52,16 +57,17 @@ You find the documentation on how to restore the data in the SQL-Backend :any:` If you want to restore the entities exported to XML, you can do:: - cont = db.Container() - with open("caosdb_data.xml") as fi: - cont = cont.from_xml(fi.read()) + cont = db.Container() + with open("caosdb_data.xml") as fi: + cont = cont.from_xml(fi.read()) cont.insert() - + User Management --------------- -The configuration of authentication mechanisms is done via the -``usersources.ini`` file (see :any:`configuration`). -We recommend the Python tools (:any:`caosdb-pylib:Administration`) for further administrative tasks (e.g. setting -user passwords). +The configuration of authentication mechanisms is done via the ``usersources.ini`` file (see +:any:`configuration`). + +We recommend the Python tools (:any:`caosdb-pylib:administration`) for further administrative tasks +(e.g. setting user passwords). diff --git a/src/doc/administration/server_side_scripting.rst b/src/doc/administration/server_side_scripting.rst index 1f2b9a1d80e11abed48ae89502ab6f80c099507e..6dc08999f38e1912e72440c609b8cde1ea7ce0bb 100644 --- a/src/doc/administration/server_side_scripting.rst +++ b/src/doc/administration/server_side_scripting.rst @@ -31,7 +31,7 @@ The script has to be executable and must be placed somewhere in one of the direc Users will need the ``SCRIPTING:EXECUTE:path:to:the:script`` permission. Here the path to the script is of course relativet to the ``SERVER_SIDE_SCRIPTING_BIN_DIRS`` where it is located. -For more information see the :doc:`specification of the API <../specs/Server-side-scripting>` +For more information see the :doc:`specification of the API <../specification/Server-side-scripting>` Environment ------------ diff --git a/src/doc/conf.py b/src/doc/conf.py index 7b6ac359abf8d88d3c60cb37001bc43e00b97872..46580733a72952fff673567ba1564903b60d584c 100644 --- a/src/doc/conf.py +++ b/src/doc/conf.py @@ -47,6 +47,7 @@ extensions = [ "recommonmark", # For markdown files. "sphinx.ext.autosectionlabel", # Allow reference sections using its title "sphinx_rtd_theme", + "sphinxcontrib.plantuml", # PlantUML diagrams ] # Add any paths that contain templates here, relative to this directory. @@ -190,14 +191,18 @@ epub_exclude_files = ['search.html'] # '<namespace_here>' : ('<base_url_here>', 'javadoc'), # } +javadoc_url_map = { + 'org.restlet': ('https://javadocs.restlet.talend.com/2.4/jse/api', 'javadoc'), +} + # -- Options for intersphinx ------------------------------------------------- # https://www.sphinx-doc.org/en/master/usage/extensions/intersphinx.html#confval-intersphinx_mapping intersphinx_mapping = { "python": ("https://docs.python.org/", None), - "caosdb-pylib": ("https://caosdb.gitlab.io/caosdb-pylib/", None), - "caosdb-mysqlbackend": ("https://caosdb.gitlab.io/caosdb-mysqlbackend/", None), + "caosdb-pylib": ("https://docs.indiscale.com/caosdb-pylib/", None), + "caosdb-mysqlbackend": ("https://docs.indiscale.com/caosdb-mysqlbackend/", None), } diff --git a/src/doc/development/benchmarking.md b/src/doc/development/benchmarking.md index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..0be781453a6f85577dd95f89844fd95f2b4141ba 100644 --- a/src/doc/development/benchmarking.md +++ b/src/doc/development/benchmarking.md @@ -0,0 +1,4 @@ +# Benchmarking CaosDB # + +Please refer to the file `doc/devel/Benchmarking.md` in the CaosDB sources for developer resources +how to do benchmarking and profiling of CaosDB. diff --git a/src/doc/development/devel.rst b/src/doc/development/devel.rst new file mode 100644 index 0000000000000000000000000000000000000000..141b045594fe6f8a58c86f758734e160e5a8c2fe --- /dev/null +++ b/src/doc/development/devel.rst @@ -0,0 +1,25 @@ +Developing CaosDB +================= + +.. toctree:: + :glob: + :maxdepth: 2 + + Structure of the Java code <structure> + Benchmarking CaosDB <benchmarking> + +CaosDB is an Open-Source project, so anyone may modify the source as they like. These pages aim to +provide some help for all CaosDB developers. + +More generally, these are the most common ways to contribute to CaosDB: + +- You found a bug, have a question, or want to request a feature? Please `create an issue + <https://gitlab.com/caosdb/caosdb-server/-/issues>`_. +- You want to contribute code? Please fork the repository and create a merge request in GitLab and + choose this repository as target. Make sure to select "Allow commits from members who can merge + the target branch" under Contribution when creating the merge request. This allows our team to + work with you on your request. +- If you have a suggestion for this `documentation <https://docs.indiscale.com/caosdb-server/>`_, + the preferred way is also a merge request as describe above (the documentation resides in + ``src/doc``). However, you can also create an issue for it. +- You can also contact the developers at *info (AT) caosdb.de*. diff --git a/src/doc/development/structure.rst b/src/doc/development/structure.rst new file mode 100644 index 0000000000000000000000000000000000000000..3848e983fca5d05871212e0985b98d539c04b7fe --- /dev/null +++ b/src/doc/development/structure.rst @@ -0,0 +1,178 @@ +CaosDB's Internal Structure +=========================== + +The CaosDB server + +- builds upon the `Restlet <https://restlet.talend.com/>`_ framework to provide a REST interface to + the network. See the :ref:`HTTP Resources` section for more information. +- uses an SQL database (MariaDB or MySQL) as the backend for data storage. This is documented in + the :ref:`MySQL Backend` section. +- has an internal scheduling framework to organize the required backend jobs. Read more on this in + the :ref:`Transactions and Schedules<transactions>` section. +- may use a number of authentication providers. Documentation for this is still to come. + +.. _HTTP Resources: + +HTTP Resources +-------------- + +HTTP resources are implemented in the :java:ref:`resource<org.caosdb.server.resource>` package, in +classes inheriting from :java:ref:`AbstractCaosDBServerResource` (which inherits Restlet's +:java:ref:`Resource<org.restlet.resource.Resource>` class). The main :java:ref:`CaosDBServer` class +defines which HTTP resource (for example ``/Entity/{specifier}``) will be handled by which class +(:java:ref:`EntityResource` in this case). + +Implementing classes need to overwrite for example the ``httpGetInChildClass()`` method (or methods +corresponding to other HTTP request methods such as POST, PUT, ...). Typically, they might call the +``execute()`` method of a :java:ref:`Transaction<org.caosdb.server.transaction.Transaction>` object. +Transactions are explained in detail in the :ref:`Transactions and Schedules<transactions>` section. + +.. uml:: + + @startuml + abstract AbstractCaosDBServerResource { + {abstract} httpGetInChildClass() + {abstract} httpPostInChildClass() + {abstract} ...InChildClass() + } + abstract RetrieveEntityResource + class EntityResource + AbstractCaosDBServerResource <|-- RetrieveEntityResource + RetrieveEntityResource <|-- EntityResource + @enduml + +.. _MySQL Backend: + +MySQL Backend +------------- + +The MySQL backend in CaosDB may be substituted by other backends, but at the time of writing this +documentation, only MySQL (MariaDB is used for testing) is implemented. There are the following +main packages which handle the backend: + +:java:ref:`backend.interfaces<interfaces>` + Interfaces which backends may implement. The main method for most interfaces is + ``execute(...)`` with arguments depending on the specific interface, and benchmarking methods + (``getBenchmark()`` and ``setTransactionBenchmark(b)`` may also be required. + +:java:ref:`backend.implementation.MySQL<MySQL>` + MySQL implementations of the interfaces. Typical "simple" implementations create a prepared SQL + statement from the arguments to ``execute(...)`` and send it to the SQL server. They may also + have methods for undoing and cleanup, using an :java:ref:`UndoHandler`. + +:java:ref:`backend.transaction<backend.transaction>` classes + Subclasses of the abstract :java:ref:`BackendTransaction` which implement the ``execute()`` + method. These classes may use specific backend implementations (like for example the MySQL + implementations) to interact with the backend database. + +For example, the structure when getting an Entity ID by name looks like this: + +.. uml:: + + @startuml + together { + abstract BackendTransaction { + HashMap impl // stores all implementations + {abstract} execute() + } + note left of BackendTransaction::impl + Stores the + implementation + for each + interface." + end note + package ...backend.interfaces { + interface GetIDByNameImpl { + {abstract} execute(String name, String role, String limit) + } + } + } + together { + package ...backend.transaction { + class GetIDByName extends BackendTransaction { + execute() + } + } + package ...backend.implementation.MySQL { + class MySQLGetIDByName implements GetIDByNameImpl { + execute(String name, String role, String limit) + } + } + } + + GetIDByName::execute --r-> MySQLGetIDByName + @enduml + +.. _transactions: + +Transactions and Schedules +-------------------------- + +In CaosDB, several client requests may be handled concurrently. This poses no problem as long as +only read-only requests are processed, but writing transactions need to block other requests. +Therefore all transactions (between their first and last access) block write transactions other than +themselves from writing to the backend, while read transactions may happen at any time, except when +a write transaction actually writes to the backend. + +.. note:: + + There is a fine distinction between write transactions on the CaosDB server and actually writing + to the backend, since even transactions which need only very short write access to the backend may + require extensive read access before, for example to check for permissions or to check if the + intended write action makes sense (linked entities must exist, they may need to be of the correct + RecordType, etc.). + +The request handling in CaosDB is organized in the following way: + +- HTTP resources usually create a :java:ref:`Transaction` object and call its + :java:ref:`Transaction.execute()` method. Entities are passed to and from the transaction via + :java:ref:`TransactionContainers<TransactionContainer>` (basically normal + :java:ref:`Containers<Container>`, enriched with some metadata). +- The Transaction keeps a :java:ref:`Schedule` of related :java:ref:`Jobs<Job>` (each also wrapping + a specific Transaction), which may be called at different stages, called + :java:ref:`TransactionStages<TransactionStage>`. +- The Transaction's ``execute()`` method, when called, in turn calls a number of methods for + initialization, checks, preparations, cleanup etc. Additionally the scheduled jobs are executed + at their specified stages, for example all jobs scheduled for ``INIT`` are executed immediately + after calling ``Transaction.init()``. Please consult the API documentation for + :java:ref:`Transaction.execute()` for details. + + Most importantly, the (abstract) method ``transaction()`` is called by ``execute()``, which in + inheriting classes typically interacts with the backend via :java:ref:`execute(BackendTransaction, + Access)<Transaction.execute(K t, Access access)>`, which in turn calls the + ``BackendTransaction``'s :java:ref:`BackendTransaction.executeTransaction()` method (just a thin + wrapper around its ``execute()`` method). + +Summarized, the classes are connected like this: + +.. uml:: + + @startuml + hide empty members + + class Container + class TransactionContainer extends Container + + abstract Transaction { + Schedule schedule + TransactionContainer container + execute() + execute(BackendTransaction t, Access access)\n // -> t.executeTransaction(t) + } + + class Schedule + class ScheduledJob + abstract Job { + TransactionStage stage + Transaction transaction + execute(BackendTransaction t)\n // -> transaction.execute(t, transaction.access) + } + + Schedule "*" *- ScheduledJob + ScheduledJob *- Job + Job o--d- Transaction + + TransactionContainer -* Transaction::container + Transaction::schedule *- Schedule + @enduml + diff --git a/src/doc/index.rst b/src/doc/index.rst index 870db0251049dfaaff6a6c3a2f2c38b9e95aff97..2ae00f7e685407325903159257d6561912f94a66 100644 --- a/src/doc/index.rst +++ b/src/doc/index.rst @@ -12,15 +12,15 @@ Welcome to caosdb-server's documentation! Concepts <concepts> Query Language <CaosDB-Query-Language> administration - development/* + Development <development/devel> specification/index.rst Glossary API documentation<_apidoc/packages> Welcome to the CaosDB, the flexible semantic data management toolkit! -This documentation helps you to :doc:`get started<getting_started>`, explains the most important -:doc:`concepts<concepts>` and offers a range of :doc:`tutorials<tutorials>`. +This documentation helps you to :doc:`get started<README_SETUP>`, explains the most important +:doc:`concepts<concepts>` and has information if you want to :doc:`develop<development/devel>` CaosDB yourself. Indices and tables diff --git a/src/doc/roles.md b/src/doc/roles.md index b70f54506e8adb3ad3caf46af94479af49370409..5f4641b7a6f6fcb1b63cdd26edb706af117ee7d6 100644 --- a/src/doc/roles.md +++ b/src/doc/roles.md @@ -10,7 +10,7 @@ users may have the same role, and there may be roles without any users. The user and their roles are always returned by the server in answers to requests and can thus be interpreted and used by clients. The most important use though -is [permission](permissions) checking in the server: Access and +is [permission](Permissions) checking in the server: Access and modification of entities can be controlled via roles, so that users of a given role are allowed or denied certain actions. Incidentally, the permission to edit the permissions @@ -32,4 +32,4 @@ There are some special roles, which are automatically assigned to users: Except for the `anonymous` role, these special roles are not returned by the server, but can nevertheless be used to define -[permissions](permissions.rst). +[permissions](Permissions). diff --git a/src/doc/specification/AbstractProperty.md b/src/doc/specification/AbstractProperty.md index d594f81967cacd708699ceaeccd9188685a7184d..a337fa5ed053742a0e0cee4e9771cee862c19d7c 100644 --- a/src/doc/specification/AbstractProperty.md +++ b/src/doc/specification/AbstractProperty.md @@ -1,7 +1,12 @@ +.. note :: + + This document has not been updated for a long time. Although it is concerned with the mostly + stable API, its content may no longer reflect the actual CaosDB behavior. + # AbstractProperty Specification ## Introduction -An `AbstractProperty` is one of the basal [objects of HeartDB](./HeartDBObject). +An `AbstractProperty` is one of the basal objects of HeartDB. An `AbstractProperty` MUST have the following _qualities_ (shortcut in brackets): * a persistent id (`id`) * an unique name (`name`) @@ -54,7 +59,7 @@ Any xml representation of an `AbstractProperty` that is retrieved from the Heart <Property id="$id" name="$name" description="$description" generator="$generator" creator="$creator" created="$created" type="file" /> '''General Notes: -* If the called Property does not exist or if the Property called without permission, the HeartDB Server will return an [Error](./Errorcodes). +* If the called Property does not exist or if the Property called without permission, the HeartDB Server will return an Error. ### POST AbstractProperty Any xml representation of an `AbstractProperty` that is to be posted to the HeartDB server MUST have exactly ONE of the following forms, depending on the `AbstractProperty's` type: @@ -80,7 +85,7 @@ Any xml representation of an `AbstractProperty` that is to be posted to the Hear * The `AbstractProperty's` `id` and timestamp (`created`) will be generated by the HeartDB Server. * The `AbstractProperty's` creator will be determined by the HeartDB Server depending on it's policy configuration. * Any given attribute beyond these will be *ignored*. -* If the `<Property/>` tag isn't compliant with these the HeartDB Server will return an [Error](./Errorcodes). +* If the `<Property/>` tag isn't compliant with these the HeartDB Server will return an Error. ---- ## Examples diff --git a/src/doc/specification/Fileserver.md b/src/doc/specification/Fileserver.md index fda2ec183c909e494a407f74eb1f4d03a5a23625..badb4b413f4ff898ee9455fce303f7bcaad90b9d 100644 --- a/src/doc/specification/Fileserver.md +++ b/src/doc/specification/Fileserver.md @@ -35,7 +35,8 @@ where ### HTTP upload stream #### Files -There is an example on file upload using cURL described in detail in [the curl section of this wiki](manuals/curl/curl-access). +There is an example on file upload using cURL described in detail in [the curl section of this +documentation](../administration/curl-access.md). File upload via HTTP is implemented in a [rfc1867](http://www.ietf.org/rfc/rfc1867.txt) consistent way. This is a de-facto standard that defines a file upload as a part of an HTML form submission. This concept shall not be amplified here. But it has to be noticed that this protocol is not designed for uploads of complete structured folders. Therefore the HeartDB file components have to impose that structure on the upload protocol. diff --git a/src/doc/specification/RecordType.md b/src/doc/specification/RecordType.md index 068c433d8c5390882176c0ff8783576127a2da2e..a7830b863712d8b1db3e54c617572ee16bf4e166 100644 --- a/src/doc/specification/RecordType.md +++ b/src/doc/specification/RecordType.md @@ -1,5 +1,5 @@ # RecordType ----- + ## Overview RecordTypes function as templates for [[Record|Records]], they provide a description for a type of Record and define which [[Property|Properties]] should be present. Properties come with an _importance_ attribute which tells the user or client program how strongly necessary the Property is. (As all other entities,) RecordTypes can be inherited from other RecordTypes (or any Entities). When RecordTypes inherit from other RecordTypes, the _inheritance_ flag tells which properties shall be inherited. diff --git a/src/doc/specification/index.rst b/src/doc/specification/index.rst index e9683072f555725ee8b74a0ae468c6a407e244b2..3609aa4a8eb6165adf6070faae26bfb8bd4ba50f 100644 --- a/src/doc/specification/index.rst +++ b/src/doc/specification/index.rst @@ -8,15 +8,13 @@ Specification :hidden: AbstractProperty - C-Client Fileserver Record - Server-side-scripting Specification of the Entity API <Specification-of-the-Entity-API> Authentication Datatype Paging RecordType - Server side scripting <Server-side-scripting-v0.1> + Server side scripting <Server-side-scripting> Specification of the Message API <Specification-of-the-Message-API> diff --git a/src/main/java/org/caosdb/server/database/BackendTransaction.java b/src/main/java/org/caosdb/server/database/BackendTransaction.java index 61aebf42a2dfd65fb9ed1c84459064e3aa274c06..965e7181d1d010528b341f6cbf34d4c0c67a9e5c 100644 --- a/src/main/java/org/caosdb/server/database/BackendTransaction.java +++ b/src/main/java/org/caosdb/server/database/BackendTransaction.java @@ -145,6 +145,7 @@ public abstract class BackendTransaction implements Undoable { protected abstract void execute(); + /** Like execute(), but with benchmarking measurement. */ public final void executeTransaction() { final long t1 = System.currentTimeMillis(); execute(); diff --git a/src/main/java/org/caosdb/server/jobs/Job.java b/src/main/java/org/caosdb/server/jobs/Job.java index 86ba49fabeedf04833f4e82214876be698aaa8cd..9432fc3916606cddbf64922850d7f2f9ff5284c2 100644 --- a/src/main/java/org/caosdb/server/jobs/Job.java +++ b/src/main/java/org/caosdb/server/jobs/Job.java @@ -60,13 +60,18 @@ import org.reflections.Reflections; * @todo Describe me. */ public abstract class Job { + /** All known Job classes, by name (actually lowercase getSimpleName()). */ + static HashMap<String, Class<? extends Job>> allClasses = null; + + private static List<Class<? extends Job>> loadAlways; + private Transaction<? extends TransactionContainer> transaction = null; private Mode mode = null; + private final TransactionStage stage; + private EntityInterface entity = null; public abstract JobTarget getTarget(); - private EntityInterface entity = null; - protected <S, T> HashMap<S, T> getCache(final String name) { return getTransaction().getCache(name); } @@ -112,9 +117,9 @@ public abstract class Job { protected Job() { if (this.getClass().isAnnotationPresent(JobAnnotation.class)) { - this.time = this.getClass().getAnnotation(JobAnnotation.class).time(); + this.stage = this.getClass().getAnnotation(JobAnnotation.class).stage(); } else { - this.time = JobExecutionTime.CHECK; + this.stage = TransactionStage.CHECK; } } @@ -230,22 +235,33 @@ public abstract class Job { } } - static HashMap<String, Class<? extends Job>> allClasses = null; - private static List<Class<? extends Job>> loadAlways; - + /** + * Create a Job object with the given parameters. + * + * <p>This static method is used by other classes to create Job objects, instead of the private + * constructor. + * + * @return The generated Job object. + */ public static Job getJob( final String job, final Mode mode, final EntityInterface entity, final Transaction<? extends TransactionContainer> transaction) { + // Fill `allClasses` with available subclasses scanJobClasspath(); + // Get matching class for Job and generate it. final Class<? extends Job> jobClass = allClasses.get(job.toLowerCase()); return getJob(jobClass, mode, entity, transaction); } + /** + * Initialize {@code allClasses} with all {@code Job} classes found in the classpath. + * + * @todo Details when this has any effect. + */ private static void scanJobClasspath() { - if (allClasses == null || loadAlways == null) { allClasses = new HashMap<>(); loadAlways = new ArrayList<>(); @@ -452,15 +468,18 @@ public abstract class Job { System.out.println(toString()); } - private final JobExecutionTime time; - - public JobExecutionTime getExecutionTime() { - return this.time; + public TransactionStage getTransactionStage() { + return this.stage; } + /** + * Return those matching jobs which are annotated with the "loadAlways" attribute. + * + * @return A list with the jobs. + */ public static List<Job> loadPermanentContainerJobs(Transaction<?> transaction) { final ArrayList<Job> jobs = new ArrayList<>(); - // load permanent jobs + // load permanent jobs: ContainerJob classes with the correct transaction for (Class<? extends Job> j : loadAlways) { if (ContainerJob.class.isAssignableFrom(j) && j.getAnnotation(JobAnnotation.class).transaction().isInstance(transaction)) { diff --git a/src/main/java/org/caosdb/server/jobs/JobAnnotation.java b/src/main/java/org/caosdb/server/jobs/JobAnnotation.java index 15193dcdd590745773f308c71b9ed3bb1351a413..bebebd75ed386c19f8abfb7c09e802dda5ec2941 100644 --- a/src/main/java/org/caosdb/server/jobs/JobAnnotation.java +++ b/src/main/java/org/caosdb/server/jobs/JobAnnotation.java @@ -26,9 +26,17 @@ import java.lang.annotation.Retention; import java.lang.annotation.RetentionPolicy; import org.caosdb.server.transaction.TransactionInterface; +/** + * Jobs may be annotated with @JobAnnotation(...). + * + * <p>Without a JobAnnotation, the Job will run at the default {@link + * org.caosdb.server.transaction.Transaction#check() CHECK} stage. + * + * @see {@link TransactionStage} + */ @Retention(RetentionPolicy.RUNTIME) public @interface JobAnnotation { - JobExecutionTime time() default JobExecutionTime.CHECK; + TransactionStage stage() default TransactionStage.CHECK; String flag() default ""; diff --git a/src/main/java/org/caosdb/server/jobs/JobExecutionTime.java b/src/main/java/org/caosdb/server/jobs/JobExecutionTime.java deleted file mode 100644 index 9374ab9b1c2f4c285b538b1721441f0bc7272600..0000000000000000000000000000000000000000 --- a/src/main/java/org/caosdb/server/jobs/JobExecutionTime.java +++ /dev/null @@ -1,35 +0,0 @@ -/* - * ** header v3.0 - * This file is a part of the CaosDB Project. - * - * Copyright (C) 2018 Research Group Biomedical Physics, - * Max-Planck-Institute for Dynamics and Self-Organization Göttingen - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of the - * License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see <https://www.gnu.org/licenses/>. - * - * ** end header - */ -package org.caosdb.server.jobs; - -public enum JobExecutionTime { - INIT, - PRE_CHECK, - CHECK, - POST_CHECK, - PRE_TRANSACTION, - TRANSACTION, - POST_TRANSACTION, - CLEANUP, - ROLL_BACK -} diff --git a/src/main/java/org/caosdb/server/jobs/Schedule.java b/src/main/java/org/caosdb/server/jobs/Schedule.java index 77bc57c9e3f4183bf572b6a95dfe16fc79b6e003..0090ec92dffb310052b5b8cc82424e2edd0288f9 100644 --- a/src/main/java/org/caosdb/server/jobs/Schedule.java +++ b/src/main/java/org/caosdb/server/jobs/Schedule.java @@ -30,13 +30,20 @@ import java.util.Map; import java.util.concurrent.CopyOnWriteArrayList; import org.caosdb.server.entity.EntityInterface; +/** + * Keeps track of Jobs. + * + * <p>The Schedule class orders jobs by {@link TransactionStage} and also assures that jobs are + * skipped when appropriate and to prevent that jobs run more than once (because sometimes they + * trigger each other). + */ public class Schedule { private final Map<Integer, List<ScheduledJob>> jobLists = new HashMap<>(); private ScheduledJob running = null; public List<ScheduledJob> addAll(final Collection<Job> jobs) { - List<ScheduledJob> result = new ArrayList<ScheduledJob>(jobs.size()); + final List<ScheduledJob> result = new ArrayList<ScheduledJob>(jobs.size()); for (final Job j : jobs) { result.add(add(j)); } @@ -44,18 +51,19 @@ public class Schedule { } public ScheduledJob add(final Job j) { - ScheduledJob ret = new ScheduledJob(j); - List<ScheduledJob> jobs = jobLists.get(ret.getExecutionTime().ordinal()); + final ScheduledJob ret = new ScheduledJob(j); + List<ScheduledJob> jobs = jobLists.get(ret.getTransactionStage().ordinal()); if (jobs == null) { jobs = new CopyOnWriteArrayList<ScheduledJob>(); - jobLists.put(ret.getExecutionTime().ordinal(), jobs); + jobLists.put(ret.getTransactionStage().ordinal(), jobs); } jobs.add(ret); return ret; } - public void runJobs(final JobExecutionTime time) { - List<ScheduledJob> jobs = this.jobLists.get(time.ordinal()); + /** Run all Jobs from the specified {@link TransactionStage}. */ + public void runJobs(final TransactionStage stage) { + final List<ScheduledJob> jobs = this.jobLists.get(stage.ordinal()); if (jobs != null) { for (final ScheduledJob scheduledJob : jobs) { runJob(scheduledJob); @@ -81,11 +89,14 @@ public class Schedule { } } + /** Run all scheduled Jobs of a given class for the given entity. */ public void runJob(final EntityInterface entity, final Class<? extends Job> jobclass) { - List<ScheduledJob> jobs = + + // the jobs of this class are in the jobList for the TransactionStage of the jobClass. + final List<ScheduledJob> jobs = jobclass.isAnnotationPresent(JobAnnotation.class) - ? this.jobLists.get(jobclass.getAnnotation(JobAnnotation.class).time().ordinal()) - : this.jobLists.get(JobExecutionTime.CHECK.ordinal()); + ? this.jobLists.get(jobclass.getAnnotation(JobAnnotation.class).stage().ordinal()) + : this.jobLists.get(TransactionStage.CHECK.ordinal()); for (final ScheduledJob scheduledJob : jobs) { if (jobclass.isInstance(scheduledJob.job)) { if (scheduledJob.job.getEntity() == entity) { diff --git a/src/main/java/org/caosdb/server/jobs/ScheduledJob.java b/src/main/java/org/caosdb/server/jobs/ScheduledJob.java index ee0c29805d7500987f5823f2c7d3eaec8f6ed50b..65bc3a1947baf5b76539f281e0d855e878a3bdfd 100644 --- a/src/main/java/org/caosdb/server/jobs/ScheduledJob.java +++ b/src/main/java/org/caosdb/server/jobs/ScheduledJob.java @@ -54,10 +54,12 @@ public class ScheduledJob { return this.job.toString(); } + /** Does not actually start the job, but only sets the startTime. */ private void start() { this.startTime = System.currentTimeMillis(); } + /** Calculate and set the runtime, and add the measurement. */ private void finish() { this.runtime += System.currentTimeMillis() - this.startTime; this.job @@ -78,8 +80,9 @@ public class ScheduledJob { return this.startTime != -1; } - public JobExecutionTime getExecutionTime() { - return this.job.getExecutionTime(); + /** Return the state of the inner Job. */ + public TransactionStage getTransactionStage() { + return this.job.getTransactionStage(); } public boolean skip() { diff --git a/src/main/java/org/caosdb/server/jobs/TransactionStage.java b/src/main/java/org/caosdb/server/jobs/TransactionStage.java new file mode 100644 index 0000000000000000000000000000000000000000..8248140d913e08acfb699cbb553a14749c8e2f1f --- /dev/null +++ b/src/main/java/org/caosdb/server/jobs/TransactionStage.java @@ -0,0 +1,63 @@ +/* + * ** header v3.0 + * This file is a part of the CaosDB Project. + * + * Copyright (C) 2018 Research Group Biomedical Physics, + * Max-Planck-Institute for Dynamics and Self-Organization Göttingen + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see <https://www.gnu.org/licenses/>. + * + * ** end header + */ +package org.caosdb.server.jobs; + +import org.caosdb.server.jobs.core.Strict; +import org.caosdb.server.transaction.Transaction; +import org.caosdb.server.utils.UndoHandler; + +/** + * Any {@link Transaction} of an Entity consists of sequence of stages. Jobs which have a {@link + * JobAnnotation} can specify the transaction stage in which they are to be executed. By default, + * any job is executed during the CHECK stage. + * <li>INIT - The transaction is being initialized (create schedule, aquire locks (one writing + * thread, many reading threads permitted). + * <li>PRE_CHECK - Prepare entities (e.g. check if any updates are to be processed, load/generate + * acl, cast objects into more specialized classes.) + * <li>CHECK - Do the actual consistency checking. + * <li>POST_CHECK - Do more consistency checking (reserved for those jobs which need the normal + * checks to be done already, e.g. the {@link Strict} job). + * <li>PRE_TRANSACTION - Prepare the entities for the transaction (e.g. for paging, translate the + * state messages into state properties). Also, in this stage, the full read and write lock is + * aquired. + * <li>TRANSACTION - Do the actual transaction. + * <li>POST_TRANSACTION - Post-process entities (Success messages, write history, transform stage + * properties into messages). + * <li>CLEANUP - Release all locks, remove temporary files, clean-up the {@link UndoHandler}s. + * <li>ROLL_BACK - In case an error occured in any of the stages, this special stage rolls-back any + * changes and calls the {@link UndoHandler#undo()} of the {@link UndoHandler}s. + * + * @see {@link Transaction#execute()}. + * @author Timm Fitschen <t.fitschen@indiscale.com> + */ +public enum TransactionStage { + INIT, + PRE_CHECK, + CHECK, + POST_CHECK, + PRE_TRANSACTION, + TRANSACTION, + POST_TRANSACTION, + CLEANUP, + ROLL_BACK +} diff --git a/src/main/java/org/caosdb/server/jobs/core/AccessControl.java b/src/main/java/org/caosdb/server/jobs/core/AccessControl.java index d1a2113a2f87018cec63d381cce64ad6c8ad09f2..408384f416e1249501d5383ea1028ced56914ded 100644 --- a/src/main/java/org/caosdb/server/jobs/core/AccessControl.java +++ b/src/main/java/org/caosdb/server/jobs/core/AccessControl.java @@ -29,12 +29,12 @@ import org.caosdb.server.entity.EntityInterface; import org.caosdb.server.entity.wrapper.Parent; import org.caosdb.server.jobs.ContainerJob; import org.caosdb.server.jobs.JobAnnotation; -import org.caosdb.server.jobs.JobExecutionTime; +import org.caosdb.server.jobs.TransactionStage; import org.caosdb.server.transaction.Retrieve; import org.caosdb.server.utils.EntityStatus; import org.caosdb.server.utils.ServerMessages; -@JobAnnotation(time = JobExecutionTime.INIT) +@JobAnnotation(stage = TransactionStage.INIT) public class AccessControl extends ContainerJob { @Override diff --git a/src/main/java/org/caosdb/server/jobs/core/Atomic.java b/src/main/java/org/caosdb/server/jobs/core/Atomic.java index 45b5de8e4d5835bd835a797290e2d3c875687e44..8639615ee6ac4ddd648a19d15262a1c526ef10c6 100644 --- a/src/main/java/org/caosdb/server/jobs/core/Atomic.java +++ b/src/main/java/org/caosdb/server/jobs/core/Atomic.java @@ -26,7 +26,7 @@ import org.caosdb.server.entity.Entity; import org.caosdb.server.entity.EntityInterface; import org.caosdb.server.jobs.ContainerJob; import org.caosdb.server.jobs.JobAnnotation; -import org.caosdb.server.jobs.JobExecutionTime; +import org.caosdb.server.jobs.TransactionStage; import org.caosdb.server.transaction.WriteTransaction; import org.caosdb.server.utils.EntityStatus; import org.caosdb.server.utils.Observable; @@ -34,7 +34,7 @@ import org.caosdb.server.utils.Observer; import org.caosdb.server.utils.ServerMessages; @JobAnnotation( - time = JobExecutionTime.PRE_TRANSACTION, + stage = TransactionStage.PRE_TRANSACTION, transaction = WriteTransaction.class, loadAlways = true) public class Atomic extends ContainerJob implements Observer { diff --git a/src/main/java/org/caosdb/server/jobs/core/CheckParValid.java b/src/main/java/org/caosdb/server/jobs/core/CheckParValid.java index 02cb96fc1214712c7a8ef1bb98327ba9fb39e9d0..e663874189ee0301a5b30452468391711624b606 100644 --- a/src/main/java/org/caosdb/server/jobs/core/CheckParValid.java +++ b/src/main/java/org/caosdb/server/jobs/core/CheckParValid.java @@ -33,7 +33,7 @@ import org.caosdb.server.entity.wrapper.Parent; import org.caosdb.server.entity.wrapper.Property; import org.caosdb.server.jobs.EntityJob; import org.caosdb.server.jobs.JobAnnotation; -import org.caosdb.server.jobs.JobExecutionTime; +import org.caosdb.server.jobs.TransactionStage; import org.caosdb.server.permissions.EntityPermission; import org.caosdb.server.utils.EntityStatus; import org.caosdb.server.utils.ServerMessages; @@ -43,7 +43,7 @@ import org.caosdb.server.utils.ServerMessages; * * @author tf */ -@JobAnnotation(time = JobExecutionTime.PRE_CHECK) +@JobAnnotation(stage = TransactionStage.PRE_CHECK) public class CheckParValid extends EntityJob { @Override public final void run() { diff --git a/src/main/java/org/caosdb/server/jobs/core/CheckPropValid.java b/src/main/java/org/caosdb/server/jobs/core/CheckPropValid.java index f95104bd2f2552a642e7e4bcd2fc13f7fd2788d4..390deedde211c0931eca1c3677ac5ff9c8ee9d8f 100644 --- a/src/main/java/org/caosdb/server/jobs/core/CheckPropValid.java +++ b/src/main/java/org/caosdb/server/jobs/core/CheckPropValid.java @@ -32,7 +32,7 @@ import org.caosdb.server.entity.Message; import org.caosdb.server.entity.wrapper.Property; import org.caosdb.server.jobs.EntityJob; import org.caosdb.server.jobs.JobAnnotation; -import org.caosdb.server.jobs.JobExecutionTime; +import org.caosdb.server.jobs.TransactionStage; import org.caosdb.server.permissions.EntityPermission; import org.caosdb.server.utils.EntityStatus; import org.caosdb.server.utils.ServerMessages; @@ -42,7 +42,7 @@ import org.caosdb.server.utils.ServerMessages; * * @author tf */ -@JobAnnotation(time = JobExecutionTime.PRE_CHECK) +@JobAnnotation(stage = TransactionStage.PRE_CHECK) public class CheckPropValid extends EntityJob { @Override public final void run() { diff --git a/src/main/java/org/caosdb/server/jobs/core/CheckStateTransition.java b/src/main/java/org/caosdb/server/jobs/core/CheckStateTransition.java index cd187cf17ad72c5431fcd3f2eb0b95890941faf9..d82a8a5be4baf265e6af263391751a68458a91f4 100644 --- a/src/main/java/org/caosdb/server/jobs/core/CheckStateTransition.java +++ b/src/main/java/org/caosdb/server/jobs/core/CheckStateTransition.java @@ -30,7 +30,7 @@ import org.caosdb.server.entity.Message; import org.caosdb.server.entity.Message.MessageType; import org.caosdb.server.entity.UpdateEntity; import org.caosdb.server.jobs.JobAnnotation; -import org.caosdb.server.jobs.JobExecutionTime; +import org.caosdb.server.jobs.TransactionStage; import org.caosdb.server.transaction.WriteTransaction; import org.caosdb.server.utils.ServerMessages; @@ -42,7 +42,7 @@ import org.caosdb.server.utils.ServerMessages; * * @author Timm Fitschen (t.fitschen@indiscale.com) */ -@JobAnnotation(time = JobExecutionTime.POST_CHECK, transaction = WriteTransaction.class) +@JobAnnotation(stage = TransactionStage.POST_CHECK, transaction = WriteTransaction.class) public class CheckStateTransition extends EntityStateJob { private static final String PERMISSION_STATE_FORCE_FINAL = "STATE:FORCE:FINAL"; diff --git a/src/main/java/org/caosdb/server/jobs/core/DebugCalls.java b/src/main/java/org/caosdb/server/jobs/core/DebugCalls.java index 8527560d02e498bf3812162a8b01b9641aad7260..8e0ae7435693ba1010f023da6c585911825f5a5e 100644 --- a/src/main/java/org/caosdb/server/jobs/core/DebugCalls.java +++ b/src/main/java/org/caosdb/server/jobs/core/DebugCalls.java @@ -25,11 +25,11 @@ package org.caosdb.server.jobs.core; import org.caosdb.server.CaosDBServer; import org.caosdb.server.jobs.FlagJob; import org.caosdb.server.jobs.JobAnnotation; -import org.caosdb.server.jobs.JobExecutionTime; +import org.caosdb.server.jobs.TransactionStage; @JobAnnotation( flag = "debug", - time = JobExecutionTime.INIT, + stage = TransactionStage.INIT, values = {DebugCalls.ex_null_p}, description = "This job for debugging and is available in debug mode only. Otherwise the flag is ignored. It throws an exception on purpose.", diff --git a/src/main/java/org/caosdb/server/jobs/core/ExecuteQuery.java b/src/main/java/org/caosdb/server/jobs/core/ExecuteQuery.java index 7f5444b36104526101ebf9b3f2aa547b3d91cbc9..399039a072d2fd7b5f54566b2f448791169a356b 100644 --- a/src/main/java/org/caosdb/server/jobs/core/ExecuteQuery.java +++ b/src/main/java/org/caosdb/server/jobs/core/ExecuteQuery.java @@ -26,13 +26,13 @@ import org.caosdb.server.entity.EntityInterface; import org.caosdb.server.entity.Message; import org.caosdb.server.jobs.FlagJob; import org.caosdb.server.jobs.JobAnnotation; -import org.caosdb.server.jobs.JobExecutionTime; +import org.caosdb.server.jobs.TransactionStage; import org.caosdb.server.query.Query; import org.caosdb.server.query.Query.ParsingException; import org.caosdb.server.utils.EntityStatus; import org.caosdb.server.utils.ServerMessages; -@JobAnnotation(flag = "query", time = JobExecutionTime.INIT) +@JobAnnotation(flag = "query", stage = TransactionStage.INIT) public class ExecuteQuery extends FlagJob { @Override diff --git a/src/main/java/org/caosdb/server/jobs/core/History.java b/src/main/java/org/caosdb/server/jobs/core/History.java index 5594b5afdb841436dbdd63f96b8e2c44faef51c6..d39a6e1f570400194b3a530c3c323a92d0374b7c 100644 --- a/src/main/java/org/caosdb/server/jobs/core/History.java +++ b/src/main/java/org/caosdb/server/jobs/core/History.java @@ -27,7 +27,7 @@ import org.caosdb.server.database.backend.transaction.RetrieveVersionHistory; import org.caosdb.server.entity.EntityInterface; import org.caosdb.server.jobs.FlagJob; import org.caosdb.server.jobs.JobAnnotation; -import org.caosdb.server.jobs.JobExecutionTime; +import org.caosdb.server.jobs.TransactionStage; import org.caosdb.server.permissions.EntityPermission; import org.caosdb.server.utils.EntityStatus; import org.caosdb.server.utils.ServerMessages; @@ -37,7 +37,7 @@ import org.caosdb.server.utils.ServerMessages; * * @author Timm Fitschen (t.fitschen@indiscale.com) */ -@JobAnnotation(time = JobExecutionTime.POST_TRANSACTION, flag = "H") +@JobAnnotation(stage = TransactionStage.POST_TRANSACTION, flag = "H") public class History extends FlagJob { @Override diff --git a/src/main/java/org/caosdb/server/jobs/core/InheritInitialState.java b/src/main/java/org/caosdb/server/jobs/core/InheritInitialState.java index bda4b8ebe885aa49df13ecf36ab14663c7ead270..39b66e35330a485bbd8eb4c40724ab3b250ccc45 100644 --- a/src/main/java/org/caosdb/server/jobs/core/InheritInitialState.java +++ b/src/main/java/org/caosdb/server/jobs/core/InheritInitialState.java @@ -5,9 +5,9 @@ import org.caosdb.server.entity.Message; import org.caosdb.server.entity.wrapper.Parent; import org.caosdb.server.entity.wrapper.Property; import org.caosdb.server.jobs.JobAnnotation; -import org.caosdb.server.jobs.JobExecutionTime; +import org.caosdb.server.jobs.TransactionStage; -@JobAnnotation(time = JobExecutionTime.CHECK) +@JobAnnotation(stage = TransactionStage.CHECK) public class InheritInitialState extends EntityStateJob { @Override diff --git a/src/main/java/org/caosdb/server/jobs/core/InitEntityStateJobs.java b/src/main/java/org/caosdb/server/jobs/core/InitEntityStateJobs.java index 7dbff695a342e5dcd2a19a2d72116c14fba78a6c..d3a1fc636761298f1b0586a4036be09ced835021 100644 --- a/src/main/java/org/caosdb/server/jobs/core/InitEntityStateJobs.java +++ b/src/main/java/org/caosdb/server/jobs/core/InitEntityStateJobs.java @@ -39,7 +39,7 @@ import org.caosdb.server.entity.UpdateEntity; import org.caosdb.server.entity.WritableEntity; import org.caosdb.server.entity.wrapper.Property; import org.caosdb.server.jobs.JobAnnotation; -import org.caosdb.server.jobs.JobExecutionTime; +import org.caosdb.server.jobs.TransactionStage; import org.caosdb.server.transaction.WriteTransaction; import org.caosdb.server.utils.EntityStatus; import org.caosdb.server.utils.Observable; @@ -57,7 +57,7 @@ import org.caosdb.server.utils.Observer; */ @JobAnnotation( loadAlways = true, - time = JobExecutionTime.INIT, + stage = TransactionStage.INIT, transaction = WriteTransaction.class) public class InitEntityStateJobs extends EntityStateJob implements Observer { diff --git a/src/main/java/org/caosdb/server/jobs/core/InsertFilesInDir.java b/src/main/java/org/caosdb/server/jobs/core/InsertFilesInDir.java index 35ce49297291accd41a7c293fc909cc0a27f9317..8133e4659d55caf29faa56eeb80ced194cd26cfc 100644 --- a/src/main/java/org/caosdb/server/jobs/core/InsertFilesInDir.java +++ b/src/main/java/org/caosdb/server/jobs/core/InsertFilesInDir.java @@ -46,7 +46,7 @@ import org.caosdb.server.entity.Role; import org.caosdb.server.jobs.FlagJob; import org.caosdb.server.jobs.Job; import org.caosdb.server.jobs.JobAnnotation; -import org.caosdb.server.jobs.JobExecutionTime; +import org.caosdb.server.jobs.TransactionStage; import org.caosdb.server.transaction.Retrieve; import org.caosdb.server.transaction.WriteTransactionInterface; import org.caosdb.server.utils.EntityStatus; @@ -57,7 +57,7 @@ import org.caosdb.server.utils.Utils; @JobAnnotation( flag = "InsertFilesInDir", loadOnDefault = false, - time = JobExecutionTime.INIT, + stage = TransactionStage.INIT, description = "For expert users only! Risk of creating spam records!\nValue of this flag might be any directory on the servers local file system which is part of the server's back-end file storage. This job will insert every readable, nonhidden file in said directory into the database and link the file with a symlink. This is useful to add a huge amount of files without actully copying them to the back-end file storage. If you call this job on a directory more than once every file that was recently added to the source directory is inserted. Every yet known file is left untouched. \nOptional parameter -e EXCLUDE: A regular expression of files which are to be ignored. \n Optional parameter -i INCLUDE: a regular expression of files which are to be included. By default, all files are included. The -e takes precedence. \nOptional parameter -p PREFIX: Stores all new files into the directory PREFIX in the server's file system.\nOptional parameter --force-allow-symlinks: Simlinks in your data are a source of problems for the database. Therefore, simlinks are ignored by default. This option allows symlinks (but still generates simlink warnings). \nPrepend/Dry run: Call this flag with a retrieve transaction (HTTP GET) and it will only count all files and list them without actually inserting them.") public class InsertFilesInDir extends FlagJob { diff --git a/src/main/java/org/caosdb/server/jobs/core/LoadContainerFlagJobs.java b/src/main/java/org/caosdb/server/jobs/core/LoadContainerFlagJobs.java index 89f381d648144e274325c5c71a57c9f4daf04895..ef8df214c804fb1d55c3ed088c5f7f6e998aca07 100644 --- a/src/main/java/org/caosdb/server/jobs/core/LoadContainerFlagJobs.java +++ b/src/main/java/org/caosdb/server/jobs/core/LoadContainerFlagJobs.java @@ -27,9 +27,9 @@ import org.caosdb.server.jobs.ContainerJob; import org.caosdb.server.jobs.FlagJob; import org.caosdb.server.jobs.Job; import org.caosdb.server.jobs.JobAnnotation; -import org.caosdb.server.jobs.JobExecutionTime; +import org.caosdb.server.jobs.TransactionStage; -@JobAnnotation(time = JobExecutionTime.INIT) +@JobAnnotation(stage = TransactionStage.INIT) public class LoadContainerFlagJobs extends ContainerJob { @Override diff --git a/src/main/java/org/caosdb/server/jobs/core/MakeStateMessage.java b/src/main/java/org/caosdb/server/jobs/core/MakeStateMessage.java index 6e98704a5f6d73199993f2ed531e6d6277934e64..855d6a139aa80fa0b67cda652708b3e56ef5eb2b 100644 --- a/src/main/java/org/caosdb/server/jobs/core/MakeStateMessage.java +++ b/src/main/java/org/caosdb/server/jobs/core/MakeStateMessage.java @@ -30,7 +30,7 @@ import org.caosdb.server.entity.Message; import org.caosdb.server.entity.wrapper.Property; import org.caosdb.server.entity.xml.ToElementable; import org.caosdb.server.jobs.JobAnnotation; -import org.caosdb.server.jobs.JobExecutionTime; +import org.caosdb.server.jobs.TransactionStage; import org.caosdb.server.transaction.Retrieve; import org.jdom2.Element; @@ -46,7 +46,7 @@ import org.jdom2.Element; @JobAnnotation( loadAlways = true, transaction = Retrieve.class, - time = JobExecutionTime.POST_TRANSACTION) + stage = TransactionStage.POST_TRANSACTION) public class MakeStateMessage extends EntityStateJob { public static final String SPARSE_FLAG = "sparseState"; diff --git a/src/main/java/org/caosdb/server/jobs/core/MakeStateProperty.java b/src/main/java/org/caosdb/server/jobs/core/MakeStateProperty.java index 56682921563d77f462903567a2c128fcd5d92555..c2aebc681b38a9902bede69883cce4025bc8afe6 100644 --- a/src/main/java/org/caosdb/server/jobs/core/MakeStateProperty.java +++ b/src/main/java/org/caosdb/server/jobs/core/MakeStateProperty.java @@ -25,14 +25,14 @@ package org.caosdb.server.jobs.core; import org.caosdb.server.entity.Message; import org.caosdb.server.jobs.JobAnnotation; -import org.caosdb.server.jobs.JobExecutionTime; +import org.caosdb.server.jobs.TransactionStage; import org.caosdb.server.transaction.WriteTransaction; /** * This job constructs an ordinary Property from the State right before the entity is being written * to the back-end and after any checks run. */ -@JobAnnotation(transaction = WriteTransaction.class, time = JobExecutionTime.PRE_TRANSACTION) +@JobAnnotation(transaction = WriteTransaction.class, stage = TransactionStage.PRE_TRANSACTION) public class MakeStateProperty extends EntityStateJob { @Override diff --git a/src/main/java/org/caosdb/server/jobs/core/NoCache.java b/src/main/java/org/caosdb/server/jobs/core/NoCache.java index a4f639392ac63a97f2f025dfe6717a664114c561..e408330ee2639d1c5d7dc5f145848e292daa9b3b 100644 --- a/src/main/java/org/caosdb/server/jobs/core/NoCache.java +++ b/src/main/java/org/caosdb/server/jobs/core/NoCache.java @@ -24,7 +24,7 @@ package org.caosdb.server.jobs.core; import org.caosdb.server.jobs.FlagJob; import org.caosdb.server.jobs.JobAnnotation; -import org.caosdb.server.jobs.JobExecutionTime; +import org.caosdb.server.jobs.TransactionStage; @JobAnnotation( flag = "cache", @@ -32,7 +32,7 @@ import org.caosdb.server.jobs.JobExecutionTime; description = "Depending on the configuraion of the server some transactions with the backend database might be cached internally for performance reasons. Disable the internal caching for this transaction with 'false'. Note: You cannot enable the caching with 'true', if the server is not configured to use caching.", values = {"true", "false"}, - time = JobExecutionTime.INIT) + stage = TransactionStage.INIT) public class NoCache extends FlagJob { @Override diff --git a/src/main/java/org/caosdb/server/jobs/core/Paging.java b/src/main/java/org/caosdb/server/jobs/core/Paging.java index 43683639f75c08be394a38ce03e91282405f333d..5f2a6ed62f48f20d09c34c8ee34190231903ffe7 100644 --- a/src/main/java/org/caosdb/server/jobs/core/Paging.java +++ b/src/main/java/org/caosdb/server/jobs/core/Paging.java @@ -27,11 +27,11 @@ import java.util.regex.Pattern; import org.caosdb.server.entity.EntityInterface; import org.caosdb.server.jobs.FlagJob; import org.caosdb.server.jobs.JobAnnotation; -import org.caosdb.server.jobs.JobExecutionTime; +import org.caosdb.server.jobs.TransactionStage; import org.caosdb.server.transaction.Retrieve; import org.caosdb.server.utils.EntityStatus; -@JobAnnotation(flag = "P", time = JobExecutionTime.PRE_TRANSACTION) +@JobAnnotation(flag = "P", stage = TransactionStage.PRE_TRANSACTION) public class Paging extends FlagJob { public static final int DEFAULT_LENGTH = 100; diff --git a/src/main/java/org/caosdb/server/jobs/core/PickUp.java b/src/main/java/org/caosdb/server/jobs/core/PickUp.java index b91fccf72d54022c6f6d19ba26c2b0974a54829f..94a80836c863ea226a88e035201a2e2fb9fd1039 100644 --- a/src/main/java/org/caosdb/server/jobs/core/PickUp.java +++ b/src/main/java/org/caosdb/server/jobs/core/PickUp.java @@ -30,12 +30,12 @@ import org.caosdb.server.entity.FileProperties; import org.caosdb.server.entity.Message; import org.caosdb.server.jobs.EntityJob; import org.caosdb.server.jobs.JobAnnotation; -import org.caosdb.server.jobs.JobExecutionTime; +import org.caosdb.server.jobs.TransactionStage; import org.caosdb.server.utils.EntityStatus; import org.caosdb.server.utils.Observable; import org.caosdb.server.utils.Observer; -@JobAnnotation(time = JobExecutionTime.INIT) +@JobAnnotation(stage = TransactionStage.INIT) public class PickUp extends EntityJob implements Observer { @Override diff --git a/src/main/java/org/caosdb/server/jobs/core/ProcessNameProperties.java b/src/main/java/org/caosdb/server/jobs/core/ProcessNameProperties.java index e7e8f0e4d6aab3f3d0b3e9392d408a88aaa0abcc..1cc12c6102f66dda224360554e4ac5c515ae418d 100644 --- a/src/main/java/org/caosdb/server/jobs/core/ProcessNameProperties.java +++ b/src/main/java/org/caosdb/server/jobs/core/ProcessNameProperties.java @@ -35,7 +35,7 @@ import org.caosdb.server.entity.wrapper.Parent; import org.caosdb.server.entity.wrapper.Property; import org.caosdb.server.jobs.EntityJob; import org.caosdb.server.jobs.JobAnnotation; -import org.caosdb.server.jobs.JobExecutionTime; +import org.caosdb.server.jobs.TransactionStage; import org.caosdb.server.utils.EntityStatus; import org.caosdb.server.utils.ServerMessages; @@ -44,7 +44,7 @@ import org.caosdb.server.utils.ServerMessages; * * @author tf */ -@JobAnnotation(time = JobExecutionTime.PRE_TRANSACTION) +@JobAnnotation(stage = TransactionStage.PRE_TRANSACTION) public class ProcessNameProperties extends EntityJob { @Override diff --git a/src/main/java/org/caosdb/server/jobs/core/RetrieveACL.java b/src/main/java/org/caosdb/server/jobs/core/RetrieveACL.java index f59b2c1ec94de846714575537516c41d18bf4629..4664c01b88d361485c342a8fcddca2ab082f0dee 100644 --- a/src/main/java/org/caosdb/server/jobs/core/RetrieveACL.java +++ b/src/main/java/org/caosdb/server/jobs/core/RetrieveACL.java @@ -27,7 +27,7 @@ import org.caosdb.server.entity.EntityInterface; import org.caosdb.server.entity.xml.ToElementable; import org.caosdb.server.jobs.FlagJob; import org.caosdb.server.jobs.JobAnnotation; -import org.caosdb.server.jobs.JobExecutionTime; +import org.caosdb.server.jobs.TransactionStage; import org.caosdb.server.permissions.EntityPermission; import org.caosdb.server.transaction.Retrieve; import org.caosdb.server.utils.EntityStatus; @@ -35,7 +35,7 @@ import org.caosdb.server.utils.ServerMessages; import org.jdom2.Element; @JobAnnotation( - time = JobExecutionTime.POST_TRANSACTION, + stage = TransactionStage.POST_TRANSACTION, flag = "ACL", description = "Adds an XML representation of the EntityACL to the output of this entity.") public class RetrieveACL extends FlagJob { diff --git a/src/main/java/org/caosdb/server/jobs/core/RetrieveAllJob.java b/src/main/java/org/caosdb/server/jobs/core/RetrieveAllJob.java index 603d861ad24b659fab262291d888420c02f6bcf5..7cd7791e0b27f1ec1d5e67e047e7f1cbe177948c 100644 --- a/src/main/java/org/caosdb/server/jobs/core/RetrieveAllJob.java +++ b/src/main/java/org/caosdb/server/jobs/core/RetrieveAllJob.java @@ -26,9 +26,9 @@ import org.caosdb.server.database.backend.transaction.RetrieveAll; import org.caosdb.server.entity.EntityInterface; import org.caosdb.server.jobs.FlagJob; import org.caosdb.server.jobs.JobAnnotation; -import org.caosdb.server.jobs.JobExecutionTime; +import org.caosdb.server.jobs.TransactionStage; -@JobAnnotation(flag = "all", time = JobExecutionTime.INIT) +@JobAnnotation(flag = "all", stage = TransactionStage.INIT) public class RetrieveAllJob extends FlagJob { @Override diff --git a/src/main/java/org/caosdb/server/jobs/core/RetrieveIdOnlyFlag.java b/src/main/java/org/caosdb/server/jobs/core/RetrieveIdOnlyFlag.java index 1712950dff9403df54f26457d0b6a763d7ce4de8..844468c5473b4d2dff6e59777f925f2701505749 100644 --- a/src/main/java/org/caosdb/server/jobs/core/RetrieveIdOnlyFlag.java +++ b/src/main/java/org/caosdb/server/jobs/core/RetrieveIdOnlyFlag.java @@ -25,10 +25,10 @@ package org.caosdb.server.jobs.core; import org.caosdb.server.entity.EntityInterface; import org.caosdb.server.jobs.FlagJob; import org.caosdb.server.jobs.JobAnnotation; -import org.caosdb.server.jobs.JobExecutionTime; +import org.caosdb.server.jobs.TransactionStage; import org.caosdb.server.utils.EntityStatus; -@JobAnnotation(flag = "IdOnly", time = JobExecutionTime.PRE_TRANSACTION) +@JobAnnotation(flag = "IdOnly", stage = TransactionStage.PRE_TRANSACTION) public class RetrieveIdOnlyFlag extends FlagJob { @Override diff --git a/src/main/java/org/caosdb/server/jobs/core/RetrieveOwner.java b/src/main/java/org/caosdb/server/jobs/core/RetrieveOwner.java index 7377805805ec55e2c3c3928cc42b0c053d8a9c9d..8b6412cee5d6e800a78509741d4f3680403008ce 100644 --- a/src/main/java/org/caosdb/server/jobs/core/RetrieveOwner.java +++ b/src/main/java/org/caosdb/server/jobs/core/RetrieveOwner.java @@ -28,7 +28,7 @@ import org.caosdb.server.entity.EntityInterface; import org.caosdb.server.entity.xml.ToElementable; import org.caosdb.server.jobs.FlagJob; import org.caosdb.server.jobs.JobAnnotation; -import org.caosdb.server.jobs.JobExecutionTime; +import org.caosdb.server.jobs.TransactionStage; import org.caosdb.server.permissions.EntityPermission; import org.caosdb.server.permissions.ResponsibleAgent; import org.caosdb.server.transaction.Retrieve; @@ -37,7 +37,7 @@ import org.caosdb.server.utils.ServerMessages; import org.jdom2.Element; @JobAnnotation( - time = JobExecutionTime.POST_TRANSACTION, + stage = TransactionStage.POST_TRANSACTION, flag = "owner", description = "Adds an XML representation of the owner(s) to the output of this entity.") public class RetrieveOwner extends FlagJob { diff --git a/src/main/java/org/caosdb/server/jobs/core/RetriveAllNames.java b/src/main/java/org/caosdb/server/jobs/core/RetriveAllNames.java index dfd69a5cb5a40a6e6be35e345f17e38f3dbb7e2e..b5787246cbafba0df31056d2da15adb330ef0c50 100644 --- a/src/main/java/org/caosdb/server/jobs/core/RetriveAllNames.java +++ b/src/main/java/org/caosdb/server/jobs/core/RetriveAllNames.java @@ -4,7 +4,7 @@ import org.caosdb.server.database.backend.transaction.GetAllNames; import org.caosdb.server.entity.EntityInterface; import org.caosdb.server.jobs.FlagJob; import org.caosdb.server.jobs.JobAnnotation; -import org.caosdb.server.jobs.JobExecutionTime; +import org.caosdb.server.jobs.TransactionStage; import org.caosdb.server.permissions.EntityPermission; import org.caosdb.server.utils.EntityStatus; @@ -15,7 +15,7 @@ import org.caosdb.server.utils.EntityStatus; * <p>The entites' status are set to VALID because the entities parents and properties do not have * to be retrieved. */ -@JobAnnotation(flag = "names", time = JobExecutionTime.INIT) +@JobAnnotation(flag = "names", stage = TransactionStage.INIT) public class RetriveAllNames extends FlagJob { @Override diff --git a/src/main/java/org/caosdb/server/jobs/core/Strict.java b/src/main/java/org/caosdb/server/jobs/core/Strict.java index aebbecf86a89476339dbdef8892b074aebc8196d..bb69063c116e5c14aa9fad2b0bbf881d2481b394 100644 --- a/src/main/java/org/caosdb/server/jobs/core/Strict.java +++ b/src/main/java/org/caosdb/server/jobs/core/Strict.java @@ -27,11 +27,11 @@ import org.caosdb.server.entity.Message; import org.caosdb.server.entity.xml.ToElementable; import org.caosdb.server.jobs.ContainerJob; import org.caosdb.server.jobs.JobAnnotation; -import org.caosdb.server.jobs.JobExecutionTime; +import org.caosdb.server.jobs.TransactionStage; import org.caosdb.server.utils.EntityStatus; import org.caosdb.server.utils.ServerMessages; -@JobAnnotation(time = JobExecutionTime.POST_CHECK) +@JobAnnotation(stage = TransactionStage.POST_CHECK) public class Strict extends ContainerJob { // TODO load strict job via unified flag interface diff --git a/src/main/java/org/caosdb/server/jobs/core/UpdateUnitConverters.java b/src/main/java/org/caosdb/server/jobs/core/UpdateUnitConverters.java index b3ef80adb64fcf3c1007a33a64bde691c9ff9c1d..0b71d2102c98188d875d52d3bbc3e0f79a23043b 100644 --- a/src/main/java/org/caosdb/server/jobs/core/UpdateUnitConverters.java +++ b/src/main/java/org/caosdb/server/jobs/core/UpdateUnitConverters.java @@ -25,12 +25,12 @@ package org.caosdb.server.jobs.core; import org.caosdb.server.database.backend.transaction.InsertLinCon; import org.caosdb.server.jobs.EntityJob; import org.caosdb.server.jobs.JobAnnotation; -import org.caosdb.server.jobs.JobExecutionTime; +import org.caosdb.server.jobs.TransactionStage; import org.caosdb.unit.Converter; import org.caosdb.unit.LinearConverter; import org.caosdb.unit.Unit; -@JobAnnotation(time = JobExecutionTime.PRE_TRANSACTION) +@JobAnnotation(stage = TransactionStage.PRE_TRANSACTION) public class UpdateUnitConverters extends EntityJob { @Override diff --git a/src/main/java/org/caosdb/server/transaction/Transaction.java b/src/main/java/org/caosdb/server/transaction/Transaction.java index 97ed4954b93c55830d1bed5f74bad6e157e70776..99c305fa17b37878cc5f3277c93623a915be4ea3 100644 --- a/src/main/java/org/caosdb/server/transaction/Transaction.java +++ b/src/main/java/org/caosdb/server/transaction/Transaction.java @@ -39,14 +39,15 @@ import org.caosdb.server.entity.Message; import org.caosdb.server.entity.Message.MessageType; import org.caosdb.server.entity.container.TransactionContainer; import org.caosdb.server.jobs.Job; -import org.caosdb.server.jobs.JobExecutionTime; import org.caosdb.server.jobs.Schedule; import org.caosdb.server.jobs.ScheduledJob; +import org.caosdb.server.jobs.TransactionStage; import org.caosdb.server.jobs.core.AccessControl; import org.caosdb.server.jobs.core.CheckDatatypePresent; import org.caosdb.server.jobs.core.CheckEntityACLRoles; import org.caosdb.server.jobs.core.Mode; import org.caosdb.server.jobs.core.PickUp; +import org.caosdb.server.permissions.EntityACL; import org.caosdb.server.utils.AbstractObservable; import org.caosdb.server.utils.Info; import org.caosdb.server.utils.Observer; @@ -73,9 +74,11 @@ public abstract class Transaction<C extends TransactionContainer> extends Abstra this(container, Info.getInstance()); } - protected Transaction(C container, Observer o) { + protected Transaction(final C container, final Observer o) { this.container = container; - if (o != null) acceptObserver(o); + if (o != null) { + acceptObserver(o); + } } public static DatabaseAccessManager getAccessManager() { @@ -86,10 +89,15 @@ public abstract class Transaction<C extends TransactionContainer> extends Abstra return this.container; } + /** + * Implementation note: Not called in this class, but may be used by subclasses. + * + * <p>E.g. in {@link Retrieve} and {@link WriteTransaction}. + */ protected void makeSchedule() throws Exception { // load flag jobs final Job loadContainerFlags = Job.getJob("LoadContainerFlagJobs", Mode.MUST, null, this); - ScheduledJob scheduledJob = this.schedule.add(loadContainerFlags); + final ScheduledJob scheduledJob = this.schedule.add(loadContainerFlags); this.schedule.runJob(scheduledJob); // AccessControl @@ -115,12 +123,42 @@ public abstract class Transaction<C extends TransactionContainer> extends Abstra } } + /** + * The main transaction execution method. + * + * <p>This method calls the following other internal methods and scheduled jobs stored in the + * {@link getSchedule() internal Schedule object}: + * + * <ol> + * <li>{@link init} - Make {@link Schedule}, resolve names to ids, aquire read access. + * <li>{@link Schedule.runJobs(INIT)} - See {@link TransactionStage#INIT}. + * <li>{@link preCheck} - Load/generate {@link EntityACL}s, check if any updates are to be + * processed. + * <li>{@link Schedule.runJobs(PRE_CHECK)} - See {@link TransactionStage#PRE_CHECK}. + * <li>{@link check} - only run the jobs in the CHECK stage, see {@link TransactionStage#CHECK}. + * <li>{@link Schedule.runJobs(POST_CHECK)} - See {@link TransactionStage#POST_CHECK}. + * <li>{@link postCheck} - currently, nothing happens here (just there for consistency). + * <li>{@link preTransaction} - acquire write access (if necessary) + * <li>{@link Schedule.runJobs(PRE_TRANSACTION)} - See {@link TransactionStage#PRE_TRANSACTION}. + * <li>{@link transaction}: This is typically the main method of a Transaction. + * <li>{@link Schedule.runJobs(POST_TRANSACTION)} - See {@link + * TransactionStage#POST_TRANSACTION}. + * <li>{@link postTransaction} - Add success messages + * <li>{@link writeHistory} - write the transaction history logs + * <li>{@link commit} - commit the changes + * <li>{@link rollBack}: Only in the case of errors - rollback any changes (also file-system + * changes). + * <li>{@link cleanUp}: Always - cleanup the transaction (e.g. remove temporary files). + * <li>{@link notifyObservers(CLEAN_UP)}: Also always - for any jobs that do their own clean-up. + * + * @see {@link TransactionStage}. + */ @Override public final void execute() throws Exception { long t1 = System.currentTimeMillis(); try { init(); - this.schedule.runJobs(JobExecutionTime.INIT); + this.schedule.runJobs(TransactionStage.INIT); getContainer() .getTransactionBenchmark() .addMeasurement( @@ -128,7 +166,7 @@ public abstract class Transaction<C extends TransactionContainer> extends Abstra t1 = System.currentTimeMillis(); preCheck(); - this.schedule.runJobs(JobExecutionTime.PRE_CHECK); + this.schedule.runJobs(TransactionStage.PRE_CHECK); getContainer() .getTransactionBenchmark() .addMeasurement( @@ -142,7 +180,7 @@ public abstract class Transaction<C extends TransactionContainer> extends Abstra this.getClass().getSimpleName() + ".check", System.currentTimeMillis() - t1); t1 = System.currentTimeMillis(); - this.schedule.runJobs(JobExecutionTime.POST_CHECK); + this.schedule.runJobs(TransactionStage.POST_CHECK); postCheck(); getContainer() .getTransactionBenchmark() @@ -151,7 +189,7 @@ public abstract class Transaction<C extends TransactionContainer> extends Abstra t1 = System.currentTimeMillis(); preTransaction(); - this.schedule.runJobs(JobExecutionTime.PRE_TRANSACTION); + this.schedule.runJobs(TransactionStage.PRE_TRANSACTION); getContainer() .getTransactionBenchmark() .addMeasurement( @@ -166,7 +204,7 @@ public abstract class Transaction<C extends TransactionContainer> extends Abstra this.getClass().getSimpleName() + ".transaction", System.currentTimeMillis() - t1); t1 = System.currentTimeMillis(); - this.schedule.runJobs(JobExecutionTime.POST_TRANSACTION); + this.schedule.runJobs(TransactionStage.POST_TRANSACTION); postTransaction(); getContainer() .getTransactionBenchmark() @@ -202,6 +240,12 @@ public abstract class Transaction<C extends TransactionContainer> extends Abstra } } + /** + * Return the internal {@link Schedule} object. + * + * <p>The Schedule stores jobs which are also triggered by this transaction (see {@link execute()} + * for details). + */ public Schedule getSchedule() { return this.schedule; } @@ -210,6 +254,7 @@ public abstract class Transaction<C extends TransactionContainer> extends Abstra return getContainer().getOwner(); } + /** Return true iff this transaction should be logged in the transaction history logs. */ public abstract boolean logHistory(); public UTCDateTime getTimestamp() { @@ -219,36 +264,46 @@ public abstract class Transaction<C extends TransactionContainer> extends Abstra // TODO move to post-transaction job private void writeHistory() throws TransactionException, Message { if (logHistory()) { - String realm = ((Principal) getTransactor().getPrincipal()).getRealm(); - String username = ((Principal) getTransactor().getPrincipal()).getUsername(); + final String realm = ((Principal) getTransactor().getPrincipal()).getRealm(); + final String username = ((Principal) getTransactor().getPrincipal()).getUsername(); execute( new InsertTransactionHistory(getContainer(), realm, username, getTimestamp()), getAccess()); } } + /** @see {@link #execute()} */ protected void rollBack() { - this.schedule.runJobs(JobExecutionTime.ROLL_BACK); + this.schedule.runJobs(TransactionStage.ROLL_BACK); } + /** @see {@link #execute()} */ protected abstract void init() throws Exception; + /** @see {@link #execute()} */ protected abstract void preCheck() throws InterruptedException, Exception; + /** @see {@link #execute()} */ protected final void check() { - this.schedule.runJobs(JobExecutionTime.CHECK); + this.schedule.runJobs(TransactionStage.CHECK); } + /** @see {@link #execute()} */ protected abstract void postCheck(); + /** @see {@link #execute()} */ protected abstract void preTransaction() throws InterruptedException; + /** @see {@link #execute()} */ protected abstract void transaction() throws Exception; + /** @see {@link #execute()} */ protected abstract void postTransaction() throws Exception; + /** @see {@link #execute()} */ protected abstract void cleanUp(); + /** @see {@link #execute()} */ protected void commit() throws Exception {} public boolean useCache() { diff --git a/src/main/java/org/caosdb/server/transaction/TransactionInterface.java b/src/main/java/org/caosdb/server/transaction/TransactionInterface.java index 1aa41b2bbc31063b2978aa9c141e75c98799f124..d56407ca854c756a956fe11d9bc98a72f05a4b50 100644 --- a/src/main/java/org/caosdb/server/transaction/TransactionInterface.java +++ b/src/main/java/org/caosdb/server/transaction/TransactionInterface.java @@ -35,6 +35,10 @@ public interface TransactionInterface { return TransactionBenchmark.getRootInstance().getBenchmark(getClass()); } + /** + * Append the BackendTransaction t to a RollBackHandler before basically calling {@code + * t.execute()}. Except for benchmarking, this method does not interact directly with this object. + */ public default <K extends BackendTransaction> K execute(K t, Access access) { final RollBackHandler handler = (RollBackHandler) access.getHelper("RollBack"); handler.append(t); diff --git a/src/main/java/org/caosdb/server/transaction/WriteTransaction.java b/src/main/java/org/caosdb/server/transaction/WriteTransaction.java index 570fce2c2b001699fdb48c1379e807ae05d62996..670ff8bd6f4e78c6f20500dc057fc0e432872255 100644 --- a/src/main/java/org/caosdb/server/transaction/WriteTransaction.java +++ b/src/main/java/org/caosdb/server/transaction/WriteTransaction.java @@ -76,8 +76,7 @@ public class WriteTransaction extends Transaction<WritableContainer> @Override protected final void preTransaction() throws InterruptedException { - // acquire strong access. No other thread can have access until - // it this strong access is released. + // Acquire strong access. No other thread can have access until this strong access is released. setAccess(getAccessManager().acquireWriteAccess(this)); } diff --git a/src/main/java/org/caosdb/server/utils/FlagInfo.java b/src/main/java/org/caosdb/server/utils/FlagInfo.java index 164c6aabe9c4fafff26f93db30d791fa91389eee..60697cb90df0bcf0da0588c580c49664ec0b0f52 100644 --- a/src/main/java/org/caosdb/server/utils/FlagInfo.java +++ b/src/main/java/org/caosdb/server/utils/FlagInfo.java @@ -55,7 +55,7 @@ public class FlagInfo { for (final JobAnnotation a : as) { final Element aElem = new Element(a.flag()); aElem.setAttribute("description", a.description()); - aElem.setAttribute("time", a.time().toString()); + aElem.setAttribute("stage", a.stage().toString()); if (!a.transaction().equals(TransactionInterface.class)) { aElem.setAttribute("transaction", a.transaction().getSimpleName()); } diff --git a/src/test/docker/Dockerfile b/src/test/docker/Dockerfile index 08a2a0884d1f154bb941388075b2bdd915125f99..f3c8b98fdab34400cccfc11f0c9211eeca37f845 100644 --- a/src/test/docker/Dockerfile +++ b/src/test/docker/Dockerfile @@ -2,10 +2,11 @@ FROM debian:buster RUN apt-get update && \ apt-get install -y \ git make mariadb-server maven openjdk-11-jdk-headless \ + plantuml \ python3-pip screen libpam0g-dev unzip curl shunit2 \ python3-sphinx \ && \ - pip3 install javasphinx recommonmark sphinx-rtd-theme + pip3 install javasphinx recommonmark sphinx-rtd-theme sphinxcontrib-plantuml # Alternative, if javasphinx fails because python3-sphinx is too recent: # (`_l` not found): diff --git a/src/test/java/org/caosdb/server/jobs/ScheduleTest.java b/src/test/java/org/caosdb/server/jobs/ScheduleTest.java index 7ad1d6112e8b7bd3a29042d5f6ca82709ae4100a..cef0ec3e076f38893ada35bbad881ccfe8a02331 100644 --- a/src/test/java/org/caosdb/server/jobs/ScheduleTest.java +++ b/src/test/java/org/caosdb/server/jobs/ScheduleTest.java @@ -46,6 +46,6 @@ public class ScheduleTest { } }); - schedule.runJobs(JobExecutionTime.ROLL_BACK); + schedule.runJobs(TransactionStage.ROLL_BACK); } }