DOC: Basic structure for manual documentation.

Also amended Changelog and Readme.

.gitignore

+# -*- mode:conf; -*-
+
 # configuration files
 /conf/ext/*
 !/conf/core/*.template
@@ -11,8 +13,11 @@
 *.jks
 
 # typical build dirs
+build/
 bin/
 target/
+_apidoc/
+
 # But include server-side scripting
 !/scripting/bin
 

.gitlab-ci.yml

@@ -41,6 +41,7 @@ build-testenv:
     - schedules
   script:
     - cd src/test/docker
+    - time docker load < /image-cache/caosdb-server-testenv.tar || true
     - docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY
       # use here general latest or specific branch latest...
     - docker pull $CI_REGISTRY_IMAGE || true
@@ -48,6 +49,8 @@ build-testenv:
       --pull
       --cache-from $CI_REGISTRY_IMAGE
       -t $CI_REGISTRY_IMAGE .
+    - docker save $CI_REGISTRY_IMAGE > image.tar;
+          mv image.tar /image-cache/caosdb-server-testenv.tar;
     - docker push $CI_REGISTRY_IMAGE
 
 # Test: run unit tests of the server
@@ -74,3 +77,21 @@ trigger_build:
       -F "variables[TriggerdBy]=SERVER"
       -F "variables[TriggerdByHash]=$CI_COMMIT_SHORT_SHA"
       -F ref=$DEPLOY_REF https://gitlab.indiscale.com/api/v4/projects/14/trigger/pipeline
+
+# Build the sphinx documentation and make it ready for deployment by Gitlab Pages
+# documentation:
+#   stage: deploy
+
+# Special job for serving a static website. See https://docs.gitlab.com/ee/ci/yaml/README.html#pages
+pages:
+  tags: [ cached-dind ]
+  stage: deploy
+  only:
+    - dev
+  script:
+    - echo "Deploying"
+    - make doc
+    - cp -r build/doc/html public
+  artifacts:
+    paths:
+      - public

CHANGELOG.md

@@ -19,6 +19,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 * New server property `SERVER_SIDE_SCRIPTING_BIN_DIRS` which accepts a comma or
   space separated list as values. The server looks for scripts in all
   directories in the order or the list and uses the first matching file.
+* Automated documentation builds: `make doc`
 
 ### Changed
 

makefile → Makefile

@@ -128,3 +128,8 @@ stop-debug-screen:
 easy-units: .m2-local
 	mvn clean
 	mvn deploy:deploy-file -DgroupId=de.timmfitschen -DartifactId=easy-units -Dversion=0.0.1-SNAPSHOT -Durl=file:./.m2-local/ -DrepositoryId=local-maven-repo -DupdateReleaseInfo=true -Dfile=./lib/easy-units-0.0.1-SNAPSHOT-jar-with-dependencies.jar
+
+# Compile the standalone documentation
+.PHONY: doc
+doc:
+	$(MAKE) -C src/doc html

README_SETUP.md

@@ -179,3 +179,20 @@ before you execute it.
 ```sh
 sed -i.bak -e "s/\(\s*\)\([^.]\)caosdb\.server/\1\2org.caosdb.server/g" FILE_TO_BE_CHANGED
 ```
+
+# Build the documentation #
+
+Stand-alone documentation is built using Sphinx: `make doc`
+
+## Requirements ##
+
+- sphinx
+- javasphinx :: `pip3 install --user javasphinx`
+  - Alternative, if javasphinx fails because python3-sphinx is too recent:
+  (`_l` not found):
+  ```
+git clone git@github.com:simgrid/javasphinx.git
+cd javasphinx
+git checkout 659209069603a
+pip3 install .
+```

conf/core/cache.ccf

+# -*- mode:conf-javaprop; -*-
+
+# Configuration for the Java Caching System (JCS) which is used by the server.  Please look at
+# http://commons.apache.org/proper/commons-jcs/getting_started/intro.html for further information.
+
 # default caching options
 jcs.default.cacheattributes=org.apache.commons.jcs.engine.CompositeCacheAttributes
 jcs.default.cacheattributes.MaxObjects=1000

conf/core/global_entity_permissions.xml

 <globalPermissions>
+    <!--
+         4-store for permissions, implemented as a mapping:
+         {Grant/Deny, priority, role} -> List of permissions
+
+         Please look at the permission documentation for more information.
+    -->
     <Grant priority="false" role="?OWNER?"><Permission name="*"/></Grant>
     <Grant priority="false" role="?OTHER?"><Permission name="RETRIEVE:*"/></Grant>
     <Grant priority="false" role="?OTHER?"><Permission name="USE:*"/></Grant>

conf/core/log4j2-debug.properties

+# This log4j2-debug.properties file is only loaded when the server runs in debug mode.
+
 # override location of log files for debugging and testing
 property.LOG_DIR = testlog
 

conf/core/log4j2-default.properties

+# This log4j2-default.properties file describes the logging settings.  See
+# https://logging.apache.org/log4j/2.x/ for more information.
+
 name = base_configuration
 status = TRACE
 verbose = true

conf/core/server.conf

@@ -145,7 +145,7 @@ MAIL_TO_FILE_HANDLER_LOC=./
 
 # --------------------------------------------------
 # Admin settings
-# # --------------------------------------------------
+# --------------------------------------------------
 # Name of the administrator of this instance
 ADMIN_NAME=CaosDB Admin
 # Email of the administrator of this instance
@@ -182,4 +182,9 @@ CHECK_ENTITY_ACL_ROLES_MODE=MUST
 # part of any Entity ACL.
 GLOBAL_ENTITY_PERMISSIONS_FILE=./conf/core/global_entity_permissions.xml
 
+# --------------------------------------------------
+# Extensions
+# --------------------------------------------------
+
+# If set to true, versioning of entities' history is enabled.
 ENTITY_VERSIONING_ENABLED=true

conf/core/usersources.ini.template

-#
+# -*- mode:conf; -*-
+
 # ** header v3.0
 # This file is a part of the CaosDB Project.
 #
@@ -20,17 +21,30 @@
 #
 # ** end header
 #
+
+# `realms` is a comma and/or space separated list of realms which users can use for authentication
 realms = PAM
+
+# This is the default realm, to be used when no other realms is specified
 defaultRealm = PAM
 
+# Each realm has one section with specific options.  The options for a specific realm can be looked
+# up in that realm's documentation.
+#
+# Hint: Realms are implemented by classes which are typically in the
+# org.caosdb.server.accessControl.Pam package and implement the UserSource interface.
+
+# Options for authentication against Linux' PAM.
 [PAM]
 class = org.caosdb.server.accessControl.Pam
+# The script which does the actual checking.
 ; pam_script = ./misc/pam_authentication/pam_authentication.sh
 default_status = ACTIVE
+# Only users which fulfill these criteria are accepted.
 ;include.user = [uncomment and put your users here]
 ;include.group = [uncomment and put your groups here]
 ;exclude.user = [uncomment and put excluded users here]
 ;exclude.group = [uncomment and put excluded groups here]
 
-;it is necessary to add at least one admin
+# It is typically necessary to add at least one admin
 ;user.[uncomment a set a username here].roles = administration

src/doc/Makefile

+# ** header v3.0
+# This file is a part of the CaosDB Project.
+#
+# Copyright (C) 2020 IndiScale GmbH <info@indiscale.com>
+# Copyright (C) 2020 Daniel Hornung <d.hornung@indiscale.com>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see <https://www.gnu.org/licenses/>.
+#
+# ** end header
+
+# This Makefile is a wrapper for sphinx scripts.
+#
+# It is based upon the autocreated makefile for Sphinx documentation.
+
+# You can set these variables from the command line, and also
+# from the environment for the first two.
+SPHINXOPTS    ?= -a
+SPHINXBUILD   ?= sphinx-build
+SPHINXAPIDOC   ?= javasphinx-apidoc
+SOURCEDIR     = .
+BUILDDIR      = ../../build/doc
+
+.PHONY: doc-help Makefile apidoc
+
+# Put it first so that "make" without argument is like "make help".
+doc-help:
+	@$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
+
+# Catch-all target: route all unknown targets to Sphinx using the new
+# "make mode" option.  $(O) is meant as a shortcut for $(SPHINXOPTS).
+%: Makefile apidoc
+	@$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
+#	sphinx-build -M html . ../../build/doc
+
+apidoc:
+	@$(SPHINXAPIDOC) -o _apidoc --update --title="CaosDB Server" ../main/

src/doc/administration/configuration.rst

+Configuration
+=============
+
+The server is configured through configuration files. There are two directories with config files:
+
+``conf/core``
+   Upstream defaults are stored here.
+``conf/ext``
+   User specific configuration should be stored here, settings in ``ext`` override settings in
+   ``core``.  Additionally, configuration files may be stored in ``*.d`` directories here, named
+   after the original config file name.  For example, the general server configuration will be
+   assembled from ``conf/core/server.conf``, ``conf/ext/server.conf`` and any ``*.conf`` files found
+   in ``conf/ext/server.conf.d``.
+
+Configuration files
+-------------------
+
+In each of these directories, the server looks for the following files:
+
+``server.conf``
+   General server configuration options.  The possible configuration options are documented inside
+   the `default file <https://gitlab.com/caosdb/caosdb-server/-/blob/dev/conf/core/server.conf>`_.
+
+``global_entity_permissions.xml``
+   :ref:`Permissions <concepts:Permissions>` which are automatically set, based on user roles.
+
+``usersources.ini``
+   This file defines possible sources which are checked when a user tries to authenticate.  Each
+   defined source has a special section, the possible options are defined separately for each user
+   source.  At the moment the best place to look for this specific documentation is at the API
+   documentation of :java:type:`UserSource` and its implementing classes.  The provided `template
+   file <https://gitlab.com/caosdb/caosdb-server/-/blob/dev/conf/core/usersources.ini.template>`_
+   also has some information.  The general concept about authentication realms is described in
+   :java:type:`UserSources`.
+
+``authtoken.yaml``
+   Configuration for dispensed authentication tokens, which can be used to authenticate to CaosDB
+   without the need of a user/password combination.  Possible use cases are server-side scripts or
+   initial setup after the server start.  There is more documentation inside the `template file
+   <https://gitlab.com/caosdb/caosdb-server/-/blob/dev/conf/core/authtoken.example.yaml>`_.
+
+``cache.ccf``
+   Configuration for the Java Caching System (JCS) which can be used by the server.  More
+   documentation is `upstream
+   <http://commons.apache.org/proper/commons-jcs/getting_started/intro.html>`_ and inside `the file
+   <https://gitlab.com/caosdb/caosdb-server/-/blob/dev/conf/core/cache.ccf>`_.
+
+``log4j2-default.properties``, ``log4j2-debug.properties``
+   Configuration for logging, following the standard described by the `log4j library
+   <https://logging.apache.org/log4j/2.x/>`_.  The ``default`` file is always loaded, in debug mode
+   the ``debug`` file iss added as well.
+
+
+Changing the configuration at runtime
+-------------------------------------
+
+Remark:
+   Only when the server is in debug mode, the general configuration can be changed at runtime.
+
+In the debug case, the server provides the ``_server_properties`` resource which allows the ``GET``
+and ``POST`` methods to access the server's properties.  The Python client library conveniently
+wraps this in the :any:`caosdb-pylib:caosdb.common.administration` module.

src/doc/concepts.rst

+===================================
+Basic concepts of the CaosDB server
+===================================
+
+The CaosDB server provides the HTTP API resources to users and client libraries.  It uses a plain
+MariaDB/MySQL database as backend for data storage, raw files are stored separately on the file
+system.
+
+
+Configuration
+-------------
+
+Administrators may configure the server through :doc:`configuration
+files<administration/configuration>`.  Additionally, configurations may be set via the API if the
+server is in debug mode.
+
+
+Permissions
+-----------
+
+CaosDB has a fine grained role based permission system.  Each interaction with the server is
+governed by the current rules of the user, by default this is the ``anonymous`` role.  The
+permissions for an action which involves one or more objects are set either manually or via default
+permissions which can be configured.  For more detailed information, there is separate
+:doc:`documentation of the permission system<permissions>`.
+
+
+
+

src/doc/conf.py

+# -*- coding: utf-8 -*-
+#
+# Configuration file for the Sphinx documentation builder.
+#
+# This file only contains a selection of the most common options. For a full
+# list see the documentation:
+# https://www.sphinx-doc.org/en/master/usage/configuration.html
+
+# -- Path setup --------------------------------------------------------------
+
+# If extensions (or modules to document with autodoc) are in another directory,
+# add these directories to sys.path here. If the directory is relative to the
+# documentation root, use os.path.abspath to make it absolute, like shown here.
+#
+# import os
+# import sys
+# sys.path.insert(0, os.path.abspath('../caosdb'))
+
+
+# -- Project information -----------------------------------------------------
+
+project = 'caosdb-server'
+copyright = '2020, IndiScale GmbH'
+author = 'Daniel Hornung'
+
+# The short X.Y version
+version = '0.X.Y'
+# The full version, including alpha/beta/rc tags
+release = '0.x.y-beta-rc2'
+
+
+# -- General configuration ---------------------------------------------------
+
+# If your documentation needs a minimal Sphinx version, state it here.
+#
+# needs_sphinx = '1.0'
+
+# Add any Sphinx extension module names here, as strings. They can be
+# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
+# ones.
+extensions = [
+    'javasphinx',
+    'sphinx.ext.autodoc',
+    'sphinx.ext.intersphinx',
+    'sphinx.ext.napoleon',     # For Google style docstrings
+    "recommonmark",            # For markdown files.
+    "sphinx.ext.autosectionlabel",  # Allow reference sections using its title
+]
+
+# Add any paths that contain templates here, relative to this directory.
+templates_path = ['_templates']
+
+# The suffix(es) of source filenames.
+# You can specify multiple suffix as a list of string:
+#
+# source_suffix = ['.rst', '.md']
+source_suffix = '.rst'
+
+# The master toctree document.
+master_doc = 'index'
+
+# The language for content autogenerated by Sphinx. Refer to documentation
+# for a list of supported languages.
+#
+# This is also used if you do content translation via gettext catalogs.
+# Usually you set "language" from the command line for these cases.
+language = None
+
+# List of patterns, relative to source directory, that match files and
+# directories to ignore when looking for source files.
+# This pattern also affects html_static_path and html_extra_path.
+exclude_patterns = []
+
+# The name of the Pygments (syntax highlighting) style to use.
+pygments_style = None
+
+
+# -- Options for HTML output -------------------------------------------------
+
+# The theme to use for HTML and HTML Help pages.  See the documentation for
+# a list of builtin themes.
+#
+html_theme = 'alabaster'
+
+# Theme options are theme-specific and customize the look and feel of a theme
+# further.  For a list of options available for each theme, see the
+# documentation.
+#
+# html_theme_options = {}
+
+# Add any paths that contain custom static files (such as style sheets) here,
+# relative to this directory. They are copied after the builtin static files,
+# so a file named "default.css" will overwrite the builtin "default.css".
+html_static_path = ['_static']
+
+# Custom sidebar templates, must be a dictionary that maps document names
+# to template names.
+#
+# The default sidebars (for documents that don't match any pattern) are
+# defined by theme itself.  Builtin themes are using these templates by
+# default: ``['localtoc.html', 'relations.html', 'sourcelink.html',
+# 'searchbox.html']``.
+#
+# html_sidebars = {}
+
+
+# -- Options for HTMLHelp output ---------------------------------------------
+
+# Output file base name for HTML help builder.
+htmlhelp_basename = 'caosdb-serverdoc'
+
+
+# -- Options for LaTeX output ------------------------------------------------
+
+latex_elements = {
+    # The paper size ('letterpaper' or 'a4paper').
+    #
+    # 'papersize': 'letterpaper',
+
+    # The font size ('10pt', '11pt' or '12pt').
+    #
+    # 'pointsize': '10pt',
+
+    # Additional stuff for the LaTeX preamble.
+    #
+    # 'preamble': '',
+
+    # Latex figure (float) alignment
+    #
+    # 'figure_align': 'htbp',
+}
+
+# Grouping the document tree into LaTeX files. List of tuples
+# (source start file, target name, title,
+#  author, documentclass [howto, manual, or own class]).
+latex_documents = [
+    (master_doc, 'caosdb-server.tex', 'caosdb-server Documentation',
+     'IndiScale GmbH', 'manual'),
+]
+
+
+# -- Options for manual page output ------------------------------------------
+
+# One entry per manual page. List of tuples
+# (source start file, name, description, authors, manual section).
+man_pages = [
+    (master_doc, 'caosdb-server', 'caosdb-server Documentation',
+     [author], 1)
+]
+
+
+# -- Options for Texinfo output ----------------------------------------------
+
+# Grouping the document tree into Texinfo files. List of tuples
+# (source start file, target name, title, author,
+#  dir menu entry, description, category)
+texinfo_documents = [
+    (master_doc, 'caosdb-server', 'caosdb-server Documentation',
+     author, 'caosdb-server', 'One line description of project.',
+     'Miscellaneous'),
+]
+
+
+# -- Options for Epub output -------------------------------------------------
+
+# Bibliographic Dublin Core info.
+epub_title = project
+
+# The unique identifier of the text. This can be a ISBN number
+# or the project homepage.
+#
+# epub_identifier = ''
+
+# A unique identification for the text.
+#
+# epub_uid = ''
+
+# A list of files that should not be packed into the epub file.
+epub_exclude_files = ['search.html']
+
+
+# -- Extension configuration -------------------------------------------------
+
+# -- Options for javasphinx --------------------------------------------------
+# See also https://bronto-javasphinx.readthedocs.io/en/latest/
+
+# javadoc_url_map = {
+#     '<namespace_here>' : ('<base_url_here>', 'javadoc'),
+# }
+
+
+# -- Options for intersphinx -------------------------------------------------
+
+# https://www.sphinx-doc.org/en/master/usage/extensions/intersphinx.html#confval-intersphinx_mapping
+intersphinx_mapping = {
+    "python": ("https://docs.python.org/", None),
+    "caosdb-pylib": ("https://caosdb.gitlab.io/caosdb-pylib/", None),
+}
+
+
+# -- Options for autodoc -----------------------------------------------------
+# TODO Which options do we want?
+autodoc_default_options = {
+    'members': None,
+    'undoc-members': None,
+}
+
+
+# -- Options for autosectionlabel --------------------------------------------
+
+autosectionlabel_prefix_document = True

src/doc/getting_started.md

+# Getting started with pycaosdb #
+
+1. Install
+2. import
+3. enjoy

src/doc/index.rst

+
+Welcome to caosdb-server's documentation!
+=========================================
+
+.. toctree::
+   :maxdepth: 2
+   :caption: Contents:
+   :hidden:
+   :glob:
+
+   Getting started <getting_started>
+   Concepts <concepts>
+   tutorials
+   administration/*
+   API documentation<_apidoc/packages>
+
+Welcome to the CaosDB, the flexible semantic data management toolkit!
+
+This documentation helps you to :doc:`get started<getting_started>`, explains the most important
+:doc:`concepts<concepts>` and offers a range of :doc:`tutorials<tutorials>`.
+
+
+Indices and tables
+==================
+
+* :ref:`genindex`
+* :ref:`modindex`
+* :ref:`search`

src/main/java/org/caosdb/server/database/backend/transaction/VersionTransaction.java

@@ -35,7 +35,7 @@ import org.caosdb.server.entity.EntityInterface;
 import org.caosdb.server.entity.Version;
 
 /**
- * Abstract base class which retrieves and caches the full, but flag version history. The
+ * Abstract base class which retrieves and caches the full, but flat version history. The
  * implementations then use the flat version history to construct either single version information
  * items (see {@link RetrieveVersionInfo}) or the complete history as a tree (see {@link
  * RetrieveVersionHistory})

src/main/java/org/caosdb/server/jobs/Job.java

@@ -57,6 +57,13 @@ import org.caosdb.server.utils.Observer;
 import org.caosdb.server.utils.ServerMessages;
 import org.reflections.Reflections;
 
+
+/**
+ * This is a Job.
+ *
+ * @todo Describe me.
+ *
+ */
 public abstract class Job extends AbstractObservable implements Observer {
   private Transaction<? extends TransactionContainer> transaction = null;
   private Mode mode = null;

src/test/docker/Dockerfile

 FROM debian:buster
 RUN apt-get update && \
-  apt-get install \
-      git make mariadb-server maven openjdk-11-jdk-headless \
-      python3-pip screen libpam0g-dev unzip curl shunit2 -y
+  apt-get install -y \
+    git make mariadb-server maven openjdk-11-jdk-headless \
+    python3-pip screen libpam0g-dev unzip curl shunit2 \
+    python3-sphinx \
+  && \
+  pip3 install javasphinx recommonmark
+
+# Alternative, if javasphinx fails because python3-sphinx is too recent:
+# (`_l` not found):
+#
+# git clone git@github.com:simgrid/javasphinx.git
+# cd javasphinx
+# git checkout 659209069603a
+# pip3 install .