diff --git a/.gitignore b/.gitignore index ddfb9ac071b823c0b1f9b2495c1e44c49290ec1b..ef0861267fc42504b306d158ecfd332a44b3f6ce 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,5 @@ +# -*- mode:conf; -*- + # dot files .* !/.git* @@ -9,6 +11,8 @@ # the build dir /public /sss_bin +/node_modules/ +/build # screen logs screenlog.* diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 2a014f451fb7b6de3dd5a8715b3aeae701d806e5..7f009150afd1b3aec6d3aed2383c6b45607335a2 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -6,6 +6,7 @@ # Copyright (C) 2019 Henrik tom Wörden # Copyright (C) 2020 Timm Fitschen (t.fitschen@indiscale.com) # Copyright (C) 2020 IndiScale GmbH (info@indiscale.com) +# Copyright (C) 2020 Daniel Hornung <d.hornung@indiscale.com> # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License as @@ -90,9 +91,25 @@ build-testenv: - cd test/docker - docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY # use here general latest or specific branch latest... - - docker pull $CI_REGISTRY_IMAGE:latest || true - docker build --pull - --cache-from $CI_REGISTRY_IMAGE:latest -t $CI_REGISTRY_IMAGE:latest . - docker push $CI_REGISTRY_IMAGE:latest + +# Build the sphinx documentation and make it ready for deployment by Gitlab Pages +# documentation: +# stage: deploy + +# Special job for serving a static website. See https://docs.gitlab.com/ee/ci/yaml/README.html#pages +pages: + tags: [ docker ] + stage: deploy + only: + - f-doc + script: + - echo "Deploying" + - make doc + - cp -r build/doc/html public + artifacts: + paths: + - public diff --git a/CHANGELOG.md b/CHANGELOG.md index c2802f2a004cb839bffcd816537bbb7a24769699..f2c54f430fd5e7f3ccd11294c8fe0daf5c7a23f1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,7 +20,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 button (edit_mode). * Plotly preview has an additional parameter for a config object, e.g., for disabling the plotly logo -- After a SELECT statement now also all referenced files can be downloaded. +* After a SELECT statement now also all referenced files can be downloaded. +* Automated documentation builds: `make doc` ### Changed (for changes in existing functionality) - enabled and enhanced autocompletion diff --git a/makefile b/Makefile similarity index 99% rename from makefile rename to Makefile index d44625927fbef317ce4afa072a616ada36df96ce..92f69dfb6234826b752a1c3355e1e812e5b2df61 100644 --- a/makefile +++ b/Makefile @@ -307,3 +307,9 @@ PYLINT ?= pylint3 PYTHON_FILES = $(subst $(ROOT_DIR)/,,$(shell find $(ROOT_DIR)/ -iname "*.py")) pylint: $(PYTHON_FILES) for f in $(PYTHON_FILES); do $(PYLINT) -d all -e E,F $$f || exit 1; done + + +# Compile the standalone documentation +.PHONY: doc +doc: + $(MAKE) -C src/doc html diff --git a/README_SETUP.md b/README_SETUP.md index 127549ec3a273b0cca08ee6a824601fbbd3172bf..6fb59c342e91e938851ba5f999dab2e099e1f187 100644 --- a/README_SETUP.md +++ b/README_SETUP.md @@ -68,3 +68,15 @@ information. # Clean * Run `make clean` to clean up everything. + +# Documentation # + +Build documentation in `build/` with `make doc`. + +## Requirements ## + +- sphinx +- sphinx-autoapi +- jsdoc (`npm install jsdoc`) +- sphinx-js +- recommonmark diff --git a/src/core/js/ext_bottom_line.js b/src/core/js/ext_bottom_line.js index d92ea4d5b4e6285469c92c583d501534abf98d21..bd7ec6a9938ff80afbd7dafdf0a734cd443678bc 100644 --- a/src/core/js/ext_bottom_line.js +++ b/src/core/js/ext_bottom_line.js @@ -154,7 +154,7 @@ var ext_bottom_line = function($, logger, is_in_view_port, load_config, getEntit * Tiff files are decompressed if necessary and converted into png by UTIF library. * * @param {HTMLElement} entity - * @return {Promise for HTMLElement} Promise for an IMG element. + * @return {Promise | HTMLElement} Promise for an IMG element. */ const _create_tiff_preview = function(entity) { const path = connection.getFileSystemPath() + getEntityPath(entity); diff --git a/src/core/js/footer.js b/src/core/js/footer.js index 79f3ed0ecb382534d7c997ce0bc230178792f677..c48c5cf19c405aea326b36aba2868008466a8329 100644 --- a/src/core/js/footer.js +++ b/src/core/js/footer.js @@ -26,7 +26,7 @@ * Call initially. * * TODO refactor to async function for better readability. - * @return + * @return something */ function footer_initOnDocumentReady() { diff --git a/src/core/js/preview.js b/src/core/js/preview.js index f74fd13ffd2fe373543c025866e91cfe6b6fd9eb..b85f7b56dc13ff94aa2c3de90eb1b464d7f1e9d6 100644 --- a/src/core/js/preview.js +++ b/src/core/js/preview.js @@ -211,8 +211,8 @@ var preview = new function() { /** * Transform the raw xml response of the server into an array of entities for preview. * - * @param {Promise for XMLDocument} xml - A Promise for the servers xml response. - * @return {Promise for HTMLElement[]} A Promise for an array of entities. + * @param {Promise | XMLDocument} xml - A Promise for the servers xml response. + * @return {Promise | HTMLElement[]} A Promise for an array of entities. */ this.processPreviewResponse = function(xml) { let xsl = preview.getEntityXsl(); @@ -222,7 +222,7 @@ var preview = new function() { /** * Retrieve the XSL script for entities from the server. * - * @return {Promise for XMLDocument} A Promise for the XSL script. + * @return {Promise | XMLDocument} A Promise for the XSL script. */ this.getEntityXsl = async function _getEntityXsl() { return transformation.retrieveEntityXsl(); @@ -679,7 +679,7 @@ var preview = new function() { * Retrieve a list of entities from the server. * * @param {String[]} entityIds - The ids of the entities which are to be retrieved. - * @return {Promise for HTMLElement[]} A Promise for an array of entities. + * @return {Promise | HTMLElement[]} A Promise for an array of entities. */ this.retrievePreviewEntities = async function _rPE(entityIds) { try { @@ -711,9 +711,9 @@ var preview = new function() { /** * Transform the xml to an array of entities. * - * @param {Promise XMLDocument} xml - The server response. - * @param {Promise XMLDocument} xsl - The xsl script. - * @return {Promise HTMLElement[]} A promise for an Array of HTMLElements. + * @param {Promise | XMLDocument} xml - The server response. + * @param {Promise | XMLDocument} xsl - The xsl script. + * @return {Promise | HTMLElement[]} A promise for an Array of HTMLElements. */ this.transformXmlToPreviews = async function _tXTP(xml, xsl) { let html = await asyncXslt(xml, xsl); diff --git a/src/core/js/webcaosdb.js b/src/core/js/webcaosdb.js index 509abee58bd66bd938788527f4f0e459ceb36e6d..55337a36a97d6a7ad42aadfe673e429d6d942a0a 100644 --- a/src/core/js/webcaosdb.js +++ b/src/core/js/webcaosdb.js @@ -1706,13 +1706,13 @@ function insertParam(xsl, name, value = null) { /** * When the page is scrolled down 100 pixels, the scroll-back button appears. * - * @return + * @return FIXME */ /** * Every initial function calling is done here. * - * @return + * @return TODO */ function initOnDocumentReady() { hintMessages.init(); diff --git a/src/doc/Makefile b/src/doc/Makefile new file mode 100644 index 0000000000000000000000000000000000000000..b3c35b4ddbc2849eb6b3bdc223095e275669ad39 --- /dev/null +++ b/src/doc/Makefile @@ -0,0 +1,51 @@ +# ** header v3.0 +# This file is a part of the CaosDB Project. +# +# Copyright (C) 2020 IndiScale GmbH <info@indiscale.com> +# Copyright (C) 2020 Daniel Hornung <d.hornung@indiscale.com> +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see <https://www.gnu.org/licenses/>. +# +# ** end header + +# This Makefile is a wrapper for sphinx scripts. +# +# It is based upon the autocreated makefile for Sphinx documentation. + +# You can set these variables from the command line, and also +# from the environment for the first two. +SPHINXOPTS ?= -a +SPHINXBUILD ?= sphinx-build +# SPHINXAPIDOC ?= javasphinx-apidoc +SOURCEDIR = . +BUILDDIR = ../../build/doc + +# npm is not always in the global PATH +NPM_PATH = $(shell npm bin) + +.PHONY: doc-help Makefile apidoc + +# Put it first so that "make" without argument is like "make help". +doc-help: + @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) + +# Catch-all target: route all unknown targets to Sphinx using the new +# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). +%: Makefile + PATH=$(NPM_PATH):$$PATH $(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) +# sphinx-build -M html . ../../build/doc + +# Not necessary in this repository, apidoc is doone with the sphinx-autoapi extension +# apidoc: +# @$(SPHINXAPIDOC) -o _apidoc --update --title="CaosDB Server" ../main/ diff --git a/src/doc/concepts.rst b/src/doc/concepts.rst new file mode 100644 index 0000000000000000000000000000000000000000..23e5fc4f6ddb666757fb9c79e192e07ffed8fb44 --- /dev/null +++ b/src/doc/concepts.rst @@ -0,0 +1,6 @@ +======================== +The concepts of pycaosdb +======================== + +Some text... + diff --git a/src/doc/conf.py b/src/doc/conf.py new file mode 100644 index 0000000000000000000000000000000000000000..56fbccb79f8e5ecbff7f70e5c7b7e2c0f56cf040 --- /dev/null +++ b/src/doc/conf.py @@ -0,0 +1,206 @@ +# -*- coding: utf-8 -*- +# +# Configuration file for the Sphinx documentation builder. +# +# This file only contains a selection of the most common options. For a full +# list see the documentation: +# https://www.sphinx-doc.org/en/master/usage/configuration.html + +# -- Path setup -------------------------------------------------------------- + +# If extensions (or modules to document with autodoc) are in another directory, +# add these directories to sys.path here. If the directory is relative to the +# documentation root, use os.path.abspath to make it absolute, like shown here. +# +# import os +# import sys +# sys.path.insert(0, os.path.abspath('../caosdb')) + + +# -- Project information ----------------------------------------------------- + +project = 'caosdb-server' +copyright = '2020, IndiScale GmbH' +author = 'Daniel Hornung' + +# The short X.Y version +version = '0.X.Y' +# The full version, including alpha/beta/rc tags +release = '0.x.y-beta-rc2' + + +# -- General configuration --------------------------------------------------- + +# If your documentation needs a minimal Sphinx version, state it here. +# +# needs_sphinx = '1.0' + +# Add any Sphinx extension module names here, as strings. They can be +# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom +# ones. +extensions = [ + 'sphinx_js', + 'sphinx.ext.todo', + "sphinx.ext.autodoc", + 'autoapi.extension', + "recommonmark", # For markdown files. + # 'sphinx.ext.intersphinx', + # 'sphinx.ext.napoleon', # For Google style docstrings +] + +# Add any paths that contain templates here, relative to this directory. +templates_path = ['_templates'] + +# The suffix(es) of source filenames. +# You can specify multiple suffix as a list of string: +# +# source_suffix = ['.rst', '.md'] +source_suffix = '.rst' + +# The master toctree document. +master_doc = 'index' + +# The language for content autogenerated by Sphinx. Refer to documentation +# for a list of supported languages. +# +# This is also used if you do content translation via gettext catalogs. +# Usually you set "language" from the command line for these cases. +language = None + +# List of patterns, relative to source directory, that match files and +# directories to ignore when looking for source files. +# This pattern also affects html_static_path and html_extra_path. +exclude_patterns = [] + +# The name of the Pygments (syntax highlighting) style to use. +pygments_style = None + + +# -- Options for HTML output ------------------------------------------------- + +# The theme to use for HTML and HTML Help pages. See the documentation for +# a list of builtin themes. +# +html_theme = 'alabaster' + +# Theme options are theme-specific and customize the look and feel of a theme +# further. For a list of options available for each theme, see the +# documentation. +# +# html_theme_options = {} + +# Add any paths that contain custom static files (such as style sheets) here, +# relative to this directory. They are copied after the builtin static files, +# so a file named "default.css" will overwrite the builtin "default.css". +html_static_path = ['_static'] + +# Custom sidebar templates, must be a dictionary that maps document names +# to template names. +# +# The default sidebars (for documents that don't match any pattern) are +# defined by theme itself. Builtin themes are using these templates by +# default: ``['localtoc.html', 'relations.html', 'sourcelink.html', +# 'searchbox.html']``. +# +# html_sidebars = {} + + +# -- Options for HTMLHelp output --------------------------------------------- + +# Output file base name for HTML help builder. +htmlhelp_basename = 'caosdb-webuidoc' + + +# -- Options for LaTeX output ------------------------------------------------ + +latex_elements = { + # The paper size ('letterpaper' or 'a4paper'). + # + # 'papersize': 'letterpaper', + + # The font size ('10pt', '11pt' or '12pt'). + # + # 'pointsize': '10pt', + + # Additional stuff for the LaTeX preamble. + # + # 'preamble': '', + + # Latex figure (float) alignment + # + # 'figure_align': 'htbp', +} + +# Grouping the document tree into LaTeX files. List of tuples +# (source start file, target name, title, +# author, documentclass [howto, manual, or own class]). +latex_documents = [ + (master_doc, 'caosdb-webui.tex', 'caosdb-webui Documentation', + 'IndiScale GmbH', 'manual'), +] + + +# -- Options for manual page output ------------------------------------------ + +# One entry per manual page. List of tuples +# (source start file, name, description, authors, manual section). +man_pages = [ + (master_doc, 'caosdb-webui', 'caosdb-webui Documentation', + [author], 1) +] + + +# -- Options for Texinfo output ---------------------------------------------- + +# Grouping the document tree into Texinfo files. List of tuples +# (source start file, target name, title, author, +# dir menu entry, description, category) +texinfo_documents = [ + (master_doc, 'caosdb-webui', 'caosdb-webui Documentation', + author, 'caosdb-webui', 'One line description of project.', + 'Miscellaneous'), +] + + +# -- Options for Epub output ------------------------------------------------- + +# Bibliographic Dublin Core info. +epub_title = project + +# The unique identifier of the text. This can be a ISBN number +# or the project homepage. +# +# epub_identifier = '' + +# A unique identification for the text. +# +# epub_uid = '' + +# A list of files that should not be packed into the epub file. +epub_exclude_files = ['search.html'] + + +# -- Extension configuration ------------------------------------------------- + +# -- Options for sphinx-js --------------------------------------------------- +# See also https://pypi.org/project/sphinx-js/ + +js_source_path = '../core/js/' +primary_domain = 'js' # Not strictly necessary? + +# -- Options for intersphinx extension --------------------------------------- + +# Example configuration for intersphinx: refer to the Python standard library. +intersphinx_mapping = {'https://docs.python.org/': None} + +# TODO Which options do we want? +autodoc_default_options = { + 'members': None, + 'undoc-members': None, +} + +# -- Options for sphinx-autoapi ---------------------------------------------- +# See also https://pypi.org/project/sphinx-js/ + +autoapi_type = 'javascript' +autoapi_dirs = ['../core/js/'] diff --git a/src/doc/getting_started.md b/src/doc/getting_started.md new file mode 100644 index 0000000000000000000000000000000000000000..6805181e73a7f338fbacb74bdd9f734360404372 --- /dev/null +++ b/src/doc/getting_started.md @@ -0,0 +1,5 @@ +# Getting started with pycaosdb # + +1. Install +2. import +3. enjoy diff --git a/src/doc/index.rst b/src/doc/index.rst new file mode 100644 index 0000000000000000000000000000000000000000..9c1f07e6ca5023d5dc25176d7ec7c68909952652 --- /dev/null +++ b/src/doc/index.rst @@ -0,0 +1,35 @@ + +Welcome to caosdb-server's documentation! +========================================= + +.. toctree:: + :maxdepth: 2 + :caption: Contents: + :hidden: + + Getting started <getting_started> + Concepts <concepts> + tutorials + API documentation<autoapi> + +Welcome to the WebUI for CaosDB, ``pycaosdb``! + +This documentation helps you to :doc:`get started<getting_started>`, explains the most important +:doc:`concepts<concepts>` and offers a range of :doc:`tutorials<tutorials>`. + +.. note:: + + TODO: Build the index (manually?) like here: https://github.com/mozilla/fathom/edit/master/docs/ + + .. js:autofunction:: input2caosdbDate + + .. js:autofunction:: getEntityVersion + + + +Indices and tables +================== + +* :ref:`genindex` +* :ref:`modindex` +* :ref:`search` diff --git a/test/docker/Dockerfile b/test/docker/Dockerfile index bccd94fe14f49a79d8a43b559b3b857ed1d7d07d..f355939cacd0f16b7038ae459584f2d015e04763 100644 --- a/test/docker/Dockerfile +++ b/test/docker/Dockerfile @@ -1,8 +1,16 @@ -FROM debian:latest -RUN apt-get update && \ - apt-get install firefox-esr gettext-base pylint3 python3-pip \ - python3-httpbin git curl x11-apps xvfb unzip -y python3-pytest +FROM debian:10 +RUN echo "deb http://deb.debian.org/debian buster-backports main" >> /etc/apt/sources.list \ + && apt-get update \ + && apt-get install -y \ + firefox-esr gettext-base pylint3 python3-pip \ + python3-httpbin git curl x11-apps xvfb unzip python3-pytest \ + python3-recommonmark \ + && apt-get install -y -t buster-backports \ + npm + RUN pip3 install caosdb RUN pip3 install pandas xlrd RUN pip3 install git+https://gitlab.com/caosdb/caosdb-advanced-user-tools.git@dev - +# For automatic documentation +RUN npm install -g jsdoc +RUN pip3 install sphinx-js sphinx-autoapi