diff --git a/.gitignore b/.gitignore
index 341bfea4f1381b6eb59e548f395859bad35520d7..b522b1da9176e59756bffe89cd4eafe0d751a23c 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,3 +1,5 @@
+# -*- mode:conf; -*-
+
# dot files
.*
!/.gitignore
@@ -11,3 +13,6 @@ __pycache__/
dist/
build/
src/caosdb/version.py
+
+# documentation
+_apidoc
\ No newline at end of file
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 1db4c0394de48cbd82d5abbe4d7c581abcb9b277..b45dde5de1e4cafdbc02a15c4b775b763d5447bd 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -4,6 +4,8 @@
# Copyright (C) 2018 Research Group Biomedical Physics,
# Max-Planck-Institute for Dynamics and Self-Organization Göttingen
# Copyright (C) 2019 Henrik tom Wörden
+# Copyright (C) 2020 Indiscale GmbH <info@indiscale.com>
+# Copyright (C) 2020 Daniel Hornung <d.hornung@indiscale.com>
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as
@@ -88,3 +90,21 @@ build-testenv:
--cache-from $CI_REGISTRY_IMAGE
-t $CI_REGISTRY_IMAGE .
- docker push $CI_REGISTRY_IMAGE
+
+# Build the sphinx documentation and make it ready for deployment by Gitlab Pages
+# documentation:
+# stage: deploy
+
+# Special job for serving a static website. See https://docs.gitlab.com/ee/ci/yaml/README.html#pages
+pages:
+ stage: deploy
+ only:
+ # TODO this should be for master only, once releases are more regularly
+ - dev
+ script:
+ - echo "Deploying"
+ - make doc
+ - cp -r build/doc/html public
+ artifacts:
+ paths:
+ - public
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 631e0133af48d279399bd9bcffe34abcaf67288b..b65d9c17e4780e91b5983200796dbc6d9ab6ad14 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -14,6 +14,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
* Versioning support (experimental). The version db.Version class can
represents particular entity versions and also the complete history of an
entity.
+* Automated documentation builds: `make doc`
### Changed ###
diff --git a/Makefile b/Makefile
new file mode 100644
index 0000000000000000000000000000000000000000..4bc3459d209936c17a445c64e77180d9559e4653
--- /dev/null
+++ b/Makefile
@@ -0,0 +1,33 @@
+# ** header v3.0
+# This file is a part of the CaosDB Project.
+#
+# Copyright (C) 2020 IndiScale GmbH <info@indiscale.com>
+# Copyright (C) 2020 Daniel Hornung <d.hornung@indiscale.com>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see <https://www.gnu.org/licenses/>.
+#
+# ** end header
+
+# This Makefile is a wrapper for several other scripts.
+
+.PHONY: help
+
+help:
+ @echo 'Type `make doc` for documentation, or `make install` for (local) installation.'
+
+doc:
+ $(MAKE) -C src/doc html
+
+install:
+ @echo "Not implemented yet, use pip for installation."
diff --git a/README_SETUP.md b/README_SETUP.md
index 290350105740a206ac47cb5bd761a3c6655b2c15..481182d02d4c25acc5c5ed9607a282b22e861448 100644
--- a/README_SETUP.md
+++ b/README_SETUP.md
@@ -1,11 +1,141 @@
-# Installation
-python3.6 is required.
-pip install . --user
-pip install tox --user
+# Getting started with PyCaosDB #
-# Run Unit Tests
+## Installation ##
+
+### Requirements ###
+
+PyCaosDB needs at least Python 3.6. Additionally, the following packages are required (they will
+typically be installed automatically):
+
+- `lxml`
+- `PyYaml`
+- `PySocks`
+
+### How to install ###
+
+#### Linux ####
+
+Make sure that Python (at least version 3.6) and pip is installed, using your system tools and
+documentation.
+
+Then open a terminal and continue in the [Generic installation](#generic-installation) section.
+
+#### Windows ####
+
+If a Python distribution is not yet installed, we recommend Anaconda Python, which you can download
+for free from [https://www.anaconda.com](https://www.anaconda.com). The "Anaconda Individual Edition" provides most of all
+packages you will ever need out of the box. If you prefer, you may also install the leaner
+"Miniconda" installer, which allows you to install packages as you need them.
+
+After installation, open an Anaconda prompt from the Windows menu and continue in the [Generic
+installation](#generic-installation) section.
+
+#### Generic installation ####
+
+To install PyCaosDB locally, use `pip3` (also called `pip` on some systems):
+
+```sh
+pip3 install --user caosdb
+```
+
+---
+
+Alternatively, obtain the sources from GitLab and install from there (`git` must be installed for
+this option):
+
+```sh
+git clone https://gitlab.com/caosdb/caosdb-pylib
+cd caosdb-pylib
+pip3 install --user .
+```
+
+## Configuration ##
+
+The configuration is done using `ini` configuration files.
+PyCaosDB tries to read from the inifile specified in the environment variable `PYCAOSDBINI` or
+alternatively in `~/.pycaosdb.ini` upon import. After that, the ini file `pycaosdb.ini` in the
+current working directory will be read additionally, if it exists.
+
+Here, we will look at the most common configuration options. For a full and
+comprehensive description please check out
+[pycaosdb.ini file](https://gitlab.com/caosdb/caosdb-pylib/-/blob/master/examples/pycaosdb.ini)
+You can download this file and use it as a starting point.
+
+
+Typically, you need to change at least the `url` and `username` fields as required.
+(Ask your CaosDB administrator or IT crowd if
+you do not know what to put there, but for the demo instances https://demo.indiscale.com, `username=admin`
+and `password=caosdb` should work).
+
+### Authentication ##
+
+The default configuration (that your are asked for your password when ever a connection is created
+can be changed by setting `password_method`:
+
+* with `password_method=input` password (and possibly user) will be queried on demand (**default**)
+* use the password manager [pass](https://www.passwordstore.org) by using `pass` as value, see also the [ArchWiki
+ entry](https://wiki.archlinux.org/index.php/Pass#Basic_usage). This also requires `password_identifier` which refers to the identifier within pass
+ for the desired password.
+* install the python package [keyring](https://pypi.org/project/keyring), to use the system keyring/wallet (macOS, GNOME, KDE,
+ Windows). The password will be queried on first usage.
+* with `password_method=plain` (**strongly discouraged**)
+
+The following illustrates the recommended options:
+
+```ini
+[Connection]
+# using "pass" password manager
+#password_method=pass
+#password_identifier=...
+
+# using the system keyring/wallet (macOS, GNOME, KDE, Windows)
+#password_method=keyring
+```
+
+### SSL Certificate ##
+In some cases (especially if you are testing CaosDB) you might need to supply
+an SSL certificate to allow SSL encryption.
+
+```ini
+[Connection]
+cacert=/path/to/caosdb.ca.pem
+```
+
+### Further Settings ##
+As mentioned above, a complete list of options can be found in the
+[pycaosdb.ini file](https://gitlab.com/caosdb/caosdb-pylib/-/blob/master/examples/pycaosdb.ini) in
+the examples folder of the source code.
+
+## Try it out ##
+
+Start Python and check whether the you can access the database. (You will be asked for the
+password):
+
+```python
+In [1]: import caosdb as db
+In [2]: db.Info()
+Please enter the password: # It's `caosdb` for the demo server.
+Out[2]: Connection to CaosDB with 501 Records.
+```
+
+Note: This setup will ask you for your password whenever a new connection is created. If you do not
+like this, check out the "Authentication" section in the [configuration documentation](configuration.md).
+
+Now would be a good time to continue with the [tutorials](tutorials.html).
+
+## Run Unit Tests
tox
-# Code Formatting
+## Code Formatting
autopep8 -i -r ./
+
+## Documentation #
+
+Build documentation in `build/` with `make doc`.
+
+### Requirements ##
+
+- `sphinx`
+- `sphinx-autoapi`
+- `recommonmark`
diff --git a/examples/pycaosdb.ini b/examples/pycaosdb.ini
index 625a3878341265637e6669291ba9d3c517906534..edc32195fbb364bb355d67b8733e8c7bccbb0d34 100644
--- a/examples/pycaosdb.ini
+++ b/examples/pycaosdb.ini
@@ -4,34 +4,66 @@
# - the location given in the env variable PYCAOSDBINI
[Connection]
-url=https://demo.indiscale.com/
-
-## For local installations, enter your server address and SSL certificate here.
+# URL of the CaosDB server
+# url=https://demo.indiscale.com/
+# For local installations this typically is
# url=https://localhost:10443/
-# cacert=/path/to/caosdb.ca.pem
-
-## If this option is set, the SSL certificate will be ignored. Use with care!
-# ssl_insecure=1
-username=admin
+# User name used for authentication with the server
+# username=admin
-## The password input method can be chosen with the `password_method` setting,
-## which by default is set to `plain`.
-##
-## DEFAULT: the password method is `plain`, with this method the password must
-## be saved as plain text.
+# The password input method defines how the password is supplied that is used
+# for authentication with the server.
+#
+# DEFAULT: `input`
+# The username is optional in this case. The password is entered directly by the user.
+# password_method=input
+#
+# OR: `plain`
+# This implies that the password must # be saved as plain text in a
+# configuration under the 'password' key.
# password_method=plain
# password=caosdb
+#
+# OR `pass`
+# The password is retrieved from the "pass" password manager
+# This password manager uses identifiers to distinguish passwords. Supply this
+# identifier under the key 'password_identifier'.
+# password_method=pass
+# password_identifier=caosdb_password
+#
+# OR: `keyring`
+# Using the system keyring/wallet (macOS, GNOME, KDE, Windows)
+# requires installation of the keyring python package (pip install keyring).
+# password_method=keyring
-## OR: `input`: username is optional, password is entered directly by the user.
-## The default password for the demo server is `caosdb`.
-password_method=input
+# Using an authentication token to connect with the server. This setting is
+# not recommended for users.
+# auth_token=TOKEN
-## OR: `pass`: password is retrieved from the "pass" password manager
-# password_method=pass
-# password_identifier=...
+# If the server's SSL certificate cannot be validated by your installed
+# certificates (default or installed by your admins), you may also need to
+# supply the matching key (pem file):
+# cacert=/path/to/caosdb.ca.pem
-## OR: `keyring`: using the system keyring/wallet (macOS, GNOME, KDE, Windows)
-## requires installation of the keyring python package:
-## pip install keyring
-# password_method=keyring
+# If this option is set, the SSL certificate of the server will not be
+# validated. This has the potential of a man-in-the-middle attack. Use with care!
+# ssl_insecure=True
+
+# You may define the ssl version to be used. It has to be the name of the
+# corresponding attribute in the Python ssl module.
+# ssl_version=PROTOCOL_TLS
+
+# The debug key allows control the verbosity. Set it to 1 or 2 in case you
+# want to see debugging output or if you want to learn more about
+# the internals of the protocol. 0 disables debugging output.
+# debug=0
+
+# You can define a socket proxy to be used.
+# This is for the case that the server sits behind a firewall which is being
+# tunnelled with a socket proxy (SOCKS4 or SOCKS5) (e.g. via ssh's -D option or
+# a dedicated proxy server).
+# socket_proxy=localhost:12345
+
+# This option is used internally and for testing. Do not override.
+# implementation=_DefaultCaosDBServerConnection
diff --git a/src/caosdb/__init__.py b/src/caosdb/__init__.py
index d59c60dba6010cdd29e6fc6c64494c6db5941d00..0135ecd16cc6eaf2ff4baa7fe75019602cadc332 100644
--- a/src/caosdb/__init__.py
+++ b/src/caosdb/__init__.py
@@ -22,6 +22,13 @@
# ** end header
#
+"""CaosDB Python bindings.
+
+Tries to read from the inifile specified in the environment variable `PYCAOSDBINI` or alternatively
+in `~/.pycaosdb.ini` upon import. After that, the ini file `pycaosdb.ini` in the current working
+directory will be read additionally, if it exists.
+"""
+
from os import environ, getcwd
# Import of the connection function (which is used to connect to the DB):
from os.path import expanduser, join
diff --git a/src/caosdb/common/__init__.py b/src/caosdb/common/__init__.py
index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..436281df8077b2cbf357537d36b21567b86ea5a2 100644
--- a/src/caosdb/common/__init__.py
+++ b/src/caosdb/common/__init__.py
@@ -0,0 +1 @@
+"""Commonly used classes for CaosDB."""
diff --git a/src/caosdb/connection/connection.py b/src/caosdb/connection/connection.py
index 65c8413c251ad42f44b77e37ac34a26b1c329c0d..703897d62a40a820a6ff578869a862f6c7c4c019 100644
--- a/src/caosdb/connection/connection.py
+++ b/src/caosdb/connection/connection.py
@@ -246,7 +246,7 @@ def _make_conf(*conf):
_DEFAULT_CONF = {
- "password_method": "plain",
+ "password_method": "input",
"implementation": _DefaultCaosDBServerConnection,
"timeout": 210,
"cacert": resource_filename("caosdb", 'cert/indiscale.ca.crt')
diff --git a/src/doc/Makefile b/src/doc/Makefile
new file mode 100644
index 0000000000000000000000000000000000000000..5458c5300efc82e55686bc1cd6934182c5c8e39a
--- /dev/null
+++ b/src/doc/Makefile
@@ -0,0 +1,47 @@
+# ** header v3.0
+# This file is a part of the CaosDB Project.
+#
+# Copyright (C) 2020 IndiScale GmbH <info@indiscale.com>
+# Copyright (C) 2020 Daniel Hornung <d.hornung@indiscale.com>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see <https://www.gnu.org/licenses/>.
+#
+# ** end header
+
+# This Makefile is a wrapper for sphinx scripts.
+#
+# It is based upon the autocreated makefile for Sphinx documentation.
+
+# You can set these variables from the command line, and also
+# from the environment for the first two.
+SPHINXOPTS ?= -a
+SPHINXBUILD ?= sphinx-build
+SPHINXAPIDOC ?= sphinx-apidoc
+PY_BASEDIR = ../caosdb
+SOURCEDIR = .
+BUILDDIR = ../../build/doc
+
+.PHONY: doc-help Makefile
+
+# Put it first so that "make" without argument is like "make help".
+doc-help:
+ @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
+
+# Catch-all target: route all unknown targets to Sphinx using the new
+# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS).
+%: Makefile apidoc
+ @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
+
+apidoc:
+ @$(SPHINXAPIDOC) -o _apidoc $(PY_BASEDIR)
diff --git a/src/doc/README_SETUP.md b/src/doc/README_SETUP.md
new file mode 120000
index 0000000000000000000000000000000000000000..88332e357f5e06f3de522768ccdcd9e513c15f62
--- /dev/null
+++ b/src/doc/README_SETUP.md
@@ -0,0 +1 @@
+../../README_SETUP.md
\ No newline at end of file
diff --git a/src/doc/concepts.rst b/src/doc/concepts.rst
new file mode 100644
index 0000000000000000000000000000000000000000..29625a0a105dacdea2183eac743d1904a7743ec7
--- /dev/null
+++ b/src/doc/concepts.rst
@@ -0,0 +1,6 @@
+========================
+The concepts of PyCaosDB
+========================
+
+- `Configuration <configuration>`
+
diff --git a/src/doc/conf.py b/src/doc/conf.py
new file mode 100644
index 0000000000000000000000000000000000000000..9e2924ae726e13aacd2f955ae1904b39ad73cbc3
--- /dev/null
+++ b/src/doc/conf.py
@@ -0,0 +1,194 @@
+# -*- coding: utf-8 -*-
+#
+# Configuration file for the Sphinx documentation builder.
+#
+# This file only contains a selection of the most common options. For a full
+# list see the documentation:
+# https://www.sphinx-doc.org/en/master/usage/configuration.html
+
+# -- Path setup --------------------------------------------------------------
+
+# If extensions (or modules to document with autodoc) are in another directory,
+# add these directories to sys.path here. If the directory is relative to the
+# documentation root, use os.path.abspath to make it absolute, like shown here.
+#
+# import os
+# import sys
+# sys.path.insert(0, os.path.abspath('../caosdb'))
+
+
+import sphinx_rtd_theme
+
+
+# -- Project information -----------------------------------------------------
+
+project = 'pycaosdb'
+copyright = '2020, IndiScale GmbH'
+author = 'Daniel Hornung'
+
+# The short X.Y version
+version = '0.4.0'
+# The full version, including alpha/beta/rc tags
+release = '0.4.0-rc'
+
+
+# -- General configuration ---------------------------------------------------
+
+# If your documentation needs a minimal Sphinx version, state it here.
+#
+# needs_sphinx = '1.0'
+
+# Add any Sphinx extension module names here, as strings. They can be
+# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
+# ones.
+extensions = [
+ 'sphinx.ext.autodoc',
+ 'sphinx.ext.intersphinx',
+ 'sphinx.ext.napoleon', # For Google style docstrings
+ "recommonmark", # For markdown files.
+ "sphinx_rtd_theme",
+]
+
+# Add any paths that contain templates here, relative to this directory.
+templates_path = ['_templates']
+
+# The suffix(es) of source filenames.
+# You can specify multiple suffix as a list of string:
+#
+source_suffix = ['.rst', '.md']
+
+# The master toctree document.
+master_doc = 'index'
+
+# The language for content autogenerated by Sphinx. Refer to documentation
+# for a list of supported languages.
+#
+# This is also used if you do content translation via gettext catalogs.
+# Usually you set "language" from the command line for these cases.
+language = None
+
+# List of patterns, relative to source directory, that match files and
+# directories to ignore when looking for source files.
+# This pattern also affects html_static_path and html_extra_path.
+exclude_patterns = []
+
+# The name of the Pygments (syntax highlighting) style to use.
+pygments_style = None
+
+
+# -- Options for HTML output -------------------------------------------------
+
+# The theme to use for HTML and HTML Help pages. See the documentation for
+# a list of builtin themes.
+#
+html_theme = "sphinx_rtd_theme"
+
+# Theme options are theme-specific and customize the look and feel of a theme
+# further. For a list of options available for each theme, see the
+# documentation.
+#
+# html_theme_options = {}
+
+# Add any paths that contain custom static files (such as style sheets) here,
+# relative to this directory. They are copied after the builtin static files,
+# so a file named "default.css" will overwrite the builtin "default.css".
+html_static_path = ['_static']
+
+# Custom sidebar templates, must be a dictionary that maps document names
+# to template names.
+#
+# The default sidebars (for documents that don't match any pattern) are
+# defined by theme itself. Builtin themes are using these templates by
+# default: ``['localtoc.html', 'relations.html', 'sourcelink.html',
+# 'searchbox.html']``.
+#
+# html_sidebars = {}
+
+
+# -- Options for HTMLHelp output ---------------------------------------------
+
+# Output file base name for HTML help builder.
+htmlhelp_basename = 'caosdb-pylibdoc'
+
+
+# -- Options for LaTeX output ------------------------------------------------
+
+latex_elements = {
+ # The paper size ('letterpaper' or 'a4paper').
+ #
+ # 'papersize': 'letterpaper',
+
+ # The font size ('10pt', '11pt' or '12pt').
+ #
+ # 'pointsize': '10pt',
+
+ # Additional stuff for the LaTeX preamble.
+ #
+ # 'preamble': '',
+
+ # Latex figure (float) alignment
+ #
+ # 'figure_align': 'htbp',
+}
+
+# Grouping the document tree into LaTeX files. List of tuples
+# (source start file, target name, title,
+# author, documentclass [howto, manual, or own class]).
+latex_documents = [
+ (master_doc, 'caosdb-pylib.tex', 'caosdb-pylib Documentation',
+ 'IndiScale GmbH', 'manual'),
+]
+
+
+# -- Options for manual page output ------------------------------------------
+
+# One entry per manual page. List of tuples
+# (source start file, name, description, authors, manual section).
+man_pages = [
+ (master_doc, 'pycaosdb', 'pycaosdb documentation',
+ [author], 1)
+]
+
+
+# -- Options for Texinfo output ----------------------------------------------
+
+# Grouping the document tree into Texinfo files. List of tuples
+# (source start file, target name, title, author,
+# dir menu entry, description, category)
+texinfo_documents = [
+ (master_doc, 'pycaosdb', 'pycaosdb documentation',
+ author, 'pycaosdb', 'One line description of project.',
+ 'Miscellaneous'),
+]
+
+
+# -- Options for Epub output -------------------------------------------------
+
+# Bibliographic Dublin Core info.
+epub_title = project
+
+# The unique identifier of the text. This can be a ISBN number
+# or the project homepage.
+#
+# epub_identifier = ''
+
+# A unique identification for the text.
+#
+# epub_uid = ''
+
+# A list of files that should not be packed into the epub file.
+epub_exclude_files = ['search.html']
+
+
+# -- Extension configuration -------------------------------------------------
+
+# -- Options for intersphinx extension ---------------------------------------
+
+# Example configuration for intersphinx: refer to the Python standard library.
+intersphinx_mapping = {'https://docs.python.org/': None}
+
+# TODO Which options do we want?
+autodoc_default_options = {
+ 'members': None,
+ 'undoc-members': None,
+}
diff --git a/src/doc/configuration.md b/src/doc/configuration.md
new file mode 100644
index 0000000000000000000000000000000000000000..b2de2781d5adff4d59cb3648cd912e142327f676
--- /dev/null
+++ b/src/doc/configuration.md
@@ -0,0 +1,54 @@
+# Configuration of PyCaosDB #
+The behavior of PyCaosDB is defined via a configuration that is provided using configuration files.
+PyCaosDB tries to read from the inifile specified in the environment variable `PYCAOSDBINI` or
+alternatively in `~/.pycaosdb.ini` upon import. After that, the ini file `pycaosdb.ini` in the
+current working directory will be read additionally, if it exists.
+
+## Authentication ##
+
+The default configuration (that your are asked for your password when ever a connection is created
+can be changed by setting `password_method`:
+
+* with `password_method=input` password (and possibly user) will be queried on demand (**default**)
+* use the password manager [pass](https://www.passwordstore.org) by using `pass` as value, see also the [ArchWiki
+ entry](https://wiki.archlinux.org/index.php/Pass#Basic_usage). This also requires `password_identifier` which refers to the identifier within pass
+ for the desired password.
+* install the python package [keyring](https://pypi.org/project/keyring), to use the system keyring/wallet (macOS, GNOME, KDE,
+ Windows). The password will be queried on first usage.
+* with `password_method=plain` (**strongly discouraged**)
+
+```ini
+[Connection]
+username=YOUR_USERNAME
+
+# password using "pass" password manager
+#password_method=pass
+#password_identifier=...
+
+# using the system keyring/wallet (macOS, GNOME, KDE, Windows)
+#password_method=keyring
+
+#discouraged: password in plain text
+#password_method=plain
+#password=YOUR_PASSWORD
+```
+
+## SSL Certificate ##
+
+You can set the pass to the ssl certificate to be used:
+
+```ini
+[Connection]
+cacert=/path/to/caosdb.ca.pem
+```
+
+## Further Settings ##
+
+`debug=0` ensures that debug information is **not** printed to the terminal every time you interact
+with CaosDB which makes the experience much less verbose. Set it to 1 or 2 in case you want to help
+debugging (which I hope will not be necessary for this tutorial) or if you want to learn more about
+the internals of the protocol.
+
+A complete list of options can be found in the
+[pycaosdb.ini file](https://gitlab.com/caosdb/caosdb-pylib/-/blob/master/examples/pycaosdb.ini) in
+the examples folder of the source code.
diff --git a/src/doc/index.rst b/src/doc/index.rst
new file mode 100644
index 0000000000000000000000000000000000000000..e8cc93aa398de36c30833bc97ca79ce20564daa6
--- /dev/null
+++ b/src/doc/index.rst
@@ -0,0 +1,27 @@
+
+Welcome to PyCaosDB's documentation!
+====================================
+
+.. toctree::
+ :maxdepth: 2
+ :caption: Contents:
+ :hidden:
+
+ Getting started <README_SETUP>
+ tutorials/index
+ Concepts <concepts>
+ Configuration <configuration>
+ API documentation<_apidoc/modules>
+
+This is the documentation for the Python client library for CaosDB, ``PyCaosDB``.
+
+This documentation helps you to :doc:`get started<getting_started>`, explains the most important
+:doc:`concepts<concepts>` and offers a range of :doc:`tutorials<tutorials>`.
+
+
+Indices and tables
+==================
+
+* :ref:`genindex`
+* :ref:`modindex`
+* :ref:`search`
diff --git a/src/doc/tutorials/basic_analysis.rst b/src/doc/tutorials/basic_analysis.rst
new file mode 100644
index 0000000000000000000000000000000000000000..cc185e0ee08f9e5ee0f890c0ab55f52972882d17
--- /dev/null
+++ b/src/doc/tutorials/basic_analysis.rst
@@ -0,0 +1,47 @@
+
+Basic Analysis
+==============
+
+If you have not yet, configure a connection with the demo instance. E.g.:
+
+>>> import caosdb as db
+>>> _ = db.configure_connection(
+... url="https://demo.indiscale.com/",
+... password_method="plain",
+... username="admin",
+... password="caosdb")
+
+A basic Analysis of data in CaosDB could start like:
+
+>>>
+>>> analyses = db.execute_query("FIND RECORD Analysis with quality_factor")
+>>> qfs = [el.get_property("quality_factor").value for el in analyses]
+
+This first retrieves all analysis records that have a ``quality_factor`` and
+then creates a Python list that contains the values. You could create a
+histogram of those for example by (**warning** this is a very boring histogram)::
+
+ import matplotlib
+ import matplotlib.pyplot as plt
+ plt.hist(qfs)
+ plt.xlabel("quality factors")
+ plt.ylabel("count")
+ plt.show()
+
+
+
+Often we are interested in table like data for our processing. And the disentangling of the property values as above is a bit annoying. Thus there is a convenience function for that.
+
+>>> from caosadvancedtools.table_converter import to_table
+>>> # Let us retrieve the data in a table like form using `SELECT`
+>>> data = db.execute_query("SELECT quality_factor FROM RECORD Analysis with quality_factor" )
+>>> table = to_table(data)
+>>> print(table)
+ quality_factor
+0 ...
+
+Summary
+-------
+
+Now you know, how you can collect query results in lists or tables that can then
+be used for further processing.
diff --git a/src/doc/tutorials/first_steps.rst b/src/doc/tutorials/first_steps.rst
new file mode 100644
index 0000000000000000000000000000000000000000..3da05b05c4d4025f6d81490550fd1482ae147fb4
--- /dev/null
+++ b/src/doc/tutorials/first_steps.rst
@@ -0,0 +1,130 @@
+First Steps
+===========
+
+You should have a working connection to a CaosDB instance now. If not, please check out the
+:doc:`Getting Started secton</getting_started>`.
+
+If you are not yet familiar with Records, RecordTypes and Properties used in CaosDB,
+please check out the respective part in the `Web Interface Tutorial`_.
+You should also know the basics of the CaosDB Query Language (a tutorial is here_).
+
+We recommend that you connect to the demo instance in order to try out the following
+examples. You can do this with
+
+>>> import caosdb as db
+>>> _ = db.configure_connection(
+... url="https://demo.indiscale.com/",
+... password_method="plain",
+... username="admin",
+... password="caosdb")
+
+or by using corresponding settings in the configuration file
+(see :doc:`Getting Started secton</getting_started>`.).
+However, you can also translate the examples to the data model that you have at hand.
+
+Let's start with a simple query.
+
+>>> response = db.execute_query("FIND RECORD Guitar")
+
+Queries work the same way as in the web interface. You simply provide the
+query string to the corresponding function (``db.execute_query``). However, the result is not
+displayed as beautifully as in the web interface (Try ``print(response)``). That is why browsing through
+data is the strength of the web interface while the automated processing of
+data is the strength of the Python client.
+
+>>> type(response)
+<class 'caosdb.common.models.Container'>
+
+As you can see the type of the returned object is Container. Containers are
+simply lists of LinkAhead objects with useful functions to interact with LinkAhead.
+Thus we can easily find out how many Records where returned:
+
+>>> len(response)
+3
+
+Let's look at the first element:
+
+>>> firstguitar = response[0]
+>>> print(type(firstguitar))
+<class 'caosdb.common.models.Record'>
+>>> print(firstguitar)
+<Record ...
+
+.. The above example needs doctest ELLIPSIS
+You see that the object is a Record. It has a Parent and two Properties.
+
+.. note::
+
+ Many useful functions and classes are directly available top level in the module::
+
+ db.Container()
+ db.Record()
+
+Accessing Properties
+--------------------
+
+Often it is necessary to access the value of a property.
+
+>>> # get the property object
+>>> print(firstguitar.get_property("price"))
+<Property id="100" name="price" datatype="DOUBLE" unit="€">48.0</Property>
+<BLANKLINE>
+>>> # the value of it
+>>> print(firstguitar.get_property("price").value)
+48.0
+>>> # What is this?
+>>> print(firstguitar.get_property(100))
+<Property id="100" name="price" datatype="DOUBLE" unit="€">48.0</Property>
+<BLANKLINE>
+
+
+Why did the second version work? In the web interface we do not realize it that easily, but there is only one thing that uniquely identifies Entities in LinkAhead: the id.
+
+In the xml output you see, that the properties have the ids 100 and 106. Often names of entities are also unique, but this is not guaranteed. Thus in many cases it is preferable or even necessary to use the id for identifying LinkAhead Entities.
+
+Ids can also come in handy when searching. Suppose you have some complicated condition for the object that you want
+
+
+>>> # This condition is not that complicated and long but let's suppose it was.
+>>> record = db.execute_query("FIND Analysis with quality_factor=0.08", unique=True)
+>>> # You can use unique=True when you only expect one result Entity. An error will be
+>>> # thrown if the number of results is unequal to 1 and the resulting object will be
+>>> # an Entity and not a Container
+>>> print(type(record))
+<class 'caosdb.common.models.Record'>
+>>> print(record.id)
+123
+
+Now we can continue using the id of the first query. This for example allows to formulate a second query with a condition involving this object without including the potentially long and complicated subquery in this one:
+
+>>> query = "FIND Guitar WHICH IS REFERENCED BY {id}".format(id=record.id)
+>>> guitar = db.execute_query(query, unique=True)
+>>> print(guitar)
+<Record id="115" ...
+
+Files
+-----
+
+You can download files (if the LinkAhead server has access to them)
+
+>>> file = db.execute_query("FIND FILE *2019-023" , unique=True)
+>>> target_path = el = file.download()
+
+The file will be saved under target_path.
+If the files are large data files, it is often a better idea to only retrieve the path of the file and access them via a local mount.
+
+Summary
+-------
+
+Now you know, how you can use Python to send queries to CaosDB and you can access
+the result Records and their properties.
+
+The next tutorial shows how to make some meaningful use of this.
+
+
+
+.. _here: https://gitlabio.something
+.. _`demo instance`: https://demo.indiscale.com
+.. _`IndiScale`: https://indiscale.com
+.. _`Web Interface Tutorial`: https://caosdb.gitlab.io/caosdb-webui/tutorials/model.html
+.. _here: https://caosdb.gitlab.io/caosdb-webui/tutorials/cql.html
diff --git a/src/doc/tutorials/index.rst b/src/doc/tutorials/index.rst
new file mode 100644
index 0000000000000000000000000000000000000000..311f7080045e8925dd26eb03c2da3b12b1dba6e4
--- /dev/null
+++ b/src/doc/tutorials/index.rst
@@ -0,0 +1,15 @@
+
+PyCaosDB Tutorials
+==================
+
+This chapter contains tutorials that lead you from the first steps to
+advanced usage of the Python client.
+
+.. toctree::
+ :maxdepth: 2
+ :caption: Contents:
+ :hidden:
+
+ first_steps
+ basic_analysis
+
diff --git a/tox.ini b/tox.ini
index f9776382db0770e3efe1b2711f50bb5b3adde1ec..94c2dc8affb280d3e7f6cff4536636432c9f7749 100644
--- a/tox.ini
+++ b/tox.ini
@@ -1,5 +1,5 @@
[tox]
-envlist=py36, py37, py38
+envlist=py36, py37, py38, py39
skip_missing_interpreters = true
[testenv]
diff --git a/unittests/docker/Dockerfile b/unittests/docker/Dockerfile
index da45618fa6f60fdcc20042dcaf3df30f4e2d166c..e41fb91b3e9ae54cff277519c03f481c21264e9e 100644
--- a/unittests/docker/Dockerfile
+++ b/unittests/docker/Dockerfile
@@ -1,7 +1,10 @@
FROM debian:latest
RUN apt-get update && \
- apt-get install pylint3 python3-pip tox git \
- curl pycodestyle -y
+ apt-get install -y \
+ pylint3 python3-pip tox git \
+ curl pycodestyle \
+ python3-sphinx
ARG COMMIT="dev"
RUN git clone -b dev https://gitlab.com/caosdb/caosdb-pylib.git && \
cd caosdb-pylib && git checkout $COMMIT && pip3 install .
+RUN pip3 install recommonmark sphinx-rtd-theme
diff --git a/unittests/test_authentication_plain.py b/unittests/test_authentication_plain.py
index 26c57e953f28a2a0ab80e35026c27ff74f18d370..10cbc418df8bd81c81568a4df0cf1e8a4ac498f8 100644
--- a/unittests/test_authentication_plain.py
+++ b/unittests/test_authentication_plain.py
@@ -27,11 +27,18 @@ Unit tests for the modul caosdb.connection.authentication.plain.
"""
from __future__ import unicode_literals
-from pytest import raises
+
from caosdb.connection.authentication.plain import PlainTextCredentialsProvider
+from pytest import raises
def test_subclass_configure():
+ # TODO I do not see the meaning of this test.
+ # It only tests, that the call of the super version of configure sets the
+ # password property. And that due to the subclassing no longer a password
+ # argument can be provided.
+ # Suggestion: Either remove this test or state in what context this test
+ # is meanigful.
"""Test the correct passing of the password argument."""
class SubClassOf(PlainTextCredentialsProvider):
"""A simple subclass of PlainTextCredentialsProvider."""
@@ -51,8 +58,8 @@ def test_subclass_configure():
with raises(TypeError) as exc_info:
instance.configure(password="OH NO!")
- assert exc_info.value.args[0] == ("configure() got multiple values for "
- "keyword argument 'password'")
+ assert exc_info.value.args[0].endswith("configure() got multiple values for "
+ "keyword argument 'password'")
def test_plain_has_logger():
diff --git a/unittests/test_concrete_property.py b/unittests/test_concrete_property.py
index b806098732c8a77a4913bbda0f8b2b8ebdc4a68c..0e5c28534c7ac404b829df575225f42e908adb01 100644
--- a/unittests/test_concrete_property.py
+++ b/unittests/test_concrete_property.py
@@ -24,17 +24,19 @@
"""Tests for the _ConcreteProperty class."""
-# pylint: disable=missing-docstring
-from nose.tools import (assert_is_not_none as there, assert_true as tru,
- assert_equal as eq)
-from caosdb.common.models import _ConcreteProperty
from caosdb import configure_connection
+from caosdb.common.models import _ConcreteProperty
from caosdb.connection.mockup import MockUpServerConnection
+# pylint: disable=missing-docstring
+from nose.tools import assert_equal as eq
+from nose.tools import assert_is_not_none as there
+from nose.tools import assert_true as tru
def setup_module():
there(_ConcreteProperty)
configure_connection(url="unittests", username="testuser",
+ password_method="plain",
password="testpassword", timeout=200,
implementation=MockUpServerConnection)
diff --git a/unittests/test_connection.py b/unittests/test_connection.py
index 5c1518c65b51087ace514ca26fa77eff53130c73..1dfd1fce730d4d4ba627eb4c27830b0c5156fb29 100644
--- a/unittests/test_connection.py
+++ b/unittests/test_connection.py
@@ -23,23 +23,27 @@
#
"""Test caosdb.connection."""
# pylint: disable=missing-docstring
-from __future__ import unicode_literals, print_function
-from builtins import bytes, str # pylint: disable=redefined-builtin
+from __future__ import print_function, unicode_literals
+
import re
-from pytest import raises
-from nose.tools import (assert_equal as eq, assert_raises as raiz, assert_true
- as tru, assert_is_not_none as there, assert_false as
- falz)
-from caosdb.exceptions import ConfigurationException, LoginFailedException
-from caosdb.connection.utils import urlencode, make_uri_path, quote
-from caosdb.connection.connection import (
- configure_connection, CaosDBServerConnection,
- _DefaultCaosDBServerConnection)
-from caosdb.connection.mockup import (MockUpServerConnection, MockUpResponse,
- _request_log_message)
-from caosdb.configuration import get_config, _reset_config
-from caosdb.connection.authentication.interface import CredentialsAuthenticator
+from builtins import bytes, str # pylint: disable=redefined-builtin
+
from caosdb import execute_query
+from caosdb.configuration import _reset_config, get_config
+from caosdb.connection.authentication.interface import CredentialsAuthenticator
+from caosdb.connection.connection import (CaosDBServerConnection,
+ _DefaultCaosDBServerConnection,
+ configure_connection)
+from caosdb.connection.mockup import (MockUpResponse, MockUpServerConnection,
+ _request_log_message)
+from caosdb.connection.utils import make_uri_path, quote, urlencode
+from caosdb.exceptions import ConfigurationException, LoginFailedException
+from nose.tools import assert_equal as eq
+from nose.tools import assert_false as falz
+from nose.tools import assert_is_not_none as there
+from nose.tools import assert_raises as raiz
+from nose.tools import assert_true as tru
+from pytest import raises
def setup_module():
@@ -96,6 +100,7 @@ def test_configure_connection():
get_config().add_section("Connection")
get_config().set("Connection", "url", "https://host.de")
get_config().set("Connection", "username", "test_username")
+ get_config().set("Connection", "password_method", "plain")
get_config().set("Connection", "password", "test_password")
get_config().set("Connection", "timeout", "200")
@@ -152,6 +157,7 @@ def test_init_response():
response = MockUpResponse(
status=200, headers={"sessionToken": "SessionToken"}, body="Body")
there(response)
+
return response
@@ -175,6 +181,7 @@ def test_getter_session_token():
def test_init_connection():
connection = MockUpServerConnection()
there(connection)
+
return connection
@@ -184,6 +191,7 @@ def test_resources_list():
assert len(connection.resources) == 1
connection.resources.append(lambda **kwargs: test_init_response())
assert len(connection.resources) == 2
+
return connection
@@ -214,6 +222,7 @@ def setup_two_resources():
connection = test_init_connection()
connection.resources.extend([r1, r2, r3])
+
return connection
diff --git a/unittests/test_entity.py b/unittests/test_entity.py
index d877e1ab7267977a7e73d65033d6f02c16ea5521..e98dfbef5b6b5e5f691e8aecc2fa7d4a86991452 100644
--- a/unittests/test_entity.py
+++ b/unittests/test_entity.py
@@ -35,6 +35,7 @@ class TestEntity(unittest.TestCase):
def setUp(self):
self.assertIsNotNone(Entity)
configure_connection(url="unittests", username="testuser",
+ password_method="plain",
password="testpassword", timeout=200,
implementation=MockUpServerConnection)
diff --git a/unittests/test_file.py b/unittests/test_file.py
index 957c578062b25c74547b599a25802938b057a2e0..3c80af7f362a7cdabe0a9ebc89cd2986d04fe242 100644
--- a/unittests/test_file.py
+++ b/unittests/test_file.py
@@ -22,16 +22,18 @@
# ** end header
#
"""Tests for the File class."""
-# pylint: disable=missing-docstring
-from nose.tools import (assert_is_not_none as there, assert_true as tru,
- assert_equal as eq)
-from caosdb import Record, File, configure_connection
+from caosdb import File, Record, configure_connection
from caosdb.connection.mockup import MockUpServerConnection
+# pylint: disable=missing-docstring
+from nose.tools import assert_equal as eq
+from nose.tools import assert_is_not_none as there
+from nose.tools import assert_true as tru
def setup_module():
there(File)
configure_connection(url="unittests", username="testuser",
+ password_method="plain",
password="testpassword", timeout=200,
implementation=MockUpServerConnection)
diff --git a/unittests/test_record_type.py b/unittests/test_record_type.py
index c105220c53579537994648b2f9bae5b4d378be13..f31c56decfc394211940296babc83200a470cc8a 100644
--- a/unittests/test_record_type.py
+++ b/unittests/test_record_type.py
@@ -22,16 +22,18 @@
# ** end header
#
"""Tests for the RecordType class."""
-# pylint: disable=missing-docstring
-from nose.tools import (assert_is_not_none as there, assert_true as tru,
- assert_equal as eq)
from caosdb import Entity, RecordType, configure_connection
from caosdb.connection.mockup import MockUpServerConnection
+# pylint: disable=missing-docstring
+from nose.tools import assert_equal as eq
+from nose.tools import assert_is_not_none as there
+from nose.tools import assert_true as tru
def setup_module():
there(RecordType)
configure_connection(url="unittests", username="testuser",
+ password_method="plain",
password="testpassword", timeout=200,
implementation=MockUpServerConnection)