Merge branch 'f-server-side-scripting' into f-socks-connection

README.md

+<!--THIS FILE HAS BEEN GENERATED BY A SCRIPT. PLEASE DON'T CHANGE IT MANUALLY.-->
+
 # Welcome
 
 This is the **CaosDB Python Client Library** repository and a part of the

examples/server_side_script.py

+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+#
+# ** header v3.0
+# This file is a part of the CaosDB Project.
+#
+# Copyright (C) 2018 Research Group Biomedical Physics,
+# Max-Planck-Institute for Dynamics and Self-Organization Göttingen
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see <https://www.gnu.org/licenses/>.
+#
+# ** end header
+#
+"""server_side_script.py.
+
+An example which implements a minimal server-side script.
+
+1) This script expects to find a *.txt file in the .upload_files dir which is
+printed to stdout.
+
+2) It executes a "Count stars" query and prints the result to stdout.
+
+3) It will return with code 0 if everything is ok, or with any code that is
+specified with the commandline option --exit
+"""
+
+import sys
+from os import listdir
+from caosdb import configure_connection, execute_query
+
+
+# parse --auth-token option and configure connection
+CODE = 0
+QUERY = "COUNT stars"
+for arg in sys.argv:
+    if arg.startswith("--auth-token="):
+        auth_token = arg[13:]
+        configure_connection(auth_token=auth_token)
+    if arg.startswith("--exit="):
+        CODE = int(arg[7:])
+    if arg.startswith("--query="):
+        QUERY = arg[8:]
+
+
+############################################################
+# 1 # find and print *.txt file ############################
+############################################################
+
+try:
+    for fname in listdir(".upload_files"):
+        if fname.endswith(".txt"):
+            with open(".upload_files/{}".format(fname)) as f:
+                print(f.read())
+except FileNotFoundError:
+    pass
+
+
+############################################################
+# 2 # query "COUNT stars" ##################################
+############################################################
+
+RESULT = execute_query(QUERY)
+print(RESULT)
+
+############################################################
+# 3 ########################################################
+############################################################
+
+sys.exit(CODE)

pytest.ini

 [pytest]
 testpaths=unittests
-addopts = -vv --cov=caosdb
+addopts=-x -vv --cov=caosdb

setup.cfg

+[aliases]
+test=pytest
 [pycodestyle]
 ignore=E501,E121,E123,E126,E226,E24,E704,W503,W504

setup.py

@@ -32,8 +32,9 @@ setup(name='PyCaosDB',
       author_email='timm.fitschen@ds.mpg.de',
       packages=find_packages('src'),
       package_dir={'': 'src'},
-      install_requires=['lxml>=3.6.4', 'coverage>=4.4.2',
+      install_requires=['lxml>=3.6.4',
                         'PyYaml>=3.12', 'future'],
       extras_require={'keyring': ['keyring>=13.0.0']},
-      tests_require=["pytest"],
+      setup_requires=["pytest-runner>=2.0,<3dev"],
+      tests_require=["pytest", "pytest-cov", "coverage>=4.4.2"],
       )

src/caosdb/__init__.py

@@ -42,5 +42,7 @@ from caosdb.common.models import (delete, execute_query, raise_errors,
                                   get_global_acl, get_known_permissions)
 # Import of convenience methods:
 import caosdb.apiutils
+
+# read configuration these files
 configure(expanduser('~/.pycaosdb.ini'))
 configure(join(getcwd(), "pycaosdb.ini"))

src/caosdb/common/administration.py

@@ -31,6 +31,71 @@ from caosdb.connection.connection import get_connection
 from caosdb.common.utils import xml2str
 
 
+def set_server_property(key, value):
+    """set_server_property.
+
+    Set a server property.
+
+    Parameters
+    ----------
+    key : str
+        The name of the server property.
+    value : str
+        The value of the server property.
+
+
+    Returns
+    -------
+    None
+    """
+    con = get_connection()
+
+    con._form_data_request(method="POST", path="_server_properties",
+                           params={key: value}).read()
+
+
+def get_server_properties():
+    """get_server_properties.
+
+    Get all server properties as a dict.
+
+    Returns
+    -------
+    dict
+        The server properties.
+    """
+    con = get_connection()
+    body = con._http_request(method="GET", path="_server_properties").response
+    xml = etree.parse(body)
+    props = dict()
+    for elem in xml.getroot():
+        props[elem.tag] = elem.text
+    return props
+
+
+def get_server_property(key):
+    """get_server_property.
+
+    Get a server property.
+
+    Parameters
+    ----------
+    key : str
+        The name of the server property
+
+    Returns
+    -------
+    value : str
+        The string value of the server property.
+
+    Raises
+    ------
+    KeyError
+        If the server property is no defined.
+    """
+    return get_server_properties()[key]
+
+
 def _retrieve_user(name, realm=None, **kwargs):
     con = get_connection()
     try:
@@ -154,6 +219,7 @@ def _delete_role(name, **kwargs):
 
 def _set_roles(username, roles, realm=None, **kwargs):
     xml = etree.Element("Roles")
+    print(roles)
     for r in roles:
         xml.append(etree.Element("Role", name=r))
 
@@ -189,7 +255,8 @@ def _get_roles(username, realm=None, **kwargs):
         e.msg = "User does not exist."
         raise
     ret = set()
-    for r in etree.fromstring(body)[0]:
+    for r in etree.fromstring(body).xpath('/Response/Roles')[0]:
+        if r.tag == "Role":
             ret.add(r.get("name"))
     return ret
 
@@ -251,11 +318,12 @@ class PermissionRule():
         xml = etree.fromstring(body)
         ret = set()
         for c in xml:
+            if c.tag in ["Grant", "Deny"]:
                 ret.add(PermissionRule._parse_element(c))
         return ret
 
     def __str__(self):
-        return self._action + "(" + self._permission + ")" + \
+        return str(self._action) + "(" + str(self._permission) + ")" + \
             ("P" if self._priority is True else "")
 
     def __repr__(self):

src/caosdb/common/models.py

@@ -1822,11 +1822,13 @@ class Container(list):
     insertion, update, and deletion which are a applied to all entities
     in the container or the whole container respectively.
     """
+
     _debug = staticmethod(
         lambda: (
             get_config().getint(
                 "Container",
-                "debug") if get_config().get(
+                "debug") if get_config().has_section("Container") and
+            get_config().get(
                 "Container",
                 "debug") is not None else 0))
 

src/caosdb/configuration.py

@@ -21,9 +21,6 @@
 #
 # ** end header
 #
-
-"""Created on 20.09.2016."""
-
 try:
     # python2
     from ConfigParser import ConfigParser
@@ -31,29 +28,10 @@ except ImportError:
     # python3
     from configparser import ConfigParser
 
-_DEFAULTS = {"Connection":
-             {"url": None,
-              "timeout": "200",
-              "username": None,
-              "password_method": "plain",
-              "debug": "0",
-              "cacert": None},
-             "Container":
-             {"debug": "0"}}
-
 
 def _reset_config():
     global _pycaosdbconf
-    pycaosdbconf = None
-    _pycaosdbconf = ConfigParser(allow_no_value=True)
-    _init_defaults(_pycaosdbconf)
-
-
-def _init_defaults(confpar):
-    for sec in _DEFAULTS.keys():
-        confpar.add_section(sec)
-        for opt in _DEFAULTS[sec].keys():
-            confpar.set(sec, opt, _DEFAULTS[sec][opt])
+    _pycaosdbconf = ConfigParser(allow_no_value=False)
 
 
 def configure(inifile):

src/caosdb/connection/connection.py

@@ -72,6 +72,14 @@ class _WrappedHTTPResponse(CaosDBHTTPResponse):
 
 
 class _DefaultCaosDBServerConnection(CaosDBServerConnection):
+    """_DefaultCaosDBServerConnection.
+
+    Methods
+    -------
+    configure
+    request
+    """
+
     def __init__(self):
         self._useragent = ("PyCaosDB - "
                            "DefaultCaosDBServerConnection")
@@ -79,6 +87,28 @@ class _DefaultCaosDBServerConnection(CaosDBServerConnection):
         self._base_path = None
 
     def request(self, method, path, headers=None, body=None, **kwargs):
+        """request.
+
+        Send a HTTP request to the server.
+
+        Parameters
+        ----------
+        method : str
+            The HTTP request method.
+        path : str
+            An URI path segment (without the 'scheme://host:port/' parts),
+            including query and frament segments.
+        headers : dict of str -> str, optional
+            HTTP request headers. (Defautl: None)
+        body : str or bytes or readable, opional
+            The body of the HTTP request. Bytes should be a utf-8 encoded
+            string.
+        **kwargs :
+            Any keyword arguments will be ignored.
+
+        Returns
+        -------
+        """
         if headers is None:
             headers = {}
         try:
@@ -91,6 +121,27 @@ class _DefaultCaosDBServerConnection(CaosDBServerConnection):
         return _WrappedHTTPResponse(self._http_con.getresponse())
 
     def configure(self, **config):
+        """configure.
+
+        Configure the http connection.
+
+        Parameters
+        ----------
+        cacert : str
+            Path to the CA certificate which will be used to identify the
+            server.
+        url : str
+            The url of the CaosDB Server, e.g.
+            `https://example.com:443/rootpath`, including a possible root path.
+        **config :
+            Any further keyword arguments are being ignored.
+
+        Raises
+        ------
+        ConnectionException
+            If no url has been specified, or if the CA certificate cannot be
+            loaded.
+        """
         context = ssl.SSLContext(ssl.PROTOCOL_TLSv1)
         context.verify_mode = ssl.CERT_REQUIRED
         if hasattr(context, "check_hostname"):
@@ -146,10 +197,36 @@ def _make_conf(*conf):
 
 
 _DEFAULT_CONF = {"password_method": "plain", "implementation":
-                 _DefaultCaosDBServerConnection}
+                 _DefaultCaosDBServerConnection, "timeout": 210}
 
 
 def _get_authenticator(**config):
+    """_get_authenticator.
+
+    Import and configure the password_method.
+
+    Parameters
+    ----------
+    password_method : str
+        The simple name of a submodule of caosdb.connection.authentication.
+        Currently, there are three valid values for this parameter: 'plain',
+        'pass', and 'keyring'.
+    **config :
+        Any other keyword arguments are passed the configre method of the
+        password_method.
+
+    Returns
+    -------
+    AbstractAuthenticator
+        An object which implements the password_method and which already
+        configured.
+
+    Raises
+    ------
+    ConnectionException
+        If the password_method string cannot be resolved to a CaosAuthenticator
+        class.
+    """
     auth_module = ("caosdb.connection.authentication." +
                    config["password_method"])
     _LOGGER.debug("import auth_module %s", auth_module)
@@ -161,29 +238,37 @@ def _get_authenticator(**config):
         return auth_provider
 
     except ImportError:
-        raise RuntimeError("Password method \"{}\" not implemented. "
+        raise ConfigurationException("Password method \"{}\" not implemented. "
                                      "Valid methods: plain, pass, or keyring."
                                      .format(config["password_method"]))
 
 
 def configure_connection(**kwargs):
-    """Configures the caosdb connection.
-
-    All paramters are optional. The default configuration is taken from the
-    `Connection` section of the configuration which can be retrieved by
-    `get_config`
-
-    Returns the Connection object.
-
-    Typical arguments are:
-    url             The URL of the CaosDB server. E.g. https://dumiatis01:8433/playground
-    username        The username
-    password        Two options:
-                        1) The plain text password
-                        2) A function which returns the password.
-    timeout         A connection timeout in seconds.
-    implementation  A class which implements CaosDBServerConnection. (Default:
+    """Configures the caosdb connection and return the Connection object.
+
+    The effective configuration is governed by the default values (see
+    'Parameters'), the global configuration (see `caosdb.get_config()`) and the
+    parameters which are passed to this function, with ascending priority.
+
+    The parameters which are listed here, are possibly not sufficient for a
+    working configuration of the connection. Check the `configure` method of
+    the implementation class and the password_method for more details.
+
+    Parameters
+    ----------
+    implementation : CaosDBServerConnection
+        The class which implements the connection. (Default:
         _DefaultCaosDBServerConnection)
+    password_method : str
+        The name of a submodule of caosdb.connection.authentication which
+        implements the AbstractAuthenticator interface. (Default: 'plain')
+    timeout : int
+        A connection timeout in seconds. (Default: 210)
+
+    Returns
+    -------
+    _Connection
+        The singleton instance of the _Connection class.
     """
     global_conf = (dict(get_config().items("Connection")) if
                    get_config().has_section("Connection") else dict())
@@ -242,7 +327,8 @@ def _handle_response_status(http_response):
 
 class _Connection(object):  # pylint: disable=useless-object-inheritance
     """This connection class provides the interface to the database connection
-    allowing for retrieval, insertion, update, etc. of entries.
+    allowing for retrieval, insertion, update, etc. of entities, files, users,
+    roles and much more.
 
     It wrapps an instance of CaosDBServerConnection which actually does the
     work (how, depends on the instance).

src/caosdb/connection/mockup.py

@@ -47,7 +47,7 @@ class MockUpResponse(CaosDBHTTPResponse):
     def __init__(self, status, headers, body):
         self._status = status
         self.headers = headers
-        self.body = StringIO(body)
+        self.response = StringIO(body)
 
     @property
     def status(self):
@@ -56,7 +56,7 @@ class MockUpResponse(CaosDBHTTPResponse):
 
     def read(self, size=-1):
         """Return the body of the response."""
-        return self.body.read(size)
+        return self.response.read(size)
 
     def getheader(self, name, default=None):
         """Get the contents of the header `name`, or `default` if there is no

src/caosdb/exceptions.py

@@ -70,9 +70,10 @@ class ClientErrorException(CaosDBException):
 class ServerErrorException(CaosDBException):
     def __init__(self, body):
         xml = etree.fromstring(body)
-        msg = xml[0].get("description")
-        if xml[0].text is not None:
-            msg = msg + "\n\n" + xml[0].text
+        error = xml.xpath('/Response/Error')[0]
+        msg = error.get("description")
+        if error.text is not None:
+            msg = msg + "\n\n" + error.text
         CaosDBException.__init__(self, msg)
 
 

unittests/test_administraction.py

+# -*- encoding: utf-8 -*-
+#
+# ** header v3.0
+# This file is a part of the CaosDB Project.
+#
+# Copyright (C) 2018 Research Group Biomedical Physics,
+# Max-Planck-Institute for Dynamics and Self-Organization Göttingen
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see <https://www.gnu.org/licenses/>.
+#
+# ** end header
+#
+"""Tests for the administration class."""
+# pylint: disable=missing-docstring
+from __future__ import unicode_literals
+from pytest import raises
+from caosdb import administration, configure_connection, get_connection
+from caosdb.connection.mockup import MockUpServerConnection, MockUpResponse
+
+
+def setup():
+    configure_connection(url="unittests", username="testuser",
+                         password="testpassword", timeout=200,
+                         implementation=MockUpServerConnection)
+
+
+def test_get_server_properties_success():
+    properties = "<Properties><TEST_PROP>TEST_VAL</TEST_PROP></Properties>"
+    get_connection()._delegate_connection.resources.append(
+        lambda **kwargs: MockUpResponse(200, {}, properties))
+    props = administration.get_server_properties()
+    assert isinstance(props, dict)
+
+
+def test_get_server_property_success():
+    properties = "<Properties><TEST_PROP>TEST_VAL</TEST_PROP></Properties>"
+    get_connection()._delegate_connection.resources.append(
+        lambda **kwargs: MockUpResponse(200, {}, properties))
+    assert "TEST_VAL" == administration.get_server_property("TEST_PROP")
+
+
+def test_get_server_property_key_error():
+    properties = "<Properties><TEST_PROP>TEST_VAL</TEST_PROP></Properties>"
+    get_connection()._delegate_connection.resources.append(
+        lambda **kwargs: MockUpResponse(200, {}, properties))
+    with raises(KeyError) as e:
+        assert administration.get_server_property("BLA")
+
+
+def test_set_server_property():
+    def check_form(**kwargs):
+        assert kwargs["path"] == "_server_properties"
+        assert kwargs["method"] == "POST"
+        assert kwargs["body"] == "TEST_PROP=TEST_VAL".encode()
+        assert kwargs["headers"]["Content-Type"] == "application/x-www-form-urlencoded"
+        return MockUpResponse(200, {}, "<Properties><TEST_PROP>TEST_VAL</TEST_PROP></Properties>")
+    get_connection()._delegate_connection.resources.append(check_form)
+    administration.set_server_property("TEST_PROP", "TEST_VAL")

unittests/test_connection.py

@@ -92,6 +92,8 @@ def test_make_uri_path():
 
 
 def test_configure_connection():
+    if not get_config().has_section("Connection"):
+        get_config().add_section("Connection")
     get_config().set("Connection", "url", "https://host.de")
     get_config().set("Connection", "username", "test_username")
     get_config().set("Connection", "password", "test_password")