WIP: https proxy

src/caosdb/cert/indiscale.ca.crt

------BEGIN CERTIFICATE-----
-MIIJ6TCCBdGgAwIBAgIIFVYzdrEDk6EwDQYJKoZIhvcNAQENBQAwgZMxCzAJBgNV
-BAYTAkRFMRUwEwYDVQQIEwxMb3dlciBTYXhvbnkxEzARBgNVBAcTCkdvZXR0aW5n
-ZW4xFzAVBgNVBAoTDkluZGlTY2FsZSBHbWJIMRwwGgYDVQQDExNJbmRpU2NhbGUg
-Um9vdCBDQSAxMSEwHwYJKoZIhvcNAQkBFhJpbmZvQGluZGlzY2FsZS5jb20wHhcN
-MTkwODA3MDAwMDAwWhcNMzQwODA2MjM1OTU5WjCBkzELMAkGA1UEBhMCREUxFTAT
-BgNVBAgTDExvd2VyIFNheG9ueTETMBEGA1UEBxMKR29ldHRpbmdlbjEXMBUGA1UE
-ChMOSW5kaVNjYWxlIEdtYkgxHDAaBgNVBAMTE0luZGlTY2FsZSBSb290IENBIDEx
-ITAfBgkqhkiG9w0BCQEWEmluZm9AaW5kaXNjYWxlLmNvbTCCBCIwDQYJKoZIhvcN
-AQEBBQADggQPADCCBAoCggQBAKxJO3XOqrUxFU3qdVyk9tmZEHwhwntcLO+kRR5t
-64/1Z/+VIPSgVN5phkSCukj2BPJITWKplWzJDAYWSvA/7cqavCtx8yP+m3AHWrRa
-CeHbtkGZ1nzwyFel3GIr93e65REeWqBE3knzem+qxTlZ2hp8/w3oxUlhy7tGxjBs
-JlekgLRDrnj4Opyb4GVjcVfcELmu3sLrrPX1wdYJrqaMQUR4BKZnbXxKdOYyX+kR
-/W2P4sihCCJh7Wy29VXHwSSCM1qEkU3REjvPEmEElCG7UpqOfg+3jaNZDqnvfskf
-okU4GuFCxSWQituyP9jm/hFVEhz59tUMYCllcjEi2jGmD2DBKpiru4t4/z0Aymf4
-Pep9hNtH1yhZMxpQeCYK9ESEE5d7do0bu/4YFp7jAg5vWZ8KlILZakmypVBFUw8I
-U/QJoJ55j95vIp+kjFdXelIVcr5La/zOR82JldaoPfyoBKObzwpwqaWQwYm8pj4p
-XkUdJTf8rpW21SSGWZm8JoFSYDfGvI61rPEjl/ohKhlG0tV6E2tCc406HNo/7pPe
-pmx/v9ZWLbYDAH7MVMB4tv6zDRE/c4KTbh5/s70VbXbAeOG6DNwegdDLDYZOv6Yw
-YQMz9NWtKGzvoFehP2vY5nGK95JVUcd90jaNaoURLB102VtxAjPIEQA1PjbQxLvC
-7A6kshlpQiN7zS/R9IgiEkYP/9gjy6mMuQVxH7C+9cqmCnXvVmpHmxXGUqk61r/B
-h12htsx5qjbbkToZYhUXBmwRq4LDtyoxNeaF2Jc+gE762obbHsSYMuSuh0kTFUUd
-uqfrI8OyzX4r1w5dYf2FEetZTT2Obyxb3Cy0btJF5+zEerBX44RulkdC+TPTMhJw
-b1jrPCACKywy9b6vJcSQ2V1+uLk7rH2JKD+fQRIKUqZZkhNKFYz5dnYYTgS45M0/
-C+vIvRnhgNSNb4efG6wyFvWEF8poDSPnJ4mM+0jHG/+cLqF/M2CMFvC+yU8Hj9YH
-B+H2L6V1QlCkpw5Ai4ji6OaQmnrsjE8EJj58vwYKsjmLGuf4j5AivkQTxfgCPGrT
-6CxSesoFmYDPSg/2eO+IfYEwnd7Rbs4aAhW8eo+lGpmK0DQxNjlejYt/Cgp7HWCq
-m/VNqWPIDMSTTqyk1GTmp67NjEZKt2ukJxI2CpL8s/9x4f3GTjNyI750pKM/uzMk
-OBKTMuWJQ6xeMR3h9RQlqlmwcErLXoUGInOTHHjRGXDI+ZBeLqT5DikcFiwbHG3+
-6FOuxXO0eqqg2tBW8cQ5kuRI0YFznipDUcfgDZt0JEkEXmRuL0nxYO35WKKdpGcF
-xFRJtO4FRB4nVWekVRuK9m47IPm6vC4eo+pCNPPoQ+FjyQ8CAwEAAaM/MD0wDAYD
-VR0TBAUwAwEB/zAdBgNVHQ4EFgQUFjE2TLaKASKEJ0LKOO+37/Hu7qowDgYDVR0P
-AQH/BAQDAgEGMA0GCSqGSIb3DQEBDQUAA4IEAQB2j1GL1G0ferWp9wmuDdF2oumn
-k+JIctRaGHaSrqHy4zjwj3Oqm2JA1ds+WfWozz+d38ZcLqSHo+g9ku5h/XOogQEU
-O4/y7j44pxIUg0EcIpMHtf7KPocPfvi9lw/4vE/3V/WKh4E09SXWgyY5tMUlEMaB
-6t8n7gg943HY2MJE1QU2wOPMXB1krFbunYxJcrUMs21w9jRWVv/wvaj6rkdvvSbU
-Yre11J+VlhC6fxx+STohQopzE6jnsaHile56b9xAmCCKcPEpWeKKBFS7pVNHEIHF
-uHWpgVjhoheEMMbYgu6l5E5K32TNYCKU49jNRWEKETjmYQSNl9dsSip+XlvaU8wQ
-VRR8UMHZPiJDW/AAHCr+bXEarZ9mSj/y+R512YtVw95zCnGUtzOJViThoIk/IAOR
-AJdnvsFmZSIKtFHpSEFYlTDq2yr1ulzbaDhuPRzita8b0cP27UvqRebZw5CvHN48
-B9a9tTYowKuJqmtjE6D00QA4xS8fRizLnx54uNmDbwf/8WavVk6MzDERwRE3OsSy
-D0dV6gy3t2AqEpVBrICrFqvgAQa4fcFcIwz3Qbt5o5uEi7acRomY57YrxrlfNTwh
-2oDQz+HQ/ZTDwZ3DrIgel7GrQ5fXrXDLL3ebtsbuIeBx8crOWQask832HcLtDVpu
-E/FdJEMMjglzIcy2dHpuODIGFmgEVfHR4DOOSBl0hfNdlrYnhC0h8/6QFswtlYFF
-8aQbGX7inK8L2in5wQ7ypeoMuXkQVYxlU1TEGmgB8aDke47MuX1FH+clsCaZ3s1E
-ka6lV6cjNYcosS718B6b2JgDUzmGBn2Sdm1xFmJM16dXp7TSmC5/fYxXuE/CynDs
-PmaUb9Ms6XUYSwKKhZ5HZdeRoNz8w62WNAeF7o7iX6IVrd/G1bJnSBN01istckyR
-BDuIkaoBQ9yvHN6Bo/J3KR08ixF1dHFPo/oSgkBxkLakb/yeslBTP/oISiFeQ4+q
-Gld1mhAvmG99dVZfoysrMjZSyghNbqwScjbYYN115lExV5ZeRtSwA7JCYE2lBjmB
-vocmz/hh/ifbmmqIvSv0NtiBnM6mNqngZEWD/rAloVOQoq0KVJJ5lUCQrBSFtR4+
-G1JGMX6b7uRp4mfdqqDE62KxxfkWBUwzUTIKGb5K42ji1Gy5li/TIWJtLNGNNQ2A
-0ui2RhwioaGGfYyomSFuAo5IPE/NF0ASjrTDW6GoNxypTSYE4/7oSoxeryafVnqN
-S0fRyrgSLiuT5tAiZ3b5Q3EFYUM2OcU3ezr/ZUabf9qIsqOnCi91SqE88BQbenot
-0HyUMdp/7QX9SyWM/azhcRiReAtkmq9pgeQA2TTZADDNTkKRljG9VeFDSwl7
------END CERTIFICATE-----

src/caosdb/common/timezone.py

@@ -12,6 +12,7 @@ class TimeZone():
     display_name : string
         A human-friendly name of the time zone:
     """
+
     def __init__(self, zone_id, offset, display_name):
         self.zone_id = zone_id
         self.offset = offset

src/caosdb/configuration.py

@@ -31,12 +31,7 @@ try:
 except ImportError:
     pass
 
-try:
-    # python2
-    from ConfigParser import ConfigParser
-except ImportError:
-    # python3
-    from configparser import ConfigParser
+from configparser import ConfigParser
 
 from os import environ, getcwd
 from os.path import expanduser, join, isfile
@@ -59,6 +54,9 @@ def configure(inifile):
         _reset_config()
     read_config = _pycaosdbconf.read(inifile)
     validate_yaml_schema(config_to_yaml(_pycaosdbconf))
+
+    if "HTTPS_PROXY" in environ:
+        _pycaosdbconf["Connection"]["https_proxy"] = environ["HTTPS_PROXY"]
     return read_config
 
 

src/caosdb/connection/connection.py

@@ -28,12 +28,15 @@ from __future__ import absolute_import, print_function, unicode_literals
 import logging
 import ssl
 import sys
+import warnings
 from builtins import str  # pylint: disable=redefined-builtin
 from errno import EPIPE as BrokenPipe
 from socket import error as SocketError
 from urllib.parse import quote, urlparse
 from requests import Session as HTTPSession
 from requests.exceptions import ConnectionError as HTTPConnectionError
+from urllib3.poolmanager import PoolManager
+from requests.adapters import HTTPAdapter
 
 from caosdb.configuration import get_config
 from caosdb.exceptions import (CaosDBException, HTTPClientError,
@@ -54,9 +57,6 @@ from pkg_resources import resource_filename
 from .interface import CaosDBHTTPResponse, CaosDBServerConnection
 from .utils import make_uri_path, parse_url, urlencode
 
-
-# pylint: disable=missing-docstring
-
 _LOGGER = logging.getLogger(__name__)
 
 
@@ -86,6 +86,19 @@ class _WrappedHTTPResponse(CaosDBHTTPResponse):
         self.response.close()
 
 
+class _SSLAdapter(HTTPAdapter):
+    """Transport adapter that allows us to use different SSL versions."""
+
+    def __init__(self, ssl_version):
+        self.ssl_version = ssl_version
+        super().__init__()
+
+    def init_poolmanager(self, connections, maxsize, block=False):
+        self.poolmanager = PoolManager(
+            num_pools=connections, maxsize=maxsize,
+            block=block, ssl_version=self.ssl_version)
+
+
 class _DefaultCaosDBServerConnection(CaosDBServerConnection):
     """_DefaultCaosDBServerConnection.
 
@@ -99,8 +112,11 @@ class _DefaultCaosDBServerConnection(CaosDBServerConnection):
         self._useragent = ("caosdb-pylib/{version} - {implementation}".format(
             version=version, implementation=type(self).__name__))
         self._base_path = None
+        self._session = None
+        self._timeout = None
+        self._verify = None
 
-    def request(self, method, path, headers=None, body=None, **kwargs):
+    def request(self, method, path, headers=None, body=None):
         """request.
 
         Send a HTTP request to the server.
@@ -117,13 +133,10 @@ class _DefaultCaosDBServerConnection(CaosDBServerConnection):
         body : str or bytes or readable, optional
             The body of the HTTP request. Bytes should be a utf-8 encoded
             string.
-        **kwargs :
-            Any keyword arguments will be ignored.
-            TODO: Why are they allowed then?
 
         Returns
         -------
-            TODO: What?
+        response : CaosDBHTTPResponse
         """
 
         if headers is None:
@@ -131,31 +144,20 @@ class _DefaultCaosDBServerConnection(CaosDBServerConnection):
         headers["User-Agent"] = self._useragent
 
         try:
-            session = HTTPSession()
-            if self.setup_fields["https_proxy"] is not None:
-                session.proxies = {
-                    "https": self.setup_fields["https_proxy"]
-                }
-            verify = True
-            if self.setup_fields["cacert"]:
-                verify = self.setup_fields["cacert"]
-            if self.setup_fields["ssl_insecure"]:
-                verify = False
-
-            url = urlparse(self.setup_fields["url_base_path"] + path)
-
-
-            response = session.request(method=method,
-                                       url=self.setup_fields["url_base_path"] + path,
-                                       verify=verify,
-                                       headers=headers, data=body, stream=True)
+            response = self._session.request(
+                method=method,
+                url=self._base_path + path,
+                headers=headers,
+                data=body,
+                verify=self._verify,
+                timeout=self._timeout,
+                stream=True)
             return _WrappedHTTPResponse(response)
         except HTTPConnectionError as conn_err:
             raise CaosDBConnectionError(
                 "Connection failed. Network or server down? " + str(conn_err)
             )
 
-
     def configure(self, **config):
         """configure.
 
@@ -183,63 +185,49 @@ class _DefaultCaosDBServerConnection(CaosDBServerConnection):
             ssl_version = getattr(ssl, config["ssl_version"])
         else:
             ssl_version = ssl.PROTOCOL_TLS
-        context = ssl.SSLContext(ssl_version)
-        context.verify_mode = ssl.CERT_REQUIRED
-
-        if config.get("ssl_insecure"):
-            _LOGGER.warning("*** Warning! ***\n"
-                            "Insecure SSL mode, certificate will not be checked! "
-                            "Please consider removing the `ssl_insecure` configuration option.\n"
-                            "****************")
-            context.verify_mode = ssl.CERT_NONE
-
-        if (not context.verify_mode == ssl.CERT_NONE and
-                hasattr(context, "check_hostname")):
-            context.check_hostname = True
-
-        if ("cacert" in config and config["cacert"] is not None and
-                config["cacert"]):
-            try:
-                context.load_verify_locations(config["cacert"])
-            except Exception as exc:
-                raise CaosDBConnectionError("Could not load the cacert in"
-                                            "`{}`: {}".format(config["cacert"],
-                                                              exc))
-
-        context.load_default_certs()
 
         if "url" in config:
-            parsed_url = parse_url(config["url"])
-            host = parsed_url.netloc
-            self._base_path = parsed_url.path
+            url = urlparse(config["url"])
+            path = url.path.strip("/")
+            if len(path) > 0:
+                path = path + "/"
+            self._base_path = url.scheme + "://" + url.netloc + "/" + path
         else:
             raise CaosDBConnectionError(
                 "No connection url specified. Please "
                 "do so via caosdb.configure_connection(...) or in a config "
                 "file.")
 
-        socket_proxy = None
-        if "socket_proxy" in config:
-            socket_proxy = config["socket_proxy"]
-
-        https_proxy = None
-        if "https_proxy" in config:
-            result = urlparse(config["https_proxy"], scheme="http")
-            if result.scheme != "http":
-                raise ValueError(
-                    "The `https_proxy` parameter or config option must "
-                    "contain a valid http uri or None")
-            https_proxy = result.scheme + "://" + result.netloc
-
-        self.setup_fields = {
-            "url_base_path": config["url"],
-            "cacert": config["cacert"] if "cacert" in config else None,
-            "ssl_insecure": config["ssl_insecure"] if "ssl_insecure" in config else False,
-            "host": host,
-            "timeout": int(config.get("timeout")),
-            "context": context,
-            "https_proxy": https_proxy,
-            "socket_proxy": socket_proxy}
+        # TODO remove in next release
+        socket_proxy = config["socket_proxy"] if "socket_proxy" in config else None
+        https_proxy = config["https_proxy"] if "https_proxy" in config else None
+
+        self._session = HTTPSession()
+        #self._session.mount(self._base_path, _SSLAdapter(ssl_version))
+        if socket_proxy is not None:
+            self._session.proxies = {
+                "https": "socks5://" + socket_proxy,
+            }
+
+        if https_proxy is not None:
+            self._session.proxies = {
+                "https": https_proxy,
+            }
+
+        verify = True
+        if "cacert" in config:
+            verify = config["cacert"]
+        if "ssl_insecure" in config and config["ssl_insecure"].lower() == "true":
+            _LOGGER.warning("*** Warning! ***\n"
+                            "Insecure SSL mode, certificate will not be checked! "
+                            "Please consider removing the `ssl_insecure` configuration option.\n"
+                            "****************")
+            verify = False
+        if verify is not None:
+            self._verify = verify
+
+        if "timeout" in config:
+            self._timeout = config["timeout"]
 
 
 def _make_conf(*conf):
@@ -270,7 +258,6 @@ _DEFAULT_CONF = {
     "password_method": "input",
     "implementation": _DefaultCaosDBServerConnection,
     "timeout": 210,
-    "cacert": resource_filename("caosdb", 'cert/indiscale.ca.crt')
 }
 
 
@@ -361,9 +348,7 @@ def configure_connection(**kwargs):
         Implies `password_method="auth_token"` if set.  An example token string would be `["O","OneTimeAuthenticationToken","anonymous",["administration"],[],1592995200000,604800000,"3ZZ4WKRB-5I7DG2Q6-ZZE6T64P-VQ","197d0d081615c52dc18fb323c300d7be077beaad4020773bb58920b55023fa6ee49355e35754a4277b9ac525c882bcd3a22e7227ba36dfcbbdbf8f15f19d1ee9",1,30000]`.
 
     https_proxy : str
-        Define a https proxy, e.g. `https://localhost:8888`. Currently,
-        authentication against the proxy and non-TLS connections are not
-        supported.  (Default: None)
+        Define a https proxy, e.g. `http://localhost:8888`. (Default: None)
 
     implementation : CaosDBServerConnection
         The class which implements the connection. (Default:
@@ -395,6 +380,11 @@ def configure_connection(**kwargs):
     local_conf = _make_conf(_DEFAULT_CONF, global_conf, kwargs)
 
     connection = _Connection.get_instance()
+
+    if "socket_proxy" in local_conf:
+        warnings.warn("Deprecated configuration option: socket_proxy. Use "
+                      "the new https_proxy option instead",
+                      DeprecationWarning, stacklevel=1)
     connection.configure(**local_conf)
 
     return connection

src/caosdb/connection/encode.py

@@ -364,6 +364,7 @@ def get_headers(params, boundary):
     headers['Content-Length'] = str(get_body_size(params, boundary))
     return headers
 
+
 class MultipartYielder(object):
     """An iterator that yields the parameters of a multipart/formdata http
     body."""
@@ -476,6 +477,7 @@ def multipart_encode(params, boundary=None, callback=None):
 
     return MultipartYielder(params, boundary, callback), headers
 
+
 class ReadableMultiparts(object):
 
     def __init__(self, multipart_yielder):