diff --git a/CHANGELOG.md b/CHANGELOG.md
index 9e4d7301fbce387aa96fdeef759c75457422ce96..acbd2fe23a3a00bdf895c069a4a3067e7df97936 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -9,6 +9,15 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
### Added ###
+* HTTP connections are allowed additionally to HTTPS connections.
+* Dependency on the `requests` package.
+* Dependency on the `python-dateutil` package.
+* `Connection.https_proxy` and `Connection.http_proxy` option of the
+ pycaosdb.ini and the `https_proxy` and `http_proxy` parameter of the
+ `configure_connection` function. See the documentation of the
+ latter for more information.
+ Note that the `HTTP_PROXY` and `HTTPS_PROXY` environment variables are
+ respected as well, unless overridden programmatically.
* `apiutils.empty_diff` function that returns `True` if the diffs of two
entities found with the `compare_entitis` function are empty, `False`
otherwise.
@@ -32,6 +41,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
### Deprecated ###
+* `Connection.socket_proxy` option of the pycaosdb.ini. Please use
+ `Connection.https_proxy` or `Connection.http_proxy` instead. The deprecated
+ option will be removed with the next minor release.
+
### Removed ###
### Fixed ###
diff --git a/setup.py b/setup.py
index e044d1f587b9010d2dd3e29f99a8345d4c8c76d4..11b734b692e7dd29e853cd50fa7456530085253a 100755
--- a/setup.py
+++ b/setup.py
@@ -97,6 +97,9 @@ def get_version_info():
if os.path.exists('.git'):
GIT_REVISION = git_version()
+ elif os.path.exists('caosdb_pylib_commit'):
+ with open('caosdb_pylib_commit', 'r') as f:
+ GIT_REVISION = f.read().strip()
elif os.path.exists('src/caosdb/version.py'):
# must be a source distribution, use existing version file
try:
@@ -171,7 +174,11 @@ def setup_package():
python_requires='>=3.8',
package_dir={'': 'src'},
install_requires=['lxml>=4.6.3',
- 'PyYAML>=5.4.1', 'future', 'PySocks>=1.6.7'],
+ "requests[socks]>=2.28.1",
+ "python-dateutil>=2.8.2",
+ 'PyYAML>=5.4.1',
+ 'future',
+ ],
extras_require={'keyring': ['keyring>=13.0.0'],
'jsonschema': ['jsonschema>=4.4.0']},
setup_requires=["pytest-runner>=2.0,<3dev"],
diff --git a/src/caosdb/cert/indiscale.ca.crt b/src/caosdb/cert/indiscale.ca.crt
deleted file mode 100644
index 08a79d60c5d34626eb96f1a92e33d0ac22494f3c..0000000000000000000000000000000000000000
--- a/src/caosdb/cert/indiscale.ca.crt
+++ /dev/null
@@ -1,55 +0,0 @@
------BEGIN CERTIFICATE-----
-MIIJ6TCCBdGgAwIBAgIIFVYzdrEDk6EwDQYJKoZIhvcNAQENBQAwgZMxCzAJBgNV
-BAYTAkRFMRUwEwYDVQQIEwxMb3dlciBTYXhvbnkxEzARBgNVBAcTCkdvZXR0aW5n
-ZW4xFzAVBgNVBAoTDkluZGlTY2FsZSBHbWJIMRwwGgYDVQQDExNJbmRpU2NhbGUg
-Um9vdCBDQSAxMSEwHwYJKoZIhvcNAQkBFhJpbmZvQGluZGlzY2FsZS5jb20wHhcN
-MTkwODA3MDAwMDAwWhcNMzQwODA2MjM1OTU5WjCBkzELMAkGA1UEBhMCREUxFTAT
-BgNVBAgTDExvd2VyIFNheG9ueTETMBEGA1UEBxMKR29ldHRpbmdlbjEXMBUGA1UE
-ChMOSW5kaVNjYWxlIEdtYkgxHDAaBgNVBAMTE0luZGlTY2FsZSBSb290IENBIDEx
-ITAfBgkqhkiG9w0BCQEWEmluZm9AaW5kaXNjYWxlLmNvbTCCBCIwDQYJKoZIhvcN
-AQEBBQADggQPADCCBAoCggQBAKxJO3XOqrUxFU3qdVyk9tmZEHwhwntcLO+kRR5t
-64/1Z/+VIPSgVN5phkSCukj2BPJITWKplWzJDAYWSvA/7cqavCtx8yP+m3AHWrRa
-CeHbtkGZ1nzwyFel3GIr93e65REeWqBE3knzem+qxTlZ2hp8/w3oxUlhy7tGxjBs
-JlekgLRDrnj4Opyb4GVjcVfcELmu3sLrrPX1wdYJrqaMQUR4BKZnbXxKdOYyX+kR
-/W2P4sihCCJh7Wy29VXHwSSCM1qEkU3REjvPEmEElCG7UpqOfg+3jaNZDqnvfskf
-okU4GuFCxSWQituyP9jm/hFVEhz59tUMYCllcjEi2jGmD2DBKpiru4t4/z0Aymf4
-Pep9hNtH1yhZMxpQeCYK9ESEE5d7do0bu/4YFp7jAg5vWZ8KlILZakmypVBFUw8I
-U/QJoJ55j95vIp+kjFdXelIVcr5La/zOR82JldaoPfyoBKObzwpwqaWQwYm8pj4p
-XkUdJTf8rpW21SSGWZm8JoFSYDfGvI61rPEjl/ohKhlG0tV6E2tCc406HNo/7pPe
-pmx/v9ZWLbYDAH7MVMB4tv6zDRE/c4KTbh5/s70VbXbAeOG6DNwegdDLDYZOv6Yw
-YQMz9NWtKGzvoFehP2vY5nGK95JVUcd90jaNaoURLB102VtxAjPIEQA1PjbQxLvC
-7A6kshlpQiN7zS/R9IgiEkYP/9gjy6mMuQVxH7C+9cqmCnXvVmpHmxXGUqk61r/B
-h12htsx5qjbbkToZYhUXBmwRq4LDtyoxNeaF2Jc+gE762obbHsSYMuSuh0kTFUUd
-uqfrI8OyzX4r1w5dYf2FEetZTT2Obyxb3Cy0btJF5+zEerBX44RulkdC+TPTMhJw
-b1jrPCACKywy9b6vJcSQ2V1+uLk7rH2JKD+fQRIKUqZZkhNKFYz5dnYYTgS45M0/
-C+vIvRnhgNSNb4efG6wyFvWEF8poDSPnJ4mM+0jHG/+cLqF/M2CMFvC+yU8Hj9YH
-B+H2L6V1QlCkpw5Ai4ji6OaQmnrsjE8EJj58vwYKsjmLGuf4j5AivkQTxfgCPGrT
-6CxSesoFmYDPSg/2eO+IfYEwnd7Rbs4aAhW8eo+lGpmK0DQxNjlejYt/Cgp7HWCq
-m/VNqWPIDMSTTqyk1GTmp67NjEZKt2ukJxI2CpL8s/9x4f3GTjNyI750pKM/uzMk
-OBKTMuWJQ6xeMR3h9RQlqlmwcErLXoUGInOTHHjRGXDI+ZBeLqT5DikcFiwbHG3+
-6FOuxXO0eqqg2tBW8cQ5kuRI0YFznipDUcfgDZt0JEkEXmRuL0nxYO35WKKdpGcF
-xFRJtO4FRB4nVWekVRuK9m47IPm6vC4eo+pCNPPoQ+FjyQ8CAwEAAaM/MD0wDAYD
-VR0TBAUwAwEB/zAdBgNVHQ4EFgQUFjE2TLaKASKEJ0LKOO+37/Hu7qowDgYDVR0P
-AQH/BAQDAgEGMA0GCSqGSIb3DQEBDQUAA4IEAQB2j1GL1G0ferWp9wmuDdF2oumn
-k+JIctRaGHaSrqHy4zjwj3Oqm2JA1ds+WfWozz+d38ZcLqSHo+g9ku5h/XOogQEU
-O4/y7j44pxIUg0EcIpMHtf7KPocPfvi9lw/4vE/3V/WKh4E09SXWgyY5tMUlEMaB
-6t8n7gg943HY2MJE1QU2wOPMXB1krFbunYxJcrUMs21w9jRWVv/wvaj6rkdvvSbU
-Yre11J+VlhC6fxx+STohQopzE6jnsaHile56b9xAmCCKcPEpWeKKBFS7pVNHEIHF
-uHWpgVjhoheEMMbYgu6l5E5K32TNYCKU49jNRWEKETjmYQSNl9dsSip+XlvaU8wQ
-VRR8UMHZPiJDW/AAHCr+bXEarZ9mSj/y+R512YtVw95zCnGUtzOJViThoIk/IAOR
-AJdnvsFmZSIKtFHpSEFYlTDq2yr1ulzbaDhuPRzita8b0cP27UvqRebZw5CvHN48
-B9a9tTYowKuJqmtjE6D00QA4xS8fRizLnx54uNmDbwf/8WavVk6MzDERwRE3OsSy
-D0dV6gy3t2AqEpVBrICrFqvgAQa4fcFcIwz3Qbt5o5uEi7acRomY57YrxrlfNTwh
-2oDQz+HQ/ZTDwZ3DrIgel7GrQ5fXrXDLL3ebtsbuIeBx8crOWQask832HcLtDVpu
-E/FdJEMMjglzIcy2dHpuODIGFmgEVfHR4DOOSBl0hfNdlrYnhC0h8/6QFswtlYFF
-8aQbGX7inK8L2in5wQ7ypeoMuXkQVYxlU1TEGmgB8aDke47MuX1FH+clsCaZ3s1E
-ka6lV6cjNYcosS718B6b2JgDUzmGBn2Sdm1xFmJM16dXp7TSmC5/fYxXuE/CynDs
-PmaUb9Ms6XUYSwKKhZ5HZdeRoNz8w62WNAeF7o7iX6IVrd/G1bJnSBN01istckyR
-BDuIkaoBQ9yvHN6Bo/J3KR08ixF1dHFPo/oSgkBxkLakb/yeslBTP/oISiFeQ4+q
-Gld1mhAvmG99dVZfoysrMjZSyghNbqwScjbYYN115lExV5ZeRtSwA7JCYE2lBjmB
-vocmz/hh/ifbmmqIvSv0NtiBnM6mNqngZEWD/rAloVOQoq0KVJJ5lUCQrBSFtR4+
-G1JGMX6b7uRp4mfdqqDE62KxxfkWBUwzUTIKGb5K42ji1Gy5li/TIWJtLNGNNQ2A
-0ui2RhwioaGGfYyomSFuAo5IPE/NF0ASjrTDW6GoNxypTSYE4/7oSoxeryafVnqN
-S0fRyrgSLiuT5tAiZ3b5Q3EFYUM2OcU3ezr/ZUabf9qIsqOnCi91SqE88BQbenot
-0HyUMdp/7QX9SyWM/azhcRiReAtkmq9pgeQA2TTZADDNTkKRljG9VeFDSwl7
------END CERTIFICATE-----
diff --git a/src/caosdb/common/administration.py b/src/caosdb/common/administration.py
index 98d4d2826da7131ef79b5c3cc9b3d9597abc0248..bd09c626d5a9e48bc9d3610f23cb46660cb10db3 100644
--- a/src/caosdb/common/administration.py
+++ b/src/caosdb/common/administration.py
@@ -76,7 +76,7 @@ def get_server_properties():
con = get_connection()
try:
body = con._http_request(
- method="GET", path="_server_properties").response
+ method="GET", path="_server_properties")
except EntityDoesNotExistError:
raise ServerConfigurationException(
"Debug mode in server is probably disabled.") from None
diff --git a/src/caosdb/common/timezone.py b/src/caosdb/common/timezone.py
index 2bd3d3d4d739118e160f7b3a35757fbb0afe70cb..8fc5e710d3cbf6f20cf81397573f972db3b22f12 100644
--- a/src/caosdb/common/timezone.py
+++ b/src/caosdb/common/timezone.py
@@ -12,6 +12,7 @@ class TimeZone():
display_name : string
A human-friendly name of the time zone:
"""
+
def __init__(self, zone_id, offset, display_name):
self.zone_id = zone_id
self.offset = offset
diff --git a/src/caosdb/configuration.py b/src/caosdb/configuration.py
index 75827df0d00d6c82251c2c04fa47413ac2801928..1c108ac1d39c135dbc90f477be8b8f2f630391ce 100644
--- a/src/caosdb/configuration.py
+++ b/src/caosdb/configuration.py
@@ -31,12 +31,7 @@ try:
except ImportError:
pass
-try:
- # python2
- from ConfigParser import ConfigParser
-except ImportError:
- # python3
- from configparser import ConfigParser
+from configparser import ConfigParser
from os import environ, getcwd
from os.path import expanduser, join, isfile
@@ -59,6 +54,11 @@ def configure(inifile):
_reset_config()
read_config = _pycaosdbconf.read(inifile)
validate_yaml_schema(config_to_yaml(_pycaosdbconf))
+
+ if "HTTPS_PROXY" in environ:
+ _pycaosdbconf["Connection"]["https_proxy"] = environ["HTTPS_PROXY"]
+ if "HTTP_PROXY" in environ:
+ _pycaosdbconf["Connection"]["http_proxy"] = environ["HTTP_PROXY"]
return read_config
diff --git a/src/caosdb/connection/SocksiPy.zip b/src/caosdb/connection/SocksiPy.zip
deleted file mode 100644
index e81f1f9393c766a3acd41b44245f9e17f090cbe5..0000000000000000000000000000000000000000
Binary files a/src/caosdb/connection/SocksiPy.zip and /dev/null differ
diff --git a/src/caosdb/connection/connection.py b/src/caosdb/connection/connection.py
index 43eb3410d8d5bdc8323a811fd2b6424fb75f3fda..aaa09dfe9d804a5465f322bfe8db5826b304211b 100644
--- a/src/caosdb/connection/connection.py
+++ b/src/caosdb/connection/connection.py
@@ -28,9 +28,15 @@ from __future__ import absolute_import, print_function, unicode_literals
import logging
import ssl
import sys
+import warnings
from builtins import str # pylint: disable=redefined-builtin
from errno import EPIPE as BrokenPipe
from socket import error as SocketError
+from urllib.parse import quote, urlparse
+from requests import Session as HTTPSession
+from requests.exceptions import ConnectionError as HTTPConnectionError
+from urllib3.poolmanager import PoolManager
+from requests.adapters import HTTPAdapter
from caosdb.configuration import get_config
from caosdb.exceptions import (CaosDBException, HTTPClientError,
@@ -49,16 +55,8 @@ except ModuleNotFoundError:
from pkg_resources import resource_filename
from .interface import CaosDBHTTPResponse, CaosDBServerConnection
-from .streaminghttp import StreamingHTTPSConnection
from .utils import make_uri_path, parse_url, urlencode
-
-try:
- from urllib.parse import quote, urlparse
-except ImportError:
- from urllib import quote
- from urlparse import urlparse
-
-# pylint: disable=missing-docstring
+from .encode import MultipartYielder, ReadableMultiparts
_LOGGER = logging.getLogger(__name__)
@@ -67,6 +65,9 @@ class _WrappedHTTPResponse(CaosDBHTTPResponse):
def __init__(self, response):
self.response = response
+ self._generator = None
+ self._buffer = b''
+ self._stream_consumed = False
@property
def reason(self):
@@ -74,21 +75,71 @@ class _WrappedHTTPResponse(CaosDBHTTPResponse):
@property
def status(self):
- return self.response.status
+ return self.response.status_code
def read(self, size=None):
- return self.response.read(size)
+ if self._stream_consumed is True:
+ raise RuntimeError("Stream is consumed")
+
+ if self._buffer is None:
+ # the buffer has been drained in the previous call.
+ self._stream_consumed = True
+ return b''
+
+ if self._generator is None and (size is None or size == 0):
+ # return full content at once
+ self._stream_consumed = True
+ return self.response.content
+
+ if len(self._buffer) >= size:
+ # still enough bytes in the buffer
+ result = chunk[:size]
+ self._buffer = chunk[size:]
+ return result
+
+ if self._generator is None:
+ # first call to this method
+ if size is None or size == 0:
+ size = 512
+ self._generator = self.response.iter_content(size)
+
+ try:
+ # read new data into the buffer
+ chunk = self._buffer + next(self._generator)
+ result = chunk[:size]
+ if len(result) == 0:
+ self._stream_consumed = True
+ self._buffer = chunk[size:]
+ return result
+ except StopIteration:
+ # drain buffer
+ result = self._buffer
+ self._buffer = None
+ return result
def getheader(self, name, default=None):
- return self.response.getheader(name=name, default=default)
+ return self.response.headers[name] if name in self.response.headers else default
def getheaders(self):
- return self.response.getheaders()
+ return self.response.headers.items()
def close(self):
self.response.close()
+class _SSLAdapter(HTTPAdapter):
+ """Transport adapter that allows us to use different SSL versions."""
+
+ def __init__(self, ssl_version):
+ self.ssl_version = ssl_version
+ super().__init__()
+
+ def init_poolmanager(self, connections, maxsize, block=False):
+ self.poolmanager = PoolManager(
+ num_pools=connections, maxsize=maxsize,
+ block=block, ssl_version=self.ssl_version)
+
+
class _DefaultCaosDBServerConnection(CaosDBServerConnection):
"""_DefaultCaosDBServerConnection.
@@ -101,10 +152,11 @@ class _DefaultCaosDBServerConnection(CaosDBServerConnection):
def __init__(self):
self._useragent = ("caosdb-pylib/{version} - {implementation}".format(
version=version, implementation=type(self).__name__))
- self._http_con = None
self._base_path = None
+ self._session = None
+ self._timeout = None
- def request(self, method, path, headers=None, body=None, **kwargs):
+ def request(self, method, path, headers=None, body=None):
"""request.
Send a HTTP request to the server.
@@ -118,38 +170,40 @@ class _DefaultCaosDBServerConnection(CaosDBServerConnection):
including query and frament segments.
headers : dict of str -> str, optional
HTTP request headers. (Defautl: None)
- body : str or bytes or readable, opional
+ body : str or bytes or readable, optional
The body of the HTTP request. Bytes should be a utf-8 encoded
string.
- **kwargs :
- Any keyword arguments will be ignored.
- TODO: Why are they allowed then?
Returns
-------
- TODO: What?
+ response : CaosDBHTTPResponse
"""
if headers is None:
headers = {}
headers["User-Agent"] = self._useragent
+
+ if path.endswith("/."):
+ path = path[:-1] + "%2E"
+
+ if isinstance(body, MultipartYielder):
+ body = ReadableMultiparts(body)
+
try:
- self._http_con = StreamingHTTPSConnection(
- # TODO looks as if configure needs to be done first.
- # That is however not assured.
- host=self.setup_fields["host"],
- timeout=self.setup_fields["timeout"],
- context=self.setup_fields["context"],
- socket_proxy=self.setup_fields["socket_proxy"])
- self._http_con.request(method=method, url=self._base_path + path,
- headers=headers, body=body)
- except SocketError as socket_err:
+ response = self._session.request(
+ method=method,
+ url=self._base_path + path,
+ headers=headers,
+ data=body,
+ timeout=self._timeout,
+ stream=True)
+
+ return _WrappedHTTPResponse(response)
+ except HTTPConnectionError as conn_err:
raise CaosDBConnectionError(
- "Connection failed. Network or server down? " + str(socket_err)
+ "Connection failed. Network or server down? " + str(conn_err)
)
- return _WrappedHTTPResponse(self._http_con.getresponse())
-
def configure(self, **config):
"""configure.
@@ -173,55 +227,69 @@ class _DefaultCaosDBServerConnection(CaosDBServerConnection):
loaded.
"""
+ if "url" not in config:
+ raise CaosDBConnectionError(
+ "No connection url specified. Please "
+ "do so via caosdb.configure_connection(...) or in a config "
+ "file.")
+ if (not config["url"].lower().startswith("https://") and not config["url"].lower().startswith("http://")):
+ raise CaosDBConnectionError("The connection url is expected "
+ "to be a http or https url and "
+ "must include the url scheme "
+ "(i.e. start with https:// or "
+ "http://).")
+
+ url = urlparse(config["url"])
+ path = url.path.strip("/")
+ if len(path) > 0:
+ path = path + "/"
+ self._base_path = url.scheme + "://" + url.netloc + "/" + path
+
+ self._session = HTTPSession()
+
+ if url.scheme == "https":
+ self._setup_ssl(config)
+
+ # TODO(tf) remove in next release
+ socket_proxy = config["socket_proxy"] if "socket_proxy" in config else None
+ if socket_proxy is not None:
+ self._session.proxies = {
+ "https": "socks5://" + socket_proxy,
+ "http": "socks5://" + socket_proxy,
+ }
+
+ if "https_proxy" in config:
+ if self._session.proxies is None:
+ self._session.proxies = {}
+ self._session.proxies["https"] = config["https_proxy"]
+
+ if "http_proxy" in config:
+ if self._session.proxies is None:
+ self._session.proxies = {}
+ self._session.proxies["http"] = config["http_proxy"]
+
+ if "timeout" in config:
+ self._timeout = config["timeout"]
+
+ def _setup_ssl(self, config):
if "ssl_version" in config and config["cacert"] is not None:
ssl_version = getattr(ssl, config["ssl_version"])
else:
ssl_version = ssl.PROTOCOL_TLS
- context = ssl.SSLContext(ssl_version)
- context.verify_mode = ssl.CERT_REQUIRED
- if config.get("ssl_insecure"):
+ self._session.mount(self._base_path, _SSLAdapter(ssl_version))
+
+ verify = True
+ if "cacert" in config:
+ verify = config["cacert"]
+ if "ssl_insecure" in config and config["ssl_insecure"].lower() == "true":
_LOGGER.warning("*** Warning! ***\n"
"Insecure SSL mode, certificate will not be checked! "
"Please consider removing the `ssl_insecure` configuration option.\n"
"****************")
- context.verify_mode = ssl.CERT_NONE
-
- if (not context.verify_mode == ssl.CERT_NONE and
- hasattr(context, "check_hostname")):
- context.check_hostname = True
-
- if ("cacert" in config and config["cacert"] is not None and
- config["cacert"]):
- try:
- context.load_verify_locations(config["cacert"])
- except Exception as exc:
- raise CaosDBConnectionError("Could not load the cacert in"
- "`{}`: {}".format(config["cacert"],
- exc))
-
- context.load_default_certs()
-
- if "url" in config:
- parsed_url = parse_url(config["url"])
- host = parsed_url.netloc
- self._base_path = parsed_url.path
- else:
- raise CaosDBConnectionError(
- "No connection url specified. Please "
- "do so via caosdb.configure_connection(...) or in a config "
- "file.")
-
- socket_proxy = None
-
- if "socket_proxy" in config:
- socket_proxy = config["socket_proxy"]
-
- self.setup_fields = {
- "host": host,
- "timeout": int(config.get("timeout")),
- "context": context,
- "socket_proxy": socket_proxy}
+ verify = False
+ if verify is not None:
+ self._session.verify = verify
def _make_conf(*conf):
@@ -252,7 +320,6 @@ _DEFAULT_CONF = {
"password_method": "input",
"implementation": _DefaultCaosDBServerConnection,
"timeout": 210,
- "cacert": resource_filename("caosdb", 'cert/indiscale.ca.crt')
}
@@ -314,6 +381,10 @@ def configure_connection(**kwargs):
Parameters
----------
+ url : str
+ The url of the CaosDB Server. HTTP and HTTPS urls are allowed. However,
+ it is **highly** recommend to avoid HTTP because passwords and
+ authentication token are send over the network in plain text.
username : str
Username for login; e.g. 'admin'.
@@ -342,6 +413,24 @@ def configure_connection(**kwargs):
An authentication token which has been issued by the CaosDB Server.
Implies `password_method="auth_token"` if set. An example token string would be `["O","OneTimeAuthenticationToken","anonymous",["administration"],[],1592995200000,604800000,"3ZZ4WKRB-5I7DG2Q6-ZZE6T64P-VQ","197d0d081615c52dc18fb323c300d7be077beaad4020773bb58920b55023fa6ee49355e35754a4277b9ac525c882bcd3a22e7227ba36dfcbbdbf8f15f19d1ee9",1,30000]`.
+ https_proxy : str, optional
+ Define a proxy for the https connections, e.g. `http://localhost:8888`,
+ `socks5://localhost:8888`, or `socks4://localhost:8888`. These are
+ either (non-TLS) HTTP proxies, SOCKS4 proxies, or SOCKS5 proxies. HTTPS
+ proxies are not supported. However, the connection will be secured
+ using TLS in the tunneled connection nonetheless. Only the connection
+ to the proxy is insecure which is why it is not recommended to use HTTP
+ proxies when authentication against the proxy is necessary. If
+ unspecified, the https_proxy option of the pycaosdb.ini or the HTTPS_PROXY
+ environment variable are being used. Use `None` to override these
+ options with a no-proxy setting.
+
+ http_proxy : str, optional
+ Define a proxy for the http connections, e.g. `http://localhost:8888`.
+ If unspecified, the http_proxy option of the pycaosdb.ini or the
+ HTTP_PROXY environment variable are being used. Use `None` to override
+ these options with a no-proxy setting.
+
implementation : CaosDBServerConnection
The class which implements the connection. (Default:
_DefaultCaosDBServerConnection)
@@ -372,6 +461,11 @@ def configure_connection(**kwargs):
local_conf = _make_conf(_DEFAULT_CONF, global_conf, kwargs)
connection = _Connection.get_instance()
+
+ if "socket_proxy" in local_conf:
+ warnings.warn("Deprecated configuration option: socket_proxy. Use "
+ "the new https_proxy option instead",
+ DeprecationWarning, stacklevel=1)
connection.configure(**local_conf)
return connection
@@ -599,7 +693,7 @@ class _Connection(object): # pylint: disable=useless-object-inheritance
method=method,
path=path,
headers=headers,
- body=body, **kwargs)
+ body=body)
_LOGGER.debug("response: %s %s", str(http_response.status),
str(http_response.getheaders()))
self._authenticator.on_response(http_response)
diff --git a/src/caosdb/connection/encode.py b/src/caosdb/connection/encode.py
index 7b092aae784a76abec0104ef7269df7ae0111b3b..0b826cc4400275a2374308ee104cdbdabb619b75 100644
--- a/src/caosdb/connection/encode.py
+++ b/src/caosdb/connection/encode.py
@@ -51,7 +51,8 @@ multipart/form-data is the standard way to upload files over HTTP
__all__ = [
'gen_boundary', 'encode_and_quote', 'MultipartParam', 'encode_string',
- 'encode_file_header', 'get_body_size', 'get_headers', 'multipart_encode'
+ 'encode_file_header', 'get_body_size', 'get_headers', 'multipart_encode',
+ 'ReadableMultiparts',
]
from urllib.parse import quote_plus
from io import UnsupportedOperation
@@ -475,3 +476,40 @@ def multipart_encode(params, boundary=None, callback=None):
params = MultipartParam.from_params(params)
return MultipartYielder(params, boundary, callback), headers
+
+
+class ReadableMultiparts(object):
+ """Wraps instances of the MultipartYielder class as a readable and withable
+ object."""
+
+ def __init__(self, multipart_yielder):
+ self.multipart_yielder = multipart_yielder
+ self.current_block = None
+ self.left_over = b''
+
+ def read(self, size=-1):
+ result = self.left_over
+ while size == -1 or len(result) < size:
+ try:
+ next_chunk = self.multipart_yielder.next()
+ if hasattr(next_chunk, "encode"):
+ next_chunk = next_chunk.encode("utf8")
+ result += next_chunk
+ except StopIteration:
+ break
+
+ if size == -1:
+ self.left_over = b''
+ return result
+
+ self.left_over = result[size:]
+ return result[:size]
+
+ def __enter__(self):
+ pass
+
+ def __exit__(self, type, value, traceback):
+ self.close()
+
+ def close(self):
+ self.multipart_yielder.reset()
diff --git a/src/caosdb/connection/streaminghttp.py b/src/caosdb/connection/streaminghttp.py
deleted file mode 100644
index 01774301b9bdb55bdbf6b56695042aaf354dba97..0000000000000000000000000000000000000000
--- a/src/caosdb/connection/streaminghttp.py
+++ /dev/null
@@ -1,152 +0,0 @@
-# -*- encoding: utf-8 -*-
-#
-# ** header v3.0
-# This file is a part of the CaosDB Project.
-#
-# Copyright (C) 2018 Research Group Biomedical Physics,
-# Max-Planck-Institute for Dynamics and Self-Organization Göttingen
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Affero General Public License as
-# published by the Free Software Foundation, either version 3 of the
-# License, or (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU Affero General Public License for more details.
-#
-# You should have received a copy of the GNU Affero General Public License
-# along with this program. If not, see <https://www.gnu.org/licenses/>.
-#
-# ** end header
-#
-# Original work Copyright (c) 2011 Chris AtLee
-# Modified work Copyright (c) 2017 Biomedical Physics, MPI for Dynamics and Self-Organization
-#
-# Permission is hereby granted, free of charge, to any person obtaining a copy
-# of this software and associated documentation files (the "Software"), to deal
-# in the Software without restriction, including without limitation the rights
-# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-# copies of the Software, and to permit persons to whom the Software is
-# furnished to do so, subject to the following conditions:
-#
-# The above copyright notice and this permission notice shall be included in
-# all copies or substantial portions of the Software.
-#
-# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
-# THE SOFTWARE.
-"""Streaming HTTP uploads module.
-
-This module extends the standard httplib and http.client HTTPConnection so that
-iterable objects can be used in the body of HTTP requests.
-
-**N.B.** You must specify a Content-Length header if using an iterable object
-since there is no way to determine in advance the total size that will be
-yielded, and there is no way to reset an interator.
-"""
-
-from __future__ import unicode_literals, print_function, absolute_import
-import socks
-import socket
-try:
- # python3
- from http import client as client
-except ImportError:
- # python2
- import httplib as client
-
-
-__all__ = ['StreamingHTTPSConnection']
-
-
-class StreamingHTTPSConnection(client.HTTPSConnection, object):
- """Subclass of `http.client.HTTSConnection` or `httplib.HTTPSConnection`
- that overrides the `send()` method to support iterable body objects."""
- # pylint: disable=unused-argument, arguments-differ
-
- def __init__(self, socket_proxy=None, **kwargs):
- if socket_proxy is not None:
- host, port = socket_proxy.split(":")
- socks.setdefaultproxy(socks.PROXY_TYPE_SOCKS5, host,
- int(port))
- socket.socket = socks.socksocket
- super(StreamingHTTPSConnection, self).__init__(**kwargs)
-
- def _send_output(self, body, **kwargs):
- """Send the currently buffered request and clear the buffer.
-
- Appends an extra \\r\\n to the buffer.
- A message_body may be specified, to be appended to the request.
-
- This method is implemented in differently in the various python
- versions (which is extremely annoying). So we provide a unified but
- relatively dumb implementaion which only serves our needs.
- """
- self._buffer.extend(("".encode("utf-8"), "".encode("utf-8")))
- headers = "\r\n".encode("utf-8").join(self._buffer)
- del self._buffer[:]
-
- self.send(headers)
- if body is not None:
- self.send(body)
-
- # pylint: disable=too-many-branches
-
- def send(self, value):
- """Send ``value`` to the server.
-
- ``value`` can be a string-like object which supports a 'encode' method,
- a file-like object that supports a .read() method, or an iterable object
- that supports a .next() method.
-
- An encode()able ``value`` will be utf-8 encoded before sending.
- """
- # Based on python 2.6's httplib.HTTPConnection.send()
- if self.sock is None:
- if self.auto_open:
- self.connect()
- else:
- raise client.NotConnected()
-
- # send the data to the server. if we get a broken pipe, then close
- # the socket. we want to reconnect when somebody tries to send again.
- #
- # NOTE: we DO propagate the error, though, because we cannot simply
- # ignore the error... the caller will know if they can retry.
- if self.debuglevel > 0:
- print("send: ", repr(value))
- try:
- blocksize = 8192
- if hasattr(value, 'read'):
- if hasattr(value, 'seek'):
- value.seek(0)
- if self.debuglevel > 0:
- print("sendIng a read()able")
- data = value.read(blocksize)
- while data:
- self.sock.sendall(data)
- data = value.read(blocksize)
- elif hasattr(value, 'next'):
- if hasattr(value, 'reset'):
- value.reset()
- if self.debuglevel > 0:
- print("sendIng an iterable")
- for data in value:
- if hasattr(data, "encode"):
- self.sock.sendall(data.encode('utf-8'))
- else:
- self.sock.sendall(data)
- else:
- if self.debuglevel > 0:
- print("sendIng a byte-like")
- self.sock.sendall(value)
- except socket.error as err:
- if err.args[0] == 32: # Broken pipe
- self.close()
- raise
diff --git a/src/caosdb/schema-pycaosdb-ini.yml b/src/caosdb/schema-pycaosdb-ini.yml
index a81bf006523ab7690ee0bf9d27e0a2d57ce8c3c3..64451a24da1587abbd9815b8ddefc4c85fa36e5a 100644
--- a/src/caosdb/schema-pycaosdb-ini.yml
+++ b/src/caosdb/schema-pycaosdb-ini.yml
@@ -14,10 +14,10 @@ schema-pycaosdb-ini:
additionalProperties: false
properties:
url:
- description: URL of the CaosDB server
+ description: "URL of the CaosDB server. Allowed are HTTP and HTTPS connections. However, since authentication tokens and sometimes even passwords are send in plain text to the server it is **highly** recommended to use HTTPS connections whenever possible. HTTP is ok for testing and debugging."
type: string
- pattern: https://[-a-zA-Z0-9\.]+(:[0-9]+)?(/)?
- examples: ["https://demo.indiscale.com/", "https://localhost:10443/"]
+ pattern: http(s)?://[-a-zA-Z0-9\.]+(:[0-9]+)?(/)?
+ examples: ["https://demo.indiscale.com/", "http://localhost:10080/"]
username:
type: string
description: User name used for authentication with the server
@@ -54,7 +54,15 @@ schema-pycaosdb-ini:
socket_proxy:
examples: ["localhost:12345"]
type: string
- description: You can define a socket proxy to be used. This is for the case that the server sits behind a firewall which is being tunnelled with a socket proxy (SOCKS4 or SOCKS5) (e.g. via ssh's -D option or a dedicated proxy server).
+ description: Deprecated. Please use https_proxy instead.
+ https_proxy:
+ examples: ["http://localhost:8888", "socks5://localhost:8888", "socks4://localhost:8888"]
+ type: string
+ description: "Define a proxy for the https connections. These are either (non-TLS) HTTP proxies, SOCKS4 proxies, or SOCKS5 proxies. HTTPS proxies are not supported. However, the connection will be secured using TLS in the tunneled connection nonetheless. Only the connection to the proxy is insecure which is why it is not recommended to use HTTP proxies when authentication against the proxy is necessary. Note: this option is overridden by the HTTPS_PROXY environment variable, if present."
+ http_proxy:
+ examples: ["http://localhost:8888", "socks5://localhost:8888", "socks4://localhost:8888"]
+ type: string
+ description: "Define a proxy for the http connections. These are either (non-TLS) HTTP proxies, SOCKS4 proxies, or SOCKS5 proxies. HTTPS proxies are not supported. Note: this option is overridden by the HTTP_PROXY environment variable, if present."
implementation:
description: This option is used internally and for testing. Do not override.
examples: [_DefaultCaosDBServerConnection]
diff --git a/src/caosdb/utils/server_side_scripting.py b/src/caosdb/utils/server_side_scripting.py
index 663178dcbda4293cb30dff88efbfb7b7302df70d..7e5ee4390ae3314792d12fd2942980aa3d9c9773 100644
--- a/src/caosdb/utils/server_side_scripting.py
+++ b/src/caosdb/utils/server_side_scripting.py
@@ -30,7 +30,8 @@ from lxml import etree
from caosdb.connection.connection import get_connection
from caosdb.connection.utils import urlencode
-from caosdb.connection.encode import MultipartParam, multipart_encode
+from caosdb.connection.encode import (MultipartParam, multipart_encode,
+ ReadableMultiparts)
def _make_params(pos_args, opts):
@@ -63,6 +64,7 @@ def _make_multipart_request(call, pos_args, opts, files):
filename=filename))
body, headers = multipart_encode(parts)
+ body = ReadableMultiparts(body)
return body, headers
diff --git a/tox.ini b/tox.ini
index e321891883b9425d24543a41f2d1283e0be52109..50c22d5716769ef2ec818f6c8fb94491ea372434 100644
--- a/tox.ini
+++ b/tox.ini
@@ -7,7 +7,6 @@ deps = .
nose
pytest
pytest-cov
- python-dateutil
jsonschema==4.0.1
commands=py.test --cov=caosdb -vv {posargs}
diff --git a/unittests/test_connection.py b/unittests/test_connection.py
index ee564ea033f9afc80522d75a85557f70819ece1e..d0408674b339e0958d6f37d6d9c2a834274986d2 100644
--- a/unittests/test_connection.py
+++ b/unittests/test_connection.py
@@ -37,7 +37,8 @@ from caosdb.connection.connection import (CaosDBServerConnection,
from caosdb.connection.mockup import (MockUpResponse, MockUpServerConnection,
_request_log_message)
from caosdb.connection.utils import make_uri_path, quote, urlencode
-from caosdb.exceptions import ConfigurationError, LoginFailedError
+from caosdb.exceptions import (ConfigurationError, LoginFailedError,
+ CaosDBConnectionError)
from nose.tools import assert_equal as eq
from nose.tools import assert_false as falz
from nose.tools import assert_is_not_none as there
@@ -46,6 +47,13 @@ from nose.tools import assert_true as tru
from pytest import raises
+def setup_function(function):
+ configure_connection(url="http://localhost:8888/some/path",
+ password_method="plain", username="test",
+ password="blub",
+ implementation=MockUpServerConnection)
+
+
def setup_module():
_reset_config()
@@ -116,6 +124,18 @@ def test_configure_connection():
tru(isinstance(c._delegate_connection, MockUpServerConnection))
+def test_configure_connection_bad_url():
+ configure_connection(url="https://localhost:8888")
+ with raises(CaosDBConnectionError) as exc_info:
+ configure_connection(url="ftp://localhost:8888")
+ assert exc_info.value.args[0].startswith(
+ "The connection url is expected to be a http or https url")
+ with raises(CaosDBConnectionError) as exc_info:
+ configure_connection(url="localhost:8888")
+ assert exc_info.value.args[0].startswith(
+ "The connection url is expected to be a http or https url")
+
+
def test_connection_interface():
with raiz(TypeError) as cm:
CaosDBServerConnection()
diff --git a/unittests/test_server_side_scripting.py b/unittests/test_server_side_scripting.py
index 1fb24d7e40bb843391a971c5f69680b541e1de0e..b699c4482d02972282167eb9683a956097ebc5e9 100644
--- a/unittests/test_server_side_scripting.py
+++ b/unittests/test_server_side_scripting.py
@@ -46,7 +46,7 @@ def setup_module():
content_type = kwargs["headers"]["Content-Type"]
if content_type.startswith("multipart/form-data; boundary"):
- parts = kwargs["body"]
+ parts = kwargs["body"].multipart_yielder
stdout = []
for part in parts:
if hasattr(part, "decode"):