Browse Source

Merge branch 'feature/UpdateUrllib3' into develop

pull/1200/head
JackDandy 6 years ago
parent
commit
8cd88e3e78
  1. 1
      CHANGES.md
  2. 55
      lib/urllib3/__init__.py
  3. 37
      lib/urllib3/_collections.py
  4. 197
      lib/urllib3/connection.py
  5. 397
      lib/urllib3/connectionpool.py
  6. 20
      lib/urllib3/contrib/_appengine_environ.py
  7. 261
      lib/urllib3/contrib/_securetransport/bindings.py
  8. 52
      lib/urllib3/contrib/_securetransport/low_level.py
  9. 118
      lib/urllib3/contrib/appengine.py
  10. 98
      lib/urllib3/contrib/ntlmpool.py
  11. 129
      lib/urllib3/contrib/pyopenssl.py
  12. 157
      lib/urllib3/contrib/securetransport.py
  13. 101
      lib/urllib3/contrib/socks.py
  14. 29
      lib/urllib3/exceptions.py
  15. 89
      lib/urllib3/fields.py
  16. 12
      lib/urllib3/filepost.py
  17. 2
      lib/urllib3/packages/__init__.py
  18. 9
      lib/urllib3/packages/backports/makefile.py
  19. 40
      lib/urllib3/packages/rfc3986/__init__.py
  20. 122
      lib/urllib3/packages/rfc3986/_mixin.py
  21. 219
      lib/urllib3/packages/rfc3986/abnf_regexp.py
  22. 10
      lib/urllib3/packages/rfc3986/api.py
  23. 47
      lib/urllib3/packages/rfc3986/builder.py
  24. 15
      lib/urllib3/packages/rfc3986/compat.py
  25. 38
      lib/urllib3/packages/rfc3986/exceptions.py
  26. 63
      lib/urllib3/packages/rfc3986/iri.py
  27. 105
      lib/urllib3/packages/rfc3986/misc.py
  28. 53
      lib/urllib3/packages/rfc3986/normalizers.py
  29. 326
      lib/urllib3/packages/rfc3986/parseresult.py
  30. 45
      lib/urllib3/packages/rfc3986/uri.py
  31. 91
      lib/urllib3/packages/rfc3986/validators.py
  32. 203
      lib/urllib3/packages/six.py
  33. 2
      lib/urllib3/packages/ssl_match_hostname/__init__.py
  34. 58
      lib/urllib3/packages/ssl_match_hostname/_implementation.py
  35. 183
      lib/urllib3/poolmanager.py
  36. 79
      lib/urllib3/request.py
  37. 176
      lib/urllib3/response.py
  38. 60
      lib/urllib3/util/__init__.py
  39. 16
      lib/urllib3/util/connection.py
  40. 52
      lib/urllib3/util/request.py
  41. 9
      lib/urllib3/util/response.py
  42. 100
      lib/urllib3/util/retry.py
  43. 153
      lib/urllib3/util/ssl_.py
  44. 79
      lib/urllib3/util/timeout.py
  45. 102
      lib/urllib3/util/url.py
  46. 3
      lib/urllib3/util/wait.py

1
CHANGES.md

@ -11,6 +11,7 @@
* Update Six compatibility library 1.12.0 (8da94b8) to 1.12.0 (aa4e90b) * Update Six compatibility library 1.12.0 (8da94b8) to 1.12.0 (aa4e90b)
* Update TZlocal 2.0.0.dev0 (b73a692) to 2.0.0b3 (410a838) * Update TZlocal 2.0.0.dev0 (b73a692) to 2.0.0b3 (410a838)
* Update unidecode module 1.0.22 (a5045ab) to 1.1.1 (632af82) * Update unidecode module 1.0.22 (a5045ab) to 1.1.1 (632af82)
* Update urllib3 release 1.25.2 (49eea80) to 1.25.3 (3387b20)
[develop changelog] [develop changelog]

55
lib/urllib3/__init__.py

@ -4,11 +4,7 @@ urllib3 - Thread-safe connection pooling and re-using.
from __future__ import absolute_import from __future__ import absolute_import
import warnings import warnings
from .connectionpool import ( from .connectionpool import HTTPConnectionPool, HTTPSConnectionPool, connection_from_url
HTTPConnectionPool,
HTTPSConnectionPool,
connection_from_url
)
from . import exceptions from . import exceptions
from .filepost import encode_multipart_formdata from .filepost import encode_multipart_formdata
@ -24,25 +20,25 @@ from .util.retry import Retry
import logging import logging
from logging import NullHandler from logging import NullHandler
__author__ = 'Andrey Petrov (andrey.petrov@shazow.net)' __author__ = "Andrey Petrov (andrey.petrov@shazow.net)"
__license__ = 'MIT' __license__ = "MIT"
__version__ = '1.25.2' __version__ = "1.25.3"
__all__ = ( __all__ = (
'HTTPConnectionPool', "HTTPConnectionPool",
'HTTPSConnectionPool', "HTTPSConnectionPool",
'PoolManager', "PoolManager",
'ProxyManager', "ProxyManager",
'HTTPResponse', "HTTPResponse",
'Retry', "Retry",
'Timeout', "Timeout",
'add_stderr_logger', "add_stderr_logger",
'connection_from_url', "connection_from_url",
'disable_warnings', "disable_warnings",
'encode_multipart_formdata', "encode_multipart_formdata",
'get_host', "get_host",
'make_headers', "make_headers",
'proxy_from_url', "proxy_from_url",
) )
logging.getLogger(__name__).addHandler(NullHandler()) logging.getLogger(__name__).addHandler(NullHandler())
@ -59,10 +55,10 @@ def add_stderr_logger(level=logging.DEBUG):
# even if urllib3 is vendored within another package. # even if urllib3 is vendored within another package.
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
handler = logging.StreamHandler() handler = logging.StreamHandler()
handler.setFormatter(logging.Formatter('%(asctime)s %(levelname)s %(message)s')) handler.setFormatter(logging.Formatter("%(asctime)s %(levelname)s %(message)s"))
logger.addHandler(handler) logger.addHandler(handler)
logger.setLevel(level) logger.setLevel(level)
logger.debug('Added a stderr logging handler to logger: %s', __name__) logger.debug("Added a stderr logging handler to logger: %s", __name__)
return handler return handler
@ -74,18 +70,17 @@ del NullHandler
# shouldn't be: otherwise, it's very hard for users to use most Python # shouldn't be: otherwise, it's very hard for users to use most Python
# mechanisms to silence them. # mechanisms to silence them.
# SecurityWarning's always go off by default. # SecurityWarning's always go off by default.
warnings.simplefilter('always', exceptions.SecurityWarning, append=True) warnings.simplefilter("always", exceptions.SecurityWarning, append=True)
# SubjectAltNameWarning's should go off once per host # SubjectAltNameWarning's should go off once per host
warnings.simplefilter('default', exceptions.SubjectAltNameWarning, append=True) warnings.simplefilter("default", exceptions.SubjectAltNameWarning, append=True)
# InsecurePlatformWarning's don't vary between requests, so we keep it default. # InsecurePlatformWarning's don't vary between requests, so we keep it default.
warnings.simplefilter('default', exceptions.InsecurePlatformWarning, warnings.simplefilter("default", exceptions.InsecurePlatformWarning, append=True)
append=True)
# SNIMissingWarnings should go off only once. # SNIMissingWarnings should go off only once.
warnings.simplefilter('default', exceptions.SNIMissingWarning, append=True) warnings.simplefilter("default", exceptions.SNIMissingWarning, append=True)
def disable_warnings(category=exceptions.HTTPWarning): def disable_warnings(category=exceptions.HTTPWarning):
""" """
Helper for quickly disabling all urllib3 warnings. Helper for quickly disabling all urllib3 warnings.
""" """
warnings.simplefilter('ignore', category) warnings.simplefilter("ignore", category)

37
lib/urllib3/_collections.py

@ -1,4 +1,5 @@
from __future__ import absolute_import from __future__ import absolute_import
try: try:
from collections.abc import Mapping, MutableMapping from collections.abc import Mapping, MutableMapping
except ImportError: except ImportError:
@ -6,6 +7,7 @@ except ImportError:
try: try:
from threading import RLock from threading import RLock
except ImportError: # Platform-specific: No threads available except ImportError: # Platform-specific: No threads available
class RLock: class RLock:
def __enter__(self): def __enter__(self):
pass pass
@ -19,7 +21,7 @@ from .exceptions import InvalidHeader
from .packages.six import iterkeys, itervalues, PY3 from .packages.six import iterkeys, itervalues, PY3
__all__ = ['RecentlyUsedContainer', 'HTTPHeaderDict'] __all__ = ["RecentlyUsedContainer", "HTTPHeaderDict"]
_Null = object() _Null = object()
@ -82,7 +84,9 @@ class RecentlyUsedContainer(MutableMapping):
return len(self._container) return len(self._container)
def __iter__(self): def __iter__(self):
raise NotImplementedError('Iteration over this class is unlikely to be threadsafe.') raise NotImplementedError(
"Iteration over this class is unlikely to be threadsafe."
)
def clear(self): def clear(self):
with self.lock: with self.lock:
@ -150,7 +154,7 @@ class HTTPHeaderDict(MutableMapping):
def __getitem__(self, key): def __getitem__(self, key):
val = self._container[key.lower()] val = self._container[key.lower()]
return ', '.join(val[1:]) return ", ".join(val[1:])
def __delitem__(self, key): def __delitem__(self, key):
del self._container[key.lower()] del self._container[key.lower()]
@ -159,12 +163,13 @@ class HTTPHeaderDict(MutableMapping):
return key.lower() in self._container return key.lower() in self._container
def __eq__(self, other): def __eq__(self, other):
if not isinstance(other, Mapping) and not hasattr(other, 'keys'): if not isinstance(other, Mapping) and not hasattr(other, "keys"):
return False return False
if not isinstance(other, type(self)): if not isinstance(other, type(self)):
other = type(self)(other) other = type(self)(other)
return (dict((k.lower(), v) for k, v in self.itermerged()) == return dict((k.lower(), v) for k, v in self.itermerged()) == dict(
dict((k.lower(), v) for k, v in other.itermerged())) (k.lower(), v) for k, v in other.itermerged()
)
def __ne__(self, other): def __ne__(self, other):
return not self.__eq__(other) return not self.__eq__(other)
@ -184,9 +189,9 @@ class HTTPHeaderDict(MutableMapping):
yield vals[0] yield vals[0]
def pop(self, key, default=__marker): def pop(self, key, default=__marker):
'''D.pop(k[,d]) -> v, remove specified key and return the corresponding value. """D.pop(k[,d]) -> v, remove specified key and return the corresponding value.
If key is not found, d is returned if given, otherwise KeyError is raised. If key is not found, d is returned if given, otherwise KeyError is raised.
''' """
# Using the MutableMapping function directly fails due to the private marker. # Using the MutableMapping function directly fails due to the private marker.
# Using ordinary dict.pop would expose the internal structures. # Using ordinary dict.pop would expose the internal structures.
# So let's reinvent the wheel. # So let's reinvent the wheel.
@ -228,8 +233,10 @@ class HTTPHeaderDict(MutableMapping):
with self.add instead of self.__setitem__ with self.add instead of self.__setitem__
""" """
if len(args) > 1: if len(args) > 1:
raise TypeError("extend() takes at most 1 positional " raise TypeError(
"arguments ({0} given)".format(len(args))) "extend() takes at most 1 positional "
"arguments ({0} given)".format(len(args))
)
other = args[0] if len(args) >= 1 else () other = args[0] if len(args) >= 1 else ()
if isinstance(other, HTTPHeaderDict): if isinstance(other, HTTPHeaderDict):
@ -295,7 +302,7 @@ class HTTPHeaderDict(MutableMapping):
"""Iterate over all headers, merging duplicate ones together.""" """Iterate over all headers, merging duplicate ones together."""
for key in self: for key in self:
val = self._container[key.lower()] val = self._container[key.lower()]
yield val[0], ', '.join(val[1:]) yield val[0], ", ".join(val[1:])
def items(self): def items(self):
return list(self.iteritems()) return list(self.iteritems())
@ -306,7 +313,7 @@ class HTTPHeaderDict(MutableMapping):
# python2.7 does not expose a proper API for exporting multiheaders # python2.7 does not expose a proper API for exporting multiheaders
# efficiently. This function re-reads raw lines from the message # efficiently. This function re-reads raw lines from the message
# object and extracts the multiheaders properly. # object and extracts the multiheaders properly.
obs_fold_continued_leaders = (' ', '\t') obs_fold_continued_leaders = (" ", "\t")
headers = [] headers = []
for line in message.headers: for line in message.headers:
@ -316,14 +323,14 @@ class HTTPHeaderDict(MutableMapping):
# in RFC-7230 S3.2.4. This indicates a multiline header, but # in RFC-7230 S3.2.4. This indicates a multiline header, but
# there exists no previous header to which we can attach it. # there exists no previous header to which we can attach it.
raise InvalidHeader( raise InvalidHeader(
'Header continuation with no previous header: %s' % line "Header continuation with no previous header: %s" % line
) )
else: else:
key, value = headers[-1] key, value = headers[-1]
headers[-1] = (key, value + ' ' + line.strip()) headers[-1] = (key, value + " " + line.strip())
continue continue
key, value = line.split(':', 1) key, value = line.split(":", 1)
headers.append((key, value.strip())) headers.append((key, value.strip()))
return cls(headers) return cls(headers)

197
lib/urllib3/connection.py

@ -11,6 +11,7 @@ from .packages.six.moves.http_client import HTTPException # noqa: F401
try: # Compiled with SSL? try: # Compiled with SSL?
import ssl import ssl
BaseSSLError = ssl.SSLError BaseSSLError = ssl.SSLError
except (ImportError, AttributeError): # Platform-specific: No SSL. except (ImportError, AttributeError): # Platform-specific: No SSL.
ssl = None ssl = None
@ -41,7 +42,7 @@ from .util.ssl_ import (
resolve_ssl_version, resolve_ssl_version,
assert_fingerprint, assert_fingerprint,
create_urllib3_context, create_urllib3_context,
ssl_wrap_socket ssl_wrap_socket,
) )
@ -51,20 +52,16 @@ from ._collections import HTTPHeaderDict
log = logging.getLogger(__name__) log = logging.getLogger(__name__)
port_by_scheme = { port_by_scheme = {"http": 80, "https": 443}
'http': 80,
'https': 443,
}
# When updating RECENT_DATE, move it to within two years of the current date, # When it comes time to update this value as a part of regular maintenance
# and not less than 6 months ago. # (ie test_recent_date is failing) update it to ~6 months before the current date.
# Example: if Today is 2018-01-01, then RECENT_DATE should be any date on or RECENT_DATE = datetime.date(2019, 1, 1)
# after 2016-01-01 (today - 2 years) AND before 2017-07-01 (today - 6 months)
RECENT_DATE = datetime.date(2017, 6, 30)
class DummyConnection(object): class DummyConnection(object):
"""Used to detect a failed ConnectionCls import.""" """Used to detect a failed ConnectionCls import."""
pass pass
@ -92,7 +89,7 @@ class HTTPConnection(_HTTPConnection, object):
Or you may want to disable the defaults by passing an empty list (e.g., ``[]``). Or you may want to disable the defaults by passing an empty list (e.g., ``[]``).
""" """
default_port = port_by_scheme['http'] default_port = port_by_scheme["http"]
#: Disable Nagle's algorithm by default. #: Disable Nagle's algorithm by default.
#: ``[(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)]`` #: ``[(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)]``
@ -103,14 +100,14 @@ class HTTPConnection(_HTTPConnection, object):
def __init__(self, *args, **kw): def __init__(self, *args, **kw):
if six.PY3: if six.PY3:
kw.pop('strict', None) kw.pop("strict", None)
# Pre-set source_address. # Pre-set source_address.
self.source_address = kw.get('source_address') self.source_address = kw.get("source_address")
#: The socket options provided by the user. If no options are #: The socket options provided by the user. If no options are
#: provided, we use the default options. #: provided, we use the default options.
self.socket_options = kw.pop('socket_options', self.default_socket_options) self.socket_options = kw.pop("socket_options", self.default_socket_options)
_HTTPConnection.__init__(self, *args, **kw) _HTTPConnection.__init__(self, *args, **kw)
@ -131,7 +128,7 @@ class HTTPConnection(_HTTPConnection, object):
those cases where it's appropriate (i.e., when doing DNS lookup to establish the those cases where it's appropriate (i.e., when doing DNS lookup to establish the
actual TCP connection across which we're going to send HTTP requests). actual TCP connection across which we're going to send HTTP requests).
""" """
return self._dns_host.rstrip('.') return self._dns_host.rstrip(".")
@host.setter @host.setter
def host(self, value): def host(self, value):
@ -150,30 +147,34 @@ class HTTPConnection(_HTTPConnection, object):
""" """
extra_kw = {} extra_kw = {}
if self.source_address: if self.source_address:
extra_kw['source_address'] = self.source_address extra_kw["source_address"] = self.source_address
if self.socket_options: if self.socket_options:
extra_kw['socket_options'] = self.socket_options extra_kw["socket_options"] = self.socket_options
try: try:
conn = connection.create_connection( conn = connection.create_connection(
(self._dns_host, self.port), self.timeout, **extra_kw) (self._dns_host, self.port), self.timeout, **extra_kw
)
except SocketTimeout: except SocketTimeout:
raise ConnectTimeoutError( raise ConnectTimeoutError(
self, "Connection to %s timed out. (connect timeout=%s)" % self,
(self.host, self.timeout)) "Connection to %s timed out. (connect timeout=%s)"
% (self.host, self.timeout),
)
except SocketError as e: except SocketError as e:
raise NewConnectionError( raise NewConnectionError(
self, "Failed to establish a new connection: %s" % e) self, "Failed to establish a new connection: %s" % e
)
return conn return conn
def _prepare_conn(self, conn): def _prepare_conn(self, conn):
self.sock = conn self.sock = conn
# Google App Engine's httplib does not define _tunnel_host # Google App Engine's httplib does not define _tunnel_host
if getattr(self, '_tunnel_host', None): if getattr(self, "_tunnel_host", None):
# TODO: Fix tunnel so it doesn't depend on self.sock state. # TODO: Fix tunnel so it doesn't depend on self.sock state.
self._tunnel() self._tunnel()
# Mark this connection as not reusable # Mark this connection as not reusable
@ -189,18 +190,15 @@ class HTTPConnection(_HTTPConnection, object):
body with chunked encoding and not as one block body with chunked encoding and not as one block
""" """
headers = HTTPHeaderDict(headers if headers is not None else {}) headers = HTTPHeaderDict(headers if headers is not None else {})
skip_accept_encoding = 'accept-encoding' in headers skip_accept_encoding = "accept-encoding" in headers
skip_host = 'host' in headers skip_host = "host" in headers
self.putrequest( self.putrequest(
method, method, url, skip_accept_encoding=skip_accept_encoding, skip_host=skip_host
url,
skip_accept_encoding=skip_accept_encoding,
skip_host=skip_host
) )
for header, value in headers.items(): for header, value in headers.items():
self.putheader(header, value) self.putheader(header, value)
if 'transfer-encoding' not in headers: if "transfer-encoding" not in headers:
self.putheader('Transfer-Encoding', 'chunked') self.putheader("Transfer-Encoding", "chunked")
self.endheaders() self.endheaders()
if body is not None: if body is not None:
@ -211,29 +209,37 @@ class HTTPConnection(_HTTPConnection, object):
if not chunk: if not chunk:
continue continue
if not isinstance(chunk, bytes): if not isinstance(chunk, bytes):
chunk = chunk.encode('utf8') chunk = chunk.encode("utf8")
len_str = hex(len(chunk))[2:] len_str = hex(len(chunk))[2:]
self.send(len_str.encode('utf-8')) self.send(len_str.encode("utf-8"))
self.send(b'\r\n') self.send(b"\r\n")
self.send(chunk) self.send(chunk)
self.send(b'\r\n') self.send(b"\r\n")
# After the if clause, to always have a closed body # After the if clause, to always have a closed body
self.send(b'0\r\n\r\n') self.send(b"0\r\n\r\n")
class HTTPSConnection(HTTPConnection): class HTTPSConnection(HTTPConnection):
default_port = port_by_scheme['https'] default_port = port_by_scheme["https"]
ssl_version = None ssl_version = None
def __init__(self, host, port=None, key_file=None, cert_file=None, def __init__(
key_password=None, strict=None, self,
timeout=socket._GLOBAL_DEFAULT_TIMEOUT, host,
ssl_context=None, server_hostname=None, **kw): port=None,
key_file=None,
HTTPConnection.__init__(self, host, port, strict=strict, cert_file=None,
timeout=timeout, **kw) key_password=None,
strict=None,
timeout=socket._GLOBAL_DEFAULT_TIMEOUT,
ssl_context=None,
server_hostname=None,
**kw
):
HTTPConnection.__init__(self, host, port, strict=strict, timeout=timeout, **kw)
self.key_file = key_file self.key_file = key_file
self.cert_file = cert_file self.cert_file = cert_file
@ -243,25 +249,40 @@ class HTTPSConnection(HTTPConnection):
# Required property for Google AppEngine 1.9.0 which otherwise causes # Required property for Google AppEngine 1.9.0 which otherwise causes
# HTTPS requests to go out as HTTP. (See Issue #356) # HTTPS requests to go out as HTTP. (See Issue #356)
self._protocol = 'https' self._protocol = "https"
def connect(self): def connect(self):
conn = self._new_conn() conn = self._new_conn()
self._prepare_conn(conn) self._prepare_conn(conn)
# Wrap socket using verification with the root certs in
# trusted_root_certs
default_ssl_context = False
if self.ssl_context is None: if self.ssl_context is None:
default_ssl_context = True
self.ssl_context = create_urllib3_context( self.ssl_context = create_urllib3_context(
ssl_version=resolve_ssl_version(None), ssl_version=resolve_ssl_version(self.ssl_version),
cert_reqs=resolve_cert_reqs(None), cert_reqs=resolve_cert_reqs(self.cert_reqs),
) )
# Try to load OS default certs if none are given.
# Works well on Windows (requires Python3.4+)
context = self.ssl_context
if (
not self.ca_certs
and not self.ca_cert_dir
and default_ssl_context
and hasattr(context, "load_default_certs")
):
context.load_default_certs()
self.sock = ssl_wrap_socket( self.sock = ssl_wrap_socket(
sock=conn, sock=conn,
keyfile=self.key_file, keyfile=self.key_file,
certfile=self.cert_file, certfile=self.cert_file,
key_password=self.key_password, key_password=self.key_password,
ssl_context=self.ssl_context, ssl_context=self.ssl_context,
server_hostname=self.server_hostname server_hostname=self.server_hostname,
) )
@ -270,16 +291,24 @@ class VerifiedHTTPSConnection(HTTPSConnection):
Based on httplib.HTTPSConnection but wraps the socket with Based on httplib.HTTPSConnection but wraps the socket with
SSL certification. SSL certification.
""" """
cert_reqs = None cert_reqs = None
ca_certs = None ca_certs = None
ca_cert_dir = None ca_cert_dir = None
ssl_version = None ssl_version = None
assert_fingerprint = None assert_fingerprint = None
def set_cert(self, key_file=None, cert_file=None, def set_cert(
cert_reqs=None, key_password=None, ca_certs=None, self,
assert_hostname=None, assert_fingerprint=None, key_file=None,
ca_cert_dir=None): cert_file=None,
cert_reqs=None,
key_password=None,
ca_certs=None,
assert_hostname=None,
assert_fingerprint=None,
ca_cert_dir=None,
):
""" """
This method should only be called once, before the connection is used. This method should only be called once, before the connection is used.
""" """
@ -306,7 +335,7 @@ class VerifiedHTTPSConnection(HTTPSConnection):
hostname = self.host hostname = self.host
# Google App Engine's httplib does not define _tunnel_host # Google App Engine's httplib does not define _tunnel_host
if getattr(self, '_tunnel_host', None): if getattr(self, "_tunnel_host", None):
self.sock = conn self.sock = conn
# Calls self._set_hostport(), so self.host is # Calls self._set_hostport(), so self.host is
# self._tunnel_host below. # self._tunnel_host below.
@ -323,15 +352,19 @@ class VerifiedHTTPSConnection(HTTPSConnection):
is_time_off = datetime.date.today() < RECENT_DATE is_time_off = datetime.date.today() < RECENT_DATE
if is_time_off: if is_time_off:
warnings.warn(( warnings.warn(
'System time is way off (before {0}). This will probably ' (
'lead to SSL verification errors').format(RECENT_DATE), "System time is way off (before {0}). This will probably "
SystemTimeWarning "lead to SSL verification errors"
).format(RECENT_DATE),
SystemTimeWarning,
) )
# Wrap socket using verification with the root certs in # Wrap socket using verification with the root certs in
# trusted_root_certs # trusted_root_certs
default_ssl_context = False
if self.ssl_context is None: if self.ssl_context is None:
default_ssl_context = True
self.ssl_context = create_urllib3_context( self.ssl_context = create_urllib3_context(
ssl_version=resolve_ssl_version(self.ssl_version), ssl_version=resolve_ssl_version(self.ssl_version),
cert_reqs=resolve_cert_reqs(self.cert_reqs), cert_reqs=resolve_cert_reqs(self.cert_reqs),
@ -339,6 +372,17 @@ class VerifiedHTTPSConnection(HTTPSConnection):
context = self.ssl_context context = self.ssl_context
context.verify_mode = resolve_cert_reqs(self.cert_reqs) context.verify_mode = resolve_cert_reqs(self.cert_reqs)
# Try to load OS default certs if none are given.
# Works well on Windows (requires Python3.4+)
if (
not self.ca_certs
and not self.ca_cert_dir
and default_ssl_context
and hasattr(context, "load_default_certs")
):
context.load_default_certs()
self.sock = ssl_wrap_socket( self.sock = ssl_wrap_socket(
sock=conn, sock=conn,
keyfile=self.key_file, keyfile=self.key_file,
@ -347,31 +391,37 @@ class VerifiedHTTPSConnection(HTTPSConnection):
ca_certs=self.ca_certs, ca_certs=self.ca_certs,
ca_cert_dir=self.ca_cert_dir, ca_cert_dir=self.ca_cert_dir,
server_hostname=server_hostname, server_hostname=server_hostname,
ssl_context=context) ssl_context=context,
)
if self.assert_fingerprint: if self.assert_fingerprint:
assert_fingerprint(self.sock.getpeercert(binary_form=True), assert_fingerprint(
self.assert_fingerprint) self.sock.getpeercert(binary_form=True), self.assert_fingerprint
elif context.verify_mode != ssl.CERT_NONE \ )
and not getattr(context, 'check_hostname', False) \ elif (
and self.assert_hostname is not False: context.verify_mode != ssl.CERT_NONE
and not getattr(context, "check_hostname", False)
and self.assert_hostname is not False
):
# While urllib3 attempts to always turn off hostname matching from # While urllib3 attempts to always turn off hostname matching from
# the TLS library, this cannot always be done. So we check whether # the TLS library, this cannot always be done. So we check whether
# the TLS Library still thinks it's matching hostnames. # the TLS Library still thinks it's matching hostnames.
cert = self.sock.getpeercert() cert = self.sock.getpeercert()
if not cert.get('subjectAltName', ()): if not cert.get("subjectAltName", ()):
warnings.warn(( warnings.warn(
'Certificate for {0} has no `subjectAltName`, falling back to check for a ' (
'`commonName` for now. This feature is being removed by major browsers and ' "Certificate for {0} has no `subjectAltName`, falling back to check for a "
'deprecated by RFC 2818. (See https://github.com/shazow/urllib3/issues/497 ' "`commonName` for now. This feature is being removed by major browsers and "
'for details.)'.format(hostname)), "deprecated by RFC 2818. (See https://github.com/shazow/urllib3/issues/497 "
SubjectAltNameWarning "for details.)".format(hostname)
),
SubjectAltNameWarning,
) )
_match_hostname(cert, self.assert_hostname or server_hostname) _match_hostname(cert, self.assert_hostname or server_hostname)
self.is_verified = ( self.is_verified = (
context.verify_mode == ssl.CERT_REQUIRED or context.verify_mode == ssl.CERT_REQUIRED
self.assert_fingerprint is not None or self.assert_fingerprint is not None
) )
@ -380,8 +430,9 @@ def _match_hostname(cert, asserted_hostname):
match_hostname(cert, asserted_hostname) match_hostname(cert, asserted_hostname)
except CertificateError as e: except CertificateError as e:
log.error( log.error(
'Certificate did not match expected hostname: %s. ' "Certificate did not match expected hostname: %s. " "Certificate: %s",
'Certificate: %s', asserted_hostname, cert asserted_hostname,
cert,
) )
# Add cert to exception and reraise so client code can inspect # Add cert to exception and reraise so client code can inspect
# the cert when catching the exception, if they want to # the cert when catching the exception, if they want to

397
lib/urllib3/connectionpool.py

@ -30,8 +30,11 @@ from .packages.rfc3986.normalizers import normalize_host
from .connection import ( from .connection import (
port_by_scheme, port_by_scheme,
DummyConnection, DummyConnection,
HTTPConnection, HTTPSConnection, VerifiedHTTPSConnection, HTTPConnection,
HTTPException, BaseSSLError, HTTPSConnection,
VerifiedHTTPSConnection,
HTTPException,
BaseSSLError,
) )
from .request import RequestMethods from .request import RequestMethods
from .response import HTTPResponse from .response import HTTPResponse
@ -71,8 +74,7 @@ class ConnectionPool(object):
self.port = port self.port = port
def __str__(self): def __str__(self):
return '%s(host=%r, port=%r)' % (type(self).__name__, return "%s(host=%r, port=%r)" % (type(self).__name__, self.host, self.port)
self.host, self.port)
def __enter__(self): def __enter__(self):
return self return self
@ -153,15 +155,24 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods):
:class:`urllib3.connection.HTTPSConnection` instances. :class:`urllib3.connection.HTTPSConnection` instances.
""" """
scheme = 'http' scheme = "http"
ConnectionCls = HTTPConnection ConnectionCls = HTTPConnection
ResponseCls = HTTPResponse ResponseCls = HTTPResponse
def __init__(self, host, port=None, strict=False, def __init__(
timeout=Timeout.DEFAULT_TIMEOUT, maxsize=1, block=False, self,
headers=None, retries=None, host,
_proxy=None, _proxy_headers=None, port=None,
**conn_kw): strict=False,
timeout=Timeout.DEFAULT_TIMEOUT,
maxsize=1,
block=False,
headers=None,
retries=None,
_proxy=None,
_proxy_headers=None,
**conn_kw
):
ConnectionPool.__init__(self, host, port) ConnectionPool.__init__(self, host, port)
RequestMethods.__init__(self, headers) RequestMethods.__init__(self, headers)
@ -195,19 +206,27 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods):
# Enable Nagle's algorithm for proxies, to avoid packet fragmentation. # Enable Nagle's algorithm for proxies, to avoid packet fragmentation.
# We cannot know if the user has added default socket options, so we cannot replace the # We cannot know if the user has added default socket options, so we cannot replace the
# list. # list.
self.conn_kw.setdefault('socket_options', []) self.conn_kw.setdefault("socket_options", [])
def _new_conn(self): def _new_conn(self):
""" """
Return a fresh :class:`HTTPConnection`. Return a fresh :class:`HTTPConnection`.
""" """
self.num_connections += 1 self.num_connections += 1
log.debug("Starting new HTTP connection (%d): %s:%s", log.debug(
self.num_connections, self.host, self.port or "80") "Starting new HTTP connection (%d): %s:%s",
self.num_connections,
conn = self.ConnectionCls(host=self.host, port=self.port, self.host,
timeout=self.timeout.connect_timeout, self.port or "80",
strict=self.strict, **self.conn_kw) )
conn = self.ConnectionCls(
host=self.host,
port=self.port,
timeout=self.timeout.connect_timeout,
strict=self.strict,
**self.conn_kw
)
return conn return conn
def _get_conn(self, timeout=None): def _get_conn(self, timeout=None):
@ -231,16 +250,17 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods):
except queue.Empty: except queue.Empty:
if self.block: if self.block:
raise EmptyPoolError(self, raise EmptyPoolError(
"Pool reached maximum size and no more " self,
"connections are allowed.") "Pool reached maximum size and no more " "connections are allowed.",
)
pass # Oh well, we'll create a new connection then pass # Oh well, we'll create a new connection then
# If this is a persistent connection, check if it got disconnected # If this is a persistent connection, check if it got disconnected
if conn and is_connection_dropped(conn): if conn and is_connection_dropped(conn):
log.debug("Resetting dropped connection: %s", self.host) log.debug("Resetting dropped connection: %s", self.host)
conn.close() conn.close()
if getattr(conn, 'auto_open', 1) == 0: if getattr(conn, "auto_open", 1) == 0:
# This is a proxied connection that has been mutated by # This is a proxied connection that has been mutated by
# httplib._tunnel() and cannot be reused (since it would # httplib._tunnel() and cannot be reused (since it would
# attempt to bypass the proxy) # attempt to bypass the proxy)
@ -270,9 +290,7 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods):
pass pass
except queue.Full: except queue.Full:
# This should never happen if self.block == True # This should never happen if self.block == True
log.warning( log.warning("Connection pool is full, discarding connection: %s", self.host)
"Connection pool is full, discarding connection: %s",
self.host)
# Connection never got put back into the pool, close it. # Connection never got put back into the pool, close it.
if conn: if conn:
@ -304,21 +322,30 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods):
"""Is the error actually a timeout? Will raise a ReadTimeout or pass""" """Is the error actually a timeout? Will raise a ReadTimeout or pass"""
if isinstance(err, SocketTimeout): if isinstance(err, SocketTimeout):
raise ReadTimeoutError(self, url, "Read timed out. (read timeout=%s)" % timeout_value) raise ReadTimeoutError(
self, url, "Read timed out. (read timeout=%s)" % timeout_value
)
# See the above comment about EAGAIN in Python 3. In Python 2 we have # See the above comment about EAGAIN in Python 3. In Python 2 we have
# to specifically catch it and throw the timeout error # to specifically catch it and throw the timeout error
if hasattr(err, 'errno') and err.errno in _blocking_errnos: if hasattr(err, "errno") and err.errno in _blocking_errnos:
raise ReadTimeoutError(self, url, "Read timed out. (read timeout=%s)" % timeout_value) raise ReadTimeoutError(
self, url, "Read timed out. (read timeout=%s)" % timeout_value
)
# Catch possible read timeouts thrown as SSL errors. If not the # Catch possible read timeouts thrown as SSL errors. If not the
# case, rethrow the original. We need to do this because of: # case, rethrow the original. We need to do this because of:
# http://bugs.python.org/issue10272 # http://bugs.python.org/issue10272
if 'timed out' in str(err) or 'did not complete (read)' in str(err): # Python < 2.7.4 if "timed out" in str(err) or "did not complete (read)" in str(
raise ReadTimeoutError(self, url, "Read timed out. (read timeout=%s)" % timeout_value) err
): # Python < 2.7.4
def _make_request(self, conn, method, url, timeout=_Default, chunked=False, raise ReadTimeoutError(
**httplib_request_kw): self, url, "Read timed out. (read timeout=%s)" % timeout_value
)
def _make_request(
self, conn, method, url, timeout=_Default, chunked=False, **httplib_request_kw
):
""" """
Perform a request on a given urllib connection object taken from our Perform a request on a given urllib connection object taken from our
pool. pool.
@ -358,7 +385,7 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods):
read_timeout = timeout_obj.read_timeout read_timeout = timeout_obj.read_timeout
# App Engine doesn't have a sock attr # App Engine doesn't have a sock attr
if getattr(conn, 'sock', None): if getattr(conn, "sock", None):
# In Python 3 socket.py will catch EAGAIN and return None when you # In Python 3 socket.py will catch EAGAIN and return None when you
# try and read into the file pointer created by http.client, which # try and read into the file pointer created by http.client, which
# instead raises a BadStatusLine exception. Instead of catching # instead raises a BadStatusLine exception. Instead of catching
@ -366,7 +393,8 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods):
# timeouts, check for a zero timeout before making the request. # timeouts, check for a zero timeout before making the request.
if read_timeout == 0: if read_timeout == 0:
raise ReadTimeoutError( raise ReadTimeoutError(
self, url, "Read timed out. (read timeout=%s)" % read_timeout) self, url, "Read timed out. (read timeout=%s)" % read_timeout
)
if read_timeout is Timeout.DEFAULT_TIMEOUT: if read_timeout is Timeout.DEFAULT_TIMEOUT:
conn.sock.settimeout(socket.getdefaulttimeout()) conn.sock.settimeout(socket.getdefaulttimeout())
else: # None or a value else: # None or a value
@ -381,26 +409,38 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods):
# Python 3 # Python 3
try: try:
httplib_response = conn.getresponse() httplib_response = conn.getresponse()
except Exception as e: except BaseException as e:
# Remove the TypeError from the exception chain in Python 3; # Remove the TypeError from the exception chain in
# otherwise it looks like a programming error was the cause. # Python 3 (including for exceptions like SystemExit).
# Otherwise it looks like a bug in the code.
six.raise_from(e, None) six.raise_from(e, None)
except (SocketTimeout, BaseSSLError, SocketError) as e: except (SocketTimeout, BaseSSLError, SocketError) as e:
self._raise_timeout(err=e, url=url, timeout_value=read_timeout) self._raise_timeout(err=e, url=url, timeout_value=read_timeout)
raise raise
# AppEngine doesn't have a version attr. # AppEngine doesn't have a version attr.
http_version = getattr(conn, '_http_vsn_str', 'HTTP/?') http_version = getattr(conn, "_http_vsn_str", "HTTP/?")
log.debug("%s://%s:%s \"%s %s %s\" %s %s", self.scheme, self.host, self.port, log.debug(
method, url, http_version, httplib_response.status, '%s://%s:%s "%s %s %s" %s %s',
httplib_response.length) self.scheme,
self.host,
self.port,
method,
url,
http_version,
httplib_response.status,
httplib_response.length,
)
try: try:
assert_header_parsing(httplib_response.msg) assert_header_parsing(httplib_response.msg)
except (HeaderParsingError, TypeError) as hpe: # Platform-specific: Python 3 except (HeaderParsingError, TypeError) as hpe: # Platform-specific: Python 3
log.warning( log.warning(
'Failed to parse headers (url=%s): %s', "Failed to parse headers (url=%s): %s",
self._absolute_url(url), hpe, exc_info=True) self._absolute_url(url),
hpe,
exc_info=True,
)
return httplib_response return httplib_response
@ -430,7 +470,7 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods):
Check if the given ``url`` is a member of the same host as this Check if the given ``url`` is a member of the same host as this
connection pool. connection pool.
""" """
if url.startswith('/'): if url.startswith("/"):
return True return True
# TODO: Add optional support for socket.gethostbyname checking. # TODO: Add optional support for socket.gethostbyname checking.
@ -446,10 +486,22 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods):
return (scheme, host, port) == (self.scheme, self.host, self.port) return (scheme, host, port) == (self.scheme, self.host, self.port)
def urlopen(self, method, url, body=None, headers=None, retries=None, def urlopen(
redirect=True, assert_same_host=True, timeout=_Default, self,
pool_timeout=None, release_conn=None, chunked=False, method,
body_pos=None, **response_kw): url,
body=None,
headers=None,
retries=None,
redirect=True,
assert_same_host=True,
timeout=_Default,
pool_timeout=None,
release_conn=None,
chunked=False,
body_pos=None,
**response_kw
):
""" """
Get a connection from the pool and perform an HTTP request. This is the Get a connection from the pool and perform an HTTP request. This is the
lowest level call for making a request, so you'll need to specify all lowest level call for making a request, so you'll need to specify all
@ -547,7 +599,7 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods):
retries = Retry.from_int(retries, redirect=redirect, default=self.retries) retries = Retry.from_int(retries, redirect=redirect, default=self.retries)
if release_conn is None: if release_conn is None:
release_conn = response_kw.get('preload_content', True) release_conn = response_kw.get("preload_content", True)
# Check host # Check host
if assert_same_host and not self.is_same_host(url): if assert_same_host and not self.is_same_host(url):
@ -569,7 +621,7 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods):
# Merge the proxy headers. Only do this in HTTP. We have to copy the # Merge the proxy headers. Only do this in HTTP. We have to copy the
# headers dict so we can safely change it without those changes being # headers dict so we can safely change it without those changes being
# reflected in anyone else's copy. # reflected in anyone else's copy.
if self.scheme == 'http': if self.scheme == "http":
headers = headers.copy() headers = headers.copy()
headers.update(self.proxy_headers) headers.update(self.proxy_headers)
@ -592,15 +644,22 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods):
conn.timeout = timeout_obj.connect_timeout conn.timeout = timeout_obj.connect_timeout
is_new_proxy_conn = self.proxy is not None and not getattr(conn, 'sock', None) is_new_proxy_conn = self.proxy is not None and not getattr(
conn, "sock", None
)
if is_new_proxy_conn: if is_new_proxy_conn:
self._prepare_proxy(conn) self._prepare_proxy(conn)
# Make the request on the httplib connection object. # Make the request on the httplib connection object.
httplib_response = self._make_request(conn, method, url, httplib_response = self._make_request(
timeout=timeout_obj, conn,
body=body, headers=headers, method,
chunked=chunked) url,
timeout=timeout_obj,
body=body,
headers=headers,
chunked=chunked,
)
# If we're going to release the connection in ``finally:``, then # If we're going to release the connection in ``finally:``, then
# the response doesn't need to know about the connection. Otherwise # the response doesn't need to know about the connection. Otherwise
@ -609,14 +668,16 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods):
response_conn = conn if not release_conn else None response_conn = conn if not release_conn else None
# Pass method to Response for length checking # Pass method to Response for length checking
response_kw['request_method'] = method response_kw["request_method"] = method
# Import httplib's response into our own wrapper object # Import httplib's response into our own wrapper object
response = self.ResponseCls.from_httplib(httplib_response, response = self.ResponseCls.from_httplib(
pool=self, httplib_response,
connection=response_conn, pool=self,
retries=retries, connection=response_conn,
**response_kw) retries=retries,
**response_kw
)
# Everything went great! # Everything went great!
clean_exit = True clean_exit = True
@ -625,20 +686,28 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods):
# Timed out by queue. # Timed out by queue.
raise EmptyPoolError(self, "No pool connections are available.") raise EmptyPoolError(self, "No pool connections are available.")
except (TimeoutError, HTTPException, SocketError, ProtocolError, except (
BaseSSLError, SSLError, CertificateError) as e: TimeoutError,
HTTPException,
SocketError,
ProtocolError,
BaseSSLError,
SSLError,
CertificateError,
) as e:
# Discard the connection for these exceptions. It will be # Discard the connection for these exceptions. It will be
# replaced during the next _get_conn() call. # replaced during the next _get_conn() call.
clean_exit = False clean_exit = False
if isinstance(e, (BaseSSLError, CertificateError)): if isinstance(e, (BaseSSLError, CertificateError)):
e = SSLError(e) e = SSLError(e)
elif isinstance(e, (SocketError, NewConnectionError)) and self.proxy: elif isinstance(e, (SocketError, NewConnectionError)) and self.proxy:
e = ProxyError('Cannot connect to proxy.', e) e = ProxyError("Cannot connect to proxy.", e)
elif isinstance(e, (SocketError, HTTPException)): elif isinstance(e, (SocketError, HTTPException)):
e = ProtocolError('Connection aborted.', e) e = ProtocolError("Connection aborted.", e)
retries = retries.increment(method, url, error=e, _pool=self, retries = retries.increment(
_stacktrace=sys.exc_info()[2]) method, url, error=e, _pool=self, _stacktrace=sys.exc_info()[2]
)
retries.sleep() retries.sleep()
# Keep track of the error for the retry warning. # Keep track of the error for the retry warning.
@ -661,28 +730,47 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods):
if not conn: if not conn:
# Try again # Try again
log.warning("Retrying (%r) after connection " log.warning(
"broken by '%r': %s", retries, err, url) "Retrying (%r) after connection " "broken by '%r': %s",
return self.urlopen(method, url, body, headers, retries, retries,
redirect, assert_same_host, err,
timeout=timeout, pool_timeout=pool_timeout, url,
release_conn=release_conn, body_pos=body_pos, )
**response_kw) return self.urlopen(
method,
url,
body,
headers,
retries,
redirect,
assert_same_host,
timeout=timeout,
pool_timeout=pool_timeout,
release_conn=release_conn,
body_pos=body_pos,
**response_kw
)
def drain_and_release_conn(response): def drain_and_release_conn(response):
try: try:
# discard any remaining response body, the connection will be # discard any remaining response body, the connection will be
# released back to the pool once the entire response is read # released back to the pool once the entire response is read
response.read() response.read()
except (TimeoutError, HTTPException, SocketError, ProtocolError, except (
BaseSSLError, SSLError): TimeoutError,
HTTPException,
SocketError,
ProtocolError,
BaseSSLError,
SSLError,
):
pass pass
# Handle redirect? # Handle redirect?
redirect_location = redirect and response.get_redirect_location() redirect_location = redirect and response.get_redirect_location()
if redirect_location: if redirect_location:
if response.status == 303: if response.status == 303:
method = 'GET' method = "GET"
try: try:
retries = retries.increment(method, url, response=response, _pool=self) retries = retries.increment(method, url, response=response, _pool=self)
@ -700,15 +788,22 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods):
retries.sleep_for_retry(response) retries.sleep_for_retry(response)
log.debug("Redirecting %s -> %s", url, redirect_location) log.debug("Redirecting %s -> %s", url, redirect_location)
return self.urlopen( return self.urlopen(
method, redirect_location, body, headers, method,
retries=retries, redirect=redirect, redirect_location,
body,
headers,
retries=retries,
redirect=redirect,
assert_same_host=assert_same_host, assert_same_host=assert_same_host,
timeout=timeout, pool_timeout=pool_timeout, timeout=timeout,
release_conn=release_conn, body_pos=body_pos, pool_timeout=pool_timeout,
**response_kw) release_conn=release_conn,
body_pos=body_pos,
**response_kw
)
# Check if we should retry the HTTP response. # Check if we should retry the HTTP response.
has_retry_after = bool(response.getheader('Retry-After')) has_retry_after = bool(response.getheader("Retry-After"))
if retries.is_retry(method, response.status, has_retry_after): if retries.is_retry(method, response.status, has_retry_after):
try: try:
retries = retries.increment(method, url, response=response, _pool=self) retries = retries.increment(method, url, response=response, _pool=self)
@ -726,12 +821,19 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods):
retries.sleep(response) retries.sleep(response)
log.debug("Retry: %s", url) log.debug("Retry: %s", url)
return self.urlopen( return self.urlopen(
method, url, body, headers, method,
retries=retries, redirect=redirect, url,
body,
headers,
retries=retries,
redirect=redirect,
assert_same_host=assert_same_host, assert_same_host=assert_same_host,
timeout=timeout, pool_timeout=pool_timeout, timeout=timeout,
pool_timeout=pool_timeout,
release_conn=release_conn, release_conn=release_conn,
body_pos=body_pos, **response_kw) body_pos=body_pos,
**response_kw
)
return response return response
@ -754,21 +856,47 @@ class HTTPSConnectionPool(HTTPConnectionPool):
the connection socket into an SSL socket. the connection socket into an SSL socket.
""" """
scheme = 'https' scheme = "https"
ConnectionCls = HTTPSConnection ConnectionCls = HTTPSConnection
def __init__(self, host, port=None, def __init__(
strict=False, timeout=Timeout.DEFAULT_TIMEOUT, maxsize=1, self,
block=False, headers=None, retries=None, host,
_proxy=None, _proxy_headers=None, port=None,
key_file=None, cert_file=None, cert_reqs=None, strict=False,
key_password=None, ca_certs=None, ssl_version=None, timeout=Timeout.DEFAULT_TIMEOUT,
assert_hostname=None, assert_fingerprint=None, maxsize=1,
ca_cert_dir=None, **conn_kw): block=False,
headers=None,
HTTPConnectionPool.__init__(self, host, port, strict, timeout, maxsize, retries=None,
block, headers, retries, _proxy, _proxy_headers, _proxy=None,
**conn_kw) _proxy_headers=None,
key_file=None,
cert_file=None,
cert_reqs=None,
key_password=None,
ca_certs=None,
ssl_version=None,
assert_hostname=None,
assert_fingerprint=None,
ca_cert_dir=None,
**conn_kw
):
HTTPConnectionPool.__init__(
self,
host,
port,
strict,
timeout,
maxsize,
block,
headers,
retries,
_proxy,
_proxy_headers,
**conn_kw
)
self.key_file = key_file self.key_file = key_file
self.cert_file = cert_file self.cert_file = cert_file
@ -787,14 +915,16 @@ class HTTPSConnectionPool(HTTPConnectionPool):
""" """
if isinstance(conn, VerifiedHTTPSConnection): if isinstance(conn, VerifiedHTTPSConnection):
conn.set_cert(key_file=self.key_file, conn.set_cert(
key_password=self.key_password, key_file=self.key_file,
cert_file=self.cert_file, key_password=self.key_password,
cert_reqs=self.cert_reqs, cert_file=self.cert_file,
ca_certs=self.ca_certs, cert_reqs=self.cert_reqs,
ca_cert_dir=self.ca_cert_dir, ca_certs=self.ca_certs,
assert_hostname=self.assert_hostname, ca_cert_dir=self.ca_cert_dir,
assert_fingerprint=self.assert_fingerprint) assert_hostname=self.assert_hostname,
assert_fingerprint=self.assert_fingerprint,
)
conn.ssl_version = self.ssl_version conn.ssl_version = self.ssl_version
return conn return conn
@ -811,12 +941,17 @@ class HTTPSConnectionPool(HTTPConnectionPool):
Return a fresh :class:`httplib.HTTPSConnection`. Return a fresh :class:`httplib.HTTPSConnection`.
""" """
self.num_connections += 1 self.num_connections += 1
log.debug("Starting new HTTPS connection (%d): %s:%s", log.debug(
self.num_connections, self.host, self.port or "443") "Starting new HTTPS connection (%d): %s:%s",
self.num_connections,
self.host,
self.port or "443",
)
if not self.ConnectionCls or self.ConnectionCls is DummyConnection: if not self.ConnectionCls or self.ConnectionCls is DummyConnection:
raise SSLError("Can't connect to HTTPS URL because the SSL " raise SSLError(
"module is not available.") "Can't connect to HTTPS URL because the SSL " "module is not available."
)
actual_host = self.host actual_host = self.host
actual_port = self.port actual_port = self.port
@ -824,11 +959,16 @@ class HTTPSConnectionPool(HTTPConnectionPool):
actual_host = self.proxy.host actual_host = self.proxy.host
actual_port = self.proxy.port actual_port = self.proxy.port
conn = self.ConnectionCls(host=actual_host, port=actual_port, conn = self.ConnectionCls(
timeout=self.timeout.connect_timeout, host=actual_host,
strict=self.strict, cert_file=self.cert_file, port=actual_port,
key_file=self.key_file, key_password=self.key_password, timeout=self.timeout.connect_timeout,
**self.conn_kw) strict=self.strict,
cert_file=self.cert_file,
key_file=self.key_file,
key_password=self.key_password,
**self.conn_kw
)
return self._prepare_conn(conn) return self._prepare_conn(conn)
@ -839,16 +979,19 @@ class HTTPSConnectionPool(HTTPConnectionPool):
super(HTTPSConnectionPool, self)._validate_conn(conn) super(HTTPSConnectionPool, self)._validate_conn(conn)
# Force connect early to allow us to validate the connection. # Force connect early to allow us to validate the connection.
if not getattr(conn, 'sock', None): # AppEngine might not have `.sock` if not getattr(conn, "sock", None): # AppEngine might not have `.sock`
conn.connect() conn.connect()
if not conn.is_verified: if not conn.is_verified:
warnings.warn(( warnings.warn(
'Unverified HTTPS request is being made. ' (
'Adding certificate verification is strongly advised. See: ' "Unverified HTTPS request is being made. "
'https://urllib3.readthedocs.io/en/latest/advanced-usage.html' "Adding certificate verification is strongly advised. See: "
'#ssl-warnings'), "https://urllib3.readthedocs.io/en/latest/advanced-usage.html"
InsecureRequestWarning) "#ssl-warnings"
),
InsecureRequestWarning,
)
def connection_from_url(url, **kw): def connection_from_url(url, **kw):
@ -873,7 +1016,7 @@ def connection_from_url(url, **kw):
""" """
scheme, host, port = get_host(url) scheme, host, port = get_host(url)
port = port or port_by_scheme.get(scheme, 80) port = port or port_by_scheme.get(scheme, 80)
if scheme == 'https': if scheme == "https":
return HTTPSConnectionPool(host, port=port, **kw) return HTTPSConnectionPool(host, port=port, **kw)
else: else:
return HTTPConnectionPool(host, port=port, **kw) return HTTPConnectionPool(host, port=port, **kw)
@ -890,8 +1033,8 @@ def _normalize_host(host, scheme):
# Instead, we need to make sure we never pass ``None`` as the port. # Instead, we need to make sure we never pass ``None`` as the port.
# However, for backward compatibility reasons we can't actually # However, for backward compatibility reasons we can't actually
# *assert* that. See http://bugs.python.org/issue28539 # *assert* that. See http://bugs.python.org/issue28539
if host.startswith('[') and host.endswith(']'): if host.startswith("[") and host.endswith("]"):
host = host.strip('[]') host = host.strip("[]")
if scheme in NORMALIZABLE_SCHEMES: if scheme in NORMALIZABLE_SCHEMES:
host = normalize_host(host) host = normalize_host(host)
return host return host

20
lib/urllib3/contrib/_appengine_environ.py

@ -6,9 +6,7 @@ import os
def is_appengine(): def is_appengine():
return (is_local_appengine() or return is_local_appengine() or is_prod_appengine() or is_prod_appengine_mvms()
is_prod_appengine() or
is_prod_appengine_mvms())
def is_appengine_sandbox(): def is_appengine_sandbox():
@ -16,15 +14,19 @@ def is_appengine_sandbox():
def is_local_appengine(): def is_local_appengine():
return ('APPENGINE_RUNTIME' in os.environ and return (
'Development/' in os.environ['SERVER_SOFTWARE']) "APPENGINE_RUNTIME" in os.environ
and "Development/" in os.environ["SERVER_SOFTWARE"]
)
def is_prod_appengine(): def is_prod_appengine():
return ('APPENGINE_RUNTIME' in os.environ and return (
'Google App Engine/' in os.environ['SERVER_SOFTWARE'] and "APPENGINE_RUNTIME" in os.environ
not is_prod_appengine_mvms()) and "Google App Engine/" in os.environ["SERVER_SOFTWARE"]
and not is_prod_appengine_mvms()
)
def is_prod_appengine_mvms(): def is_prod_appengine_mvms():
return os.environ.get('GAE_VM', False) == 'true' return os.environ.get("GAE_VM", False) == "true"

261
lib/urllib3/contrib/_securetransport/bindings.py

@ -34,29 +34,35 @@ from __future__ import absolute_import
import platform import platform
from ctypes.util import find_library from ctypes.util import find_library
from ctypes import ( from ctypes import (
c_void_p, c_int32, c_char_p, c_size_t, c_byte, c_uint32, c_ulong, c_long, c_void_p,
c_bool c_int32,
c_char_p,
c_size_t,
c_byte,
c_uint32,
c_ulong,
c_long,
c_bool,
) )
from ctypes import CDLL, POINTER, CFUNCTYPE from ctypes import CDLL, POINTER, CFUNCTYPE
security_path = find_library('Security') security_path = find_library("Security")
if not security_path: if not security_path:
raise ImportError('The library Security could not be found') raise ImportError("The library Security could not be found")
core_foundation_path = find_library('CoreFoundation') core_foundation_path = find_library("CoreFoundation")
if not core_foundation_path: if not core_foundation_path:
raise ImportError('The library CoreFoundation could not be found') raise ImportError("The library CoreFoundation could not be found")
version = platform.mac_ver()[0] version = platform.mac_ver()[0]
version_info = tuple(map(int, version.split('.'))) version_info = tuple(map(int, version.split(".")))
if version_info < (10, 8): if version_info < (10, 8):
raise OSError( raise OSError(
'Only OS X 10.8 and newer are supported, not %s.%s' % ( "Only OS X 10.8 and newer are supported, not %s.%s"
version_info[0], version_info[1] % (version_info[0], version_info[1])
)
) )
Security = CDLL(security_path, use_errno=True) Security = CDLL(security_path, use_errno=True)
@ -129,27 +135,19 @@ try:
Security.SecKeyGetTypeID.argtypes = [] Security.SecKeyGetTypeID.argtypes = []
Security.SecKeyGetTypeID.restype = CFTypeID Security.SecKeyGetTypeID.restype = CFTypeID
Security.SecCertificateCreateWithData.argtypes = [ Security.SecCertificateCreateWithData.argtypes = [CFAllocatorRef, CFDataRef]
CFAllocatorRef,
CFDataRef
]
Security.SecCertificateCreateWithData.restype = SecCertificateRef Security.SecCertificateCreateWithData.restype = SecCertificateRef
Security.SecCertificateCopyData.argtypes = [ Security.SecCertificateCopyData.argtypes = [SecCertificateRef]
SecCertificateRef
]
Security.SecCertificateCopyData.restype = CFDataRef Security.SecCertificateCopyData.restype = CFDataRef
Security.SecCopyErrorMessageString.argtypes = [ Security.SecCopyErrorMessageString.argtypes = [OSStatus, c_void_p]
OSStatus,
c_void_p
]
Security.SecCopyErrorMessageString.restype = CFStringRef Security.SecCopyErrorMessageString.restype = CFStringRef
Security.SecIdentityCreateWithCertificate.argtypes = [ Security.SecIdentityCreateWithCertificate.argtypes = [
CFTypeRef, CFTypeRef,
SecCertificateRef, SecCertificateRef,
POINTER(SecIdentityRef) POINTER(SecIdentityRef),
] ]
Security.SecIdentityCreateWithCertificate.restype = OSStatus Security.SecIdentityCreateWithCertificate.restype = OSStatus
@ -159,201 +157,126 @@ try:
c_void_p, c_void_p,
Boolean, Boolean,
c_void_p, c_void_p,
POINTER(SecKeychainRef) POINTER(SecKeychainRef),
] ]
Security.SecKeychainCreate.restype = OSStatus Security.SecKeychainCreate.restype = OSStatus
Security.SecKeychainDelete.argtypes = [ Security.SecKeychainDelete.argtypes = [SecKeychainRef]
SecKeychainRef
]
Security.SecKeychainDelete.restype = OSStatus Security.SecKeychainDelete.restype = OSStatus
Security.SecPKCS12Import.argtypes = [ Security.SecPKCS12Import.argtypes = [
CFDataRef, CFDataRef,
CFDictionaryRef, CFDictionaryRef,
POINTER(CFArrayRef) POINTER(CFArrayRef),
] ]
Security.SecPKCS12Import.restype = OSStatus Security.SecPKCS12Import.restype = OSStatus
SSLReadFunc = CFUNCTYPE(OSStatus, SSLConnectionRef, c_void_p, POINTER(c_size_t)) SSLReadFunc = CFUNCTYPE(OSStatus, SSLConnectionRef, c_void_p, POINTER(c_size_t))
SSLWriteFunc = CFUNCTYPE(OSStatus, SSLConnectionRef, POINTER(c_byte), POINTER(c_size_t)) SSLWriteFunc = CFUNCTYPE(
OSStatus, SSLConnectionRef, POINTER(c_byte), POINTER(c_size_t)
)
Security.SSLSetIOFuncs.argtypes = [ Security.SSLSetIOFuncs.argtypes = [SSLContextRef, SSLReadFunc, SSLWriteFunc]
SSLContextRef,
SSLReadFunc,
SSLWriteFunc
]
Security.SSLSetIOFuncs.restype = OSStatus Security.SSLSetIOFuncs.restype = OSStatus
Security.SSLSetPeerID.argtypes = [ Security.SSLSetPeerID.argtypes = [SSLContextRef, c_char_p, c_size_t]
SSLContextRef,
c_char_p,
c_size_t
]
Security.SSLSetPeerID.restype = OSStatus Security.SSLSetPeerID.restype = OSStatus
Security.SSLSetCertificate.argtypes = [ Security.SSLSetCertificate.argtypes = [SSLContextRef, CFArrayRef]
SSLContextRef,
CFArrayRef
]
Security.SSLSetCertificate.restype = OSStatus Security.SSLSetCertificate.restype = OSStatus
Security.SSLSetCertificateAuthorities.argtypes = [ Security.SSLSetCertificateAuthorities.argtypes = [SSLContextRef, CFTypeRef, Boolean]
SSLContextRef,
CFTypeRef,
Boolean
]
Security.SSLSetCertificateAuthorities.restype = OSStatus Security.SSLSetCertificateAuthorities.restype = OSStatus
Security.SSLSetConnection.argtypes = [ Security.SSLSetConnection.argtypes = [SSLContextRef, SSLConnectionRef]
SSLContextRef,
SSLConnectionRef
]
Security.SSLSetConnection.restype = OSStatus Security.SSLSetConnection.restype = OSStatus
Security.SSLSetPeerDomainName.argtypes = [ Security.SSLSetPeerDomainName.argtypes = [SSLContextRef, c_char_p, c_size_t]
SSLContextRef,
c_char_p,
c_size_t
]
Security.SSLSetPeerDomainName.restype = OSStatus Security.SSLSetPeerDomainName.restype = OSStatus
Security.SSLHandshake.argtypes = [ Security.SSLHandshake.argtypes = [SSLContextRef]
SSLContextRef
]
Security.SSLHandshake.restype = OSStatus Security.SSLHandshake.restype = OSStatus
Security.SSLRead.argtypes = [ Security.SSLRead.argtypes = [SSLContextRef, c_char_p, c_size_t, POINTER(c_size_t)]
SSLContextRef,
c_char_p,
c_size_t,
POINTER(c_size_t)
]
Security.SSLRead.restype = OSStatus Security.SSLRead.restype = OSStatus
Security.SSLWrite.argtypes = [ Security.SSLWrite.argtypes = [SSLContextRef, c_char_p, c_size_t, POINTER(c_size_t)]
SSLContextRef,
c_char_p,
c_size_t,
POINTER(c_size_t)
]
Security.SSLWrite.restype = OSStatus Security.SSLWrite.restype = OSStatus
Security.SSLClose.argtypes = [ Security.SSLClose.argtypes = [SSLContextRef]
SSLContextRef
]
Security.SSLClose.restype = OSStatus Security.SSLClose.restype = OSStatus
Security.SSLGetNumberSupportedCiphers.argtypes = [ Security.SSLGetNumberSupportedCiphers.argtypes = [SSLContextRef, POINTER(c_size_t)]
SSLContextRef,
POINTER(c_size_t)
]
Security.SSLGetNumberSupportedCiphers.restype = OSStatus Security.SSLGetNumberSupportedCiphers.restype = OSStatus
Security.SSLGetSupportedCiphers.argtypes = [ Security.SSLGetSupportedCiphers.argtypes = [
SSLContextRef, SSLContextRef,
POINTER(SSLCipherSuite), POINTER(SSLCipherSuite),
POINTER(c_size_t) POINTER(c_size_t),
] ]
Security.SSLGetSupportedCiphers.restype = OSStatus Security.SSLGetSupportedCiphers.restype = OSStatus
Security.SSLSetEnabledCiphers.argtypes = [ Security.SSLSetEnabledCiphers.argtypes = [
SSLContextRef, SSLContextRef,
POINTER(SSLCipherSuite), POINTER(SSLCipherSuite),
c_size_t c_size_t,
] ]
Security.SSLSetEnabledCiphers.restype = OSStatus Security.SSLSetEnabledCiphers.restype = OSStatus
Security.SSLGetNumberEnabledCiphers.argtype = [ Security.SSLGetNumberEnabledCiphers.argtype = [SSLContextRef, POINTER(c_size_t)]
SSLContextRef,
POINTER(c_size_t)
]
Security.SSLGetNumberEnabledCiphers.restype = OSStatus Security.SSLGetNumberEnabledCiphers.restype = OSStatus
Security.SSLGetEnabledCiphers.argtypes = [ Security.SSLGetEnabledCiphers.argtypes = [
SSLContextRef, SSLContextRef,
POINTER(SSLCipherSuite), POINTER(SSLCipherSuite),
POINTER(c_size_t) POINTER(c_size_t),
] ]
Security.SSLGetEnabledCiphers.restype = OSStatus Security.SSLGetEnabledCiphers.restype = OSStatus
Security.SSLGetNegotiatedCipher.argtypes = [ Security.SSLGetNegotiatedCipher.argtypes = [SSLContextRef, POINTER(SSLCipherSuite)]
SSLContextRef,
POINTER(SSLCipherSuite)
]
Security.SSLGetNegotiatedCipher.restype = OSStatus Security.SSLGetNegotiatedCipher.restype = OSStatus
Security.SSLGetNegotiatedProtocolVersion.argtypes = [ Security.SSLGetNegotiatedProtocolVersion.argtypes = [
SSLContextRef, SSLContextRef,
POINTER(SSLProtocol) POINTER(SSLProtocol),
] ]
Security.SSLGetNegotiatedProtocolVersion.restype = OSStatus Security.SSLGetNegotiatedProtocolVersion.restype = OSStatus
Security.SSLCopyPeerTrust.argtypes = [ Security.SSLCopyPeerTrust.argtypes = [SSLContextRef, POINTER(SecTrustRef)]
SSLContextRef,
POINTER(SecTrustRef)
]
Security.SSLCopyPeerTrust.restype = OSStatus Security.SSLCopyPeerTrust.restype = OSStatus
Security.SecTrustSetAnchorCertificates.argtypes = [ Security.SecTrustSetAnchorCertificates.argtypes = [SecTrustRef, CFArrayRef]
SecTrustRef,
CFArrayRef
]
Security.SecTrustSetAnchorCertificates.restype = OSStatus Security.SecTrustSetAnchorCertificates.restype = OSStatus
Security.SecTrustSetAnchorCertificatesOnly.argstypes = [ Security.SecTrustSetAnchorCertificatesOnly.argstypes = [SecTrustRef, Boolean]
SecTrustRef,
Boolean
]
Security.SecTrustSetAnchorCertificatesOnly.restype = OSStatus Security.SecTrustSetAnchorCertificatesOnly.restype = OSStatus
Security.SecTrustEvaluate.argtypes = [ Security.SecTrustEvaluate.argtypes = [SecTrustRef, POINTER(SecTrustResultType)]
SecTrustRef,
POINTER(SecTrustResultType)
]
Security.SecTrustEvaluate.restype = OSStatus Security.SecTrustEvaluate.restype = OSStatus
Security.SecTrustGetCertificateCount.argtypes = [ Security.SecTrustGetCertificateCount.argtypes = [SecTrustRef]
SecTrustRef
]
Security.SecTrustGetCertificateCount.restype = CFIndex Security.SecTrustGetCertificateCount.restype = CFIndex
Security.SecTrustGetCertificateAtIndex.argtypes = [ Security.SecTrustGetCertificateAtIndex.argtypes = [SecTrustRef, CFIndex]
SecTrustRef,
CFIndex
]
Security.SecTrustGetCertificateAtIndex.restype = SecCertificateRef Security.SecTrustGetCertificateAtIndex.restype = SecCertificateRef
Security.SSLCreateContext.argtypes = [ Security.SSLCreateContext.argtypes = [
CFAllocatorRef, CFAllocatorRef,
SSLProtocolSide, SSLProtocolSide,
SSLConnectionType SSLConnectionType,
] ]
Security.SSLCreateContext.restype = SSLContextRef Security.SSLCreateContext.restype = SSLContextRef
Security.SSLSetSessionOption.argtypes = [ Security.SSLSetSessionOption.argtypes = [SSLContextRef, SSLSessionOption, Boolean]
SSLContextRef,
SSLSessionOption,
Boolean
]
Security.SSLSetSessionOption.restype = OSStatus Security.SSLSetSessionOption.restype = OSStatus
Security.SSLSetProtocolVersionMin.argtypes = [ Security.SSLSetProtocolVersionMin.argtypes = [SSLContextRef, SSLProtocol]
SSLContextRef,
SSLProtocol
]
Security.SSLSetProtocolVersionMin.restype = OSStatus Security.SSLSetProtocolVersionMin.restype = OSStatus
Security.SSLSetProtocolVersionMax.argtypes = [ Security.SSLSetProtocolVersionMax.argtypes = [SSLContextRef, SSLProtocol]
SSLContextRef,
SSLProtocol
]
Security.SSLSetProtocolVersionMax.restype = OSStatus Security.SSLSetProtocolVersionMax.restype = OSStatus
Security.SecCopyErrorMessageString.argtypes = [ Security.SecCopyErrorMessageString.argtypes = [OSStatus, c_void_p]
OSStatus,
c_void_p
]
Security.SecCopyErrorMessageString.restype = CFStringRef Security.SecCopyErrorMessageString.restype = CFStringRef
Security.SSLReadFunc = SSLReadFunc Security.SSLReadFunc = SSLReadFunc
@ -369,64 +292,47 @@ try:
Security.OSStatus = OSStatus Security.OSStatus = OSStatus
Security.kSecImportExportPassphrase = CFStringRef.in_dll( Security.kSecImportExportPassphrase = CFStringRef.in_dll(
Security, 'kSecImportExportPassphrase' Security, "kSecImportExportPassphrase"
) )
Security.kSecImportItemIdentity = CFStringRef.in_dll( Security.kSecImportItemIdentity = CFStringRef.in_dll(
Security, 'kSecImportItemIdentity' Security, "kSecImportItemIdentity"
) )
# CoreFoundation time! # CoreFoundation time!
CoreFoundation.CFRetain.argtypes = [ CoreFoundation.CFRetain.argtypes = [CFTypeRef]
CFTypeRef
]
CoreFoundation.CFRetain.restype = CFTypeRef CoreFoundation.CFRetain.restype = CFTypeRef
CoreFoundation.CFRelease.argtypes = [ CoreFoundation.CFRelease.argtypes = [CFTypeRef]
CFTypeRef
]
CoreFoundation.CFRelease.restype = None CoreFoundation.CFRelease.restype = None
CoreFoundation.CFGetTypeID.argtypes = [ CoreFoundation.CFGetTypeID.argtypes = [CFTypeRef]
CFTypeRef
]
CoreFoundation.CFGetTypeID.restype = CFTypeID CoreFoundation.CFGetTypeID.restype = CFTypeID
CoreFoundation.CFStringCreateWithCString.argtypes = [ CoreFoundation.CFStringCreateWithCString.argtypes = [
CFAllocatorRef, CFAllocatorRef,
c_char_p, c_char_p,
CFStringEncoding CFStringEncoding,
] ]
CoreFoundation.CFStringCreateWithCString.restype = CFStringRef CoreFoundation.CFStringCreateWithCString.restype = CFStringRef
CoreFoundation.CFStringGetCStringPtr.argtypes = [ CoreFoundation.CFStringGetCStringPtr.argtypes = [CFStringRef, CFStringEncoding]
CFStringRef,
CFStringEncoding
]
CoreFoundation.CFStringGetCStringPtr.restype = c_char_p CoreFoundation.CFStringGetCStringPtr.restype = c_char_p
CoreFoundation.CFStringGetCString.argtypes = [ CoreFoundation.CFStringGetCString.argtypes = [
CFStringRef, CFStringRef,
c_char_p, c_char_p,
CFIndex, CFIndex,
CFStringEncoding CFStringEncoding,
] ]
CoreFoundation.CFStringGetCString.restype = c_bool CoreFoundation.CFStringGetCString.restype = c_bool
CoreFoundation.CFDataCreate.argtypes = [ CoreFoundation.CFDataCreate.argtypes = [CFAllocatorRef, c_char_p, CFIndex]
CFAllocatorRef,
c_char_p,
CFIndex
]
CoreFoundation.CFDataCreate.restype = CFDataRef CoreFoundation.CFDataCreate.restype = CFDataRef
CoreFoundation.CFDataGetLength.argtypes = [ CoreFoundation.CFDataGetLength.argtypes = [CFDataRef]
CFDataRef
]
CoreFoundation.CFDataGetLength.restype = CFIndex CoreFoundation.CFDataGetLength.restype = CFIndex
CoreFoundation.CFDataGetBytePtr.argtypes = [ CoreFoundation.CFDataGetBytePtr.argtypes = [CFDataRef]
CFDataRef
]
CoreFoundation.CFDataGetBytePtr.restype = c_void_p CoreFoundation.CFDataGetBytePtr.restype = c_void_p
CoreFoundation.CFDictionaryCreate.argtypes = [ CoreFoundation.CFDictionaryCreate.argtypes = [
@ -435,14 +341,11 @@ try:
POINTER(CFTypeRef), POINTER(CFTypeRef),
CFIndex, CFIndex,
CFDictionaryKeyCallBacks, CFDictionaryKeyCallBacks,
CFDictionaryValueCallBacks CFDictionaryValueCallBacks,
] ]
CoreFoundation.CFDictionaryCreate.restype = CFDictionaryRef CoreFoundation.CFDictionaryCreate.restype = CFDictionaryRef
CoreFoundation.CFDictionaryGetValue.argtypes = [ CoreFoundation.CFDictionaryGetValue.argtypes = [CFDictionaryRef, CFTypeRef]
CFDictionaryRef,
CFTypeRef
]
CoreFoundation.CFDictionaryGetValue.restype = CFTypeRef CoreFoundation.CFDictionaryGetValue.restype = CFTypeRef
CoreFoundation.CFArrayCreate.argtypes = [ CoreFoundation.CFArrayCreate.argtypes = [
@ -456,36 +359,30 @@ try:
CoreFoundation.CFArrayCreateMutable.argtypes = [ CoreFoundation.CFArrayCreateMutable.argtypes = [
CFAllocatorRef, CFAllocatorRef,
CFIndex, CFIndex,
CFArrayCallBacks CFArrayCallBacks,
] ]
CoreFoundation.CFArrayCreateMutable.restype = CFMutableArrayRef CoreFoundation.CFArrayCreateMutable.restype = CFMutableArrayRef
CoreFoundation.CFArrayAppendValue.argtypes = [ CoreFoundation.CFArrayAppendValue.argtypes = [CFMutableArrayRef, c_void_p]
CFMutableArrayRef,
c_void_p
]
CoreFoundation.CFArrayAppendValue.restype = None CoreFoundation.CFArrayAppendValue.restype = None
CoreFoundation.CFArrayGetCount.argtypes = [ CoreFoundation.CFArrayGetCount.argtypes = [CFArrayRef]
CFArrayRef
]
CoreFoundation.CFArrayGetCount.restype = CFIndex CoreFoundation.CFArrayGetCount.restype = CFIndex
CoreFoundation.CFArrayGetValueAtIndex.argtypes = [ CoreFoundation.CFArrayGetValueAtIndex.argtypes = [CFArrayRef, CFIndex]
CFArrayRef,
CFIndex
]
CoreFoundation.CFArrayGetValueAtIndex.restype = c_void_p CoreFoundation.CFArrayGetValueAtIndex.restype = c_void_p
CoreFoundation.kCFAllocatorDefault = CFAllocatorRef.in_dll( CoreFoundation.kCFAllocatorDefault = CFAllocatorRef.in_dll(
CoreFoundation, 'kCFAllocatorDefault' CoreFoundation, "kCFAllocatorDefault"
)
CoreFoundation.kCFTypeArrayCallBacks = c_void_p.in_dll(
CoreFoundation, "kCFTypeArrayCallBacks"
) )
CoreFoundation.kCFTypeArrayCallBacks = c_void_p.in_dll(CoreFoundation, 'kCFTypeArrayCallBacks')
CoreFoundation.kCFTypeDictionaryKeyCallBacks = c_void_p.in_dll( CoreFoundation.kCFTypeDictionaryKeyCallBacks = c_void_p.in_dll(
CoreFoundation, 'kCFTypeDictionaryKeyCallBacks' CoreFoundation, "kCFTypeDictionaryKeyCallBacks"
) )
CoreFoundation.kCFTypeDictionaryValueCallBacks = c_void_p.in_dll( CoreFoundation.kCFTypeDictionaryValueCallBacks = c_void_p.in_dll(
CoreFoundation, 'kCFTypeDictionaryValueCallBacks' CoreFoundation, "kCFTypeDictionaryValueCallBacks"
) )
CoreFoundation.CFTypeRef = CFTypeRef CoreFoundation.CFTypeRef = CFTypeRef
@ -494,7 +391,7 @@ try:
CoreFoundation.CFDictionaryRef = CFDictionaryRef CoreFoundation.CFDictionaryRef = CFDictionaryRef
except (AttributeError): except (AttributeError):
raise ImportError('Error initializing ctypes') raise ImportError("Error initializing ctypes")
class CFConst(object): class CFConst(object):
@ -502,6 +399,7 @@ class CFConst(object):
A class object that acts as essentially a namespace for CoreFoundation A class object that acts as essentially a namespace for CoreFoundation
constants. constants.
""" """
kCFStringEncodingUTF8 = CFStringEncoding(0x08000100) kCFStringEncodingUTF8 = CFStringEncoding(0x08000100)
@ -509,6 +407,7 @@ class SecurityConst(object):
""" """
A class object that acts as essentially a namespace for Security constants. A class object that acts as essentially a namespace for Security constants.
""" """
kSSLSessionOptionBreakOnServerAuth = 0 kSSLSessionOptionBreakOnServerAuth = 0
kSSLProtocol2 = 1 kSSLProtocol2 = 1

52
lib/urllib3/contrib/_securetransport/low_level.py

@ -66,22 +66,18 @@ def _cf_string_to_unicode(value):
value_as_void_p = ctypes.cast(value, ctypes.POINTER(ctypes.c_void_p)) value_as_void_p = ctypes.cast(value, ctypes.POINTER(ctypes.c_void_p))
string = CoreFoundation.CFStringGetCStringPtr( string = CoreFoundation.CFStringGetCStringPtr(
value_as_void_p, value_as_void_p, CFConst.kCFStringEncodingUTF8
CFConst.kCFStringEncodingUTF8
) )
if string is None: if string is None:
buffer = ctypes.create_string_buffer(1024) buffer = ctypes.create_string_buffer(1024)
result = CoreFoundation.CFStringGetCString( result = CoreFoundation.CFStringGetCString(
value_as_void_p, value_as_void_p, buffer, 1024, CFConst.kCFStringEncodingUTF8
buffer,
1024,
CFConst.kCFStringEncodingUTF8
) )
if not result: if not result:
raise OSError('Error copying C string from CFStringRef') raise OSError("Error copying C string from CFStringRef")
string = buffer.value string = buffer.value
if string is not None: if string is not None:
string = string.decode('utf-8') string = string.decode("utf-8")
return string return string
@ -97,8 +93,8 @@ def _assert_no_error(error, exception_class=None):
output = _cf_string_to_unicode(cf_error_string) output = _cf_string_to_unicode(cf_error_string)
CoreFoundation.CFRelease(cf_error_string) CoreFoundation.CFRelease(cf_error_string)
if output is None or output == u'': if output is None or output == u"":
output = u'OSStatus %s' % error output = u"OSStatus %s" % error
if exception_class is None: if exception_class is None:
exception_class = ssl.SSLError exception_class = ssl.SSLError
@ -115,8 +111,7 @@ def _cert_array_from_pem(pem_bundle):
pem_bundle = pem_bundle.replace(b"\r\n", b"\n") pem_bundle = pem_bundle.replace(b"\r\n", b"\n")
der_certs = [ der_certs = [
base64.b64decode(match.group(1)) base64.b64decode(match.group(1)) for match in _PEM_CERTS_RE.finditer(pem_bundle)
for match in _PEM_CERTS_RE.finditer(pem_bundle)
] ]
if not der_certs: if not der_certs:
raise ssl.SSLError("No root certificates specified") raise ssl.SSLError("No root certificates specified")
@ -124,7 +119,7 @@ def _cert_array_from_pem(pem_bundle):
cert_array = CoreFoundation.CFArrayCreateMutable( cert_array = CoreFoundation.CFArrayCreateMutable(
CoreFoundation.kCFAllocatorDefault, CoreFoundation.kCFAllocatorDefault,
0, 0,
ctypes.byref(CoreFoundation.kCFTypeArrayCallBacks) ctypes.byref(CoreFoundation.kCFTypeArrayCallBacks),
) )
if not cert_array: if not cert_array:
raise ssl.SSLError("Unable to allocate memory!") raise ssl.SSLError("Unable to allocate memory!")
@ -186,21 +181,16 @@ def _temporary_keychain():
# some random bytes to password-protect the keychain we're creating, so we # some random bytes to password-protect the keychain we're creating, so we
# ask for 40 random bytes. # ask for 40 random bytes.
random_bytes = os.urandom(40) random_bytes = os.urandom(40)
filename = base64.b16encode(random_bytes[:8]).decode('utf-8') filename = base64.b16encode(random_bytes[:8]).decode("utf-8")
password = base64.b16encode(random_bytes[8:]) # Must be valid UTF-8 password = base64.b16encode(random_bytes[8:]) # Must be valid UTF-8
tempdirectory = tempfile.mkdtemp() tempdirectory = tempfile.mkdtemp()
keychain_path = os.path.join(tempdirectory, filename).encode('utf-8') keychain_path = os.path.join(tempdirectory, filename).encode("utf-8")
# We now want to create the keychain itself. # We now want to create the keychain itself.
keychain = Security.SecKeychainRef() keychain = Security.SecKeychainRef()
status = Security.SecKeychainCreate( status = Security.SecKeychainCreate(
keychain_path, keychain_path, len(password), password, False, None, ctypes.byref(keychain)
len(password),
password,
False,
None,
ctypes.byref(keychain)
) )
_assert_no_error(status) _assert_no_error(status)
@ -219,14 +209,12 @@ def _load_items_from_file(keychain, path):
identities = [] identities = []
result_array = None result_array = None
with open(path, 'rb') as f: with open(path, "rb") as f:
raw_filedata = f.read() raw_filedata = f.read()
try: try:
filedata = CoreFoundation.CFDataCreate( filedata = CoreFoundation.CFDataCreate(
CoreFoundation.kCFAllocatorDefault, CoreFoundation.kCFAllocatorDefault, raw_filedata, len(raw_filedata)
raw_filedata,
len(raw_filedata)
) )
result_array = CoreFoundation.CFArrayRef() result_array = CoreFoundation.CFArrayRef()
result = Security.SecItemImport( result = Security.SecItemImport(
@ -237,7 +225,7 @@ def _load_items_from_file(keychain, path):
0, # import flags 0, # import flags
None, # key params, can include passphrase in the future None, # key params, can include passphrase in the future
keychain, # The keychain to insert into keychain, # The keychain to insert into
ctypes.byref(result_array) # Results ctypes.byref(result_array), # Results
) )
_assert_no_error(result) _assert_no_error(result)
@ -247,9 +235,7 @@ def _load_items_from_file(keychain, path):
# keychain already has them! # keychain already has them!
result_count = CoreFoundation.CFArrayGetCount(result_array) result_count = CoreFoundation.CFArrayGetCount(result_array)
for index in range(result_count): for index in range(result_count):
item = CoreFoundation.CFArrayGetValueAtIndex( item = CoreFoundation.CFArrayGetValueAtIndex(result_array, index)
result_array, index
)
item = ctypes.cast(item, CoreFoundation.CFTypeRef) item = ctypes.cast(item, CoreFoundation.CFTypeRef)
if _is_cert(item): if _is_cert(item):
@ -307,9 +293,7 @@ def _load_client_cert_chain(keychain, *paths):
try: try:
for file_path in paths: for file_path in paths:
new_identities, new_certs = _load_items_from_file( new_identities, new_certs = _load_items_from_file(keychain, file_path)
keychain, file_path
)
identities.extend(new_identities) identities.extend(new_identities)
certificates.extend(new_certs) certificates.extend(new_certs)
@ -318,9 +302,7 @@ def _load_client_cert_chain(keychain, *paths):
if not identities: if not identities:
new_identity = Security.SecIdentityRef() new_identity = Security.SecIdentityRef()
status = Security.SecIdentityCreateWithCertificate( status = Security.SecIdentityCreateWithCertificate(
keychain, keychain, certificates[0], ctypes.byref(new_identity)
certificates[0],
ctypes.byref(new_identity)
) )
_assert_no_error(status) _assert_no_error(status)
identities.append(new_identity) identities.append(new_identity)

118
lib/urllib3/contrib/appengine.py

@ -50,7 +50,7 @@ from ..exceptions import (
MaxRetryError, MaxRetryError,
ProtocolError, ProtocolError,
TimeoutError, TimeoutError,
SSLError SSLError,
) )
from ..request import RequestMethods from ..request import RequestMethods
@ -96,23 +96,31 @@ class AppEngineManager(RequestMethods):
Beyond those cases, it will raise normal urllib3 errors. Beyond those cases, it will raise normal urllib3 errors.
""" """
def __init__(self, headers=None, retries=None, validate_certificate=True, def __init__(
urlfetch_retries=True): self,
headers=None,
retries=None,
validate_certificate=True,
urlfetch_retries=True,
):
if not urlfetch: if not urlfetch:
raise AppEnginePlatformError( raise AppEnginePlatformError(
"URLFetch is not available in this environment.") "URLFetch is not available in this environment."
)
if is_prod_appengine_mvms(): if is_prod_appengine_mvms():
raise AppEnginePlatformError( raise AppEnginePlatformError(
"Use normal urllib3.PoolManager instead of AppEngineManager" "Use normal urllib3.PoolManager instead of AppEngineManager"
"on Managed VMs, as using URLFetch is not necessary in " "on Managed VMs, as using URLFetch is not necessary in "
"this environment.") "this environment."
)
warnings.warn( warnings.warn(
"urllib3 is using URLFetch on Google App Engine sandbox instead " "urllib3 is using URLFetch on Google App Engine sandbox instead "
"of sockets. To use sockets directly instead of URLFetch see " "of sockets. To use sockets directly instead of URLFetch see "
"https://urllib3.readthedocs.io/en/latest/reference/urllib3.contrib.html.", "https://urllib3.readthedocs.io/en/latest/reference/urllib3.contrib.html.",
AppEnginePlatformWarning) AppEnginePlatformWarning,
)
RequestMethods.__init__(self, headers) RequestMethods.__init__(self, headers)
self.validate_certificate = validate_certificate self.validate_certificate = validate_certificate
@ -127,17 +135,22 @@ class AppEngineManager(RequestMethods):
# Return False to re-raise any potential exceptions # Return False to re-raise any potential exceptions
return False return False
def urlopen(self, method, url, body=None, headers=None, def urlopen(
retries=None, redirect=True, timeout=Timeout.DEFAULT_TIMEOUT, self,
**response_kw): method,
url,
body=None,
headers=None,
retries=None,
redirect=True,
timeout=Timeout.DEFAULT_TIMEOUT,
**response_kw
):
retries = self._get_retries(retries, redirect) retries = self._get_retries(retries, redirect)
try: try:
follow_redirects = ( follow_redirects = redirect and retries.redirect != 0 and retries.total
redirect and
retries.redirect != 0 and
retries.total)
response = urlfetch.fetch( response = urlfetch.fetch(
url, url,
payload=body, payload=body,
@ -152,44 +165,52 @@ class AppEngineManager(RequestMethods):
raise TimeoutError(self, e) raise TimeoutError(self, e)
except urlfetch.InvalidURLError as e: except urlfetch.InvalidURLError as e:
if 'too large' in str(e): if "too large" in str(e):
raise AppEnginePlatformError( raise AppEnginePlatformError(
"URLFetch request too large, URLFetch only " "URLFetch request too large, URLFetch only "
"supports requests up to 10mb in size.", e) "supports requests up to 10mb in size.",
e,
)
raise ProtocolError(e) raise ProtocolError(e)
except urlfetch.DownloadError as e: except urlfetch.DownloadError as e:
if 'Too many redirects' in str(e): if "Too many redirects" in str(e):
raise MaxRetryError(self, url, reason=e) raise MaxRetryError(self, url, reason=e)
raise ProtocolError(e) raise ProtocolError(e)
except urlfetch.ResponseTooLargeError as e: except urlfetch.ResponseTooLargeError as e:
raise AppEnginePlatformError( raise AppEnginePlatformError(
"URLFetch response too large, URLFetch only supports" "URLFetch response too large, URLFetch only supports"
"responses up to 32mb in size.", e) "responses up to 32mb in size.",
e,
)
except urlfetch.SSLCertificateError as e: except urlfetch.SSLCertificateError as e:
raise SSLError(e) raise SSLError(e)
except urlfetch.InvalidMethodError as e: except urlfetch.InvalidMethodError as e:
raise AppEnginePlatformError( raise AppEnginePlatformError(
"URLFetch does not support method: %s" % method, e) "URLFetch does not support method: %s" % method, e
)
http_response = self._urlfetch_response_to_http_response( http_response = self._urlfetch_response_to_http_response(
response, retries=retries, **response_kw) response, retries=retries, **response_kw
)
# Handle redirect? # Handle redirect?
redirect_location = redirect and http_response.get_redirect_location() redirect_location = redirect and http_response.get_redirect_location()
if redirect_location: if redirect_location:
# Check for redirect response # Check for redirect response
if (self.urlfetch_retries and retries.raise_on_redirect): if self.urlfetch_retries and retries.raise_on_redirect:
raise MaxRetryError(self, url, "too many redirects") raise MaxRetryError(self, url, "too many redirects")
else: else:
if http_response.status == 303: if http_response.status == 303:
method = 'GET' method = "GET"
try: try:
retries = retries.increment(method, url, response=http_response, _pool=self) retries = retries.increment(
method, url, response=http_response, _pool=self
)
except MaxRetryError: except MaxRetryError:
if retries.raise_on_redirect: if retries.raise_on_redirect:
raise MaxRetryError(self, url, "too many redirects") raise MaxRetryError(self, url, "too many redirects")
@ -199,22 +220,32 @@ class AppEngineManager(RequestMethods):
log.debug("Redirecting %s -> %s", url, redirect_location) log.debug("Redirecting %s -> %s", url, redirect_location)
redirect_url = urljoin(url, redirect_location) redirect_url = urljoin(url, redirect_location)
return self.urlopen( return self.urlopen(
method, redirect_url, body, headers, method,
retries=retries, redirect=redirect, redirect_url,
timeout=timeout, **response_kw) body,
headers,
retries=retries,
redirect=redirect,
timeout=timeout,
**response_kw
)
# Check if we should retry the HTTP response. # Check if we should retry the HTTP response.
has_retry_after = bool(http_response.getheader('Retry-After')) has_retry_after = bool(http_response.getheader("Retry-After"))
if retries.is_retry(method, http_response.status, has_retry_after): if retries.is_retry(method, http_response.status, has_retry_after):
retries = retries.increment( retries = retries.increment(method, url, response=http_response, _pool=self)
method, url, response=http_response, _pool=self)
log.debug("Retry: %s", url) log.debug("Retry: %s", url)
retries.sleep(http_response) retries.sleep(http_response)
return self.urlopen( return self.urlopen(
method, url, method,
body=body, headers=headers, url,
retries=retries, redirect=redirect, body=body,
timeout=timeout, **response_kw) headers=headers,
retries=retries,
redirect=redirect,
timeout=timeout,
**response_kw
)
return http_response return http_response
@ -223,18 +254,18 @@ class AppEngineManager(RequestMethods):
if is_prod_appengine(): if is_prod_appengine():
# Production GAE handles deflate encoding automatically, but does # Production GAE handles deflate encoding automatically, but does
# not remove the encoding header. # not remove the encoding header.
content_encoding = urlfetch_resp.headers.get('content-encoding') content_encoding = urlfetch_resp.headers.get("content-encoding")
if content_encoding == 'deflate': if content_encoding == "deflate":
del urlfetch_resp.headers['content-encoding'] del urlfetch_resp.headers["content-encoding"]
transfer_encoding = urlfetch_resp.headers.get('transfer-encoding') transfer_encoding = urlfetch_resp.headers.get("transfer-encoding")
# We have a full response's content, # We have a full response's content,
# so let's make sure we don't report ourselves as chunked data. # so let's make sure we don't report ourselves as chunked data.
if transfer_encoding == 'chunked': if transfer_encoding == "chunked":
encodings = transfer_encoding.split(",") encodings = transfer_encoding.split(",")
encodings.remove('chunked') encodings.remove("chunked")
urlfetch_resp.headers['transfer-encoding'] = ','.join(encodings) urlfetch_resp.headers["transfer-encoding"] = ",".join(encodings)
original_response = HTTPResponse( original_response = HTTPResponse(
# In order for decoding to work, we must present the content as # In order for decoding to work, we must present the content as
@ -262,20 +293,21 @@ class AppEngineManager(RequestMethods):
warnings.warn( warnings.warn(
"URLFetch does not support granular timeout settings, " "URLFetch does not support granular timeout settings, "
"reverting to total or default URLFetch timeout.", "reverting to total or default URLFetch timeout.",
AppEnginePlatformWarning) AppEnginePlatformWarning,
)
return timeout.total return timeout.total
return timeout return timeout
def _get_retries(self, retries, redirect): def _get_retries(self, retries, redirect):
if not isinstance(retries, Retry): if not isinstance(retries, Retry):
retries = Retry.from_int( retries = Retry.from_int(retries, redirect=redirect, default=self.retries)
retries, redirect=redirect, default=self.retries)
if retries.connect or retries.read or retries.redirect: if retries.connect or retries.read or retries.redirect:
warnings.warn( warnings.warn(
"URLFetch only supports total retries and does not " "URLFetch only supports total retries and does not "
"recognize connect, read, or redirect retry parameters.", "recognize connect, read, or redirect retry parameters.",
AppEnginePlatformWarning) AppEnginePlatformWarning,
)
return retries return retries

98
lib/urllib3/contrib/ntlmpool.py

@ -20,7 +20,7 @@ class NTLMConnectionPool(HTTPSConnectionPool):
Implements an NTLM authentication version of an urllib3 connection pool Implements an NTLM authentication version of an urllib3 connection pool
""" """
scheme = 'https' scheme = "https"
def __init__(self, user, pw, authurl, *args, **kwargs): def __init__(self, user, pw, authurl, *args, **kwargs):
""" """
@ -31,7 +31,7 @@ class NTLMConnectionPool(HTTPSConnectionPool):
super(NTLMConnectionPool, self).__init__(*args, **kwargs) super(NTLMConnectionPool, self).__init__(*args, **kwargs)
self.authurl = authurl self.authurl = authurl
self.rawuser = user self.rawuser = user
user_parts = user.split('\\', 1) user_parts = user.split("\\", 1)
self.domain = user_parts[0].upper() self.domain = user_parts[0].upper()
self.user = user_parts[1] self.user = user_parts[1]
self.pw = pw self.pw = pw
@ -40,72 +40,84 @@ class NTLMConnectionPool(HTTPSConnectionPool):
# Performs the NTLM handshake that secures the connection. The socket # Performs the NTLM handshake that secures the connection. The socket
# must be kept open while requests are performed. # must be kept open while requests are performed.
self.num_connections += 1 self.num_connections += 1
log.debug('Starting NTLM HTTPS connection no. %d: https://%s%s', log.debug(
self.num_connections, self.host, self.authurl) "Starting NTLM HTTPS connection no. %d: https://%s%s",
self.num_connections,
self.host,
self.authurl,
)
headers = {'Connection': 'Keep-Alive'} headers = {"Connection": "Keep-Alive"}
req_header = 'Authorization' req_header = "Authorization"
resp_header = 'www-authenticate' resp_header = "www-authenticate"
conn = HTTPSConnection(host=self.host, port=self.port) conn = HTTPSConnection(host=self.host, port=self.port)
# Send negotiation message # Send negotiation message
headers[req_header] = ( headers[req_header] = "NTLM %s" % ntlm.create_NTLM_NEGOTIATE_MESSAGE(
'NTLM %s' % ntlm.create_NTLM_NEGOTIATE_MESSAGE(self.rawuser)) self.rawuser
log.debug('Request headers: %s', headers) )
conn.request('GET', self.authurl, None, headers) log.debug("Request headers: %s", headers)
conn.request("GET", self.authurl, None, headers)
res = conn.getresponse() res = conn.getresponse()
reshdr = dict(res.getheaders()) reshdr = dict(res.getheaders())
log.debug('Response status: %s %s', res.status, res.reason) log.debug("Response status: %s %s", res.status, res.reason)
log.debug('Response headers: %s', reshdr) log.debug("Response headers: %s", reshdr)
log.debug('Response data: %s [...]', res.read(100)) log.debug("Response data: %s [...]", res.read(100))
# Remove the reference to the socket, so that it can not be closed by # Remove the reference to the socket, so that it can not be closed by
# the response object (we want to keep the socket open) # the response object (we want to keep the socket open)
res.fp = None res.fp = None
# Server should respond with a challenge message # Server should respond with a challenge message
auth_header_values = reshdr[resp_header].split(', ') auth_header_values = reshdr[resp_header].split(", ")
auth_header_value = None auth_header_value = None
for s in auth_header_values: for s in auth_header_values:
if s[:5] == 'NTLM ': if s[:5] == "NTLM ":
auth_header_value = s[5:] auth_header_value = s[5:]
if auth_header_value is None: if auth_header_value is None:
raise Exception('Unexpected %s response header: %s' % raise Exception(
(resp_header, reshdr[resp_header])) "Unexpected %s response header: %s" % (resp_header, reshdr[resp_header])
)
# Send authentication message # Send authentication message
ServerChallenge, NegotiateFlags = \ ServerChallenge, NegotiateFlags = ntlm.parse_NTLM_CHALLENGE_MESSAGE(
ntlm.parse_NTLM_CHALLENGE_MESSAGE(auth_header_value) auth_header_value
auth_msg = ntlm.create_NTLM_AUTHENTICATE_MESSAGE(ServerChallenge, )
self.user, auth_msg = ntlm.create_NTLM_AUTHENTICATE_MESSAGE(
self.domain, ServerChallenge, self.user, self.domain, self.pw, NegotiateFlags
self.pw, )
NegotiateFlags) headers[req_header] = "NTLM %s" % auth_msg
headers[req_header] = 'NTLM %s' % auth_msg log.debug("Request headers: %s", headers)
log.debug('Request headers: %s', headers) conn.request("GET", self.authurl, None, headers)
conn.request('GET', self.authurl, None, headers)
res = conn.getresponse() res = conn.getresponse()
log.debug('Response status: %s %s', res.status, res.reason) log.debug("Response status: %s %s", res.status, res.reason)
log.debug('Response headers: %s', dict(res.getheaders())) log.debug("Response headers: %s", dict(res.getheaders()))
log.debug('Response data: %s [...]', res.read()[:100]) log.debug("Response data: %s [...]", res.read()[:100])
if res.status != 200: if res.status != 200:
if res.status == 401: if res.status == 401:
raise Exception('Server rejected request: wrong ' raise Exception(
'username or password') "Server rejected request: wrong " "username or password"
raise Exception('Wrong server response: %s %s' % )
(res.status, res.reason)) raise Exception("Wrong server response: %s %s" % (res.status, res.reason))
res.fp = None res.fp = None
log.debug('Connection established') log.debug("Connection established")
return conn return conn
def urlopen(self, method, url, body=None, headers=None, retries=3, def urlopen(
redirect=True, assert_same_host=True): self,
method,
url,
body=None,
headers=None,
retries=3,
redirect=True,
assert_same_host=True,
):
if headers is None: if headers is None:
headers = {} headers = {}
headers['Connection'] = 'Keep-Alive' headers["Connection"] = "Keep-Alive"
return super(NTLMConnectionPool, self).urlopen(method, url, body, return super(NTLMConnectionPool, self).urlopen(
headers, retries, method, url, body, headers, retries, redirect, assert_same_host
redirect, )
assert_same_host)

129
lib/urllib3/contrib/pyopenssl.py

@ -47,6 +47,7 @@ import OpenSSL.SSL
from cryptography import x509 from cryptography import x509
from cryptography.hazmat.backends.openssl import backend as openssl_backend from cryptography.hazmat.backends.openssl import backend as openssl_backend
from cryptography.hazmat.backends.openssl.x509 import _Certificate from cryptography.hazmat.backends.openssl.x509 import _Certificate
try: try:
from cryptography.x509 import UnsupportedExtension from cryptography.x509 import UnsupportedExtension
except ImportError: except ImportError:
@ -54,6 +55,7 @@ except ImportError:
class UnsupportedExtension(Exception): class UnsupportedExtension(Exception):
pass pass
from socket import timeout, error as SocketError from socket import timeout, error as SocketError
from io import BytesIO from io import BytesIO
@ -71,7 +73,7 @@ import sys
from .. import util from .. import util
__all__ = ['inject_into_urllib3', 'extract_from_urllib3'] __all__ = ["inject_into_urllib3", "extract_from_urllib3"]
# SNI always works. # SNI always works.
HAS_SNI = True HAS_SNI = True
@ -82,25 +84,23 @@ _openssl_versions = {
ssl.PROTOCOL_TLSv1: OpenSSL.SSL.TLSv1_METHOD, ssl.PROTOCOL_TLSv1: OpenSSL.SSL.TLSv1_METHOD,
} }
if hasattr(ssl, 'PROTOCOL_SSLv3') and hasattr(OpenSSL.SSL, 'SSLv3_METHOD'): if hasattr(ssl, "PROTOCOL_SSLv3") and hasattr(OpenSSL.SSL, "SSLv3_METHOD"):
_openssl_versions[ssl.PROTOCOL_SSLv3] = OpenSSL.SSL.SSLv3_METHOD _openssl_versions[ssl.PROTOCOL_SSLv3] = OpenSSL.SSL.SSLv3_METHOD
if hasattr(ssl, 'PROTOCOL_TLSv1_1') and hasattr(OpenSSL.SSL, 'TLSv1_1_METHOD'): if hasattr(ssl, "PROTOCOL_TLSv1_1") and hasattr(OpenSSL.SSL, "TLSv1_1_METHOD"):
_openssl_versions[ssl.PROTOCOL_TLSv1_1] = OpenSSL.SSL.TLSv1_1_METHOD _openssl_versions[ssl.PROTOCOL_TLSv1_1] = OpenSSL.SSL.TLSv1_1_METHOD
if hasattr(ssl, 'PROTOCOL_TLSv1_2') and hasattr(OpenSSL.SSL, 'TLSv1_2_METHOD'): if hasattr(ssl, "PROTOCOL_TLSv1_2") and hasattr(OpenSSL.SSL, "TLSv1_2_METHOD"):
_openssl_versions[ssl.PROTOCOL_TLSv1_2] = OpenSSL.SSL.TLSv1_2_METHOD _openssl_versions[ssl.PROTOCOL_TLSv1_2] = OpenSSL.SSL.TLSv1_2_METHOD
_stdlib_to_openssl_verify = { _stdlib_to_openssl_verify = {
ssl.CERT_NONE: OpenSSL.SSL.VERIFY_NONE, ssl.CERT_NONE: OpenSSL.SSL.VERIFY_NONE,
ssl.CERT_OPTIONAL: OpenSSL.SSL.VERIFY_PEER, ssl.CERT_OPTIONAL: OpenSSL.SSL.VERIFY_PEER,
ssl.CERT_REQUIRED: ssl.CERT_REQUIRED: OpenSSL.SSL.VERIFY_PEER
OpenSSL.SSL.VERIFY_PEER + OpenSSL.SSL.VERIFY_FAIL_IF_NO_PEER_CERT, + OpenSSL.SSL.VERIFY_FAIL_IF_NO_PEER_CERT,
} }
_openssl_to_stdlib_verify = dict( _openssl_to_stdlib_verify = dict((v, k) for k, v in _stdlib_to_openssl_verify.items())
(v, k) for k, v in _stdlib_to_openssl_verify.items()
)
# OpenSSL will only write 16K at a time # OpenSSL will only write 16K at a time
SSL_WRITE_BLOCKSIZE = 16384 SSL_WRITE_BLOCKSIZE = 16384
@ -113,7 +113,7 @@ log = logging.getLogger(__name__)
def inject_into_urllib3(): def inject_into_urllib3():
'Monkey-patch urllib3 with PyOpenSSL-backed SSL-support.' "Monkey-patch urllib3 with PyOpenSSL-backed SSL-support."
_validate_dependencies_met() _validate_dependencies_met()
@ -126,7 +126,7 @@ def inject_into_urllib3():
def extract_from_urllib3(): def extract_from_urllib3():
'Undo monkey-patching by :func:`inject_into_urllib3`.' "Undo monkey-patching by :func:`inject_into_urllib3`."
util.SSLContext = orig_util_SSLContext util.SSLContext = orig_util_SSLContext
util.ssl_.SSLContext = orig_util_SSLContext util.ssl_.SSLContext = orig_util_SSLContext
@ -143,17 +143,23 @@ def _validate_dependencies_met():
""" """
# Method added in `cryptography==1.1`; not available in older versions # Method added in `cryptography==1.1`; not available in older versions
from cryptography.x509.extensions import Extensions from cryptography.x509.extensions import Extensions
if getattr(Extensions, "get_extension_for_class", None) is None: if getattr(Extensions, "get_extension_for_class", None) is None:
raise ImportError("'cryptography' module missing required functionality. " raise ImportError(
"Try upgrading to v1.3.4 or newer.") "'cryptography' module missing required functionality. "
"Try upgrading to v1.3.4 or newer."
)
# pyOpenSSL 0.14 and above use cryptography for OpenSSL bindings. The _x509 # pyOpenSSL 0.14 and above use cryptography for OpenSSL bindings. The _x509
# attribute is only present on those versions. # attribute is only present on those versions.
from OpenSSL.crypto import X509 from OpenSSL.crypto import X509
x509 = X509() x509 = X509()
if getattr(x509, "_x509", None) is None: if getattr(x509, "_x509", None) is None:
raise ImportError("'pyOpenSSL' module missing required functionality. " raise ImportError(
"Try upgrading to v0.14 or newer.") "'pyOpenSSL' module missing required functionality. "
"Try upgrading to v0.14 or newer."
)
def _dnsname_to_stdlib(name): def _dnsname_to_stdlib(name):
@ -169,6 +175,7 @@ def _dnsname_to_stdlib(name):
If the name cannot be idna-encoded then we return None signalling that If the name cannot be idna-encoded then we return None signalling that
the name given should be skipped. the name given should be skipped.
""" """
def idna_encode(name): def idna_encode(name):
""" """
Borrowed wholesale from the Python Cryptography Project. It turns out Borrowed wholesale from the Python Cryptography Project. It turns out
@ -178,23 +185,23 @@ def _dnsname_to_stdlib(name):
import idna import idna
try: try:
for prefix in [u'*.', u'.']: for prefix in [u"*.", u"."]:
if name.startswith(prefix): if name.startswith(prefix):
name = name[len(prefix):] name = name[len(prefix) :]
return prefix.encode('ascii') + idna.encode(name) return prefix.encode("ascii") + idna.encode(name)
return idna.encode(name) return idna.encode(name)
except idna.core.IDNAError: except idna.core.IDNAError:
return None return None
# Don't send IPv6 addresses through the IDNA encoder. # Don't send IPv6 addresses through the IDNA encoder.
if ':' in name: if ":" in name:
return name return name
name = idna_encode(name) name = idna_encode(name)
if name is None: if name is None:
return None return None
elif sys.version_info >= (3, 0): elif sys.version_info >= (3, 0):
name = name.decode('utf-8') name = name.decode("utf-8")
return name return name
@ -213,14 +220,16 @@ def get_subj_alt_name(peer_cert):
# We want to find the SAN extension. Ask Cryptography to locate it (it's # We want to find the SAN extension. Ask Cryptography to locate it (it's
# faster than looping in Python) # faster than looping in Python)
try: try:
ext = cert.extensions.get_extension_for_class( ext = cert.extensions.get_extension_for_class(x509.SubjectAlternativeName).value
x509.SubjectAlternativeName
).value
except x509.ExtensionNotFound: except x509.ExtensionNotFound:
# No such extension, return the empty list. # No such extension, return the empty list.
return [] return []
except (x509.DuplicateExtension, UnsupportedExtension, except (
x509.UnsupportedGeneralNameType, UnicodeError) as e: x509.DuplicateExtension,
UnsupportedExtension,
x509.UnsupportedGeneralNameType,
UnicodeError,
) as e:
# A problem has been found with the quality of the certificate. Assume # A problem has been found with the quality of the certificate. Assume
# no SAN field is present. # no SAN field is present.
log.warning( log.warning(
@ -239,23 +248,23 @@ def get_subj_alt_name(peer_cert):
# does with certificates, and so we need to attempt to do the same. # does with certificates, and so we need to attempt to do the same.
# We also want to skip over names which cannot be idna encoded. # We also want to skip over names which cannot be idna encoded.
names = [ names = [
('DNS', name) for name in map(_dnsname_to_stdlib, ext.get_values_for_type(x509.DNSName)) ("DNS", name)
for name in map(_dnsname_to_stdlib, ext.get_values_for_type(x509.DNSName))
if name is not None if name is not None
] ]
names.extend( names.extend(
('IP Address', str(name)) ("IP Address", str(name)) for name in ext.get_values_for_type(x509.IPAddress)
for name in ext.get_values_for_type(x509.IPAddress)
) )
return names return names
class WrappedSocket(object): class WrappedSocket(object):
'''API-compatibility wrapper for Python OpenSSL's Connection-class. """API-compatibility wrapper for Python OpenSSL's Connection-class.
Note: _makefile_refs, _drop() and _reuse() are needed for the garbage Note: _makefile_refs, _drop() and _reuse() are needed for the garbage
collector of pypy. collector of pypy.
''' """
def __init__(self, connection, socket, suppress_ragged_eofs=True): def __init__(self, connection, socket, suppress_ragged_eofs=True):
self.connection = connection self.connection = connection
@ -278,18 +287,18 @@ class WrappedSocket(object):
try: try:
data = self.connection.recv(*args, **kwargs) data = self.connection.recv(*args, **kwargs)
except OpenSSL.SSL.SysCallError as e: except OpenSSL.SSL.SysCallError as e:
if self.suppress_ragged_eofs and e.args == (-1, 'Unexpected EOF'): if self.suppress_ragged_eofs and e.args == (-1, "Unexpected EOF"):
return b'' return b""
else: else:
raise SocketError(str(e)) raise SocketError(str(e))
except OpenSSL.SSL.ZeroReturnError: except OpenSSL.SSL.ZeroReturnError:
if self.connection.get_shutdown() == OpenSSL.SSL.RECEIVED_SHUTDOWN: if self.connection.get_shutdown() == OpenSSL.SSL.RECEIVED_SHUTDOWN:
return b'' return b""
else: else:
raise raise
except OpenSSL.SSL.WantReadError: except OpenSSL.SSL.WantReadError:
if not util.wait_for_read(self.socket, self.socket.gettimeout()): if not util.wait_for_read(self.socket, self.socket.gettimeout()):
raise timeout('The read operation timed out') raise timeout("The read operation timed out")
else: else:
return self.recv(*args, **kwargs) return self.recv(*args, **kwargs)
@ -303,7 +312,7 @@ class WrappedSocket(object):
try: try:
return self.connection.recv_into(*args, **kwargs) return self.connection.recv_into(*args, **kwargs)
except OpenSSL.SSL.SysCallError as e: except OpenSSL.SSL.SysCallError as e:
if self.suppress_ragged_eofs and e.args == (-1, 'Unexpected EOF'): if self.suppress_ragged_eofs and e.args == (-1, "Unexpected EOF"):
return 0 return 0
else: else:
raise SocketError(str(e)) raise SocketError(str(e))
@ -314,7 +323,7 @@ class WrappedSocket(object):
raise raise
except OpenSSL.SSL.WantReadError: except OpenSSL.SSL.WantReadError:
if not util.wait_for_read(self.socket, self.socket.gettimeout()): if not util.wait_for_read(self.socket, self.socket.gettimeout()):
raise timeout('The read operation timed out') raise timeout("The read operation timed out")
else: else:
return self.recv_into(*args, **kwargs) return self.recv_into(*args, **kwargs)
@ -339,7 +348,9 @@ class WrappedSocket(object):
def sendall(self, data): def sendall(self, data):
total_sent = 0 total_sent = 0
while total_sent < len(data): while total_sent < len(data):
sent = self._send_until_done(data[total_sent:total_sent + SSL_WRITE_BLOCKSIZE]) sent = self._send_until_done(
data[total_sent : total_sent + SSL_WRITE_BLOCKSIZE]
)
total_sent += sent total_sent += sent
def shutdown(self): def shutdown(self):
@ -363,15 +374,11 @@ class WrappedSocket(object):
return x509 return x509
if binary_form: if binary_form:
return OpenSSL.crypto.dump_certificate( return OpenSSL.crypto.dump_certificate(OpenSSL.crypto.FILETYPE_ASN1, x509)
OpenSSL.crypto.FILETYPE_ASN1,
x509)
return { return {
'subject': ( "subject": ((("commonName", x509.get_subject().CN),),),
(('commonName', x509.get_subject().CN),), "subjectAltName": get_subj_alt_name(x509),
),
'subjectAltName': get_subj_alt_name(x509)
} }
def version(self): def version(self):
@ -388,9 +395,12 @@ class WrappedSocket(object):
if _fileobject: # Platform-specific: Python 2 if _fileobject: # Platform-specific: Python 2
def makefile(self, mode, bufsize=-1): def makefile(self, mode, bufsize=-1):
self._makefile_refs += 1 self._makefile_refs += 1
return _fileobject(self, mode, bufsize, close=True) return _fileobject(self, mode, bufsize, close=True)
else: # Platform-specific: Python 3 else: # Platform-specific: Python 3
makefile = backport_makefile makefile = backport_makefile
@ -403,6 +413,7 @@ class PyOpenSSLContext(object):
for translating the interface of the standard library ``SSLContext`` object for translating the interface of the standard library ``SSLContext`` object
to calls into PyOpenSSL. to calls into PyOpenSSL.
""" """
def __init__(self, protocol): def __init__(self, protocol):
self.protocol = _openssl_versions[protocol] self.protocol = _openssl_versions[protocol]
self._ctx = OpenSSL.SSL.Context(self.protocol) self._ctx = OpenSSL.SSL.Context(self.protocol)
@ -424,24 +435,21 @@ class PyOpenSSLContext(object):
@verify_mode.setter @verify_mode.setter
def verify_mode(self, value): def verify_mode(self, value):
self._ctx.set_verify( self._ctx.set_verify(_stdlib_to_openssl_verify[value], _verify_callback)
_stdlib_to_openssl_verify[value],
_verify_callback
)
def set_default_verify_paths(self): def set_default_verify_paths(self):
self._ctx.set_default_verify_paths() self._ctx.set_default_verify_paths()
def set_ciphers(self, ciphers): def set_ciphers(self, ciphers):
if isinstance(ciphers, six.text_type): if isinstance(ciphers, six.text_type):
ciphers = ciphers.encode('utf-8') ciphers = ciphers.encode("utf-8")
self._ctx.set_cipher_list(ciphers) self._ctx.set_cipher_list(ciphers)
def load_verify_locations(self, cafile=None, capath=None, cadata=None): def load_verify_locations(self, cafile=None, capath=None, cadata=None):
if cafile is not None: if cafile is not None:
cafile = cafile.encode('utf-8') cafile = cafile.encode("utf-8")
if capath is not None: if capath is not None:
capath = capath.encode('utf-8') capath = capath.encode("utf-8")
self._ctx.load_verify_locations(cafile, capath) self._ctx.load_verify_locations(cafile, capath)
if cadata is not None: if cadata is not None:
self._ctx.load_verify_locations(BytesIO(cadata)) self._ctx.load_verify_locations(BytesIO(cadata))
@ -450,17 +458,22 @@ class PyOpenSSLContext(object):
self._ctx.use_certificate_chain_file(certfile) self._ctx.use_certificate_chain_file(certfile)
if password is not None: if password is not None:
if not isinstance(password, six.binary_type): if not isinstance(password, six.binary_type):
password = password.encode('utf-8') password = password.encode("utf-8")
self._ctx.set_passwd_cb(lambda *_: password) self._ctx.set_passwd_cb(lambda *_: password)
self._ctx.use_privatekey_file(keyfile or certfile) self._ctx.use_privatekey_file(keyfile or certfile)
def wrap_socket(self, sock, server_side=False, def wrap_socket(
do_handshake_on_connect=True, suppress_ragged_eofs=True, self,
server_hostname=None): sock,
server_side=False,
do_handshake_on_connect=True,
suppress_ragged_eofs=True,
server_hostname=None,
):
cnx = OpenSSL.SSL.Connection(self._ctx, sock) cnx = OpenSSL.SSL.Connection(self._ctx, sock)
if isinstance(server_hostname, six.text_type): # Platform-specific: Python 3 if isinstance(server_hostname, six.text_type): # Platform-specific: Python 3
server_hostname = server_hostname.encode('utf-8') server_hostname = server_hostname.encode("utf-8")
if server_hostname is not None: if server_hostname is not None:
cnx.set_tlsext_host_name(server_hostname) cnx.set_tlsext_host_name(server_hostname)
@ -472,10 +485,10 @@ class PyOpenSSLContext(object):
cnx.do_handshake() cnx.do_handshake()
except OpenSSL.SSL.WantReadError: except OpenSSL.SSL.WantReadError:
if not util.wait_for_read(sock, sock.gettimeout()): if not util.wait_for_read(sock, sock.gettimeout()):
raise timeout('select timed out') raise timeout("select timed out")
continue continue
except OpenSSL.SSL.Error as e: except OpenSSL.SSL.Error as e:
raise ssl.SSLError('bad handshake: %r' % e) raise ssl.SSLError("bad handshake: %r" % e)
break break
return WrappedSocket(cnx, sock) return WrappedSocket(cnx, sock)

157
lib/urllib3/contrib/securetransport.py

@ -62,12 +62,12 @@ import threading
import weakref import weakref
from .. import util from .. import util
from ._securetransport.bindings import ( from ._securetransport.bindings import Security, SecurityConst, CoreFoundation
Security, SecurityConst, CoreFoundation
)
from ._securetransport.low_level import ( from ._securetransport.low_level import (
_assert_no_error, _cert_array_from_pem, _temporary_keychain, _assert_no_error,
_load_client_cert_chain _cert_array_from_pem,
_temporary_keychain,
_load_client_cert_chain,
) )
try: # Platform-specific: Python 2 try: # Platform-specific: Python 2
@ -76,7 +76,7 @@ except ImportError: # Platform-specific: Python 3
_fileobject = None _fileobject = None
from ..packages.backports.makefile import backport_makefile from ..packages.backports.makefile import backport_makefile
__all__ = ['inject_into_urllib3', 'extract_from_urllib3'] __all__ = ["inject_into_urllib3", "extract_from_urllib3"]
# SNI always works # SNI always works
HAS_SNI = True HAS_SNI = True
@ -147,28 +147,36 @@ CIPHER_SUITES = [
# TLSv1 and a high of TLSv1.3. For everything else, we pin to that version. # TLSv1 and a high of TLSv1.3. For everything else, we pin to that version.
# TLSv1 to 1.2 are supported on macOS 10.8+ and TLSv1.3 is macOS 10.13+ # TLSv1 to 1.2 are supported on macOS 10.8+ and TLSv1.3 is macOS 10.13+
_protocol_to_min_max = { _protocol_to_min_max = {
util.PROTOCOL_TLS: (SecurityConst.kTLSProtocol1, SecurityConst.kTLSProtocolMaxSupported), util.PROTOCOL_TLS: (
SecurityConst.kTLSProtocol1,
SecurityConst.kTLSProtocolMaxSupported,
)
} }
if hasattr(ssl, "PROTOCOL_SSLv2"): if hasattr(ssl, "PROTOCOL_SSLv2"):
_protocol_to_min_max[ssl.PROTOCOL_SSLv2] = ( _protocol_to_min_max[ssl.PROTOCOL_SSLv2] = (
SecurityConst.kSSLProtocol2, SecurityConst.kSSLProtocol2 SecurityConst.kSSLProtocol2,
SecurityConst.kSSLProtocol2,
) )
if hasattr(ssl, "PROTOCOL_SSLv3"): if hasattr(ssl, "PROTOCOL_SSLv3"):
_protocol_to_min_max[ssl.PROTOCOL_SSLv3] = ( _protocol_to_min_max[ssl.PROTOCOL_SSLv3] = (
SecurityConst.kSSLProtocol3, SecurityConst.kSSLProtocol3 SecurityConst.kSSLProtocol3,
SecurityConst.kSSLProtocol3,
) )
if hasattr(ssl, "PROTOCOL_TLSv1"): if hasattr(ssl, "PROTOCOL_TLSv1"):
_protocol_to_min_max[ssl.PROTOCOL_TLSv1] = ( _protocol_to_min_max[ssl.PROTOCOL_TLSv1] = (
SecurityConst.kTLSProtocol1, SecurityConst.kTLSProtocol1 SecurityConst.kTLSProtocol1,
SecurityConst.kTLSProtocol1,
) )
if hasattr(ssl, "PROTOCOL_TLSv1_1"): if hasattr(ssl, "PROTOCOL_TLSv1_1"):
_protocol_to_min_max[ssl.PROTOCOL_TLSv1_1] = ( _protocol_to_min_max[ssl.PROTOCOL_TLSv1_1] = (
SecurityConst.kTLSProtocol11, SecurityConst.kTLSProtocol11 SecurityConst.kTLSProtocol11,
SecurityConst.kTLSProtocol11,
) )
if hasattr(ssl, "PROTOCOL_TLSv1_2"): if hasattr(ssl, "PROTOCOL_TLSv1_2"):
_protocol_to_min_max[ssl.PROTOCOL_TLSv1_2] = ( _protocol_to_min_max[ssl.PROTOCOL_TLSv1_2] = (
SecurityConst.kTLSProtocol12, SecurityConst.kTLSProtocol12 SecurityConst.kTLSProtocol12,
SecurityConst.kTLSProtocol12,
) )
@ -218,7 +226,7 @@ def _read_callback(connection_id, data_buffer, data_length_pointer):
while read_count < requested_length: while read_count < requested_length:
if timeout is None or timeout >= 0: if timeout is None or timeout >= 0:
if not util.wait_for_read(base_socket, timeout): if not util.wait_for_read(base_socket, timeout):
raise socket.error(errno.EAGAIN, 'timed out') raise socket.error(errno.EAGAIN, "timed out")
remaining = requested_length - read_count remaining = requested_length - read_count
buffer = (ctypes.c_char * remaining).from_address( buffer = (ctypes.c_char * remaining).from_address(
@ -274,7 +282,7 @@ def _write_callback(connection_id, data_buffer, data_length_pointer):
while sent < bytes_to_write: while sent < bytes_to_write:
if timeout is None or timeout >= 0: if timeout is None or timeout >= 0:
if not util.wait_for_write(base_socket, timeout): if not util.wait_for_write(base_socket, timeout):
raise socket.error(errno.EAGAIN, 'timed out') raise socket.error(errno.EAGAIN, "timed out")
chunk_sent = base_socket.send(data) chunk_sent = base_socket.send(data)
sent += chunk_sent sent += chunk_sent
@ -316,6 +324,7 @@ class WrappedSocket(object):
Note: _makefile_refs, _drop(), and _reuse() are needed for the garbage Note: _makefile_refs, _drop(), and _reuse() are needed for the garbage
collector of PyPy. collector of PyPy.
""" """
def __init__(self, socket): def __init__(self, socket):
self.socket = socket self.socket = socket
self.context = None self.context = None
@ -380,7 +389,7 @@ class WrappedSocket(object):
# We want data in memory, so load it up. # We want data in memory, so load it up.
if os.path.isfile(trust_bundle): if os.path.isfile(trust_bundle):
with open(trust_bundle, 'rb') as f: with open(trust_bundle, "rb") as f:
trust_bundle = f.read() trust_bundle = f.read()
cert_array = None cert_array = None
@ -394,9 +403,7 @@ class WrappedSocket(object):
# created for this connection, shove our CAs into it, tell ST to # created for this connection, shove our CAs into it, tell ST to
# ignore everything else it knows, and then ask if it can build a # ignore everything else it knows, and then ask if it can build a
# chain. This is a buuuunch of code. # chain. This is a buuuunch of code.
result = Security.SSLCopyPeerTrust( result = Security.SSLCopyPeerTrust(self.context, ctypes.byref(trust))
self.context, ctypes.byref(trust)
)
_assert_no_error(result) _assert_no_error(result)
if not trust: if not trust:
raise ssl.SSLError("Failed to copy trust reference") raise ssl.SSLError("Failed to copy trust reference")
@ -408,9 +415,7 @@ class WrappedSocket(object):
_assert_no_error(result) _assert_no_error(result)
trust_result = Security.SecTrustResultType() trust_result = Security.SecTrustResultType()
result = Security.SecTrustEvaluate( result = Security.SecTrustEvaluate(trust, ctypes.byref(trust_result))
trust, ctypes.byref(trust_result)
)
_assert_no_error(result) _assert_no_error(result)
finally: finally:
if trust: if trust:
@ -422,23 +427,24 @@ class WrappedSocket(object):
# Ok, now we can look at what the result was. # Ok, now we can look at what the result was.
successes = ( successes = (
SecurityConst.kSecTrustResultUnspecified, SecurityConst.kSecTrustResultUnspecified,
SecurityConst.kSecTrustResultProceed SecurityConst.kSecTrustResultProceed,
) )
if trust_result.value not in successes: if trust_result.value not in successes:
raise ssl.SSLError( raise ssl.SSLError(
"certificate verify failed, error code: %d" % "certificate verify failed, error code: %d" % trust_result.value
trust_result.value
) )
def handshake(self, def handshake(
server_hostname, self,
verify, server_hostname,
trust_bundle, verify,
min_version, trust_bundle,
max_version, min_version,
client_cert, max_version,
client_key, client_cert,
client_key_passphrase): client_key,
client_key_passphrase,
):
""" """
Actually performs the TLS handshake. This is run automatically by Actually performs the TLS handshake. This is run automatically by
wrapped socket, and shouldn't be needed in user code. wrapped socket, and shouldn't be needed in user code.
@ -468,7 +474,7 @@ class WrappedSocket(object):
# If we have a server hostname, we should set that too. # If we have a server hostname, we should set that too.
if server_hostname: if server_hostname:
if not isinstance(server_hostname, bytes): if not isinstance(server_hostname, bytes):
server_hostname = server_hostname.encode('utf-8') server_hostname = server_hostname.encode("utf-8")
result = Security.SSLSetPeerDomainName( result = Security.SSLSetPeerDomainName(
self.context, server_hostname, len(server_hostname) self.context, server_hostname, len(server_hostname)
@ -488,7 +494,9 @@ class WrappedSocket(object):
# was added in macOS 10.13 along with kTLSProtocol13. # was added in macOS 10.13 along with kTLSProtocol13.
result = Security.SSLSetProtocolVersionMax(self.context, max_version) result = Security.SSLSetProtocolVersionMax(self.context, max_version)
if result != 0 and max_version == SecurityConst.kTLSProtocolMaxSupported: if result != 0 and max_version == SecurityConst.kTLSProtocolMaxSupported:
result = Security.SSLSetProtocolVersionMax(self.context, SecurityConst.kTLSProtocol12) result = Security.SSLSetProtocolVersionMax(
self.context, SecurityConst.kTLSProtocol12
)
_assert_no_error(result) _assert_no_error(result)
# If there's a trust DB, we need to use it. We do that by telling # If there's a trust DB, we need to use it. We do that by telling
@ -497,9 +505,7 @@ class WrappedSocket(object):
# authing in that case. # authing in that case.
if not verify or trust_bundle is not None: if not verify or trust_bundle is not None:
result = Security.SSLSetSessionOption( result = Security.SSLSetSessionOption(
self.context, self.context, SecurityConst.kSSLSessionOptionBreakOnServerAuth, True
SecurityConst.kSSLSessionOptionBreakOnServerAuth,
True
) )
_assert_no_error(result) _assert_no_error(result)
@ -509,9 +515,7 @@ class WrappedSocket(object):
self._client_cert_chain = _load_client_cert_chain( self._client_cert_chain = _load_client_cert_chain(
self._keychain, client_cert, client_key self._keychain, client_cert, client_key
) )
result = Security.SSLSetCertificate( result = Security.SSLSetCertificate(self.context, self._client_cert_chain)
self.context, self._client_cert_chain
)
_assert_no_error(result) _assert_no_error(result)
while True: while True:
@ -562,7 +566,7 @@ class WrappedSocket(object):
# There are some result codes that we want to treat as "not always # There are some result codes that we want to treat as "not always
# errors". Specifically, those are errSSLWouldBlock, # errors". Specifically, those are errSSLWouldBlock,
# errSSLClosedGraceful, and errSSLClosedNoNotify. # errSSLClosedGraceful, and errSSLClosedNoNotify.
if (result == SecurityConst.errSSLWouldBlock): if result == SecurityConst.errSSLWouldBlock:
# If we didn't process any bytes, then this was just a time out. # If we didn't process any bytes, then this was just a time out.
# However, we can get errSSLWouldBlock in situations when we *did* # However, we can get errSSLWouldBlock in situations when we *did*
# read some data, and in those cases we should just read "short" # read some data, and in those cases we should just read "short"
@ -570,7 +574,10 @@ class WrappedSocket(object):
if processed_bytes.value == 0: if processed_bytes.value == 0:
# Timed out, no data read. # Timed out, no data read.
raise socket.timeout("recv timed out") raise socket.timeout("recv timed out")
elif result in (SecurityConst.errSSLClosedGraceful, SecurityConst.errSSLClosedNoNotify): elif result in (
SecurityConst.errSSLClosedGraceful,
SecurityConst.errSSLClosedNoNotify,
):
# The remote peer has closed this connection. We should do so as # The remote peer has closed this connection. We should do so as
# well. Note that we don't actually return here because in # well. Note that we don't actually return here because in
# principle this could actually be fired along with return data. # principle this could actually be fired along with return data.
@ -609,7 +616,7 @@ class WrappedSocket(object):
def sendall(self, data): def sendall(self, data):
total_sent = 0 total_sent = 0
while total_sent < len(data): while total_sent < len(data):
sent = self.send(data[total_sent:total_sent + SSL_WRITE_BLOCKSIZE]) sent = self.send(data[total_sent : total_sent + SSL_WRITE_BLOCKSIZE])
total_sent += sent total_sent += sent
def shutdown(self): def shutdown(self):
@ -656,18 +663,14 @@ class WrappedSocket(object):
# instead to just flag to urllib3 that it shouldn't do its own hostname # instead to just flag to urllib3 that it shouldn't do its own hostname
# validation when using SecureTransport. # validation when using SecureTransport.
if not binary_form: if not binary_form:
raise ValueError( raise ValueError("SecureTransport only supports dumping binary certs")
"SecureTransport only supports dumping binary certs"
)
trust = Security.SecTrustRef() trust = Security.SecTrustRef()
certdata = None certdata = None
der_bytes = None der_bytes = None
try: try:
# Grab the trust store. # Grab the trust store.
result = Security.SSLCopyPeerTrust( result = Security.SSLCopyPeerTrust(self.context, ctypes.byref(trust))
self.context, ctypes.byref(trust)
)
_assert_no_error(result) _assert_no_error(result)
if not trust: if not trust:
# Probably we haven't done the handshake yet. No biggie. # Probably we haven't done the handshake yet. No biggie.
@ -699,22 +702,24 @@ class WrappedSocket(object):
def version(self): def version(self):
protocol = Security.SSLProtocol() protocol = Security.SSLProtocol()
result = Security.SSLGetNegotiatedProtocolVersion(self.context, ctypes.byref(protocol)) result = Security.SSLGetNegotiatedProtocolVersion(
self.context, ctypes.byref(protocol)
)
_assert_no_error(result) _assert_no_error(result)
if protocol.value == SecurityConst.kTLSProtocol13: if protocol.value == SecurityConst.kTLSProtocol13:
return 'TLSv1.3' return "TLSv1.3"
elif protocol.value == SecurityConst.kTLSProtocol12: elif protocol.value == SecurityConst.kTLSProtocol12:
return 'TLSv1.2' return "TLSv1.2"
elif protocol.value == SecurityConst.kTLSProtocol11: elif protocol.value == SecurityConst.kTLSProtocol11:
return 'TLSv1.1' return "TLSv1.1"
elif protocol.value == SecurityConst.kTLSProtocol1: elif protocol.value == SecurityConst.kTLSProtocol1:
return 'TLSv1' return "TLSv1"
elif protocol.value == SecurityConst.kSSLProtocol3: elif protocol.value == SecurityConst.kSSLProtocol3:
return 'SSLv3' return "SSLv3"
elif protocol.value == SecurityConst.kSSLProtocol2: elif protocol.value == SecurityConst.kSSLProtocol2:
return 'SSLv2' return "SSLv2"
else: else:
raise ssl.SSLError('Unknown TLS version: %r' % protocol) raise ssl.SSLError("Unknown TLS version: %r" % protocol)
def _reuse(self): def _reuse(self):
self._makefile_refs += 1 self._makefile_refs += 1
@ -727,16 +732,21 @@ class WrappedSocket(object):
if _fileobject: # Platform-specific: Python 2 if _fileobject: # Platform-specific: Python 2
def makefile(self, mode, bufsize=-1): def makefile(self, mode, bufsize=-1):
self._makefile_refs += 1 self._makefile_refs += 1
return _fileobject(self, mode, bufsize, close=True) return _fileobject(self, mode, bufsize, close=True)
else: # Platform-specific: Python 3 else: # Platform-specific: Python 3
def makefile(self, mode="r", buffering=None, *args, **kwargs): def makefile(self, mode="r", buffering=None, *args, **kwargs):
# We disable buffering with SecureTransport because it conflicts with # We disable buffering with SecureTransport because it conflicts with
# the buffering that ST does internally (see issue #1153 for more). # the buffering that ST does internally (see issue #1153 for more).
buffering = 0 buffering = 0
return backport_makefile(self, mode, buffering, *args, **kwargs) return backport_makefile(self, mode, buffering, *args, **kwargs)
WrappedSocket.makefile = makefile WrappedSocket.makefile = makefile
@ -746,6 +756,7 @@ class SecureTransportContext(object):
interface of the standard library ``SSLContext`` object to calls into interface of the standard library ``SSLContext`` object to calls into
SecureTransport. SecureTransport.
""" """
def __init__(self, protocol): def __init__(self, protocol):
self._min_version, self._max_version = _protocol_to_min_max[protocol] self._min_version, self._max_version = _protocol_to_min_max[protocol]
self._options = 0 self._options = 0
@ -812,16 +823,12 @@ class SecureTransportContext(object):
def set_ciphers(self, ciphers): def set_ciphers(self, ciphers):
# For now, we just require the default cipher string. # For now, we just require the default cipher string.
if ciphers != util.ssl_.DEFAULT_CIPHERS: if ciphers != util.ssl_.DEFAULT_CIPHERS:
raise ValueError( raise ValueError("SecureTransport doesn't support custom cipher strings")
"SecureTransport doesn't support custom cipher strings"
)
def load_verify_locations(self, cafile=None, capath=None, cadata=None): def load_verify_locations(self, cafile=None, capath=None, cadata=None):
# OK, we only really support cadata and cafile. # OK, we only really support cadata and cafile.
if capath is not None: if capath is not None:
raise ValueError( raise ValueError("SecureTransport does not support cert directories")
"SecureTransport does not support cert directories"
)
self._trust_bundle = cafile or cadata self._trust_bundle = cafile or cadata
@ -830,9 +837,14 @@ class SecureTransportContext(object):
self._client_key = keyfile self._client_key = keyfile
self._client_cert_passphrase = password self._client_cert_passphrase = password
def wrap_socket(self, sock, server_side=False, def wrap_socket(
do_handshake_on_connect=True, suppress_ragged_eofs=True, self,
server_hostname=None): sock,
server_side=False,
do_handshake_on_connect=True,
suppress_ragged_eofs=True,
server_hostname=None,
):
# So, what do we do here? Firstly, we assert some properties. This is a # So, what do we do here? Firstly, we assert some properties. This is a
# stripped down shim, so there is some functionality we don't support. # stripped down shim, so there is some functionality we don't support.
# See PEP 543 for the real deal. # See PEP 543 for the real deal.
@ -846,8 +858,13 @@ class SecureTransportContext(object):
# Now we can handshake # Now we can handshake
wrapped_socket.handshake( wrapped_socket.handshake(
server_hostname, self._verify, self._trust_bundle, server_hostname,
self._min_version, self._max_version, self._client_cert, self._verify,
self._client_key, self._client_key_passphrase self._trust_bundle,
self._min_version,
self._max_version,
self._client_cert,
self._client_key,
self._client_key_passphrase,
) )
return wrapped_socket return wrapped_socket

101
lib/urllib3/contrib/socks.py

@ -42,23 +42,20 @@ except ImportError:
import warnings import warnings
from ..exceptions import DependencyWarning from ..exceptions import DependencyWarning
warnings.warn(( warnings.warn(
'SOCKS support in urllib3 requires the installation of optional ' (
'dependencies: specifically, PySocks. For more information, see ' "SOCKS support in urllib3 requires the installation of optional "
'https://urllib3.readthedocs.io/en/latest/contrib.html#socks-proxies' "dependencies: specifically, PySocks. For more information, see "
"https://urllib3.readthedocs.io/en/latest/contrib.html#socks-proxies"
), ),
DependencyWarning DependencyWarning,
) )
raise raise
from socket import error as SocketError, timeout as SocketTimeout from socket import error as SocketError, timeout as SocketTimeout
from ..connection import ( from ..connection import HTTPConnection, HTTPSConnection
HTTPConnection, HTTPSConnection from ..connectionpool import HTTPConnectionPool, HTTPSConnectionPool
)
from ..connectionpool import (
HTTPConnectionPool, HTTPSConnectionPool
)
from ..exceptions import ConnectTimeoutError, NewConnectionError from ..exceptions import ConnectTimeoutError, NewConnectionError
from ..poolmanager import PoolManager from ..poolmanager import PoolManager
from ..util.url import parse_url from ..util.url import parse_url
@ -73,8 +70,9 @@ class SOCKSConnection(HTTPConnection):
""" """
A plain-text HTTP connection that connects via a SOCKS proxy. A plain-text HTTP connection that connects via a SOCKS proxy.
""" """
def __init__(self, *args, **kwargs): def __init__(self, *args, **kwargs):
self._socks_options = kwargs.pop('_socks_options') self._socks_options = kwargs.pop("_socks_options")
super(SOCKSConnection, self).__init__(*args, **kwargs) super(SOCKSConnection, self).__init__(*args, **kwargs)
def _new_conn(self): def _new_conn(self):
@ -83,28 +81,30 @@ class SOCKSConnection(HTTPConnection):
""" """
extra_kw = {} extra_kw = {}
if self.source_address: if self.source_address:
extra_kw['source_address'] = self.source_address extra_kw["source_address"] = self.source_address
if self.socket_options: if self.socket_options:
extra_kw['socket_options'] = self.socket_options extra_kw["socket_options"] = self.socket_options
try: try:
conn = socks.create_connection( conn = socks.create_connection(
(self.host, self.port), (self.host, self.port),
proxy_type=self._socks_options['socks_version'], proxy_type=self._socks_options["socks_version"],
proxy_addr=self._socks_options['proxy_host'], proxy_addr=self._socks_options["proxy_host"],
proxy_port=self._socks_options['proxy_port'], proxy_port=self._socks_options["proxy_port"],
proxy_username=self._socks_options['username'], proxy_username=self._socks_options["username"],
proxy_password=self._socks_options['password'], proxy_password=self._socks_options["password"],
proxy_rdns=self._socks_options['rdns'], proxy_rdns=self._socks_options["rdns"],
timeout=self.timeout, timeout=self.timeout,
**extra_kw **extra_kw
) )
except SocketTimeout: except SocketTimeout:
raise ConnectTimeoutError( raise ConnectTimeoutError(
self, "Connection to %s timed out. (connect timeout=%s)" % self,
(self.host, self.timeout)) "Connection to %s timed out. (connect timeout=%s)"
% (self.host, self.timeout),
)
except socks.ProxyError as e: except socks.ProxyError as e:
# This is fragile as hell, but it seems to be the only way to raise # This is fragile as hell, but it seems to be the only way to raise
@ -114,23 +114,22 @@ class SOCKSConnection(HTTPConnection):
if isinstance(error, SocketTimeout): if isinstance(error, SocketTimeout):
raise ConnectTimeoutError( raise ConnectTimeoutError(
self, self,
"Connection to %s timed out. (connect timeout=%s)" % "Connection to %s timed out. (connect timeout=%s)"
(self.host, self.timeout) % (self.host, self.timeout),
) )
else: else:
raise NewConnectionError( raise NewConnectionError(
self, self, "Failed to establish a new connection: %s" % error
"Failed to establish a new connection: %s" % error
) )
else: else:
raise NewConnectionError( raise NewConnectionError(
self, self, "Failed to establish a new connection: %s" % e
"Failed to establish a new connection: %s" % e
) )
except SocketError as e: # Defensive: PySocks should catch all these. except SocketError as e: # Defensive: PySocks should catch all these.
raise NewConnectionError( raise NewConnectionError(
self, "Failed to establish a new connection: %s" % e) self, "Failed to establish a new connection: %s" % e
)
return conn return conn
@ -156,47 +155,53 @@ class SOCKSProxyManager(PoolManager):
A version of the urllib3 ProxyManager that routes connections via the A version of the urllib3 ProxyManager that routes connections via the
defined SOCKS proxy. defined SOCKS proxy.
""" """
pool_classes_by_scheme = { pool_classes_by_scheme = {
'http': SOCKSHTTPConnectionPool, "http": SOCKSHTTPConnectionPool,
'https': SOCKSHTTPSConnectionPool, "https": SOCKSHTTPSConnectionPool,
} }
def __init__(self, proxy_url, username=None, password=None, def __init__(
num_pools=10, headers=None, **connection_pool_kw): self,
proxy_url,
username=None,
password=None,
num_pools=10,
headers=None,
**connection_pool_kw
):
parsed = parse_url(proxy_url) parsed = parse_url(proxy_url)
if username is None and password is None and parsed.auth is not None: if username is None and password is None and parsed.auth is not None:
split = parsed.auth.split(':') split = parsed.auth.split(":")
if len(split) == 2: if len(split) == 2:
username, password = split username, password = split
if parsed.scheme == 'socks5': if parsed.scheme == "socks5":
socks_version = socks.PROXY_TYPE_SOCKS5 socks_version = socks.PROXY_TYPE_SOCKS5
rdns = False rdns = False
elif parsed.scheme == 'socks5h': elif parsed.scheme == "socks5h":
socks_version = socks.PROXY_TYPE_SOCKS5 socks_version = socks.PROXY_TYPE_SOCKS5
rdns = True rdns = True
elif parsed.scheme == 'socks4': elif parsed.scheme == "socks4":
socks_version = socks.PROXY_TYPE_SOCKS4 socks_version = socks.PROXY_TYPE_SOCKS4
rdns = False rdns = False
elif parsed.scheme == 'socks4a': elif parsed.scheme == "socks4a":
socks_version = socks.PROXY_TYPE_SOCKS4 socks_version = socks.PROXY_TYPE_SOCKS4
rdns = True rdns = True
else: else:
raise ValueError( raise ValueError("Unable to determine SOCKS version from %s" % proxy_url)
"Unable to determine SOCKS version from %s" % proxy_url
)
self.proxy_url = proxy_url self.proxy_url = proxy_url
socks_options = { socks_options = {
'socks_version': socks_version, "socks_version": socks_version,
'proxy_host': parsed.host, "proxy_host": parsed.host,
'proxy_port': parsed.port, "proxy_port": parsed.port,
'username': username, "username": username,
'password': password, "password": password,
'rdns': rdns "rdns": rdns,
} }
connection_pool_kw['_socks_options'] = socks_options connection_pool_kw["_socks_options"] = socks_options
super(SOCKSProxyManager, self).__init__( super(SOCKSProxyManager, self).__init__(
num_pools, headers, **connection_pool_kw num_pools, headers, **connection_pool_kw

29
lib/urllib3/exceptions.py

@ -1,7 +1,6 @@
from __future__ import absolute_import from __future__ import absolute_import
from .packages.six.moves.http_client import ( from .packages.six.moves.http_client import IncompleteRead as httplib_IncompleteRead
IncompleteRead as httplib_IncompleteRead
)
# Base Exceptions # Base Exceptions
@ -17,6 +16,7 @@ class HTTPWarning(Warning):
class PoolError(HTTPError): class PoolError(HTTPError):
"Base exception for errors caused within a pool." "Base exception for errors caused within a pool."
def __init__(self, pool, message): def __init__(self, pool, message):
self.pool = pool self.pool = pool
HTTPError.__init__(self, "%s: %s" % (pool, message)) HTTPError.__init__(self, "%s: %s" % (pool, message))
@ -28,6 +28,7 @@ class PoolError(HTTPError):
class RequestError(PoolError): class RequestError(PoolError):
"Base exception for PoolErrors that have associated URLs." "Base exception for PoolErrors that have associated URLs."
def __init__(self, pool, url, message): def __init__(self, pool, url, message):
self.url = url self.url = url
PoolError.__init__(self, pool, message) PoolError.__init__(self, pool, message)
@ -63,6 +64,7 @@ ConnectionError = ProtocolError
# Leaf Exceptions # Leaf Exceptions
class MaxRetryError(RequestError): class MaxRetryError(RequestError):
"""Raised when the maximum number of retries is exceeded. """Raised when the maximum number of retries is exceeded.
@ -76,8 +78,7 @@ class MaxRetryError(RequestError):
def __init__(self, pool, url, reason=None): def __init__(self, pool, url, reason=None):
self.reason = reason self.reason = reason
message = "Max retries exceeded with url: %s (Caused by %r)" % ( message = "Max retries exceeded with url: %s (Caused by %r)" % (url, reason)
url, reason)
RequestError.__init__(self, pool, url, message) RequestError.__init__(self, pool, url, message)
@ -93,6 +94,7 @@ class HostChangedError(RequestError):
class TimeoutStateError(HTTPError): class TimeoutStateError(HTTPError):
""" Raised when passing an invalid state to a timeout """ """ Raised when passing an invalid state to a timeout """
pass pass
@ -102,6 +104,7 @@ class TimeoutError(HTTPError):
Catching this error will catch both :exc:`ReadTimeoutErrors Catching this error will catch both :exc:`ReadTimeoutErrors
<ReadTimeoutError>` and :exc:`ConnectTimeoutErrors <ConnectTimeoutError>`. <ReadTimeoutError>` and :exc:`ConnectTimeoutErrors <ConnectTimeoutError>`.
""" """
pass pass
@ -149,8 +152,8 @@ class LocationParseError(LocationValueError):
class ResponseError(HTTPError): class ResponseError(HTTPError):
"Used as a container for an error reason supplied in a MaxRetryError." "Used as a container for an error reason supplied in a MaxRetryError."
GENERIC_ERROR = 'too many error responses' GENERIC_ERROR = "too many error responses"
SPECIFIC_ERROR = 'too many {status_code} error responses' SPECIFIC_ERROR = "too many {status_code} error responses"
class SecurityWarning(HTTPWarning): class SecurityWarning(HTTPWarning):
@ -188,6 +191,7 @@ class DependencyWarning(HTTPWarning):
Warned when an attempt is made to import a module with missing optional Warned when an attempt is made to import a module with missing optional
dependencies. dependencies.
""" """
pass pass
@ -201,6 +205,7 @@ class BodyNotHttplibCompatible(HTTPError):
Body should be httplib.HTTPResponse like (have an fp attribute which Body should be httplib.HTTPResponse like (have an fp attribute which
returns raw chunks) for read_chunked(). returns raw chunks) for read_chunked().
""" """
pass pass
@ -212,12 +217,15 @@ class IncompleteRead(HTTPError, httplib_IncompleteRead):
for `partial` to avoid creating large objects on streamed for `partial` to avoid creating large objects on streamed
reads. reads.
""" """
def __init__(self, partial, expected): def __init__(self, partial, expected):
super(IncompleteRead, self).__init__(partial, expected) super(IncompleteRead, self).__init__(partial, expected)
def __repr__(self): def __repr__(self):
return ('IncompleteRead(%i bytes read, ' return "IncompleteRead(%i bytes read, " "%i more expected)" % (
'%i more expected)' % (self.partial, self.expected)) self.partial,
self.expected,
)
class InvalidHeader(HTTPError): class InvalidHeader(HTTPError):
@ -236,8 +244,9 @@ class ProxySchemeUnknown(AssertionError, ValueError):
class HeaderParsingError(HTTPError): class HeaderParsingError(HTTPError):
"Raised by assert_header_parsing, but we convert it to a log.warning statement." "Raised by assert_header_parsing, but we convert it to a log.warning statement."
def __init__(self, defects, unparsed_data): def __init__(self, defects, unparsed_data):
message = '%s, unparsed data: %r' % (defects or 'Unknown', unparsed_data) message = "%s, unparsed data: %r" % (defects or "Unknown", unparsed_data)
super(HeaderParsingError, self).__init__(message) super(HeaderParsingError, self).__init__(message)

89
lib/urllib3/fields.py

@ -6,7 +6,7 @@ import re
from .packages import six from .packages import six
def guess_content_type(filename, default='application/octet-stream'): def guess_content_type(filename, default="application/octet-stream"):
""" """
Guess the "Content-Type" of a file. Guess the "Content-Type" of a file.
@ -41,22 +41,22 @@ def format_header_param_rfc2231(name, value):
if not any(ch in value for ch in '"\\\r\n'): if not any(ch in value for ch in '"\\\r\n'):
result = u'%s="%s"' % (name, value) result = u'%s="%s"' % (name, value)
try: try:
result.encode('ascii') result.encode("ascii")
except (UnicodeEncodeError, UnicodeDecodeError): except (UnicodeEncodeError, UnicodeDecodeError):
pass pass
else: else:
return result return result
if not six.PY3: # Python 2: if not six.PY3: # Python 2:
value = value.encode('utf-8') value = value.encode("utf-8")
# encode_rfc2231 accepts an encoded string and returns an ascii-encoded # encode_rfc2231 accepts an encoded string and returns an ascii-encoded
# string in Python 2 but accepts and returns unicode strings in Python 3 # string in Python 2 but accepts and returns unicode strings in Python 3
value = email.utils.encode_rfc2231(value, 'utf-8') value = email.utils.encode_rfc2231(value, "utf-8")
value = '%s*=%s' % (name, value) value = "%s*=%s" % (name, value)
if not six.PY3: # Python 2: if not six.PY3: # Python 2:
value = value.decode('utf-8') value = value.decode("utf-8")
return value return value
@ -69,23 +69,21 @@ _HTML5_REPLACEMENTS = {
} }
# All control characters from 0x00 to 0x1F *except* 0x1B. # All control characters from 0x00 to 0x1F *except* 0x1B.
_HTML5_REPLACEMENTS.update({ _HTML5_REPLACEMENTS.update(
six.unichr(cc): u"%{:02X}".format(cc) {
for cc six.unichr(cc): u"%{:02X}".format(cc)
in range(0x00, 0x1F+1) for cc in range(0x00, 0x1F + 1)
if cc not in (0x1B,) if cc not in (0x1B,)
}) }
)
def _replace_multiple(value, needles_and_replacements): def _replace_multiple(value, needles_and_replacements):
def replacer(match): def replacer(match):
return needles_and_replacements[match.group(0)] return needles_and_replacements[match.group(0)]
pattern = re.compile( pattern = re.compile(
r"|".join([ r"|".join([re.escape(needle) for needle in needles_and_replacements.keys()])
re.escape(needle) for needle in needles_and_replacements.keys()
])
) )
result = pattern.sub(replacer, value) result = pattern.sub(replacer, value)
@ -140,13 +138,15 @@ class RequestField(object):
An optional callable that is used to encode and format the headers. By An optional callable that is used to encode and format the headers. By
default, this is :func:`format_header_param_html5`. default, this is :func:`format_header_param_html5`.
""" """
def __init__( def __init__(
self, self,
name, name,
data, data,
filename=None, filename=None,
headers=None, headers=None,
header_formatter=format_header_param_html5): header_formatter=format_header_param_html5,
):
self._name = name self._name = name
self._filename = filename self._filename = filename
self.data = data self.data = data
@ -156,11 +156,7 @@ class RequestField(object):
self.header_formatter = header_formatter self.header_formatter = header_formatter
@classmethod @classmethod
def from_tuples( def from_tuples(cls, fieldname, value, header_formatter=format_header_param_html5):
cls,
fieldname,
value,
header_formatter=format_header_param_html5):
""" """
A :class:`~urllib3.fields.RequestField` factory from old-style tuple parameters. A :class:`~urllib3.fields.RequestField` factory from old-style tuple parameters.
@ -189,7 +185,8 @@ class RequestField(object):
data = value data = value
request_param = cls( request_param = cls(
fieldname, data, filename=filename, header_formatter=header_formatter) fieldname, data, filename=filename, header_formatter=header_formatter
)
request_param.make_multipart(content_type=content_type) request_param.make_multipart(content_type=content_type)
return request_param return request_param
@ -227,7 +224,7 @@ class RequestField(object):
if value is not None: if value is not None:
parts.append(self._render_part(name, value)) parts.append(self._render_part(name, value))
return u'; '.join(parts) return u"; ".join(parts)
def render_headers(self): def render_headers(self):
""" """
@ -235,21 +232,22 @@ class RequestField(object):
""" """
lines = [] lines = []
sort_keys = ['Content-Disposition', 'Content-Type', 'Content-Location'] sort_keys = ["Content-Disposition", "Content-Type", "Content-Location"]
for sort_key in sort_keys: for sort_key in sort_keys:
if self.headers.get(sort_key, False): if self.headers.get(sort_key, False):
lines.append(u'%s: %s' % (sort_key, self.headers[sort_key])) lines.append(u"%s: %s" % (sort_key, self.headers[sort_key]))
for header_name, header_value in self.headers.items(): for header_name, header_value in self.headers.items():
if header_name not in sort_keys: if header_name not in sort_keys:
if header_value: if header_value:
lines.append(u'%s: %s' % (header_name, header_value)) lines.append(u"%s: %s" % (header_name, header_value))
lines.append(u'\r\n') lines.append(u"\r\n")
return u'\r\n'.join(lines) return u"\r\n".join(lines)
def make_multipart(self, content_disposition=None, content_type=None, def make_multipart(
content_location=None): self, content_disposition=None, content_type=None, content_location=None
):
""" """
Makes this request field into a multipart request field. Makes this request field into a multipart request field.
@ -262,11 +260,14 @@ class RequestField(object):
The 'Content-Location' of the request body. The 'Content-Location' of the request body.
""" """
self.headers['Content-Disposition'] = content_disposition or u'form-data' self.headers["Content-Disposition"] = content_disposition or u"form-data"
self.headers['Content-Disposition'] += u'; '.join([ self.headers["Content-Disposition"] += u"; ".join(
u'', self._render_parts( [
((u'name', self._name), (u'filename', self._filename)) u"",
) self._render_parts(
]) ((u"name", self._name), (u"filename", self._filename))
self.headers['Content-Type'] = content_type ),
self.headers['Content-Location'] = content_location ]
)
self.headers["Content-Type"] = content_type
self.headers["Content-Location"] = content_location

12
lib/urllib3/filepost.py

@ -9,7 +9,7 @@ from .packages import six
from .packages.six import b from .packages.six import b
from .fields import RequestField from .fields import RequestField
writer = codecs.lookup('utf-8')[3] writer = codecs.lookup("utf-8")[3]
def choose_boundary(): def choose_boundary():
@ -18,7 +18,7 @@ def choose_boundary():
""" """
boundary = binascii.hexlify(os.urandom(16)) boundary = binascii.hexlify(os.urandom(16))
if six.PY3: if six.PY3:
boundary = boundary.decode('ascii') boundary = boundary.decode("ascii")
return boundary return boundary
@ -76,7 +76,7 @@ def encode_multipart_formdata(fields, boundary=None):
boundary = choose_boundary() boundary = choose_boundary()
for field in iter_field_objects(fields): for field in iter_field_objects(fields):
body.write(b('--%s\r\n' % (boundary))) body.write(b("--%s\r\n" % (boundary)))
writer(body).write(field.render_headers()) writer(body).write(field.render_headers())
data = field.data data = field.data
@ -89,10 +89,10 @@ def encode_multipart_formdata(fields, boundary=None):
else: else:
body.write(data) body.write(data)
body.write(b'\r\n') body.write(b"\r\n")
body.write(b('--%s--\r\n' % (boundary))) body.write(b("--%s--\r\n" % (boundary)))
content_type = str('multipart/form-data; boundary=%s' % boundary) content_type = str("multipart/form-data; boundary=%s" % boundary)
return body.getvalue(), content_type return body.getvalue(), content_type

2
lib/urllib3/packages/__init__.py

@ -2,4 +2,4 @@ from __future__ import absolute_import
from . import ssl_match_hostname from . import ssl_match_hostname
__all__ = ('ssl_match_hostname', ) __all__ = ("ssl_match_hostname",)

9
lib/urllib3/packages/backports/makefile.py

@ -11,15 +11,14 @@ import io
from socket import SocketIO from socket import SocketIO
def backport_makefile(self, mode="r", buffering=None, encoding=None, def backport_makefile(
errors=None, newline=None): self, mode="r", buffering=None, encoding=None, errors=None, newline=None
):
""" """
Backport of ``socket.makefile`` from Python 3.5. Backport of ``socket.makefile`` from Python 3.5.
""" """
if not set(mode) <= {"r", "w", "b"}: if not set(mode) <= {"r", "w", "b"}:
raise ValueError( raise ValueError("invalid mode %r (only r, w, b allowed)" % (mode,))
"invalid mode %r (only r, w, b allowed)" % (mode,)
)
writing = "w" in mode writing = "w" in mode
reading = "r" in mode or not writing reading = "r" in mode or not writing
assert reading or writing assert reading or writing

40
lib/urllib3/packages/rfc3986/__init__.py

@ -31,26 +31,26 @@ from .api import URIReference
from .api import urlparse from .api import urlparse
from .parseresult import ParseResult from .parseresult import ParseResult
__title__ = 'rfc3986' __title__ = "rfc3986"
__author__ = 'Ian Stapleton Cordasco' __author__ = "Ian Stapleton Cordasco"
__author_email__ = 'graffatcolmingov@gmail.com' __author_email__ = "graffatcolmingov@gmail.com"
__license__ = 'Apache v2.0' __license__ = "Apache v2.0"
__copyright__ = 'Copyright 2014 Rackspace' __copyright__ = "Copyright 2014 Rackspace"
__version__ = '1.3.1' __version__ = "1.3.2"
__all__ = ( __all__ = (
'ParseResult', "ParseResult",
'URIReference', "URIReference",
'IRIReference', "IRIReference",
'is_valid_uri', "is_valid_uri",
'normalize_uri', "normalize_uri",
'uri_reference', "uri_reference",
'iri_reference', "iri_reference",
'urlparse', "urlparse",
'__title__', "__title__",
'__author__', "__author__",
'__author_email__', "__author_email__",
'__license__', "__license__",
'__copyright__', "__copyright__",
'__version__', "__version__",
) )

122
lib/urllib3/packages/rfc3986/_mixin.py

@ -26,7 +26,7 @@ class URIMixin(object):
If the authority is not ``None`` and can not be parsed. If the authority is not ``None`` and can not be parsed.
""" """
if not self.authority: if not self.authority:
return {'userinfo': None, 'host': None, 'port': None} return {"userinfo": None, "host": None, "port": None}
match = self._match_subauthority() match = self._match_subauthority()
@ -40,10 +40,13 @@ class URIMixin(object):
# We had a match, now let's ensure that it is actually a valid host # We had a match, now let's ensure that it is actually a valid host
# address if it is IPv4 # address if it is IPv4
matches = match.groupdict() matches = match.groupdict()
host = matches.get('host') host = matches.get("host")
if (host and misc.IPv4_MATCHER.match(host) and not if (
validators.valid_ipv4_host_address(host)): host
and misc.IPv4_MATCHER.match(host)
and not validators.valid_ipv4_host_address(host)
):
# If we have a host, it appears to be IPv4 and it does not have # If we have a host, it appears to be IPv4 and it does not have
# valid bytes, it is an InvalidAuthority. # valid bytes, it is an InvalidAuthority.
raise exc.InvalidAuthority(self.authority.encode(self.encoding)) raise exc.InvalidAuthority(self.authority.encode(self.encoding))
@ -60,7 +63,7 @@ class URIMixin(object):
authority = self.authority_info() authority = self.authority_info()
except exc.InvalidAuthority: except exc.InvalidAuthority:
return None return None
return authority['host'] return authority["host"]
@property @property
def port(self): def port(self):
@ -69,7 +72,7 @@ class URIMixin(object):
authority = self.authority_info() authority = self.authority_info()
except exc.InvalidAuthority: except exc.InvalidAuthority:
return None return None
return authority['port'] return authority["port"]
@property @property
def userinfo(self): def userinfo(self):
@ -78,7 +81,7 @@ class URIMixin(object):
authority = self.authority_info() authority = self.authority_info()
except exc.InvalidAuthority: except exc.InvalidAuthority:
return None return None
return authority['userinfo'] return authority["userinfo"]
def is_absolute(self): def is_absolute(self):
"""Determine if this URI Reference is an absolute URI. """Determine if this URI Reference is an absolute URI.
@ -110,16 +113,18 @@ class URIMixin(object):
:returns: ``True`` if the URI is valid. ``False`` otherwise. :returns: ``True`` if the URI is valid. ``False`` otherwise.
:rtype: bool :rtype: bool
""" """
warnings.warn("Please use rfc3986.validators.Validator instead. " warnings.warn(
"This method will be eventually removed.", "Please use rfc3986.validators.Validator instead. "
DeprecationWarning) "This method will be eventually removed.",
DeprecationWarning,
)
validators = [ validators = [
(self.scheme_is_valid, kwargs.get('require_scheme', False)), (self.scheme_is_valid, kwargs.get("require_scheme", False)),
(self.authority_is_valid, kwargs.get('require_authority', False)), (self.authority_is_valid, kwargs.get("require_authority", False)),
(self.path_is_valid, kwargs.get('require_path', False)), (self.path_is_valid, kwargs.get("require_path", False)),
(self.query_is_valid, kwargs.get('require_query', False)), (self.query_is_valid, kwargs.get("require_query", False)),
(self.fragment_is_valid, kwargs.get('require_fragment', False)), (self.fragment_is_valid, kwargs.get("require_fragment", False)),
] ]
return all(v(r) for v, r in validators) return all(v(r) for v, r in validators)
def authority_is_valid(self, require=False): def authority_is_valid(self, require=False):
@ -136,18 +141,18 @@ class URIMixin(object):
:rtype: :rtype:
bool bool
""" """
warnings.warn("Please use rfc3986.validators.Validator instead. " warnings.warn(
"This method will be eventually removed.", "Please use rfc3986.validators.Validator instead. "
DeprecationWarning) "This method will be eventually removed.",
DeprecationWarning,
)
try: try:
self.authority_info() self.authority_info()
except exc.InvalidAuthority: except exc.InvalidAuthority:
return False return False
return validators.authority_is_valid( return validators.authority_is_valid(
self.authority, self.authority, host=self.host, require=require
host=self.host,
require=require,
) )
def scheme_is_valid(self, require=False): def scheme_is_valid(self, require=False):
@ -162,9 +167,11 @@ class URIMixin(object):
:returns: ``True`` if the scheme is valid. ``False`` otherwise. :returns: ``True`` if the scheme is valid. ``False`` otherwise.
:rtype: bool :rtype: bool
""" """
warnings.warn("Please use rfc3986.validators.Validator instead. " warnings.warn(
"This method will be eventually removed.", "Please use rfc3986.validators.Validator instead. "
DeprecationWarning) "This method will be eventually removed.",
DeprecationWarning,
)
return validators.scheme_is_valid(self.scheme, require) return validators.scheme_is_valid(self.scheme, require)
def path_is_valid(self, require=False): def path_is_valid(self, require=False):
@ -179,9 +186,11 @@ class URIMixin(object):
:returns: ``True`` if the path is valid. ``False`` otherwise. :returns: ``True`` if the path is valid. ``False`` otherwise.
:rtype: bool :rtype: bool
""" """
warnings.warn("Please use rfc3986.validators.Validator instead. " warnings.warn(
"This method will be eventually removed.", "Please use rfc3986.validators.Validator instead. "
DeprecationWarning) "This method will be eventually removed.",
DeprecationWarning,
)
return validators.path_is_valid(self.path, require) return validators.path_is_valid(self.path, require)
def query_is_valid(self, require=False): def query_is_valid(self, require=False):
@ -196,9 +205,11 @@ class URIMixin(object):
:returns: ``True`` if the query is valid. ``False`` otherwise. :returns: ``True`` if the query is valid. ``False`` otherwise.
:rtype: bool :rtype: bool
""" """
warnings.warn("Please use rfc3986.validators.Validator instead. " warnings.warn(
"This method will be eventually removed.", "Please use rfc3986.validators.Validator instead. "
DeprecationWarning) "This method will be eventually removed.",
DeprecationWarning,
)
return validators.query_is_valid(self.query, require) return validators.query_is_valid(self.query, require)
def fragment_is_valid(self, require=False): def fragment_is_valid(self, require=False):
@ -213,9 +224,11 @@ class URIMixin(object):
:returns: ``True`` if the fragment is valid. ``False`` otherwise. :returns: ``True`` if the fragment is valid. ``False`` otherwise.
:rtype: bool :rtype: bool
""" """
warnings.warn("Please use rfc3986.validators.Validator instead. " warnings.warn(
"This method will be eventually removed.", "Please use rfc3986.validators.Validator instead. "
DeprecationWarning) "This method will be eventually removed.",
DeprecationWarning,
)
return validators.fragment_is_valid(self.fragment, require) return validators.fragment_is_valid(self.fragment, require)
def normalized_equality(self, other_ref): def normalized_equality(self, other_ref):
@ -269,7 +282,7 @@ class URIMixin(object):
if resolving.authority is not None: if resolving.authority is not None:
target = resolving.copy_with( target = resolving.copy_with(
scheme=base_uri.scheme, scheme=base_uri.scheme,
path=normalizers.normalize_path(resolving.path) path=normalizers.normalize_path(resolving.path),
) )
else: else:
if resolving.path is None: if resolving.path is None:
@ -281,10 +294,10 @@ class URIMixin(object):
scheme=base_uri.scheme, scheme=base_uri.scheme,
authority=base_uri.authority, authority=base_uri.authority,
path=base_uri.path, path=base_uri.path,
query=query query=query,
) )
else: else:
if resolving.path.startswith('/'): if resolving.path.startswith("/"):
path = normalizers.normalize_path(resolving.path) path = normalizers.normalize_path(resolving.path)
else: else:
path = normalizers.normalize_path( path = normalizers.normalize_path(
@ -294,7 +307,7 @@ class URIMixin(object):
scheme=base_uri.scheme, scheme=base_uri.scheme,
authority=base_uri.authority, authority=base_uri.authority,
path=path, path=path,
query=resolving.query query=resolving.query,
) )
return target return target
@ -307,20 +320,25 @@ class URIMixin(object):
# See http://tools.ietf.org/html/rfc3986#section-5.3 # See http://tools.ietf.org/html/rfc3986#section-5.3
result_list = [] result_list = []
if self.scheme: if self.scheme:
result_list.extend([self.scheme, ':']) result_list.extend([self.scheme, ":"])
if self.authority: if self.authority:
result_list.extend(['//', self.authority]) result_list.extend(["//", self.authority])
if self.path: if self.path:
result_list.append(self.path) result_list.append(self.path)
if self.query is not None: if self.query is not None:
result_list.extend(['?', self.query]) result_list.extend(["?", self.query])
if self.fragment is not None: if self.fragment is not None:
result_list.extend(['#', self.fragment]) result_list.extend(["#", self.fragment])
return ''.join(result_list) return "".join(result_list)
def copy_with(self, scheme=misc.UseExisting, authority=misc.UseExisting, def copy_with(
path=misc.UseExisting, query=misc.UseExisting, self,
fragment=misc.UseExisting): scheme=misc.UseExisting,
authority=misc.UseExisting,
path=misc.UseExisting,
query=misc.UseExisting,
fragment=misc.UseExisting,
):
"""Create a copy of this reference with the new components. """Create a copy of this reference with the new components.
:param str scheme: :param str scheme:
@ -339,11 +357,11 @@ class URIMixin(object):
URIReference URIReference
""" """
attributes = { attributes = {
'scheme': scheme, "scheme": scheme,
'authority': authority, "authority": authority,
'path': path, "path": path,
'query': query, "query": query,
'fragment': fragment, "fragment": fragment,
} }
for key, value in list(attributes.items()): for key, value in list(attributes.items()):
if value is misc.UseExisting: if value is misc.UseExisting:

219
lib/urllib3/packages/rfc3986/abnf_regexp.py

@ -24,35 +24,35 @@ SUB_DELIMITERS_SET = set(SUB_DELIMITERS)
# Escape the '*' for use in regular expressions # Escape the '*' for use in regular expressions
SUB_DELIMITERS_RE = r"!$&'()\*+,;=" SUB_DELIMITERS_RE = r"!$&'()\*+,;="
RESERVED_CHARS_SET = GENERIC_DELIMITERS_SET.union(SUB_DELIMITERS_SET) RESERVED_CHARS_SET = GENERIC_DELIMITERS_SET.union(SUB_DELIMITERS_SET)
ALPHA = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz' ALPHA = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz"
DIGIT = '0123456789' DIGIT = "0123456789"
# https://tools.ietf.org/html/rfc3986#section-2.3 # https://tools.ietf.org/html/rfc3986#section-2.3
UNRESERVED = UNRESERVED_CHARS = ALPHA + DIGIT + r'._!-' UNRESERVED = UNRESERVED_CHARS = ALPHA + DIGIT + r"._!-"
UNRESERVED_CHARS_SET = set(UNRESERVED_CHARS) UNRESERVED_CHARS_SET = set(UNRESERVED_CHARS)
NON_PCT_ENCODED_SET = RESERVED_CHARS_SET.union(UNRESERVED_CHARS_SET) NON_PCT_ENCODED_SET = RESERVED_CHARS_SET.union(UNRESERVED_CHARS_SET)
# We need to escape the '-' in this case: # We need to escape the '-' in this case:
UNRESERVED_RE = r'A-Za-z0-9._~\-' UNRESERVED_RE = r"A-Za-z0-9._~\-"
# Percent encoded character values # Percent encoded character values
PERCENT_ENCODED = PCT_ENCODED = '%[A-Fa-f0-9]{2}' PERCENT_ENCODED = PCT_ENCODED = "%[A-Fa-f0-9]{2}"
PCHAR = '([' + UNRESERVED_RE + SUB_DELIMITERS_RE + ':@]|%s)' % PCT_ENCODED PCHAR = "([" + UNRESERVED_RE + SUB_DELIMITERS_RE + ":@]|%s)" % PCT_ENCODED
# NOTE(sigmavirus24): We're going to use more strict regular expressions # NOTE(sigmavirus24): We're going to use more strict regular expressions
# than appear in Appendix B for scheme. This will prevent over-eager # than appear in Appendix B for scheme. This will prevent over-eager
# consuming of items that aren't schemes. # consuming of items that aren't schemes.
SCHEME_RE = '[a-zA-Z][a-zA-Z0-9+.-]*' SCHEME_RE = "[a-zA-Z][a-zA-Z0-9+.-]*"
_AUTHORITY_RE = '[^/?#]*' _AUTHORITY_RE = "[^/?#]*"
_PATH_RE = '[^?#]*' _PATH_RE = "[^?#]*"
_QUERY_RE = '[^#]*' _QUERY_RE = "[^#]*"
_FRAGMENT_RE = '.*' _FRAGMENT_RE = ".*"
# Extracted from http://tools.ietf.org/html/rfc3986#appendix-B # Extracted from http://tools.ietf.org/html/rfc3986#appendix-B
COMPONENT_PATTERN_DICT = { COMPONENT_PATTERN_DICT = {
'scheme': SCHEME_RE, "scheme": SCHEME_RE,
'authority': _AUTHORITY_RE, "authority": _AUTHORITY_RE,
'path': _PATH_RE, "path": _PATH_RE,
'query': _QUERY_RE, "query": _QUERY_RE,
'fragment': _FRAGMENT_RE, "fragment": _FRAGMENT_RE,
} }
# See http://tools.ietf.org/html/rfc3986#appendix-B # See http://tools.ietf.org/html/rfc3986#appendix-B
@ -61,9 +61,9 @@ COMPONENT_PATTERN_DICT = {
# modified to ignore other matches that are not important to the parsing of # modified to ignore other matches that are not important to the parsing of
# the reference so we can also simply use SRE_Match#groups. # the reference so we can also simply use SRE_Match#groups.
URL_PARSING_RE = ( URL_PARSING_RE = (
r'(?:(?P<scheme>{scheme}):)?(?://(?P<authority>{authority}))?' r"(?:(?P<scheme>{scheme}):)?(?://(?P<authority>{authority}))?"
r'(?P<path>{path})(?:\?(?P<query>{query}))?' r"(?P<path>{path})(?:\?(?P<query>{query}))?"
r'(?:#(?P<fragment>{fragment}))?' r"(?:#(?P<fragment>{fragment}))?"
).format(**COMPONENT_PATTERN_DICT) ).format(**COMPONENT_PATTERN_DICT)
@ -73,71 +73,58 @@ URL_PARSING_RE = (
# Host patterns, see: http://tools.ietf.org/html/rfc3986#section-3.2.2 # Host patterns, see: http://tools.ietf.org/html/rfc3986#section-3.2.2
# The pattern for a regular name, e.g., www.google.com, api.github.com # The pattern for a regular name, e.g., www.google.com, api.github.com
REGULAR_NAME_RE = REG_NAME = '((?:{0}|[{1}])*)'.format( REGULAR_NAME_RE = REG_NAME = "((?:{0}|[{1}])*)".format(
'%[0-9A-Fa-f]{2}', SUB_DELIMITERS_RE + UNRESERVED_RE "%[0-9A-Fa-f]{2}", SUB_DELIMITERS_RE + UNRESERVED_RE
) )
# The pattern for an IPv4 address, e.g., 192.168.255.255, 127.0.0.1, # The pattern for an IPv4 address, e.g., 192.168.255.255, 127.0.0.1,
IPv4_RE = r'([0-9]{1,3}\.){3}[0-9]{1,3}' IPv4_RE = r"([0-9]{1,3}\.){3}[0-9]{1,3}"
# Hexadecimal characters used in each piece of an IPv6 address # Hexadecimal characters used in each piece of an IPv6 address
HEXDIG_RE = '[0-9A-Fa-f]{1,4}' HEXDIG_RE = "[0-9A-Fa-f]{1,4}"
# Least-significant 32 bits of an IPv6 address # Least-significant 32 bits of an IPv6 address
LS32_RE = '({hex}:{hex}|{ipv4})'.format(hex=HEXDIG_RE, ipv4=IPv4_RE) LS32_RE = "({hex}:{hex}|{ipv4})".format(hex=HEXDIG_RE, ipv4=IPv4_RE)
# Substitutions into the following patterns for IPv6 patterns defined # Substitutions into the following patterns for IPv6 patterns defined
# http://tools.ietf.org/html/rfc3986#page-20 # http://tools.ietf.org/html/rfc3986#page-20
_subs = {'hex': HEXDIG_RE, 'ls32': LS32_RE} _subs = {"hex": HEXDIG_RE, "ls32": LS32_RE}
# Below: h16 = hexdig, see: https://tools.ietf.org/html/rfc5234 for details # Below: h16 = hexdig, see: https://tools.ietf.org/html/rfc5234 for details
# about ABNF (Augmented Backus-Naur Form) use in the comments # about ABNF (Augmented Backus-Naur Form) use in the comments
variations = [ variations = [
# 6( h16 ":" ) ls32 # 6( h16 ":" ) ls32
'(%(hex)s:){6}%(ls32)s' % _subs, "(%(hex)s:){6}%(ls32)s" % _subs,
# "::" 5( h16 ":" ) ls32 # "::" 5( h16 ":" ) ls32
'::(%(hex)s:){5}%(ls32)s' % _subs, "::(%(hex)s:){5}%(ls32)s" % _subs,
# [ h16 ] "::" 4( h16 ":" ) ls32 # [ h16 ] "::" 4( h16 ":" ) ls32
'(%(hex)s)?::(%(hex)s:){4}%(ls32)s' % _subs, "(%(hex)s)?::(%(hex)s:){4}%(ls32)s" % _subs,
# [ *1( h16 ":" ) h16 ] "::" 3( h16 ":" ) ls32 # [ *1( h16 ":" ) h16 ] "::" 3( h16 ":" ) ls32
'((%(hex)s:)?%(hex)s)?::(%(hex)s:){3}%(ls32)s' % _subs, "((%(hex)s:)?%(hex)s)?::(%(hex)s:){3}%(ls32)s" % _subs,
# [ *2( h16 ":" ) h16 ] "::" 2( h16 ":" ) ls32 # [ *2( h16 ":" ) h16 ] "::" 2( h16 ":" ) ls32
'((%(hex)s:){0,2}%(hex)s)?::(%(hex)s:){2}%(ls32)s' % _subs, "((%(hex)s:){0,2}%(hex)s)?::(%(hex)s:){2}%(ls32)s" % _subs,
# [ *3( h16 ":" ) h16 ] "::" h16 ":" ls32 # [ *3( h16 ":" ) h16 ] "::" h16 ":" ls32
'((%(hex)s:){0,3}%(hex)s)?::%(hex)s:%(ls32)s' % _subs, "((%(hex)s:){0,3}%(hex)s)?::%(hex)s:%(ls32)s" % _subs,
# [ *4( h16 ":" ) h16 ] "::" ls32 # [ *4( h16 ":" ) h16 ] "::" ls32
'((%(hex)s:){0,4}%(hex)s)?::%(ls32)s' % _subs, "((%(hex)s:){0,4}%(hex)s)?::%(ls32)s" % _subs,
# [ *5( h16 ":" ) h16 ] "::" h16 # [ *5( h16 ":" ) h16 ] "::" h16
'((%(hex)s:){0,5}%(hex)s)?::%(hex)s' % _subs, "((%(hex)s:){0,5}%(hex)s)?::%(hex)s" % _subs,
# [ *6( h16 ":" ) h16 ] "::" # [ *6( h16 ":" ) h16 ] "::"
'((%(hex)s:){0,6}%(hex)s)?::' % _subs, "((%(hex)s:){0,6}%(hex)s)?::" % _subs,
] ]
IPv6_RE = '(({0})|({1})|({2})|({3})|({4})|({5})|({6})|({7})|({8}))'.format( IPv6_RE = "(({0})|({1})|({2})|({3})|({4})|({5})|({6})|({7})|({8}))".format(*variations)
*variations
)
IPv_FUTURE_RE = r'v[0-9A-Fa-f]+\.[%s]+' % ( IPv_FUTURE_RE = r"v[0-9A-Fa-f]+\.[%s]+" % (UNRESERVED_RE + SUB_DELIMITERS_RE + ":")
UNRESERVED_RE + SUB_DELIMITERS_RE + ':'
)
# RFC 6874 Zone ID ABNF # RFC 6874 Zone ID ABNF
ZONE_ID = '(?:[' + UNRESERVED_RE + ']|' + PCT_ENCODED + ')+' ZONE_ID = "(?:[" + UNRESERVED_RE + "]|" + PCT_ENCODED + ")+"
IPv6_ADDRZ_RFC4007_RE = IPv6_RE + '(?:(?:%25|%)' + ZONE_ID + ')?' IPv6_ADDRZ_RFC4007_RE = IPv6_RE + "(?:(?:%25|%)" + ZONE_ID + ")?"
IPv6_ADDRZ_RE = IPv6_RE + '(?:%25' + ZONE_ID + ')?' IPv6_ADDRZ_RE = IPv6_RE + "(?:%25" + ZONE_ID + ")?"
IP_LITERAL_RE = r'\[({0}|{1})\]'.format( IP_LITERAL_RE = r"\[({0}|{1})\]".format(IPv6_ADDRZ_RFC4007_RE, IPv_FUTURE_RE)
IPv6_ADDRZ_RFC4007_RE,
IPv_FUTURE_RE,
)
# Pattern for matching the host piece of the authority # Pattern for matching the host piece of the authority
HOST_RE = HOST_PATTERN = '({0}|{1}|{2})'.format( HOST_RE = HOST_PATTERN = "({0}|{1}|{2})".format(REG_NAME, IPv4_RE, IP_LITERAL_RE)
REG_NAME, USERINFO_RE = "^([" + UNRESERVED_RE + SUB_DELIMITERS_RE + ":]|%s)+" % (PCT_ENCODED)
IPv4_RE, PORT_RE = "[0-9]{1,5}"
IP_LITERAL_RE,
)
USERINFO_RE = '^([' + UNRESERVED_RE + SUB_DELIMITERS_RE + ':]|%s)+' % (
PCT_ENCODED
)
PORT_RE = '[0-9]{1,5}'
# #################### # ####################
# Path Matcher Section # Path Matcher Section
@ -146,25 +133,29 @@ PORT_RE = '[0-9]{1,5}'
# See http://tools.ietf.org/html/rfc3986#section-3.3 for more information # See http://tools.ietf.org/html/rfc3986#section-3.3 for more information
# about the path patterns defined below. # about the path patterns defined below.
segments = { segments = {
'segment': PCHAR + '*', "segment": PCHAR + "*",
# Non-zero length segment # Non-zero length segment
'segment-nz': PCHAR + '+', "segment-nz": PCHAR + "+",
# Non-zero length segment without ":" # Non-zero length segment without ":"
'segment-nz-nc': PCHAR.replace(':', '') + '+' "segment-nz-nc": PCHAR.replace(":", "") + "+",
} }
# Path types taken from Section 3.3 (linked above) # Path types taken from Section 3.3 (linked above)
PATH_EMPTY = '^$' PATH_EMPTY = "^$"
PATH_ROOTLESS = '%(segment-nz)s(/%(segment)s)*' % segments PATH_ROOTLESS = "%(segment-nz)s(/%(segment)s)*" % segments
PATH_NOSCHEME = '%(segment-nz-nc)s(/%(segment)s)*' % segments PATH_NOSCHEME = "%(segment-nz-nc)s(/%(segment)s)*" % segments
PATH_ABSOLUTE = '/(%s)?' % PATH_ROOTLESS PATH_ABSOLUTE = "/(%s)?" % PATH_ROOTLESS
PATH_ABEMPTY = '(/%(segment)s)*' % segments PATH_ABEMPTY = "(/%(segment)s)*" % segments
PATH_RE = '^(%s|%s|%s|%s|%s)$' % ( PATH_RE = "^(%s|%s|%s|%s|%s)$" % (
PATH_ABEMPTY, PATH_ABSOLUTE, PATH_NOSCHEME, PATH_ROOTLESS, PATH_EMPTY PATH_ABEMPTY,
PATH_ABSOLUTE,
PATH_NOSCHEME,
PATH_ROOTLESS,
PATH_EMPTY,
) )
FRAGMENT_RE = QUERY_RE = ( FRAGMENT_RE = QUERY_RE = (
'^([/?:@' + UNRESERVED_RE + SUB_DELIMITERS_RE + ']|%s)*$' % PCT_ENCODED "^([/?:@" + UNRESERVED_RE + SUB_DELIMITERS_RE + "]|%s)*$" % PCT_ENCODED
) )
# ########################## # ##########################
@ -172,8 +163,8 @@ FRAGMENT_RE = QUERY_RE = (
# ########################## # ##########################
# See http://tools.ietf.org/html/rfc3986#section-4.2 for details # See http://tools.ietf.org/html/rfc3986#section-4.2 for details
RELATIVE_PART_RE = '(//%s%s|%s|%s|%s)' % ( RELATIVE_PART_RE = "(//%s%s|%s|%s|%s)" % (
COMPONENT_PATTERN_DICT['authority'], COMPONENT_PATTERN_DICT["authority"],
PATH_ABEMPTY, PATH_ABEMPTY,
PATH_ABSOLUTE, PATH_ABSOLUTE,
PATH_NOSCHEME, PATH_NOSCHEME,
@ -181,8 +172,8 @@ RELATIVE_PART_RE = '(//%s%s|%s|%s|%s)' % (
) )
# See http://tools.ietf.org/html/rfc3986#section-3 for definition # See http://tools.ietf.org/html/rfc3986#section-3 for definition
HIER_PART_RE = '(//%s%s|%s|%s|%s)' % ( HIER_PART_RE = "(//%s%s|%s|%s|%s)" % (
COMPONENT_PATTERN_DICT['authority'], COMPONENT_PATTERN_DICT["authority"],
PATH_ABEMPTY, PATH_ABEMPTY,
PATH_ABSOLUTE, PATH_ABSOLUTE,
PATH_ROOTLESS, PATH_ROOTLESS,
@ -195,71 +186,75 @@ HIER_PART_RE = '(//%s%s|%s|%s|%s)' % (
# Only wide-unicode gets the high-ranges of UCSCHAR # Only wide-unicode gets the high-ranges of UCSCHAR
if sys.maxunicode > 0xFFFF: # pragma: no cover if sys.maxunicode > 0xFFFF: # pragma: no cover
IPRIVATE = u'\uE000-\uF8FF\U000F0000-\U000FFFFD\U00100000-\U0010FFFD' IPRIVATE = u"\uE000-\uF8FF\U000F0000-\U000FFFFD\U00100000-\U0010FFFD"
UCSCHAR_RE = ( UCSCHAR_RE = (
u'\u00A0-\uD7FF\uF900-\uFDCF\uFDF0-\uFFEF' u"\u00A0-\uD7FF\uF900-\uFDCF\uFDF0-\uFFEF"
u'\U00010000-\U0001FFFD\U00020000-\U0002FFFD' u"\U00010000-\U0001FFFD\U00020000-\U0002FFFD"
u'\U00030000-\U0003FFFD\U00040000-\U0004FFFD' u"\U00030000-\U0003FFFD\U00040000-\U0004FFFD"
u'\U00050000-\U0005FFFD\U00060000-\U0006FFFD' u"\U00050000-\U0005FFFD\U00060000-\U0006FFFD"
u'\U00070000-\U0007FFFD\U00080000-\U0008FFFD' u"\U00070000-\U0007FFFD\U00080000-\U0008FFFD"
u'\U00090000-\U0009FFFD\U000A0000-\U000AFFFD' u"\U00090000-\U0009FFFD\U000A0000-\U000AFFFD"
u'\U000B0000-\U000BFFFD\U000C0000-\U000CFFFD' u"\U000B0000-\U000BFFFD\U000C0000-\U000CFFFD"
u'\U000D0000-\U000DFFFD\U000E1000-\U000EFFFD' u"\U000D0000-\U000DFFFD\U000E1000-\U000EFFFD"
) )
else: # pragma: no cover else: # pragma: no cover
IPRIVATE = u'\uE000-\uF8FF' IPRIVATE = u"\uE000-\uF8FF"
UCSCHAR_RE = ( UCSCHAR_RE = u"\u00A0-\uD7FF\uF900-\uFDCF\uFDF0-\uFFEF"
u'\u00A0-\uD7FF\uF900-\uFDCF\uFDF0-\uFFEF'
)
IUNRESERVED_RE = u'A-Za-z0-9\\._~\\-' + UCSCHAR_RE IUNRESERVED_RE = u"A-Za-z0-9\\._~\\-" + UCSCHAR_RE
IPCHAR = u'([' + IUNRESERVED_RE + SUB_DELIMITERS_RE + u':@]|%s)' % PCT_ENCODED IPCHAR = u"([" + IUNRESERVED_RE + SUB_DELIMITERS_RE + u":@]|%s)" % PCT_ENCODED
isegments = { isegments = {
'isegment': IPCHAR + u'*', "isegment": IPCHAR + u"*",
# Non-zero length segment # Non-zero length segment
'isegment-nz': IPCHAR + u'+', "isegment-nz": IPCHAR + u"+",
# Non-zero length segment without ":" # Non-zero length segment without ":"
'isegment-nz-nc': IPCHAR.replace(':', '') + u'+' "isegment-nz-nc": IPCHAR.replace(":", "") + u"+",
} }
IPATH_ROOTLESS = u'%(isegment-nz)s(/%(isegment)s)*' % isegments IPATH_ROOTLESS = u"%(isegment-nz)s(/%(isegment)s)*" % isegments
IPATH_NOSCHEME = u'%(isegment-nz-nc)s(/%(isegment)s)*' % isegments IPATH_NOSCHEME = u"%(isegment-nz-nc)s(/%(isegment)s)*" % isegments
IPATH_ABSOLUTE = u'/(?:%s)?' % IPATH_ROOTLESS IPATH_ABSOLUTE = u"/(?:%s)?" % IPATH_ROOTLESS
IPATH_ABEMPTY = u'(?:/%(isegment)s)*' % isegments IPATH_ABEMPTY = u"(?:/%(isegment)s)*" % isegments
IPATH_RE = u'^(?:%s|%s|%s|%s|%s)$' % ( IPATH_RE = u"^(?:%s|%s|%s|%s|%s)$" % (
IPATH_ABEMPTY, IPATH_ABSOLUTE, IPATH_NOSCHEME, IPATH_ROOTLESS, PATH_EMPTY IPATH_ABEMPTY,
IPATH_ABSOLUTE,
IPATH_NOSCHEME,
IPATH_ROOTLESS,
PATH_EMPTY,
) )
IREGULAR_NAME_RE = IREG_NAME = u'(?:{0}|[{1}])*'.format( IREGULAR_NAME_RE = IREG_NAME = u"(?:{0}|[{1}])*".format(
u'%[0-9A-Fa-f]{2}', SUB_DELIMITERS_RE + IUNRESERVED_RE u"%[0-9A-Fa-f]{2}", SUB_DELIMITERS_RE + IUNRESERVED_RE
) )
IHOST_RE = IHOST_PATTERN = u'({0}|{1}|{2})'.format( IHOST_RE = IHOST_PATTERN = u"({0}|{1}|{2})".format(IREG_NAME, IPv4_RE, IP_LITERAL_RE)
IREG_NAME,
IPv4_RE,
IP_LITERAL_RE,
)
IUSERINFO_RE = u'^(?:[' + IUNRESERVED_RE + SUB_DELIMITERS_RE + u':]|%s)+' % ( IUSERINFO_RE = (
PCT_ENCODED u"^(?:[" + IUNRESERVED_RE + SUB_DELIMITERS_RE + u":]|%s)+" % (PCT_ENCODED)
) )
IFRAGMENT_RE = (u'^(?:[/?:@' + IUNRESERVED_RE + SUB_DELIMITERS_RE IFRAGMENT_RE = (
+ u']|%s)*$' % PCT_ENCODED) u"^(?:[/?:@" + IUNRESERVED_RE + SUB_DELIMITERS_RE + u"]|%s)*$" % PCT_ENCODED
IQUERY_RE = (u'^(?:[/?:@' + IUNRESERVED_RE + SUB_DELIMITERS_RE )
+ IPRIVATE + u']|%s)*$' % PCT_ENCODED) IQUERY_RE = (
u"^(?:[/?:@"
+ IUNRESERVED_RE
+ SUB_DELIMITERS_RE
+ IPRIVATE
+ u"]|%s)*$" % PCT_ENCODED
)
IRELATIVE_PART_RE = u'(//%s%s|%s|%s|%s)' % ( IRELATIVE_PART_RE = u"(//%s%s|%s|%s|%s)" % (
COMPONENT_PATTERN_DICT['authority'], COMPONENT_PATTERN_DICT["authority"],
IPATH_ABEMPTY, IPATH_ABEMPTY,
IPATH_ABSOLUTE, IPATH_ABSOLUTE,
IPATH_NOSCHEME, IPATH_NOSCHEME,
PATH_EMPTY, PATH_EMPTY,
) )
IHIER_PART_RE = u'(//%s%s|%s|%s|%s)' % ( IHIER_PART_RE = u"(//%s%s|%s|%s|%s)" % (
COMPONENT_PATTERN_DICT['authority'], COMPONENT_PATTERN_DICT["authority"],
IPATH_ABEMPTY, IPATH_ABEMPTY,
IPATH_ABSOLUTE, IPATH_ABSOLUTE,
IPATH_ROOTLESS, IPATH_ROOTLESS,

10
lib/urllib3/packages/rfc3986/api.py

@ -24,7 +24,7 @@ from .parseresult import ParseResult
from .uri import URIReference from .uri import URIReference
def uri_reference(uri, encoding='utf-8'): def uri_reference(uri, encoding="utf-8"):
"""Parse a URI string into a URIReference. """Parse a URI string into a URIReference.
This is a convenience function. You could achieve the same end by using This is a convenience function. You could achieve the same end by using
@ -38,7 +38,7 @@ def uri_reference(uri, encoding='utf-8'):
return URIReference.from_string(uri, encoding) return URIReference.from_string(uri, encoding)
def iri_reference(iri, encoding='utf-8'): def iri_reference(iri, encoding="utf-8"):
"""Parse a IRI string into an IRIReference. """Parse a IRI string into an IRIReference.
This is a convenience function. You could achieve the same end by using This is a convenience function. You could achieve the same end by using
@ -52,7 +52,7 @@ def iri_reference(iri, encoding='utf-8'):
return IRIReference.from_string(iri, encoding) return IRIReference.from_string(iri, encoding)
def is_valid_uri(uri, encoding='utf-8', **kwargs): def is_valid_uri(uri, encoding="utf-8", **kwargs):
"""Determine if the URI given is valid. """Determine if the URI given is valid.
This is a convenience function. You could use either This is a convenience function. You could use either
@ -77,7 +77,7 @@ def is_valid_uri(uri, encoding='utf-8', **kwargs):
return URIReference.from_string(uri, encoding).is_valid(**kwargs) return URIReference.from_string(uri, encoding).is_valid(**kwargs)
def normalize_uri(uri, encoding='utf-8'): def normalize_uri(uri, encoding="utf-8"):
"""Normalize the given URI. """Normalize the given URI.
This is a convenience function. You could use either This is a convenience function. You could use either
@ -93,7 +93,7 @@ def normalize_uri(uri, encoding='utf-8'):
return normalized_reference.unsplit() return normalized_reference.unsplit()
def urlparse(uri, encoding='utf-8'): def urlparse(uri, encoding="utf-8"):
"""Parse a given URI and return a ParseResult. """Parse a given URI and return a ParseResult.
This is a partial replacement of the standard library's urlparse function. This is a partial replacement of the standard library's urlparse function.

47
lib/urllib3/packages/rfc3986/builder.py

@ -29,8 +29,16 @@ class URIBuilder(object):
""" """
def __init__(self, scheme=None, userinfo=None, host=None, port=None, def __init__(
path=None, query=None, fragment=None): self,
scheme=None,
userinfo=None,
host=None,
port=None,
path=None,
query=None,
fragment=None,
):
"""Initialize our URI builder. """Initialize our URI builder.
:param str scheme: :param str scheme:
@ -58,9 +66,11 @@ class URIBuilder(object):
def __repr__(self): def __repr__(self):
"""Provide a convenient view of our builder object.""" """Provide a convenient view of our builder object."""
formatstr = ('URIBuilder(scheme={b.scheme}, userinfo={b.userinfo}, ' formatstr = (
'host={b.host}, port={b.port}, path={b.path}, ' "URIBuilder(scheme={b.scheme}, userinfo={b.userinfo}, "
'query={b.query}, fragment={b.fragment})') "host={b.host}, port={b.port}, path={b.path}, "
"query={b.query}, fragment={b.fragment})"
)
return formatstr.format(b=self) return formatstr.format(b=self)
def add_scheme(self, scheme): def add_scheme(self, scheme):
@ -101,13 +111,12 @@ class URIBuilder(object):
port=None, path=None, query=None, fragment=None) port=None, path=None, query=None, fragment=None)
""" """
if username is None: if username is None:
raise ValueError('Username cannot be None') raise ValueError("Username cannot be None")
userinfo = normalizers.normalize_username(username) userinfo = normalizers.normalize_username(username)
if password is not None: if password is not None:
userinfo = '{}:{}'.format( userinfo = "{}:{}".format(
userinfo, userinfo, normalizers.normalize_password(password)
normalizers.normalize_password(password),
) )
return URIBuilder( return URIBuilder(
@ -157,23 +166,19 @@ class URIBuilder(object):
port_int = int(port) port_int = int(port)
if port_int < 0: if port_int < 0:
raise ValueError( raise ValueError(
'ports are not allowed to be negative. You provided {}'.format( "ports are not allowed to be negative. You provided {}".format(port_int)
port_int,
)
) )
if port_int > 65535: if port_int > 65535:
raise ValueError( raise ValueError(
'ports are not allowed to be larger than 65535. ' "ports are not allowed to be larger than 65535. "
'You provided {}'.format( "You provided {}".format(port_int)
port_int,
)
) )
return URIBuilder( return URIBuilder(
scheme=self.scheme, scheme=self.scheme,
userinfo=self.userinfo, userinfo=self.userinfo,
host=self.host, host=self.host,
port='{}'.format(port_int), port="{}".format(port_int),
path=self.path, path=self.path,
query=self.query, query=self.query,
fragment=self.fragment, fragment=self.fragment,
@ -193,8 +198,8 @@ class URIBuilder(object):
path='/checkout.php', query=None, fragment=None) path='/checkout.php', query=None, fragment=None)
""" """
if not path.startswith('/'): if not path.startswith("/"):
path = '/{}'.format(path) path = "/{}".format(path)
return URIBuilder( return URIBuilder(
scheme=self.scheme, scheme=self.scheme,
@ -289,9 +294,7 @@ class URIBuilder(object):
""" """
return uri.URIReference( return uri.URIReference(
self.scheme, self.scheme,
normalizers.normalize_authority( normalizers.normalize_authority((self.userinfo, self.host, self.port)),
(self.userinfo, self.host, self.port)
),
self.path, self.path,
self.query, self.query,
self.fragment, self.fragment,

15
lib/urllib3/packages/rfc3986/compat.py

@ -25,12 +25,7 @@ try:
except ImportError: # Python 2.x except ImportError: # Python 2.x
from urllib import urlencode from urllib import urlencode
__all__ = ( __all__ = ("to_bytes", "to_str", "urlquote", "urlencode")
'to_bytes',
'to_str',
'urlquote',
'urlencode',
)
PY3 = (3, 0) <= sys.version_info < (4, 0) PY3 = (3, 0) <= sys.version_info < (4, 0)
PY2 = (2, 6) <= sys.version_info < (2, 8) PY2 = (2, 6) <= sys.version_info < (2, 8)
@ -40,15 +35,15 @@ if PY3:
unicode = str # Python 3.x unicode = str # Python 3.x
def to_str(b, encoding='utf-8'): def to_str(b, encoding="utf-8"):
"""Ensure that b is text in the specified encoding.""" """Ensure that b is text in the specified encoding."""
if hasattr(b, 'decode') and not isinstance(b, unicode): if hasattr(b, "decode") and not isinstance(b, unicode):
b = b.decode(encoding) b = b.decode(encoding)
return b return b
def to_bytes(s, encoding='utf-8'): def to_bytes(s, encoding="utf-8"):
"""Ensure that s is converted to bytes from the encoding.""" """Ensure that s is converted to bytes from the encoding."""
if hasattr(s, 'encode') and not isinstance(s, bytes): if hasattr(s, "encode") and not isinstance(s, bytes):
s = s.encode(encoding) s = s.encode(encoding)
return s return s

38
lib/urllib3/packages/rfc3986/exceptions.py

@ -16,8 +16,8 @@ class InvalidAuthority(RFC3986Exception):
def __init__(self, authority): def __init__(self, authority):
"""Initialize the exception with the invalid authority.""" """Initialize the exception with the invalid authority."""
super(InvalidAuthority, self).__init__( super(InvalidAuthority, self).__init__(
u"The authority ({0}) is not valid.".format( u"The authority ({0}) is not valid.".format(compat.to_str(authority))
compat.to_str(authority))) )
class InvalidPort(RFC3986Exception): class InvalidPort(RFC3986Exception):
@ -25,8 +25,7 @@ class InvalidPort(RFC3986Exception):
def __init__(self, port): def __init__(self, port):
"""Initialize the exception with the invalid port.""" """Initialize the exception with the invalid port."""
super(InvalidPort, self).__init__( super(InvalidPort, self).__init__('The port ("{0}") is not valid.'.format(port))
'The port ("{0}") is not valid.'.format(port))
class ResolutionError(RFC3986Exception): class ResolutionError(RFC3986Exception):
@ -35,7 +34,8 @@ class ResolutionError(RFC3986Exception):
def __init__(self, uri): def __init__(self, uri):
"""Initialize the error with the failed URI.""" """Initialize the error with the failed URI."""
super(ResolutionError, self).__init__( super(ResolutionError, self).__init__(
"{0} is not an absolute URI.".format(uri.unsplit())) "{0} is not an absolute URI.".format(uri.unsplit())
)
class ValidationError(RFC3986Exception): class ValidationError(RFC3986Exception):
@ -49,17 +49,15 @@ class MissingComponentError(ValidationError):
def __init__(self, uri, *component_names): def __init__(self, uri, *component_names):
"""Initialize the error with the missing component name.""" """Initialize the error with the missing component name."""
verb = 'was' verb = "was"
if len(component_names) > 1: if len(component_names) > 1:
verb = 'were' verb = "were"
self.uri = uri self.uri = uri
self.components = sorted(component_names) self.components = sorted(component_names)
components = ', '.join(self.components) components = ", ".join(self.components)
super(MissingComponentError, self).__init__( super(MissingComponentError, self).__init__(
"{} {} required but missing".format(components, verb), "{} {} required but missing".format(components, verb), uri, self.components
uri,
self.components,
) )
@ -70,7 +68,7 @@ class UnpermittedComponentError(ValidationError):
"""Initialize the error with the unpermitted component.""" """Initialize the error with the unpermitted component."""
super(UnpermittedComponentError, self).__init__( super(UnpermittedComponentError, self).__init__(
"{} was required to be one of {!r} but was {!r}".format( "{} was required to be one of {!r} but was {!r}".format(
component_name, list(sorted(allowed_values)), component_value, component_name, list(sorted(allowed_values)), component_value
), ),
component_name, component_name,
component_value, component_value,
@ -86,11 +84,9 @@ class PasswordForbidden(ValidationError):
def __init__(self, uri): def __init__(self, uri):
"""Initialize the error with the URI that failed validation.""" """Initialize the error with the URI that failed validation."""
unsplit = getattr(uri, 'unsplit', lambda: uri) unsplit = getattr(uri, "unsplit", lambda: uri)
super(PasswordForbidden, self).__init__( super(PasswordForbidden, self).__init__(
'"{}" contained a password when validation forbade it'.format( '"{}" contained a password when validation forbade it'.format(unsplit())
unsplit()
)
) )
self.uri = uri self.uri = uri
@ -100,17 +96,15 @@ class InvalidComponentsError(ValidationError):
def __init__(self, uri, *component_names): def __init__(self, uri, *component_names):
"""Initialize the error with the invalid component name(s).""" """Initialize the error with the invalid component name(s)."""
verb = 'was' verb = "was"
if len(component_names) > 1: if len(component_names) > 1:
verb = 'were' verb = "were"
self.uri = uri self.uri = uri
self.components = sorted(component_names) self.components = sorted(component_names)
components = ', '.join(self.components) components = ", ".join(self.components)
super(InvalidComponentsError, self).__init__( super(InvalidComponentsError, self).__init__(
"{} {} found to be invalid".format(components, verb), "{} {} found to be invalid".format(components, verb), uri, self.components
uri,
self.components,
) )

63
lib/urllib3/packages/rfc3986/iri.py

@ -29,8 +29,7 @@ except ImportError: # pragma: no cover
idna = None idna = None
class IRIReference(namedtuple('IRIReference', misc.URI_COMPONENTS), class IRIReference(namedtuple("IRIReference", misc.URI_COMPONENTS), uri.URIMixin):
uri.URIMixin):
"""Immutable object representing a parsed IRI Reference. """Immutable object representing a parsed IRI Reference.
Can be encoded into an URIReference object via the procedure Can be encoded into an URIReference object via the procedure
@ -43,16 +42,11 @@ class IRIReference(namedtuple('IRIReference', misc.URI_COMPONENTS),
slots = () slots = ()
def __new__(cls, scheme, authority, path, query, fragment, def __new__(cls, scheme, authority, path, query, fragment, encoding="utf-8"):
encoding='utf-8'):
"""Create a new IRIReference.""" """Create a new IRIReference."""
ref = super(IRIReference, cls).__new__( ref = super(IRIReference, cls).__new__(
cls, cls, scheme or None, authority or None, path or None, query, fragment
scheme or None, )
authority or None,
path or None,
query,
fragment)
ref.encoding = encoding ref.encoding = encoding
return ref return ref
@ -66,8 +60,10 @@ class IRIReference(namedtuple('IRIReference', misc.URI_COMPONENTS),
other_ref = self.__class__.from_string(other) other_ref = self.__class__.from_string(other)
except TypeError: except TypeError:
raise TypeError( raise TypeError(
'Unable to compare {0}() to {1}()'.format( "Unable to compare {0}() to {1}()".format(
type(self).__name__, type(other).__name__)) type(self).__name__, type(other).__name__
)
)
# See http://tools.ietf.org/html/rfc3986#section-6.2 # See http://tools.ietf.org/html/rfc3986#section-6.2
return tuple(self) == tuple(other_ref) return tuple(self) == tuple(other_ref)
@ -76,7 +72,7 @@ class IRIReference(namedtuple('IRIReference', misc.URI_COMPONENTS),
return misc.ISUBAUTHORITY_MATCHER.match(self.authority) return misc.ISUBAUTHORITY_MATCHER.match(self.authority)
@classmethod @classmethod
def from_string(cls, iri_string, encoding='utf-8'): def from_string(cls, iri_string, encoding="utf-8"):
"""Parse a IRI reference from the given unicode IRI string. """Parse a IRI reference from the given unicode IRI string.
:param str iri_string: Unicode IRI to be parsed into a reference. :param str iri_string: Unicode IRI to be parsed into a reference.
@ -87,10 +83,11 @@ class IRIReference(namedtuple('IRIReference', misc.URI_COMPONENTS),
split_iri = misc.IRI_MATCHER.match(iri_string).groupdict() split_iri = misc.IRI_MATCHER.match(iri_string).groupdict()
return cls( return cls(
split_iri['scheme'], split_iri['authority'], split_iri["scheme"],
normalizers.encode_component(split_iri['path'], encoding), split_iri["authority"],
normalizers.encode_component(split_iri['query'], encoding), normalizers.encode_component(split_iri["path"], encoding),
normalizers.encode_component(split_iri['fragment'], encoding), normalizers.encode_component(split_iri["query"], encoding),
normalizers.encode_component(split_iri["fragment"], encoding),
encoding, encoding,
) )
@ -120,28 +117,34 @@ class IRIReference(namedtuple('IRIReference', misc.URI_COMPONENTS),
def idna_encoder(name): def idna_encoder(name):
if any(ord(c) > 128 for c in name): if any(ord(c) > 128 for c in name):
try: try:
return idna.encode(name.lower(), return idna.encode(
strict=True, name.lower(), strict=True, std3_rules=True
std3_rules=True) )
except idna.IDNAError: except idna.IDNAError:
raise exceptions.InvalidAuthority(self.authority) raise exceptions.InvalidAuthority(self.authority)
return name return name
authority = "" authority = ""
if self.host: if self.host:
authority = ".".join([compat.to_str(idna_encoder(part)) authority = ".".join(
for part in self.host.split(".")]) [compat.to_str(idna_encoder(part)) for part in self.host.split(".")]
)
if self.userinfo is not None: if self.userinfo is not None:
authority = (normalizers.encode_component( authority = (
self.userinfo, self.encoding) + '@' + authority) normalizers.encode_component(self.userinfo, self.encoding)
+ "@"
+ authority
)
if self.port is not None: if self.port is not None:
authority += ":" + str(self.port) authority += ":" + str(self.port)
return uri.URIReference(self.scheme, return uri.URIReference(
authority, self.scheme,
path=self.path, authority,
query=self.query, path=self.path,
fragment=self.fragment, query=self.query,
encoding=self.encoding) fragment=self.fragment,
encoding=self.encoding,
)

105
lib/urllib3/packages/rfc3986/misc.py

@ -25,16 +25,16 @@ from . import abnf_regexp
# These are enumerated for the named tuple used as a superclass of # These are enumerated for the named tuple used as a superclass of
# URIReference # URIReference
URI_COMPONENTS = ['scheme', 'authority', 'path', 'query', 'fragment'] URI_COMPONENTS = ["scheme", "authority", "path", "query", "fragment"]
important_characters = { important_characters = {
'generic_delimiters': abnf_regexp.GENERIC_DELIMITERS, "generic_delimiters": abnf_regexp.GENERIC_DELIMITERS,
'sub_delimiters': abnf_regexp.SUB_DELIMITERS, "sub_delimiters": abnf_regexp.SUB_DELIMITERS,
# We need to escape the '*' in this case # We need to escape the '*' in this case
're_sub_delimiters': abnf_regexp.SUB_DELIMITERS_RE, "re_sub_delimiters": abnf_regexp.SUB_DELIMITERS_RE,
'unreserved_chars': abnf_regexp.UNRESERVED_CHARS, "unreserved_chars": abnf_regexp.UNRESERVED_CHARS,
# We need to escape the '-' in this case: # We need to escape the '-' in this case:
're_unreserved': abnf_regexp.UNRESERVED_RE, "re_unreserved": abnf_regexp.UNRESERVED_RE,
} }
# For details about delimiters and reserved characters, see: # For details about delimiters and reserved characters, see:
@ -49,23 +49,21 @@ NON_PCT_ENCODED = abnf_regexp.NON_PCT_ENCODED_SET
URI_MATCHER = re.compile(abnf_regexp.URL_PARSING_RE) URI_MATCHER = re.compile(abnf_regexp.URL_PARSING_RE)
SUBAUTHORITY_MATCHER = re.compile(( SUBAUTHORITY_MATCHER = re.compile(
'^(?:(?P<userinfo>{0})@)?' # userinfo (
'(?P<host>{1})' # host "^(?:(?P<userinfo>{0})@)?" # userinfo
':?(?P<port>{2})?$' # port "(?P<host>{1})" # host
).format(abnf_regexp.USERINFO_RE, ":?(?P<port>{2})?$" # port
abnf_regexp.HOST_PATTERN, ).format(abnf_regexp.USERINFO_RE, abnf_regexp.HOST_PATTERN, abnf_regexp.PORT_RE)
abnf_regexp.PORT_RE)) )
HOST_MATCHER = re.compile('^' + abnf_regexp.HOST_RE + '$') HOST_MATCHER = re.compile("^" + abnf_regexp.HOST_RE + "$")
IPv4_MATCHER = re.compile('^' + abnf_regexp.IPv4_RE + '$') IPv4_MATCHER = re.compile("^" + abnf_regexp.IPv4_RE + "$")
IPv6_MATCHER = re.compile(r'^\[' + abnf_regexp.IPv6_ADDRZ_RFC4007_RE + r'\]$') IPv6_MATCHER = re.compile(r"^\[" + abnf_regexp.IPv6_ADDRZ_RFC4007_RE + r"\]$")
# Used by host validator # Used by host validator
IPv6_NO_RFC4007_MATCHER = re.compile(r'^\[%s\]$' % ( IPv6_NO_RFC4007_MATCHER = re.compile(r"^\[%s\]$" % (abnf_regexp.IPv6_ADDRZ_RE))
abnf_regexp.IPv6_ADDRZ_RE
))
# Matcher used to validate path components # Matcher used to validate path components
PATH_MATCHER = re.compile(abnf_regexp.PATH_RE) PATH_MATCHER = re.compile(abnf_regexp.PATH_RE)
@ -80,20 +78,22 @@ QUERY_MATCHER = re.compile(abnf_regexp.QUERY_RE)
FRAGMENT_MATCHER = QUERY_MATCHER FRAGMENT_MATCHER = QUERY_MATCHER
# Scheme validation, see: http://tools.ietf.org/html/rfc3986#section-3.1 # Scheme validation, see: http://tools.ietf.org/html/rfc3986#section-3.1
SCHEME_MATCHER = re.compile('^{0}$'.format(abnf_regexp.SCHEME_RE)) SCHEME_MATCHER = re.compile("^{0}$".format(abnf_regexp.SCHEME_RE))
RELATIVE_REF_MATCHER = re.compile(r'^%s(\?%s)?(#%s)?$' % ( RELATIVE_REF_MATCHER = re.compile(
abnf_regexp.RELATIVE_PART_RE, r"^%s(\?%s)?(#%s)?$"
abnf_regexp.QUERY_RE, % (abnf_regexp.RELATIVE_PART_RE, abnf_regexp.QUERY_RE, abnf_regexp.FRAGMENT_RE)
abnf_regexp.FRAGMENT_RE, )
))
# See http://tools.ietf.org/html/rfc3986#section-4.3 # See http://tools.ietf.org/html/rfc3986#section-4.3
ABSOLUTE_URI_MATCHER = re.compile(r'^%s:%s(\?%s)?$' % ( ABSOLUTE_URI_MATCHER = re.compile(
abnf_regexp.COMPONENT_PATTERN_DICT['scheme'], r"^%s:%s(\?%s)?$"
abnf_regexp.HIER_PART_RE, % (
abnf_regexp.QUERY_RE[1:-1], abnf_regexp.COMPONENT_PATTERN_DICT["scheme"],
)) abnf_regexp.HIER_PART_RE,
abnf_regexp.QUERY_RE[1:-1],
)
)
# ############### # ###############
# IRIs / RFC 3987 # IRIs / RFC 3987
@ -101,46 +101,25 @@ ABSOLUTE_URI_MATCHER = re.compile(r'^%s:%s(\?%s)?$' % (
IRI_MATCHER = re.compile(abnf_regexp.URL_PARSING_RE, re.UNICODE) IRI_MATCHER = re.compile(abnf_regexp.URL_PARSING_RE, re.UNICODE)
ISUBAUTHORITY_MATCHER = re.compile(( ISUBAUTHORITY_MATCHER = re.compile(
u'^(?:(?P<userinfo>{0})@)?' # iuserinfo (
u'(?P<host>{1})' # ihost u"^(?:(?P<userinfo>{0})@)?" # iuserinfo
u':?(?P<port>{2})?$' # port u"(?P<host>{1})" # ihost
).format(abnf_regexp.IUSERINFO_RE, u":?(?P<port>{2})?$" # port
abnf_regexp.IHOST_RE, ).format(abnf_regexp.IUSERINFO_RE, abnf_regexp.IHOST_RE, abnf_regexp.PORT_RE),
abnf_regexp.PORT_RE), re.UNICODE) re.UNICODE,
)
IHOST_MATCHER = re.compile('^' + abnf_regexp.IHOST_RE + '$', re.UNICODE)
IPATH_MATCHER = re.compile(abnf_regexp.IPATH_RE, re.UNICODE)
IQUERY_MATCHER = re.compile(abnf_regexp.IQUERY_RE, re.UNICODE)
IFRAGMENT_MATCHER = re.compile(abnf_regexp.IFRAGMENT_RE, re.UNICODE)
RELATIVE_IRI_MATCHER = re.compile(u'^%s(?:\\?%s)?(?:%s)?$' % (
abnf_regexp.IRELATIVE_PART_RE,
abnf_regexp.IQUERY_RE,
abnf_regexp.IFRAGMENT_RE
), re.UNICODE)
ABSOLUTE_IRI_MATCHER = re.compile(u'^%s:%s(?:\\?%s)?$' % (
abnf_regexp.COMPONENT_PATTERN_DICT['scheme'],
abnf_regexp.IHIER_PART_RE,
abnf_regexp.IQUERY_RE[1:-1]
), re.UNICODE)
# Path merger as defined in http://tools.ietf.org/html/rfc3986#section-5.2.3 # Path merger as defined in http://tools.ietf.org/html/rfc3986#section-5.2.3
def merge_paths(base_uri, relative_path): def merge_paths(base_uri, relative_path):
"""Merge a base URI's path with a relative URI's path.""" """Merge a base URI's path with a relative URI's path."""
if base_uri.path is None and base_uri.authority is not None: if base_uri.path is None and base_uri.authority is not None:
return '/' + relative_path return "/" + relative_path
else: else:
path = base_uri.path or '' path = base_uri.path or ""
index = path.rfind('/') index = path.rfind("/")
return path[:index] + '/' + relative_path return path[:index] + "/" + relative_path
UseExisting = object() UseExisting = object()

53
lib/urllib3/packages/rfc3986/normalizers.py

@ -27,13 +27,13 @@ def normalize_scheme(scheme):
def normalize_authority(authority): def normalize_authority(authority):
"""Normalize an authority tuple to a string.""" """Normalize an authority tuple to a string."""
userinfo, host, port = authority userinfo, host, port = authority
result = '' result = ""
if userinfo: if userinfo:
result += normalize_percent_characters(userinfo) + '@' result += normalize_percent_characters(userinfo) + "@"
if host: if host:
result += normalize_host(host) result += normalize_host(host)
if port: if port:
result += ':' + port result += ":" + port
return result return result
@ -50,16 +50,19 @@ def normalize_password(password):
def normalize_host(host): def normalize_host(host):
"""Normalize a host string.""" """Normalize a host string."""
if misc.IPv6_MATCHER.match(host): if misc.IPv6_MATCHER.match(host):
percent = host.find('%') percent = host.find("%")
if percent != -1: if percent != -1:
percent_25 = host.find('%25') percent_25 = host.find("%25")
# Replace RFC 4007 IPv6 Zone ID delimiter '%' with '%25' # Replace RFC 4007 IPv6 Zone ID delimiter '%' with '%25'
# from RFC 6874. If the host is '[<IPv6 addr>%25]' then we # from RFC 6874. If the host is '[<IPv6 addr>%25]' then we
# assume RFC 4007 and normalize to '[<IPV6 addr>%2525]' # assume RFC 4007 and normalize to '[<IPV6 addr>%2525]'
if percent_25 == -1 or percent < percent_25 or \ if (
(percent == percent_25 and percent_25 == len(host) - 4): percent_25 == -1
host = host.replace('%', '%25', 1) or percent < percent_25
or (percent == percent_25 and percent_25 == len(host) - 4)
):
host = host.replace("%", "%25", 1)
# Don't normalize the casing of the Zone ID # Don't normalize the casing of the Zone ID
return host[:percent].lower() + host[percent:] return host[:percent].lower() + host[percent:]
@ -90,7 +93,7 @@ def normalize_fragment(fragment):
return normalize_percent_characters(fragment) return normalize_percent_characters(fragment)
PERCENT_MATCHER = re.compile('%[A-Fa-f0-9]{2}') PERCENT_MATCHER = re.compile("%[A-Fa-f0-9]{2}")
def normalize_percent_characters(s): def normalize_percent_characters(s):
@ -111,15 +114,15 @@ def remove_dot_segments(s):
See also Section 5.2.4 of :rfc:`3986`. See also Section 5.2.4 of :rfc:`3986`.
""" """
# See http://tools.ietf.org/html/rfc3986#section-5.2.4 for pseudo-code # See http://tools.ietf.org/html/rfc3986#section-5.2.4 for pseudo-code
segments = s.split('/') # Turn the path into a list of segments segments = s.split("/") # Turn the path into a list of segments
output = [] # Initialize the variable to use to store output output = [] # Initialize the variable to use to store output
for segment in segments: for segment in segments:
# '.' is the current directory, so ignore it, it is superfluous # '.' is the current directory, so ignore it, it is superfluous
if segment == '.': if segment == ".":
continue continue
# Anything other than '..', should be appended to the output # Anything other than '..', should be appended to the output
elif segment != '..': elif segment != "..":
output.append(segment) output.append(segment)
# In this case segment == '..', if we can, we should pop the last # In this case segment == '..', if we can, we should pop the last
# element # element
@ -128,15 +131,15 @@ def remove_dot_segments(s):
# If the path starts with '/' and the output is empty or the first string # If the path starts with '/' and the output is empty or the first string
# is non-empty # is non-empty
if s.startswith('/') and (not output or output[0]): if s.startswith("/") and (not output or output[0]):
output.insert(0, '') output.insert(0, "")
# If the path starts with '/.' or '/..' ensure we add one more empty # If the path starts with '/.' or '/..' ensure we add one more empty
# string to add a trailing '/' # string to add a trailing '/'
if s.endswith(('/.', '/..')): if s.endswith(("/.", "/..")):
output.append('') output.append("")
return '/'.join(output) return "/".join(output)
def encode_component(uri_component, encoding): def encode_component(uri_component, encoding):
@ -146,22 +149,24 @@ def encode_component(uri_component, encoding):
# Try to see if the component we're encoding is already percent-encoded # Try to see if the component we're encoding is already percent-encoded
# so we can skip all '%' characters but still encode all others. # so we can skip all '%' characters but still encode all others.
percent_encodings = len(PERCENT_MATCHER.findall( percent_encodings = len(
compat.to_str(uri_component, encoding))) PERCENT_MATCHER.findall(compat.to_str(uri_component, encoding))
)
uri_bytes = compat.to_bytes(uri_component, encoding) uri_bytes = compat.to_bytes(uri_component, encoding)
is_percent_encoded = percent_encodings == uri_bytes.count(b'%') is_percent_encoded = percent_encodings == uri_bytes.count(b"%")
encoded_uri = bytearray() encoded_uri = bytearray()
for i in range(0, len(uri_bytes)): for i in range(0, len(uri_bytes)):
# Will return a single character bytestring on both Python 2 & 3 # Will return a single character bytestring on both Python 2 & 3
byte = uri_bytes[i:i+1] byte = uri_bytes[i : i + 1]
byte_ord = ord(byte) byte_ord = ord(byte)
if ((is_percent_encoded and byte == b'%') if (is_percent_encoded and byte == b"%") or (
or (byte_ord < 128 and byte.decode() in misc.NON_PCT_ENCODED)): byte_ord < 128 and byte.decode() in misc.NON_PCT_ENCODED
):
encoded_uri.extend(byte) encoded_uri.extend(byte)
continue continue
encoded_uri.extend('%{0:02x}'.format(byte_ord).encode().upper()) encoded_uri.extend("%{0:02x}".format(byte_ord).encode().upper())
return encoded_uri.decode(encoding) return encoded_uri.decode(encoding)

326
lib/urllib3/packages/rfc3986/parseresult.py

@ -21,27 +21,25 @@ from . import misc
from . import normalizers from . import normalizers
from . import uri from . import uri
__all__ = ('ParseResult', 'ParseResultBytes') __all__ = ("ParseResult", "ParseResultBytes")
PARSED_COMPONENTS = ('scheme', 'userinfo', 'host', 'port', 'path', 'query', PARSED_COMPONENTS = ("scheme", "userinfo", "host", "port", "path", "query", "fragment")
'fragment')
class ParseResultMixin(object): class ParseResultMixin(object):
def _generate_authority(self, attributes): def _generate_authority(self, attributes):
# I swear I did not align the comparisons below. That's just how they # I swear I did not align the comparisons below. That's just how they
# happened to align based on pep8 and attribute lengths. # happened to align based on pep8 and attribute lengths.
userinfo, host, port = (attributes[p] userinfo, host, port = (attributes[p] for p in ("userinfo", "host", "port"))
for p in ('userinfo', 'host', 'port')) if self.userinfo != userinfo or self.host != host or self.port != port:
if (self.userinfo != userinfo or
self.host != host or
self.port != port):
if port: if port:
port = '{0}'.format(port) port = "{0}".format(port)
return normalizers.normalize_authority( return normalizers.normalize_authority(
(compat.to_str(userinfo, self.encoding), (
compat.to_str(host, self.encoding), compat.to_str(userinfo, self.encoding),
port) compat.to_str(host, self.encoding),
port,
)
) )
return self.authority return self.authority
@ -65,8 +63,7 @@ class ParseResultMixin(object):
return self.query return self.query
class ParseResult(namedtuple('ParseResult', PARSED_COMPONENTS), class ParseResult(namedtuple("ParseResult", PARSED_COMPONENTS), ParseResultMixin):
ParseResultMixin):
"""Implementation of urlparse compatibility class. """Implementation of urlparse compatibility class.
This uses the URIReference logic to handle compatibility with the This uses the URIReference logic to handle compatibility with the
@ -75,8 +72,18 @@ class ParseResult(namedtuple('ParseResult', PARSED_COMPONENTS),
slots = () slots = ()
def __new__(cls, scheme, userinfo, host, port, path, query, fragment, def __new__(
uri_ref, encoding='utf-8'): cls,
scheme,
userinfo,
host,
port,
path,
query,
fragment,
uri_ref,
encoding="utf-8",
):
"""Create a new ParseResult.""" """Create a new ParseResult."""
parse_result = super(ParseResult, cls).__new__( parse_result = super(ParseResult, cls).__new__(
cls, cls,
@ -86,42 +93,57 @@ class ParseResult(namedtuple('ParseResult', PARSED_COMPONENTS),
port or None, port or None,
path or None, path or None,
query, query,
fragment) fragment,
)
parse_result.encoding = encoding parse_result.encoding = encoding
parse_result.reference = uri_ref parse_result.reference = uri_ref
return parse_result return parse_result
@classmethod @classmethod
def from_parts(cls, scheme=None, userinfo=None, host=None, port=None, def from_parts(
path=None, query=None, fragment=None, encoding='utf-8'): cls,
scheme=None,
userinfo=None,
host=None,
port=None,
path=None,
query=None,
fragment=None,
encoding="utf-8",
):
"""Create a ParseResult instance from its parts.""" """Create a ParseResult instance from its parts."""
authority = '' authority = ""
if userinfo is not None: if userinfo is not None:
authority += userinfo + '@' authority += userinfo + "@"
if host is not None: if host is not None:
authority += host authority += host
if port is not None: if port is not None:
authority += ':{0}'.format(port) authority += ":{0}".format(port)
uri_ref = uri.URIReference(scheme=scheme, uri_ref = uri.URIReference(
authority=authority, scheme=scheme,
path=path, authority=authority,
query=query, path=path,
fragment=fragment, query=query,
encoding=encoding).normalize() fragment=fragment,
encoding=encoding,
).normalize()
userinfo, host, port = authority_from(uri_ref, strict=True) userinfo, host, port = authority_from(uri_ref, strict=True)
return cls(scheme=uri_ref.scheme, return cls(
userinfo=userinfo, scheme=uri_ref.scheme,
host=host, userinfo=userinfo,
port=port, host=host,
path=uri_ref.path, port=port,
query=uri_ref.query, path=uri_ref.path,
fragment=uri_ref.fragment, query=uri_ref.query,
uri_ref=uri_ref, fragment=uri_ref.fragment,
encoding=encoding) uri_ref=uri_ref,
encoding=encoding,
)
@classmethod @classmethod
def from_string(cls, uri_string, encoding='utf-8', strict=True, def from_string(
lazy_normalize=True): cls, uri_string, encoding="utf-8", strict=True, lazy_normalize=True
):
"""Parse a URI from the given unicode URI string. """Parse a URI from the given unicode URI string.
:param str uri_string: Unicode URI to be parsed into a reference. :param str uri_string: Unicode URI to be parsed into a reference.
@ -136,53 +158,65 @@ class ParseResult(namedtuple('ParseResult', PARSED_COMPONENTS),
reference = reference.normalize() reference = reference.normalize()
userinfo, host, port = authority_from(reference, strict) userinfo, host, port = authority_from(reference, strict)
return cls(scheme=reference.scheme, return cls(
userinfo=userinfo, scheme=reference.scheme,
host=host, userinfo=userinfo,
port=port, host=host,
path=reference.path, port=port,
query=reference.query, path=reference.path,
fragment=reference.fragment, query=reference.query,
uri_ref=reference, fragment=reference.fragment,
encoding=encoding) uri_ref=reference,
encoding=encoding,
)
@property @property
def authority(self): def authority(self):
"""Return the normalized authority.""" """Return the normalized authority."""
return self.reference.authority return self.reference.authority
def copy_with(self, scheme=misc.UseExisting, userinfo=misc.UseExisting, def copy_with(
host=misc.UseExisting, port=misc.UseExisting, self,
path=misc.UseExisting, query=misc.UseExisting, scheme=misc.UseExisting,
fragment=misc.UseExisting): userinfo=misc.UseExisting,
host=misc.UseExisting,
port=misc.UseExisting,
path=misc.UseExisting,
query=misc.UseExisting,
fragment=misc.UseExisting,
):
"""Create a copy of this instance replacing with specified parts.""" """Create a copy of this instance replacing with specified parts."""
attributes = zip(PARSED_COMPONENTS, attributes = zip(
(scheme, userinfo, host, port, path, query, fragment)) PARSED_COMPONENTS, (scheme, userinfo, host, port, path, query, fragment)
)
attrs_dict = {} attrs_dict = {}
for name, value in attributes: for name, value in attributes:
if value is misc.UseExisting: if value is misc.UseExisting:
value = getattr(self, name) value = getattr(self, name)
attrs_dict[name] = value attrs_dict[name] = value
authority = self._generate_authority(attrs_dict) authority = self._generate_authority(attrs_dict)
ref = self.reference.copy_with(scheme=attrs_dict['scheme'], ref = self.reference.copy_with(
authority=authority, scheme=attrs_dict["scheme"],
path=attrs_dict['path'], authority=authority,
query=attrs_dict['query'], path=attrs_dict["path"],
fragment=attrs_dict['fragment']) query=attrs_dict["query"],
fragment=attrs_dict["fragment"],
)
return ParseResult(uri_ref=ref, encoding=self.encoding, **attrs_dict) return ParseResult(uri_ref=ref, encoding=self.encoding, **attrs_dict)
def encode(self, encoding=None): def encode(self, encoding=None):
"""Convert to an instance of ParseResultBytes.""" """Convert to an instance of ParseResultBytes."""
encoding = encoding or self.encoding encoding = encoding or self.encoding
attrs = dict( attrs = dict(
zip(PARSED_COMPONENTS, zip(
(attr.encode(encoding) if hasattr(attr, 'encode') else attr PARSED_COMPONENTS,
for attr in self))) (
return ParseResultBytes( attr.encode(encoding) if hasattr(attr, "encode") else attr
uri_ref=self.reference, for attr in self
encoding=encoding, ),
**attrs )
) )
return ParseResultBytes(uri_ref=self.reference, encoding=encoding, **attrs)
def unsplit(self, use_idna=False): def unsplit(self, use_idna=False):
"""Create a URI string from the components. """Create a URI string from the components.
@ -192,18 +226,30 @@ class ParseResult(namedtuple('ParseResult', PARSED_COMPONENTS),
""" """
parse_result = self parse_result = self
if use_idna and self.host: if use_idna and self.host:
hostbytes = self.host.encode('idna') hostbytes = self.host.encode("idna")
host = hostbytes.decode(self.encoding) host = hostbytes.decode(self.encoding)
parse_result = self.copy_with(host=host) parse_result = self.copy_with(host=host)
return parse_result.reference.unsplit() return parse_result.reference.unsplit()
class ParseResultBytes(namedtuple('ParseResultBytes', PARSED_COMPONENTS), class ParseResultBytes(
ParseResultMixin): namedtuple("ParseResultBytes", PARSED_COMPONENTS), ParseResultMixin
):
"""Compatibility shim for the urlparse.ParseResultBytes object.""" """Compatibility shim for the urlparse.ParseResultBytes object."""
def __new__(cls, scheme, userinfo, host, port, path, query, fragment, def __new__(
uri_ref, encoding='utf-8', lazy_normalize=True): cls,
scheme,
userinfo,
host,
port,
path,
query,
fragment,
uri_ref,
encoding="utf-8",
lazy_normalize=True,
):
"""Create a new ParseResultBytes instance.""" """Create a new ParseResultBytes instance."""
parse_result = super(ParseResultBytes, cls).__new__( parse_result = super(ParseResultBytes, cls).__new__(
cls, cls,
@ -213,48 +259,63 @@ class ParseResultBytes(namedtuple('ParseResultBytes', PARSED_COMPONENTS),
port or None, port or None,
path or None, path or None,
query or None, query or None,
fragment or None) fragment or None,
)
parse_result.encoding = encoding parse_result.encoding = encoding
parse_result.reference = uri_ref parse_result.reference = uri_ref
parse_result.lazy_normalize = lazy_normalize parse_result.lazy_normalize = lazy_normalize
return parse_result return parse_result
@classmethod @classmethod
def from_parts(cls, scheme=None, userinfo=None, host=None, port=None, def from_parts(
path=None, query=None, fragment=None, encoding='utf-8', cls,
lazy_normalize=True): scheme=None,
userinfo=None,
host=None,
port=None,
path=None,
query=None,
fragment=None,
encoding="utf-8",
lazy_normalize=True,
):
"""Create a ParseResult instance from its parts.""" """Create a ParseResult instance from its parts."""
authority = '' authority = ""
if userinfo is not None: if userinfo is not None:
authority += userinfo + '@' authority += userinfo + "@"
if host is not None: if host is not None:
authority += host authority += host
if port is not None: if port is not None:
authority += ':{0}'.format(int(port)) authority += ":{0}".format(int(port))
uri_ref = uri.URIReference(scheme=scheme, uri_ref = uri.URIReference(
authority=authority, scheme=scheme,
path=path, authority=authority,
query=query, path=path,
fragment=fragment, query=query,
encoding=encoding) fragment=fragment,
encoding=encoding,
)
if not lazy_normalize: if not lazy_normalize:
uri_ref = uri_ref.normalize() uri_ref = uri_ref.normalize()
to_bytes = compat.to_bytes to_bytes = compat.to_bytes
userinfo, host, port = authority_from(uri_ref, strict=True) userinfo, host, port = authority_from(uri_ref, strict=True)
return cls(scheme=to_bytes(scheme, encoding), return cls(
userinfo=to_bytes(userinfo, encoding), scheme=to_bytes(scheme, encoding),
host=to_bytes(host, encoding), userinfo=to_bytes(userinfo, encoding),
port=port, host=to_bytes(host, encoding),
path=to_bytes(path, encoding), port=port,
query=to_bytes(query, encoding), path=to_bytes(path, encoding),
fragment=to_bytes(fragment, encoding), query=to_bytes(query, encoding),
uri_ref=uri_ref, fragment=to_bytes(fragment, encoding),
encoding=encoding, uri_ref=uri_ref,
lazy_normalize=lazy_normalize) encoding=encoding,
lazy_normalize=lazy_normalize,
)
@classmethod @classmethod
def from_string(cls, uri_string, encoding='utf-8', strict=True, def from_string(
lazy_normalize=True): cls, uri_string, encoding="utf-8", strict=True, lazy_normalize=True
):
"""Parse a URI from the given unicode URI string. """Parse a URI from the given unicode URI string.
:param str uri_string: Unicode URI to be parsed into a reference. :param str uri_string: Unicode URI to be parsed into a reference.
@ -270,44 +331,54 @@ class ParseResultBytes(namedtuple('ParseResultBytes', PARSED_COMPONENTS),
userinfo, host, port = authority_from(reference, strict) userinfo, host, port = authority_from(reference, strict)
to_bytes = compat.to_bytes to_bytes = compat.to_bytes
return cls(scheme=to_bytes(reference.scheme, encoding), return cls(
userinfo=to_bytes(userinfo, encoding), scheme=to_bytes(reference.scheme, encoding),
host=to_bytes(host, encoding), userinfo=to_bytes(userinfo, encoding),
port=port, host=to_bytes(host, encoding),
path=to_bytes(reference.path, encoding), port=port,
query=to_bytes(reference.query, encoding), path=to_bytes(reference.path, encoding),
fragment=to_bytes(reference.fragment, encoding), query=to_bytes(reference.query, encoding),
uri_ref=reference, fragment=to_bytes(reference.fragment, encoding),
encoding=encoding, uri_ref=reference,
lazy_normalize=lazy_normalize) encoding=encoding,
lazy_normalize=lazy_normalize,
)
@property @property
def authority(self): def authority(self):
"""Return the normalized authority.""" """Return the normalized authority."""
return self.reference.authority.encode(self.encoding) return self.reference.authority.encode(self.encoding)
def copy_with(self, scheme=misc.UseExisting, userinfo=misc.UseExisting, def copy_with(
host=misc.UseExisting, port=misc.UseExisting, self,
path=misc.UseExisting, query=misc.UseExisting, scheme=misc.UseExisting,
fragment=misc.UseExisting, lazy_normalize=True): userinfo=misc.UseExisting,
host=misc.UseExisting,
port=misc.UseExisting,
path=misc.UseExisting,
query=misc.UseExisting,
fragment=misc.UseExisting,
lazy_normalize=True,
):
"""Create a copy of this instance replacing with specified parts.""" """Create a copy of this instance replacing with specified parts."""
attributes = zip(PARSED_COMPONENTS, attributes = zip(
(scheme, userinfo, host, port, path, query, fragment)) PARSED_COMPONENTS, (scheme, userinfo, host, port, path, query, fragment)
)
attrs_dict = {} attrs_dict = {}
for name, value in attributes: for name, value in attributes:
if value is misc.UseExisting: if value is misc.UseExisting:
value = getattr(self, name) value = getattr(self, name)
if not isinstance(value, bytes) and hasattr(value, 'encode'): if not isinstance(value, bytes) and hasattr(value, "encode"):
value = value.encode(self.encoding) value = value.encode(self.encoding)
attrs_dict[name] = value attrs_dict[name] = value
authority = self._generate_authority(attrs_dict) authority = self._generate_authority(attrs_dict)
to_str = compat.to_str to_str = compat.to_str
ref = self.reference.copy_with( ref = self.reference.copy_with(
scheme=to_str(attrs_dict['scheme'], self.encoding), scheme=to_str(attrs_dict["scheme"], self.encoding),
authority=to_str(authority, self.encoding), authority=to_str(authority, self.encoding),
path=to_str(attrs_dict['path'], self.encoding), path=to_str(attrs_dict["path"], self.encoding),
query=to_str(attrs_dict['query'], self.encoding), query=to_str(attrs_dict["query"], self.encoding),
fragment=to_str(attrs_dict['fragment'], self.encoding) fragment=to_str(attrs_dict["fragment"], self.encoding),
) )
if not lazy_normalize: if not lazy_normalize:
ref = ref.normalize() ref = ref.normalize()
@ -329,7 +400,7 @@ class ParseResultBytes(namedtuple('ParseResultBytes', PARSED_COMPONENTS),
# self.host is bytes, to encode to idna, we need to decode it # self.host is bytes, to encode to idna, we need to decode it
# first # first
host = self.host.decode(self.encoding) host = self.host.decode(self.encoding)
hostbytes = host.encode('idna') hostbytes = host.encode("idna")
parse_result = self.copy_with(host=hostbytes) parse_result = self.copy_with(host=hostbytes)
if self.lazy_normalize: if self.lazy_normalize:
parse_result = parse_result.copy_with(lazy_normalize=False) parse_result = parse_result.copy_with(lazy_normalize=False)
@ -345,16 +416,16 @@ def split_authority(authority):
# Set-up rest in case there is no userinfo portion # Set-up rest in case there is no userinfo portion
rest = authority rest = authority
if '@' in authority: if "@" in authority:
userinfo, rest = authority.rsplit('@', 1) userinfo, rest = authority.rsplit("@", 1)
# Handle IPv6 host addresses # Handle IPv6 host addresses
if rest.startswith('['): if rest.startswith("["):
host, rest = rest.split(']', 1) host, rest = rest.split("]", 1)
host += ']' host += "]"
if ':' in rest: if ":" in rest:
extra_host, port = rest.split(':', 1) extra_host, port = rest.split(":", 1)
elif not host and rest: elif not host and rest:
host = rest host = rest
@ -374,8 +445,9 @@ def authority_from(reference, strict):
else: else:
# Thanks to Richard Barrell for this idea: # Thanks to Richard Barrell for this idea:
# https://twitter.com/0x2ba22e11/status/617338811975139328 # https://twitter.com/0x2ba22e11/status/617338811975139328
userinfo, host, port = (subauthority.get(p) userinfo, host, port = (
for p in ('userinfo', 'host', 'port')) subauthority.get(p) for p in ("userinfo", "host", "port")
)
if port: if port:
try: try:

45
lib/urllib3/packages/rfc3986/uri.py

@ -22,7 +22,7 @@ from . import normalizers
from ._mixin import URIMixin from ._mixin import URIMixin
class URIReference(namedtuple('URIReference', misc.URI_COMPONENTS), URIMixin): class URIReference(namedtuple("URIReference", misc.URI_COMPONENTS), URIMixin):
"""Immutable object representing a parsed URI Reference. """Immutable object representing a parsed URI Reference.
.. note:: .. note::
@ -82,16 +82,11 @@ class URIReference(namedtuple('URIReference', misc.URI_COMPONENTS), URIMixin):
slots = () slots = ()
def __new__(cls, scheme, authority, path, query, fragment, def __new__(cls, scheme, authority, path, query, fragment, encoding="utf-8"):
encoding='utf-8'):
"""Create a new URIReference.""" """Create a new URIReference."""
ref = super(URIReference, cls).__new__( ref = super(URIReference, cls).__new__(
cls, cls, scheme or None, authority or None, path or None, query, fragment
scheme or None, )
authority or None,
path or None,
query,
fragment)
ref.encoding = encoding ref.encoding = encoding
return ref return ref
@ -107,8 +102,10 @@ class URIReference(namedtuple('URIReference', misc.URI_COMPONENTS), URIMixin):
other_ref = URIReference.from_string(other) other_ref = URIReference.from_string(other)
except TypeError: except TypeError:
raise TypeError( raise TypeError(
'Unable to compare URIReference() to {0}()'.format( "Unable to compare URIReference() to {0}()".format(
type(other).__name__)) type(other).__name__
)
)
# See http://tools.ietf.org/html/rfc3986#section-6.2 # See http://tools.ietf.org/html/rfc3986#section-6.2
naive_equality = tuple(self) == tuple(other_ref) naive_equality = tuple(self) == tuple(other_ref)
@ -125,16 +122,17 @@ class URIReference(namedtuple('URIReference', misc.URI_COMPONENTS), URIMixin):
""" """
# See http://tools.ietf.org/html/rfc3986#section-6.2.2 for logic in # See http://tools.ietf.org/html/rfc3986#section-6.2.2 for logic in
# this method. # this method.
return URIReference(normalizers.normalize_scheme(self.scheme or ''), return URIReference(
normalizers.normalize_authority( normalizers.normalize_scheme(self.scheme or ""),
(self.userinfo, self.host, self.port)), normalizers.normalize_authority((self.userinfo, self.host, self.port)),
normalizers.normalize_path(self.path or ''), normalizers.normalize_path(self.path or ""),
normalizers.normalize_query(self.query), normalizers.normalize_query(self.query),
normalizers.normalize_fragment(self.fragment), normalizers.normalize_fragment(self.fragment),
self.encoding) self.encoding,
)
@classmethod @classmethod
def from_string(cls, uri_string, encoding='utf-8'): def from_string(cls, uri_string, encoding="utf-8"):
"""Parse a URI reference from the given unicode URI string. """Parse a URI reference from the given unicode URI string.
:param str uri_string: Unicode URI to be parsed into a reference. :param str uri_string: Unicode URI to be parsed into a reference.
@ -145,9 +143,10 @@ class URIReference(namedtuple('URIReference', misc.URI_COMPONENTS), URIMixin):
split_uri = misc.URI_MATCHER.match(uri_string).groupdict() split_uri = misc.URI_MATCHER.match(uri_string).groupdict()
return cls( return cls(
split_uri['scheme'], split_uri['authority'], split_uri["scheme"],
normalizers.encode_component(split_uri['path'], encoding), split_uri["authority"],
normalizers.encode_component(split_uri['query'], encoding), normalizers.encode_component(split_uri["path"], encoding),
normalizers.encode_component(split_uri['fragment'], encoding), normalizers.encode_component(split_uri["query"], encoding),
normalizers.encode_component(split_uri["fragment"], encoding),
encoding, encoding,
) )

91
lib/urllib3/packages/rfc3986/validators.py

@ -45,15 +45,9 @@ class Validator(object):
""" """
COMPONENT_NAMES = frozenset([ COMPONENT_NAMES = frozenset(
'scheme', ["scheme", "userinfo", "host", "port", "path", "query", "fragment"]
'userinfo', )
'host',
'port',
'path',
'query',
'fragment',
])
def __init__(self): def __init__(self):
"""Initialize our default validations.""" """Initialize our default validations."""
@ -62,13 +56,13 @@ class Validator(object):
self.allowed_ports = set() self.allowed_ports = set()
self.allow_password = True self.allow_password = True
self.required_components = { self.required_components = {
'scheme': False, "scheme": False,
'userinfo': False, "userinfo": False,
'host': False, "host": False,
'port': False, "port": False,
'path': False, "path": False,
'query': False, "query": False,
'fragment': False, "fragment": False,
} }
self.validated_components = self.required_components.copy() self.validated_components = self.required_components.copy()
@ -165,12 +159,8 @@ class Validator(object):
components = [c.lower() for c in components] components = [c.lower() for c in components]
for component in components: for component in components:
if component not in self.COMPONENT_NAMES: if component not in self.COMPONENT_NAMES:
raise ValueError( raise ValueError('"{}" is not a valid component'.format(component))
'"{}" is not a valid component'.format(component) self.validated_components.update({component: True for component in components})
)
self.validated_components.update({
component: True for component in components
})
return self return self
def require_presence_of(self, *components): def require_presence_of(self, *components):
@ -190,12 +180,8 @@ class Validator(object):
components = [c.lower() for c in components] components = [c.lower() for c in components]
for component in components: for component in components:
if component not in self.COMPONENT_NAMES: if component not in self.COMPONENT_NAMES:
raise ValueError( raise ValueError('"{}" is not a valid component'.format(component))
'"{}" is not a valid component'.format(component) self.required_components.update({component: True for component in components})
)
self.required_components.update({
component: True for component in components
})
return self return self
def validate(self, uri): def validate(self, uri):
@ -235,9 +221,9 @@ class Validator(object):
if validated_components: if validated_components:
ensure_components_are_valid(uri, validated_components) ensure_components_are_valid(uri, validated_components)
ensure_one_of(self.allowed_schemes, uri, 'scheme') ensure_one_of(self.allowed_schemes, uri, "scheme")
ensure_one_of(self.allowed_hosts, uri, 'host') ensure_one_of(self.allowed_hosts, uri, "host")
ensure_one_of(self.allowed_ports, uri, 'port') ensure_one_of(self.allowed_ports, uri, "port")
def check_password(uri): def check_password(uri):
@ -245,7 +231,7 @@ def check_password(uri):
userinfo = uri.userinfo userinfo = uri.userinfo
if not userinfo: if not userinfo:
return return
credentials = userinfo.split(':', 1) credentials = userinfo.split(":", 1)
if len(credentials) <= 1: if len(credentials) <= 1:
return return
raise exceptions.PasswordForbidden(uri) raise exceptions.PasswordForbidden(uri)
@ -255,18 +241,18 @@ def ensure_one_of(allowed_values, uri, attribute):
"""Assert that the uri's attribute is one of the allowed values.""" """Assert that the uri's attribute is one of the allowed values."""
value = getattr(uri, attribute) value = getattr(uri, attribute)
if value is not None and allowed_values and value not in allowed_values: if value is not None and allowed_values and value not in allowed_values:
raise exceptions.UnpermittedComponentError( raise exceptions.UnpermittedComponentError(attribute, value, allowed_values)
attribute, value, allowed_values,
)
def ensure_required_components_exist(uri, required_components): def ensure_required_components_exist(uri, required_components):
"""Assert that all required components are present in the URI.""" """Assert that all required components are present in the URI."""
missing_components = sorted([ missing_components = sorted(
component [
for component in required_components component
if getattr(uri, component) is None for component in required_components
]) if getattr(uri, component) is None
]
)
if missing_components: if missing_components:
raise exceptions.MissingComponentError(uri, *missing_components) raise exceptions.MissingComponentError(uri, *missing_components)
@ -282,8 +268,7 @@ def is_valid(value, matcher, require):
Whether or not the value is required. Whether or not the value is required.
""" """
if require: if require:
return (value is not None return value is not None and matcher.match(value)
and matcher.match(value))
# require is False and value is not None # require is False and value is not None
return value is None or matcher.match(value) return value is None or matcher.match(value)
@ -393,17 +378,17 @@ def valid_ipv4_host_address(host):
"""Determine if the given host is a valid IPv4 address.""" """Determine if the given host is a valid IPv4 address."""
# If the host exists, and it might be IPv4, check each byte in the # If the host exists, and it might be IPv4, check each byte in the
# address. # address.
return all([0 <= int(byte, base=10) <= 255 for byte in host.split('.')]) return all([0 <= int(byte, base=10) <= 255 for byte in host.split(".")])
_COMPONENT_VALIDATORS = { _COMPONENT_VALIDATORS = {
'scheme': scheme_is_valid, "scheme": scheme_is_valid,
'path': path_is_valid, "path": path_is_valid,
'query': query_is_valid, "query": query_is_valid,
'fragment': fragment_is_valid, "fragment": fragment_is_valid,
} }
_SUBAUTHORITY_VALIDATORS = set(['userinfo', 'host', 'port']) _SUBAUTHORITY_VALIDATORS = set(["userinfo", "host", "port"])
def subauthority_component_is_valid(uri, component): def subauthority_component_is_valid(uri, component):
@ -415,19 +400,19 @@ def subauthority_component_is_valid(uri, component):
# If we can parse the authority into sub-components and we're not # If we can parse the authority into sub-components and we're not
# validating the port, we can assume it's valid. # validating the port, we can assume it's valid.
if component == 'host': if component == "host":
return host_is_valid(subauthority_dict['host']) return host_is_valid(subauthority_dict["host"])
elif component != 'port': elif component != "port":
return True return True
try: try:
port = int(subauthority_dict['port']) port = int(subauthority_dict["port"])
except TypeError: except TypeError:
# If the port wasn't provided it'll be None and int(None) raises a # If the port wasn't provided it'll be None and int(None) raises a
# TypeError # TypeError
return True return True
return (0 <= port <= 65535) return 0 <= port <= 65535
def ensure_components_are_valid(uri, validated_components): def ensure_components_are_valid(uri, validated_components):

203
lib/urllib3/packages/six.py

@ -38,15 +38,15 @@ PY3 = sys.version_info[0] == 3
PY34 = sys.version_info[0:2] >= (3, 4) PY34 = sys.version_info[0:2] >= (3, 4)
if PY3: if PY3:
string_types = str, string_types = (str,)
integer_types = int, integer_types = (int,)
class_types = type, class_types = (type,)
text_type = str text_type = str
binary_type = bytes binary_type = bytes
MAXSIZE = sys.maxsize MAXSIZE = sys.maxsize
else: else:
string_types = basestring, string_types = (basestring,)
integer_types = (int, long) integer_types = (int, long)
class_types = (type, types.ClassType) class_types = (type, types.ClassType)
text_type = unicode text_type = unicode
@ -58,9 +58,9 @@ else:
else: else:
# It's possible to have sizeof(long) != sizeof(Py_ssize_t). # It's possible to have sizeof(long) != sizeof(Py_ssize_t).
class X(object): class X(object):
def __len__(self): def __len__(self):
return 1 << 31 return 1 << 31
try: try:
len(X()) len(X())
except OverflowError: except OverflowError:
@ -84,7 +84,6 @@ def _import_module(name):
class _LazyDescr(object): class _LazyDescr(object):
def __init__(self, name): def __init__(self, name):
self.name = name self.name = name
@ -101,7 +100,6 @@ class _LazyDescr(object):
class MovedModule(_LazyDescr): class MovedModule(_LazyDescr):
def __init__(self, name, old, new=None): def __init__(self, name, old, new=None):
super(MovedModule, self).__init__(name) super(MovedModule, self).__init__(name)
if PY3: if PY3:
@ -122,7 +120,6 @@ class MovedModule(_LazyDescr):
class _LazyModule(types.ModuleType): class _LazyModule(types.ModuleType):
def __init__(self, name): def __init__(self, name):
super(_LazyModule, self).__init__(name) super(_LazyModule, self).__init__(name)
self.__doc__ = self.__class__.__doc__ self.__doc__ = self.__class__.__doc__
@ -137,7 +134,6 @@ class _LazyModule(types.ModuleType):
class MovedAttribute(_LazyDescr): class MovedAttribute(_LazyDescr):
def __init__(self, name, old_mod, new_mod, old_attr=None, new_attr=None): def __init__(self, name, old_mod, new_mod, old_attr=None, new_attr=None):
super(MovedAttribute, self).__init__(name) super(MovedAttribute, self).__init__(name)
if PY3: if PY3:
@ -221,28 +217,35 @@ class _SixMetaPathImporter(object):
Required, if is_package is implemented""" Required, if is_package is implemented"""
self.__get_module(fullname) # eventually raises ImportError self.__get_module(fullname) # eventually raises ImportError
return None return None
get_source = get_code # same as get_code get_source = get_code # same as get_code
_importer = _SixMetaPathImporter(__name__) _importer = _SixMetaPathImporter(__name__)
class _MovedItems(_LazyModule): class _MovedItems(_LazyModule):
"""Lazy loading of moved objects""" """Lazy loading of moved objects"""
__path__ = [] # mark as package __path__ = [] # mark as package
_moved_attributes = [ _moved_attributes = [
MovedAttribute("cStringIO", "cStringIO", "io", "StringIO"), MovedAttribute("cStringIO", "cStringIO", "io", "StringIO"),
MovedAttribute("filter", "itertools", "builtins", "ifilter", "filter"), MovedAttribute("filter", "itertools", "builtins", "ifilter", "filter"),
MovedAttribute("filterfalse", "itertools", "itertools", "ifilterfalse", "filterfalse"), MovedAttribute(
"filterfalse", "itertools", "itertools", "ifilterfalse", "filterfalse"
),
MovedAttribute("input", "__builtin__", "builtins", "raw_input", "input"), MovedAttribute("input", "__builtin__", "builtins", "raw_input", "input"),
MovedAttribute("intern", "__builtin__", "sys"), MovedAttribute("intern", "__builtin__", "sys"),
MovedAttribute("map", "itertools", "builtins", "imap", "map"), MovedAttribute("map", "itertools", "builtins", "imap", "map"),
MovedAttribute("getcwd", "os", "os", "getcwdu", "getcwd"), MovedAttribute("getcwd", "os", "os", "getcwdu", "getcwd"),
MovedAttribute("getcwdb", "os", "os", "getcwd", "getcwdb"), MovedAttribute("getcwdb", "os", "os", "getcwd", "getcwdb"),
MovedAttribute("range", "__builtin__", "builtins", "xrange", "range"), MovedAttribute("range", "__builtin__", "builtins", "xrange", "range"),
MovedAttribute("reload_module", "__builtin__", "importlib" if PY34 else "imp", "reload"), MovedAttribute(
"reload_module", "__builtin__", "importlib" if PY34 else "imp", "reload"
),
MovedAttribute("reduce", "__builtin__", "functools"), MovedAttribute("reduce", "__builtin__", "functools"),
MovedAttribute("shlex_quote", "pipes", "shlex", "quote"), MovedAttribute("shlex_quote", "pipes", "shlex", "quote"),
MovedAttribute("StringIO", "StringIO", "io"), MovedAttribute("StringIO", "StringIO", "io"),
@ -251,7 +254,9 @@ _moved_attributes = [
MovedAttribute("UserString", "UserString", "collections"), MovedAttribute("UserString", "UserString", "collections"),
MovedAttribute("xrange", "__builtin__", "builtins", "xrange", "range"), MovedAttribute("xrange", "__builtin__", "builtins", "xrange", "range"),
MovedAttribute("zip", "itertools", "builtins", "izip", "zip"), MovedAttribute("zip", "itertools", "builtins", "izip", "zip"),
MovedAttribute("zip_longest", "itertools", "itertools", "izip_longest", "zip_longest"), MovedAttribute(
"zip_longest", "itertools", "itertools", "izip_longest", "zip_longest"
),
MovedModule("builtins", "__builtin__"), MovedModule("builtins", "__builtin__"),
MovedModule("configparser", "ConfigParser"), MovedModule("configparser", "ConfigParser"),
MovedModule("copyreg", "copy_reg"), MovedModule("copyreg", "copy_reg"),
@ -263,7 +268,9 @@ _moved_attributes = [
MovedModule("html_parser", "HTMLParser", "html.parser"), MovedModule("html_parser", "HTMLParser", "html.parser"),
MovedModule("http_client", "httplib", "http.client"), MovedModule("http_client", "httplib", "http.client"),
MovedModule("email_mime_multipart", "email.MIMEMultipart", "email.mime.multipart"), MovedModule("email_mime_multipart", "email.MIMEMultipart", "email.mime.multipart"),
MovedModule("email_mime_nonmultipart", "email.MIMENonMultipart", "email.mime.nonmultipart"), MovedModule(
"email_mime_nonmultipart", "email.MIMENonMultipart", "email.mime.nonmultipart"
),
MovedModule("email_mime_text", "email.MIMEText", "email.mime.text"), MovedModule("email_mime_text", "email.MIMEText", "email.mime.text"),
MovedModule("email_mime_base", "email.MIMEBase", "email.mime.base"), MovedModule("email_mime_base", "email.MIMEBase", "email.mime.base"),
MovedModule("BaseHTTPServer", "BaseHTTPServer", "http.server"), MovedModule("BaseHTTPServer", "BaseHTTPServer", "http.server"),
@ -283,15 +290,12 @@ _moved_attributes = [
MovedModule("tkinter_ttk", "ttk", "tkinter.ttk"), MovedModule("tkinter_ttk", "ttk", "tkinter.ttk"),
MovedModule("tkinter_constants", "Tkconstants", "tkinter.constants"), MovedModule("tkinter_constants", "Tkconstants", "tkinter.constants"),
MovedModule("tkinter_dnd", "Tkdnd", "tkinter.dnd"), MovedModule("tkinter_dnd", "Tkdnd", "tkinter.dnd"),
MovedModule("tkinter_colorchooser", "tkColorChooser", MovedModule("tkinter_colorchooser", "tkColorChooser", "tkinter.colorchooser"),
"tkinter.colorchooser"), MovedModule("tkinter_commondialog", "tkCommonDialog", "tkinter.commondialog"),
MovedModule("tkinter_commondialog", "tkCommonDialog",
"tkinter.commondialog"),
MovedModule("tkinter_tkfiledialog", "tkFileDialog", "tkinter.filedialog"), MovedModule("tkinter_tkfiledialog", "tkFileDialog", "tkinter.filedialog"),
MovedModule("tkinter_font", "tkFont", "tkinter.font"), MovedModule("tkinter_font", "tkFont", "tkinter.font"),
MovedModule("tkinter_messagebox", "tkMessageBox", "tkinter.messagebox"), MovedModule("tkinter_messagebox", "tkMessageBox", "tkinter.messagebox"),
MovedModule("tkinter_tksimpledialog", "tkSimpleDialog", MovedModule("tkinter_tksimpledialog", "tkSimpleDialog", "tkinter.simpledialog"),
"tkinter.simpledialog"),
MovedModule("urllib_parse", __name__ + ".moves.urllib_parse", "urllib.parse"), MovedModule("urllib_parse", __name__ + ".moves.urllib_parse", "urllib.parse"),
MovedModule("urllib_error", __name__ + ".moves.urllib_error", "urllib.error"), MovedModule("urllib_error", __name__ + ".moves.urllib_error", "urllib.error"),
MovedModule("urllib", __name__ + ".moves.urllib", __name__ + ".moves.urllib"), MovedModule("urllib", __name__ + ".moves.urllib", __name__ + ".moves.urllib"),
@ -301,9 +305,7 @@ _moved_attributes = [
] ]
# Add windows specific modules. # Add windows specific modules.
if sys.platform == "win32": if sys.platform == "win32":
_moved_attributes += [ _moved_attributes += [MovedModule("winreg", "_winreg")]
MovedModule("winreg", "_winreg"),
]
for attr in _moved_attributes: for attr in _moved_attributes:
setattr(_MovedItems, attr.name, attr) setattr(_MovedItems, attr.name, attr)
@ -353,8 +355,11 @@ del attr
Module_six_moves_urllib_parse._moved_attributes = _urllib_parse_moved_attributes Module_six_moves_urllib_parse._moved_attributes = _urllib_parse_moved_attributes
_importer._add_module(Module_six_moves_urllib_parse(__name__ + ".moves.urllib_parse"), _importer._add_module(
"moves.urllib_parse", "moves.urllib.parse") Module_six_moves_urllib_parse(__name__ + ".moves.urllib_parse"),
"moves.urllib_parse",
"moves.urllib.parse",
)
class Module_six_moves_urllib_error(_LazyModule): class Module_six_moves_urllib_error(_LazyModule):
@ -373,8 +378,11 @@ del attr
Module_six_moves_urllib_error._moved_attributes = _urllib_error_moved_attributes Module_six_moves_urllib_error._moved_attributes = _urllib_error_moved_attributes
_importer._add_module(Module_six_moves_urllib_error(__name__ + ".moves.urllib.error"), _importer._add_module(
"moves.urllib_error", "moves.urllib.error") Module_six_moves_urllib_error(__name__ + ".moves.urllib.error"),
"moves.urllib_error",
"moves.urllib.error",
)
class Module_six_moves_urllib_request(_LazyModule): class Module_six_moves_urllib_request(_LazyModule):
@ -423,8 +431,11 @@ del attr
Module_six_moves_urllib_request._moved_attributes = _urllib_request_moved_attributes Module_six_moves_urllib_request._moved_attributes = _urllib_request_moved_attributes
_importer._add_module(Module_six_moves_urllib_request(__name__ + ".moves.urllib.request"), _importer._add_module(
"moves.urllib_request", "moves.urllib.request") Module_six_moves_urllib_request(__name__ + ".moves.urllib.request"),
"moves.urllib_request",
"moves.urllib.request",
)
class Module_six_moves_urllib_response(_LazyModule): class Module_six_moves_urllib_response(_LazyModule):
@ -444,8 +455,11 @@ del attr
Module_six_moves_urllib_response._moved_attributes = _urllib_response_moved_attributes Module_six_moves_urllib_response._moved_attributes = _urllib_response_moved_attributes
_importer._add_module(Module_six_moves_urllib_response(__name__ + ".moves.urllib.response"), _importer._add_module(
"moves.urllib_response", "moves.urllib.response") Module_six_moves_urllib_response(__name__ + ".moves.urllib.response"),
"moves.urllib_response",
"moves.urllib.response",
)
class Module_six_moves_urllib_robotparser(_LazyModule): class Module_six_moves_urllib_robotparser(_LazyModule):
@ -454,21 +468,27 @@ class Module_six_moves_urllib_robotparser(_LazyModule):
_urllib_robotparser_moved_attributes = [ _urllib_robotparser_moved_attributes = [
MovedAttribute("RobotFileParser", "robotparser", "urllib.robotparser"), MovedAttribute("RobotFileParser", "robotparser", "urllib.robotparser")
] ]
for attr in _urllib_robotparser_moved_attributes: for attr in _urllib_robotparser_moved_attributes:
setattr(Module_six_moves_urllib_robotparser, attr.name, attr) setattr(Module_six_moves_urllib_robotparser, attr.name, attr)
del attr del attr
Module_six_moves_urllib_robotparser._moved_attributes = _urllib_robotparser_moved_attributes Module_six_moves_urllib_robotparser._moved_attributes = (
_urllib_robotparser_moved_attributes
)
_importer._add_module(Module_six_moves_urllib_robotparser(__name__ + ".moves.urllib.robotparser"), _importer._add_module(
"moves.urllib_robotparser", "moves.urllib.robotparser") Module_six_moves_urllib_robotparser(__name__ + ".moves.urllib.robotparser"),
"moves.urllib_robotparser",
"moves.urllib.robotparser",
)
class Module_six_moves_urllib(types.ModuleType): class Module_six_moves_urllib(types.ModuleType):
"""Create a six.moves.urllib namespace that resembles the Python 3 namespace""" """Create a six.moves.urllib namespace that resembles the Python 3 namespace"""
__path__ = [] # mark as package __path__ = [] # mark as package
parse = _importer._get_module("moves.urllib_parse") parse = _importer._get_module("moves.urllib_parse")
error = _importer._get_module("moves.urllib_error") error = _importer._get_module("moves.urllib_error")
@ -477,10 +497,12 @@ class Module_six_moves_urllib(types.ModuleType):
robotparser = _importer._get_module("moves.urllib_robotparser") robotparser = _importer._get_module("moves.urllib_robotparser")
def __dir__(self): def __dir__(self):
return ['parse', 'error', 'request', 'response', 'robotparser'] return ["parse", "error", "request", "response", "robotparser"]
_importer._add_module(Module_six_moves_urllib(__name__ + ".moves.urllib"), _importer._add_module(
"moves.urllib") Module_six_moves_urllib(__name__ + ".moves.urllib"), "moves.urllib"
)
def add_move(move): def add_move(move):
@ -520,19 +542,24 @@ else:
try: try:
advance_iterator = next advance_iterator = next
except NameError: except NameError:
def advance_iterator(it): def advance_iterator(it):
return it.next() return it.next()
next = advance_iterator next = advance_iterator
try: try:
callable = callable callable = callable
except NameError: except NameError:
def callable(obj): def callable(obj):
return any("__call__" in klass.__dict__ for klass in type(obj).__mro__) return any("__call__" in klass.__dict__ for klass in type(obj).__mro__)
if PY3: if PY3:
def get_unbound_function(unbound): def get_unbound_function(unbound):
return unbound return unbound
@ -543,6 +570,7 @@ if PY3:
Iterator = object Iterator = object
else: else:
def get_unbound_function(unbound): def get_unbound_function(unbound):
return unbound.im_func return unbound.im_func
@ -553,13 +581,13 @@ else:
return types.MethodType(func, None, cls) return types.MethodType(func, None, cls)
class Iterator(object): class Iterator(object):
def next(self): def next(self):
return type(self).__next__(self) return type(self).__next__(self)
callable = callable callable = callable
_add_doc(get_unbound_function, _add_doc(
"""Get the function out of a possibly unbound function""") get_unbound_function, """Get the function out of a possibly unbound function"""
)
get_method_function = operator.attrgetter(_meth_func) get_method_function = operator.attrgetter(_meth_func)
@ -571,6 +599,7 @@ get_function_globals = operator.attrgetter(_func_globals)
if PY3: if PY3:
def iterkeys(d, **kw): def iterkeys(d, **kw):
return iter(d.keys(**kw)) return iter(d.keys(**kw))
@ -589,6 +618,7 @@ if PY3:
viewitems = operator.methodcaller("items") viewitems = operator.methodcaller("items")
else: else:
def iterkeys(d, **kw): def iterkeys(d, **kw):
return d.iterkeys(**kw) return d.iterkeys(**kw)
@ -609,26 +639,30 @@ else:
_add_doc(iterkeys, "Return an iterator over the keys of a dictionary.") _add_doc(iterkeys, "Return an iterator over the keys of a dictionary.")
_add_doc(itervalues, "Return an iterator over the values of a dictionary.") _add_doc(itervalues, "Return an iterator over the values of a dictionary.")
_add_doc(iteritems, _add_doc(iteritems, "Return an iterator over the (key, value) pairs of a dictionary.")
"Return an iterator over the (key, value) pairs of a dictionary.") _add_doc(
_add_doc(iterlists, iterlists, "Return an iterator over the (key, [values]) pairs of a dictionary."
"Return an iterator over the (key, [values]) pairs of a dictionary.") )
if PY3: if PY3:
def b(s): def b(s):
return s.encode("latin-1") return s.encode("latin-1")
def u(s): def u(s):
return s return s
unichr = chr unichr = chr
import struct import struct
int2byte = struct.Struct(">B").pack int2byte = struct.Struct(">B").pack
del struct del struct
byte2int = operator.itemgetter(0) byte2int = operator.itemgetter(0)
indexbytes = operator.getitem indexbytes = operator.getitem
iterbytes = iter iterbytes = iter
import io import io
StringIO = io.StringIO StringIO = io.StringIO
BytesIO = io.BytesIO BytesIO = io.BytesIO
_assertCountEqual = "assertCountEqual" _assertCountEqual = "assertCountEqual"
@ -639,12 +673,15 @@ if PY3:
_assertRaisesRegex = "assertRaisesRegex" _assertRaisesRegex = "assertRaisesRegex"
_assertRegex = "assertRegex" _assertRegex = "assertRegex"
else: else:
def b(s): def b(s):
return s return s
# Workaround for standalone backslash # Workaround for standalone backslash
def u(s): def u(s):
return unicode(s.replace(r'\\', r'\\\\'), "unicode_escape") return unicode(s.replace(r"\\", r"\\\\"), "unicode_escape")
unichr = unichr unichr = unichr
int2byte = chr int2byte = chr
@ -653,8 +690,10 @@ else:
def indexbytes(buf, i): def indexbytes(buf, i):
return ord(buf[i]) return ord(buf[i])
iterbytes = functools.partial(itertools.imap, ord) iterbytes = functools.partial(itertools.imap, ord)
import StringIO import StringIO
StringIO = BytesIO = StringIO.StringIO StringIO = BytesIO = StringIO.StringIO
_assertCountEqual = "assertItemsEqual" _assertCountEqual = "assertItemsEqual"
_assertRaisesRegex = "assertRaisesRegexp" _assertRaisesRegex = "assertRaisesRegexp"
@ -685,7 +724,9 @@ if PY3:
raise value.with_traceback(tb) raise value.with_traceback(tb)
raise value raise value
else: else:
def exec_(_code_, _globs_=None, _locs_=None): def exec_(_code_, _globs_=None, _locs_=None):
"""Execute code in a namespace.""" """Execute code in a namespace."""
if _globs_ is None: if _globs_ is None:
@ -698,28 +739,36 @@ else:
_locs_ = _globs_ _locs_ = _globs_
exec("""exec _code_ in _globs_, _locs_""") exec("""exec _code_ in _globs_, _locs_""")
exec_("""def reraise(tp, value, tb=None): exec_(
"""def reraise(tp, value, tb=None):
raise tp, value, tb raise tp, value, tb
""") """
)
if sys.version_info[:2] == (3, 2): if sys.version_info[:2] == (3, 2):
exec_("""def raise_from(value, from_value): exec_(
"""def raise_from(value, from_value):
if from_value is None: if from_value is None:
raise value raise value
raise value from from_value raise value from from_value
""") """
)
elif sys.version_info[:2] > (3, 2): elif sys.version_info[:2] > (3, 2):
exec_("""def raise_from(value, from_value): exec_(
"""def raise_from(value, from_value):
raise value from from_value raise value from from_value
""") """
)
else: else:
def raise_from(value, from_value): def raise_from(value, from_value):
raise value raise value
print_ = getattr(moves.builtins, "print", None) print_ = getattr(moves.builtins, "print", None)
if print_ is None: if print_ is None:
def print_(*args, **kwargs): def print_(*args, **kwargs):
"""The new-style print function for Python 2.4 and 2.5.""" """The new-style print function for Python 2.4 and 2.5."""
fp = kwargs.pop("file", sys.stdout) fp = kwargs.pop("file", sys.stdout)
@ -730,14 +779,17 @@ if print_ is None:
if not isinstance(data, basestring): if not isinstance(data, basestring):
data = str(data) data = str(data)
# If the file has an encoding, encode unicode with it. # If the file has an encoding, encode unicode with it.
if (isinstance(fp, file) and if (
isinstance(data, unicode) and isinstance(fp, file)
fp.encoding is not None): and isinstance(data, unicode)
and fp.encoding is not None
):
errors = getattr(fp, "errors", None) errors = getattr(fp, "errors", None)
if errors is None: if errors is None:
errors = "strict" errors = "strict"
data = data.encode(fp.encoding, errors) data = data.encode(fp.encoding, errors)
fp.write(data) fp.write(data)
want_unicode = False want_unicode = False
sep = kwargs.pop("sep", None) sep = kwargs.pop("sep", None)
if sep is not None: if sep is not None:
@ -773,6 +825,8 @@ if print_ is None:
write(sep) write(sep)
write(arg) write(arg)
write(end) write(end)
if sys.version_info[:2] < (3, 3): if sys.version_info[:2] < (3, 3):
_print = print_ _print = print_
@ -783,16 +837,24 @@ if sys.version_info[:2] < (3, 3):
if flush and fp is not None: if flush and fp is not None:
fp.flush() fp.flush()
_add_doc(reraise, """Reraise an exception.""") _add_doc(reraise, """Reraise an exception.""")
if sys.version_info[0:2] < (3, 4): if sys.version_info[0:2] < (3, 4):
def wraps(wrapped, assigned=functools.WRAPPER_ASSIGNMENTS,
updated=functools.WRAPPER_UPDATES): def wraps(
wrapped,
assigned=functools.WRAPPER_ASSIGNMENTS,
updated=functools.WRAPPER_UPDATES,
):
def wrapper(f): def wrapper(f):
f = functools.wraps(wrapped, assigned, updated)(f) f = functools.wraps(wrapped, assigned, updated)(f)
f.__wrapped__ = wrapped f.__wrapped__ = wrapped
return f return f
return wrapper return wrapper
else: else:
wraps = functools.wraps wraps = functools.wraps
@ -803,25 +865,27 @@ def with_metaclass(meta, *bases):
# metaclass for one level of class instantiation that replaces itself with # metaclass for one level of class instantiation that replaces itself with
# the actual metaclass. # the actual metaclass.
class metaclass(meta): class metaclass(meta):
def __new__(cls, name, this_bases, d): def __new__(cls, name, this_bases, d):
return meta(name, bases, d) return meta(name, bases, d)
return type.__new__(metaclass, 'temporary_class', (), {})
return type.__new__(metaclass, "temporary_class", (), {})
def add_metaclass(metaclass): def add_metaclass(metaclass):
"""Class decorator for creating a class with a metaclass.""" """Class decorator for creating a class with a metaclass."""
def wrapper(cls): def wrapper(cls):
orig_vars = cls.__dict__.copy() orig_vars = cls.__dict__.copy()
slots = orig_vars.get('__slots__') slots = orig_vars.get("__slots__")
if slots is not None: if slots is not None:
if isinstance(slots, str): if isinstance(slots, str):
slots = [slots] slots = [slots]
for slots_var in slots: for slots_var in slots:
orig_vars.pop(slots_var) orig_vars.pop(slots_var)
orig_vars.pop('__dict__', None) orig_vars.pop("__dict__", None)
orig_vars.pop('__weakref__', None) orig_vars.pop("__weakref__", None)
return metaclass(cls.__name__, cls.__bases__, orig_vars) return metaclass(cls.__name__, cls.__bases__, orig_vars)
return wrapper return wrapper
@ -834,12 +898,13 @@ def python_2_unicode_compatible(klass):
returning text and apply this decorator to the class. returning text and apply this decorator to the class.
""" """
if PY2: if PY2:
if '__str__' not in klass.__dict__: if "__str__" not in klass.__dict__:
raise ValueError("@python_2_unicode_compatible cannot be applied " raise ValueError(
"to %s because it doesn't define __str__()." % "@python_2_unicode_compatible cannot be applied "
klass.__name__) "to %s because it doesn't define __str__()." % klass.__name__
)
klass.__unicode__ = klass.__str__ klass.__unicode__ = klass.__str__
klass.__str__ = lambda self: self.__unicode__().encode('utf-8') klass.__str__ = lambda self: self.__unicode__().encode("utf-8")
return klass return klass
@ -859,8 +924,10 @@ if sys.meta_path:
# be floating around. Therefore, we can't use isinstance() to check for # be floating around. Therefore, we can't use isinstance() to check for
# the six meta path importer, since the other six instance will have # the six meta path importer, since the other six instance will have
# inserted an importer with different class. # inserted an importer with different class.
if (type(importer).__name__ == "_SixMetaPathImporter" and if (
importer.name == __name__): type(importer).__name__ == "_SixMetaPathImporter"
and importer.name == __name__
):
del sys.meta_path[i] del sys.meta_path[i]
break break
del i, importer del i, importer

2
lib/urllib3/packages/ssl_match_hostname/__init__.py

@ -16,4 +16,4 @@ except ImportError:
from ._implementation import CertificateError, match_hostname from ._implementation import CertificateError, match_hostname
# Not needed, but documenting what we provide. # Not needed, but documenting what we provide.
__all__ = ('CertificateError', 'match_hostname') __all__ = ("CertificateError", "match_hostname")

58
lib/urllib3/packages/ssl_match_hostname/_implementation.py

@ -15,7 +15,7 @@ try:
except ImportError: except ImportError:
ipaddress = None ipaddress = None
__version__ = '3.5.0.1' __version__ = "3.5.0.1"
class CertificateError(ValueError): class CertificateError(ValueError):
@ -33,18 +33,19 @@ def _dnsname_match(dn, hostname, max_wildcards=1):
# Ported from python3-syntax: # Ported from python3-syntax:
# leftmost, *remainder = dn.split(r'.') # leftmost, *remainder = dn.split(r'.')
parts = dn.split(r'.') parts = dn.split(r".")
leftmost = parts[0] leftmost = parts[0]
remainder = parts[1:] remainder = parts[1:]
wildcards = leftmost.count('*') wildcards = leftmost.count("*")
if wildcards > max_wildcards: if wildcards > max_wildcards:
# Issue #17980: avoid denials of service by refusing more # Issue #17980: avoid denials of service by refusing more
# than one wildcard per fragment. A survey of established # than one wildcard per fragment. A survey of established
# policy among SSL implementations showed it to be a # policy among SSL implementations showed it to be a
# reasonable choice. # reasonable choice.
raise CertificateError( raise CertificateError(
"too many wildcards in certificate DNS name: " + repr(dn)) "too many wildcards in certificate DNS name: " + repr(dn)
)
# speed up common case w/o wildcards # speed up common case w/o wildcards
if not wildcards: if not wildcards:
@ -53,11 +54,11 @@ def _dnsname_match(dn, hostname, max_wildcards=1):
# RFC 6125, section 6.4.3, subitem 1. # RFC 6125, section 6.4.3, subitem 1.
# The client SHOULD NOT attempt to match a presented identifier in which # The client SHOULD NOT attempt to match a presented identifier in which
# the wildcard character comprises a label other than the left-most label. # the wildcard character comprises a label other than the left-most label.
if leftmost == '*': if leftmost == "*":
# When '*' is a fragment by itself, it matches a non-empty dotless # When '*' is a fragment by itself, it matches a non-empty dotless
# fragment. # fragment.
pats.append('[^.]+') pats.append("[^.]+")
elif leftmost.startswith('xn--') or hostname.startswith('xn--'): elif leftmost.startswith("xn--") or hostname.startswith("xn--"):
# RFC 6125, section 6.4.3, subitem 3. # RFC 6125, section 6.4.3, subitem 3.
# The client SHOULD NOT attempt to match a presented identifier # The client SHOULD NOT attempt to match a presented identifier
# where the wildcard character is embedded within an A-label or # where the wildcard character is embedded within an A-label or
@ -65,21 +66,22 @@ def _dnsname_match(dn, hostname, max_wildcards=1):
pats.append(re.escape(leftmost)) pats.append(re.escape(leftmost))
else: else:
# Otherwise, '*' matches any dotless string, e.g. www* # Otherwise, '*' matches any dotless string, e.g. www*
pats.append(re.escape(leftmost).replace(r'\*', '[^.]*')) pats.append(re.escape(leftmost).replace(r"\*", "[^.]*"))
# add the remaining fragments, ignore any wildcards # add the remaining fragments, ignore any wildcards
for frag in remainder: for frag in remainder:
pats.append(re.escape(frag)) pats.append(re.escape(frag))
pat = re.compile(r'\A' + r'\.'.join(pats) + r'\Z', re.IGNORECASE) pat = re.compile(r"\A" + r"\.".join(pats) + r"\Z", re.IGNORECASE)
return pat.match(hostname) return pat.match(hostname)
def _to_unicode(obj): def _to_unicode(obj):
if isinstance(obj, str) and sys.version_info < (3,): if isinstance(obj, str) and sys.version_info < (3,):
obj = unicode(obj, encoding='ascii', errors='strict') obj = unicode(obj, encoding="ascii", errors="strict")
return obj return obj
def _ipaddress_match(ipname, host_ip): def _ipaddress_match(ipname, host_ip):
"""Exact matching of IP addresses. """Exact matching of IP addresses.
@ -101,9 +103,11 @@ def match_hostname(cert, hostname):
returns nothing. returns nothing.
""" """
if not cert: if not cert:
raise ValueError("empty or no certificate, match_hostname needs a " raise ValueError(
"SSL socket or SSL context with either " "empty or no certificate, match_hostname needs a "
"CERT_OPTIONAL or CERT_REQUIRED") "SSL socket or SSL context with either "
"CERT_OPTIONAL or CERT_REQUIRED"
)
try: try:
# Divergence from upstream: ipaddress can't handle byte str # Divergence from upstream: ipaddress can't handle byte str
host_ip = ipaddress.ip_address(_to_unicode(hostname)) host_ip = ipaddress.ip_address(_to_unicode(hostname))
@ -122,35 +126,37 @@ def match_hostname(cert, hostname):
else: else:
raise raise
dnsnames = [] dnsnames = []
san = cert.get('subjectAltName', ()) san = cert.get("subjectAltName", ())
for key, value in san: for key, value in san:
if key == 'DNS': if key == "DNS":
if host_ip is None and _dnsname_match(value, hostname): if host_ip is None and _dnsname_match(value, hostname):
return return
dnsnames.append(value) dnsnames.append(value)
elif key == 'IP Address': elif key == "IP Address":
if host_ip is not None and _ipaddress_match(value, host_ip): if host_ip is not None and _ipaddress_match(value, host_ip):
return return
dnsnames.append(value) dnsnames.append(value)
if not dnsnames: if not dnsnames:
# The subject is only checked when there is no dNSName entry # The subject is only checked when there is no dNSName entry
# in subjectAltName # in subjectAltName
for sub in cert.get('subject', ()): for sub in cert.get("subject", ()):
for key, value in sub: for key, value in sub:
# XXX according to RFC 2818, the most specific Common Name # XXX according to RFC 2818, the most specific Common Name
# must be used. # must be used.
if key == 'commonName': if key == "commonName":
if _dnsname_match(value, hostname): if _dnsname_match(value, hostname):
return return
dnsnames.append(value) dnsnames.append(value)
if len(dnsnames) > 1: if len(dnsnames) > 1:
raise CertificateError("hostname %r " raise CertificateError(
"doesn't match either of %s" "hostname %r "
% (hostname, ', '.join(map(repr, dnsnames)))) "doesn't match either of %s" % (hostname, ", ".join(map(repr, dnsnames)))
)
elif len(dnsnames) == 1: elif len(dnsnames) == 1:
raise CertificateError("hostname %r " raise CertificateError(
"doesn't match %r" "hostname %r " "doesn't match %r" % (hostname, dnsnames[0])
% (hostname, dnsnames[0])) )
else: else:
raise CertificateError("no appropriate commonName or " raise CertificateError(
"subjectAltName fields were found") "no appropriate commonName or " "subjectAltName fields were found"
)

183
lib/urllib3/poolmanager.py

@ -14,48 +14,55 @@ from .util.url import parse_url
from .util.retry import Retry from .util.retry import Retry
__all__ = ['PoolManager', 'ProxyManager', 'proxy_from_url'] __all__ = ["PoolManager", "ProxyManager", "proxy_from_url"]
log = logging.getLogger(__name__) log = logging.getLogger(__name__)
SSL_KEYWORDS = ('key_file', 'cert_file', 'cert_reqs', 'ca_certs', SSL_KEYWORDS = (
'ssl_version', 'ca_cert_dir', 'ssl_context', "key_file",
'key_password') "cert_file",
"cert_reqs",
"ca_certs",
"ssl_version",
"ca_cert_dir",
"ssl_context",
"key_password",
)
# All known keyword arguments that could be provided to the pool manager, its # All known keyword arguments that could be provided to the pool manager, its
# pools, or the underlying connections. This is used to construct a pool key. # pools, or the underlying connections. This is used to construct a pool key.
_key_fields = ( _key_fields = (
'key_scheme', # str "key_scheme", # str
'key_host', # str "key_host", # str
'key_port', # int "key_port", # int
'key_timeout', # int or float or Timeout "key_timeout", # int or float or Timeout
'key_retries', # int or Retry "key_retries", # int or Retry
'key_strict', # bool "key_strict", # bool
'key_block', # bool "key_block", # bool
'key_source_address', # str "key_source_address", # str
'key_key_file', # str "key_key_file", # str
'key_key_password', # str "key_key_password", # str
'key_cert_file', # str "key_cert_file", # str
'key_cert_reqs', # str "key_cert_reqs", # str
'key_ca_certs', # str "key_ca_certs", # str
'key_ssl_version', # str "key_ssl_version", # str
'key_ca_cert_dir', # str "key_ca_cert_dir", # str
'key_ssl_context', # instance of ssl.SSLContext or urllib3.util.ssl_.SSLContext "key_ssl_context", # instance of ssl.SSLContext or urllib3.util.ssl_.SSLContext
'key_maxsize', # int "key_maxsize", # int
'key_headers', # dict "key_headers", # dict
'key__proxy', # parsed proxy url "key__proxy", # parsed proxy url
'key__proxy_headers', # dict "key__proxy_headers", # dict
'key_socket_options', # list of (level (int), optname (int), value (int or str)) tuples "key_socket_options", # list of (level (int), optname (int), value (int or str)) tuples
'key__socks_options', # dict "key__socks_options", # dict
'key_assert_hostname', # bool or string "key_assert_hostname", # bool or string
'key_assert_fingerprint', # str "key_assert_fingerprint", # str
'key_server_hostname', # str "key_server_hostname", # str
) )
#: The namedtuple class used to construct keys for the connection pool. #: The namedtuple class used to construct keys for the connection pool.
#: All custom key schemes should include the fields in this key at a minimum. #: All custom key schemes should include the fields in this key at a minimum.
PoolKey = collections.namedtuple('PoolKey', _key_fields) PoolKey = collections.namedtuple("PoolKey", _key_fields)
def _default_key_normalizer(key_class, request_context): def _default_key_normalizer(key_class, request_context):
@ -80,24 +87,24 @@ def _default_key_normalizer(key_class, request_context):
""" """
# Since we mutate the dictionary, make a copy first # Since we mutate the dictionary, make a copy first
context = request_context.copy() context = request_context.copy()
context['scheme'] = context['scheme'].lower() context["scheme"] = context["scheme"].lower()
context['host'] = context['host'].lower() context["host"] = context["host"].lower()
# These are both dictionaries and need to be transformed into frozensets # These are both dictionaries and need to be transformed into frozensets
for key in ('headers', '_proxy_headers', '_socks_options'): for key in ("headers", "_proxy_headers", "_socks_options"):
if key in context and context[key] is not None: if key in context and context[key] is not None:
context[key] = frozenset(context[key].items()) context[key] = frozenset(context[key].items())
# The socket_options key may be a list and needs to be transformed into a # The socket_options key may be a list and needs to be transformed into a
# tuple. # tuple.
socket_opts = context.get('socket_options') socket_opts = context.get("socket_options")
if socket_opts is not None: if socket_opts is not None:
context['socket_options'] = tuple(socket_opts) context["socket_options"] = tuple(socket_opts)
# Map the kwargs to the names in the namedtuple - this is necessary since # Map the kwargs to the names in the namedtuple - this is necessary since
# namedtuples can't have fields starting with '_'. # namedtuples can't have fields starting with '_'.
for key in list(context.keys()): for key in list(context.keys()):
context['key_' + key] = context.pop(key) context["key_" + key] = context.pop(key)
# Default to ``None`` for keys missing from the context # Default to ``None`` for keys missing from the context
for field in key_class._fields: for field in key_class._fields:
@ -112,14 +119,11 @@ def _default_key_normalizer(key_class, request_context):
#: Each PoolManager makes a copy of this dictionary so they can be configured #: Each PoolManager makes a copy of this dictionary so they can be configured
#: globally here, or individually on the instance. #: globally here, or individually on the instance.
key_fn_by_scheme = { key_fn_by_scheme = {
'http': functools.partial(_default_key_normalizer, PoolKey), "http": functools.partial(_default_key_normalizer, PoolKey),
'https': functools.partial(_default_key_normalizer, PoolKey), "https": functools.partial(_default_key_normalizer, PoolKey),
} }
pool_classes_by_scheme = { pool_classes_by_scheme = {"http": HTTPConnectionPool, "https": HTTPSConnectionPool}
'http': HTTPConnectionPool,
'https': HTTPSConnectionPool,
}
class PoolManager(RequestMethods): class PoolManager(RequestMethods):
@ -155,8 +159,7 @@ class PoolManager(RequestMethods):
def __init__(self, num_pools=10, headers=None, **connection_pool_kw): def __init__(self, num_pools=10, headers=None, **connection_pool_kw):
RequestMethods.__init__(self, headers) RequestMethods.__init__(self, headers)
self.connection_pool_kw = connection_pool_kw self.connection_pool_kw = connection_pool_kw
self.pools = RecentlyUsedContainer(num_pools, self.pools = RecentlyUsedContainer(num_pools, dispose_func=lambda p: p.close())
dispose_func=lambda p: p.close())
# Locally set the pool classes and keys so other PoolManagers can # Locally set the pool classes and keys so other PoolManagers can
# override them. # override them.
@ -189,10 +192,10 @@ class PoolManager(RequestMethods):
# this function has historically only used the scheme, host, and port # this function has historically only used the scheme, host, and port
# in the positional args. When an API change is acceptable these can # in the positional args. When an API change is acceptable these can
# be removed. # be removed.
for key in ('scheme', 'host', 'port'): for key in ("scheme", "host", "port"):
request_context.pop(key, None) request_context.pop(key, None)
if scheme == 'http': if scheme == "http":
for kw in SSL_KEYWORDS: for kw in SSL_KEYWORDS:
request_context.pop(kw, None) request_context.pop(kw, None)
@ -207,7 +210,7 @@ class PoolManager(RequestMethods):
""" """
self.pools.clear() self.pools.clear()
def connection_from_host(self, host, port=None, scheme='http', pool_kwargs=None): def connection_from_host(self, host, port=None, scheme="http", pool_kwargs=None):
""" """
Get a :class:`ConnectionPool` based on the host, port, and scheme. Get a :class:`ConnectionPool` based on the host, port, and scheme.
@ -222,11 +225,11 @@ class PoolManager(RequestMethods):
raise LocationValueError("No host specified.") raise LocationValueError("No host specified.")
request_context = self._merge_pool_kwargs(pool_kwargs) request_context = self._merge_pool_kwargs(pool_kwargs)
request_context['scheme'] = scheme or 'http' request_context["scheme"] = scheme or "http"
if not port: if not port:
port = port_by_scheme.get(request_context['scheme'].lower(), 80) port = port_by_scheme.get(request_context["scheme"].lower(), 80)
request_context['port'] = port request_context["port"] = port
request_context['host'] = host request_context["host"] = host
return self.connection_from_context(request_context) return self.connection_from_context(request_context)
@ -237,7 +240,7 @@ class PoolManager(RequestMethods):
``request_context`` must at least contain the ``scheme`` key and its ``request_context`` must at least contain the ``scheme`` key and its
value must be a key in ``key_fn_by_scheme`` instance variable. value must be a key in ``key_fn_by_scheme`` instance variable.
""" """
scheme = request_context['scheme'].lower() scheme = request_context["scheme"].lower()
pool_key_constructor = self.key_fn_by_scheme[scheme] pool_key_constructor = self.key_fn_by_scheme[scheme]
pool_key = pool_key_constructor(request_context) pool_key = pool_key_constructor(request_context)
@ -259,9 +262,9 @@ class PoolManager(RequestMethods):
return pool return pool
# Make a fresh ConnectionPool of the desired type # Make a fresh ConnectionPool of the desired type
scheme = request_context['scheme'] scheme = request_context["scheme"]
host = request_context['host'] host = request_context["host"]
port = request_context['port'] port = request_context["port"]
pool = self._new_pool(scheme, host, port, request_context=request_context) pool = self._new_pool(scheme, host, port, request_context=request_context)
self.pools[pool_key] = pool self.pools[pool_key] = pool
@ -279,8 +282,9 @@ class PoolManager(RequestMethods):
not used. not used.
""" """
u = parse_url(url) u = parse_url(url)
return self.connection_from_host(u.host, port=u.port, scheme=u.scheme, return self.connection_from_host(
pool_kwargs=pool_kwargs) u.host, port=u.port, scheme=u.scheme, pool_kwargs=pool_kwargs
)
def _merge_pool_kwargs(self, override): def _merge_pool_kwargs(self, override):
""" """
@ -314,11 +318,11 @@ class PoolManager(RequestMethods):
u = parse_url(url) u = parse_url(url)
conn = self.connection_from_host(u.host, port=u.port, scheme=u.scheme) conn = self.connection_from_host(u.host, port=u.port, scheme=u.scheme)
kw['assert_same_host'] = False kw["assert_same_host"] = False
kw['redirect'] = False kw["redirect"] = False
if 'headers' not in kw: if "headers" not in kw:
kw['headers'] = self.headers.copy() kw["headers"] = self.headers.copy()
if self.proxy is not None and u.scheme == "http": if self.proxy is not None and u.scheme == "http":
response = conn.urlopen(method, url, **kw) response = conn.urlopen(method, url, **kw)
@ -334,21 +338,22 @@ class PoolManager(RequestMethods):
# RFC 7231, Section 6.4.4 # RFC 7231, Section 6.4.4
if response.status == 303: if response.status == 303:
method = 'GET' method = "GET"
retries = kw.get('retries') retries = kw.get("retries")
if not isinstance(retries, Retry): if not isinstance(retries, Retry):
retries = Retry.from_int(retries, redirect=redirect) retries = Retry.from_int(retries, redirect=redirect)
# Strip headers marked as unsafe to forward to the redirected location. # Strip headers marked as unsafe to forward to the redirected location.
# Check remove_headers_on_redirect to avoid a potential network call within # Check remove_headers_on_redirect to avoid a potential network call within
# conn.is_same_host() which may use socket.gethostbyname() in the future. # conn.is_same_host() which may use socket.gethostbyname() in the future.
if (retries.remove_headers_on_redirect if retries.remove_headers_on_redirect and not conn.is_same_host(
and not conn.is_same_host(redirect_location)): redirect_location
headers = list(six.iterkeys(kw['headers'])) ):
headers = list(six.iterkeys(kw["headers"]))
for header in headers: for header in headers:
if header.lower() in retries.remove_headers_on_redirect: if header.lower() in retries.remove_headers_on_redirect:
kw['headers'].pop(header, None) kw["headers"].pop(header, None)
try: try:
retries = retries.increment(method, url, response=response, _pool=conn) retries = retries.increment(method, url, response=response, _pool=conn)
@ -357,8 +362,8 @@ class PoolManager(RequestMethods):
raise raise
return response return response
kw['retries'] = retries kw["retries"] = retries
kw['redirect'] = redirect kw["redirect"] = redirect
log.info("Redirecting %s -> %s", url, redirect_location) log.info("Redirecting %s -> %s", url, redirect_location)
return self.urlopen(method, redirect_location, **kw) return self.urlopen(method, redirect_location, **kw)
@ -391,12 +396,21 @@ class ProxyManager(PoolManager):
""" """
def __init__(self, proxy_url, num_pools=10, headers=None, def __init__(
proxy_headers=None, **connection_pool_kw): self,
proxy_url,
num_pools=10,
headers=None,
proxy_headers=None,
**connection_pool_kw
):
if isinstance(proxy_url, HTTPConnectionPool): if isinstance(proxy_url, HTTPConnectionPool):
proxy_url = '%s://%s:%i' % (proxy_url.scheme, proxy_url.host, proxy_url = "%s://%s:%i" % (
proxy_url.port) proxy_url.scheme,
proxy_url.host,
proxy_url.port,
)
proxy = parse_url(proxy_url) proxy = parse_url(proxy_url)
if not proxy.port: if not proxy.port:
port = port_by_scheme.get(proxy.scheme, 80) port = port_by_scheme.get(proxy.scheme, 80)
@ -408,30 +422,31 @@ class ProxyManager(PoolManager):
self.proxy = proxy self.proxy = proxy
self.proxy_headers = proxy_headers or {} self.proxy_headers = proxy_headers or {}
connection_pool_kw['_proxy'] = self.proxy connection_pool_kw["_proxy"] = self.proxy
connection_pool_kw['_proxy_headers'] = self.proxy_headers connection_pool_kw["_proxy_headers"] = self.proxy_headers
super(ProxyManager, self).__init__( super(ProxyManager, self).__init__(num_pools, headers, **connection_pool_kw)
num_pools, headers, **connection_pool_kw)
def connection_from_host(self, host, port=None, scheme='http', pool_kwargs=None): def connection_from_host(self, host, port=None, scheme="http", pool_kwargs=None):
if scheme == "https": if scheme == "https":
return super(ProxyManager, self).connection_from_host( return super(ProxyManager, self).connection_from_host(
host, port, scheme, pool_kwargs=pool_kwargs) host, port, scheme, pool_kwargs=pool_kwargs
)
return super(ProxyManager, self).connection_from_host( return super(ProxyManager, self).connection_from_host(
self.proxy.host, self.proxy.port, self.proxy.scheme, pool_kwargs=pool_kwargs) self.proxy.host, self.proxy.port, self.proxy.scheme, pool_kwargs=pool_kwargs
)
def _set_proxy_headers(self, url, headers=None): def _set_proxy_headers(self, url, headers=None):
""" """
Sets headers needed by proxies: specifically, the Accept and Host Sets headers needed by proxies: specifically, the Accept and Host
headers. Only sets headers not provided by the user. headers. Only sets headers not provided by the user.
""" """
headers_ = {'Accept': '*/*'} headers_ = {"Accept": "*/*"}
netloc = parse_url(url).netloc netloc = parse_url(url).netloc
if netloc: if netloc:
headers_['Host'] = netloc headers_["Host"] = netloc
if headers: if headers:
headers_.update(headers) headers_.update(headers)
@ -445,8 +460,8 @@ class ProxyManager(PoolManager):
# For proxied HTTPS requests, httplib sets the necessary headers # For proxied HTTPS requests, httplib sets the necessary headers
# on the CONNECT to the proxy. For HTTP, we'll definitely # on the CONNECT to the proxy. For HTTP, we'll definitely
# need to set 'Host' at the very least. # need to set 'Host' at the very least.
headers = kw.get('headers', self.headers) headers = kw.get("headers", self.headers)
kw['headers'] = self._set_proxy_headers(url, headers) kw["headers"] = self._set_proxy_headers(url, headers)
return super(ProxyManager, self).urlopen(method, url, redirect=redirect, **kw) return super(ProxyManager, self).urlopen(method, url, redirect=redirect, **kw)

79
lib/urllib3/request.py

@ -4,7 +4,7 @@ from .filepost import encode_multipart_formdata
from .packages.six.moves.urllib.parse import urlencode from .packages.six.moves.urllib.parse import urlencode
__all__ = ['RequestMethods'] __all__ = ["RequestMethods"]
class RequestMethods(object): class RequestMethods(object):
@ -36,16 +36,25 @@ class RequestMethods(object):
explicitly. explicitly.
""" """
_encode_url_methods = {'DELETE', 'GET', 'HEAD', 'OPTIONS'} _encode_url_methods = {"DELETE", "GET", "HEAD", "OPTIONS"}
def __init__(self, headers=None): def __init__(self, headers=None):
self.headers = headers or {} self.headers = headers or {}
def urlopen(self, method, url, body=None, headers=None, def urlopen(
encode_multipart=True, multipart_boundary=None, self,
**kw): # Abstract method,
raise NotImplementedError("Classes extending RequestMethods must implement " url,
"their own ``urlopen`` method.") body=None,
headers=None,
encode_multipart=True,
multipart_boundary=None,
**kw
): # Abstract
raise NotImplementedError(
"Classes extending RequestMethods must implement "
"their own ``urlopen`` method."
)
def request(self, method, url, fields=None, headers=None, **urlopen_kw): def request(self, method, url, fields=None, headers=None, **urlopen_kw):
""" """
@ -60,19 +69,18 @@ class RequestMethods(object):
""" """
method = method.upper() method = method.upper()
urlopen_kw['request_url'] = url urlopen_kw["request_url"] = url
if method in self._encode_url_methods: if method in self._encode_url_methods:
return self.request_encode_url(method, url, fields=fields, return self.request_encode_url(
headers=headers, method, url, fields=fields, headers=headers, **urlopen_kw
**urlopen_kw) )
else: else:
return self.request_encode_body(method, url, fields=fields, return self.request_encode_body(
headers=headers, method, url, fields=fields, headers=headers, **urlopen_kw
**urlopen_kw) )
def request_encode_url(self, method, url, fields=None, headers=None, def request_encode_url(self, method, url, fields=None, headers=None, **urlopen_kw):
**urlopen_kw):
""" """
Make a request using :meth:`urlopen` with the ``fields`` encoded in Make a request using :meth:`urlopen` with the ``fields`` encoded in
the url. This is useful for request methods like GET, HEAD, DELETE, etc. the url. This is useful for request methods like GET, HEAD, DELETE, etc.
@ -80,17 +88,24 @@ class RequestMethods(object):
if headers is None: if headers is None:
headers = self.headers headers = self.headers
extra_kw = {'headers': headers} extra_kw = {"headers": headers}
extra_kw.update(urlopen_kw) extra_kw.update(urlopen_kw)
if fields: if fields:
url += '?' + urlencode(fields) url += "?" + urlencode(fields)
return self.urlopen(method, url, **extra_kw) return self.urlopen(method, url, **extra_kw)
def request_encode_body(self, method, url, fields=None, headers=None, def request_encode_body(
encode_multipart=True, multipart_boundary=None, self,
**urlopen_kw): method,
url,
fields=None,
headers=None,
encode_multipart=True,
multipart_boundary=None,
**urlopen_kw
):
""" """
Make a request using :meth:`urlopen` with the ``fields`` encoded in Make a request using :meth:`urlopen` with the ``fields`` encoded in
the body. This is useful for request methods like POST, PUT, PATCH, etc. the body. This is useful for request methods like POST, PUT, PATCH, etc.
@ -129,22 +144,28 @@ class RequestMethods(object):
if headers is None: if headers is None:
headers = self.headers headers = self.headers
extra_kw = {'headers': {}} extra_kw = {"headers": {}}
if fields: if fields:
if 'body' in urlopen_kw: if "body" in urlopen_kw:
raise TypeError( raise TypeError(
"request got values for both 'fields' and 'body', can only specify one.") "request got values for both 'fields' and 'body', can only specify one."
)
if encode_multipart: if encode_multipart:
body, content_type = encode_multipart_formdata(fields, boundary=multipart_boundary) body, content_type = encode_multipart_formdata(
fields, boundary=multipart_boundary
)
else: else:
body, content_type = urlencode(fields), 'application/x-www-form-urlencoded' body, content_type = (
urlencode(fields),
"application/x-www-form-urlencoded",
)
extra_kw['body'] = body extra_kw["body"] = body
extra_kw['headers'] = {'Content-Type': content_type} extra_kw["headers"] = {"Content-Type": content_type}
extra_kw['headers'].update(headers) extra_kw["headers"].update(headers)
extra_kw.update(urlopen_kw) extra_kw.update(urlopen_kw)
return self.urlopen(method, url, **extra_kw) return self.urlopen(method, url, **extra_kw)

176
lib/urllib3/response.py

@ -13,8 +13,13 @@ except ImportError:
from ._collections import HTTPHeaderDict from ._collections import HTTPHeaderDict
from .exceptions import ( from .exceptions import (
BodyNotHttplibCompatible, ProtocolError, DecodeError, ReadTimeoutError, BodyNotHttplibCompatible,
ResponseNotChunked, IncompleteRead, InvalidHeader ProtocolError,
DecodeError,
ReadTimeoutError,
ResponseNotChunked,
IncompleteRead,
InvalidHeader,
) )
from .packages.six import string_types as basestring, PY3 from .packages.six import string_types as basestring, PY3
from .packages.six.moves import http_client as httplib from .packages.six.moves import http_client as httplib
@ -25,10 +30,9 @@ log = logging.getLogger(__name__)
class DeflateDecoder(object): class DeflateDecoder(object):
def __init__(self): def __init__(self):
self._first_try = True self._first_try = True
self._data = b'' self._data = b""
self._obj = zlib.decompressobj() self._obj = zlib.decompressobj()
def __getattr__(self, name): def __getattr__(self, name):
@ -65,7 +69,6 @@ class GzipDecoderState(object):
class GzipDecoder(object): class GzipDecoder(object):
def __init__(self): def __init__(self):
self._obj = zlib.decompressobj(16 + zlib.MAX_WBITS) self._obj = zlib.decompressobj(16 + zlib.MAX_WBITS)
self._state = GzipDecoderState.FIRST_MEMBER self._state = GzipDecoderState.FIRST_MEMBER
@ -96,6 +99,7 @@ class GzipDecoder(object):
if brotli is not None: if brotli is not None:
class BrotliDecoder(object): class BrotliDecoder(object):
# Supports both 'brotlipy' and 'Brotli' packages # Supports both 'brotlipy' and 'Brotli' packages
# since they share an import name. The top branches # since they share an import name. The top branches
@ -104,14 +108,14 @@ if brotli is not None:
self._obj = brotli.Decompressor() self._obj = brotli.Decompressor()
def decompress(self, data): def decompress(self, data):
if hasattr(self._obj, 'decompress'): if hasattr(self._obj, "decompress"):
return self._obj.decompress(data) return self._obj.decompress(data)
return self._obj.process(data) return self._obj.process(data)
def flush(self): def flush(self):
if hasattr(self._obj, 'flush'): if hasattr(self._obj, "flush"):
return self._obj.flush() return self._obj.flush()
return b'' return b""
class MultiDecoder(object): class MultiDecoder(object):
@ -124,7 +128,7 @@ class MultiDecoder(object):
""" """
def __init__(self, modes): def __init__(self, modes):
self._decoders = [_get_decoder(m.strip()) for m in modes.split(',')] self._decoders = [_get_decoder(m.strip()) for m in modes.split(",")]
def flush(self): def flush(self):
return self._decoders[0].flush() return self._decoders[0].flush()
@ -136,13 +140,13 @@ class MultiDecoder(object):
def _get_decoder(mode): def _get_decoder(mode):
if ',' in mode: if "," in mode:
return MultiDecoder(mode) return MultiDecoder(mode)
if mode == 'gzip': if mode == "gzip":
return GzipDecoder() return GzipDecoder()
if brotli is not None and mode == 'br': if brotli is not None and mode == "br":
return BrotliDecoder() return BrotliDecoder()
return DeflateDecoder() return DeflateDecoder()
@ -181,16 +185,30 @@ class HTTPResponse(io.IOBase):
value of Content-Length header, if present. Otherwise, raise error. value of Content-Length header, if present. Otherwise, raise error.
""" """
CONTENT_DECODERS = ['gzip', 'deflate'] CONTENT_DECODERS = ["gzip", "deflate"]
if brotli is not None: if brotli is not None:
CONTENT_DECODERS += ['br'] CONTENT_DECODERS += ["br"]
REDIRECT_STATUSES = [301, 302, 303, 307, 308] REDIRECT_STATUSES = [301, 302, 303, 307, 308]
def __init__(self, body='', headers=None, status=0, version=0, reason=None, def __init__(
strict=0, preload_content=True, decode_content=True, self,
original_response=None, pool=None, connection=None, msg=None, body="",
retries=None, enforce_content_length=False, headers=None,
request_method=None, request_url=None): status=0,
version=0,
reason=None,
strict=0,
preload_content=True,
decode_content=True,
original_response=None,
pool=None,
connection=None,
msg=None,
retries=None,
enforce_content_length=False,
request_method=None,
request_url=None,
):
if isinstance(headers, HTTPHeaderDict): if isinstance(headers, HTTPHeaderDict):
self.headers = headers self.headers = headers
@ -218,13 +236,13 @@ class HTTPResponse(io.IOBase):
self._pool = pool self._pool = pool
self._connection = connection self._connection = connection
if hasattr(body, 'read'): if hasattr(body, "read"):
self._fp = body self._fp = body
# Are we using the chunked-style of transfer encoding? # Are we using the chunked-style of transfer encoding?
self.chunked = False self.chunked = False
self.chunk_left = None self.chunk_left = None
tr_enc = self.headers.get('transfer-encoding', '').lower() tr_enc = self.headers.get("transfer-encoding", "").lower()
# Don't incur the penalty of creating a list and then discarding it # Don't incur the penalty of creating a list and then discarding it
encodings = (enc.strip() for enc in tr_enc.split(",")) encodings = (enc.strip() for enc in tr_enc.split(","))
if "chunked" in encodings: if "chunked" in encodings:
@ -246,7 +264,7 @@ class HTTPResponse(io.IOBase):
location. ``False`` if not a redirect status code. location. ``False`` if not a redirect status code.
""" """
if self.status in self.REDIRECT_STATUSES: if self.status in self.REDIRECT_STATUSES:
return self.headers.get('location') return self.headers.get("location")
return False return False
@ -285,18 +303,20 @@ class HTTPResponse(io.IOBase):
""" """
Set initial length value for Response content if available. Set initial length value for Response content if available.
""" """
length = self.headers.get('content-length') length = self.headers.get("content-length")
if length is not None: if length is not None:
if self.chunked: if self.chunked:
# This Response will fail with an IncompleteRead if it can't be # This Response will fail with an IncompleteRead if it can't be
# received as chunked. This method falls back to attempt reading # received as chunked. This method falls back to attempt reading
# the response before raising an exception. # the response before raising an exception.
log.warning("Received response with both Content-Length and " log.warning(
"Transfer-Encoding set. This is expressly forbidden " "Received response with both Content-Length and "
"by RFC 7230 sec 3.3.2. Ignoring Content-Length and " "Transfer-Encoding set. This is expressly forbidden "
"attempting to process response as Transfer-Encoding: " "by RFC 7230 sec 3.3.2. Ignoring Content-Length and "
"chunked.") "attempting to process response as Transfer-Encoding: "
"chunked."
)
return None return None
try: try:
@ -305,10 +325,12 @@ class HTTPResponse(io.IOBase):
# (e.g. Content-Length: 42, 42). This line ensures the values # (e.g. Content-Length: 42, 42). This line ensures the values
# are all valid ints and that as long as the `set` length is 1, # are all valid ints and that as long as the `set` length is 1,
# all values are the same. Otherwise, the header is invalid. # all values are the same. Otherwise, the header is invalid.
lengths = set([int(val) for val in length.split(',')]) lengths = set([int(val) for val in length.split(",")])
if len(lengths) > 1: if len(lengths) > 1:
raise InvalidHeader("Content-Length contained multiple " raise InvalidHeader(
"unmatching values (%s)" % length) "Content-Length contained multiple "
"unmatching values (%s)" % length
)
length = lengths.pop() length = lengths.pop()
except ValueError: except ValueError:
length = None length = None
@ -324,7 +346,7 @@ class HTTPResponse(io.IOBase):
status = 0 status = 0
# Check for responses that shouldn't include a body # Check for responses that shouldn't include a body
if status in (204, 304) or 100 <= status < 200 or request_method == 'HEAD': if status in (204, 304) or 100 <= status < 200 or request_method == "HEAD":
length = 0 length = 0
return length return length
@ -335,14 +357,16 @@ class HTTPResponse(io.IOBase):
""" """
# Note: content-encoding value should be case-insensitive, per RFC 7230 # Note: content-encoding value should be case-insensitive, per RFC 7230
# Section 3.2 # Section 3.2
content_encoding = self.headers.get('content-encoding', '').lower() content_encoding = self.headers.get("content-encoding", "").lower()
if self._decoder is None: if self._decoder is None:
if content_encoding in self.CONTENT_DECODERS: if content_encoding in self.CONTENT_DECODERS:
self._decoder = _get_decoder(content_encoding) self._decoder = _get_decoder(content_encoding)
elif ',' in content_encoding: elif "," in content_encoding:
encodings = [ encodings = [
e.strip() for e in content_encoding.split(',') e.strip()
if e.strip() in self.CONTENT_DECODERS] for e in content_encoding.split(",")
if e.strip() in self.CONTENT_DECODERS
]
if len(encodings): if len(encodings):
self._decoder = _get_decoder(content_encoding) self._decoder = _get_decoder(content_encoding)
@ -361,10 +385,12 @@ class HTTPResponse(io.IOBase):
if self._decoder: if self._decoder:
data = self._decoder.decompress(data) data = self._decoder.decompress(data)
except self.DECODER_ERROR_CLASSES as e: except self.DECODER_ERROR_CLASSES as e:
content_encoding = self.headers.get('content-encoding', '').lower() content_encoding = self.headers.get("content-encoding", "").lower()
raise DecodeError( raise DecodeError(
"Received response with content-encoding: %s, but " "Received response with content-encoding: %s, but "
"failed to decode it." % content_encoding, e) "failed to decode it." % content_encoding,
e,
)
if flush_decoder: if flush_decoder:
data += self._flush_decoder() data += self._flush_decoder()
@ -376,10 +402,10 @@ class HTTPResponse(io.IOBase):
being used. being used.
""" """
if self._decoder: if self._decoder:
buf = self._decoder.decompress(b'') buf = self._decoder.decompress(b"")
return buf + self._decoder.flush() return buf + self._decoder.flush()
return b'' return b""
@contextmanager @contextmanager
def _error_catcher(self): def _error_catcher(self):
@ -399,20 +425,20 @@ class HTTPResponse(io.IOBase):
except SocketTimeout: except SocketTimeout:
# FIXME: Ideally we'd like to include the url in the ReadTimeoutError but # FIXME: Ideally we'd like to include the url in the ReadTimeoutError but
# there is yet no clean way to get at it from this context. # there is yet no clean way to get at it from this context.
raise ReadTimeoutError(self._pool, None, 'Read timed out.') raise ReadTimeoutError(self._pool, None, "Read timed out.")
except BaseSSLError as e: except BaseSSLError as e:
# FIXME: Is there a better way to differentiate between SSLErrors? # FIXME: Is there a better way to differentiate between SSLErrors?
if 'read operation timed out' not in str(e): # Defensive: if "read operation timed out" not in str(e): # Defensive:
# This shouldn't happen but just in case we're missing an edge # This shouldn't happen but just in case we're missing an edge
# case, let's avoid swallowing SSL errors. # case, let's avoid swallowing SSL errors.
raise raise
raise ReadTimeoutError(self._pool, None, 'Read timed out.') raise ReadTimeoutError(self._pool, None, "Read timed out.")
except (HTTPException, SocketError) as e: except (HTTPException, SocketError) as e:
# This includes IncompleteRead. # This includes IncompleteRead.
raise ProtocolError('Connection broken: %r' % e, e) raise ProtocolError("Connection broken: %r" % e, e)
# If no exception is thrown, we should avoid cleaning up # If no exception is thrown, we should avoid cleaning up
# unnecessarily. # unnecessarily.
@ -477,7 +503,9 @@ class HTTPResponse(io.IOBase):
else: else:
cache_content = False cache_content = False
data = self._fp.read(amt) data = self._fp.read(amt)
if amt != 0 and not data: # Platform-specific: Buggy versions of Python. if (
amt != 0 and not data
): # Platform-specific: Buggy versions of Python.
# Close the connection when no data is returned # Close the connection when no data is returned
# #
# This is redundant to what httplib/http.client _should_ # This is redundant to what httplib/http.client _should_
@ -487,7 +515,10 @@ class HTTPResponse(io.IOBase):
# no harm in redundantly calling close. # no harm in redundantly calling close.
self._fp.close() self._fp.close()
flush_decoder = True flush_decoder = True
if self.enforce_content_length and self.length_remaining not in (0, None): if self.enforce_content_length and self.length_remaining not in (
0,
None,
):
# This is an edge case that httplib failed to cover due # This is an edge case that httplib failed to cover due
# to concerns of backward compatibility. We're # to concerns of backward compatibility. We're
# addressing it here to make sure IncompleteRead is # addressing it here to make sure IncompleteRead is
@ -507,7 +538,7 @@ class HTTPResponse(io.IOBase):
return data return data
def stream(self, amt=2**16, decode_content=None): def stream(self, amt=2 ** 16, decode_content=None):
""" """
A generator wrapper for the read() method. A call will block until A generator wrapper for the read() method. A call will block until
``amt`` bytes have been read from the connection or until the ``amt`` bytes have been read from the connection or until the
@ -552,15 +583,17 @@ class HTTPResponse(io.IOBase):
headers = HTTPHeaderDict.from_httplib(headers) headers = HTTPHeaderDict.from_httplib(headers)
# HTTPResponse objects in Python 3 don't have a .strict attribute # HTTPResponse objects in Python 3 don't have a .strict attribute
strict = getattr(r, 'strict', 0) strict = getattr(r, "strict", 0)
resp = ResponseCls(body=r, resp = ResponseCls(
headers=headers, body=r,
status=r.status, headers=headers,
version=r.version, status=r.status,
reason=r.reason, version=r.version,
strict=strict, reason=r.reason,
original_response=r, strict=strict,
**response_kw) original_response=r,
**response_kw
)
return resp return resp
# Backwards-compatibility methods for httplib.HTTPResponse # Backwards-compatibility methods for httplib.HTTPResponse
@ -586,9 +619,9 @@ class HTTPResponse(io.IOBase):
def closed(self): def closed(self):
if self._fp is None: if self._fp is None:
return True return True
elif hasattr(self._fp, 'isclosed'): elif hasattr(self._fp, "isclosed"):
return self._fp.isclosed() return self._fp.isclosed()
elif hasattr(self._fp, 'closed'): elif hasattr(self._fp, "closed"):
return self._fp.closed return self._fp.closed
else: else:
return True return True
@ -599,11 +632,13 @@ class HTTPResponse(io.IOBase):
elif hasattr(self._fp, "fileno"): elif hasattr(self._fp, "fileno"):
return self._fp.fileno() return self._fp.fileno()
else: else:
raise IOError("The file-like object this HTTPResponse is wrapped " raise IOError(
"around has no file descriptor") "The file-like object this HTTPResponse is wrapped "
"around has no file descriptor"
)
def flush(self): def flush(self):
if self._fp is not None and hasattr(self._fp, 'flush'): if self._fp is not None and hasattr(self._fp, "flush"):
return self._fp.flush() return self._fp.flush()
def readable(self): def readable(self):
@ -616,7 +651,7 @@ class HTTPResponse(io.IOBase):
if len(temp) == 0: if len(temp) == 0:
return 0 return 0
else: else:
b[:len(temp)] = temp b[: len(temp)] = temp
return len(temp) return len(temp)
def supports_chunked_reads(self): def supports_chunked_reads(self):
@ -626,7 +661,7 @@ class HTTPResponse(io.IOBase):
attribute. If it is present we assume it returns raw chunks as attribute. If it is present we assume it returns raw chunks as
processed by read_chunked(). processed by read_chunked().
""" """
return hasattr(self._fp, 'fp') return hasattr(self._fp, "fp")
def _update_chunk_length(self): def _update_chunk_length(self):
# First, we'll figure out length of a chunk and then # First, we'll figure out length of a chunk and then
@ -634,7 +669,7 @@ class HTTPResponse(io.IOBase):
if self.chunk_left is not None: if self.chunk_left is not None:
return return
line = self._fp.fp.readline() line = self._fp.fp.readline()
line = line.split(b';', 1)[0] line = line.split(b";", 1)[0]
try: try:
self.chunk_left = int(line, 16) self.chunk_left = int(line, 16)
except ValueError: except ValueError:
@ -683,11 +718,13 @@ class HTTPResponse(io.IOBase):
if not self.chunked: if not self.chunked:
raise ResponseNotChunked( raise ResponseNotChunked(
"Response is not chunked. " "Response is not chunked. "
"Header 'transfer-encoding: chunked' is missing.") "Header 'transfer-encoding: chunked' is missing."
)
if not self.supports_chunked_reads(): if not self.supports_chunked_reads():
raise BodyNotHttplibCompatible( raise BodyNotHttplibCompatible(
"Body should be httplib.HTTPResponse like. " "Body should be httplib.HTTPResponse like. "
"It should have have an fp attribute which returns raw chunks.") "It should have have an fp attribute which returns raw chunks."
)
with self._error_catcher(): with self._error_catcher():
# Don't bother reading the body of a HEAD request. # Don't bother reading the body of a HEAD request.
@ -705,8 +742,9 @@ class HTTPResponse(io.IOBase):
if self.chunk_left == 0: if self.chunk_left == 0:
break break
chunk = self._handle_chunk(amt) chunk = self._handle_chunk(amt)
decoded = self._decode(chunk, decode_content=decode_content, decoded = self._decode(
flush_decoder=False) chunk, decode_content=decode_content, flush_decoder=False
)
if decoded: if decoded:
yield decoded yield decoded
@ -724,7 +762,7 @@ class HTTPResponse(io.IOBase):
if not line: if not line:
# Some sites may not end with '\r\n'. # Some sites may not end with '\r\n'.
break break
if line == b'\r\n': if line == b"\r\n":
break break
# We read everything; close the "file". # We read everything; close the "file".

60
lib/urllib3/util/__init__.py

@ -1,4 +1,5 @@
from __future__ import absolute_import from __future__ import absolute_import
# For backwards compatibility, provide imports that used to be here. # For backwards compatibility, provide imports that used to be here.
from .connection import is_connection_dropped from .connection import is_connection_dropped
from .request import make_headers from .request import make_headers
@ -14,43 +15,32 @@ from .ssl_ import (
ssl_wrap_socket, ssl_wrap_socket,
PROTOCOL_TLS, PROTOCOL_TLS,
) )
from .timeout import ( from .timeout import current_time, Timeout
current_time,
Timeout,
)
from .retry import Retry from .retry import Retry
from .url import ( from .url import get_host, parse_url, split_first, Url
get_host, from .wait import wait_for_read, wait_for_write
parse_url,
split_first,
Url,
)
from .wait import (
wait_for_read,
wait_for_write
)
__all__ = ( __all__ = (
'HAS_SNI', "HAS_SNI",
'IS_PYOPENSSL', "IS_PYOPENSSL",
'IS_SECURETRANSPORT', "IS_SECURETRANSPORT",
'SSLContext', "SSLContext",
'PROTOCOL_TLS', "PROTOCOL_TLS",
'Retry', "Retry",
'Timeout', "Timeout",
'Url', "Url",
'assert_fingerprint', "assert_fingerprint",
'current_time', "current_time",
'is_connection_dropped', "is_connection_dropped",
'is_fp_closed', "is_fp_closed",
'get_host', "get_host",
'parse_url', "parse_url",
'make_headers', "make_headers",
'resolve_cert_reqs', "resolve_cert_reqs",
'resolve_ssl_version', "resolve_ssl_version",
'split_first', "split_first",
'ssl_wrap_socket', "ssl_wrap_socket",
'wait_for_read', "wait_for_read",
'wait_for_write' "wait_for_write",
) )

16
lib/urllib3/util/connection.py

@ -14,7 +14,7 @@ def is_connection_dropped(conn): # Platform-specific
Note: For platforms like AppEngine, this will always return ``False`` to Note: For platforms like AppEngine, this will always return ``False`` to
let the platform handle connection recycling transparently for us. let the platform handle connection recycling transparently for us.
""" """
sock = getattr(conn, 'sock', False) sock = getattr(conn, "sock", False)
if sock is False: # Platform-specific: AppEngine if sock is False: # Platform-specific: AppEngine
return False return False
if sock is None: # Connection already closed (such as by httplib). if sock is None: # Connection already closed (such as by httplib).
@ -30,8 +30,12 @@ def is_connection_dropped(conn): # Platform-specific
# library test suite. Added to its signature is only `socket_options`. # library test suite. Added to its signature is only `socket_options`.
# One additional modification is that we avoid binding to IPv6 servers # One additional modification is that we avoid binding to IPv6 servers
# discovered in DNS if the system doesn't have IPv6 functionality. # discovered in DNS if the system doesn't have IPv6 functionality.
def create_connection(address, timeout=socket._GLOBAL_DEFAULT_TIMEOUT, def create_connection(
source_address=None, socket_options=None): address,
timeout=socket._GLOBAL_DEFAULT_TIMEOUT,
source_address=None,
socket_options=None,
):
"""Connect to *address* and return the socket object. """Connect to *address* and return the socket object.
Convenience function. Connect to *address* (a 2-tuple ``(host, Convenience function. Connect to *address* (a 2-tuple ``(host,
@ -45,8 +49,8 @@ def create_connection(address, timeout=socket._GLOBAL_DEFAULT_TIMEOUT,
""" """
host, port = address host, port = address
if host.startswith('['): if host.startswith("["):
host = host.strip('[]') host = host.strip("[]")
err = None err = None
# Using the value from allowed_gai_family() in the context of getaddrinfo lets # Using the value from allowed_gai_family() in the context of getaddrinfo lets
@ -131,4 +135,4 @@ def _has_ipv6(host):
return has_ipv6 return has_ipv6
HAS_IPV6 = _has_ipv6('::1') HAS_IPV6 = _has_ipv6("::1")

52
lib/urllib3/util/request.py

@ -4,19 +4,25 @@ from base64 import b64encode
from ..packages.six import b, integer_types from ..packages.six import b, integer_types
from ..exceptions import UnrewindableBodyError from ..exceptions import UnrewindableBodyError
ACCEPT_ENCODING = 'gzip,deflate' ACCEPT_ENCODING = "gzip,deflate"
try: try:
import brotli as _unused_module_brotli # noqa: F401 import brotli as _unused_module_brotli # noqa: F401
except ImportError: except ImportError:
pass pass
else: else:
ACCEPT_ENCODING += ',br' ACCEPT_ENCODING += ",br"
_FAILEDTELL = object() _FAILEDTELL = object()
def make_headers(keep_alive=None, accept_encoding=None, user_agent=None, def make_headers(
basic_auth=None, proxy_basic_auth=None, disable_cache=None): keep_alive=None,
accept_encoding=None,
user_agent=None,
basic_auth=None,
proxy_basic_auth=None,
disable_cache=None,
):
""" """
Shortcuts for generating request headers. Shortcuts for generating request headers.
@ -56,27 +62,27 @@ def make_headers(keep_alive=None, accept_encoding=None, user_agent=None,
if isinstance(accept_encoding, str): if isinstance(accept_encoding, str):
pass pass
elif isinstance(accept_encoding, list): elif isinstance(accept_encoding, list):
accept_encoding = ','.join(accept_encoding) accept_encoding = ",".join(accept_encoding)
else: else:
accept_encoding = ACCEPT_ENCODING accept_encoding = ACCEPT_ENCODING
headers['accept-encoding'] = accept_encoding headers["accept-encoding"] = accept_encoding
if user_agent: if user_agent:
headers['user-agent'] = user_agent headers["user-agent"] = user_agent
if keep_alive: if keep_alive:
headers['connection'] = 'keep-alive' headers["connection"] = "keep-alive"
if basic_auth: if basic_auth:
headers['authorization'] = 'Basic ' + \ headers["authorization"] = "Basic " + b64encode(b(basic_auth)).decode("utf-8")
b64encode(b(basic_auth)).decode('utf-8')
if proxy_basic_auth: if proxy_basic_auth:
headers['proxy-authorization'] = 'Basic ' + \ headers["proxy-authorization"] = "Basic " + b64encode(
b64encode(b(proxy_basic_auth)).decode('utf-8') b(proxy_basic_auth)
).decode("utf-8")
if disable_cache: if disable_cache:
headers['cache-control'] = 'no-cache' headers["cache-control"] = "no-cache"
return headers return headers
@ -88,7 +94,7 @@ def set_file_position(body, pos):
""" """
if pos is not None: if pos is not None:
rewind_body(body, pos) rewind_body(body, pos)
elif getattr(body, 'tell', None) is not None: elif getattr(body, "tell", None) is not None:
try: try:
pos = body.tell() pos = body.tell()
except (IOError, OSError): except (IOError, OSError):
@ -110,16 +116,20 @@ def rewind_body(body, body_pos):
:param int pos: :param int pos:
Position to seek to in file. Position to seek to in file.
""" """
body_seek = getattr(body, 'seek', None) body_seek = getattr(body, "seek", None)
if body_seek is not None and isinstance(body_pos, integer_types): if body_seek is not None and isinstance(body_pos, integer_types):
try: try:
body_seek(body_pos) body_seek(body_pos)
except (IOError, OSError): except (IOError, OSError):
raise UnrewindableBodyError("An error occurred when rewinding request " raise UnrewindableBodyError(
"body for redirect/retry.") "An error occurred when rewinding request " "body for redirect/retry."
)
elif body_pos is _FAILEDTELL: elif body_pos is _FAILEDTELL:
raise UnrewindableBodyError("Unable to record file position for rewinding " raise UnrewindableBodyError(
"request body during a redirect/retry.") "Unable to record file position for rewinding "
"request body during a redirect/retry."
)
else: else:
raise ValueError("body_pos must be of type integer, " raise ValueError(
"instead it was %s." % type(body_pos)) "body_pos must be of type integer, " "instead it was %s." % type(body_pos)
)

9
lib/urllib3/util/response.py

@ -52,11 +52,10 @@ def assert_header_parsing(headers):
# This will fail silently if we pass in the wrong kind of parameter. # This will fail silently if we pass in the wrong kind of parameter.
# To make debugging easier add an explicit check. # To make debugging easier add an explicit check.
if not isinstance(headers, httplib.HTTPMessage): if not isinstance(headers, httplib.HTTPMessage):
raise TypeError('expected httplib.Message, got {0}.'.format( raise TypeError("expected httplib.Message, got {0}.".format(type(headers)))
type(headers)))
defects = getattr(headers, 'defects', None) defects = getattr(headers, "defects", None)
get_payload = getattr(headers, 'get_payload', None) get_payload = getattr(headers, "get_payload", None)
unparsed_data = None unparsed_data = None
if get_payload: if get_payload:
@ -84,4 +83,4 @@ def is_response_to_head(response):
method = response._method method = response._method
if isinstance(method, int): # Platform-specific: Appengine if isinstance(method, int): # Platform-specific: Appengine
return method == 3 return method == 3
return method.upper() == 'HEAD' return method.upper() == "HEAD"

100
lib/urllib3/util/retry.py

@ -21,8 +21,9 @@ log = logging.getLogger(__name__)
# Data structure for representing the metadata of requests that result in a retry. # Data structure for representing the metadata of requests that result in a retry.
RequestHistory = namedtuple('RequestHistory', ["method", "url", "error", RequestHistory = namedtuple(
"status", "redirect_location"]) "RequestHistory", ["method", "url", "error", "status", "redirect_location"]
)
class Retry(object): class Retry(object):
@ -146,21 +147,33 @@ class Retry(object):
request. request.
""" """
DEFAULT_METHOD_WHITELIST = frozenset([ DEFAULT_METHOD_WHITELIST = frozenset(
'HEAD', 'GET', 'PUT', 'DELETE', 'OPTIONS', 'TRACE']) ["HEAD", "GET", "PUT", "DELETE", "OPTIONS", "TRACE"]
)
RETRY_AFTER_STATUS_CODES = frozenset([413, 429, 503]) RETRY_AFTER_STATUS_CODES = frozenset([413, 429, 503])
DEFAULT_REDIRECT_HEADERS_BLACKLIST = frozenset(['Authorization']) DEFAULT_REDIRECT_HEADERS_BLACKLIST = frozenset(["Authorization"])
#: Maximum backoff time. #: Maximum backoff time.
BACKOFF_MAX = 120 BACKOFF_MAX = 120
def __init__(self, total=10, connect=None, read=None, redirect=None, status=None, def __init__(
method_whitelist=DEFAULT_METHOD_WHITELIST, status_forcelist=None, self,
backoff_factor=0, raise_on_redirect=True, raise_on_status=True, total=10,
history=None, respect_retry_after_header=True, connect=None,
remove_headers_on_redirect=DEFAULT_REDIRECT_HEADERS_BLACKLIST): read=None,
redirect=None,
status=None,
method_whitelist=DEFAULT_METHOD_WHITELIST,
status_forcelist=None,
backoff_factor=0,
raise_on_redirect=True,
raise_on_status=True,
history=None,
respect_retry_after_header=True,
remove_headers_on_redirect=DEFAULT_REDIRECT_HEADERS_BLACKLIST,
):
self.total = total self.total = total
self.connect = connect self.connect = connect
@ -179,20 +192,25 @@ class Retry(object):
self.raise_on_status = raise_on_status self.raise_on_status = raise_on_status
self.history = history or tuple() self.history = history or tuple()
self.respect_retry_after_header = respect_retry_after_header self.respect_retry_after_header = respect_retry_after_header
self.remove_headers_on_redirect = frozenset([ self.remove_headers_on_redirect = frozenset(
h.lower() for h in remove_headers_on_redirect]) [h.lower() for h in remove_headers_on_redirect]
)
def new(self, **kw): def new(self, **kw):
params = dict( params = dict(
total=self.total, total=self.total,
connect=self.connect, read=self.read, redirect=self.redirect, status=self.status, connect=self.connect,
read=self.read,
redirect=self.redirect,
status=self.status,
method_whitelist=self.method_whitelist, method_whitelist=self.method_whitelist,
status_forcelist=self.status_forcelist, status_forcelist=self.status_forcelist,
backoff_factor=self.backoff_factor, backoff_factor=self.backoff_factor,
raise_on_redirect=self.raise_on_redirect, raise_on_redirect=self.raise_on_redirect,
raise_on_status=self.raise_on_status, raise_on_status=self.raise_on_status,
history=self.history, history=self.history,
remove_headers_on_redirect=self.remove_headers_on_redirect remove_headers_on_redirect=self.remove_headers_on_redirect,
respect_retry_after_header=self.respect_retry_after_header,
) )
params.update(kw) params.update(kw)
return type(self)(**params) return type(self)(**params)
@ -217,8 +235,11 @@ class Retry(object):
:rtype: float :rtype: float
""" """
# We want to consider only the last consecutive errors sequence (Ignore redirects). # We want to consider only the last consecutive errors sequence (Ignore redirects).
consecutive_errors_len = len(list(takewhile(lambda x: x.redirect_location is None, consecutive_errors_len = len(
reversed(self.history)))) list(
takewhile(lambda x: x.redirect_location is None, reversed(self.history))
)
)
if consecutive_errors_len <= 1: if consecutive_errors_len <= 1:
return 0 return 0
@ -274,7 +295,7 @@ class Retry(object):
this method will return immediately. this method will return immediately.
""" """
if response: if self.respect_retry_after_header and response:
slept = self.sleep_for_retry(response) slept = self.sleep_for_retry(response)
if slept: if slept:
return return
@ -315,8 +336,12 @@ class Retry(object):
if self.status_forcelist and status_code in self.status_forcelist: if self.status_forcelist and status_code in self.status_forcelist:
return True return True
return (self.total and self.respect_retry_after_header and return (
has_retry_after and (status_code in self.RETRY_AFTER_STATUS_CODES)) self.total
and self.respect_retry_after_header
and has_retry_after
and (status_code in self.RETRY_AFTER_STATUS_CODES)
)
def is_exhausted(self): def is_exhausted(self):
""" Are we out of retries? """ """ Are we out of retries? """
@ -327,8 +352,15 @@ class Retry(object):
return min(retry_counts) < 0 return min(retry_counts) < 0
def increment(self, method=None, url=None, response=None, error=None, def increment(
_pool=None, _stacktrace=None): self,
method=None,
url=None,
response=None,
error=None,
_pool=None,
_stacktrace=None,
):
""" Return a new Retry object with incremented retry counters. """ Return a new Retry object with incremented retry counters.
:param response: A response object, or None, if the server did not :param response: A response object, or None, if the server did not
@ -351,7 +383,7 @@ class Retry(object):
read = self.read read = self.read
redirect = self.redirect redirect = self.redirect
status_count = self.status status_count = self.status
cause = 'unknown' cause = "unknown"
status = None status = None
redirect_location = None redirect_location = None
@ -373,7 +405,7 @@ class Retry(object):
# Redirect retry? # Redirect retry?
if redirect is not None: if redirect is not None:
redirect -= 1 redirect -= 1
cause = 'too many redirects' cause = "too many redirects"
redirect_location = response.get_redirect_location() redirect_location = response.get_redirect_location()
status = response.status status = response.status
@ -384,16 +416,21 @@ class Retry(object):
if response and response.status: if response and response.status:
if status_count is not None: if status_count is not None:
status_count -= 1 status_count -= 1
cause = ResponseError.SPECIFIC_ERROR.format( cause = ResponseError.SPECIFIC_ERROR.format(status_code=response.status)
status_code=response.status)
status = response.status status = response.status
history = self.history + (RequestHistory(method, url, error, status, redirect_location),) history = self.history + (
RequestHistory(method, url, error, status, redirect_location),
)
new_retry = self.new( new_retry = self.new(
total=total, total=total,
connect=connect, read=read, redirect=redirect, status=status_count, connect=connect,
history=history) read=read,
redirect=redirect,
status=status_count,
history=history,
)
if new_retry.is_exhausted(): if new_retry.is_exhausted():
raise MaxRetryError(_pool, url, error or ResponseError(cause)) raise MaxRetryError(_pool, url, error or ResponseError(cause))
@ -403,9 +440,10 @@ class Retry(object):
return new_retry return new_retry
def __repr__(self): def __repr__(self):
return ('{cls.__name__}(total={self.total}, connect={self.connect}, ' return (
'read={self.read}, redirect={self.redirect}, status={self.status})').format( "{cls.__name__}(total={self.total}, connect={self.connect}, "
cls=type(self), self=self) "read={self.read}, redirect={self.redirect}, status={self.status})"
).format(cls=type(self), self=self)
# For backwards compatibility (equivalent to pre-v1.9): # For backwards compatibility (equivalent to pre-v1.9):

153
lib/urllib3/util/ssl_.py

@ -18,11 +18,7 @@ IS_PYOPENSSL = False
IS_SECURETRANSPORT = False IS_SECURETRANSPORT = False
# Maps the length of a digest to a possible hash function producing this digest # Maps the length of a digest to a possible hash function producing this digest
HASHFUNC_MAP = { HASHFUNC_MAP = {32: md5, 40: sha1, 64: sha256}
32: md5,
40: sha1,
64: sha256,
}
def _const_compare_digest_backport(a, b): def _const_compare_digest_backport(a, b):
@ -38,17 +34,13 @@ def _const_compare_digest_backport(a, b):
return result == 0 return result == 0
_const_compare_digest = getattr(hmac, 'compare_digest', _const_compare_digest = getattr(hmac, "compare_digest", _const_compare_digest_backport)
_const_compare_digest_backport)
# Borrow rfc3986's regular expressions for IPv4 # Borrow rfc3986's regular expressions for IPv4
# and IPv6 addresses for use in is_ipaddress() # and IPv6 addresses for use in is_ipaddress()
_IP_ADDRESS_REGEX = re.compile( _IP_ADDRESS_REGEX = re.compile(
r'^(?:%s|%s|%s)$' % ( r"^(?:%s|%s|%s)$"
abnf_regexp.IPv4_RE, % (abnf_regexp.IPv4_RE, abnf_regexp.IPv6_RE, abnf_regexp.IPv6_ADDRZ_RFC4007_RE)
abnf_regexp.IPv6_RE,
abnf_regexp.IPv6_ADDRZ_RFC4007_RE
)
) )
try: # Test for SSL features try: # Test for SSL features
@ -60,10 +52,12 @@ except ImportError:
try: # Platform-specific: Python 3.6 try: # Platform-specific: Python 3.6
from ssl import PROTOCOL_TLS from ssl import PROTOCOL_TLS
PROTOCOL_SSLv23 = PROTOCOL_TLS PROTOCOL_SSLv23 = PROTOCOL_TLS
except ImportError: except ImportError:
try: try:
from ssl import PROTOCOL_SSLv23 as PROTOCOL_TLS from ssl import PROTOCOL_SSLv23 as PROTOCOL_TLS
PROTOCOL_SSLv23 = PROTOCOL_TLS PROTOCOL_SSLv23 = PROTOCOL_TLS
except ImportError: except ImportError:
PROTOCOL_SSLv23 = PROTOCOL_TLS = 2 PROTOCOL_SSLv23 = PROTOCOL_TLS = 2
@ -93,26 +87,29 @@ except ImportError:
# insecure ciphers for security reasons. # insecure ciphers for security reasons.
# - NOTE: TLS 1.3 cipher suites are managed through a different interface # - NOTE: TLS 1.3 cipher suites are managed through a different interface
# not exposed by CPython (yet!) and are enabled by default if they're available. # not exposed by CPython (yet!) and are enabled by default if they're available.
DEFAULT_CIPHERS = ':'.join([ DEFAULT_CIPHERS = ":".join(
'ECDHE+AESGCM', [
'ECDHE+CHACHA20', "ECDHE+AESGCM",
'DHE+AESGCM', "ECDHE+CHACHA20",
'DHE+CHACHA20', "DHE+AESGCM",
'ECDH+AESGCM', "DHE+CHACHA20",
'DH+AESGCM', "ECDH+AESGCM",
'ECDH+AES', "DH+AESGCM",
'DH+AES', "ECDH+AES",
'RSA+AESGCM', "DH+AES",
'RSA+AES', "RSA+AESGCM",
'!aNULL', "RSA+AES",
'!eNULL', "!aNULL",
'!MD5', "!eNULL",
'!DSS', "!MD5",
]) "!DSS",
]
)
try: try:
from ssl import SSLContext # Modern SSL? from ssl import SSLContext # Modern SSL?
except ImportError: except ImportError:
class SSLContext(object): # Platform-specific: Python 2 class SSLContext(object): # Platform-specific: Python 2
def __init__(self, protocol_version): def __init__(self, protocol_version):
self.protocol = protocol_version self.protocol = protocol_version
@ -140,21 +137,21 @@ except ImportError:
def wrap_socket(self, socket, server_hostname=None, server_side=False): def wrap_socket(self, socket, server_hostname=None, server_side=False):
warnings.warn( warnings.warn(
'A true SSLContext object is not available. This prevents ' "A true SSLContext object is not available. This prevents "
'urllib3 from configuring SSL appropriately and may cause ' "urllib3 from configuring SSL appropriately and may cause "
'certain SSL connections to fail. You can upgrade to a newer ' "certain SSL connections to fail. You can upgrade to a newer "
'version of Python to solve this. For more information, see ' "version of Python to solve this. For more information, see "
'https://urllib3.readthedocs.io/en/latest/advanced-usage.html' "https://urllib3.readthedocs.io/en/latest/advanced-usage.html"
'#ssl-warnings', "#ssl-warnings",
InsecurePlatformWarning InsecurePlatformWarning,
) )
kwargs = { kwargs = {
'keyfile': self.keyfile, "keyfile": self.keyfile,
'certfile': self.certfile, "certfile": self.certfile,
'ca_certs': self.ca_certs, "ca_certs": self.ca_certs,
'cert_reqs': self.verify_mode, "cert_reqs": self.verify_mode,
'ssl_version': self.protocol, "ssl_version": self.protocol,
'server_side': server_side, "server_side": server_side,
} }
return wrap_socket(socket, ciphers=self.ciphers, **kwargs) return wrap_socket(socket, ciphers=self.ciphers, **kwargs)
@ -169,12 +166,11 @@ def assert_fingerprint(cert, fingerprint):
Fingerprint as string of hexdigits, can be interspersed by colons. Fingerprint as string of hexdigits, can be interspersed by colons.
""" """
fingerprint = fingerprint.replace(':', '').lower() fingerprint = fingerprint.replace(":", "").lower()
digest_length = len(fingerprint) digest_length = len(fingerprint)
hashfunc = HASHFUNC_MAP.get(digest_length) hashfunc = HASHFUNC_MAP.get(digest_length)
if not hashfunc: if not hashfunc:
raise SSLError( raise SSLError("Fingerprint of invalid length: {0}".format(fingerprint))
'Fingerprint of invalid length: {0}'.format(fingerprint))
# We need encode() here for py32; works on py2 and p33. # We need encode() here for py32; works on py2 and p33.
fingerprint_bytes = unhexlify(fingerprint.encode()) fingerprint_bytes = unhexlify(fingerprint.encode())
@ -182,8 +178,11 @@ def assert_fingerprint(cert, fingerprint):
cert_digest = hashfunc(cert).digest() cert_digest = hashfunc(cert).digest()
if not _const_compare_digest(cert_digest, fingerprint_bytes): if not _const_compare_digest(cert_digest, fingerprint_bytes):
raise SSLError('Fingerprints did not match. Expected "{0}", got "{1}".' raise SSLError(
.format(fingerprint, hexlify(cert_digest))) 'Fingerprints did not match. Expected "{0}", got "{1}".'.format(
fingerprint, hexlify(cert_digest)
)
)
def resolve_cert_reqs(candidate): def resolve_cert_reqs(candidate):
@ -203,7 +202,7 @@ def resolve_cert_reqs(candidate):
if isinstance(candidate, str): if isinstance(candidate, str):
res = getattr(ssl, candidate, None) res = getattr(ssl, candidate, None)
if res is None: if res is None:
res = getattr(ssl, 'CERT_' + candidate) res = getattr(ssl, "CERT_" + candidate)
return res return res
return candidate return candidate
@ -219,14 +218,15 @@ def resolve_ssl_version(candidate):
if isinstance(candidate, str): if isinstance(candidate, str):
res = getattr(ssl, candidate, None) res = getattr(ssl, candidate, None)
if res is None: if res is None:
res = getattr(ssl, 'PROTOCOL_' + candidate) res = getattr(ssl, "PROTOCOL_" + candidate)
return res return res
return candidate return candidate
def create_urllib3_context(ssl_version=None, cert_reqs=None, def create_urllib3_context(
options=None, ciphers=None): ssl_version=None, cert_reqs=None, options=None, ciphers=None
):
"""All arguments have the same meaning as ``ssl_wrap_socket``. """All arguments have the same meaning as ``ssl_wrap_socket``.
By default, this function does a lot of the same work that By default, this function does a lot of the same work that
@ -280,17 +280,28 @@ def create_urllib3_context(ssl_version=None, cert_reqs=None,
context.options |= options context.options |= options
context.verify_mode = cert_reqs context.verify_mode = cert_reqs
if getattr(context, 'check_hostname', None) is not None: # Platform-specific: Python 3.2 if (
getattr(context, "check_hostname", None) is not None
): # Platform-specific: Python 3.2
# We do our own verification, including fingerprints and alternative # We do our own verification, including fingerprints and alternative
# hostnames. So disable it here # hostnames. So disable it here
context.check_hostname = False context.check_hostname = False
return context return context
def ssl_wrap_socket(sock, keyfile=None, certfile=None, cert_reqs=None, def ssl_wrap_socket(
ca_certs=None, server_hostname=None, sock,
ssl_version=None, ciphers=None, ssl_context=None, keyfile=None,
ca_cert_dir=None, key_password=None): certfile=None,
cert_reqs=None,
ca_certs=None,
server_hostname=None,
ssl_version=None,
ciphers=None,
ssl_context=None,
ca_cert_dir=None,
key_password=None,
):
""" """
All arguments except for server_hostname, ssl_context, and ca_cert_dir have All arguments except for server_hostname, ssl_context, and ca_cert_dir have
the same meaning as they do when using :func:`ssl.wrap_socket`. the same meaning as they do when using :func:`ssl.wrap_socket`.
@ -314,8 +325,7 @@ def ssl_wrap_socket(sock, keyfile=None, certfile=None, cert_reqs=None,
# Note: This branch of code and all the variables in it are no longer # Note: This branch of code and all the variables in it are no longer
# used by urllib3 itself. We should consider deprecating and removing # used by urllib3 itself. We should consider deprecating and removing
# this code. # this code.
context = create_urllib3_context(ssl_version, cert_reqs, context = create_urllib3_context(ssl_version, cert_reqs, ciphers=ciphers)
ciphers=ciphers)
if ca_certs or ca_cert_dir: if ca_certs or ca_cert_dir:
try: try:
@ -329,7 +339,7 @@ def ssl_wrap_socket(sock, keyfile=None, certfile=None, cert_reqs=None,
raise SSLError(e) raise SSLError(e)
raise raise
elif ssl_context is None and hasattr(context, 'load_default_certs'): elif ssl_context is None and hasattr(context, "load_default_certs"):
# try to load OS default certs; works well on Windows (require Python3.4+) # try to load OS default certs; works well on Windows (require Python3.4+)
context.load_default_certs() context.load_default_certs()
@ -349,20 +359,21 @@ def ssl_wrap_socket(sock, keyfile=None, certfile=None, cert_reqs=None,
# extension should not be used according to RFC3546 Section 3.1 # extension should not be used according to RFC3546 Section 3.1
# We shouldn't warn the user if SNI isn't available but we would # We shouldn't warn the user if SNI isn't available but we would
# not be using SNI anyways due to IP address for server_hostname. # not be using SNI anyways due to IP address for server_hostname.
if ((server_hostname is not None and not is_ipaddress(server_hostname)) if (
or IS_SECURETRANSPORT): server_hostname is not None and not is_ipaddress(server_hostname)
) or IS_SECURETRANSPORT:
if HAS_SNI and server_hostname is not None: if HAS_SNI and server_hostname is not None:
return context.wrap_socket(sock, server_hostname=server_hostname) return context.wrap_socket(sock, server_hostname=server_hostname)
warnings.warn( warnings.warn(
'An HTTPS request has been made, but the SNI (Server Name ' "An HTTPS request has been made, but the SNI (Server Name "
'Indication) extension to TLS is not available on this platform. ' "Indication) extension to TLS is not available on this platform. "
'This may cause the server to present an incorrect TLS ' "This may cause the server to present an incorrect TLS "
'certificate, which can cause validation failures. You can upgrade to ' "certificate, which can cause validation failures. You can upgrade to "
'a newer version of Python to solve this. For more information, see ' "a newer version of Python to solve this. For more information, see "
'https://urllib3.readthedocs.io/en/latest/advanced-usage.html' "https://urllib3.readthedocs.io/en/latest/advanced-usage.html"
'#ssl-warnings', "#ssl-warnings",
SNIMissingWarning SNIMissingWarning,
) )
return context.wrap_socket(sock) return context.wrap_socket(sock)
@ -377,16 +388,16 @@ def is_ipaddress(hostname):
""" """
if six.PY3 and isinstance(hostname, bytes): if six.PY3 and isinstance(hostname, bytes):
# IDN A-label bytes are ASCII compatible. # IDN A-label bytes are ASCII compatible.
hostname = hostname.decode('ascii') hostname = hostname.decode("ascii")
return _IP_ADDRESS_REGEX.match(hostname) is not None return _IP_ADDRESS_REGEX.match(hostname) is not None
def _is_key_file_encrypted(key_file): def _is_key_file_encrypted(key_file):
"""Detects if a key file is encrypted or not.""" """Detects if a key file is encrypted or not."""
with open(key_file, 'r') as f: with open(key_file, "r") as f:
for line in f: for line in f:
# Look for Proc-Type: 4,ENCRYPTED # Look for Proc-Type: 4,ENCRYPTED
if 'ENCRYPTED' in line: if "ENCRYPTED" in line:
return True return True
return False return False

79
lib/urllib3/util/timeout.py

@ -1,4 +1,5 @@
from __future__ import absolute_import from __future__ import absolute_import
# The default socket timeout, used by httplib to indicate that no timeout was # The default socket timeout, used by httplib to indicate that no timeout was
# specified by the user # specified by the user
from socket import _GLOBAL_DEFAULT_TIMEOUT from socket import _GLOBAL_DEFAULT_TIMEOUT
@ -45,19 +46,20 @@ class Timeout(object):
:type total: integer, float, or None :type total: integer, float, or None
:param connect: :param connect:
The maximum amount of time to wait for a connection attempt to a server The maximum amount of time (in seconds) to wait for a connection
to succeed. Omitting the parameter will default the connect timeout to attempt to a server to succeed. Omitting the parameter will default the
the system default, probably `the global default timeout in socket.py connect timeout to the system default, probably `the global default
timeout in socket.py
<http://hg.python.org/cpython/file/603b4d593758/Lib/socket.py#l535>`_. <http://hg.python.org/cpython/file/603b4d593758/Lib/socket.py#l535>`_.
None will set an infinite timeout for connection attempts. None will set an infinite timeout for connection attempts.
:type connect: integer, float, or None :type connect: integer, float, or None
:param read: :param read:
The maximum amount of time to wait between consecutive The maximum amount of time (in seconds) to wait between consecutive
read operations for a response from the server. Omitting read operations for a response from the server. Omitting the parameter
the parameter will default the read timeout to the system will default the read timeout to the system default, probably `the
default, probably `the global default timeout in socket.py global default timeout in socket.py
<http://hg.python.org/cpython/file/603b4d593758/Lib/socket.py#l535>`_. <http://hg.python.org/cpython/file/603b4d593758/Lib/socket.py#l535>`_.
None will set an infinite timeout. None will set an infinite timeout.
@ -91,14 +93,18 @@ class Timeout(object):
DEFAULT_TIMEOUT = _GLOBAL_DEFAULT_TIMEOUT DEFAULT_TIMEOUT = _GLOBAL_DEFAULT_TIMEOUT
def __init__(self, total=None, connect=_Default, read=_Default): def __init__(self, total=None, connect=_Default, read=_Default):
self._connect = self._validate_timeout(connect, 'connect') self._connect = self._validate_timeout(connect, "connect")
self._read = self._validate_timeout(read, 'read') self._read = self._validate_timeout(read, "read")
self.total = self._validate_timeout(total, 'total') self.total = self._validate_timeout(total, "total")
self._start_connect = None self._start_connect = None
def __str__(self): def __str__(self):
return '%s(connect=%r, read=%r, total=%r)' % ( return "%s(connect=%r, read=%r, total=%r)" % (
type(self).__name__, self._connect, self._read, self.total) type(self).__name__,
self._connect,
self._read,
self.total,
)
@classmethod @classmethod
def _validate_timeout(cls, value, name): def _validate_timeout(cls, value, name):
@ -118,23 +124,31 @@ class Timeout(object):
return value return value
if isinstance(value, bool): if isinstance(value, bool):
raise ValueError("Timeout cannot be a boolean value. It must " raise ValueError(
"be an int, float or None.") "Timeout cannot be a boolean value. It must "
"be an int, float or None."
)
try: try:
float(value) float(value)
except (TypeError, ValueError): except (TypeError, ValueError):
raise ValueError("Timeout value %s was %s, but it must be an " raise ValueError(
"int, float or None." % (name, value)) "Timeout value %s was %s, but it must be an "
"int, float or None." % (name, value)
)
try: try:
if value <= 0: if value <= 0:
raise ValueError("Attempted to set %s timeout to %s, but the " raise ValueError(
"timeout cannot be set to a value less " "Attempted to set %s timeout to %s, but the "
"than or equal to 0." % (name, value)) "timeout cannot be set to a value less "
"than or equal to 0." % (name, value)
)
except TypeError: except TypeError:
# Python 3 # Python 3
raise ValueError("Timeout value %s was %s, but it must be an " raise ValueError(
"int, float or None." % (name, value)) "Timeout value %s was %s, but it must be an "
"int, float or None." % (name, value)
)
return value return value
@ -166,8 +180,7 @@ class Timeout(object):
# We can't use copy.deepcopy because that will also create a new object # We can't use copy.deepcopy because that will also create a new object
# for _GLOBAL_DEFAULT_TIMEOUT, which socket.py uses as a sentinel to # for _GLOBAL_DEFAULT_TIMEOUT, which socket.py uses as a sentinel to
# detect the user default. # detect the user default.
return Timeout(connect=self._connect, read=self._read, return Timeout(connect=self._connect, read=self._read, total=self.total)
total=self.total)
def start_connect(self): def start_connect(self):
""" Start the timeout clock, used during a connect() attempt """ Start the timeout clock, used during a connect() attempt
@ -183,14 +196,15 @@ class Timeout(object):
def get_connect_duration(self): def get_connect_duration(self):
""" Gets the time elapsed since the call to :meth:`start_connect`. """ Gets the time elapsed since the call to :meth:`start_connect`.
:return: Elapsed time. :return: Elapsed time in seconds.
:rtype: float :rtype: float
:raises urllib3.exceptions.TimeoutStateError: if you attempt :raises urllib3.exceptions.TimeoutStateError: if you attempt
to get duration for a timer that hasn't been started. to get duration for a timer that hasn't been started.
""" """
if self._start_connect is None: if self._start_connect is None:
raise TimeoutStateError("Can't get connect duration for timer " raise TimeoutStateError(
"that has not started.") "Can't get connect duration for timer " "that has not started."
)
return current_time() - self._start_connect return current_time() - self._start_connect
@property @property
@ -228,15 +242,16 @@ class Timeout(object):
:raises urllib3.exceptions.TimeoutStateError: If :meth:`start_connect` :raises urllib3.exceptions.TimeoutStateError: If :meth:`start_connect`
has not yet been called on this object. has not yet been called on this object.
""" """
if (self.total is not None and if (
self.total is not self.DEFAULT_TIMEOUT and self.total is not None
self._read is not None and and self.total is not self.DEFAULT_TIMEOUT
self._read is not self.DEFAULT_TIMEOUT): and self._read is not None
and self._read is not self.DEFAULT_TIMEOUT
):
# In case the connect timeout has not yet been established. # In case the connect timeout has not yet been established.
if self._start_connect is None: if self._start_connect is None:
return self._read return self._read
return max(0, min(self.total - self.get_connect_duration(), return max(0, min(self.total - self.get_connect_duration(), self._read))
self._read))
elif self.total is not None and self.total is not self.DEFAULT_TIMEOUT: elif self.total is not None and self.total is not self.DEFAULT_TIMEOUT:
return max(0, self.total - self.get_connect_duration()) return max(0, self.total - self.get_connect_duration())
else: else:

102
lib/urllib3/util/url.py

@ -9,35 +9,47 @@ from ..packages.rfc3986.validators import Validator
from ..packages.rfc3986 import abnf_regexp, normalizers, compat, misc from ..packages.rfc3986 import abnf_regexp, normalizers, compat, misc
url_attrs = ['scheme', 'auth', 'host', 'port', 'path', 'query', 'fragment'] url_attrs = ["scheme", "auth", "host", "port", "path", "query", "fragment"]
# We only want to normalize urls with an HTTP(S) scheme. # We only want to normalize urls with an HTTP(S) scheme.
# urllib3 infers URLs without a scheme (None) to be http. # urllib3 infers URLs without a scheme (None) to be http.
NORMALIZABLE_SCHEMES = ('http', 'https', None) NORMALIZABLE_SCHEMES = ("http", "https", None)
# Regex for detecting URLs with schemes. RFC 3986 Section 3.1 # Regex for detecting URLs with schemes. RFC 3986 Section 3.1
SCHEME_REGEX = re.compile(r"^(?:[a-zA-Z][a-zA-Z0-9+\-]*:|/)") SCHEME_REGEX = re.compile(r"^(?:[a-zA-Z][a-zA-Z0-9+\-]*:|/)")
PATH_CHARS = abnf_regexp.UNRESERVED_CHARS_SET | abnf_regexp.SUB_DELIMITERS_SET | {':', '@', '/'} PATH_CHARS = (
QUERY_CHARS = FRAGMENT_CHARS = PATH_CHARS | {'?'} abnf_regexp.UNRESERVED_CHARS_SET | abnf_regexp.SUB_DELIMITERS_SET | {":", "@", "/"}
)
QUERY_CHARS = FRAGMENT_CHARS = PATH_CHARS | {"?"}
class Url(namedtuple('Url', url_attrs)): class Url(namedtuple("Url", url_attrs)):
""" """
Data structure for representing an HTTP URL. Used as a return value for Data structure for representing an HTTP URL. Used as a return value for
:func:`parse_url`. Both the scheme and host are normalized as they are :func:`parse_url`. Both the scheme and host are normalized as they are
both case-insensitive according to RFC 3986. both case-insensitive according to RFC 3986.
""" """
__slots__ = () __slots__ = ()
def __new__(cls, scheme=None, auth=None, host=None, port=None, path=None, def __new__(
query=None, fragment=None): cls,
if path and not path.startswith('/'): scheme=None,
path = '/' + path auth=None,
host=None,
port=None,
path=None,
query=None,
fragment=None,
):
if path and not path.startswith("/"):
path = "/" + path
if scheme is not None: if scheme is not None:
scheme = scheme.lower() scheme = scheme.lower()
return super(Url, cls).__new__(cls, scheme, auth, host, port, path, return super(Url, cls).__new__(
query, fragment) cls, scheme, auth, host, port, path, query, fragment
)
@property @property
def hostname(self): def hostname(self):
@ -47,10 +59,10 @@ class Url(namedtuple('Url', url_attrs)):
@property @property
def request_uri(self): def request_uri(self):
"""Absolute path including the query string.""" """Absolute path including the query string."""
uri = self.path or '/' uri = self.path or "/"
if self.query is not None: if self.query is not None:
uri += '?' + self.query uri += "?" + self.query
return uri return uri
@ -58,7 +70,7 @@ class Url(namedtuple('Url', url_attrs)):
def netloc(self): def netloc(self):
"""Network location including host and port""" """Network location including host and port"""
if self.port: if self.port:
return '%s:%d' % (self.host, self.port) return "%s:%d" % (self.host, self.port)
return self.host return self.host
@property @property
@ -81,23 +93,23 @@ class Url(namedtuple('Url', url_attrs)):
'http://username:password@host.com:80/path?query#fragment' 'http://username:password@host.com:80/path?query#fragment'
""" """
scheme, auth, host, port, path, query, fragment = self scheme, auth, host, port, path, query, fragment = self
url = u'' url = u""
# We use "is not None" we want things to happen with empty strings (or 0 port) # We use "is not None" we want things to happen with empty strings (or 0 port)
if scheme is not None: if scheme is not None:
url += scheme + u'://' url += scheme + u"://"
if auth is not None: if auth is not None:
url += auth + u'@' url += auth + u"@"
if host is not None: if host is not None:
url += host url += host
if port is not None: if port is not None:
url += u':' + str(port) url += u":" + str(port)
if path is not None: if path is not None:
url += path url += path
if query is not None: if query is not None:
url += u'?' + query url += u"?" + query
if fragment is not None: if fragment is not None:
url += u'#' + fragment url += u"#" + fragment
return url return url
@ -135,12 +147,12 @@ def split_first(s, delims):
min_delim = d min_delim = d
if min_idx is None or min_idx < 0: if min_idx is None or min_idx < 0:
return s, '', None return s, "", None
return s[:min_idx], s[min_idx + 1:], min_delim return s[:min_idx], s[min_idx + 1 :], min_delim
def _encode_invalid_chars(component, allowed_chars, encoding='utf-8'): def _encode_invalid_chars(component, allowed_chars, encoding="utf-8"):
"""Percent-encodes a URI component without reapplying """Percent-encodes a URI component without reapplying
onto an already percent-encoded component. Based on onto an already percent-encoded component. Based on
rfc3986.normalizers.encode_component() rfc3986.normalizers.encode_component()
@ -150,23 +162,25 @@ def _encode_invalid_chars(component, allowed_chars, encoding='utf-8'):
# Try to see if the component we're encoding is already percent-encoded # Try to see if the component we're encoding is already percent-encoded
# so we can skip all '%' characters but still encode all others. # so we can skip all '%' characters but still encode all others.
percent_encodings = len(normalizers.PERCENT_MATCHER.findall( percent_encodings = len(
compat.to_str(component, encoding))) normalizers.PERCENT_MATCHER.findall(compat.to_str(component, encoding))
)
uri_bytes = component.encode('utf-8', 'surrogatepass') uri_bytes = component.encode("utf-8", "surrogatepass")
is_percent_encoded = percent_encodings == uri_bytes.count(b'%') is_percent_encoded = percent_encodings == uri_bytes.count(b"%")
encoded_component = bytearray() encoded_component = bytearray()
for i in range(0, len(uri_bytes)): for i in range(0, len(uri_bytes)):
# Will return a single character bytestring on both Python 2 & 3 # Will return a single character bytestring on both Python 2 & 3
byte = uri_bytes[i:i+1] byte = uri_bytes[i : i + 1]
byte_ord = ord(byte) byte_ord = ord(byte)
if ((is_percent_encoded and byte == b'%') if (is_percent_encoded and byte == b"%") or (
or (byte_ord < 128 and byte.decode() in allowed_chars)): byte_ord < 128 and byte.decode() in allowed_chars
):
encoded_component.extend(byte) encoded_component.extend(byte)
continue continue
encoded_component.extend('%{0:02x}'.format(byte_ord).encode().upper()) encoded_component.extend("%{0:02x}".format(byte_ord).encode().upper())
return encoded_component.decode(encoding) return encoded_component.decode(encoding)
@ -209,7 +223,9 @@ def parse_url(url):
try: try:
import idna import idna
except ImportError: except ImportError:
raise LocationParseError("Unable to parse URL without the 'idna' module") raise LocationParseError(
"Unable to parse URL without the 'idna' module"
)
try: try:
return idna.encode(name.lower(), strict=True, std3_rules=True) return idna.encode(name.lower(), strict=True, std3_rules=True)
except idna.IDNAError: except idna.IDNAError:
@ -219,10 +235,11 @@ def parse_url(url):
try: try:
split_iri = misc.IRI_MATCHER.match(compat.to_str(url)).groupdict() split_iri = misc.IRI_MATCHER.match(compat.to_str(url)).groupdict()
iri_ref = rfc3986.IRIReference( iri_ref = rfc3986.IRIReference(
split_iri['scheme'], split_iri['authority'], split_iri["scheme"],
_encode_invalid_chars(split_iri['path'], PATH_CHARS), split_iri["authority"],
_encode_invalid_chars(split_iri['query'], QUERY_CHARS), _encode_invalid_chars(split_iri["path"], PATH_CHARS),
_encode_invalid_chars(split_iri['fragment'], FRAGMENT_CHARS) _encode_invalid_chars(split_iri["query"], QUERY_CHARS),
_encode_invalid_chars(split_iri["fragment"], FRAGMENT_CHARS),
) )
has_authority = iri_ref.authority is not None has_authority = iri_ref.authority is not None
uri_ref = iri_ref.encode(idna_encoder=idna_encode) uri_ref = iri_ref.encode(idna_encoder=idna_encode)
@ -243,9 +260,7 @@ def parse_url(url):
# normalization has completed. # normalization has completed.
validator = Validator() validator = Validator()
try: try:
validator.check_validity_of( validator.check_validity_of(*validator.COMPONENT_NAMES).validate(uri_ref)
*validator.COMPONENT_NAMES
).validate(uri_ref)
except ValidationError: except ValidationError:
return six.raise_from(LocationParseError(url), None) return six.raise_from(LocationParseError(url), None)
@ -255,8 +270,7 @@ def parse_url(url):
# TODO: Remove this when we break backwards compatibility. # TODO: Remove this when we break backwards compatibility.
path = uri_ref.path path = uri_ref.path
if not path: if not path:
if (uri_ref.query is not None if uri_ref.query is not None or uri_ref.fragment is not None:
or uri_ref.fragment is not None):
path = "" path = ""
else: else:
path = None path = None
@ -267,7 +281,7 @@ def parse_url(url):
if x is None: if x is None:
return None return None
elif not is_string and not isinstance(x, six.binary_type): elif not is_string and not isinstance(x, six.binary_type):
return x.encode('utf-8') return x.encode("utf-8")
return x return x
return Url( return Url(
@ -277,7 +291,7 @@ def parse_url(url):
port=int(uri_ref.port) if uri_ref.port is not None else None, port=int(uri_ref.port) if uri_ref.port is not None else None,
path=to_input_type(path), path=to_input_type(path),
query=to_input_type(uri_ref.query), query=to_input_type(uri_ref.query),
fragment=to_input_type(uri_ref.fragment) fragment=to_input_type(uri_ref.fragment),
) )
@ -286,4 +300,4 @@ def get_host(url):
Deprecated. Use :func:`parse_url` instead. Deprecated. Use :func:`parse_url` instead.
""" """
p = parse_url(url) p = parse_url(url)
return p.scheme or 'http', p.hostname, p.port return p.scheme or "http", p.hostname, p.port

3
lib/urllib3/util/wait.py

@ -2,6 +2,7 @@ import errno
from functools import partial from functools import partial
import select import select
import sys import sys
try: try:
from time import monotonic from time import monotonic
except ImportError: except ImportError:
@ -40,6 +41,8 @@ if sys.version_info >= (3, 5):
# Modern Python, that retries syscalls by default # Modern Python, that retries syscalls by default
def _retry_on_intr(fn, timeout): def _retry_on_intr(fn, timeout):
return fn(timeout) return fn(timeout)
else: else:
# Old and broken Pythons. # Old and broken Pythons.
def _retry_on_intr(fn, timeout): def _retry_on_intr(fn, timeout):

Loading…
Cancel
Save