diff --git a/CHANGES.md b/CHANGES.md index 415535f..74fea8d 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -11,6 +11,7 @@ * Update Six compatibility library 1.12.0 (8da94b8) to 1.12.0 (aa4e90b) * Update TZlocal 2.0.0.dev0 (b73a692) to 2.0.0b3 (410a838) * Update unidecode module 1.0.22 (a5045ab) to 1.1.1 (632af82) +* Update urllib3 release 1.25.2 (49eea80) to 1.25.3 (3387b20) [develop changelog] diff --git a/lib/urllib3/__init__.py b/lib/urllib3/__init__.py index eb91588..63e5026 100644 --- a/lib/urllib3/__init__.py +++ b/lib/urllib3/__init__.py @@ -4,11 +4,7 @@ urllib3 - Thread-safe connection pooling and re-using. from __future__ import absolute_import import warnings -from .connectionpool import ( - HTTPConnectionPool, - HTTPSConnectionPool, - connection_from_url -) +from .connectionpool import HTTPConnectionPool, HTTPSConnectionPool, connection_from_url from . import exceptions from .filepost import encode_multipart_formdata @@ -24,25 +20,25 @@ from .util.retry import Retry import logging from logging import NullHandler -__author__ = 'Andrey Petrov (andrey.petrov@shazow.net)' -__license__ = 'MIT' -__version__ = '1.25.2' +__author__ = "Andrey Petrov (andrey.petrov@shazow.net)" +__license__ = "MIT" +__version__ = "1.25.3" __all__ = ( - 'HTTPConnectionPool', - 'HTTPSConnectionPool', - 'PoolManager', - 'ProxyManager', - 'HTTPResponse', - 'Retry', - 'Timeout', - 'add_stderr_logger', - 'connection_from_url', - 'disable_warnings', - 'encode_multipart_formdata', - 'get_host', - 'make_headers', - 'proxy_from_url', + "HTTPConnectionPool", + "HTTPSConnectionPool", + "PoolManager", + "ProxyManager", + "HTTPResponse", + "Retry", + "Timeout", + "add_stderr_logger", + "connection_from_url", + "disable_warnings", + "encode_multipart_formdata", + "get_host", + "make_headers", + "proxy_from_url", ) logging.getLogger(__name__).addHandler(NullHandler()) @@ -59,10 +55,10 @@ def add_stderr_logger(level=logging.DEBUG): # even if urllib3 is vendored within another package. logger = logging.getLogger(__name__) handler = logging.StreamHandler() - handler.setFormatter(logging.Formatter('%(asctime)s %(levelname)s %(message)s')) + handler.setFormatter(logging.Formatter("%(asctime)s %(levelname)s %(message)s")) logger.addHandler(handler) logger.setLevel(level) - logger.debug('Added a stderr logging handler to logger: %s', __name__) + logger.debug("Added a stderr logging handler to logger: %s", __name__) return handler @@ -74,18 +70,17 @@ del NullHandler # shouldn't be: otherwise, it's very hard for users to use most Python # mechanisms to silence them. # SecurityWarning's always go off by default. -warnings.simplefilter('always', exceptions.SecurityWarning, append=True) +warnings.simplefilter("always", exceptions.SecurityWarning, append=True) # SubjectAltNameWarning's should go off once per host -warnings.simplefilter('default', exceptions.SubjectAltNameWarning, append=True) +warnings.simplefilter("default", exceptions.SubjectAltNameWarning, append=True) # InsecurePlatformWarning's don't vary between requests, so we keep it default. -warnings.simplefilter('default', exceptions.InsecurePlatformWarning, - append=True) +warnings.simplefilter("default", exceptions.InsecurePlatformWarning, append=True) # SNIMissingWarnings should go off only once. -warnings.simplefilter('default', exceptions.SNIMissingWarning, append=True) +warnings.simplefilter("default", exceptions.SNIMissingWarning, append=True) def disable_warnings(category=exceptions.HTTPWarning): """ Helper for quickly disabling all urllib3 warnings. """ - warnings.simplefilter('ignore', category) + warnings.simplefilter("ignore", category) diff --git a/lib/urllib3/_collections.py b/lib/urllib3/_collections.py index 34f2381..019d151 100644 --- a/lib/urllib3/_collections.py +++ b/lib/urllib3/_collections.py @@ -1,4 +1,5 @@ from __future__ import absolute_import + try: from collections.abc import Mapping, MutableMapping except ImportError: @@ -6,6 +7,7 @@ except ImportError: try: from threading import RLock except ImportError: # Platform-specific: No threads available + class RLock: def __enter__(self): pass @@ -19,7 +21,7 @@ from .exceptions import InvalidHeader from .packages.six import iterkeys, itervalues, PY3 -__all__ = ['RecentlyUsedContainer', 'HTTPHeaderDict'] +__all__ = ["RecentlyUsedContainer", "HTTPHeaderDict"] _Null = object() @@ -82,7 +84,9 @@ class RecentlyUsedContainer(MutableMapping): return len(self._container) def __iter__(self): - raise NotImplementedError('Iteration over this class is unlikely to be threadsafe.') + raise NotImplementedError( + "Iteration over this class is unlikely to be threadsafe." + ) def clear(self): with self.lock: @@ -150,7 +154,7 @@ class HTTPHeaderDict(MutableMapping): def __getitem__(self, key): val = self._container[key.lower()] - return ', '.join(val[1:]) + return ", ".join(val[1:]) def __delitem__(self, key): del self._container[key.lower()] @@ -159,12 +163,13 @@ class HTTPHeaderDict(MutableMapping): return key.lower() in self._container def __eq__(self, other): - if not isinstance(other, Mapping) and not hasattr(other, 'keys'): + if not isinstance(other, Mapping) and not hasattr(other, "keys"): return False if not isinstance(other, type(self)): other = type(self)(other) - return (dict((k.lower(), v) for k, v in self.itermerged()) == - dict((k.lower(), v) for k, v in other.itermerged())) + return dict((k.lower(), v) for k, v in self.itermerged()) == dict( + (k.lower(), v) for k, v in other.itermerged() + ) def __ne__(self, other): return not self.__eq__(other) @@ -184,9 +189,9 @@ class HTTPHeaderDict(MutableMapping): yield vals[0] def pop(self, key, default=__marker): - '''D.pop(k[,d]) -> v, remove specified key and return the corresponding value. + """D.pop(k[,d]) -> v, remove specified key and return the corresponding value. If key is not found, d is returned if given, otherwise KeyError is raised. - ''' + """ # Using the MutableMapping function directly fails due to the private marker. # Using ordinary dict.pop would expose the internal structures. # So let's reinvent the wheel. @@ -228,8 +233,10 @@ class HTTPHeaderDict(MutableMapping): with self.add instead of self.__setitem__ """ if len(args) > 1: - raise TypeError("extend() takes at most 1 positional " - "arguments ({0} given)".format(len(args))) + raise TypeError( + "extend() takes at most 1 positional " + "arguments ({0} given)".format(len(args)) + ) other = args[0] if len(args) >= 1 else () if isinstance(other, HTTPHeaderDict): @@ -295,7 +302,7 @@ class HTTPHeaderDict(MutableMapping): """Iterate over all headers, merging duplicate ones together.""" for key in self: val = self._container[key.lower()] - yield val[0], ', '.join(val[1:]) + yield val[0], ", ".join(val[1:]) def items(self): return list(self.iteritems()) @@ -306,7 +313,7 @@ class HTTPHeaderDict(MutableMapping): # python2.7 does not expose a proper API for exporting multiheaders # efficiently. This function re-reads raw lines from the message # object and extracts the multiheaders properly. - obs_fold_continued_leaders = (' ', '\t') + obs_fold_continued_leaders = (" ", "\t") headers = [] for line in message.headers: @@ -316,14 +323,14 @@ class HTTPHeaderDict(MutableMapping): # in RFC-7230 S3.2.4. This indicates a multiline header, but # there exists no previous header to which we can attach it. raise InvalidHeader( - 'Header continuation with no previous header: %s' % line + "Header continuation with no previous header: %s" % line ) else: key, value = headers[-1] - headers[-1] = (key, value + ' ' + line.strip()) + headers[-1] = (key, value + " " + line.strip()) continue - key, value = line.split(':', 1) + key, value = line.split(":", 1) headers.append((key, value.strip())) return cls(headers) diff --git a/lib/urllib3/connection.py b/lib/urllib3/connection.py index f816ee8..a138bb2 100644 --- a/lib/urllib3/connection.py +++ b/lib/urllib3/connection.py @@ -11,6 +11,7 @@ from .packages.six.moves.http_client import HTTPException # noqa: F401 try: # Compiled with SSL? import ssl + BaseSSLError = ssl.SSLError except (ImportError, AttributeError): # Platform-specific: No SSL. ssl = None @@ -41,7 +42,7 @@ from .util.ssl_ import ( resolve_ssl_version, assert_fingerprint, create_urllib3_context, - ssl_wrap_socket + ssl_wrap_socket, ) @@ -51,20 +52,16 @@ from ._collections import HTTPHeaderDict log = logging.getLogger(__name__) -port_by_scheme = { - 'http': 80, - 'https': 443, -} +port_by_scheme = {"http": 80, "https": 443} -# When updating RECENT_DATE, move it to within two years of the current date, -# and not less than 6 months ago. -# Example: if Today is 2018-01-01, then RECENT_DATE should be any date on or -# after 2016-01-01 (today - 2 years) AND before 2017-07-01 (today - 6 months) -RECENT_DATE = datetime.date(2017, 6, 30) +# When it comes time to update this value as a part of regular maintenance +# (ie test_recent_date is failing) update it to ~6 months before the current date. +RECENT_DATE = datetime.date(2019, 1, 1) class DummyConnection(object): """Used to detect a failed ConnectionCls import.""" + pass @@ -92,7 +89,7 @@ class HTTPConnection(_HTTPConnection, object): Or you may want to disable the defaults by passing an empty list (e.g., ``[]``). """ - default_port = port_by_scheme['http'] + default_port = port_by_scheme["http"] #: Disable Nagle's algorithm by default. #: ``[(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)]`` @@ -103,14 +100,14 @@ class HTTPConnection(_HTTPConnection, object): def __init__(self, *args, **kw): if six.PY3: - kw.pop('strict', None) + kw.pop("strict", None) # Pre-set source_address. - self.source_address = kw.get('source_address') + self.source_address = kw.get("source_address") #: The socket options provided by the user. If no options are #: provided, we use the default options. - self.socket_options = kw.pop('socket_options', self.default_socket_options) + self.socket_options = kw.pop("socket_options", self.default_socket_options) _HTTPConnection.__init__(self, *args, **kw) @@ -131,7 +128,7 @@ class HTTPConnection(_HTTPConnection, object): those cases where it's appropriate (i.e., when doing DNS lookup to establish the actual TCP connection across which we're going to send HTTP requests). """ - return self._dns_host.rstrip('.') + return self._dns_host.rstrip(".") @host.setter def host(self, value): @@ -150,30 +147,34 @@ class HTTPConnection(_HTTPConnection, object): """ extra_kw = {} if self.source_address: - extra_kw['source_address'] = self.source_address + extra_kw["source_address"] = self.source_address if self.socket_options: - extra_kw['socket_options'] = self.socket_options + extra_kw["socket_options"] = self.socket_options try: conn = connection.create_connection( - (self._dns_host, self.port), self.timeout, **extra_kw) + (self._dns_host, self.port), self.timeout, **extra_kw + ) except SocketTimeout: raise ConnectTimeoutError( - self, "Connection to %s timed out. (connect timeout=%s)" % - (self.host, self.timeout)) + self, + "Connection to %s timed out. (connect timeout=%s)" + % (self.host, self.timeout), + ) except SocketError as e: raise NewConnectionError( - self, "Failed to establish a new connection: %s" % e) + self, "Failed to establish a new connection: %s" % e + ) return conn def _prepare_conn(self, conn): self.sock = conn # Google App Engine's httplib does not define _tunnel_host - if getattr(self, '_tunnel_host', None): + if getattr(self, "_tunnel_host", None): # TODO: Fix tunnel so it doesn't depend on self.sock state. self._tunnel() # Mark this connection as not reusable @@ -189,18 +190,15 @@ class HTTPConnection(_HTTPConnection, object): body with chunked encoding and not as one block """ headers = HTTPHeaderDict(headers if headers is not None else {}) - skip_accept_encoding = 'accept-encoding' in headers - skip_host = 'host' in headers + skip_accept_encoding = "accept-encoding" in headers + skip_host = "host" in headers self.putrequest( - method, - url, - skip_accept_encoding=skip_accept_encoding, - skip_host=skip_host + method, url, skip_accept_encoding=skip_accept_encoding, skip_host=skip_host ) for header, value in headers.items(): self.putheader(header, value) - if 'transfer-encoding' not in headers: - self.putheader('Transfer-Encoding', 'chunked') + if "transfer-encoding" not in headers: + self.putheader("Transfer-Encoding", "chunked") self.endheaders() if body is not None: @@ -211,29 +209,37 @@ class HTTPConnection(_HTTPConnection, object): if not chunk: continue if not isinstance(chunk, bytes): - chunk = chunk.encode('utf8') + chunk = chunk.encode("utf8") len_str = hex(len(chunk))[2:] - self.send(len_str.encode('utf-8')) - self.send(b'\r\n') + self.send(len_str.encode("utf-8")) + self.send(b"\r\n") self.send(chunk) - self.send(b'\r\n') + self.send(b"\r\n") # After the if clause, to always have a closed body - self.send(b'0\r\n\r\n') + self.send(b"0\r\n\r\n") class HTTPSConnection(HTTPConnection): - default_port = port_by_scheme['https'] + default_port = port_by_scheme["https"] ssl_version = None - def __init__(self, host, port=None, key_file=None, cert_file=None, - key_password=None, strict=None, - timeout=socket._GLOBAL_DEFAULT_TIMEOUT, - ssl_context=None, server_hostname=None, **kw): - - HTTPConnection.__init__(self, host, port, strict=strict, - timeout=timeout, **kw) + def __init__( + self, + host, + port=None, + key_file=None, + cert_file=None, + key_password=None, + strict=None, + timeout=socket._GLOBAL_DEFAULT_TIMEOUT, + ssl_context=None, + server_hostname=None, + **kw + ): + + HTTPConnection.__init__(self, host, port, strict=strict, timeout=timeout, **kw) self.key_file = key_file self.cert_file = cert_file @@ -243,25 +249,40 @@ class HTTPSConnection(HTTPConnection): # Required property for Google AppEngine 1.9.0 which otherwise causes # HTTPS requests to go out as HTTP. (See Issue #356) - self._protocol = 'https' + self._protocol = "https" def connect(self): conn = self._new_conn() self._prepare_conn(conn) + # Wrap socket using verification with the root certs in + # trusted_root_certs + default_ssl_context = False if self.ssl_context is None: + default_ssl_context = True self.ssl_context = create_urllib3_context( - ssl_version=resolve_ssl_version(None), - cert_reqs=resolve_cert_reqs(None), + ssl_version=resolve_ssl_version(self.ssl_version), + cert_reqs=resolve_cert_reqs(self.cert_reqs), ) + # Try to load OS default certs if none are given. + # Works well on Windows (requires Python3.4+) + context = self.ssl_context + if ( + not self.ca_certs + and not self.ca_cert_dir + and default_ssl_context + and hasattr(context, "load_default_certs") + ): + context.load_default_certs() + self.sock = ssl_wrap_socket( sock=conn, keyfile=self.key_file, certfile=self.cert_file, key_password=self.key_password, ssl_context=self.ssl_context, - server_hostname=self.server_hostname + server_hostname=self.server_hostname, ) @@ -270,16 +291,24 @@ class VerifiedHTTPSConnection(HTTPSConnection): Based on httplib.HTTPSConnection but wraps the socket with SSL certification. """ + cert_reqs = None ca_certs = None ca_cert_dir = None ssl_version = None assert_fingerprint = None - def set_cert(self, key_file=None, cert_file=None, - cert_reqs=None, key_password=None, ca_certs=None, - assert_hostname=None, assert_fingerprint=None, - ca_cert_dir=None): + def set_cert( + self, + key_file=None, + cert_file=None, + cert_reqs=None, + key_password=None, + ca_certs=None, + assert_hostname=None, + assert_fingerprint=None, + ca_cert_dir=None, + ): """ This method should only be called once, before the connection is used. """ @@ -306,7 +335,7 @@ class VerifiedHTTPSConnection(HTTPSConnection): hostname = self.host # Google App Engine's httplib does not define _tunnel_host - if getattr(self, '_tunnel_host', None): + if getattr(self, "_tunnel_host", None): self.sock = conn # Calls self._set_hostport(), so self.host is # self._tunnel_host below. @@ -323,15 +352,19 @@ class VerifiedHTTPSConnection(HTTPSConnection): is_time_off = datetime.date.today() < RECENT_DATE if is_time_off: - warnings.warn(( - 'System time is way off (before {0}). This will probably ' - 'lead to SSL verification errors').format(RECENT_DATE), - SystemTimeWarning + warnings.warn( + ( + "System time is way off (before {0}). This will probably " + "lead to SSL verification errors" + ).format(RECENT_DATE), + SystemTimeWarning, ) # Wrap socket using verification with the root certs in # trusted_root_certs + default_ssl_context = False if self.ssl_context is None: + default_ssl_context = True self.ssl_context = create_urllib3_context( ssl_version=resolve_ssl_version(self.ssl_version), cert_reqs=resolve_cert_reqs(self.cert_reqs), @@ -339,6 +372,17 @@ class VerifiedHTTPSConnection(HTTPSConnection): context = self.ssl_context context.verify_mode = resolve_cert_reqs(self.cert_reqs) + + # Try to load OS default certs if none are given. + # Works well on Windows (requires Python3.4+) + if ( + not self.ca_certs + and not self.ca_cert_dir + and default_ssl_context + and hasattr(context, "load_default_certs") + ): + context.load_default_certs() + self.sock = ssl_wrap_socket( sock=conn, keyfile=self.key_file, @@ -347,31 +391,37 @@ class VerifiedHTTPSConnection(HTTPSConnection): ca_certs=self.ca_certs, ca_cert_dir=self.ca_cert_dir, server_hostname=server_hostname, - ssl_context=context) + ssl_context=context, + ) if self.assert_fingerprint: - assert_fingerprint(self.sock.getpeercert(binary_form=True), - self.assert_fingerprint) - elif context.verify_mode != ssl.CERT_NONE \ - and not getattr(context, 'check_hostname', False) \ - and self.assert_hostname is not False: + assert_fingerprint( + self.sock.getpeercert(binary_form=True), self.assert_fingerprint + ) + elif ( + context.verify_mode != ssl.CERT_NONE + and not getattr(context, "check_hostname", False) + and self.assert_hostname is not False + ): # While urllib3 attempts to always turn off hostname matching from # the TLS library, this cannot always be done. So we check whether # the TLS Library still thinks it's matching hostnames. cert = self.sock.getpeercert() - if not cert.get('subjectAltName', ()): - warnings.warn(( - 'Certificate for {0} has no `subjectAltName`, falling back to check for a ' - '`commonName` for now. This feature is being removed by major browsers and ' - 'deprecated by RFC 2818. (See https://github.com/shazow/urllib3/issues/497 ' - 'for details.)'.format(hostname)), - SubjectAltNameWarning + if not cert.get("subjectAltName", ()): + warnings.warn( + ( + "Certificate for {0} has no `subjectAltName`, falling back to check for a " + "`commonName` for now. This feature is being removed by major browsers and " + "deprecated by RFC 2818. (See https://github.com/shazow/urllib3/issues/497 " + "for details.)".format(hostname) + ), + SubjectAltNameWarning, ) _match_hostname(cert, self.assert_hostname or server_hostname) self.is_verified = ( - context.verify_mode == ssl.CERT_REQUIRED or - self.assert_fingerprint is not None + context.verify_mode == ssl.CERT_REQUIRED + or self.assert_fingerprint is not None ) @@ -380,8 +430,9 @@ def _match_hostname(cert, asserted_hostname): match_hostname(cert, asserted_hostname) except CertificateError as e: log.error( - 'Certificate did not match expected hostname: %s. ' - 'Certificate: %s', asserted_hostname, cert + "Certificate did not match expected hostname: %s. " "Certificate: %s", + asserted_hostname, + cert, ) # Add cert to exception and reraise so client code can inspect # the cert when catching the exception, if they want to diff --git a/lib/urllib3/connectionpool.py b/lib/urllib3/connectionpool.py index 157568a..ee53c4b 100644 --- a/lib/urllib3/connectionpool.py +++ b/lib/urllib3/connectionpool.py @@ -30,8 +30,11 @@ from .packages.rfc3986.normalizers import normalize_host from .connection import ( port_by_scheme, DummyConnection, - HTTPConnection, HTTPSConnection, VerifiedHTTPSConnection, - HTTPException, BaseSSLError, + HTTPConnection, + HTTPSConnection, + VerifiedHTTPSConnection, + HTTPException, + BaseSSLError, ) from .request import RequestMethods from .response import HTTPResponse @@ -71,8 +74,7 @@ class ConnectionPool(object): self.port = port def __str__(self): - return '%s(host=%r, port=%r)' % (type(self).__name__, - self.host, self.port) + return "%s(host=%r, port=%r)" % (type(self).__name__, self.host, self.port) def __enter__(self): return self @@ -153,15 +155,24 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods): :class:`urllib3.connection.HTTPSConnection` instances. """ - scheme = 'http' + scheme = "http" ConnectionCls = HTTPConnection ResponseCls = HTTPResponse - def __init__(self, host, port=None, strict=False, - timeout=Timeout.DEFAULT_TIMEOUT, maxsize=1, block=False, - headers=None, retries=None, - _proxy=None, _proxy_headers=None, - **conn_kw): + def __init__( + self, + host, + port=None, + strict=False, + timeout=Timeout.DEFAULT_TIMEOUT, + maxsize=1, + block=False, + headers=None, + retries=None, + _proxy=None, + _proxy_headers=None, + **conn_kw + ): ConnectionPool.__init__(self, host, port) RequestMethods.__init__(self, headers) @@ -195,19 +206,27 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods): # Enable Nagle's algorithm for proxies, to avoid packet fragmentation. # We cannot know if the user has added default socket options, so we cannot replace the # list. - self.conn_kw.setdefault('socket_options', []) + self.conn_kw.setdefault("socket_options", []) def _new_conn(self): """ Return a fresh :class:`HTTPConnection`. """ self.num_connections += 1 - log.debug("Starting new HTTP connection (%d): %s:%s", - self.num_connections, self.host, self.port or "80") - - conn = self.ConnectionCls(host=self.host, port=self.port, - timeout=self.timeout.connect_timeout, - strict=self.strict, **self.conn_kw) + log.debug( + "Starting new HTTP connection (%d): %s:%s", + self.num_connections, + self.host, + self.port or "80", + ) + + conn = self.ConnectionCls( + host=self.host, + port=self.port, + timeout=self.timeout.connect_timeout, + strict=self.strict, + **self.conn_kw + ) return conn def _get_conn(self, timeout=None): @@ -231,16 +250,17 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods): except queue.Empty: if self.block: - raise EmptyPoolError(self, - "Pool reached maximum size and no more " - "connections are allowed.") + raise EmptyPoolError( + self, + "Pool reached maximum size and no more " "connections are allowed.", + ) pass # Oh well, we'll create a new connection then # If this is a persistent connection, check if it got disconnected if conn and is_connection_dropped(conn): log.debug("Resetting dropped connection: %s", self.host) conn.close() - if getattr(conn, 'auto_open', 1) == 0: + if getattr(conn, "auto_open", 1) == 0: # This is a proxied connection that has been mutated by # httplib._tunnel() and cannot be reused (since it would # attempt to bypass the proxy) @@ -270,9 +290,7 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods): pass except queue.Full: # This should never happen if self.block == True - log.warning( - "Connection pool is full, discarding connection: %s", - self.host) + log.warning("Connection pool is full, discarding connection: %s", self.host) # Connection never got put back into the pool, close it. if conn: @@ -304,21 +322,30 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods): """Is the error actually a timeout? Will raise a ReadTimeout or pass""" if isinstance(err, SocketTimeout): - raise ReadTimeoutError(self, url, "Read timed out. (read timeout=%s)" % timeout_value) + raise ReadTimeoutError( + self, url, "Read timed out. (read timeout=%s)" % timeout_value + ) # See the above comment about EAGAIN in Python 3. In Python 2 we have # to specifically catch it and throw the timeout error - if hasattr(err, 'errno') and err.errno in _blocking_errnos: - raise ReadTimeoutError(self, url, "Read timed out. (read timeout=%s)" % timeout_value) + if hasattr(err, "errno") and err.errno in _blocking_errnos: + raise ReadTimeoutError( + self, url, "Read timed out. (read timeout=%s)" % timeout_value + ) # Catch possible read timeouts thrown as SSL errors. If not the # case, rethrow the original. We need to do this because of: # http://bugs.python.org/issue10272 - if 'timed out' in str(err) or 'did not complete (read)' in str(err): # Python < 2.7.4 - raise ReadTimeoutError(self, url, "Read timed out. (read timeout=%s)" % timeout_value) - - def _make_request(self, conn, method, url, timeout=_Default, chunked=False, - **httplib_request_kw): + if "timed out" in str(err) or "did not complete (read)" in str( + err + ): # Python < 2.7.4 + raise ReadTimeoutError( + self, url, "Read timed out. (read timeout=%s)" % timeout_value + ) + + def _make_request( + self, conn, method, url, timeout=_Default, chunked=False, **httplib_request_kw + ): """ Perform a request on a given urllib connection object taken from our pool. @@ -358,7 +385,7 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods): read_timeout = timeout_obj.read_timeout # App Engine doesn't have a sock attr - if getattr(conn, 'sock', None): + if getattr(conn, "sock", None): # In Python 3 socket.py will catch EAGAIN and return None when you # try and read into the file pointer created by http.client, which # instead raises a BadStatusLine exception. Instead of catching @@ -366,7 +393,8 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods): # timeouts, check for a zero timeout before making the request. if read_timeout == 0: raise ReadTimeoutError( - self, url, "Read timed out. (read timeout=%s)" % read_timeout) + self, url, "Read timed out. (read timeout=%s)" % read_timeout + ) if read_timeout is Timeout.DEFAULT_TIMEOUT: conn.sock.settimeout(socket.getdefaulttimeout()) else: # None or a value @@ -381,26 +409,38 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods): # Python 3 try: httplib_response = conn.getresponse() - except Exception as e: - # Remove the TypeError from the exception chain in Python 3; - # otherwise it looks like a programming error was the cause. + except BaseException as e: + # Remove the TypeError from the exception chain in + # Python 3 (including for exceptions like SystemExit). + # Otherwise it looks like a bug in the code. six.raise_from(e, None) except (SocketTimeout, BaseSSLError, SocketError) as e: self._raise_timeout(err=e, url=url, timeout_value=read_timeout) raise # AppEngine doesn't have a version attr. - http_version = getattr(conn, '_http_vsn_str', 'HTTP/?') - log.debug("%s://%s:%s \"%s %s %s\" %s %s", self.scheme, self.host, self.port, - method, url, http_version, httplib_response.status, - httplib_response.length) + http_version = getattr(conn, "_http_vsn_str", "HTTP/?") + log.debug( + '%s://%s:%s "%s %s %s" %s %s', + self.scheme, + self.host, + self.port, + method, + url, + http_version, + httplib_response.status, + httplib_response.length, + ) try: assert_header_parsing(httplib_response.msg) except (HeaderParsingError, TypeError) as hpe: # Platform-specific: Python 3 log.warning( - 'Failed to parse headers (url=%s): %s', - self._absolute_url(url), hpe, exc_info=True) + "Failed to parse headers (url=%s): %s", + self._absolute_url(url), + hpe, + exc_info=True, + ) return httplib_response @@ -430,7 +470,7 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods): Check if the given ``url`` is a member of the same host as this connection pool. """ - if url.startswith('/'): + if url.startswith("/"): return True # TODO: Add optional support for socket.gethostbyname checking. @@ -446,10 +486,22 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods): return (scheme, host, port) == (self.scheme, self.host, self.port) - def urlopen(self, method, url, body=None, headers=None, retries=None, - redirect=True, assert_same_host=True, timeout=_Default, - pool_timeout=None, release_conn=None, chunked=False, - body_pos=None, **response_kw): + def urlopen( + self, + method, + url, + body=None, + headers=None, + retries=None, + redirect=True, + assert_same_host=True, + timeout=_Default, + pool_timeout=None, + release_conn=None, + chunked=False, + body_pos=None, + **response_kw + ): """ Get a connection from the pool and perform an HTTP request. This is the lowest level call for making a request, so you'll need to specify all @@ -547,7 +599,7 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods): retries = Retry.from_int(retries, redirect=redirect, default=self.retries) if release_conn is None: - release_conn = response_kw.get('preload_content', True) + release_conn = response_kw.get("preload_content", True) # Check host if assert_same_host and not self.is_same_host(url): @@ -569,7 +621,7 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods): # Merge the proxy headers. Only do this in HTTP. We have to copy the # headers dict so we can safely change it without those changes being # reflected in anyone else's copy. - if self.scheme == 'http': + if self.scheme == "http": headers = headers.copy() headers.update(self.proxy_headers) @@ -592,15 +644,22 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods): conn.timeout = timeout_obj.connect_timeout - is_new_proxy_conn = self.proxy is not None and not getattr(conn, 'sock', None) + is_new_proxy_conn = self.proxy is not None and not getattr( + conn, "sock", None + ) if is_new_proxy_conn: self._prepare_proxy(conn) # Make the request on the httplib connection object. - httplib_response = self._make_request(conn, method, url, - timeout=timeout_obj, - body=body, headers=headers, - chunked=chunked) + httplib_response = self._make_request( + conn, + method, + url, + timeout=timeout_obj, + body=body, + headers=headers, + chunked=chunked, + ) # If we're going to release the connection in ``finally:``, then # the response doesn't need to know about the connection. Otherwise @@ -609,14 +668,16 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods): response_conn = conn if not release_conn else None # Pass method to Response for length checking - response_kw['request_method'] = method + response_kw["request_method"] = method # Import httplib's response into our own wrapper object - response = self.ResponseCls.from_httplib(httplib_response, - pool=self, - connection=response_conn, - retries=retries, - **response_kw) + response = self.ResponseCls.from_httplib( + httplib_response, + pool=self, + connection=response_conn, + retries=retries, + **response_kw + ) # Everything went great! clean_exit = True @@ -625,20 +686,28 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods): # Timed out by queue. raise EmptyPoolError(self, "No pool connections are available.") - except (TimeoutError, HTTPException, SocketError, ProtocolError, - BaseSSLError, SSLError, CertificateError) as e: + except ( + TimeoutError, + HTTPException, + SocketError, + ProtocolError, + BaseSSLError, + SSLError, + CertificateError, + ) as e: # Discard the connection for these exceptions. It will be # replaced during the next _get_conn() call. clean_exit = False if isinstance(e, (BaseSSLError, CertificateError)): e = SSLError(e) elif isinstance(e, (SocketError, NewConnectionError)) and self.proxy: - e = ProxyError('Cannot connect to proxy.', e) + e = ProxyError("Cannot connect to proxy.", e) elif isinstance(e, (SocketError, HTTPException)): - e = ProtocolError('Connection aborted.', e) + e = ProtocolError("Connection aborted.", e) - retries = retries.increment(method, url, error=e, _pool=self, - _stacktrace=sys.exc_info()[2]) + retries = retries.increment( + method, url, error=e, _pool=self, _stacktrace=sys.exc_info()[2] + ) retries.sleep() # Keep track of the error for the retry warning. @@ -661,28 +730,47 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods): if not conn: # Try again - log.warning("Retrying (%r) after connection " - "broken by '%r': %s", retries, err, url) - return self.urlopen(method, url, body, headers, retries, - redirect, assert_same_host, - timeout=timeout, pool_timeout=pool_timeout, - release_conn=release_conn, body_pos=body_pos, - **response_kw) + log.warning( + "Retrying (%r) after connection " "broken by '%r': %s", + retries, + err, + url, + ) + return self.urlopen( + method, + url, + body, + headers, + retries, + redirect, + assert_same_host, + timeout=timeout, + pool_timeout=pool_timeout, + release_conn=release_conn, + body_pos=body_pos, + **response_kw + ) def drain_and_release_conn(response): try: # discard any remaining response body, the connection will be # released back to the pool once the entire response is read response.read() - except (TimeoutError, HTTPException, SocketError, ProtocolError, - BaseSSLError, SSLError): + except ( + TimeoutError, + HTTPException, + SocketError, + ProtocolError, + BaseSSLError, + SSLError, + ): pass # Handle redirect? redirect_location = redirect and response.get_redirect_location() if redirect_location: if response.status == 303: - method = 'GET' + method = "GET" try: retries = retries.increment(method, url, response=response, _pool=self) @@ -700,15 +788,22 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods): retries.sleep_for_retry(response) log.debug("Redirecting %s -> %s", url, redirect_location) return self.urlopen( - method, redirect_location, body, headers, - retries=retries, redirect=redirect, + method, + redirect_location, + body, + headers, + retries=retries, + redirect=redirect, assert_same_host=assert_same_host, - timeout=timeout, pool_timeout=pool_timeout, - release_conn=release_conn, body_pos=body_pos, - **response_kw) + timeout=timeout, + pool_timeout=pool_timeout, + release_conn=release_conn, + body_pos=body_pos, + **response_kw + ) # Check if we should retry the HTTP response. - has_retry_after = bool(response.getheader('Retry-After')) + has_retry_after = bool(response.getheader("Retry-After")) if retries.is_retry(method, response.status, has_retry_after): try: retries = retries.increment(method, url, response=response, _pool=self) @@ -726,12 +821,19 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods): retries.sleep(response) log.debug("Retry: %s", url) return self.urlopen( - method, url, body, headers, - retries=retries, redirect=redirect, + method, + url, + body, + headers, + retries=retries, + redirect=redirect, assert_same_host=assert_same_host, - timeout=timeout, pool_timeout=pool_timeout, + timeout=timeout, + pool_timeout=pool_timeout, release_conn=release_conn, - body_pos=body_pos, **response_kw) + body_pos=body_pos, + **response_kw + ) return response @@ -754,21 +856,47 @@ class HTTPSConnectionPool(HTTPConnectionPool): the connection socket into an SSL socket. """ - scheme = 'https' + scheme = "https" ConnectionCls = HTTPSConnection - def __init__(self, host, port=None, - strict=False, timeout=Timeout.DEFAULT_TIMEOUT, maxsize=1, - block=False, headers=None, retries=None, - _proxy=None, _proxy_headers=None, - key_file=None, cert_file=None, cert_reqs=None, - key_password=None, ca_certs=None, ssl_version=None, - assert_hostname=None, assert_fingerprint=None, - ca_cert_dir=None, **conn_kw): - - HTTPConnectionPool.__init__(self, host, port, strict, timeout, maxsize, - block, headers, retries, _proxy, _proxy_headers, - **conn_kw) + def __init__( + self, + host, + port=None, + strict=False, + timeout=Timeout.DEFAULT_TIMEOUT, + maxsize=1, + block=False, + headers=None, + retries=None, + _proxy=None, + _proxy_headers=None, + key_file=None, + cert_file=None, + cert_reqs=None, + key_password=None, + ca_certs=None, + ssl_version=None, + assert_hostname=None, + assert_fingerprint=None, + ca_cert_dir=None, + **conn_kw + ): + + HTTPConnectionPool.__init__( + self, + host, + port, + strict, + timeout, + maxsize, + block, + headers, + retries, + _proxy, + _proxy_headers, + **conn_kw + ) self.key_file = key_file self.cert_file = cert_file @@ -787,14 +915,16 @@ class HTTPSConnectionPool(HTTPConnectionPool): """ if isinstance(conn, VerifiedHTTPSConnection): - conn.set_cert(key_file=self.key_file, - key_password=self.key_password, - cert_file=self.cert_file, - cert_reqs=self.cert_reqs, - ca_certs=self.ca_certs, - ca_cert_dir=self.ca_cert_dir, - assert_hostname=self.assert_hostname, - assert_fingerprint=self.assert_fingerprint) + conn.set_cert( + key_file=self.key_file, + key_password=self.key_password, + cert_file=self.cert_file, + cert_reqs=self.cert_reqs, + ca_certs=self.ca_certs, + ca_cert_dir=self.ca_cert_dir, + assert_hostname=self.assert_hostname, + assert_fingerprint=self.assert_fingerprint, + ) conn.ssl_version = self.ssl_version return conn @@ -811,12 +941,17 @@ class HTTPSConnectionPool(HTTPConnectionPool): Return a fresh :class:`httplib.HTTPSConnection`. """ self.num_connections += 1 - log.debug("Starting new HTTPS connection (%d): %s:%s", - self.num_connections, self.host, self.port or "443") + log.debug( + "Starting new HTTPS connection (%d): %s:%s", + self.num_connections, + self.host, + self.port or "443", + ) if not self.ConnectionCls or self.ConnectionCls is DummyConnection: - raise SSLError("Can't connect to HTTPS URL because the SSL " - "module is not available.") + raise SSLError( + "Can't connect to HTTPS URL because the SSL " "module is not available." + ) actual_host = self.host actual_port = self.port @@ -824,11 +959,16 @@ class HTTPSConnectionPool(HTTPConnectionPool): actual_host = self.proxy.host actual_port = self.proxy.port - conn = self.ConnectionCls(host=actual_host, port=actual_port, - timeout=self.timeout.connect_timeout, - strict=self.strict, cert_file=self.cert_file, - key_file=self.key_file, key_password=self.key_password, - **self.conn_kw) + conn = self.ConnectionCls( + host=actual_host, + port=actual_port, + timeout=self.timeout.connect_timeout, + strict=self.strict, + cert_file=self.cert_file, + key_file=self.key_file, + key_password=self.key_password, + **self.conn_kw + ) return self._prepare_conn(conn) @@ -839,16 +979,19 @@ class HTTPSConnectionPool(HTTPConnectionPool): super(HTTPSConnectionPool, self)._validate_conn(conn) # Force connect early to allow us to validate the connection. - if not getattr(conn, 'sock', None): # AppEngine might not have `.sock` + if not getattr(conn, "sock", None): # AppEngine might not have `.sock` conn.connect() if not conn.is_verified: - warnings.warn(( - 'Unverified HTTPS request is being made. ' - 'Adding certificate verification is strongly advised. See: ' - 'https://urllib3.readthedocs.io/en/latest/advanced-usage.html' - '#ssl-warnings'), - InsecureRequestWarning) + warnings.warn( + ( + "Unverified HTTPS request is being made. " + "Adding certificate verification is strongly advised. See: " + "https://urllib3.readthedocs.io/en/latest/advanced-usage.html" + "#ssl-warnings" + ), + InsecureRequestWarning, + ) def connection_from_url(url, **kw): @@ -873,7 +1016,7 @@ def connection_from_url(url, **kw): """ scheme, host, port = get_host(url) port = port or port_by_scheme.get(scheme, 80) - if scheme == 'https': + if scheme == "https": return HTTPSConnectionPool(host, port=port, **kw) else: return HTTPConnectionPool(host, port=port, **kw) @@ -890,8 +1033,8 @@ def _normalize_host(host, scheme): # Instead, we need to make sure we never pass ``None`` as the port. # However, for backward compatibility reasons we can't actually # *assert* that. See http://bugs.python.org/issue28539 - if host.startswith('[') and host.endswith(']'): - host = host.strip('[]') + if host.startswith("[") and host.endswith("]"): + host = host.strip("[]") if scheme in NORMALIZABLE_SCHEMES: host = normalize_host(host) return host diff --git a/lib/urllib3/contrib/_appengine_environ.py b/lib/urllib3/contrib/_appengine_environ.py index f3e0094..c909010 100644 --- a/lib/urllib3/contrib/_appengine_environ.py +++ b/lib/urllib3/contrib/_appengine_environ.py @@ -6,9 +6,7 @@ import os def is_appengine(): - return (is_local_appengine() or - is_prod_appengine() or - is_prod_appengine_mvms()) + return is_local_appengine() or is_prod_appengine() or is_prod_appengine_mvms() def is_appengine_sandbox(): @@ -16,15 +14,19 @@ def is_appengine_sandbox(): def is_local_appengine(): - return ('APPENGINE_RUNTIME' in os.environ and - 'Development/' in os.environ['SERVER_SOFTWARE']) + return ( + "APPENGINE_RUNTIME" in os.environ + and "Development/" in os.environ["SERVER_SOFTWARE"] + ) def is_prod_appengine(): - return ('APPENGINE_RUNTIME' in os.environ and - 'Google App Engine/' in os.environ['SERVER_SOFTWARE'] and - not is_prod_appengine_mvms()) + return ( + "APPENGINE_RUNTIME" in os.environ + and "Google App Engine/" in os.environ["SERVER_SOFTWARE"] + and not is_prod_appengine_mvms() + ) def is_prod_appengine_mvms(): - return os.environ.get('GAE_VM', False) == 'true' + return os.environ.get("GAE_VM", False) == "true" diff --git a/lib/urllib3/contrib/_securetransport/bindings.py b/lib/urllib3/contrib/_securetransport/bindings.py index be34215..b46e1e3 100644 --- a/lib/urllib3/contrib/_securetransport/bindings.py +++ b/lib/urllib3/contrib/_securetransport/bindings.py @@ -34,29 +34,35 @@ from __future__ import absolute_import import platform from ctypes.util import find_library from ctypes import ( - c_void_p, c_int32, c_char_p, c_size_t, c_byte, c_uint32, c_ulong, c_long, - c_bool + c_void_p, + c_int32, + c_char_p, + c_size_t, + c_byte, + c_uint32, + c_ulong, + c_long, + c_bool, ) from ctypes import CDLL, POINTER, CFUNCTYPE -security_path = find_library('Security') +security_path = find_library("Security") if not security_path: - raise ImportError('The library Security could not be found') + raise ImportError("The library Security could not be found") -core_foundation_path = find_library('CoreFoundation') +core_foundation_path = find_library("CoreFoundation") if not core_foundation_path: - raise ImportError('The library CoreFoundation could not be found') + raise ImportError("The library CoreFoundation could not be found") version = platform.mac_ver()[0] -version_info = tuple(map(int, version.split('.'))) +version_info = tuple(map(int, version.split("."))) if version_info < (10, 8): raise OSError( - 'Only OS X 10.8 and newer are supported, not %s.%s' % ( - version_info[0], version_info[1] - ) + "Only OS X 10.8 and newer are supported, not %s.%s" + % (version_info[0], version_info[1]) ) Security = CDLL(security_path, use_errno=True) @@ -129,27 +135,19 @@ try: Security.SecKeyGetTypeID.argtypes = [] Security.SecKeyGetTypeID.restype = CFTypeID - Security.SecCertificateCreateWithData.argtypes = [ - CFAllocatorRef, - CFDataRef - ] + Security.SecCertificateCreateWithData.argtypes = [CFAllocatorRef, CFDataRef] Security.SecCertificateCreateWithData.restype = SecCertificateRef - Security.SecCertificateCopyData.argtypes = [ - SecCertificateRef - ] + Security.SecCertificateCopyData.argtypes = [SecCertificateRef] Security.SecCertificateCopyData.restype = CFDataRef - Security.SecCopyErrorMessageString.argtypes = [ - OSStatus, - c_void_p - ] + Security.SecCopyErrorMessageString.argtypes = [OSStatus, c_void_p] Security.SecCopyErrorMessageString.restype = CFStringRef Security.SecIdentityCreateWithCertificate.argtypes = [ CFTypeRef, SecCertificateRef, - POINTER(SecIdentityRef) + POINTER(SecIdentityRef), ] Security.SecIdentityCreateWithCertificate.restype = OSStatus @@ -159,201 +157,126 @@ try: c_void_p, Boolean, c_void_p, - POINTER(SecKeychainRef) + POINTER(SecKeychainRef), ] Security.SecKeychainCreate.restype = OSStatus - Security.SecKeychainDelete.argtypes = [ - SecKeychainRef - ] + Security.SecKeychainDelete.argtypes = [SecKeychainRef] Security.SecKeychainDelete.restype = OSStatus Security.SecPKCS12Import.argtypes = [ CFDataRef, CFDictionaryRef, - POINTER(CFArrayRef) + POINTER(CFArrayRef), ] Security.SecPKCS12Import.restype = OSStatus SSLReadFunc = CFUNCTYPE(OSStatus, SSLConnectionRef, c_void_p, POINTER(c_size_t)) - SSLWriteFunc = CFUNCTYPE(OSStatus, SSLConnectionRef, POINTER(c_byte), POINTER(c_size_t)) + SSLWriteFunc = CFUNCTYPE( + OSStatus, SSLConnectionRef, POINTER(c_byte), POINTER(c_size_t) + ) - Security.SSLSetIOFuncs.argtypes = [ - SSLContextRef, - SSLReadFunc, - SSLWriteFunc - ] + Security.SSLSetIOFuncs.argtypes = [SSLContextRef, SSLReadFunc, SSLWriteFunc] Security.SSLSetIOFuncs.restype = OSStatus - Security.SSLSetPeerID.argtypes = [ - SSLContextRef, - c_char_p, - c_size_t - ] + Security.SSLSetPeerID.argtypes = [SSLContextRef, c_char_p, c_size_t] Security.SSLSetPeerID.restype = OSStatus - Security.SSLSetCertificate.argtypes = [ - SSLContextRef, - CFArrayRef - ] + Security.SSLSetCertificate.argtypes = [SSLContextRef, CFArrayRef] Security.SSLSetCertificate.restype = OSStatus - Security.SSLSetCertificateAuthorities.argtypes = [ - SSLContextRef, - CFTypeRef, - Boolean - ] + Security.SSLSetCertificateAuthorities.argtypes = [SSLContextRef, CFTypeRef, Boolean] Security.SSLSetCertificateAuthorities.restype = OSStatus - Security.SSLSetConnection.argtypes = [ - SSLContextRef, - SSLConnectionRef - ] + Security.SSLSetConnection.argtypes = [SSLContextRef, SSLConnectionRef] Security.SSLSetConnection.restype = OSStatus - Security.SSLSetPeerDomainName.argtypes = [ - SSLContextRef, - c_char_p, - c_size_t - ] + Security.SSLSetPeerDomainName.argtypes = [SSLContextRef, c_char_p, c_size_t] Security.SSLSetPeerDomainName.restype = OSStatus - Security.SSLHandshake.argtypes = [ - SSLContextRef - ] + Security.SSLHandshake.argtypes = [SSLContextRef] Security.SSLHandshake.restype = OSStatus - Security.SSLRead.argtypes = [ - SSLContextRef, - c_char_p, - c_size_t, - POINTER(c_size_t) - ] + Security.SSLRead.argtypes = [SSLContextRef, c_char_p, c_size_t, POINTER(c_size_t)] Security.SSLRead.restype = OSStatus - Security.SSLWrite.argtypes = [ - SSLContextRef, - c_char_p, - c_size_t, - POINTER(c_size_t) - ] + Security.SSLWrite.argtypes = [SSLContextRef, c_char_p, c_size_t, POINTER(c_size_t)] Security.SSLWrite.restype = OSStatus - Security.SSLClose.argtypes = [ - SSLContextRef - ] + Security.SSLClose.argtypes = [SSLContextRef] Security.SSLClose.restype = OSStatus - Security.SSLGetNumberSupportedCiphers.argtypes = [ - SSLContextRef, - POINTER(c_size_t) - ] + Security.SSLGetNumberSupportedCiphers.argtypes = [SSLContextRef, POINTER(c_size_t)] Security.SSLGetNumberSupportedCiphers.restype = OSStatus Security.SSLGetSupportedCiphers.argtypes = [ SSLContextRef, POINTER(SSLCipherSuite), - POINTER(c_size_t) + POINTER(c_size_t), ] Security.SSLGetSupportedCiphers.restype = OSStatus Security.SSLSetEnabledCiphers.argtypes = [ SSLContextRef, POINTER(SSLCipherSuite), - c_size_t + c_size_t, ] Security.SSLSetEnabledCiphers.restype = OSStatus - Security.SSLGetNumberEnabledCiphers.argtype = [ - SSLContextRef, - POINTER(c_size_t) - ] + Security.SSLGetNumberEnabledCiphers.argtype = [SSLContextRef, POINTER(c_size_t)] Security.SSLGetNumberEnabledCiphers.restype = OSStatus Security.SSLGetEnabledCiphers.argtypes = [ SSLContextRef, POINTER(SSLCipherSuite), - POINTER(c_size_t) + POINTER(c_size_t), ] Security.SSLGetEnabledCiphers.restype = OSStatus - Security.SSLGetNegotiatedCipher.argtypes = [ - SSLContextRef, - POINTER(SSLCipherSuite) - ] + Security.SSLGetNegotiatedCipher.argtypes = [SSLContextRef, POINTER(SSLCipherSuite)] Security.SSLGetNegotiatedCipher.restype = OSStatus Security.SSLGetNegotiatedProtocolVersion.argtypes = [ SSLContextRef, - POINTER(SSLProtocol) + POINTER(SSLProtocol), ] Security.SSLGetNegotiatedProtocolVersion.restype = OSStatus - Security.SSLCopyPeerTrust.argtypes = [ - SSLContextRef, - POINTER(SecTrustRef) - ] + Security.SSLCopyPeerTrust.argtypes = [SSLContextRef, POINTER(SecTrustRef)] Security.SSLCopyPeerTrust.restype = OSStatus - Security.SecTrustSetAnchorCertificates.argtypes = [ - SecTrustRef, - CFArrayRef - ] + Security.SecTrustSetAnchorCertificates.argtypes = [SecTrustRef, CFArrayRef] Security.SecTrustSetAnchorCertificates.restype = OSStatus - Security.SecTrustSetAnchorCertificatesOnly.argstypes = [ - SecTrustRef, - Boolean - ] + Security.SecTrustSetAnchorCertificatesOnly.argstypes = [SecTrustRef, Boolean] Security.SecTrustSetAnchorCertificatesOnly.restype = OSStatus - Security.SecTrustEvaluate.argtypes = [ - SecTrustRef, - POINTER(SecTrustResultType) - ] + Security.SecTrustEvaluate.argtypes = [SecTrustRef, POINTER(SecTrustResultType)] Security.SecTrustEvaluate.restype = OSStatus - Security.SecTrustGetCertificateCount.argtypes = [ - SecTrustRef - ] + Security.SecTrustGetCertificateCount.argtypes = [SecTrustRef] Security.SecTrustGetCertificateCount.restype = CFIndex - Security.SecTrustGetCertificateAtIndex.argtypes = [ - SecTrustRef, - CFIndex - ] + Security.SecTrustGetCertificateAtIndex.argtypes = [SecTrustRef, CFIndex] Security.SecTrustGetCertificateAtIndex.restype = SecCertificateRef Security.SSLCreateContext.argtypes = [ CFAllocatorRef, SSLProtocolSide, - SSLConnectionType + SSLConnectionType, ] Security.SSLCreateContext.restype = SSLContextRef - Security.SSLSetSessionOption.argtypes = [ - SSLContextRef, - SSLSessionOption, - Boolean - ] + Security.SSLSetSessionOption.argtypes = [SSLContextRef, SSLSessionOption, Boolean] Security.SSLSetSessionOption.restype = OSStatus - Security.SSLSetProtocolVersionMin.argtypes = [ - SSLContextRef, - SSLProtocol - ] + Security.SSLSetProtocolVersionMin.argtypes = [SSLContextRef, SSLProtocol] Security.SSLSetProtocolVersionMin.restype = OSStatus - Security.SSLSetProtocolVersionMax.argtypes = [ - SSLContextRef, - SSLProtocol - ] + Security.SSLSetProtocolVersionMax.argtypes = [SSLContextRef, SSLProtocol] Security.SSLSetProtocolVersionMax.restype = OSStatus - Security.SecCopyErrorMessageString.argtypes = [ - OSStatus, - c_void_p - ] + Security.SecCopyErrorMessageString.argtypes = [OSStatus, c_void_p] Security.SecCopyErrorMessageString.restype = CFStringRef Security.SSLReadFunc = SSLReadFunc @@ -369,64 +292,47 @@ try: Security.OSStatus = OSStatus Security.kSecImportExportPassphrase = CFStringRef.in_dll( - Security, 'kSecImportExportPassphrase' + Security, "kSecImportExportPassphrase" ) Security.kSecImportItemIdentity = CFStringRef.in_dll( - Security, 'kSecImportItemIdentity' + Security, "kSecImportItemIdentity" ) # CoreFoundation time! - CoreFoundation.CFRetain.argtypes = [ - CFTypeRef - ] + CoreFoundation.CFRetain.argtypes = [CFTypeRef] CoreFoundation.CFRetain.restype = CFTypeRef - CoreFoundation.CFRelease.argtypes = [ - CFTypeRef - ] + CoreFoundation.CFRelease.argtypes = [CFTypeRef] CoreFoundation.CFRelease.restype = None - CoreFoundation.CFGetTypeID.argtypes = [ - CFTypeRef - ] + CoreFoundation.CFGetTypeID.argtypes = [CFTypeRef] CoreFoundation.CFGetTypeID.restype = CFTypeID CoreFoundation.CFStringCreateWithCString.argtypes = [ CFAllocatorRef, c_char_p, - CFStringEncoding + CFStringEncoding, ] CoreFoundation.CFStringCreateWithCString.restype = CFStringRef - CoreFoundation.CFStringGetCStringPtr.argtypes = [ - CFStringRef, - CFStringEncoding - ] + CoreFoundation.CFStringGetCStringPtr.argtypes = [CFStringRef, CFStringEncoding] CoreFoundation.CFStringGetCStringPtr.restype = c_char_p CoreFoundation.CFStringGetCString.argtypes = [ CFStringRef, c_char_p, CFIndex, - CFStringEncoding + CFStringEncoding, ] CoreFoundation.CFStringGetCString.restype = c_bool - CoreFoundation.CFDataCreate.argtypes = [ - CFAllocatorRef, - c_char_p, - CFIndex - ] + CoreFoundation.CFDataCreate.argtypes = [CFAllocatorRef, c_char_p, CFIndex] CoreFoundation.CFDataCreate.restype = CFDataRef - CoreFoundation.CFDataGetLength.argtypes = [ - CFDataRef - ] + CoreFoundation.CFDataGetLength.argtypes = [CFDataRef] CoreFoundation.CFDataGetLength.restype = CFIndex - CoreFoundation.CFDataGetBytePtr.argtypes = [ - CFDataRef - ] + CoreFoundation.CFDataGetBytePtr.argtypes = [CFDataRef] CoreFoundation.CFDataGetBytePtr.restype = c_void_p CoreFoundation.CFDictionaryCreate.argtypes = [ @@ -435,14 +341,11 @@ try: POINTER(CFTypeRef), CFIndex, CFDictionaryKeyCallBacks, - CFDictionaryValueCallBacks + CFDictionaryValueCallBacks, ] CoreFoundation.CFDictionaryCreate.restype = CFDictionaryRef - CoreFoundation.CFDictionaryGetValue.argtypes = [ - CFDictionaryRef, - CFTypeRef - ] + CoreFoundation.CFDictionaryGetValue.argtypes = [CFDictionaryRef, CFTypeRef] CoreFoundation.CFDictionaryGetValue.restype = CFTypeRef CoreFoundation.CFArrayCreate.argtypes = [ @@ -456,36 +359,30 @@ try: CoreFoundation.CFArrayCreateMutable.argtypes = [ CFAllocatorRef, CFIndex, - CFArrayCallBacks + CFArrayCallBacks, ] CoreFoundation.CFArrayCreateMutable.restype = CFMutableArrayRef - CoreFoundation.CFArrayAppendValue.argtypes = [ - CFMutableArrayRef, - c_void_p - ] + CoreFoundation.CFArrayAppendValue.argtypes = [CFMutableArrayRef, c_void_p] CoreFoundation.CFArrayAppendValue.restype = None - CoreFoundation.CFArrayGetCount.argtypes = [ - CFArrayRef - ] + CoreFoundation.CFArrayGetCount.argtypes = [CFArrayRef] CoreFoundation.CFArrayGetCount.restype = CFIndex - CoreFoundation.CFArrayGetValueAtIndex.argtypes = [ - CFArrayRef, - CFIndex - ] + CoreFoundation.CFArrayGetValueAtIndex.argtypes = [CFArrayRef, CFIndex] CoreFoundation.CFArrayGetValueAtIndex.restype = c_void_p CoreFoundation.kCFAllocatorDefault = CFAllocatorRef.in_dll( - CoreFoundation, 'kCFAllocatorDefault' + CoreFoundation, "kCFAllocatorDefault" + ) + CoreFoundation.kCFTypeArrayCallBacks = c_void_p.in_dll( + CoreFoundation, "kCFTypeArrayCallBacks" ) - CoreFoundation.kCFTypeArrayCallBacks = c_void_p.in_dll(CoreFoundation, 'kCFTypeArrayCallBacks') CoreFoundation.kCFTypeDictionaryKeyCallBacks = c_void_p.in_dll( - CoreFoundation, 'kCFTypeDictionaryKeyCallBacks' + CoreFoundation, "kCFTypeDictionaryKeyCallBacks" ) CoreFoundation.kCFTypeDictionaryValueCallBacks = c_void_p.in_dll( - CoreFoundation, 'kCFTypeDictionaryValueCallBacks' + CoreFoundation, "kCFTypeDictionaryValueCallBacks" ) CoreFoundation.CFTypeRef = CFTypeRef @@ -494,7 +391,7 @@ try: CoreFoundation.CFDictionaryRef = CFDictionaryRef except (AttributeError): - raise ImportError('Error initializing ctypes') + raise ImportError("Error initializing ctypes") class CFConst(object): @@ -502,6 +399,7 @@ class CFConst(object): A class object that acts as essentially a namespace for CoreFoundation constants. """ + kCFStringEncodingUTF8 = CFStringEncoding(0x08000100) @@ -509,6 +407,7 @@ class SecurityConst(object): """ A class object that acts as essentially a namespace for Security constants. """ + kSSLSessionOptionBreakOnServerAuth = 0 kSSLProtocol2 = 1 diff --git a/lib/urllib3/contrib/_securetransport/low_level.py b/lib/urllib3/contrib/_securetransport/low_level.py index b13cd9e..e60168c 100644 --- a/lib/urllib3/contrib/_securetransport/low_level.py +++ b/lib/urllib3/contrib/_securetransport/low_level.py @@ -66,22 +66,18 @@ def _cf_string_to_unicode(value): value_as_void_p = ctypes.cast(value, ctypes.POINTER(ctypes.c_void_p)) string = CoreFoundation.CFStringGetCStringPtr( - value_as_void_p, - CFConst.kCFStringEncodingUTF8 + value_as_void_p, CFConst.kCFStringEncodingUTF8 ) if string is None: buffer = ctypes.create_string_buffer(1024) result = CoreFoundation.CFStringGetCString( - value_as_void_p, - buffer, - 1024, - CFConst.kCFStringEncodingUTF8 + value_as_void_p, buffer, 1024, CFConst.kCFStringEncodingUTF8 ) if not result: - raise OSError('Error copying C string from CFStringRef') + raise OSError("Error copying C string from CFStringRef") string = buffer.value if string is not None: - string = string.decode('utf-8') + string = string.decode("utf-8") return string @@ -97,8 +93,8 @@ def _assert_no_error(error, exception_class=None): output = _cf_string_to_unicode(cf_error_string) CoreFoundation.CFRelease(cf_error_string) - if output is None or output == u'': - output = u'OSStatus %s' % error + if output is None or output == u"": + output = u"OSStatus %s" % error if exception_class is None: exception_class = ssl.SSLError @@ -115,8 +111,7 @@ def _cert_array_from_pem(pem_bundle): pem_bundle = pem_bundle.replace(b"\r\n", b"\n") der_certs = [ - base64.b64decode(match.group(1)) - for match in _PEM_CERTS_RE.finditer(pem_bundle) + base64.b64decode(match.group(1)) for match in _PEM_CERTS_RE.finditer(pem_bundle) ] if not der_certs: raise ssl.SSLError("No root certificates specified") @@ -124,7 +119,7 @@ def _cert_array_from_pem(pem_bundle): cert_array = CoreFoundation.CFArrayCreateMutable( CoreFoundation.kCFAllocatorDefault, 0, - ctypes.byref(CoreFoundation.kCFTypeArrayCallBacks) + ctypes.byref(CoreFoundation.kCFTypeArrayCallBacks), ) if not cert_array: raise ssl.SSLError("Unable to allocate memory!") @@ -186,21 +181,16 @@ def _temporary_keychain(): # some random bytes to password-protect the keychain we're creating, so we # ask for 40 random bytes. random_bytes = os.urandom(40) - filename = base64.b16encode(random_bytes[:8]).decode('utf-8') + filename = base64.b16encode(random_bytes[:8]).decode("utf-8") password = base64.b16encode(random_bytes[8:]) # Must be valid UTF-8 tempdirectory = tempfile.mkdtemp() - keychain_path = os.path.join(tempdirectory, filename).encode('utf-8') + keychain_path = os.path.join(tempdirectory, filename).encode("utf-8") # We now want to create the keychain itself. keychain = Security.SecKeychainRef() status = Security.SecKeychainCreate( - keychain_path, - len(password), - password, - False, - None, - ctypes.byref(keychain) + keychain_path, len(password), password, False, None, ctypes.byref(keychain) ) _assert_no_error(status) @@ -219,14 +209,12 @@ def _load_items_from_file(keychain, path): identities = [] result_array = None - with open(path, 'rb') as f: + with open(path, "rb") as f: raw_filedata = f.read() try: filedata = CoreFoundation.CFDataCreate( - CoreFoundation.kCFAllocatorDefault, - raw_filedata, - len(raw_filedata) + CoreFoundation.kCFAllocatorDefault, raw_filedata, len(raw_filedata) ) result_array = CoreFoundation.CFArrayRef() result = Security.SecItemImport( @@ -237,7 +225,7 @@ def _load_items_from_file(keychain, path): 0, # import flags None, # key params, can include passphrase in the future keychain, # The keychain to insert into - ctypes.byref(result_array) # Results + ctypes.byref(result_array), # Results ) _assert_no_error(result) @@ -247,9 +235,7 @@ def _load_items_from_file(keychain, path): # keychain already has them! result_count = CoreFoundation.CFArrayGetCount(result_array) for index in range(result_count): - item = CoreFoundation.CFArrayGetValueAtIndex( - result_array, index - ) + item = CoreFoundation.CFArrayGetValueAtIndex(result_array, index) item = ctypes.cast(item, CoreFoundation.CFTypeRef) if _is_cert(item): @@ -307,9 +293,7 @@ def _load_client_cert_chain(keychain, *paths): try: for file_path in paths: - new_identities, new_certs = _load_items_from_file( - keychain, file_path - ) + new_identities, new_certs = _load_items_from_file(keychain, file_path) identities.extend(new_identities) certificates.extend(new_certs) @@ -318,9 +302,7 @@ def _load_client_cert_chain(keychain, *paths): if not identities: new_identity = Security.SecIdentityRef() status = Security.SecIdentityCreateWithCertificate( - keychain, - certificates[0], - ctypes.byref(new_identity) + keychain, certificates[0], ctypes.byref(new_identity) ) _assert_no_error(status) identities.append(new_identity) diff --git a/lib/urllib3/contrib/appengine.py b/lib/urllib3/contrib/appengine.py index 2952f11..01c9140 100644 --- a/lib/urllib3/contrib/appengine.py +++ b/lib/urllib3/contrib/appengine.py @@ -50,7 +50,7 @@ from ..exceptions import ( MaxRetryError, ProtocolError, TimeoutError, - SSLError + SSLError, ) from ..request import RequestMethods @@ -96,23 +96,31 @@ class AppEngineManager(RequestMethods): Beyond those cases, it will raise normal urllib3 errors. """ - def __init__(self, headers=None, retries=None, validate_certificate=True, - urlfetch_retries=True): + def __init__( + self, + headers=None, + retries=None, + validate_certificate=True, + urlfetch_retries=True, + ): if not urlfetch: raise AppEnginePlatformError( - "URLFetch is not available in this environment.") + "URLFetch is not available in this environment." + ) if is_prod_appengine_mvms(): raise AppEnginePlatformError( "Use normal urllib3.PoolManager instead of AppEngineManager" "on Managed VMs, as using URLFetch is not necessary in " - "this environment.") + "this environment." + ) warnings.warn( "urllib3 is using URLFetch on Google App Engine sandbox instead " "of sockets. To use sockets directly instead of URLFetch see " "https://urllib3.readthedocs.io/en/latest/reference/urllib3.contrib.html.", - AppEnginePlatformWarning) + AppEnginePlatformWarning, + ) RequestMethods.__init__(self, headers) self.validate_certificate = validate_certificate @@ -127,17 +135,22 @@ class AppEngineManager(RequestMethods): # Return False to re-raise any potential exceptions return False - def urlopen(self, method, url, body=None, headers=None, - retries=None, redirect=True, timeout=Timeout.DEFAULT_TIMEOUT, - **response_kw): + def urlopen( + self, + method, + url, + body=None, + headers=None, + retries=None, + redirect=True, + timeout=Timeout.DEFAULT_TIMEOUT, + **response_kw + ): retries = self._get_retries(retries, redirect) try: - follow_redirects = ( - redirect and - retries.redirect != 0 and - retries.total) + follow_redirects = redirect and retries.redirect != 0 and retries.total response = urlfetch.fetch( url, payload=body, @@ -152,44 +165,52 @@ class AppEngineManager(RequestMethods): raise TimeoutError(self, e) except urlfetch.InvalidURLError as e: - if 'too large' in str(e): + if "too large" in str(e): raise AppEnginePlatformError( "URLFetch request too large, URLFetch only " - "supports requests up to 10mb in size.", e) + "supports requests up to 10mb in size.", + e, + ) raise ProtocolError(e) except urlfetch.DownloadError as e: - if 'Too many redirects' in str(e): + if "Too many redirects" in str(e): raise MaxRetryError(self, url, reason=e) raise ProtocolError(e) except urlfetch.ResponseTooLargeError as e: raise AppEnginePlatformError( "URLFetch response too large, URLFetch only supports" - "responses up to 32mb in size.", e) + "responses up to 32mb in size.", + e, + ) except urlfetch.SSLCertificateError as e: raise SSLError(e) except urlfetch.InvalidMethodError as e: raise AppEnginePlatformError( - "URLFetch does not support method: %s" % method, e) + "URLFetch does not support method: %s" % method, e + ) http_response = self._urlfetch_response_to_http_response( - response, retries=retries, **response_kw) + response, retries=retries, **response_kw + ) # Handle redirect? redirect_location = redirect and http_response.get_redirect_location() if redirect_location: # Check for redirect response - if (self.urlfetch_retries and retries.raise_on_redirect): + if self.urlfetch_retries and retries.raise_on_redirect: raise MaxRetryError(self, url, "too many redirects") else: if http_response.status == 303: - method = 'GET' + method = "GET" try: - retries = retries.increment(method, url, response=http_response, _pool=self) + retries = retries.increment( + method, url, response=http_response, _pool=self + ) except MaxRetryError: if retries.raise_on_redirect: raise MaxRetryError(self, url, "too many redirects") @@ -199,22 +220,32 @@ class AppEngineManager(RequestMethods): log.debug("Redirecting %s -> %s", url, redirect_location) redirect_url = urljoin(url, redirect_location) return self.urlopen( - method, redirect_url, body, headers, - retries=retries, redirect=redirect, - timeout=timeout, **response_kw) + method, + redirect_url, + body, + headers, + retries=retries, + redirect=redirect, + timeout=timeout, + **response_kw + ) # Check if we should retry the HTTP response. - has_retry_after = bool(http_response.getheader('Retry-After')) + has_retry_after = bool(http_response.getheader("Retry-After")) if retries.is_retry(method, http_response.status, has_retry_after): - retries = retries.increment( - method, url, response=http_response, _pool=self) + retries = retries.increment(method, url, response=http_response, _pool=self) log.debug("Retry: %s", url) retries.sleep(http_response) return self.urlopen( - method, url, - body=body, headers=headers, - retries=retries, redirect=redirect, - timeout=timeout, **response_kw) + method, + url, + body=body, + headers=headers, + retries=retries, + redirect=redirect, + timeout=timeout, + **response_kw + ) return http_response @@ -223,18 +254,18 @@ class AppEngineManager(RequestMethods): if is_prod_appengine(): # Production GAE handles deflate encoding automatically, but does # not remove the encoding header. - content_encoding = urlfetch_resp.headers.get('content-encoding') + content_encoding = urlfetch_resp.headers.get("content-encoding") - if content_encoding == 'deflate': - del urlfetch_resp.headers['content-encoding'] + if content_encoding == "deflate": + del urlfetch_resp.headers["content-encoding"] - transfer_encoding = urlfetch_resp.headers.get('transfer-encoding') + transfer_encoding = urlfetch_resp.headers.get("transfer-encoding") # We have a full response's content, # so let's make sure we don't report ourselves as chunked data. - if transfer_encoding == 'chunked': + if transfer_encoding == "chunked": encodings = transfer_encoding.split(",") - encodings.remove('chunked') - urlfetch_resp.headers['transfer-encoding'] = ','.join(encodings) + encodings.remove("chunked") + urlfetch_resp.headers["transfer-encoding"] = ",".join(encodings) original_response = HTTPResponse( # In order for decoding to work, we must present the content as @@ -262,20 +293,21 @@ class AppEngineManager(RequestMethods): warnings.warn( "URLFetch does not support granular timeout settings, " "reverting to total or default URLFetch timeout.", - AppEnginePlatformWarning) + AppEnginePlatformWarning, + ) return timeout.total return timeout def _get_retries(self, retries, redirect): if not isinstance(retries, Retry): - retries = Retry.from_int( - retries, redirect=redirect, default=self.retries) + retries = Retry.from_int(retries, redirect=redirect, default=self.retries) if retries.connect or retries.read or retries.redirect: warnings.warn( "URLFetch only supports total retries and does not " "recognize connect, read, or redirect retry parameters.", - AppEnginePlatformWarning) + AppEnginePlatformWarning, + ) return retries diff --git a/lib/urllib3/contrib/ntlmpool.py b/lib/urllib3/contrib/ntlmpool.py index 8ea127c..9c96be2 100644 --- a/lib/urllib3/contrib/ntlmpool.py +++ b/lib/urllib3/contrib/ntlmpool.py @@ -20,7 +20,7 @@ class NTLMConnectionPool(HTTPSConnectionPool): Implements an NTLM authentication version of an urllib3 connection pool """ - scheme = 'https' + scheme = "https" def __init__(self, user, pw, authurl, *args, **kwargs): """ @@ -31,7 +31,7 @@ class NTLMConnectionPool(HTTPSConnectionPool): super(NTLMConnectionPool, self).__init__(*args, **kwargs) self.authurl = authurl self.rawuser = user - user_parts = user.split('\\', 1) + user_parts = user.split("\\", 1) self.domain = user_parts[0].upper() self.user = user_parts[1] self.pw = pw @@ -40,72 +40,84 @@ class NTLMConnectionPool(HTTPSConnectionPool): # Performs the NTLM handshake that secures the connection. The socket # must be kept open while requests are performed. self.num_connections += 1 - log.debug('Starting NTLM HTTPS connection no. %d: https://%s%s', - self.num_connections, self.host, self.authurl) + log.debug( + "Starting NTLM HTTPS connection no. %d: https://%s%s", + self.num_connections, + self.host, + self.authurl, + ) - headers = {'Connection': 'Keep-Alive'} - req_header = 'Authorization' - resp_header = 'www-authenticate' + headers = {"Connection": "Keep-Alive"} + req_header = "Authorization" + resp_header = "www-authenticate" conn = HTTPSConnection(host=self.host, port=self.port) # Send negotiation message - headers[req_header] = ( - 'NTLM %s' % ntlm.create_NTLM_NEGOTIATE_MESSAGE(self.rawuser)) - log.debug('Request headers: %s', headers) - conn.request('GET', self.authurl, None, headers) + headers[req_header] = "NTLM %s" % ntlm.create_NTLM_NEGOTIATE_MESSAGE( + self.rawuser + ) + log.debug("Request headers: %s", headers) + conn.request("GET", self.authurl, None, headers) res = conn.getresponse() reshdr = dict(res.getheaders()) - log.debug('Response status: %s %s', res.status, res.reason) - log.debug('Response headers: %s', reshdr) - log.debug('Response data: %s [...]', res.read(100)) + log.debug("Response status: %s %s", res.status, res.reason) + log.debug("Response headers: %s", reshdr) + log.debug("Response data: %s [...]", res.read(100)) # Remove the reference to the socket, so that it can not be closed by # the response object (we want to keep the socket open) res.fp = None # Server should respond with a challenge message - auth_header_values = reshdr[resp_header].split(', ') + auth_header_values = reshdr[resp_header].split(", ") auth_header_value = None for s in auth_header_values: - if s[:5] == 'NTLM ': + if s[:5] == "NTLM ": auth_header_value = s[5:] if auth_header_value is None: - raise Exception('Unexpected %s response header: %s' % - (resp_header, reshdr[resp_header])) + raise Exception( + "Unexpected %s response header: %s" % (resp_header, reshdr[resp_header]) + ) # Send authentication message - ServerChallenge, NegotiateFlags = \ - ntlm.parse_NTLM_CHALLENGE_MESSAGE(auth_header_value) - auth_msg = ntlm.create_NTLM_AUTHENTICATE_MESSAGE(ServerChallenge, - self.user, - self.domain, - self.pw, - NegotiateFlags) - headers[req_header] = 'NTLM %s' % auth_msg - log.debug('Request headers: %s', headers) - conn.request('GET', self.authurl, None, headers) + ServerChallenge, NegotiateFlags = ntlm.parse_NTLM_CHALLENGE_MESSAGE( + auth_header_value + ) + auth_msg = ntlm.create_NTLM_AUTHENTICATE_MESSAGE( + ServerChallenge, self.user, self.domain, self.pw, NegotiateFlags + ) + headers[req_header] = "NTLM %s" % auth_msg + log.debug("Request headers: %s", headers) + conn.request("GET", self.authurl, None, headers) res = conn.getresponse() - log.debug('Response status: %s %s', res.status, res.reason) - log.debug('Response headers: %s', dict(res.getheaders())) - log.debug('Response data: %s [...]', res.read()[:100]) + log.debug("Response status: %s %s", res.status, res.reason) + log.debug("Response headers: %s", dict(res.getheaders())) + log.debug("Response data: %s [...]", res.read()[:100]) if res.status != 200: if res.status == 401: - raise Exception('Server rejected request: wrong ' - 'username or password') - raise Exception('Wrong server response: %s %s' % - (res.status, res.reason)) + raise Exception( + "Server rejected request: wrong " "username or password" + ) + raise Exception("Wrong server response: %s %s" % (res.status, res.reason)) res.fp = None - log.debug('Connection established') + log.debug("Connection established") return conn - def urlopen(self, method, url, body=None, headers=None, retries=3, - redirect=True, assert_same_host=True): + def urlopen( + self, + method, + url, + body=None, + headers=None, + retries=3, + redirect=True, + assert_same_host=True, + ): if headers is None: headers = {} - headers['Connection'] = 'Keep-Alive' - return super(NTLMConnectionPool, self).urlopen(method, url, body, - headers, retries, - redirect, - assert_same_host) + headers["Connection"] = "Keep-Alive" + return super(NTLMConnectionPool, self).urlopen( + method, url, body, headers, retries, redirect, assert_same_host + ) diff --git a/lib/urllib3/contrib/pyopenssl.py b/lib/urllib3/contrib/pyopenssl.py index 821c174..3051ef3 100644 --- a/lib/urllib3/contrib/pyopenssl.py +++ b/lib/urllib3/contrib/pyopenssl.py @@ -47,6 +47,7 @@ import OpenSSL.SSL from cryptography import x509 from cryptography.hazmat.backends.openssl import backend as openssl_backend from cryptography.hazmat.backends.openssl.x509 import _Certificate + try: from cryptography.x509 import UnsupportedExtension except ImportError: @@ -54,6 +55,7 @@ except ImportError: class UnsupportedExtension(Exception): pass + from socket import timeout, error as SocketError from io import BytesIO @@ -71,7 +73,7 @@ import sys from .. import util -__all__ = ['inject_into_urllib3', 'extract_from_urllib3'] +__all__ = ["inject_into_urllib3", "extract_from_urllib3"] # SNI always works. HAS_SNI = True @@ -82,25 +84,23 @@ _openssl_versions = { ssl.PROTOCOL_TLSv1: OpenSSL.SSL.TLSv1_METHOD, } -if hasattr(ssl, 'PROTOCOL_SSLv3') and hasattr(OpenSSL.SSL, 'SSLv3_METHOD'): +if hasattr(ssl, "PROTOCOL_SSLv3") and hasattr(OpenSSL.SSL, "SSLv3_METHOD"): _openssl_versions[ssl.PROTOCOL_SSLv3] = OpenSSL.SSL.SSLv3_METHOD -if hasattr(ssl, 'PROTOCOL_TLSv1_1') and hasattr(OpenSSL.SSL, 'TLSv1_1_METHOD'): +if hasattr(ssl, "PROTOCOL_TLSv1_1") and hasattr(OpenSSL.SSL, "TLSv1_1_METHOD"): _openssl_versions[ssl.PROTOCOL_TLSv1_1] = OpenSSL.SSL.TLSv1_1_METHOD -if hasattr(ssl, 'PROTOCOL_TLSv1_2') and hasattr(OpenSSL.SSL, 'TLSv1_2_METHOD'): +if hasattr(ssl, "PROTOCOL_TLSv1_2") and hasattr(OpenSSL.SSL, "TLSv1_2_METHOD"): _openssl_versions[ssl.PROTOCOL_TLSv1_2] = OpenSSL.SSL.TLSv1_2_METHOD _stdlib_to_openssl_verify = { ssl.CERT_NONE: OpenSSL.SSL.VERIFY_NONE, ssl.CERT_OPTIONAL: OpenSSL.SSL.VERIFY_PEER, - ssl.CERT_REQUIRED: - OpenSSL.SSL.VERIFY_PEER + OpenSSL.SSL.VERIFY_FAIL_IF_NO_PEER_CERT, + ssl.CERT_REQUIRED: OpenSSL.SSL.VERIFY_PEER + + OpenSSL.SSL.VERIFY_FAIL_IF_NO_PEER_CERT, } -_openssl_to_stdlib_verify = dict( - (v, k) for k, v in _stdlib_to_openssl_verify.items() -) +_openssl_to_stdlib_verify = dict((v, k) for k, v in _stdlib_to_openssl_verify.items()) # OpenSSL will only write 16K at a time SSL_WRITE_BLOCKSIZE = 16384 @@ -113,7 +113,7 @@ log = logging.getLogger(__name__) def inject_into_urllib3(): - 'Monkey-patch urllib3 with PyOpenSSL-backed SSL-support.' + "Monkey-patch urllib3 with PyOpenSSL-backed SSL-support." _validate_dependencies_met() @@ -126,7 +126,7 @@ def inject_into_urllib3(): def extract_from_urllib3(): - 'Undo monkey-patching by :func:`inject_into_urllib3`.' + "Undo monkey-patching by :func:`inject_into_urllib3`." util.SSLContext = orig_util_SSLContext util.ssl_.SSLContext = orig_util_SSLContext @@ -143,17 +143,23 @@ def _validate_dependencies_met(): """ # Method added in `cryptography==1.1`; not available in older versions from cryptography.x509.extensions import Extensions + if getattr(Extensions, "get_extension_for_class", None) is None: - raise ImportError("'cryptography' module missing required functionality. " - "Try upgrading to v1.3.4 or newer.") + raise ImportError( + "'cryptography' module missing required functionality. " + "Try upgrading to v1.3.4 or newer." + ) # pyOpenSSL 0.14 and above use cryptography for OpenSSL bindings. The _x509 # attribute is only present on those versions. from OpenSSL.crypto import X509 + x509 = X509() if getattr(x509, "_x509", None) is None: - raise ImportError("'pyOpenSSL' module missing required functionality. " - "Try upgrading to v0.14 or newer.") + raise ImportError( + "'pyOpenSSL' module missing required functionality. " + "Try upgrading to v0.14 or newer." + ) def _dnsname_to_stdlib(name): @@ -169,6 +175,7 @@ def _dnsname_to_stdlib(name): If the name cannot be idna-encoded then we return None signalling that the name given should be skipped. """ + def idna_encode(name): """ Borrowed wholesale from the Python Cryptography Project. It turns out @@ -178,23 +185,23 @@ def _dnsname_to_stdlib(name): import idna try: - for prefix in [u'*.', u'.']: + for prefix in [u"*.", u"."]: if name.startswith(prefix): - name = name[len(prefix):] - return prefix.encode('ascii') + idna.encode(name) + name = name[len(prefix) :] + return prefix.encode("ascii") + idna.encode(name) return idna.encode(name) except idna.core.IDNAError: return None # Don't send IPv6 addresses through the IDNA encoder. - if ':' in name: + if ":" in name: return name name = idna_encode(name) if name is None: return None elif sys.version_info >= (3, 0): - name = name.decode('utf-8') + name = name.decode("utf-8") return name @@ -213,14 +220,16 @@ def get_subj_alt_name(peer_cert): # We want to find the SAN extension. Ask Cryptography to locate it (it's # faster than looping in Python) try: - ext = cert.extensions.get_extension_for_class( - x509.SubjectAlternativeName - ).value + ext = cert.extensions.get_extension_for_class(x509.SubjectAlternativeName).value except x509.ExtensionNotFound: # No such extension, return the empty list. return [] - except (x509.DuplicateExtension, UnsupportedExtension, - x509.UnsupportedGeneralNameType, UnicodeError) as e: + except ( + x509.DuplicateExtension, + UnsupportedExtension, + x509.UnsupportedGeneralNameType, + UnicodeError, + ) as e: # A problem has been found with the quality of the certificate. Assume # no SAN field is present. log.warning( @@ -239,23 +248,23 @@ def get_subj_alt_name(peer_cert): # does with certificates, and so we need to attempt to do the same. # We also want to skip over names which cannot be idna encoded. names = [ - ('DNS', name) for name in map(_dnsname_to_stdlib, ext.get_values_for_type(x509.DNSName)) + ("DNS", name) + for name in map(_dnsname_to_stdlib, ext.get_values_for_type(x509.DNSName)) if name is not None ] names.extend( - ('IP Address', str(name)) - for name in ext.get_values_for_type(x509.IPAddress) + ("IP Address", str(name)) for name in ext.get_values_for_type(x509.IPAddress) ) return names class WrappedSocket(object): - '''API-compatibility wrapper for Python OpenSSL's Connection-class. + """API-compatibility wrapper for Python OpenSSL's Connection-class. Note: _makefile_refs, _drop() and _reuse() are needed for the garbage collector of pypy. - ''' + """ def __init__(self, connection, socket, suppress_ragged_eofs=True): self.connection = connection @@ -278,18 +287,18 @@ class WrappedSocket(object): try: data = self.connection.recv(*args, **kwargs) except OpenSSL.SSL.SysCallError as e: - if self.suppress_ragged_eofs and e.args == (-1, 'Unexpected EOF'): - return b'' + if self.suppress_ragged_eofs and e.args == (-1, "Unexpected EOF"): + return b"" else: raise SocketError(str(e)) except OpenSSL.SSL.ZeroReturnError: if self.connection.get_shutdown() == OpenSSL.SSL.RECEIVED_SHUTDOWN: - return b'' + return b"" else: raise except OpenSSL.SSL.WantReadError: if not util.wait_for_read(self.socket, self.socket.gettimeout()): - raise timeout('The read operation timed out') + raise timeout("The read operation timed out") else: return self.recv(*args, **kwargs) @@ -303,7 +312,7 @@ class WrappedSocket(object): try: return self.connection.recv_into(*args, **kwargs) except OpenSSL.SSL.SysCallError as e: - if self.suppress_ragged_eofs and e.args == (-1, 'Unexpected EOF'): + if self.suppress_ragged_eofs and e.args == (-1, "Unexpected EOF"): return 0 else: raise SocketError(str(e)) @@ -314,7 +323,7 @@ class WrappedSocket(object): raise except OpenSSL.SSL.WantReadError: if not util.wait_for_read(self.socket, self.socket.gettimeout()): - raise timeout('The read operation timed out') + raise timeout("The read operation timed out") else: return self.recv_into(*args, **kwargs) @@ -339,7 +348,9 @@ class WrappedSocket(object): def sendall(self, data): total_sent = 0 while total_sent < len(data): - sent = self._send_until_done(data[total_sent:total_sent + SSL_WRITE_BLOCKSIZE]) + sent = self._send_until_done( + data[total_sent : total_sent + SSL_WRITE_BLOCKSIZE] + ) total_sent += sent def shutdown(self): @@ -363,15 +374,11 @@ class WrappedSocket(object): return x509 if binary_form: - return OpenSSL.crypto.dump_certificate( - OpenSSL.crypto.FILETYPE_ASN1, - x509) + return OpenSSL.crypto.dump_certificate(OpenSSL.crypto.FILETYPE_ASN1, x509) return { - 'subject': ( - (('commonName', x509.get_subject().CN),), - ), - 'subjectAltName': get_subj_alt_name(x509) + "subject": ((("commonName", x509.get_subject().CN),),), + "subjectAltName": get_subj_alt_name(x509), } def version(self): @@ -388,9 +395,12 @@ class WrappedSocket(object): if _fileobject: # Platform-specific: Python 2 + def makefile(self, mode, bufsize=-1): self._makefile_refs += 1 return _fileobject(self, mode, bufsize, close=True) + + else: # Platform-specific: Python 3 makefile = backport_makefile @@ -403,6 +413,7 @@ class PyOpenSSLContext(object): for translating the interface of the standard library ``SSLContext`` object to calls into PyOpenSSL. """ + def __init__(self, protocol): self.protocol = _openssl_versions[protocol] self._ctx = OpenSSL.SSL.Context(self.protocol) @@ -424,24 +435,21 @@ class PyOpenSSLContext(object): @verify_mode.setter def verify_mode(self, value): - self._ctx.set_verify( - _stdlib_to_openssl_verify[value], - _verify_callback - ) + self._ctx.set_verify(_stdlib_to_openssl_verify[value], _verify_callback) def set_default_verify_paths(self): self._ctx.set_default_verify_paths() def set_ciphers(self, ciphers): if isinstance(ciphers, six.text_type): - ciphers = ciphers.encode('utf-8') + ciphers = ciphers.encode("utf-8") self._ctx.set_cipher_list(ciphers) def load_verify_locations(self, cafile=None, capath=None, cadata=None): if cafile is not None: - cafile = cafile.encode('utf-8') + cafile = cafile.encode("utf-8") if capath is not None: - capath = capath.encode('utf-8') + capath = capath.encode("utf-8") self._ctx.load_verify_locations(cafile, capath) if cadata is not None: self._ctx.load_verify_locations(BytesIO(cadata)) @@ -450,17 +458,22 @@ class PyOpenSSLContext(object): self._ctx.use_certificate_chain_file(certfile) if password is not None: if not isinstance(password, six.binary_type): - password = password.encode('utf-8') + password = password.encode("utf-8") self._ctx.set_passwd_cb(lambda *_: password) self._ctx.use_privatekey_file(keyfile or certfile) - def wrap_socket(self, sock, server_side=False, - do_handshake_on_connect=True, suppress_ragged_eofs=True, - server_hostname=None): + def wrap_socket( + self, + sock, + server_side=False, + do_handshake_on_connect=True, + suppress_ragged_eofs=True, + server_hostname=None, + ): cnx = OpenSSL.SSL.Connection(self._ctx, sock) if isinstance(server_hostname, six.text_type): # Platform-specific: Python 3 - server_hostname = server_hostname.encode('utf-8') + server_hostname = server_hostname.encode("utf-8") if server_hostname is not None: cnx.set_tlsext_host_name(server_hostname) @@ -472,10 +485,10 @@ class PyOpenSSLContext(object): cnx.do_handshake() except OpenSSL.SSL.WantReadError: if not util.wait_for_read(sock, sock.gettimeout()): - raise timeout('select timed out') + raise timeout("select timed out") continue except OpenSSL.SSL.Error as e: - raise ssl.SSLError('bad handshake: %r' % e) + raise ssl.SSLError("bad handshake: %r" % e) break return WrappedSocket(cnx, sock) diff --git a/lib/urllib3/contrib/securetransport.py b/lib/urllib3/contrib/securetransport.py index 4dc4848..24e6b5c 100644 --- a/lib/urllib3/contrib/securetransport.py +++ b/lib/urllib3/contrib/securetransport.py @@ -62,12 +62,12 @@ import threading import weakref from .. import util -from ._securetransport.bindings import ( - Security, SecurityConst, CoreFoundation -) +from ._securetransport.bindings import Security, SecurityConst, CoreFoundation from ._securetransport.low_level import ( - _assert_no_error, _cert_array_from_pem, _temporary_keychain, - _load_client_cert_chain + _assert_no_error, + _cert_array_from_pem, + _temporary_keychain, + _load_client_cert_chain, ) try: # Platform-specific: Python 2 @@ -76,7 +76,7 @@ except ImportError: # Platform-specific: Python 3 _fileobject = None from ..packages.backports.makefile import backport_makefile -__all__ = ['inject_into_urllib3', 'extract_from_urllib3'] +__all__ = ["inject_into_urllib3", "extract_from_urllib3"] # SNI always works HAS_SNI = True @@ -147,28 +147,36 @@ CIPHER_SUITES = [ # TLSv1 and a high of TLSv1.3. For everything else, we pin to that version. # TLSv1 to 1.2 are supported on macOS 10.8+ and TLSv1.3 is macOS 10.13+ _protocol_to_min_max = { - util.PROTOCOL_TLS: (SecurityConst.kTLSProtocol1, SecurityConst.kTLSProtocolMaxSupported), + util.PROTOCOL_TLS: ( + SecurityConst.kTLSProtocol1, + SecurityConst.kTLSProtocolMaxSupported, + ) } if hasattr(ssl, "PROTOCOL_SSLv2"): _protocol_to_min_max[ssl.PROTOCOL_SSLv2] = ( - SecurityConst.kSSLProtocol2, SecurityConst.kSSLProtocol2 + SecurityConst.kSSLProtocol2, + SecurityConst.kSSLProtocol2, ) if hasattr(ssl, "PROTOCOL_SSLv3"): _protocol_to_min_max[ssl.PROTOCOL_SSLv3] = ( - SecurityConst.kSSLProtocol3, SecurityConst.kSSLProtocol3 + SecurityConst.kSSLProtocol3, + SecurityConst.kSSLProtocol3, ) if hasattr(ssl, "PROTOCOL_TLSv1"): _protocol_to_min_max[ssl.PROTOCOL_TLSv1] = ( - SecurityConst.kTLSProtocol1, SecurityConst.kTLSProtocol1 + SecurityConst.kTLSProtocol1, + SecurityConst.kTLSProtocol1, ) if hasattr(ssl, "PROTOCOL_TLSv1_1"): _protocol_to_min_max[ssl.PROTOCOL_TLSv1_1] = ( - SecurityConst.kTLSProtocol11, SecurityConst.kTLSProtocol11 + SecurityConst.kTLSProtocol11, + SecurityConst.kTLSProtocol11, ) if hasattr(ssl, "PROTOCOL_TLSv1_2"): _protocol_to_min_max[ssl.PROTOCOL_TLSv1_2] = ( - SecurityConst.kTLSProtocol12, SecurityConst.kTLSProtocol12 + SecurityConst.kTLSProtocol12, + SecurityConst.kTLSProtocol12, ) @@ -218,7 +226,7 @@ def _read_callback(connection_id, data_buffer, data_length_pointer): while read_count < requested_length: if timeout is None or timeout >= 0: if not util.wait_for_read(base_socket, timeout): - raise socket.error(errno.EAGAIN, 'timed out') + raise socket.error(errno.EAGAIN, "timed out") remaining = requested_length - read_count buffer = (ctypes.c_char * remaining).from_address( @@ -274,7 +282,7 @@ def _write_callback(connection_id, data_buffer, data_length_pointer): while sent < bytes_to_write: if timeout is None or timeout >= 0: if not util.wait_for_write(base_socket, timeout): - raise socket.error(errno.EAGAIN, 'timed out') + raise socket.error(errno.EAGAIN, "timed out") chunk_sent = base_socket.send(data) sent += chunk_sent @@ -316,6 +324,7 @@ class WrappedSocket(object): Note: _makefile_refs, _drop(), and _reuse() are needed for the garbage collector of PyPy. """ + def __init__(self, socket): self.socket = socket self.context = None @@ -380,7 +389,7 @@ class WrappedSocket(object): # We want data in memory, so load it up. if os.path.isfile(trust_bundle): - with open(trust_bundle, 'rb') as f: + with open(trust_bundle, "rb") as f: trust_bundle = f.read() cert_array = None @@ -394,9 +403,7 @@ class WrappedSocket(object): # created for this connection, shove our CAs into it, tell ST to # ignore everything else it knows, and then ask if it can build a # chain. This is a buuuunch of code. - result = Security.SSLCopyPeerTrust( - self.context, ctypes.byref(trust) - ) + result = Security.SSLCopyPeerTrust(self.context, ctypes.byref(trust)) _assert_no_error(result) if not trust: raise ssl.SSLError("Failed to copy trust reference") @@ -408,9 +415,7 @@ class WrappedSocket(object): _assert_no_error(result) trust_result = Security.SecTrustResultType() - result = Security.SecTrustEvaluate( - trust, ctypes.byref(trust_result) - ) + result = Security.SecTrustEvaluate(trust, ctypes.byref(trust_result)) _assert_no_error(result) finally: if trust: @@ -422,23 +427,24 @@ class WrappedSocket(object): # Ok, now we can look at what the result was. successes = ( SecurityConst.kSecTrustResultUnspecified, - SecurityConst.kSecTrustResultProceed + SecurityConst.kSecTrustResultProceed, ) if trust_result.value not in successes: raise ssl.SSLError( - "certificate verify failed, error code: %d" % - trust_result.value + "certificate verify failed, error code: %d" % trust_result.value ) - def handshake(self, - server_hostname, - verify, - trust_bundle, - min_version, - max_version, - client_cert, - client_key, - client_key_passphrase): + def handshake( + self, + server_hostname, + verify, + trust_bundle, + min_version, + max_version, + client_cert, + client_key, + client_key_passphrase, + ): """ Actually performs the TLS handshake. This is run automatically by wrapped socket, and shouldn't be needed in user code. @@ -468,7 +474,7 @@ class WrappedSocket(object): # If we have a server hostname, we should set that too. if server_hostname: if not isinstance(server_hostname, bytes): - server_hostname = server_hostname.encode('utf-8') + server_hostname = server_hostname.encode("utf-8") result = Security.SSLSetPeerDomainName( self.context, server_hostname, len(server_hostname) @@ -488,7 +494,9 @@ class WrappedSocket(object): # was added in macOS 10.13 along with kTLSProtocol13. result = Security.SSLSetProtocolVersionMax(self.context, max_version) if result != 0 and max_version == SecurityConst.kTLSProtocolMaxSupported: - result = Security.SSLSetProtocolVersionMax(self.context, SecurityConst.kTLSProtocol12) + result = Security.SSLSetProtocolVersionMax( + self.context, SecurityConst.kTLSProtocol12 + ) _assert_no_error(result) # If there's a trust DB, we need to use it. We do that by telling @@ -497,9 +505,7 @@ class WrappedSocket(object): # authing in that case. if not verify or trust_bundle is not None: result = Security.SSLSetSessionOption( - self.context, - SecurityConst.kSSLSessionOptionBreakOnServerAuth, - True + self.context, SecurityConst.kSSLSessionOptionBreakOnServerAuth, True ) _assert_no_error(result) @@ -509,9 +515,7 @@ class WrappedSocket(object): self._client_cert_chain = _load_client_cert_chain( self._keychain, client_cert, client_key ) - result = Security.SSLSetCertificate( - self.context, self._client_cert_chain - ) + result = Security.SSLSetCertificate(self.context, self._client_cert_chain) _assert_no_error(result) while True: @@ -562,7 +566,7 @@ class WrappedSocket(object): # There are some result codes that we want to treat as "not always # errors". Specifically, those are errSSLWouldBlock, # errSSLClosedGraceful, and errSSLClosedNoNotify. - if (result == SecurityConst.errSSLWouldBlock): + if result == SecurityConst.errSSLWouldBlock: # If we didn't process any bytes, then this was just a time out. # However, we can get errSSLWouldBlock in situations when we *did* # read some data, and in those cases we should just read "short" @@ -570,7 +574,10 @@ class WrappedSocket(object): if processed_bytes.value == 0: # Timed out, no data read. raise socket.timeout("recv timed out") - elif result in (SecurityConst.errSSLClosedGraceful, SecurityConst.errSSLClosedNoNotify): + elif result in ( + SecurityConst.errSSLClosedGraceful, + SecurityConst.errSSLClosedNoNotify, + ): # The remote peer has closed this connection. We should do so as # well. Note that we don't actually return here because in # principle this could actually be fired along with return data. @@ -609,7 +616,7 @@ class WrappedSocket(object): def sendall(self, data): total_sent = 0 while total_sent < len(data): - sent = self.send(data[total_sent:total_sent + SSL_WRITE_BLOCKSIZE]) + sent = self.send(data[total_sent : total_sent + SSL_WRITE_BLOCKSIZE]) total_sent += sent def shutdown(self): @@ -656,18 +663,14 @@ class WrappedSocket(object): # instead to just flag to urllib3 that it shouldn't do its own hostname # validation when using SecureTransport. if not binary_form: - raise ValueError( - "SecureTransport only supports dumping binary certs" - ) + raise ValueError("SecureTransport only supports dumping binary certs") trust = Security.SecTrustRef() certdata = None der_bytes = None try: # Grab the trust store. - result = Security.SSLCopyPeerTrust( - self.context, ctypes.byref(trust) - ) + result = Security.SSLCopyPeerTrust(self.context, ctypes.byref(trust)) _assert_no_error(result) if not trust: # Probably we haven't done the handshake yet. No biggie. @@ -699,22 +702,24 @@ class WrappedSocket(object): def version(self): protocol = Security.SSLProtocol() - result = Security.SSLGetNegotiatedProtocolVersion(self.context, ctypes.byref(protocol)) + result = Security.SSLGetNegotiatedProtocolVersion( + self.context, ctypes.byref(protocol) + ) _assert_no_error(result) if protocol.value == SecurityConst.kTLSProtocol13: - return 'TLSv1.3' + return "TLSv1.3" elif protocol.value == SecurityConst.kTLSProtocol12: - return 'TLSv1.2' + return "TLSv1.2" elif protocol.value == SecurityConst.kTLSProtocol11: - return 'TLSv1.1' + return "TLSv1.1" elif protocol.value == SecurityConst.kTLSProtocol1: - return 'TLSv1' + return "TLSv1" elif protocol.value == SecurityConst.kSSLProtocol3: - return 'SSLv3' + return "SSLv3" elif protocol.value == SecurityConst.kSSLProtocol2: - return 'SSLv2' + return "SSLv2" else: - raise ssl.SSLError('Unknown TLS version: %r' % protocol) + raise ssl.SSLError("Unknown TLS version: %r" % protocol) def _reuse(self): self._makefile_refs += 1 @@ -727,16 +732,21 @@ class WrappedSocket(object): if _fileobject: # Platform-specific: Python 2 + def makefile(self, mode, bufsize=-1): self._makefile_refs += 1 return _fileobject(self, mode, bufsize, close=True) + + else: # Platform-specific: Python 3 + def makefile(self, mode="r", buffering=None, *args, **kwargs): # We disable buffering with SecureTransport because it conflicts with # the buffering that ST does internally (see issue #1153 for more). buffering = 0 return backport_makefile(self, mode, buffering, *args, **kwargs) + WrappedSocket.makefile = makefile @@ -746,6 +756,7 @@ class SecureTransportContext(object): interface of the standard library ``SSLContext`` object to calls into SecureTransport. """ + def __init__(self, protocol): self._min_version, self._max_version = _protocol_to_min_max[protocol] self._options = 0 @@ -812,16 +823,12 @@ class SecureTransportContext(object): def set_ciphers(self, ciphers): # For now, we just require the default cipher string. if ciphers != util.ssl_.DEFAULT_CIPHERS: - raise ValueError( - "SecureTransport doesn't support custom cipher strings" - ) + raise ValueError("SecureTransport doesn't support custom cipher strings") def load_verify_locations(self, cafile=None, capath=None, cadata=None): # OK, we only really support cadata and cafile. if capath is not None: - raise ValueError( - "SecureTransport does not support cert directories" - ) + raise ValueError("SecureTransport does not support cert directories") self._trust_bundle = cafile or cadata @@ -830,9 +837,14 @@ class SecureTransportContext(object): self._client_key = keyfile self._client_cert_passphrase = password - def wrap_socket(self, sock, server_side=False, - do_handshake_on_connect=True, suppress_ragged_eofs=True, - server_hostname=None): + def wrap_socket( + self, + sock, + server_side=False, + do_handshake_on_connect=True, + suppress_ragged_eofs=True, + server_hostname=None, + ): # So, what do we do here? Firstly, we assert some properties. This is a # stripped down shim, so there is some functionality we don't support. # See PEP 543 for the real deal. @@ -846,8 +858,13 @@ class SecureTransportContext(object): # Now we can handshake wrapped_socket.handshake( - server_hostname, self._verify, self._trust_bundle, - self._min_version, self._max_version, self._client_cert, - self._client_key, self._client_key_passphrase + server_hostname, + self._verify, + self._trust_bundle, + self._min_version, + self._max_version, + self._client_cert, + self._client_key, + self._client_key_passphrase, ) return wrapped_socket diff --git a/lib/urllib3/contrib/socks.py b/lib/urllib3/contrib/socks.py index 636d261..9e97f7a 100644 --- a/lib/urllib3/contrib/socks.py +++ b/lib/urllib3/contrib/socks.py @@ -42,23 +42,20 @@ except ImportError: import warnings from ..exceptions import DependencyWarning - warnings.warn(( - 'SOCKS support in urllib3 requires the installation of optional ' - 'dependencies: specifically, PySocks. For more information, see ' - 'https://urllib3.readthedocs.io/en/latest/contrib.html#socks-proxies' + warnings.warn( + ( + "SOCKS support in urllib3 requires the installation of optional " + "dependencies: specifically, PySocks. For more information, see " + "https://urllib3.readthedocs.io/en/latest/contrib.html#socks-proxies" ), - DependencyWarning + DependencyWarning, ) raise from socket import error as SocketError, timeout as SocketTimeout -from ..connection import ( - HTTPConnection, HTTPSConnection -) -from ..connectionpool import ( - HTTPConnectionPool, HTTPSConnectionPool -) +from ..connection import HTTPConnection, HTTPSConnection +from ..connectionpool import HTTPConnectionPool, HTTPSConnectionPool from ..exceptions import ConnectTimeoutError, NewConnectionError from ..poolmanager import PoolManager from ..util.url import parse_url @@ -73,8 +70,9 @@ class SOCKSConnection(HTTPConnection): """ A plain-text HTTP connection that connects via a SOCKS proxy. """ + def __init__(self, *args, **kwargs): - self._socks_options = kwargs.pop('_socks_options') + self._socks_options = kwargs.pop("_socks_options") super(SOCKSConnection, self).__init__(*args, **kwargs) def _new_conn(self): @@ -83,28 +81,30 @@ class SOCKSConnection(HTTPConnection): """ extra_kw = {} if self.source_address: - extra_kw['source_address'] = self.source_address + extra_kw["source_address"] = self.source_address if self.socket_options: - extra_kw['socket_options'] = self.socket_options + extra_kw["socket_options"] = self.socket_options try: conn = socks.create_connection( (self.host, self.port), - proxy_type=self._socks_options['socks_version'], - proxy_addr=self._socks_options['proxy_host'], - proxy_port=self._socks_options['proxy_port'], - proxy_username=self._socks_options['username'], - proxy_password=self._socks_options['password'], - proxy_rdns=self._socks_options['rdns'], + proxy_type=self._socks_options["socks_version"], + proxy_addr=self._socks_options["proxy_host"], + proxy_port=self._socks_options["proxy_port"], + proxy_username=self._socks_options["username"], + proxy_password=self._socks_options["password"], + proxy_rdns=self._socks_options["rdns"], timeout=self.timeout, **extra_kw ) except SocketTimeout: raise ConnectTimeoutError( - self, "Connection to %s timed out. (connect timeout=%s)" % - (self.host, self.timeout)) + self, + "Connection to %s timed out. (connect timeout=%s)" + % (self.host, self.timeout), + ) except socks.ProxyError as e: # This is fragile as hell, but it seems to be the only way to raise @@ -114,23 +114,22 @@ class SOCKSConnection(HTTPConnection): if isinstance(error, SocketTimeout): raise ConnectTimeoutError( self, - "Connection to %s timed out. (connect timeout=%s)" % - (self.host, self.timeout) + "Connection to %s timed out. (connect timeout=%s)" + % (self.host, self.timeout), ) else: raise NewConnectionError( - self, - "Failed to establish a new connection: %s" % error + self, "Failed to establish a new connection: %s" % error ) else: raise NewConnectionError( - self, - "Failed to establish a new connection: %s" % e + self, "Failed to establish a new connection: %s" % e ) except SocketError as e: # Defensive: PySocks should catch all these. raise NewConnectionError( - self, "Failed to establish a new connection: %s" % e) + self, "Failed to establish a new connection: %s" % e + ) return conn @@ -156,47 +155,53 @@ class SOCKSProxyManager(PoolManager): A version of the urllib3 ProxyManager that routes connections via the defined SOCKS proxy. """ + pool_classes_by_scheme = { - 'http': SOCKSHTTPConnectionPool, - 'https': SOCKSHTTPSConnectionPool, + "http": SOCKSHTTPConnectionPool, + "https": SOCKSHTTPSConnectionPool, } - def __init__(self, proxy_url, username=None, password=None, - num_pools=10, headers=None, **connection_pool_kw): + def __init__( + self, + proxy_url, + username=None, + password=None, + num_pools=10, + headers=None, + **connection_pool_kw + ): parsed = parse_url(proxy_url) if username is None and password is None and parsed.auth is not None: - split = parsed.auth.split(':') + split = parsed.auth.split(":") if len(split) == 2: username, password = split - if parsed.scheme == 'socks5': + if parsed.scheme == "socks5": socks_version = socks.PROXY_TYPE_SOCKS5 rdns = False - elif parsed.scheme == 'socks5h': + elif parsed.scheme == "socks5h": socks_version = socks.PROXY_TYPE_SOCKS5 rdns = True - elif parsed.scheme == 'socks4': + elif parsed.scheme == "socks4": socks_version = socks.PROXY_TYPE_SOCKS4 rdns = False - elif parsed.scheme == 'socks4a': + elif parsed.scheme == "socks4a": socks_version = socks.PROXY_TYPE_SOCKS4 rdns = True else: - raise ValueError( - "Unable to determine SOCKS version from %s" % proxy_url - ) + raise ValueError("Unable to determine SOCKS version from %s" % proxy_url) self.proxy_url = proxy_url socks_options = { - 'socks_version': socks_version, - 'proxy_host': parsed.host, - 'proxy_port': parsed.port, - 'username': username, - 'password': password, - 'rdns': rdns + "socks_version": socks_version, + "proxy_host": parsed.host, + "proxy_port": parsed.port, + "username": username, + "password": password, + "rdns": rdns, } - connection_pool_kw['_socks_options'] = socks_options + connection_pool_kw["_socks_options"] = socks_options super(SOCKSProxyManager, self).__init__( num_pools, headers, **connection_pool_kw diff --git a/lib/urllib3/exceptions.py b/lib/urllib3/exceptions.py index 7bbaa98..93d93fb 100644 --- a/lib/urllib3/exceptions.py +++ b/lib/urllib3/exceptions.py @@ -1,7 +1,6 @@ from __future__ import absolute_import -from .packages.six.moves.http_client import ( - IncompleteRead as httplib_IncompleteRead -) +from .packages.six.moves.http_client import IncompleteRead as httplib_IncompleteRead + # Base Exceptions @@ -17,6 +16,7 @@ class HTTPWarning(Warning): class PoolError(HTTPError): "Base exception for errors caused within a pool." + def __init__(self, pool, message): self.pool = pool HTTPError.__init__(self, "%s: %s" % (pool, message)) @@ -28,6 +28,7 @@ class PoolError(HTTPError): class RequestError(PoolError): "Base exception for PoolErrors that have associated URLs." + def __init__(self, pool, url, message): self.url = url PoolError.__init__(self, pool, message) @@ -63,6 +64,7 @@ ConnectionError = ProtocolError # Leaf Exceptions + class MaxRetryError(RequestError): """Raised when the maximum number of retries is exceeded. @@ -76,8 +78,7 @@ class MaxRetryError(RequestError): def __init__(self, pool, url, reason=None): self.reason = reason - message = "Max retries exceeded with url: %s (Caused by %r)" % ( - url, reason) + message = "Max retries exceeded with url: %s (Caused by %r)" % (url, reason) RequestError.__init__(self, pool, url, message) @@ -93,6 +94,7 @@ class HostChangedError(RequestError): class TimeoutStateError(HTTPError): """ Raised when passing an invalid state to a timeout """ + pass @@ -102,6 +104,7 @@ class TimeoutError(HTTPError): Catching this error will catch both :exc:`ReadTimeoutErrors ` and :exc:`ConnectTimeoutErrors `. """ + pass @@ -149,8 +152,8 @@ class LocationParseError(LocationValueError): class ResponseError(HTTPError): "Used as a container for an error reason supplied in a MaxRetryError." - GENERIC_ERROR = 'too many error responses' - SPECIFIC_ERROR = 'too many {status_code} error responses' + GENERIC_ERROR = "too many error responses" + SPECIFIC_ERROR = "too many {status_code} error responses" class SecurityWarning(HTTPWarning): @@ -188,6 +191,7 @@ class DependencyWarning(HTTPWarning): Warned when an attempt is made to import a module with missing optional dependencies. """ + pass @@ -201,6 +205,7 @@ class BodyNotHttplibCompatible(HTTPError): Body should be httplib.HTTPResponse like (have an fp attribute which returns raw chunks) for read_chunked(). """ + pass @@ -212,12 +217,15 @@ class IncompleteRead(HTTPError, httplib_IncompleteRead): for `partial` to avoid creating large objects on streamed reads. """ + def __init__(self, partial, expected): super(IncompleteRead, self).__init__(partial, expected) def __repr__(self): - return ('IncompleteRead(%i bytes read, ' - '%i more expected)' % (self.partial, self.expected)) + return "IncompleteRead(%i bytes read, " "%i more expected)" % ( + self.partial, + self.expected, + ) class InvalidHeader(HTTPError): @@ -236,8 +244,9 @@ class ProxySchemeUnknown(AssertionError, ValueError): class HeaderParsingError(HTTPError): "Raised by assert_header_parsing, but we convert it to a log.warning statement." + def __init__(self, defects, unparsed_data): - message = '%s, unparsed data: %r' % (defects or 'Unknown', unparsed_data) + message = "%s, unparsed data: %r" % (defects or "Unknown", unparsed_data) super(HeaderParsingError, self).__init__(message) diff --git a/lib/urllib3/fields.py b/lib/urllib3/fields.py index 6a9a5a7..534436f 100644 --- a/lib/urllib3/fields.py +++ b/lib/urllib3/fields.py @@ -6,7 +6,7 @@ import re from .packages import six -def guess_content_type(filename, default='application/octet-stream'): +def guess_content_type(filename, default="application/octet-stream"): """ Guess the "Content-Type" of a file. @@ -41,22 +41,22 @@ def format_header_param_rfc2231(name, value): if not any(ch in value for ch in '"\\\r\n'): result = u'%s="%s"' % (name, value) try: - result.encode('ascii') + result.encode("ascii") except (UnicodeEncodeError, UnicodeDecodeError): pass else: return result if not six.PY3: # Python 2: - value = value.encode('utf-8') + value = value.encode("utf-8") # encode_rfc2231 accepts an encoded string and returns an ascii-encoded # string in Python 2 but accepts and returns unicode strings in Python 3 - value = email.utils.encode_rfc2231(value, 'utf-8') - value = '%s*=%s' % (name, value) + value = email.utils.encode_rfc2231(value, "utf-8") + value = "%s*=%s" % (name, value) if not six.PY3: # Python 2: - value = value.decode('utf-8') + value = value.decode("utf-8") return value @@ -69,23 +69,21 @@ _HTML5_REPLACEMENTS = { } # All control characters from 0x00 to 0x1F *except* 0x1B. -_HTML5_REPLACEMENTS.update({ - six.unichr(cc): u"%{:02X}".format(cc) - for cc - in range(0x00, 0x1F+1) - if cc not in (0x1B,) -}) +_HTML5_REPLACEMENTS.update( + { + six.unichr(cc): u"%{:02X}".format(cc) + for cc in range(0x00, 0x1F + 1) + if cc not in (0x1B,) + } +) def _replace_multiple(value, needles_and_replacements): - def replacer(match): return needles_and_replacements[match.group(0)] pattern = re.compile( - r"|".join([ - re.escape(needle) for needle in needles_and_replacements.keys() - ]) + r"|".join([re.escape(needle) for needle in needles_and_replacements.keys()]) ) result = pattern.sub(replacer, value) @@ -140,13 +138,15 @@ class RequestField(object): An optional callable that is used to encode and format the headers. By default, this is :func:`format_header_param_html5`. """ + def __init__( - self, - name, - data, - filename=None, - headers=None, - header_formatter=format_header_param_html5): + self, + name, + data, + filename=None, + headers=None, + header_formatter=format_header_param_html5, + ): self._name = name self._filename = filename self.data = data @@ -156,11 +156,7 @@ class RequestField(object): self.header_formatter = header_formatter @classmethod - def from_tuples( - cls, - fieldname, - value, - header_formatter=format_header_param_html5): + def from_tuples(cls, fieldname, value, header_formatter=format_header_param_html5): """ A :class:`~urllib3.fields.RequestField` factory from old-style tuple parameters. @@ -189,7 +185,8 @@ class RequestField(object): data = value request_param = cls( - fieldname, data, filename=filename, header_formatter=header_formatter) + fieldname, data, filename=filename, header_formatter=header_formatter + ) request_param.make_multipart(content_type=content_type) return request_param @@ -227,7 +224,7 @@ class RequestField(object): if value is not None: parts.append(self._render_part(name, value)) - return u'; '.join(parts) + return u"; ".join(parts) def render_headers(self): """ @@ -235,21 +232,22 @@ class RequestField(object): """ lines = [] - sort_keys = ['Content-Disposition', 'Content-Type', 'Content-Location'] + sort_keys = ["Content-Disposition", "Content-Type", "Content-Location"] for sort_key in sort_keys: if self.headers.get(sort_key, False): - lines.append(u'%s: %s' % (sort_key, self.headers[sort_key])) + lines.append(u"%s: %s" % (sort_key, self.headers[sort_key])) for header_name, header_value in self.headers.items(): if header_name not in sort_keys: if header_value: - lines.append(u'%s: %s' % (header_name, header_value)) + lines.append(u"%s: %s" % (header_name, header_value)) - lines.append(u'\r\n') - return u'\r\n'.join(lines) + lines.append(u"\r\n") + return u"\r\n".join(lines) - def make_multipart(self, content_disposition=None, content_type=None, - content_location=None): + def make_multipart( + self, content_disposition=None, content_type=None, content_location=None + ): """ Makes this request field into a multipart request field. @@ -262,11 +260,14 @@ class RequestField(object): The 'Content-Location' of the request body. """ - self.headers['Content-Disposition'] = content_disposition or u'form-data' - self.headers['Content-Disposition'] += u'; '.join([ - u'', self._render_parts( - ((u'name', self._name), (u'filename', self._filename)) - ) - ]) - self.headers['Content-Type'] = content_type - self.headers['Content-Location'] = content_location + self.headers["Content-Disposition"] = content_disposition or u"form-data" + self.headers["Content-Disposition"] += u"; ".join( + [ + u"", + self._render_parts( + ((u"name", self._name), (u"filename", self._filename)) + ), + ] + ) + self.headers["Content-Type"] = content_type + self.headers["Content-Location"] = content_location diff --git a/lib/urllib3/filepost.py b/lib/urllib3/filepost.py index 78f1e19..459e790 100644 --- a/lib/urllib3/filepost.py +++ b/lib/urllib3/filepost.py @@ -9,7 +9,7 @@ from .packages import six from .packages.six import b from .fields import RequestField -writer = codecs.lookup('utf-8')[3] +writer = codecs.lookup("utf-8")[3] def choose_boundary(): @@ -18,7 +18,7 @@ def choose_boundary(): """ boundary = binascii.hexlify(os.urandom(16)) if six.PY3: - boundary = boundary.decode('ascii') + boundary = boundary.decode("ascii") return boundary @@ -76,7 +76,7 @@ def encode_multipart_formdata(fields, boundary=None): boundary = choose_boundary() for field in iter_field_objects(fields): - body.write(b('--%s\r\n' % (boundary))) + body.write(b("--%s\r\n" % (boundary))) writer(body).write(field.render_headers()) data = field.data @@ -89,10 +89,10 @@ def encode_multipart_formdata(fields, boundary=None): else: body.write(data) - body.write(b'\r\n') + body.write(b"\r\n") - body.write(b('--%s--\r\n' % (boundary))) + body.write(b("--%s--\r\n" % (boundary))) - content_type = str('multipart/form-data; boundary=%s' % boundary) + content_type = str("multipart/form-data; boundary=%s" % boundary) return body.getvalue(), content_type diff --git a/lib/urllib3/packages/__init__.py b/lib/urllib3/packages/__init__.py index 170e974..fce4caa 100644 --- a/lib/urllib3/packages/__init__.py +++ b/lib/urllib3/packages/__init__.py @@ -2,4 +2,4 @@ from __future__ import absolute_import from . import ssl_match_hostname -__all__ = ('ssl_match_hostname', ) +__all__ = ("ssl_match_hostname",) diff --git a/lib/urllib3/packages/backports/makefile.py b/lib/urllib3/packages/backports/makefile.py index 740db37..a3156a6 100644 --- a/lib/urllib3/packages/backports/makefile.py +++ b/lib/urllib3/packages/backports/makefile.py @@ -11,15 +11,14 @@ import io from socket import SocketIO -def backport_makefile(self, mode="r", buffering=None, encoding=None, - errors=None, newline=None): +def backport_makefile( + self, mode="r", buffering=None, encoding=None, errors=None, newline=None +): """ Backport of ``socket.makefile`` from Python 3.5. """ if not set(mode) <= {"r", "w", "b"}: - raise ValueError( - "invalid mode %r (only r, w, b allowed)" % (mode,) - ) + raise ValueError("invalid mode %r (only r, w, b allowed)" % (mode,)) writing = "w" in mode reading = "r" in mode or not writing assert reading or writing diff --git a/lib/urllib3/packages/rfc3986/__init__.py b/lib/urllib3/packages/rfc3986/__init__.py index 9d3c3bc..d953d2b 100644 --- a/lib/urllib3/packages/rfc3986/__init__.py +++ b/lib/urllib3/packages/rfc3986/__init__.py @@ -31,26 +31,26 @@ from .api import URIReference from .api import urlparse from .parseresult import ParseResult -__title__ = 'rfc3986' -__author__ = 'Ian Stapleton Cordasco' -__author_email__ = 'graffatcolmingov@gmail.com' -__license__ = 'Apache v2.0' -__copyright__ = 'Copyright 2014 Rackspace' -__version__ = '1.3.1' +__title__ = "rfc3986" +__author__ = "Ian Stapleton Cordasco" +__author_email__ = "graffatcolmingov@gmail.com" +__license__ = "Apache v2.0" +__copyright__ = "Copyright 2014 Rackspace" +__version__ = "1.3.2" __all__ = ( - 'ParseResult', - 'URIReference', - 'IRIReference', - 'is_valid_uri', - 'normalize_uri', - 'uri_reference', - 'iri_reference', - 'urlparse', - '__title__', - '__author__', - '__author_email__', - '__license__', - '__copyright__', - '__version__', + "ParseResult", + "URIReference", + "IRIReference", + "is_valid_uri", + "normalize_uri", + "uri_reference", + "iri_reference", + "urlparse", + "__title__", + "__author__", + "__author_email__", + "__license__", + "__copyright__", + "__version__", ) diff --git a/lib/urllib3/packages/rfc3986/_mixin.py b/lib/urllib3/packages/rfc3986/_mixin.py index 543925c..4ddcb2d 100644 --- a/lib/urllib3/packages/rfc3986/_mixin.py +++ b/lib/urllib3/packages/rfc3986/_mixin.py @@ -26,7 +26,7 @@ class URIMixin(object): If the authority is not ``None`` and can not be parsed. """ if not self.authority: - return {'userinfo': None, 'host': None, 'port': None} + return {"userinfo": None, "host": None, "port": None} match = self._match_subauthority() @@ -40,10 +40,13 @@ class URIMixin(object): # We had a match, now let's ensure that it is actually a valid host # address if it is IPv4 matches = match.groupdict() - host = matches.get('host') + host = matches.get("host") - if (host and misc.IPv4_MATCHER.match(host) and not - validators.valid_ipv4_host_address(host)): + if ( + host + and misc.IPv4_MATCHER.match(host) + and not validators.valid_ipv4_host_address(host) + ): # If we have a host, it appears to be IPv4 and it does not have # valid bytes, it is an InvalidAuthority. raise exc.InvalidAuthority(self.authority.encode(self.encoding)) @@ -60,7 +63,7 @@ class URIMixin(object): authority = self.authority_info() except exc.InvalidAuthority: return None - return authority['host'] + return authority["host"] @property def port(self): @@ -69,7 +72,7 @@ class URIMixin(object): authority = self.authority_info() except exc.InvalidAuthority: return None - return authority['port'] + return authority["port"] @property def userinfo(self): @@ -78,7 +81,7 @@ class URIMixin(object): authority = self.authority_info() except exc.InvalidAuthority: return None - return authority['userinfo'] + return authority["userinfo"] def is_absolute(self): """Determine if this URI Reference is an absolute URI. @@ -110,16 +113,18 @@ class URIMixin(object): :returns: ``True`` if the URI is valid. ``False`` otherwise. :rtype: bool """ - warnings.warn("Please use rfc3986.validators.Validator instead. " - "This method will be eventually removed.", - DeprecationWarning) + warnings.warn( + "Please use rfc3986.validators.Validator instead. " + "This method will be eventually removed.", + DeprecationWarning, + ) validators = [ - (self.scheme_is_valid, kwargs.get('require_scheme', False)), - (self.authority_is_valid, kwargs.get('require_authority', False)), - (self.path_is_valid, kwargs.get('require_path', False)), - (self.query_is_valid, kwargs.get('require_query', False)), - (self.fragment_is_valid, kwargs.get('require_fragment', False)), - ] + (self.scheme_is_valid, kwargs.get("require_scheme", False)), + (self.authority_is_valid, kwargs.get("require_authority", False)), + (self.path_is_valid, kwargs.get("require_path", False)), + (self.query_is_valid, kwargs.get("require_query", False)), + (self.fragment_is_valid, kwargs.get("require_fragment", False)), + ] return all(v(r) for v, r in validators) def authority_is_valid(self, require=False): @@ -136,18 +141,18 @@ class URIMixin(object): :rtype: bool """ - warnings.warn("Please use rfc3986.validators.Validator instead. " - "This method will be eventually removed.", - DeprecationWarning) + warnings.warn( + "Please use rfc3986.validators.Validator instead. " + "This method will be eventually removed.", + DeprecationWarning, + ) try: self.authority_info() except exc.InvalidAuthority: return False return validators.authority_is_valid( - self.authority, - host=self.host, - require=require, + self.authority, host=self.host, require=require ) def scheme_is_valid(self, require=False): @@ -162,9 +167,11 @@ class URIMixin(object): :returns: ``True`` if the scheme is valid. ``False`` otherwise. :rtype: bool """ - warnings.warn("Please use rfc3986.validators.Validator instead. " - "This method will be eventually removed.", - DeprecationWarning) + warnings.warn( + "Please use rfc3986.validators.Validator instead. " + "This method will be eventually removed.", + DeprecationWarning, + ) return validators.scheme_is_valid(self.scheme, require) def path_is_valid(self, require=False): @@ -179,9 +186,11 @@ class URIMixin(object): :returns: ``True`` if the path is valid. ``False`` otherwise. :rtype: bool """ - warnings.warn("Please use rfc3986.validators.Validator instead. " - "This method will be eventually removed.", - DeprecationWarning) + warnings.warn( + "Please use rfc3986.validators.Validator instead. " + "This method will be eventually removed.", + DeprecationWarning, + ) return validators.path_is_valid(self.path, require) def query_is_valid(self, require=False): @@ -196,9 +205,11 @@ class URIMixin(object): :returns: ``True`` if the query is valid. ``False`` otherwise. :rtype: bool """ - warnings.warn("Please use rfc3986.validators.Validator instead. " - "This method will be eventually removed.", - DeprecationWarning) + warnings.warn( + "Please use rfc3986.validators.Validator instead. " + "This method will be eventually removed.", + DeprecationWarning, + ) return validators.query_is_valid(self.query, require) def fragment_is_valid(self, require=False): @@ -213,9 +224,11 @@ class URIMixin(object): :returns: ``True`` if the fragment is valid. ``False`` otherwise. :rtype: bool """ - warnings.warn("Please use rfc3986.validators.Validator instead. " - "This method will be eventually removed.", - DeprecationWarning) + warnings.warn( + "Please use rfc3986.validators.Validator instead. " + "This method will be eventually removed.", + DeprecationWarning, + ) return validators.fragment_is_valid(self.fragment, require) def normalized_equality(self, other_ref): @@ -269,7 +282,7 @@ class URIMixin(object): if resolving.authority is not None: target = resolving.copy_with( scheme=base_uri.scheme, - path=normalizers.normalize_path(resolving.path) + path=normalizers.normalize_path(resolving.path), ) else: if resolving.path is None: @@ -281,10 +294,10 @@ class URIMixin(object): scheme=base_uri.scheme, authority=base_uri.authority, path=base_uri.path, - query=query + query=query, ) else: - if resolving.path.startswith('/'): + if resolving.path.startswith("/"): path = normalizers.normalize_path(resolving.path) else: path = normalizers.normalize_path( @@ -294,7 +307,7 @@ class URIMixin(object): scheme=base_uri.scheme, authority=base_uri.authority, path=path, - query=resolving.query + query=resolving.query, ) return target @@ -307,20 +320,25 @@ class URIMixin(object): # See http://tools.ietf.org/html/rfc3986#section-5.3 result_list = [] if self.scheme: - result_list.extend([self.scheme, ':']) + result_list.extend([self.scheme, ":"]) if self.authority: - result_list.extend(['//', self.authority]) + result_list.extend(["//", self.authority]) if self.path: result_list.append(self.path) if self.query is not None: - result_list.extend(['?', self.query]) + result_list.extend(["?", self.query]) if self.fragment is not None: - result_list.extend(['#', self.fragment]) - return ''.join(result_list) - - def copy_with(self, scheme=misc.UseExisting, authority=misc.UseExisting, - path=misc.UseExisting, query=misc.UseExisting, - fragment=misc.UseExisting): + result_list.extend(["#", self.fragment]) + return "".join(result_list) + + def copy_with( + self, + scheme=misc.UseExisting, + authority=misc.UseExisting, + path=misc.UseExisting, + query=misc.UseExisting, + fragment=misc.UseExisting, + ): """Create a copy of this reference with the new components. :param str scheme: @@ -339,11 +357,11 @@ class URIMixin(object): URIReference """ attributes = { - 'scheme': scheme, - 'authority': authority, - 'path': path, - 'query': query, - 'fragment': fragment, + "scheme": scheme, + "authority": authority, + "path": path, + "query": query, + "fragment": fragment, } for key, value in list(attributes.items()): if value is misc.UseExisting: diff --git a/lib/urllib3/packages/rfc3986/abnf_regexp.py b/lib/urllib3/packages/rfc3986/abnf_regexp.py index 24c9c3d..c461443 100644 --- a/lib/urllib3/packages/rfc3986/abnf_regexp.py +++ b/lib/urllib3/packages/rfc3986/abnf_regexp.py @@ -24,35 +24,35 @@ SUB_DELIMITERS_SET = set(SUB_DELIMITERS) # Escape the '*' for use in regular expressions SUB_DELIMITERS_RE = r"!$&'()\*+,;=" RESERVED_CHARS_SET = GENERIC_DELIMITERS_SET.union(SUB_DELIMITERS_SET) -ALPHA = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz' -DIGIT = '0123456789' +ALPHA = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz" +DIGIT = "0123456789" # https://tools.ietf.org/html/rfc3986#section-2.3 -UNRESERVED = UNRESERVED_CHARS = ALPHA + DIGIT + r'._!-' +UNRESERVED = UNRESERVED_CHARS = ALPHA + DIGIT + r"._!-" UNRESERVED_CHARS_SET = set(UNRESERVED_CHARS) NON_PCT_ENCODED_SET = RESERVED_CHARS_SET.union(UNRESERVED_CHARS_SET) # We need to escape the '-' in this case: -UNRESERVED_RE = r'A-Za-z0-9._~\-' +UNRESERVED_RE = r"A-Za-z0-9._~\-" # Percent encoded character values -PERCENT_ENCODED = PCT_ENCODED = '%[A-Fa-f0-9]{2}' -PCHAR = '([' + UNRESERVED_RE + SUB_DELIMITERS_RE + ':@]|%s)' % PCT_ENCODED +PERCENT_ENCODED = PCT_ENCODED = "%[A-Fa-f0-9]{2}" +PCHAR = "([" + UNRESERVED_RE + SUB_DELIMITERS_RE + ":@]|%s)" % PCT_ENCODED # NOTE(sigmavirus24): We're going to use more strict regular expressions # than appear in Appendix B for scheme. This will prevent over-eager # consuming of items that aren't schemes. -SCHEME_RE = '[a-zA-Z][a-zA-Z0-9+.-]*' -_AUTHORITY_RE = '[^/?#]*' -_PATH_RE = '[^?#]*' -_QUERY_RE = '[^#]*' -_FRAGMENT_RE = '.*' +SCHEME_RE = "[a-zA-Z][a-zA-Z0-9+.-]*" +_AUTHORITY_RE = "[^/?#]*" +_PATH_RE = "[^?#]*" +_QUERY_RE = "[^#]*" +_FRAGMENT_RE = ".*" # Extracted from http://tools.ietf.org/html/rfc3986#appendix-B COMPONENT_PATTERN_DICT = { - 'scheme': SCHEME_RE, - 'authority': _AUTHORITY_RE, - 'path': _PATH_RE, - 'query': _QUERY_RE, - 'fragment': _FRAGMENT_RE, + "scheme": SCHEME_RE, + "authority": _AUTHORITY_RE, + "path": _PATH_RE, + "query": _QUERY_RE, + "fragment": _FRAGMENT_RE, } # See http://tools.ietf.org/html/rfc3986#appendix-B @@ -61,9 +61,9 @@ COMPONENT_PATTERN_DICT = { # modified to ignore other matches that are not important to the parsing of # the reference so we can also simply use SRE_Match#groups. URL_PARSING_RE = ( - r'(?:(?P{scheme}):)?(?://(?P{authority}))?' - r'(?P{path})(?:\?(?P{query}))?' - r'(?:#(?P{fragment}))?' + r"(?:(?P{scheme}):)?(?://(?P{authority}))?" + r"(?P{path})(?:\?(?P{query}))?" + r"(?:#(?P{fragment}))?" ).format(**COMPONENT_PATTERN_DICT) @@ -73,71 +73,58 @@ URL_PARSING_RE = ( # Host patterns, see: http://tools.ietf.org/html/rfc3986#section-3.2.2 # The pattern for a regular name, e.g., www.google.com, api.github.com -REGULAR_NAME_RE = REG_NAME = '((?:{0}|[{1}])*)'.format( - '%[0-9A-Fa-f]{2}', SUB_DELIMITERS_RE + UNRESERVED_RE +REGULAR_NAME_RE = REG_NAME = "((?:{0}|[{1}])*)".format( + "%[0-9A-Fa-f]{2}", SUB_DELIMITERS_RE + UNRESERVED_RE ) # The pattern for an IPv4 address, e.g., 192.168.255.255, 127.0.0.1, -IPv4_RE = r'([0-9]{1,3}\.){3}[0-9]{1,3}' +IPv4_RE = r"([0-9]{1,3}\.){3}[0-9]{1,3}" # Hexadecimal characters used in each piece of an IPv6 address -HEXDIG_RE = '[0-9A-Fa-f]{1,4}' +HEXDIG_RE = "[0-9A-Fa-f]{1,4}" # Least-significant 32 bits of an IPv6 address -LS32_RE = '({hex}:{hex}|{ipv4})'.format(hex=HEXDIG_RE, ipv4=IPv4_RE) +LS32_RE = "({hex}:{hex}|{ipv4})".format(hex=HEXDIG_RE, ipv4=IPv4_RE) # Substitutions into the following patterns for IPv6 patterns defined # http://tools.ietf.org/html/rfc3986#page-20 -_subs = {'hex': HEXDIG_RE, 'ls32': LS32_RE} +_subs = {"hex": HEXDIG_RE, "ls32": LS32_RE} # Below: h16 = hexdig, see: https://tools.ietf.org/html/rfc5234 for details # about ABNF (Augmented Backus-Naur Form) use in the comments variations = [ # 6( h16 ":" ) ls32 - '(%(hex)s:){6}%(ls32)s' % _subs, + "(%(hex)s:){6}%(ls32)s" % _subs, # "::" 5( h16 ":" ) ls32 - '::(%(hex)s:){5}%(ls32)s' % _subs, + "::(%(hex)s:){5}%(ls32)s" % _subs, # [ h16 ] "::" 4( h16 ":" ) ls32 - '(%(hex)s)?::(%(hex)s:){4}%(ls32)s' % _subs, + "(%(hex)s)?::(%(hex)s:){4}%(ls32)s" % _subs, # [ *1( h16 ":" ) h16 ] "::" 3( h16 ":" ) ls32 - '((%(hex)s:)?%(hex)s)?::(%(hex)s:){3}%(ls32)s' % _subs, + "((%(hex)s:)?%(hex)s)?::(%(hex)s:){3}%(ls32)s" % _subs, # [ *2( h16 ":" ) h16 ] "::" 2( h16 ":" ) ls32 - '((%(hex)s:){0,2}%(hex)s)?::(%(hex)s:){2}%(ls32)s' % _subs, + "((%(hex)s:){0,2}%(hex)s)?::(%(hex)s:){2}%(ls32)s" % _subs, # [ *3( h16 ":" ) h16 ] "::" h16 ":" ls32 - '((%(hex)s:){0,3}%(hex)s)?::%(hex)s:%(ls32)s' % _subs, + "((%(hex)s:){0,3}%(hex)s)?::%(hex)s:%(ls32)s" % _subs, # [ *4( h16 ":" ) h16 ] "::" ls32 - '((%(hex)s:){0,4}%(hex)s)?::%(ls32)s' % _subs, + "((%(hex)s:){0,4}%(hex)s)?::%(ls32)s" % _subs, # [ *5( h16 ":" ) h16 ] "::" h16 - '((%(hex)s:){0,5}%(hex)s)?::%(hex)s' % _subs, + "((%(hex)s:){0,5}%(hex)s)?::%(hex)s" % _subs, # [ *6( h16 ":" ) h16 ] "::" - '((%(hex)s:){0,6}%(hex)s)?::' % _subs, + "((%(hex)s:){0,6}%(hex)s)?::" % _subs, ] -IPv6_RE = '(({0})|({1})|({2})|({3})|({4})|({5})|({6})|({7})|({8}))'.format( - *variations -) +IPv6_RE = "(({0})|({1})|({2})|({3})|({4})|({5})|({6})|({7})|({8}))".format(*variations) -IPv_FUTURE_RE = r'v[0-9A-Fa-f]+\.[%s]+' % ( - UNRESERVED_RE + SUB_DELIMITERS_RE + ':' -) +IPv_FUTURE_RE = r"v[0-9A-Fa-f]+\.[%s]+" % (UNRESERVED_RE + SUB_DELIMITERS_RE + ":") # RFC 6874 Zone ID ABNF -ZONE_ID = '(?:[' + UNRESERVED_RE + ']|' + PCT_ENCODED + ')+' +ZONE_ID = "(?:[" + UNRESERVED_RE + "]|" + PCT_ENCODED + ")+" -IPv6_ADDRZ_RFC4007_RE = IPv6_RE + '(?:(?:%25|%)' + ZONE_ID + ')?' -IPv6_ADDRZ_RE = IPv6_RE + '(?:%25' + ZONE_ID + ')?' +IPv6_ADDRZ_RFC4007_RE = IPv6_RE + "(?:(?:%25|%)" + ZONE_ID + ")?" +IPv6_ADDRZ_RE = IPv6_RE + "(?:%25" + ZONE_ID + ")?" -IP_LITERAL_RE = r'\[({0}|{1})\]'.format( - IPv6_ADDRZ_RFC4007_RE, - IPv_FUTURE_RE, -) +IP_LITERAL_RE = r"\[({0}|{1})\]".format(IPv6_ADDRZ_RFC4007_RE, IPv_FUTURE_RE) # Pattern for matching the host piece of the authority -HOST_RE = HOST_PATTERN = '({0}|{1}|{2})'.format( - REG_NAME, - IPv4_RE, - IP_LITERAL_RE, -) -USERINFO_RE = '^([' + UNRESERVED_RE + SUB_DELIMITERS_RE + ':]|%s)+' % ( - PCT_ENCODED -) -PORT_RE = '[0-9]{1,5}' +HOST_RE = HOST_PATTERN = "({0}|{1}|{2})".format(REG_NAME, IPv4_RE, IP_LITERAL_RE) +USERINFO_RE = "^([" + UNRESERVED_RE + SUB_DELIMITERS_RE + ":]|%s)+" % (PCT_ENCODED) +PORT_RE = "[0-9]{1,5}" # #################### # Path Matcher Section @@ -146,25 +133,29 @@ PORT_RE = '[0-9]{1,5}' # See http://tools.ietf.org/html/rfc3986#section-3.3 for more information # about the path patterns defined below. segments = { - 'segment': PCHAR + '*', + "segment": PCHAR + "*", # Non-zero length segment - 'segment-nz': PCHAR + '+', + "segment-nz": PCHAR + "+", # Non-zero length segment without ":" - 'segment-nz-nc': PCHAR.replace(':', '') + '+' + "segment-nz-nc": PCHAR.replace(":", "") + "+", } # Path types taken from Section 3.3 (linked above) -PATH_EMPTY = '^$' -PATH_ROOTLESS = '%(segment-nz)s(/%(segment)s)*' % segments -PATH_NOSCHEME = '%(segment-nz-nc)s(/%(segment)s)*' % segments -PATH_ABSOLUTE = '/(%s)?' % PATH_ROOTLESS -PATH_ABEMPTY = '(/%(segment)s)*' % segments -PATH_RE = '^(%s|%s|%s|%s|%s)$' % ( - PATH_ABEMPTY, PATH_ABSOLUTE, PATH_NOSCHEME, PATH_ROOTLESS, PATH_EMPTY +PATH_EMPTY = "^$" +PATH_ROOTLESS = "%(segment-nz)s(/%(segment)s)*" % segments +PATH_NOSCHEME = "%(segment-nz-nc)s(/%(segment)s)*" % segments +PATH_ABSOLUTE = "/(%s)?" % PATH_ROOTLESS +PATH_ABEMPTY = "(/%(segment)s)*" % segments +PATH_RE = "^(%s|%s|%s|%s|%s)$" % ( + PATH_ABEMPTY, + PATH_ABSOLUTE, + PATH_NOSCHEME, + PATH_ROOTLESS, + PATH_EMPTY, ) FRAGMENT_RE = QUERY_RE = ( - '^([/?:@' + UNRESERVED_RE + SUB_DELIMITERS_RE + ']|%s)*$' % PCT_ENCODED + "^([/?:@" + UNRESERVED_RE + SUB_DELIMITERS_RE + "]|%s)*$" % PCT_ENCODED ) # ########################## @@ -172,8 +163,8 @@ FRAGMENT_RE = QUERY_RE = ( # ########################## # See http://tools.ietf.org/html/rfc3986#section-4.2 for details -RELATIVE_PART_RE = '(//%s%s|%s|%s|%s)' % ( - COMPONENT_PATTERN_DICT['authority'], +RELATIVE_PART_RE = "(//%s%s|%s|%s|%s)" % ( + COMPONENT_PATTERN_DICT["authority"], PATH_ABEMPTY, PATH_ABSOLUTE, PATH_NOSCHEME, @@ -181,8 +172,8 @@ RELATIVE_PART_RE = '(//%s%s|%s|%s|%s)' % ( ) # See http://tools.ietf.org/html/rfc3986#section-3 for definition -HIER_PART_RE = '(//%s%s|%s|%s|%s)' % ( - COMPONENT_PATTERN_DICT['authority'], +HIER_PART_RE = "(//%s%s|%s|%s|%s)" % ( + COMPONENT_PATTERN_DICT["authority"], PATH_ABEMPTY, PATH_ABSOLUTE, PATH_ROOTLESS, @@ -195,71 +186,75 @@ HIER_PART_RE = '(//%s%s|%s|%s|%s)' % ( # Only wide-unicode gets the high-ranges of UCSCHAR if sys.maxunicode > 0xFFFF: # pragma: no cover - IPRIVATE = u'\uE000-\uF8FF\U000F0000-\U000FFFFD\U00100000-\U0010FFFD' + IPRIVATE = u"\uE000-\uF8FF\U000F0000-\U000FFFFD\U00100000-\U0010FFFD" UCSCHAR_RE = ( - u'\u00A0-\uD7FF\uF900-\uFDCF\uFDF0-\uFFEF' - u'\U00010000-\U0001FFFD\U00020000-\U0002FFFD' - u'\U00030000-\U0003FFFD\U00040000-\U0004FFFD' - u'\U00050000-\U0005FFFD\U00060000-\U0006FFFD' - u'\U00070000-\U0007FFFD\U00080000-\U0008FFFD' - u'\U00090000-\U0009FFFD\U000A0000-\U000AFFFD' - u'\U000B0000-\U000BFFFD\U000C0000-\U000CFFFD' - u'\U000D0000-\U000DFFFD\U000E1000-\U000EFFFD' + u"\u00A0-\uD7FF\uF900-\uFDCF\uFDF0-\uFFEF" + u"\U00010000-\U0001FFFD\U00020000-\U0002FFFD" + u"\U00030000-\U0003FFFD\U00040000-\U0004FFFD" + u"\U00050000-\U0005FFFD\U00060000-\U0006FFFD" + u"\U00070000-\U0007FFFD\U00080000-\U0008FFFD" + u"\U00090000-\U0009FFFD\U000A0000-\U000AFFFD" + u"\U000B0000-\U000BFFFD\U000C0000-\U000CFFFD" + u"\U000D0000-\U000DFFFD\U000E1000-\U000EFFFD" ) else: # pragma: no cover - IPRIVATE = u'\uE000-\uF8FF' - UCSCHAR_RE = ( - u'\u00A0-\uD7FF\uF900-\uFDCF\uFDF0-\uFFEF' - ) + IPRIVATE = u"\uE000-\uF8FF" + UCSCHAR_RE = u"\u00A0-\uD7FF\uF900-\uFDCF\uFDF0-\uFFEF" -IUNRESERVED_RE = u'A-Za-z0-9\\._~\\-' + UCSCHAR_RE -IPCHAR = u'([' + IUNRESERVED_RE + SUB_DELIMITERS_RE + u':@]|%s)' % PCT_ENCODED +IUNRESERVED_RE = u"A-Za-z0-9\\._~\\-" + UCSCHAR_RE +IPCHAR = u"([" + IUNRESERVED_RE + SUB_DELIMITERS_RE + u":@]|%s)" % PCT_ENCODED isegments = { - 'isegment': IPCHAR + u'*', + "isegment": IPCHAR + u"*", # Non-zero length segment - 'isegment-nz': IPCHAR + u'+', + "isegment-nz": IPCHAR + u"+", # Non-zero length segment without ":" - 'isegment-nz-nc': IPCHAR.replace(':', '') + u'+' + "isegment-nz-nc": IPCHAR.replace(":", "") + u"+", } -IPATH_ROOTLESS = u'%(isegment-nz)s(/%(isegment)s)*' % isegments -IPATH_NOSCHEME = u'%(isegment-nz-nc)s(/%(isegment)s)*' % isegments -IPATH_ABSOLUTE = u'/(?:%s)?' % IPATH_ROOTLESS -IPATH_ABEMPTY = u'(?:/%(isegment)s)*' % isegments -IPATH_RE = u'^(?:%s|%s|%s|%s|%s)$' % ( - IPATH_ABEMPTY, IPATH_ABSOLUTE, IPATH_NOSCHEME, IPATH_ROOTLESS, PATH_EMPTY +IPATH_ROOTLESS = u"%(isegment-nz)s(/%(isegment)s)*" % isegments +IPATH_NOSCHEME = u"%(isegment-nz-nc)s(/%(isegment)s)*" % isegments +IPATH_ABSOLUTE = u"/(?:%s)?" % IPATH_ROOTLESS +IPATH_ABEMPTY = u"(?:/%(isegment)s)*" % isegments +IPATH_RE = u"^(?:%s|%s|%s|%s|%s)$" % ( + IPATH_ABEMPTY, + IPATH_ABSOLUTE, + IPATH_NOSCHEME, + IPATH_ROOTLESS, + PATH_EMPTY, ) -IREGULAR_NAME_RE = IREG_NAME = u'(?:{0}|[{1}])*'.format( - u'%[0-9A-Fa-f]{2}', SUB_DELIMITERS_RE + IUNRESERVED_RE +IREGULAR_NAME_RE = IREG_NAME = u"(?:{0}|[{1}])*".format( + u"%[0-9A-Fa-f]{2}", SUB_DELIMITERS_RE + IUNRESERVED_RE ) -IHOST_RE = IHOST_PATTERN = u'({0}|{1}|{2})'.format( - IREG_NAME, - IPv4_RE, - IP_LITERAL_RE, -) +IHOST_RE = IHOST_PATTERN = u"({0}|{1}|{2})".format(IREG_NAME, IPv4_RE, IP_LITERAL_RE) -IUSERINFO_RE = u'^(?:[' + IUNRESERVED_RE + SUB_DELIMITERS_RE + u':]|%s)+' % ( - PCT_ENCODED +IUSERINFO_RE = ( + u"^(?:[" + IUNRESERVED_RE + SUB_DELIMITERS_RE + u":]|%s)+" % (PCT_ENCODED) ) -IFRAGMENT_RE = (u'^(?:[/?:@' + IUNRESERVED_RE + SUB_DELIMITERS_RE - + u']|%s)*$' % PCT_ENCODED) -IQUERY_RE = (u'^(?:[/?:@' + IUNRESERVED_RE + SUB_DELIMITERS_RE - + IPRIVATE + u']|%s)*$' % PCT_ENCODED) +IFRAGMENT_RE = ( + u"^(?:[/?:@" + IUNRESERVED_RE + SUB_DELIMITERS_RE + u"]|%s)*$" % PCT_ENCODED +) +IQUERY_RE = ( + u"^(?:[/?:@" + + IUNRESERVED_RE + + SUB_DELIMITERS_RE + + IPRIVATE + + u"]|%s)*$" % PCT_ENCODED +) -IRELATIVE_PART_RE = u'(//%s%s|%s|%s|%s)' % ( - COMPONENT_PATTERN_DICT['authority'], +IRELATIVE_PART_RE = u"(//%s%s|%s|%s|%s)" % ( + COMPONENT_PATTERN_DICT["authority"], IPATH_ABEMPTY, IPATH_ABSOLUTE, IPATH_NOSCHEME, PATH_EMPTY, ) -IHIER_PART_RE = u'(//%s%s|%s|%s|%s)' % ( - COMPONENT_PATTERN_DICT['authority'], +IHIER_PART_RE = u"(//%s%s|%s|%s|%s)" % ( + COMPONENT_PATTERN_DICT["authority"], IPATH_ABEMPTY, IPATH_ABSOLUTE, IPATH_ROOTLESS, diff --git a/lib/urllib3/packages/rfc3986/api.py b/lib/urllib3/packages/rfc3986/api.py index ddc4a1c..1e098b3 100644 --- a/lib/urllib3/packages/rfc3986/api.py +++ b/lib/urllib3/packages/rfc3986/api.py @@ -24,7 +24,7 @@ from .parseresult import ParseResult from .uri import URIReference -def uri_reference(uri, encoding='utf-8'): +def uri_reference(uri, encoding="utf-8"): """Parse a URI string into a URIReference. This is a convenience function. You could achieve the same end by using @@ -38,7 +38,7 @@ def uri_reference(uri, encoding='utf-8'): return URIReference.from_string(uri, encoding) -def iri_reference(iri, encoding='utf-8'): +def iri_reference(iri, encoding="utf-8"): """Parse a IRI string into an IRIReference. This is a convenience function. You could achieve the same end by using @@ -52,7 +52,7 @@ def iri_reference(iri, encoding='utf-8'): return IRIReference.from_string(iri, encoding) -def is_valid_uri(uri, encoding='utf-8', **kwargs): +def is_valid_uri(uri, encoding="utf-8", **kwargs): """Determine if the URI given is valid. This is a convenience function. You could use either @@ -77,7 +77,7 @@ def is_valid_uri(uri, encoding='utf-8', **kwargs): return URIReference.from_string(uri, encoding).is_valid(**kwargs) -def normalize_uri(uri, encoding='utf-8'): +def normalize_uri(uri, encoding="utf-8"): """Normalize the given URI. This is a convenience function. You could use either @@ -93,7 +93,7 @@ def normalize_uri(uri, encoding='utf-8'): return normalized_reference.unsplit() -def urlparse(uri, encoding='utf-8'): +def urlparse(uri, encoding="utf-8"): """Parse a given URI and return a ParseResult. This is a partial replacement of the standard library's urlparse function. diff --git a/lib/urllib3/packages/rfc3986/builder.py b/lib/urllib3/packages/rfc3986/builder.py index 7934279..bbabfaf 100644 --- a/lib/urllib3/packages/rfc3986/builder.py +++ b/lib/urllib3/packages/rfc3986/builder.py @@ -29,8 +29,16 @@ class URIBuilder(object): """ - def __init__(self, scheme=None, userinfo=None, host=None, port=None, - path=None, query=None, fragment=None): + def __init__( + self, + scheme=None, + userinfo=None, + host=None, + port=None, + path=None, + query=None, + fragment=None, + ): """Initialize our URI builder. :param str scheme: @@ -58,9 +66,11 @@ class URIBuilder(object): def __repr__(self): """Provide a convenient view of our builder object.""" - formatstr = ('URIBuilder(scheme={b.scheme}, userinfo={b.userinfo}, ' - 'host={b.host}, port={b.port}, path={b.path}, ' - 'query={b.query}, fragment={b.fragment})') + formatstr = ( + "URIBuilder(scheme={b.scheme}, userinfo={b.userinfo}, " + "host={b.host}, port={b.port}, path={b.path}, " + "query={b.query}, fragment={b.fragment})" + ) return formatstr.format(b=self) def add_scheme(self, scheme): @@ -101,13 +111,12 @@ class URIBuilder(object): port=None, path=None, query=None, fragment=None) """ if username is None: - raise ValueError('Username cannot be None') + raise ValueError("Username cannot be None") userinfo = normalizers.normalize_username(username) if password is not None: - userinfo = '{}:{}'.format( - userinfo, - normalizers.normalize_password(password), + userinfo = "{}:{}".format( + userinfo, normalizers.normalize_password(password) ) return URIBuilder( @@ -157,23 +166,19 @@ class URIBuilder(object): port_int = int(port) if port_int < 0: raise ValueError( - 'ports are not allowed to be negative. You provided {}'.format( - port_int, - ) + "ports are not allowed to be negative. You provided {}".format(port_int) ) if port_int > 65535: raise ValueError( - 'ports are not allowed to be larger than 65535. ' - 'You provided {}'.format( - port_int, - ) + "ports are not allowed to be larger than 65535. " + "You provided {}".format(port_int) ) return URIBuilder( scheme=self.scheme, userinfo=self.userinfo, host=self.host, - port='{}'.format(port_int), + port="{}".format(port_int), path=self.path, query=self.query, fragment=self.fragment, @@ -193,8 +198,8 @@ class URIBuilder(object): path='/checkout.php', query=None, fragment=None) """ - if not path.startswith('/'): - path = '/{}'.format(path) + if not path.startswith("/"): + path = "/{}".format(path) return URIBuilder( scheme=self.scheme, @@ -289,9 +294,7 @@ class URIBuilder(object): """ return uri.URIReference( self.scheme, - normalizers.normalize_authority( - (self.userinfo, self.host, self.port) - ), + normalizers.normalize_authority((self.userinfo, self.host, self.port)), self.path, self.query, self.fragment, diff --git a/lib/urllib3/packages/rfc3986/compat.py b/lib/urllib3/packages/rfc3986/compat.py index 8968c38..36e490a 100644 --- a/lib/urllib3/packages/rfc3986/compat.py +++ b/lib/urllib3/packages/rfc3986/compat.py @@ -25,12 +25,7 @@ try: except ImportError: # Python 2.x from urllib import urlencode -__all__ = ( - 'to_bytes', - 'to_str', - 'urlquote', - 'urlencode', -) +__all__ = ("to_bytes", "to_str", "urlquote", "urlencode") PY3 = (3, 0) <= sys.version_info < (4, 0) PY2 = (2, 6) <= sys.version_info < (2, 8) @@ -40,15 +35,15 @@ if PY3: unicode = str # Python 3.x -def to_str(b, encoding='utf-8'): +def to_str(b, encoding="utf-8"): """Ensure that b is text in the specified encoding.""" - if hasattr(b, 'decode') and not isinstance(b, unicode): + if hasattr(b, "decode") and not isinstance(b, unicode): b = b.decode(encoding) return b -def to_bytes(s, encoding='utf-8'): +def to_bytes(s, encoding="utf-8"): """Ensure that s is converted to bytes from the encoding.""" - if hasattr(s, 'encode') and not isinstance(s, bytes): + if hasattr(s, "encode") and not isinstance(s, bytes): s = s.encode(encoding) return s diff --git a/lib/urllib3/packages/rfc3986/exceptions.py b/lib/urllib3/packages/rfc3986/exceptions.py index da8ca7c..865f9bd 100644 --- a/lib/urllib3/packages/rfc3986/exceptions.py +++ b/lib/urllib3/packages/rfc3986/exceptions.py @@ -16,8 +16,8 @@ class InvalidAuthority(RFC3986Exception): def __init__(self, authority): """Initialize the exception with the invalid authority.""" super(InvalidAuthority, self).__init__( - u"The authority ({0}) is not valid.".format( - compat.to_str(authority))) + u"The authority ({0}) is not valid.".format(compat.to_str(authority)) + ) class InvalidPort(RFC3986Exception): @@ -25,8 +25,7 @@ class InvalidPort(RFC3986Exception): def __init__(self, port): """Initialize the exception with the invalid port.""" - super(InvalidPort, self).__init__( - 'The port ("{0}") is not valid.'.format(port)) + super(InvalidPort, self).__init__('The port ("{0}") is not valid.'.format(port)) class ResolutionError(RFC3986Exception): @@ -35,7 +34,8 @@ class ResolutionError(RFC3986Exception): def __init__(self, uri): """Initialize the error with the failed URI.""" super(ResolutionError, self).__init__( - "{0} is not an absolute URI.".format(uri.unsplit())) + "{0} is not an absolute URI.".format(uri.unsplit()) + ) class ValidationError(RFC3986Exception): @@ -49,17 +49,15 @@ class MissingComponentError(ValidationError): def __init__(self, uri, *component_names): """Initialize the error with the missing component name.""" - verb = 'was' + verb = "was" if len(component_names) > 1: - verb = 'were' + verb = "were" self.uri = uri self.components = sorted(component_names) - components = ', '.join(self.components) + components = ", ".join(self.components) super(MissingComponentError, self).__init__( - "{} {} required but missing".format(components, verb), - uri, - self.components, + "{} {} required but missing".format(components, verb), uri, self.components ) @@ -70,7 +68,7 @@ class UnpermittedComponentError(ValidationError): """Initialize the error with the unpermitted component.""" super(UnpermittedComponentError, self).__init__( "{} was required to be one of {!r} but was {!r}".format( - component_name, list(sorted(allowed_values)), component_value, + component_name, list(sorted(allowed_values)), component_value ), component_name, component_value, @@ -86,11 +84,9 @@ class PasswordForbidden(ValidationError): def __init__(self, uri): """Initialize the error with the URI that failed validation.""" - unsplit = getattr(uri, 'unsplit', lambda: uri) + unsplit = getattr(uri, "unsplit", lambda: uri) super(PasswordForbidden, self).__init__( - '"{}" contained a password when validation forbade it'.format( - unsplit() - ) + '"{}" contained a password when validation forbade it'.format(unsplit()) ) self.uri = uri @@ -100,17 +96,15 @@ class InvalidComponentsError(ValidationError): def __init__(self, uri, *component_names): """Initialize the error with the invalid component name(s).""" - verb = 'was' + verb = "was" if len(component_names) > 1: - verb = 'were' + verb = "were" self.uri = uri self.components = sorted(component_names) - components = ', '.join(self.components) + components = ", ".join(self.components) super(InvalidComponentsError, self).__init__( - "{} {} found to be invalid".format(components, verb), - uri, - self.components, + "{} {} found to be invalid".format(components, verb), uri, self.components ) diff --git a/lib/urllib3/packages/rfc3986/iri.py b/lib/urllib3/packages/rfc3986/iri.py index 9c01fe1..a15e838 100644 --- a/lib/urllib3/packages/rfc3986/iri.py +++ b/lib/urllib3/packages/rfc3986/iri.py @@ -29,8 +29,7 @@ except ImportError: # pragma: no cover idna = None -class IRIReference(namedtuple('IRIReference', misc.URI_COMPONENTS), - uri.URIMixin): +class IRIReference(namedtuple("IRIReference", misc.URI_COMPONENTS), uri.URIMixin): """Immutable object representing a parsed IRI Reference. Can be encoded into an URIReference object via the procedure @@ -43,16 +42,11 @@ class IRIReference(namedtuple('IRIReference', misc.URI_COMPONENTS), slots = () - def __new__(cls, scheme, authority, path, query, fragment, - encoding='utf-8'): + def __new__(cls, scheme, authority, path, query, fragment, encoding="utf-8"): """Create a new IRIReference.""" ref = super(IRIReference, cls).__new__( - cls, - scheme or None, - authority or None, - path or None, - query, - fragment) + cls, scheme or None, authority or None, path or None, query, fragment + ) ref.encoding = encoding return ref @@ -66,8 +60,10 @@ class IRIReference(namedtuple('IRIReference', misc.URI_COMPONENTS), other_ref = self.__class__.from_string(other) except TypeError: raise TypeError( - 'Unable to compare {0}() to {1}()'.format( - type(self).__name__, type(other).__name__)) + "Unable to compare {0}() to {1}()".format( + type(self).__name__, type(other).__name__ + ) + ) # See http://tools.ietf.org/html/rfc3986#section-6.2 return tuple(self) == tuple(other_ref) @@ -76,7 +72,7 @@ class IRIReference(namedtuple('IRIReference', misc.URI_COMPONENTS), return misc.ISUBAUTHORITY_MATCHER.match(self.authority) @classmethod - def from_string(cls, iri_string, encoding='utf-8'): + def from_string(cls, iri_string, encoding="utf-8"): """Parse a IRI reference from the given unicode IRI string. :param str iri_string: Unicode IRI to be parsed into a reference. @@ -87,10 +83,11 @@ class IRIReference(namedtuple('IRIReference', misc.URI_COMPONENTS), split_iri = misc.IRI_MATCHER.match(iri_string).groupdict() return cls( - split_iri['scheme'], split_iri['authority'], - normalizers.encode_component(split_iri['path'], encoding), - normalizers.encode_component(split_iri['query'], encoding), - normalizers.encode_component(split_iri['fragment'], encoding), + split_iri["scheme"], + split_iri["authority"], + normalizers.encode_component(split_iri["path"], encoding), + normalizers.encode_component(split_iri["query"], encoding), + normalizers.encode_component(split_iri["fragment"], encoding), encoding, ) @@ -120,28 +117,34 @@ class IRIReference(namedtuple('IRIReference', misc.URI_COMPONENTS), def idna_encoder(name): if any(ord(c) > 128 for c in name): try: - return idna.encode(name.lower(), - strict=True, - std3_rules=True) + return idna.encode( + name.lower(), strict=True, std3_rules=True + ) except idna.IDNAError: raise exceptions.InvalidAuthority(self.authority) return name authority = "" if self.host: - authority = ".".join([compat.to_str(idna_encoder(part)) - for part in self.host.split(".")]) + authority = ".".join( + [compat.to_str(idna_encoder(part)) for part in self.host.split(".")] + ) if self.userinfo is not None: - authority = (normalizers.encode_component( - self.userinfo, self.encoding) + '@' + authority) + authority = ( + normalizers.encode_component(self.userinfo, self.encoding) + + "@" + + authority + ) if self.port is not None: authority += ":" + str(self.port) - return uri.URIReference(self.scheme, - authority, - path=self.path, - query=self.query, - fragment=self.fragment, - encoding=self.encoding) + return uri.URIReference( + self.scheme, + authority, + path=self.path, + query=self.query, + fragment=self.fragment, + encoding=self.encoding, + ) diff --git a/lib/urllib3/packages/rfc3986/misc.py b/lib/urllib3/packages/rfc3986/misc.py index 00f9f3b..353a629 100644 --- a/lib/urllib3/packages/rfc3986/misc.py +++ b/lib/urllib3/packages/rfc3986/misc.py @@ -25,16 +25,16 @@ from . import abnf_regexp # These are enumerated for the named tuple used as a superclass of # URIReference -URI_COMPONENTS = ['scheme', 'authority', 'path', 'query', 'fragment'] +URI_COMPONENTS = ["scheme", "authority", "path", "query", "fragment"] important_characters = { - 'generic_delimiters': abnf_regexp.GENERIC_DELIMITERS, - 'sub_delimiters': abnf_regexp.SUB_DELIMITERS, + "generic_delimiters": abnf_regexp.GENERIC_DELIMITERS, + "sub_delimiters": abnf_regexp.SUB_DELIMITERS, # We need to escape the '*' in this case - 're_sub_delimiters': abnf_regexp.SUB_DELIMITERS_RE, - 'unreserved_chars': abnf_regexp.UNRESERVED_CHARS, + "re_sub_delimiters": abnf_regexp.SUB_DELIMITERS_RE, + "unreserved_chars": abnf_regexp.UNRESERVED_CHARS, # We need to escape the '-' in this case: - 're_unreserved': abnf_regexp.UNRESERVED_RE, + "re_unreserved": abnf_regexp.UNRESERVED_RE, } # For details about delimiters and reserved characters, see: @@ -49,23 +49,21 @@ NON_PCT_ENCODED = abnf_regexp.NON_PCT_ENCODED_SET URI_MATCHER = re.compile(abnf_regexp.URL_PARSING_RE) -SUBAUTHORITY_MATCHER = re.compile(( - '^(?:(?P{0})@)?' # userinfo - '(?P{1})' # host - ':?(?P{2})?$' # port - ).format(abnf_regexp.USERINFO_RE, - abnf_regexp.HOST_PATTERN, - abnf_regexp.PORT_RE)) +SUBAUTHORITY_MATCHER = re.compile( + ( + "^(?:(?P{0})@)?" # userinfo + "(?P{1})" # host + ":?(?P{2})?$" # port + ).format(abnf_regexp.USERINFO_RE, abnf_regexp.HOST_PATTERN, abnf_regexp.PORT_RE) +) -HOST_MATCHER = re.compile('^' + abnf_regexp.HOST_RE + '$') -IPv4_MATCHER = re.compile('^' + abnf_regexp.IPv4_RE + '$') -IPv6_MATCHER = re.compile(r'^\[' + abnf_regexp.IPv6_ADDRZ_RFC4007_RE + r'\]$') +HOST_MATCHER = re.compile("^" + abnf_regexp.HOST_RE + "$") +IPv4_MATCHER = re.compile("^" + abnf_regexp.IPv4_RE + "$") +IPv6_MATCHER = re.compile(r"^\[" + abnf_regexp.IPv6_ADDRZ_RFC4007_RE + r"\]$") # Used by host validator -IPv6_NO_RFC4007_MATCHER = re.compile(r'^\[%s\]$' % ( - abnf_regexp.IPv6_ADDRZ_RE -)) +IPv6_NO_RFC4007_MATCHER = re.compile(r"^\[%s\]$" % (abnf_regexp.IPv6_ADDRZ_RE)) # Matcher used to validate path components PATH_MATCHER = re.compile(abnf_regexp.PATH_RE) @@ -80,20 +78,22 @@ QUERY_MATCHER = re.compile(abnf_regexp.QUERY_RE) FRAGMENT_MATCHER = QUERY_MATCHER # Scheme validation, see: http://tools.ietf.org/html/rfc3986#section-3.1 -SCHEME_MATCHER = re.compile('^{0}$'.format(abnf_regexp.SCHEME_RE)) +SCHEME_MATCHER = re.compile("^{0}$".format(abnf_regexp.SCHEME_RE)) -RELATIVE_REF_MATCHER = re.compile(r'^%s(\?%s)?(#%s)?$' % ( - abnf_regexp.RELATIVE_PART_RE, - abnf_regexp.QUERY_RE, - abnf_regexp.FRAGMENT_RE, -)) +RELATIVE_REF_MATCHER = re.compile( + r"^%s(\?%s)?(#%s)?$" + % (abnf_regexp.RELATIVE_PART_RE, abnf_regexp.QUERY_RE, abnf_regexp.FRAGMENT_RE) +) # See http://tools.ietf.org/html/rfc3986#section-4.3 -ABSOLUTE_URI_MATCHER = re.compile(r'^%s:%s(\?%s)?$' % ( - abnf_regexp.COMPONENT_PATTERN_DICT['scheme'], - abnf_regexp.HIER_PART_RE, - abnf_regexp.QUERY_RE[1:-1], -)) +ABSOLUTE_URI_MATCHER = re.compile( + r"^%s:%s(\?%s)?$" + % ( + abnf_regexp.COMPONENT_PATTERN_DICT["scheme"], + abnf_regexp.HIER_PART_RE, + abnf_regexp.QUERY_RE[1:-1], + ) +) # ############### # IRIs / RFC 3987 @@ -101,46 +101,25 @@ ABSOLUTE_URI_MATCHER = re.compile(r'^%s:%s(\?%s)?$' % ( IRI_MATCHER = re.compile(abnf_regexp.URL_PARSING_RE, re.UNICODE) -ISUBAUTHORITY_MATCHER = re.compile(( - u'^(?:(?P{0})@)?' # iuserinfo - u'(?P{1})' # ihost - u':?(?P{2})?$' # port - ).format(abnf_regexp.IUSERINFO_RE, - abnf_regexp.IHOST_RE, - abnf_regexp.PORT_RE), re.UNICODE) - - -IHOST_MATCHER = re.compile('^' + abnf_regexp.IHOST_RE + '$', re.UNICODE) - -IPATH_MATCHER = re.compile(abnf_regexp.IPATH_RE, re.UNICODE) - -IQUERY_MATCHER = re.compile(abnf_regexp.IQUERY_RE, re.UNICODE) - -IFRAGMENT_MATCHER = re.compile(abnf_regexp.IFRAGMENT_RE, re.UNICODE) - - -RELATIVE_IRI_MATCHER = re.compile(u'^%s(?:\\?%s)?(?:%s)?$' % ( - abnf_regexp.IRELATIVE_PART_RE, - abnf_regexp.IQUERY_RE, - abnf_regexp.IFRAGMENT_RE -), re.UNICODE) - -ABSOLUTE_IRI_MATCHER = re.compile(u'^%s:%s(?:\\?%s)?$' % ( - abnf_regexp.COMPONENT_PATTERN_DICT['scheme'], - abnf_regexp.IHIER_PART_RE, - abnf_regexp.IQUERY_RE[1:-1] -), re.UNICODE) +ISUBAUTHORITY_MATCHER = re.compile( + ( + u"^(?:(?P{0})@)?" # iuserinfo + u"(?P{1})" # ihost + u":?(?P{2})?$" # port + ).format(abnf_regexp.IUSERINFO_RE, abnf_regexp.IHOST_RE, abnf_regexp.PORT_RE), + re.UNICODE, +) # Path merger as defined in http://tools.ietf.org/html/rfc3986#section-5.2.3 def merge_paths(base_uri, relative_path): """Merge a base URI's path with a relative URI's path.""" if base_uri.path is None and base_uri.authority is not None: - return '/' + relative_path + return "/" + relative_path else: - path = base_uri.path or '' - index = path.rfind('/') - return path[:index] + '/' + relative_path + path = base_uri.path or "" + index = path.rfind("/") + return path[:index] + "/" + relative_path UseExisting = object() diff --git a/lib/urllib3/packages/rfc3986/normalizers.py b/lib/urllib3/packages/rfc3986/normalizers.py index 2eb1bb3..0d702b6 100644 --- a/lib/urllib3/packages/rfc3986/normalizers.py +++ b/lib/urllib3/packages/rfc3986/normalizers.py @@ -27,13 +27,13 @@ def normalize_scheme(scheme): def normalize_authority(authority): """Normalize an authority tuple to a string.""" userinfo, host, port = authority - result = '' + result = "" if userinfo: - result += normalize_percent_characters(userinfo) + '@' + result += normalize_percent_characters(userinfo) + "@" if host: result += normalize_host(host) if port: - result += ':' + port + result += ":" + port return result @@ -50,16 +50,19 @@ def normalize_password(password): def normalize_host(host): """Normalize a host string.""" if misc.IPv6_MATCHER.match(host): - percent = host.find('%') + percent = host.find("%") if percent != -1: - percent_25 = host.find('%25') + percent_25 = host.find("%25") # Replace RFC 4007 IPv6 Zone ID delimiter '%' with '%25' # from RFC 6874. If the host is '[%25]' then we # assume RFC 4007 and normalize to '[%2525]' - if percent_25 == -1 or percent < percent_25 or \ - (percent == percent_25 and percent_25 == len(host) - 4): - host = host.replace('%', '%25', 1) + if ( + percent_25 == -1 + or percent < percent_25 + or (percent == percent_25 and percent_25 == len(host) - 4) + ): + host = host.replace("%", "%25", 1) # Don't normalize the casing of the Zone ID return host[:percent].lower() + host[percent:] @@ -90,7 +93,7 @@ def normalize_fragment(fragment): return normalize_percent_characters(fragment) -PERCENT_MATCHER = re.compile('%[A-Fa-f0-9]{2}') +PERCENT_MATCHER = re.compile("%[A-Fa-f0-9]{2}") def normalize_percent_characters(s): @@ -111,15 +114,15 @@ def remove_dot_segments(s): See also Section 5.2.4 of :rfc:`3986`. """ # See http://tools.ietf.org/html/rfc3986#section-5.2.4 for pseudo-code - segments = s.split('/') # Turn the path into a list of segments + segments = s.split("/") # Turn the path into a list of segments output = [] # Initialize the variable to use to store output for segment in segments: # '.' is the current directory, so ignore it, it is superfluous - if segment == '.': + if segment == ".": continue # Anything other than '..', should be appended to the output - elif segment != '..': + elif segment != "..": output.append(segment) # In this case segment == '..', if we can, we should pop the last # element @@ -128,15 +131,15 @@ def remove_dot_segments(s): # If the path starts with '/' and the output is empty or the first string # is non-empty - if s.startswith('/') and (not output or output[0]): - output.insert(0, '') + if s.startswith("/") and (not output or output[0]): + output.insert(0, "") # If the path starts with '/.' or '/..' ensure we add one more empty # string to add a trailing '/' - if s.endswith(('/.', '/..')): - output.append('') + if s.endswith(("/.", "/..")): + output.append("") - return '/'.join(output) + return "/".join(output) def encode_component(uri_component, encoding): @@ -146,22 +149,24 @@ def encode_component(uri_component, encoding): # Try to see if the component we're encoding is already percent-encoded # so we can skip all '%' characters but still encode all others. - percent_encodings = len(PERCENT_MATCHER.findall( - compat.to_str(uri_component, encoding))) + percent_encodings = len( + PERCENT_MATCHER.findall(compat.to_str(uri_component, encoding)) + ) uri_bytes = compat.to_bytes(uri_component, encoding) - is_percent_encoded = percent_encodings == uri_bytes.count(b'%') + is_percent_encoded = percent_encodings == uri_bytes.count(b"%") encoded_uri = bytearray() for i in range(0, len(uri_bytes)): # Will return a single character bytestring on both Python 2 & 3 - byte = uri_bytes[i:i+1] + byte = uri_bytes[i : i + 1] byte_ord = ord(byte) - if ((is_percent_encoded and byte == b'%') - or (byte_ord < 128 and byte.decode() in misc.NON_PCT_ENCODED)): + if (is_percent_encoded and byte == b"%") or ( + byte_ord < 128 and byte.decode() in misc.NON_PCT_ENCODED + ): encoded_uri.extend(byte) continue - encoded_uri.extend('%{0:02x}'.format(byte_ord).encode().upper()) + encoded_uri.extend("%{0:02x}".format(byte_ord).encode().upper()) return encoded_uri.decode(encoding) diff --git a/lib/urllib3/packages/rfc3986/parseresult.py b/lib/urllib3/packages/rfc3986/parseresult.py index 0a73456..74d12c2 100644 --- a/lib/urllib3/packages/rfc3986/parseresult.py +++ b/lib/urllib3/packages/rfc3986/parseresult.py @@ -21,27 +21,25 @@ from . import misc from . import normalizers from . import uri -__all__ = ('ParseResult', 'ParseResultBytes') +__all__ = ("ParseResult", "ParseResultBytes") -PARSED_COMPONENTS = ('scheme', 'userinfo', 'host', 'port', 'path', 'query', - 'fragment') +PARSED_COMPONENTS = ("scheme", "userinfo", "host", "port", "path", "query", "fragment") class ParseResultMixin(object): def _generate_authority(self, attributes): # I swear I did not align the comparisons below. That's just how they # happened to align based on pep8 and attribute lengths. - userinfo, host, port = (attributes[p] - for p in ('userinfo', 'host', 'port')) - if (self.userinfo != userinfo or - self.host != host or - self.port != port): + userinfo, host, port = (attributes[p] for p in ("userinfo", "host", "port")) + if self.userinfo != userinfo or self.host != host or self.port != port: if port: - port = '{0}'.format(port) + port = "{0}".format(port) return normalizers.normalize_authority( - (compat.to_str(userinfo, self.encoding), - compat.to_str(host, self.encoding), - port) + ( + compat.to_str(userinfo, self.encoding), + compat.to_str(host, self.encoding), + port, + ) ) return self.authority @@ -65,8 +63,7 @@ class ParseResultMixin(object): return self.query -class ParseResult(namedtuple('ParseResult', PARSED_COMPONENTS), - ParseResultMixin): +class ParseResult(namedtuple("ParseResult", PARSED_COMPONENTS), ParseResultMixin): """Implementation of urlparse compatibility class. This uses the URIReference logic to handle compatibility with the @@ -75,8 +72,18 @@ class ParseResult(namedtuple('ParseResult', PARSED_COMPONENTS), slots = () - def __new__(cls, scheme, userinfo, host, port, path, query, fragment, - uri_ref, encoding='utf-8'): + def __new__( + cls, + scheme, + userinfo, + host, + port, + path, + query, + fragment, + uri_ref, + encoding="utf-8", + ): """Create a new ParseResult.""" parse_result = super(ParseResult, cls).__new__( cls, @@ -86,42 +93,57 @@ class ParseResult(namedtuple('ParseResult', PARSED_COMPONENTS), port or None, path or None, query, - fragment) + fragment, + ) parse_result.encoding = encoding parse_result.reference = uri_ref return parse_result @classmethod - def from_parts(cls, scheme=None, userinfo=None, host=None, port=None, - path=None, query=None, fragment=None, encoding='utf-8'): + def from_parts( + cls, + scheme=None, + userinfo=None, + host=None, + port=None, + path=None, + query=None, + fragment=None, + encoding="utf-8", + ): """Create a ParseResult instance from its parts.""" - authority = '' + authority = "" if userinfo is not None: - authority += userinfo + '@' + authority += userinfo + "@" if host is not None: authority += host if port is not None: - authority += ':{0}'.format(port) - uri_ref = uri.URIReference(scheme=scheme, - authority=authority, - path=path, - query=query, - fragment=fragment, - encoding=encoding).normalize() + authority += ":{0}".format(port) + uri_ref = uri.URIReference( + scheme=scheme, + authority=authority, + path=path, + query=query, + fragment=fragment, + encoding=encoding, + ).normalize() userinfo, host, port = authority_from(uri_ref, strict=True) - return cls(scheme=uri_ref.scheme, - userinfo=userinfo, - host=host, - port=port, - path=uri_ref.path, - query=uri_ref.query, - fragment=uri_ref.fragment, - uri_ref=uri_ref, - encoding=encoding) + return cls( + scheme=uri_ref.scheme, + userinfo=userinfo, + host=host, + port=port, + path=uri_ref.path, + query=uri_ref.query, + fragment=uri_ref.fragment, + uri_ref=uri_ref, + encoding=encoding, + ) @classmethod - def from_string(cls, uri_string, encoding='utf-8', strict=True, - lazy_normalize=True): + def from_string( + cls, uri_string, encoding="utf-8", strict=True, lazy_normalize=True + ): """Parse a URI from the given unicode URI string. :param str uri_string: Unicode URI to be parsed into a reference. @@ -136,53 +158,65 @@ class ParseResult(namedtuple('ParseResult', PARSED_COMPONENTS), reference = reference.normalize() userinfo, host, port = authority_from(reference, strict) - return cls(scheme=reference.scheme, - userinfo=userinfo, - host=host, - port=port, - path=reference.path, - query=reference.query, - fragment=reference.fragment, - uri_ref=reference, - encoding=encoding) + return cls( + scheme=reference.scheme, + userinfo=userinfo, + host=host, + port=port, + path=reference.path, + query=reference.query, + fragment=reference.fragment, + uri_ref=reference, + encoding=encoding, + ) @property def authority(self): """Return the normalized authority.""" return self.reference.authority - def copy_with(self, scheme=misc.UseExisting, userinfo=misc.UseExisting, - host=misc.UseExisting, port=misc.UseExisting, - path=misc.UseExisting, query=misc.UseExisting, - fragment=misc.UseExisting): + def copy_with( + self, + scheme=misc.UseExisting, + userinfo=misc.UseExisting, + host=misc.UseExisting, + port=misc.UseExisting, + path=misc.UseExisting, + query=misc.UseExisting, + fragment=misc.UseExisting, + ): """Create a copy of this instance replacing with specified parts.""" - attributes = zip(PARSED_COMPONENTS, - (scheme, userinfo, host, port, path, query, fragment)) + attributes = zip( + PARSED_COMPONENTS, (scheme, userinfo, host, port, path, query, fragment) + ) attrs_dict = {} for name, value in attributes: if value is misc.UseExisting: value = getattr(self, name) attrs_dict[name] = value authority = self._generate_authority(attrs_dict) - ref = self.reference.copy_with(scheme=attrs_dict['scheme'], - authority=authority, - path=attrs_dict['path'], - query=attrs_dict['query'], - fragment=attrs_dict['fragment']) + ref = self.reference.copy_with( + scheme=attrs_dict["scheme"], + authority=authority, + path=attrs_dict["path"], + query=attrs_dict["query"], + fragment=attrs_dict["fragment"], + ) return ParseResult(uri_ref=ref, encoding=self.encoding, **attrs_dict) def encode(self, encoding=None): """Convert to an instance of ParseResultBytes.""" encoding = encoding or self.encoding attrs = dict( - zip(PARSED_COMPONENTS, - (attr.encode(encoding) if hasattr(attr, 'encode') else attr - for attr in self))) - return ParseResultBytes( - uri_ref=self.reference, - encoding=encoding, - **attrs + zip( + PARSED_COMPONENTS, + ( + attr.encode(encoding) if hasattr(attr, "encode") else attr + for attr in self + ), + ) ) + return ParseResultBytes(uri_ref=self.reference, encoding=encoding, **attrs) def unsplit(self, use_idna=False): """Create a URI string from the components. @@ -192,18 +226,30 @@ class ParseResult(namedtuple('ParseResult', PARSED_COMPONENTS), """ parse_result = self if use_idna and self.host: - hostbytes = self.host.encode('idna') + hostbytes = self.host.encode("idna") host = hostbytes.decode(self.encoding) parse_result = self.copy_with(host=host) return parse_result.reference.unsplit() -class ParseResultBytes(namedtuple('ParseResultBytes', PARSED_COMPONENTS), - ParseResultMixin): +class ParseResultBytes( + namedtuple("ParseResultBytes", PARSED_COMPONENTS), ParseResultMixin +): """Compatibility shim for the urlparse.ParseResultBytes object.""" - def __new__(cls, scheme, userinfo, host, port, path, query, fragment, - uri_ref, encoding='utf-8', lazy_normalize=True): + def __new__( + cls, + scheme, + userinfo, + host, + port, + path, + query, + fragment, + uri_ref, + encoding="utf-8", + lazy_normalize=True, + ): """Create a new ParseResultBytes instance.""" parse_result = super(ParseResultBytes, cls).__new__( cls, @@ -213,48 +259,63 @@ class ParseResultBytes(namedtuple('ParseResultBytes', PARSED_COMPONENTS), port or None, path or None, query or None, - fragment or None) + fragment or None, + ) parse_result.encoding = encoding parse_result.reference = uri_ref parse_result.lazy_normalize = lazy_normalize return parse_result @classmethod - def from_parts(cls, scheme=None, userinfo=None, host=None, port=None, - path=None, query=None, fragment=None, encoding='utf-8', - lazy_normalize=True): + def from_parts( + cls, + scheme=None, + userinfo=None, + host=None, + port=None, + path=None, + query=None, + fragment=None, + encoding="utf-8", + lazy_normalize=True, + ): """Create a ParseResult instance from its parts.""" - authority = '' + authority = "" if userinfo is not None: - authority += userinfo + '@' + authority += userinfo + "@" if host is not None: authority += host if port is not None: - authority += ':{0}'.format(int(port)) - uri_ref = uri.URIReference(scheme=scheme, - authority=authority, - path=path, - query=query, - fragment=fragment, - encoding=encoding) + authority += ":{0}".format(int(port)) + uri_ref = uri.URIReference( + scheme=scheme, + authority=authority, + path=path, + query=query, + fragment=fragment, + encoding=encoding, + ) if not lazy_normalize: uri_ref = uri_ref.normalize() to_bytes = compat.to_bytes userinfo, host, port = authority_from(uri_ref, strict=True) - return cls(scheme=to_bytes(scheme, encoding), - userinfo=to_bytes(userinfo, encoding), - host=to_bytes(host, encoding), - port=port, - path=to_bytes(path, encoding), - query=to_bytes(query, encoding), - fragment=to_bytes(fragment, encoding), - uri_ref=uri_ref, - encoding=encoding, - lazy_normalize=lazy_normalize) + return cls( + scheme=to_bytes(scheme, encoding), + userinfo=to_bytes(userinfo, encoding), + host=to_bytes(host, encoding), + port=port, + path=to_bytes(path, encoding), + query=to_bytes(query, encoding), + fragment=to_bytes(fragment, encoding), + uri_ref=uri_ref, + encoding=encoding, + lazy_normalize=lazy_normalize, + ) @classmethod - def from_string(cls, uri_string, encoding='utf-8', strict=True, - lazy_normalize=True): + def from_string( + cls, uri_string, encoding="utf-8", strict=True, lazy_normalize=True + ): """Parse a URI from the given unicode URI string. :param str uri_string: Unicode URI to be parsed into a reference. @@ -270,44 +331,54 @@ class ParseResultBytes(namedtuple('ParseResultBytes', PARSED_COMPONENTS), userinfo, host, port = authority_from(reference, strict) to_bytes = compat.to_bytes - return cls(scheme=to_bytes(reference.scheme, encoding), - userinfo=to_bytes(userinfo, encoding), - host=to_bytes(host, encoding), - port=port, - path=to_bytes(reference.path, encoding), - query=to_bytes(reference.query, encoding), - fragment=to_bytes(reference.fragment, encoding), - uri_ref=reference, - encoding=encoding, - lazy_normalize=lazy_normalize) + return cls( + scheme=to_bytes(reference.scheme, encoding), + userinfo=to_bytes(userinfo, encoding), + host=to_bytes(host, encoding), + port=port, + path=to_bytes(reference.path, encoding), + query=to_bytes(reference.query, encoding), + fragment=to_bytes(reference.fragment, encoding), + uri_ref=reference, + encoding=encoding, + lazy_normalize=lazy_normalize, + ) @property def authority(self): """Return the normalized authority.""" return self.reference.authority.encode(self.encoding) - def copy_with(self, scheme=misc.UseExisting, userinfo=misc.UseExisting, - host=misc.UseExisting, port=misc.UseExisting, - path=misc.UseExisting, query=misc.UseExisting, - fragment=misc.UseExisting, lazy_normalize=True): + def copy_with( + self, + scheme=misc.UseExisting, + userinfo=misc.UseExisting, + host=misc.UseExisting, + port=misc.UseExisting, + path=misc.UseExisting, + query=misc.UseExisting, + fragment=misc.UseExisting, + lazy_normalize=True, + ): """Create a copy of this instance replacing with specified parts.""" - attributes = zip(PARSED_COMPONENTS, - (scheme, userinfo, host, port, path, query, fragment)) + attributes = zip( + PARSED_COMPONENTS, (scheme, userinfo, host, port, path, query, fragment) + ) attrs_dict = {} for name, value in attributes: if value is misc.UseExisting: value = getattr(self, name) - if not isinstance(value, bytes) and hasattr(value, 'encode'): + if not isinstance(value, bytes) and hasattr(value, "encode"): value = value.encode(self.encoding) attrs_dict[name] = value authority = self._generate_authority(attrs_dict) to_str = compat.to_str ref = self.reference.copy_with( - scheme=to_str(attrs_dict['scheme'], self.encoding), + scheme=to_str(attrs_dict["scheme"], self.encoding), authority=to_str(authority, self.encoding), - path=to_str(attrs_dict['path'], self.encoding), - query=to_str(attrs_dict['query'], self.encoding), - fragment=to_str(attrs_dict['fragment'], self.encoding) + path=to_str(attrs_dict["path"], self.encoding), + query=to_str(attrs_dict["query"], self.encoding), + fragment=to_str(attrs_dict["fragment"], self.encoding), ) if not lazy_normalize: ref = ref.normalize() @@ -329,7 +400,7 @@ class ParseResultBytes(namedtuple('ParseResultBytes', PARSED_COMPONENTS), # self.host is bytes, to encode to idna, we need to decode it # first host = self.host.decode(self.encoding) - hostbytes = host.encode('idna') + hostbytes = host.encode("idna") parse_result = self.copy_with(host=hostbytes) if self.lazy_normalize: parse_result = parse_result.copy_with(lazy_normalize=False) @@ -345,16 +416,16 @@ def split_authority(authority): # Set-up rest in case there is no userinfo portion rest = authority - if '@' in authority: - userinfo, rest = authority.rsplit('@', 1) + if "@" in authority: + userinfo, rest = authority.rsplit("@", 1) # Handle IPv6 host addresses - if rest.startswith('['): - host, rest = rest.split(']', 1) - host += ']' + if rest.startswith("["): + host, rest = rest.split("]", 1) + host += "]" - if ':' in rest: - extra_host, port = rest.split(':', 1) + if ":" in rest: + extra_host, port = rest.split(":", 1) elif not host and rest: host = rest @@ -374,8 +445,9 @@ def authority_from(reference, strict): else: # Thanks to Richard Barrell for this idea: # https://twitter.com/0x2ba22e11/status/617338811975139328 - userinfo, host, port = (subauthority.get(p) - for p in ('userinfo', 'host', 'port')) + userinfo, host, port = ( + subauthority.get(p) for p in ("userinfo", "host", "port") + ) if port: try: diff --git a/lib/urllib3/packages/rfc3986/uri.py b/lib/urllib3/packages/rfc3986/uri.py index d1d7150..61104ab 100644 --- a/lib/urllib3/packages/rfc3986/uri.py +++ b/lib/urllib3/packages/rfc3986/uri.py @@ -22,7 +22,7 @@ from . import normalizers from ._mixin import URIMixin -class URIReference(namedtuple('URIReference', misc.URI_COMPONENTS), URIMixin): +class URIReference(namedtuple("URIReference", misc.URI_COMPONENTS), URIMixin): """Immutable object representing a parsed URI Reference. .. note:: @@ -82,16 +82,11 @@ class URIReference(namedtuple('URIReference', misc.URI_COMPONENTS), URIMixin): slots = () - def __new__(cls, scheme, authority, path, query, fragment, - encoding='utf-8'): + def __new__(cls, scheme, authority, path, query, fragment, encoding="utf-8"): """Create a new URIReference.""" ref = super(URIReference, cls).__new__( - cls, - scheme or None, - authority or None, - path or None, - query, - fragment) + cls, scheme or None, authority or None, path or None, query, fragment + ) ref.encoding = encoding return ref @@ -107,8 +102,10 @@ class URIReference(namedtuple('URIReference', misc.URI_COMPONENTS), URIMixin): other_ref = URIReference.from_string(other) except TypeError: raise TypeError( - 'Unable to compare URIReference() to {0}()'.format( - type(other).__name__)) + "Unable to compare URIReference() to {0}()".format( + type(other).__name__ + ) + ) # See http://tools.ietf.org/html/rfc3986#section-6.2 naive_equality = tuple(self) == tuple(other_ref) @@ -125,16 +122,17 @@ class URIReference(namedtuple('URIReference', misc.URI_COMPONENTS), URIMixin): """ # See http://tools.ietf.org/html/rfc3986#section-6.2.2 for logic in # this method. - return URIReference(normalizers.normalize_scheme(self.scheme or ''), - normalizers.normalize_authority( - (self.userinfo, self.host, self.port)), - normalizers.normalize_path(self.path or ''), - normalizers.normalize_query(self.query), - normalizers.normalize_fragment(self.fragment), - self.encoding) + return URIReference( + normalizers.normalize_scheme(self.scheme or ""), + normalizers.normalize_authority((self.userinfo, self.host, self.port)), + normalizers.normalize_path(self.path or ""), + normalizers.normalize_query(self.query), + normalizers.normalize_fragment(self.fragment), + self.encoding, + ) @classmethod - def from_string(cls, uri_string, encoding='utf-8'): + def from_string(cls, uri_string, encoding="utf-8"): """Parse a URI reference from the given unicode URI string. :param str uri_string: Unicode URI to be parsed into a reference. @@ -145,9 +143,10 @@ class URIReference(namedtuple('URIReference', misc.URI_COMPONENTS), URIMixin): split_uri = misc.URI_MATCHER.match(uri_string).groupdict() return cls( - split_uri['scheme'], split_uri['authority'], - normalizers.encode_component(split_uri['path'], encoding), - normalizers.encode_component(split_uri['query'], encoding), - normalizers.encode_component(split_uri['fragment'], encoding), + split_uri["scheme"], + split_uri["authority"], + normalizers.encode_component(split_uri["path"], encoding), + normalizers.encode_component(split_uri["query"], encoding), + normalizers.encode_component(split_uri["fragment"], encoding), encoding, ) diff --git a/lib/urllib3/packages/rfc3986/validators.py b/lib/urllib3/packages/rfc3986/validators.py index 7fc9721..a60ae91 100644 --- a/lib/urllib3/packages/rfc3986/validators.py +++ b/lib/urllib3/packages/rfc3986/validators.py @@ -45,15 +45,9 @@ class Validator(object): """ - COMPONENT_NAMES = frozenset([ - 'scheme', - 'userinfo', - 'host', - 'port', - 'path', - 'query', - 'fragment', - ]) + COMPONENT_NAMES = frozenset( + ["scheme", "userinfo", "host", "port", "path", "query", "fragment"] + ) def __init__(self): """Initialize our default validations.""" @@ -62,13 +56,13 @@ class Validator(object): self.allowed_ports = set() self.allow_password = True self.required_components = { - 'scheme': False, - 'userinfo': False, - 'host': False, - 'port': False, - 'path': False, - 'query': False, - 'fragment': False, + "scheme": False, + "userinfo": False, + "host": False, + "port": False, + "path": False, + "query": False, + "fragment": False, } self.validated_components = self.required_components.copy() @@ -165,12 +159,8 @@ class Validator(object): components = [c.lower() for c in components] for component in components: if component not in self.COMPONENT_NAMES: - raise ValueError( - '"{}" is not a valid component'.format(component) - ) - self.validated_components.update({ - component: True for component in components - }) + raise ValueError('"{}" is not a valid component'.format(component)) + self.validated_components.update({component: True for component in components}) return self def require_presence_of(self, *components): @@ -190,12 +180,8 @@ class Validator(object): components = [c.lower() for c in components] for component in components: if component not in self.COMPONENT_NAMES: - raise ValueError( - '"{}" is not a valid component'.format(component) - ) - self.required_components.update({ - component: True for component in components - }) + raise ValueError('"{}" is not a valid component'.format(component)) + self.required_components.update({component: True for component in components}) return self def validate(self, uri): @@ -235,9 +221,9 @@ class Validator(object): if validated_components: ensure_components_are_valid(uri, validated_components) - ensure_one_of(self.allowed_schemes, uri, 'scheme') - ensure_one_of(self.allowed_hosts, uri, 'host') - ensure_one_of(self.allowed_ports, uri, 'port') + ensure_one_of(self.allowed_schemes, uri, "scheme") + ensure_one_of(self.allowed_hosts, uri, "host") + ensure_one_of(self.allowed_ports, uri, "port") def check_password(uri): @@ -245,7 +231,7 @@ def check_password(uri): userinfo = uri.userinfo if not userinfo: return - credentials = userinfo.split(':', 1) + credentials = userinfo.split(":", 1) if len(credentials) <= 1: return raise exceptions.PasswordForbidden(uri) @@ -255,18 +241,18 @@ def ensure_one_of(allowed_values, uri, attribute): """Assert that the uri's attribute is one of the allowed values.""" value = getattr(uri, attribute) if value is not None and allowed_values and value not in allowed_values: - raise exceptions.UnpermittedComponentError( - attribute, value, allowed_values, - ) + raise exceptions.UnpermittedComponentError(attribute, value, allowed_values) def ensure_required_components_exist(uri, required_components): """Assert that all required components are present in the URI.""" - missing_components = sorted([ - component - for component in required_components - if getattr(uri, component) is None - ]) + missing_components = sorted( + [ + component + for component in required_components + if getattr(uri, component) is None + ] + ) if missing_components: raise exceptions.MissingComponentError(uri, *missing_components) @@ -282,8 +268,7 @@ def is_valid(value, matcher, require): Whether or not the value is required. """ if require: - return (value is not None - and matcher.match(value)) + return value is not None and matcher.match(value) # require is False and value is not None return value is None or matcher.match(value) @@ -393,17 +378,17 @@ def valid_ipv4_host_address(host): """Determine if the given host is a valid IPv4 address.""" # If the host exists, and it might be IPv4, check each byte in the # address. - return all([0 <= int(byte, base=10) <= 255 for byte in host.split('.')]) + return all([0 <= int(byte, base=10) <= 255 for byte in host.split(".")]) _COMPONENT_VALIDATORS = { - 'scheme': scheme_is_valid, - 'path': path_is_valid, - 'query': query_is_valid, - 'fragment': fragment_is_valid, + "scheme": scheme_is_valid, + "path": path_is_valid, + "query": query_is_valid, + "fragment": fragment_is_valid, } -_SUBAUTHORITY_VALIDATORS = set(['userinfo', 'host', 'port']) +_SUBAUTHORITY_VALIDATORS = set(["userinfo", "host", "port"]) def subauthority_component_is_valid(uri, component): @@ -415,19 +400,19 @@ def subauthority_component_is_valid(uri, component): # If we can parse the authority into sub-components and we're not # validating the port, we can assume it's valid. - if component == 'host': - return host_is_valid(subauthority_dict['host']) - elif component != 'port': + if component == "host": + return host_is_valid(subauthority_dict["host"]) + elif component != "port": return True try: - port = int(subauthority_dict['port']) + port = int(subauthority_dict["port"]) except TypeError: # If the port wasn't provided it'll be None and int(None) raises a # TypeError return True - return (0 <= port <= 65535) + return 0 <= port <= 65535 def ensure_components_are_valid(uri, validated_components): diff --git a/lib/urllib3/packages/six.py b/lib/urllib3/packages/six.py index 190c023..26bbb43 100644 --- a/lib/urllib3/packages/six.py +++ b/lib/urllib3/packages/six.py @@ -38,15 +38,15 @@ PY3 = sys.version_info[0] == 3 PY34 = sys.version_info[0:2] >= (3, 4) if PY3: - string_types = str, - integer_types = int, - class_types = type, + string_types = (str,) + integer_types = (int,) + class_types = (type,) text_type = str binary_type = bytes MAXSIZE = sys.maxsize else: - string_types = basestring, + string_types = (basestring,) integer_types = (int, long) class_types = (type, types.ClassType) text_type = unicode @@ -58,9 +58,9 @@ else: else: # It's possible to have sizeof(long) != sizeof(Py_ssize_t). class X(object): - def __len__(self): return 1 << 31 + try: len(X()) except OverflowError: @@ -84,7 +84,6 @@ def _import_module(name): class _LazyDescr(object): - def __init__(self, name): self.name = name @@ -101,7 +100,6 @@ class _LazyDescr(object): class MovedModule(_LazyDescr): - def __init__(self, name, old, new=None): super(MovedModule, self).__init__(name) if PY3: @@ -122,7 +120,6 @@ class MovedModule(_LazyDescr): class _LazyModule(types.ModuleType): - def __init__(self, name): super(_LazyModule, self).__init__(name) self.__doc__ = self.__class__.__doc__ @@ -137,7 +134,6 @@ class _LazyModule(types.ModuleType): class MovedAttribute(_LazyDescr): - def __init__(self, name, old_mod, new_mod, old_attr=None, new_attr=None): super(MovedAttribute, self).__init__(name) if PY3: @@ -221,28 +217,35 @@ class _SixMetaPathImporter(object): Required, if is_package is implemented""" self.__get_module(fullname) # eventually raises ImportError return None + get_source = get_code # same as get_code + _importer = _SixMetaPathImporter(__name__) class _MovedItems(_LazyModule): """Lazy loading of moved objects""" + __path__ = [] # mark as package _moved_attributes = [ MovedAttribute("cStringIO", "cStringIO", "io", "StringIO"), MovedAttribute("filter", "itertools", "builtins", "ifilter", "filter"), - MovedAttribute("filterfalse", "itertools", "itertools", "ifilterfalse", "filterfalse"), + MovedAttribute( + "filterfalse", "itertools", "itertools", "ifilterfalse", "filterfalse" + ), MovedAttribute("input", "__builtin__", "builtins", "raw_input", "input"), MovedAttribute("intern", "__builtin__", "sys"), MovedAttribute("map", "itertools", "builtins", "imap", "map"), MovedAttribute("getcwd", "os", "os", "getcwdu", "getcwd"), MovedAttribute("getcwdb", "os", "os", "getcwd", "getcwdb"), MovedAttribute("range", "__builtin__", "builtins", "xrange", "range"), - MovedAttribute("reload_module", "__builtin__", "importlib" if PY34 else "imp", "reload"), + MovedAttribute( + "reload_module", "__builtin__", "importlib" if PY34 else "imp", "reload" + ), MovedAttribute("reduce", "__builtin__", "functools"), MovedAttribute("shlex_quote", "pipes", "shlex", "quote"), MovedAttribute("StringIO", "StringIO", "io"), @@ -251,7 +254,9 @@ _moved_attributes = [ MovedAttribute("UserString", "UserString", "collections"), MovedAttribute("xrange", "__builtin__", "builtins", "xrange", "range"), MovedAttribute("zip", "itertools", "builtins", "izip", "zip"), - MovedAttribute("zip_longest", "itertools", "itertools", "izip_longest", "zip_longest"), + MovedAttribute( + "zip_longest", "itertools", "itertools", "izip_longest", "zip_longest" + ), MovedModule("builtins", "__builtin__"), MovedModule("configparser", "ConfigParser"), MovedModule("copyreg", "copy_reg"), @@ -263,7 +268,9 @@ _moved_attributes = [ MovedModule("html_parser", "HTMLParser", "html.parser"), MovedModule("http_client", "httplib", "http.client"), MovedModule("email_mime_multipart", "email.MIMEMultipart", "email.mime.multipart"), - MovedModule("email_mime_nonmultipart", "email.MIMENonMultipart", "email.mime.nonmultipart"), + MovedModule( + "email_mime_nonmultipart", "email.MIMENonMultipart", "email.mime.nonmultipart" + ), MovedModule("email_mime_text", "email.MIMEText", "email.mime.text"), MovedModule("email_mime_base", "email.MIMEBase", "email.mime.base"), MovedModule("BaseHTTPServer", "BaseHTTPServer", "http.server"), @@ -283,15 +290,12 @@ _moved_attributes = [ MovedModule("tkinter_ttk", "ttk", "tkinter.ttk"), MovedModule("tkinter_constants", "Tkconstants", "tkinter.constants"), MovedModule("tkinter_dnd", "Tkdnd", "tkinter.dnd"), - MovedModule("tkinter_colorchooser", "tkColorChooser", - "tkinter.colorchooser"), - MovedModule("tkinter_commondialog", "tkCommonDialog", - "tkinter.commondialog"), + MovedModule("tkinter_colorchooser", "tkColorChooser", "tkinter.colorchooser"), + MovedModule("tkinter_commondialog", "tkCommonDialog", "tkinter.commondialog"), MovedModule("tkinter_tkfiledialog", "tkFileDialog", "tkinter.filedialog"), MovedModule("tkinter_font", "tkFont", "tkinter.font"), MovedModule("tkinter_messagebox", "tkMessageBox", "tkinter.messagebox"), - MovedModule("tkinter_tksimpledialog", "tkSimpleDialog", - "tkinter.simpledialog"), + MovedModule("tkinter_tksimpledialog", "tkSimpleDialog", "tkinter.simpledialog"), MovedModule("urllib_parse", __name__ + ".moves.urllib_parse", "urllib.parse"), MovedModule("urllib_error", __name__ + ".moves.urllib_error", "urllib.error"), MovedModule("urllib", __name__ + ".moves.urllib", __name__ + ".moves.urllib"), @@ -301,9 +305,7 @@ _moved_attributes = [ ] # Add windows specific modules. if sys.platform == "win32": - _moved_attributes += [ - MovedModule("winreg", "_winreg"), - ] + _moved_attributes += [MovedModule("winreg", "_winreg")] for attr in _moved_attributes: setattr(_MovedItems, attr.name, attr) @@ -353,8 +355,11 @@ del attr Module_six_moves_urllib_parse._moved_attributes = _urllib_parse_moved_attributes -_importer._add_module(Module_six_moves_urllib_parse(__name__ + ".moves.urllib_parse"), - "moves.urllib_parse", "moves.urllib.parse") +_importer._add_module( + Module_six_moves_urllib_parse(__name__ + ".moves.urllib_parse"), + "moves.urllib_parse", + "moves.urllib.parse", +) class Module_six_moves_urllib_error(_LazyModule): @@ -373,8 +378,11 @@ del attr Module_six_moves_urllib_error._moved_attributes = _urllib_error_moved_attributes -_importer._add_module(Module_six_moves_urllib_error(__name__ + ".moves.urllib.error"), - "moves.urllib_error", "moves.urllib.error") +_importer._add_module( + Module_six_moves_urllib_error(__name__ + ".moves.urllib.error"), + "moves.urllib_error", + "moves.urllib.error", +) class Module_six_moves_urllib_request(_LazyModule): @@ -423,8 +431,11 @@ del attr Module_six_moves_urllib_request._moved_attributes = _urllib_request_moved_attributes -_importer._add_module(Module_six_moves_urllib_request(__name__ + ".moves.urllib.request"), - "moves.urllib_request", "moves.urllib.request") +_importer._add_module( + Module_six_moves_urllib_request(__name__ + ".moves.urllib.request"), + "moves.urllib_request", + "moves.urllib.request", +) class Module_six_moves_urllib_response(_LazyModule): @@ -444,8 +455,11 @@ del attr Module_six_moves_urllib_response._moved_attributes = _urllib_response_moved_attributes -_importer._add_module(Module_six_moves_urllib_response(__name__ + ".moves.urllib.response"), - "moves.urllib_response", "moves.urllib.response") +_importer._add_module( + Module_six_moves_urllib_response(__name__ + ".moves.urllib.response"), + "moves.urllib_response", + "moves.urllib.response", +) class Module_six_moves_urllib_robotparser(_LazyModule): @@ -454,21 +468,27 @@ class Module_six_moves_urllib_robotparser(_LazyModule): _urllib_robotparser_moved_attributes = [ - MovedAttribute("RobotFileParser", "robotparser", "urllib.robotparser"), + MovedAttribute("RobotFileParser", "robotparser", "urllib.robotparser") ] for attr in _urllib_robotparser_moved_attributes: setattr(Module_six_moves_urllib_robotparser, attr.name, attr) del attr -Module_six_moves_urllib_robotparser._moved_attributes = _urllib_robotparser_moved_attributes +Module_six_moves_urllib_robotparser._moved_attributes = ( + _urllib_robotparser_moved_attributes +) -_importer._add_module(Module_six_moves_urllib_robotparser(__name__ + ".moves.urllib.robotparser"), - "moves.urllib_robotparser", "moves.urllib.robotparser") +_importer._add_module( + Module_six_moves_urllib_robotparser(__name__ + ".moves.urllib.robotparser"), + "moves.urllib_robotparser", + "moves.urllib.robotparser", +) class Module_six_moves_urllib(types.ModuleType): """Create a six.moves.urllib namespace that resembles the Python 3 namespace""" + __path__ = [] # mark as package parse = _importer._get_module("moves.urllib_parse") error = _importer._get_module("moves.urllib_error") @@ -477,10 +497,12 @@ class Module_six_moves_urllib(types.ModuleType): robotparser = _importer._get_module("moves.urllib_robotparser") def __dir__(self): - return ['parse', 'error', 'request', 'response', 'robotparser'] + return ["parse", "error", "request", "response", "robotparser"] + -_importer._add_module(Module_six_moves_urllib(__name__ + ".moves.urllib"), - "moves.urllib") +_importer._add_module( + Module_six_moves_urllib(__name__ + ".moves.urllib"), "moves.urllib" +) def add_move(move): @@ -520,19 +542,24 @@ else: try: advance_iterator = next except NameError: + def advance_iterator(it): return it.next() + + next = advance_iterator try: callable = callable except NameError: + def callable(obj): return any("__call__" in klass.__dict__ for klass in type(obj).__mro__) if PY3: + def get_unbound_function(unbound): return unbound @@ -543,6 +570,7 @@ if PY3: Iterator = object else: + def get_unbound_function(unbound): return unbound.im_func @@ -553,13 +581,13 @@ else: return types.MethodType(func, None, cls) class Iterator(object): - def next(self): return type(self).__next__(self) callable = callable -_add_doc(get_unbound_function, - """Get the function out of a possibly unbound function""") +_add_doc( + get_unbound_function, """Get the function out of a possibly unbound function""" +) get_method_function = operator.attrgetter(_meth_func) @@ -571,6 +599,7 @@ get_function_globals = operator.attrgetter(_func_globals) if PY3: + def iterkeys(d, **kw): return iter(d.keys(**kw)) @@ -589,6 +618,7 @@ if PY3: viewitems = operator.methodcaller("items") else: + def iterkeys(d, **kw): return d.iterkeys(**kw) @@ -609,26 +639,30 @@ else: _add_doc(iterkeys, "Return an iterator over the keys of a dictionary.") _add_doc(itervalues, "Return an iterator over the values of a dictionary.") -_add_doc(iteritems, - "Return an iterator over the (key, value) pairs of a dictionary.") -_add_doc(iterlists, - "Return an iterator over the (key, [values]) pairs of a dictionary.") +_add_doc(iteritems, "Return an iterator over the (key, value) pairs of a dictionary.") +_add_doc( + iterlists, "Return an iterator over the (key, [values]) pairs of a dictionary." +) if PY3: + def b(s): return s.encode("latin-1") def u(s): return s + unichr = chr import struct + int2byte = struct.Struct(">B").pack del struct byte2int = operator.itemgetter(0) indexbytes = operator.getitem iterbytes = iter import io + StringIO = io.StringIO BytesIO = io.BytesIO _assertCountEqual = "assertCountEqual" @@ -639,12 +673,15 @@ if PY3: _assertRaisesRegex = "assertRaisesRegex" _assertRegex = "assertRegex" else: + def b(s): return s + # Workaround for standalone backslash def u(s): - return unicode(s.replace(r'\\', r'\\\\'), "unicode_escape") + return unicode(s.replace(r"\\", r"\\\\"), "unicode_escape") + unichr = unichr int2byte = chr @@ -653,8 +690,10 @@ else: def indexbytes(buf, i): return ord(buf[i]) + iterbytes = functools.partial(itertools.imap, ord) import StringIO + StringIO = BytesIO = StringIO.StringIO _assertCountEqual = "assertItemsEqual" _assertRaisesRegex = "assertRaisesRegexp" @@ -685,7 +724,9 @@ if PY3: raise value.with_traceback(tb) raise value + else: + def exec_(_code_, _globs_=None, _locs_=None): """Execute code in a namespace.""" if _globs_ is None: @@ -698,28 +739,36 @@ else: _locs_ = _globs_ exec("""exec _code_ in _globs_, _locs_""") - exec_("""def reraise(tp, value, tb=None): + exec_( + """def reraise(tp, value, tb=None): raise tp, value, tb -""") +""" + ) if sys.version_info[:2] == (3, 2): - exec_("""def raise_from(value, from_value): + exec_( + """def raise_from(value, from_value): if from_value is None: raise value raise value from from_value -""") +""" + ) elif sys.version_info[:2] > (3, 2): - exec_("""def raise_from(value, from_value): + exec_( + """def raise_from(value, from_value): raise value from from_value -""") +""" + ) else: + def raise_from(value, from_value): raise value print_ = getattr(moves.builtins, "print", None) if print_ is None: + def print_(*args, **kwargs): """The new-style print function for Python 2.4 and 2.5.""" fp = kwargs.pop("file", sys.stdout) @@ -730,14 +779,17 @@ if print_ is None: if not isinstance(data, basestring): data = str(data) # If the file has an encoding, encode unicode with it. - if (isinstance(fp, file) and - isinstance(data, unicode) and - fp.encoding is not None): + if ( + isinstance(fp, file) + and isinstance(data, unicode) + and fp.encoding is not None + ): errors = getattr(fp, "errors", None) if errors is None: errors = "strict" data = data.encode(fp.encoding, errors) fp.write(data) + want_unicode = False sep = kwargs.pop("sep", None) if sep is not None: @@ -773,6 +825,8 @@ if print_ is None: write(sep) write(arg) write(end) + + if sys.version_info[:2] < (3, 3): _print = print_ @@ -783,16 +837,24 @@ if sys.version_info[:2] < (3, 3): if flush and fp is not None: fp.flush() + _add_doc(reraise, """Reraise an exception.""") if sys.version_info[0:2] < (3, 4): - def wraps(wrapped, assigned=functools.WRAPPER_ASSIGNMENTS, - updated=functools.WRAPPER_UPDATES): + + def wraps( + wrapped, + assigned=functools.WRAPPER_ASSIGNMENTS, + updated=functools.WRAPPER_UPDATES, + ): def wrapper(f): f = functools.wraps(wrapped, assigned, updated)(f) f.__wrapped__ = wrapped return f + return wrapper + + else: wraps = functools.wraps @@ -803,25 +865,27 @@ def with_metaclass(meta, *bases): # metaclass for one level of class instantiation that replaces itself with # the actual metaclass. class metaclass(meta): - def __new__(cls, name, this_bases, d): return meta(name, bases, d) - return type.__new__(metaclass, 'temporary_class', (), {}) + + return type.__new__(metaclass, "temporary_class", (), {}) def add_metaclass(metaclass): """Class decorator for creating a class with a metaclass.""" + def wrapper(cls): orig_vars = cls.__dict__.copy() - slots = orig_vars.get('__slots__') + slots = orig_vars.get("__slots__") if slots is not None: if isinstance(slots, str): slots = [slots] for slots_var in slots: orig_vars.pop(slots_var) - orig_vars.pop('__dict__', None) - orig_vars.pop('__weakref__', None) + orig_vars.pop("__dict__", None) + orig_vars.pop("__weakref__", None) return metaclass(cls.__name__, cls.__bases__, orig_vars) + return wrapper @@ -834,12 +898,13 @@ def python_2_unicode_compatible(klass): returning text and apply this decorator to the class. """ if PY2: - if '__str__' not in klass.__dict__: - raise ValueError("@python_2_unicode_compatible cannot be applied " - "to %s because it doesn't define __str__()." % - klass.__name__) + if "__str__" not in klass.__dict__: + raise ValueError( + "@python_2_unicode_compatible cannot be applied " + "to %s because it doesn't define __str__()." % klass.__name__ + ) klass.__unicode__ = klass.__str__ - klass.__str__ = lambda self: self.__unicode__().encode('utf-8') + klass.__str__ = lambda self: self.__unicode__().encode("utf-8") return klass @@ -859,8 +924,10 @@ if sys.meta_path: # be floating around. Therefore, we can't use isinstance() to check for # the six meta path importer, since the other six instance will have # inserted an importer with different class. - if (type(importer).__name__ == "_SixMetaPathImporter" and - importer.name == __name__): + if ( + type(importer).__name__ == "_SixMetaPathImporter" + and importer.name == __name__ + ): del sys.meta_path[i] break del i, importer diff --git a/lib/urllib3/packages/ssl_match_hostname/__init__.py b/lib/urllib3/packages/ssl_match_hostname/__init__.py index d6594eb..75b6bb1 100644 --- a/lib/urllib3/packages/ssl_match_hostname/__init__.py +++ b/lib/urllib3/packages/ssl_match_hostname/__init__.py @@ -16,4 +16,4 @@ except ImportError: from ._implementation import CertificateError, match_hostname # Not needed, but documenting what we provide. -__all__ = ('CertificateError', 'match_hostname') +__all__ = ("CertificateError", "match_hostname") diff --git a/lib/urllib3/packages/ssl_match_hostname/_implementation.py b/lib/urllib3/packages/ssl_match_hostname/_implementation.py index d6e66c0..2d8e7a1 100644 --- a/lib/urllib3/packages/ssl_match_hostname/_implementation.py +++ b/lib/urllib3/packages/ssl_match_hostname/_implementation.py @@ -15,7 +15,7 @@ try: except ImportError: ipaddress = None -__version__ = '3.5.0.1' +__version__ = "3.5.0.1" class CertificateError(ValueError): @@ -33,18 +33,19 @@ def _dnsname_match(dn, hostname, max_wildcards=1): # Ported from python3-syntax: # leftmost, *remainder = dn.split(r'.') - parts = dn.split(r'.') + parts = dn.split(r".") leftmost = parts[0] remainder = parts[1:] - wildcards = leftmost.count('*') + wildcards = leftmost.count("*") if wildcards > max_wildcards: # Issue #17980: avoid denials of service by refusing more # than one wildcard per fragment. A survey of established # policy among SSL implementations showed it to be a # reasonable choice. raise CertificateError( - "too many wildcards in certificate DNS name: " + repr(dn)) + "too many wildcards in certificate DNS name: " + repr(dn) + ) # speed up common case w/o wildcards if not wildcards: @@ -53,11 +54,11 @@ def _dnsname_match(dn, hostname, max_wildcards=1): # RFC 6125, section 6.4.3, subitem 1. # The client SHOULD NOT attempt to match a presented identifier in which # the wildcard character comprises a label other than the left-most label. - if leftmost == '*': + if leftmost == "*": # When '*' is a fragment by itself, it matches a non-empty dotless # fragment. - pats.append('[^.]+') - elif leftmost.startswith('xn--') or hostname.startswith('xn--'): + pats.append("[^.]+") + elif leftmost.startswith("xn--") or hostname.startswith("xn--"): # RFC 6125, section 6.4.3, subitem 3. # The client SHOULD NOT attempt to match a presented identifier # where the wildcard character is embedded within an A-label or @@ -65,21 +66,22 @@ def _dnsname_match(dn, hostname, max_wildcards=1): pats.append(re.escape(leftmost)) else: # Otherwise, '*' matches any dotless string, e.g. www* - pats.append(re.escape(leftmost).replace(r'\*', '[^.]*')) + pats.append(re.escape(leftmost).replace(r"\*", "[^.]*")) # add the remaining fragments, ignore any wildcards for frag in remainder: pats.append(re.escape(frag)) - pat = re.compile(r'\A' + r'\.'.join(pats) + r'\Z', re.IGNORECASE) + pat = re.compile(r"\A" + r"\.".join(pats) + r"\Z", re.IGNORECASE) return pat.match(hostname) def _to_unicode(obj): if isinstance(obj, str) and sys.version_info < (3,): - obj = unicode(obj, encoding='ascii', errors='strict') + obj = unicode(obj, encoding="ascii", errors="strict") return obj + def _ipaddress_match(ipname, host_ip): """Exact matching of IP addresses. @@ -101,9 +103,11 @@ def match_hostname(cert, hostname): returns nothing. """ if not cert: - raise ValueError("empty or no certificate, match_hostname needs a " - "SSL socket or SSL context with either " - "CERT_OPTIONAL or CERT_REQUIRED") + raise ValueError( + "empty or no certificate, match_hostname needs a " + "SSL socket or SSL context with either " + "CERT_OPTIONAL or CERT_REQUIRED" + ) try: # Divergence from upstream: ipaddress can't handle byte str host_ip = ipaddress.ip_address(_to_unicode(hostname)) @@ -122,35 +126,37 @@ def match_hostname(cert, hostname): else: raise dnsnames = [] - san = cert.get('subjectAltName', ()) + san = cert.get("subjectAltName", ()) for key, value in san: - if key == 'DNS': + if key == "DNS": if host_ip is None and _dnsname_match(value, hostname): return dnsnames.append(value) - elif key == 'IP Address': + elif key == "IP Address": if host_ip is not None and _ipaddress_match(value, host_ip): return dnsnames.append(value) if not dnsnames: # The subject is only checked when there is no dNSName entry # in subjectAltName - for sub in cert.get('subject', ()): + for sub in cert.get("subject", ()): for key, value in sub: # XXX according to RFC 2818, the most specific Common Name # must be used. - if key == 'commonName': + if key == "commonName": if _dnsname_match(value, hostname): return dnsnames.append(value) if len(dnsnames) > 1: - raise CertificateError("hostname %r " - "doesn't match either of %s" - % (hostname, ', '.join(map(repr, dnsnames)))) + raise CertificateError( + "hostname %r " + "doesn't match either of %s" % (hostname, ", ".join(map(repr, dnsnames))) + ) elif len(dnsnames) == 1: - raise CertificateError("hostname %r " - "doesn't match %r" - % (hostname, dnsnames[0])) + raise CertificateError( + "hostname %r " "doesn't match %r" % (hostname, dnsnames[0]) + ) else: - raise CertificateError("no appropriate commonName or " - "subjectAltName fields were found") + raise CertificateError( + "no appropriate commonName or " "subjectAltName fields were found" + ) diff --git a/lib/urllib3/poolmanager.py b/lib/urllib3/poolmanager.py index a6ade6e..242a2f8 100644 --- a/lib/urllib3/poolmanager.py +++ b/lib/urllib3/poolmanager.py @@ -14,48 +14,55 @@ from .util.url import parse_url from .util.retry import Retry -__all__ = ['PoolManager', 'ProxyManager', 'proxy_from_url'] +__all__ = ["PoolManager", "ProxyManager", "proxy_from_url"] log = logging.getLogger(__name__) -SSL_KEYWORDS = ('key_file', 'cert_file', 'cert_reqs', 'ca_certs', - 'ssl_version', 'ca_cert_dir', 'ssl_context', - 'key_password') +SSL_KEYWORDS = ( + "key_file", + "cert_file", + "cert_reqs", + "ca_certs", + "ssl_version", + "ca_cert_dir", + "ssl_context", + "key_password", +) # All known keyword arguments that could be provided to the pool manager, its # pools, or the underlying connections. This is used to construct a pool key. _key_fields = ( - 'key_scheme', # str - 'key_host', # str - 'key_port', # int - 'key_timeout', # int or float or Timeout - 'key_retries', # int or Retry - 'key_strict', # bool - 'key_block', # bool - 'key_source_address', # str - 'key_key_file', # str - 'key_key_password', # str - 'key_cert_file', # str - 'key_cert_reqs', # str - 'key_ca_certs', # str - 'key_ssl_version', # str - 'key_ca_cert_dir', # str - 'key_ssl_context', # instance of ssl.SSLContext or urllib3.util.ssl_.SSLContext - 'key_maxsize', # int - 'key_headers', # dict - 'key__proxy', # parsed proxy url - 'key__proxy_headers', # dict - 'key_socket_options', # list of (level (int), optname (int), value (int or str)) tuples - 'key__socks_options', # dict - 'key_assert_hostname', # bool or string - 'key_assert_fingerprint', # str - 'key_server_hostname', # str + "key_scheme", # str + "key_host", # str + "key_port", # int + "key_timeout", # int or float or Timeout + "key_retries", # int or Retry + "key_strict", # bool + "key_block", # bool + "key_source_address", # str + "key_key_file", # str + "key_key_password", # str + "key_cert_file", # str + "key_cert_reqs", # str + "key_ca_certs", # str + "key_ssl_version", # str + "key_ca_cert_dir", # str + "key_ssl_context", # instance of ssl.SSLContext or urllib3.util.ssl_.SSLContext + "key_maxsize", # int + "key_headers", # dict + "key__proxy", # parsed proxy url + "key__proxy_headers", # dict + "key_socket_options", # list of (level (int), optname (int), value (int or str)) tuples + "key__socks_options", # dict + "key_assert_hostname", # bool or string + "key_assert_fingerprint", # str + "key_server_hostname", # str ) #: The namedtuple class used to construct keys for the connection pool. #: All custom key schemes should include the fields in this key at a minimum. -PoolKey = collections.namedtuple('PoolKey', _key_fields) +PoolKey = collections.namedtuple("PoolKey", _key_fields) def _default_key_normalizer(key_class, request_context): @@ -80,24 +87,24 @@ def _default_key_normalizer(key_class, request_context): """ # Since we mutate the dictionary, make a copy first context = request_context.copy() - context['scheme'] = context['scheme'].lower() - context['host'] = context['host'].lower() + context["scheme"] = context["scheme"].lower() + context["host"] = context["host"].lower() # These are both dictionaries and need to be transformed into frozensets - for key in ('headers', '_proxy_headers', '_socks_options'): + for key in ("headers", "_proxy_headers", "_socks_options"): if key in context and context[key] is not None: context[key] = frozenset(context[key].items()) # The socket_options key may be a list and needs to be transformed into a # tuple. - socket_opts = context.get('socket_options') + socket_opts = context.get("socket_options") if socket_opts is not None: - context['socket_options'] = tuple(socket_opts) + context["socket_options"] = tuple(socket_opts) # Map the kwargs to the names in the namedtuple - this is necessary since # namedtuples can't have fields starting with '_'. for key in list(context.keys()): - context['key_' + key] = context.pop(key) + context["key_" + key] = context.pop(key) # Default to ``None`` for keys missing from the context for field in key_class._fields: @@ -112,14 +119,11 @@ def _default_key_normalizer(key_class, request_context): #: Each PoolManager makes a copy of this dictionary so they can be configured #: globally here, or individually on the instance. key_fn_by_scheme = { - 'http': functools.partial(_default_key_normalizer, PoolKey), - 'https': functools.partial(_default_key_normalizer, PoolKey), + "http": functools.partial(_default_key_normalizer, PoolKey), + "https": functools.partial(_default_key_normalizer, PoolKey), } -pool_classes_by_scheme = { - 'http': HTTPConnectionPool, - 'https': HTTPSConnectionPool, -} +pool_classes_by_scheme = {"http": HTTPConnectionPool, "https": HTTPSConnectionPool} class PoolManager(RequestMethods): @@ -155,8 +159,7 @@ class PoolManager(RequestMethods): def __init__(self, num_pools=10, headers=None, **connection_pool_kw): RequestMethods.__init__(self, headers) self.connection_pool_kw = connection_pool_kw - self.pools = RecentlyUsedContainer(num_pools, - dispose_func=lambda p: p.close()) + self.pools = RecentlyUsedContainer(num_pools, dispose_func=lambda p: p.close()) # Locally set the pool classes and keys so other PoolManagers can # override them. @@ -189,10 +192,10 @@ class PoolManager(RequestMethods): # this function has historically only used the scheme, host, and port # in the positional args. When an API change is acceptable these can # be removed. - for key in ('scheme', 'host', 'port'): + for key in ("scheme", "host", "port"): request_context.pop(key, None) - if scheme == 'http': + if scheme == "http": for kw in SSL_KEYWORDS: request_context.pop(kw, None) @@ -207,7 +210,7 @@ class PoolManager(RequestMethods): """ self.pools.clear() - def connection_from_host(self, host, port=None, scheme='http', pool_kwargs=None): + def connection_from_host(self, host, port=None, scheme="http", pool_kwargs=None): """ Get a :class:`ConnectionPool` based on the host, port, and scheme. @@ -222,11 +225,11 @@ class PoolManager(RequestMethods): raise LocationValueError("No host specified.") request_context = self._merge_pool_kwargs(pool_kwargs) - request_context['scheme'] = scheme or 'http' + request_context["scheme"] = scheme or "http" if not port: - port = port_by_scheme.get(request_context['scheme'].lower(), 80) - request_context['port'] = port - request_context['host'] = host + port = port_by_scheme.get(request_context["scheme"].lower(), 80) + request_context["port"] = port + request_context["host"] = host return self.connection_from_context(request_context) @@ -237,7 +240,7 @@ class PoolManager(RequestMethods): ``request_context`` must at least contain the ``scheme`` key and its value must be a key in ``key_fn_by_scheme`` instance variable. """ - scheme = request_context['scheme'].lower() + scheme = request_context["scheme"].lower() pool_key_constructor = self.key_fn_by_scheme[scheme] pool_key = pool_key_constructor(request_context) @@ -259,9 +262,9 @@ class PoolManager(RequestMethods): return pool # Make a fresh ConnectionPool of the desired type - scheme = request_context['scheme'] - host = request_context['host'] - port = request_context['port'] + scheme = request_context["scheme"] + host = request_context["host"] + port = request_context["port"] pool = self._new_pool(scheme, host, port, request_context=request_context) self.pools[pool_key] = pool @@ -279,8 +282,9 @@ class PoolManager(RequestMethods): not used. """ u = parse_url(url) - return self.connection_from_host(u.host, port=u.port, scheme=u.scheme, - pool_kwargs=pool_kwargs) + return self.connection_from_host( + u.host, port=u.port, scheme=u.scheme, pool_kwargs=pool_kwargs + ) def _merge_pool_kwargs(self, override): """ @@ -314,11 +318,11 @@ class PoolManager(RequestMethods): u = parse_url(url) conn = self.connection_from_host(u.host, port=u.port, scheme=u.scheme) - kw['assert_same_host'] = False - kw['redirect'] = False + kw["assert_same_host"] = False + kw["redirect"] = False - if 'headers' not in kw: - kw['headers'] = self.headers.copy() + if "headers" not in kw: + kw["headers"] = self.headers.copy() if self.proxy is not None and u.scheme == "http": response = conn.urlopen(method, url, **kw) @@ -334,21 +338,22 @@ class PoolManager(RequestMethods): # RFC 7231, Section 6.4.4 if response.status == 303: - method = 'GET' + method = "GET" - retries = kw.get('retries') + retries = kw.get("retries") if not isinstance(retries, Retry): retries = Retry.from_int(retries, redirect=redirect) # Strip headers marked as unsafe to forward to the redirected location. # Check remove_headers_on_redirect to avoid a potential network call within # conn.is_same_host() which may use socket.gethostbyname() in the future. - if (retries.remove_headers_on_redirect - and not conn.is_same_host(redirect_location)): - headers = list(six.iterkeys(kw['headers'])) + if retries.remove_headers_on_redirect and not conn.is_same_host( + redirect_location + ): + headers = list(six.iterkeys(kw["headers"])) for header in headers: if header.lower() in retries.remove_headers_on_redirect: - kw['headers'].pop(header, None) + kw["headers"].pop(header, None) try: retries = retries.increment(method, url, response=response, _pool=conn) @@ -357,8 +362,8 @@ class PoolManager(RequestMethods): raise return response - kw['retries'] = retries - kw['redirect'] = redirect + kw["retries"] = retries + kw["redirect"] = redirect log.info("Redirecting %s -> %s", url, redirect_location) return self.urlopen(method, redirect_location, **kw) @@ -391,12 +396,21 @@ class ProxyManager(PoolManager): """ - def __init__(self, proxy_url, num_pools=10, headers=None, - proxy_headers=None, **connection_pool_kw): + def __init__( + self, + proxy_url, + num_pools=10, + headers=None, + proxy_headers=None, + **connection_pool_kw + ): if isinstance(proxy_url, HTTPConnectionPool): - proxy_url = '%s://%s:%i' % (proxy_url.scheme, proxy_url.host, - proxy_url.port) + proxy_url = "%s://%s:%i" % ( + proxy_url.scheme, + proxy_url.host, + proxy_url.port, + ) proxy = parse_url(proxy_url) if not proxy.port: port = port_by_scheme.get(proxy.scheme, 80) @@ -408,30 +422,31 @@ class ProxyManager(PoolManager): self.proxy = proxy self.proxy_headers = proxy_headers or {} - connection_pool_kw['_proxy'] = self.proxy - connection_pool_kw['_proxy_headers'] = self.proxy_headers + connection_pool_kw["_proxy"] = self.proxy + connection_pool_kw["_proxy_headers"] = self.proxy_headers - super(ProxyManager, self).__init__( - num_pools, headers, **connection_pool_kw) + super(ProxyManager, self).__init__(num_pools, headers, **connection_pool_kw) - def connection_from_host(self, host, port=None, scheme='http', pool_kwargs=None): + def connection_from_host(self, host, port=None, scheme="http", pool_kwargs=None): if scheme == "https": return super(ProxyManager, self).connection_from_host( - host, port, scheme, pool_kwargs=pool_kwargs) + host, port, scheme, pool_kwargs=pool_kwargs + ) return super(ProxyManager, self).connection_from_host( - self.proxy.host, self.proxy.port, self.proxy.scheme, pool_kwargs=pool_kwargs) + self.proxy.host, self.proxy.port, self.proxy.scheme, pool_kwargs=pool_kwargs + ) def _set_proxy_headers(self, url, headers=None): """ Sets headers needed by proxies: specifically, the Accept and Host headers. Only sets headers not provided by the user. """ - headers_ = {'Accept': '*/*'} + headers_ = {"Accept": "*/*"} netloc = parse_url(url).netloc if netloc: - headers_['Host'] = netloc + headers_["Host"] = netloc if headers: headers_.update(headers) @@ -445,8 +460,8 @@ class ProxyManager(PoolManager): # For proxied HTTPS requests, httplib sets the necessary headers # on the CONNECT to the proxy. For HTTP, we'll definitely # need to set 'Host' at the very least. - headers = kw.get('headers', self.headers) - kw['headers'] = self._set_proxy_headers(url, headers) + headers = kw.get("headers", self.headers) + kw["headers"] = self._set_proxy_headers(url, headers) return super(ProxyManager, self).urlopen(method, url, redirect=redirect, **kw) diff --git a/lib/urllib3/request.py b/lib/urllib3/request.py index 8f2f44b..55f160b 100644 --- a/lib/urllib3/request.py +++ b/lib/urllib3/request.py @@ -4,7 +4,7 @@ from .filepost import encode_multipart_formdata from .packages.six.moves.urllib.parse import urlencode -__all__ = ['RequestMethods'] +__all__ = ["RequestMethods"] class RequestMethods(object): @@ -36,16 +36,25 @@ class RequestMethods(object): explicitly. """ - _encode_url_methods = {'DELETE', 'GET', 'HEAD', 'OPTIONS'} + _encode_url_methods = {"DELETE", "GET", "HEAD", "OPTIONS"} def __init__(self, headers=None): self.headers = headers or {} - def urlopen(self, method, url, body=None, headers=None, - encode_multipart=True, multipart_boundary=None, - **kw): # Abstract - raise NotImplementedError("Classes extending RequestMethods must implement " - "their own ``urlopen`` method.") + def urlopen( + self, + method, + url, + body=None, + headers=None, + encode_multipart=True, + multipart_boundary=None, + **kw + ): # Abstract + raise NotImplementedError( + "Classes extending RequestMethods must implement " + "their own ``urlopen`` method." + ) def request(self, method, url, fields=None, headers=None, **urlopen_kw): """ @@ -60,19 +69,18 @@ class RequestMethods(object): """ method = method.upper() - urlopen_kw['request_url'] = url + urlopen_kw["request_url"] = url if method in self._encode_url_methods: - return self.request_encode_url(method, url, fields=fields, - headers=headers, - **urlopen_kw) + return self.request_encode_url( + method, url, fields=fields, headers=headers, **urlopen_kw + ) else: - return self.request_encode_body(method, url, fields=fields, - headers=headers, - **urlopen_kw) + return self.request_encode_body( + method, url, fields=fields, headers=headers, **urlopen_kw + ) - def request_encode_url(self, method, url, fields=None, headers=None, - **urlopen_kw): + def request_encode_url(self, method, url, fields=None, headers=None, **urlopen_kw): """ Make a request using :meth:`urlopen` with the ``fields`` encoded in the url. This is useful for request methods like GET, HEAD, DELETE, etc. @@ -80,17 +88,24 @@ class RequestMethods(object): if headers is None: headers = self.headers - extra_kw = {'headers': headers} + extra_kw = {"headers": headers} extra_kw.update(urlopen_kw) if fields: - url += '?' + urlencode(fields) + url += "?" + urlencode(fields) return self.urlopen(method, url, **extra_kw) - def request_encode_body(self, method, url, fields=None, headers=None, - encode_multipart=True, multipart_boundary=None, - **urlopen_kw): + def request_encode_body( + self, + method, + url, + fields=None, + headers=None, + encode_multipart=True, + multipart_boundary=None, + **urlopen_kw + ): """ Make a request using :meth:`urlopen` with the ``fields`` encoded in the body. This is useful for request methods like POST, PUT, PATCH, etc. @@ -129,22 +144,28 @@ class RequestMethods(object): if headers is None: headers = self.headers - extra_kw = {'headers': {}} + extra_kw = {"headers": {}} if fields: - if 'body' in urlopen_kw: + if "body" in urlopen_kw: raise TypeError( - "request got values for both 'fields' and 'body', can only specify one.") + "request got values for both 'fields' and 'body', can only specify one." + ) if encode_multipart: - body, content_type = encode_multipart_formdata(fields, boundary=multipart_boundary) + body, content_type = encode_multipart_formdata( + fields, boundary=multipart_boundary + ) else: - body, content_type = urlencode(fields), 'application/x-www-form-urlencoded' + body, content_type = ( + urlencode(fields), + "application/x-www-form-urlencoded", + ) - extra_kw['body'] = body - extra_kw['headers'] = {'Content-Type': content_type} + extra_kw["body"] = body + extra_kw["headers"] = {"Content-Type": content_type} - extra_kw['headers'].update(headers) + extra_kw["headers"].update(headers) extra_kw.update(urlopen_kw) return self.urlopen(method, url, **extra_kw) diff --git a/lib/urllib3/response.py b/lib/urllib3/response.py index 4f85793..50de013 100644 --- a/lib/urllib3/response.py +++ b/lib/urllib3/response.py @@ -13,8 +13,13 @@ except ImportError: from ._collections import HTTPHeaderDict from .exceptions import ( - BodyNotHttplibCompatible, ProtocolError, DecodeError, ReadTimeoutError, - ResponseNotChunked, IncompleteRead, InvalidHeader + BodyNotHttplibCompatible, + ProtocolError, + DecodeError, + ReadTimeoutError, + ResponseNotChunked, + IncompleteRead, + InvalidHeader, ) from .packages.six import string_types as basestring, PY3 from .packages.six.moves import http_client as httplib @@ -25,10 +30,9 @@ log = logging.getLogger(__name__) class DeflateDecoder(object): - def __init__(self): self._first_try = True - self._data = b'' + self._data = b"" self._obj = zlib.decompressobj() def __getattr__(self, name): @@ -65,7 +69,6 @@ class GzipDecoderState(object): class GzipDecoder(object): - def __init__(self): self._obj = zlib.decompressobj(16 + zlib.MAX_WBITS) self._state = GzipDecoderState.FIRST_MEMBER @@ -96,6 +99,7 @@ class GzipDecoder(object): if brotli is not None: + class BrotliDecoder(object): # Supports both 'brotlipy' and 'Brotli' packages # since they share an import name. The top branches @@ -104,14 +108,14 @@ if brotli is not None: self._obj = brotli.Decompressor() def decompress(self, data): - if hasattr(self._obj, 'decompress'): + if hasattr(self._obj, "decompress"): return self._obj.decompress(data) return self._obj.process(data) def flush(self): - if hasattr(self._obj, 'flush'): + if hasattr(self._obj, "flush"): return self._obj.flush() - return b'' + return b"" class MultiDecoder(object): @@ -124,7 +128,7 @@ class MultiDecoder(object): """ def __init__(self, modes): - self._decoders = [_get_decoder(m.strip()) for m in modes.split(',')] + self._decoders = [_get_decoder(m.strip()) for m in modes.split(",")] def flush(self): return self._decoders[0].flush() @@ -136,13 +140,13 @@ class MultiDecoder(object): def _get_decoder(mode): - if ',' in mode: + if "," in mode: return MultiDecoder(mode) - if mode == 'gzip': + if mode == "gzip": return GzipDecoder() - if brotli is not None and mode == 'br': + if brotli is not None and mode == "br": return BrotliDecoder() return DeflateDecoder() @@ -181,16 +185,30 @@ class HTTPResponse(io.IOBase): value of Content-Length header, if present. Otherwise, raise error. """ - CONTENT_DECODERS = ['gzip', 'deflate'] + CONTENT_DECODERS = ["gzip", "deflate"] if brotli is not None: - CONTENT_DECODERS += ['br'] + CONTENT_DECODERS += ["br"] REDIRECT_STATUSES = [301, 302, 303, 307, 308] - def __init__(self, body='', headers=None, status=0, version=0, reason=None, - strict=0, preload_content=True, decode_content=True, - original_response=None, pool=None, connection=None, msg=None, - retries=None, enforce_content_length=False, - request_method=None, request_url=None): + def __init__( + self, + body="", + headers=None, + status=0, + version=0, + reason=None, + strict=0, + preload_content=True, + decode_content=True, + original_response=None, + pool=None, + connection=None, + msg=None, + retries=None, + enforce_content_length=False, + request_method=None, + request_url=None, + ): if isinstance(headers, HTTPHeaderDict): self.headers = headers @@ -218,13 +236,13 @@ class HTTPResponse(io.IOBase): self._pool = pool self._connection = connection - if hasattr(body, 'read'): + if hasattr(body, "read"): self._fp = body # Are we using the chunked-style of transfer encoding? self.chunked = False self.chunk_left = None - tr_enc = self.headers.get('transfer-encoding', '').lower() + tr_enc = self.headers.get("transfer-encoding", "").lower() # Don't incur the penalty of creating a list and then discarding it encodings = (enc.strip() for enc in tr_enc.split(",")) if "chunked" in encodings: @@ -246,7 +264,7 @@ class HTTPResponse(io.IOBase): location. ``False`` if not a redirect status code. """ if self.status in self.REDIRECT_STATUSES: - return self.headers.get('location') + return self.headers.get("location") return False @@ -285,18 +303,20 @@ class HTTPResponse(io.IOBase): """ Set initial length value for Response content if available. """ - length = self.headers.get('content-length') + length = self.headers.get("content-length") if length is not None: if self.chunked: # This Response will fail with an IncompleteRead if it can't be # received as chunked. This method falls back to attempt reading # the response before raising an exception. - log.warning("Received response with both Content-Length and " - "Transfer-Encoding set. This is expressly forbidden " - "by RFC 7230 sec 3.3.2. Ignoring Content-Length and " - "attempting to process response as Transfer-Encoding: " - "chunked.") + log.warning( + "Received response with both Content-Length and " + "Transfer-Encoding set. This is expressly forbidden " + "by RFC 7230 sec 3.3.2. Ignoring Content-Length and " + "attempting to process response as Transfer-Encoding: " + "chunked." + ) return None try: @@ -305,10 +325,12 @@ class HTTPResponse(io.IOBase): # (e.g. Content-Length: 42, 42). This line ensures the values # are all valid ints and that as long as the `set` length is 1, # all values are the same. Otherwise, the header is invalid. - lengths = set([int(val) for val in length.split(',')]) + lengths = set([int(val) for val in length.split(",")]) if len(lengths) > 1: - raise InvalidHeader("Content-Length contained multiple " - "unmatching values (%s)" % length) + raise InvalidHeader( + "Content-Length contained multiple " + "unmatching values (%s)" % length + ) length = lengths.pop() except ValueError: length = None @@ -324,7 +346,7 @@ class HTTPResponse(io.IOBase): status = 0 # Check for responses that shouldn't include a body - if status in (204, 304) or 100 <= status < 200 or request_method == 'HEAD': + if status in (204, 304) or 100 <= status < 200 or request_method == "HEAD": length = 0 return length @@ -335,14 +357,16 @@ class HTTPResponse(io.IOBase): """ # Note: content-encoding value should be case-insensitive, per RFC 7230 # Section 3.2 - content_encoding = self.headers.get('content-encoding', '').lower() + content_encoding = self.headers.get("content-encoding", "").lower() if self._decoder is None: if content_encoding in self.CONTENT_DECODERS: self._decoder = _get_decoder(content_encoding) - elif ',' in content_encoding: + elif "," in content_encoding: encodings = [ - e.strip() for e in content_encoding.split(',') - if e.strip() in self.CONTENT_DECODERS] + e.strip() + for e in content_encoding.split(",") + if e.strip() in self.CONTENT_DECODERS + ] if len(encodings): self._decoder = _get_decoder(content_encoding) @@ -361,10 +385,12 @@ class HTTPResponse(io.IOBase): if self._decoder: data = self._decoder.decompress(data) except self.DECODER_ERROR_CLASSES as e: - content_encoding = self.headers.get('content-encoding', '').lower() + content_encoding = self.headers.get("content-encoding", "").lower() raise DecodeError( "Received response with content-encoding: %s, but " - "failed to decode it." % content_encoding, e) + "failed to decode it." % content_encoding, + e, + ) if flush_decoder: data += self._flush_decoder() @@ -376,10 +402,10 @@ class HTTPResponse(io.IOBase): being used. """ if self._decoder: - buf = self._decoder.decompress(b'') + buf = self._decoder.decompress(b"") return buf + self._decoder.flush() - return b'' + return b"" @contextmanager def _error_catcher(self): @@ -399,20 +425,20 @@ class HTTPResponse(io.IOBase): except SocketTimeout: # FIXME: Ideally we'd like to include the url in the ReadTimeoutError but # there is yet no clean way to get at it from this context. - raise ReadTimeoutError(self._pool, None, 'Read timed out.') + raise ReadTimeoutError(self._pool, None, "Read timed out.") except BaseSSLError as e: # FIXME: Is there a better way to differentiate between SSLErrors? - if 'read operation timed out' not in str(e): # Defensive: + if "read operation timed out" not in str(e): # Defensive: # This shouldn't happen but just in case we're missing an edge # case, let's avoid swallowing SSL errors. raise - raise ReadTimeoutError(self._pool, None, 'Read timed out.') + raise ReadTimeoutError(self._pool, None, "Read timed out.") except (HTTPException, SocketError) as e: # This includes IncompleteRead. - raise ProtocolError('Connection broken: %r' % e, e) + raise ProtocolError("Connection broken: %r" % e, e) # If no exception is thrown, we should avoid cleaning up # unnecessarily. @@ -477,7 +503,9 @@ class HTTPResponse(io.IOBase): else: cache_content = False data = self._fp.read(amt) - if amt != 0 and not data: # Platform-specific: Buggy versions of Python. + if ( + amt != 0 and not data + ): # Platform-specific: Buggy versions of Python. # Close the connection when no data is returned # # This is redundant to what httplib/http.client _should_ @@ -487,7 +515,10 @@ class HTTPResponse(io.IOBase): # no harm in redundantly calling close. self._fp.close() flush_decoder = True - if self.enforce_content_length and self.length_remaining not in (0, None): + if self.enforce_content_length and self.length_remaining not in ( + 0, + None, + ): # This is an edge case that httplib failed to cover due # to concerns of backward compatibility. We're # addressing it here to make sure IncompleteRead is @@ -507,7 +538,7 @@ class HTTPResponse(io.IOBase): return data - def stream(self, amt=2**16, decode_content=None): + def stream(self, amt=2 ** 16, decode_content=None): """ A generator wrapper for the read() method. A call will block until ``amt`` bytes have been read from the connection or until the @@ -552,15 +583,17 @@ class HTTPResponse(io.IOBase): headers = HTTPHeaderDict.from_httplib(headers) # HTTPResponse objects in Python 3 don't have a .strict attribute - strict = getattr(r, 'strict', 0) - resp = ResponseCls(body=r, - headers=headers, - status=r.status, - version=r.version, - reason=r.reason, - strict=strict, - original_response=r, - **response_kw) + strict = getattr(r, "strict", 0) + resp = ResponseCls( + body=r, + headers=headers, + status=r.status, + version=r.version, + reason=r.reason, + strict=strict, + original_response=r, + **response_kw + ) return resp # Backwards-compatibility methods for httplib.HTTPResponse @@ -586,9 +619,9 @@ class HTTPResponse(io.IOBase): def closed(self): if self._fp is None: return True - elif hasattr(self._fp, 'isclosed'): + elif hasattr(self._fp, "isclosed"): return self._fp.isclosed() - elif hasattr(self._fp, 'closed'): + elif hasattr(self._fp, "closed"): return self._fp.closed else: return True @@ -599,11 +632,13 @@ class HTTPResponse(io.IOBase): elif hasattr(self._fp, "fileno"): return self._fp.fileno() else: - raise IOError("The file-like object this HTTPResponse is wrapped " - "around has no file descriptor") + raise IOError( + "The file-like object this HTTPResponse is wrapped " + "around has no file descriptor" + ) def flush(self): - if self._fp is not None and hasattr(self._fp, 'flush'): + if self._fp is not None and hasattr(self._fp, "flush"): return self._fp.flush() def readable(self): @@ -616,7 +651,7 @@ class HTTPResponse(io.IOBase): if len(temp) == 0: return 0 else: - b[:len(temp)] = temp + b[: len(temp)] = temp return len(temp) def supports_chunked_reads(self): @@ -626,7 +661,7 @@ class HTTPResponse(io.IOBase): attribute. If it is present we assume it returns raw chunks as processed by read_chunked(). """ - return hasattr(self._fp, 'fp') + return hasattr(self._fp, "fp") def _update_chunk_length(self): # First, we'll figure out length of a chunk and then @@ -634,7 +669,7 @@ class HTTPResponse(io.IOBase): if self.chunk_left is not None: return line = self._fp.fp.readline() - line = line.split(b';', 1)[0] + line = line.split(b";", 1)[0] try: self.chunk_left = int(line, 16) except ValueError: @@ -683,11 +718,13 @@ class HTTPResponse(io.IOBase): if not self.chunked: raise ResponseNotChunked( "Response is not chunked. " - "Header 'transfer-encoding: chunked' is missing.") + "Header 'transfer-encoding: chunked' is missing." + ) if not self.supports_chunked_reads(): raise BodyNotHttplibCompatible( "Body should be httplib.HTTPResponse like. " - "It should have have an fp attribute which returns raw chunks.") + "It should have have an fp attribute which returns raw chunks." + ) with self._error_catcher(): # Don't bother reading the body of a HEAD request. @@ -705,8 +742,9 @@ class HTTPResponse(io.IOBase): if self.chunk_left == 0: break chunk = self._handle_chunk(amt) - decoded = self._decode(chunk, decode_content=decode_content, - flush_decoder=False) + decoded = self._decode( + chunk, decode_content=decode_content, flush_decoder=False + ) if decoded: yield decoded @@ -724,7 +762,7 @@ class HTTPResponse(io.IOBase): if not line: # Some sites may not end with '\r\n'. break - if line == b'\r\n': + if line == b"\r\n": break # We read everything; close the "file". diff --git a/lib/urllib3/util/__init__.py b/lib/urllib3/util/__init__.py index 2914bb4..a96c73a 100644 --- a/lib/urllib3/util/__init__.py +++ b/lib/urllib3/util/__init__.py @@ -1,4 +1,5 @@ from __future__ import absolute_import + # For backwards compatibility, provide imports that used to be here. from .connection import is_connection_dropped from .request import make_headers @@ -14,43 +15,32 @@ from .ssl_ import ( ssl_wrap_socket, PROTOCOL_TLS, ) -from .timeout import ( - current_time, - Timeout, -) +from .timeout import current_time, Timeout from .retry import Retry -from .url import ( - get_host, - parse_url, - split_first, - Url, -) -from .wait import ( - wait_for_read, - wait_for_write -) +from .url import get_host, parse_url, split_first, Url +from .wait import wait_for_read, wait_for_write __all__ = ( - 'HAS_SNI', - 'IS_PYOPENSSL', - 'IS_SECURETRANSPORT', - 'SSLContext', - 'PROTOCOL_TLS', - 'Retry', - 'Timeout', - 'Url', - 'assert_fingerprint', - 'current_time', - 'is_connection_dropped', - 'is_fp_closed', - 'get_host', - 'parse_url', - 'make_headers', - 'resolve_cert_reqs', - 'resolve_ssl_version', - 'split_first', - 'ssl_wrap_socket', - 'wait_for_read', - 'wait_for_write' + "HAS_SNI", + "IS_PYOPENSSL", + "IS_SECURETRANSPORT", + "SSLContext", + "PROTOCOL_TLS", + "Retry", + "Timeout", + "Url", + "assert_fingerprint", + "current_time", + "is_connection_dropped", + "is_fp_closed", + "get_host", + "parse_url", + "make_headers", + "resolve_cert_reqs", + "resolve_ssl_version", + "split_first", + "ssl_wrap_socket", + "wait_for_read", + "wait_for_write", ) diff --git a/lib/urllib3/util/connection.py b/lib/urllib3/util/connection.py index 5ad70b2..0e11126 100644 --- a/lib/urllib3/util/connection.py +++ b/lib/urllib3/util/connection.py @@ -14,7 +14,7 @@ def is_connection_dropped(conn): # Platform-specific Note: For platforms like AppEngine, this will always return ``False`` to let the platform handle connection recycling transparently for us. """ - sock = getattr(conn, 'sock', False) + sock = getattr(conn, "sock", False) if sock is False: # Platform-specific: AppEngine return False if sock is None: # Connection already closed (such as by httplib). @@ -30,8 +30,12 @@ def is_connection_dropped(conn): # Platform-specific # library test suite. Added to its signature is only `socket_options`. # One additional modification is that we avoid binding to IPv6 servers # discovered in DNS if the system doesn't have IPv6 functionality. -def create_connection(address, timeout=socket._GLOBAL_DEFAULT_TIMEOUT, - source_address=None, socket_options=None): +def create_connection( + address, + timeout=socket._GLOBAL_DEFAULT_TIMEOUT, + source_address=None, + socket_options=None, +): """Connect to *address* and return the socket object. Convenience function. Connect to *address* (a 2-tuple ``(host, @@ -45,8 +49,8 @@ def create_connection(address, timeout=socket._GLOBAL_DEFAULT_TIMEOUT, """ host, port = address - if host.startswith('['): - host = host.strip('[]') + if host.startswith("["): + host = host.strip("[]") err = None # Using the value from allowed_gai_family() in the context of getaddrinfo lets @@ -131,4 +135,4 @@ def _has_ipv6(host): return has_ipv6 -HAS_IPV6 = _has_ipv6('::1') +HAS_IPV6 = _has_ipv6("::1") diff --git a/lib/urllib3/util/request.py b/lib/urllib3/util/request.py index 280b853..262a6d6 100644 --- a/lib/urllib3/util/request.py +++ b/lib/urllib3/util/request.py @@ -4,19 +4,25 @@ from base64 import b64encode from ..packages.six import b, integer_types from ..exceptions import UnrewindableBodyError -ACCEPT_ENCODING = 'gzip,deflate' +ACCEPT_ENCODING = "gzip,deflate" try: import brotli as _unused_module_brotli # noqa: F401 except ImportError: pass else: - ACCEPT_ENCODING += ',br' + ACCEPT_ENCODING += ",br" _FAILEDTELL = object() -def make_headers(keep_alive=None, accept_encoding=None, user_agent=None, - basic_auth=None, proxy_basic_auth=None, disable_cache=None): +def make_headers( + keep_alive=None, + accept_encoding=None, + user_agent=None, + basic_auth=None, + proxy_basic_auth=None, + disable_cache=None, +): """ Shortcuts for generating request headers. @@ -56,27 +62,27 @@ def make_headers(keep_alive=None, accept_encoding=None, user_agent=None, if isinstance(accept_encoding, str): pass elif isinstance(accept_encoding, list): - accept_encoding = ','.join(accept_encoding) + accept_encoding = ",".join(accept_encoding) else: accept_encoding = ACCEPT_ENCODING - headers['accept-encoding'] = accept_encoding + headers["accept-encoding"] = accept_encoding if user_agent: - headers['user-agent'] = user_agent + headers["user-agent"] = user_agent if keep_alive: - headers['connection'] = 'keep-alive' + headers["connection"] = "keep-alive" if basic_auth: - headers['authorization'] = 'Basic ' + \ - b64encode(b(basic_auth)).decode('utf-8') + headers["authorization"] = "Basic " + b64encode(b(basic_auth)).decode("utf-8") if proxy_basic_auth: - headers['proxy-authorization'] = 'Basic ' + \ - b64encode(b(proxy_basic_auth)).decode('utf-8') + headers["proxy-authorization"] = "Basic " + b64encode( + b(proxy_basic_auth) + ).decode("utf-8") if disable_cache: - headers['cache-control'] = 'no-cache' + headers["cache-control"] = "no-cache" return headers @@ -88,7 +94,7 @@ def set_file_position(body, pos): """ if pos is not None: rewind_body(body, pos) - elif getattr(body, 'tell', None) is not None: + elif getattr(body, "tell", None) is not None: try: pos = body.tell() except (IOError, OSError): @@ -110,16 +116,20 @@ def rewind_body(body, body_pos): :param int pos: Position to seek to in file. """ - body_seek = getattr(body, 'seek', None) + body_seek = getattr(body, "seek", None) if body_seek is not None and isinstance(body_pos, integer_types): try: body_seek(body_pos) except (IOError, OSError): - raise UnrewindableBodyError("An error occurred when rewinding request " - "body for redirect/retry.") + raise UnrewindableBodyError( + "An error occurred when rewinding request " "body for redirect/retry." + ) elif body_pos is _FAILEDTELL: - raise UnrewindableBodyError("Unable to record file position for rewinding " - "request body during a redirect/retry.") + raise UnrewindableBodyError( + "Unable to record file position for rewinding " + "request body during a redirect/retry." + ) else: - raise ValueError("body_pos must be of type integer, " - "instead it was %s." % type(body_pos)) + raise ValueError( + "body_pos must be of type integer, " "instead it was %s." % type(body_pos) + ) diff --git a/lib/urllib3/util/response.py b/lib/urllib3/util/response.py index 3d54864..715868d 100644 --- a/lib/urllib3/util/response.py +++ b/lib/urllib3/util/response.py @@ -52,11 +52,10 @@ def assert_header_parsing(headers): # This will fail silently if we pass in the wrong kind of parameter. # To make debugging easier add an explicit check. if not isinstance(headers, httplib.HTTPMessage): - raise TypeError('expected httplib.Message, got {0}.'.format( - type(headers))) + raise TypeError("expected httplib.Message, got {0}.".format(type(headers))) - defects = getattr(headers, 'defects', None) - get_payload = getattr(headers, 'get_payload', None) + defects = getattr(headers, "defects", None) + get_payload = getattr(headers, "get_payload", None) unparsed_data = None if get_payload: @@ -84,4 +83,4 @@ def is_response_to_head(response): method = response._method if isinstance(method, int): # Platform-specific: Appengine return method == 3 - return method.upper() == 'HEAD' + return method.upper() == "HEAD" diff --git a/lib/urllib3/util/retry.py b/lib/urllib3/util/retry.py index 02429ee..5a049fe 100644 --- a/lib/urllib3/util/retry.py +++ b/lib/urllib3/util/retry.py @@ -21,8 +21,9 @@ log = logging.getLogger(__name__) # Data structure for representing the metadata of requests that result in a retry. -RequestHistory = namedtuple('RequestHistory', ["method", "url", "error", - "status", "redirect_location"]) +RequestHistory = namedtuple( + "RequestHistory", ["method", "url", "error", "status", "redirect_location"] +) class Retry(object): @@ -146,21 +147,33 @@ class Retry(object): request. """ - DEFAULT_METHOD_WHITELIST = frozenset([ - 'HEAD', 'GET', 'PUT', 'DELETE', 'OPTIONS', 'TRACE']) + DEFAULT_METHOD_WHITELIST = frozenset( + ["HEAD", "GET", "PUT", "DELETE", "OPTIONS", "TRACE"] + ) RETRY_AFTER_STATUS_CODES = frozenset([413, 429, 503]) - DEFAULT_REDIRECT_HEADERS_BLACKLIST = frozenset(['Authorization']) + DEFAULT_REDIRECT_HEADERS_BLACKLIST = frozenset(["Authorization"]) #: Maximum backoff time. BACKOFF_MAX = 120 - def __init__(self, total=10, connect=None, read=None, redirect=None, status=None, - method_whitelist=DEFAULT_METHOD_WHITELIST, status_forcelist=None, - backoff_factor=0, raise_on_redirect=True, raise_on_status=True, - history=None, respect_retry_after_header=True, - remove_headers_on_redirect=DEFAULT_REDIRECT_HEADERS_BLACKLIST): + def __init__( + self, + total=10, + connect=None, + read=None, + redirect=None, + status=None, + method_whitelist=DEFAULT_METHOD_WHITELIST, + status_forcelist=None, + backoff_factor=0, + raise_on_redirect=True, + raise_on_status=True, + history=None, + respect_retry_after_header=True, + remove_headers_on_redirect=DEFAULT_REDIRECT_HEADERS_BLACKLIST, + ): self.total = total self.connect = connect @@ -179,20 +192,25 @@ class Retry(object): self.raise_on_status = raise_on_status self.history = history or tuple() self.respect_retry_after_header = respect_retry_after_header - self.remove_headers_on_redirect = frozenset([ - h.lower() for h in remove_headers_on_redirect]) + self.remove_headers_on_redirect = frozenset( + [h.lower() for h in remove_headers_on_redirect] + ) def new(self, **kw): params = dict( total=self.total, - connect=self.connect, read=self.read, redirect=self.redirect, status=self.status, + connect=self.connect, + read=self.read, + redirect=self.redirect, + status=self.status, method_whitelist=self.method_whitelist, status_forcelist=self.status_forcelist, backoff_factor=self.backoff_factor, raise_on_redirect=self.raise_on_redirect, raise_on_status=self.raise_on_status, history=self.history, - remove_headers_on_redirect=self.remove_headers_on_redirect + remove_headers_on_redirect=self.remove_headers_on_redirect, + respect_retry_after_header=self.respect_retry_after_header, ) params.update(kw) return type(self)(**params) @@ -217,8 +235,11 @@ class Retry(object): :rtype: float """ # We want to consider only the last consecutive errors sequence (Ignore redirects). - consecutive_errors_len = len(list(takewhile(lambda x: x.redirect_location is None, - reversed(self.history)))) + consecutive_errors_len = len( + list( + takewhile(lambda x: x.redirect_location is None, reversed(self.history)) + ) + ) if consecutive_errors_len <= 1: return 0 @@ -274,7 +295,7 @@ class Retry(object): this method will return immediately. """ - if response: + if self.respect_retry_after_header and response: slept = self.sleep_for_retry(response) if slept: return @@ -315,8 +336,12 @@ class Retry(object): if self.status_forcelist and status_code in self.status_forcelist: return True - return (self.total and self.respect_retry_after_header and - has_retry_after and (status_code in self.RETRY_AFTER_STATUS_CODES)) + return ( + self.total + and self.respect_retry_after_header + and has_retry_after + and (status_code in self.RETRY_AFTER_STATUS_CODES) + ) def is_exhausted(self): """ Are we out of retries? """ @@ -327,8 +352,15 @@ class Retry(object): return min(retry_counts) < 0 - def increment(self, method=None, url=None, response=None, error=None, - _pool=None, _stacktrace=None): + def increment( + self, + method=None, + url=None, + response=None, + error=None, + _pool=None, + _stacktrace=None, + ): """ Return a new Retry object with incremented retry counters. :param response: A response object, or None, if the server did not @@ -351,7 +383,7 @@ class Retry(object): read = self.read redirect = self.redirect status_count = self.status - cause = 'unknown' + cause = "unknown" status = None redirect_location = None @@ -373,7 +405,7 @@ class Retry(object): # Redirect retry? if redirect is not None: redirect -= 1 - cause = 'too many redirects' + cause = "too many redirects" redirect_location = response.get_redirect_location() status = response.status @@ -384,16 +416,21 @@ class Retry(object): if response and response.status: if status_count is not None: status_count -= 1 - cause = ResponseError.SPECIFIC_ERROR.format( - status_code=response.status) + cause = ResponseError.SPECIFIC_ERROR.format(status_code=response.status) status = response.status - history = self.history + (RequestHistory(method, url, error, status, redirect_location),) + history = self.history + ( + RequestHistory(method, url, error, status, redirect_location), + ) new_retry = self.new( total=total, - connect=connect, read=read, redirect=redirect, status=status_count, - history=history) + connect=connect, + read=read, + redirect=redirect, + status=status_count, + history=history, + ) if new_retry.is_exhausted(): raise MaxRetryError(_pool, url, error or ResponseError(cause)) @@ -403,9 +440,10 @@ class Retry(object): return new_retry def __repr__(self): - return ('{cls.__name__}(total={self.total}, connect={self.connect}, ' - 'read={self.read}, redirect={self.redirect}, status={self.status})').format( - cls=type(self), self=self) + return ( + "{cls.__name__}(total={self.total}, connect={self.connect}, " + "read={self.read}, redirect={self.redirect}, status={self.status})" + ).format(cls=type(self), self=self) # For backwards compatibility (equivalent to pre-v1.9): diff --git a/lib/urllib3/util/ssl_.py b/lib/urllib3/util/ssl_.py index f271ce9..2fd4ca5 100644 --- a/lib/urllib3/util/ssl_.py +++ b/lib/urllib3/util/ssl_.py @@ -18,11 +18,7 @@ IS_PYOPENSSL = False IS_SECURETRANSPORT = False # Maps the length of a digest to a possible hash function producing this digest -HASHFUNC_MAP = { - 32: md5, - 40: sha1, - 64: sha256, -} +HASHFUNC_MAP = {32: md5, 40: sha1, 64: sha256} def _const_compare_digest_backport(a, b): @@ -38,17 +34,13 @@ def _const_compare_digest_backport(a, b): return result == 0 -_const_compare_digest = getattr(hmac, 'compare_digest', - _const_compare_digest_backport) +_const_compare_digest = getattr(hmac, "compare_digest", _const_compare_digest_backport) # Borrow rfc3986's regular expressions for IPv4 # and IPv6 addresses for use in is_ipaddress() _IP_ADDRESS_REGEX = re.compile( - r'^(?:%s|%s|%s)$' % ( - abnf_regexp.IPv4_RE, - abnf_regexp.IPv6_RE, - abnf_regexp.IPv6_ADDRZ_RFC4007_RE - ) + r"^(?:%s|%s|%s)$" + % (abnf_regexp.IPv4_RE, abnf_regexp.IPv6_RE, abnf_regexp.IPv6_ADDRZ_RFC4007_RE) ) try: # Test for SSL features @@ -60,10 +52,12 @@ except ImportError: try: # Platform-specific: Python 3.6 from ssl import PROTOCOL_TLS + PROTOCOL_SSLv23 = PROTOCOL_TLS except ImportError: try: from ssl import PROTOCOL_SSLv23 as PROTOCOL_TLS + PROTOCOL_SSLv23 = PROTOCOL_TLS except ImportError: PROTOCOL_SSLv23 = PROTOCOL_TLS = 2 @@ -93,26 +87,29 @@ except ImportError: # insecure ciphers for security reasons. # - NOTE: TLS 1.3 cipher suites are managed through a different interface # not exposed by CPython (yet!) and are enabled by default if they're available. -DEFAULT_CIPHERS = ':'.join([ - 'ECDHE+AESGCM', - 'ECDHE+CHACHA20', - 'DHE+AESGCM', - 'DHE+CHACHA20', - 'ECDH+AESGCM', - 'DH+AESGCM', - 'ECDH+AES', - 'DH+AES', - 'RSA+AESGCM', - 'RSA+AES', - '!aNULL', - '!eNULL', - '!MD5', - '!DSS', -]) +DEFAULT_CIPHERS = ":".join( + [ + "ECDHE+AESGCM", + "ECDHE+CHACHA20", + "DHE+AESGCM", + "DHE+CHACHA20", + "ECDH+AESGCM", + "DH+AESGCM", + "ECDH+AES", + "DH+AES", + "RSA+AESGCM", + "RSA+AES", + "!aNULL", + "!eNULL", + "!MD5", + "!DSS", + ] +) try: from ssl import SSLContext # Modern SSL? except ImportError: + class SSLContext(object): # Platform-specific: Python 2 def __init__(self, protocol_version): self.protocol = protocol_version @@ -140,21 +137,21 @@ except ImportError: def wrap_socket(self, socket, server_hostname=None, server_side=False): warnings.warn( - 'A true SSLContext object is not available. This prevents ' - 'urllib3 from configuring SSL appropriately and may cause ' - 'certain SSL connections to fail. You can upgrade to a newer ' - 'version of Python to solve this. For more information, see ' - 'https://urllib3.readthedocs.io/en/latest/advanced-usage.html' - '#ssl-warnings', - InsecurePlatformWarning + "A true SSLContext object is not available. This prevents " + "urllib3 from configuring SSL appropriately and may cause " + "certain SSL connections to fail. You can upgrade to a newer " + "version of Python to solve this. For more information, see " + "https://urllib3.readthedocs.io/en/latest/advanced-usage.html" + "#ssl-warnings", + InsecurePlatformWarning, ) kwargs = { - 'keyfile': self.keyfile, - 'certfile': self.certfile, - 'ca_certs': self.ca_certs, - 'cert_reqs': self.verify_mode, - 'ssl_version': self.protocol, - 'server_side': server_side, + "keyfile": self.keyfile, + "certfile": self.certfile, + "ca_certs": self.ca_certs, + "cert_reqs": self.verify_mode, + "ssl_version": self.protocol, + "server_side": server_side, } return wrap_socket(socket, ciphers=self.ciphers, **kwargs) @@ -169,12 +166,11 @@ def assert_fingerprint(cert, fingerprint): Fingerprint as string of hexdigits, can be interspersed by colons. """ - fingerprint = fingerprint.replace(':', '').lower() + fingerprint = fingerprint.replace(":", "").lower() digest_length = len(fingerprint) hashfunc = HASHFUNC_MAP.get(digest_length) if not hashfunc: - raise SSLError( - 'Fingerprint of invalid length: {0}'.format(fingerprint)) + raise SSLError("Fingerprint of invalid length: {0}".format(fingerprint)) # We need encode() here for py32; works on py2 and p33. fingerprint_bytes = unhexlify(fingerprint.encode()) @@ -182,8 +178,11 @@ def assert_fingerprint(cert, fingerprint): cert_digest = hashfunc(cert).digest() if not _const_compare_digest(cert_digest, fingerprint_bytes): - raise SSLError('Fingerprints did not match. Expected "{0}", got "{1}".' - .format(fingerprint, hexlify(cert_digest))) + raise SSLError( + 'Fingerprints did not match. Expected "{0}", got "{1}".'.format( + fingerprint, hexlify(cert_digest) + ) + ) def resolve_cert_reqs(candidate): @@ -203,7 +202,7 @@ def resolve_cert_reqs(candidate): if isinstance(candidate, str): res = getattr(ssl, candidate, None) if res is None: - res = getattr(ssl, 'CERT_' + candidate) + res = getattr(ssl, "CERT_" + candidate) return res return candidate @@ -219,14 +218,15 @@ def resolve_ssl_version(candidate): if isinstance(candidate, str): res = getattr(ssl, candidate, None) if res is None: - res = getattr(ssl, 'PROTOCOL_' + candidate) + res = getattr(ssl, "PROTOCOL_" + candidate) return res return candidate -def create_urllib3_context(ssl_version=None, cert_reqs=None, - options=None, ciphers=None): +def create_urllib3_context( + ssl_version=None, cert_reqs=None, options=None, ciphers=None +): """All arguments have the same meaning as ``ssl_wrap_socket``. By default, this function does a lot of the same work that @@ -280,17 +280,28 @@ def create_urllib3_context(ssl_version=None, cert_reqs=None, context.options |= options context.verify_mode = cert_reqs - if getattr(context, 'check_hostname', None) is not None: # Platform-specific: Python 3.2 + if ( + getattr(context, "check_hostname", None) is not None + ): # Platform-specific: Python 3.2 # We do our own verification, including fingerprints and alternative # hostnames. So disable it here context.check_hostname = False return context -def ssl_wrap_socket(sock, keyfile=None, certfile=None, cert_reqs=None, - ca_certs=None, server_hostname=None, - ssl_version=None, ciphers=None, ssl_context=None, - ca_cert_dir=None, key_password=None): +def ssl_wrap_socket( + sock, + keyfile=None, + certfile=None, + cert_reqs=None, + ca_certs=None, + server_hostname=None, + ssl_version=None, + ciphers=None, + ssl_context=None, + ca_cert_dir=None, + key_password=None, +): """ All arguments except for server_hostname, ssl_context, and ca_cert_dir have the same meaning as they do when using :func:`ssl.wrap_socket`. @@ -314,8 +325,7 @@ def ssl_wrap_socket(sock, keyfile=None, certfile=None, cert_reqs=None, # Note: This branch of code and all the variables in it are no longer # used by urllib3 itself. We should consider deprecating and removing # this code. - context = create_urllib3_context(ssl_version, cert_reqs, - ciphers=ciphers) + context = create_urllib3_context(ssl_version, cert_reqs, ciphers=ciphers) if ca_certs or ca_cert_dir: try: @@ -329,7 +339,7 @@ def ssl_wrap_socket(sock, keyfile=None, certfile=None, cert_reqs=None, raise SSLError(e) raise - elif ssl_context is None and hasattr(context, 'load_default_certs'): + elif ssl_context is None and hasattr(context, "load_default_certs"): # try to load OS default certs; works well on Windows (require Python3.4+) context.load_default_certs() @@ -349,20 +359,21 @@ def ssl_wrap_socket(sock, keyfile=None, certfile=None, cert_reqs=None, # extension should not be used according to RFC3546 Section 3.1 # We shouldn't warn the user if SNI isn't available but we would # not be using SNI anyways due to IP address for server_hostname. - if ((server_hostname is not None and not is_ipaddress(server_hostname)) - or IS_SECURETRANSPORT): + if ( + server_hostname is not None and not is_ipaddress(server_hostname) + ) or IS_SECURETRANSPORT: if HAS_SNI and server_hostname is not None: return context.wrap_socket(sock, server_hostname=server_hostname) warnings.warn( - 'An HTTPS request has been made, but the SNI (Server Name ' - 'Indication) extension to TLS is not available on this platform. ' - 'This may cause the server to present an incorrect TLS ' - 'certificate, which can cause validation failures. You can upgrade to ' - 'a newer version of Python to solve this. For more information, see ' - 'https://urllib3.readthedocs.io/en/latest/advanced-usage.html' - '#ssl-warnings', - SNIMissingWarning + "An HTTPS request has been made, but the SNI (Server Name " + "Indication) extension to TLS is not available on this platform. " + "This may cause the server to present an incorrect TLS " + "certificate, which can cause validation failures. You can upgrade to " + "a newer version of Python to solve this. For more information, see " + "https://urllib3.readthedocs.io/en/latest/advanced-usage.html" + "#ssl-warnings", + SNIMissingWarning, ) return context.wrap_socket(sock) @@ -377,16 +388,16 @@ def is_ipaddress(hostname): """ if six.PY3 and isinstance(hostname, bytes): # IDN A-label bytes are ASCII compatible. - hostname = hostname.decode('ascii') + hostname = hostname.decode("ascii") return _IP_ADDRESS_REGEX.match(hostname) is not None def _is_key_file_encrypted(key_file): """Detects if a key file is encrypted or not.""" - with open(key_file, 'r') as f: + with open(key_file, "r") as f: for line in f: # Look for Proc-Type: 4,ENCRYPTED - if 'ENCRYPTED' in line: + if "ENCRYPTED" in line: return True return False diff --git a/lib/urllib3/util/timeout.py b/lib/urllib3/util/timeout.py index a4d004a..c1dc1e9 100644 --- a/lib/urllib3/util/timeout.py +++ b/lib/urllib3/util/timeout.py @@ -1,4 +1,5 @@ from __future__ import absolute_import + # The default socket timeout, used by httplib to indicate that no timeout was # specified by the user from socket import _GLOBAL_DEFAULT_TIMEOUT @@ -45,19 +46,20 @@ class Timeout(object): :type total: integer, float, or None :param connect: - The maximum amount of time to wait for a connection attempt to a server - to succeed. Omitting the parameter will default the connect timeout to - the system default, probably `the global default timeout in socket.py + The maximum amount of time (in seconds) to wait for a connection + attempt to a server to succeed. Omitting the parameter will default the + connect timeout to the system default, probably `the global default + timeout in socket.py `_. None will set an infinite timeout for connection attempts. :type connect: integer, float, or None :param read: - The maximum amount of time to wait between consecutive - read operations for a response from the server. Omitting - the parameter will default the read timeout to the system - default, probably `the global default timeout in socket.py + The maximum amount of time (in seconds) to wait between consecutive + read operations for a response from the server. Omitting the parameter + will default the read timeout to the system default, probably `the + global default timeout in socket.py `_. None will set an infinite timeout. @@ -91,14 +93,18 @@ class Timeout(object): DEFAULT_TIMEOUT = _GLOBAL_DEFAULT_TIMEOUT def __init__(self, total=None, connect=_Default, read=_Default): - self._connect = self._validate_timeout(connect, 'connect') - self._read = self._validate_timeout(read, 'read') - self.total = self._validate_timeout(total, 'total') + self._connect = self._validate_timeout(connect, "connect") + self._read = self._validate_timeout(read, "read") + self.total = self._validate_timeout(total, "total") self._start_connect = None def __str__(self): - return '%s(connect=%r, read=%r, total=%r)' % ( - type(self).__name__, self._connect, self._read, self.total) + return "%s(connect=%r, read=%r, total=%r)" % ( + type(self).__name__, + self._connect, + self._read, + self.total, + ) @classmethod def _validate_timeout(cls, value, name): @@ -118,23 +124,31 @@ class Timeout(object): return value if isinstance(value, bool): - raise ValueError("Timeout cannot be a boolean value. It must " - "be an int, float or None.") + raise ValueError( + "Timeout cannot be a boolean value. It must " + "be an int, float or None." + ) try: float(value) except (TypeError, ValueError): - raise ValueError("Timeout value %s was %s, but it must be an " - "int, float or None." % (name, value)) + raise ValueError( + "Timeout value %s was %s, but it must be an " + "int, float or None." % (name, value) + ) try: if value <= 0: - raise ValueError("Attempted to set %s timeout to %s, but the " - "timeout cannot be set to a value less " - "than or equal to 0." % (name, value)) + raise ValueError( + "Attempted to set %s timeout to %s, but the " + "timeout cannot be set to a value less " + "than or equal to 0." % (name, value) + ) except TypeError: # Python 3 - raise ValueError("Timeout value %s was %s, but it must be an " - "int, float or None." % (name, value)) + raise ValueError( + "Timeout value %s was %s, but it must be an " + "int, float or None." % (name, value) + ) return value @@ -166,8 +180,7 @@ class Timeout(object): # We can't use copy.deepcopy because that will also create a new object # for _GLOBAL_DEFAULT_TIMEOUT, which socket.py uses as a sentinel to # detect the user default. - return Timeout(connect=self._connect, read=self._read, - total=self.total) + return Timeout(connect=self._connect, read=self._read, total=self.total) def start_connect(self): """ Start the timeout clock, used during a connect() attempt @@ -183,14 +196,15 @@ class Timeout(object): def get_connect_duration(self): """ Gets the time elapsed since the call to :meth:`start_connect`. - :return: Elapsed time. + :return: Elapsed time in seconds. :rtype: float :raises urllib3.exceptions.TimeoutStateError: if you attempt to get duration for a timer that hasn't been started. """ if self._start_connect is None: - raise TimeoutStateError("Can't get connect duration for timer " - "that has not started.") + raise TimeoutStateError( + "Can't get connect duration for timer " "that has not started." + ) return current_time() - self._start_connect @property @@ -228,15 +242,16 @@ class Timeout(object): :raises urllib3.exceptions.TimeoutStateError: If :meth:`start_connect` has not yet been called on this object. """ - if (self.total is not None and - self.total is not self.DEFAULT_TIMEOUT and - self._read is not None and - self._read is not self.DEFAULT_TIMEOUT): + if ( + self.total is not None + and self.total is not self.DEFAULT_TIMEOUT + and self._read is not None + and self._read is not self.DEFAULT_TIMEOUT + ): # In case the connect timeout has not yet been established. if self._start_connect is None: return self._read - return max(0, min(self.total - self.get_connect_duration(), - self._read)) + return max(0, min(self.total - self.get_connect_duration(), self._read)) elif self.total is not None and self.total is not self.DEFAULT_TIMEOUT: return max(0, self.total - self.get_connect_duration()) else: diff --git a/lib/urllib3/util/url.py b/lib/urllib3/util/url.py index 0bc6ced..f225cd8 100644 --- a/lib/urllib3/util/url.py +++ b/lib/urllib3/util/url.py @@ -9,35 +9,47 @@ from ..packages.rfc3986.validators import Validator from ..packages.rfc3986 import abnf_regexp, normalizers, compat, misc -url_attrs = ['scheme', 'auth', 'host', 'port', 'path', 'query', 'fragment'] +url_attrs = ["scheme", "auth", "host", "port", "path", "query", "fragment"] # We only want to normalize urls with an HTTP(S) scheme. # urllib3 infers URLs without a scheme (None) to be http. -NORMALIZABLE_SCHEMES = ('http', 'https', None) +NORMALIZABLE_SCHEMES = ("http", "https", None) # Regex for detecting URLs with schemes. RFC 3986 Section 3.1 SCHEME_REGEX = re.compile(r"^(?:[a-zA-Z][a-zA-Z0-9+\-]*:|/)") -PATH_CHARS = abnf_regexp.UNRESERVED_CHARS_SET | abnf_regexp.SUB_DELIMITERS_SET | {':', '@', '/'} -QUERY_CHARS = FRAGMENT_CHARS = PATH_CHARS | {'?'} +PATH_CHARS = ( + abnf_regexp.UNRESERVED_CHARS_SET | abnf_regexp.SUB_DELIMITERS_SET | {":", "@", "/"} +) +QUERY_CHARS = FRAGMENT_CHARS = PATH_CHARS | {"?"} -class Url(namedtuple('Url', url_attrs)): +class Url(namedtuple("Url", url_attrs)): """ Data structure for representing an HTTP URL. Used as a return value for :func:`parse_url`. Both the scheme and host are normalized as they are both case-insensitive according to RFC 3986. """ + __slots__ = () - def __new__(cls, scheme=None, auth=None, host=None, port=None, path=None, - query=None, fragment=None): - if path and not path.startswith('/'): - path = '/' + path + def __new__( + cls, + scheme=None, + auth=None, + host=None, + port=None, + path=None, + query=None, + fragment=None, + ): + if path and not path.startswith("/"): + path = "/" + path if scheme is not None: scheme = scheme.lower() - return super(Url, cls).__new__(cls, scheme, auth, host, port, path, - query, fragment) + return super(Url, cls).__new__( + cls, scheme, auth, host, port, path, query, fragment + ) @property def hostname(self): @@ -47,10 +59,10 @@ class Url(namedtuple('Url', url_attrs)): @property def request_uri(self): """Absolute path including the query string.""" - uri = self.path or '/' + uri = self.path or "/" if self.query is not None: - uri += '?' + self.query + uri += "?" + self.query return uri @@ -58,7 +70,7 @@ class Url(namedtuple('Url', url_attrs)): def netloc(self): """Network location including host and port""" if self.port: - return '%s:%d' % (self.host, self.port) + return "%s:%d" % (self.host, self.port) return self.host @property @@ -81,23 +93,23 @@ class Url(namedtuple('Url', url_attrs)): 'http://username:password@host.com:80/path?query#fragment' """ scheme, auth, host, port, path, query, fragment = self - url = u'' + url = u"" # We use "is not None" we want things to happen with empty strings (or 0 port) if scheme is not None: - url += scheme + u'://' + url += scheme + u"://" if auth is not None: - url += auth + u'@' + url += auth + u"@" if host is not None: url += host if port is not None: - url += u':' + str(port) + url += u":" + str(port) if path is not None: url += path if query is not None: - url += u'?' + query + url += u"?" + query if fragment is not None: - url += u'#' + fragment + url += u"#" + fragment return url @@ -135,12 +147,12 @@ def split_first(s, delims): min_delim = d if min_idx is None or min_idx < 0: - return s, '', None + return s, "", None - return s[:min_idx], s[min_idx + 1:], min_delim + return s[:min_idx], s[min_idx + 1 :], min_delim -def _encode_invalid_chars(component, allowed_chars, encoding='utf-8'): +def _encode_invalid_chars(component, allowed_chars, encoding="utf-8"): """Percent-encodes a URI component without reapplying onto an already percent-encoded component. Based on rfc3986.normalizers.encode_component() @@ -150,23 +162,25 @@ def _encode_invalid_chars(component, allowed_chars, encoding='utf-8'): # Try to see if the component we're encoding is already percent-encoded # so we can skip all '%' characters but still encode all others. - percent_encodings = len(normalizers.PERCENT_MATCHER.findall( - compat.to_str(component, encoding))) + percent_encodings = len( + normalizers.PERCENT_MATCHER.findall(compat.to_str(component, encoding)) + ) - uri_bytes = component.encode('utf-8', 'surrogatepass') - is_percent_encoded = percent_encodings == uri_bytes.count(b'%') + uri_bytes = component.encode("utf-8", "surrogatepass") + is_percent_encoded = percent_encodings == uri_bytes.count(b"%") encoded_component = bytearray() for i in range(0, len(uri_bytes)): # Will return a single character bytestring on both Python 2 & 3 - byte = uri_bytes[i:i+1] + byte = uri_bytes[i : i + 1] byte_ord = ord(byte) - if ((is_percent_encoded and byte == b'%') - or (byte_ord < 128 and byte.decode() in allowed_chars)): + if (is_percent_encoded and byte == b"%") or ( + byte_ord < 128 and byte.decode() in allowed_chars + ): encoded_component.extend(byte) continue - encoded_component.extend('%{0:02x}'.format(byte_ord).encode().upper()) + encoded_component.extend("%{0:02x}".format(byte_ord).encode().upper()) return encoded_component.decode(encoding) @@ -209,7 +223,9 @@ def parse_url(url): try: import idna except ImportError: - raise LocationParseError("Unable to parse URL without the 'idna' module") + raise LocationParseError( + "Unable to parse URL without the 'idna' module" + ) try: return idna.encode(name.lower(), strict=True, std3_rules=True) except idna.IDNAError: @@ -219,10 +235,11 @@ def parse_url(url): try: split_iri = misc.IRI_MATCHER.match(compat.to_str(url)).groupdict() iri_ref = rfc3986.IRIReference( - split_iri['scheme'], split_iri['authority'], - _encode_invalid_chars(split_iri['path'], PATH_CHARS), - _encode_invalid_chars(split_iri['query'], QUERY_CHARS), - _encode_invalid_chars(split_iri['fragment'], FRAGMENT_CHARS) + split_iri["scheme"], + split_iri["authority"], + _encode_invalid_chars(split_iri["path"], PATH_CHARS), + _encode_invalid_chars(split_iri["query"], QUERY_CHARS), + _encode_invalid_chars(split_iri["fragment"], FRAGMENT_CHARS), ) has_authority = iri_ref.authority is not None uri_ref = iri_ref.encode(idna_encoder=idna_encode) @@ -243,9 +260,7 @@ def parse_url(url): # normalization has completed. validator = Validator() try: - validator.check_validity_of( - *validator.COMPONENT_NAMES - ).validate(uri_ref) + validator.check_validity_of(*validator.COMPONENT_NAMES).validate(uri_ref) except ValidationError: return six.raise_from(LocationParseError(url), None) @@ -255,8 +270,7 @@ def parse_url(url): # TODO: Remove this when we break backwards compatibility. path = uri_ref.path if not path: - if (uri_ref.query is not None - or uri_ref.fragment is not None): + if uri_ref.query is not None or uri_ref.fragment is not None: path = "" else: path = None @@ -267,7 +281,7 @@ def parse_url(url): if x is None: return None elif not is_string and not isinstance(x, six.binary_type): - return x.encode('utf-8') + return x.encode("utf-8") return x return Url( @@ -277,7 +291,7 @@ def parse_url(url): port=int(uri_ref.port) if uri_ref.port is not None else None, path=to_input_type(path), query=to_input_type(uri_ref.query), - fragment=to_input_type(uri_ref.fragment) + fragment=to_input_type(uri_ref.fragment), ) @@ -286,4 +300,4 @@ def get_host(url): Deprecated. Use :func:`parse_url` instead. """ p = parse_url(url) - return p.scheme or 'http', p.hostname, p.port + return p.scheme or "http", p.hostname, p.port diff --git a/lib/urllib3/util/wait.py b/lib/urllib3/util/wait.py index 4db71ba..d71d2fd 100644 --- a/lib/urllib3/util/wait.py +++ b/lib/urllib3/util/wait.py @@ -2,6 +2,7 @@ import errno from functools import partial import select import sys + try: from time import monotonic except ImportError: @@ -40,6 +41,8 @@ if sys.version_info >= (3, 5): # Modern Python, that retries syscalls by default def _retry_on_intr(fn, timeout): return fn(timeout) + + else: # Old and broken Pythons. def _retry_on_intr(fn, timeout):