Ruud
11 years ago
107 changed files with 8925 additions and 8425 deletions
@ -0,0 +1,8 @@ |
|||
If you are planning to submit a pull request to requests with any changes in |
|||
this library do not go any further. These are independent libraries which we |
|||
vendor into requests. Any changes necessary to these libraries must be made in |
|||
them and submitted as separate pull requests to those libraries. |
|||
|
|||
urllib3 pull requests go here: https://github.com/shazow/urllib3 |
|||
|
|||
chardet pull requests go here: https://github.com/chardet/chardet |
|||
@ -1,7 +0,0 @@ |
|||
''' |
|||
support ';python -m charade <file1> [file2] ...' package execution syntax (2.7+) |
|||
''' |
|||
|
|||
from charade import charade_cli |
|||
|
|||
charade_cli() |
|||
@ -0,0 +1,46 @@ |
|||
#!/usr/bin/env python |
|||
""" |
|||
Script which takes one or more file paths and reports on their detected |
|||
encodings |
|||
|
|||
Example:: |
|||
|
|||
% chardetect somefile someotherfile |
|||
somefile: windows-1252 with confidence 0.5 |
|||
someotherfile: ascii with confidence 1.0 |
|||
|
|||
If no paths are provided, it takes its input from stdin. |
|||
|
|||
""" |
|||
from io import open |
|||
from sys import argv, stdin |
|||
|
|||
from chardet.universaldetector import UniversalDetector |
|||
|
|||
|
|||
def description_of(file, name='stdin'): |
|||
"""Return a string describing the probable encoding of a file.""" |
|||
u = UniversalDetector() |
|||
for line in file: |
|||
u.feed(line) |
|||
u.close() |
|||
result = u.result |
|||
if result['encoding']: |
|||
return '%s: %s with confidence %s' % (name, |
|||
result['encoding'], |
|||
result['confidence']) |
|||
else: |
|||
return '%s: no result' % name |
|||
|
|||
|
|||
def main(): |
|||
if len(argv) <= 1: |
|||
print(description_of(stdin)) |
|||
else: |
|||
for path in argv[1:]: |
|||
with open(path, 'rb') as f: |
|||
print(description_of(f, path)) |
|||
|
|||
|
|||
if __name__ == '__main__': |
|||
main() |
|||
@ -1,643 +0,0 @@ |
|||
# urllib3/util.py |
|||
# Copyright 2008-2013 Andrey Petrov and contributors (see CONTRIBUTORS.txt) |
|||
# |
|||
# This module is part of urllib3 and is released under |
|||
# the MIT License: http://www.opensource.org/licenses/mit-license.php |
|||
|
|||
|
|||
from base64 import b64encode |
|||
from binascii import hexlify, unhexlify |
|||
from collections import namedtuple |
|||
from hashlib import md5, sha1 |
|||
from socket import error as SocketError, _GLOBAL_DEFAULT_TIMEOUT |
|||
import time |
|||
|
|||
try: |
|||
from select import poll, POLLIN |
|||
except ImportError: # `poll` doesn't exist on OSX and other platforms |
|||
poll = False |
|||
try: |
|||
from select import select |
|||
except ImportError: # `select` doesn't exist on AppEngine. |
|||
select = False |
|||
|
|||
try: # Test for SSL features |
|||
SSLContext = None |
|||
HAS_SNI = False |
|||
|
|||
import ssl |
|||
from ssl import wrap_socket, CERT_NONE, PROTOCOL_SSLv23 |
|||
from ssl import SSLContext # Modern SSL? |
|||
from ssl import HAS_SNI # Has SNI? |
|||
except ImportError: |
|||
pass |
|||
|
|||
from .packages import six |
|||
from .exceptions import LocationParseError, SSLError, TimeoutStateError |
|||
|
|||
|
|||
_Default = object() |
|||
# The default timeout to use for socket connections. This is the attribute used |
|||
# by httplib to define the default timeout |
|||
|
|||
|
|||
def current_time(): |
|||
""" |
|||
Retrieve the current time, this function is mocked out in unit testing. |
|||
""" |
|||
return time.time() |
|||
|
|||
|
|||
class Timeout(object): |
|||
""" |
|||
Utility object for storing timeout values. |
|||
|
|||
Example usage: |
|||
|
|||
.. code-block:: python |
|||
|
|||
timeout = urllib3.util.Timeout(connect=2.0, read=7.0) |
|||
pool = HTTPConnectionPool('www.google.com', 80, timeout=timeout) |
|||
pool.request(...) # Etc, etc |
|||
|
|||
:param connect: |
|||
The maximum amount of time to wait for a connection attempt to a server |
|||
to succeed. Omitting the parameter will default the connect timeout to |
|||
the system default, probably `the global default timeout in socket.py |
|||
<http://hg.python.org/cpython/file/603b4d593758/Lib/socket.py#l535>`_. |
|||
None will set an infinite timeout for connection attempts. |
|||
|
|||
:type connect: integer, float, or None |
|||
|
|||
:param read: |
|||
The maximum amount of time to wait between consecutive |
|||
read operations for a response from the server. Omitting |
|||
the parameter will default the read timeout to the system |
|||
default, probably `the global default timeout in socket.py |
|||
<http://hg.python.org/cpython/file/603b4d593758/Lib/socket.py#l535>`_. |
|||
None will set an infinite timeout. |
|||
|
|||
:type read: integer, float, or None |
|||
|
|||
:param total: |
|||
This combines the connect and read timeouts into one; the read timeout |
|||
will be set to the time leftover from the connect attempt. In the |
|||
event that both a connect timeout and a total are specified, or a read |
|||
timeout and a total are specified, the shorter timeout will be applied. |
|||
|
|||
Defaults to None. |
|||
|
|||
:type total: integer, float, or None |
|||
|
|||
.. note:: |
|||
|
|||
Many factors can affect the total amount of time for urllib3 to return |
|||
an HTTP response. Specifically, Python's DNS resolver does not obey the |
|||
timeout specified on the socket. Other factors that can affect total |
|||
request time include high CPU load, high swap, the program running at a |
|||
low priority level, or other behaviors. The observed running time for |
|||
urllib3 to return a response may be greater than the value passed to |
|||
`total`. |
|||
|
|||
In addition, the read and total timeouts only measure the time between |
|||
read operations on the socket connecting the client and the server, |
|||
not the total amount of time for the request to return a complete |
|||
response. For most requests, the timeout is raised because the server |
|||
has not sent the first byte in the specified time. This is not always |
|||
the case; if a server streams one byte every fifteen seconds, a timeout |
|||
of 20 seconds will not ever trigger, even though the request will |
|||
take several minutes to complete. |
|||
|
|||
If your goal is to cut off any request after a set amount of wall clock |
|||
time, consider having a second "watcher" thread to cut off a slow |
|||
request. |
|||
""" |
|||
|
|||
#: A sentinel object representing the default timeout value |
|||
DEFAULT_TIMEOUT = _GLOBAL_DEFAULT_TIMEOUT |
|||
|
|||
def __init__(self, total=None, connect=_Default, read=_Default): |
|||
self._connect = self._validate_timeout(connect, 'connect') |
|||
self._read = self._validate_timeout(read, 'read') |
|||
self.total = self._validate_timeout(total, 'total') |
|||
self._start_connect = None |
|||
|
|||
def __str__(self): |
|||
return '%s(connect=%r, read=%r, total=%r)' % ( |
|||
type(self).__name__, self._connect, self._read, self.total) |
|||
|
|||
|
|||
@classmethod |
|||
def _validate_timeout(cls, value, name): |
|||
""" Check that a timeout attribute is valid |
|||
|
|||
:param value: The timeout value to validate |
|||
:param name: The name of the timeout attribute to validate. This is used |
|||
for clear error messages |
|||
:return: the value |
|||
:raises ValueError: if the type is not an integer or a float, or if it |
|||
is a numeric value less than zero |
|||
""" |
|||
if value is _Default: |
|||
return cls.DEFAULT_TIMEOUT |
|||
|
|||
if value is None or value is cls.DEFAULT_TIMEOUT: |
|||
return value |
|||
|
|||
try: |
|||
float(value) |
|||
except (TypeError, ValueError): |
|||
raise ValueError("Timeout value %s was %s, but it must be an " |
|||
"int or float." % (name, value)) |
|||
|
|||
try: |
|||
if value < 0: |
|||
raise ValueError("Attempted to set %s timeout to %s, but the " |
|||
"timeout cannot be set to a value less " |
|||
"than 0." % (name, value)) |
|||
except TypeError: # Python 3 |
|||
raise ValueError("Timeout value %s was %s, but it must be an " |
|||
"int or float." % (name, value)) |
|||
|
|||
return value |
|||
|
|||
@classmethod |
|||
def from_float(cls, timeout): |
|||
""" Create a new Timeout from a legacy timeout value. |
|||
|
|||
The timeout value used by httplib.py sets the same timeout on the |
|||
connect(), and recv() socket requests. This creates a :class:`Timeout` |
|||
object that sets the individual timeouts to the ``timeout`` value passed |
|||
to this function. |
|||
|
|||
:param timeout: The legacy timeout value |
|||
:type timeout: integer, float, sentinel default object, or None |
|||
:return: a Timeout object |
|||
:rtype: :class:`Timeout` |
|||
""" |
|||
return Timeout(read=timeout, connect=timeout) |
|||
|
|||
def clone(self): |
|||
""" Create a copy of the timeout object |
|||
|
|||
Timeout properties are stored per-pool but each request needs a fresh |
|||
Timeout object to ensure each one has its own start/stop configured. |
|||
|
|||
:return: a copy of the timeout object |
|||
:rtype: :class:`Timeout` |
|||
""" |
|||
# We can't use copy.deepcopy because that will also create a new object |
|||
# for _GLOBAL_DEFAULT_TIMEOUT, which socket.py uses as a sentinel to |
|||
# detect the user default. |
|||
return Timeout(connect=self._connect, read=self._read, |
|||
total=self.total) |
|||
|
|||
def start_connect(self): |
|||
""" Start the timeout clock, used during a connect() attempt |
|||
|
|||
:raises urllib3.exceptions.TimeoutStateError: if you attempt |
|||
to start a timer that has been started already. |
|||
""" |
|||
if self._start_connect is not None: |
|||
raise TimeoutStateError("Timeout timer has already been started.") |
|||
self._start_connect = current_time() |
|||
return self._start_connect |
|||
|
|||
def get_connect_duration(self): |
|||
""" Gets the time elapsed since the call to :meth:`start_connect`. |
|||
|
|||
:return: the elapsed time |
|||
:rtype: float |
|||
:raises urllib3.exceptions.TimeoutStateError: if you attempt |
|||
to get duration for a timer that hasn't been started. |
|||
""" |
|||
if self._start_connect is None: |
|||
raise TimeoutStateError("Can't get connect duration for timer " |
|||
"that has not started.") |
|||
return current_time() - self._start_connect |
|||
|
|||
@property |
|||
def connect_timeout(self): |
|||
""" Get the value to use when setting a connection timeout. |
|||
|
|||
This will be a positive float or integer, the value None |
|||
(never timeout), or the default system timeout. |
|||
|
|||
:return: the connect timeout |
|||
:rtype: int, float, :attr:`Timeout.DEFAULT_TIMEOUT` or None |
|||
""" |
|||
if self.total is None: |
|||
return self._connect |
|||
|
|||
if self._connect is None or self._connect is self.DEFAULT_TIMEOUT: |
|||
return self.total |
|||
|
|||
return min(self._connect, self.total) |
|||
|
|||
@property |
|||
def read_timeout(self): |
|||
""" Get the value for the read timeout. |
|||
|
|||
This assumes some time has elapsed in the connection timeout and |
|||
computes the read timeout appropriately. |
|||
|
|||
If self.total is set, the read timeout is dependent on the amount of |
|||
time taken by the connect timeout. If the connection time has not been |
|||
established, a :exc:`~urllib3.exceptions.TimeoutStateError` will be |
|||
raised. |
|||
|
|||
:return: the value to use for the read timeout |
|||
:rtype: int, float, :attr:`Timeout.DEFAULT_TIMEOUT` or None |
|||
:raises urllib3.exceptions.TimeoutStateError: If :meth:`start_connect` |
|||
has not yet been called on this object. |
|||
""" |
|||
if (self.total is not None and |
|||
self.total is not self.DEFAULT_TIMEOUT and |
|||
self._read is not None and |
|||
self._read is not self.DEFAULT_TIMEOUT): |
|||
# in case the connect timeout has not yet been established. |
|||
if self._start_connect is None: |
|||
return self._read |
|||
return max(0, min(self.total - self.get_connect_duration(), |
|||
self._read)) |
|||
elif self.total is not None and self.total is not self.DEFAULT_TIMEOUT: |
|||
return max(0, self.total - self.get_connect_duration()) |
|||
else: |
|||
return self._read |
|||
|
|||
|
|||
class Url(namedtuple('Url', ['scheme', 'auth', 'host', 'port', 'path', 'query', 'fragment'])): |
|||
""" |
|||
Datastructure for representing an HTTP URL. Used as a return value for |
|||
:func:`parse_url`. |
|||
""" |
|||
slots = () |
|||
|
|||
def __new__(cls, scheme=None, auth=None, host=None, port=None, path=None, query=None, fragment=None): |
|||
return super(Url, cls).__new__(cls, scheme, auth, host, port, path, query, fragment) |
|||
|
|||
@property |
|||
def hostname(self): |
|||
"""For backwards-compatibility with urlparse. We're nice like that.""" |
|||
return self.host |
|||
|
|||
@property |
|||
def request_uri(self): |
|||
"""Absolute path including the query string.""" |
|||
uri = self.path or '/' |
|||
|
|||
if self.query is not None: |
|||
uri += '?' + self.query |
|||
|
|||
return uri |
|||
|
|||
@property |
|||
def netloc(self): |
|||
"""Network location including host and port""" |
|||
if self.port: |
|||
return '%s:%d' % (self.host, self.port) |
|||
return self.host |
|||
|
|||
|
|||
def split_first(s, delims): |
|||
""" |
|||
Given a string and an iterable of delimiters, split on the first found |
|||
delimiter. Return two split parts and the matched delimiter. |
|||
|
|||
If not found, then the first part is the full input string. |
|||
|
|||
Example: :: |
|||
|
|||
>>> split_first('foo/bar?baz', '?/=') |
|||
('foo', 'bar?baz', '/') |
|||
>>> split_first('foo/bar?baz', '123') |
|||
('foo/bar?baz', '', None) |
|||
|
|||
Scales linearly with number of delims. Not ideal for large number of delims. |
|||
""" |
|||
min_idx = None |
|||
min_delim = None |
|||
for d in delims: |
|||
idx = s.find(d) |
|||
if idx < 0: |
|||
continue |
|||
|
|||
if min_idx is None or idx < min_idx: |
|||
min_idx = idx |
|||
min_delim = d |
|||
|
|||
if min_idx is None or min_idx < 0: |
|||
return s, '', None |
|||
|
|||
return s[:min_idx], s[min_idx+1:], min_delim |
|||
|
|||
|
|||
def parse_url(url): |
|||
""" |
|||
Given a url, return a parsed :class:`.Url` namedtuple. Best-effort is |
|||
performed to parse incomplete urls. Fields not provided will be None. |
|||
|
|||
Partly backwards-compatible with :mod:`urlparse`. |
|||
|
|||
Example: :: |
|||
|
|||
>>> parse_url('http://google.com/mail/') |
|||
Url(scheme='http', host='google.com', port=None, path='/', ...) |
|||
>>> parse_url('google.com:80') |
|||
Url(scheme=None, host='google.com', port=80, path=None, ...) |
|||
>>> parse_url('/foo?bar') |
|||
Url(scheme=None, host=None, port=None, path='/foo', query='bar', ...) |
|||
""" |
|||
|
|||
# While this code has overlap with stdlib's urlparse, it is much |
|||
# simplified for our needs and less annoying. |
|||
# Additionally, this implementations does silly things to be optimal |
|||
# on CPython. |
|||
|
|||
scheme = None |
|||
auth = None |
|||
host = None |
|||
port = None |
|||
path = None |
|||
fragment = None |
|||
query = None |
|||
|
|||
# Scheme |
|||
if '://' in url: |
|||
scheme, url = url.split('://', 1) |
|||
|
|||
# Find the earliest Authority Terminator |
|||
# (http://tools.ietf.org/html/rfc3986#section-3.2) |
|||
url, path_, delim = split_first(url, ['/', '?', '#']) |
|||
|
|||
if delim: |
|||
# Reassemble the path |
|||
path = delim + path_ |
|||
|
|||
# Auth |
|||
if '@' in url: |
|||
# Last '@' denotes end of auth part |
|||
auth, url = url.rsplit('@', 1) |
|||
|
|||
# IPv6 |
|||
if url and url[0] == '[': |
|||
host, url = url.split(']', 1) |
|||
host += ']' |
|||
|
|||
# Port |
|||
if ':' in url: |
|||
_host, port = url.split(':', 1) |
|||
|
|||
if not host: |
|||
host = _host |
|||
|
|||
if port: |
|||
# If given, ports must be integers. |
|||
if not port.isdigit(): |
|||
raise LocationParseError("Failed to parse: %s" % url) |
|||
port = int(port) |
|||
else: |
|||
# Blank ports are cool, too. (rfc3986#section-3.2.3) |
|||
port = None |
|||
|
|||
elif not host and url: |
|||
host = url |
|||
|
|||
if not path: |
|||
return Url(scheme, auth, host, port, path, query, fragment) |
|||
|
|||
# Fragment |
|||
if '#' in path: |
|||
path, fragment = path.split('#', 1) |
|||
|
|||
# Query |
|||
if '?' in path: |
|||
path, query = path.split('?', 1) |
|||
|
|||
return Url(scheme, auth, host, port, path, query, fragment) |
|||
|
|||
|
|||
def get_host(url): |
|||
""" |
|||
Deprecated. Use :func:`.parse_url` instead. |
|||
""" |
|||
p = parse_url(url) |
|||
return p.scheme or 'http', p.hostname, p.port |
|||
|
|||
|
|||
def make_headers(keep_alive=None, accept_encoding=None, user_agent=None, |
|||
basic_auth=None, proxy_basic_auth=None): |
|||
""" |
|||
Shortcuts for generating request headers. |
|||
|
|||
:param keep_alive: |
|||
If ``True``, adds 'connection: keep-alive' header. |
|||
|
|||
:param accept_encoding: |
|||
Can be a boolean, list, or string. |
|||
``True`` translates to 'gzip,deflate'. |
|||
List will get joined by comma. |
|||
String will be used as provided. |
|||
|
|||
:param user_agent: |
|||
String representing the user-agent you want, such as |
|||
"python-urllib3/0.6" |
|||
|
|||
:param basic_auth: |
|||
Colon-separated username:password string for 'authorization: basic ...' |
|||
auth header. |
|||
|
|||
:param proxy_basic_auth: |
|||
Colon-separated username:password string for 'proxy-authorization: basic ...' |
|||
auth header. |
|||
|
|||
Example: :: |
|||
|
|||
>>> make_headers(keep_alive=True, user_agent="Batman/1.0") |
|||
{'connection': 'keep-alive', 'user-agent': 'Batman/1.0'} |
|||
>>> make_headers(accept_encoding=True) |
|||
{'accept-encoding': 'gzip,deflate'} |
|||
""" |
|||
headers = {} |
|||
if accept_encoding: |
|||
if isinstance(accept_encoding, str): |
|||
pass |
|||
elif isinstance(accept_encoding, list): |
|||
accept_encoding = ','.join(accept_encoding) |
|||
else: |
|||
accept_encoding = 'gzip,deflate' |
|||
headers['accept-encoding'] = accept_encoding |
|||
|
|||
if user_agent: |
|||
headers['user-agent'] = user_agent |
|||
|
|||
if keep_alive: |
|||
headers['connection'] = 'keep-alive' |
|||
|
|||
if basic_auth: |
|||
headers['authorization'] = 'Basic ' + \ |
|||
b64encode(six.b(basic_auth)).decode('utf-8') |
|||
|
|||
if proxy_basic_auth: |
|||
headers['proxy-authorization'] = 'Basic ' + \ |
|||
b64encode(six.b(proxy_basic_auth)).decode('utf-8') |
|||
|
|||
return headers |
|||
|
|||
|
|||
def is_connection_dropped(conn): # Platform-specific |
|||
""" |
|||
Returns True if the connection is dropped and should be closed. |
|||
|
|||
:param conn: |
|||
:class:`httplib.HTTPConnection` object. |
|||
|
|||
Note: For platforms like AppEngine, this will always return ``False`` to |
|||
let the platform handle connection recycling transparently for us. |
|||
""" |
|||
sock = getattr(conn, 'sock', False) |
|||
if not sock: # Platform-specific: AppEngine |
|||
return False |
|||
|
|||
if not poll: |
|||
if not select: # Platform-specific: AppEngine |
|||
return False |
|||
|
|||
try: |
|||
return select([sock], [], [], 0.0)[0] |
|||
except SocketError: |
|||
return True |
|||
|
|||
# This version is better on platforms that support it. |
|||
p = poll() |
|||
p.register(sock, POLLIN) |
|||
for (fno, ev) in p.poll(0.0): |
|||
if fno == sock.fileno(): |
|||
# Either data is buffered (bad), or the connection is dropped. |
|||
return True |
|||
|
|||
|
|||
def resolve_cert_reqs(candidate): |
|||
""" |
|||
Resolves the argument to a numeric constant, which can be passed to |
|||
the wrap_socket function/method from the ssl module. |
|||
Defaults to :data:`ssl.CERT_NONE`. |
|||
If given a string it is assumed to be the name of the constant in the |
|||
:mod:`ssl` module or its abbrevation. |
|||
(So you can specify `REQUIRED` instead of `CERT_REQUIRED`. |
|||
If it's neither `None` nor a string we assume it is already the numeric |
|||
constant which can directly be passed to wrap_socket. |
|||
""" |
|||
if candidate is None: |
|||
return CERT_NONE |
|||
|
|||
if isinstance(candidate, str): |
|||
res = getattr(ssl, candidate, None) |
|||
if res is None: |
|||
res = getattr(ssl, 'CERT_' + candidate) |
|||
return res |
|||
|
|||
return candidate |
|||
|
|||
|
|||
def resolve_ssl_version(candidate): |
|||
""" |
|||
like resolve_cert_reqs |
|||
""" |
|||
if candidate is None: |
|||
return PROTOCOL_SSLv23 |
|||
|
|||
if isinstance(candidate, str): |
|||
res = getattr(ssl, candidate, None) |
|||
if res is None: |
|||
res = getattr(ssl, 'PROTOCOL_' + candidate) |
|||
return res |
|||
|
|||
return candidate |
|||
|
|||
|
|||
def assert_fingerprint(cert, fingerprint): |
|||
""" |
|||
Checks if given fingerprint matches the supplied certificate. |
|||
|
|||
:param cert: |
|||
Certificate as bytes object. |
|||
:param fingerprint: |
|||
Fingerprint as string of hexdigits, can be interspersed by colons. |
|||
""" |
|||
|
|||
# Maps the length of a digest to a possible hash function producing |
|||
# this digest. |
|||
hashfunc_map = { |
|||
16: md5, |
|||
20: sha1 |
|||
} |
|||
|
|||
fingerprint = fingerprint.replace(':', '').lower() |
|||
|
|||
digest_length, rest = divmod(len(fingerprint), 2) |
|||
|
|||
if rest or digest_length not in hashfunc_map: |
|||
raise SSLError('Fingerprint is of invalid length.') |
|||
|
|||
# We need encode() here for py32; works on py2 and p33. |
|||
fingerprint_bytes = unhexlify(fingerprint.encode()) |
|||
|
|||
hashfunc = hashfunc_map[digest_length] |
|||
|
|||
cert_digest = hashfunc(cert).digest() |
|||
|
|||
if not cert_digest == fingerprint_bytes: |
|||
raise SSLError('Fingerprints did not match. Expected "{0}", got "{1}".' |
|||
.format(hexlify(fingerprint_bytes), |
|||
hexlify(cert_digest))) |
|||
|
|||
def is_fp_closed(obj): |
|||
""" |
|||
Checks whether a given file-like object is closed. |
|||
|
|||
:param obj: |
|||
The file-like object to check. |
|||
""" |
|||
if hasattr(obj, 'fp'): |
|||
# Object is a container for another file-like object that gets released |
|||
# on exhaustion (e.g. HTTPResponse) |
|||
return obj.fp is None |
|||
|
|||
return obj.closed |
|||
|
|||
|
|||
if SSLContext is not None: # Python 3.2+ |
|||
def ssl_wrap_socket(sock, keyfile=None, certfile=None, cert_reqs=None, |
|||
ca_certs=None, server_hostname=None, |
|||
ssl_version=None): |
|||
""" |
|||
All arguments except `server_hostname` have the same meaning as for |
|||
:func:`ssl.wrap_socket` |
|||
|
|||
:param server_hostname: |
|||
Hostname of the expected certificate |
|||
""" |
|||
context = SSLContext(ssl_version) |
|||
context.verify_mode = cert_reqs |
|||
if ca_certs: |
|||
try: |
|||
context.load_verify_locations(ca_certs) |
|||
# Py32 raises IOError |
|||
# Py33 raises FileNotFoundError |
|||
except Exception as e: # Reraise as SSLError |
|||
raise SSLError(e) |
|||
if certfile: |
|||
# FIXME: This block needs a test. |
|||
context.load_cert_chain(certfile, keyfile) |
|||
if HAS_SNI: # Platform-specific: OpenSSL with enabled SNI |
|||
return context.wrap_socket(sock, server_hostname=server_hostname) |
|||
return context.wrap_socket(sock) |
|||
|
|||
else: # Python 3.1 and earlier |
|||
def ssl_wrap_socket(sock, keyfile=None, certfile=None, cert_reqs=None, |
|||
ca_certs=None, server_hostname=None, |
|||
ssl_version=None): |
|||
return wrap_socket(sock, keyfile=keyfile, certfile=certfile, |
|||
ca_certs=ca_certs, cert_reqs=cert_reqs, |
|||
ssl_version=ssl_version) |
|||
@ -0,0 +1,27 @@ |
|||
# urllib3/util/__init__.py |
|||
# Copyright 2008-2014 Andrey Petrov and contributors (see CONTRIBUTORS.txt) |
|||
# |
|||
# This module is part of urllib3 and is released under |
|||
# the MIT License: http://www.opensource.org/licenses/mit-license.php |
|||
|
|||
from .connection import is_connection_dropped |
|||
from .request import make_headers |
|||
from .response import is_fp_closed |
|||
from .ssl_ import ( |
|||
SSLContext, |
|||
HAS_SNI, |
|||
assert_fingerprint, |
|||
resolve_cert_reqs, |
|||
resolve_ssl_version, |
|||
ssl_wrap_socket, |
|||
) |
|||
from .timeout import ( |
|||
current_time, |
|||
Timeout, |
|||
) |
|||
from .url import ( |
|||
get_host, |
|||
parse_url, |
|||
split_first, |
|||
Url, |
|||
) |
|||
@ -0,0 +1,45 @@ |
|||
from socket import error as SocketError |
|||
try: |
|||
from select import poll, POLLIN |
|||
except ImportError: # `poll` doesn't exist on OSX and other platforms |
|||
poll = False |
|||
try: |
|||
from select import select |
|||
except ImportError: # `select` doesn't exist on AppEngine. |
|||
select = False |
|||
|
|||
def is_connection_dropped(conn): # Platform-specific |
|||
""" |
|||
Returns True if the connection is dropped and should be closed. |
|||
|
|||
:param conn: |
|||
:class:`httplib.HTTPConnection` object. |
|||
|
|||
Note: For platforms like AppEngine, this will always return ``False`` to |
|||
let the platform handle connection recycling transparently for us. |
|||
""" |
|||
sock = getattr(conn, 'sock', False) |
|||
if sock is False: # Platform-specific: AppEngine |
|||
return False |
|||
if sock is None: # Connection already closed (such as by httplib). |
|||
return False |
|||
|
|||
if not poll: |
|||
if not select: # Platform-specific: AppEngine |
|||
return False |
|||
|
|||
try: |
|||
return select([sock], [], [], 0.0)[0] |
|||
except SocketError: |
|||
return True |
|||
|
|||
# This version is better on platforms that support it. |
|||
p = poll() |
|||
p.register(sock, POLLIN) |
|||
for (fno, ev) in p.poll(0.0): |
|||
if fno == sock.fileno(): |
|||
# Either data is buffered (bad), or the connection is dropped. |
|||
return True |
|||
|
|||
|
|||
|
|||
@ -0,0 +1,68 @@ |
|||
from base64 import b64encode |
|||
|
|||
from ..packages import six |
|||
|
|||
|
|||
ACCEPT_ENCODING = 'gzip,deflate' |
|||
|
|||
|
|||
def make_headers(keep_alive=None, accept_encoding=None, user_agent=None, |
|||
basic_auth=None, proxy_basic_auth=None): |
|||
""" |
|||
Shortcuts for generating request headers. |
|||
|
|||
:param keep_alive: |
|||
If ``True``, adds 'connection: keep-alive' header. |
|||
|
|||
:param accept_encoding: |
|||
Can be a boolean, list, or string. |
|||
``True`` translates to 'gzip,deflate'. |
|||
List will get joined by comma. |
|||
String will be used as provided. |
|||
|
|||
:param user_agent: |
|||
String representing the user-agent you want, such as |
|||
"python-urllib3/0.6" |
|||
|
|||
:param basic_auth: |
|||
Colon-separated username:password string for 'authorization: basic ...' |
|||
auth header. |
|||
|
|||
:param proxy_basic_auth: |
|||
Colon-separated username:password string for 'proxy-authorization: basic ...' |
|||
auth header. |
|||
|
|||
Example: :: |
|||
|
|||
>>> make_headers(keep_alive=True, user_agent="Batman/1.0") |
|||
{'connection': 'keep-alive', 'user-agent': 'Batman/1.0'} |
|||
>>> make_headers(accept_encoding=True) |
|||
{'accept-encoding': 'gzip,deflate'} |
|||
""" |
|||
headers = {} |
|||
if accept_encoding: |
|||
if isinstance(accept_encoding, str): |
|||
pass |
|||
elif isinstance(accept_encoding, list): |
|||
accept_encoding = ','.join(accept_encoding) |
|||
else: |
|||
accept_encoding = ACCEPT_ENCODING |
|||
headers['accept-encoding'] = accept_encoding |
|||
|
|||
if user_agent: |
|||
headers['user-agent'] = user_agent |
|||
|
|||
if keep_alive: |
|||
headers['connection'] = 'keep-alive' |
|||
|
|||
if basic_auth: |
|||
headers['authorization'] = 'Basic ' + \ |
|||
b64encode(six.b(basic_auth)).decode('utf-8') |
|||
|
|||
if proxy_basic_auth: |
|||
headers['proxy-authorization'] = 'Basic ' + \ |
|||
b64encode(six.b(proxy_basic_auth)).decode('utf-8') |
|||
|
|||
return headers |
|||
|
|||
|
|||
@ -0,0 +1,13 @@ |
|||
def is_fp_closed(obj): |
|||
""" |
|||
Checks whether a given file-like object is closed. |
|||
|
|||
:param obj: |
|||
The file-like object to check. |
|||
""" |
|||
if hasattr(obj, 'fp'): |
|||
# Object is a container for another file-like object that gets released |
|||
# on exhaustion (e.g. HTTPResponse) |
|||
return obj.fp is None |
|||
|
|||
return obj.closed |
|||
@ -0,0 +1,133 @@ |
|||
from binascii import hexlify, unhexlify |
|||
from hashlib import md5, sha1 |
|||
|
|||
from ..exceptions import SSLError |
|||
|
|||
|
|||
try: # Test for SSL features |
|||
SSLContext = None |
|||
HAS_SNI = False |
|||
|
|||
import ssl |
|||
from ssl import wrap_socket, CERT_NONE, PROTOCOL_SSLv23 |
|||
from ssl import SSLContext # Modern SSL? |
|||
from ssl import HAS_SNI # Has SNI? |
|||
except ImportError: |
|||
pass |
|||
|
|||
|
|||
def assert_fingerprint(cert, fingerprint): |
|||
""" |
|||
Checks if given fingerprint matches the supplied certificate. |
|||
|
|||
:param cert: |
|||
Certificate as bytes object. |
|||
:param fingerprint: |
|||
Fingerprint as string of hexdigits, can be interspersed by colons. |
|||
""" |
|||
|
|||
# Maps the length of a digest to a possible hash function producing |
|||
# this digest. |
|||
hashfunc_map = { |
|||
16: md5, |
|||
20: sha1 |
|||
} |
|||
|
|||
fingerprint = fingerprint.replace(':', '').lower() |
|||
|
|||
digest_length, rest = divmod(len(fingerprint), 2) |
|||
|
|||
if rest or digest_length not in hashfunc_map: |
|||
raise SSLError('Fingerprint is of invalid length.') |
|||
|
|||
# We need encode() here for py32; works on py2 and p33. |
|||
fingerprint_bytes = unhexlify(fingerprint.encode()) |
|||
|
|||
hashfunc = hashfunc_map[digest_length] |
|||
|
|||
cert_digest = hashfunc(cert).digest() |
|||
|
|||
if not cert_digest == fingerprint_bytes: |
|||
raise SSLError('Fingerprints did not match. Expected "{0}", got "{1}".' |
|||
.format(hexlify(fingerprint_bytes), |
|||
hexlify(cert_digest))) |
|||
|
|||
|
|||
def resolve_cert_reqs(candidate): |
|||
""" |
|||
Resolves the argument to a numeric constant, which can be passed to |
|||
the wrap_socket function/method from the ssl module. |
|||
Defaults to :data:`ssl.CERT_NONE`. |
|||
If given a string it is assumed to be the name of the constant in the |
|||
:mod:`ssl` module or its abbrevation. |
|||
(So you can specify `REQUIRED` instead of `CERT_REQUIRED`. |
|||
If it's neither `None` nor a string we assume it is already the numeric |
|||
constant which can directly be passed to wrap_socket. |
|||
""" |
|||
if candidate is None: |
|||
return CERT_NONE |
|||
|
|||
if isinstance(candidate, str): |
|||
res = getattr(ssl, candidate, None) |
|||
if res is None: |
|||
res = getattr(ssl, 'CERT_' + candidate) |
|||
return res |
|||
|
|||
return candidate |
|||
|
|||
|
|||
def resolve_ssl_version(candidate): |
|||
""" |
|||
like resolve_cert_reqs |
|||
""" |
|||
if candidate is None: |
|||
return PROTOCOL_SSLv23 |
|||
|
|||
if isinstance(candidate, str): |
|||
res = getattr(ssl, candidate, None) |
|||
if res is None: |
|||
res = getattr(ssl, 'PROTOCOL_' + candidate) |
|||
return res |
|||
|
|||
return candidate |
|||
|
|||
|
|||
if SSLContext is not None: # Python 3.2+ |
|||
def ssl_wrap_socket(sock, keyfile=None, certfile=None, cert_reqs=None, |
|||
ca_certs=None, server_hostname=None, |
|||
ssl_version=None): |
|||
""" |
|||
All arguments except `server_hostname` have the same meaning as for |
|||
:func:`ssl.wrap_socket` |
|||
|
|||
:param server_hostname: |
|||
Hostname of the expected certificate |
|||
""" |
|||
context = SSLContext(ssl_version) |
|||
context.verify_mode = cert_reqs |
|||
|
|||
# Disable TLS compression to migitate CRIME attack (issue #309) |
|||
OP_NO_COMPRESSION = 0x20000 |
|||
context.options |= OP_NO_COMPRESSION |
|||
|
|||
if ca_certs: |
|||
try: |
|||
context.load_verify_locations(ca_certs) |
|||
# Py32 raises IOError |
|||
# Py33 raises FileNotFoundError |
|||
except Exception as e: # Reraise as SSLError |
|||
raise SSLError(e) |
|||
if certfile: |
|||
# FIXME: This block needs a test. |
|||
context.load_cert_chain(certfile, keyfile) |
|||
if HAS_SNI: # Platform-specific: OpenSSL with enabled SNI |
|||
return context.wrap_socket(sock, server_hostname=server_hostname) |
|||
return context.wrap_socket(sock) |
|||
|
|||
else: # Python 3.1 and earlier |
|||
def ssl_wrap_socket(sock, keyfile=None, certfile=None, cert_reqs=None, |
|||
ca_certs=None, server_hostname=None, |
|||
ssl_version=None): |
|||
return wrap_socket(sock, keyfile=keyfile, certfile=certfile, |
|||
ca_certs=ca_certs, cert_reqs=cert_reqs, |
|||
ssl_version=ssl_version) |
|||
@ -0,0 +1,234 @@ |
|||
from socket import _GLOBAL_DEFAULT_TIMEOUT |
|||
import time |
|||
|
|||
from ..exceptions import TimeoutStateError |
|||
|
|||
|
|||
def current_time(): |
|||
""" |
|||
Retrieve the current time, this function is mocked out in unit testing. |
|||
""" |
|||
return time.time() |
|||
|
|||
|
|||
_Default = object() |
|||
# The default timeout to use for socket connections. This is the attribute used |
|||
# by httplib to define the default timeout |
|||
|
|||
|
|||
class Timeout(object): |
|||
""" |
|||
Utility object for storing timeout values. |
|||
|
|||
Example usage: |
|||
|
|||
.. code-block:: python |
|||
|
|||
timeout = urllib3.util.Timeout(connect=2.0, read=7.0) |
|||
pool = HTTPConnectionPool('www.google.com', 80, timeout=timeout) |
|||
pool.request(...) # Etc, etc |
|||
|
|||
:param connect: |
|||
The maximum amount of time to wait for a connection attempt to a server |
|||
to succeed. Omitting the parameter will default the connect timeout to |
|||
the system default, probably `the global default timeout in socket.py |
|||
<http://hg.python.org/cpython/file/603b4d593758/Lib/socket.py#l535>`_. |
|||
None will set an infinite timeout for connection attempts. |
|||
|
|||
:type connect: integer, float, or None |
|||
|
|||
:param read: |
|||
The maximum amount of time to wait between consecutive |
|||
read operations for a response from the server. Omitting |
|||
the parameter will default the read timeout to the system |
|||
default, probably `the global default timeout in socket.py |
|||
<http://hg.python.org/cpython/file/603b4d593758/Lib/socket.py#l535>`_. |
|||
None will set an infinite timeout. |
|||
|
|||
:type read: integer, float, or None |
|||
|
|||
:param total: |
|||
This combines the connect and read timeouts into one; the read timeout |
|||
will be set to the time leftover from the connect attempt. In the |
|||
event that both a connect timeout and a total are specified, or a read |
|||
timeout and a total are specified, the shorter timeout will be applied. |
|||
|
|||
Defaults to None. |
|||
|
|||
:type total: integer, float, or None |
|||
|
|||
.. note:: |
|||
|
|||
Many factors can affect the total amount of time for urllib3 to return |
|||
an HTTP response. Specifically, Python's DNS resolver does not obey the |
|||
timeout specified on the socket. Other factors that can affect total |
|||
request time include high CPU load, high swap, the program running at a |
|||
low priority level, or other behaviors. The observed running time for |
|||
urllib3 to return a response may be greater than the value passed to |
|||
`total`. |
|||
|
|||
In addition, the read and total timeouts only measure the time between |
|||
read operations on the socket connecting the client and the server, |
|||
not the total amount of time for the request to return a complete |
|||
response. For most requests, the timeout is raised because the server |
|||
has not sent the first byte in the specified time. This is not always |
|||
the case; if a server streams one byte every fifteen seconds, a timeout |
|||
of 20 seconds will not ever trigger, even though the request will |
|||
take several minutes to complete. |
|||
|
|||
If your goal is to cut off any request after a set amount of wall clock |
|||
time, consider having a second "watcher" thread to cut off a slow |
|||
request. |
|||
""" |
|||
|
|||
#: A sentinel object representing the default timeout value |
|||
DEFAULT_TIMEOUT = _GLOBAL_DEFAULT_TIMEOUT |
|||
|
|||
def __init__(self, total=None, connect=_Default, read=_Default): |
|||
self._connect = self._validate_timeout(connect, 'connect') |
|||
self._read = self._validate_timeout(read, 'read') |
|||
self.total = self._validate_timeout(total, 'total') |
|||
self._start_connect = None |
|||
|
|||
def __str__(self): |
|||
return '%s(connect=%r, read=%r, total=%r)' % ( |
|||
type(self).__name__, self._connect, self._read, self.total) |
|||
|
|||
|
|||
@classmethod |
|||
def _validate_timeout(cls, value, name): |
|||
""" Check that a timeout attribute is valid |
|||
|
|||
:param value: The timeout value to validate |
|||
:param name: The name of the timeout attribute to validate. This is used |
|||
for clear error messages |
|||
:return: the value |
|||
:raises ValueError: if the type is not an integer or a float, or if it |
|||
is a numeric value less than zero |
|||
""" |
|||
if value is _Default: |
|||
return cls.DEFAULT_TIMEOUT |
|||
|
|||
if value is None or value is cls.DEFAULT_TIMEOUT: |
|||
return value |
|||
|
|||
try: |
|||
float(value) |
|||
except (TypeError, ValueError): |
|||
raise ValueError("Timeout value %s was %s, but it must be an " |
|||
"int or float." % (name, value)) |
|||
|
|||
try: |
|||
if value < 0: |
|||
raise ValueError("Attempted to set %s timeout to %s, but the " |
|||
"timeout cannot be set to a value less " |
|||
"than 0." % (name, value)) |
|||
except TypeError: # Python 3 |
|||
raise ValueError("Timeout value %s was %s, but it must be an " |
|||
"int or float." % (name, value)) |
|||
|
|||
return value |
|||
|
|||
@classmethod |
|||
def from_float(cls, timeout): |
|||
""" Create a new Timeout from a legacy timeout value. |
|||
|
|||
The timeout value used by httplib.py sets the same timeout on the |
|||
connect(), and recv() socket requests. This creates a :class:`Timeout` |
|||
object that sets the individual timeouts to the ``timeout`` value passed |
|||
to this function. |
|||
|
|||
:param timeout: The legacy timeout value |
|||
:type timeout: integer, float, sentinel default object, or None |
|||
:return: a Timeout object |
|||
:rtype: :class:`Timeout` |
|||
""" |
|||
return Timeout(read=timeout, connect=timeout) |
|||
|
|||
def clone(self): |
|||
""" Create a copy of the timeout object |
|||
|
|||
Timeout properties are stored per-pool but each request needs a fresh |
|||
Timeout object to ensure each one has its own start/stop configured. |
|||
|
|||
:return: a copy of the timeout object |
|||
:rtype: :class:`Timeout` |
|||
""" |
|||
# We can't use copy.deepcopy because that will also create a new object |
|||
# for _GLOBAL_DEFAULT_TIMEOUT, which socket.py uses as a sentinel to |
|||
# detect the user default. |
|||
return Timeout(connect=self._connect, read=self._read, |
|||
total=self.total) |
|||
|
|||
def start_connect(self): |
|||
""" Start the timeout clock, used during a connect() attempt |
|||
|
|||
:raises urllib3.exceptions.TimeoutStateError: if you attempt |
|||
to start a timer that has been started already. |
|||
""" |
|||
if self._start_connect is not None: |
|||
raise TimeoutStateError("Timeout timer has already been started.") |
|||
self._start_connect = current_time() |
|||
return self._start_connect |
|||
|
|||
def get_connect_duration(self): |
|||
""" Gets the time elapsed since the call to :meth:`start_connect`. |
|||
|
|||
:return: the elapsed time |
|||
:rtype: float |
|||
:raises urllib3.exceptions.TimeoutStateError: if you attempt |
|||
to get duration for a timer that hasn't been started. |
|||
""" |
|||
if self._start_connect is None: |
|||
raise TimeoutStateError("Can't get connect duration for timer " |
|||
"that has not started.") |
|||
return current_time() - self._start_connect |
|||
|
|||
@property |
|||
def connect_timeout(self): |
|||
""" Get the value to use when setting a connection timeout. |
|||
|
|||
This will be a positive float or integer, the value None |
|||
(never timeout), or the default system timeout. |
|||
|
|||
:return: the connect timeout |
|||
:rtype: int, float, :attr:`Timeout.DEFAULT_TIMEOUT` or None |
|||
""" |
|||
if self.total is None: |
|||
return self._connect |
|||
|
|||
if self._connect is None or self._connect is self.DEFAULT_TIMEOUT: |
|||
return self.total |
|||
|
|||
return min(self._connect, self.total) |
|||
|
|||
@property |
|||
def read_timeout(self): |
|||
""" Get the value for the read timeout. |
|||
|
|||
This assumes some time has elapsed in the connection timeout and |
|||
computes the read timeout appropriately. |
|||
|
|||
If self.total is set, the read timeout is dependent on the amount of |
|||
time taken by the connect timeout. If the connection time has not been |
|||
established, a :exc:`~urllib3.exceptions.TimeoutStateError` will be |
|||
raised. |
|||
|
|||
:return: the value to use for the read timeout |
|||
:rtype: int, float, :attr:`Timeout.DEFAULT_TIMEOUT` or None |
|||
:raises urllib3.exceptions.TimeoutStateError: If :meth:`start_connect` |
|||
has not yet been called on this object. |
|||
""" |
|||
if (self.total is not None and |
|||
self.total is not self.DEFAULT_TIMEOUT and |
|||
self._read is not None and |
|||
self._read is not self.DEFAULT_TIMEOUT): |
|||
# in case the connect timeout has not yet been established. |
|||
if self._start_connect is None: |
|||
return self._read |
|||
return max(0, min(self.total - self.get_connect_duration(), |
|||
self._read)) |
|||
elif self.total is not None and self.total is not self.DEFAULT_TIMEOUT: |
|||
return max(0, self.total - self.get_connect_duration()) |
|||
else: |
|||
return self._read |
|||
@ -0,0 +1,162 @@ |
|||
from collections import namedtuple |
|||
|
|||
from ..exceptions import LocationParseError |
|||
|
|||
|
|||
class Url(namedtuple('Url', ['scheme', 'auth', 'host', 'port', 'path', 'query', 'fragment'])): |
|||
""" |
|||
Datastructure for representing an HTTP URL. Used as a return value for |
|||
:func:`parse_url`. |
|||
""" |
|||
slots = () |
|||
|
|||
def __new__(cls, scheme=None, auth=None, host=None, port=None, path=None, query=None, fragment=None): |
|||
return super(Url, cls).__new__(cls, scheme, auth, host, port, path, query, fragment) |
|||
|
|||
@property |
|||
def hostname(self): |
|||
"""For backwards-compatibility with urlparse. We're nice like that.""" |
|||
return self.host |
|||
|
|||
@property |
|||
def request_uri(self): |
|||
"""Absolute path including the query string.""" |
|||
uri = self.path or '/' |
|||
|
|||
if self.query is not None: |
|||
uri += '?' + self.query |
|||
|
|||
return uri |
|||
|
|||
@property |
|||
def netloc(self): |
|||
"""Network location including host and port""" |
|||
if self.port: |
|||
return '%s:%d' % (self.host, self.port) |
|||
return self.host |
|||
|
|||
|
|||
def split_first(s, delims): |
|||
""" |
|||
Given a string and an iterable of delimiters, split on the first found |
|||
delimiter. Return two split parts and the matched delimiter. |
|||
|
|||
If not found, then the first part is the full input string. |
|||
|
|||
Example: :: |
|||
|
|||
>>> split_first('foo/bar?baz', '?/=') |
|||
('foo', 'bar?baz', '/') |
|||
>>> split_first('foo/bar?baz', '123') |
|||
('foo/bar?baz', '', None) |
|||
|
|||
Scales linearly with number of delims. Not ideal for large number of delims. |
|||
""" |
|||
min_idx = None |
|||
min_delim = None |
|||
for d in delims: |
|||
idx = s.find(d) |
|||
if idx < 0: |
|||
continue |
|||
|
|||
if min_idx is None or idx < min_idx: |
|||
min_idx = idx |
|||
min_delim = d |
|||
|
|||
if min_idx is None or min_idx < 0: |
|||
return s, '', None |
|||
|
|||
return s[:min_idx], s[min_idx+1:], min_delim |
|||
|
|||
|
|||
def parse_url(url): |
|||
""" |
|||
Given a url, return a parsed :class:`.Url` namedtuple. Best-effort is |
|||
performed to parse incomplete urls. Fields not provided will be None. |
|||
|
|||
Partly backwards-compatible with :mod:`urlparse`. |
|||
|
|||
Example: :: |
|||
|
|||
>>> parse_url('http://google.com/mail/') |
|||
Url(scheme='http', host='google.com', port=None, path='/', ...) |
|||
>>> parse_url('google.com:80') |
|||
Url(scheme=None, host='google.com', port=80, path=None, ...) |
|||
>>> parse_url('/foo?bar') |
|||
Url(scheme=None, host=None, port=None, path='/foo', query='bar', ...) |
|||
""" |
|||
|
|||
# While this code has overlap with stdlib's urlparse, it is much |
|||
# simplified for our needs and less annoying. |
|||
# Additionally, this implementations does silly things to be optimal |
|||
# on CPython. |
|||
|
|||
scheme = None |
|||
auth = None |
|||
host = None |
|||
port = None |
|||
path = None |
|||
fragment = None |
|||
query = None |
|||
|
|||
# Scheme |
|||
if '://' in url: |
|||
scheme, url = url.split('://', 1) |
|||
|
|||
# Find the earliest Authority Terminator |
|||
# (http://tools.ietf.org/html/rfc3986#section-3.2) |
|||
url, path_, delim = split_first(url, ['/', '?', '#']) |
|||
|
|||
if delim: |
|||
# Reassemble the path |
|||
path = delim + path_ |
|||
|
|||
# Auth |
|||
if '@' in url: |
|||
# Last '@' denotes end of auth part |
|||
auth, url = url.rsplit('@', 1) |
|||
|
|||
# IPv6 |
|||
if url and url[0] == '[': |
|||
host, url = url.split(']', 1) |
|||
host += ']' |
|||
|
|||
# Port |
|||
if ':' in url: |
|||
_host, port = url.split(':', 1) |
|||
|
|||
if not host: |
|||
host = _host |
|||
|
|||
if port: |
|||
# If given, ports must be integers. |
|||
if not port.isdigit(): |
|||
raise LocationParseError(url) |
|||
port = int(port) |
|||
else: |
|||
# Blank ports are cool, too. (rfc3986#section-3.2.3) |
|||
port = None |
|||
|
|||
elif not host and url: |
|||
host = url |
|||
|
|||
if not path: |
|||
return Url(scheme, auth, host, port, path, query, fragment) |
|||
|
|||
# Fragment |
|||
if '#' in path: |
|||
path, fragment = path.split('#', 1) |
|||
|
|||
# Query |
|||
if '?' in path: |
|||
path, query = path.split('?', 1) |
|||
|
|||
return Url(scheme, auth, host, port, path, query, fragment) |
|||
|
|||
|
|||
def get_host(url): |
|||
""" |
|||
Deprecated. Use :func:`.parse_url` instead. |
|||
""" |
|||
p = parse_url(url) |
|||
return p.scheme or 'http', p.hostname, p.port |
|||
Loading…
Reference in new issue