|
|
@ -13,7 +13,7 @@ from collections import Mapping |
|
|
from datetime import datetime |
|
|
from datetime import datetime |
|
|
|
|
|
|
|
|
from .auth import _basic_auth_str |
|
|
from .auth import _basic_auth_str |
|
|
from .compat import cookielib, OrderedDict, urljoin, urlparse, builtin_str |
|
|
from .compat import cookielib, OrderedDict, urljoin, urlparse |
|
|
from .cookies import ( |
|
|
from .cookies import ( |
|
|
cookiejar_from_dict, extract_cookies_to_jar, RequestsCookieJar, merge_cookies) |
|
|
cookiejar_from_dict, extract_cookies_to_jar, RequestsCookieJar, merge_cookies) |
|
|
from .models import Request, PreparedRequest, DEFAULT_REDIRECT_LIMIT |
|
|
from .models import Request, PreparedRequest, DEFAULT_REDIRECT_LIMIT |
|
|
@ -21,6 +21,7 @@ from .hooks import default_hooks, dispatch_hook |
|
|
from .utils import to_key_val_list, default_headers, to_native_string |
|
|
from .utils import to_key_val_list, default_headers, to_native_string |
|
|
from .exceptions import ( |
|
|
from .exceptions import ( |
|
|
TooManyRedirects, InvalidSchema, ChunkedEncodingError, ContentDecodingError) |
|
|
TooManyRedirects, InvalidSchema, ChunkedEncodingError, ContentDecodingError) |
|
|
|
|
|
from .packages.urllib3._collections import RecentlyUsedContainer |
|
|
from .structures import CaseInsensitiveDict |
|
|
from .structures import CaseInsensitiveDict |
|
|
|
|
|
|
|
|
from .adapters import HTTPAdapter |
|
|
from .adapters import HTTPAdapter |
|
|
@ -35,6 +36,8 @@ from .status_codes import codes |
|
|
# formerly defined here, reexposed here for backward compatibility |
|
|
# formerly defined here, reexposed here for backward compatibility |
|
|
from .models import REDIRECT_STATI |
|
|
from .models import REDIRECT_STATI |
|
|
|
|
|
|
|
|
|
|
|
REDIRECT_CACHE_SIZE = 1000 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def merge_setting(request_setting, session_setting, dict_class=OrderedDict): |
|
|
def merge_setting(request_setting, session_setting, dict_class=OrderedDict): |
|
|
""" |
|
|
""" |
|
|
@ -128,14 +131,14 @@ class SessionRedirectMixin(object): |
|
|
# Facilitate relative 'location' headers, as allowed by RFC 7231. |
|
|
# Facilitate relative 'location' headers, as allowed by RFC 7231. |
|
|
# (e.g. '/path/to/resource' instead of 'http://domain.tld/path/to/resource') |
|
|
# (e.g. '/path/to/resource' instead of 'http://domain.tld/path/to/resource') |
|
|
# Compliant with RFC3986, we percent encode the url. |
|
|
# Compliant with RFC3986, we percent encode the url. |
|
|
if not urlparse(url).netloc: |
|
|
if not parsed.netloc: |
|
|
url = urljoin(resp.url, requote_uri(url)) |
|
|
url = urljoin(resp.url, requote_uri(url)) |
|
|
else: |
|
|
else: |
|
|
url = requote_uri(url) |
|
|
url = requote_uri(url) |
|
|
|
|
|
|
|
|
prepared_request.url = to_native_string(url) |
|
|
prepared_request.url = to_native_string(url) |
|
|
# cache the url |
|
|
# Cache the url, unless it redirects to itself. |
|
|
if resp.is_permanent_redirect: |
|
|
if resp.is_permanent_redirect and req.url != prepared_request.url: |
|
|
self.redirect_cache[req.url] = prepared_request.url |
|
|
self.redirect_cache[req.url] = prepared_request.url |
|
|
|
|
|
|
|
|
# http://tools.ietf.org/html/rfc7231#section-6.4.4 |
|
|
# http://tools.ietf.org/html/rfc7231#section-6.4.4 |
|
|
@ -271,9 +274,10 @@ class Session(SessionRedirectMixin): |
|
|
""" |
|
|
""" |
|
|
|
|
|
|
|
|
__attrs__ = [ |
|
|
__attrs__ = [ |
|
|
'headers', 'cookies', 'auth', 'timeout', 'proxies', 'hooks', |
|
|
'headers', 'cookies', 'auth', 'proxies', 'hooks', 'params', 'verify', |
|
|
'params', 'verify', 'cert', 'prefetch', 'adapters', 'stream', |
|
|
'cert', 'prefetch', 'adapters', 'stream', 'trust_env', |
|
|
'trust_env', 'max_redirects', 'redirect_cache'] |
|
|
'max_redirects', |
|
|
|
|
|
] |
|
|
|
|
|
|
|
|
def __init__(self): |
|
|
def __init__(self): |
|
|
|
|
|
|
|
|
@ -326,7 +330,8 @@ class Session(SessionRedirectMixin): |
|
|
self.mount('https://', HTTPAdapter()) |
|
|
self.mount('https://', HTTPAdapter()) |
|
|
self.mount('http://', HTTPAdapter()) |
|
|
self.mount('http://', HTTPAdapter()) |
|
|
|
|
|
|
|
|
self.redirect_cache = {} |
|
|
# Only store 1000 redirects to prevent using infinite memory |
|
|
|
|
|
self.redirect_cache = RecentlyUsedContainer(REDIRECT_CACHE_SIZE) |
|
|
|
|
|
|
|
|
def __enter__(self): |
|
|
def __enter__(self): |
|
|
return self |
|
|
return self |
|
|
@ -365,6 +370,7 @@ class Session(SessionRedirectMixin): |
|
|
url=request.url, |
|
|
url=request.url, |
|
|
files=request.files, |
|
|
files=request.files, |
|
|
data=request.data, |
|
|
data=request.data, |
|
|
|
|
|
json=request.json, |
|
|
headers=merge_setting(request.headers, self.headers, dict_class=CaseInsensitiveDict), |
|
|
headers=merge_setting(request.headers, self.headers, dict_class=CaseInsensitiveDict), |
|
|
params=merge_setting(request.params, self.params), |
|
|
params=merge_setting(request.params, self.params), |
|
|
auth=merge_setting(auth, self.auth), |
|
|
auth=merge_setting(auth, self.auth), |
|
|
@ -386,7 +392,8 @@ class Session(SessionRedirectMixin): |
|
|
hooks=None, |
|
|
hooks=None, |
|
|
stream=None, |
|
|
stream=None, |
|
|
verify=None, |
|
|
verify=None, |
|
|
cert=None): |
|
|
cert=None, |
|
|
|
|
|
json=None): |
|
|
"""Constructs a :class:`Request <Request>`, prepares it and sends it. |
|
|
"""Constructs a :class:`Request <Request>`, prepares it and sends it. |
|
|
Returns :class:`Response <Response>` object. |
|
|
Returns :class:`Response <Response>` object. |
|
|
|
|
|
|
|
|
@ -396,17 +403,22 @@ class Session(SessionRedirectMixin): |
|
|
string for the :class:`Request`. |
|
|
string for the :class:`Request`. |
|
|
:param data: (optional) Dictionary or bytes to send in the body of the |
|
|
:param data: (optional) Dictionary or bytes to send in the body of the |
|
|
:class:`Request`. |
|
|
:class:`Request`. |
|
|
|
|
|
:param json: (optional) json to send in the body of the |
|
|
|
|
|
:class:`Request`. |
|
|
:param headers: (optional) Dictionary of HTTP Headers to send with the |
|
|
:param headers: (optional) Dictionary of HTTP Headers to send with the |
|
|
:class:`Request`. |
|
|
:class:`Request`. |
|
|
:param cookies: (optional) Dict or CookieJar object to send with the |
|
|
:param cookies: (optional) Dict or CookieJar object to send with the |
|
|
:class:`Request`. |
|
|
:class:`Request`. |
|
|
:param files: (optional) Dictionary of 'filename': file-like-objects |
|
|
:param files: (optional) Dictionary of ``'filename': file-like-objects`` |
|
|
for multipart encoding upload. |
|
|
for multipart encoding upload. |
|
|
:param auth: (optional) Auth tuple or callable to enable |
|
|
:param auth: (optional) Auth tuple or callable to enable |
|
|
Basic/Digest/Custom HTTP Auth. |
|
|
Basic/Digest/Custom HTTP Auth. |
|
|
:param timeout: (optional) Float describing the timeout of the |
|
|
:param timeout: (optional) How long to wait for the server to send |
|
|
request in seconds. |
|
|
data before giving up, as a float, or a (`connect timeout, read |
|
|
:param allow_redirects: (optional) Boolean. Set to True by default. |
|
|
timeout <user/advanced.html#timeouts>`_) tuple. |
|
|
|
|
|
:type timeout: float or tuple |
|
|
|
|
|
:param allow_redirects: (optional) Set to True by default. |
|
|
|
|
|
:type allow_redirects: bool |
|
|
:param proxies: (optional) Dictionary mapping protocol to the URL of |
|
|
:param proxies: (optional) Dictionary mapping protocol to the URL of |
|
|
the proxy. |
|
|
the proxy. |
|
|
:param stream: (optional) whether to immediately download the response |
|
|
:param stream: (optional) whether to immediately download the response |
|
|
@ -417,7 +429,7 @@ class Session(SessionRedirectMixin): |
|
|
If Tuple, ('cert', 'key') pair. |
|
|
If Tuple, ('cert', 'key') pair. |
|
|
""" |
|
|
""" |
|
|
|
|
|
|
|
|
method = builtin_str(method) |
|
|
method = to_native_string(method) |
|
|
|
|
|
|
|
|
# Create the Request. |
|
|
# Create the Request. |
|
|
req = Request( |
|
|
req = Request( |
|
|
@ -426,6 +438,7 @@ class Session(SessionRedirectMixin): |
|
|
headers = headers, |
|
|
headers = headers, |
|
|
files = files, |
|
|
files = files, |
|
|
data = data or {}, |
|
|
data = data or {}, |
|
|
|
|
|
json = json, |
|
|
params = params or {}, |
|
|
params = params or {}, |
|
|
auth = auth, |
|
|
auth = auth, |
|
|
cookies = cookies, |
|
|
cookies = cookies, |
|
|
@ -479,15 +492,16 @@ class Session(SessionRedirectMixin): |
|
|
kwargs.setdefault('allow_redirects', False) |
|
|
kwargs.setdefault('allow_redirects', False) |
|
|
return self.request('HEAD', url, **kwargs) |
|
|
return self.request('HEAD', url, **kwargs) |
|
|
|
|
|
|
|
|
def post(self, url, data=None, **kwargs): |
|
|
def post(self, url, data=None, json=None, **kwargs): |
|
|
"""Sends a POST request. Returns :class:`Response` object. |
|
|
"""Sends a POST request. Returns :class:`Response` object. |
|
|
|
|
|
|
|
|
:param url: URL for the new :class:`Request` object. |
|
|
:param url: URL for the new :class:`Request` object. |
|
|
:param data: (optional) Dictionary, bytes, or file-like object to send in the body of the :class:`Request`. |
|
|
:param data: (optional) Dictionary, bytes, or file-like object to send in the body of the :class:`Request`. |
|
|
|
|
|
:param json: (optional) json to send in the body of the :class:`Request`. |
|
|
:param \*\*kwargs: Optional arguments that ``request`` takes. |
|
|
:param \*\*kwargs: Optional arguments that ``request`` takes. |
|
|
""" |
|
|
""" |
|
|
|
|
|
|
|
|
return self.request('POST', url, data=data, **kwargs) |
|
|
return self.request('POST', url, data=data, json=json, **kwargs) |
|
|
|
|
|
|
|
|
def put(self, url, data=None, **kwargs): |
|
|
def put(self, url, data=None, **kwargs): |
|
|
"""Sends a PUT request. Returns :class:`Response` object. |
|
|
"""Sends a PUT request. Returns :class:`Response` object. |
|
|
@ -532,12 +546,13 @@ class Session(SessionRedirectMixin): |
|
|
if not isinstance(request, PreparedRequest): |
|
|
if not isinstance(request, PreparedRequest): |
|
|
raise ValueError('You can only send PreparedRequests.') |
|
|
raise ValueError('You can only send PreparedRequests.') |
|
|
|
|
|
|
|
|
redirect_count = 0 |
|
|
checked_urls = set() |
|
|
while request.url in self.redirect_cache: |
|
|
while request.url in self.redirect_cache: |
|
|
redirect_count += 1 |
|
|
checked_urls.add(request.url) |
|
|
if redirect_count > self.max_redirects: |
|
|
new_url = self.redirect_cache.get(request.url) |
|
|
raise TooManyRedirects |
|
|
if new_url in checked_urls: |
|
|
request.url = self.redirect_cache.get(request.url) |
|
|
break |
|
|
|
|
|
request.url = new_url |
|
|
|
|
|
|
|
|
# Set up variables needed for resolve_redirects and dispatching of hooks |
|
|
# Set up variables needed for resolve_redirects and dispatching of hooks |
|
|
allow_redirects = kwargs.pop('allow_redirects', True) |
|
|
allow_redirects = kwargs.pop('allow_redirects', True) |
|
|
@ -647,12 +662,19 @@ class Session(SessionRedirectMixin): |
|
|
self.adapters[key] = self.adapters.pop(key) |
|
|
self.adapters[key] = self.adapters.pop(key) |
|
|
|
|
|
|
|
|
def __getstate__(self): |
|
|
def __getstate__(self): |
|
|
return dict((attr, getattr(self, attr, None)) for attr in self.__attrs__) |
|
|
state = dict((attr, getattr(self, attr, None)) for attr in self.__attrs__) |
|
|
|
|
|
state['redirect_cache'] = dict(self.redirect_cache) |
|
|
|
|
|
return state |
|
|
|
|
|
|
|
|
def __setstate__(self, state): |
|
|
def __setstate__(self, state): |
|
|
|
|
|
redirect_cache = state.pop('redirect_cache', {}) |
|
|
for attr, value in state.items(): |
|
|
for attr, value in state.items(): |
|
|
setattr(self, attr, value) |
|
|
setattr(self, attr, value) |
|
|
|
|
|
|
|
|
|
|
|
self.redirect_cache = RecentlyUsedContainer(REDIRECT_CACHE_SIZE) |
|
|
|
|
|
for redirect, to in redirect_cache.items(): |
|
|
|
|
|
self.redirect_cache[redirect] = to |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def session(): |
|
|
def session(): |
|
|
"""Returns a :class:`Session` for context-management.""" |
|
|
"""Returns a :class:`Session` for context-management.""" |
|
|
|
Ruud
10 years ago