You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
92 lines
1.9 KiB
92 lines
1.9 KiB
14 years ago
|
# -*- coding: utf-8 -*-
|
||
|
|
|
||
|
|
"""
|
||
|
|
requests.async
|
||
|
|
~~~~~~~~~~~~~~
|
||
|
|
|
||
|
|
This module contains an asynchronous replica of ``requests.api``, powered
|
||
|
|
by gevent. All API methods return a ``Request`` instance (as opposed to
|
||
|
|
``Response``). A list of requests can be sent with ``map()``.
|
||
|
|
"""
|
||
|
|
|
||
|
|
try:
|
||
|
|
import gevent
|
||
|
|
from gevent import monkey as curious_george
|
||
|
|
from gevent.pool import Pool
|
||
|
|
except ImportError:
|
||
|
|
raise RuntimeError('Gevent is required for requests.async.')
|
||
|
|
|
||
|
|
# Monkey-patch.
|
||
|
|
curious_george.patch_all(thread=False)
|
||
|
|
|
||
|
|
from . import api
|
||
|
|
|
||
|
|
|
||
|
|
__all__ = (
|
||
|
|
'map',
|
||
|
|
'get', 'options', 'head', 'post', 'put', 'patch', 'delete', 'request'
|
||
|
|
)
|
||
|
|
|
||
|
|
|
||
|
|
def patched(f):
|
||
|
|
"""Patches a given API function to not send."""
|
||
|
|
|
||
|
|
def wrapped(*args, **kwargs):
|
||
|
|
|
||
|
|
kwargs['return_response'] = False
|
||
|
|
kwargs['prefetch'] = True
|
||
|
|
|
||
|
|
return f(*args, **kwargs)
|
||
|
|
|
||
|
|
return wrapped
|
||
|
|
|
||
|
|
|
||
|
|
def send(r, pools=None):
|
||
|
|
"""Sends a given Request object."""
|
||
|
|
|
||
|
|
if pools:
|
||
|
|
r._pools = pools
|
||
|
|
|
||
|
|
r.send()
|
||
|
|
|
||
|
|
return r.response
|
||
|
|
|
||
|
|
|
||
|
|
# Patched requests.api functions.
|
||
|
|
get = patched(api.get)
|
||
|
|
options = patched(api.options)
|
||
|
|
head = patched(api.head)
|
||
|
|
post = patched(api.post)
|
||
|
|
put = patched(api.put)
|
||
|
|
patch = patched(api.patch)
|
||
|
|
delete = patched(api.delete)
|
||
|
|
request = patched(api.request)
|
||
|
|
|
||
|
|
|
||
|
|
def map(requests, prefetch=True, size=None):
|
||
|
|
"""Concurrently converts a list of Requests to Responses.
|
||
|
|
|
||
|
|
:param requests: a collection of Request objects.
|
||
|
|
:param prefetch: If False, the content will not be downloaded immediately.
|
||
|
|
:param size: Specifies the number of requests to make at a time. If None, no throttling occurs.
|
||
|
|
"""
|
||
|
|
|
||
|
|
requests = list(requests)
|
||
|
|
|
||
|
|
if size:
|
||
|
|
pool = Pool(size)
|
||
|
|
pool.map(send, requests)
|
||
|
|
pool.join()
|
||
|
|
else:
|
||
|
|
jobs = [gevent.spawn(send, r) for r in requests]
|
||
|
|
gevent.joinall(jobs)
|
||
|
|
|
||
|
|
if prefetch:
|
||
|
|
[r.response.content for r in requests]
|
||
|
|
|
||
|
|
return [r.response for r in requests]
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
|