Ruud
13 years ago
65 changed files with 16219 additions and 378 deletions
@ -1,23 +0,0 @@ |
|||
from .main import Moovee |
|||
|
|||
def start(): |
|||
return Moovee() |
|||
|
|||
config = [{ |
|||
'name': 'moovee', |
|||
'groups': [ |
|||
{ |
|||
'tab': 'searcher', |
|||
'subtab': 'providers', |
|||
'name': '#alt.binaries.moovee', |
|||
'description': 'SD movies only', |
|||
'options': [ |
|||
{ |
|||
'name': 'enabled', |
|||
'type': 'enabler', |
|||
'default': False, |
|||
}, |
|||
], |
|||
}, |
|||
], |
|||
}] |
|||
@ -1,66 +0,0 @@ |
|||
from couchpotato.core.event import fireEvent |
|||
from couchpotato.core.helpers.encoding import tryUrlencode |
|||
from couchpotato.core.helpers.variable import getTitle |
|||
from couchpotato.core.logger import CPLog |
|||
from couchpotato.core.providers.nzb.base import NZBProvider |
|||
from dateutil.parser import parse |
|||
import re |
|||
import time |
|||
|
|||
log = CPLog(__name__) |
|||
|
|||
|
|||
class Moovee(NZBProvider): |
|||
|
|||
urls = { |
|||
'download': 'http://85.214.105.230/get_nzb.php?id=%s§ion=moovee', |
|||
'search': 'http://abmoovee.allfilled.com/search.php?q=%s&Search=Search', |
|||
} |
|||
|
|||
regex = '<td class="cell_reqid">(?P<reqid>.*?)</td>.+?<td class="cell_request">(?P<title>.*?)</td>.+?<td class="cell_statuschange">(?P<age>.*?)</td>' |
|||
|
|||
http_time_between_calls = 2 # Seconds |
|||
|
|||
def search(self, movie, quality): |
|||
|
|||
results = [] |
|||
if self.isDisabled() or not self.isAvailable(self.urls['search']) or quality.get('hd', False): |
|||
return results |
|||
|
|||
q = '%s %s' % (getTitle(movie['library']), quality.get('identifier')) |
|||
url = self.urls['search'] % tryUrlencode(q) |
|||
|
|||
cache_key = 'moovee.%s' % q |
|||
data = self.getCache(cache_key, url) |
|||
if data: |
|||
match = re.compile(self.regex, re.DOTALL).finditer(data) |
|||
|
|||
for nzb in match: |
|||
new = { |
|||
'id': nzb.group('reqid'), |
|||
'name': nzb.group('title'), |
|||
'type': 'nzb', |
|||
'provider': self.getName(), |
|||
'age': self.calculateAge(time.mktime(parse(nzb.group('age')).timetuple())), |
|||
'size': None, |
|||
'url': self.urls['download'] % (nzb.group('reqid')), |
|||
'detail_url': '', |
|||
'description': '', |
|||
'check_nzb': False, |
|||
} |
|||
|
|||
new['score'] = fireEvent('score.calculate', new, movie, single = True) |
|||
is_correct_movie = fireEvent('searcher.correct_movie', |
|||
nzb = new, movie = movie, quality = quality, |
|||
imdb_results = False, single_category = False, single = True) |
|||
if is_correct_movie: |
|||
results.append(new) |
|||
self.found(new) |
|||
|
|||
return results |
|||
|
|||
def belongsTo(self, url, host = None): |
|||
match = re.match('http://85\.214\.105\.230/get_nzb\.php\?id=[0-9]*§ion=moovee', url) |
|||
if match: |
|||
return self |
|||
return |
|||
@ -1,23 +0,0 @@ |
|||
from .main import X264 |
|||
|
|||
def start(): |
|||
return X264() |
|||
|
|||
config = [{ |
|||
'name': 'x264', |
|||
'groups': [ |
|||
{ |
|||
'tab': 'searcher', |
|||
'subtab': 'providers', |
|||
'name': '#alt.binaries.hdtv.x264', |
|||
'description': 'HD movies only', |
|||
'options': [ |
|||
{ |
|||
'name': 'enabled', |
|||
'type': 'enabler', |
|||
'default': False, |
|||
}, |
|||
], |
|||
}, |
|||
], |
|||
}] |
|||
@ -1,70 +0,0 @@ |
|||
from couchpotato.core.event import fireEvent |
|||
from couchpotato.core.helpers.encoding import tryUrlencode |
|||
from couchpotato.core.helpers.variable import tryInt, getTitle |
|||
from couchpotato.core.logger import CPLog |
|||
from couchpotato.core.providers.nzb.base import NZBProvider |
|||
import re |
|||
|
|||
log = CPLog(__name__) |
|||
|
|||
|
|||
class X264(NZBProvider): |
|||
|
|||
urls = { |
|||
'download': 'http://85.214.105.230/get_nzb.php?id=%s§ion=hd', |
|||
'search': 'http://85.214.105.230/x264/requests.php?release=%s&status=FILLED&age=1300&sort=ID', |
|||
} |
|||
|
|||
regex = '<tr class="req_filled"><td class="reqid">(?P<id>.*?)</td><td class="release">(?P<title>.*?)</td>.+?<td class="age">(?P<age>.*?)</td>' |
|||
|
|||
http_time_between_calls = 2 # Seconds |
|||
|
|||
def search(self, movie, quality): |
|||
|
|||
results = [] |
|||
if self.isDisabled() or not self.isAvailable(self.urls['search'].split('requests')[0]) or not quality.get('hd', False): |
|||
return results |
|||
|
|||
q = '%s %s %s' % (getTitle(movie['library']), movie['library']['year'], quality.get('identifier')) |
|||
url = self.urls['search'] % tryUrlencode(q) |
|||
|
|||
cache_key = 'x264.%s.%s' % (movie['library']['identifier'], quality.get('identifier')) |
|||
data = self.getCache(cache_key, url) |
|||
if data: |
|||
match = re.compile(self.regex, re.DOTALL).finditer(data) |
|||
|
|||
for nzb in match: |
|||
try: |
|||
age_match = re.match('((?P<day>\d+)d)', nzb.group('age')) |
|||
age = age_match.group('day') |
|||
except: |
|||
age = 1 |
|||
|
|||
new = { |
|||
'id': nzb.group('id'), |
|||
'name': nzb.group('title'), |
|||
'type': 'nzb', |
|||
'provider': self.getName(), |
|||
'age': tryInt(age), |
|||
'size': None, |
|||
'url': self.urls['download'] % (nzb.group('id')), |
|||
'detail_url': '', |
|||
'description': '', |
|||
'check_nzb': False, |
|||
} |
|||
|
|||
new['score'] = fireEvent('score.calculate', new, movie, single = True) |
|||
is_correct_movie = fireEvent('searcher.correct_movie', |
|||
nzb = new, movie = movie, quality = quality, |
|||
imdb_results = False, single_category = False, single = True) |
|||
if is_correct_movie: |
|||
results.append(new) |
|||
self.found(new) |
|||
|
|||
return results |
|||
|
|||
def belongsTo(self, url, host = None): |
|||
match = re.match('http://85\.214\.105\.230/get_nzb\.php\?id=[0-9]*§ion=hd', url) |
|||
if match: |
|||
return self |
|||
return |
|||
@ -0,0 +1,27 @@ |
|||
#!/usr/bin/env python |
|||
# |
|||
# Copyright 2009 Facebook |
|||
# |
|||
# Licensed under the Apache License, Version 2.0 (the "License"); you may |
|||
# not use this file except in compliance with the License. You may obtain |
|||
# a copy of the License at |
|||
# |
|||
# http://www.apache.org/licenses/LICENSE-2.0 |
|||
# |
|||
# Unless required by applicable law or agreed to in writing, software |
|||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT |
|||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the |
|||
# License for the specific language governing permissions and limitations |
|||
# under the License. |
|||
|
|||
"""The Tornado web server and tools.""" |
|||
|
|||
# version is a human-readable version number. |
|||
|
|||
# version_info is a four-tuple for programmatic comparison. The first |
|||
# three numbers are the components of the version number. The fourth |
|||
# is zero for an official release, positive for a development branch, |
|||
# or negative for a release candidate (after the base version number |
|||
# has been incremented) |
|||
version = "2.2.1" |
|||
version_info = (2, 2, 1, 0) |
|||
File diff suppressed because it is too large
@ -0,0 +1,250 @@ |
|||
#!/usr/bin/env python |
|||
# |
|||
# Copyright 2009 Facebook |
|||
# |
|||
# Licensed under the Apache License, Version 2.0 (the "License"); you may |
|||
# not use this file except in compliance with the License. You may obtain |
|||
# a copy of the License at |
|||
# |
|||
# http://www.apache.org/licenses/LICENSE-2.0 |
|||
# |
|||
# Unless required by applicable law or agreed to in writing, software |
|||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT |
|||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the |
|||
# License for the specific language governing permissions and limitations |
|||
# under the License. |
|||
|
|||
"""A module to automatically restart the server when a module is modified. |
|||
|
|||
Most applications should not call this module directly. Instead, pass the |
|||
keyword argument ``debug=True`` to the `tornado.web.Application` constructor. |
|||
This will enable autoreload mode as well as checking for changes to templates |
|||
and static resources. |
|||
|
|||
This module depends on IOLoop, so it will not work in WSGI applications |
|||
and Google AppEngine. It also will not work correctly when HTTPServer's |
|||
multi-process mode is used. |
|||
""" |
|||
|
|||
from __future__ import with_statement |
|||
|
|||
import functools |
|||
import logging |
|||
import os |
|||
import pkgutil |
|||
import sys |
|||
import types |
|||
import subprocess |
|||
|
|||
from tornado import ioloop |
|||
from tornado import process |
|||
|
|||
try: |
|||
import signal |
|||
except ImportError: |
|||
signal = None |
|||
|
|||
def start(io_loop=None, check_time=500): |
|||
"""Restarts the process automatically when a module is modified. |
|||
|
|||
We run on the I/O loop, and restarting is a destructive operation, |
|||
so will terminate any pending requests. |
|||
""" |
|||
io_loop = io_loop or ioloop.IOLoop.instance() |
|||
add_reload_hook(functools.partial(_close_all_fds, io_loop)) |
|||
modify_times = {} |
|||
callback = functools.partial(_reload_on_update, modify_times) |
|||
scheduler = ioloop.PeriodicCallback(callback, check_time, io_loop=io_loop) |
|||
scheduler.start() |
|||
|
|||
def wait(): |
|||
"""Wait for a watched file to change, then restart the process. |
|||
|
|||
Intended to be used at the end of scripts like unit test runners, |
|||
to run the tests again after any source file changes (but see also |
|||
the command-line interface in `main`) |
|||
""" |
|||
io_loop = ioloop.IOLoop() |
|||
start(io_loop) |
|||
io_loop.start() |
|||
|
|||
_watched_files = set() |
|||
|
|||
def watch(filename): |
|||
"""Add a file to the watch list. |
|||
|
|||
All imported modules are watched by default. |
|||
""" |
|||
_watched_files.add(filename) |
|||
|
|||
_reload_hooks = [] |
|||
|
|||
def add_reload_hook(fn): |
|||
"""Add a function to be called before reloading the process. |
|||
|
|||
Note that for open file and socket handles it is generally |
|||
preferable to set the ``FD_CLOEXEC`` flag (using `fcntl` or |
|||
`tornado.platform.auto.set_close_exec`) instead of using a reload |
|||
hook to close them. |
|||
""" |
|||
_reload_hooks.append(fn) |
|||
|
|||
def _close_all_fds(io_loop): |
|||
for fd in io_loop._handlers.keys(): |
|||
try: |
|||
os.close(fd) |
|||
except Exception: |
|||
pass |
|||
|
|||
_reload_attempted = False |
|||
|
|||
def _reload_on_update(modify_times): |
|||
if _reload_attempted: |
|||
# We already tried to reload and it didn't work, so don't try again. |
|||
return |
|||
if process.task_id() is not None: |
|||
# We're in a child process created by fork_processes. If child |
|||
# processes restarted themselves, they'd all restart and then |
|||
# all call fork_processes again. |
|||
return |
|||
for module in sys.modules.values(): |
|||
# Some modules play games with sys.modules (e.g. email/__init__.py |
|||
# in the standard library), and occasionally this can cause strange |
|||
# failures in getattr. Just ignore anything that's not an ordinary |
|||
# module. |
|||
if not isinstance(module, types.ModuleType): continue |
|||
path = getattr(module, "__file__", None) |
|||
if not path: continue |
|||
if path.endswith(".pyc") or path.endswith(".pyo"): |
|||
path = path[:-1] |
|||
_check_file(modify_times, path) |
|||
for path in _watched_files: |
|||
_check_file(modify_times, path) |
|||
|
|||
def _check_file(modify_times, path): |
|||
try: |
|||
modified = os.stat(path).st_mtime |
|||
except Exception: |
|||
return |
|||
if path not in modify_times: |
|||
modify_times[path] = modified |
|||
return |
|||
if modify_times[path] != modified: |
|||
logging.info("%s modified; restarting server", path) |
|||
_reload() |
|||
|
|||
def _reload(): |
|||
global _reload_attempted |
|||
_reload_attempted = True |
|||
for fn in _reload_hooks: |
|||
fn() |
|||
if hasattr(signal, "setitimer"): |
|||
# Clear the alarm signal set by |
|||
# ioloop.set_blocking_log_threshold so it doesn't fire |
|||
# after the exec. |
|||
signal.setitimer(signal.ITIMER_REAL, 0, 0) |
|||
if sys.platform == 'win32': |
|||
# os.execv is broken on Windows and can't properly parse command line |
|||
# arguments and executable name if they contain whitespaces. subprocess |
|||
# fixes that behavior. |
|||
subprocess.Popen([sys.executable] + sys.argv) |
|||
sys.exit(0) |
|||
else: |
|||
try: |
|||
os.execv(sys.executable, [sys.executable] + sys.argv) |
|||
except OSError: |
|||
# Mac OS X versions prior to 10.6 do not support execv in |
|||
# a process that contains multiple threads. Instead of |
|||
# re-executing in the current process, start a new one |
|||
# and cause the current process to exit. This isn't |
|||
# ideal since the new process is detached from the parent |
|||
# terminal and thus cannot easily be killed with ctrl-C, |
|||
# but it's better than not being able to autoreload at |
|||
# all. |
|||
# Unfortunately the errno returned in this case does not |
|||
# appear to be consistent, so we can't easily check for |
|||
# this error specifically. |
|||
os.spawnv(os.P_NOWAIT, sys.executable, |
|||
[sys.executable] + sys.argv) |
|||
sys.exit(0) |
|||
|
|||
_USAGE = """\ |
|||
Usage: |
|||
python -m tornado.autoreload -m module.to.run [args...] |
|||
python -m tornado.autoreload path/to/script.py [args...] |
|||
""" |
|||
def main(): |
|||
"""Command-line wrapper to re-run a script whenever its source changes. |
|||
|
|||
Scripts may be specified by filename or module name:: |
|||
|
|||
python -m tornado.autoreload -m tornado.test.runtests |
|||
python -m tornado.autoreload tornado/test/runtests.py |
|||
|
|||
Running a script with this wrapper is similar to calling |
|||
`tornado.autoreload.wait` at the end of the script, but this wrapper |
|||
can catch import-time problems like syntax errors that would otherwise |
|||
prevent the script from reaching its call to `wait`. |
|||
""" |
|||
original_argv = sys.argv |
|||
sys.argv = sys.argv[:] |
|||
if len(sys.argv) >= 3 and sys.argv[1] == "-m": |
|||
mode = "module" |
|||
module = sys.argv[2] |
|||
del sys.argv[1:3] |
|||
elif len(sys.argv) >= 2: |
|||
mode = "script" |
|||
script = sys.argv[1] |
|||
sys.argv = sys.argv[1:] |
|||
else: |
|||
print >>sys.stderr, _USAGE |
|||
sys.exit(1) |
|||
|
|||
try: |
|||
if mode == "module": |
|||
import runpy |
|||
runpy.run_module(module, run_name="__main__", alter_sys=True) |
|||
elif mode == "script": |
|||
with open(script) as f: |
|||
global __file__ |
|||
__file__ = script |
|||
# Use globals as our "locals" dictionary so that |
|||
# something that tries to import __main__ (e.g. the unittest |
|||
# module) will see the right things. |
|||
exec f.read() in globals(), globals() |
|||
except SystemExit, e: |
|||
logging.info("Script exited with status %s", e.code) |
|||
except Exception, e: |
|||
logging.warning("Script exited with uncaught exception", exc_info=True) |
|||
if isinstance(e, SyntaxError): |
|||
watch(e.filename) |
|||
else: |
|||
logging.info("Script exited normally") |
|||
# restore sys.argv so subsequent executions will include autoreload |
|||
sys.argv = original_argv |
|||
|
|||
if mode == 'module': |
|||
# runpy did a fake import of the module as __main__, but now it's |
|||
# no longer in sys.modules. Figure out where it is and watch it. |
|||
watch(pkgutil.get_loader(module).get_filename()) |
|||
|
|||
wait() |
|||
|
|||
|
|||
if __name__ == "__main__": |
|||
# If this module is run with "python -m tornado.autoreload", the current |
|||
# directory is automatically prepended to sys.path, but not if it is |
|||
# run as "path/to/tornado/autoreload.py". The processing for "-m" rewrites |
|||
# the former to the latter, so subsequent executions won't have the same |
|||
# path as the original. Modify os.environ here to ensure that the |
|||
# re-executed process will have the same path. |
|||
# Conversely, when run as path/to/tornado/autoreload.py, the directory |
|||
# containing autoreload.py gets added to the path, but we don't want |
|||
# tornado modules importable at top level, so remove it. |
|||
path_prefix = '.' + os.pathsep |
|||
if (sys.path[0] == '' and |
|||
not os.environ.get("PYTHONPATH", "").startswith(path_prefix)): |
|||
os.environ["PYTHONPATH"] = path_prefix + os.environ.get("PYTHONPATH", "") |
|||
elif sys.path[0] == os.path.dirname(__file__): |
|||
del sys.path[0] |
|||
main() |
|||
File diff suppressed because it is too large
@ -0,0 +1,435 @@ |
|||
#!/usr/bin/env python |
|||
# |
|||
# Copyright 2009 Facebook |
|||
# |
|||
# Licensed under the Apache License, Version 2.0 (the "License"); you may |
|||
# not use this file except in compliance with the License. You may obtain |
|||
# a copy of the License at |
|||
# |
|||
# http://www.apache.org/licenses/LICENSE-2.0 |
|||
# |
|||
# Unless required by applicable law or agreed to in writing, software |
|||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT |
|||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the |
|||
# License for the specific language governing permissions and limitations |
|||
# under the License. |
|||
|
|||
"""Blocking and non-blocking HTTP client implementations using pycurl.""" |
|||
|
|||
from __future__ import with_statement |
|||
|
|||
import cStringIO |
|||
import collections |
|||
import logging |
|||
import pycurl |
|||
import threading |
|||
import time |
|||
|
|||
from tornado import httputil |
|||
from tornado import ioloop |
|||
from tornado import stack_context |
|||
|
|||
from tornado.escape import utf8 |
|||
from tornado.httpclient import HTTPRequest, HTTPResponse, HTTPError, AsyncHTTPClient, main |
|||
|
|||
class CurlAsyncHTTPClient(AsyncHTTPClient): |
|||
def initialize(self, io_loop=None, max_clients=10, |
|||
max_simultaneous_connections=None): |
|||
self.io_loop = io_loop |
|||
self._multi = pycurl.CurlMulti() |
|||
self._multi.setopt(pycurl.M_TIMERFUNCTION, self._set_timeout) |
|||
self._multi.setopt(pycurl.M_SOCKETFUNCTION, self._handle_socket) |
|||
self._curls = [_curl_create(max_simultaneous_connections) |
|||
for i in xrange(max_clients)] |
|||
self._free_list = self._curls[:] |
|||
self._requests = collections.deque() |
|||
self._fds = {} |
|||
self._timeout = None |
|||
|
|||
try: |
|||
self._socket_action = self._multi.socket_action |
|||
except AttributeError: |
|||
# socket_action is found in pycurl since 7.18.2 (it's been |
|||
# in libcurl longer than that but wasn't accessible to |
|||
# python). |
|||
logging.warning("socket_action method missing from pycurl; " |
|||
"falling back to socket_all. Upgrading " |
|||
"libcurl and pycurl will improve performance") |
|||
self._socket_action = \ |
|||
lambda fd, action: self._multi.socket_all() |
|||
|
|||
# libcurl has bugs that sometimes cause it to not report all |
|||
# relevant file descriptors and timeouts to TIMERFUNCTION/ |
|||
# SOCKETFUNCTION. Mitigate the effects of such bugs by |
|||
# forcing a periodic scan of all active requests. |
|||
self._force_timeout_callback = ioloop.PeriodicCallback( |
|||
self._handle_force_timeout, 1000, io_loop=io_loop) |
|||
self._force_timeout_callback.start() |
|||
|
|||
def close(self): |
|||
self._force_timeout_callback.stop() |
|||
for curl in self._curls: |
|||
curl.close() |
|||
self._multi.close() |
|||
self._closed = True |
|||
super(CurlAsyncHTTPClient, self).close() |
|||
|
|||
def fetch(self, request, callback, **kwargs): |
|||
if not isinstance(request, HTTPRequest): |
|||
request = HTTPRequest(url=request, **kwargs) |
|||
self._requests.append((request, stack_context.wrap(callback))) |
|||
self._process_queue() |
|||
self._set_timeout(0) |
|||
|
|||
def _handle_socket(self, event, fd, multi, data): |
|||
"""Called by libcurl when it wants to change the file descriptors |
|||
it cares about. |
|||
""" |
|||
event_map = { |
|||
pycurl.POLL_NONE: ioloop.IOLoop.NONE, |
|||
pycurl.POLL_IN: ioloop.IOLoop.READ, |
|||
pycurl.POLL_OUT: ioloop.IOLoop.WRITE, |
|||
pycurl.POLL_INOUT: ioloop.IOLoop.READ | ioloop.IOLoop.WRITE |
|||
} |
|||
if event == pycurl.POLL_REMOVE: |
|||
self.io_loop.remove_handler(fd) |
|||
del self._fds[fd] |
|||
else: |
|||
ioloop_event = event_map[event] |
|||
if fd not in self._fds: |
|||
self._fds[fd] = ioloop_event |
|||
self.io_loop.add_handler(fd, self._handle_events, |
|||
ioloop_event) |
|||
else: |
|||
self._fds[fd] = ioloop_event |
|||
self.io_loop.update_handler(fd, ioloop_event) |
|||
|
|||
def _set_timeout(self, msecs): |
|||
"""Called by libcurl to schedule a timeout.""" |
|||
if self._timeout is not None: |
|||
self.io_loop.remove_timeout(self._timeout) |
|||
self._timeout = self.io_loop.add_timeout( |
|||
time.time() + msecs/1000.0, self._handle_timeout) |
|||
|
|||
def _handle_events(self, fd, events): |
|||
"""Called by IOLoop when there is activity on one of our |
|||
file descriptors. |
|||
""" |
|||
action = 0 |
|||
if events & ioloop.IOLoop.READ: action |= pycurl.CSELECT_IN |
|||
if events & ioloop.IOLoop.WRITE: action |= pycurl.CSELECT_OUT |
|||
while True: |
|||
try: |
|||
ret, num_handles = self._socket_action(fd, action) |
|||
except pycurl.error, e: |
|||
ret = e.args[0] |
|||
if ret != pycurl.E_CALL_MULTI_PERFORM: |
|||
break |
|||
self._finish_pending_requests() |
|||
|
|||
def _handle_timeout(self): |
|||
"""Called by IOLoop when the requested timeout has passed.""" |
|||
with stack_context.NullContext(): |
|||
self._timeout = None |
|||
while True: |
|||
try: |
|||
ret, num_handles = self._socket_action( |
|||
pycurl.SOCKET_TIMEOUT, 0) |
|||
except pycurl.error, e: |
|||
ret = e.args[0] |
|||
if ret != pycurl.E_CALL_MULTI_PERFORM: |
|||
break |
|||
self._finish_pending_requests() |
|||
|
|||
# In theory, we shouldn't have to do this because curl will |
|||
# call _set_timeout whenever the timeout changes. However, |
|||
# sometimes after _handle_timeout we will need to reschedule |
|||
# immediately even though nothing has changed from curl's |
|||
# perspective. This is because when socket_action is |
|||
# called with SOCKET_TIMEOUT, libcurl decides internally which |
|||
# timeouts need to be processed by using a monotonic clock |
|||
# (where available) while tornado uses python's time.time() |
|||
# to decide when timeouts have occurred. When those clocks |
|||
# disagree on elapsed time (as they will whenever there is an |
|||
# NTP adjustment), tornado might call _handle_timeout before |
|||
# libcurl is ready. After each timeout, resync the scheduled |
|||
# timeout with libcurl's current state. |
|||
new_timeout = self._multi.timeout() |
|||
if new_timeout != -1: |
|||
self._set_timeout(new_timeout) |
|||
|
|||
def _handle_force_timeout(self): |
|||
"""Called by IOLoop periodically to ask libcurl to process any |
|||
events it may have forgotten about. |
|||
""" |
|||
with stack_context.NullContext(): |
|||
while True: |
|||
try: |
|||
ret, num_handles = self._multi.socket_all() |
|||
except pycurl.error, e: |
|||
ret = e.args[0] |
|||
if ret != pycurl.E_CALL_MULTI_PERFORM: |
|||
break |
|||
self._finish_pending_requests() |
|||
|
|||
def _finish_pending_requests(self): |
|||
"""Process any requests that were completed by the last |
|||
call to multi.socket_action. |
|||
""" |
|||
while True: |
|||
num_q, ok_list, err_list = self._multi.info_read() |
|||
for curl in ok_list: |
|||
self._finish(curl) |
|||
for curl, errnum, errmsg in err_list: |
|||
self._finish(curl, errnum, errmsg) |
|||
if num_q == 0: |
|||
break |
|||
self._process_queue() |
|||
|
|||
def _process_queue(self): |
|||
with stack_context.NullContext(): |
|||
while True: |
|||
started = 0 |
|||
while self._free_list and self._requests: |
|||
started += 1 |
|||
curl = self._free_list.pop() |
|||
(request, callback) = self._requests.popleft() |
|||
curl.info = { |
|||
"headers": httputil.HTTPHeaders(), |
|||
"buffer": cStringIO.StringIO(), |
|||
"request": request, |
|||
"callback": callback, |
|||
"curl_start_time": time.time(), |
|||
} |
|||
# Disable IPv6 to mitigate the effects of this bug |
|||
# on curl versions <= 7.21.0 |
|||
# http://sourceforge.net/tracker/?func=detail&aid=3017819&group_id=976&atid=100976 |
|||
if pycurl.version_info()[2] <= 0x71500: # 7.21.0 |
|||
curl.setopt(pycurl.IPRESOLVE, pycurl.IPRESOLVE_V4) |
|||
_curl_setup_request(curl, request, curl.info["buffer"], |
|||
curl.info["headers"]) |
|||
self._multi.add_handle(curl) |
|||
|
|||
if not started: |
|||
break |
|||
|
|||
def _finish(self, curl, curl_error=None, curl_message=None): |
|||
info = curl.info |
|||
curl.info = None |
|||
self._multi.remove_handle(curl) |
|||
self._free_list.append(curl) |
|||
buffer = info["buffer"] |
|||
if curl_error: |
|||
error = CurlError(curl_error, curl_message) |
|||
code = error.code |
|||
effective_url = None |
|||
buffer.close() |
|||
buffer = None |
|||
else: |
|||
error = None |
|||
code = curl.getinfo(pycurl.HTTP_CODE) |
|||
effective_url = curl.getinfo(pycurl.EFFECTIVE_URL) |
|||
buffer.seek(0) |
|||
# the various curl timings are documented at |
|||
# http://curl.haxx.se/libcurl/c/curl_easy_getinfo.html |
|||
time_info = dict( |
|||
queue=info["curl_start_time"] - info["request"].start_time, |
|||
namelookup=curl.getinfo(pycurl.NAMELOOKUP_TIME), |
|||
connect=curl.getinfo(pycurl.CONNECT_TIME), |
|||
pretransfer=curl.getinfo(pycurl.PRETRANSFER_TIME), |
|||
starttransfer=curl.getinfo(pycurl.STARTTRANSFER_TIME), |
|||
total=curl.getinfo(pycurl.TOTAL_TIME), |
|||
redirect=curl.getinfo(pycurl.REDIRECT_TIME), |
|||
) |
|||
try: |
|||
info["callback"](HTTPResponse( |
|||
request=info["request"], code=code, headers=info["headers"], |
|||
buffer=buffer, effective_url=effective_url, error=error, |
|||
request_time=time.time() - info["curl_start_time"], |
|||
time_info=time_info)) |
|||
except Exception: |
|||
self.handle_callback_exception(info["callback"]) |
|||
|
|||
|
|||
def handle_callback_exception(self, callback): |
|||
self.io_loop.handle_callback_exception(callback) |
|||
|
|||
|
|||
class CurlError(HTTPError): |
|||
def __init__(self, errno, message): |
|||
HTTPError.__init__(self, 599, message) |
|||
self.errno = errno |
|||
|
|||
|
|||
def _curl_create(max_simultaneous_connections=None): |
|||
curl = pycurl.Curl() |
|||
if logging.getLogger().isEnabledFor(logging.DEBUG): |
|||
curl.setopt(pycurl.VERBOSE, 1) |
|||
curl.setopt(pycurl.DEBUGFUNCTION, _curl_debug) |
|||
curl.setopt(pycurl.MAXCONNECTS, max_simultaneous_connections or 5) |
|||
return curl |
|||
|
|||
|
|||
def _curl_setup_request(curl, request, buffer, headers): |
|||
curl.setopt(pycurl.URL, utf8(request.url)) |
|||
|
|||
# libcurl's magic "Expect: 100-continue" behavior causes delays |
|||
# with servers that don't support it (which include, among others, |
|||
# Google's OpenID endpoint). Additionally, this behavior has |
|||
# a bug in conjunction with the curl_multi_socket_action API |
|||
# (https://sourceforge.net/tracker/?func=detail&atid=100976&aid=3039744&group_id=976), |
|||
# which increases the delays. It's more trouble than it's worth, |
|||
# so just turn off the feature (yes, setting Expect: to an empty |
|||
# value is the official way to disable this) |
|||
if "Expect" not in request.headers: |
|||
request.headers["Expect"] = "" |
|||
|
|||
# libcurl adds Pragma: no-cache by default; disable that too |
|||
if "Pragma" not in request.headers: |
|||
request.headers["Pragma"] = "" |
|||
|
|||
# Request headers may be either a regular dict or HTTPHeaders object |
|||
if isinstance(request.headers, httputil.HTTPHeaders): |
|||
curl.setopt(pycurl.HTTPHEADER, |
|||
[utf8("%s: %s" % i) for i in request.headers.get_all()]) |
|||
else: |
|||
curl.setopt(pycurl.HTTPHEADER, |
|||
[utf8("%s: %s" % i) for i in request.headers.iteritems()]) |
|||
|
|||
if request.header_callback: |
|||
curl.setopt(pycurl.HEADERFUNCTION, request.header_callback) |
|||
else: |
|||
curl.setopt(pycurl.HEADERFUNCTION, |
|||
lambda line: _curl_header_callback(headers, line)) |
|||
if request.streaming_callback: |
|||
curl.setopt(pycurl.WRITEFUNCTION, request.streaming_callback) |
|||
else: |
|||
curl.setopt(pycurl.WRITEFUNCTION, buffer.write) |
|||
curl.setopt(pycurl.FOLLOWLOCATION, request.follow_redirects) |
|||
curl.setopt(pycurl.MAXREDIRS, request.max_redirects) |
|||
curl.setopt(pycurl.CONNECTTIMEOUT_MS, int(1000 * request.connect_timeout)) |
|||
curl.setopt(pycurl.TIMEOUT_MS, int(1000 * request.request_timeout)) |
|||
if request.user_agent: |
|||
curl.setopt(pycurl.USERAGENT, utf8(request.user_agent)) |
|||
else: |
|||
curl.setopt(pycurl.USERAGENT, "Mozilla/5.0 (compatible; pycurl)") |
|||
if request.network_interface: |
|||
curl.setopt(pycurl.INTERFACE, request.network_interface) |
|||
if request.use_gzip: |
|||
curl.setopt(pycurl.ENCODING, "gzip,deflate") |
|||
else: |
|||
curl.setopt(pycurl.ENCODING, "none") |
|||
if request.proxy_host and request.proxy_port: |
|||
curl.setopt(pycurl.PROXY, request.proxy_host) |
|||
curl.setopt(pycurl.PROXYPORT, request.proxy_port) |
|||
if request.proxy_username: |
|||
credentials = '%s:%s' % (request.proxy_username, |
|||
request.proxy_password) |
|||
curl.setopt(pycurl.PROXYUSERPWD, credentials) |
|||
else: |
|||
curl.setopt(pycurl.PROXY, '') |
|||
if request.validate_cert: |
|||
curl.setopt(pycurl.SSL_VERIFYPEER, 1) |
|||
curl.setopt(pycurl.SSL_VERIFYHOST, 2) |
|||
else: |
|||
curl.setopt(pycurl.SSL_VERIFYPEER, 0) |
|||
curl.setopt(pycurl.SSL_VERIFYHOST, 0) |
|||
if request.ca_certs is not None: |
|||
curl.setopt(pycurl.CAINFO, request.ca_certs) |
|||
else: |
|||
# There is no way to restore pycurl.CAINFO to its default value |
|||
# (Using unsetopt makes it reject all certificates). |
|||
# I don't see any way to read the default value from python so it |
|||
# can be restored later. We'll have to just leave CAINFO untouched |
|||
# if no ca_certs file was specified, and require that if any |
|||
# request uses a custom ca_certs file, they all must. |
|||
pass |
|||
|
|||
if request.allow_ipv6 is False: |
|||
# Curl behaves reasonably when DNS resolution gives an ipv6 address |
|||
# that we can't reach, so allow ipv6 unless the user asks to disable. |
|||
# (but see version check in _process_queue above) |
|||
curl.setopt(pycurl.IPRESOLVE, pycurl.IPRESOLVE_V4) |
|||
|
|||
# Set the request method through curl's irritating interface which makes |
|||
# up names for almost every single method |
|||
curl_options = { |
|||
"GET": pycurl.HTTPGET, |
|||
"POST": pycurl.POST, |
|||
"PUT": pycurl.UPLOAD, |
|||
"HEAD": pycurl.NOBODY, |
|||
} |
|||
custom_methods = set(["DELETE"]) |
|||
for o in curl_options.values(): |
|||
curl.setopt(o, False) |
|||
if request.method in curl_options: |
|||
curl.unsetopt(pycurl.CUSTOMREQUEST) |
|||
curl.setopt(curl_options[request.method], True) |
|||
elif request.allow_nonstandard_methods or request.method in custom_methods: |
|||
curl.setopt(pycurl.CUSTOMREQUEST, request.method) |
|||
else: |
|||
raise KeyError('unknown method ' + request.method) |
|||
|
|||
# Handle curl's cryptic options for every individual HTTP method |
|||
if request.method in ("POST", "PUT"): |
|||
request_buffer = cStringIO.StringIO(utf8(request.body)) |
|||
curl.setopt(pycurl.READFUNCTION, request_buffer.read) |
|||
if request.method == "POST": |
|||
def ioctl(cmd): |
|||
if cmd == curl.IOCMD_RESTARTREAD: |
|||
request_buffer.seek(0) |
|||
curl.setopt(pycurl.IOCTLFUNCTION, ioctl) |
|||
curl.setopt(pycurl.POSTFIELDSIZE, len(request.body)) |
|||
else: |
|||
curl.setopt(pycurl.INFILESIZE, len(request.body)) |
|||
|
|||
if request.auth_username is not None: |
|||
userpwd = "%s:%s" % (request.auth_username, request.auth_password or '') |
|||
curl.setopt(pycurl.HTTPAUTH, pycurl.HTTPAUTH_BASIC) |
|||
curl.setopt(pycurl.USERPWD, utf8(userpwd)) |
|||
logging.debug("%s %s (username: %r)", request.method, request.url, |
|||
request.auth_username) |
|||
else: |
|||
curl.unsetopt(pycurl.USERPWD) |
|||
logging.debug("%s %s", request.method, request.url) |
|||
|
|||
if request.client_key is not None or request.client_cert is not None: |
|||
raise ValueError("Client certificate not supported with curl_httpclient") |
|||
|
|||
if threading.activeCount() > 1: |
|||
# libcurl/pycurl is not thread-safe by default. When multiple threads |
|||
# are used, signals should be disabled. This has the side effect |
|||
# of disabling DNS timeouts in some environments (when libcurl is |
|||
# not linked against ares), so we don't do it when there is only one |
|||
# thread. Applications that use many short-lived threads may need |
|||
# to set NOSIGNAL manually in a prepare_curl_callback since |
|||
# there may not be any other threads running at the time we call |
|||
# threading.activeCount. |
|||
curl.setopt(pycurl.NOSIGNAL, 1) |
|||
if request.prepare_curl_callback is not None: |
|||
request.prepare_curl_callback(curl) |
|||
|
|||
|
|||
def _curl_header_callback(headers, header_line): |
|||
# header_line as returned by curl includes the end-of-line characters. |
|||
header_line = header_line.strip() |
|||
if header_line.startswith("HTTP/"): |
|||
headers.clear() |
|||
return |
|||
if not header_line: |
|||
return |
|||
headers.parse_line(header_line) |
|||
|
|||
def _curl_debug(debug_type, debug_msg): |
|||
debug_types = ('I', '<', '>', '<', '>') |
|||
if debug_type == 0: |
|||
logging.debug('%s', debug_msg.strip()) |
|||
elif debug_type in (1, 2): |
|||
for line in debug_msg.splitlines(): |
|||
logging.debug('%s %s', debug_types[debug_type], line) |
|||
elif debug_type == 4: |
|||
logging.debug('%s %r', debug_types[debug_type], debug_msg) |
|||
|
|||
if __name__ == "__main__": |
|||
AsyncHTTPClient.configure(CurlAsyncHTTPClient) |
|||
main() |
|||
@ -0,0 +1,229 @@ |
|||
#!/usr/bin/env python |
|||
# |
|||
# Copyright 2009 Facebook |
|||
# |
|||
# Licensed under the Apache License, Version 2.0 (the "License"); you may |
|||
# not use this file except in compliance with the License. You may obtain |
|||
# a copy of the License at |
|||
# |
|||
# http://www.apache.org/licenses/LICENSE-2.0 |
|||
# |
|||
# Unless required by applicable law or agreed to in writing, software |
|||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT |
|||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the |
|||
# License for the specific language governing permissions and limitations |
|||
# under the License. |
|||
|
|||
"""A lightweight wrapper around MySQLdb.""" |
|||
|
|||
import copy |
|||
import MySQLdb.constants |
|||
import MySQLdb.converters |
|||
import MySQLdb.cursors |
|||
import itertools |
|||
import logging |
|||
import time |
|||
|
|||
class Connection(object): |
|||
"""A lightweight wrapper around MySQLdb DB-API connections. |
|||
|
|||
The main value we provide is wrapping rows in a dict/object so that |
|||
columns can be accessed by name. Typical usage:: |
|||
|
|||
db = database.Connection("localhost", "mydatabase") |
|||
for article in db.query("SELECT * FROM articles"): |
|||
print article.title |
|||
|
|||
Cursors are hidden by the implementation, but other than that, the methods |
|||
are very similar to the DB-API. |
|||
|
|||
We explicitly set the timezone to UTC and the character encoding to |
|||
UTF-8 on all connections to avoid time zone and encoding errors. |
|||
""" |
|||
def __init__(self, host, database, user=None, password=None, |
|||
max_idle_time=7*3600): |
|||
self.host = host |
|||
self.database = database |
|||
self.max_idle_time = max_idle_time |
|||
|
|||
args = dict(conv=CONVERSIONS, use_unicode=True, charset="utf8", |
|||
db=database, init_command='SET time_zone = "+0:00"', |
|||
sql_mode="TRADITIONAL") |
|||
if user is not None: |
|||
args["user"] = user |
|||
if password is not None: |
|||
args["passwd"] = password |
|||
|
|||
# We accept a path to a MySQL socket file or a host(:port) string |
|||
if "/" in host: |
|||
args["unix_socket"] = host |
|||
else: |
|||
self.socket = None |
|||
pair = host.split(":") |
|||
if len(pair) == 2: |
|||
args["host"] = pair[0] |
|||
args["port"] = int(pair[1]) |
|||
else: |
|||
args["host"] = host |
|||
args["port"] = 3306 |
|||
|
|||
self._db = None |
|||
self._db_args = args |
|||
self._last_use_time = time.time() |
|||
try: |
|||
self.reconnect() |
|||
except Exception: |
|||
logging.error("Cannot connect to MySQL on %s", self.host, |
|||
exc_info=True) |
|||
|
|||
def __del__(self): |
|||
self.close() |
|||
|
|||
def close(self): |
|||
"""Closes this database connection.""" |
|||
if getattr(self, "_db", None) is not None: |
|||
self._db.close() |
|||
self._db = None |
|||
|
|||
def reconnect(self): |
|||
"""Closes the existing database connection and re-opens it.""" |
|||
self.close() |
|||
self._db = MySQLdb.connect(**self._db_args) |
|||
self._db.autocommit(True) |
|||
|
|||
def iter(self, query, *parameters): |
|||
"""Returns an iterator for the given query and parameters.""" |
|||
self._ensure_connected() |
|||
cursor = MySQLdb.cursors.SSCursor(self._db) |
|||
try: |
|||
self._execute(cursor, query, parameters) |
|||
column_names = [d[0] for d in cursor.description] |
|||
for row in cursor: |
|||
yield Row(zip(column_names, row)) |
|||
finally: |
|||
cursor.close() |
|||
|
|||
def query(self, query, *parameters): |
|||
"""Returns a row list for the given query and parameters.""" |
|||
cursor = self._cursor() |
|||
try: |
|||
self._execute(cursor, query, parameters) |
|||
column_names = [d[0] for d in cursor.description] |
|||
return [Row(itertools.izip(column_names, row)) for row in cursor] |
|||
finally: |
|||
cursor.close() |
|||
|
|||
def get(self, query, *parameters): |
|||
"""Returns the first row returned for the given query.""" |
|||
rows = self.query(query, *parameters) |
|||
if not rows: |
|||
return None |
|||
elif len(rows) > 1: |
|||
raise Exception("Multiple rows returned for Database.get() query") |
|||
else: |
|||
return rows[0] |
|||
|
|||
# rowcount is a more reasonable default return value than lastrowid, |
|||
# but for historical compatibility execute() must return lastrowid. |
|||
def execute(self, query, *parameters): |
|||
"""Executes the given query, returning the lastrowid from the query.""" |
|||
return self.execute_lastrowid(query, *parameters) |
|||
|
|||
def execute_lastrowid(self, query, *parameters): |
|||
"""Executes the given query, returning the lastrowid from the query.""" |
|||
cursor = self._cursor() |
|||
try: |
|||
self._execute(cursor, query, parameters) |
|||
return cursor.lastrowid |
|||
finally: |
|||
cursor.close() |
|||
|
|||
def execute_rowcount(self, query, *parameters): |
|||
"""Executes the given query, returning the rowcount from the query.""" |
|||
cursor = self._cursor() |
|||
try: |
|||
self._execute(cursor, query, parameters) |
|||
return cursor.rowcount |
|||
finally: |
|||
cursor.close() |
|||
|
|||
def executemany(self, query, parameters): |
|||
"""Executes the given query against all the given param sequences. |
|||
|
|||
We return the lastrowid from the query. |
|||
""" |
|||
return self.executemany_lastrowid(query, parameters) |
|||
|
|||
def executemany_lastrowid(self, query, parameters): |
|||
"""Executes the given query against all the given param sequences. |
|||
|
|||
We return the lastrowid from the query. |
|||
""" |
|||
cursor = self._cursor() |
|||
try: |
|||
cursor.executemany(query, parameters) |
|||
return cursor.lastrowid |
|||
finally: |
|||
cursor.close() |
|||
|
|||
def executemany_rowcount(self, query, parameters): |
|||
"""Executes the given query against all the given param sequences. |
|||
|
|||
We return the rowcount from the query. |
|||
""" |
|||
cursor = self._cursor() |
|||
try: |
|||
cursor.executemany(query, parameters) |
|||
return cursor.rowcount |
|||
finally: |
|||
cursor.close() |
|||
|
|||
def _ensure_connected(self): |
|||
# Mysql by default closes client connections that are idle for |
|||
# 8 hours, but the client library does not report this fact until |
|||
# you try to perform a query and it fails. Protect against this |
|||
# case by preemptively closing and reopening the connection |
|||
# if it has been idle for too long (7 hours by default). |
|||
if (self._db is None or |
|||
(time.time() - self._last_use_time > self.max_idle_time)): |
|||
self.reconnect() |
|||
self._last_use_time = time.time() |
|||
|
|||
def _cursor(self): |
|||
self._ensure_connected() |
|||
return self._db.cursor() |
|||
|
|||
def _execute(self, cursor, query, parameters): |
|||
try: |
|||
return cursor.execute(query, parameters) |
|||
except OperationalError: |
|||
logging.error("Error connecting to MySQL on %s", self.host) |
|||
self.close() |
|||
raise |
|||
|
|||
|
|||
class Row(dict): |
|||
"""A dict that allows for object-like property access syntax.""" |
|||
def __getattr__(self, name): |
|||
try: |
|||
return self[name] |
|||
except KeyError: |
|||
raise AttributeError(name) |
|||
|
|||
|
|||
# Fix the access conversions to properly recognize unicode/binary |
|||
FIELD_TYPE = MySQLdb.constants.FIELD_TYPE |
|||
FLAG = MySQLdb.constants.FLAG |
|||
CONVERSIONS = copy.copy(MySQLdb.converters.conversions) |
|||
|
|||
field_types = [FIELD_TYPE.BLOB, FIELD_TYPE.STRING, FIELD_TYPE.VAR_STRING] |
|||
if 'VARCHAR' in vars(FIELD_TYPE): |
|||
field_types.append(FIELD_TYPE.VARCHAR) |
|||
|
|||
for field_type in field_types: |
|||
CONVERSIONS[field_type] = [(FLAG.BINARY, str)] + CONVERSIONS[field_type] |
|||
|
|||
|
|||
# Alias some common MySQL exceptions |
|||
IntegrityError = MySQLdb.IntegrityError |
|||
OperationalError = MySQLdb.OperationalError |
|||
@ -0,0 +1,112 @@ |
|||
/*
|
|||
* Copyright 2009 Facebook |
|||
* |
|||
* Licensed under the Apache License, Version 2.0 (the "License"); you may |
|||
* not use this file except in compliance with the License. You may obtain |
|||
* a copy of the License at |
|||
* |
|||
* http://www.apache.org/licenses/LICENSE-2.0
|
|||
* |
|||
* Unless required by applicable law or agreed to in writing, software |
|||
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT |
|||
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the |
|||
* License for the specific language governing permissions and limitations |
|||
* under the License. |
|||
*/ |
|||
|
|||
#include "Python.h" |
|||
#include <string.h> |
|||
#include <sys/epoll.h> |
|||
|
|||
#define MAX_EVENTS 24 |
|||
|
|||
/*
|
|||
* Simple wrapper around epoll_create. |
|||
*/ |
|||
static PyObject* _epoll_create(void) { |
|||
int fd = epoll_create(MAX_EVENTS); |
|||
if (fd == -1) { |
|||
PyErr_SetFromErrno(PyExc_Exception); |
|||
return NULL; |
|||
} |
|||
|
|||
return PyInt_FromLong(fd); |
|||
} |
|||
|
|||
/*
|
|||
* Simple wrapper around epoll_ctl. We throw an exception if the call fails |
|||
* rather than returning the error code since it is an infrequent (and likely |
|||
* catastrophic) event when it does happen. |
|||
*/ |
|||
static PyObject* _epoll_ctl(PyObject* self, PyObject* args) { |
|||
int epfd, op, fd, events; |
|||
struct epoll_event event; |
|||
|
|||
if (!PyArg_ParseTuple(args, "iiiI", &epfd, &op, &fd, &events)) { |
|||
return NULL; |
|||
} |
|||
|
|||
memset(&event, 0, sizeof(event)); |
|||
event.events = events; |
|||
event.data.fd = fd; |
|||
if (epoll_ctl(epfd, op, fd, &event) == -1) { |
|||
PyErr_SetFromErrno(PyExc_OSError); |
|||
return NULL; |
|||
} |
|||
|
|||
Py_INCREF(Py_None); |
|||
return Py_None; |
|||
} |
|||
|
|||
/*
|
|||
* Simple wrapper around epoll_wait. We return None if the call times out and |
|||
* throw an exception if an error occurs. Otherwise, we return a list of |
|||
* (fd, event) tuples. |
|||
*/ |
|||
static PyObject* _epoll_wait(PyObject* self, PyObject* args) { |
|||
struct epoll_event events[MAX_EVENTS]; |
|||
int epfd, timeout, num_events, i; |
|||
PyObject* list; |
|||
PyObject* tuple; |
|||
|
|||
if (!PyArg_ParseTuple(args, "ii", &epfd, &timeout)) { |
|||
return NULL; |
|||
} |
|||
|
|||
Py_BEGIN_ALLOW_THREADS |
|||
num_events = epoll_wait(epfd, events, MAX_EVENTS, timeout); |
|||
Py_END_ALLOW_THREADS |
|||
if (num_events == -1) { |
|||
PyErr_SetFromErrno(PyExc_Exception); |
|||
return NULL; |
|||
} |
|||
|
|||
list = PyList_New(num_events); |
|||
for (i = 0; i < num_events; i++) { |
|||
tuple = PyTuple_New(2); |
|||
PyTuple_SET_ITEM(tuple, 0, PyInt_FromLong(events[i].data.fd)); |
|||
PyTuple_SET_ITEM(tuple, 1, PyInt_FromLong(events[i].events)); |
|||
PyList_SET_ITEM(list, i, tuple); |
|||
} |
|||
return list; |
|||
} |
|||
|
|||
/*
|
|||
* Our method declararations |
|||
*/ |
|||
static PyMethodDef kEpollMethods[] = { |
|||
{"epoll_create", (PyCFunction)_epoll_create, METH_NOARGS, |
|||
"Create an epoll file descriptor"}, |
|||
{"epoll_ctl", _epoll_ctl, METH_VARARGS, |
|||
"Control an epoll file descriptor"}, |
|||
{"epoll_wait", _epoll_wait, METH_VARARGS, |
|||
"Wait for events on an epoll file descriptor"}, |
|||
{NULL, NULL, 0, NULL} |
|||
}; |
|||
|
|||
/*
|
|||
* Module initialization |
|||
*/ |
|||
PyMODINIT_FUNC initepoll(void) { |
|||
Py_InitModule("epoll", kEpollMethods); |
|||
} |
|||
@ -0,0 +1,327 @@ |
|||
#!/usr/bin/env python |
|||
# |
|||
# Copyright 2009 Facebook |
|||
# |
|||
# Licensed under the Apache License, Version 2.0 (the "License"); you may |
|||
# not use this file except in compliance with the License. You may obtain |
|||
# a copy of the License at |
|||
# |
|||
# http://www.apache.org/licenses/LICENSE-2.0 |
|||
# |
|||
# Unless required by applicable law or agreed to in writing, software |
|||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT |
|||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the |
|||
# License for the specific language governing permissions and limitations |
|||
# under the License. |
|||
|
|||
"""Escaping/unescaping methods for HTML, JSON, URLs, and others. |
|||
|
|||
Also includes a few other miscellaneous string manipulation functions that |
|||
have crept in over time. |
|||
""" |
|||
|
|||
import htmlentitydefs |
|||
import re |
|||
import sys |
|||
import urllib |
|||
|
|||
# Python3 compatibility: On python2.5, introduce the bytes alias from 2.6 |
|||
try: bytes |
|||
except Exception: bytes = str |
|||
|
|||
try: |
|||
from urlparse import parse_qs # Python 2.6+ |
|||
except ImportError: |
|||
from cgi import parse_qs |
|||
|
|||
# json module is in the standard library as of python 2.6; fall back to |
|||
# simplejson if present for older versions. |
|||
try: |
|||
import json |
|||
assert hasattr(json, "loads") and hasattr(json, "dumps") |
|||
_json_decode = json.loads |
|||
_json_encode = json.dumps |
|||
except Exception: |
|||
try: |
|||
import simplejson |
|||
_json_decode = lambda s: simplejson.loads(_unicode(s)) |
|||
_json_encode = lambda v: simplejson.dumps(v) |
|||
except ImportError: |
|||
try: |
|||
# For Google AppEngine |
|||
from django.utils import simplejson |
|||
_json_decode = lambda s: simplejson.loads(_unicode(s)) |
|||
_json_encode = lambda v: simplejson.dumps(v) |
|||
except ImportError: |
|||
def _json_decode(s): |
|||
raise NotImplementedError( |
|||
"A JSON parser is required, e.g., simplejson at " |
|||
"http://pypi.python.org/pypi/simplejson/") |
|||
_json_encode = _json_decode |
|||
|
|||
|
|||
_XHTML_ESCAPE_RE = re.compile('[&<>"]') |
|||
_XHTML_ESCAPE_DICT = {'&': '&', '<': '<', '>': '>', '"': '"'} |
|||
def xhtml_escape(value): |
|||
"""Escapes a string so it is valid within XML or XHTML.""" |
|||
return _XHTML_ESCAPE_RE.sub(lambda match: _XHTML_ESCAPE_DICT[match.group(0)], |
|||
to_basestring(value)) |
|||
|
|||
|
|||
def xhtml_unescape(value): |
|||
"""Un-escapes an XML-escaped string.""" |
|||
return re.sub(r"&(#?)(\w+?);", _convert_entity, _unicode(value)) |
|||
|
|||
|
|||
def json_encode(value): |
|||
"""JSON-encodes the given Python object.""" |
|||
# JSON permits but does not require forward slashes to be escaped. |
|||
# This is useful when json data is emitted in a <script> tag |
|||
# in HTML, as it prevents </script> tags from prematurely terminating |
|||
# the javscript. Some json libraries do this escaping by default, |
|||
# although python's standard library does not, so we do it here. |
|||
# http://stackoverflow.com/questions/1580647/json-why-are-forward-slashes-escaped |
|||
return _json_encode(recursive_unicode(value)).replace("</", "<\\/") |
|||
|
|||
|
|||
def json_decode(value): |
|||
"""Returns Python objects for the given JSON string.""" |
|||
return _json_decode(to_basestring(value)) |
|||
|
|||
|
|||
def squeeze(value): |
|||
"""Replace all sequences of whitespace chars with a single space.""" |
|||
return re.sub(r"[\x00-\x20]+", " ", value).strip() |
|||
|
|||
|
|||
def url_escape(value): |
|||
"""Returns a valid URL-encoded version of the given value.""" |
|||
return urllib.quote_plus(utf8(value)) |
|||
|
|||
# python 3 changed things around enough that we need two separate |
|||
# implementations of url_unescape. We also need our own implementation |
|||
# of parse_qs since python 3's version insists on decoding everything. |
|||
if sys.version_info[0] < 3: |
|||
def url_unescape(value, encoding='utf-8'): |
|||
"""Decodes the given value from a URL. |
|||
|
|||
The argument may be either a byte or unicode string. |
|||
|
|||
If encoding is None, the result will be a byte string. Otherwise, |
|||
the result is a unicode string in the specified encoding. |
|||
""" |
|||
if encoding is None: |
|||
return urllib.unquote_plus(utf8(value)) |
|||
else: |
|||
return unicode(urllib.unquote_plus(utf8(value)), encoding) |
|||
|
|||
parse_qs_bytes = parse_qs |
|||
else: |
|||
def url_unescape(value, encoding='utf-8'): |
|||
"""Decodes the given value from a URL. |
|||
|
|||
The argument may be either a byte or unicode string. |
|||
|
|||
If encoding is None, the result will be a byte string. Otherwise, |
|||
the result is a unicode string in the specified encoding. |
|||
""" |
|||
if encoding is None: |
|||
return urllib.parse.unquote_to_bytes(value) |
|||
else: |
|||
return urllib.unquote_plus(to_basestring(value), encoding=encoding) |
|||
|
|||
def parse_qs_bytes(qs, keep_blank_values=False, strict_parsing=False): |
|||
"""Parses a query string like urlparse.parse_qs, but returns the |
|||
values as byte strings. |
|||
|
|||
Keys still become type str (interpreted as latin1 in python3!) |
|||
because it's too painful to keep them as byte strings in |
|||
python3 and in practice they're nearly always ascii anyway. |
|||
""" |
|||
# This is gross, but python3 doesn't give us another way. |
|||
# Latin1 is the universal donor of character encodings. |
|||
result = parse_qs(qs, keep_blank_values, strict_parsing, |
|||
encoding='latin1', errors='strict') |
|||
encoded = {} |
|||
for k,v in result.iteritems(): |
|||
encoded[k] = [i.encode('latin1') for i in v] |
|||
return encoded |
|||
|
|||
|
|||
|
|||
_UTF8_TYPES = (bytes, type(None)) |
|||
def utf8(value): |
|||
"""Converts a string argument to a byte string. |
|||
|
|||
If the argument is already a byte string or None, it is returned unchanged. |
|||
Otherwise it must be a unicode string and is encoded as utf8. |
|||
""" |
|||
if isinstance(value, _UTF8_TYPES): |
|||
return value |
|||
assert isinstance(value, unicode) |
|||
return value.encode("utf-8") |
|||
|
|||
_TO_UNICODE_TYPES = (unicode, type(None)) |
|||
def to_unicode(value): |
|||
"""Converts a string argument to a unicode string. |
|||
|
|||
If the argument is already a unicode string or None, it is returned |
|||
unchanged. Otherwise it must be a byte string and is decoded as utf8. |
|||
""" |
|||
if isinstance(value, _TO_UNICODE_TYPES): |
|||
return value |
|||
assert isinstance(value, bytes) |
|||
return value.decode("utf-8") |
|||
|
|||
# to_unicode was previously named _unicode not because it was private, |
|||
# but to avoid conflicts with the built-in unicode() function/type |
|||
_unicode = to_unicode |
|||
|
|||
# When dealing with the standard library across python 2 and 3 it is |
|||
# sometimes useful to have a direct conversion to the native string type |
|||
if str is unicode: |
|||
native_str = to_unicode |
|||
else: |
|||
native_str = utf8 |
|||
|
|||
_BASESTRING_TYPES = (basestring, type(None)) |
|||
def to_basestring(value): |
|||
"""Converts a string argument to a subclass of basestring. |
|||
|
|||
In python2, byte and unicode strings are mostly interchangeable, |
|||
so functions that deal with a user-supplied argument in combination |
|||
with ascii string constants can use either and should return the type |
|||
the user supplied. In python3, the two types are not interchangeable, |
|||
so this method is needed to convert byte strings to unicode. |
|||
""" |
|||
if isinstance(value, _BASESTRING_TYPES): |
|||
return value |
|||
assert isinstance(value, bytes) |
|||
return value.decode("utf-8") |
|||
|
|||
def recursive_unicode(obj): |
|||
"""Walks a simple data structure, converting byte strings to unicode. |
|||
|
|||
Supports lists, tuples, and dictionaries. |
|||
""" |
|||
if isinstance(obj, dict): |
|||
return dict((recursive_unicode(k), recursive_unicode(v)) for (k,v) in obj.iteritems()) |
|||
elif isinstance(obj, list): |
|||
return list(recursive_unicode(i) for i in obj) |
|||
elif isinstance(obj, tuple): |
|||
return tuple(recursive_unicode(i) for i in obj) |
|||
elif isinstance(obj, bytes): |
|||
return to_unicode(obj) |
|||
else: |
|||
return obj |
|||
|
|||
# I originally used the regex from |
|||
# http://daringfireball.net/2010/07/improved_regex_for_matching_urls |
|||
# but it gets all exponential on certain patterns (such as too many trailing |
|||
# dots), causing the regex matcher to never return. |
|||
# This regex should avoid those problems. |
|||
_URL_RE = re.compile(ur"""\b((?:([\w-]+):(/{1,3})|www[.])(?:(?:(?:[^\s&()]|&|")*(?:[^!"#$%&'()*+,.:;<=>?@\[\]^`{|}~\s]))|(?:\((?:[^\s&()]|&|")*\)))+)""") |
|||
|
|||
|
|||
def linkify(text, shorten=False, extra_params="", |
|||
require_protocol=False, permitted_protocols=["http", "https"]): |
|||
"""Converts plain text into HTML with links. |
|||
|
|||
For example: ``linkify("Hello http://tornadoweb.org!")`` would return |
|||
``Hello <a href="http://tornadoweb.org">http://tornadoweb.org</a>!`` |
|||
|
|||
Parameters: |
|||
|
|||
shorten: Long urls will be shortened for display. |
|||
|
|||
extra_params: Extra text to include in the link tag, |
|||
e.g. linkify(text, extra_params='rel="nofollow" class="external"') |
|||
|
|||
require_protocol: Only linkify urls which include a protocol. If this is |
|||
False, urls such as www.facebook.com will also be linkified. |
|||
|
|||
permitted_protocols: List (or set) of protocols which should be linkified, |
|||
e.g. linkify(text, permitted_protocols=["http", "ftp", "mailto"]). |
|||
It is very unsafe to include protocols such as "javascript". |
|||
""" |
|||
if extra_params: |
|||
extra_params = " " + extra_params.strip() |
|||
|
|||
def make_link(m): |
|||
url = m.group(1) |
|||
proto = m.group(2) |
|||
if require_protocol and not proto: |
|||
return url # not protocol, no linkify |
|||
|
|||
if proto and proto not in permitted_protocols: |
|||
return url # bad protocol, no linkify |
|||
|
|||
href = m.group(1) |
|||
if not proto: |
|||
href = "http://" + href # no proto specified, use http |
|||
|
|||
params = extra_params |
|||
|
|||
# clip long urls. max_len is just an approximation |
|||
max_len = 30 |
|||
if shorten and len(url) > max_len: |
|||
before_clip = url |
|||
if proto: |
|||
proto_len = len(proto) + 1 + len(m.group(3) or "") # +1 for : |
|||
else: |
|||
proto_len = 0 |
|||
|
|||
parts = url[proto_len:].split("/") |
|||
if len(parts) > 1: |
|||
# Grab the whole host part plus the first bit of the path |
|||
# The path is usually not that interesting once shortened |
|||
# (no more slug, etc), so it really just provides a little |
|||
# extra indication of shortening. |
|||
url = url[:proto_len] + parts[0] + "/" + \ |
|||
parts[1][:8].split('?')[0].split('.')[0] |
|||
|
|||
if len(url) > max_len * 1.5: # still too long |
|||
url = url[:max_len] |
|||
|
|||
if url != before_clip: |
|||
amp = url.rfind('&') |
|||
# avoid splitting html char entities |
|||
if amp > max_len - 5: |
|||
url = url[:amp] |
|||
url += "..." |
|||
|
|||
if len(url) >= len(before_clip): |
|||
url = before_clip |
|||
else: |
|||
# full url is visible on mouse-over (for those who don't |
|||
# have a status bar, such as Safari by default) |
|||
params += ' title="%s"' % href |
|||
|
|||
return u'<a href="%s"%s>%s</a>' % (href, params, url) |
|||
|
|||
# First HTML-escape so that our strings are all safe. |
|||
# The regex is modified to avoid character entites other than & so |
|||
# that we won't pick up ", etc. |
|||
text = _unicode(xhtml_escape(text)) |
|||
return _URL_RE.sub(make_link, text) |
|||
|
|||
|
|||
def _convert_entity(m): |
|||
if m.group(1) == "#": |
|||
try: |
|||
return unichr(int(m.group(2))) |
|||
except ValueError: |
|||
return "&#%s;" % m.group(2) |
|||
try: |
|||
return _HTML_UNICODE_MAP[m.group(2)] |
|||
except KeyError: |
|||
return "&%s;" % m.group(2) |
|||
|
|||
|
|||
def _build_unicode_map(): |
|||
unicode_map = {} |
|||
for name, value in htmlentitydefs.name2codepoint.iteritems(): |
|||
unicode_map[name] = unichr(value) |
|||
return unicode_map |
|||
|
|||
_HTML_UNICODE_MAP = _build_unicode_map() |
|||
@ -0,0 +1,382 @@ |
|||
"""``tornado.gen`` is a generator-based interface to make it easier to |
|||
work in an asynchronous environment. Code using the ``gen`` module |
|||
is technically asynchronous, but it is written as a single generator |
|||
instead of a collection of separate functions. |
|||
|
|||
For example, the following asynchronous handler:: |
|||
|
|||
class AsyncHandler(RequestHandler): |
|||
@asynchronous |
|||
def get(self): |
|||
http_client = AsyncHTTPClient() |
|||
http_client.fetch("http://example.com", |
|||
callback=self.on_fetch) |
|||
|
|||
def on_fetch(self, response): |
|||
do_something_with_response(response) |
|||
self.render("template.html") |
|||
|
|||
could be written with ``gen`` as:: |
|||
|
|||
class GenAsyncHandler(RequestHandler): |
|||
@asynchronous |
|||
@gen.engine |
|||
def get(self): |
|||
http_client = AsyncHTTPClient() |
|||
response = yield gen.Task(http_client.fetch, "http://example.com") |
|||
do_something_with_response(response) |
|||
self.render("template.html") |
|||
|
|||
`Task` works with any function that takes a ``callback`` keyword |
|||
argument. You can also yield a list of ``Tasks``, which will be |
|||
started at the same time and run in parallel; a list of results will |
|||
be returned when they are all finished:: |
|||
|
|||
def get(self): |
|||
http_client = AsyncHTTPClient() |
|||
response1, response2 = yield [gen.Task(http_client.fetch, url1), |
|||
gen.Task(http_client.fetch, url2)] |
|||
|
|||
For more complicated interfaces, `Task` can be split into two parts: |
|||
`Callback` and `Wait`:: |
|||
|
|||
class GenAsyncHandler2(RequestHandler): |
|||
@asynchronous |
|||
@gen.engine |
|||
def get(self): |
|||
http_client = AsyncHTTPClient() |
|||
http_client.fetch("http://example.com", |
|||
callback=(yield gen.Callback("key")) |
|||
response = yield gen.Wait("key") |
|||
do_something_with_response(response) |
|||
self.render("template.html") |
|||
|
|||
The ``key`` argument to `Callback` and `Wait` allows for multiple |
|||
asynchronous operations to be started at different times and proceed |
|||
in parallel: yield several callbacks with different keys, then wait |
|||
for them once all the async operations have started. |
|||
|
|||
The result of a `Wait` or `Task` yield expression depends on how the callback |
|||
was run. If it was called with no arguments, the result is ``None``. If |
|||
it was called with one argument, the result is that argument. If it was |
|||
called with more than one argument or any keyword arguments, the result |
|||
is an `Arguments` object, which is a named tuple ``(args, kwargs)``. |
|||
""" |
|||
from __future__ import with_statement |
|||
|
|||
import functools |
|||
import operator |
|||
import sys |
|||
import types |
|||
|
|||
from tornado.stack_context import ExceptionStackContext |
|||
|
|||
class KeyReuseError(Exception): pass |
|||
class UnknownKeyError(Exception): pass |
|||
class LeakedCallbackError(Exception): pass |
|||
class BadYieldError(Exception): pass |
|||
|
|||
def engine(func): |
|||
"""Decorator for asynchronous generators. |
|||
|
|||
Any generator that yields objects from this module must be wrapped |
|||
in this decorator. The decorator only works on functions that are |
|||
already asynchronous. For `~tornado.web.RequestHandler` |
|||
``get``/``post``/etc methods, this means that both the |
|||
`tornado.web.asynchronous` and `tornado.gen.engine` decorators |
|||
must be used (for proper exception handling, ``asynchronous`` |
|||
should come before ``gen.engine``). In most other cases, it means |
|||
that it doesn't make sense to use ``gen.engine`` on functions that |
|||
don't already take a callback argument. |
|||
""" |
|||
@functools.wraps(func) |
|||
def wrapper(*args, **kwargs): |
|||
runner = None |
|||
def handle_exception(typ, value, tb): |
|||
# if the function throws an exception before its first "yield" |
|||
# (or is not a generator at all), the Runner won't exist yet. |
|||
# However, in that case we haven't reached anything asynchronous |
|||
# yet, so we can just let the exception propagate. |
|||
if runner is not None: |
|||
return runner.handle_exception(typ, value, tb) |
|||
return False |
|||
with ExceptionStackContext(handle_exception): |
|||
gen = func(*args, **kwargs) |
|||
if isinstance(gen, types.GeneratorType): |
|||
runner = Runner(gen) |
|||
runner.run() |
|||
return |
|||
assert gen is None, gen |
|||
# no yield, so we're done |
|||
return wrapper |
|||
|
|||
class YieldPoint(object): |
|||
"""Base class for objects that may be yielded from the generator.""" |
|||
def start(self, runner): |
|||
"""Called by the runner after the generator has yielded. |
|||
|
|||
No other methods will be called on this object before ``start``. |
|||
""" |
|||
raise NotImplementedError() |
|||
|
|||
def is_ready(self): |
|||
"""Called by the runner to determine whether to resume the generator. |
|||
|
|||
Returns a boolean; may be called more than once. |
|||
""" |
|||
raise NotImplementedError() |
|||
|
|||
def get_result(self): |
|||
"""Returns the value to use as the result of the yield expression. |
|||
|
|||
This method will only be called once, and only after `is_ready` |
|||
has returned true. |
|||
""" |
|||
raise NotImplementedError() |
|||
|
|||
class Callback(YieldPoint): |
|||
"""Returns a callable object that will allow a matching `Wait` to proceed. |
|||
|
|||
The key may be any value suitable for use as a dictionary key, and is |
|||
used to match ``Callbacks`` to their corresponding ``Waits``. The key |
|||
must be unique among outstanding callbacks within a single run of the |
|||
generator function, but may be reused across different runs of the same |
|||
function (so constants generally work fine). |
|||
|
|||
The callback may be called with zero or one arguments; if an argument |
|||
is given it will be returned by `Wait`. |
|||
""" |
|||
def __init__(self, key): |
|||
self.key = key |
|||
|
|||
def start(self, runner): |
|||
self.runner = runner |
|||
runner.register_callback(self.key) |
|||
|
|||
def is_ready(self): |
|||
return True |
|||
|
|||
def get_result(self): |
|||
return self.runner.result_callback(self.key) |
|||
|
|||
class Wait(YieldPoint): |
|||
"""Returns the argument passed to the result of a previous `Callback`.""" |
|||
def __init__(self, key): |
|||
self.key = key |
|||
|
|||
def start(self, runner): |
|||
self.runner = runner |
|||
|
|||
def is_ready(self): |
|||
return self.runner.is_ready(self.key) |
|||
|
|||
def get_result(self): |
|||
return self.runner.pop_result(self.key) |
|||
|
|||
class WaitAll(YieldPoint): |
|||
"""Returns the results of multiple previous `Callbacks`. |
|||
|
|||
The argument is a sequence of `Callback` keys, and the result is |
|||
a list of results in the same order. |
|||
|
|||
`WaitAll` is equivalent to yielding a list of `Wait` objects. |
|||
""" |
|||
def __init__(self, keys): |
|||
self.keys = keys |
|||
|
|||
def start(self, runner): |
|||
self.runner = runner |
|||
|
|||
def is_ready(self): |
|||
return all(self.runner.is_ready(key) for key in self.keys) |
|||
|
|||
def get_result(self): |
|||
return [self.runner.pop_result(key) for key in self.keys] |
|||
|
|||
|
|||
class Task(YieldPoint): |
|||
"""Runs a single asynchronous operation. |
|||
|
|||
Takes a function (and optional additional arguments) and runs it with |
|||
those arguments plus a ``callback`` keyword argument. The argument passed |
|||
to the callback is returned as the result of the yield expression. |
|||
|
|||
A `Task` is equivalent to a `Callback`/`Wait` pair (with a unique |
|||
key generated automatically):: |
|||
|
|||
result = yield gen.Task(func, args) |
|||
|
|||
func(args, callback=(yield gen.Callback(key))) |
|||
result = yield gen.Wait(key) |
|||
""" |
|||
def __init__(self, func, *args, **kwargs): |
|||
assert "callback" not in kwargs |
|||
self.args = args |
|||
self.kwargs = kwargs |
|||
self.func = func |
|||
|
|||
def start(self, runner): |
|||
self.runner = runner |
|||
self.key = object() |
|||
runner.register_callback(self.key) |
|||
self.kwargs["callback"] = runner.result_callback(self.key) |
|||
self.func(*self.args, **self.kwargs) |
|||
|
|||
def is_ready(self): |
|||
return self.runner.is_ready(self.key) |
|||
|
|||
def get_result(self): |
|||
return self.runner.pop_result(self.key) |
|||
|
|||
class Multi(YieldPoint): |
|||
"""Runs multiple asynchronous operations in parallel. |
|||
|
|||
Takes a list of ``Tasks`` or other ``YieldPoints`` and returns a list of |
|||
their responses. It is not necessary to call `Multi` explicitly, |
|||
since the engine will do so automatically when the generator yields |
|||
a list of ``YieldPoints``. |
|||
""" |
|||
def __init__(self, children): |
|||
assert all(isinstance(i, YieldPoint) for i in children) |
|||
self.children = children |
|||
|
|||
def start(self, runner): |
|||
for i in self.children: |
|||
i.start(runner) |
|||
|
|||
def is_ready(self): |
|||
return all(i.is_ready() for i in self.children) |
|||
|
|||
def get_result(self): |
|||
return [i.get_result() for i in self.children] |
|||
|
|||
class _NullYieldPoint(YieldPoint): |
|||
def start(self, runner): |
|||
pass |
|||
def is_ready(self): |
|||
return True |
|||
def get_result(self): |
|||
return None |
|||
|
|||
class Runner(object): |
|||
"""Internal implementation of `tornado.gen.engine`. |
|||
|
|||
Maintains information about pending callbacks and their results. |
|||
""" |
|||
def __init__(self, gen): |
|||
self.gen = gen |
|||
self.yield_point = _NullYieldPoint() |
|||
self.pending_callbacks = set() |
|||
self.results = {} |
|||
self.running = False |
|||
self.finished = False |
|||
self.exc_info = None |
|||
self.had_exception = False |
|||
|
|||
def register_callback(self, key): |
|||
"""Adds ``key`` to the list of callbacks.""" |
|||
if key in self.pending_callbacks: |
|||
raise KeyReuseError("key %r is already pending" % key) |
|||
self.pending_callbacks.add(key) |
|||
|
|||
def is_ready(self, key): |
|||
"""Returns true if a result is available for ``key``.""" |
|||
if key not in self.pending_callbacks: |
|||
raise UnknownKeyError("key %r is not pending" % key) |
|||
return key in self.results |
|||
|
|||
def set_result(self, key, result): |
|||
"""Sets the result for ``key`` and attempts to resume the generator.""" |
|||
self.results[key] = result |
|||
self.run() |
|||
|
|||
def pop_result(self, key): |
|||
"""Returns the result for ``key`` and unregisters it.""" |
|||
self.pending_callbacks.remove(key) |
|||
return self.results.pop(key) |
|||
|
|||
def run(self): |
|||
"""Starts or resumes the generator, running until it reaches a |
|||
yield point that is not ready. |
|||
""" |
|||
if self.running or self.finished: |
|||
return |
|||
try: |
|||
self.running = True |
|||
while True: |
|||
if self.exc_info is None: |
|||
try: |
|||
if not self.yield_point.is_ready(): |
|||
return |
|||
next = self.yield_point.get_result() |
|||
except Exception: |
|||
self.exc_info = sys.exc_info() |
|||
try: |
|||
if self.exc_info is not None: |
|||
self.had_exception = True |
|||
exc_info = self.exc_info |
|||
self.exc_info = None |
|||
yielded = self.gen.throw(*exc_info) |
|||
else: |
|||
yielded = self.gen.send(next) |
|||
except StopIteration: |
|||
self.finished = True |
|||
if self.pending_callbacks and not self.had_exception: |
|||
# If we ran cleanly without waiting on all callbacks |
|||
# raise an error (really more of a warning). If we |
|||
# had an exception then some callbacks may have been |
|||
# orphaned, so skip the check in that case. |
|||
raise LeakedCallbackError( |
|||
"finished without waiting for callbacks %r" % |
|||
self.pending_callbacks) |
|||
return |
|||
except Exception: |
|||
self.finished = True |
|||
raise |
|||
if isinstance(yielded, list): |
|||
yielded = Multi(yielded) |
|||
if isinstance(yielded, YieldPoint): |
|||
self.yield_point = yielded |
|||
try: |
|||
self.yield_point.start(self) |
|||
except Exception: |
|||
self.exc_info = sys.exc_info() |
|||
else: |
|||
self.exc_info = (BadYieldError("yielded unknown object %r" % yielded),) |
|||
finally: |
|||
self.running = False |
|||
|
|||
def result_callback(self, key): |
|||
def inner(*args, **kwargs): |
|||
if kwargs or len(args) > 1: |
|||
result = Arguments(args, kwargs) |
|||
elif args: |
|||
result = args[0] |
|||
else: |
|||
result = None |
|||
self.set_result(key, result) |
|||
return inner |
|||
|
|||
def handle_exception(self, typ, value, tb): |
|||
if not self.running and not self.finished: |
|||
self.exc_info = (typ, value, tb) |
|||
self.run() |
|||
return True |
|||
else: |
|||
return False |
|||
|
|||
# in python 2.6+ this could be a collections.namedtuple |
|||
class Arguments(tuple): |
|||
"""The result of a yield expression whose callback had more than one |
|||
argument (or keyword arguments). |
|||
|
|||
The `Arguments` object can be used as a tuple ``(args, kwargs)`` |
|||
or an object with attributes ``args`` and ``kwargs``. |
|||
""" |
|||
__slots__ = () |
|||
|
|||
def __new__(cls, args, kwargs): |
|||
return tuple.__new__(cls, (args, kwargs)) |
|||
|
|||
args = property(operator.itemgetter(0)) |
|||
kwargs = property(operator.itemgetter(1)) |
|||
@ -0,0 +1,417 @@ |
|||
"""Blocking and non-blocking HTTP client interfaces. |
|||
|
|||
This module defines a common interface shared by two implementations, |
|||
`simple_httpclient` and `curl_httpclient`. Applications may either |
|||
instantiate their chosen implementation class directly or use the |
|||
`AsyncHTTPClient` class from this module, which selects an implementation |
|||
that can be overridden with the `AsyncHTTPClient.configure` method. |
|||
|
|||
The default implementation is `simple_httpclient`, and this is expected |
|||
to be suitable for most users' needs. However, some applications may wish |
|||
to switch to `curl_httpclient` for reasons such as the following: |
|||
|
|||
* `curl_httpclient` has some features not found in `simple_httpclient`, |
|||
including support for HTTP proxies and the ability to use a specified |
|||
network interface. |
|||
|
|||
* `curl_httpclient` is more likely to be compatible with sites that are |
|||
not-quite-compliant with the HTTP spec, or sites that use little-exercised |
|||
features of HTTP. |
|||
|
|||
* `simple_httpclient` only supports SSL on Python 2.6 and above. |
|||
|
|||
* `curl_httpclient` is faster |
|||
|
|||
* `curl_httpclient` was the default prior to Tornado 2.0. |
|||
|
|||
Note that if you are using `curl_httpclient`, it is highly recommended that |
|||
you use a recent version of ``libcurl`` and ``pycurl``. Currently the minimum |
|||
supported version is 7.18.2, and the recommended version is 7.21.1 or newer. |
|||
""" |
|||
|
|||
import calendar |
|||
import email.utils |
|||
import httplib |
|||
import time |
|||
import weakref |
|||
|
|||
from tornado.escape import utf8 |
|||
from tornado import httputil |
|||
from tornado.ioloop import IOLoop |
|||
from tornado.util import import_object, bytes_type |
|||
|
|||
class HTTPClient(object): |
|||
"""A blocking HTTP client. |
|||
|
|||
This interface is provided for convenience and testing; most applications |
|||
that are running an IOLoop will want to use `AsyncHTTPClient` instead. |
|||
Typical usage looks like this:: |
|||
|
|||
http_client = httpclient.HTTPClient() |
|||
try: |
|||
response = http_client.fetch("http://www.google.com/") |
|||
print response.body |
|||
except httpclient.HTTPError, e: |
|||
print "Error:", e |
|||
""" |
|||
def __init__(self, async_client_class=None): |
|||
self._io_loop = IOLoop() |
|||
if async_client_class is None: |
|||
async_client_class = AsyncHTTPClient |
|||
self._async_client = async_client_class(self._io_loop) |
|||
self._response = None |
|||
self._closed = False |
|||
|
|||
def __del__(self): |
|||
self.close() |
|||
|
|||
def close(self): |
|||
"""Closes the HTTPClient, freeing any resources used.""" |
|||
if not self._closed: |
|||
self._async_client.close() |
|||
self._io_loop.close() |
|||
self._closed = True |
|||
|
|||
def fetch(self, request, **kwargs): |
|||
"""Executes a request, returning an `HTTPResponse`. |
|||
|
|||
The request may be either a string URL or an `HTTPRequest` object. |
|||
If it is a string, we construct an `HTTPRequest` using any additional |
|||
kwargs: ``HTTPRequest(request, **kwargs)`` |
|||
|
|||
If an error occurs during the fetch, we raise an `HTTPError`. |
|||
""" |
|||
def callback(response): |
|||
self._response = response |
|||
self._io_loop.stop() |
|||
self._async_client.fetch(request, callback, **kwargs) |
|||
self._io_loop.start() |
|||
response = self._response |
|||
self._response = None |
|||
response.rethrow() |
|||
return response |
|||
|
|||
class AsyncHTTPClient(object): |
|||
"""An non-blocking HTTP client. |
|||
|
|||
Example usage:: |
|||
|
|||
import ioloop |
|||
|
|||
def handle_request(response): |
|||
if response.error: |
|||
print "Error:", response.error |
|||
else: |
|||
print response.body |
|||
ioloop.IOLoop.instance().stop() |
|||
|
|||
http_client = httpclient.AsyncHTTPClient() |
|||
http_client.fetch("http://www.google.com/", handle_request) |
|||
ioloop.IOLoop.instance().start() |
|||
|
|||
The constructor for this class is magic in several respects: It actually |
|||
creates an instance of an implementation-specific subclass, and instances |
|||
are reused as a kind of pseudo-singleton (one per IOLoop). The keyword |
|||
argument force_instance=True can be used to suppress this singleton |
|||
behavior. Constructor arguments other than io_loop and force_instance |
|||
are deprecated. The implementation subclass as well as arguments to |
|||
its constructor can be set with the static method configure() |
|||
""" |
|||
_impl_class = None |
|||
_impl_kwargs = None |
|||
|
|||
@classmethod |
|||
def _async_clients(cls): |
|||
assert cls is not AsyncHTTPClient, "should only be called on subclasses" |
|||
if not hasattr(cls, '_async_client_dict'): |
|||
cls._async_client_dict = weakref.WeakKeyDictionary() |
|||
return cls._async_client_dict |
|||
|
|||
def __new__(cls, io_loop=None, max_clients=10, force_instance=False, |
|||
**kwargs): |
|||
io_loop = io_loop or IOLoop.instance() |
|||
if cls is AsyncHTTPClient: |
|||
if cls._impl_class is None: |
|||
from tornado.simple_httpclient import SimpleAsyncHTTPClient |
|||
AsyncHTTPClient._impl_class = SimpleAsyncHTTPClient |
|||
impl = AsyncHTTPClient._impl_class |
|||
else: |
|||
impl = cls |
|||
if io_loop in impl._async_clients() and not force_instance: |
|||
return impl._async_clients()[io_loop] |
|||
else: |
|||
instance = super(AsyncHTTPClient, cls).__new__(impl) |
|||
args = {} |
|||
if cls._impl_kwargs: |
|||
args.update(cls._impl_kwargs) |
|||
args.update(kwargs) |
|||
instance.initialize(io_loop, max_clients, **args) |
|||
if not force_instance: |
|||
impl._async_clients()[io_loop] = instance |
|||
return instance |
|||
|
|||
def close(self): |
|||
"""Destroys this http client, freeing any file descriptors used. |
|||
Not needed in normal use, but may be helpful in unittests that |
|||
create and destroy http clients. No other methods may be called |
|||
on the AsyncHTTPClient after close(). |
|||
""" |
|||
if self._async_clients().get(self.io_loop) is self: |
|||
del self._async_clients()[self.io_loop] |
|||
|
|||
def fetch(self, request, callback, **kwargs): |
|||
"""Executes a request, calling callback with an `HTTPResponse`. |
|||
|
|||
The request may be either a string URL or an `HTTPRequest` object. |
|||
If it is a string, we construct an `HTTPRequest` using any additional |
|||
kwargs: ``HTTPRequest(request, **kwargs)`` |
|||
|
|||
If an error occurs during the fetch, the HTTPResponse given to the |
|||
callback has a non-None error attribute that contains the exception |
|||
encountered during the request. You can call response.rethrow() to |
|||
throw the exception (if any) in the callback. |
|||
""" |
|||
raise NotImplementedError() |
|||
|
|||
@staticmethod |
|||
def configure(impl, **kwargs): |
|||
"""Configures the AsyncHTTPClient subclass to use. |
|||
|
|||
AsyncHTTPClient() actually creates an instance of a subclass. |
|||
This method may be called with either a class object or the |
|||
fully-qualified name of such a class (or None to use the default, |
|||
SimpleAsyncHTTPClient) |
|||
|
|||
If additional keyword arguments are given, they will be passed |
|||
to the constructor of each subclass instance created. The |
|||
keyword argument max_clients determines the maximum number of |
|||
simultaneous fetch() operations that can execute in parallel |
|||
on each IOLoop. Additional arguments may be supported depending |
|||
on the implementation class in use. |
|||
|
|||
Example:: |
|||
|
|||
AsyncHTTPClient.configure("tornado.curl_httpclient.CurlAsyncHTTPClient") |
|||
""" |
|||
if isinstance(impl, (unicode, bytes_type)): |
|||
impl = import_object(impl) |
|||
if impl is not None and not issubclass(impl, AsyncHTTPClient): |
|||
raise ValueError("Invalid AsyncHTTPClient implementation") |
|||
AsyncHTTPClient._impl_class = impl |
|||
AsyncHTTPClient._impl_kwargs = kwargs |
|||
|
|||
class HTTPRequest(object): |
|||
"""HTTP client request object.""" |
|||
def __init__(self, url, method="GET", headers=None, body=None, |
|||
auth_username=None, auth_password=None, |
|||
connect_timeout=20.0, request_timeout=20.0, |
|||
if_modified_since=None, follow_redirects=True, |
|||
max_redirects=5, user_agent=None, use_gzip=True, |
|||
network_interface=None, streaming_callback=None, |
|||
header_callback=None, prepare_curl_callback=None, |
|||
proxy_host=None, proxy_port=None, proxy_username=None, |
|||
proxy_password='', allow_nonstandard_methods=False, |
|||
validate_cert=True, ca_certs=None, |
|||
allow_ipv6=None, |
|||
client_key=None, client_cert=None): |
|||
"""Creates an `HTTPRequest`. |
|||
|
|||
All parameters except `url` are optional. |
|||
|
|||
:arg string url: URL to fetch |
|||
:arg string method: HTTP method, e.g. "GET" or "POST" |
|||
:arg headers: Additional HTTP headers to pass on the request |
|||
:type headers: `~tornado.httputil.HTTPHeaders` or `dict` |
|||
:arg string auth_username: Username for HTTP "Basic" authentication |
|||
:arg string auth_password: Password for HTTP "Basic" authentication |
|||
:arg float connect_timeout: Timeout for initial connection in seconds |
|||
:arg float request_timeout: Timeout for entire request in seconds |
|||
:arg datetime if_modified_since: Timestamp for ``If-Modified-Since`` |
|||
header |
|||
:arg bool follow_redirects: Should redirects be followed automatically |
|||
or return the 3xx response? |
|||
:arg int max_redirects: Limit for `follow_redirects` |
|||
:arg string user_agent: String to send as ``User-Agent`` header |
|||
:arg bool use_gzip: Request gzip encoding from the server |
|||
:arg string network_interface: Network interface to use for request |
|||
:arg callable streaming_callback: If set, `streaming_callback` will |
|||
be run with each chunk of data as it is received, and |
|||
`~HTTPResponse.body` and `~HTTPResponse.buffer` will be empty in |
|||
the final response. |
|||
:arg callable header_callback: If set, `header_callback` will |
|||
be run with each header line as it is received, and |
|||
`~HTTPResponse.headers` will be empty in the final response. |
|||
:arg callable prepare_curl_callback: If set, will be called with |
|||
a `pycurl.Curl` object to allow the application to make additional |
|||
`setopt` calls. |
|||
:arg string proxy_host: HTTP proxy hostname. To use proxies, |
|||
`proxy_host` and `proxy_port` must be set; `proxy_username` and |
|||
`proxy_pass` are optional. Proxies are currently only support |
|||
with `curl_httpclient`. |
|||
:arg int proxy_port: HTTP proxy port |
|||
:arg string proxy_username: HTTP proxy username |
|||
:arg string proxy_password: HTTP proxy password |
|||
:arg bool allow_nonstandard_methods: Allow unknown values for `method` |
|||
argument? |
|||
:arg bool validate_cert: For HTTPS requests, validate the server's |
|||
certificate? |
|||
:arg string ca_certs: filename of CA certificates in PEM format, |
|||
or None to use defaults. Note that in `curl_httpclient`, if |
|||
any request uses a custom `ca_certs` file, they all must (they |
|||
don't have to all use the same `ca_certs`, but it's not possible |
|||
to mix requests with ca_certs and requests that use the defaults. |
|||
:arg bool allow_ipv6: Use IPv6 when available? Default is false in |
|||
`simple_httpclient` and true in `curl_httpclient` |
|||
:arg string client_key: Filename for client SSL key, if any |
|||
:arg string client_cert: Filename for client SSL certificate, if any |
|||
""" |
|||
if headers is None: |
|||
headers = httputil.HTTPHeaders() |
|||
if if_modified_since: |
|||
timestamp = calendar.timegm(if_modified_since.utctimetuple()) |
|||
headers["If-Modified-Since"] = email.utils.formatdate( |
|||
timestamp, localtime=False, usegmt=True) |
|||
self.proxy_host = proxy_host |
|||
self.proxy_port = proxy_port |
|||
self.proxy_username = proxy_username |
|||
self.proxy_password = proxy_password |
|||
self.url = url |
|||
self.method = method |
|||
self.headers = headers |
|||
self.body = utf8(body) |
|||
self.auth_username = auth_username |
|||
self.auth_password = auth_password |
|||
self.connect_timeout = connect_timeout |
|||
self.request_timeout = request_timeout |
|||
self.follow_redirects = follow_redirects |
|||
self.max_redirects = max_redirects |
|||
self.user_agent = user_agent |
|||
self.use_gzip = use_gzip |
|||
self.network_interface = network_interface |
|||
self.streaming_callback = streaming_callback |
|||
self.header_callback = header_callback |
|||
self.prepare_curl_callback = prepare_curl_callback |
|||
self.allow_nonstandard_methods = allow_nonstandard_methods |
|||
self.validate_cert = validate_cert |
|||
self.ca_certs = ca_certs |
|||
self.allow_ipv6 = allow_ipv6 |
|||
self.client_key = client_key |
|||
self.client_cert = client_cert |
|||
self.start_time = time.time() |
|||
|
|||
|
|||
class HTTPResponse(object): |
|||
"""HTTP Response object. |
|||
|
|||
Attributes: |
|||
|
|||
* request: HTTPRequest object |
|||
|
|||
* code: numeric HTTP status code, e.g. 200 or 404 |
|||
|
|||
* headers: httputil.HTTPHeaders object |
|||
|
|||
* buffer: cStringIO object for response body |
|||
|
|||
* body: respose body as string (created on demand from self.buffer) |
|||
|
|||
* error: Exception object, if any |
|||
|
|||
* request_time: seconds from request start to finish |
|||
|
|||
* time_info: dictionary of diagnostic timing information from the request. |
|||
Available data are subject to change, but currently uses timings |
|||
available from http://curl.haxx.se/libcurl/c/curl_easy_getinfo.html, |
|||
plus 'queue', which is the delay (if any) introduced by waiting for |
|||
a slot under AsyncHTTPClient's max_clients setting. |
|||
""" |
|||
def __init__(self, request, code, headers={}, buffer=None, |
|||
effective_url=None, error=None, request_time=None, |
|||
time_info={}): |
|||
self.request = request |
|||
self.code = code |
|||
self.headers = headers |
|||
self.buffer = buffer |
|||
self._body = None |
|||
if effective_url is None: |
|||
self.effective_url = request.url |
|||
else: |
|||
self.effective_url = effective_url |
|||
if error is None: |
|||
if self.code < 200 or self.code >= 300: |
|||
self.error = HTTPError(self.code, response=self) |
|||
else: |
|||
self.error = None |
|||
else: |
|||
self.error = error |
|||
self.request_time = request_time |
|||
self.time_info = time_info |
|||
|
|||
def _get_body(self): |
|||
if self.buffer is None: |
|||
return None |
|||
elif self._body is None: |
|||
self._body = self.buffer.getvalue() |
|||
|
|||
return self._body |
|||
|
|||
body = property(_get_body) |
|||
|
|||
def rethrow(self): |
|||
"""If there was an error on the request, raise an `HTTPError`.""" |
|||
if self.error: |
|||
raise self.error |
|||
|
|||
def __repr__(self): |
|||
args = ",".join("%s=%r" % i for i in self.__dict__.iteritems()) |
|||
return "%s(%s)" % (self.__class__.__name__, args) |
|||
|
|||
|
|||
class HTTPError(Exception): |
|||
"""Exception thrown for an unsuccessful HTTP request. |
|||
|
|||
Attributes: |
|||
|
|||
code - HTTP error integer error code, e.g. 404. Error code 599 is |
|||
used when no HTTP response was received, e.g. for a timeout. |
|||
|
|||
response - HTTPResponse object, if any. |
|||
|
|||
Note that if follow_redirects is False, redirects become HTTPErrors, |
|||
and you can look at error.response.headers['Location'] to see the |
|||
destination of the redirect. |
|||
""" |
|||
def __init__(self, code, message=None, response=None): |
|||
self.code = code |
|||
message = message or httplib.responses.get(code, "Unknown") |
|||
self.response = response |
|||
Exception.__init__(self, "HTTP %d: %s" % (self.code, message)) |
|||
|
|||
|
|||
def main(): |
|||
from tornado.options import define, options, parse_command_line |
|||
define("print_headers", type=bool, default=False) |
|||
define("print_body", type=bool, default=True) |
|||
define("follow_redirects", type=bool, default=True) |
|||
define("validate_cert", type=bool, default=True) |
|||
args = parse_command_line() |
|||
client = HTTPClient() |
|||
for arg in args: |
|||
try: |
|||
response = client.fetch(arg, |
|||
follow_redirects=options.follow_redirects, |
|||
validate_cert=options.validate_cert, |
|||
) |
|||
except HTTPError, e: |
|||
if e.response is not None: |
|||
response = e.response |
|||
else: |
|||
raise |
|||
if options.print_headers: |
|||
print response.headers |
|||
if options.print_body: |
|||
print response.body |
|||
client.close() |
|||
|
|||
if __name__ == "__main__": |
|||
main() |
|||
@ -0,0 +1,476 @@ |
|||
#!/usr/bin/env python |
|||
# |
|||
# Copyright 2009 Facebook |
|||
# |
|||
# Licensed under the Apache License, Version 2.0 (the "License"); you may |
|||
# not use this file except in compliance with the License. You may obtain |
|||
# a copy of the License at |
|||
# |
|||
# http://www.apache.org/licenses/LICENSE-2.0 |
|||
# |
|||
# Unless required by applicable law or agreed to in writing, software |
|||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT |
|||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the |
|||
# License for the specific language governing permissions and limitations |
|||
# under the License. |
|||
|
|||
"""A non-blocking, single-threaded HTTP server. |
|||
|
|||
Typical applications have little direct interaction with the `HTTPServer` |
|||
class except to start a server at the beginning of the process |
|||
(and even that is often done indirectly via `tornado.web.Application.listen`). |
|||
|
|||
This module also defines the `HTTPRequest` class which is exposed via |
|||
`tornado.web.RequestHandler.request`. |
|||
""" |
|||
|
|||
import Cookie |
|||
import logging |
|||
import socket |
|||
import time |
|||
import urlparse |
|||
|
|||
from tornado.escape import utf8, native_str, parse_qs_bytes |
|||
from tornado import httputil |
|||
from tornado import iostream |
|||
from tornado.netutil import TCPServer |
|||
from tornado import stack_context |
|||
from tornado.util import b, bytes_type |
|||
|
|||
try: |
|||
import ssl # Python 2.6+ |
|||
except ImportError: |
|||
ssl = None |
|||
|
|||
class HTTPServer(TCPServer): |
|||
r"""A non-blocking, single-threaded HTTP server. |
|||
|
|||
A server is defined by a request callback that takes an HTTPRequest |
|||
instance as an argument and writes a valid HTTP response with |
|||
`HTTPRequest.write`. `HTTPRequest.finish` finishes the request (but does |
|||
not necessarily close the connection in the case of HTTP/1.1 keep-alive |
|||
requests). A simple example server that echoes back the URI you |
|||
requested:: |
|||
|
|||
import httpserver |
|||
import ioloop |
|||
|
|||
def handle_request(request): |
|||
message = "You requested %s\n" % request.uri |
|||
request.write("HTTP/1.1 200 OK\r\nContent-Length: %d\r\n\r\n%s" % ( |
|||
len(message), message)) |
|||
request.finish() |
|||
|
|||
http_server = httpserver.HTTPServer(handle_request) |
|||
http_server.listen(8888) |
|||
ioloop.IOLoop.instance().start() |
|||
|
|||
`HTTPServer` is a very basic connection handler. Beyond parsing the |
|||
HTTP request body and headers, the only HTTP semantics implemented |
|||
in `HTTPServer` is HTTP/1.1 keep-alive connections. We do not, however, |
|||
implement chunked encoding, so the request callback must provide a |
|||
``Content-Length`` header or implement chunked encoding for HTTP/1.1 |
|||
requests for the server to run correctly for HTTP/1.1 clients. If |
|||
the request handler is unable to do this, you can provide the |
|||
``no_keep_alive`` argument to the `HTTPServer` constructor, which will |
|||
ensure the connection is closed on every request no matter what HTTP |
|||
version the client is using. |
|||
|
|||
If ``xheaders`` is ``True``, we support the ``X-Real-Ip`` and ``X-Scheme`` |
|||
headers, which override the remote IP and HTTP scheme for all requests. |
|||
These headers are useful when running Tornado behind a reverse proxy or |
|||
load balancer. |
|||
|
|||
`HTTPServer` can serve SSL traffic with Python 2.6+ and OpenSSL. |
|||
To make this server serve SSL traffic, send the ssl_options dictionary |
|||
argument with the arguments required for the `ssl.wrap_socket` method, |
|||
including "certfile" and "keyfile":: |
|||
|
|||
HTTPServer(applicaton, ssl_options={ |
|||
"certfile": os.path.join(data_dir, "mydomain.crt"), |
|||
"keyfile": os.path.join(data_dir, "mydomain.key"), |
|||
}) |
|||
|
|||
`HTTPServer` initialization follows one of three patterns (the |
|||
initialization methods are defined on `tornado.netutil.TCPServer`): |
|||
|
|||
1. `~tornado.netutil.TCPServer.listen`: simple single-process:: |
|||
|
|||
server = HTTPServer(app) |
|||
server.listen(8888) |
|||
IOLoop.instance().start() |
|||
|
|||
In many cases, `tornado.web.Application.listen` can be used to avoid |
|||
the need to explicitly create the `HTTPServer`. |
|||
|
|||
2. `~tornado.netutil.TCPServer.bind`/`~tornado.netutil.TCPServer.start`: |
|||
simple multi-process:: |
|||
|
|||
server = HTTPServer(app) |
|||
server.bind(8888) |
|||
server.start(0) # Forks multiple sub-processes |
|||
IOLoop.instance().start() |
|||
|
|||
When using this interface, an `IOLoop` must *not* be passed |
|||
to the `HTTPServer` constructor. `start` will always start |
|||
the server on the default singleton `IOLoop`. |
|||
|
|||
3. `~tornado.netutil.TCPServer.add_sockets`: advanced multi-process:: |
|||
|
|||
sockets = tornado.netutil.bind_sockets(8888) |
|||
tornado.process.fork_processes(0) |
|||
server = HTTPServer(app) |
|||
server.add_sockets(sockets) |
|||
IOLoop.instance().start() |
|||
|
|||
The `add_sockets` interface is more complicated, but it can be |
|||
used with `tornado.process.fork_processes` to give you more |
|||
flexibility in when the fork happens. `add_sockets` can |
|||
also be used in single-process servers if you want to create |
|||
your listening sockets in some way other than |
|||
`tornado.netutil.bind_sockets`. |
|||
|
|||
""" |
|||
def __init__(self, request_callback, no_keep_alive=False, io_loop=None, |
|||
xheaders=False, ssl_options=None, **kwargs): |
|||
self.request_callback = request_callback |
|||
self.no_keep_alive = no_keep_alive |
|||
self.xheaders = xheaders |
|||
TCPServer.__init__(self, io_loop=io_loop, ssl_options=ssl_options, |
|||
**kwargs) |
|||
|
|||
def handle_stream(self, stream, address): |
|||
HTTPConnection(stream, address, self.request_callback, |
|||
self.no_keep_alive, self.xheaders) |
|||
|
|||
class _BadRequestException(Exception): |
|||
"""Exception class for malformed HTTP requests.""" |
|||
pass |
|||
|
|||
class HTTPConnection(object): |
|||
"""Handles a connection to an HTTP client, executing HTTP requests. |
|||
|
|||
We parse HTTP headers and bodies, and execute the request callback |
|||
until the HTTP conection is closed. |
|||
""" |
|||
def __init__(self, stream, address, request_callback, no_keep_alive=False, |
|||
xheaders=False): |
|||
self.stream = stream |
|||
if self.stream.socket.family not in (socket.AF_INET, socket.AF_INET6): |
|||
# Unix (or other) socket; fake the remote address |
|||
address = ('0.0.0.0', 0) |
|||
self.address = address |
|||
self.request_callback = request_callback |
|||
self.no_keep_alive = no_keep_alive |
|||
self.xheaders = xheaders |
|||
self._request = None |
|||
self._request_finished = False |
|||
# Save stack context here, outside of any request. This keeps |
|||
# contexts from one request from leaking into the next. |
|||
self._header_callback = stack_context.wrap(self._on_headers) |
|||
self.stream.read_until(b("\r\n\r\n"), self._header_callback) |
|||
self._write_callback = None |
|||
|
|||
def write(self, chunk, callback=None): |
|||
"""Writes a chunk of output to the stream.""" |
|||
assert self._request, "Request closed" |
|||
if not self.stream.closed(): |
|||
self._write_callback = stack_context.wrap(callback) |
|||
self.stream.write(chunk, self._on_write_complete) |
|||
|
|||
def finish(self): |
|||
"""Finishes the request.""" |
|||
assert self._request, "Request closed" |
|||
self._request_finished = True |
|||
if not self.stream.writing(): |
|||
self._finish_request() |
|||
|
|||
def _on_write_complete(self): |
|||
if self._write_callback is not None: |
|||
callback = self._write_callback |
|||
self._write_callback = None |
|||
callback() |
|||
# _on_write_complete is enqueued on the IOLoop whenever the |
|||
# IOStream's write buffer becomes empty, but it's possible for |
|||
# another callback that runs on the IOLoop before it to |
|||
# simultaneously write more data and finish the request. If |
|||
# there is still data in the IOStream, a future |
|||
# _on_write_complete will be responsible for calling |
|||
# _finish_request. |
|||
if self._request_finished and not self.stream.writing(): |
|||
self._finish_request() |
|||
|
|||
def _finish_request(self): |
|||
if self.no_keep_alive: |
|||
disconnect = True |
|||
else: |
|||
connection_header = self._request.headers.get("Connection") |
|||
if connection_header is not None: |
|||
connection_header = connection_header.lower() |
|||
if self._request.supports_http_1_1(): |
|||
disconnect = connection_header == "close" |
|||
elif ("Content-Length" in self._request.headers |
|||
or self._request.method in ("HEAD", "GET")): |
|||
disconnect = connection_header != "keep-alive" |
|||
else: |
|||
disconnect = True |
|||
self._request = None |
|||
self._request_finished = False |
|||
if disconnect: |
|||
self.stream.close() |
|||
return |
|||
self.stream.read_until(b("\r\n\r\n"), self._header_callback) |
|||
|
|||
def _on_headers(self, data): |
|||
try: |
|||
data = native_str(data.decode('latin1')) |
|||
eol = data.find("\r\n") |
|||
start_line = data[:eol] |
|||
try: |
|||
method, uri, version = start_line.split(" ") |
|||
except ValueError: |
|||
raise _BadRequestException("Malformed HTTP request line") |
|||
if not version.startswith("HTTP/"): |
|||
raise _BadRequestException("Malformed HTTP version in HTTP Request-Line") |
|||
headers = httputil.HTTPHeaders.parse(data[eol:]) |
|||
self._request = HTTPRequest( |
|||
connection=self, method=method, uri=uri, version=version, |
|||
headers=headers, remote_ip=self.address[0]) |
|||
|
|||
content_length = headers.get("Content-Length") |
|||
if content_length: |
|||
content_length = int(content_length) |
|||
if content_length > self.stream.max_buffer_size: |
|||
raise _BadRequestException("Content-Length too long") |
|||
if headers.get("Expect") == "100-continue": |
|||
self.stream.write(b("HTTP/1.1 100 (Continue)\r\n\r\n")) |
|||
self.stream.read_bytes(content_length, self._on_request_body) |
|||
return |
|||
|
|||
self.request_callback(self._request) |
|||
except _BadRequestException, e: |
|||
logging.info("Malformed HTTP request from %s: %s", |
|||
self.address[0], e) |
|||
self.stream.close() |
|||
return |
|||
|
|||
def _on_request_body(self, data): |
|||
self._request.body = data |
|||
content_type = self._request.headers.get("Content-Type", "") |
|||
if self._request.method in ("POST", "PUT"): |
|||
if content_type.startswith("application/x-www-form-urlencoded"): |
|||
arguments = parse_qs_bytes(native_str(self._request.body)) |
|||
for name, values in arguments.iteritems(): |
|||
values = [v for v in values if v] |
|||
if values: |
|||
self._request.arguments.setdefault(name, []).extend( |
|||
values) |
|||
elif content_type.startswith("multipart/form-data"): |
|||
fields = content_type.split(";") |
|||
for field in fields: |
|||
k, sep, v = field.strip().partition("=") |
|||
if k == "boundary" and v: |
|||
httputil.parse_multipart_form_data( |
|||
utf8(v), data, |
|||
self._request.arguments, |
|||
self._request.files) |
|||
break |
|||
else: |
|||
logging.warning("Invalid multipart/form-data") |
|||
self.request_callback(self._request) |
|||
|
|||
|
|||
class HTTPRequest(object): |
|||
"""A single HTTP request. |
|||
|
|||
All attributes are type `str` unless otherwise noted. |
|||
|
|||
.. attribute:: method |
|||
|
|||
HTTP request method, e.g. "GET" or "POST" |
|||
|
|||
.. attribute:: uri |
|||
|
|||
The requested uri. |
|||
|
|||
.. attribute:: path |
|||
|
|||
The path portion of `uri` |
|||
|
|||
.. attribute:: query |
|||
|
|||
The query portion of `uri` |
|||
|
|||
.. attribute:: version |
|||
|
|||
HTTP version specified in request, e.g. "HTTP/1.1" |
|||
|
|||
.. attribute:: headers |
|||
|
|||
`HTTPHeader` dictionary-like object for request headers. Acts like |
|||
a case-insensitive dictionary with additional methods for repeated |
|||
headers. |
|||
|
|||
.. attribute:: body |
|||
|
|||
Request body, if present, as a byte string. |
|||
|
|||
.. attribute:: remote_ip |
|||
|
|||
Client's IP address as a string. If `HTTPServer.xheaders` is set, |
|||
will pass along the real IP address provided by a load balancer |
|||
in the ``X-Real-Ip`` header |
|||
|
|||
.. attribute:: protocol |
|||
|
|||
The protocol used, either "http" or "https". If `HTTPServer.xheaders` |
|||
is set, will pass along the protocol used by a load balancer if |
|||
reported via an ``X-Scheme`` header. |
|||
|
|||
.. attribute:: host |
|||
|
|||
The requested hostname, usually taken from the ``Host`` header. |
|||
|
|||
.. attribute:: arguments |
|||
|
|||
GET/POST arguments are available in the arguments property, which |
|||
maps arguments names to lists of values (to support multiple values |
|||
for individual names). Names are of type `str`, while arguments |
|||
are byte strings. Note that this is different from |
|||
`RequestHandler.get_argument`, which returns argument values as |
|||
unicode strings. |
|||
|
|||
.. attribute:: files |
|||
|
|||
File uploads are available in the files property, which maps file |
|||
names to lists of :class:`HTTPFile`. |
|||
|
|||
.. attribute:: connection |
|||
|
|||
An HTTP request is attached to a single HTTP connection, which can |
|||
be accessed through the "connection" attribute. Since connections |
|||
are typically kept open in HTTP/1.1, multiple requests can be handled |
|||
sequentially on a single connection. |
|||
""" |
|||
def __init__(self, method, uri, version="HTTP/1.0", headers=None, |
|||
body=None, remote_ip=None, protocol=None, host=None, |
|||
files=None, connection=None): |
|||
self.method = method |
|||
self.uri = uri |
|||
self.version = version |
|||
self.headers = headers or httputil.HTTPHeaders() |
|||
self.body = body or "" |
|||
if connection and connection.xheaders: |
|||
# Squid uses X-Forwarded-For, others use X-Real-Ip |
|||
self.remote_ip = self.headers.get( |
|||
"X-Real-Ip", self.headers.get("X-Forwarded-For", remote_ip)) |
|||
if not self._valid_ip(self.remote_ip): |
|||
self.remote_ip = remote_ip |
|||
# AWS uses X-Forwarded-Proto |
|||
self.protocol = self.headers.get( |
|||
"X-Scheme", self.headers.get("X-Forwarded-Proto", protocol)) |
|||
if self.protocol not in ("http", "https"): |
|||
self.protocol = "http" |
|||
else: |
|||
self.remote_ip = remote_ip |
|||
if protocol: |
|||
self.protocol = protocol |
|||
elif connection and isinstance(connection.stream, |
|||
iostream.SSLIOStream): |
|||
self.protocol = "https" |
|||
else: |
|||
self.protocol = "http" |
|||
self.host = host or self.headers.get("Host") or "127.0.0.1" |
|||
self.files = files or {} |
|||
self.connection = connection |
|||
self._start_time = time.time() |
|||
self._finish_time = None |
|||
|
|||
scheme, netloc, path, query, fragment = urlparse.urlsplit(native_str(uri)) |
|||
self.path = path |
|||
self.query = query |
|||
arguments = parse_qs_bytes(query) |
|||
self.arguments = {} |
|||
for name, values in arguments.iteritems(): |
|||
values = [v for v in values if v] |
|||
if values: self.arguments[name] = values |
|||
|
|||
def supports_http_1_1(self): |
|||
"""Returns True if this request supports HTTP/1.1 semantics""" |
|||
return self.version == "HTTP/1.1" |
|||
|
|||
@property |
|||
def cookies(self): |
|||
"""A dictionary of Cookie.Morsel objects.""" |
|||
if not hasattr(self, "_cookies"): |
|||
self._cookies = Cookie.SimpleCookie() |
|||
if "Cookie" in self.headers: |
|||
try: |
|||
self._cookies.load( |
|||
native_str(self.headers["Cookie"])) |
|||
except Exception: |
|||
self._cookies = {} |
|||
return self._cookies |
|||
|
|||
def write(self, chunk, callback=None): |
|||
"""Writes the given chunk to the response stream.""" |
|||
assert isinstance(chunk, bytes_type) |
|||
self.connection.write(chunk, callback=callback) |
|||
|
|||
def finish(self): |
|||
"""Finishes this HTTP request on the open connection.""" |
|||
self.connection.finish() |
|||
self._finish_time = time.time() |
|||
|
|||
def full_url(self): |
|||
"""Reconstructs the full URL for this request.""" |
|||
return self.protocol + "://" + self.host + self.uri |
|||
|
|||
def request_time(self): |
|||
"""Returns the amount of time it took for this request to execute.""" |
|||
if self._finish_time is None: |
|||
return time.time() - self._start_time |
|||
else: |
|||
return self._finish_time - self._start_time |
|||
|
|||
def get_ssl_certificate(self): |
|||
"""Returns the client's SSL certificate, if any. |
|||
|
|||
To use client certificates, the HTTPServer must have been constructed |
|||
with cert_reqs set in ssl_options, e.g.:: |
|||
|
|||
server = HTTPServer(app, |
|||
ssl_options=dict( |
|||
certfile="foo.crt", |
|||
keyfile="foo.key", |
|||
cert_reqs=ssl.CERT_REQUIRED, |
|||
ca_certs="cacert.crt")) |
|||
|
|||
The return value is a dictionary, see SSLSocket.getpeercert() in |
|||
the standard library for more details. |
|||
http://docs.python.org/library/ssl.html#sslsocket-objects |
|||
""" |
|||
try: |
|||
return self.connection.stream.socket.getpeercert() |
|||
except ssl.SSLError: |
|||
return None |
|||
|
|||
def __repr__(self): |
|||
attrs = ("protocol", "host", "method", "uri", "version", "remote_ip", |
|||
"body") |
|||
args = ", ".join(["%s=%r" % (n, getattr(self, n)) for n in attrs]) |
|||
return "%s(%s, headers=%s)" % ( |
|||
self.__class__.__name__, args, dict(self.headers)) |
|||
|
|||
def _valid_ip(self, ip): |
|||
try: |
|||
res = socket.getaddrinfo(ip, 0, socket.AF_UNSPEC, |
|||
socket.SOCK_STREAM, |
|||
0, socket.AI_NUMERICHOST) |
|||
return bool(res) |
|||
except socket.gaierror, e: |
|||
if e.args[0] == socket.EAI_NONAME: |
|||
return False |
|||
raise |
|||
return True |
|||
|
|||
@ -0,0 +1,280 @@ |
|||
#!/usr/bin/env python |
|||
# |
|||
# Copyright 2009 Facebook |
|||
# |
|||
# Licensed under the Apache License, Version 2.0 (the "License"); you may |
|||
# not use this file except in compliance with the License. You may obtain |
|||
# a copy of the License at |
|||
# |
|||
# http://www.apache.org/licenses/LICENSE-2.0 |
|||
# |
|||
# Unless required by applicable law or agreed to in writing, software |
|||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT |
|||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the |
|||
# License for the specific language governing permissions and limitations |
|||
# under the License. |
|||
|
|||
"""HTTP utility code shared by clients and servers.""" |
|||
|
|||
import logging |
|||
import urllib |
|||
import re |
|||
|
|||
from tornado.util import b, ObjectDict |
|||
|
|||
class HTTPHeaders(dict): |
|||
"""A dictionary that maintains Http-Header-Case for all keys. |
|||
|
|||
Supports multiple values per key via a pair of new methods, |
|||
add() and get_list(). The regular dictionary interface returns a single |
|||
value per key, with multiple values joined by a comma. |
|||
|
|||
>>> h = HTTPHeaders({"content-type": "text/html"}) |
|||
>>> h.keys() |
|||
['Content-Type'] |
|||
>>> h["Content-Type"] |
|||
'text/html' |
|||
|
|||
>>> h.add("Set-Cookie", "A=B") |
|||
>>> h.add("Set-Cookie", "C=D") |
|||
>>> h["set-cookie"] |
|||
'A=B,C=D' |
|||
>>> h.get_list("set-cookie") |
|||
['A=B', 'C=D'] |
|||
|
|||
>>> for (k,v) in sorted(h.get_all()): |
|||
... print '%s: %s' % (k,v) |
|||
... |
|||
Content-Type: text/html |
|||
Set-Cookie: A=B |
|||
Set-Cookie: C=D |
|||
""" |
|||
def __init__(self, *args, **kwargs): |
|||
# Don't pass args or kwargs to dict.__init__, as it will bypass |
|||
# our __setitem__ |
|||
dict.__init__(self) |
|||
self._as_list = {} |
|||
self._last_key = None |
|||
self.update(*args, **kwargs) |
|||
|
|||
# new public methods |
|||
|
|||
def add(self, name, value): |
|||
"""Adds a new value for the given key.""" |
|||
norm_name = HTTPHeaders._normalize_name(name) |
|||
self._last_key = norm_name |
|||
if norm_name in self: |
|||
# bypass our override of __setitem__ since it modifies _as_list |
|||
dict.__setitem__(self, norm_name, self[norm_name] + ',' + value) |
|||
self._as_list[norm_name].append(value) |
|||
else: |
|||
self[norm_name] = value |
|||
|
|||
def get_list(self, name): |
|||
"""Returns all values for the given header as a list.""" |
|||
norm_name = HTTPHeaders._normalize_name(name) |
|||
return self._as_list.get(norm_name, []) |
|||
|
|||
def get_all(self): |
|||
"""Returns an iterable of all (name, value) pairs. |
|||
|
|||
If a header has multiple values, multiple pairs will be |
|||
returned with the same name. |
|||
""" |
|||
for name, list in self._as_list.iteritems(): |
|||
for value in list: |
|||
yield (name, value) |
|||
|
|||
def parse_line(self, line): |
|||
"""Updates the dictionary with a single header line. |
|||
|
|||
>>> h = HTTPHeaders() |
|||
>>> h.parse_line("Content-Type: text/html") |
|||
>>> h.get('content-type') |
|||
'text/html' |
|||
""" |
|||
if line[0].isspace(): |
|||
# continuation of a multi-line header |
|||
new_part = ' ' + line.lstrip() |
|||
self._as_list[self._last_key][-1] += new_part |
|||
dict.__setitem__(self, self._last_key, |
|||
self[self._last_key] + new_part) |
|||
else: |
|||
name, value = line.split(":", 1) |
|||
self.add(name, value.strip()) |
|||
|
|||
@classmethod |
|||
def parse(cls, headers): |
|||
"""Returns a dictionary from HTTP header text. |
|||
|
|||
>>> h = HTTPHeaders.parse("Content-Type: text/html\\r\\nContent-Length: 42\\r\\n") |
|||
>>> sorted(h.iteritems()) |
|||
[('Content-Length', '42'), ('Content-Type', 'text/html')] |
|||
""" |
|||
h = cls() |
|||
for line in headers.splitlines(): |
|||
if line: |
|||
h.parse_line(line) |
|||
return h |
|||
|
|||
# dict implementation overrides |
|||
|
|||
def __setitem__(self, name, value): |
|||
norm_name = HTTPHeaders._normalize_name(name) |
|||
dict.__setitem__(self, norm_name, value) |
|||
self._as_list[norm_name] = [value] |
|||
|
|||
def __getitem__(self, name): |
|||
return dict.__getitem__(self, HTTPHeaders._normalize_name(name)) |
|||
|
|||
def __delitem__(self, name): |
|||
norm_name = HTTPHeaders._normalize_name(name) |
|||
dict.__delitem__(self, norm_name) |
|||
del self._as_list[norm_name] |
|||
|
|||
def __contains__(self, name): |
|||
norm_name = HTTPHeaders._normalize_name(name) |
|||
return dict.__contains__(self, norm_name) |
|||
|
|||
def get(self, name, default=None): |
|||
return dict.get(self, HTTPHeaders._normalize_name(name), default) |
|||
|
|||
def update(self, *args, **kwargs): |
|||
# dict.update bypasses our __setitem__ |
|||
for k, v in dict(*args, **kwargs).iteritems(): |
|||
self[k] = v |
|||
|
|||
_NORMALIZED_HEADER_RE = re.compile(r'^[A-Z0-9][a-z0-9]*(-[A-Z0-9][a-z0-9]*)*$') |
|||
_normalized_headers = {} |
|||
|
|||
@staticmethod |
|||
def _normalize_name(name): |
|||
"""Converts a name to Http-Header-Case. |
|||
|
|||
>>> HTTPHeaders._normalize_name("coNtent-TYPE") |
|||
'Content-Type' |
|||
""" |
|||
try: |
|||
return HTTPHeaders._normalized_headers[name] |
|||
except KeyError: |
|||
if HTTPHeaders._NORMALIZED_HEADER_RE.match(name): |
|||
normalized = name |
|||
else: |
|||
normalized = "-".join([w.capitalize() for w in name.split("-")]) |
|||
HTTPHeaders._normalized_headers[name] = normalized |
|||
return normalized |
|||
|
|||
|
|||
def url_concat(url, args): |
|||
"""Concatenate url and argument dictionary regardless of whether |
|||
url has existing query parameters. |
|||
|
|||
>>> url_concat("http://example.com/foo?a=b", dict(c="d")) |
|||
'http://example.com/foo?a=b&c=d' |
|||
""" |
|||
if not args: return url |
|||
if url[-1] not in ('?', '&'): |
|||
url += '&' if ('?' in url) else '?' |
|||
return url + urllib.urlencode(args) |
|||
|
|||
|
|||
class HTTPFile(ObjectDict): |
|||
"""Represents an HTTP file. For backwards compatibility, its instance |
|||
attributes are also accessible as dictionary keys. |
|||
|
|||
:ivar filename: |
|||
:ivar body: |
|||
:ivar content_type: The content_type comes from the provided HTTP header |
|||
and should not be trusted outright given that it can be easily forged. |
|||
""" |
|||
pass |
|||
|
|||
|
|||
def parse_multipart_form_data(boundary, data, arguments, files): |
|||
"""Parses a multipart/form-data body. |
|||
|
|||
The boundary and data parameters are both byte strings. |
|||
The dictionaries given in the arguments and files parameters |
|||
will be updated with the contents of the body. |
|||
""" |
|||
# The standard allows for the boundary to be quoted in the header, |
|||
# although it's rare (it happens at least for google app engine |
|||
# xmpp). I think we're also supposed to handle backslash-escapes |
|||
# here but I'll save that until we see a client that uses them |
|||
# in the wild. |
|||
if boundary.startswith(b('"')) and boundary.endswith(b('"')): |
|||
boundary = boundary[1:-1] |
|||
if data.endswith(b("\r\n")): |
|||
footer_length = len(boundary) + 6 |
|||
else: |
|||
footer_length = len(boundary) + 4 |
|||
parts = data[:-footer_length].split(b("--") + boundary + b("\r\n")) |
|||
for part in parts: |
|||
if not part: continue |
|||
eoh = part.find(b("\r\n\r\n")) |
|||
if eoh == -1: |
|||
logging.warning("multipart/form-data missing headers") |
|||
continue |
|||
headers = HTTPHeaders.parse(part[:eoh].decode("utf-8")) |
|||
disp_header = headers.get("Content-Disposition", "") |
|||
disposition, disp_params = _parse_header(disp_header) |
|||
if disposition != "form-data" or not part.endswith(b("\r\n")): |
|||
logging.warning("Invalid multipart/form-data") |
|||
continue |
|||
value = part[eoh + 4:-2] |
|||
if not disp_params.get("name"): |
|||
logging.warning("multipart/form-data value missing name") |
|||
continue |
|||
name = disp_params["name"] |
|||
if disp_params.get("filename"): |
|||
ctype = headers.get("Content-Type", "application/unknown") |
|||
files.setdefault(name, []).append(HTTPFile( |
|||
filename=disp_params["filename"], body=value, |
|||
content_type=ctype)) |
|||
else: |
|||
arguments.setdefault(name, []).append(value) |
|||
|
|||
|
|||
# _parseparam and _parse_header are copied and modified from python2.7's cgi.py |
|||
# The original 2.7 version of this code did not correctly support some |
|||
# combinations of semicolons and double quotes. |
|||
def _parseparam(s): |
|||
while s[:1] == ';': |
|||
s = s[1:] |
|||
end = s.find(';') |
|||
while end > 0 and (s.count('"', 0, end) - s.count('\\"', 0, end)) % 2: |
|||
end = s.find(';', end + 1) |
|||
if end < 0: |
|||
end = len(s) |
|||
f = s[:end] |
|||
yield f.strip() |
|||
s = s[end:] |
|||
|
|||
def _parse_header(line): |
|||
"""Parse a Content-type like header. |
|||
|
|||
Return the main content-type and a dictionary of options. |
|||
|
|||
""" |
|||
parts = _parseparam(';' + line) |
|||
key = parts.next() |
|||
pdict = {} |
|||
for p in parts: |
|||
i = p.find('=') |
|||
if i >= 0: |
|||
name = p[:i].strip().lower() |
|||
value = p[i+1:].strip() |
|||
if len(value) >= 2 and value[0] == value[-1] == '"': |
|||
value = value[1:-1] |
|||
value = value.replace('\\\\', '\\').replace('\\"', '"') |
|||
pdict[name] = value |
|||
return key, pdict |
|||
|
|||
|
|||
def doctests(): |
|||
import doctest |
|||
return doctest.DocTestSuite() |
|||
|
|||
if __name__ == "__main__": |
|||
import doctest |
|||
doctest.testmod() |
|||
@ -0,0 +1,643 @@ |
|||
#!/usr/bin/env python |
|||
# |
|||
# Copyright 2009 Facebook |
|||
# |
|||
# Licensed under the Apache License, Version 2.0 (the "License"); you may |
|||
# not use this file except in compliance with the License. You may obtain |
|||
# a copy of the License at |
|||
# |
|||
# http://www.apache.org/licenses/LICENSE-2.0 |
|||
# |
|||
# Unless required by applicable law or agreed to in writing, software |
|||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT |
|||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the |
|||
# License for the specific language governing permissions and limitations |
|||
# under the License. |
|||
|
|||
"""An I/O event loop for non-blocking sockets. |
|||
|
|||
Typical applications will use a single `IOLoop` object, in the |
|||
`IOLoop.instance` singleton. The `IOLoop.start` method should usually |
|||
be called at the end of the ``main()`` function. Atypical applications may |
|||
use more than one `IOLoop`, such as one `IOLoop` per thread, or per `unittest` |
|||
case. |
|||
|
|||
In addition to I/O events, the `IOLoop` can also schedule time-based events. |
|||
`IOLoop.add_timeout` is a non-blocking alternative to `time.sleep`. |
|||
""" |
|||
|
|||
from __future__ import with_statement |
|||
|
|||
import datetime |
|||
import errno |
|||
import heapq |
|||
import os |
|||
import logging |
|||
import select |
|||
import thread |
|||
import threading |
|||
import time |
|||
import traceback |
|||
|
|||
from tornado import stack_context |
|||
|
|||
try: |
|||
import signal |
|||
except ImportError: |
|||
signal = None |
|||
|
|||
from tornado.platform.auto import set_close_exec, Waker |
|||
|
|||
|
|||
class IOLoop(object): |
|||
"""A level-triggered I/O loop. |
|||
|
|||
We use epoll (Linux) or kqueue (BSD and Mac OS X; requires python |
|||
2.6+) if they are available, or else we fall back on select(). If |
|||
you are implementing a system that needs to handle thousands of |
|||
simultaneous connections, you should use a system that supports either |
|||
epoll or queue. |
|||
|
|||
Example usage for a simple TCP server:: |
|||
|
|||
import errno |
|||
import functools |
|||
import ioloop |
|||
import socket |
|||
|
|||
def connection_ready(sock, fd, events): |
|||
while True: |
|||
try: |
|||
connection, address = sock.accept() |
|||
except socket.error, e: |
|||
if e.args[0] not in (errno.EWOULDBLOCK, errno.EAGAIN): |
|||
raise |
|||
return |
|||
connection.setblocking(0) |
|||
handle_connection(connection, address) |
|||
|
|||
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM, 0) |
|||
sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) |
|||
sock.setblocking(0) |
|||
sock.bind(("", port)) |
|||
sock.listen(128) |
|||
|
|||
io_loop = ioloop.IOLoop.instance() |
|||
callback = functools.partial(connection_ready, sock) |
|||
io_loop.add_handler(sock.fileno(), callback, io_loop.READ) |
|||
io_loop.start() |
|||
|
|||
""" |
|||
# Constants from the epoll module |
|||
_EPOLLIN = 0x001 |
|||
_EPOLLPRI = 0x002 |
|||
_EPOLLOUT = 0x004 |
|||
_EPOLLERR = 0x008 |
|||
_EPOLLHUP = 0x010 |
|||
_EPOLLRDHUP = 0x2000 |
|||
_EPOLLONESHOT = (1 << 30) |
|||
_EPOLLET = (1 << 31) |
|||
|
|||
# Our events map exactly to the epoll events |
|||
NONE = 0 |
|||
READ = _EPOLLIN |
|||
WRITE = _EPOLLOUT |
|||
ERROR = _EPOLLERR | _EPOLLHUP |
|||
|
|||
def __init__(self, impl=None): |
|||
self._impl = impl or _poll() |
|||
if hasattr(self._impl, 'fileno'): |
|||
set_close_exec(self._impl.fileno()) |
|||
self._handlers = {} |
|||
self._events = {} |
|||
self._callbacks = [] |
|||
self._callback_lock = threading.Lock() |
|||
self._timeouts = [] |
|||
self._running = False |
|||
self._stopped = False |
|||
self._thread_ident = None |
|||
self._blocking_signal_threshold = None |
|||
|
|||
# Create a pipe that we send bogus data to when we want to wake |
|||
# the I/O loop when it is idle |
|||
self._waker = Waker() |
|||
self.add_handler(self._waker.fileno(), |
|||
lambda fd, events: self._waker.consume(), |
|||
self.READ) |
|||
|
|||
@staticmethod |
|||
def instance(): |
|||
"""Returns a global IOLoop instance. |
|||
|
|||
Most single-threaded applications have a single, global IOLoop. |
|||
Use this method instead of passing around IOLoop instances |
|||
throughout your code. |
|||
|
|||
A common pattern for classes that depend on IOLoops is to use |
|||
a default argument to enable programs with multiple IOLoops |
|||
but not require the argument for simpler applications:: |
|||
|
|||
class MyClass(object): |
|||
def __init__(self, io_loop=None): |
|||
self.io_loop = io_loop or IOLoop.instance() |
|||
""" |
|||
if not hasattr(IOLoop, "_instance"): |
|||
IOLoop._instance = IOLoop() |
|||
return IOLoop._instance |
|||
|
|||
@staticmethod |
|||
def initialized(): |
|||
"""Returns true if the singleton instance has been created.""" |
|||
return hasattr(IOLoop, "_instance") |
|||
|
|||
def install(self): |
|||
"""Installs this IOloop object as the singleton instance. |
|||
|
|||
This is normally not necessary as `instance()` will create |
|||
an IOLoop on demand, but you may want to call `install` to use |
|||
a custom subclass of IOLoop. |
|||
""" |
|||
assert not IOLoop.initialized() |
|||
IOLoop._instance = self |
|||
|
|||
def close(self, all_fds=False): |
|||
"""Closes the IOLoop, freeing any resources used. |
|||
|
|||
If ``all_fds`` is true, all file descriptors registered on the |
|||
IOLoop will be closed (not just the ones created by the IOLoop itself. |
|||
""" |
|||
self.remove_handler(self._waker.fileno()) |
|||
if all_fds: |
|||
for fd in self._handlers.keys()[:]: |
|||
try: |
|||
os.close(fd) |
|||
except Exception: |
|||
logging.debug("error closing fd %s", fd, exc_info=True) |
|||
self._waker.close() |
|||
self._impl.close() |
|||
|
|||
def add_handler(self, fd, handler, events): |
|||
"""Registers the given handler to receive the given events for fd.""" |
|||
self._handlers[fd] = stack_context.wrap(handler) |
|||
self._impl.register(fd, events | self.ERROR) |
|||
|
|||
def update_handler(self, fd, events): |
|||
"""Changes the events we listen for fd.""" |
|||
self._impl.modify(fd, events | self.ERROR) |
|||
|
|||
def remove_handler(self, fd): |
|||
"""Stop listening for events on fd.""" |
|||
self._handlers.pop(fd, None) |
|||
self._events.pop(fd, None) |
|||
try: |
|||
self._impl.unregister(fd) |
|||
except (OSError, IOError): |
|||
logging.debug("Error deleting fd from IOLoop", exc_info=True) |
|||
|
|||
def set_blocking_signal_threshold(self, seconds, action): |
|||
"""Sends a signal if the ioloop is blocked for more than s seconds. |
|||
|
|||
Pass seconds=None to disable. Requires python 2.6 on a unixy |
|||
platform. |
|||
|
|||
The action parameter is a python signal handler. Read the |
|||
documentation for the python 'signal' module for more information. |
|||
If action is None, the process will be killed if it is blocked for |
|||
too long. |
|||
""" |
|||
if not hasattr(signal, "setitimer"): |
|||
logging.error("set_blocking_signal_threshold requires a signal module " |
|||
"with the setitimer method") |
|||
return |
|||
self._blocking_signal_threshold = seconds |
|||
if seconds is not None: |
|||
signal.signal(signal.SIGALRM, |
|||
action if action is not None else signal.SIG_DFL) |
|||
|
|||
def set_blocking_log_threshold(self, seconds): |
|||
"""Logs a stack trace if the ioloop is blocked for more than s seconds. |
|||
Equivalent to set_blocking_signal_threshold(seconds, self.log_stack) |
|||
""" |
|||
self.set_blocking_signal_threshold(seconds, self.log_stack) |
|||
|
|||
def log_stack(self, signal, frame): |
|||
"""Signal handler to log the stack trace of the current thread. |
|||
|
|||
For use with set_blocking_signal_threshold. |
|||
""" |
|||
logging.warning('IOLoop blocked for %f seconds in\n%s', |
|||
self._blocking_signal_threshold, |
|||
''.join(traceback.format_stack(frame))) |
|||
|
|||
def start(self): |
|||
"""Starts the I/O loop. |
|||
|
|||
The loop will run until one of the I/O handlers calls stop(), which |
|||
will make the loop stop after the current event iteration completes. |
|||
""" |
|||
if self._stopped: |
|||
self._stopped = False |
|||
return |
|||
self._thread_ident = thread.get_ident() |
|||
self._running = True |
|||
while True: |
|||
poll_timeout = 3600.0 |
|||
|
|||
# Prevent IO event starvation by delaying new callbacks |
|||
# to the next iteration of the event loop. |
|||
with self._callback_lock: |
|||
callbacks = self._callbacks |
|||
self._callbacks = [] |
|||
for callback in callbacks: |
|||
self._run_callback(callback) |
|||
|
|||
if self._timeouts: |
|||
now = time.time() |
|||
while self._timeouts: |
|||
if self._timeouts[0].callback is None: |
|||
# the timeout was cancelled |
|||
heapq.heappop(self._timeouts) |
|||
elif self._timeouts[0].deadline <= now: |
|||
timeout = heapq.heappop(self._timeouts) |
|||
self._run_callback(timeout.callback) |
|||
else: |
|||
seconds = self._timeouts[0].deadline - now |
|||
poll_timeout = min(seconds, poll_timeout) |
|||
break |
|||
|
|||
if self._callbacks: |
|||
# If any callbacks or timeouts called add_callback, |
|||
# we don't want to wait in poll() before we run them. |
|||
poll_timeout = 0.0 |
|||
|
|||
if not self._running: |
|||
break |
|||
|
|||
if self._blocking_signal_threshold is not None: |
|||
# clear alarm so it doesn't fire while poll is waiting for |
|||
# events. |
|||
signal.setitimer(signal.ITIMER_REAL, 0, 0) |
|||
|
|||
try: |
|||
event_pairs = self._impl.poll(poll_timeout) |
|||
except Exception, e: |
|||
# Depending on python version and IOLoop implementation, |
|||
# different exception types may be thrown and there are |
|||
# two ways EINTR might be signaled: |
|||
# * e.errno == errno.EINTR |
|||
# * e.args is like (errno.EINTR, 'Interrupted system call') |
|||
if (getattr(e, 'errno', None) == errno.EINTR or |
|||
(isinstance(getattr(e, 'args', None), tuple) and |
|||
len(e.args) == 2 and e.args[0] == errno.EINTR)): |
|||
continue |
|||
else: |
|||
raise |
|||
|
|||
if self._blocking_signal_threshold is not None: |
|||
signal.setitimer(signal.ITIMER_REAL, |
|||
self._blocking_signal_threshold, 0) |
|||
|
|||
# Pop one fd at a time from the set of pending fds and run |
|||
# its handler. Since that handler may perform actions on |
|||
# other file descriptors, there may be reentrant calls to |
|||
# this IOLoop that update self._events |
|||
self._events.update(event_pairs) |
|||
while self._events: |
|||
fd, events = self._events.popitem() |
|||
try: |
|||
self._handlers[fd](fd, events) |
|||
except (OSError, IOError), e: |
|||
if e.args[0] == errno.EPIPE: |
|||
# Happens when the client closes the connection |
|||
pass |
|||
else: |
|||
logging.error("Exception in I/O handler for fd %s", |
|||
fd, exc_info=True) |
|||
except Exception: |
|||
logging.error("Exception in I/O handler for fd %s", |
|||
fd, exc_info=True) |
|||
# reset the stopped flag so another start/stop pair can be issued |
|||
self._stopped = False |
|||
if self._blocking_signal_threshold is not None: |
|||
signal.setitimer(signal.ITIMER_REAL, 0, 0) |
|||
|
|||
def stop(self): |
|||
"""Stop the loop after the current event loop iteration is complete. |
|||
If the event loop is not currently running, the next call to start() |
|||
will return immediately. |
|||
|
|||
To use asynchronous methods from otherwise-synchronous code (such as |
|||
unit tests), you can start and stop the event loop like this:: |
|||
|
|||
ioloop = IOLoop() |
|||
async_method(ioloop=ioloop, callback=ioloop.stop) |
|||
ioloop.start() |
|||
|
|||
ioloop.start() will return after async_method has run its callback, |
|||
whether that callback was invoked before or after ioloop.start. |
|||
""" |
|||
self._running = False |
|||
self._stopped = True |
|||
self._waker.wake() |
|||
|
|||
def running(self): |
|||
"""Returns true if this IOLoop is currently running.""" |
|||
return self._running |
|||
|
|||
def add_timeout(self, deadline, callback): |
|||
"""Calls the given callback at the time deadline from the I/O loop. |
|||
|
|||
Returns a handle that may be passed to remove_timeout to cancel. |
|||
|
|||
``deadline`` may be a number denoting a unix timestamp (as returned |
|||
by ``time.time()`` or a ``datetime.timedelta`` object for a deadline |
|||
relative to the current time. |
|||
|
|||
Note that it is not safe to call `add_timeout` from other threads. |
|||
Instead, you must use `add_callback` to transfer control to the |
|||
IOLoop's thread, and then call `add_timeout` from there. |
|||
""" |
|||
timeout = _Timeout(deadline, stack_context.wrap(callback)) |
|||
heapq.heappush(self._timeouts, timeout) |
|||
return timeout |
|||
|
|||
def remove_timeout(self, timeout): |
|||
"""Cancels a pending timeout. |
|||
|
|||
The argument is a handle as returned by add_timeout. |
|||
""" |
|||
# Removing from a heap is complicated, so just leave the defunct |
|||
# timeout object in the queue (see discussion in |
|||
# http://docs.python.org/library/heapq.html). |
|||
# If this turns out to be a problem, we could add a garbage |
|||
# collection pass whenever there are too many dead timeouts. |
|||
timeout.callback = None |
|||
|
|||
def add_callback(self, callback): |
|||
"""Calls the given callback on the next I/O loop iteration. |
|||
|
|||
It is safe to call this method from any thread at any time. |
|||
Note that this is the *only* method in IOLoop that makes this |
|||
guarantee; all other interaction with the IOLoop must be done |
|||
from that IOLoop's thread. add_callback() may be used to transfer |
|||
control from other threads to the IOLoop's thread. |
|||
""" |
|||
with self._callback_lock: |
|||
list_empty = not self._callbacks |
|||
self._callbacks.append(stack_context.wrap(callback)) |
|||
if list_empty and thread.get_ident() != self._thread_ident: |
|||
# If we're in the IOLoop's thread, we know it's not currently |
|||
# polling. If we're not, and we added the first callback to an |
|||
# empty list, we may need to wake it up (it may wake up on its |
|||
# own, but an occasional extra wake is harmless). Waking |
|||
# up a polling IOLoop is relatively expensive, so we try to |
|||
# avoid it when we can. |
|||
self._waker.wake() |
|||
|
|||
def _run_callback(self, callback): |
|||
try: |
|||
callback() |
|||
except Exception: |
|||
self.handle_callback_exception(callback) |
|||
|
|||
def handle_callback_exception(self, callback): |
|||
"""This method is called whenever a callback run by the IOLoop |
|||
throws an exception. |
|||
|
|||
By default simply logs the exception as an error. Subclasses |
|||
may override this method to customize reporting of exceptions. |
|||
|
|||
The exception itself is not passed explicitly, but is available |
|||
in sys.exc_info. |
|||
""" |
|||
logging.error("Exception in callback %r", callback, exc_info=True) |
|||
|
|||
|
|||
class _Timeout(object): |
|||
"""An IOLoop timeout, a UNIX timestamp and a callback""" |
|||
|
|||
# Reduce memory overhead when there are lots of pending callbacks |
|||
__slots__ = ['deadline', 'callback'] |
|||
|
|||
def __init__(self, deadline, callback): |
|||
if isinstance(deadline, (int, long, float)): |
|||
self.deadline = deadline |
|||
elif isinstance(deadline, datetime.timedelta): |
|||
self.deadline = time.time() + _Timeout.timedelta_to_seconds(deadline) |
|||
else: |
|||
raise TypeError("Unsupported deadline %r" % deadline) |
|||
self.callback = callback |
|||
|
|||
@staticmethod |
|||
def timedelta_to_seconds(td): |
|||
"""Equivalent to td.total_seconds() (introduced in python 2.7).""" |
|||
return (td.microseconds + (td.seconds + td.days * 24 * 3600) * 10**6) / float(10**6) |
|||
|
|||
# Comparison methods to sort by deadline, with object id as a tiebreaker |
|||
# to guarantee a consistent ordering. The heapq module uses __le__ |
|||
# in python2.5, and __lt__ in 2.6+ (sort() and most other comparisons |
|||
# use __lt__). |
|||
def __lt__(self, other): |
|||
return ((self.deadline, id(self)) < |
|||
(other.deadline, id(other))) |
|||
|
|||
def __le__(self, other): |
|||
return ((self.deadline, id(self)) <= |
|||
(other.deadline, id(other))) |
|||
|
|||
|
|||
class PeriodicCallback(object): |
|||
"""Schedules the given callback to be called periodically. |
|||
|
|||
The callback is called every callback_time milliseconds. |
|||
|
|||
`start` must be called after the PeriodicCallback is created. |
|||
""" |
|||
def __init__(self, callback, callback_time, io_loop=None): |
|||
self.callback = callback |
|||
self.callback_time = callback_time |
|||
self.io_loop = io_loop or IOLoop.instance() |
|||
self._running = False |
|||
self._timeout = None |
|||
|
|||
def start(self): |
|||
"""Starts the timer.""" |
|||
self._running = True |
|||
self._next_timeout = time.time() |
|||
self._schedule_next() |
|||
|
|||
def stop(self): |
|||
"""Stops the timer.""" |
|||
self._running = False |
|||
if self._timeout is not None: |
|||
self.io_loop.remove_timeout(self._timeout) |
|||
self._timeout = None |
|||
|
|||
def _run(self): |
|||
if not self._running: return |
|||
try: |
|||
self.callback() |
|||
except Exception: |
|||
logging.error("Error in periodic callback", exc_info=True) |
|||
self._schedule_next() |
|||
|
|||
def _schedule_next(self): |
|||
if self._running: |
|||
current_time = time.time() |
|||
while self._next_timeout <= current_time: |
|||
self._next_timeout += self.callback_time / 1000.0 |
|||
self._timeout = self.io_loop.add_timeout(self._next_timeout, self._run) |
|||
|
|||
|
|||
class _EPoll(object): |
|||
"""An epoll-based event loop using our C module for Python 2.5 systems""" |
|||
_EPOLL_CTL_ADD = 1 |
|||
_EPOLL_CTL_DEL = 2 |
|||
_EPOLL_CTL_MOD = 3 |
|||
|
|||
def __init__(self): |
|||
self._epoll_fd = epoll.epoll_create() |
|||
|
|||
def fileno(self): |
|||
return self._epoll_fd |
|||
|
|||
def close(self): |
|||
os.close(self._epoll_fd) |
|||
|
|||
def register(self, fd, events): |
|||
epoll.epoll_ctl(self._epoll_fd, self._EPOLL_CTL_ADD, fd, events) |
|||
|
|||
def modify(self, fd, events): |
|||
epoll.epoll_ctl(self._epoll_fd, self._EPOLL_CTL_MOD, fd, events) |
|||
|
|||
def unregister(self, fd): |
|||
epoll.epoll_ctl(self._epoll_fd, self._EPOLL_CTL_DEL, fd, 0) |
|||
|
|||
def poll(self, timeout): |
|||
return epoll.epoll_wait(self._epoll_fd, int(timeout * 1000)) |
|||
|
|||
|
|||
class _KQueue(object): |
|||
"""A kqueue-based event loop for BSD/Mac systems.""" |
|||
def __init__(self): |
|||
self._kqueue = select.kqueue() |
|||
self._active = {} |
|||
|
|||
def fileno(self): |
|||
return self._kqueue.fileno() |
|||
|
|||
def close(self): |
|||
self._kqueue.close() |
|||
|
|||
def register(self, fd, events): |
|||
self._control(fd, events, select.KQ_EV_ADD) |
|||
self._active[fd] = events |
|||
|
|||
def modify(self, fd, events): |
|||
self.unregister(fd) |
|||
self.register(fd, events) |
|||
|
|||
def unregister(self, fd): |
|||
events = self._active.pop(fd) |
|||
self._control(fd, events, select.KQ_EV_DELETE) |
|||
|
|||
def _control(self, fd, events, flags): |
|||
kevents = [] |
|||
if events & IOLoop.WRITE: |
|||
kevents.append(select.kevent( |
|||
fd, filter=select.KQ_FILTER_WRITE, flags=flags)) |
|||
if events & IOLoop.READ or not kevents: |
|||
# Always read when there is not a write |
|||
kevents.append(select.kevent( |
|||
fd, filter=select.KQ_FILTER_READ, flags=flags)) |
|||
# Even though control() takes a list, it seems to return EINVAL |
|||
# on Mac OS X (10.6) when there is more than one event in the list. |
|||
for kevent in kevents: |
|||
self._kqueue.control([kevent], 0) |
|||
|
|||
def poll(self, timeout): |
|||
kevents = self._kqueue.control(None, 1000, timeout) |
|||
events = {} |
|||
for kevent in kevents: |
|||
fd = kevent.ident |
|||
if kevent.filter == select.KQ_FILTER_READ: |
|||
events[fd] = events.get(fd, 0) | IOLoop.READ |
|||
if kevent.filter == select.KQ_FILTER_WRITE: |
|||
if kevent.flags & select.KQ_EV_EOF: |
|||
# If an asynchronous connection is refused, kqueue |
|||
# returns a write event with the EOF flag set. |
|||
# Turn this into an error for consistency with the |
|||
# other IOLoop implementations. |
|||
# Note that for read events, EOF may be returned before |
|||
# all data has been consumed from the socket buffer, |
|||
# so we only check for EOF on write events. |
|||
events[fd] = IOLoop.ERROR |
|||
else: |
|||
events[fd] = events.get(fd, 0) | IOLoop.WRITE |
|||
if kevent.flags & select.KQ_EV_ERROR: |
|||
events[fd] = events.get(fd, 0) | IOLoop.ERROR |
|||
return events.items() |
|||
|
|||
|
|||
class _Select(object): |
|||
"""A simple, select()-based IOLoop implementation for non-Linux systems""" |
|||
def __init__(self): |
|||
self.read_fds = set() |
|||
self.write_fds = set() |
|||
self.error_fds = set() |
|||
self.fd_sets = (self.read_fds, self.write_fds, self.error_fds) |
|||
|
|||
def close(self): |
|||
pass |
|||
|
|||
def register(self, fd, events): |
|||
if events & IOLoop.READ: self.read_fds.add(fd) |
|||
if events & IOLoop.WRITE: self.write_fds.add(fd) |
|||
if events & IOLoop.ERROR: |
|||
self.error_fds.add(fd) |
|||
# Closed connections are reported as errors by epoll and kqueue, |
|||
# but as zero-byte reads by select, so when errors are requested |
|||
# we need to listen for both read and error. |
|||
self.read_fds.add(fd) |
|||
|
|||
def modify(self, fd, events): |
|||
self.unregister(fd) |
|||
self.register(fd, events) |
|||
|
|||
def unregister(self, fd): |
|||
self.read_fds.discard(fd) |
|||
self.write_fds.discard(fd) |
|||
self.error_fds.discard(fd) |
|||
|
|||
def poll(self, timeout): |
|||
readable, writeable, errors = select.select( |
|||
self.read_fds, self.write_fds, self.error_fds, timeout) |
|||
events = {} |
|||
for fd in readable: |
|||
events[fd] = events.get(fd, 0) | IOLoop.READ |
|||
for fd in writeable: |
|||
events[fd] = events.get(fd, 0) | IOLoop.WRITE |
|||
for fd in errors: |
|||
events[fd] = events.get(fd, 0) | IOLoop.ERROR |
|||
return events.items() |
|||
|
|||
|
|||
# Choose a poll implementation. Use epoll if it is available, fall back to |
|||
# select() for non-Linux platforms |
|||
if hasattr(select, "epoll"): |
|||
# Python 2.6+ on Linux |
|||
_poll = select.epoll |
|||
elif hasattr(select, "kqueue"): |
|||
# Python 2.6+ on BSD or Mac |
|||
_poll = _KQueue |
|||
else: |
|||
try: |
|||
# Linux systems with our C module installed |
|||
import epoll |
|||
_poll = _EPoll |
|||
except Exception: |
|||
# All other systems |
|||
import sys |
|||
if "linux" in sys.platform: |
|||
logging.warning("epoll module not found; using select()") |
|||
_poll = _Select |
|||
@ -0,0 +1,728 @@ |
|||
#!/usr/bin/env python |
|||
# |
|||
# Copyright 2009 Facebook |
|||
# |
|||
# Licensed under the Apache License, Version 2.0 (the "License"); you may |
|||
# not use this file except in compliance with the License. You may obtain |
|||
# a copy of the License at |
|||
# |
|||
# http://www.apache.org/licenses/LICENSE-2.0 |
|||
# |
|||
# Unless required by applicable law or agreed to in writing, software |
|||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT |
|||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the |
|||
# License for the specific language governing permissions and limitations |
|||
# under the License. |
|||
|
|||
"""A utility class to write to and read from a non-blocking socket.""" |
|||
|
|||
from __future__ import with_statement |
|||
|
|||
import collections |
|||
import errno |
|||
import logging |
|||
import socket |
|||
import sys |
|||
import re |
|||
|
|||
from tornado import ioloop |
|||
from tornado import stack_context |
|||
from tornado.util import b, bytes_type |
|||
|
|||
try: |
|||
import ssl # Python 2.6+ |
|||
except ImportError: |
|||
ssl = None |
|||
|
|||
class IOStream(object): |
|||
r"""A utility class to write to and read from a non-blocking socket. |
|||
|
|||
We support a non-blocking ``write()`` and a family of ``read_*()`` methods. |
|||
All of the methods take callbacks (since writing and reading are |
|||
non-blocking and asynchronous). |
|||
|
|||
The socket parameter may either be connected or unconnected. For |
|||
server operations the socket is the result of calling socket.accept(). |
|||
For client operations the socket is created with socket.socket(), |
|||
and may either be connected before passing it to the IOStream or |
|||
connected with IOStream.connect. |
|||
|
|||
A very simple (and broken) HTTP client using this class:: |
|||
|
|||
from tornado import ioloop |
|||
from tornado import iostream |
|||
import socket |
|||
|
|||
def send_request(): |
|||
stream.write("GET / HTTP/1.0\r\nHost: friendfeed.com\r\n\r\n") |
|||
stream.read_until("\r\n\r\n", on_headers) |
|||
|
|||
def on_headers(data): |
|||
headers = {} |
|||
for line in data.split("\r\n"): |
|||
parts = line.split(":") |
|||
if len(parts) == 2: |
|||
headers[parts[0].strip()] = parts[1].strip() |
|||
stream.read_bytes(int(headers["Content-Length"]), on_body) |
|||
|
|||
def on_body(data): |
|||
print data |
|||
stream.close() |
|||
ioloop.IOLoop.instance().stop() |
|||
|
|||
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM, 0) |
|||
stream = iostream.IOStream(s) |
|||
stream.connect(("friendfeed.com", 80), send_request) |
|||
ioloop.IOLoop.instance().start() |
|||
|
|||
""" |
|||
def __init__(self, socket, io_loop=None, max_buffer_size=104857600, |
|||
read_chunk_size=4096): |
|||
self.socket = socket |
|||
self.socket.setblocking(False) |
|||
self.io_loop = io_loop or ioloop.IOLoop.instance() |
|||
self.max_buffer_size = max_buffer_size |
|||
self.read_chunk_size = read_chunk_size |
|||
self._read_buffer = collections.deque() |
|||
self._write_buffer = collections.deque() |
|||
self._read_buffer_size = 0 |
|||
self._write_buffer_frozen = False |
|||
self._read_delimiter = None |
|||
self._read_regex = None |
|||
self._read_bytes = None |
|||
self._read_until_close = False |
|||
self._read_callback = None |
|||
self._streaming_callback = None |
|||
self._write_callback = None |
|||
self._close_callback = None |
|||
self._connect_callback = None |
|||
self._connecting = False |
|||
self._state = None |
|||
self._pending_callbacks = 0 |
|||
|
|||
def connect(self, address, callback=None): |
|||
"""Connects the socket to a remote address without blocking. |
|||
|
|||
May only be called if the socket passed to the constructor was |
|||
not previously connected. The address parameter is in the |
|||
same format as for socket.connect, i.e. a (host, port) tuple. |
|||
If callback is specified, it will be called when the |
|||
connection is completed. |
|||
|
|||
Note that it is safe to call IOStream.write while the |
|||
connection is pending, in which case the data will be written |
|||
as soon as the connection is ready. Calling IOStream read |
|||
methods before the socket is connected works on some platforms |
|||
but is non-portable. |
|||
""" |
|||
self._connecting = True |
|||
try: |
|||
self.socket.connect(address) |
|||
except socket.error, e: |
|||
# In non-blocking mode we expect connect() to raise an |
|||
# exception with EINPROGRESS or EWOULDBLOCK. |
|||
# |
|||
# On freebsd, other errors such as ECONNREFUSED may be |
|||
# returned immediately when attempting to connect to |
|||
# localhost, so handle them the same way as an error |
|||
# reported later in _handle_connect. |
|||
if e.args[0] not in (errno.EINPROGRESS, errno.EWOULDBLOCK): |
|||
logging.warning("Connect error on fd %d: %s", |
|||
self.socket.fileno(), e) |
|||
self.close() |
|||
return |
|||
self._connect_callback = stack_context.wrap(callback) |
|||
self._add_io_state(self.io_loop.WRITE) |
|||
|
|||
def read_until_regex(self, regex, callback): |
|||
"""Call callback when we read the given regex pattern.""" |
|||
assert not self._read_callback, "Already reading" |
|||
self._read_regex = re.compile(regex) |
|||
self._read_callback = stack_context.wrap(callback) |
|||
while True: |
|||
# See if we've already got the data from a previous read |
|||
if self._read_from_buffer(): |
|||
return |
|||
self._check_closed() |
|||
if self._read_to_buffer() == 0: |
|||
break |
|||
self._add_io_state(self.io_loop.READ) |
|||
|
|||
def read_until(self, delimiter, callback): |
|||
"""Call callback when we read the given delimiter.""" |
|||
assert not self._read_callback, "Already reading" |
|||
self._read_delimiter = delimiter |
|||
self._read_callback = stack_context.wrap(callback) |
|||
while True: |
|||
# See if we've already got the data from a previous read |
|||
if self._read_from_buffer(): |
|||
return |
|||
self._check_closed() |
|||
if self._read_to_buffer() == 0: |
|||
break |
|||
self._add_io_state(self.io_loop.READ) |
|||
|
|||
def read_bytes(self, num_bytes, callback, streaming_callback=None): |
|||
"""Call callback when we read the given number of bytes. |
|||
|
|||
If a ``streaming_callback`` is given, it will be called with chunks |
|||
of data as they become available, and the argument to the final |
|||
``callback`` will be empty. |
|||
""" |
|||
assert not self._read_callback, "Already reading" |
|||
assert isinstance(num_bytes, (int, long)) |
|||
self._read_bytes = num_bytes |
|||
self._read_callback = stack_context.wrap(callback) |
|||
self._streaming_callback = stack_context.wrap(streaming_callback) |
|||
while True: |
|||
if self._read_from_buffer(): |
|||
return |
|||
self._check_closed() |
|||
if self._read_to_buffer() == 0: |
|||
break |
|||
self._add_io_state(self.io_loop.READ) |
|||
|
|||
def read_until_close(self, callback, streaming_callback=None): |
|||
"""Reads all data from the socket until it is closed. |
|||
|
|||
If a ``streaming_callback`` is given, it will be called with chunks |
|||
of data as they become available, and the argument to the final |
|||
``callback`` will be empty. |
|||
|
|||
Subject to ``max_buffer_size`` limit from `IOStream` constructor if |
|||
a ``streaming_callback`` is not used. |
|||
""" |
|||
assert not self._read_callback, "Already reading" |
|||
if self.closed(): |
|||
self._run_callback(callback, self._consume(self._read_buffer_size)) |
|||
return |
|||
self._read_until_close = True |
|||
self._read_callback = stack_context.wrap(callback) |
|||
self._streaming_callback = stack_context.wrap(streaming_callback) |
|||
self._add_io_state(self.io_loop.READ) |
|||
|
|||
def write(self, data, callback=None): |
|||
"""Write the given data to this stream. |
|||
|
|||
If callback is given, we call it when all of the buffered write |
|||
data has been successfully written to the stream. If there was |
|||
previously buffered write data and an old write callback, that |
|||
callback is simply overwritten with this new callback. |
|||
""" |
|||
assert isinstance(data, bytes_type) |
|||
self._check_closed() |
|||
if data: |
|||
# We use bool(_write_buffer) as a proxy for write_buffer_size>0, |
|||
# so never put empty strings in the buffer. |
|||
self._write_buffer.append(data) |
|||
self._write_callback = stack_context.wrap(callback) |
|||
self._handle_write() |
|||
if self._write_buffer: |
|||
self._add_io_state(self.io_loop.WRITE) |
|||
self._maybe_add_error_listener() |
|||
|
|||
def set_close_callback(self, callback): |
|||
"""Call the given callback when the stream is closed.""" |
|||
self._close_callback = stack_context.wrap(callback) |
|||
|
|||
def close(self): |
|||
"""Close this stream.""" |
|||
if self.socket is not None: |
|||
if self._read_until_close: |
|||
callback = self._read_callback |
|||
self._read_callback = None |
|||
self._read_until_close = False |
|||
self._run_callback(callback, |
|||
self._consume(self._read_buffer_size)) |
|||
if self._state is not None: |
|||
self.io_loop.remove_handler(self.socket.fileno()) |
|||
self._state = None |
|||
self.socket.close() |
|||
self.socket = None |
|||
if self._close_callback and self._pending_callbacks == 0: |
|||
# if there are pending callbacks, don't run the close callback |
|||
# until they're done (see _maybe_add_error_handler) |
|||
cb = self._close_callback |
|||
self._close_callback = None |
|||
self._run_callback(cb) |
|||
|
|||
def reading(self): |
|||
"""Returns true if we are currently reading from the stream.""" |
|||
return self._read_callback is not None |
|||
|
|||
def writing(self): |
|||
"""Returns true if we are currently writing to the stream.""" |
|||
return bool(self._write_buffer) |
|||
|
|||
def closed(self): |
|||
"""Returns true if the stream has been closed.""" |
|||
return self.socket is None |
|||
|
|||
def _handle_events(self, fd, events): |
|||
if not self.socket: |
|||
logging.warning("Got events for closed stream %d", fd) |
|||
return |
|||
try: |
|||
if events & self.io_loop.READ: |
|||
self._handle_read() |
|||
if not self.socket: |
|||
return |
|||
if events & self.io_loop.WRITE: |
|||
if self._connecting: |
|||
self._handle_connect() |
|||
self._handle_write() |
|||
if not self.socket: |
|||
return |
|||
if events & self.io_loop.ERROR: |
|||
# We may have queued up a user callback in _handle_read or |
|||
# _handle_write, so don't close the IOStream until those |
|||
# callbacks have had a chance to run. |
|||
self.io_loop.add_callback(self.close) |
|||
return |
|||
state = self.io_loop.ERROR |
|||
if self.reading(): |
|||
state |= self.io_loop.READ |
|||
if self.writing(): |
|||
state |= self.io_loop.WRITE |
|||
if state == self.io_loop.ERROR: |
|||
state |= self.io_loop.READ |
|||
if state != self._state: |
|||
assert self._state is not None, \ |
|||
"shouldn't happen: _handle_events without self._state" |
|||
self._state = state |
|||
self.io_loop.update_handler(self.socket.fileno(), self._state) |
|||
except Exception: |
|||
logging.error("Uncaught exception, closing connection.", |
|||
exc_info=True) |
|||
self.close() |
|||
raise |
|||
|
|||
def _run_callback(self, callback, *args): |
|||
def wrapper(): |
|||
self._pending_callbacks -= 1 |
|||
try: |
|||
callback(*args) |
|||
except Exception: |
|||
logging.error("Uncaught exception, closing connection.", |
|||
exc_info=True) |
|||
# Close the socket on an uncaught exception from a user callback |
|||
# (It would eventually get closed when the socket object is |
|||
# gc'd, but we don't want to rely on gc happening before we |
|||
# run out of file descriptors) |
|||
self.close() |
|||
# Re-raise the exception so that IOLoop.handle_callback_exception |
|||
# can see it and log the error |
|||
raise |
|||
self._maybe_add_error_listener() |
|||
# We schedule callbacks to be run on the next IOLoop iteration |
|||
# rather than running them directly for several reasons: |
|||
# * Prevents unbounded stack growth when a callback calls an |
|||
# IOLoop operation that immediately runs another callback |
|||
# * Provides a predictable execution context for e.g. |
|||
# non-reentrant mutexes |
|||
# * Ensures that the try/except in wrapper() is run outside |
|||
# of the application's StackContexts |
|||
with stack_context.NullContext(): |
|||
# stack_context was already captured in callback, we don't need to |
|||
# capture it again for IOStream's wrapper. This is especially |
|||
# important if the callback was pre-wrapped before entry to |
|||
# IOStream (as in HTTPConnection._header_callback), as we could |
|||
# capture and leak the wrong context here. |
|||
self._pending_callbacks += 1 |
|||
self.io_loop.add_callback(wrapper) |
|||
|
|||
def _handle_read(self): |
|||
while True: |
|||
try: |
|||
# Read from the socket until we get EWOULDBLOCK or equivalent. |
|||
# SSL sockets do some internal buffering, and if the data is |
|||
# sitting in the SSL object's buffer select() and friends |
|||
# can't see it; the only way to find out if it's there is to |
|||
# try to read it. |
|||
result = self._read_to_buffer() |
|||
except Exception: |
|||
self.close() |
|||
return |
|||
if result == 0: |
|||
break |
|||
else: |
|||
if self._read_from_buffer(): |
|||
return |
|||
|
|||
def _read_from_socket(self): |
|||
"""Attempts to read from the socket. |
|||
|
|||
Returns the data read or None if there is nothing to read. |
|||
May be overridden in subclasses. |
|||
""" |
|||
try: |
|||
chunk = self.socket.recv(self.read_chunk_size) |
|||
except socket.error, e: |
|||
if e.args[0] in (errno.EWOULDBLOCK, errno.EAGAIN): |
|||
return None |
|||
else: |
|||
raise |
|||
if not chunk: |
|||
self.close() |
|||
return None |
|||
return chunk |
|||
|
|||
def _read_to_buffer(self): |
|||
"""Reads from the socket and appends the result to the read buffer. |
|||
|
|||
Returns the number of bytes read. Returns 0 if there is nothing |
|||
to read (i.e. the read returns EWOULDBLOCK or equivalent). On |
|||
error closes the socket and raises an exception. |
|||
""" |
|||
try: |
|||
chunk = self._read_from_socket() |
|||
except socket.error, e: |
|||
# ssl.SSLError is a subclass of socket.error |
|||
logging.warning("Read error on %d: %s", |
|||
self.socket.fileno(), e) |
|||
self.close() |
|||
raise |
|||
if chunk is None: |
|||
return 0 |
|||
self._read_buffer.append(chunk) |
|||
self._read_buffer_size += len(chunk) |
|||
if self._read_buffer_size >= self.max_buffer_size: |
|||
logging.error("Reached maximum read buffer size") |
|||
self.close() |
|||
raise IOError("Reached maximum read buffer size") |
|||
return len(chunk) |
|||
|
|||
def _read_from_buffer(self): |
|||
"""Attempts to complete the currently-pending read from the buffer. |
|||
|
|||
Returns True if the read was completed. |
|||
""" |
|||
if self._read_bytes is not None: |
|||
if self._streaming_callback is not None and self._read_buffer_size: |
|||
bytes_to_consume = min(self._read_bytes, self._read_buffer_size) |
|||
self._read_bytes -= bytes_to_consume |
|||
self._run_callback(self._streaming_callback, |
|||
self._consume(bytes_to_consume)) |
|||
if self._read_buffer_size >= self._read_bytes: |
|||
num_bytes = self._read_bytes |
|||
callback = self._read_callback |
|||
self._read_callback = None |
|||
self._streaming_callback = None |
|||
self._read_bytes = None |
|||
self._run_callback(callback, self._consume(num_bytes)) |
|||
return True |
|||
elif self._read_delimiter is not None: |
|||
# Multi-byte delimiters (e.g. '\r\n') may straddle two |
|||
# chunks in the read buffer, so we can't easily find them |
|||
# without collapsing the buffer. However, since protocols |
|||
# using delimited reads (as opposed to reads of a known |
|||
# length) tend to be "line" oriented, the delimiter is likely |
|||
# to be in the first few chunks. Merge the buffer gradually |
|||
# since large merges are relatively expensive and get undone in |
|||
# consume(). |
|||
loc = -1 |
|||
if self._read_buffer: |
|||
loc = self._read_buffer[0].find(self._read_delimiter) |
|||
while loc == -1 and len(self._read_buffer) > 1: |
|||
# Grow by doubling, but don't split the second chunk just |
|||
# because the first one is small. |
|||
new_len = max(len(self._read_buffer[0]) * 2, |
|||
(len(self._read_buffer[0]) + |
|||
len(self._read_buffer[1]))) |
|||
_merge_prefix(self._read_buffer, new_len) |
|||
loc = self._read_buffer[0].find(self._read_delimiter) |
|||
if loc != -1: |
|||
callback = self._read_callback |
|||
delimiter_len = len(self._read_delimiter) |
|||
self._read_callback = None |
|||
self._streaming_callback = None |
|||
self._read_delimiter = None |
|||
self._run_callback(callback, |
|||
self._consume(loc + delimiter_len)) |
|||
return True |
|||
elif self._read_regex is not None: |
|||
m = None |
|||
if self._read_buffer: |
|||
m = self._read_regex.search(self._read_buffer[0]) |
|||
while m is None and len(self._read_buffer) > 1: |
|||
# Grow by doubling, but don't split the second chunk just |
|||
# because the first one is small. |
|||
new_len = max(len(self._read_buffer[0]) * 2, |
|||
(len(self._read_buffer[0]) + |
|||
len(self._read_buffer[1]))) |
|||
_merge_prefix(self._read_buffer, new_len) |
|||
m = self._read_regex.search(self._read_buffer[0]) |
|||
_merge_prefix(self._read_buffer, sys.maxint) |
|||
m = self._read_regex.search(self._read_buffer[0]) |
|||
if m: |
|||
callback = self._read_callback |
|||
self._read_callback = None |
|||
self._streaming_callback = None |
|||
self._read_regex = None |
|||
self._run_callback(callback, self._consume(m.end())) |
|||
return True |
|||
elif self._read_until_close: |
|||
if self._streaming_callback is not None and self._read_buffer_size: |
|||
self._run_callback(self._streaming_callback, |
|||
self._consume(self._read_buffer_size)) |
|||
return False |
|||
|
|||
def _handle_connect(self): |
|||
err = self.socket.getsockopt(socket.SOL_SOCKET, socket.SO_ERROR) |
|||
if err != 0: |
|||
# IOLoop implementations may vary: some of them return |
|||
# an error state before the socket becomes writable, so |
|||
# in that case a connection failure would be handled by the |
|||
# error path in _handle_events instead of here. |
|||
logging.warning("Connect error on fd %d: %s", |
|||
self.socket.fileno(), errno.errorcode[err]) |
|||
self.close() |
|||
return |
|||
if self._connect_callback is not None: |
|||
callback = self._connect_callback |
|||
self._connect_callback = None |
|||
self._run_callback(callback) |
|||
self._connecting = False |
|||
|
|||
def _handle_write(self): |
|||
while self._write_buffer: |
|||
try: |
|||
if not self._write_buffer_frozen: |
|||
# On windows, socket.send blows up if given a |
|||
# write buffer that's too large, instead of just |
|||
# returning the number of bytes it was able to |
|||
# process. Therefore we must not call socket.send |
|||
# with more than 128KB at a time. |
|||
_merge_prefix(self._write_buffer, 128 * 1024) |
|||
num_bytes = self.socket.send(self._write_buffer[0]) |
|||
if num_bytes == 0: |
|||
# With OpenSSL, if we couldn't write the entire buffer, |
|||
# the very same string object must be used on the |
|||
# next call to send. Therefore we suppress |
|||
# merging the write buffer after an incomplete send. |
|||
# A cleaner solution would be to set |
|||
# SSL_MODE_ACCEPT_MOVING_WRITE_BUFFER, but this is |
|||
# not yet accessible from python |
|||
# (http://bugs.python.org/issue8240) |
|||
self._write_buffer_frozen = True |
|||
break |
|||
self._write_buffer_frozen = False |
|||
_merge_prefix(self._write_buffer, num_bytes) |
|||
self._write_buffer.popleft() |
|||
except socket.error, e: |
|||
if e.args[0] in (errno.EWOULDBLOCK, errno.EAGAIN): |
|||
self._write_buffer_frozen = True |
|||
break |
|||
else: |
|||
logging.warning("Write error on %d: %s", |
|||
self.socket.fileno(), e) |
|||
self.close() |
|||
return |
|||
if not self._write_buffer and self._write_callback: |
|||
callback = self._write_callback |
|||
self._write_callback = None |
|||
self._run_callback(callback) |
|||
|
|||
def _consume(self, loc): |
|||
if loc == 0: |
|||
return b("") |
|||
_merge_prefix(self._read_buffer, loc) |
|||
self._read_buffer_size -= loc |
|||
return self._read_buffer.popleft() |
|||
|
|||
def _check_closed(self): |
|||
if not self.socket: |
|||
raise IOError("Stream is closed") |
|||
|
|||
def _maybe_add_error_listener(self): |
|||
if self._state is None and self._pending_callbacks == 0: |
|||
if self.socket is None: |
|||
cb = self._close_callback |
|||
if cb is not None: |
|||
self._close_callback = None |
|||
self._run_callback(cb) |
|||
else: |
|||
self._add_io_state(ioloop.IOLoop.READ) |
|||
|
|||
def _add_io_state(self, state): |
|||
"""Adds `state` (IOLoop.{READ,WRITE} flags) to our event handler. |
|||
|
|||
Implementation notes: Reads and writes have a fast path and a |
|||
slow path. The fast path reads synchronously from socket |
|||
buffers, while the slow path uses `_add_io_state` to schedule |
|||
an IOLoop callback. Note that in both cases, the callback is |
|||
run asynchronously with `_run_callback`. |
|||
|
|||
To detect closed connections, we must have called |
|||
`_add_io_state` at some point, but we want to delay this as |
|||
much as possible so we don't have to set an `IOLoop.ERROR` |
|||
listener that will be overwritten by the next slow-path |
|||
operation. As long as there are callbacks scheduled for |
|||
fast-path ops, those callbacks may do more reads. |
|||
If a sequence of fast-path ops do not end in a slow-path op, |
|||
(e.g. for an @asynchronous long-poll request), we must add |
|||
the error handler. This is done in `_run_callback` and `write` |
|||
(since the write callback is optional so we can have a |
|||
fast-path write with no `_run_callback`) |
|||
""" |
|||
if self.socket is None: |
|||
# connection has been closed, so there can be no future events |
|||
return |
|||
if self._state is None: |
|||
self._state = ioloop.IOLoop.ERROR | state |
|||
with stack_context.NullContext(): |
|||
self.io_loop.add_handler( |
|||
self.socket.fileno(), self._handle_events, self._state) |
|||
elif not self._state & state: |
|||
self._state = self._state | state |
|||
self.io_loop.update_handler(self.socket.fileno(), self._state) |
|||
|
|||
|
|||
class SSLIOStream(IOStream): |
|||
"""A utility class to write to and read from a non-blocking SSL socket. |
|||
|
|||
If the socket passed to the constructor is already connected, |
|||
it should be wrapped with:: |
|||
|
|||
ssl.wrap_socket(sock, do_handshake_on_connect=False, **kwargs) |
|||
|
|||
before constructing the SSLIOStream. Unconnected sockets will be |
|||
wrapped when IOStream.connect is finished. |
|||
""" |
|||
def __init__(self, *args, **kwargs): |
|||
"""Creates an SSLIOStream. |
|||
|
|||
If a dictionary is provided as keyword argument ssl_options, |
|||
it will be used as additional keyword arguments to ssl.wrap_socket. |
|||
""" |
|||
self._ssl_options = kwargs.pop('ssl_options', {}) |
|||
super(SSLIOStream, self).__init__(*args, **kwargs) |
|||
self._ssl_accepting = True |
|||
self._handshake_reading = False |
|||
self._handshake_writing = False |
|||
|
|||
def reading(self): |
|||
return self._handshake_reading or super(SSLIOStream, self).reading() |
|||
|
|||
def writing(self): |
|||
return self._handshake_writing or super(SSLIOStream, self).writing() |
|||
|
|||
def _do_ssl_handshake(self): |
|||
# Based on code from test_ssl.py in the python stdlib |
|||
try: |
|||
self._handshake_reading = False |
|||
self._handshake_writing = False |
|||
self.socket.do_handshake() |
|||
except ssl.SSLError, err: |
|||
if err.args[0] == ssl.SSL_ERROR_WANT_READ: |
|||
self._handshake_reading = True |
|||
return |
|||
elif err.args[0] == ssl.SSL_ERROR_WANT_WRITE: |
|||
self._handshake_writing = True |
|||
return |
|||
elif err.args[0] in (ssl.SSL_ERROR_EOF, |
|||
ssl.SSL_ERROR_ZERO_RETURN): |
|||
return self.close() |
|||
elif err.args[0] == ssl.SSL_ERROR_SSL: |
|||
logging.warning("SSL Error on %d: %s", self.socket.fileno(), err) |
|||
return self.close() |
|||
raise |
|||
except socket.error, err: |
|||
if err.args[0] == errno.ECONNABORTED: |
|||
return self.close() |
|||
else: |
|||
self._ssl_accepting = False |
|||
super(SSLIOStream, self)._handle_connect() |
|||
|
|||
def _handle_read(self): |
|||
if self._ssl_accepting: |
|||
self._do_ssl_handshake() |
|||
return |
|||
super(SSLIOStream, self)._handle_read() |
|||
|
|||
def _handle_write(self): |
|||
if self._ssl_accepting: |
|||
self._do_ssl_handshake() |
|||
return |
|||
super(SSLIOStream, self)._handle_write() |
|||
|
|||
def _handle_connect(self): |
|||
self.socket = ssl.wrap_socket(self.socket, |
|||
do_handshake_on_connect=False, |
|||
**self._ssl_options) |
|||
# Don't call the superclass's _handle_connect (which is responsible |
|||
# for telling the application that the connection is complete) |
|||
# until we've completed the SSL handshake (so certificates are |
|||
# available, etc). |
|||
|
|||
|
|||
def _read_from_socket(self): |
|||
if self._ssl_accepting: |
|||
# If the handshake hasn't finished yet, there can't be anything |
|||
# to read (attempting to read may or may not raise an exception |
|||
# depending on the SSL version) |
|||
return None |
|||
try: |
|||
# SSLSocket objects have both a read() and recv() method, |
|||
# while regular sockets only have recv(). |
|||
# The recv() method blocks (at least in python 2.6) if it is |
|||
# called when there is nothing to read, so we have to use |
|||
# read() instead. |
|||
chunk = self.socket.read(self.read_chunk_size) |
|||
except ssl.SSLError, e: |
|||
# SSLError is a subclass of socket.error, so this except |
|||
# block must come first. |
|||
if e.args[0] == ssl.SSL_ERROR_WANT_READ: |
|||
return None |
|||
else: |
|||
raise |
|||
except socket.error, e: |
|||
if e.args[0] in (errno.EWOULDBLOCK, errno.EAGAIN): |
|||
return None |
|||
else: |
|||
raise |
|||
if not chunk: |
|||
self.close() |
|||
return None |
|||
return chunk |
|||
|
|||
def _merge_prefix(deque, size): |
|||
"""Replace the first entries in a deque of strings with a single |
|||
string of up to size bytes. |
|||
|
|||
>>> d = collections.deque(['abc', 'de', 'fghi', 'j']) |
|||
>>> _merge_prefix(d, 5); print d |
|||
deque(['abcde', 'fghi', 'j']) |
|||
|
|||
Strings will be split as necessary to reach the desired size. |
|||
>>> _merge_prefix(d, 7); print d |
|||
deque(['abcdefg', 'hi', 'j']) |
|||
|
|||
>>> _merge_prefix(d, 3); print d |
|||
deque(['abc', 'defg', 'hi', 'j']) |
|||
|
|||
>>> _merge_prefix(d, 100); print d |
|||
deque(['abcdefghij']) |
|||
""" |
|||
if len(deque) == 1 and len(deque[0]) <= size: |
|||
return |
|||
prefix = [] |
|||
remaining = size |
|||
while deque and remaining > 0: |
|||
chunk = deque.popleft() |
|||
if len(chunk) > remaining: |
|||
deque.appendleft(chunk[remaining:]) |
|||
chunk = chunk[:remaining] |
|||
prefix.append(chunk) |
|||
remaining -= len(chunk) |
|||
# This data structure normally just contains byte strings, but |
|||
# the unittest gets messy if it doesn't use the default str() type, |
|||
# so do the merge based on the type of data that's actually present. |
|||
if prefix: |
|||
deque.appendleft(type(prefix[0])().join(prefix)) |
|||
if not deque: |
|||
deque.appendleft(b("")) |
|||
|
|||
def doctests(): |
|||
import doctest |
|||
return doctest.DocTestSuite() |
|||
@ -0,0 +1,472 @@ |
|||
#!/usr/bin/env python |
|||
# |
|||
# Copyright 2009 Facebook |
|||
# |
|||
# Licensed under the Apache License, Version 2.0 (the "License"); you may |
|||
# not use this file except in compliance with the License. You may obtain |
|||
# a copy of the License at |
|||
# |
|||
# http://www.apache.org/licenses/LICENSE-2.0 |
|||
# |
|||
# Unless required by applicable law or agreed to in writing, software |
|||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT |
|||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the |
|||
# License for the specific language governing permissions and limitations |
|||
# under the License. |
|||
|
|||
"""Translation methods for generating localized strings. |
|||
|
|||
To load a locale and generate a translated string:: |
|||
|
|||
user_locale = locale.get("es_LA") |
|||
print user_locale.translate("Sign out") |
|||
|
|||
locale.get() returns the closest matching locale, not necessarily the |
|||
specific locale you requested. You can support pluralization with |
|||
additional arguments to translate(), e.g.:: |
|||
|
|||
people = [...] |
|||
message = user_locale.translate( |
|||
"%(list)s is online", "%(list)s are online", len(people)) |
|||
print message % {"list": user_locale.list(people)} |
|||
|
|||
The first string is chosen if len(people) == 1, otherwise the second |
|||
string is chosen. |
|||
|
|||
Applications should call one of load_translations (which uses a simple |
|||
CSV format) or load_gettext_translations (which uses the .mo format |
|||
supported by gettext and related tools). If neither method is called, |
|||
the locale.translate method will simply return the original string. |
|||
""" |
|||
|
|||
import csv |
|||
import datetime |
|||
import logging |
|||
import os |
|||
import re |
|||
|
|||
_default_locale = "en_US" |
|||
_translations = {} |
|||
_supported_locales = frozenset([_default_locale]) |
|||
_use_gettext = False |
|||
|
|||
def get(*locale_codes): |
|||
"""Returns the closest match for the given locale codes. |
|||
|
|||
We iterate over all given locale codes in order. If we have a tight |
|||
or a loose match for the code (e.g., "en" for "en_US"), we return |
|||
the locale. Otherwise we move to the next code in the list. |
|||
|
|||
By default we return en_US if no translations are found for any of |
|||
the specified locales. You can change the default locale with |
|||
set_default_locale() below. |
|||
""" |
|||
return Locale.get_closest(*locale_codes) |
|||
|
|||
|
|||
def set_default_locale(code): |
|||
"""Sets the default locale, used in get_closest_locale(). |
|||
|
|||
The default locale is assumed to be the language used for all strings |
|||
in the system. The translations loaded from disk are mappings from |
|||
the default locale to the destination locale. Consequently, you don't |
|||
need to create a translation file for the default locale. |
|||
""" |
|||
global _default_locale |
|||
global _supported_locales |
|||
_default_locale = code |
|||
_supported_locales = frozenset(_translations.keys() + [_default_locale]) |
|||
|
|||
|
|||
def load_translations(directory): |
|||
u"""Loads translations from CSV files in a directory. |
|||
|
|||
Translations are strings with optional Python-style named placeholders |
|||
(e.g., "My name is %(name)s") and their associated translations. |
|||
|
|||
The directory should have translation files of the form LOCALE.csv, |
|||
e.g. es_GT.csv. The CSV files should have two or three columns: string, |
|||
translation, and an optional plural indicator. Plural indicators should |
|||
be one of "plural" or "singular". A given string can have both singular |
|||
and plural forms. For example "%(name)s liked this" may have a |
|||
different verb conjugation depending on whether %(name)s is one |
|||
name or a list of names. There should be two rows in the CSV file for |
|||
that string, one with plural indicator "singular", and one "plural". |
|||
For strings with no verbs that would change on translation, simply |
|||
use "unknown" or the empty string (or don't include the column at all). |
|||
|
|||
The file is read using the csv module in the default "excel" dialect. |
|||
In this format there should not be spaces after the commas. |
|||
|
|||
Example translation es_LA.csv: |
|||
|
|||
"I love you","Te amo" |
|||
"%(name)s liked this","A %(name)s les gust\u00f3 esto","plural" |
|||
"%(name)s liked this","A %(name)s le gust\u00f3 esto","singular" |
|||
|
|||
""" |
|||
global _translations |
|||
global _supported_locales |
|||
_translations = {} |
|||
for path in os.listdir(directory): |
|||
if not path.endswith(".csv"): continue |
|||
locale, extension = path.split(".") |
|||
if not re.match("[a-z]+(_[A-Z]+)?$", locale): |
|||
logging.error("Unrecognized locale %r (path: %s)", locale, |
|||
os.path.join(directory, path)) |
|||
continue |
|||
f = open(os.path.join(directory, path), "r") |
|||
_translations[locale] = {} |
|||
for i, row in enumerate(csv.reader(f)): |
|||
if not row or len(row) < 2: continue |
|||
row = [c.decode("utf-8").strip() for c in row] |
|||
english, translation = row[:2] |
|||
if len(row) > 2: |
|||
plural = row[2] or "unknown" |
|||
else: |
|||
plural = "unknown" |
|||
if plural not in ("plural", "singular", "unknown"): |
|||
logging.error("Unrecognized plural indicator %r in %s line %d", |
|||
plural, path, i + 1) |
|||
continue |
|||
_translations[locale].setdefault(plural, {})[english] = translation |
|||
f.close() |
|||
_supported_locales = frozenset(_translations.keys() + [_default_locale]) |
|||
logging.info("Supported locales: %s", sorted(_supported_locales)) |
|||
|
|||
def load_gettext_translations(directory, domain): |
|||
"""Loads translations from gettext's locale tree |
|||
|
|||
Locale tree is similar to system's /usr/share/locale, like: |
|||
|
|||
{directory}/{lang}/LC_MESSAGES/{domain}.mo |
|||
|
|||
Three steps are required to have you app translated: |
|||
|
|||
1. Generate POT translation file |
|||
xgettext --language=Python --keyword=_:1,2 -d cyclone file1.py file2.html etc |
|||
|
|||
2. Merge against existing POT file: |
|||
msgmerge old.po cyclone.po > new.po |
|||
|
|||
3. Compile: |
|||
msgfmt cyclone.po -o {directory}/pt_BR/LC_MESSAGES/cyclone.mo |
|||
""" |
|||
import gettext |
|||
global _translations |
|||
global _supported_locales |
|||
global _use_gettext |
|||
_translations = {} |
|||
for lang in os.listdir(directory): |
|||
if lang.startswith('.'): continue # skip .svn, etc |
|||
if os.path.isfile(os.path.join(directory, lang)): continue |
|||
try: |
|||
os.stat(os.path.join(directory, lang, "LC_MESSAGES", domain+".mo")) |
|||
_translations[lang] = gettext.translation(domain, directory, |
|||
languages=[lang]) |
|||
except Exception, e: |
|||
logging.error("Cannot load translation for '%s': %s", lang, str(e)) |
|||
continue |
|||
_supported_locales = frozenset(_translations.keys() + [_default_locale]) |
|||
_use_gettext = True |
|||
logging.info("Supported locales: %s", sorted(_supported_locales)) |
|||
|
|||
|
|||
def get_supported_locales(cls): |
|||
"""Returns a list of all the supported locale codes.""" |
|||
return _supported_locales |
|||
|
|||
|
|||
class Locale(object): |
|||
"""Object representing a locale. |
|||
|
|||
After calling one of `load_translations` or `load_gettext_translations`, |
|||
call `get` or `get_closest` to get a Locale object. |
|||
""" |
|||
@classmethod |
|||
def get_closest(cls, *locale_codes): |
|||
"""Returns the closest match for the given locale code.""" |
|||
for code in locale_codes: |
|||
if not code: continue |
|||
code = code.replace("-", "_") |
|||
parts = code.split("_") |
|||
if len(parts) > 2: |
|||
continue |
|||
elif len(parts) == 2: |
|||
code = parts[0].lower() + "_" + parts[1].upper() |
|||
if code in _supported_locales: |
|||
return cls.get(code) |
|||
if parts[0].lower() in _supported_locales: |
|||
return cls.get(parts[0].lower()) |
|||
return cls.get(_default_locale) |
|||
|
|||
@classmethod |
|||
def get(cls, code): |
|||
"""Returns the Locale for the given locale code. |
|||
|
|||
If it is not supported, we raise an exception. |
|||
""" |
|||
if not hasattr(cls, "_cache"): |
|||
cls._cache = {} |
|||
if code not in cls._cache: |
|||
assert code in _supported_locales |
|||
translations = _translations.get(code, None) |
|||
if translations is None: |
|||
locale = CSVLocale(code, {}) |
|||
elif _use_gettext: |
|||
locale = GettextLocale(code, translations) |
|||
else: |
|||
locale = CSVLocale(code, translations) |
|||
cls._cache[code] = locale |
|||
return cls._cache[code] |
|||
|
|||
def __init__(self, code, translations): |
|||
self.code = code |
|||
self.name = LOCALE_NAMES.get(code, {}).get("name", u"Unknown") |
|||
self.rtl = False |
|||
for prefix in ["fa", "ar", "he"]: |
|||
if self.code.startswith(prefix): |
|||
self.rtl = True |
|||
break |
|||
self.translations = translations |
|||
|
|||
# Initialize strings for date formatting |
|||
_ = self.translate |
|||
self._months = [ |
|||
_("January"), _("February"), _("March"), _("April"), |
|||
_("May"), _("June"), _("July"), _("August"), |
|||
_("September"), _("October"), _("November"), _("December")] |
|||
self._weekdays = [ |
|||
_("Monday"), _("Tuesday"), _("Wednesday"), _("Thursday"), |
|||
_("Friday"), _("Saturday"), _("Sunday")] |
|||
|
|||
def translate(self, message, plural_message=None, count=None): |
|||
"""Returns the translation for the given message for this locale. |
|||
|
|||
If plural_message is given, you must also provide count. We return |
|||
plural_message when count != 1, and we return the singular form |
|||
for the given message when count == 1. |
|||
""" |
|||
raise NotImplementedError() |
|||
|
|||
def format_date(self, date, gmt_offset=0, relative=True, shorter=False, |
|||
full_format=False): |
|||
"""Formats the given date (which should be GMT). |
|||
|
|||
By default, we return a relative time (e.g., "2 minutes ago"). You |
|||
can return an absolute date string with relative=False. |
|||
|
|||
You can force a full format date ("July 10, 1980") with |
|||
full_format=True. |
|||
|
|||
This method is primarily intended for dates in the past. |
|||
For dates in the future, we fall back to full format. |
|||
""" |
|||
if self.code.startswith("ru"): |
|||
relative = False |
|||
if type(date) in (int, long, float): |
|||
date = datetime.datetime.utcfromtimestamp(date) |
|||
now = datetime.datetime.utcnow() |
|||
if date > now: |
|||
if relative and (date - now).seconds < 60: |
|||
# Due to click skew, things are some things slightly |
|||
# in the future. Round timestamps in the immediate |
|||
# future down to now in relative mode. |
|||
date = now |
|||
else: |
|||
# Otherwise, future dates always use the full format. |
|||
full_format = True |
|||
local_date = date - datetime.timedelta(minutes=gmt_offset) |
|||
local_now = now - datetime.timedelta(minutes=gmt_offset) |
|||
local_yesterday = local_now - datetime.timedelta(hours=24) |
|||
difference = now - date |
|||
seconds = difference.seconds |
|||
days = difference.days |
|||
|
|||
_ = self.translate |
|||
format = None |
|||
if not full_format: |
|||
if relative and days == 0: |
|||
if seconds < 50: |
|||
return _("1 second ago", "%(seconds)d seconds ago", |
|||
seconds) % { "seconds": seconds } |
|||
|
|||
if seconds < 50 * 60: |
|||
minutes = round(seconds / 60.0) |
|||
return _("1 minute ago", "%(minutes)d minutes ago", |
|||
minutes) % { "minutes": minutes } |
|||
|
|||
hours = round(seconds / (60.0 * 60)) |
|||
return _("1 hour ago", "%(hours)d hours ago", |
|||
hours) % { "hours": hours } |
|||
|
|||
if days == 0: |
|||
format = _("%(time)s") |
|||
elif days == 1 and local_date.day == local_yesterday.day and \ |
|||
relative: |
|||
format = _("yesterday") if shorter else \ |
|||
_("yesterday at %(time)s") |
|||
elif days < 5: |
|||
format = _("%(weekday)s") if shorter else \ |
|||
_("%(weekday)s at %(time)s") |
|||
elif days < 334: # 11mo, since confusing for same month last year |
|||
format = _("%(month_name)s %(day)s") if shorter else \ |
|||
_("%(month_name)s %(day)s at %(time)s") |
|||
|
|||
if format is None: |
|||
format = _("%(month_name)s %(day)s, %(year)s") if shorter else \ |
|||
_("%(month_name)s %(day)s, %(year)s at %(time)s") |
|||
|
|||
tfhour_clock = self.code not in ("en", "en_US", "zh_CN") |
|||
if tfhour_clock: |
|||
str_time = "%d:%02d" % (local_date.hour, local_date.minute) |
|||
elif self.code == "zh_CN": |
|||
str_time = "%s%d:%02d" % ( |
|||
(u'\u4e0a\u5348', u'\u4e0b\u5348')[local_date.hour >= 12], |
|||
local_date.hour % 12 or 12, local_date.minute) |
|||
else: |
|||
str_time = "%d:%02d %s" % ( |
|||
local_date.hour % 12 or 12, local_date.minute, |
|||
("am", "pm")[local_date.hour >= 12]) |
|||
|
|||
return format % { |
|||
"month_name": self._months[local_date.month - 1], |
|||
"weekday": self._weekdays[local_date.weekday()], |
|||
"day": str(local_date.day), |
|||
"year": str(local_date.year), |
|||
"time": str_time |
|||
} |
|||
|
|||
def format_day(self, date, gmt_offset=0, dow=True): |
|||
"""Formats the given date as a day of week. |
|||
|
|||
Example: "Monday, January 22". You can remove the day of week with |
|||
dow=False. |
|||
""" |
|||
local_date = date - datetime.timedelta(minutes=gmt_offset) |
|||
_ = self.translate |
|||
if dow: |
|||
return _("%(weekday)s, %(month_name)s %(day)s") % { |
|||
"month_name": self._months[local_date.month - 1], |
|||
"weekday": self._weekdays[local_date.weekday()], |
|||
"day": str(local_date.day), |
|||
} |
|||
else: |
|||
return _("%(month_name)s %(day)s") % { |
|||
"month_name": self._months[local_date.month - 1], |
|||
"day": str(local_date.day), |
|||
} |
|||
|
|||
def list(self, parts): |
|||
"""Returns a comma-separated list for the given list of parts. |
|||
|
|||
The format is, e.g., "A, B and C", "A and B" or just "A" for lists |
|||
of size 1. |
|||
""" |
|||
_ = self.translate |
|||
if len(parts) == 0: return "" |
|||
if len(parts) == 1: return parts[0] |
|||
comma = u' \u0648 ' if self.code.startswith("fa") else u", " |
|||
return _("%(commas)s and %(last)s") % { |
|||
"commas": comma.join(parts[:-1]), |
|||
"last": parts[len(parts) - 1], |
|||
} |
|||
|
|||
def friendly_number(self, value): |
|||
"""Returns a comma-separated number for the given integer.""" |
|||
if self.code not in ("en", "en_US"): |
|||
return str(value) |
|||
value = str(value) |
|||
parts = [] |
|||
while value: |
|||
parts.append(value[-3:]) |
|||
value = value[:-3] |
|||
return ",".join(reversed(parts)) |
|||
|
|||
class CSVLocale(Locale): |
|||
"""Locale implementation using tornado's CSV translation format.""" |
|||
def translate(self, message, plural_message=None, count=None): |
|||
if plural_message is not None: |
|||
assert count is not None |
|||
if count != 1: |
|||
message = plural_message |
|||
message_dict = self.translations.get("plural", {}) |
|||
else: |
|||
message_dict = self.translations.get("singular", {}) |
|||
else: |
|||
message_dict = self.translations.get("unknown", {}) |
|||
return message_dict.get(message, message) |
|||
|
|||
class GettextLocale(Locale): |
|||
"""Locale implementation using the gettext module.""" |
|||
def translate(self, message, plural_message=None, count=None): |
|||
if plural_message is not None: |
|||
assert count is not None |
|||
return self.translations.ungettext(message, plural_message, count) |
|||
else: |
|||
return self.translations.ugettext(message) |
|||
|
|||
LOCALE_NAMES = { |
|||
"af_ZA": {"name_en": u"Afrikaans", "name": u"Afrikaans"}, |
|||
"am_ET": {"name_en": u"Amharic", "name": u'\u12a0\u121b\u122d\u129b'}, |
|||
"ar_AR": {"name_en": u"Arabic", "name": u"\u0627\u0644\u0639\u0631\u0628\u064a\u0629"}, |
|||
"bg_BG": {"name_en": u"Bulgarian", "name": u"\u0411\u044a\u043b\u0433\u0430\u0440\u0441\u043a\u0438"}, |
|||
"bn_IN": {"name_en": u"Bengali", "name": u"\u09ac\u09be\u0982\u09b2\u09be"}, |
|||
"bs_BA": {"name_en": u"Bosnian", "name": u"Bosanski"}, |
|||
"ca_ES": {"name_en": u"Catalan", "name": u"Catal\xe0"}, |
|||
"cs_CZ": {"name_en": u"Czech", "name": u"\u010ce\u0161tina"}, |
|||
"cy_GB": {"name_en": u"Welsh", "name": u"Cymraeg"}, |
|||
"da_DK": {"name_en": u"Danish", "name": u"Dansk"}, |
|||
"de_DE": {"name_en": u"German", "name": u"Deutsch"}, |
|||
"el_GR": {"name_en": u"Greek", "name": u"\u0395\u03bb\u03bb\u03b7\u03bd\u03b9\u03ba\u03ac"}, |
|||
"en_GB": {"name_en": u"English (UK)", "name": u"English (UK)"}, |
|||
"en_US": {"name_en": u"English (US)", "name": u"English (US)"}, |
|||
"es_ES": {"name_en": u"Spanish (Spain)", "name": u"Espa\xf1ol (Espa\xf1a)"}, |
|||
"es_LA": {"name_en": u"Spanish", "name": u"Espa\xf1ol"}, |
|||
"et_EE": {"name_en": u"Estonian", "name": u"Eesti"}, |
|||
"eu_ES": {"name_en": u"Basque", "name": u"Euskara"}, |
|||
"fa_IR": {"name_en": u"Persian", "name": u"\u0641\u0627\u0631\u0633\u06cc"}, |
|||
"fi_FI": {"name_en": u"Finnish", "name": u"Suomi"}, |
|||
"fr_CA": {"name_en": u"French (Canada)", "name": u"Fran\xe7ais (Canada)"}, |
|||
"fr_FR": {"name_en": u"French", "name": u"Fran\xe7ais"}, |
|||
"ga_IE": {"name_en": u"Irish", "name": u"Gaeilge"}, |
|||
"gl_ES": {"name_en": u"Galician", "name": u"Galego"}, |
|||
"he_IL": {"name_en": u"Hebrew", "name": u"\u05e2\u05d1\u05e8\u05d9\u05ea"}, |
|||
"hi_IN": {"name_en": u"Hindi", "name": u"\u0939\u093f\u0928\u094d\u0926\u0940"}, |
|||
"hr_HR": {"name_en": u"Croatian", "name": u"Hrvatski"}, |
|||
"hu_HU": {"name_en": u"Hungarian", "name": u"Magyar"}, |
|||
"id_ID": {"name_en": u"Indonesian", "name": u"Bahasa Indonesia"}, |
|||
"is_IS": {"name_en": u"Icelandic", "name": u"\xcdslenska"}, |
|||
"it_IT": {"name_en": u"Italian", "name": u"Italiano"}, |
|||
"ja_JP": {"name_en": u"Japanese", "name": u"\u65e5\u672c\u8a9e"}, |
|||
"ko_KR": {"name_en": u"Korean", "name": u"\ud55c\uad6d\uc5b4"}, |
|||
"lt_LT": {"name_en": u"Lithuanian", "name": u"Lietuvi\u0173"}, |
|||
"lv_LV": {"name_en": u"Latvian", "name": u"Latvie\u0161u"}, |
|||
"mk_MK": {"name_en": u"Macedonian", "name": u"\u041c\u0430\u043a\u0435\u0434\u043e\u043d\u0441\u043a\u0438"}, |
|||
"ml_IN": {"name_en": u"Malayalam", "name": u"\u0d2e\u0d32\u0d2f\u0d3e\u0d33\u0d02"}, |
|||
"ms_MY": {"name_en": u"Malay", "name": u"Bahasa Melayu"}, |
|||
"nb_NO": {"name_en": u"Norwegian (bokmal)", "name": u"Norsk (bokm\xe5l)"}, |
|||
"nl_NL": {"name_en": u"Dutch", "name": u"Nederlands"}, |
|||
"nn_NO": {"name_en": u"Norwegian (nynorsk)", "name": u"Norsk (nynorsk)"}, |
|||
"pa_IN": {"name_en": u"Punjabi", "name": u"\u0a2a\u0a70\u0a1c\u0a3e\u0a2c\u0a40"}, |
|||
"pl_PL": {"name_en": u"Polish", "name": u"Polski"}, |
|||
"pt_BR": {"name_en": u"Portuguese (Brazil)", "name": u"Portugu\xeas (Brasil)"}, |
|||
"pt_PT": {"name_en": u"Portuguese (Portugal)", "name": u"Portugu\xeas (Portugal)"}, |
|||
"ro_RO": {"name_en": u"Romanian", "name": u"Rom\xe2n\u0103"}, |
|||
"ru_RU": {"name_en": u"Russian", "name": u"\u0420\u0443\u0441\u0441\u043a\u0438\u0439"}, |
|||
"sk_SK": {"name_en": u"Slovak", "name": u"Sloven\u010dina"}, |
|||
"sl_SI": {"name_en": u"Slovenian", "name": u"Sloven\u0161\u010dina"}, |
|||
"sq_AL": {"name_en": u"Albanian", "name": u"Shqip"}, |
|||
"sr_RS": {"name_en": u"Serbian", "name": u"\u0421\u0440\u043f\u0441\u043a\u0438"}, |
|||
"sv_SE": {"name_en": u"Swedish", "name": u"Svenska"}, |
|||
"sw_KE": {"name_en": u"Swahili", "name": u"Kiswahili"}, |
|||
"ta_IN": {"name_en": u"Tamil", "name": u"\u0ba4\u0bae\u0bbf\u0bb4\u0bcd"}, |
|||
"te_IN": {"name_en": u"Telugu", "name": u"\u0c24\u0c46\u0c32\u0c41\u0c17\u0c41"}, |
|||
"th_TH": {"name_en": u"Thai", "name": u"\u0e20\u0e32\u0e29\u0e32\u0e44\u0e17\u0e22"}, |
|||
"tl_PH": {"name_en": u"Filipino", "name": u"Filipino"}, |
|||
"tr_TR": {"name_en": u"Turkish", "name": u"T\xfcrk\xe7e"}, |
|||
"uk_UA": {"name_en": u"Ukraini ", "name": u"\u0423\u043a\u0440\u0430\u0457\u043d\u0441\u044c\u043a\u0430"}, |
|||
"vi_VN": {"name_en": u"Vietnamese", "name": u"Ti\u1ebfng Vi\u1ec7t"}, |
|||
"zh_CN": {"name_en": u"Chinese (Simplified)", "name": u"\u4e2d\u6587(\u7b80\u4f53)"}, |
|||
"zh_TW": {"name_en": u"Chinese (Traditional)", "name": u"\u4e2d\u6587(\u7e41\u9ad4)"}, |
|||
} |
|||
@ -0,0 +1,314 @@ |
|||
#!/usr/bin/env python |
|||
# |
|||
# Copyright 2011 Facebook |
|||
# |
|||
# Licensed under the Apache License, Version 2.0 (the "License"); you may |
|||
# not use this file except in compliance with the License. You may obtain |
|||
# a copy of the License at |
|||
# |
|||
# http://www.apache.org/licenses/LICENSE-2.0 |
|||
# |
|||
# Unless required by applicable law or agreed to in writing, software |
|||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT |
|||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the |
|||
# License for the specific language governing permissions and limitations |
|||
# under the License. |
|||
|
|||
"""Miscellaneous network utility code.""" |
|||
|
|||
import errno |
|||
import logging |
|||
import os |
|||
import socket |
|||
import stat |
|||
|
|||
from tornado import process |
|||
from tornado.ioloop import IOLoop |
|||
from tornado.iostream import IOStream, SSLIOStream |
|||
from tornado.platform.auto import set_close_exec |
|||
|
|||
try: |
|||
import ssl # Python 2.6+ |
|||
except ImportError: |
|||
ssl = None |
|||
|
|||
class TCPServer(object): |
|||
r"""A non-blocking, single-threaded TCP server. |
|||
|
|||
To use `TCPServer`, define a subclass which overrides the `handle_stream` |
|||
method. |
|||
|
|||
`TCPServer` can serve SSL traffic with Python 2.6+ and OpenSSL. |
|||
To make this server serve SSL traffic, send the ssl_options dictionary |
|||
argument with the arguments required for the `ssl.wrap_socket` method, |
|||
including "certfile" and "keyfile":: |
|||
|
|||
TCPServer(ssl_options={ |
|||
"certfile": os.path.join(data_dir, "mydomain.crt"), |
|||
"keyfile": os.path.join(data_dir, "mydomain.key"), |
|||
}) |
|||
|
|||
`TCPServer` initialization follows one of three patterns: |
|||
|
|||
1. `listen`: simple single-process:: |
|||
|
|||
server = TCPServer() |
|||
server.listen(8888) |
|||
IOLoop.instance().start() |
|||
|
|||
2. `bind`/`start`: simple multi-process:: |
|||
|
|||
server = TCPServer() |
|||
server.bind(8888) |
|||
server.start(0) # Forks multiple sub-processes |
|||
IOLoop.instance().start() |
|||
|
|||
When using this interface, an `IOLoop` must *not* be passed |
|||
to the `TCPServer` constructor. `start` will always start |
|||
the server on the default singleton `IOLoop`. |
|||
|
|||
3. `add_sockets`: advanced multi-process:: |
|||
|
|||
sockets = bind_sockets(8888) |
|||
tornado.process.fork_processes(0) |
|||
server = TCPServer() |
|||
server.add_sockets(sockets) |
|||
IOLoop.instance().start() |
|||
|
|||
The `add_sockets` interface is more complicated, but it can be |
|||
used with `tornado.process.fork_processes` to give you more |
|||
flexibility in when the fork happens. `add_sockets` can |
|||
also be used in single-process servers if you want to create |
|||
your listening sockets in some way other than |
|||
`bind_sockets`. |
|||
""" |
|||
def __init__(self, io_loop=None, ssl_options=None): |
|||
self.io_loop = io_loop |
|||
self.ssl_options = ssl_options |
|||
self._sockets = {} # fd -> socket object |
|||
self._pending_sockets = [] |
|||
self._started = False |
|||
|
|||
def listen(self, port, address=""): |
|||
"""Starts accepting connections on the given port. |
|||
|
|||
This method may be called more than once to listen on multiple ports. |
|||
`listen` takes effect immediately; it is not necessary to call |
|||
`TCPServer.start` afterwards. It is, however, necessary to start |
|||
the `IOLoop`. |
|||
""" |
|||
sockets = bind_sockets(port, address=address) |
|||
self.add_sockets(sockets) |
|||
|
|||
def add_sockets(self, sockets): |
|||
"""Makes this server start accepting connections on the given sockets. |
|||
|
|||
The ``sockets`` parameter is a list of socket objects such as |
|||
those returned by `bind_sockets`. |
|||
`add_sockets` is typically used in combination with that |
|||
method and `tornado.process.fork_processes` to provide greater |
|||
control over the initialization of a multi-process server. |
|||
""" |
|||
if self.io_loop is None: |
|||
self.io_loop = IOLoop.instance() |
|||
|
|||
for sock in sockets: |
|||
self._sockets[sock.fileno()] = sock |
|||
add_accept_handler(sock, self._handle_connection, |
|||
io_loop=self.io_loop) |
|||
|
|||
def add_socket(self, socket): |
|||
"""Singular version of `add_sockets`. Takes a single socket object.""" |
|||
self.add_sockets([socket]) |
|||
|
|||
def bind(self, port, address=None, family=socket.AF_UNSPEC, backlog=128): |
|||
"""Binds this server to the given port on the given address. |
|||
|
|||
To start the server, call `start`. If you want to run this server |
|||
in a single process, you can call `listen` as a shortcut to the |
|||
sequence of `bind` and `start` calls. |
|||
|
|||
Address may be either an IP address or hostname. If it's a hostname, |
|||
the server will listen on all IP addresses associated with the |
|||
name. Address may be an empty string or None to listen on all |
|||
available interfaces. Family may be set to either ``socket.AF_INET`` |
|||
or ``socket.AF_INET6`` to restrict to ipv4 or ipv6 addresses, otherwise |
|||
both will be used if available. |
|||
|
|||
The ``backlog`` argument has the same meaning as for |
|||
`socket.listen`. |
|||
|
|||
This method may be called multiple times prior to `start` to listen |
|||
on multiple ports or interfaces. |
|||
""" |
|||
sockets = bind_sockets(port, address=address, family=family, |
|||
backlog=backlog) |
|||
if self._started: |
|||
self.add_sockets(sockets) |
|||
else: |
|||
self._pending_sockets.extend(sockets) |
|||
|
|||
def start(self, num_processes=1): |
|||
"""Starts this server in the IOLoop. |
|||
|
|||
By default, we run the server in this process and do not fork any |
|||
additional child process. |
|||
|
|||
If num_processes is ``None`` or <= 0, we detect the number of cores |
|||
available on this machine and fork that number of child |
|||
processes. If num_processes is given and > 1, we fork that |
|||
specific number of sub-processes. |
|||
|
|||
Since we use processes and not threads, there is no shared memory |
|||
between any server code. |
|||
|
|||
Note that multiple processes are not compatible with the autoreload |
|||
module (or the ``debug=True`` option to `tornado.web.Application`). |
|||
When using multiple processes, no IOLoops can be created or |
|||
referenced until after the call to ``TCPServer.start(n)``. |
|||
""" |
|||
assert not self._started |
|||
self._started = True |
|||
if num_processes != 1: |
|||
process.fork_processes(num_processes) |
|||
sockets = self._pending_sockets |
|||
self._pending_sockets = [] |
|||
self.add_sockets(sockets) |
|||
|
|||
def stop(self): |
|||
"""Stops listening for new connections. |
|||
|
|||
Requests currently in progress may still continue after the |
|||
server is stopped. |
|||
""" |
|||
for fd, sock in self._sockets.iteritems(): |
|||
self.io_loop.remove_handler(fd) |
|||
sock.close() |
|||
|
|||
def handle_stream(self, stream, address): |
|||
"""Override to handle a new `IOStream` from an incoming connection.""" |
|||
raise NotImplementedError() |
|||
|
|||
def _handle_connection(self, connection, address): |
|||
if self.ssl_options is not None: |
|||
assert ssl, "Python 2.6+ and OpenSSL required for SSL" |
|||
try: |
|||
connection = ssl.wrap_socket(connection, |
|||
server_side=True, |
|||
do_handshake_on_connect=False, |
|||
**self.ssl_options) |
|||
except ssl.SSLError, err: |
|||
if err.args[0] == ssl.SSL_ERROR_EOF: |
|||
return connection.close() |
|||
else: |
|||
raise |
|||
except socket.error, err: |
|||
if err.args[0] == errno.ECONNABORTED: |
|||
return connection.close() |
|||
else: |
|||
raise |
|||
try: |
|||
if self.ssl_options is not None: |
|||
stream = SSLIOStream(connection, io_loop=self.io_loop) |
|||
else: |
|||
stream = IOStream(connection, io_loop=self.io_loop) |
|||
self.handle_stream(stream, address) |
|||
except Exception: |
|||
logging.error("Error in connection callback", exc_info=True) |
|||
|
|||
|
|||
def bind_sockets(port, address=None, family=socket.AF_UNSPEC, backlog=128): |
|||
"""Creates listening sockets bound to the given port and address. |
|||
|
|||
Returns a list of socket objects (multiple sockets are returned if |
|||
the given address maps to multiple IP addresses, which is most common |
|||
for mixed IPv4 and IPv6 use). |
|||
|
|||
Address may be either an IP address or hostname. If it's a hostname, |
|||
the server will listen on all IP addresses associated with the |
|||
name. Address may be an empty string or None to listen on all |
|||
available interfaces. Family may be set to either socket.AF_INET |
|||
or socket.AF_INET6 to restrict to ipv4 or ipv6 addresses, otherwise |
|||
both will be used if available. |
|||
|
|||
The ``backlog`` argument has the same meaning as for |
|||
``socket.listen()``. |
|||
""" |
|||
sockets = [] |
|||
if address == "": |
|||
address = None |
|||
flags = socket.AI_PASSIVE |
|||
for res in set(socket.getaddrinfo(address, port, family, socket.SOCK_STREAM, |
|||
0, flags)): |
|||
af, socktype, proto, canonname, sockaddr = res |
|||
sock = socket.socket(af, socktype, proto) |
|||
set_close_exec(sock.fileno()) |
|||
sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) |
|||
if af == socket.AF_INET6: |
|||
# On linux, ipv6 sockets accept ipv4 too by default, |
|||
# but this makes it impossible to bind to both |
|||
# 0.0.0.0 in ipv4 and :: in ipv6. On other systems, |
|||
# separate sockets *must* be used to listen for both ipv4 |
|||
# and ipv6. For consistency, always disable ipv4 on our |
|||
# ipv6 sockets and use a separate ipv4 socket when needed. |
|||
# |
|||
# Python 2.x on windows doesn't have IPPROTO_IPV6. |
|||
if hasattr(socket, "IPPROTO_IPV6"): |
|||
sock.setsockopt(socket.IPPROTO_IPV6, socket.IPV6_V6ONLY, 1) |
|||
sock.setblocking(0) |
|||
sock.bind(sockaddr) |
|||
sock.listen(backlog) |
|||
sockets.append(sock) |
|||
return sockets |
|||
|
|||
if hasattr(socket, 'AF_UNIX'): |
|||
def bind_unix_socket(file, mode=0600, backlog=128): |
|||
"""Creates a listening unix socket. |
|||
|
|||
If a socket with the given name already exists, it will be deleted. |
|||
If any other file with that name exists, an exception will be |
|||
raised. |
|||
|
|||
Returns a socket object (not a list of socket objects like |
|||
`bind_sockets`) |
|||
""" |
|||
sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) |
|||
set_close_exec(sock.fileno()) |
|||
sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) |
|||
sock.setblocking(0) |
|||
try: |
|||
st = os.stat(file) |
|||
except OSError, err: |
|||
if err.errno != errno.ENOENT: |
|||
raise |
|||
else: |
|||
if stat.S_ISSOCK(st.st_mode): |
|||
os.remove(file) |
|||
else: |
|||
raise ValueError("File %s exists and is not a socket", file) |
|||
sock.bind(file) |
|||
os.chmod(file, mode) |
|||
sock.listen(backlog) |
|||
return sock |
|||
|
|||
def add_accept_handler(sock, callback, io_loop=None): |
|||
"""Adds an ``IOLoop`` event handler to accept new connections on ``sock``. |
|||
|
|||
When a connection is accepted, ``callback(connection, address)`` will |
|||
be run (``connection`` is a socket object, and ``address`` is the |
|||
address of the other end of the connection). Note that this signature |
|||
is different from the ``callback(fd, events)`` signature used for |
|||
``IOLoop`` handlers. |
|||
""" |
|||
if io_loop is None: |
|||
io_loop = IOLoop.instance() |
|||
def accept_handler(fd, events): |
|||
while True: |
|||
try: |
|||
connection, address = sock.accept() |
|||
except socket.error, e: |
|||
if e.args[0] in (errno.EWOULDBLOCK, errno.EAGAIN): |
|||
return |
|||
raise |
|||
callback(connection, address) |
|||
io_loop.add_handler(sock.fileno(), accept_handler, IOLoop.READ) |
|||
@ -0,0 +1,422 @@ |
|||
#!/usr/bin/env python |
|||
# |
|||
# Copyright 2009 Facebook |
|||
# |
|||
# Licensed under the Apache License, Version 2.0 (the "License"); you may |
|||
# not use this file except in compliance with the License. You may obtain |
|||
# a copy of the License at |
|||
# |
|||
# http://www.apache.org/licenses/LICENSE-2.0 |
|||
# |
|||
# Unless required by applicable law or agreed to in writing, software |
|||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT |
|||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the |
|||
# License for the specific language governing permissions and limitations |
|||
# under the License. |
|||
|
|||
"""A command line parsing module that lets modules define their own options. |
|||
|
|||
Each module defines its own options, e.g.:: |
|||
|
|||
from tornado.options import define, options |
|||
|
|||
define("mysql_host", default="127.0.0.1:3306", help="Main user DB") |
|||
define("memcache_hosts", default="127.0.0.1:11011", multiple=True, |
|||
help="Main user memcache servers") |
|||
|
|||
def connect(): |
|||
db = database.Connection(options.mysql_host) |
|||
... |
|||
|
|||
The main() method of your application does not need to be aware of all of |
|||
the options used throughout your program; they are all automatically loaded |
|||
when the modules are loaded. Your main() method can parse the command line |
|||
or parse a config file with:: |
|||
|
|||
import tornado.options |
|||
tornado.options.parse_config_file("/etc/server.conf") |
|||
tornado.options.parse_command_line() |
|||
|
|||
Command line formats are what you would expect ("--myoption=myvalue"). |
|||
Config files are just Python files. Global names become options, e.g.:: |
|||
|
|||
myoption = "myvalue" |
|||
myotheroption = "myothervalue" |
|||
|
|||
We support datetimes, timedeltas, ints, and floats (just pass a 'type' |
|||
kwarg to define). We also accept multi-value options. See the documentation |
|||
for define() below. |
|||
""" |
|||
|
|||
import datetime |
|||
import logging |
|||
import logging.handlers |
|||
import re |
|||
import sys |
|||
import time |
|||
|
|||
from tornado.escape import _unicode |
|||
|
|||
# For pretty log messages, if available |
|||
try: |
|||
import curses |
|||
except ImportError: |
|||
curses = None |
|||
|
|||
|
|||
def define(name, default=None, type=None, help=None, metavar=None, |
|||
multiple=False, group=None): |
|||
"""Defines a new command line option. |
|||
|
|||
If type is given (one of str, float, int, datetime, or timedelta) |
|||
or can be inferred from the default, we parse the command line |
|||
arguments based on the given type. If multiple is True, we accept |
|||
comma-separated values, and the option value is always a list. |
|||
|
|||
For multi-value integers, we also accept the syntax x:y, which |
|||
turns into range(x, y) - very useful for long integer ranges. |
|||
|
|||
help and metavar are used to construct the automatically generated |
|||
command line help string. The help message is formatted like:: |
|||
|
|||
--name=METAVAR help string |
|||
|
|||
group is used to group the defined options in logical groups. By default, |
|||
command line options are grouped by the defined file. |
|||
|
|||
Command line option names must be unique globally. They can be parsed |
|||
from the command line with parse_command_line() or parsed from a |
|||
config file with parse_config_file. |
|||
""" |
|||
if name in options: |
|||
raise Error("Option %r already defined in %s", name, |
|||
options[name].file_name) |
|||
frame = sys._getframe(0) |
|||
options_file = frame.f_code.co_filename |
|||
file_name = frame.f_back.f_code.co_filename |
|||
if file_name == options_file: file_name = "" |
|||
if type is None: |
|||
if not multiple and default is not None: |
|||
type = default.__class__ |
|||
else: |
|||
type = str |
|||
if group: |
|||
group_name = group |
|||
else: |
|||
group_name = file_name |
|||
options[name] = _Option(name, file_name=file_name, default=default, |
|||
type=type, help=help, metavar=metavar, |
|||
multiple=multiple, group_name=group_name) |
|||
|
|||
|
|||
def parse_command_line(args=None): |
|||
"""Parses all options given on the command line. |
|||
|
|||
We return all command line arguments that are not options as a list. |
|||
""" |
|||
if args is None: args = sys.argv |
|||
remaining = [] |
|||
for i in xrange(1, len(args)): |
|||
# All things after the last option are command line arguments |
|||
if not args[i].startswith("-"): |
|||
remaining = args[i:] |
|||
break |
|||
if args[i] == "--": |
|||
remaining = args[i+1:] |
|||
break |
|||
arg = args[i].lstrip("-") |
|||
name, equals, value = arg.partition("=") |
|||
name = name.replace('-', '_') |
|||
if not name in options: |
|||
print_help() |
|||
raise Error('Unrecognized command line option: %r' % name) |
|||
option = options[name] |
|||
if not equals: |
|||
if option.type == bool: |
|||
value = "true" |
|||
else: |
|||
raise Error('Option %r requires a value' % name) |
|||
option.parse(value) |
|||
if options.help: |
|||
print_help() |
|||
sys.exit(0) |
|||
|
|||
# Set up log level and pretty console logging by default |
|||
if options.logging != 'none': |
|||
logging.getLogger().setLevel(getattr(logging, options.logging.upper())) |
|||
enable_pretty_logging() |
|||
|
|||
return remaining |
|||
|
|||
|
|||
def parse_config_file(path): |
|||
"""Parses and loads the Python config file at the given path.""" |
|||
config = {} |
|||
execfile(path, config, config) |
|||
for name in config: |
|||
if name in options: |
|||
options[name].set(config[name]) |
|||
|
|||
|
|||
def print_help(file=sys.stdout): |
|||
"""Prints all the command line options to stdout.""" |
|||
print >> file, "Usage: %s [OPTIONS]" % sys.argv[0] |
|||
print >> file, "" |
|||
print >> file, "Options:" |
|||
by_group = {} |
|||
for option in options.itervalues(): |
|||
by_group.setdefault(option.group_name, []).append(option) |
|||
|
|||
for filename, o in sorted(by_group.items()): |
|||
if filename: print >> file, filename |
|||
o.sort(key=lambda option: option.name) |
|||
for option in o: |
|||
prefix = option.name |
|||
if option.metavar: |
|||
prefix += "=" + option.metavar |
|||
print >> file, " --%-30s %s" % (prefix, option.help or "") |
|||
print >> file |
|||
|
|||
|
|||
class _Options(dict): |
|||
"""Our global program options, an dictionary with object-like access.""" |
|||
@classmethod |
|||
def instance(cls): |
|||
if not hasattr(cls, "_instance"): |
|||
cls._instance = cls() |
|||
return cls._instance |
|||
|
|||
def __getattr__(self, name): |
|||
if isinstance(self.get(name), _Option): |
|||
return self[name].value() |
|||
raise AttributeError("Unrecognized option %r" % name) |
|||
|
|||
|
|||
class _Option(object): |
|||
def __init__(self, name, default=None, type=str, help=None, metavar=None, |
|||
multiple=False, file_name=None, group_name=None): |
|||
if default is None and multiple: |
|||
default = [] |
|||
self.name = name |
|||
self.type = type |
|||
self.help = help |
|||
self.metavar = metavar |
|||
self.multiple = multiple |
|||
self.file_name = file_name |
|||
self.group_name = group_name |
|||
self.default = default |
|||
self._value = None |
|||
|
|||
def value(self): |
|||
return self.default if self._value is None else self._value |
|||
|
|||
def parse(self, value): |
|||
_parse = { |
|||
datetime.datetime: self._parse_datetime, |
|||
datetime.timedelta: self._parse_timedelta, |
|||
bool: self._parse_bool, |
|||
str: self._parse_string, |
|||
}.get(self.type, self.type) |
|||
if self.multiple: |
|||
if self._value is None: |
|||
self._value = [] |
|||
for part in value.split(","): |
|||
if self.type in (int, long): |
|||
# allow ranges of the form X:Y (inclusive at both ends) |
|||
lo, _, hi = part.partition(":") |
|||
lo = _parse(lo) |
|||
hi = _parse(hi) if hi else lo |
|||
self._value.extend(range(lo, hi+1)) |
|||
else: |
|||
self._value.append(_parse(part)) |
|||
else: |
|||
self._value = _parse(value) |
|||
return self.value() |
|||
|
|||
def set(self, value): |
|||
if self.multiple: |
|||
if not isinstance(value, list): |
|||
raise Error("Option %r is required to be a list of %s" % |
|||
(self.name, self.type.__name__)) |
|||
for item in value: |
|||
if item != None and not isinstance(item, self.type): |
|||
raise Error("Option %r is required to be a list of %s" % |
|||
(self.name, self.type.__name__)) |
|||
else: |
|||
if value != None and not isinstance(value, self.type): |
|||
raise Error("Option %r is required to be a %s" % |
|||
(self.name, self.type.__name__)) |
|||
self._value = value |
|||
|
|||
# Supported date/time formats in our options |
|||
_DATETIME_FORMATS = [ |
|||
"%a %b %d %H:%M:%S %Y", |
|||
"%Y-%m-%d %H:%M:%S", |
|||
"%Y-%m-%d %H:%M", |
|||
"%Y-%m-%dT%H:%M", |
|||
"%Y%m%d %H:%M:%S", |
|||
"%Y%m%d %H:%M", |
|||
"%Y-%m-%d", |
|||
"%Y%m%d", |
|||
"%H:%M:%S", |
|||
"%H:%M", |
|||
] |
|||
|
|||
def _parse_datetime(self, value): |
|||
for format in self._DATETIME_FORMATS: |
|||
try: |
|||
return datetime.datetime.strptime(value, format) |
|||
except ValueError: |
|||
pass |
|||
raise Error('Unrecognized date/time format: %r' % value) |
|||
|
|||
_TIMEDELTA_ABBREVS = [ |
|||
('hours', ['h']), |
|||
('minutes', ['m', 'min']), |
|||
('seconds', ['s', 'sec']), |
|||
('milliseconds', ['ms']), |
|||
('microseconds', ['us']), |
|||
('days', ['d']), |
|||
('weeks', ['w']), |
|||
] |
|||
|
|||
_TIMEDELTA_ABBREV_DICT = dict( |
|||
(abbrev, full) for full, abbrevs in _TIMEDELTA_ABBREVS |
|||
for abbrev in abbrevs) |
|||
|
|||
_FLOAT_PATTERN = r'[-+]?(?:\d+(?:\.\d*)?|\.\d+)(?:[eE][-+]?\d+)?' |
|||
|
|||
_TIMEDELTA_PATTERN = re.compile( |
|||
r'\s*(%s)\s*(\w*)\s*' % _FLOAT_PATTERN, re.IGNORECASE) |
|||
|
|||
def _parse_timedelta(self, value): |
|||
try: |
|||
sum = datetime.timedelta() |
|||
start = 0 |
|||
while start < len(value): |
|||
m = self._TIMEDELTA_PATTERN.match(value, start) |
|||
if not m: |
|||
raise Exception() |
|||
num = float(m.group(1)) |
|||
units = m.group(2) or 'seconds' |
|||
units = self._TIMEDELTA_ABBREV_DICT.get(units, units) |
|||
sum += datetime.timedelta(**{units: num}) |
|||
start = m.end() |
|||
return sum |
|||
except Exception: |
|||
raise |
|||
|
|||
def _parse_bool(self, value): |
|||
return value.lower() not in ("false", "0", "f") |
|||
|
|||
def _parse_string(self, value): |
|||
return _unicode(value) |
|||
|
|||
|
|||
class Error(Exception): |
|||
"""Exception raised by errors in the options module.""" |
|||
pass |
|||
|
|||
|
|||
def enable_pretty_logging(): |
|||
"""Turns on formatted logging output as configured. |
|||
|
|||
This is called automatically by `parse_command_line`. |
|||
""" |
|||
root_logger = logging.getLogger() |
|||
if options.log_file_prefix: |
|||
channel = logging.handlers.RotatingFileHandler( |
|||
filename=options.log_file_prefix, |
|||
maxBytes=options.log_file_max_size, |
|||
backupCount=options.log_file_num_backups) |
|||
channel.setFormatter(_LogFormatter(color=False)) |
|||
root_logger.addHandler(channel) |
|||
|
|||
if (options.log_to_stderr or |
|||
(options.log_to_stderr is None and not root_logger.handlers)): |
|||
# Set up color if we are in a tty and curses is installed |
|||
color = False |
|||
if curses and sys.stderr.isatty(): |
|||
try: |
|||
curses.setupterm() |
|||
if curses.tigetnum("colors") > 0: |
|||
color = True |
|||
except Exception: |
|||
pass |
|||
channel = logging.StreamHandler() |
|||
channel.setFormatter(_LogFormatter(color=color)) |
|||
root_logger.addHandler(channel) |
|||
|
|||
|
|||
|
|||
class _LogFormatter(logging.Formatter): |
|||
def __init__(self, color, *args, **kwargs): |
|||
logging.Formatter.__init__(self, *args, **kwargs) |
|||
self._color = color |
|||
if color: |
|||
# The curses module has some str/bytes confusion in |
|||
# python3. Until version 3.2.3, most methods return |
|||
# bytes, but only accept strings. In addition, we want to |
|||
# output these strings with the logging module, which |
|||
# works with unicode strings. The explicit calls to |
|||
# unicode() below are harmless in python2 but will do the |
|||
# right conversion in python 3. |
|||
fg_color = (curses.tigetstr("setaf") or |
|||
curses.tigetstr("setf") or "") |
|||
if (3, 0) < sys.version_info < (3, 2, 3): |
|||
fg_color = unicode(fg_color, "ascii") |
|||
self._colors = { |
|||
logging.DEBUG: unicode(curses.tparm(fg_color, 4), # Blue |
|||
"ascii"), |
|||
logging.INFO: unicode(curses.tparm(fg_color, 2), # Green |
|||
"ascii"), |
|||
logging.WARNING: unicode(curses.tparm(fg_color, 3), # Yellow |
|||
"ascii"), |
|||
logging.ERROR: unicode(curses.tparm(fg_color, 1), # Red |
|||
"ascii"), |
|||
} |
|||
self._normal = unicode(curses.tigetstr("sgr0"), "ascii") |
|||
|
|||
def format(self, record): |
|||
try: |
|||
record.message = record.getMessage() |
|||
except Exception, e: |
|||
record.message = "Bad message (%r): %r" % (e, record.__dict__) |
|||
record.asctime = time.strftime( |
|||
"%y%m%d %H:%M:%S", self.converter(record.created)) |
|||
prefix = '[%(levelname)1.1s %(asctime)s %(module)s:%(lineno)d]' % \ |
|||
record.__dict__ |
|||
if self._color: |
|||
prefix = (self._colors.get(record.levelno, self._normal) + |
|||
prefix + self._normal) |
|||
formatted = prefix + " " + record.message |
|||
if record.exc_info: |
|||
if not record.exc_text: |
|||
record.exc_text = self.formatException(record.exc_info) |
|||
if record.exc_text: |
|||
formatted = formatted.rstrip() + "\n" + record.exc_text |
|||
return formatted.replace("\n", "\n ") |
|||
|
|||
|
|||
options = _Options.instance() |
|||
|
|||
|
|||
# Default options |
|||
define("help", type=bool, help="show this help information") |
|||
define("logging", default="info", |
|||
help=("Set the Python log level. If 'none', tornado won't touch the " |
|||
"logging configuration."), |
|||
metavar="debug|info|warning|error|none") |
|||
define("log_to_stderr", type=bool, default=None, |
|||
help=("Send log output to stderr (colorized if possible). " |
|||
"By default use stderr if --log_file_prefix is not set and " |
|||
"no other logging is configured.")) |
|||
define("log_file_prefix", type=str, default=None, metavar="PATH", |
|||
help=("Path prefix for log files. " |
|||
"Note that if you are running multiple tornado processes, " |
|||
"log_file_prefix must be different for each of them (e.g. " |
|||
"include the port number)")) |
|||
define("log_file_max_size", type=int, default=100 * 1000 * 1000, |
|||
help="max size of log files before rollover") |
|||
define("log_file_num_backups", type=int, default=10, |
|||
help="number of log files to keep") |
|||
@ -0,0 +1,31 @@ |
|||
#!/usr/bin/env python |
|||
# |
|||
# Copyright 2011 Facebook |
|||
# |
|||
# Licensed under the Apache License, Version 2.0 (the "License"); you may |
|||
# not use this file except in compliance with the License. You may obtain |
|||
# a copy of the License at |
|||
# |
|||
# http://www.apache.org/licenses/LICENSE-2.0 |
|||
# |
|||
# Unless required by applicable law or agreed to in writing, software |
|||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT |
|||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the |
|||
# License for the specific language governing permissions and limitations |
|||
# under the License. |
|||
|
|||
"""Implementation of platform-specific functionality. |
|||
|
|||
For each function or class described in `tornado.platform.interface`, |
|||
the appropriate platform-specific implementation exists in this module. |
|||
Most code that needs access to this functionality should do e.g.:: |
|||
|
|||
from tornado.platform.auto import set_close_exec |
|||
""" |
|||
|
|||
import os |
|||
|
|||
if os.name == 'nt': |
|||
from tornado.platform.windows import set_close_exec, Waker |
|||
else: |
|||
from tornado.platform.posix import set_close_exec, Waker |
|||
@ -0,0 +1,57 @@ |
|||
#!/usr/bin/env python |
|||
# |
|||
# Copyright 2011 Facebook |
|||
# |
|||
# Licensed under the Apache License, Version 2.0 (the "License"); you may |
|||
# not use this file except in compliance with the License. You may obtain |
|||
# a copy of the License at |
|||
# |
|||
# http://www.apache.org/licenses/LICENSE-2.0 |
|||
# |
|||
# Unless required by applicable law or agreed to in writing, software |
|||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT |
|||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the |
|||
# License for the specific language governing permissions and limitations |
|||
# under the License. |
|||
|
|||
"""Interfaces for platform-specific functionality. |
|||
|
|||
This module exists primarily for documentation purposes and as base classes |
|||
for other tornado.platform modules. Most code should import the appropriate |
|||
implementation from `tornado.platform.auto`. |
|||
""" |
|||
|
|||
def set_close_exec(fd): |
|||
"""Sets the close-on-exec bit (``FD_CLOEXEC``)for a file descriptor.""" |
|||
raise NotImplementedError() |
|||
|
|||
class Waker(object): |
|||
"""A socket-like object that can wake another thread from ``select()``. |
|||
|
|||
The `~tornado.ioloop.IOLoop` will add the Waker's `fileno()` to |
|||
its ``select`` (or ``epoll`` or ``kqueue``) calls. When another |
|||
thread wants to wake up the loop, it calls `wake`. Once it has woken |
|||
up, it will call `consume` to do any necessary per-wake cleanup. When |
|||
the ``IOLoop`` is closed, it closes its waker too. |
|||
""" |
|||
def fileno(self): |
|||
"""Returns a file descriptor for this waker. |
|||
|
|||
Must be suitable for use with ``select()`` or equivalent on the |
|||
local platform. |
|||
""" |
|||
raise NotImplementedError() |
|||
|
|||
def wake(self): |
|||
"""Triggers activity on the waker's file descriptor.""" |
|||
raise NotImplementedError() |
|||
|
|||
def consume(self): |
|||
"""Called after the listen has woken up to do any necessary cleanup.""" |
|||
raise NotImplementedError() |
|||
|
|||
def close(self): |
|||
"""Closes the waker's file descriptor(s).""" |
|||
raise NotImplementedError() |
|||
|
|||
|
|||
@ -0,0 +1,62 @@ |
|||
#!/usr/bin/env python |
|||
# |
|||
# Copyright 2011 Facebook |
|||
# |
|||
# Licensed under the Apache License, Version 2.0 (the "License"); you may |
|||
# not use this file except in compliance with the License. You may obtain |
|||
# a copy of the License at |
|||
# |
|||
# http://www.apache.org/licenses/LICENSE-2.0 |
|||
# |
|||
# Unless required by applicable law or agreed to in writing, software |
|||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT |
|||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the |
|||
# License for the specific language governing permissions and limitations |
|||
# under the License. |
|||
|
|||
"""Posix implementations of platform-specific functionality.""" |
|||
|
|||
import fcntl |
|||
import os |
|||
|
|||
from tornado.platform import interface |
|||
from tornado.util import b |
|||
|
|||
def set_close_exec(fd): |
|||
flags = fcntl.fcntl(fd, fcntl.F_GETFD) |
|||
fcntl.fcntl(fd, fcntl.F_SETFD, flags | fcntl.FD_CLOEXEC) |
|||
|
|||
def _set_nonblocking(fd): |
|||
flags = fcntl.fcntl(fd, fcntl.F_GETFL) |
|||
fcntl.fcntl(fd, fcntl.F_SETFL, flags | os.O_NONBLOCK) |
|||
|
|||
class Waker(interface.Waker): |
|||
def __init__(self): |
|||
r, w = os.pipe() |
|||
_set_nonblocking(r) |
|||
_set_nonblocking(w) |
|||
set_close_exec(r) |
|||
set_close_exec(w) |
|||
self.reader = os.fdopen(r, "rb", 0) |
|||
self.writer = os.fdopen(w, "wb", 0) |
|||
|
|||
def fileno(self): |
|||
return self.reader.fileno() |
|||
|
|||
def wake(self): |
|||
try: |
|||
self.writer.write(b("x")) |
|||
except IOError: |
|||
pass |
|||
|
|||
def consume(self): |
|||
try: |
|||
while True: |
|||
result = self.reader.read() |
|||
if not result: break; |
|||
except IOError: |
|||
pass |
|||
|
|||
def close(self): |
|||
self.reader.close() |
|||
self.writer.close() |
|||
@ -0,0 +1,330 @@ |
|||
# Author: Ovidiu Predescu |
|||
# Date: July 2011 |
|||
# |
|||
# Licensed under the Apache License, Version 2.0 (the "License"); you may |
|||
# not use this file except in compliance with the License. You may obtain |
|||
# a copy of the License at |
|||
# |
|||
# http://www.apache.org/licenses/LICENSE-2.0 |
|||
# |
|||
# Unless required by applicable law or agreed to in writing, software |
|||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT |
|||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the |
|||
# License for the specific language governing permissions and limitations |
|||
# under the License. |
|||
|
|||
# Note: This module's docs are not currently extracted automatically, |
|||
# so changes must be made manually to twisted.rst |
|||
# TODO: refactor doc build process to use an appropriate virtualenv |
|||
"""A Twisted reactor built on the Tornado IOLoop. |
|||
|
|||
This module lets you run applications and libraries written for |
|||
Twisted in a Tornado application. To use it, simply call `install` at |
|||
the beginning of the application:: |
|||
|
|||
import tornado.platform.twisted |
|||
tornado.platform.twisted.install() |
|||
from twisted.internet import reactor |
|||
|
|||
When the app is ready to start, call `IOLoop.instance().start()` |
|||
instead of `reactor.run()`. This will allow you to use a mixture of |
|||
Twisted and Tornado code in the same process. |
|||
|
|||
It is also possible to create a non-global reactor by calling |
|||
`tornado.platform.twisted.TornadoReactor(io_loop)`. However, if |
|||
the `IOLoop` and reactor are to be short-lived (such as those used in |
|||
unit tests), additional cleanup may be required. Specifically, it is |
|||
recommended to call:: |
|||
|
|||
reactor.fireSystemEvent('shutdown') |
|||
reactor.disconnectAll() |
|||
|
|||
before closing the `IOLoop`. |
|||
|
|||
This module has been tested with Twisted versions 11.0.0 and 11.1.0. |
|||
""" |
|||
|
|||
from __future__ import with_statement, absolute_import |
|||
|
|||
import functools |
|||
import logging |
|||
import time |
|||
|
|||
from twisted.internet.posixbase import PosixReactorBase |
|||
from twisted.internet.interfaces import \ |
|||
IReactorFDSet, IDelayedCall, IReactorTime |
|||
from twisted.python import failure, log |
|||
from twisted.internet import error |
|||
|
|||
from zope.interface import implements |
|||
|
|||
import tornado |
|||
import tornado.ioloop |
|||
from tornado.stack_context import NullContext |
|||
from tornado.ioloop import IOLoop |
|||
|
|||
|
|||
class TornadoDelayedCall(object): |
|||
"""DelayedCall object for Tornado.""" |
|||
implements(IDelayedCall) |
|||
|
|||
def __init__(self, reactor, seconds, f, *args, **kw): |
|||
self._reactor = reactor |
|||
self._func = functools.partial(f, *args, **kw) |
|||
self._time = self._reactor.seconds() + seconds |
|||
self._timeout = self._reactor._io_loop.add_timeout(self._time, |
|||
self._called) |
|||
self._active = True |
|||
|
|||
def _called(self): |
|||
self._active = False |
|||
self._reactor._removeDelayedCall(self) |
|||
try: |
|||
self._func() |
|||
except: |
|||
logging.error("_called caught exception", exc_info=True) |
|||
|
|||
def getTime(self): |
|||
return self._time |
|||
|
|||
def cancel(self): |
|||
self._active = False |
|||
self._reactor._io_loop.remove_timeout(self._timeout) |
|||
self._reactor._removeDelayedCall(self) |
|||
|
|||
def delay(self, seconds): |
|||
self._reactor._io_loop.remove_timeout(self._timeout) |
|||
self._time += seconds |
|||
self._timeout = self._reactor._io_loop.add_timeout(self._time, |
|||
self._called) |
|||
|
|||
def reset(self, seconds): |
|||
self._reactor._io_loop.remove_timeout(self._timeout) |
|||
self._time = self._reactor.seconds() + seconds |
|||
self._timeout = self._reactor._io_loop.add_timeout(self._time, |
|||
self._called) |
|||
|
|||
def active(self): |
|||
return self._active |
|||
|
|||
class TornadoReactor(PosixReactorBase): |
|||
"""Twisted reactor built on the Tornado IOLoop. |
|||
|
|||
Since it is intented to be used in applications where the top-level |
|||
event loop is ``io_loop.start()`` rather than ``reactor.run()``, |
|||
it is implemented a little differently than other Twisted reactors. |
|||
We override `mainLoop` instead of `doIteration` and must implement |
|||
timed call functionality on top of `IOLoop.add_timeout` rather than |
|||
using the implementation in `PosixReactorBase`. |
|||
""" |
|||
implements(IReactorTime, IReactorFDSet) |
|||
|
|||
def __init__(self, io_loop=None): |
|||
if not io_loop: |
|||
io_loop = tornado.ioloop.IOLoop.instance() |
|||
self._io_loop = io_loop |
|||
self._readers = {} # map of reader objects to fd |
|||
self._writers = {} # map of writer objects to fd |
|||
self._fds = {} # a map of fd to a (reader, writer) tuple |
|||
self._delayedCalls = {} |
|||
PosixReactorBase.__init__(self) |
|||
|
|||
# IOLoop.start() bypasses some of the reactor initialization. |
|||
# Fire off the necessary events if they weren't already triggered |
|||
# by reactor.run(). |
|||
def start_if_necessary(): |
|||
if not self._started: |
|||
self.fireSystemEvent('startup') |
|||
self._io_loop.add_callback(start_if_necessary) |
|||
|
|||
# IReactorTime |
|||
def seconds(self): |
|||
return time.time() |
|||
|
|||
def callLater(self, seconds, f, *args, **kw): |
|||
dc = TornadoDelayedCall(self, seconds, f, *args, **kw) |
|||
self._delayedCalls[dc] = True |
|||
return dc |
|||
|
|||
def getDelayedCalls(self): |
|||
return [x for x in self._delayedCalls if x._active] |
|||
|
|||
def _removeDelayedCall(self, dc): |
|||
if dc in self._delayedCalls: |
|||
del self._delayedCalls[dc] |
|||
|
|||
# IReactorThreads |
|||
def callFromThread(self, f, *args, **kw): |
|||
"""See `twisted.internet.interfaces.IReactorThreads.callFromThread`""" |
|||
assert callable(f), "%s is not callable" % f |
|||
p = functools.partial(f, *args, **kw) |
|||
self._io_loop.add_callback(p) |
|||
|
|||
# We don't need the waker code from the super class, Tornado uses |
|||
# its own waker. |
|||
def installWaker(self): |
|||
pass |
|||
|
|||
def wakeUp(self): |
|||
pass |
|||
|
|||
# IReactorFDSet |
|||
def _invoke_callback(self, fd, events): |
|||
(reader, writer) = self._fds[fd] |
|||
if reader: |
|||
err = None |
|||
if reader.fileno() == -1: |
|||
err = error.ConnectionLost() |
|||
elif events & IOLoop.READ: |
|||
err = log.callWithLogger(reader, reader.doRead) |
|||
if err is None and events & IOLoop.ERROR: |
|||
err = error.ConnectionLost() |
|||
if err is not None: |
|||
self.removeReader(reader) |
|||
reader.readConnectionLost(failure.Failure(err)) |
|||
if writer: |
|||
err = None |
|||
if writer.fileno() == -1: |
|||
err = error.ConnectionLost() |
|||
elif events & IOLoop.WRITE: |
|||
err = log.callWithLogger(writer, writer.doWrite) |
|||
if err is None and events & IOLoop.ERROR: |
|||
err = error.ConnectionLost() |
|||
if err is not None: |
|||
self.removeWriter(writer) |
|||
writer.writeConnectionLost(failure.Failure(err)) |
|||
|
|||
def addReader(self, reader): |
|||
"""Add a FileDescriptor for notification of data available to read.""" |
|||
if reader in self._readers: |
|||
# Don't add the reader if it's already there |
|||
return |
|||
fd = reader.fileno() |
|||
self._readers[reader] = fd |
|||
if fd in self._fds: |
|||
(_, writer) = self._fds[fd] |
|||
self._fds[fd] = (reader, writer) |
|||
if writer: |
|||
# We already registered this fd for write events, |
|||
# update it for read events as well. |
|||
self._io_loop.update_handler(fd, IOLoop.READ | IOLoop.WRITE) |
|||
else: |
|||
with NullContext(): |
|||
self._fds[fd] = (reader, None) |
|||
self._io_loop.add_handler(fd, self._invoke_callback, |
|||
IOLoop.READ) |
|||
|
|||
def addWriter(self, writer): |
|||
"""Add a FileDescriptor for notification of data available to write.""" |
|||
if writer in self._writers: |
|||
return |
|||
fd = writer.fileno() |
|||
self._writers[writer] = fd |
|||
if fd in self._fds: |
|||
(reader, _) = self._fds[fd] |
|||
self._fds[fd] = (reader, writer) |
|||
if reader: |
|||
# We already registered this fd for read events, |
|||
# update it for write events as well. |
|||
self._io_loop.update_handler(fd, IOLoop.READ | IOLoop.WRITE) |
|||
else: |
|||
with NullContext(): |
|||
self._fds[fd] = (None, writer) |
|||
self._io_loop.add_handler(fd, self._invoke_callback, |
|||
IOLoop.WRITE) |
|||
|
|||
def removeReader(self, reader): |
|||
"""Remove a Selectable for notification of data available to read.""" |
|||
if reader in self._readers: |
|||
fd = self._readers.pop(reader) |
|||
(_, writer) = self._fds[fd] |
|||
if writer: |
|||
# We have a writer so we need to update the IOLoop for |
|||
# write events only. |
|||
self._fds[fd] = (None, writer) |
|||
self._io_loop.update_handler(fd, IOLoop.WRITE) |
|||
else: |
|||
# Since we have no writer registered, we remove the |
|||
# entry from _fds and unregister the handler from the |
|||
# IOLoop |
|||
del self._fds[fd] |
|||
self._io_loop.remove_handler(fd) |
|||
|
|||
def removeWriter(self, writer): |
|||
"""Remove a Selectable for notification of data available to write.""" |
|||
if writer in self._writers: |
|||
fd = self._writers.pop(writer) |
|||
(reader, _) = self._fds[fd] |
|||
if reader: |
|||
# We have a reader so we need to update the IOLoop for |
|||
# read events only. |
|||
self._fds[fd] = (reader, None) |
|||
self._io_loop.update_handler(fd, IOLoop.READ) |
|||
else: |
|||
# Since we have no reader registered, we remove the |
|||
# entry from the _fds and unregister the handler from |
|||
# the IOLoop. |
|||
del self._fds[fd] |
|||
self._io_loop.remove_handler(fd) |
|||
|
|||
def removeAll(self): |
|||
return self._removeAll(self._readers, self._writers) |
|||
|
|||
def getReaders(self): |
|||
return self._readers.keys() |
|||
|
|||
def getWriters(self): |
|||
return self._writers.keys() |
|||
|
|||
# The following functions are mainly used in twisted-style test cases; |
|||
# it is expected that most users of the TornadoReactor will call |
|||
# IOLoop.start() instead of Reactor.run(). |
|||
def stop(self): |
|||
PosixReactorBase.stop(self) |
|||
self._io_loop.stop() |
|||
|
|||
def crash(self): |
|||
PosixReactorBase.crash(self) |
|||
self._io_loop.stop() |
|||
|
|||
def doIteration(self, delay): |
|||
raise NotImplementedError("doIteration") |
|||
|
|||
def mainLoop(self): |
|||
self._io_loop.start() |
|||
if self._stopped: |
|||
self.fireSystemEvent("shutdown") |
|||
|
|||
class _TestReactor(TornadoReactor): |
|||
"""Subclass of TornadoReactor for use in unittests. |
|||
|
|||
This can't go in the test.py file because of import-order dependencies |
|||
with the Twisted reactor test builder. |
|||
""" |
|||
def __init__(self): |
|||
# always use a new ioloop |
|||
super(_TestReactor, self).__init__(IOLoop()) |
|||
|
|||
def listenTCP(self, port, factory, backlog=50, interface=''): |
|||
# default to localhost to avoid firewall prompts on the mac |
|||
if not interface: |
|||
interface = '127.0.0.1' |
|||
return super(_TestReactor, self).listenTCP( |
|||
port, factory, backlog=backlog, interface=interface) |
|||
|
|||
def listenUDP(self, port, protocol, interface='', maxPacketSize=8192): |
|||
if not interface: |
|||
interface = '127.0.0.1' |
|||
return super(_TestReactor, self).listenUDP( |
|||
port, protocol, interface=interface, maxPacketSize=maxPacketSize) |
|||
|
|||
|
|||
|
|||
def install(io_loop=None): |
|||
"""Install this package as the default Twisted reactor.""" |
|||
if not io_loop: |
|||
io_loop = tornado.ioloop.IOLoop.instance() |
|||
reactor = TornadoReactor(io_loop) |
|||
from twisted.internet.main import installReactor |
|||
installReactor(reactor) |
|||
return reactor |
|||
@ -0,0 +1,97 @@ |
|||
# NOTE: win32 support is currently experimental, and not recommended |
|||
# for production use. |
|||
|
|||
import ctypes |
|||
import ctypes.wintypes |
|||
import socket |
|||
import errno |
|||
|
|||
from tornado.platform import interface |
|||
from tornado.util import b |
|||
|
|||
# See: http://msdn.microsoft.com/en-us/library/ms724935(VS.85).aspx |
|||
SetHandleInformation = ctypes.windll.kernel32.SetHandleInformation |
|||
SetHandleInformation.argtypes = (ctypes.wintypes.HANDLE, ctypes.wintypes.DWORD, ctypes.wintypes.DWORD) |
|||
SetHandleInformation.restype = ctypes.wintypes.BOOL |
|||
|
|||
HANDLE_FLAG_INHERIT = 0x00000001 |
|||
|
|||
|
|||
def set_close_exec(fd): |
|||
success = SetHandleInformation(fd, HANDLE_FLAG_INHERIT, 0) |
|||
if not success: |
|||
raise ctypes.GetLastError() |
|||
|
|||
|
|||
class Waker(interface.Waker): |
|||
"""Create an OS independent asynchronous pipe""" |
|||
def __init__(self): |
|||
# Based on Zope async.py: http://svn.zope.org/zc.ngi/trunk/src/zc/ngi/async.py |
|||
|
|||
self.writer = socket.socket() |
|||
# Disable buffering -- pulling the trigger sends 1 byte, |
|||
# and we want that sent immediately, to wake up ASAP. |
|||
self.writer.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1) |
|||
|
|||
count = 0 |
|||
while 1: |
|||
count += 1 |
|||
# Bind to a local port; for efficiency, let the OS pick |
|||
# a free port for us. |
|||
# Unfortunately, stress tests showed that we may not |
|||
# be able to connect to that port ("Address already in |
|||
# use") despite that the OS picked it. This appears |
|||
# to be a race bug in the Windows socket implementation. |
|||
# So we loop until a connect() succeeds (almost always |
|||
# on the first try). See the long thread at |
|||
# http://mail.zope.org/pipermail/zope/2005-July/160433.html |
|||
# for hideous details. |
|||
a = socket.socket() |
|||
a.bind(("127.0.0.1", 0)) |
|||
connect_address = a.getsockname() # assigned (host, port) pair |
|||
a.listen(1) |
|||
try: |
|||
self.writer.connect(connect_address) |
|||
break # success |
|||
except socket.error, detail: |
|||
if detail[0] != errno.WSAEADDRINUSE: |
|||
# "Address already in use" is the only error |
|||
# I've seen on two WinXP Pro SP2 boxes, under |
|||
# Pythons 2.3.5 and 2.4.1. |
|||
raise |
|||
# (10048, 'Address already in use') |
|||
# assert count <= 2 # never triggered in Tim's tests |
|||
if count >= 10: # I've never seen it go above 2 |
|||
a.close() |
|||
self.writer.close() |
|||
raise socket.error("Cannot bind trigger!") |
|||
# Close `a` and try again. Note: I originally put a short |
|||
# sleep() here, but it didn't appear to help or hurt. |
|||
a.close() |
|||
|
|||
self.reader, addr = a.accept() |
|||
self.reader.setblocking(0) |
|||
self.writer.setblocking(0) |
|||
a.close() |
|||
self.reader_fd = self.reader.fileno() |
|||
|
|||
def fileno(self): |
|||
return self.reader.fileno() |
|||
|
|||
def wake(self): |
|||
try: |
|||
self.writer.send(b("x")) |
|||
except IOError: |
|||
pass |
|||
|
|||
def consume(self): |
|||
try: |
|||
while True: |
|||
result = self.reader.recv(1024) |
|||
if not result: break |
|||
except IOError: |
|||
pass |
|||
|
|||
def close(self): |
|||
self.reader.close() |
|||
self.writer.close() |
|||
@ -0,0 +1,149 @@ |
|||
#!/usr/bin/env python |
|||
# |
|||
# Copyright 2011 Facebook |
|||
# |
|||
# Licensed under the Apache License, Version 2.0 (the "License"); you may |
|||
# not use this file except in compliance with the License. You may obtain |
|||
# a copy of the License at |
|||
# |
|||
# http://www.apache.org/licenses/LICENSE-2.0 |
|||
# |
|||
# Unless required by applicable law or agreed to in writing, software |
|||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT |
|||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the |
|||
# License for the specific language governing permissions and limitations |
|||
# under the License. |
|||
|
|||
"""Utilities for working with multiple processes.""" |
|||
|
|||
import errno |
|||
import logging |
|||
import os |
|||
import sys |
|||
import time |
|||
|
|||
from binascii import hexlify |
|||
|
|||
from tornado import ioloop |
|||
|
|||
try: |
|||
import multiprocessing # Python 2.6+ |
|||
except ImportError: |
|||
multiprocessing = None |
|||
|
|||
def cpu_count(): |
|||
"""Returns the number of processors on this machine.""" |
|||
if multiprocessing is not None: |
|||
try: |
|||
return multiprocessing.cpu_count() |
|||
except NotImplementedError: |
|||
pass |
|||
try: |
|||
return os.sysconf("SC_NPROCESSORS_CONF") |
|||
except ValueError: |
|||
pass |
|||
logging.error("Could not detect number of processors; assuming 1") |
|||
return 1 |
|||
|
|||
def _reseed_random(): |
|||
if 'random' not in sys.modules: |
|||
return |
|||
import random |
|||
# If os.urandom is available, this method does the same thing as |
|||
# random.seed (at least as of python 2.6). If os.urandom is not |
|||
# available, we mix in the pid in addition to a timestamp. |
|||
try: |
|||
seed = long(hexlify(os.urandom(16)), 16) |
|||
except NotImplementedError: |
|||
seed = int(time.time() * 1000) ^ os.getpid() |
|||
random.seed(seed) |
|||
|
|||
|
|||
_task_id = None |
|||
|
|||
def fork_processes(num_processes, max_restarts=100): |
|||
"""Starts multiple worker processes. |
|||
|
|||
If ``num_processes`` is None or <= 0, we detect the number of cores |
|||
available on this machine and fork that number of child |
|||
processes. If ``num_processes`` is given and > 0, we fork that |
|||
specific number of sub-processes. |
|||
|
|||
Since we use processes and not threads, there is no shared memory |
|||
between any server code. |
|||
|
|||
Note that multiple processes are not compatible with the autoreload |
|||
module (or the debug=True option to `tornado.web.Application`). |
|||
When using multiple processes, no IOLoops can be created or |
|||
referenced until after the call to ``fork_processes``. |
|||
|
|||
In each child process, ``fork_processes`` returns its *task id*, a |
|||
number between 0 and ``num_processes``. Processes that exit |
|||
abnormally (due to a signal or non-zero exit status) are restarted |
|||
with the same id (up to ``max_restarts`` times). In the parent |
|||
process, ``fork_processes`` returns None if all child processes |
|||
have exited normally, but will otherwise only exit by throwing an |
|||
exception. |
|||
""" |
|||
global _task_id |
|||
assert _task_id is None |
|||
if num_processes is None or num_processes <= 0: |
|||
num_processes = cpu_count() |
|||
if ioloop.IOLoop.initialized(): |
|||
raise RuntimeError("Cannot run in multiple processes: IOLoop instance " |
|||
"has already been initialized. You cannot call " |
|||
"IOLoop.instance() before calling start_processes()") |
|||
logging.info("Starting %d processes", num_processes) |
|||
children = {} |
|||
def start_child(i): |
|||
pid = os.fork() |
|||
if pid == 0: |
|||
# child process |
|||
_reseed_random() |
|||
global _task_id |
|||
_task_id = i |
|||
return i |
|||
else: |
|||
children[pid] = i |
|||
return None |
|||
for i in range(num_processes): |
|||
id = start_child(i) |
|||
if id is not None: return id |
|||
num_restarts = 0 |
|||
while children: |
|||
try: |
|||
pid, status = os.wait() |
|||
except OSError, e: |
|||
if e.errno == errno.EINTR: |
|||
continue |
|||
raise |
|||
if pid not in children: |
|||
continue |
|||
id = children.pop(pid) |
|||
if os.WIFSIGNALED(status): |
|||
logging.warning("child %d (pid %d) killed by signal %d, restarting", |
|||
id, pid, os.WTERMSIG(status)) |
|||
elif os.WEXITSTATUS(status) != 0: |
|||
logging.warning("child %d (pid %d) exited with status %d, restarting", |
|||
id, pid, os.WEXITSTATUS(status)) |
|||
else: |
|||
logging.info("child %d (pid %d) exited normally", id, pid) |
|||
continue |
|||
num_restarts += 1 |
|||
if num_restarts > max_restarts: |
|||
raise RuntimeError("Too many child restarts, giving up") |
|||
new_id = start_child(id) |
|||
if new_id is not None: return new_id |
|||
# All child processes exited cleanly, so exit the master process |
|||
# instead of just returning to right after the call to |
|||
# fork_processes (which will probably just start up another IOLoop |
|||
# unless the caller checks the return value). |
|||
sys.exit(0) |
|||
|
|||
def task_id(): |
|||
"""Returns the current task id, if any. |
|||
|
|||
Returns None if this process was not created by `fork_processes`. |
|||
""" |
|||
global _task_id |
|||
return _task_id |
|||
@ -0,0 +1,509 @@ |
|||
#!/usr/bin/env python |
|||
from __future__ import with_statement |
|||
|
|||
from tornado.escape import utf8, _unicode, native_str |
|||
from tornado.httpclient import HTTPRequest, HTTPResponse, HTTPError, AsyncHTTPClient, main |
|||
from tornado.httputil import HTTPHeaders |
|||
from tornado.iostream import IOStream, SSLIOStream |
|||
from tornado import stack_context |
|||
from tornado.util import b |
|||
|
|||
import base64 |
|||
import collections |
|||
import contextlib |
|||
import copy |
|||
import functools |
|||
import logging |
|||
import os.path |
|||
import re |
|||
import socket |
|||
import sys |
|||
import time |
|||
import urlparse |
|||
import zlib |
|||
|
|||
try: |
|||
from io import BytesIO # python 3 |
|||
except ImportError: |
|||
from cStringIO import StringIO as BytesIO # python 2 |
|||
|
|||
try: |
|||
import ssl # python 2.6+ |
|||
except ImportError: |
|||
ssl = None |
|||
|
|||
_DEFAULT_CA_CERTS = os.path.dirname(__file__) + '/ca-certificates.crt' |
|||
|
|||
class SimpleAsyncHTTPClient(AsyncHTTPClient): |
|||
"""Non-blocking HTTP client with no external dependencies. |
|||
|
|||
This class implements an HTTP 1.1 client on top of Tornado's IOStreams. |
|||
It does not currently implement all applicable parts of the HTTP |
|||
specification, but it does enough to work with major web service APIs |
|||
(mostly tested against the Twitter API so far). |
|||
|
|||
This class has not been tested extensively in production and |
|||
should be considered somewhat experimental as of the release of |
|||
tornado 1.2. It is intended to become the default AsyncHTTPClient |
|||
implementation in a future release. It may either be used |
|||
directly, or to facilitate testing of this class with an existing |
|||
application, setting the environment variable |
|||
USE_SIMPLE_HTTPCLIENT=1 will cause this class to transparently |
|||
replace tornado.httpclient.AsyncHTTPClient. |
|||
|
|||
Some features found in the curl-based AsyncHTTPClient are not yet |
|||
supported. In particular, proxies are not supported, connections |
|||
are not reused, and callers cannot select the network interface to be |
|||
used. |
|||
|
|||
Python 2.6 or higher is required for HTTPS support. Users of Python 2.5 |
|||
should use the curl-based AsyncHTTPClient if HTTPS support is required. |
|||
|
|||
""" |
|||
def initialize(self, io_loop=None, max_clients=10, |
|||
max_simultaneous_connections=None, |
|||
hostname_mapping=None, max_buffer_size=104857600): |
|||
"""Creates a AsyncHTTPClient. |
|||
|
|||
Only a single AsyncHTTPClient instance exists per IOLoop |
|||
in order to provide limitations on the number of pending connections. |
|||
force_instance=True may be used to suppress this behavior. |
|||
|
|||
max_clients is the number of concurrent requests that can be in |
|||
progress. max_simultaneous_connections has no effect and is accepted |
|||
only for compatibility with the curl-based AsyncHTTPClient. Note |
|||
that these arguments are only used when the client is first created, |
|||
and will be ignored when an existing client is reused. |
|||
|
|||
hostname_mapping is a dictionary mapping hostnames to IP addresses. |
|||
It can be used to make local DNS changes when modifying system-wide |
|||
settings like /etc/hosts is not possible or desirable (e.g. in |
|||
unittests). |
|||
|
|||
max_buffer_size is the number of bytes that can be read by IOStream. It |
|||
defaults to 100mb. |
|||
""" |
|||
self.io_loop = io_loop |
|||
self.max_clients = max_clients |
|||
self.queue = collections.deque() |
|||
self.active = {} |
|||
self.hostname_mapping = hostname_mapping |
|||
self.max_buffer_size = max_buffer_size |
|||
|
|||
def fetch(self, request, callback, **kwargs): |
|||
if not isinstance(request, HTTPRequest): |
|||
request = HTTPRequest(url=request, **kwargs) |
|||
if not isinstance(request.headers, HTTPHeaders): |
|||
request.headers = HTTPHeaders(request.headers) |
|||
callback = stack_context.wrap(callback) |
|||
self.queue.append((request, callback)) |
|||
self._process_queue() |
|||
if self.queue: |
|||
logging.debug("max_clients limit reached, request queued. " |
|||
"%d active, %d queued requests." % ( |
|||
len(self.active), len(self.queue))) |
|||
|
|||
def _process_queue(self): |
|||
with stack_context.NullContext(): |
|||
while self.queue and len(self.active) < self.max_clients: |
|||
request, callback = self.queue.popleft() |
|||
key = object() |
|||
self.active[key] = (request, callback) |
|||
_HTTPConnection(self.io_loop, self, request, |
|||
functools.partial(self._release_fetch, key), |
|||
callback, |
|||
self.max_buffer_size) |
|||
|
|||
def _release_fetch(self, key): |
|||
del self.active[key] |
|||
self._process_queue() |
|||
|
|||
|
|||
|
|||
class _HTTPConnection(object): |
|||
_SUPPORTED_METHODS = set(["GET", "HEAD", "POST", "PUT", "DELETE"]) |
|||
|
|||
def __init__(self, io_loop, client, request, release_callback, |
|||
final_callback, max_buffer_size): |
|||
self.start_time = time.time() |
|||
self.io_loop = io_loop |
|||
self.client = client |
|||
self.request = request |
|||
self.release_callback = release_callback |
|||
self.final_callback = final_callback |
|||
self.code = None |
|||
self.headers = None |
|||
self.chunks = None |
|||
self._decompressor = None |
|||
# Timeout handle returned by IOLoop.add_timeout |
|||
self._timeout = None |
|||
with stack_context.StackContext(self.cleanup): |
|||
parsed = urlparse.urlsplit(_unicode(self.request.url)) |
|||
if ssl is None and parsed.scheme == "https": |
|||
raise ValueError("HTTPS requires either python2.6+ or " |
|||
"curl_httpclient") |
|||
if parsed.scheme not in ("http", "https"): |
|||
raise ValueError("Unsupported url scheme: %s" % |
|||
self.request.url) |
|||
# urlsplit results have hostname and port results, but they |
|||
# didn't support ipv6 literals until python 2.7. |
|||
netloc = parsed.netloc |
|||
if "@" in netloc: |
|||
userpass, _, netloc = netloc.rpartition("@") |
|||
match = re.match(r'^(.+):(\d+)$', netloc) |
|||
if match: |
|||
host = match.group(1) |
|||
port = int(match.group(2)) |
|||
else: |
|||
host = netloc |
|||
port = 443 if parsed.scheme == "https" else 80 |
|||
if re.match(r'^\[.*\]$', host): |
|||
# raw ipv6 addresses in urls are enclosed in brackets |
|||
host = host[1:-1] |
|||
if self.client.hostname_mapping is not None: |
|||
host = self.client.hostname_mapping.get(host, host) |
|||
|
|||
if request.allow_ipv6: |
|||
af = socket.AF_UNSPEC |
|||
else: |
|||
# We only try the first IP we get from getaddrinfo, |
|||
# so restrict to ipv4 by default. |
|||
af = socket.AF_INET |
|||
|
|||
addrinfo = socket.getaddrinfo(host, port, af, socket.SOCK_STREAM, |
|||
0, 0) |
|||
af, socktype, proto, canonname, sockaddr = addrinfo[0] |
|||
|
|||
if parsed.scheme == "https": |
|||
ssl_options = {} |
|||
if request.validate_cert: |
|||
ssl_options["cert_reqs"] = ssl.CERT_REQUIRED |
|||
if request.ca_certs is not None: |
|||
ssl_options["ca_certs"] = request.ca_certs |
|||
else: |
|||
ssl_options["ca_certs"] = _DEFAULT_CA_CERTS |
|||
if request.client_key is not None: |
|||
ssl_options["keyfile"] = request.client_key |
|||
if request.client_cert is not None: |
|||
ssl_options["certfile"] = request.client_cert |
|||
|
|||
# SSL interoperability is tricky. We want to disable |
|||
# SSLv2 for security reasons; it wasn't disabled by default |
|||
# until openssl 1.0. The best way to do this is to use |
|||
# the SSL_OP_NO_SSLv2, but that wasn't exposed to python |
|||
# until 3.2. Python 2.7 adds the ciphers argument, which |
|||
# can also be used to disable SSLv2. As a last resort |
|||
# on python 2.6, we set ssl_version to SSLv3. This is |
|||
# more narrow than we'd like since it also breaks |
|||
# compatibility with servers configured for TLSv1 only, |
|||
# but nearly all servers support SSLv3: |
|||
# http://blog.ivanristic.com/2011/09/ssl-survey-protocol-support.html |
|||
if sys.version_info >= (2,7): |
|||
ssl_options["ciphers"] = "DEFAULT:!SSLv2" |
|||
else: |
|||
# This is really only necessary for pre-1.0 versions |
|||
# of openssl, but python 2.6 doesn't expose version |
|||
# information. |
|||
ssl_options["ssl_version"] = ssl.PROTOCOL_SSLv3 |
|||
|
|||
self.stream = SSLIOStream(socket.socket(af, socktype, proto), |
|||
io_loop=self.io_loop, |
|||
ssl_options=ssl_options, |
|||
max_buffer_size=max_buffer_size) |
|||
else: |
|||
self.stream = IOStream(socket.socket(af, socktype, proto), |
|||
io_loop=self.io_loop, |
|||
max_buffer_size=max_buffer_size) |
|||
timeout = min(request.connect_timeout, request.request_timeout) |
|||
if timeout: |
|||
self._timeout = self.io_loop.add_timeout( |
|||
self.start_time + timeout, |
|||
self._on_timeout) |
|||
self.stream.set_close_callback(self._on_close) |
|||
self.stream.connect(sockaddr, |
|||
functools.partial(self._on_connect, parsed)) |
|||
|
|||
def _on_timeout(self): |
|||
self._timeout = None |
|||
self._run_callback(HTTPResponse(self.request, 599, |
|||
request_time=time.time() - self.start_time, |
|||
error=HTTPError(599, "Timeout"))) |
|||
self.stream.close() |
|||
|
|||
def _on_connect(self, parsed): |
|||
if self._timeout is not None: |
|||
self.io_loop.remove_timeout(self._timeout) |
|||
self._timeout = None |
|||
if self.request.request_timeout: |
|||
self._timeout = self.io_loop.add_timeout( |
|||
self.start_time + self.request.request_timeout, |
|||
self._on_timeout) |
|||
if (self.request.validate_cert and |
|||
isinstance(self.stream, SSLIOStream)): |
|||
match_hostname(self.stream.socket.getpeercert(), |
|||
parsed.hostname) |
|||
if (self.request.method not in self._SUPPORTED_METHODS and |
|||
not self.request.allow_nonstandard_methods): |
|||
raise KeyError("unknown method %s" % self.request.method) |
|||
for key in ('network_interface', |
|||
'proxy_host', 'proxy_port', |
|||
'proxy_username', 'proxy_password'): |
|||
if getattr(self.request, key, None): |
|||
raise NotImplementedError('%s not supported' % key) |
|||
if "Host" not in self.request.headers: |
|||
self.request.headers["Host"] = parsed.netloc |
|||
username, password = None, None |
|||
if parsed.username is not None: |
|||
username, password = parsed.username, parsed.password |
|||
elif self.request.auth_username is not None: |
|||
username = self.request.auth_username |
|||
password = self.request.auth_password or '' |
|||
if username is not None: |
|||
auth = utf8(username) + b(":") + utf8(password) |
|||
self.request.headers["Authorization"] = (b("Basic ") + |
|||
base64.b64encode(auth)) |
|||
if self.request.user_agent: |
|||
self.request.headers["User-Agent"] = self.request.user_agent |
|||
if not self.request.allow_nonstandard_methods: |
|||
if self.request.method in ("POST", "PUT"): |
|||
assert self.request.body is not None |
|||
else: |
|||
assert self.request.body is None |
|||
if self.request.body is not None: |
|||
self.request.headers["Content-Length"] = str(len( |
|||
self.request.body)) |
|||
if (self.request.method == "POST" and |
|||
"Content-Type" not in self.request.headers): |
|||
self.request.headers["Content-Type"] = "application/x-www-form-urlencoded" |
|||
if self.request.use_gzip: |
|||
self.request.headers["Accept-Encoding"] = "gzip" |
|||
req_path = ((parsed.path or '/') + |
|||
(('?' + parsed.query) if parsed.query else '')) |
|||
request_lines = [utf8("%s %s HTTP/1.1" % (self.request.method, |
|||
req_path))] |
|||
for k, v in self.request.headers.get_all(): |
|||
line = utf8(k) + b(": ") + utf8(v) |
|||
if b('\n') in line: |
|||
raise ValueError('Newline in header: ' + repr(line)) |
|||
request_lines.append(line) |
|||
self.stream.write(b("\r\n").join(request_lines) + b("\r\n\r\n")) |
|||
if self.request.body is not None: |
|||
self.stream.write(self.request.body) |
|||
self.stream.read_until_regex(b("\r?\n\r?\n"), self._on_headers) |
|||
|
|||
def _release(self): |
|||
if self.release_callback is not None: |
|||
release_callback = self.release_callback |
|||
self.release_callback = None |
|||
release_callback() |
|||
|
|||
def _run_callback(self, response): |
|||
self._release() |
|||
if self.final_callback is not None: |
|||
final_callback = self.final_callback |
|||
self.final_callback = None |
|||
final_callback(response) |
|||
|
|||
@contextlib.contextmanager |
|||
def cleanup(self): |
|||
try: |
|||
yield |
|||
except Exception, e: |
|||
logging.warning("uncaught exception", exc_info=True) |
|||
self._run_callback(HTTPResponse(self.request, 599, error=e, |
|||
request_time=time.time() - self.start_time, |
|||
)) |
|||
|
|||
def _on_close(self): |
|||
self._run_callback(HTTPResponse( |
|||
self.request, 599, |
|||
request_time=time.time() - self.start_time, |
|||
error=HTTPError(599, "Connection closed"))) |
|||
|
|||
def _on_headers(self, data): |
|||
data = native_str(data.decode("latin1")) |
|||
first_line, _, header_data = data.partition("\n") |
|||
match = re.match("HTTP/1.[01] ([0-9]+)", first_line) |
|||
assert match |
|||
self.code = int(match.group(1)) |
|||
self.headers = HTTPHeaders.parse(header_data) |
|||
|
|||
if "Content-Length" in self.headers: |
|||
if "," in self.headers["Content-Length"]: |
|||
# Proxies sometimes cause Content-Length headers to get |
|||
# duplicated. If all the values are identical then we can |
|||
# use them but if they differ it's an error. |
|||
pieces = re.split(r',\s*', self.headers["Content-Length"]) |
|||
if any(i != pieces[0] for i in pieces): |
|||
raise ValueError("Multiple unequal Content-Lengths: %r" % |
|||
self.headers["Content-Length"]) |
|||
self.headers["Content-Length"] = pieces[0] |
|||
content_length = int(self.headers["Content-Length"]) |
|||
else: |
|||
content_length = None |
|||
|
|||
if self.request.header_callback is not None: |
|||
for k, v in self.headers.get_all(): |
|||
self.request.header_callback("%s: %s\r\n" % (k, v)) |
|||
|
|||
if self.request.method == "HEAD": |
|||
# HEAD requests never have content, even though they may have |
|||
# content-length headers |
|||
self._on_body(b("")) |
|||
return |
|||
if 100 <= self.code < 200 or self.code in (204, 304): |
|||
# These response codes never have bodies |
|||
# http://www.w3.org/Protocols/rfc2616/rfc2616-sec4.html#sec4.3 |
|||
assert "Transfer-Encoding" not in self.headers |
|||
assert content_length in (None, 0) |
|||
self._on_body(b("")) |
|||
return |
|||
|
|||
if (self.request.use_gzip and |
|||
self.headers.get("Content-Encoding") == "gzip"): |
|||
# Magic parameter makes zlib module understand gzip header |
|||
# http://stackoverflow.com/questions/1838699/how-can-i-decompress-a-gzip-stream-with-zlib |
|||
self._decompressor = zlib.decompressobj(16+zlib.MAX_WBITS) |
|||
if self.headers.get("Transfer-Encoding") == "chunked": |
|||
self.chunks = [] |
|||
self.stream.read_until(b("\r\n"), self._on_chunk_length) |
|||
elif content_length is not None: |
|||
self.stream.read_bytes(content_length, self._on_body) |
|||
else: |
|||
self.stream.read_until_close(self._on_body) |
|||
|
|||
def _on_body(self, data): |
|||
if self._timeout is not None: |
|||
self.io_loop.remove_timeout(self._timeout) |
|||
self._timeout = None |
|||
original_request = getattr(self.request, "original_request", |
|||
self.request) |
|||
if (self.request.follow_redirects and |
|||
self.request.max_redirects > 0 and |
|||
self.code in (301, 302, 303, 307)): |
|||
new_request = copy.copy(self.request) |
|||
new_request.url = urlparse.urljoin(self.request.url, |
|||
self.headers["Location"]) |
|||
new_request.max_redirects -= 1 |
|||
del new_request.headers["Host"] |
|||
# http://www.w3.org/Protocols/rfc2616/rfc2616-sec10.html#sec10.3.4 |
|||
# client SHOULD make a GET request |
|||
if self.code == 303: |
|||
new_request.method = "GET" |
|||
new_request.body = None |
|||
for h in ["Content-Length", "Content-Type", |
|||
"Content-Encoding", "Transfer-Encoding"]: |
|||
try: |
|||
del self.request.headers[h] |
|||
except KeyError: |
|||
pass |
|||
new_request.original_request = original_request |
|||
final_callback = self.final_callback |
|||
self.final_callback = None |
|||
self._release() |
|||
self.client.fetch(new_request, final_callback) |
|||
self.stream.close() |
|||
return |
|||
if self._decompressor: |
|||
data = self._decompressor.decompress(data) |
|||
if self.request.streaming_callback: |
|||
if self.chunks is None: |
|||
# if chunks is not None, we already called streaming_callback |
|||
# in _on_chunk_data |
|||
self.request.streaming_callback(data) |
|||
buffer = BytesIO() |
|||
else: |
|||
buffer = BytesIO(data) # TODO: don't require one big string? |
|||
response = HTTPResponse(original_request, |
|||
self.code, headers=self.headers, |
|||
request_time=time.time() - self.start_time, |
|||
buffer=buffer, |
|||
effective_url=self.request.url) |
|||
self._run_callback(response) |
|||
self.stream.close() |
|||
|
|||
def _on_chunk_length(self, data): |
|||
# TODO: "chunk extensions" http://tools.ietf.org/html/rfc2616#section-3.6.1 |
|||
length = int(data.strip(), 16) |
|||
if length == 0: |
|||
# all the data has been decompressed, so we don't need to |
|||
# decompress again in _on_body |
|||
self._decompressor = None |
|||
self._on_body(b('').join(self.chunks)) |
|||
else: |
|||
self.stream.read_bytes(length + 2, # chunk ends with \r\n |
|||
self._on_chunk_data) |
|||
|
|||
def _on_chunk_data(self, data): |
|||
assert data[-2:] == b("\r\n") |
|||
chunk = data[:-2] |
|||
if self._decompressor: |
|||
chunk = self._decompressor.decompress(chunk) |
|||
if self.request.streaming_callback is not None: |
|||
self.request.streaming_callback(chunk) |
|||
else: |
|||
self.chunks.append(chunk) |
|||
self.stream.read_until(b("\r\n"), self._on_chunk_length) |
|||
|
|||
|
|||
# match_hostname was added to the standard library ssl module in python 3.2. |
|||
# The following code was backported for older releases and copied from |
|||
# https://bitbucket.org/brandon/backports.ssl_match_hostname |
|||
class CertificateError(ValueError): |
|||
pass |
|||
|
|||
def _dnsname_to_pat(dn): |
|||
pats = [] |
|||
for frag in dn.split(r'.'): |
|||
if frag == '*': |
|||
# When '*' is a fragment by itself, it matches a non-empty dotless |
|||
# fragment. |
|||
pats.append('[^.]+') |
|||
else: |
|||
# Otherwise, '*' matches any dotless fragment. |
|||
frag = re.escape(frag) |
|||
pats.append(frag.replace(r'\*', '[^.]*')) |
|||
return re.compile(r'\A' + r'\.'.join(pats) + r'\Z', re.IGNORECASE) |
|||
|
|||
def match_hostname(cert, hostname): |
|||
"""Verify that *cert* (in decoded format as returned by |
|||
SSLSocket.getpeercert()) matches the *hostname*. RFC 2818 rules |
|||
are mostly followed, but IP addresses are not accepted for *hostname*. |
|||
|
|||
CertificateError is raised on failure. On success, the function |
|||
returns nothing. |
|||
""" |
|||
if not cert: |
|||
raise ValueError("empty or no certificate") |
|||
dnsnames = [] |
|||
san = cert.get('subjectAltName', ()) |
|||
for key, value in san: |
|||
if key == 'DNS': |
|||
if _dnsname_to_pat(value).match(hostname): |
|||
return |
|||
dnsnames.append(value) |
|||
if not san: |
|||
# The subject is only checked when subjectAltName is empty |
|||
for sub in cert.get('subject', ()): |
|||
for key, value in sub: |
|||
# XXX according to RFC 2818, the most specific Common Name |
|||
# must be used. |
|||
if key == 'commonName': |
|||
if _dnsname_to_pat(value).match(hostname): |
|||
return |
|||
dnsnames.append(value) |
|||
if len(dnsnames) > 1: |
|||
raise CertificateError("hostname %r " |
|||
"doesn't match either of %s" |
|||
% (hostname, ', '.join(map(repr, dnsnames)))) |
|||
elif len(dnsnames) == 1: |
|||
raise CertificateError("hostname %r " |
|||
"doesn't match %r" |
|||
% (hostname, dnsnames[0])) |
|||
else: |
|||
raise CertificateError("no appropriate commonName or " |
|||
"subjectAltName fields were found") |
|||
|
|||
if __name__ == "__main__": |
|||
AsyncHTTPClient.configure(SimpleAsyncHTTPClient) |
|||
main() |
|||
@ -0,0 +1,244 @@ |
|||
#!/usr/bin/env python |
|||
# |
|||
# Copyright 2010 Facebook |
|||
# |
|||
# Licensed under the Apache License, Version 2.0 (the "License"); you may |
|||
# not use this file except in compliance with the License. You may obtain |
|||
# a copy of the License at |
|||
# |
|||
# http://www.apache.org/licenses/LICENSE-2.0 |
|||
# |
|||
# Unless required by applicable law or agreed to in writing, software |
|||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT |
|||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the |
|||
# License for the specific language governing permissions and limitations |
|||
# under the License. |
|||
|
|||
'''StackContext allows applications to maintain threadlocal-like state |
|||
that follows execution as it moves to other execution contexts. |
|||
|
|||
The motivating examples are to eliminate the need for explicit |
|||
async_callback wrappers (as in tornado.web.RequestHandler), and to |
|||
allow some additional context to be kept for logging. |
|||
|
|||
This is slightly magic, but it's an extension of the idea that an exception |
|||
handler is a kind of stack-local state and when that stack is suspended |
|||
and resumed in a new context that state needs to be preserved. StackContext |
|||
shifts the burden of restoring that state from each call site (e.g. |
|||
wrapping each AsyncHTTPClient callback in async_callback) to the mechanisms |
|||
that transfer control from one context to another (e.g. AsyncHTTPClient |
|||
itself, IOLoop, thread pools, etc). |
|||
|
|||
Example usage:: |
|||
|
|||
@contextlib.contextmanager |
|||
def die_on_error(): |
|||
try: |
|||
yield |
|||
except Exception: |
|||
logging.error("exception in asynchronous operation",exc_info=True) |
|||
sys.exit(1) |
|||
|
|||
with StackContext(die_on_error): |
|||
# Any exception thrown here *or in callback and its desendents* |
|||
# will cause the process to exit instead of spinning endlessly |
|||
# in the ioloop. |
|||
http_client.fetch(url, callback) |
|||
ioloop.start() |
|||
|
|||
Most applications shouln't have to work with `StackContext` directly. |
|||
Here are a few rules of thumb for when it's necessary: |
|||
|
|||
* If you're writing an asynchronous library that doesn't rely on a |
|||
stack_context-aware library like `tornado.ioloop` or `tornado.iostream` |
|||
(for example, if you're writing a thread pool), use |
|||
`stack_context.wrap()` before any asynchronous operations to capture the |
|||
stack context from where the operation was started. |
|||
|
|||
* If you're writing an asynchronous library that has some shared |
|||
resources (such as a connection pool), create those shared resources |
|||
within a ``with stack_context.NullContext():`` block. This will prevent |
|||
``StackContexts`` from leaking from one request to another. |
|||
|
|||
* If you want to write something like an exception handler that will |
|||
persist across asynchronous calls, create a new `StackContext` (or |
|||
`ExceptionStackContext`), and make your asynchronous calls in a ``with`` |
|||
block that references your `StackContext`. |
|||
''' |
|||
|
|||
from __future__ import with_statement |
|||
|
|||
import contextlib |
|||
import functools |
|||
import itertools |
|||
import sys |
|||
import threading |
|||
|
|||
class _State(threading.local): |
|||
def __init__(self): |
|||
self.contexts = () |
|||
_state = _State() |
|||
|
|||
class StackContext(object): |
|||
'''Establishes the given context as a StackContext that will be transferred. |
|||
|
|||
Note that the parameter is a callable that returns a context |
|||
manager, not the context itself. That is, where for a |
|||
non-transferable context manager you would say:: |
|||
|
|||
with my_context(): |
|||
|
|||
StackContext takes the function itself rather than its result:: |
|||
|
|||
with StackContext(my_context): |
|||
''' |
|||
def __init__(self, context_factory): |
|||
self.context_factory = context_factory |
|||
|
|||
# Note that some of this code is duplicated in ExceptionStackContext |
|||
# below. ExceptionStackContext is more common and doesn't need |
|||
# the full generality of this class. |
|||
def __enter__(self): |
|||
self.old_contexts = _state.contexts |
|||
# _state.contexts is a tuple of (class, arg) pairs |
|||
_state.contexts = (self.old_contexts + |
|||
((StackContext, self.context_factory),)) |
|||
try: |
|||
self.context = self.context_factory() |
|||
self.context.__enter__() |
|||
except Exception: |
|||
_state.contexts = self.old_contexts |
|||
raise |
|||
|
|||
def __exit__(self, type, value, traceback): |
|||
try: |
|||
return self.context.__exit__(type, value, traceback) |
|||
finally: |
|||
_state.contexts = self.old_contexts |
|||
|
|||
class ExceptionStackContext(object): |
|||
'''Specialization of StackContext for exception handling. |
|||
|
|||
The supplied exception_handler function will be called in the |
|||
event of an uncaught exception in this context. The semantics are |
|||
similar to a try/finally clause, and intended use cases are to log |
|||
an error, close a socket, or similar cleanup actions. The |
|||
exc_info triple (type, value, traceback) will be passed to the |
|||
exception_handler function. |
|||
|
|||
If the exception handler returns true, the exception will be |
|||
consumed and will not be propagated to other exception handlers. |
|||
''' |
|||
def __init__(self, exception_handler): |
|||
self.exception_handler = exception_handler |
|||
|
|||
def __enter__(self): |
|||
self.old_contexts = _state.contexts |
|||
_state.contexts = (self.old_contexts + |
|||
((ExceptionStackContext, self.exception_handler),)) |
|||
|
|||
def __exit__(self, type, value, traceback): |
|||
try: |
|||
if type is not None: |
|||
return self.exception_handler(type, value, traceback) |
|||
finally: |
|||
_state.contexts = self.old_contexts |
|||
|
|||
class NullContext(object): |
|||
'''Resets the StackContext. |
|||
|
|||
Useful when creating a shared resource on demand (e.g. an AsyncHTTPClient) |
|||
where the stack that caused the creating is not relevant to future |
|||
operations. |
|||
''' |
|||
def __enter__(self): |
|||
self.old_contexts = _state.contexts |
|||
_state.contexts = () |
|||
|
|||
def __exit__(self, type, value, traceback): |
|||
_state.contexts = self.old_contexts |
|||
|
|||
class _StackContextWrapper(functools.partial): |
|||
pass |
|||
|
|||
def wrap(fn): |
|||
'''Returns a callable object that will restore the current StackContext |
|||
when executed. |
|||
|
|||
Use this whenever saving a callback to be executed later in a |
|||
different execution context (either in a different thread or |
|||
asynchronously in the same thread). |
|||
''' |
|||
if fn is None or fn.__class__ is _StackContextWrapper: |
|||
return fn |
|||
# functools.wraps doesn't appear to work on functools.partial objects |
|||
#@functools.wraps(fn) |
|||
def wrapped(callback, contexts, *args, **kwargs): |
|||
if contexts is _state.contexts or not contexts: |
|||
callback(*args, **kwargs) |
|||
return |
|||
if not _state.contexts: |
|||
new_contexts = [cls(arg) for (cls, arg) in contexts] |
|||
# If we're moving down the stack, _state.contexts is a prefix |
|||
# of contexts. For each element of contexts not in that prefix, |
|||
# create a new StackContext object. |
|||
# If we're moving up the stack (or to an entirely different stack), |
|||
# _state.contexts will have elements not in contexts. Use |
|||
# NullContext to clear the state and then recreate from contexts. |
|||
elif (len(_state.contexts) > len(contexts) or |
|||
any(a[1] is not b[1] |
|||
for a, b in itertools.izip(_state.contexts, contexts))): |
|||
# contexts have been removed or changed, so start over |
|||
new_contexts = ([NullContext()] + |
|||
[cls(arg) for (cls,arg) in contexts]) |
|||
else: |
|||
new_contexts = [cls(arg) |
|||
for (cls, arg) in contexts[len(_state.contexts):]] |
|||
if len(new_contexts) > 1: |
|||
with _nested(*new_contexts): |
|||
callback(*args, **kwargs) |
|||
elif new_contexts: |
|||
with new_contexts[0]: |
|||
callback(*args, **kwargs) |
|||
else: |
|||
callback(*args, **kwargs) |
|||
if _state.contexts: |
|||
return _StackContextWrapper(wrapped, fn, _state.contexts) |
|||
else: |
|||
return _StackContextWrapper(fn) |
|||
|
|||
@contextlib.contextmanager |
|||
def _nested(*managers): |
|||
"""Support multiple context managers in a single with-statement. |
|||
|
|||
Copied from the python 2.6 standard library. It's no longer present |
|||
in python 3 because the with statement natively supports multiple |
|||
context managers, but that doesn't help if the list of context |
|||
managers is not known until runtime. |
|||
""" |
|||
exits = [] |
|||
vars = [] |
|||
exc = (None, None, None) |
|||
try: |
|||
for mgr in managers: |
|||
exit = mgr.__exit__ |
|||
enter = mgr.__enter__ |
|||
vars.append(enter()) |
|||
exits.append(exit) |
|||
yield vars |
|||
except: |
|||
exc = sys.exc_info() |
|||
finally: |
|||
while exits: |
|||
exit = exits.pop() |
|||
try: |
|||
if exit(*exc): |
|||
exc = (None, None, None) |
|||
except: |
|||
exc = sys.exc_info() |
|||
if exc != (None, None, None): |
|||
# Don't rely on sys.exc_info() still containing |
|||
# the right information. Another exception may |
|||
# have been raised and caught by an exit method |
|||
raise exc[0], exc[1], exc[2] |
|||
|
|||
@ -0,0 +1,826 @@ |
|||
#!/usr/bin/env python |
|||
# |
|||
# Copyright 2009 Facebook |
|||
# |
|||
# Licensed under the Apache License, Version 2.0 (the "License"); you may |
|||
# not use this file except in compliance with the License. You may obtain |
|||
# a copy of the License at |
|||
# |
|||
# http://www.apache.org/licenses/LICENSE-2.0 |
|||
# |
|||
# Unless required by applicable law or agreed to in writing, software |
|||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT |
|||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the |
|||
# License for the specific language governing permissions and limitations |
|||
# under the License. |
|||
|
|||
"""A simple template system that compiles templates to Python code. |
|||
|
|||
Basic usage looks like:: |
|||
|
|||
t = template.Template("<html>{{ myvalue }}</html>") |
|||
print t.generate(myvalue="XXX") |
|||
|
|||
Loader is a class that loads templates from a root directory and caches |
|||
the compiled templates:: |
|||
|
|||
loader = template.Loader("/home/btaylor") |
|||
print loader.load("test.html").generate(myvalue="XXX") |
|||
|
|||
We compile all templates to raw Python. Error-reporting is currently... uh, |
|||
interesting. Syntax for the templates:: |
|||
|
|||
### base.html |
|||
<html> |
|||
<head> |
|||
<title>{% block title %}Default title{% end %}</title> |
|||
</head> |
|||
<body> |
|||
<ul> |
|||
{% for student in students %} |
|||
{% block student %} |
|||
<li>{{ escape(student.name) }}</li> |
|||
{% end %} |
|||
{% end %} |
|||
</ul> |
|||
</body> |
|||
</html> |
|||
|
|||
### bold.html |
|||
{% extends "base.html" %} |
|||
|
|||
{% block title %}A bolder title{% end %} |
|||
|
|||
{% block student %} |
|||
<li><span style="bold">{{ escape(student.name) }}</span></li> |
|||
{% end %} |
|||
|
|||
Unlike most other template systems, we do not put any restrictions on the |
|||
expressions you can include in your statements. if and for blocks get |
|||
translated exactly into Python, you can do complex expressions like:: |
|||
|
|||
{% for student in [p for p in people if p.student and p.age > 23] %} |
|||
<li>{{ escape(student.name) }}</li> |
|||
{% end %} |
|||
|
|||
Translating directly to Python means you can apply functions to expressions |
|||
easily, like the escape() function in the examples above. You can pass |
|||
functions in to your template just like any other variable:: |
|||
|
|||
### Python code |
|||
def add(x, y): |
|||
return x + y |
|||
template.execute(add=add) |
|||
|
|||
### The template |
|||
{{ add(1, 2) }} |
|||
|
|||
We provide the functions escape(), url_escape(), json_encode(), and squeeze() |
|||
to all templates by default. |
|||
|
|||
Typical applications do not create `Template` or `Loader` instances by |
|||
hand, but instead use the `render` and `render_string` methods of |
|||
`tornado.web.RequestHandler`, which load templates automatically based |
|||
on the ``template_path`` `Application` setting. |
|||
|
|||
Syntax Reference |
|||
---------------- |
|||
|
|||
Template expressions are surrounded by double curly braces: ``{{ ... }}``. |
|||
The contents may be any python expression, which will be escaped according |
|||
to the current autoescape setting and inserted into the output. Other |
|||
template directives use ``{% %}``. These tags may be escaped as ``{{!`` |
|||
and ``{%!`` if you need to include a literal ``{{`` or ``{%`` in the output. |
|||
|
|||
To comment out a section so that it is omitted from the output, surround it |
|||
with ``{# ... #}``. |
|||
|
|||
``{% apply *function* %}...{% end %}`` |
|||
Applies a function to the output of all template code between ``apply`` |
|||
and ``end``:: |
|||
|
|||
{% apply linkify %}{{name}} said: {{message}}{% end %} |
|||
|
|||
``{% autoescape *function* %}`` |
|||
Sets the autoescape mode for the current file. This does not affect |
|||
other files, even those referenced by ``{% include %}``. Note that |
|||
autoescaping can also be configured globally, at the `Application` |
|||
or `Loader`.:: |
|||
|
|||
{% autoescape xhtml_escape %} |
|||
{% autoescape None %} |
|||
|
|||
``{% block *name* %}...{% end %}`` |
|||
Indicates a named, replaceable block for use with ``{% extends %}``. |
|||
Blocks in the parent template will be replaced with the contents of |
|||
the same-named block in a child template.:: |
|||
|
|||
<!-- base.html --> |
|||
<title>{% block title %}Default title{% end %}</title> |
|||
|
|||
<!-- mypage.html --> |
|||
{% extends "base.html" %} |
|||
{% block title %}My page title{% end %} |
|||
|
|||
``{% comment ... %}`` |
|||
A comment which will be removed from the template output. Note that |
|||
there is no ``{% end %}`` tag; the comment goes from the word ``comment`` |
|||
to the closing ``%}`` tag. |
|||
|
|||
``{% extends *filename* %}`` |
|||
Inherit from another template. Templates that use ``extends`` should |
|||
contain one or more ``block`` tags to replace content from the parent |
|||
template. Anything in the child template not contained in a ``block`` |
|||
tag will be ignored. For an example, see the ``{% block %}`` tag. |
|||
|
|||
``{% for *var* in *expr* %}...{% end %}`` |
|||
Same as the python ``for`` statement. |
|||
|
|||
``{% from *x* import *y* %}`` |
|||
Same as the python ``import`` statement. |
|||
|
|||
``{% if *condition* %}...{% elif *condition* %}...{% else %}...{% end %}`` |
|||
Conditional statement - outputs the first section whose condition is |
|||
true. (The ``elif`` and ``else`` sections are optional) |
|||
|
|||
``{% import *module* %}`` |
|||
Same as the python ``import`` statement. |
|||
|
|||
``{% include *filename* %}`` |
|||
Includes another template file. The included file can see all the local |
|||
variables as if it were copied directly to the point of the ``include`` |
|||
directive (the ``{% autoescape %}`` directive is an exception). |
|||
Alternately, ``{% module Template(filename, **kwargs) %}`` may be used |
|||
to include another template with an isolated namespace. |
|||
|
|||
``{% module *expr* %}`` |
|||
Renders a `~tornado.web.UIModule`. The output of the ``UIModule`` is |
|||
not escaped:: |
|||
|
|||
{% module Template("foo.html", arg=42) %} |
|||
|
|||
``{% raw *expr* %}`` |
|||
Outputs the result of the given expression without autoescaping. |
|||
|
|||
``{% set *x* = *y* %}`` |
|||
Sets a local variable. |
|||
|
|||
``{% try %}...{% except %}...{% finally %}...{% end %}`` |
|||
Same as the python ``try`` statement. |
|||
|
|||
``{% while *condition* %}... {% end %}`` |
|||
Same as the python ``while`` statement. |
|||
""" |
|||
|
|||
from __future__ import with_statement |
|||
|
|||
import cStringIO |
|||
import datetime |
|||
import linecache |
|||
import logging |
|||
import os.path |
|||
import posixpath |
|||
import re |
|||
import threading |
|||
|
|||
from tornado import escape |
|||
from tornado.util import bytes_type, ObjectDict |
|||
|
|||
_DEFAULT_AUTOESCAPE = "xhtml_escape" |
|||
_UNSET = object() |
|||
|
|||
class Template(object): |
|||
"""A compiled template. |
|||
|
|||
We compile into Python from the given template_string. You can generate |
|||
the template from variables with generate(). |
|||
""" |
|||
def __init__(self, template_string, name="<string>", loader=None, |
|||
compress_whitespace=None, autoescape=_UNSET): |
|||
self.name = name |
|||
if compress_whitespace is None: |
|||
compress_whitespace = name.endswith(".html") or \ |
|||
name.endswith(".js") |
|||
if autoescape is not _UNSET: |
|||
self.autoescape = autoescape |
|||
elif loader: |
|||
self.autoescape = loader.autoescape |
|||
else: |
|||
self.autoescape = _DEFAULT_AUTOESCAPE |
|||
self.namespace = loader.namespace if loader else {} |
|||
reader = _TemplateReader(name, escape.native_str(template_string)) |
|||
self.file = _File(self, _parse(reader, self)) |
|||
self.code = self._generate_python(loader, compress_whitespace) |
|||
self.loader = loader |
|||
try: |
|||
# Under python2.5, the fake filename used here must match |
|||
# the module name used in __name__ below. |
|||
self.compiled = compile( |
|||
escape.to_unicode(self.code), |
|||
"%s.generated.py" % self.name.replace('.','_'), |
|||
"exec") |
|||
except Exception: |
|||
formatted_code = _format_code(self.code).rstrip() |
|||
logging.error("%s code:\n%s", self.name, formatted_code) |
|||
raise |
|||
|
|||
def generate(self, **kwargs): |
|||
"""Generate this template with the given arguments.""" |
|||
namespace = { |
|||
"escape": escape.xhtml_escape, |
|||
"xhtml_escape": escape.xhtml_escape, |
|||
"url_escape": escape.url_escape, |
|||
"json_encode": escape.json_encode, |
|||
"squeeze": escape.squeeze, |
|||
"linkify": escape.linkify, |
|||
"datetime": datetime, |
|||
"_utf8": escape.utf8, # for internal use |
|||
"_string_types": (unicode, bytes_type), |
|||
# __name__ and __loader__ allow the traceback mechanism to find |
|||
# the generated source code. |
|||
"__name__": self.name.replace('.', '_'), |
|||
"__loader__": ObjectDict(get_source=lambda name: self.code), |
|||
} |
|||
namespace.update(self.namespace) |
|||
namespace.update(kwargs) |
|||
exec self.compiled in namespace |
|||
execute = namespace["_execute"] |
|||
# Clear the traceback module's cache of source data now that |
|||
# we've generated a new template (mainly for this module's |
|||
# unittests, where different tests reuse the same name). |
|||
linecache.clearcache() |
|||
try: |
|||
return execute() |
|||
except Exception: |
|||
formatted_code = _format_code(self.code).rstrip() |
|||
logging.error("%s code:\n%s", self.name, formatted_code) |
|||
raise |
|||
|
|||
def _generate_python(self, loader, compress_whitespace): |
|||
buffer = cStringIO.StringIO() |
|||
try: |
|||
# named_blocks maps from names to _NamedBlock objects |
|||
named_blocks = {} |
|||
ancestors = self._get_ancestors(loader) |
|||
ancestors.reverse() |
|||
for ancestor in ancestors: |
|||
ancestor.find_named_blocks(loader, named_blocks) |
|||
self.file.find_named_blocks(loader, named_blocks) |
|||
writer = _CodeWriter(buffer, named_blocks, loader, ancestors[0].template, |
|||
compress_whitespace) |
|||
ancestors[0].generate(writer) |
|||
return buffer.getvalue() |
|||
finally: |
|||
buffer.close() |
|||
|
|||
def _get_ancestors(self, loader): |
|||
ancestors = [self.file] |
|||
for chunk in self.file.body.chunks: |
|||
if isinstance(chunk, _ExtendsBlock): |
|||
if not loader: |
|||
raise ParseError("{% extends %} block found, but no " |
|||
"template loader") |
|||
template = loader.load(chunk.name, self.name) |
|||
ancestors.extend(template._get_ancestors(loader)) |
|||
return ancestors |
|||
|
|||
|
|||
class BaseLoader(object): |
|||
"""Base class for template loaders.""" |
|||
def __init__(self, autoescape=_DEFAULT_AUTOESCAPE, namespace=None): |
|||
"""Creates a template loader. |
|||
|
|||
root_directory may be the empty string if this loader does not |
|||
use the filesystem. |
|||
|
|||
autoescape must be either None or a string naming a function |
|||
in the template namespace, such as "xhtml_escape". |
|||
""" |
|||
self.autoescape = autoescape |
|||
self.namespace = namespace or {} |
|||
self.templates = {} |
|||
# self.lock protects self.templates. It's a reentrant lock |
|||
# because templates may load other templates via `include` or |
|||
# `extends`. Note that thanks to the GIL this code would be safe |
|||
# even without the lock, but could lead to wasted work as multiple |
|||
# threads tried to compile the same template simultaneously. |
|||
self.lock = threading.RLock() |
|||
|
|||
def reset(self): |
|||
"""Resets the cache of compiled templates.""" |
|||
with self.lock: |
|||
self.templates = {} |
|||
|
|||
def resolve_path(self, name, parent_path=None): |
|||
"""Converts a possibly-relative path to absolute (used internally).""" |
|||
raise NotImplementedError() |
|||
|
|||
def load(self, name, parent_path=None): |
|||
"""Loads a template.""" |
|||
name = self.resolve_path(name, parent_path=parent_path) |
|||
with self.lock: |
|||
if name not in self.templates: |
|||
self.templates[name] = self._create_template(name) |
|||
return self.templates[name] |
|||
|
|||
def _create_template(self, name): |
|||
raise NotImplementedError() |
|||
|
|||
class Loader(BaseLoader): |
|||
"""A template loader that loads from a single root directory. |
|||
|
|||
You must use a template loader to use template constructs like |
|||
{% extends %} and {% include %}. Loader caches all templates after |
|||
they are loaded the first time. |
|||
""" |
|||
def __init__(self, root_directory, **kwargs): |
|||
super(Loader, self).__init__(**kwargs) |
|||
self.root = os.path.abspath(root_directory) |
|||
|
|||
def resolve_path(self, name, parent_path=None): |
|||
if parent_path and not parent_path.startswith("<") and \ |
|||
not parent_path.startswith("/") and \ |
|||
not name.startswith("/"): |
|||
current_path = os.path.join(self.root, parent_path) |
|||
file_dir = os.path.dirname(os.path.abspath(current_path)) |
|||
relative_path = os.path.abspath(os.path.join(file_dir, name)) |
|||
if relative_path.startswith(self.root): |
|||
name = relative_path[len(self.root) + 1:] |
|||
return name |
|||
|
|||
def _create_template(self, name): |
|||
path = os.path.join(self.root, name) |
|||
f = open(path, "r") |
|||
template = Template(f.read(), name=name, loader=self) |
|||
f.close() |
|||
return template |
|||
|
|||
|
|||
class DictLoader(BaseLoader): |
|||
"""A template loader that loads from a dictionary.""" |
|||
def __init__(self, dict, **kwargs): |
|||
super(DictLoader, self).__init__(**kwargs) |
|||
self.dict = dict |
|||
|
|||
def resolve_path(self, name, parent_path=None): |
|||
if parent_path and not parent_path.startswith("<") and \ |
|||
not parent_path.startswith("/") and \ |
|||
not name.startswith("/"): |
|||
file_dir = posixpath.dirname(parent_path) |
|||
name = posixpath.normpath(posixpath.join(file_dir, name)) |
|||
return name |
|||
|
|||
def _create_template(self, name): |
|||
return Template(self.dict[name], name=name, loader=self) |
|||
|
|||
|
|||
class _Node(object): |
|||
def each_child(self): |
|||
return () |
|||
|
|||
def generate(self, writer): |
|||
raise NotImplementedError() |
|||
|
|||
def find_named_blocks(self, loader, named_blocks): |
|||
for child in self.each_child(): |
|||
child.find_named_blocks(loader, named_blocks) |
|||
|
|||
|
|||
class _File(_Node): |
|||
def __init__(self, template, body): |
|||
self.template = template |
|||
self.body = body |
|||
self.line = 0 |
|||
|
|||
def generate(self, writer): |
|||
writer.write_line("def _execute():", self.line) |
|||
with writer.indent(): |
|||
writer.write_line("_buffer = []", self.line) |
|||
writer.write_line("_append = _buffer.append", self.line) |
|||
self.body.generate(writer) |
|||
writer.write_line("return _utf8('').join(_buffer)", self.line) |
|||
|
|||
def each_child(self): |
|||
return (self.body,) |
|||
|
|||
|
|||
|
|||
class _ChunkList(_Node): |
|||
def __init__(self, chunks): |
|||
self.chunks = chunks |
|||
|
|||
def generate(self, writer): |
|||
for chunk in self.chunks: |
|||
chunk.generate(writer) |
|||
|
|||
def each_child(self): |
|||
return self.chunks |
|||
|
|||
|
|||
class _NamedBlock(_Node): |
|||
def __init__(self, name, body, template, line): |
|||
self.name = name |
|||
self.body = body |
|||
self.template = template |
|||
self.line = line |
|||
|
|||
def each_child(self): |
|||
return (self.body,) |
|||
|
|||
def generate(self, writer): |
|||
block = writer.named_blocks[self.name] |
|||
with writer.include(block.template, self.line): |
|||
block.body.generate(writer) |
|||
|
|||
def find_named_blocks(self, loader, named_blocks): |
|||
named_blocks[self.name] = self |
|||
_Node.find_named_blocks(self, loader, named_blocks) |
|||
|
|||
|
|||
class _ExtendsBlock(_Node): |
|||
def __init__(self, name): |
|||
self.name = name |
|||
|
|||
|
|||
class _IncludeBlock(_Node): |
|||
def __init__(self, name, reader, line): |
|||
self.name = name |
|||
self.template_name = reader.name |
|||
self.line = line |
|||
|
|||
def find_named_blocks(self, loader, named_blocks): |
|||
included = loader.load(self.name, self.template_name) |
|||
included.file.find_named_blocks(loader, named_blocks) |
|||
|
|||
def generate(self, writer): |
|||
included = writer.loader.load(self.name, self.template_name) |
|||
with writer.include(included, self.line): |
|||
included.file.body.generate(writer) |
|||
|
|||
|
|||
class _ApplyBlock(_Node): |
|||
def __init__(self, method, line, body=None): |
|||
self.method = method |
|||
self.line = line |
|||
self.body = body |
|||
|
|||
def each_child(self): |
|||
return (self.body,) |
|||
|
|||
def generate(self, writer): |
|||
method_name = "apply%d" % writer.apply_counter |
|||
writer.apply_counter += 1 |
|||
writer.write_line("def %s():" % method_name, self.line) |
|||
with writer.indent(): |
|||
writer.write_line("_buffer = []", self.line) |
|||
writer.write_line("_append = _buffer.append", self.line) |
|||
self.body.generate(writer) |
|||
writer.write_line("return _utf8('').join(_buffer)", self.line) |
|||
writer.write_line("_append(%s(%s()))" % ( |
|||
self.method, method_name), self.line) |
|||
|
|||
|
|||
class _ControlBlock(_Node): |
|||
def __init__(self, statement, line, body=None): |
|||
self.statement = statement |
|||
self.line = line |
|||
self.body = body |
|||
|
|||
def each_child(self): |
|||
return (self.body,) |
|||
|
|||
def generate(self, writer): |
|||
writer.write_line("%s:" % self.statement, self.line) |
|||
with writer.indent(): |
|||
self.body.generate(writer) |
|||
|
|||
|
|||
class _IntermediateControlBlock(_Node): |
|||
def __init__(self, statement, line): |
|||
self.statement = statement |
|||
self.line = line |
|||
|
|||
def generate(self, writer): |
|||
writer.write_line("%s:" % self.statement, self.line, writer.indent_size() - 1) |
|||
|
|||
|
|||
class _Statement(_Node): |
|||
def __init__(self, statement, line): |
|||
self.statement = statement |
|||
self.line = line |
|||
|
|||
def generate(self, writer): |
|||
writer.write_line(self.statement, self.line) |
|||
|
|||
|
|||
class _Expression(_Node): |
|||
def __init__(self, expression, line, raw=False): |
|||
self.expression = expression |
|||
self.line = line |
|||
self.raw = raw |
|||
|
|||
def generate(self, writer): |
|||
writer.write_line("_tmp = %s" % self.expression, self.line) |
|||
writer.write_line("if isinstance(_tmp, _string_types):" |
|||
" _tmp = _utf8(_tmp)", self.line) |
|||
writer.write_line("else: _tmp = _utf8(str(_tmp))", self.line) |
|||
if not self.raw and writer.current_template.autoescape is not None: |
|||
# In python3 functions like xhtml_escape return unicode, |
|||
# so we have to convert to utf8 again. |
|||
writer.write_line("_tmp = _utf8(%s(_tmp))" % |
|||
writer.current_template.autoescape, self.line) |
|||
writer.write_line("_append(_tmp)", self.line) |
|||
|
|||
class _Module(_Expression): |
|||
def __init__(self, expression, line): |
|||
super(_Module, self).__init__("_modules." + expression, line, |
|||
raw=True) |
|||
|
|||
class _Text(_Node): |
|||
def __init__(self, value, line): |
|||
self.value = value |
|||
self.line = line |
|||
|
|||
def generate(self, writer): |
|||
value = self.value |
|||
|
|||
# Compress lots of white space to a single character. If the whitespace |
|||
# breaks a line, have it continue to break a line, but just with a |
|||
# single \n character |
|||
if writer.compress_whitespace and "<pre>" not in value: |
|||
value = re.sub(r"([\t ]+)", " ", value) |
|||
value = re.sub(r"(\s*\n\s*)", "\n", value) |
|||
|
|||
if value: |
|||
writer.write_line('_append(%r)' % escape.utf8(value), self.line) |
|||
|
|||
|
|||
class ParseError(Exception): |
|||
"""Raised for template syntax errors.""" |
|||
pass |
|||
|
|||
|
|||
class _CodeWriter(object): |
|||
def __init__(self, file, named_blocks, loader, current_template, |
|||
compress_whitespace): |
|||
self.file = file |
|||
self.named_blocks = named_blocks |
|||
self.loader = loader |
|||
self.current_template = current_template |
|||
self.compress_whitespace = compress_whitespace |
|||
self.apply_counter = 0 |
|||
self.include_stack = [] |
|||
self._indent = 0 |
|||
|
|||
def indent_size(self): |
|||
return self._indent |
|||
|
|||
def indent(self): |
|||
class Indenter(object): |
|||
def __enter__(_): |
|||
self._indent += 1 |
|||
return self |
|||
|
|||
def __exit__(_, *args): |
|||
assert self._indent > 0 |
|||
self._indent -= 1 |
|||
|
|||
return Indenter() |
|||
|
|||
def include(self, template, line): |
|||
self.include_stack.append((self.current_template, line)) |
|||
self.current_template = template |
|||
|
|||
class IncludeTemplate(object): |
|||
def __enter__(_): |
|||
return self |
|||
|
|||
def __exit__(_, *args): |
|||
self.current_template = self.include_stack.pop()[0] |
|||
|
|||
return IncludeTemplate() |
|||
|
|||
def write_line(self, line, line_number, indent=None): |
|||
if indent == None: |
|||
indent = self._indent |
|||
line_comment = ' # %s:%d' % (self.current_template.name, line_number) |
|||
if self.include_stack: |
|||
ancestors = ["%s:%d" % (tmpl.name, lineno) |
|||
for (tmpl, lineno) in self.include_stack] |
|||
line_comment += ' (via %s)' % ', '.join(reversed(ancestors)) |
|||
print >> self.file, " "*indent + line + line_comment |
|||
|
|||
|
|||
class _TemplateReader(object): |
|||
def __init__(self, name, text): |
|||
self.name = name |
|||
self.text = text |
|||
self.line = 1 |
|||
self.pos = 0 |
|||
|
|||
def find(self, needle, start=0, end=None): |
|||
assert start >= 0, start |
|||
pos = self.pos |
|||
start += pos |
|||
if end is None: |
|||
index = self.text.find(needle, start) |
|||
else: |
|||
end += pos |
|||
assert end >= start |
|||
index = self.text.find(needle, start, end) |
|||
if index != -1: |
|||
index -= pos |
|||
return index |
|||
|
|||
def consume(self, count=None): |
|||
if count is None: |
|||
count = len(self.text) - self.pos |
|||
newpos = self.pos + count |
|||
self.line += self.text.count("\n", self.pos, newpos) |
|||
s = self.text[self.pos:newpos] |
|||
self.pos = newpos |
|||
return s |
|||
|
|||
def remaining(self): |
|||
return len(self.text) - self.pos |
|||
|
|||
def __len__(self): |
|||
return self.remaining() |
|||
|
|||
def __getitem__(self, key): |
|||
if type(key) is slice: |
|||
size = len(self) |
|||
start, stop, step = key.indices(size) |
|||
if start is None: start = self.pos |
|||
else: start += self.pos |
|||
if stop is not None: stop += self.pos |
|||
return self.text[slice(start, stop, step)] |
|||
elif key < 0: |
|||
return self.text[key] |
|||
else: |
|||
return self.text[self.pos + key] |
|||
|
|||
def __str__(self): |
|||
return self.text[self.pos:] |
|||
|
|||
|
|||
def _format_code(code): |
|||
lines = code.splitlines() |
|||
format = "%%%dd %%s\n" % len(repr(len(lines) + 1)) |
|||
return "".join([format % (i + 1, line) for (i, line) in enumerate(lines)]) |
|||
|
|||
|
|||
def _parse(reader, template, in_block=None): |
|||
body = _ChunkList([]) |
|||
while True: |
|||
# Find next template directive |
|||
curly = 0 |
|||
while True: |
|||
curly = reader.find("{", curly) |
|||
if curly == -1 or curly + 1 == reader.remaining(): |
|||
# EOF |
|||
if in_block: |
|||
raise ParseError("Missing {%% end %%} block for %s" % |
|||
in_block) |
|||
body.chunks.append(_Text(reader.consume(), reader.line)) |
|||
return body |
|||
# If the first curly brace is not the start of a special token, |
|||
# start searching from the character after it |
|||
if reader[curly + 1] not in ("{", "%", "#"): |
|||
curly += 1 |
|||
continue |
|||
# When there are more than 2 curlies in a row, use the |
|||
# innermost ones. This is useful when generating languages |
|||
# like latex where curlies are also meaningful |
|||
if (curly + 2 < reader.remaining() and |
|||
reader[curly + 1] == '{' and reader[curly + 2] == '{'): |
|||
curly += 1 |
|||
continue |
|||
break |
|||
|
|||
# Append any text before the special token |
|||
if curly > 0: |
|||
cons = reader.consume(curly) |
|||
body.chunks.append(_Text(cons, reader.line)) |
|||
|
|||
start_brace = reader.consume(2) |
|||
line = reader.line |
|||
|
|||
# Template directives may be escaped as "{{!" or "{%!". |
|||
# In this case output the braces and consume the "!". |
|||
# This is especially useful in conjunction with jquery templates, |
|||
# which also use double braces. |
|||
if reader.remaining() and reader[0] == "!": |
|||
reader.consume(1) |
|||
body.chunks.append(_Text(start_brace, line)) |
|||
continue |
|||
|
|||
# Comment |
|||
if start_brace == "{#": |
|||
end = reader.find("#}") |
|||
if end == -1: |
|||
raise ParseError("Missing end expression #} on line %d" % line) |
|||
contents = reader.consume(end).strip() |
|||
reader.consume(2) |
|||
continue |
|||
|
|||
# Expression |
|||
if start_brace == "{{": |
|||
end = reader.find("}}") |
|||
if end == -1: |
|||
raise ParseError("Missing end expression }} on line %d" % line) |
|||
contents = reader.consume(end).strip() |
|||
reader.consume(2) |
|||
if not contents: |
|||
raise ParseError("Empty expression on line %d" % line) |
|||
body.chunks.append(_Expression(contents, line)) |
|||
continue |
|||
|
|||
# Block |
|||
assert start_brace == "{%", start_brace |
|||
end = reader.find("%}") |
|||
if end == -1: |
|||
raise ParseError("Missing end block %%} on line %d" % line) |
|||
contents = reader.consume(end).strip() |
|||
reader.consume(2) |
|||
if not contents: |
|||
raise ParseError("Empty block tag ({%% %%}) on line %d" % line) |
|||
|
|||
operator, space, suffix = contents.partition(" ") |
|||
suffix = suffix.strip() |
|||
|
|||
# Intermediate ("else", "elif", etc) blocks |
|||
intermediate_blocks = { |
|||
"else": set(["if", "for", "while"]), |
|||
"elif": set(["if"]), |
|||
"except": set(["try"]), |
|||
"finally": set(["try"]), |
|||
} |
|||
allowed_parents = intermediate_blocks.get(operator) |
|||
if allowed_parents is not None: |
|||
if not in_block: |
|||
raise ParseError("%s outside %s block" % |
|||
(operator, allowed_parents)) |
|||
if in_block not in allowed_parents: |
|||
raise ParseError("%s block cannot be attached to %s block" % (operator, in_block)) |
|||
body.chunks.append(_IntermediateControlBlock(contents, line)) |
|||
continue |
|||
|
|||
# End tag |
|||
elif operator == "end": |
|||
if not in_block: |
|||
raise ParseError("Extra {%% end %%} block on line %d" % line) |
|||
return body |
|||
|
|||
elif operator in ("extends", "include", "set", "import", "from", |
|||
"comment", "autoescape", "raw", "module"): |
|||
if operator == "comment": |
|||
continue |
|||
if operator == "extends": |
|||
suffix = suffix.strip('"').strip("'") |
|||
if not suffix: |
|||
raise ParseError("extends missing file path on line %d" % line) |
|||
block = _ExtendsBlock(suffix) |
|||
elif operator in ("import", "from"): |
|||
if not suffix: |
|||
raise ParseError("import missing statement on line %d" % line) |
|||
block = _Statement(contents, line) |
|||
elif operator == "include": |
|||
suffix = suffix.strip('"').strip("'") |
|||
if not suffix: |
|||
raise ParseError("include missing file path on line %d" % line) |
|||
block = _IncludeBlock(suffix, reader, line) |
|||
elif operator == "set": |
|||
if not suffix: |
|||
raise ParseError("set missing statement on line %d" % line) |
|||
block = _Statement(suffix, line) |
|||
elif operator == "autoescape": |
|||
fn = suffix.strip() |
|||
if fn == "None": fn = None |
|||
template.autoescape = fn |
|||
continue |
|||
elif operator == "raw": |
|||
block = _Expression(suffix, line, raw=True) |
|||
elif operator == "module": |
|||
block = _Module(suffix, line) |
|||
body.chunks.append(block) |
|||
continue |
|||
|
|||
elif operator in ("apply", "block", "try", "if", "for", "while"): |
|||
# parse inner body recursively |
|||
block_body = _parse(reader, template, operator) |
|||
if operator == "apply": |
|||
if not suffix: |
|||
raise ParseError("apply missing method name on line %d" % line) |
|||
block = _ApplyBlock(suffix, line, block_body) |
|||
elif operator == "block": |
|||
if not suffix: |
|||
raise ParseError("block missing name on line %d" % line) |
|||
block = _NamedBlock(suffix, block_body, template, line) |
|||
else: |
|||
block = _ControlBlock(contents, line, block_body) |
|||
body.chunks.append(block) |
|||
continue |
|||
|
|||
else: |
|||
raise ParseError("unknown operator: %r" % operator) |
|||
@ -0,0 +1,382 @@ |
|||
#!/usr/bin/env python |
|||
"""Support classes for automated testing. |
|||
|
|||
This module contains three parts: |
|||
|
|||
* `AsyncTestCase`/`AsyncHTTPTestCase`: Subclasses of unittest.TestCase |
|||
with additional support for testing asynchronous (IOLoop-based) code. |
|||
|
|||
* `LogTrapTestCase`: Subclass of unittest.TestCase that discards log output |
|||
from tests that pass and only produces output for failing tests. |
|||
|
|||
* `main()`: A simple test runner (wrapper around unittest.main()) with support |
|||
for the tornado.autoreload module to rerun the tests when code changes. |
|||
|
|||
These components may be used together or independently. In particular, |
|||
it is safe to combine AsyncTestCase and LogTrapTestCase via multiple |
|||
inheritance. See the docstrings for each class/function below for more |
|||
information. |
|||
""" |
|||
|
|||
from __future__ import with_statement |
|||
|
|||
from cStringIO import StringIO |
|||
try: |
|||
from tornado.httpclient import AsyncHTTPClient |
|||
from tornado.httpserver import HTTPServer |
|||
from tornado.ioloop import IOLoop |
|||
except ImportError: |
|||
# These modules are not importable on app engine. Parts of this module |
|||
# won't work, but e.g. LogTrapTestCase and main() will. |
|||
AsyncHTTPClient = None |
|||
HTTPServer = None |
|||
IOLoop = None |
|||
from tornado.stack_context import StackContext, NullContext |
|||
import contextlib |
|||
import logging |
|||
import signal |
|||
import sys |
|||
import time |
|||
import unittest |
|||
|
|||
_next_port = 10000 |
|||
def get_unused_port(): |
|||
"""Returns a (hopefully) unused port number.""" |
|||
global _next_port |
|||
port = _next_port |
|||
_next_port = _next_port + 1 |
|||
return port |
|||
|
|||
class AsyncTestCase(unittest.TestCase): |
|||
"""TestCase subclass for testing IOLoop-based asynchronous code. |
|||
|
|||
The unittest framework is synchronous, so the test must be complete |
|||
by the time the test method returns. This method provides the stop() |
|||
and wait() methods for this purpose. The test method itself must call |
|||
self.wait(), and asynchronous callbacks should call self.stop() to signal |
|||
completion. |
|||
|
|||
By default, a new IOLoop is constructed for each test and is available |
|||
as self.io_loop. This IOLoop should be used in the construction of |
|||
HTTP clients/servers, etc. If the code being tested requires a |
|||
global IOLoop, subclasses should override get_new_ioloop to return it. |
|||
|
|||
The IOLoop's start and stop methods should not be called directly. |
|||
Instead, use self.stop self.wait. Arguments passed to self.stop are |
|||
returned from self.wait. It is possible to have multiple |
|||
wait/stop cycles in the same test. |
|||
|
|||
Example:: |
|||
|
|||
# This test uses an asynchronous style similar to most async |
|||
# application code. |
|||
class MyTestCase(AsyncTestCase): |
|||
def test_http_fetch(self): |
|||
client = AsyncHTTPClient(self.io_loop) |
|||
client.fetch("http://www.tornadoweb.org/", self.handle_fetch) |
|||
self.wait() |
|||
|
|||
def handle_fetch(self, response): |
|||
# Test contents of response (failures and exceptions here |
|||
# will cause self.wait() to throw an exception and end the |
|||
# test). |
|||
# Exceptions thrown here are magically propagated to |
|||
# self.wait() in test_http_fetch() via stack_context. |
|||
self.assertIn("FriendFeed", response.body) |
|||
self.stop() |
|||
|
|||
# This test uses the argument passing between self.stop and self.wait |
|||
# for a simpler, more synchronous style. |
|||
# This style is recommended over the preceding example because it |
|||
# keeps the assertions in the test method itself, and is therefore |
|||
# less sensitive to the subtleties of stack_context. |
|||
class MyTestCase2(AsyncTestCase): |
|||
def test_http_fetch(self): |
|||
client = AsyncHTTPClient(self.io_loop) |
|||
client.fetch("http://www.tornadoweb.org/", self.stop) |
|||
response = self.wait() |
|||
# Test contents of response |
|||
self.assertIn("FriendFeed", response.body) |
|||
""" |
|||
def __init__(self, *args, **kwargs): |
|||
super(AsyncTestCase, self).__init__(*args, **kwargs) |
|||
self.__stopped = False |
|||
self.__running = False |
|||
self.__failure = None |
|||
self.__stop_args = None |
|||
|
|||
def setUp(self): |
|||
super(AsyncTestCase, self).setUp() |
|||
self.io_loop = self.get_new_ioloop() |
|||
|
|||
def tearDown(self): |
|||
if (not IOLoop.initialized() or |
|||
self.io_loop is not IOLoop.instance()): |
|||
# Try to clean up any file descriptors left open in the ioloop. |
|||
# This avoids leaks, especially when tests are run repeatedly |
|||
# in the same process with autoreload (because curl does not |
|||
# set FD_CLOEXEC on its file descriptors) |
|||
self.io_loop.close(all_fds=True) |
|||
super(AsyncTestCase, self).tearDown() |
|||
|
|||
def get_new_ioloop(self): |
|||
'''Creates a new IOLoop for this test. May be overridden in |
|||
subclasses for tests that require a specific IOLoop (usually |
|||
the singleton). |
|||
''' |
|||
return IOLoop() |
|||
|
|||
@contextlib.contextmanager |
|||
def _stack_context(self): |
|||
try: |
|||
yield |
|||
except Exception: |
|||
self.__failure = sys.exc_info() |
|||
self.stop() |
|||
|
|||
def run(self, result=None): |
|||
with StackContext(self._stack_context): |
|||
super(AsyncTestCase, self).run(result) |
|||
|
|||
def stop(self, _arg=None, **kwargs): |
|||
'''Stops the ioloop, causing one pending (or future) call to wait() |
|||
to return. |
|||
|
|||
Keyword arguments or a single positional argument passed to stop() are |
|||
saved and will be returned by wait(). |
|||
''' |
|||
assert _arg is None or not kwargs |
|||
self.__stop_args = kwargs or _arg |
|||
if self.__running: |
|||
self.io_loop.stop() |
|||
self.__running = False |
|||
self.__stopped = True |
|||
|
|||
def wait(self, condition=None, timeout=5): |
|||
"""Runs the IOLoop until stop is called or timeout has passed. |
|||
|
|||
In the event of a timeout, an exception will be thrown. |
|||
|
|||
If condition is not None, the IOLoop will be restarted after stop() |
|||
until condition() returns true. |
|||
""" |
|||
if not self.__stopped: |
|||
if timeout: |
|||
def timeout_func(): |
|||
try: |
|||
raise self.failureException( |
|||
'Async operation timed out after %d seconds' % |
|||
timeout) |
|||
except Exception: |
|||
self.__failure = sys.exc_info() |
|||
self.stop() |
|||
self.io_loop.add_timeout(time.time() + timeout, timeout_func) |
|||
while True: |
|||
self.__running = True |
|||
with NullContext(): |
|||
# Wipe out the StackContext that was established in |
|||
# self.run() so that all callbacks executed inside the |
|||
# IOLoop will re-run it. |
|||
self.io_loop.start() |
|||
if (self.__failure is not None or |
|||
condition is None or condition()): |
|||
break |
|||
assert self.__stopped |
|||
self.__stopped = False |
|||
if self.__failure is not None: |
|||
# 2to3 isn't smart enough to convert three-argument raise |
|||
# statements correctly in some cases. |
|||
if isinstance(self.__failure[1], self.__failure[0]): |
|||
raise self.__failure[1], None, self.__failure[2] |
|||
else: |
|||
raise self.__failure[0], self.__failure[1], self.__failure[2] |
|||
result = self.__stop_args |
|||
self.__stop_args = None |
|||
return result |
|||
|
|||
|
|||
class AsyncHTTPTestCase(AsyncTestCase): |
|||
'''A test case that starts up an HTTP server. |
|||
|
|||
Subclasses must override get_app(), which returns the |
|||
tornado.web.Application (or other HTTPServer callback) to be tested. |
|||
Tests will typically use the provided self.http_client to fetch |
|||
URLs from this server. |
|||
|
|||
Example:: |
|||
|
|||
class MyHTTPTest(AsyncHTTPTestCase): |
|||
def get_app(self): |
|||
return Application([('/', MyHandler)...]) |
|||
|
|||
def test_homepage(self): |
|||
# The following two lines are equivalent to |
|||
# response = self.fetch('/') |
|||
# but are shown in full here to demonstrate explicit use |
|||
# of self.stop and self.wait. |
|||
self.http_client.fetch(self.get_url('/'), self.stop) |
|||
response = self.wait() |
|||
# test contents of response |
|||
''' |
|||
def setUp(self): |
|||
super(AsyncHTTPTestCase, self).setUp() |
|||
self.__port = None |
|||
|
|||
self.http_client = AsyncHTTPClient(io_loop=self.io_loop) |
|||
self._app = self.get_app() |
|||
self.http_server = HTTPServer(self._app, io_loop=self.io_loop, |
|||
**self.get_httpserver_options()) |
|||
self.http_server.listen(self.get_http_port(), address="127.0.0.1") |
|||
|
|||
def get_app(self): |
|||
"""Should be overridden by subclasses to return a |
|||
tornado.web.Application or other HTTPServer callback. |
|||
""" |
|||
raise NotImplementedError() |
|||
|
|||
def fetch(self, path, **kwargs): |
|||
"""Convenience method to synchronously fetch a url. |
|||
|
|||
The given path will be appended to the local server's host and port. |
|||
Any additional kwargs will be passed directly to |
|||
AsyncHTTPClient.fetch (and so could be used to pass method="POST", |
|||
body="...", etc). |
|||
""" |
|||
self.http_client.fetch(self.get_url(path), self.stop, **kwargs) |
|||
return self.wait() |
|||
|
|||
def get_httpserver_options(self): |
|||
"""May be overridden by subclasses to return additional |
|||
keyword arguments for HTTPServer. |
|||
""" |
|||
return {} |
|||
|
|||
def get_http_port(self): |
|||
"""Returns the port used by the HTTPServer. |
|||
|
|||
A new port is chosen for each test. |
|||
""" |
|||
if self.__port is None: |
|||
self.__port = get_unused_port() |
|||
return self.__port |
|||
|
|||
def get_url(self, path): |
|||
"""Returns an absolute url for the given path on the test server.""" |
|||
return 'http://localhost:%s%s' % (self.get_http_port(), path) |
|||
|
|||
def tearDown(self): |
|||
self.http_server.stop() |
|||
self.http_client.close() |
|||
super(AsyncHTTPTestCase, self).tearDown() |
|||
|
|||
class LogTrapTestCase(unittest.TestCase): |
|||
"""A test case that captures and discards all logging output |
|||
if the test passes. |
|||
|
|||
Some libraries can produce a lot of logging output even when |
|||
the test succeeds, so this class can be useful to minimize the noise. |
|||
Simply use it as a base class for your test case. It is safe to combine |
|||
with AsyncTestCase via multiple inheritance |
|||
("class MyTestCase(AsyncHTTPTestCase, LogTrapTestCase):") |
|||
|
|||
This class assumes that only one log handler is configured and that |
|||
it is a StreamHandler. This is true for both logging.basicConfig |
|||
and the "pretty logging" configured by tornado.options. |
|||
""" |
|||
def run(self, result=None): |
|||
logger = logging.getLogger() |
|||
if len(logger.handlers) > 1: |
|||
# Multiple handlers have been defined. It gets messy to handle |
|||
# this, especially since the handlers may have different |
|||
# formatters. Just leave the logging alone in this case. |
|||
super(LogTrapTestCase, self).run(result) |
|||
return |
|||
if not logger.handlers: |
|||
logging.basicConfig() |
|||
self.assertEqual(len(logger.handlers), 1) |
|||
handler = logger.handlers[0] |
|||
assert isinstance(handler, logging.StreamHandler) |
|||
old_stream = handler.stream |
|||
try: |
|||
handler.stream = StringIO() |
|||
logging.info("RUNNING TEST: " + str(self)) |
|||
old_error_count = len(result.failures) + len(result.errors) |
|||
super(LogTrapTestCase, self).run(result) |
|||
new_error_count = len(result.failures) + len(result.errors) |
|||
if new_error_count != old_error_count: |
|||
old_stream.write(handler.stream.getvalue()) |
|||
finally: |
|||
handler.stream = old_stream |
|||
|
|||
def main(): |
|||
"""A simple test runner. |
|||
|
|||
This test runner is essentially equivalent to `unittest.main` from |
|||
the standard library, but adds support for tornado-style option |
|||
parsing and log formatting. |
|||
|
|||
The easiest way to run a test is via the command line:: |
|||
|
|||
python -m tornado.testing tornado.test.stack_context_test |
|||
|
|||
See the standard library unittest module for ways in which tests can |
|||
be specified. |
|||
|
|||
Projects with many tests may wish to define a test script like |
|||
tornado/test/runtests.py. This script should define a method all() |
|||
which returns a test suite and then call tornado.testing.main(). |
|||
Note that even when a test script is used, the all() test suite may |
|||
be overridden by naming a single test on the command line:: |
|||
|
|||
# Runs all tests |
|||
tornado/test/runtests.py |
|||
# Runs one test |
|||
tornado/test/runtests.py tornado.test.stack_context_test |
|||
|
|||
""" |
|||
from tornado.options import define, options, parse_command_line |
|||
|
|||
define('autoreload', type=bool, default=False, |
|||
help="DEPRECATED: use tornado.autoreload.main instead") |
|||
define('httpclient', type=str, default=None) |
|||
define('exception_on_interrupt', type=bool, default=True, |
|||
help=("If true (default), ctrl-c raises a KeyboardInterrupt " |
|||
"exception. This prints a stack trace but cannot interrupt " |
|||
"certain operations. If false, the process is more reliably " |
|||
"killed, but does not print a stack trace.")) |
|||
argv = [sys.argv[0]] + parse_command_line(sys.argv) |
|||
|
|||
if options.httpclient: |
|||
from tornado.httpclient import AsyncHTTPClient |
|||
AsyncHTTPClient.configure(options.httpclient) |
|||
|
|||
if not options.exception_on_interrupt: |
|||
signal.signal(signal.SIGINT, signal.SIG_DFL) |
|||
|
|||
if __name__ == '__main__' and len(argv) == 1: |
|||
print >> sys.stderr, "No tests specified" |
|||
sys.exit(1) |
|||
try: |
|||
# In order to be able to run tests by their fully-qualified name |
|||
# on the command line without importing all tests here, |
|||
# module must be set to None. Python 3.2's unittest.main ignores |
|||
# defaultTest if no module is given (it tries to do its own |
|||
# test discovery, which is incompatible with auto2to3), so don't |
|||
# set module if we're not asking for a specific test. |
|||
if len(argv) > 1: |
|||
unittest.main(module=None, argv=argv) |
|||
else: |
|||
unittest.main(defaultTest="all", argv=argv) |
|||
except SystemExit, e: |
|||
if e.code == 0: |
|||
logging.info('PASS') |
|||
else: |
|||
logging.error('FAIL') |
|||
if not options.autoreload: |
|||
raise |
|||
if options.autoreload: |
|||
import tornado.autoreload |
|||
tornado.autoreload.wait() |
|||
|
|||
if __name__ == '__main__': |
|||
main() |
|||
@ -0,0 +1,47 @@ |
|||
"""Miscellaneous utility functions.""" |
|||
|
|||
class ObjectDict(dict): |
|||
"""Makes a dictionary behave like an object.""" |
|||
def __getattr__(self, name): |
|||
try: |
|||
return self[name] |
|||
except KeyError: |
|||
raise AttributeError(name) |
|||
|
|||
def __setattr__(self, name, value): |
|||
self[name] = value |
|||
|
|||
|
|||
def import_object(name): |
|||
"""Imports an object by name. |
|||
|
|||
import_object('x.y.z') is equivalent to 'from x.y import z'. |
|||
|
|||
>>> import tornado.escape |
|||
>>> import_object('tornado.escape') is tornado.escape |
|||
True |
|||
>>> import_object('tornado.escape.utf8') is tornado.escape.utf8 |
|||
True |
|||
""" |
|||
parts = name.split('.') |
|||
obj = __import__('.'.join(parts[:-1]), None, None, [parts[-1]], 0) |
|||
return getattr(obj, parts[-1]) |
|||
|
|||
# Fake byte literal support: In python 2.6+, you can say b"foo" to get |
|||
# a byte literal (str in 2.x, bytes in 3.x). There's no way to do this |
|||
# in a way that supports 2.5, though, so we need a function wrapper |
|||
# to convert our string literals. b() should only be applied to literal |
|||
# latin1 strings. Once we drop support for 2.5, we can remove this function |
|||
# and just use byte literals. |
|||
if str is unicode: |
|||
def b(s): |
|||
return s.encode('latin1') |
|||
bytes_type = bytes |
|||
else: |
|||
def b(s): |
|||
return s |
|||
bytes_type = str |
|||
|
|||
def doctests(): |
|||
import doctest |
|||
return doctest.DocTestSuite() |
|||
File diff suppressed because it is too large
@ -0,0 +1,650 @@ |
|||
"""Server-side implementation of the WebSocket protocol. |
|||
|
|||
`WebSockets <http://dev.w3.org/html5/websockets/>`_ allow for bidirectional |
|||
communication between the browser and server. |
|||
|
|||
.. warning:: |
|||
|
|||
The WebSocket protocol was recently finalized as `RFC 6455 |
|||
<http://tools.ietf.org/html/rfc6455>`_ and is not yet supported in |
|||
all browsers. Refer to http://caniuse.com/websockets for details |
|||
on compatibility. In addition, during development the protocol |
|||
went through several incompatible versions, and some browsers only |
|||
support older versions. By default this module only supports the |
|||
latest version of the protocol, but optional support for an older |
|||
version (known as "draft 76" or "hixie-76") can be enabled by |
|||
overriding `WebSocketHandler.allow_draft76` (see that method's |
|||
documentation for caveats). |
|||
""" |
|||
# Author: Jacob Kristhammar, 2010 |
|||
|
|||
import array |
|||
import functools |
|||
import hashlib |
|||
import logging |
|||
import struct |
|||
import time |
|||
import base64 |
|||
import tornado.escape |
|||
import tornado.web |
|||
|
|||
from tornado.util import bytes_type, b |
|||
|
|||
class WebSocketHandler(tornado.web.RequestHandler): |
|||
"""Subclass this class to create a basic WebSocket handler. |
|||
|
|||
Override on_message to handle incoming messages. You can also override |
|||
open and on_close to handle opened and closed connections. |
|||
|
|||
See http://dev.w3.org/html5/websockets/ for details on the |
|||
JavaScript interface. The protocol is specified at |
|||
http://tools.ietf.org/html/rfc6455. |
|||
|
|||
Here is an example Web Socket handler that echos back all received messages |
|||
back to the client:: |
|||
|
|||
class EchoWebSocket(websocket.WebSocketHandler): |
|||
def open(self): |
|||
print "WebSocket opened" |
|||
|
|||
def on_message(self, message): |
|||
self.write_message(u"You said: " + message) |
|||
|
|||
def on_close(self): |
|||
print "WebSocket closed" |
|||
|
|||
Web Sockets are not standard HTTP connections. The "handshake" is HTTP, |
|||
but after the handshake, the protocol is message-based. Consequently, |
|||
most of the Tornado HTTP facilities are not available in handlers of this |
|||
type. The only communication methods available to you are write_message() |
|||
and close(). Likewise, your request handler class should |
|||
implement open() method rather than get() or post(). |
|||
|
|||
If you map the handler above to "/websocket" in your application, you can |
|||
invoke it in JavaScript with:: |
|||
|
|||
var ws = new WebSocket("ws://localhost:8888/websocket"); |
|||
ws.onopen = function() { |
|||
ws.send("Hello, world"); |
|||
}; |
|||
ws.onmessage = function (evt) { |
|||
alert(evt.data); |
|||
}; |
|||
|
|||
This script pops up an alert box that says "You said: Hello, world". |
|||
""" |
|||
def __init__(self, application, request, **kwargs): |
|||
tornado.web.RequestHandler.__init__(self, application, request, |
|||
**kwargs) |
|||
self.stream = request.connection.stream |
|||
self.ws_connection = None |
|||
|
|||
def _execute(self, transforms, *args, **kwargs): |
|||
self.open_args = args |
|||
self.open_kwargs = kwargs |
|||
|
|||
# Websocket only supports GET method |
|||
if self.request.method != 'GET': |
|||
self.stream.write(tornado.escape.utf8( |
|||
"HTTP/1.1 405 Method Not Allowed\r\n\r\n" |
|||
)) |
|||
self.stream.close() |
|||
return |
|||
|
|||
# Upgrade header should be present and should be equal to WebSocket |
|||
if self.request.headers.get("Upgrade", "").lower() != 'websocket': |
|||
self.stream.write(tornado.escape.utf8( |
|||
"HTTP/1.1 400 Bad Request\r\n\r\n" |
|||
"Can \"Upgrade\" only to \"WebSocket\"." |
|||
)) |
|||
self.stream.close() |
|||
return |
|||
|
|||
# Connection header should be upgrade. Some proxy servers/load balancers |
|||
# might mess with it. |
|||
headers = self.request.headers |
|||
connection = map(lambda s: s.strip().lower(), headers.get("Connection", "").split(",")) |
|||
if 'upgrade' not in connection: |
|||
self.stream.write(tornado.escape.utf8( |
|||
"HTTP/1.1 400 Bad Request\r\n\r\n" |
|||
"\"Connection\" must be \"Upgrade\"." |
|||
)) |
|||
self.stream.close() |
|||
return |
|||
|
|||
# The difference between version 8 and 13 is that in 8 the |
|||
# client sends a "Sec-Websocket-Origin" header and in 13 it's |
|||
# simply "Origin". |
|||
if self.request.headers.get("Sec-WebSocket-Version") in ("7", "8", "13"): |
|||
self.ws_connection = WebSocketProtocol13(self) |
|||
self.ws_connection.accept_connection() |
|||
elif (self.allow_draft76() and |
|||
"Sec-WebSocket-Version" not in self.request.headers): |
|||
self.ws_connection = WebSocketProtocol76(self) |
|||
self.ws_connection.accept_connection() |
|||
else: |
|||
self.stream.write(tornado.escape.utf8( |
|||
"HTTP/1.1 426 Upgrade Required\r\n" |
|||
"Sec-WebSocket-Version: 8\r\n\r\n")) |
|||
self.stream.close() |
|||
|
|||
def write_message(self, message, binary=False): |
|||
"""Sends the given message to the client of this Web Socket. |
|||
|
|||
The message may be either a string or a dict (which will be |
|||
encoded as json). If the ``binary`` argument is false, the |
|||
message will be sent as utf8; in binary mode any byte string |
|||
is allowed. |
|||
""" |
|||
if isinstance(message, dict): |
|||
message = tornado.escape.json_encode(message) |
|||
self.ws_connection.write_message(message, binary=binary) |
|||
|
|||
def select_subprotocol(self, subprotocols): |
|||
"""Invoked when a new WebSocket requests specific subprotocols. |
|||
|
|||
``subprotocols`` is a list of strings identifying the |
|||
subprotocols proposed by the client. This method may be |
|||
overridden to return one of those strings to select it, or |
|||
``None`` to not select a subprotocol. Failure to select a |
|||
subprotocol does not automatically abort the connection, |
|||
although clients may close the connection if none of their |
|||
proposed subprotocols was selected. |
|||
""" |
|||
return None |
|||
|
|||
def open(self): |
|||
"""Invoked when a new WebSocket is opened. |
|||
|
|||
The arguments to `open` are extracted from the `tornado.web.URLSpec` |
|||
regular expression, just like the arguments to |
|||
`tornado.web.RequestHandler.get`. |
|||
""" |
|||
pass |
|||
|
|||
def on_message(self, message): |
|||
"""Handle incoming messages on the WebSocket |
|||
|
|||
This method must be overridden. |
|||
""" |
|||
raise NotImplementedError |
|||
|
|||
def on_close(self): |
|||
"""Invoked when the WebSocket is closed.""" |
|||
pass |
|||
|
|||
def close(self): |
|||
"""Closes this Web Socket. |
|||
|
|||
Once the close handshake is successful the socket will be closed. |
|||
""" |
|||
self.ws_connection.close() |
|||
|
|||
def allow_draft76(self): |
|||
"""Override to enable support for the older "draft76" protocol. |
|||
|
|||
The draft76 version of the websocket protocol is disabled by |
|||
default due to security concerns, but it can be enabled by |
|||
overriding this method to return True. |
|||
|
|||
Connections using the draft76 protocol do not support the |
|||
``binary=True`` flag to `write_message`. |
|||
|
|||
Support for the draft76 protocol is deprecated and will be |
|||
removed in a future version of Tornado. |
|||
""" |
|||
return False |
|||
|
|||
def get_websocket_scheme(self): |
|||
"""Return the url scheme used for this request, either "ws" or "wss". |
|||
|
|||
This is normally decided by HTTPServer, but applications |
|||
may wish to override this if they are using an SSL proxy |
|||
that does not provide the X-Scheme header as understood |
|||
by HTTPServer. |
|||
|
|||
Note that this is only used by the draft76 protocol. |
|||
""" |
|||
return "wss" if self.request.protocol == "https" else "ws" |
|||
|
|||
def async_callback(self, callback, *args, **kwargs): |
|||
"""Wrap callbacks with this if they are used on asynchronous requests. |
|||
|
|||
Catches exceptions properly and closes this WebSocket if an exception |
|||
is uncaught. (Note that this is usually unnecessary thanks to |
|||
`tornado.stack_context`) |
|||
""" |
|||
return self.ws_connection.async_callback(callback, *args, **kwargs) |
|||
|
|||
def _not_supported(self, *args, **kwargs): |
|||
raise Exception("Method not supported for Web Sockets") |
|||
|
|||
def on_connection_close(self): |
|||
if self.ws_connection: |
|||
self.ws_connection.on_connection_close() |
|||
self.ws_connection = None |
|||
self.on_close() |
|||
|
|||
|
|||
for method in ["write", "redirect", "set_header", "send_error", "set_cookie", |
|||
"set_status", "flush", "finish"]: |
|||
setattr(WebSocketHandler, method, WebSocketHandler._not_supported) |
|||
|
|||
|
|||
class WebSocketProtocol(object): |
|||
"""Base class for WebSocket protocol versions. |
|||
""" |
|||
def __init__(self, handler): |
|||
self.handler = handler |
|||
self.request = handler.request |
|||
self.stream = handler.stream |
|||
self.client_terminated = False |
|||
self.server_terminated = False |
|||
|
|||
def async_callback(self, callback, *args, **kwargs): |
|||
"""Wrap callbacks with this if they are used on asynchronous requests. |
|||
|
|||
Catches exceptions properly and closes this WebSocket if an exception |
|||
is uncaught. |
|||
""" |
|||
if args or kwargs: |
|||
callback = functools.partial(callback, *args, **kwargs) |
|||
def wrapper(*args, **kwargs): |
|||
try: |
|||
return callback(*args, **kwargs) |
|||
except Exception: |
|||
logging.error("Uncaught exception in %s", |
|||
self.request.path, exc_info=True) |
|||
self._abort() |
|||
return wrapper |
|||
|
|||
def on_connection_close(self): |
|||
self._abort() |
|||
|
|||
def _abort(self): |
|||
"""Instantly aborts the WebSocket connection by closing the socket""" |
|||
self.client_terminated = True |
|||
self.server_terminated = True |
|||
self.stream.close() # forcibly tear down the connection |
|||
self.close() # let the subclass cleanup |
|||
|
|||
|
|||
class WebSocketProtocol76(WebSocketProtocol): |
|||
"""Implementation of the WebSockets protocol, version hixie-76. |
|||
|
|||
This class provides basic functionality to process WebSockets requests as |
|||
specified in |
|||
http://tools.ietf.org/html/draft-hixie-thewebsocketprotocol-76 |
|||
""" |
|||
def __init__(self, handler): |
|||
WebSocketProtocol.__init__(self, handler) |
|||
self.challenge = None |
|||
self._waiting = None |
|||
|
|||
def accept_connection(self): |
|||
try: |
|||
self._handle_websocket_headers() |
|||
except ValueError: |
|||
logging.debug("Malformed WebSocket request received") |
|||
self._abort() |
|||
return |
|||
|
|||
scheme = self.handler.get_websocket_scheme() |
|||
|
|||
# draft76 only allows a single subprotocol |
|||
subprotocol_header = '' |
|||
subprotocol = self.request.headers.get("Sec-WebSocket-Protocol", None) |
|||
if subprotocol: |
|||
selected = self.handler.select_subprotocol([subprotocol]) |
|||
if selected: |
|||
assert selected == subprotocol |
|||
subprotocol_header = "Sec-WebSocket-Protocol: %s\r\n" % selected |
|||
|
|||
# Write the initial headers before attempting to read the challenge. |
|||
# This is necessary when using proxies (such as HAProxy), which |
|||
# need to see the Upgrade headers before passing through the |
|||
# non-HTTP traffic that follows. |
|||
self.stream.write(tornado.escape.utf8( |
|||
"HTTP/1.1 101 WebSocket Protocol Handshake\r\n" |
|||
"Upgrade: WebSocket\r\n" |
|||
"Connection: Upgrade\r\n" |
|||
"Server: TornadoServer/%(version)s\r\n" |
|||
"Sec-WebSocket-Origin: %(origin)s\r\n" |
|||
"Sec-WebSocket-Location: %(scheme)s://%(host)s%(uri)s\r\n" |
|||
"%(subprotocol)s" |
|||
"\r\n" % (dict( |
|||
version=tornado.version, |
|||
origin=self.request.headers["Origin"], |
|||
scheme=scheme, |
|||
host=self.request.host, |
|||
uri=self.request.uri, |
|||
subprotocol=subprotocol_header)))) |
|||
self.stream.read_bytes(8, self._handle_challenge) |
|||
|
|||
def challenge_response(self, challenge): |
|||
"""Generates the challenge response that's needed in the handshake |
|||
|
|||
The challenge parameter should be the raw bytes as sent from the |
|||
client. |
|||
""" |
|||
key_1 = self.request.headers.get("Sec-Websocket-Key1") |
|||
key_2 = self.request.headers.get("Sec-Websocket-Key2") |
|||
try: |
|||
part_1 = self._calculate_part(key_1) |
|||
part_2 = self._calculate_part(key_2) |
|||
except ValueError: |
|||
raise ValueError("Invalid Keys/Challenge") |
|||
return self._generate_challenge_response(part_1, part_2, challenge) |
|||
|
|||
def _handle_challenge(self, challenge): |
|||
try: |
|||
challenge_response = self.challenge_response(challenge) |
|||
except ValueError: |
|||
logging.debug("Malformed key data in WebSocket request") |
|||
self._abort() |
|||
return |
|||
self._write_response(challenge_response) |
|||
|
|||
def _write_response(self, challenge): |
|||
self.stream.write(challenge) |
|||
self.async_callback(self.handler.open)(*self.handler.open_args, **self.handler.open_kwargs) |
|||
self._receive_message() |
|||
|
|||
def _handle_websocket_headers(self): |
|||
"""Verifies all invariant- and required headers |
|||
|
|||
If a header is missing or have an incorrect value ValueError will be |
|||
raised |
|||
""" |
|||
fields = ("Origin", "Host", "Sec-Websocket-Key1", |
|||
"Sec-Websocket-Key2") |
|||
if not all(map(lambda f: self.request.headers.get(f), fields)): |
|||
raise ValueError("Missing/Invalid WebSocket headers") |
|||
|
|||
def _calculate_part(self, key): |
|||
"""Processes the key headers and calculates their key value. |
|||
|
|||
Raises ValueError when feed invalid key.""" |
|||
number = int(''.join(c for c in key if c.isdigit())) |
|||
spaces = len([c for c in key if c.isspace()]) |
|||
try: |
|||
key_number = number // spaces |
|||
except (ValueError, ZeroDivisionError): |
|||
raise ValueError |
|||
return struct.pack(">I", key_number) |
|||
|
|||
def _generate_challenge_response(self, part_1, part_2, part_3): |
|||
m = hashlib.md5() |
|||
m.update(part_1) |
|||
m.update(part_2) |
|||
m.update(part_3) |
|||
return m.digest() |
|||
|
|||
def _receive_message(self): |
|||
self.stream.read_bytes(1, self._on_frame_type) |
|||
|
|||
def _on_frame_type(self, byte): |
|||
frame_type = ord(byte) |
|||
if frame_type == 0x00: |
|||
self.stream.read_until(b("\xff"), self._on_end_delimiter) |
|||
elif frame_type == 0xff: |
|||
self.stream.read_bytes(1, self._on_length_indicator) |
|||
else: |
|||
self._abort() |
|||
|
|||
def _on_end_delimiter(self, frame): |
|||
if not self.client_terminated: |
|||
self.async_callback(self.handler.on_message)( |
|||
frame[:-1].decode("utf-8", "replace")) |
|||
if not self.client_terminated: |
|||
self._receive_message() |
|||
|
|||
def _on_length_indicator(self, byte): |
|||
if ord(byte) != 0x00: |
|||
self._abort() |
|||
return |
|||
self.client_terminated = True |
|||
self.close() |
|||
|
|||
def write_message(self, message, binary=False): |
|||
"""Sends the given message to the client of this Web Socket.""" |
|||
if binary: |
|||
raise ValueError( |
|||
"Binary messages not supported by this version of websockets") |
|||
if isinstance(message, unicode): |
|||
message = message.encode("utf-8") |
|||
assert isinstance(message, bytes_type) |
|||
self.stream.write(b("\x00") + message + b("\xff")) |
|||
|
|||
def close(self): |
|||
"""Closes the WebSocket connection.""" |
|||
if not self.server_terminated: |
|||
if not self.stream.closed(): |
|||
self.stream.write("\xff\x00") |
|||
self.server_terminated = True |
|||
if self.client_terminated: |
|||
if self._waiting is not None: |
|||
self.stream.io_loop.remove_timeout(self._waiting) |
|||
self._waiting = None |
|||
self.stream.close() |
|||
elif self._waiting is None: |
|||
self._waiting = self.stream.io_loop.add_timeout( |
|||
time.time() + 5, self._abort) |
|||
|
|||
|
|||
class WebSocketProtocol13(WebSocketProtocol): |
|||
"""Implementation of the WebSocket protocol from RFC 6455. |
|||
|
|||
This class supports versions 7 and 8 of the protocol in addition to the |
|||
final version 13. |
|||
""" |
|||
def __init__(self, handler): |
|||
WebSocketProtocol.__init__(self, handler) |
|||
self._final_frame = False |
|||
self._frame_opcode = None |
|||
self._frame_mask = None |
|||
self._frame_length = None |
|||
self._fragmented_message_buffer = None |
|||
self._fragmented_message_opcode = None |
|||
self._waiting = None |
|||
|
|||
def accept_connection(self): |
|||
try: |
|||
self._handle_websocket_headers() |
|||
self._accept_connection() |
|||
except ValueError: |
|||
logging.debug("Malformed WebSocket request received") |
|||
self._abort() |
|||
return |
|||
|
|||
def _handle_websocket_headers(self): |
|||
"""Verifies all invariant- and required headers |
|||
|
|||
If a header is missing or have an incorrect value ValueError will be |
|||
raised |
|||
""" |
|||
fields = ("Host", "Sec-Websocket-Key", "Sec-Websocket-Version") |
|||
if not all(map(lambda f: self.request.headers.get(f), fields)): |
|||
raise ValueError("Missing/Invalid WebSocket headers") |
|||
|
|||
def _challenge_response(self): |
|||
sha1 = hashlib.sha1() |
|||
sha1.update(tornado.escape.utf8( |
|||
self.request.headers.get("Sec-Websocket-Key"))) |
|||
sha1.update(b("258EAFA5-E914-47DA-95CA-C5AB0DC85B11")) # Magic value |
|||
return tornado.escape.native_str(base64.b64encode(sha1.digest())) |
|||
|
|||
def _accept_connection(self): |
|||
subprotocol_header = '' |
|||
subprotocols = self.request.headers.get("Sec-WebSocket-Protocol", '') |
|||
subprotocols = [s.strip() for s in subprotocols.split(',')] |
|||
if subprotocols: |
|||
selected = self.handler.select_subprotocol(subprotocols) |
|||
if selected: |
|||
assert selected in subprotocols |
|||
subprotocol_header = "Sec-WebSocket-Protocol: %s\r\n" % selected |
|||
|
|||
self.stream.write(tornado.escape.utf8( |
|||
"HTTP/1.1 101 Switching Protocols\r\n" |
|||
"Upgrade: websocket\r\n" |
|||
"Connection: Upgrade\r\n" |
|||
"Sec-WebSocket-Accept: %s\r\n" |
|||
"%s" |
|||
"\r\n" % (self._challenge_response(), subprotocol_header))) |
|||
|
|||
self.async_callback(self.handler.open)(*self.handler.open_args, **self.handler.open_kwargs) |
|||
self._receive_frame() |
|||
|
|||
def _write_frame(self, fin, opcode, data): |
|||
if fin: |
|||
finbit = 0x80 |
|||
else: |
|||
finbit = 0 |
|||
frame = struct.pack("B", finbit | opcode) |
|||
l = len(data) |
|||
if l < 126: |
|||
frame += struct.pack("B", l) |
|||
elif l <= 0xFFFF: |
|||
frame += struct.pack("!BH", 126, l) |
|||
else: |
|||
frame += struct.pack("!BQ", 127, l) |
|||
frame += data |
|||
self.stream.write(frame) |
|||
|
|||
def write_message(self, message, binary=False): |
|||
"""Sends the given message to the client of this Web Socket.""" |
|||
if binary: |
|||
opcode = 0x2 |
|||
else: |
|||
opcode = 0x1 |
|||
message = tornado.escape.utf8(message) |
|||
assert isinstance(message, bytes_type) |
|||
self._write_frame(True, opcode, message) |
|||
|
|||
def _receive_frame(self): |
|||
self.stream.read_bytes(2, self._on_frame_start) |
|||
|
|||
def _on_frame_start(self, data): |
|||
header, payloadlen = struct.unpack("BB", data) |
|||
self._final_frame = header & 0x80 |
|||
reserved_bits = header & 0x70 |
|||
self._frame_opcode = header & 0xf |
|||
self._frame_opcode_is_control = self._frame_opcode & 0x8 |
|||
if reserved_bits: |
|||
# client is using as-yet-undefined extensions; abort |
|||
self._abort() |
|||
return |
|||
if not (payloadlen & 0x80): |
|||
# Unmasked frame -> abort connection |
|||
self._abort() |
|||
return |
|||
payloadlen = payloadlen & 0x7f |
|||
if self._frame_opcode_is_control and payloadlen >= 126: |
|||
# control frames must have payload < 126 |
|||
self._abort() |
|||
return |
|||
if payloadlen < 126: |
|||
self._frame_length = payloadlen |
|||
self.stream.read_bytes(4, self._on_masking_key) |
|||
elif payloadlen == 126: |
|||
self.stream.read_bytes(2, self._on_frame_length_16) |
|||
elif payloadlen == 127: |
|||
self.stream.read_bytes(8, self._on_frame_length_64) |
|||
|
|||
def _on_frame_length_16(self, data): |
|||
self._frame_length = struct.unpack("!H", data)[0]; |
|||
self.stream.read_bytes(4, self._on_masking_key); |
|||
|
|||
def _on_frame_length_64(self, data): |
|||
self._frame_length = struct.unpack("!Q", data)[0]; |
|||
self.stream.read_bytes(4, self._on_masking_key); |
|||
|
|||
def _on_masking_key(self, data): |
|||
self._frame_mask = array.array("B", data) |
|||
self.stream.read_bytes(self._frame_length, self._on_frame_data) |
|||
|
|||
def _on_frame_data(self, data): |
|||
unmasked = array.array("B", data) |
|||
for i in xrange(len(data)): |
|||
unmasked[i] = unmasked[i] ^ self._frame_mask[i % 4] |
|||
|
|||
if self._frame_opcode_is_control: |
|||
# control frames may be interleaved with a series of fragmented |
|||
# data frames, so control frames must not interact with |
|||
# self._fragmented_* |
|||
if not self._final_frame: |
|||
# control frames must not be fragmented |
|||
self._abort() |
|||
return |
|||
opcode = self._frame_opcode |
|||
elif self._frame_opcode == 0: # continuation frame |
|||
if self._fragmented_message_buffer is None: |
|||
# nothing to continue |
|||
self._abort() |
|||
return |
|||
self._fragmented_message_buffer += unmasked |
|||
if self._final_frame: |
|||
opcode = self._fragmented_message_opcode |
|||
unmasked = self._fragmented_message_buffer |
|||
self._fragmented_message_buffer = None |
|||
else: # start of new data message |
|||
if self._fragmented_message_buffer is not None: |
|||
# can't start new message until the old one is finished |
|||
self._abort() |
|||
return |
|||
if self._final_frame: |
|||
opcode = self._frame_opcode |
|||
else: |
|||
self._fragmented_message_opcode = self._frame_opcode |
|||
self._fragmented_message_buffer = unmasked |
|||
|
|||
if self._final_frame: |
|||
self._handle_message(opcode, unmasked.tostring()) |
|||
|
|||
if not self.client_terminated: |
|||
self._receive_frame() |
|||
|
|||
|
|||
def _handle_message(self, opcode, data): |
|||
if self.client_terminated: return |
|||
|
|||
if opcode == 0x1: |
|||
# UTF-8 data |
|||
try: |
|||
decoded = data.decode("utf-8") |
|||
except UnicodeDecodeError: |
|||
self._abort() |
|||
return |
|||
self.async_callback(self.handler.on_message)(decoded) |
|||
elif opcode == 0x2: |
|||
# Binary data |
|||
self.async_callback(self.handler.on_message)(data) |
|||
elif opcode == 0x8: |
|||
# Close |
|||
self.client_terminated = True |
|||
self.close() |
|||
elif opcode == 0x9: |
|||
# Ping |
|||
self._write_frame(True, 0xA, data) |
|||
elif opcode == 0xA: |
|||
# Pong |
|||
pass |
|||
else: |
|||
self._abort() |
|||
|
|||
def close(self): |
|||
"""Closes the WebSocket connection.""" |
|||
if not self.server_terminated: |
|||
if not self.stream.closed(): |
|||
self._write_frame(True, 0x8, b("")) |
|||
self.server_terminated = True |
|||
if self.client_terminated: |
|||
if self._waiting is not None: |
|||
self.stream.io_loop.remove_timeout(self._waiting) |
|||
self._waiting = None |
|||
self.stream.close() |
|||
elif self._waiting is None: |
|||
# Give the client a few seconds to complete a clean shutdown, |
|||
# otherwise just close the connection. |
|||
self._waiting = self.stream.io_loop.add_timeout( |
|||
time.time() + 5, self._abort) |
|||
@ -0,0 +1,296 @@ |
|||
#!/usr/bin/env python |
|||
# |
|||
# Copyright 2009 Facebook |
|||
# |
|||
# Licensed under the Apache License, Version 2.0 (the "License"); you may |
|||
# not use this file except in compliance with the License. You may obtain |
|||
# a copy of the License at |
|||
# |
|||
# http://www.apache.org/licenses/LICENSE-2.0 |
|||
# |
|||
# Unless required by applicable law or agreed to in writing, software |
|||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT |
|||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the |
|||
# License for the specific language governing permissions and limitations |
|||
# under the License. |
|||
|
|||
"""WSGI support for the Tornado web framework. |
|||
|
|||
WSGI is the Python standard for web servers, and allows for interoperability |
|||
between Tornado and other Python web frameworks and servers. This module |
|||
provides WSGI support in two ways: |
|||
|
|||
* `WSGIApplication` is a version of `tornado.web.Application` that can run |
|||
inside a WSGI server. This is useful for running a Tornado app on another |
|||
HTTP server, such as Google App Engine. See the `WSGIApplication` class |
|||
documentation for limitations that apply. |
|||
* `WSGIContainer` lets you run other WSGI applications and frameworks on the |
|||
Tornado HTTP server. For example, with this class you can mix Django |
|||
and Tornado handlers in a single server. |
|||
""" |
|||
|
|||
import Cookie |
|||
import cgi |
|||
import httplib |
|||
import logging |
|||
import sys |
|||
import time |
|||
import tornado |
|||
import urllib |
|||
|
|||
from tornado import escape |
|||
from tornado import httputil |
|||
from tornado import web |
|||
from tornado.escape import native_str, utf8 |
|||
from tornado.util import b |
|||
|
|||
try: |
|||
from io import BytesIO # python 3 |
|||
except ImportError: |
|||
from cStringIO import StringIO as BytesIO # python 2 |
|||
|
|||
class WSGIApplication(web.Application): |
|||
"""A WSGI equivalent of `tornado.web.Application`. |
|||
|
|||
WSGIApplication is very similar to web.Application, except no |
|||
asynchronous methods are supported (since WSGI does not support |
|||
non-blocking requests properly). If you call self.flush() or other |
|||
asynchronous methods in your request handlers running in a |
|||
WSGIApplication, we throw an exception. |
|||
|
|||
Example usage:: |
|||
|
|||
import tornado.web |
|||
import tornado.wsgi |
|||
import wsgiref.simple_server |
|||
|
|||
class MainHandler(tornado.web.RequestHandler): |
|||
def get(self): |
|||
self.write("Hello, world") |
|||
|
|||
if __name__ == "__main__": |
|||
application = tornado.wsgi.WSGIApplication([ |
|||
(r"/", MainHandler), |
|||
]) |
|||
server = wsgiref.simple_server.make_server('', 8888, application) |
|||
server.serve_forever() |
|||
|
|||
See the 'appengine' demo for an example of using this module to run |
|||
a Tornado app on Google AppEngine. |
|||
|
|||
Since no asynchronous methods are available for WSGI applications, the |
|||
httpclient and auth modules are both not available for WSGI applications. |
|||
We support the same interface, but handlers running in a WSGIApplication |
|||
do not support flush() or asynchronous methods. |
|||
""" |
|||
def __init__(self, handlers=None, default_host="", **settings): |
|||
web.Application.__init__(self, handlers, default_host, transforms=[], |
|||
wsgi=True, **settings) |
|||
|
|||
def __call__(self, environ, start_response): |
|||
handler = web.Application.__call__(self, HTTPRequest(environ)) |
|||
assert handler._finished |
|||
status = str(handler._status_code) + " " + \ |
|||
httplib.responses[handler._status_code] |
|||
headers = handler._headers.items() |
|||
for cookie_dict in getattr(handler, "_new_cookies", []): |
|||
for cookie in cookie_dict.values(): |
|||
headers.append(("Set-Cookie", cookie.OutputString(None))) |
|||
start_response(status, |
|||
[(native_str(k), native_str(v)) for (k,v) in headers]) |
|||
return handler._write_buffer |
|||
|
|||
|
|||
class HTTPRequest(object): |
|||
"""Mimics `tornado.httpserver.HTTPRequest` for WSGI applications.""" |
|||
def __init__(self, environ): |
|||
"""Parses the given WSGI environ to construct the request.""" |
|||
self.method = environ["REQUEST_METHOD"] |
|||
self.path = urllib.quote(environ.get("SCRIPT_NAME", "")) |
|||
self.path += urllib.quote(environ.get("PATH_INFO", "")) |
|||
self.uri = self.path |
|||
self.arguments = {} |
|||
self.query = environ.get("QUERY_STRING", "") |
|||
if self.query: |
|||
self.uri += "?" + self.query |
|||
arguments = cgi.parse_qs(self.query) |
|||
for name, values in arguments.iteritems(): |
|||
values = [v for v in values if v] |
|||
if values: self.arguments[name] = values |
|||
self.version = "HTTP/1.1" |
|||
self.headers = httputil.HTTPHeaders() |
|||
if environ.get("CONTENT_TYPE"): |
|||
self.headers["Content-Type"] = environ["CONTENT_TYPE"] |
|||
if environ.get("CONTENT_LENGTH"): |
|||
self.headers["Content-Length"] = environ["CONTENT_LENGTH"] |
|||
for key in environ: |
|||
if key.startswith("HTTP_"): |
|||
self.headers[key[5:].replace("_", "-")] = environ[key] |
|||
if self.headers.get("Content-Length"): |
|||
self.body = environ["wsgi.input"].read( |
|||
int(self.headers["Content-Length"])) |
|||
else: |
|||
self.body = "" |
|||
self.protocol = environ["wsgi.url_scheme"] |
|||
self.remote_ip = environ.get("REMOTE_ADDR", "") |
|||
if environ.get("HTTP_HOST"): |
|||
self.host = environ["HTTP_HOST"] |
|||
else: |
|||
self.host = environ["SERVER_NAME"] |
|||
|
|||
# Parse request body |
|||
self.files = {} |
|||
content_type = self.headers.get("Content-Type", "") |
|||
if content_type.startswith("application/x-www-form-urlencoded"): |
|||
for name, values in cgi.parse_qs(self.body).iteritems(): |
|||
self.arguments.setdefault(name, []).extend(values) |
|||
elif content_type.startswith("multipart/form-data"): |
|||
if 'boundary=' in content_type: |
|||
boundary = content_type.split('boundary=',1)[1] |
|||
if boundary: |
|||
httputil.parse_multipart_form_data( |
|||
utf8(boundary), self.body, self.arguments, self.files) |
|||
else: |
|||
logging.warning("Invalid multipart/form-data") |
|||
|
|||
self._start_time = time.time() |
|||
self._finish_time = None |
|||
|
|||
def supports_http_1_1(self): |
|||
"""Returns True if this request supports HTTP/1.1 semantics""" |
|||
return self.version == "HTTP/1.1" |
|||
|
|||
@property |
|||
def cookies(self): |
|||
"""A dictionary of Cookie.Morsel objects.""" |
|||
if not hasattr(self, "_cookies"): |
|||
self._cookies = Cookie.SimpleCookie() |
|||
if "Cookie" in self.headers: |
|||
try: |
|||
self._cookies.load( |
|||
native_str(self.headers["Cookie"])) |
|||
except Exception: |
|||
self._cookies = None |
|||
return self._cookies |
|||
|
|||
def full_url(self): |
|||
"""Reconstructs the full URL for this request.""" |
|||
return self.protocol + "://" + self.host + self.uri |
|||
|
|||
def request_time(self): |
|||
"""Returns the amount of time it took for this request to execute.""" |
|||
if self._finish_time is None: |
|||
return time.time() - self._start_time |
|||
else: |
|||
return self._finish_time - self._start_time |
|||
|
|||
|
|||
class WSGIContainer(object): |
|||
r"""Makes a WSGI-compatible function runnable on Tornado's HTTP server. |
|||
|
|||
Wrap a WSGI function in a WSGIContainer and pass it to HTTPServer to |
|||
run it. For example:: |
|||
|
|||
def simple_app(environ, start_response): |
|||
status = "200 OK" |
|||
response_headers = [("Content-type", "text/plain")] |
|||
start_response(status, response_headers) |
|||
return ["Hello world!\n"] |
|||
|
|||
container = tornado.wsgi.WSGIContainer(simple_app) |
|||
http_server = tornado.httpserver.HTTPServer(container) |
|||
http_server.listen(8888) |
|||
tornado.ioloop.IOLoop.instance().start() |
|||
|
|||
This class is intended to let other frameworks (Django, web.py, etc) |
|||
run on the Tornado HTTP server and I/O loop. |
|||
|
|||
The `tornado.web.FallbackHandler` class is often useful for mixing |
|||
Tornado and WSGI apps in the same server. See |
|||
https://github.com/bdarnell/django-tornado-demo for a complete example. |
|||
""" |
|||
def __init__(self, wsgi_application): |
|||
self.wsgi_application = wsgi_application |
|||
|
|||
def __call__(self, request): |
|||
data = {} |
|||
response = [] |
|||
def start_response(status, response_headers, exc_info=None): |
|||
data["status"] = status |
|||
data["headers"] = response_headers |
|||
return response.append |
|||
app_response = self.wsgi_application( |
|||
WSGIContainer.environ(request), start_response) |
|||
response.extend(app_response) |
|||
body = b("").join(response) |
|||
if hasattr(app_response, "close"): |
|||
app_response.close() |
|||
if not data: raise Exception("WSGI app did not call start_response") |
|||
|
|||
status_code = int(data["status"].split()[0]) |
|||
headers = data["headers"] |
|||
header_set = set(k.lower() for (k,v) in headers) |
|||
body = escape.utf8(body) |
|||
if "content-length" not in header_set: |
|||
headers.append(("Content-Length", str(len(body)))) |
|||
if "content-type" not in header_set: |
|||
headers.append(("Content-Type", "text/html; charset=UTF-8")) |
|||
if "server" not in header_set: |
|||
headers.append(("Server", "TornadoServer/%s" % tornado.version)) |
|||
|
|||
parts = [escape.utf8("HTTP/1.1 " + data["status"] + "\r\n")] |
|||
for key, value in headers: |
|||
parts.append(escape.utf8(key) + b(": ") + escape.utf8(value) + b("\r\n")) |
|||
parts.append(b("\r\n")) |
|||
parts.append(body) |
|||
request.write(b("").join(parts)) |
|||
request.finish() |
|||
self._log(status_code, request) |
|||
|
|||
@staticmethod |
|||
def environ(request): |
|||
"""Converts a `tornado.httpserver.HTTPRequest` to a WSGI environment. |
|||
""" |
|||
hostport = request.host.split(":") |
|||
if len(hostport) == 2: |
|||
host = hostport[0] |
|||
port = int(hostport[1]) |
|||
else: |
|||
host = request.host |
|||
port = 443 if request.protocol == "https" else 80 |
|||
environ = { |
|||
"REQUEST_METHOD": request.method, |
|||
"SCRIPT_NAME": "", |
|||
"PATH_INFO": urllib.unquote(request.path), |
|||
"QUERY_STRING": request.query, |
|||
"REMOTE_ADDR": request.remote_ip, |
|||
"SERVER_NAME": host, |
|||
"SERVER_PORT": str(port), |
|||
"SERVER_PROTOCOL": request.version, |
|||
"wsgi.version": (1, 0), |
|||
"wsgi.url_scheme": request.protocol, |
|||
"wsgi.input": BytesIO(escape.utf8(request.body)), |
|||
"wsgi.errors": sys.stderr, |
|||
"wsgi.multithread": False, |
|||
"wsgi.multiprocess": True, |
|||
"wsgi.run_once": False, |
|||
} |
|||
if "Content-Type" in request.headers: |
|||
environ["CONTENT_TYPE"] = request.headers.pop("Content-Type") |
|||
if "Content-Length" in request.headers: |
|||
environ["CONTENT_LENGTH"] = request.headers.pop("Content-Length") |
|||
for key, value in request.headers.iteritems(): |
|||
environ["HTTP_" + key.replace("-", "_").upper()] = value |
|||
return environ |
|||
|
|||
def _log(self, status_code, request): |
|||
if status_code < 400: |
|||
log_method = logging.info |
|||
elif status_code < 500: |
|||
log_method = logging.warning |
|||
else: |
|||
log_method = logging.error |
|||
request_time = 1000.0 * request.request_time() |
|||
summary = request.method + " " + request.uri + " (" + \ |
|||
request.remote_ip + ")" |
|||
log_method("%d %s %.2fms", status_code, summary, request_time) |
|||
Loading…
Reference in new issue