From 1510e37652a8eae74cd5346c918b3799bc87601b Mon Sep 17 00:00:00 2001 From: Ruud Date: Sun, 11 Jan 2015 16:18:22 +0100 Subject: [PATCH] Update Tornado --- libs/tornado/httpclient.py | 13 ++++++++----- libs/tornado/httpserver.py | 28 ++++------------------------ libs/tornado/httputil.py | 16 +++++++++++++++- libs/tornado/iostream.py | 2 +- libs/tornado/netutil.py | 4 ++-- libs/tornado/options.py | 7 +++++++ libs/tornado/platform/kqueue.py | 3 +-- libs/tornado/platform/select.py | 2 +- libs/tornado/simple_httpclient.py | 8 ++------ libs/tornado/testing.py | 4 ++++ libs/tornado/web.py | 9 +++++---- libs/tornado/websocket.py | 2 +- libs/tornado/wsgi.py | 2 +- 13 files changed, 52 insertions(+), 48 deletions(-) diff --git a/libs/tornado/httpclient.py b/libs/tornado/httpclient.py index df42951..6ea872d 100755 --- a/libs/tornado/httpclient.py +++ b/libs/tornado/httpclient.py @@ -95,7 +95,8 @@ class HTTPClient(object): If it is a string, we construct an `HTTPRequest` using any additional kwargs: ``HTTPRequest(request, **kwargs)`` - If an error occurs during the fetch, we raise an `HTTPError`. + If an error occurs during the fetch, we raise an `HTTPError` unless + the ``raise_error`` keyword argument is set to False. """ response = self._io_loop.run_sync(functools.partial( self._async_client.fetch, request, **kwargs)) @@ -200,7 +201,7 @@ class AsyncHTTPClient(Configurable): raise RuntimeError("inconsistent AsyncHTTPClient cache") del self._instance_cache[self.io_loop] - def fetch(self, request, callback=None, **kwargs): + def fetch(self, request, callback=None, raise_error=True, **kwargs): """Executes a request, asynchronously returning an `HTTPResponse`. The request may be either a string URL or an `HTTPRequest` object. @@ -208,8 +209,10 @@ class AsyncHTTPClient(Configurable): kwargs: ``HTTPRequest(request, **kwargs)`` This method returns a `.Future` whose result is an - `HTTPResponse`. The ``Future`` will raise an `HTTPError` if - the request returned a non-200 response code. + `HTTPResponse`. By default, the ``Future`` will raise an `HTTPError` + if the request returned a non-200 response code. Instead, if + ``raise_error`` is set to False, the response will always be + returned regardless of the response code. If a ``callback`` is given, it will be invoked with the `HTTPResponse`. In the callback interface, `HTTPError` is not automatically raised. @@ -243,7 +246,7 @@ class AsyncHTTPClient(Configurable): future.add_done_callback(handle_future) def handle_response(response): - if response.error: + if raise_error and response.error: future.set_exception(response.error) else: future.set_result(response) diff --git a/libs/tornado/httpserver.py b/libs/tornado/httpserver.py index d4c990c..47c7472 100755 --- a/libs/tornado/httpserver.py +++ b/libs/tornado/httpserver.py @@ -42,30 +42,10 @@ from tornado.tcpserver import TCPServer class HTTPServer(TCPServer, httputil.HTTPServerConnectionDelegate): r"""A non-blocking, single-threaded HTTP server. - A server is defined by either a request callback that takes a - `.HTTPServerRequest` as an argument or a `.HTTPServerConnectionDelegate` - instance. - - A simple example server that echoes back the URI you requested:: - - import tornado.httpserver - import tornado.ioloop - from tornado import httputil - - def handle_request(request): - message = "You requested %s\n" % request.uri - request.connection.write_headers( - httputil.ResponseStartLine('HTTP/1.1', 200, 'OK'), - httputil.HTTPHeaders({"Content-Length": str(len(message))})) - request.connection.write(message) - request.connection.finish() - - http_server = tornado.httpserver.HTTPServer(handle_request) - http_server.listen(8888) - tornado.ioloop.IOLoop.instance().start() - - Applications should use the methods of `.HTTPConnection` to write - their response. + A server is defined by a subclass of `.HTTPServerConnectionDelegate`, + or, for backwards compatibility, a callback that takes an + `.HTTPServerRequest` as an argument. The delegate is usually a + `tornado.web.Application`. `HTTPServer` supports keep-alive connections by default (automatically for HTTP/1.1, or for HTTP/1.0 when the client diff --git a/libs/tornado/httputil.py b/libs/tornado/httputil.py index f5c9c04..88389fe 100755 --- a/libs/tornado/httputil.py +++ b/libs/tornado/httputil.py @@ -331,7 +331,7 @@ class HTTPServerRequest(object): self.uri = uri self.version = version self.headers = headers or HTTPHeaders() - self.body = body or "" + self.body = body or b"" # set remote IP and protocol context = getattr(connection, 'context', None) @@ -873,3 +873,17 @@ def _encode_header(key, pdict): def doctests(): import doctest return doctest.DocTestSuite() + +def split_host_and_port(netloc): + """Returns ``(host, port)`` tuple from ``netloc``. + + Returned ``port`` will be ``None`` if not present. + """ + match = re.match(r'^(.+):(\d+)$', netloc) + if match: + host = match.group(1) + port = int(match.group(2)) + else: + host = netloc + port = None + return (host, port) diff --git a/libs/tornado/iostream.py b/libs/tornado/iostream.py index eced6d6..2d5df99 100755 --- a/libs/tornado/iostream.py +++ b/libs/tornado/iostream.py @@ -331,7 +331,7 @@ class BaseIOStream(object): if data: if (self.max_write_buffer_size is not None and self._write_buffer_size + len(data) > self.max_write_buffer_size): - raise StreamBufferFullError("Reached maximum read buffer size") + raise StreamBufferFullError("Reached maximum write buffer size") # Break up large contiguous strings before inserting them in the # write buffer, so we don't have to recopy the entire thing # as we slice off pieces to send to the socket. diff --git a/libs/tornado/netutil.py b/libs/tornado/netutil.py index f147c97..e85f62b 100755 --- a/libs/tornado/netutil.py +++ b/libs/tornado/netutil.py @@ -20,7 +20,7 @@ from __future__ import absolute_import, division, print_function, with_statement import errno import os -import platform +import sys import socket import stat @@ -105,7 +105,7 @@ def bind_sockets(port, address=None, family=socket.AF_UNSPEC, for res in set(socket.getaddrinfo(address, port, family, socket.SOCK_STREAM, 0, flags)): af, socktype, proto, canonname, sockaddr = res - if (platform.system() == 'Darwin' and address == 'localhost' and + if (sys.platform == 'darwin' and address == 'localhost' and af == socket.AF_INET6 and sockaddr[3] != 0): # Mac OS X includes a link-local address fe80::1%lo0 in the # getaddrinfo results for 'localhost'. However, the firewall diff --git a/libs/tornado/options.py b/libs/tornado/options.py index 5e23e29..c855407 100755 --- a/libs/tornado/options.py +++ b/libs/tornado/options.py @@ -204,6 +204,13 @@ class OptionParser(object): (name, self._options[name].file_name)) frame = sys._getframe(0) options_file = frame.f_code.co_filename + + # Can be called directly, or through top level define() fn, in which + # case, step up above that frame to look for real caller. + if (frame.f_back.f_code.co_filename == options_file and + frame.f_back.f_code.co_name == 'define'): + frame = frame.f_back + file_name = frame.f_back.f_code.co_filename if file_name == options_file: file_name = "" diff --git a/libs/tornado/platform/kqueue.py b/libs/tornado/platform/kqueue.py index de8c046..f8f3e4a 100755 --- a/libs/tornado/platform/kqueue.py +++ b/libs/tornado/platform/kqueue.py @@ -54,8 +54,7 @@ class _KQueue(object): if events & IOLoop.WRITE: kevents.append(select.kevent( fd, filter=select.KQ_FILTER_WRITE, flags=flags)) - if events & IOLoop.READ or not kevents: - # Always read when there is not a write + if events & IOLoop.READ: kevents.append(select.kevent( fd, filter=select.KQ_FILTER_READ, flags=flags)) # Even though control() takes a list, it seems to return EINVAL diff --git a/libs/tornado/platform/select.py b/libs/tornado/platform/select.py index 9a87956..1e12655 100755 --- a/libs/tornado/platform/select.py +++ b/libs/tornado/platform/select.py @@ -47,7 +47,7 @@ class _Select(object): # Closed connections are reported as errors by epoll and kqueue, # but as zero-byte reads by select, so when errors are requested # we need to listen for both read and error. - self.read_fds.add(fd) + #self.read_fds.add(fd) def modify(self, fd, events): self.unregister(fd) diff --git a/libs/tornado/simple_httpclient.py b/libs/tornado/simple_httpclient.py index e60c434..7c915e9 100755 --- a/libs/tornado/simple_httpclient.py +++ b/libs/tornado/simple_httpclient.py @@ -193,12 +193,8 @@ class _HTTPConnection(httputil.HTTPMessageDelegate): netloc = self.parsed.netloc if "@" in netloc: userpass, _, netloc = netloc.rpartition("@") - match = re.match(r'^(.+):(\d+)$', netloc) - if match: - host = match.group(1) - port = int(match.group(2)) - else: - host = netloc + host, port = httputil.split_host_and_port(netloc) + if port is None: port = 443 if self.parsed.scheme == "https" else 80 if re.match(r'^\[.*\]$', host): # raw ipv6 addresses in urls are enclosed in brackets diff --git a/libs/tornado/testing.py b/libs/tornado/testing.py index 4d85abe..4511863 100755 --- a/libs/tornado/testing.py +++ b/libs/tornado/testing.py @@ -19,6 +19,7 @@ try: from tornado.simple_httpclient import SimpleAsyncHTTPClient from tornado.ioloop import IOLoop, TimeoutError from tornado import netutil + from tornado.process import Subprocess except ImportError: # These modules are not importable on app engine. Parts of this module # won't work, but e.g. LogTrapTestCase and main() will. @@ -28,6 +29,7 @@ except ImportError: IOLoop = None netutil = None SimpleAsyncHTTPClient = None + Subprocess = None from tornado.log import gen_log, app_log from tornado.stack_context import ExceptionStackContext from tornado.util import raise_exc_info, basestring_type @@ -214,6 +216,8 @@ class AsyncTestCase(unittest.TestCase): self.io_loop.make_current() def tearDown(self): + # Clean up Subprocess, so it can be used again with a new ioloop. + Subprocess.uninitialize() self.io_loop.clear_current() if (not IOLoop.initialized() or self.io_loop is not IOLoop.instance()): diff --git a/libs/tornado/web.py b/libs/tornado/web.py index a038265..2d1dac0 100755 --- a/libs/tornado/web.py +++ b/libs/tornado/web.py @@ -85,6 +85,7 @@ from tornado import stack_context from tornado import template from tornado.escape import utf8, _unicode from tornado.util import import_object, ObjectDict, raise_exc_info, unicode_type, _websocket_mask +from tornado.httputil import split_host_and_port try: @@ -1477,7 +1478,7 @@ def asynchronous(method): with stack_context.ExceptionStackContext( self._stack_context_handle_exception): result = method(self, *args, **kwargs) - if isinstance(result, Future): + if is_future(result): # If @asynchronous is used with @gen.coroutine, (but # not @gen.engine), we can automatically finish the # request when the future resolves. Additionally, @@ -1518,7 +1519,7 @@ def stream_request_body(cls): the entire body has been read. There is a subtle interaction between ``data_received`` and asynchronous - ``prepare``: The first call to ``data_recieved`` may occur at any point + ``prepare``: The first call to ``data_received`` may occur at any point after the call to ``prepare`` has returned *or yielded*. """ if not issubclass(cls, RequestHandler): @@ -1729,7 +1730,7 @@ class Application(httputil.HTTPServerConnectionDelegate): self.transforms.append(transform_class) def _get_host_handlers(self, request): - host = request.host.lower().split(':')[0] + host = split_host_and_port(request.host.lower())[0] matches = [] for pattern, handlers in self.handlers: if pattern.match(host): @@ -1845,7 +1846,7 @@ class _RequestDispatcher(httputil.HTTPMessageDelegate): handlers = app._get_host_handlers(self.request) if not handlers: self.handler_class = RedirectHandler - self.handler_kwargs = dict(url="http://" + app.default_host + "/") + self.handler_kwargs = dict(url="%s://%s/" % (self.request.protocol, app.default_host)) return for spec in handlers: match = spec.regex.match(self.request.path) diff --git a/libs/tornado/websocket.py b/libs/tornado/websocket.py index d960b0e..5c762ad 100755 --- a/libs/tornado/websocket.py +++ b/libs/tornado/websocket.py @@ -229,7 +229,7 @@ class WebSocketHandler(tornado.web.RequestHandler): """ return None - def open(self): + def open(self, *args, **kwargs): """Invoked when a new WebSocket is opened. The arguments to `open` are extracted from the `tornado.web.URLSpec` diff --git a/libs/tornado/wsgi.py b/libs/tornado/wsgi.py index f3aa665..e7e07fb 100755 --- a/libs/tornado/wsgi.py +++ b/libs/tornado/wsgi.py @@ -207,7 +207,7 @@ class WSGIAdapter(object): body = environ["wsgi.input"].read( int(headers["Content-Length"])) else: - body = "" + body = b"" protocol = environ["wsgi.url_scheme"] remote_ip = environ.get("REMOTE_ADDR", "") if environ.get("HTTP_HOST"):