Tornado update

libs/tornado/__init__.py

@@ -25,5 +25,5 @@ from __future__ import absolute_import, division, print_function, with_statement
 # is zero for an official release, positive for a development branch,
 # or negative for a release candidate or beta (after the base version
 # number has been incremented)
-version = "3.1b1"
+version = "3.2.dev2"
-version_info = (3, 1, 0, -98)
+version_info = (3, 2, 0, -99)

libs/tornado/auth.py

@@ -56,7 +56,7 @@ import hmac
 import time
 import uuid
 
-from tornado.concurrent import Future, chain_future, return_future
+from tornado.concurrent import TracebackFuture, chain_future, return_future
 from tornado import gen
 from tornado import httpclient
 from tornado import escape
@@ -99,7 +99,7 @@ def _auth_return_future(f):
 
     @functools.wraps(f)
     def wrapper(*args, **kwargs):
-        future = Future()
+        future = TracebackFuture()
         callback, args, kwargs = replacer.replace(future, args, kwargs)
         if callback is not None:
             future.add_done_callback(
@@ -306,10 +306,10 @@ class OAuthMixin(object):
         """Redirects the user to obtain OAuth authorization for this service.
 
         The ``callback_uri`` may be omitted if you have previously
-        registered a callback URI with the third-party service.  For some
+        registered a callback URI with the third-party service.  For
-        sevices (including Twitter and Friendfeed), you must use a
+        some sevices (including Friendfeed), you must use a
-        previously-registered callback URI and cannot specify a callback
+        previously-registered callback URI and cannot specify a
-        via this method.
+        callback via this method.
 
         This method sets a cookie called ``_oauth_request_token`` which is
         subsequently used (and cleared) in `get_authenticated_user` for
@@ -1158,7 +1158,7 @@ class FacebookMixin(object):
 class FacebookGraphMixin(OAuth2Mixin):
     """Facebook authentication using the new Graph API and OAuth2."""
     _OAUTH_ACCESS_TOKEN_URL = "https://graph.facebook.com/oauth/access_token?"
-    _OAUTH_AUTHORIZE_URL = "https://graph.facebook.com/oauth/authorize?"
+    _OAUTH_AUTHORIZE_URL = "https://www.facebook.com/dialog/oauth?"
     _OAUTH_NO_CALLBACKS = False
     _FACEBOOK_BASE_URL = "https://graph.facebook.com"
 

libs/tornado/escape.py

@@ -49,8 +49,9 @@ try:
 except NameError:
     unichr = chr
 
-_XHTML_ESCAPE_RE = re.compile('[&<>"]')
+_XHTML_ESCAPE_RE = re.compile('[&<>"\']')
-_XHTML_ESCAPE_DICT = {'&': '&amp;', '<': '&lt;', '>': '&gt;', '"': '&quot;'}
+_XHTML_ESCAPE_DICT = {'&': '&amp;', '<': '&lt;', '>': '&gt;', '"': '&quot;',
+                      '\'': '&#39;'}
 
 
 def xhtml_escape(value):

libs/tornado/httpclient.py

@@ -33,7 +33,7 @@ import functools
 import time
 import weakref
 
-from tornado.concurrent import Future
+from tornado.concurrent import TracebackFuture
 from tornado.escape import utf8
 from tornado import httputil, stack_context
 from tornado.ioloop import IOLoop
@@ -144,9 +144,16 @@ class AsyncHTTPClient(Configurable):
 
     def close(self):
         """Destroys this HTTP client, freeing any file descriptors used.
-        Not needed in normal use, but may be helpful in unittests that
+
-        create and destroy http clients.  No other methods may be called
+        This method is **not needed in normal use** due to the way
-        on the `AsyncHTTPClient` after ``close()``.
+        that `AsyncHTTPClient` objects are transparently reused.
+        ``close()`` is generally only necessary when either the
+        `.IOLoop` is also being closed, or the ``force_instance=True``
+        argument was used when creating the `AsyncHTTPClient`.
+
+        No other methods may be called on the `AsyncHTTPClient` after
+        ``close()``.
+
         """
         if self._async_clients().get(self.io_loop) is self:
             del self._async_clients()[self.io_loop]
@@ -174,7 +181,7 @@ class AsyncHTTPClient(Configurable):
         # where normal dicts get converted to HTTPHeaders objects.
         request.headers = httputil.HTTPHeaders(request.headers)
         request = _RequestProxy(request, self.defaults)
-        future = Future()
+        future = TracebackFuture()
         if callback is not None:
             callback = stack_context.wrap(callback)
 

libs/tornado/ioloop.py

@@ -59,6 +59,9 @@ except ImportError:
 from tornado.platform.auto import set_close_exec, Waker
 
 
+_POLL_TIMEOUT = 3600.0
+
+
 class TimeoutError(Exception):
     pass
 
@@ -356,7 +359,7 @@ class IOLoop(Configurable):
                 if isinstance(result, Future):
                     future_cell[0] = result
                 else:
-                    future_cell[0] = Future()
+                    future_cell[0] = TracebackFuture()
                     future_cell[0].set_result(result)
             self.add_future(future_cell[0], lambda future: self.stop())
         self.add_callback(run)
@@ -596,7 +599,7 @@ class PollIOLoop(IOLoop):
                 pass
 
         while True:
-            poll_timeout = 3600.0
+            poll_timeout = _POLL_TIMEOUT
 
             # Prevent IO event starvation by delaying new callbacks
             # to the next iteration of the event loop.
@@ -605,6 +608,9 @@ class PollIOLoop(IOLoop):
                 self._callbacks = []
             for callback in callbacks:
                 self._run_callback(callback)
+            # Closures may be holding on to a lot of memory, so allow
+            # them to be freed before we go into our poll wait.
+            callbacks = callback = None
 
             if self._timeouts:
                 now = self.time()
@@ -616,6 +622,7 @@ class PollIOLoop(IOLoop):
                     elif self._timeouts[0].deadline <= now:
                         timeout = heapq.heappop(self._timeouts)
                         self._run_callback(timeout.callback)
+                        del timeout
                     else:
                         seconds = self._timeouts[0].deadline - now
                         poll_timeout = min(seconds, poll_timeout)
@@ -675,11 +682,9 @@ class PollIOLoop(IOLoop):
                         # Happens when the client closes the connection
                         pass
                     else:
-                        app_log.error("Exception in I/O handler for fd %s",
+                        self.handle_callback_exception(self._handlers.get(fd))
-                                      fd, exc_info=True)
                 except Exception:
-                    app_log.error("Exception in I/O handler for fd %s",
+                    self.handle_callback_exception(self._handlers.get(fd))
-                                  fd, exc_info=True)
         # reset the stopped flag so another start/stop pair can be issued
         self._stopped = False
         if self._blocking_signal_threshold is not None:
@@ -717,14 +722,14 @@ class PollIOLoop(IOLoop):
             list_empty = not self._callbacks
             self._callbacks.append(functools.partial(
                 stack_context.wrap(callback), *args, **kwargs))
-        if list_empty and thread.get_ident() != self._thread_ident:
+            if list_empty and thread.get_ident() != self._thread_ident:
-            # If we're in the IOLoop's thread, we know it's not currently
+                # If we're in the IOLoop's thread, we know it's not currently
-            # polling.  If we're not, and we added the first callback to an
+                # polling.  If we're not, and we added the first callback to an
-            # empty list, we may need to wake it up (it may wake up on its
+                # empty list, we may need to wake it up (it may wake up on its
-            # own, but an occasional extra wake is harmless).  Waking
+                # own, but an occasional extra wake is harmless).  Waking
-            # up a polling IOLoop is relatively expensive, so we try to
+                # up a polling IOLoop is relatively expensive, so we try to
-            # avoid it when we can.
+                # avoid it when we can.
-            self._waker.wake()
+                self._waker.wake()
 
     def add_callback_from_signal(self, callback, *args, **kwargs):
         with stack_context.NullContext():
@@ -813,7 +818,7 @@ class PeriodicCallback(object):
         try:
             self.callback()
         except Exception:
-            app_log.error("Error in periodic callback", exc_info=True)
+            self.io_loop.handle_callback_exception(self.callback)
         self._schedule_next()
 
     def _schedule_next(self):

libs/tornado/iostream.py

@@ -46,6 +46,14 @@ try:
 except ImportError:
     _set_nonblocking = None
 
+# These errnos indicate that a non-blocking operation must be retried
+# at a later time.  On most platforms they're the same value, but on
+# some they differ.
+_ERRNO_WOULDBLOCK = (errno.EWOULDBLOCK, errno.EAGAIN)
+
+# These errnos indicate that a connection has been abruptly terminated.
+# They should be caught and handled less noisily than other errors.
+_ERRNO_CONNRESET = (errno.ECONNRESET, errno.ECONNABORTED, errno.EPIPE)
 
 class StreamClosedError(IOError):
     """Exception raised by `IOStream` methods when the stream is closed.
@@ -257,15 +265,19 @@ class BaseIOStream(object):
         self._maybe_run_close_callback()
 
     def _maybe_run_close_callback(self):
-        if (self.closed() and self._close_callback and
+        # If there are pending callbacks, don't run the close callback
-                self._pending_callbacks == 0):
+        # until they're done (see _maybe_add_error_handler)
-            # if there are pending callbacks, don't run the close callback
+        if self.closed() and self._pending_callbacks == 0:
-            # until they're done (see _maybe_add_error_handler)
+            if self._close_callback is not None:
-            cb = self._close_callback
+                cb = self._close_callback
-            self._close_callback = None
+                self._close_callback = None
-            self._run_callback(cb)
+                self._run_callback(cb)
             # Delete any unfinished callbacks to break up reference cycles.
             self._read_callback = self._write_callback = None
+            # Clear the buffers so they can be cleared immediately even
+            # if the IOStream object is kept alive by a reference cycle.
+            # TODO: Clear the read buffer too; it currently breaks some tests.
+            self._write_buffer = None
 
     def reading(self):
         """Returns true if we are currently reading from the stream."""
@@ -447,7 +459,7 @@ class BaseIOStream(object):
             chunk = self.read_from_fd()
         except (socket.error, IOError, OSError) as e:
             # ssl.SSLError is a subclass of socket.error
-            if e.args[0] == errno.ECONNRESET:
+            if e.args[0] in _ERRNO_CONNRESET:
                 # Treat ECONNRESET as a connection close rather than
                 # an error to minimize log spam  (the exception will
                 # be available on self.error for apps that care).
@@ -550,12 +562,12 @@ class BaseIOStream(object):
                 self._write_buffer_frozen = False
                 _merge_prefix(self._write_buffer, num_bytes)
                 self._write_buffer.popleft()
-            except socket.error as e:
+            except (socket.error, IOError, OSError) as e:
-                if e.args[0] in (errno.EWOULDBLOCK, errno.EAGAIN):
+                if e.args[0] in _ERRNO_WOULDBLOCK:
                     self._write_buffer_frozen = True
                     break
                 else:
-                    if e.args[0] not in (errno.EPIPE, errno.ECONNRESET):
+                    if e.args[0] not in _ERRNO_CONNRESET:
                         # Broken pipe errors are usually caused by connection
                         # reset, and its better to not log EPIPE errors to
                         # minimize log spam
@@ -682,7 +694,7 @@ class IOStream(BaseIOStream):
         try:
             chunk = self.socket.recv(self.read_chunk_size)
         except socket.error as e:
-            if e.args[0] in (errno.EWOULDBLOCK, errno.EAGAIN):
+            if e.args[0] in _ERRNO_WOULDBLOCK:
                 return None
             else:
                 raise
@@ -725,7 +737,8 @@ class IOStream(BaseIOStream):
             # returned immediately when attempting to connect to
             # localhost, so handle them the same way as an error
             # reported later in _handle_connect.
-            if e.args[0] not in (errno.EINPROGRESS, errno.EWOULDBLOCK):
+            if (e.args[0] != errno.EINPROGRESS and
+                e.args[0] not in _ERRNO_WOULDBLOCK):
                 gen_log.warning("Connect error on fd %d: %s",
                                 self.socket.fileno(), e)
                 self.close(exc_info=True)
@@ -789,6 +802,17 @@ class SSLIOStream(IOStream):
         self._ssl_connect_callback = None
         self._server_hostname = None
 
+        # If the socket is already connected, attempt to start the handshake.
+        try:
+            self.socket.getpeername()
+        except socket.error:
+            pass
+        else:
+            # Indirectly start the handshake, which will run on the next
+            # IOLoop iteration and then the real IO state will be set in
+            # _handle_events.
+            self._add_io_state(self.io_loop.WRITE)
+
     def reading(self):
         return self._handshake_reading or super(SSLIOStream, self).reading()
 
@@ -821,7 +845,7 @@ class SSLIOStream(IOStream):
                 return self.close(exc_info=True)
             raise
         except socket.error as err:
-            if err.args[0] in (errno.ECONNABORTED, errno.ECONNRESET):
+            if err.args[0] in _ERRNO_CONNRESET:
                 return self.close(exc_info=True)
         except AttributeError:
             # On Linux, if the connection was reset before the call to
@@ -917,7 +941,7 @@ class SSLIOStream(IOStream):
             else:
                 raise
         except socket.error as e:
-            if e.args[0] in (errno.EWOULDBLOCK, errno.EAGAIN):
+            if e.args[0] in _ERRNO_WOULDBLOCK:
                 return None
             else:
                 raise
@@ -953,7 +977,7 @@ class PipeIOStream(BaseIOStream):
         try:
             chunk = os.read(self.fd, self.read_chunk_size)
         except (IOError, OSError) as e:
-            if e.args[0] in (errno.EWOULDBLOCK, errno.EAGAIN):
+            if e.args[0] in _ERRNO_WOULDBLOCK:
                 return None
             elif e.args[0] == errno.EBADF:
                 # If the writing half of a pipe is closed, select will

libs/tornado/netutil.py

@@ -159,6 +159,10 @@ def is_valid_ip(ip):
 
     Supports IPv4 and IPv6.
     """
+    if not ip or '\x00' in ip:
+        # getaddrinfo resolves empty strings to localhost, and truncates
+        # on zero bytes.
+        return False
     try:
         res = socket.getaddrinfo(ip, 0, socket.AF_UNSPEC,
                                  socket.SOCK_STREAM,

libs/tornado/process.py

@@ -190,23 +190,34 @@ class Subprocess(object):
 
     def __init__(self, *args, **kwargs):
         self.io_loop = kwargs.pop('io_loop', None) or ioloop.IOLoop.current()
+        # All FDs we create should be closed on error; those in to_close
+        # should be closed in the parent process on success.
+        pipe_fds = []
         to_close = []
         if kwargs.get('stdin') is Subprocess.STREAM:
             in_r, in_w = _pipe_cloexec()
             kwargs['stdin'] = in_r
+            pipe_fds.extend((in_r, in_w))
             to_close.append(in_r)
             self.stdin = PipeIOStream(in_w, io_loop=self.io_loop)
         if kwargs.get('stdout') is Subprocess.STREAM:
             out_r, out_w = _pipe_cloexec()
             kwargs['stdout'] = out_w
+            pipe_fds.extend((out_r, out_w))
             to_close.append(out_w)
             self.stdout = PipeIOStream(out_r, io_loop=self.io_loop)
         if kwargs.get('stderr') is Subprocess.STREAM:
             err_r, err_w = _pipe_cloexec()
             kwargs['stderr'] = err_w
+            pipe_fds.extend((err_r, err_w))
             to_close.append(err_w)
             self.stderr = PipeIOStream(err_r, io_loop=self.io_loop)
-        self.proc = subprocess.Popen(*args, **kwargs)
+        try:
+            self.proc = subprocess.Popen(*args, **kwargs)
+        except:
+            for fd in pipe_fds:
+                os.close(fd)
+            raise
         for fd in to_close:
             os.close(fd)
         for attr in ['stdin', 'stdout', 'stderr', 'pid']:

libs/tornado/template.py

@@ -169,6 +169,10 @@ with ``{# ... #}``.
 
         {% module Template("foo.html", arg=42) %}
 
+    ``UIModules`` are a feature of the `tornado.web.RequestHandler`
+    class (and specifically its ``render`` method) and will not work
+    when the template system is used on its own in other contexts.
+
 ``{% raw *expr* %}``
     Outputs the result of the given expression without autoescaping.
 

libs/tornado/web.py

@@ -437,15 +437,25 @@ class RequestHandler(object):
             morsel[k] = v
 
     def clear_cookie(self, name, path="/", domain=None):
-        """Deletes the cookie with the given name."""
+        """Deletes the cookie with the given name.
+
+        Due to limitations of the cookie protocol, you must pass the same
+        path and domain to clear a cookie as were used when that cookie
+        was set (but there is no way to find out on the server side
+        which values were used for a given cookie).
+        """
         expires = datetime.datetime.utcnow() - datetime.timedelta(days=365)
         self.set_cookie(name, value="", path=path, expires=expires,
                         domain=domain)
 
-    def clear_all_cookies(self):
+    def clear_all_cookies(self, path="/", domain=None):
-        """Deletes all the cookies the user sent with this request."""
+        """Deletes all the cookies the user sent with this request.
+
+        See `clear_cookie` for more information on the path and domain
+        parameters.
+        """
         for name in self.request.cookies:
-            self.clear_cookie(name)
+            self.clear_cookie(name, path=path, domain=domain)
 
     def set_secure_cookie(self, name, value, expires_days=30, **kwargs):
         """Signs and timestamps a cookie so it cannot be forged.
@@ -751,10 +761,10 @@ class RequestHandler(object):
 
         if hasattr(self.request, "connection"):
             # Now that the request is finished, clear the callback we
-            # set on the IOStream (which would otherwise prevent the
+            # set on the HTTPConnection (which would otherwise prevent the
             # garbage collection of the RequestHandler when there
             # are keepalive connections)
-            self.request.connection.stream.set_close_callback(None)
+            self.request.connection.set_close_callback(None)
 
         if not self.application._wsgi:
             self.flush(include_footers=True)
@@ -1142,7 +1152,7 @@ class RequestHandler(object):
             elif isinstance(result, Future):
                 if result.done():
                     if result.result() is not None:
-                        raise ValueError('Expected None, got %r' % result)
+                        raise ValueError('Expected None, got %r' % result.result())
                     callback()
                 else:
                     # Delayed import of IOLoop because it's not available
@@ -1827,6 +1837,10 @@ class StaticFileHandler(RequestHandler):
                 return
             if start is not None and start < 0:
                 start += size
+            if end is not None and end > size:
+                # Clients sometimes blindly use a large range to limit their
+                # download size; cap the endpoint at the actual file size.
+                end = size
             # Note: only return HTTP 206 if less than the entire range has been
             # requested. Not only is this semantically correct, but Chrome
             # refuses to play audio if it gets an HTTP 206 in response to
@@ -2305,9 +2319,12 @@ class UIModule(object):
         self.handler = handler
         self.request = handler.request
         self.ui = handler.ui
-        self.current_user = handler.current_user
         self.locale = handler.locale
 
+    @property
+    def current_user(self):
+        return self.handler.current_user
+
     def render(self, *args, **kwargs):
         """Overridden in subclasses to return this module's output."""
         raise NotImplementedError()

libs/tornado/websocket.py

@@ -31,7 +31,7 @@ import time
 import tornado.escape
 import tornado.web
 
-from tornado.concurrent import Future
+from tornado.concurrent import TracebackFuture
 from tornado.escape import utf8, native_str
 from tornado import httpclient
 from tornado.ioloop import IOLoop
@@ -51,6 +51,10 @@ class WebSocketError(Exception):
     pass
 
 
+class WebSocketClosedError(WebSocketError):
+    pass
+
+
 class WebSocketHandler(tornado.web.RequestHandler):
     """Subclass this class to create a basic WebSocket handler.
 
@@ -160,6 +164,8 @@ class WebSocketHandler(tornado.web.RequestHandler):
         message will be sent as utf8; in binary mode any byte string
         is allowed.
         """
+        if self.ws_connection is None:
+            raise WebSocketClosedError()
         if isinstance(message, dict):
             message = tornado.escape.json_encode(message)
         self.ws_connection.write_message(message, binary=binary)
@@ -195,6 +201,8 @@ class WebSocketHandler(tornado.web.RequestHandler):
 
     def ping(self, data):
         """Send ping frame to the remote end."""
+        if self.ws_connection is None:
+            raise WebSocketClosedError()
         self.ws_connection.write_ping(data)
 
     def on_pong(self, data):
@@ -210,8 +218,9 @@ class WebSocketHandler(tornado.web.RequestHandler):
 
         Once the close handshake is successful the socket will be closed.
         """
-        self.ws_connection.close()
+        if self.ws_connection:
-        self.ws_connection = None
+            self.ws_connection.close()
+            self.ws_connection = None
 
     def allow_draft76(self):
         """Override to enable support for the older "draft76" protocol.
@@ -764,7 +773,7 @@ class WebSocketProtocol13(WebSocketProtocol):
 class WebSocketClientConnection(simple_httpclient._HTTPConnection):
     """WebSocket client connection."""
     def __init__(self, io_loop, request):
-        self.connect_future = Future()
+        self.connect_future = TracebackFuture()
         self.read_future = None
         self.read_queue = collections.deque()
         self.key = base64.b64encode(os.urandom(16))
@@ -825,7 +834,7 @@ class WebSocketClientConnection(simple_httpclient._HTTPConnection):
         ready.
         """
         assert self.read_future is None
-        future = Future()
+        future = TracebackFuture()
         if self.read_queue:
             future.set_result(self.read_queue.popleft())
         else:

libs/tornado/wsgi.py

@@ -242,10 +242,12 @@ class WSGIContainer(object):
             return response.append
         app_response = self.wsgi_application(
             WSGIContainer.environ(request), start_response)
-        response.extend(app_response)
+        try:
-        body = b"".join(response)
+            response.extend(app_response)
-        if hasattr(app_response, "close"):
+            body = b"".join(response)
-            app_response.close()
+        finally:
+            if hasattr(app_response, "close"):
+                app_response.close()
         if not data:
             raise Exception("WSGI app did not call start_response")