Tornado 3.2.1

libs/tornado/__init__.py

@@ -25,5 +25,5 @@ from __future__ import absolute_import, division, print_function, with_statement
 # is zero for an official release, positive for a development branch,
 # or negative for a release candidate or beta (after the base version
 # number has been incremented)
-version = "3.2"
-version_info = (3, 2, 0, 0)
+version = "3.3.dev1"
+version_info = (3, 3, 0, -100)

libs/tornado/auth.py

@@ -73,6 +73,11 @@ try:
 except ImportError:
     import urllib as urllib_parse  # py2
 
+try:
+    long  # py2
+except NameError:
+    long = int  # py3
+
 
 class AuthError(Exception):
     pass

libs/tornado/autoreload.py

@@ -14,7 +14,7 @@
 # License for the specific language governing permissions and limitations
 # under the License.
 
-"""xAutomatically restart the server when a source file is modified.
+"""Automatically restart the server when a source file is modified.
 
 Most applications should not access this module directly.  Instead,
 pass the keyword argument ``autoreload=True`` to the

libs/tornado/concurrent.py

@@ -40,52 +40,132 @@ class ReturnValueIgnoredError(Exception):
     pass
 
 
-class _DummyFuture(object):
+class Future(object):
+    """Placeholder for an asynchronous result.
+
+    A ``Future`` encapsulates the result of an asynchronous
+    operation.  In synchronous applications ``Futures`` are used
+    to wait for the result from a thread or process pool; in
+    Tornado they are normally used with `.IOLoop.add_future` or by
+    yielding them in a `.gen.coroutine`.
+
+    `tornado.concurrent.Future` is similar to
+    `concurrent.futures.Future`, but not thread-safe (and therefore
+    faster for use with single-threaded event loops).
+
+    In addition to ``exception`` and ``set_exception``, methods ``exc_info``
+    and ``set_exc_info`` are supported to capture tracebacks in Python 2.
+    The traceback is automatically available in Python 3, but in the
+    Python 2 futures backport this information is discarded.
+    This functionality was previously available in a separate class
+    ``TracebackFuture``, which is now a deprecated alias for this class.
+
+    .. versionchanged:: 3.3
+       `tornado.concurrent.Future` is always a thread-unsafe ``Future``
+       with support for the ``exc_info`` methods.  Previously it would
+       be an alias for the thread-safe `concurrent.futures.Future`
+       if that package was available and fall back to the thread-unsafe
+       implementation if it was not.
+
+    """
     def __init__(self):
         self._done = False
         self._result = None
         self._exception = None
+        self._exc_info = None
         self._callbacks = []
 
     def cancel(self):
+        """Cancel the operation, if possible.
+
+        Tornado ``Futures`` do not support cancellation, so this method always
+        returns False.
+        """
         return False
 
     def cancelled(self):
+        """Returns True if the operation has been cancelled.
+
+        Tornado ``Futures`` do not support cancellation, so this method
+        always returns False.
+        """
         return False
 
     def running(self):
+        """Returns True if this operation is currently running."""
         return not self._done
 
     def done(self):
+        """Returns True if the future has finished running."""
         return self._done
 
     def result(self, timeout=None):
-        self._check_done()
-        if self._exception:
+        """If the operation succeeded, return its result.  If it failed,
+        re-raise its exception.
+        """
+        if self._result is not None:
+            return self._result
+        if self._exc_info is not None:
+            raise_exc_info(self._exc_info)
+        elif self._exception is not None:
             raise self._exception
+        self._check_done()
         return self._result
 
     def exception(self, timeout=None):
-        self._check_done()
-        if self._exception:
+        """If the operation raised an exception, return the `Exception`
+        object.  Otherwise returns None.
+        """
+        if self._exception is not None:
             return self._exception
         else:
+            self._check_done()
             return None
 
     def add_done_callback(self, fn):
+        """Attaches the given callback to the `Future`.
+
+        It will be invoked with the `Future` as its argument when the Future
+        has finished running and its result is available.  In Tornado
+        consider using `.IOLoop.add_future` instead of calling
+        `add_done_callback` directly.
+        """
         if self._done:
             fn(self)
         else:
             self._callbacks.append(fn)
 
     def set_result(self, result):
+        """Sets the result of a ``Future``.
+
+        It is undefined to call any of the ``set`` methods more than once
+        on the same object.
+        """
         self._result = result
         self._set_done()
 
     def set_exception(self, exception):
+        """Sets the exception of a ``Future.``"""
         self._exception = exception
         self._set_done()
 
+    def exc_info(self):
+        """Returns a tuple in the same format as `sys.exc_info` or None.
+
+        .. versionadded:: 3.3
+        """
+        return self._exc_info
+
+    def set_exc_info(self, exc_info):
+        """Sets the exception information of a ``Future.``
+
+        Preserves tracebacks on Python 2.
+
+        .. versionadded:: 3.3
+        """
+        self._exc_info = exc_info
+        self.set_exception(exc_info[1])
+
     def _check_done(self):
         if not self._done:
             raise Exception("DummyFuture does not support blocking for results")
@@ -97,38 +177,16 @@ class _DummyFuture(object):
             cb(self)
         self._callbacks = None
 
+TracebackFuture = Future
+
 if futures is None:
-    Future = _DummyFuture
+    FUTURES = Future
 else:
-    Future = futures.Future
+    FUTURES = (futures.Future, Future)
 
 
-class TracebackFuture(Future):
-    """Subclass of `Future` which can store a traceback with
-    exceptions.
-
-    The traceback is automatically available in Python 3, but in the
-    Python 2 futures backport this information is discarded.
-    """
-    def __init__(self):
-        super(TracebackFuture, self).__init__()
-        self.__exc_info = None
-
-    def exc_info(self):
-        return self.__exc_info
-
-    def set_exc_info(self, exc_info):
-        """Traceback-aware replacement for
-        `~concurrent.futures.Future.set_exception`.
-        """
-        self.__exc_info = exc_info
-        self.set_exception(exc_info[1])
-
-    def result(self, timeout=None):
-        if self.__exc_info is not None:
-            raise_exc_info(self.__exc_info)
-        else:
-            return super(TracebackFuture, self).result(timeout=timeout)
+def is_future(x):
+    return isinstance(x, FUTURES)
 
 
 class DummyExecutor(object):
@@ -254,10 +312,13 @@ def return_future(f):
 def chain_future(a, b):
     """Chain two futures together so that when one completes, so does the other.
 
-    The result (success or failure) of ``a`` will be copied to ``b``.
+    The result (success or failure) of ``a`` will be copied to ``b``, unless
+    ``b`` has already been completed or cancelled by the time ``a`` finishes.
     """
     def copy(future):
         assert future is a
+        if b.done():
+            return
         if (isinstance(a, TracebackFuture) and isinstance(b, TracebackFuture)
                 and a.exc_info() is not None):
             b.set_exc_info(a.exc_info())

libs/tornado/curl_httpclient.py

@@ -268,6 +268,7 @@ class CurlAsyncHTTPClient(AsyncHTTPClient):
             info["callback"](HTTPResponse(
                 request=info["request"], code=code, headers=info["headers"],
                 buffer=buffer, effective_url=effective_url, error=error,
+                reason=info['headers'].get("X-Http-Reason", None),
                 request_time=time.time() - info["curl_start_time"],
                 time_info=time_info))
         except Exception:
@@ -470,7 +471,11 @@ def _curl_header_callback(headers, header_line):
     header_line = header_line.strip()
     if header_line.startswith("HTTP/"):
         headers.clear()
-        return
+        try:
+            (__, __, reason) = httputil.parse_response_start_line(header_line)
+            header_line = "X-Http-Reason: %s" % reason
+        except httputil.HTTPInputException:
+            return
     if not header_line:
         return
     headers.parse_line(header_line)

libs/tornado/escape.py

@@ -75,7 +75,7 @@ def xhtml_unescape(value):
 
 
 # The fact that json_encode wraps json.dumps is an implementation detail.
-# Please see https://github.com/facebook/tornado/pull/706
+# Please see https://github.com/tornadoweb/tornado/pull/706
 # before sending a pull request that adds **kwargs to this function.
 def json_encode(value):
     """JSON-encodes the given Python object."""

libs/tornado/gen.py

@@ -87,9 +87,9 @@ import itertools
 import sys
 import types
 
-from tornado.concurrent import Future, TracebackFuture
+from tornado.concurrent import Future, TracebackFuture, is_future, chain_future
 from tornado.ioloop import IOLoop
-from tornado.stack_context import ExceptionStackContext, wrap
+from tornado import stack_context
 
 
 class KeyReuseError(Exception):
@@ -112,6 +112,10 @@ class ReturnValueIgnoredError(Exception):
     pass
 
 
+class TimeoutError(Exception):
+    """Exception raised by ``with_timeout``."""
+
+
 def engine(func):
     """Callback-oriented decorator for asynchronous generators.
 
@@ -129,45 +133,20 @@ def engine(func):
     `~tornado.web.RequestHandler` :ref:`HTTP verb methods <verbs>`,
     which use ``self.finish()`` in place of a callback argument.
     """
+    func = _make_coroutine_wrapper(func, replace_callback=False)
     @functools.wraps(func)
     def wrapper(*args, **kwargs):
-        runner = None
-
-        def handle_exception(typ, value, tb):
-            # if the function throws an exception before its first "yield"
-            # (or is not a generator at all), the Runner won't exist yet.
-            # However, in that case we haven't reached anything asynchronous
-            # yet, so we can just let the exception propagate.
-            if runner is not None:
-                return runner.handle_exception(typ, value, tb)
-            return False
-        with ExceptionStackContext(handle_exception) as deactivate:
-            try:
-                result = func(*args, **kwargs)
-            except (Return, StopIteration) as e:
-                result = getattr(e, 'value', None)
-            else:
-                if isinstance(result, types.GeneratorType):
-                    def final_callback(value):
-                        if value is not None:
-                            raise ReturnValueIgnoredError(
-                                "@gen.engine functions cannot return values: "
-                                "%r" % (value,))
-                        assert value is None
-                        deactivate()
-                    runner = Runner(result, final_callback)
-                    runner.run()
-                    return
-            if result is not None:
+        future = func(*args, **kwargs)
+        def final_callback(future):
+            if future.result() is not None:
                 raise ReturnValueIgnoredError(
                     "@gen.engine functions cannot return values: %r" %
-                    (result,))
-            deactivate()
-            # no yield, so we're done
+                    (future.result(),))
+        future.add_done_callback(final_callback)
     return wrapper
 
 
-def coroutine(func):
+def coroutine(func, replace_callback=True):
     """Decorator for asynchronous generators.
 
     Any generator that yields objects from this module must be wrapped
@@ -191,43 +170,56 @@ def coroutine(func):
     From the caller's perspective, ``@gen.coroutine`` is similar to
     the combination of ``@return_future`` and ``@gen.engine``.
     """
+    return _make_coroutine_wrapper(func, replace_callback=True)
+
+
+def _make_coroutine_wrapper(func, replace_callback):
+    """The inner workings of ``@gen.coroutine`` and ``@gen.engine``.
+
+    The two decorators differ in their treatment of the ``callback``
+    argument, so we cannot simply implement ``@engine`` in terms of
+    ``@coroutine``.
+    """
     @functools.wraps(func)
     def wrapper(*args, **kwargs):
-        runner = None
         future = TracebackFuture()
 
-        if 'callback' in kwargs:
+        if replace_callback and 'callback' in kwargs:
             callback = kwargs.pop('callback')
             IOLoop.current().add_future(
                 future, lambda future: callback(future.result()))
 
-        def handle_exception(typ, value, tb):
-            try:
-                if runner is not None and runner.handle_exception(typ, value, tb):
-                    return True
-            except Exception:
-                typ, value, tb = sys.exc_info()
-            future.set_exc_info((typ, value, tb))
-            return True
-        with ExceptionStackContext(handle_exception) as deactivate:
-            try:
-                result = func(*args, **kwargs)
-            except (Return, StopIteration) as e:
-                result = getattr(e, 'value', None)
-            except Exception:
-                deactivate()
-                future.set_exc_info(sys.exc_info())
+        try:
+            result = func(*args, **kwargs)
+        except (Return, StopIteration) as e:
+            result = getattr(e, 'value', None)
+        except Exception:
+            future.set_exc_info(sys.exc_info())
+            return future
+        else:
+            if isinstance(result, types.GeneratorType):
+                # Inline the first iteration of Runner.run.  This lets us
+                # avoid the cost of creating a Runner when the coroutine
+                # never actually yields, which in turn allows us to
+                # use "optional" coroutines in critical path code without
+                # performance penalty for the synchronous case.
+                try:
+                    orig_stack_contexts = stack_context._state.contexts
+                    yielded = next(result)
+                    if stack_context._state.contexts is not orig_stack_contexts:
+                        yielded = TracebackFuture()
+                        yielded.set_exception(
+                            stack_context.StackContextInconsistentError(
+                                'stack_context inconsistency (probably caused '
+                                'by yield within a "with StackContext" block)'))
+                except (StopIteration, Return) as e:
+                    future.set_result(getattr(e, 'value', None))
+                except Exception:
+                    future.set_exc_info(sys.exc_info())
+                else:
+                    Runner(result, future, yielded)
                 return future
-            else:
-                if isinstance(result, types.GeneratorType):
-                    def final_callback(value):
-                        deactivate()
-                        future.set_result(value)
-                    runner = Runner(result, final_callback)
-                    runner.run()
-                    return future
-            deactivate()
-            future.set_result(result)
+        future.set_result(result)
         return future
     return wrapper
 
@@ -348,7 +340,7 @@ class WaitAll(YieldPoint):
         return [self.runner.pop_result(key) for key in self.keys]
 
 
-class Task(YieldPoint):
+def Task(func, *args, **kwargs):
     """Runs a single asynchronous operation.
 
     Takes a function (and optional additional arguments) and runs it with
@@ -362,25 +354,25 @@ class Task(YieldPoint):
 
         func(args, callback=(yield gen.Callback(key)))
         result = yield gen.Wait(key)
-    """
-    def __init__(self, func, *args, **kwargs):
-        assert "callback" not in kwargs
-        self.args = args
-        self.kwargs = kwargs
-        self.func = func
-
-    def start(self, runner):
-        self.runner = runner
-        self.key = object()
-        runner.register_callback(self.key)
-        self.kwargs["callback"] = runner.result_callback(self.key)
-        self.func(*self.args, **self.kwargs)
 
-    def is_ready(self):
-        return self.runner.is_ready(self.key)
-
-    def get_result(self):
-        return self.runner.pop_result(self.key)
+    .. versionchanged:: 3.3
+       ``gen.Task`` is now a function that returns a `.Future`, instead of
+       a subclass of `YieldPoint`.  It still behaves the same way when
+       yielded.
+    """
+    future = Future()
+    def handle_exception(typ, value, tb):
+        if future.done():
+            return False
+        future.set_exc_info((typ, value, tb))
+        return True
+    def set_result(result):
+        if future.done():
+            return
+        future.set_result(result)
+    with stack_context.ExceptionStackContext(handle_exception):
+        func(*args, callback=_argument_adapter(set_result), **kwargs)
+    return future
 
 
 class YieldFuture(YieldPoint):
@@ -414,10 +406,14 @@ class YieldFuture(YieldPoint):
 class Multi(YieldPoint):
     """Runs multiple asynchronous operations in parallel.
 
-    Takes a list of ``Tasks`` or other ``YieldPoints`` and returns a list of
+    Takes a list of ``YieldPoints`` or ``Futures`` and returns a list of
     their responses.  It is not necessary to call `Multi` explicitly,
     since the engine will do so automatically when the generator yields
-    a list of ``YieldPoints``.
+    a list of ``YieldPoints`` or a mixture of ``YieldPoints`` and ``Futures``.
+
+    Instead of a list, the argument may also be a dictionary whose values are
+    Futures, in which case a parallel dictionary is returned mapping the same
+    keys to their results.
     """
     def __init__(self, children):
         self.keys = None
@@ -426,7 +422,7 @@ class Multi(YieldPoint):
             children = children.values()
         self.children = []
         for i in children:
-            if isinstance(i, Future):
+            if is_future(i):
                 i = YieldFuture(i)
             self.children.append(i)
         assert all(isinstance(i, YieldPoint) for i in self.children)
@@ -450,18 +446,127 @@ class Multi(YieldPoint):
             return list(result)
 
 
-class _NullYieldPoint(YieldPoint):
-    def start(self, runner):
-        pass
+def multi_future(children):
+    """Wait for multiple asynchronous futures in parallel.
 
-    def is_ready(self):
-        return True
+    Takes a list of ``Futures`` (but *not* other ``YieldPoints``) and returns
+    a new Future that resolves when all the other Futures are done.
+    If all the ``Futures`` succeeded, the returned Future's result is a list
+    of their results.  If any failed, the returned Future raises the exception
+    of the first one to fail.
 
-    def get_result(self):
-        return None
+    Instead of a list, the argument may also be a dictionary whose values are
+    Futures, in which case a parallel dictionary is returned mapping the same
+    keys to their results.
 
+    It is not necessary to call `multi_future` explcitly, since the engine will
+    do so automatically when the generator yields a list of `Futures`.
+    This function is faster than the `Multi` `YieldPoint` because it does not
+    require the creation of a stack context.
 
-_null_yield_point = _NullYieldPoint()
+    .. versionadded:: 3.3
+    """
+    if isinstance(children, dict):
+        keys = list(children.keys())
+        children = children.values()
+    else:
+        keys = None
+    assert all(is_future(i) for i in children)
+    unfinished_children = set(children)
+
+    future = Future()
+    if not children:
+        future.set_result({} if keys is not None else [])
+    def callback(f):
+        unfinished_children.remove(f)
+        if not unfinished_children:
+            try:
+                result_list = [i.result() for i in children]
+            except Exception:
+                future.set_exc_info(sys.exc_info())
+            else:
+                if keys is not None:
+                    future.set_result(dict(zip(keys, result_list)))
+                else:
+                    future.set_result(result_list)
+    for f in children:
+        f.add_done_callback(callback)
+    return future
+
+
+def maybe_future(x):
+    """Converts ``x`` into a `.Future`.
+
+    If ``x`` is already a `.Future`, it is simply returned; otherwise
+    it is wrapped in a new `.Future`.  This is suitable for use as
+    ``result = yield gen.maybe_future(f())`` when you don't know whether
+    ``f()`` returns a `.Future` or not.
+    """
+    if is_future(x):
+        return x
+    else:
+        fut = Future()
+        fut.set_result(x)
+        return fut
+
+
+def with_timeout(timeout, future, io_loop=None):
+    """Wraps a `.Future` in a timeout.
+
+    Raises `TimeoutError` if the input future does not complete before
+    ``timeout``, which may be specified in any form allowed by
+    `.IOLoop.add_timeout` (i.e. a `datetime.timedelta` or an absolute time
+    relative to `.IOLoop.time`)
+
+    Currently only supports Futures, not other `YieldPoint` classes.
+
+    .. versionadded:: 3.3
+    """
+    # TODO: allow yield points in addition to futures?
+    # Tricky to do with stack_context semantics.
+    #
+    # It's tempting to optimize this by cancelling the input future on timeout
+    # instead of creating a new one, but A) we can't know if we are the only
+    # one waiting on the input future, so cancelling it might disrupt other
+    # callers and B) concurrent futures can only be cancelled while they are
+    # in the queue, so cancellation cannot reliably bound our waiting time.
+    result = Future()
+    chain_future(future, result)
+    if io_loop is None:
+        io_loop = IOLoop.current()
+    timeout_handle = io_loop.add_timeout(
+        timeout,
+        lambda: result.set_exception(TimeoutError("Timeout")))
+    if isinstance(future, Future):
+        # We know this future will resolve on the IOLoop, so we don't
+        # need the extra thread-safety of IOLoop.add_future (and we also
+        # don't care about StackContext here.
+        future.add_done_callback(
+            lambda future: io_loop.remove_timeout(timeout_handle))
+    else:
+        # concurrent.futures.Futures may resolve on any thread, so we
+        # need to route them back to the IOLoop.
+        io_loop.add_future(
+            future, lambda future: io_loop.remove_timeout(timeout_handle))
+    return result
+
+
+_null_future = Future()
+_null_future.set_result(None)
+
+moment = Future()
+moment.__doc__ = \
+    """A special object which may be yielded to allow the IOLoop to run for
+one iteration.
+
+This is not needed in normal use but it can be helpful in long-running
+coroutines that are likely to yield Futures that are ready instantly.
+
+Usage: ``yield gen.moment``
+
+.. versionadded:: 3.3
+"""
+moment.set_result(None)
 
 
 class Runner(object):
@@ -469,35 +574,55 @@ class Runner(object):
 
     Maintains information about pending callbacks and their results.
 
-    ``final_callback`` is run after the generator exits.
+    The results of the generator are stored in ``result_future`` (a
+    `.TracebackFuture`)
     """
-    def __init__(self, gen, final_callback):
+    def __init__(self, gen, result_future, first_yielded):
         self.gen = gen
-        self.final_callback = final_callback
-        self.yield_point = _null_yield_point
-        self.pending_callbacks = set()
-        self.results = {}
+        self.result_future = result_future
+        self.future = _null_future
+        self.yield_point = None
+        self.pending_callbacks = None
+        self.results = None
         self.running = False
         self.finished = False
-        self.exc_info = None
         self.had_exception = False
+        self.io_loop = IOLoop.current()
+        # For efficiency, we do not create a stack context until we
+        # reach a YieldPoint (stack contexts are required for the historical
+        # semantics of YieldPoints, but not for Futures).  When we have
+        # done so, this field will be set and must be called at the end
+        # of the coroutine.
+        self.stack_context_deactivate = None
+        if self.handle_yield(first_yielded):
+            self.run()
 
     def register_callback(self, key):
         """Adds ``key`` to the list of callbacks."""
+        if self.pending_callbacks is None:
+            # Lazily initialize the old-style YieldPoint data structures.
+            self.pending_callbacks = set()
+            self.results = {}
         if key in self.pending_callbacks:
             raise KeyReuseError("key %r is already pending" % (key,))
         self.pending_callbacks.add(key)
 
     def is_ready(self, key):
         """Returns true if a result is available for ``key``."""
-        if key not in self.pending_callbacks:
+        if self.pending_callbacks is None or key not in self.pending_callbacks:
             raise UnknownKeyError("key %r is not pending" % (key,))
         return key in self.results
 
     def set_result(self, key, result):
         """Sets the result for ``key`` and attempts to resume the generator."""
         self.results[key] = result
-        self.run()
+        if self.yield_point is not None and self.yield_point.is_ready():
+            try:
+                self.future.set_result(self.yield_point.get_result())
+            except:
+                self.future.set_exc_info(sys.exc_info())
+            self.yield_point = None
+            self.run()
 
     def pop_result(self, key):
         """Returns the result for ``key`` and unregisters it."""
@@ -513,25 +638,27 @@ class Runner(object):
         try:
             self.running = True
             while True:
-                if self.exc_info is None:
+                future = self.future
+                if not future.done():
+                    return
+                self.future = None
+                try:
+                    orig_stack_contexts = stack_context._state.contexts
                     try:
-                        if not self.yield_point.is_ready():
-                            return
-                        next = self.yield_point.get_result()
-                        self.yield_point = None
+                        value = future.result()
                     except Exception:
-                        self.exc_info = sys.exc_info()
-                try:
-                    if self.exc_info is not None:
                         self.had_exception = True
-                        exc_info = self.exc_info
-                        self.exc_info = None
-                        yielded = self.gen.throw(*exc_info)
+                        yielded = self.gen.throw(*sys.exc_info())
                     else:
-                        yielded = self.gen.send(next)
+                        yielded = self.gen.send(value)
+                    if stack_context._state.contexts is not orig_stack_contexts:
+                        self.gen.throw(
+                            stack_context.StackContextInconsistentError(
+                                'stack_context inconsistency (probably caused '
+                                'by yield within a "with StackContext" block)'))
                 except (StopIteration, Return) as e:
                     self.finished = True
-                    self.yield_point = _null_yield_point
+                    self.future = _null_future
                     if self.pending_callbacks and not self.had_exception:
                         # If we ran cleanly without waiting on all callbacks
                         # raise an error (really more of a warning).  If we
@@ -540,46 +667,105 @@ class Runner(object):
                         raise LeakedCallbackError(
                             "finished without waiting for callbacks %r" %
                             self.pending_callbacks)
-                    self.final_callback(getattr(e, 'value', None))
-                    self.final_callback = None
+                    self.result_future.set_result(getattr(e, 'value', None))
+                    self.result_future = None
+                    self._deactivate_stack_context()
                     return
                 except Exception:
                     self.finished = True
-                    self.yield_point = _null_yield_point
-                    raise
-                if isinstance(yielded, (list, dict)):
-                    yielded = Multi(yielded)
-                elif isinstance(yielded, Future):
-                    yielded = YieldFuture(yielded)
-                if isinstance(yielded, YieldPoint):
-                    self.yield_point = yielded
-                    try:
-                        self.yield_point.start(self)
-                    except Exception:
-                        self.exc_info = sys.exc_info()
-                else:
-                    self.exc_info = (BadYieldError(
-                        "yielded unknown object %r" % (yielded,)),)
+                    self.future = _null_future
+                    self.result_future.set_exc_info(sys.exc_info())
+                    self.result_future = None
+                    self._deactivate_stack_context()
+                    return
+                if not self.handle_yield(yielded):
+                    return
         finally:
             self.running = False
 
-    def result_callback(self, key):
-        def inner(*args, **kwargs):
-            if kwargs or len(args) > 1:
-                result = Arguments(args, kwargs)
-            elif args:
-                result = args[0]
+    def handle_yield(self, yielded):
+        if isinstance(yielded, list):
+            if all(is_future(f) for f in yielded):
+                yielded = multi_future(yielded)
+            else:
+                yielded = Multi(yielded)
+        elif isinstance(yielded, dict):
+            if all(is_future(f) for f in yielded.values()):
+                yielded = multi_future(yielded)
             else:
-                result = None
-            self.set_result(key, result)
-        return wrap(inner)
+                yielded = Multi(yielded)
+
+        if isinstance(yielded, YieldPoint):
+            self.future = TracebackFuture()
+            def start_yield_point():
+                try:
+                    yielded.start(self)
+                    if yielded.is_ready():
+                        self.future.set_result(
+                            yielded.get_result())
+                    else:
+                        self.yield_point = yielded
+                except Exception:
+                    self.future = TracebackFuture()
+                    self.future.set_exc_info(sys.exc_info())
+            if self.stack_context_deactivate is None:
+                # Start a stack context if this is the first
+                # YieldPoint we've seen.
+                with stack_context.ExceptionStackContext(
+                        self.handle_exception) as deactivate:
+                    self.stack_context_deactivate = deactivate
+                    def cb():
+                        start_yield_point()
+                        self.run()
+                    self.io_loop.add_callback(cb)
+                    return False
+            else:
+                start_yield_point()
+        elif is_future(yielded):
+            self.future = yielded
+            if not self.future.done() or self.future is moment:
+                self.io_loop.add_future(
+                    self.future, lambda f: self.run())
+                return False
+        else:
+            self.future = TracebackFuture()
+            self.future.set_exception(BadYieldError(
+                "yielded unknown object %r" % (yielded,)))
+        return True
+
+    def result_callback(self, key):
+        return stack_context.wrap(_argument_adapter(
+            functools.partial(self.set_result, key)))
 
     def handle_exception(self, typ, value, tb):
         if not self.running and not self.finished:
-            self.exc_info = (typ, value, tb)
+            self.future = TracebackFuture()
+            self.future.set_exc_info((typ, value, tb))
             self.run()
             return True
         else:
             return False
 
+    def _deactivate_stack_context(self):
+        if self.stack_context_deactivate is not None:
+            self.stack_context_deactivate()
+            self.stack_context_deactivate = None
+
 Arguments = collections.namedtuple('Arguments', ['args', 'kwargs'])
+
+
+def _argument_adapter(callback):
+    """Returns a function that when invoked runs ``callback`` with one arg.
+
+    If the function returned by this function is called with exactly
+    one argument, that argument is passed to ``callback``.  Otherwise
+    the args tuple and kwargs dict are wrapped in an `Arguments` object.
+    """
+    def wrapper(*args, **kwargs):
+        if kwargs or len(args) > 1:
+            callback(Arguments(args, kwargs))
+        elif args:
+            callback(args[0])
+        else:
+            callback(None)
+    return wrapper

libs/tornado/http1connection.py

@@ -0,0 +1,624 @@
+#!/usr/bin/env python
+#
+# Copyright 2014 Facebook
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+"""Client and server implementations of HTTP/1.x.
+
+.. versionadded:: 3.3
+"""
+
+from __future__ import absolute_import, division, print_function, with_statement
+
+from tornado.concurrent import Future
+from tornado.escape import native_str, utf8
+from tornado import gen
+from tornado import httputil
+from tornado import iostream
+from tornado.log import gen_log, app_log
+from tornado import stack_context
+from tornado.util import GzipDecompressor
+
+
+class HTTP1ConnectionParameters(object):
+    """Parameters for `.HTTP1Connection` and `.HTTP1ServerConnection`.
+    """
+    def __init__(self, no_keep_alive=False, chunk_size=None,
+                 max_header_size=None, header_timeout=None, max_body_size=None,
+                 body_timeout=None, use_gzip=False):
+        """
+        :arg bool no_keep_alive: If true, always close the connection after
+            one request.
+        :arg int chunk_size: how much data to read into memory at once
+        :arg int max_header_size:  maximum amount of data for HTTP headers
+        :arg float header_timeout: how long to wait for all headers (seconds)
+        :arg int max_body_size: maximum amount of data for body
+        :arg float body_timeout: how long to wait while reading body (seconds)
+        :arg bool use_gzip: if true, decode incoming ``Content-Encoding: gzip``
+        """
+        self.no_keep_alive = no_keep_alive
+        self.chunk_size = chunk_size or 65536
+        self.max_header_size = max_header_size or 65536
+        self.header_timeout = header_timeout
+        self.max_body_size = max_body_size
+        self.body_timeout = body_timeout
+        self.use_gzip = use_gzip
+
+
+class HTTP1Connection(httputil.HTTPConnection):
+    """Implements the HTTP/1.x protocol.
+
+    This class can be on its own for clients, or via `HTTP1ServerConnection`
+    for servers.
+    """
+    def __init__(self, stream, is_client, params=None, context=None):
+        """
+        :arg stream: an `.IOStream`
+        :arg bool is_client: client or server
+        :arg params: a `.HTTP1ConnectionParameters` instance or ``None``
+        :arg context: an opaque application-defined object that can be accessed
+            as ``connection.context``.
+        """
+        self.is_client = is_client
+        self.stream = stream
+        if params is None:
+            params = HTTP1ConnectionParameters()
+        self.params = params
+        self.context = context
+        self.no_keep_alive = params.no_keep_alive
+        # The body limits can be altered by the delegate, so save them
+        # here instead of just referencing self.params later.
+        self._max_body_size = (self.params.max_body_size or
+                               self.stream.max_buffer_size)
+        self._body_timeout = self.params.body_timeout
+        # _write_finished is set to True when finish() has been called,
+        # i.e. there will be no more data sent.  Data may still be in the
+        # stream's write buffer.
+        self._write_finished = False
+        # True when we have read the entire incoming body.
+        self._read_finished = False
+        # _finish_future resolves when all data has been written and flushed
+        # to the IOStream.
+        self._finish_future = Future()
+        # If true, the connection should be closed after this request
+        # (after the response has been written in the server side,
+        # and after it has been read in the client)
+        self._disconnect_on_finish = False
+        self._clear_callbacks()
+        # Save the start lines after we read or write them; they
+        # affect later processing (e.g. 304 responses and HEAD methods
+        # have content-length but no bodies)
+        self._request_start_line = None
+        self._response_start_line = None
+        self._request_headers = None
+        # True if we are writing output with chunked encoding.
+        self._chunking_output = None
+        # While reading a body with a content-length, this is the
+        # amount left to read.
+        self._expected_content_remaining = None
+        # A Future for our outgoing writes, returned by IOStream.write.
+        self._pending_write = None
+
+    def read_response(self, delegate):
+        """Read a single HTTP response.
+
+        Typical client-mode usage is to write a request using `write_headers`,
+        `write`, and `finish`, and then call ``read_response``.
+
+        :arg delegate: a `.HTTPMessageDelegate`
+
+        Returns a `.Future` that resolves to None after the full response has
+        been read.
+        """
+        if self.params.use_gzip:
+            delegate = _GzipMessageDelegate(delegate, self.params.chunk_size)
+        return self._read_message(delegate)
+
+    @gen.coroutine
+    def _read_message(self, delegate):
+        need_delegate_close = False
+        try:
+            header_future = self.stream.read_until_regex(
+                b"\r?\n\r?\n",
+                max_bytes=self.params.max_header_size)
+            if self.params.header_timeout is None:
+                header_data = yield header_future
+            else:
+                try:
+                    header_data = yield gen.with_timeout(
+                        self.stream.io_loop.time() + self.params.header_timeout,
+                        header_future,
+                        io_loop=self.stream.io_loop)
+                except gen.TimeoutError:
+                    self.close()
+                    raise gen.Return(False)
+            start_line, headers = self._parse_headers(header_data)
+            if self.is_client:
+                start_line = httputil.parse_response_start_line(start_line)
+                self._response_start_line = start_line
+            else:
+                start_line = httputil.parse_request_start_line(start_line)
+                self._request_start_line = start_line
+                self._request_headers = headers
+
+            self._disconnect_on_finish = not self._can_keep_alive(
+                start_line, headers)
+            need_delegate_close = True
+            header_future = delegate.headers_received(start_line, headers)
+            if header_future is not None:
+                yield header_future
+            if self.stream is None:
+                # We've been detached.
+                need_delegate_close = False
+                raise gen.Return(False)
+            skip_body = False
+            if self.is_client:
+                if (self._request_start_line is not None and
+                        self._request_start_line.method == 'HEAD'):
+                    skip_body = True
+                code = start_line.code
+                if code == 304:
+                    skip_body = True
+                if code >= 100 and code < 200:
+                    # TODO: client delegates will get headers_received twice
+                    # in the case of a 100-continue.  Document or change?
+                    yield self._read_message(delegate)
+            else:
+                if (headers.get("Expect") == "100-continue" and
+                        not self._write_finished):
+                    self.stream.write(b"HTTP/1.1 100 (Continue)\r\n\r\n")
+            if not skip_body:
+                body_future = self._read_body(headers, delegate)
+                if body_future is not None:
+                    if self._body_timeout is None:
+                        yield body_future
+                    else:
+                        try:
+                            yield gen.with_timeout(
+                                self.stream.io_loop.time() + self._body_timeout,
+                                body_future, self.stream.io_loop)
+                        except gen.TimeoutError:
+                            gen_log.info("Timeout reading body from %s",
+                                         self.context)
+                            self.stream.close()
+                            raise gen.Return(False)
+            self._read_finished = True
+            if not self._write_finished or self.is_client:
+                need_delegate_close = False
+                delegate.finish()
+            # If we're waiting for the application to produce an asynchronous
+            # response, and we're not detached, register a close callback
+            # on the stream (we didn't need one while we were reading)
+            if (not self._finish_future.done() and
+                    self.stream is not None and
+                    not self.stream.closed()):
+                self.stream.set_close_callback(self._on_connection_close)
+                yield self._finish_future
+            if self.is_client and self._disconnect_on_finish:
+                self.close()
+            if self.stream is None:
+                raise gen.Return(False)
+        except httputil.HTTPInputException as e:
+            gen_log.info("Malformed HTTP message from %s: %s",
+                         self.context, e)
+            self.close()
+            raise gen.Return(False)
+        finally:
+            if need_delegate_close:
+                delegate.on_connection_close()
+            self._clear_callbacks()
+        raise gen.Return(True)
+
+    def _clear_callbacks(self):
+        """Clears the callback attributes.
+
+        This allows the request handler to be garbage collected more
+        quickly in CPython by breaking up reference cycles.
+        """
+        self._write_callback = None
+        self._write_future = None
+        self._close_callback = None
+        if self.stream is not None:
+            self.stream.set_close_callback(None)
+
+    def set_close_callback(self, callback):
+        """Sets a callback that will be run when the connection is closed.
+
+        .. deprecated:: 3.3
+            Use `.HTTPMessageDelegate.on_connection_close` instead.
+        """
+        self._close_callback = stack_context.wrap(callback)
+
+    def _on_connection_close(self):
+        # Note that this callback is only registered on the IOStream
+        # when we have finished reading the request and are waiting for
+        # the application to produce its response.
+        if self._close_callback is not None:
+            callback = self._close_callback
+            self._close_callback = None
+            callback()
+        if not self._finish_future.done():
+            self._finish_future.set_result(None)
+        self._clear_callbacks()
+
+    def close(self):
+        if self.stream is not None:
+            self.stream.close()
+        self._clear_callbacks()
+        if not self._finish_future.done():
+            self._finish_future.set_result(None)
+
+    def detach(self):
+        """Take control of the underlying stream.
+
+        Returns the underlying `.IOStream` object and stops all further
+        HTTP processing.  May only be called during
+        `.HTTPMessageDelegate.headers_received`.  Intended for implementing
+        protocols like websockets that tunnel over an HTTP handshake.
+        """
+        self._clear_callbacks()
+        stream = self.stream
+        self.stream = None
+        return stream
+
+    def set_body_timeout(self, timeout):
+        """Sets the body timeout for a single request.
+
+        Overrides the value from `.HTTP1ConnectionParameters`.
+        """
+        self._body_timeout = timeout
+
+    def set_max_body_size(self, max_body_size):
+        """Sets the body size limit for a single request.
+
+        Overrides the value from `.HTTP1ConnectionParameters`.
+        """
+        self._max_body_size = max_body_size
+
+    def write_headers(self, start_line, headers, chunk=None, callback=None):
+        """Implements `.HTTPConnection.write_headers`."""
+        if self.is_client:
+            self._request_start_line = start_line
+            # Client requests with a non-empty body must have either a
+            # Content-Length or a Transfer-Encoding.
+            self._chunking_output = (
+                start_line.method in ('POST', 'PUT', 'PATCH') and
+                'Content-Length' not in headers and
+                'Transfer-Encoding' not in headers)
+        else:
+            self._response_start_line = start_line
+            self._chunking_output = (
+                # TODO: should this use
+                # self._request_start_line.version or
+                # start_line.version?
+                self._request_start_line.version == 'HTTP/1.1' and
+                # 304 responses have no body (not even a zero-length body), and so
+                # should not have either Content-Length or Transfer-Encoding.
+                # headers.
+                start_line.code != 304 and
+                # No need to chunk the output if a Content-Length is specified.
+                'Content-Length' not in headers and
+                # Applications are discouraged from touching Transfer-Encoding,
+                # but if they do, leave it alone.
+                'Transfer-Encoding' not in headers)
+            # If a 1.0 client asked for keep-alive, add the header.
+            if (self._request_start_line.version == 'HTTP/1.0' and
+                (self._request_headers.get('Connection', '').lower()
+                 == 'keep-alive')):
+                headers['Connection'] = 'Keep-Alive'
+        if self._chunking_output:
+            headers['Transfer-Encoding'] = 'chunked'
+        if (not self.is_client and
+            (self._request_start_line.method == 'HEAD' or
+             start_line.code == 304)):
+            self._expected_content_remaining = 0
+        elif 'Content-Length' in headers:
+            self._expected_content_remaining = int(headers['Content-Length'])
+        else:
+            self._expected_content_remaining = None
+        lines = [utf8("%s %s %s" % start_line)]
+        lines.extend([utf8(n) + b": " + utf8(v) for n, v in headers.get_all()])
+        for line in lines:
+            if b'\n' in line:
+                raise ValueError('Newline in header: ' + repr(line))
+        future = None
+        if self.stream.closed():
+            future = self._write_future = Future()
+            future.set_exception(iostream.StreamClosedError())
+        else:
+            if callback is not None:
+                self._write_callback = stack_context.wrap(callback)
+            else:
+                future = self._write_future = Future()
+            data = b"\r\n".join(lines) + b"\r\n\r\n"
+            if chunk:
+                data += self._format_chunk(chunk)
+            self._pending_write = self.stream.write(data)
+            self._pending_write.add_done_callback(self._on_write_complete)
+        return future
+
+    def _format_chunk(self, chunk):
+        if self._expected_content_remaining is not None:
+            self._expected_content_remaining -= len(chunk)
+            if self._expected_content_remaining < 0:
+                # Close the stream now to stop further framing errors.
+                self.stream.close()
+                raise httputil.HTTPOutputException(
+                    "Tried to write more data than Content-Length")
+        if self._chunking_output and chunk:
+            # Don't write out empty chunks because that means END-OF-STREAM
+            # with chunked encoding
+            return utf8("%x" % len(chunk)) + b"\r\n" + chunk + b"\r\n"
+        else:
+            return chunk
+
+    def write(self, chunk, callback=None):
+        """Implements `.HTTPConnection.write`.
+
+        For backwards compatibility is is allowed but deprecated to
+        skip `write_headers` and instead call `write()` with a
+        pre-encoded header block.
+        """
+        future = None
+        if self.stream.closed():
+            future = self._write_future = Future()
+            self._write_future.set_exception(iostream.StreamClosedError())
+        else:
+            if callback is not None:
+                self._write_callback = stack_context.wrap(callback)
+            else:
+                future = self._write_future = Future()
+            self._pending_write = self.stream.write(self._format_chunk(chunk))
+            self._pending_write.add_done_callback(self._on_write_complete)
+        return future
+
+    def finish(self):
+        """Implements `.HTTPConnection.finish`."""
+        if (self._expected_content_remaining is not None and
+                self._expected_content_remaining != 0 and
+                not self.stream.closed()):
+            self.stream.close()
+            raise httputil.HTTPOutputException(
+                "Tried to write %d bytes less than Content-Length" %
+                self._expected_content_remaining)
+        if self._chunking_output:
+            if not self.stream.closed():
+                self._pending_write = self.stream.write(b"0\r\n\r\n")
+                self._pending_write.add_done_callback(self._on_write_complete)
+        self._write_finished = True
+        # If the app finished the request while we're still reading,
+        # divert any remaining data away from the delegate and
+        # close the connection when we're done sending our response.
+        # Closing the connection is the only way to avoid reading the
+        # whole input body.
+        if not self._read_finished:
+            self._disconnect_on_finish = True
+        # No more data is coming, so instruct TCP to send any remaining
+        # data immediately instead of waiting for a full packet or ack.
+        self.stream.set_nodelay(True)
+        if self._pending_write is None:
+            self._finish_request(None)
+        else:
+            self._pending_write.add_done_callback(self._finish_request)
+
+    def _on_write_complete(self, future):
+        if self._write_callback is not None:
+            callback = self._write_callback
+            self._write_callback = None
+            self.stream.io_loop.add_callback(callback)
+        if self._write_future is not None:
+            future = self._write_future
+            self._write_future = None
+            future.set_result(None)
+
+    def _can_keep_alive(self, start_line, headers):
+        if self.params.no_keep_alive:
+            return False
+        connection_header = headers.get("Connection")
+        if connection_header is not None:
+            connection_header = connection_header.lower()
+        if start_line.version == "HTTP/1.1":
+            return connection_header != "close"
+        elif ("Content-Length" in headers
+              or start_line.method in ("HEAD", "GET")):
+            return connection_header == "keep-alive"
+        return False
+
+    def _finish_request(self, future):
+        self._clear_callbacks()
+        if not self.is_client and self._disconnect_on_finish:
+            self.close()
+            return
+        # Turn Nagle's algorithm back on, leaving the stream in its
+        # default state for the next request.
+        self.stream.set_nodelay(False)
+        if not self._finish_future.done():
+            self._finish_future.set_result(None)
+
+    def _parse_headers(self, data):
+        data = native_str(data.decode('latin1'))
+        eol = data.find("\r\n")
+        start_line = data[:eol]
+        try:
+            headers = httputil.HTTPHeaders.parse(data[eol:])
+        except ValueError:
+            # probably form split() if there was no ':' in the line
+            raise httputil.HTTPInputException("Malformed HTTP headers: %r" %
+                                              data[eol:100])
+        return start_line, headers
+
+    def _read_body(self, headers, delegate):
+        content_length = headers.get("Content-Length")
+        if content_length:
+            content_length = int(content_length)
+            if content_length > self._max_body_size:
+                raise httputil.HTTPInputException("Content-Length too long")
+            return self._read_fixed_body(content_length, delegate)
+        if headers.get("Transfer-Encoding") == "chunked":
+            return self._read_chunked_body(delegate)
+        if self.is_client:
+            return self._read_body_until_close(delegate)
+        return None
+
+    @gen.coroutine
+    def _read_fixed_body(self, content_length, delegate):
+        while content_length > 0:
+            body = yield self.stream.read_bytes(
+                min(self.params.chunk_size, content_length), partial=True)
+            content_length -= len(body)
+            if not self._write_finished or self.is_client:
+                yield gen.maybe_future(delegate.data_received(body))
+
+    @gen.coroutine
+    def _read_chunked_body(self, delegate):
+        # TODO: "chunk extensions" http://tools.ietf.org/html/rfc2616#section-3.6.1
+        total_size = 0
+        while True:
+            chunk_len = yield self.stream.read_until(b"\r\n", max_bytes=64)
+            chunk_len = int(chunk_len.strip(), 16)
+            if chunk_len == 0:
+                return
+            total_size += chunk_len
+            if total_size > self._max_body_size:
+                raise httputil.HTTPInputException("chunked body too large")
+            bytes_to_read = chunk_len
+            while bytes_to_read:
+                chunk = yield self.stream.read_bytes(
+                    min(bytes_to_read, self.params.chunk_size), partial=True)
+                bytes_to_read -= len(chunk)
+                if not self._write_finished or self.is_client:
+                    yield gen.maybe_future(
+                        delegate.data_received(chunk))
+            # chunk ends with \r\n
+            crlf = yield self.stream.read_bytes(2)
+            assert crlf == b"\r\n"
+
+    @gen.coroutine
+    def _read_body_until_close(self, delegate):
+        body = yield self.stream.read_until_close()
+        if not self._write_finished or self.is_client:
+            delegate.data_received(body)
+
+
+class _GzipMessageDelegate(httputil.HTTPMessageDelegate):
+    """Wraps an `HTTPMessageDelegate` to decode ``Content-Encoding: gzip``.
+    """
+    def __init__(self, delegate, chunk_size):
+        self._delegate = delegate
+        self._chunk_size = chunk_size
+        self._decompressor = None
+
+    def headers_received(self, start_line, headers):
+        if headers.get("Content-Encoding") == "gzip":
+            self._decompressor = GzipDecompressor()
+            # Downstream delegates will only see uncompressed data,
+            # so rename the content-encoding header.
+            # (but note that curl_httpclient doesn't do this).
+            headers.add("X-Consumed-Content-Encoding",
+                        headers["Content-Encoding"])
+            del headers["Content-Encoding"]
+        return self._delegate.headers_received(start_line, headers)
+
+    @gen.coroutine
+    def data_received(self, chunk):
+        if self._decompressor:
+            compressed_data = chunk
+            while compressed_data:
+                decompressed = self._decompressor.decompress(
+                    compressed_data, self._chunk_size)
+                if decompressed:
+                    yield gen.maybe_future(
+                        self._delegate.data_received(decompressed))
+                compressed_data = self._decompressor.unconsumed_tail
+        else:
+            yield gen.maybe_future(self._delegate.data_received(chunk))
+
+    def finish(self):
+        if self._decompressor is not None:
+            tail = self._decompressor.flush()
+            if tail:
+                # I believe the tail will always be empty (i.e.
+                # decompress will return all it can).  The purpose
+                # of the flush call is to detect errors such
+                # as truncated input.  But in case it ever returns
+                # anything, treat it as an extra chunk
+                self._delegate.data_received(tail)
+        return self._delegate.finish()
+
+
+class HTTP1ServerConnection(object):
+    """An HTTP/1.x server."""
+    def __init__(self, stream, params=None, context=None):
+        """
+        :arg stream: an `.IOStream`
+        :arg params: a `.HTTP1ConnectionParameters` or None
+        :arg context: an opaque application-defined object that is accessible
+            as ``connection.context``
+        """
+        self.stream = stream
+        if params is None:
+            params = HTTP1ConnectionParameters()
+        self.params = params
+        self.context = context
+        self._serving_future = None
+
+    @gen.coroutine
+    def close(self):
+        """Closes the connection.
+
+        Returns a `.Future` that resolves after the serving loop has exited.
+        """
+        self.stream.close()
+        # Block until the serving loop is done, but ignore any exceptions
+        # (start_serving is already responsible for logging them).
+        try:
+            yield self._serving_future
+        except Exception:
+            pass
+
+    def start_serving(self, delegate):
+        """Starts serving requests on this connection.
+
+        :arg delegate: a `.HTTPServerConnectionDelegate`
+        """
+        assert isinstance(delegate, httputil.HTTPServerConnectionDelegate)
+        self._serving_future = self._server_request_loop(delegate)
+        # Register the future on the IOLoop so its errors get logged.
+        self.stream.io_loop.add_future(self._serving_future,
+                                       lambda f: f.result())
+
+    @gen.coroutine
+    def _server_request_loop(self, delegate):
+        try:
+            while True:
+                conn = HTTP1Connection(self.stream, False,
+                                       self.params, self.context)
+                request_delegate = delegate.start_request(self, conn)
+                try:
+                    ret = yield conn.read_response(request_delegate)
+                except (iostream.StreamClosedError,
+                        iostream.UnsatisfiableReadError):
+                    return
+                except Exception:
+                    # TODO: this is probably too broad; it would be better to
+                    # wrap all delegate calls in something that writes to app_log,
+                    # and then errors that reach this point can be gen_log.
+                    app_log.error("Uncaught exception", exc_info=True)
+                    conn.close()
+                    return
+                if not ret:
+                    return
+                yield gen.moment
+        finally:
+            delegate.on_close(self)

libs/tornado/httpclient.py

@@ -25,6 +25,11 @@ to switch to ``curl_httpclient`` for reasons such as the following:
 Note that if you are using ``curl_httpclient``, it is highly recommended that
 you use a recent version of ``libcurl`` and ``pycurl``.  Currently the minimum
 supported version is 7.18.2, and the recommended version is 7.21.1 or newer.
+It is highly recommended that your ``libcurl`` installation is built with
+asynchronous DNS resolver (threaded or c-ares), otherwise you may encounter
+various problems with request timeouts (for more information, see
+http://curl.haxx.se/libcurl/c/curl_easy_setopt.html#CURLOPTCONNECTTIMEOUTMS
+and comments in curl_httpclient.py).
 """
 
 from __future__ import absolute_import, division, print_function, with_statement
@@ -34,7 +39,7 @@ import time
 import weakref
 
 from tornado.concurrent import TracebackFuture
-from tornado.escape import utf8
+from tornado.escape import utf8, native_str
 from tornado import httputil, stack_context
 from tornado.ioloop import IOLoop
 from tornado.util import Configurable
@@ -166,7 +171,7 @@ class AsyncHTTPClient(Configurable):
         kwargs: ``HTTPRequest(request, **kwargs)``
 
         This method returns a `.Future` whose result is an
-        `HTTPResponse`.  The ``Future`` wil raise an `HTTPError` if
+        `HTTPResponse`.  The ``Future`` will raise an `HTTPError` if
         the request returned a non-200 response code.
 
         If a ``callback`` is given, it will be invoked with the `HTTPResponse`.
@@ -259,14 +264,27 @@ class HTTPRequest(object):
                  proxy_password=None, allow_nonstandard_methods=None,
                  validate_cert=None, ca_certs=None,
                  allow_ipv6=None,
-                 client_key=None, client_cert=None):
+                 client_key=None, client_cert=None, body_producer=None,
+                 expect_100_continue=False):
         r"""All parameters except ``url`` are optional.
 
         :arg string url: URL to fetch
         :arg string method: HTTP method, e.g. "GET" or "POST"
         :arg headers: Additional HTTP headers to pass on the request
-        :arg body: HTTP body to pass on the request
         :type headers: `~tornado.httputil.HTTPHeaders` or `dict`
+        :arg body: HTTP request body as a string (byte or unicode; if unicode
+           the utf-8 encoding will be used)
+        :arg body_producer: Callable used for lazy/asynchronous request bodies.
+           It is called with one argument, a ``write`` function, and should
+           return a `.Future`.  It should call the write function with new
+           data as it becomes available.  The write function returns a
+           `.Future` which can be used for flow control.
+           Only one of ``body`` and ``body_producer`` may
+           be specified.  ``body_producer`` is not supported on
+           ``curl_httpclient``.  When using ``body_producer`` it is recommended
+           to pass a ``Content-Length`` in the headers as otherwise chunked
+           encoding will be used, and many servers do not support chunked
+           encoding on requests.  New in Tornado 3.3
         :arg string auth_username: Username for HTTP authentication
         :arg string auth_password: Password for HTTP authentication
         :arg string auth_mode: Authentication mode; default is "basic".
@@ -319,6 +337,11 @@ class HTTPRequest(object):
            note below when used with ``curl_httpclient``.
         :arg string client_cert: Filename for client SSL certificate, if any.
            See note below when used with ``curl_httpclient``.
+        :arg bool expect_100_continue: If true, send the
+           ``Expect: 100-continue`` header and wait for a continue response
+           before sending the request body.  Only supported with
+           simple_httpclient.
+
 
         .. note::
 
@@ -334,6 +357,9 @@ class HTTPRequest(object):
 
         .. versionadded:: 3.1
            The ``auth_mode`` argument.
+
+        .. versionadded:: 3.3
+           The ``body_producer`` and ``expect_100_continue`` arguments.
         """
         # Note that some of these attributes go through property setters
         # defined below.
@@ -348,6 +374,7 @@ class HTTPRequest(object):
         self.url = url
         self.method = method
         self.body = body
+        self.body_producer = body_producer
         self.auth_username = auth_username
         self.auth_password = auth_password
         self.auth_mode = auth_mode
@@ -367,6 +394,7 @@ class HTTPRequest(object):
         self.allow_ipv6 = allow_ipv6
         self.client_key = client_key
         self.client_cert = client_cert
+        self.expect_100_continue = expect_100_continue
         self.start_time = time.time()
 
     @property
@@ -389,6 +417,14 @@ class HTTPRequest(object):
         self._body = utf8(value)
 
     @property
+    def body_producer(self):
+        return self._body_producer
+
+    @body_producer.setter
+    def body_producer(self, value):
+        self._body_producer = stack_context.wrap(value)
+
+    @property
     def streaming_callback(self):
         return self._streaming_callback
 
@@ -423,8 +459,6 @@ class HTTPResponse(object):
     * code: numeric HTTP status code, e.g. 200 or 404
 
     * reason: human-readable reason phrase describing the status code
-      (with curl_httpclient, this is a default value rather than the
-      server's actual response)
 
     * headers: `tornado.httputil.HTTPHeaders` object
 
@@ -466,7 +500,8 @@ class HTTPResponse(object):
             self.effective_url = effective_url
         if error is None:
             if self.code < 200 or self.code >= 300:
-                self.error = HTTPError(self.code, response=self)
+                self.error = HTTPError(self.code, message=self.reason,
+                                       response=self)
             else:
                 self.error = None
         else:
@@ -556,7 +591,7 @@ def main():
         if options.print_headers:
             print(response.headers)
         if options.print_body:
-            print(response.body)
+            print(native_str(response.body))
     client.close()
 
 if __name__ == "__main__":

libs/tornado/httpserver.py

@@ -20,70 +20,55 @@ Typical applications have little direct interaction with the `HTTPServer`
 class except to start a server at the beginning of the process
 (and even that is often done indirectly via `tornado.web.Application.listen`).
 
-This module also defines the `HTTPRequest` class which is exposed via
-`tornado.web.RequestHandler.request`.
+.. versionchanged:: 3.3
+
+   The ``HTTPRequest`` class that used to live in this module has been moved
+   to `tornado.httputil.HTTPServerRequest`.  The old name remains as an alias.
 """
 
 from __future__ import absolute_import, division, print_function, with_statement
 
 import socket
-import ssl
-import time
-import copy
 
-from tornado.escape import native_str, parse_qs_bytes
+from tornado.escape import native_str
+from tornado.http1connection import HTTP1ServerConnection, HTTP1ConnectionParameters
+from tornado import gen
 from tornado import httputil
 from tornado import iostream
-from tornado.log import gen_log
 from tornado import netutil
 from tornado.tcpserver import TCPServer
-from tornado import stack_context
-from tornado.util import bytes_type
-
-try:
-    import Cookie  # py2
-except ImportError:
-    import http.cookies as Cookie  # py3
 
 
-class HTTPServer(TCPServer):
+class HTTPServer(TCPServer, httputil.HTTPServerConnectionDelegate):
     r"""A non-blocking, single-threaded HTTP server.
 
-    A server is defined by a request callback that takes an HTTPRequest
-    instance as an argument and writes a valid HTTP response with
-    `HTTPRequest.write`. `HTTPRequest.finish` finishes the request (but does
-    not necessarily close the connection in the case of HTTP/1.1 keep-alive
-    requests). A simple example server that echoes back the URI you
-    requested::
+    A server is defined by either a request callback that takes a
+    `.HTTPServerRequest` as an argument or a `.HTTPServerConnectionDelegate`
+    instance.
+
+    A simple example server that echoes back the URI you requested::
 
         import tornado.httpserver
         import tornado.ioloop
 
         def handle_request(request):
            message = "You requested %s\n" % request.uri
-           request.write("HTTP/1.1 200 OK\r\nContent-Length: %d\r\n\r\n%s" % (
-                         len(message), message))
-           request.finish()
+           request.connection.write_headers(
+               httputil.ResponseStartLine('HTTP/1.1', 200, 'OK'),
+               {"Content-Length": str(len(message))})
+           request.connection.write(message)
+           request.connection.finish()
 
         http_server = tornado.httpserver.HTTPServer(handle_request)
         http_server.listen(8888)
         tornado.ioloop.IOLoop.instance().start()
 
-    `HTTPServer` is a very basic connection handler.  It parses the request
-    headers and body, but the request callback is responsible for producing
-    the response exactly as it will appear on the wire.  This affords
-    maximum flexibility for applications to implement whatever parts
-    of HTTP responses are required.
+    Applications should use the methods of `.HTTPConnection` to write
+    their response.
 
     `HTTPServer` supports keep-alive connections by default
     (automatically for HTTP/1.1, or for HTTP/1.0 when the client
-    requests ``Connection: keep-alive``).  This means that the request
-    callback must generate a properly-framed response, using either
-    the ``Content-Length`` header or ``Transfer-Encoding: chunked``.
-    Applications that are unable to frame their responses properly
-    should instead return a ``Connection: close`` header in each
-    response and pass ``no_keep_alive=True`` to the `HTTPServer`
-    constructor.
+    requests ``Connection: keep-alive``).
 
     If ``xheaders`` is ``True``, we support the
     ``X-Real-Ip``/``X-Forwarded-For`` and
@@ -143,407 +128,169 @@ class HTTPServer(TCPServer):
        servers if you want to create your listening sockets in some
        way other than `tornado.netutil.bind_sockets`.
 
+    .. versionchanged:: 3.3
+       Added ``gzip``, ``chunk_size``, ``max_header_size``,
+       ``idle_connection_timeout``, ``body_timeout``, ``max_body_size``
+       arguments.  Added support for `.HTTPServerConnectionDelegate`
+       instances as ``request_callback``.
     """
     def __init__(self, request_callback, no_keep_alive=False, io_loop=None,
-                 xheaders=False, ssl_options=None, protocol=None, **kwargs):
+                 xheaders=False, ssl_options=None, protocol=None, gzip=False,
+                 chunk_size=None, max_header_size=None,
+                 idle_connection_timeout=None, body_timeout=None,
+                 max_body_size=None, max_buffer_size=None):
         self.request_callback = request_callback
         self.no_keep_alive = no_keep_alive
         self.xheaders = xheaders
         self.protocol = protocol
+        self.conn_params = HTTP1ConnectionParameters(
+            use_gzip=gzip,
+            chunk_size=chunk_size,
+            max_header_size=max_header_size,
+            header_timeout=idle_connection_timeout or 3600,
+            max_body_size=max_body_size,
+            body_timeout=body_timeout)
         TCPServer.__init__(self, io_loop=io_loop, ssl_options=ssl_options,
-                           **kwargs)
+                           max_buffer_size=max_buffer_size,
+                           read_chunk_size=chunk_size)
+        self._connections = set()
 
-    def handle_stream(self, stream, address):
-        HTTPConnection(stream, address, self.request_callback,
-                       self.no_keep_alive, self.xheaders, self.protocol)
+    @gen.coroutine
+    def close_all_connections(self):
+        while self._connections:
+            # Peek at an arbitrary element of the set
+            conn = next(iter(self._connections))
+            yield conn.close()
 
+    def handle_stream(self, stream, address):
+        context = _HTTPRequestContext(stream, address,
+                                      self.protocol)
+        conn = HTTP1ServerConnection(
+            stream, self.conn_params, context)
+        self._connections.add(conn)
+        conn.start_serving(self)
 
-class _BadRequestException(Exception):
-    """Exception class for malformed HTTP requests."""
-    pass
+    def start_request(self, server_conn, request_conn):
+        return _ServerRequestAdapter(self, request_conn)
 
+    def on_close(self, server_conn):
+        self._connections.remove(server_conn)
 
-class HTTPConnection(object):
-    """Handles a connection to an HTTP client, executing HTTP requests.
 
-    We parse HTTP headers and bodies, and execute the request callback
-    until the HTTP conection is closed.
-    """
-    def __init__(self, stream, address, request_callback, no_keep_alive=False,
-                 xheaders=False, protocol=None):
-        self.stream = stream
+class _HTTPRequestContext(object):
+    def __init__(self, stream, address, protocol):
         self.address = address
+        self.protocol = protocol
         # Save the socket's address family now so we know how to
         # interpret self.address even after the stream is closed
         # and its socket attribute replaced with None.
-        self.address_family = stream.socket.family
-        self.request_callback = request_callback
-        self.no_keep_alive = no_keep_alive
-        self.xheaders = xheaders
-        self.protocol = protocol
-        self._clear_request_state()
-        # Save stack context here, outside of any request.  This keeps
-        # contexts from one request from leaking into the next.
-        self._header_callback = stack_context.wrap(self._on_headers)
-        self.stream.set_close_callback(self._on_connection_close)
-        self.stream.read_until(b"\r\n\r\n", self._header_callback)
-
-    def _clear_request_state(self):
-        """Clears the per-request state.
-
-        This is run in between requests to allow the previous handler
-        to be garbage collected (and prevent spurious close callbacks),
-        and when the connection is closed (to break up cycles and
-        facilitate garbage collection in cpython).
-        """
-        self._request = None
-        self._request_finished = False
-        self._write_callback = None
-        self._close_callback = None
-
-    def set_close_callback(self, callback):
-        """Sets a callback that will be run when the connection is closed.
-
-        Use this instead of accessing
-        `HTTPConnection.stream.set_close_callback
-        <.BaseIOStream.set_close_callback>` directly (which was the
-        recommended approach prior to Tornado 3.0).
-        """
-        self._close_callback = stack_context.wrap(callback)
-
-    def _on_connection_close(self):
-        if self._close_callback is not None:
-            callback = self._close_callback
-            self._close_callback = None
-            callback()
-        # Delete any unfinished callbacks to break up reference cycles.
-        self._header_callback = None
-        self._clear_request_state()
-
-    def close(self):
-        self.stream.close()
-        # Remove this reference to self, which would otherwise cause a
-        # cycle and delay garbage collection of this connection.
-        self._header_callback = None
-        self._clear_request_state()
-
-    def write(self, chunk, callback=None):
-        """Writes a chunk of output to the stream."""
-        if not self.stream.closed():
-            self._write_callback = stack_context.wrap(callback)
-            self.stream.write(chunk, self._on_write_complete)
-
-    def finish(self):
-        """Finishes the request."""
-        self._request_finished = True
-        # No more data is coming, so instruct TCP to send any remaining
-        # data immediately instead of waiting for a full packet or ack.
-        self.stream.set_nodelay(True)
-        if not self.stream.writing():
-            self._finish_request()
-
-    def _on_write_complete(self):
-        if self._write_callback is not None:
-            callback = self._write_callback
-            self._write_callback = None
-            callback()
-        # _on_write_complete is enqueued on the IOLoop whenever the
-        # IOStream's write buffer becomes empty, but it's possible for
-        # another callback that runs on the IOLoop before it to
-        # simultaneously write more data and finish the request.  If
-        # there is still data in the IOStream, a future
-        # _on_write_complete will be responsible for calling
-        # _finish_request.
-        if self._request_finished and not self.stream.writing():
-            self._finish_request()
-
-    def _finish_request(self):
-        if self.no_keep_alive or self._request is None:
-            disconnect = True
+        if stream.socket is not None:
+            self.address_family = stream.socket.family
         else:
-            connection_header = self._request.headers.get("Connection")
-            if connection_header is not None:
-                connection_header = connection_header.lower()
-            if self._request.supports_http_1_1():
-                disconnect = connection_header == "close"
-            elif ("Content-Length" in self._request.headers
-                    or self._request.method in ("HEAD", "GET")):
-                disconnect = connection_header != "keep-alive"
-            else:
-                disconnect = True
-        self._clear_request_state()
-        if disconnect:
-            self.close()
-            return
-        try:
-            # Use a try/except instead of checking stream.closed()
-            # directly, because in some cases the stream doesn't discover
-            # that it's closed until you try to read from it.
-            self.stream.read_until(b"\r\n\r\n", self._header_callback)
-
-            # Turn Nagle's algorithm back on, leaving the stream in its
-            # default state for the next request.
-            self.stream.set_nodelay(False)
-        except iostream.StreamClosedError:
-            self.close()
-
-    def _on_headers(self, data):
-        try:
-            data = native_str(data.decode('latin1'))
-            eol = data.find("\r\n")
-            start_line = data[:eol]
-            try:
-                method, uri, version = start_line.split(" ")
-            except ValueError:
-                raise _BadRequestException("Malformed HTTP request line")
-            if not version.startswith("HTTP/"):
-                raise _BadRequestException("Malformed HTTP version in HTTP Request-Line")
-            try:
-                headers = httputil.HTTPHeaders.parse(data[eol:])
-            except ValueError:
-                # Probably from split() if there was no ':' in the line
-                raise _BadRequestException("Malformed HTTP headers")
-
-            # HTTPRequest wants an IP, not a full socket address
-            if self.address_family in (socket.AF_INET, socket.AF_INET6):
-                remote_ip = self.address[0]
-            else:
-                # Unix (or other) socket; fake the remote address
-                remote_ip = '0.0.0.0'
-
-            self._request = HTTPRequest(
-                connection=self, method=method, uri=uri, version=version,
-                headers=headers, remote_ip=remote_ip, protocol=self.protocol)
-
-            content_length = headers.get("Content-Length")
-            if content_length:
-                content_length = int(content_length)
-                if content_length > self.stream.max_buffer_size:
-                    raise _BadRequestException("Content-Length too long")
-                if headers.get("Expect") == "100-continue":
-                    self.stream.write(b"HTTP/1.1 100 (Continue)\r\n\r\n")
-                self.stream.read_bytes(content_length, self._on_request_body)
-                return
-
-            self.request_callback(self._request)
-        except _BadRequestException as e:
-            gen_log.info("Malformed HTTP request from %r: %s",
-                         self.address, e)
-            self.close()
-            return
-
-    def _on_request_body(self, data):
-        self._request.body = data
-        if self._request.method in ("POST", "PATCH", "PUT"):
-            httputil.parse_body_arguments(
-                self._request.headers.get("Content-Type", ""), data,
-                self._request.body_arguments, self._request.files)
-
-            for k, v in self._request.body_arguments.items():
-                self._request.arguments.setdefault(k, []).extend(v)
-        self.request_callback(self._request)
-
-
-class HTTPRequest(object):
-    """A single HTTP request.
-
-    All attributes are type `str` unless otherwise noted.
-
-    .. attribute:: method
-
-       HTTP request method, e.g. "GET" or "POST"
-
-    .. attribute:: uri
-
-       The requested uri.
-
-    .. attribute:: path
-
-       The path portion of `uri`
-
-    .. attribute:: query
-
-       The query portion of `uri`
-
-    .. attribute:: version
-
-       HTTP version specified in request, e.g. "HTTP/1.1"
-
-    .. attribute:: headers
-
-       `.HTTPHeaders` dictionary-like object for request headers.  Acts like
-       a case-insensitive dictionary with additional methods for repeated
-       headers.
-
-    .. attribute:: body
-
-       Request body, if present, as a byte string.
-
-    .. attribute:: remote_ip
-
-       Client's IP address as a string.  If ``HTTPServer.xheaders`` is set,
-       will pass along the real IP address provided by a load balancer
-       in the ``X-Real-Ip`` or ``X-Forwarded-For`` header.
-
-    .. versionchanged:: 3.1
-       The list format of ``X-Forwarded-For`` is now supported.
-
-    .. attribute:: protocol
-
-       The protocol used, either "http" or "https".  If ``HTTPServer.xheaders``
-       is set, will pass along the protocol used by a load balancer if
-       reported via an ``X-Scheme`` header.
-
-    .. attribute:: host
-
-       The requested hostname, usually taken from the ``Host`` header.
-
-    .. attribute:: arguments
-
-       GET/POST arguments are available in the arguments property, which
-       maps arguments names to lists of values (to support multiple values
-       for individual names). Names are of type `str`, while arguments
-       are byte strings.  Note that this is different from
-       `.RequestHandler.get_argument`, which returns argument values as
-       unicode strings.
-
-    .. attribute:: query_arguments
-
-       Same format as ``arguments``, but contains only arguments extracted
-       from the query string.
-
-       .. versionadded:: 3.2
-
-    .. attribute:: body_arguments
-
-       Same format as ``arguments``, but contains only arguments extracted
-       from the request body.
-
-       .. versionadded:: 3.2
-
-    .. attribute:: files
-
-       File uploads are available in the files property, which maps file
-       names to lists of `.HTTPFile`.
-
-    .. attribute:: connection
-
-       An HTTP request is attached to a single HTTP connection, which can
-       be accessed through the "connection" attribute. Since connections
-       are typically kept open in HTTP/1.1, multiple requests can be handled
-       sequentially on a single connection.
-    """
-    def __init__(self, method, uri, version="HTTP/1.0", headers=None,
-                 body=None, remote_ip=None, protocol=None, host=None,
-                 files=None, connection=None):
-        self.method = method
-        self.uri = uri
-        self.version = version
-        self.headers = headers or httputil.HTTPHeaders()
-        self.body = body or ""
-
-        # set remote IP and protocol
-        self.remote_ip = remote_ip
+            self.address_family = None
+        # In HTTPServerRequest we want an IP, not a full socket address.
+        if (self.address_family in (socket.AF_INET, socket.AF_INET6) and
+                address is not None):
+            self.remote_ip = address[0]
+        else:
+            # Unix (or other) socket; fake the remote address.
+            self.remote_ip = '0.0.0.0'
         if protocol:
             self.protocol = protocol
-        elif connection and isinstance(connection.stream,
-                                       iostream.SSLIOStream):
+        elif isinstance(stream, iostream.SSLIOStream):
             self.protocol = "https"
         else:
             self.protocol = "http"
+        self._orig_remote_ip = self.remote_ip
+        self._orig_protocol = self.protocol
+
+    def __str__(self):
+        if self.address_family in (socket.AF_INET, socket.AF_INET6):
+            return self.remote_ip
+        elif isinstance(self.address, bytes):
+            # Python 3 with the -bb option warns about str(bytes),
+            # so convert it explicitly.
+            # Unix socket addresses are str on mac but bytes on linux.
+            return native_str(self.address)
+        else:
+            return str(self.address)
+
+    def _apply_xheaders(self, headers):
+        """Rewrite the ``remote_ip`` and ``protocol`` fields."""
+        # Squid uses X-Forwarded-For, others use X-Real-Ip
+        ip = headers.get("X-Forwarded-For", self.remote_ip)
+        ip = ip.split(',')[-1].strip()
+        ip = headers.get("X-Real-Ip", ip)
+        if netutil.is_valid_ip(ip):
+            self.remote_ip = ip
+        # AWS uses X-Forwarded-Proto
+        proto_header = headers.get(
+            "X-Scheme", headers.get("X-Forwarded-Proto",
+                                    self.protocol))
+        if proto_header in ("http", "https"):
+            self.protocol = proto_header
+
+    def _unapply_xheaders(self):
+        """Undo changes from `_apply_xheaders`.
+
+        Xheaders are per-request so they should not leak to the next
+        request on the same connection.
+        """
+        self.remote_ip = self._orig_remote_ip
+        self.protocol = self._orig_protocol
 
-        # xheaders can override the defaults
-        if connection and connection.xheaders:
-            # Squid uses X-Forwarded-For, others use X-Real-Ip
-            ip = self.headers.get("X-Forwarded-For", self.remote_ip)
-            ip = ip.split(',')[-1].strip()
-            ip = self.headers.get(
-                "X-Real-Ip", ip)
-            if netutil.is_valid_ip(ip):
-                self.remote_ip = ip
-            # AWS uses X-Forwarded-Proto
-            proto = self.headers.get(
-                "X-Scheme", self.headers.get("X-Forwarded-Proto", self.protocol))
-            if proto in ("http", "https"):
-                self.protocol = proto
-
-        self.host = host or self.headers.get("Host") or "127.0.0.1"
-        self.files = files or {}
+
+class _ServerRequestAdapter(httputil.HTTPMessageDelegate):
+    """Adapts the `HTTPMessageDelegate` interface to the interface expected
+    by our clients.
+    """
+    def __init__(self, server, connection):
+        self.server = server
         self.connection = connection
-        self._start_time = time.time()
-        self._finish_time = None
-
-        self.path, sep, self.query = uri.partition('?')
-        self.arguments = parse_qs_bytes(self.query, keep_blank_values=True)
-        self.query_arguments = copy.deepcopy(self.arguments)
-        self.body_arguments = {}
-
-    def supports_http_1_1(self):
-        """Returns True if this request supports HTTP/1.1 semantics"""
-        return self.version == "HTTP/1.1"
-
-    @property
-    def cookies(self):
-        """A dictionary of Cookie.Morsel objects."""
-        if not hasattr(self, "_cookies"):
-            self._cookies = Cookie.SimpleCookie()
-            if "Cookie" in self.headers:
-                try:
-                    self._cookies.load(
-                        native_str(self.headers["Cookie"]))
-                except Exception:
-                    self._cookies = {}
-        return self._cookies
-
-    def write(self, chunk, callback=None):
-        """Writes the given chunk to the response stream."""
-        assert isinstance(chunk, bytes_type)
-        self.connection.write(chunk, callback=callback)
+        self.request = None
+        if isinstance(server.request_callback,
+                      httputil.HTTPServerConnectionDelegate):
+            self.delegate = server.request_callback.start_request(connection)
+            self._chunks = None
+        else:
+            self.delegate = None
+            self._chunks = []
+
+    def headers_received(self, start_line, headers):
+        if self.server.xheaders:
+            self.connection.context._apply_xheaders(headers)
+        if self.delegate is None:
+            self.request = httputil.HTTPServerRequest(
+                connection=self.connection, start_line=start_line,
+                headers=headers)
+        else:
+            return self.delegate.headers_received(start_line, headers)
+
+    def data_received(self, chunk):
+        if self.delegate is None:
+            self._chunks.append(chunk)
+        else:
+            return self.delegate.data_received(chunk)
 
     def finish(self):
-        """Finishes this HTTP request on the open connection."""
-        self.connection.finish()
-        self._finish_time = time.time()
-
-    def full_url(self):
-        """Reconstructs the full URL for this request."""
-        return self.protocol + "://" + self.host + self.uri
-
-    def request_time(self):
-        """Returns the amount of time it took for this request to execute."""
-        if self._finish_time is None:
-            return time.time() - self._start_time
+        if self.delegate is None:
+            self.request.body = b''.join(self._chunks)
+            self.request._parse_body()
+            self.server.request_callback(self.request)
         else:
-            return self._finish_time - self._start_time
-
-    def get_ssl_certificate(self, binary_form=False):
-        """Returns the client's SSL certificate, if any.
-
-        To use client certificates, the HTTPServer must have been constructed
-        with cert_reqs set in ssl_options, e.g.::
-
-            server = HTTPServer(app,
-                ssl_options=dict(
-                    certfile="foo.crt",
-                    keyfile="foo.key",
-                    cert_reqs=ssl.CERT_REQUIRED,
-                    ca_certs="cacert.crt"))
-
-        By default, the return value is a dictionary (or None, if no
-        client certificate is present).  If ``binary_form`` is true, a
-        DER-encoded form of the certificate is returned instead.  See
-        SSLSocket.getpeercert() in the standard library for more
-        details.
-        http://docs.python.org/library/ssl.html#sslsocket-objects
-        """
-        try:
-            return self.connection.stream.socket.getpeercert(
-                binary_form=binary_form)
-        except ssl.SSLError:
-            return None
-
-    def __repr__(self):
-        attrs = ("protocol", "host", "method", "uri", "version", "remote_ip")
-        args = ", ".join(["%s=%r" % (n, getattr(self, n)) for n in attrs])
-        return "%s(%s, headers=%s)" % (
-            self.__class__.__name__, args, dict(self.headers))
+            self.delegate.finish()
+        self._cleanup()
+
+    def on_connection_close(self):
+        if self.delegate is None:
+            self._chunks = None
+        else:
+            self.delegate.on_connection_close()
+        self._cleanup()
+
+    def _cleanup(self):
+        if self.server.xheaders:
+            self.connection.context._unapply_xheaders()
+
+
+HTTPRequest = httputil.HTTPServerRequest

libs/tornado/httputil.py

@@ -14,20 +14,31 @@
 # License for the specific language governing permissions and limitations
 # under the License.
 
-"""HTTP utility code shared by clients and servers."""
+"""HTTP utility code shared by clients and servers.
+
+This module also defines the `HTTPServerRequest` class which is exposed
+via `tornado.web.RequestHandler.request`.
+"""
 
 from __future__ import absolute_import, division, print_function, with_statement
 
 import calendar
 import collections
+import copy
 import datetime
 import email.utils
 import numbers
+import re
 import time
 
 from tornado.escape import native_str, parse_qs_bytes, utf8
 from tornado.log import gen_log
-from tornado.util import ObjectDict
+from tornado.util import ObjectDict, bytes_type
+
+try:
+    import Cookie  # py2
+except ImportError:
+    import http.cookies as Cookie  # py3
 
 try:
     from httplib import responses  # py2
@@ -43,6 +54,13 @@ try:
 except ImportError:
     from urllib.parse import urlencode  # py3
 
+try:
+    from ssl import SSLError
+except ImportError:
+    # ssl is unavailable on app engine.
+    class SSLError(Exception):
+        pass
+
 
 class _NormalizedHeaderCache(dict):
     """Dynamic cached mapping of header names to Http-Header-Case.
@@ -212,6 +230,337 @@ class HTTPHeaders(dict):
         return HTTPHeaders(self)
 
 
+class HTTPServerRequest(object):
+    """A single HTTP request.
+
+    All attributes are type `str` unless otherwise noted.
+
+    .. attribute:: method
+
+       HTTP request method, e.g. "GET" or "POST"
+
+    .. attribute:: uri
+
+       The requested uri.
+
+    .. attribute:: path
+
+       The path portion of `uri`
+
+    .. attribute:: query
+
+       The query portion of `uri`
+
+    .. attribute:: version
+
+       HTTP version specified in request, e.g. "HTTP/1.1"
+
+    .. attribute:: headers
+
+       `.HTTPHeaders` dictionary-like object for request headers.  Acts like
+       a case-insensitive dictionary with additional methods for repeated
+       headers.
+
+    .. attribute:: body
+
+       Request body, if present, as a byte string.
+
+    .. attribute:: remote_ip
+
+       Client's IP address as a string.  If ``HTTPServer.xheaders`` is set,
+       will pass along the real IP address provided by a load balancer
+       in the ``X-Real-Ip`` or ``X-Forwarded-For`` header.
+
+    .. versionchanged:: 3.1
+       The list format of ``X-Forwarded-For`` is now supported.
+
+    .. attribute:: protocol
+
+       The protocol used, either "http" or "https".  If ``HTTPServer.xheaders``
+       is set, will pass along the protocol used by a load balancer if
+       reported via an ``X-Scheme`` header.
+
+    .. attribute:: host
+
+       The requested hostname, usually taken from the ``Host`` header.
+
+    .. attribute:: arguments
+
+       GET/POST arguments are available in the arguments property, which
+       maps arguments names to lists of values (to support multiple values
+       for individual names). Names are of type `str`, while arguments
+       are byte strings.  Note that this is different from
+       `.RequestHandler.get_argument`, which returns argument values as
+       unicode strings.
+
+    .. attribute:: query_arguments
+
+       Same format as ``arguments``, but contains only arguments extracted
+       from the query string.
+
+       .. versionadded:: 3.2
+
+    .. attribute:: body_arguments
+
+       Same format as ``arguments``, but contains only arguments extracted
+       from the request body.
+
+       .. versionadded:: 3.2
+
+    .. attribute:: files
+
+       File uploads are available in the files property, which maps file
+       names to lists of `.HTTPFile`.
+
+    .. attribute:: connection
+
+       An HTTP request is attached to a single HTTP connection, which can
+       be accessed through the "connection" attribute. Since connections
+       are typically kept open in HTTP/1.1, multiple requests can be handled
+       sequentially on a single connection.
+
+    .. versionchanged:: 3.3
+       Moved from ``tornado.httpserver.HTTPRequest``.
+    """
+    def __init__(self, method=None, uri=None, version="HTTP/1.0", headers=None,
+                 body=None, host=None, files=None, connection=None,
+                 start_line=None):
+        if start_line is not None:
+            method, uri, version = start_line
+        self.method = method
+        self.uri = uri
+        self.version = version
+        self.headers = headers or HTTPHeaders()
+        self.body = body or ""
+
+        # set remote IP and protocol
+        context = getattr(connection, 'context', None)
+        self.remote_ip = getattr(context, 'remote_ip')
+        self.protocol = getattr(context, 'protocol', "http")
+
+        self.host = host or self.headers.get("Host") or "127.0.0.1"
+        self.files = files or {}
+        self.connection = connection
+        self._start_time = time.time()
+        self._finish_time = None
+
+        self.path, sep, self.query = uri.partition('?')
+        self.arguments = parse_qs_bytes(self.query, keep_blank_values=True)
+        self.query_arguments = copy.deepcopy(self.arguments)
+        self.body_arguments = {}
+
+    def supports_http_1_1(self):
+        """Returns True if this request supports HTTP/1.1 semantics.
+
+        .. deprecated:: 3.3
+           Applications are less likely to need this information with the
+           introduction of `.HTTPConnection`.  If you still need it, access
+           the ``version`` attribute directly.
+        """
+        return self.version == "HTTP/1.1"
+
+    @property
+    def cookies(self):
+        """A dictionary of Cookie.Morsel objects."""
+        if not hasattr(self, "_cookies"):
+            self._cookies = Cookie.SimpleCookie()
+            if "Cookie" in self.headers:
+                try:
+                    self._cookies.load(
+                        native_str(self.headers["Cookie"]))
+                except Exception:
+                    self._cookies = {}
+        return self._cookies
+
+    def write(self, chunk, callback=None):
+        """Writes the given chunk to the response stream.
+
+        .. deprecated:: 3.3
+           Use ``request.connection`` and the `.HTTPConnection` methods
+           to write the response.
+        """
+        assert isinstance(chunk, bytes_type)
+        self.connection.write(chunk, callback=callback)
+
+    def finish(self):
+        """Finishes this HTTP request on the open connection.
+
+        .. deprecated:: 3.3
+           Use ``request.connection`` and the `.HTTPConnection` methods
+           to write the response.
+        """
+        self.connection.finish()
+        self._finish_time = time.time()
+
+    def full_url(self):
+        """Reconstructs the full URL for this request."""
+        return self.protocol + "://" + self.host + self.uri
+
+    def request_time(self):
+        """Returns the amount of time it took for this request to execute."""
+        if self._finish_time is None:
+            return time.time() - self._start_time
+        else:
+            return self._finish_time - self._start_time
+
+    def get_ssl_certificate(self, binary_form=False):
+        """Returns the client's SSL certificate, if any.
+
+        To use client certificates, the HTTPServer must have been constructed
+        with cert_reqs set in ssl_options, e.g.::
+
+            server = HTTPServer(app,
+                ssl_options=dict(
+                    certfile="foo.crt",
+                    keyfile="foo.key",
+                    cert_reqs=ssl.CERT_REQUIRED,
+                    ca_certs="cacert.crt"))
+
+        By default, the return value is a dictionary (or None, if no
+        client certificate is present).  If ``binary_form`` is true, a
+        DER-encoded form of the certificate is returned instead.  See
+        SSLSocket.getpeercert() in the standard library for more
+        details.
+        http://docs.python.org/library/ssl.html#sslsocket-objects
+        """
+        try:
+            return self.connection.stream.socket.getpeercert(
+                binary_form=binary_form)
+        except SSLError:
+            return None
+
+    def _parse_body(self):
+        parse_body_arguments(
+            self.headers.get("Content-Type", ""), self.body,
+            self.body_arguments, self.files,
+            self.headers)
+
+        for k, v in self.body_arguments.items():
+            self.arguments.setdefault(k, []).extend(v)
+
+    def __repr__(self):
+        attrs = ("protocol", "host", "method", "uri", "version", "remote_ip")
+        args = ", ".join(["%s=%r" % (n, getattr(self, n)) for n in attrs])
+        return "%s(%s, headers=%s)" % (
+            self.__class__.__name__, args, dict(self.headers))
+
+
+class HTTPInputException(Exception):
+    """Exception class for malformed HTTP requests or responses
+    from remote sources.
+
+    .. versionadded:: 3.3
+    """
+    pass
+
+
+class HTTPOutputException(Exception):
+    """Exception class for errors in HTTP output.
+
+    .. versionadded:: 3.3
+    """
+    pass
+
+
+class HTTPServerConnectionDelegate(object):
+    """Implement this interface to handle requests from `.HTTPServer`.
+
+    .. versionadded:: 3.3
+    """
+    def start_request(self, server_conn, request_conn):
+        """This method is called by the server when a new request has started.
+
+        :arg server_conn: is an opaque object representing the long-lived
+            (e.g. tcp-level) connection.
+        :arg request_conn: is a `.HTTPConnection` object for a single
+            request/response exchange.
+
+        This method should return a `.HTTPMessageDelegate`.
+        """
+        raise NotImplementedError()
+
+    def on_close(self, server_conn):
+        """This method is called when a connection has been closed.
+
+        :arg server_conn: is a server connection that has previously been
+            passed to ``start_request``.
+        """
+        pass
+
+
+class HTTPMessageDelegate(object):
+    """Implement this interface to handle an HTTP request or response.
+
+    .. versionadded:: 3.3
+    """
+    def headers_received(self, start_line, headers):
+        """Called when the HTTP headers have been received and parsed.
+
+        :arg start_line: a `.RequestStartLine` or `.ResponseStartLine`
+            depending on whether this is a client or server message.
+        :arg headers: a `.HTTPHeaders` instance.
+
+        Some `.HTTPConnection` methods can only be called during
+        ``headers_received``.
+
+        May return a `.Future`; if it does the body will not be read
+        until it is done.
+        """
+        pass
+
+    def data_received(self, chunk):
+        """Called when a chunk of data has been received.
+
+        May return a `.Future` for flow control.
+        """
+        pass
+
+    def finish(self):
+        """Called after the last chunk of data has been received."""
+        pass
+
+    def on_connection_close(self):
+        """Called if the connection is closed without finishing the request.
+
+        If ``headers_received`` is called, either ``finish`` or
+        ``on_connection_close`` will be called, but not both.
+        """
+        pass
+
+
+class HTTPConnection(object):
+    """Applications use this interface to write their responses.
+
+    .. versionadded:: 3.3
+    """
+    def write_headers(self, start_line, headers, chunk=None, callback=None):
+        """Write an HTTP header block.
+
+        :arg start_line: a `.RequestStartLine` or `.ResponseStartLine`.
+        :arg headers: a `.HTTPHeaders` instance.
+        :arg chunk: the first (optional) chunk of data.  This is an optimization
+            so that small responses can be written in the same call as their
+            headers.
+        :arg callback: a callback to be run when the write is complete.
+
+        Returns a `.Future` if no callback is given.
+        """
+        raise NotImplementedError()
+
+    def write(self, chunk, callback=None):
+        """Writes a chunk of body data.
+
+        The callback will be run when the write is complete.  If no callback
+        is given, returns a Future.
+        """
+        raise NotImplementedError()
+
+    def finish(self):
+        """Indicates that the last body data has been written.
+        """
+        raise NotImplementedError()
+
+
 def url_concat(url, args):
     """Concatenate url and argument dictionary regardless of whether
     url has existing query parameters.
@@ -310,7 +659,7 @@ def _int_or_none(val):
     return int(val)
 
 
-def parse_body_arguments(content_type, body, arguments, files):
+def parse_body_arguments(content_type, body, arguments, files, headers=None):
     """Parses a form request body.
 
     Supports ``application/x-www-form-urlencoded`` and
@@ -319,6 +668,10 @@ def parse_body_arguments(content_type, body, arguments, files):
     and ``files`` parameters are dictionaries that will be updated
     with the parsed contents.
     """
+    if headers and 'Content-Encoding' in headers:
+        gen_log.warning("Unsupported Content-Encoding: %s",
+                        headers['Content-Encoding'])
+        return
     if content_type.startswith("application/x-www-form-urlencoded"):
         try:
             uri_arguments = parse_qs_bytes(native_str(body), keep_blank_values=True)
@@ -405,6 +758,48 @@ def format_timestamp(ts):
         raise TypeError("unknown timestamp type: %r" % ts)
     return email.utils.formatdate(ts, usegmt=True)
 
+
+RequestStartLine = collections.namedtuple(
+    'RequestStartLine', ['method', 'path', 'version'])
+
+
+def parse_request_start_line(line):
+    """Returns a (method, path, version) tuple for an HTTP 1.x request line.
+
+    The response is a `collections.namedtuple`.
+
+    >>> parse_request_start_line("GET /foo HTTP/1.1")
+    RequestStartLine(method='GET', path='/foo', version='HTTP/1.1')
+    """
+    try:
+        method, path, version = line.split(" ")
+    except ValueError:
+        raise HTTPInputException("Malformed HTTP request line")
+    if not version.startswith("HTTP/"):
+        raise HTTPInputException(
+            "Malformed HTTP version in HTTP Request-Line: %r" % version)
+    return RequestStartLine(method, path, version)
+
+
+ResponseStartLine = collections.namedtuple(
+    'ResponseStartLine', ['version', 'code', 'reason'])
+
+
+def parse_response_start_line(line):
+    """Returns a (version, code, reason) tuple for an HTTP 1.x response line.
+
+    The response is a `collections.namedtuple`.
+
+    >>> parse_response_start_line("HTTP/1.1 200 OK")
+    ResponseStartLine(version='HTTP/1.1', code=200, reason='OK')
+    """
+    line = native_str(line)
+    match = re.match("(HTTP/1.[01]) ([0-9]+) ([^\r]*)", line)
+    if not match:
+        raise HTTPInputException("Error parsing response start line")
+    return ResponseStartLine(match.group(1), int(match.group(2)),
+                             match.group(3))
+
 # _parseparam and _parse_header are copied and modified from python2.7's cgi.py
 # The original 2.7 version of this code did not correctly support some
 # combinations of semicolons and double quotes.

libs/tornado/ioloop.py

@@ -41,10 +41,11 @@ import threading
 import time
 import traceback
 
-from tornado.concurrent import Future, TracebackFuture
+from tornado.concurrent import TracebackFuture, is_future
 from tornado.log import app_log, gen_log
 from tornado import stack_context
 from tornado.util import Configurable
+from tornado.util import errno_from_exception
 
 try:
     import signal
@@ -157,6 +158,15 @@ class IOLoop(Configurable):
         IOLoop._instance = self
 
     @staticmethod
+    def clear_instance():
+        """Clear the global `IOLoop` instance.
+
+        .. versionadded:: 3.3
+        """
+        if hasattr(IOLoop, "_instance"):
+            del IOLoop._instance
+
+    @staticmethod
     def current():
         """Returns the current thread's `IOLoop`.
 
@@ -244,21 +254,40 @@ class IOLoop(Configurable):
         raise NotImplementedError()
 
     def add_handler(self, fd, handler, events):
-        """Registers the given handler to receive the given events for fd.
+        """Registers the given handler to receive the given events for ``fd``.
+
+        The ``fd`` argument may either be an integer file descriptor or
+        a file-like object with a ``fileno()`` method (and optionally a
+        ``close()`` method, which may be called when the `IOLoop` is shut
+        down).
 
         The ``events`` argument is a bitwise or of the constants
         ``IOLoop.READ``, ``IOLoop.WRITE``, and ``IOLoop.ERROR``.
 
         When an event occurs, ``handler(fd, events)`` will be run.
+
+        .. versionchanged:: 3.3
+           Added the ability to pass file-like objects in addition to
+           raw file descriptors.
         """
         raise NotImplementedError()
 
     def update_handler(self, fd, events):
-        """Changes the events we listen for fd."""
+        """Changes the events we listen for ``fd``.
+
+        .. versionchanged:: 3.3
+           Added the ability to pass file-like objects in addition to
+           raw file descriptors.
+        """
         raise NotImplementedError()
 
     def remove_handler(self, fd):
-        """Stop listening for events on fd."""
+        """Stop listening for events on ``fd``.
+
+        .. versionchanged:: 3.3
+           Added the ability to pass file-like objects in addition to
+           raw file descriptors.
+        """
         raise NotImplementedError()
 
     def set_blocking_signal_threshold(self, seconds, action):
@@ -372,7 +401,7 @@ class IOLoop(Configurable):
                 future_cell[0] = TracebackFuture()
                 future_cell[0].set_exc_info(sys.exc_info())
             else:
-                if isinstance(result, Future):
+                if is_future(result):
                     future_cell[0] = result
                 else:
                     future_cell[0] = TracebackFuture()
@@ -463,7 +492,7 @@ class IOLoop(Configurable):
         The callback is invoked with one argument, the
         `.Future`.
         """
-        assert isinstance(future, Future)
+        assert is_future(future)
         callback = stack_context.wrap(callback)
         future.add_done_callback(
             lambda future: self.add_callback(callback, future))
@@ -490,6 +519,47 @@ class IOLoop(Configurable):
         """
         app_log.error("Exception in callback %r", callback, exc_info=True)
 
+    def split_fd(self, fd):
+        """Returns an (fd, obj) pair from an ``fd`` parameter.
+
+        We accept both raw file descriptors and file-like objects as
+        input to `add_handler` and related methods.  When a file-like
+        object is passed, we must retain the object itself so we can
+        close it correctly when the `IOLoop` shuts down, but the
+        poller interfaces favor file descriptors (they will accept
+        file-like objects and call ``fileno()`` for you, but they
+        always return the descriptor itself).
+
+        This method is provided for use by `IOLoop` subclasses and should
+        not generally be used by application code.
+
+        .. versionadded:: 3.3
+        """
+        try:
+            return fd.fileno(), fd
+        except AttributeError:
+            return fd, fd
+
+    def close_fd(self, fd):
+        """Utility method to close an ``fd``.
+
+        If ``fd`` is a file-like object, we close it directly; otherwise
+        we use `os.close`.
+
+        This method is provided for use by `IOLoop` subclasses (in
+        implementations of ``IOLoop.close(all_fds=True)`` and should
+        not generally be used by application code.
+
+        .. versionadded:: 3.3
+        """
+        try:
+            try:
+                fd.close()
+            except AttributeError:
+                os.close(fd)
+        except OSError:
+            pass
+
 
 class PollIOLoop(IOLoop):
     """Base class for IOLoops built around a select-like function.
@@ -528,26 +598,24 @@ class PollIOLoop(IOLoop):
             self._closing = True
         self.remove_handler(self._waker.fileno())
         if all_fds:
-            for fd in self._handlers.keys():
-                try:
-                    close_method = getattr(fd, 'close', None)
-                    if close_method is not None:
-                        close_method()
-                    else:
-                        os.close(fd)
-                except Exception:
-                    gen_log.debug("error closing fd %s", fd, exc_info=True)
+            for fd, handler in self._handlers.values():
+                self.close_fd(fd)
         self._waker.close()
         self._impl.close()
+        self._callbacks = None
+        self._timeouts = None
 
     def add_handler(self, fd, handler, events):
-        self._handlers[fd] = stack_context.wrap(handler)
+        fd, obj = self.split_fd(fd)
+        self._handlers[fd] = (obj, stack_context.wrap(handler))
         self._impl.register(fd, events | self.ERROR)
 
     def update_handler(self, fd, events):
+        fd, obj = self.split_fd(fd)
         self._impl.modify(fd, events | self.ERROR)
 
     def remove_handler(self, fd):
+        fd, obj = self.split_fd(fd)
         self._handlers.pop(fd, None)
         self._events.pop(fd, None)
         try:
@@ -566,6 +634,8 @@ class PollIOLoop(IOLoop):
                           action if action is not None else signal.SIG_DFL)
 
     def start(self):
+        if self._running:
+            raise RuntimeError("IOLoop is already running")
         self._setup_logging()
         if self._stopped:
             self._stopped = False
@@ -666,9 +736,7 @@ class PollIOLoop(IOLoop):
                     # two ways EINTR might be signaled:
                     # * e.errno == errno.EINTR
                     # * e.args is like (errno.EINTR, 'Interrupted system call')
-                    if (getattr(e, 'errno', None) == errno.EINTR or
-                        (isinstance(getattr(e, 'args', None), tuple) and
-                         len(e.args) == 2 and e.args[0] == errno.EINTR)):
+                    if errno_from_exception(e) == errno.EINTR:
                         continue
                     else:
                         raise
@@ -685,15 +753,17 @@ class PollIOLoop(IOLoop):
                 while self._events:
                     fd, events = self._events.popitem()
                     try:
-                        self._handlers[fd](fd, events)
+                        fd_obj, handler_func = self._handlers[fd]
+                        handler_func(fd_obj, events)
                     except (OSError, IOError) as e:
-                        if e.args[0] == errno.EPIPE:
+                        if errno_from_exception(e) == errno.EPIPE:
                             # Happens when the client closes the connection
                             pass
                         else:
                             self.handle_callback_exception(self._handlers.get(fd))
                     except Exception:
                         self.handle_callback_exception(self._handlers.get(fd))
+                fd_obj = handler_func = None
 
         finally:
             # reset the stopped flag so another start/stop pair can be issued
@@ -771,7 +841,11 @@ class _Timeout(object):
         if isinstance(deadline, numbers.Real):
             self.deadline = deadline
         elif isinstance(deadline, datetime.timedelta):
-            self.deadline = io_loop.time() + _Timeout.timedelta_to_seconds(deadline)
+            now = io_loop.time()
+            try:
+                self.deadline = now + deadline.total_seconds()
+            except AttributeError:  # py2.6
+                self.deadline = now + _Timeout.timedelta_to_seconds(deadline)
         else:
             raise TypeError("Unsupported deadline %r" % deadline)
         self.callback = callback

libs/tornado/iostream.py

@@ -31,21 +31,27 @@ import errno
 import numbers
 import os
 import socket
-import ssl
 import sys
 import re
 
+from tornado.concurrent import TracebackFuture
 from tornado import ioloop
 from tornado.log import gen_log, app_log
 from tornado.netutil import ssl_wrap_socket, ssl_match_hostname, SSLCertificateError
 from tornado import stack_context
-from tornado.util import bytes_type
+from tornado.util import bytes_type, errno_from_exception
 
 try:
     from tornado.platform.posix import _set_nonblocking
 except ImportError:
     _set_nonblocking = None
 
+try:
+    import ssl
+except ImportError:
+    # ssl is not available on Google App Engine
+    ssl = None
+
 # These errnos indicate that a non-blocking operation must be retried
 # at a later time.  On most platforms they're the same value, but on
 # some they differ.
@@ -66,12 +72,31 @@ class StreamClosedError(IOError):
     pass
 
 
+class UnsatisfiableReadError(Exception):
+    """Exception raised when a read cannot be satisfied.
+
+    Raised by ``read_until`` and ``read_until_regex`` with a ``max_bytes``
+    argument.
+    """
+    pass
+
+
+class StreamBufferFullError(Exception):
+    """Exception raised by `IOStream` methods when the buffer is full.
+    """
+
+
 class BaseIOStream(object):
     """A utility class to write to and read from a non-blocking file or socket.
 
     We support a non-blocking ``write()`` and a family of ``read_*()`` methods.
-    All of the methods take callbacks (since writing and reading are
-    non-blocking and asynchronous).
+    All of the methods take an optional ``callback`` argument and return a
+    `.Future` only if no callback is given.  When the operation completes,
+    the callback will be run or the `.Future` will resolve with the data
+    read (or ``None`` for ``write()``).  All outstanding ``Futures`` will
+    resolve with a `StreamClosedError` when the stream is closed; users
+    of the callback interface will be notified via
+    `.BaseIOStream.set_close_callback` instead.
 
     When a stream is closed due to an error, the IOStream's ``error``
     attribute contains the exception object.
@@ -80,24 +105,48 @@ class BaseIOStream(object):
     `read_from_fd`, and optionally `get_fd_error`.
     """
     def __init__(self, io_loop=None, max_buffer_size=None,
-                 read_chunk_size=4096):
+                 read_chunk_size=None, max_write_buffer_size=None):
+        """`BaseIOStream` constructor.
+
+        :arg io_loop: The `.IOLoop` to use; defaults to `.IOLoop.current`.
+        :arg max_buffer_size: Maximum amount of incoming data to buffer;
+            defaults to 100MB.
+        :arg read_chunk_size: Amount of data to read at one time from the
+            underlying transport; defaults to 64KB.
+        :arg max_write_buffer_size: Amount of outgoing data to buffer;
+            defaults to unlimited.
+
+        .. versionchanged:: 3.3
+           Add the ``max_write_buffer_size`` parameter.  Changed default
+           ``read_chunk_size`` to 64KB.
+        """
         self.io_loop = io_loop or ioloop.IOLoop.current()
         self.max_buffer_size = max_buffer_size or 104857600
-        self.read_chunk_size = read_chunk_size
+        # A chunk size that is too close to max_buffer_size can cause
+        # spurious failures.
+        self.read_chunk_size = min(read_chunk_size or 65536,
+                                   self.max_buffer_size // 2)
+        self.max_write_buffer_size = max_write_buffer_size
         self.error = None
         self._read_buffer = collections.deque()
         self._write_buffer = collections.deque()
         self._read_buffer_size = 0
+        self._write_buffer_size = 0
         self._write_buffer_frozen = False
         self._read_delimiter = None
         self._read_regex = None
+        self._read_max_bytes = None
         self._read_bytes = None
+        self._read_partial = False
         self._read_until_close = False
         self._read_callback = None
+        self._read_future = None
         self._streaming_callback = None
         self._write_callback = None
+        self._write_future = None
         self._close_callback = None
         self._connect_callback = None
+        self._connect_future = None
         self._connecting = False
         self._state = None
         self._pending_callbacks = 0
@@ -142,98 +191,162 @@ class BaseIOStream(object):
         """
         return None
 
-    def read_until_regex(self, regex, callback):
-        """Run ``callback`` when we read the given regex pattern.
+    def read_until_regex(self, regex, callback=None, max_bytes=None):
+        """Asynchronously read until we have matched the given regex.
+
+        The result includes the data that matches the regex and anything
+        that came before it.  If a callback is given, it will be run
+        with the data as an argument; if not, this method returns a
+        `.Future`.
 
-        The callback will get the data read (including the data that
-        matched the regex and anything that came before it) as an argument.
+        If ``max_bytes`` is not None, the connection will be closed
+        if more than ``max_bytes`` bytes have been read and the regex is
+        not satisfied.
+
+        .. versionchanged:: 3.3
+            Added the ``max_bytes`` argument.  The ``callback`` argument is
+            now optional and a `.Future` will be returned if it is omitted.
         """
-        self._set_read_callback(callback)
+        future = self._set_read_callback(callback)
         self._read_regex = re.compile(regex)
-        self._try_inline_read()
+        self._read_max_bytes = max_bytes
+        try:
+            self._try_inline_read()
+        except UnsatisfiableReadError as e:
+            # Handle this the same way as in _handle_events.
+            gen_log.info("Unsatisfiable read, closing connection: %s" % e)
+            self.close(exc_info=True)
+            return future
+        return future
 
-    def read_until(self, delimiter, callback):
-        """Run ``callback`` when we read the given delimiter.
+    def read_until(self, delimiter, callback=None, max_bytes=None):
+        """Asynchronously read until we have found the given delimiter.
 
-        The callback will get the data read (including the delimiter)
-        as an argument.
+        The result includes all the data read including the delimiter.
+        If a callback is given, it will be run with the data as an argument;
+        if not, this method returns a `.Future`.
+
+        If ``max_bytes`` is not None, the connection will be closed
+        if more than ``max_bytes`` bytes have been read and the delimiter
+        is not found.
+
+        .. versionchanged:: 3.3
+            Added the ``max_bytes`` argument.  The ``callback`` argument is
+            now optional and a `.Future` will be returned if it is omitted.
         """
-        self._set_read_callback(callback)
+        future = self._set_read_callback(callback)
         self._read_delimiter = delimiter
-        self._try_inline_read()
+        self._read_max_bytes = max_bytes
+        try:
+            self._try_inline_read()
+        except UnsatisfiableReadError as e:
+            # Handle this the same way as in _handle_events.
+            gen_log.info("Unsatisfiable read, closing connection: %s" % e)
+            self.close(exc_info=True)
+            return future
+        return future
 
-    def read_bytes(self, num_bytes, callback, streaming_callback=None):
-        """Run callback when we read the given number of bytes.
+    def read_bytes(self, num_bytes, callback=None, streaming_callback=None,
+                   partial=False):
+        """Asynchronously read a number of bytes.
 
         If a ``streaming_callback`` is given, it will be called with chunks
-        of data as they become available, and the argument to the final
-        ``callback`` will be empty.  Otherwise, the ``callback`` gets
-        the data as an argument.
+        of data as they become available, and the final result will be empty.
+        Otherwise, the result is all the data that was read.
+        If a callback is given, it will be run with the data as an argument;
+        if not, this method returns a `.Future`.
+
+        If ``partial`` is true, the callback is run as soon as we have
+        any bytes to return (but never more than ``num_bytes``)
+
+        .. versionchanged:: 3.3
+            Added the ``partial`` argument.  The callback argument is now
+            optional and a `.Future` will be returned if it is omitted.
         """
-        self._set_read_callback(callback)
+        future = self._set_read_callback(callback)
         assert isinstance(num_bytes, numbers.Integral)
         self._read_bytes = num_bytes
+        self._read_partial = partial
         self._streaming_callback = stack_context.wrap(streaming_callback)
         self._try_inline_read()
+        return future
 
-    def read_until_close(self, callback, streaming_callback=None):
-        """Reads all data from the socket until it is closed.
+    def read_until_close(self, callback=None, streaming_callback=None):
+        """Asynchronously reads all data from the socket until it is closed.
 
         If a ``streaming_callback`` is given, it will be called with chunks
-        of data as they become available, and the argument to the final
-        ``callback`` will be empty.  Otherwise, the ``callback`` gets the
-        data as an argument.
-
-        Subject to ``max_buffer_size`` limit from `IOStream` constructor if
-        a ``streaming_callback`` is not used.
+        of data as they become available, and the final result will be empty.
+        Otherwise, the result is all the data that was read.
+        If a callback is given, it will be run with the data as an argument;
+        if not, this method returns a `.Future`.
+
+        .. versionchanged:: 3.3
+            The callback argument is now optional and a `.Future` will
+            be returned if it is omitted.
         """
-        self._set_read_callback(callback)
+        future = self._set_read_callback(callback)
         self._streaming_callback = stack_context.wrap(streaming_callback)
         if self.closed():
             if self._streaming_callback is not None:
-                self._run_callback(self._streaming_callback,
-                                   self._consume(self._read_buffer_size))
-            self._run_callback(self._read_callback,
-                               self._consume(self._read_buffer_size))
-            self._streaming_callback = None
-            self._read_callback = None
-            return
+                self._run_read_callback(self._read_buffer_size, True)
+            self._run_read_callback(self._read_buffer_size, False)
+            return future
         self._read_until_close = True
-        self._streaming_callback = stack_context.wrap(streaming_callback)
         self._try_inline_read()
+        return future
 
     def write(self, data, callback=None):
-        """Write the given data to this stream.
+        """Asynchronously write the given data to this stream.
 
         If ``callback`` is given, we call it when all of the buffered write
         data has been successfully written to the stream. If there was
         previously buffered write data and an old write callback, that
         callback is simply overwritten with this new callback.
+
+        If no ``callback`` is given, this method returns a `.Future` that
+        resolves (with a result of ``None``) when the write has been
+        completed.  If `write` is called again before that `.Future` has
+        resolved, the previous future will be orphaned and will never resolve.
+
+        .. versionchanged:: 3.3
+            Now returns a `.Future` if no callback is given.
         """
         assert isinstance(data, bytes_type)
         self._check_closed()
         # We use bool(_write_buffer) as a proxy for write_buffer_size>0,
         # so never put empty strings in the buffer.
         if data:
+            if (self.max_write_buffer_size is not None and
+                    self._write_buffer_size + len(data) > self.max_write_buffer_size):
+                raise StreamBufferFullError("Reached maximum read buffer size")
             # Break up large contiguous strings before inserting them in the
             # write buffer, so we don't have to recopy the entire thing
             # as we slice off pieces to send to the socket.
             WRITE_BUFFER_CHUNK_SIZE = 128 * 1024
-            if len(data) > WRITE_BUFFER_CHUNK_SIZE:
-                for i in range(0, len(data), WRITE_BUFFER_CHUNK_SIZE):
-                    self._write_buffer.append(data[i:i + WRITE_BUFFER_CHUNK_SIZE])
-            else:
-                self._write_buffer.append(data)
-        self._write_callback = stack_context.wrap(callback)
+            for i in range(0, len(data), WRITE_BUFFER_CHUNK_SIZE):
+                self._write_buffer.append(data[i:i + WRITE_BUFFER_CHUNK_SIZE])
+            self._write_buffer_size += len(data)
+        if callback is not None:
+            self._write_callback = stack_context.wrap(callback)
+            future = None
+        else:
+            future = self._write_future = TracebackFuture()
         if not self._connecting:
             self._handle_write()
             if self._write_buffer:
                 self._add_io_state(self.io_loop.WRITE)
             self._maybe_add_error_listener()
+        return future
 
     def set_close_callback(self, callback):
-        """Call the given callback when the stream is closed."""
+        """Call the given callback when the stream is closed.
+
+        This is not necessary for applications that use the `.Future`
+        interface; all outstanding ``Futures`` will resolve with a
+        `StreamClosedError` when the stream is closed.
+        """
         self._close_callback = stack_context.wrap(callback)
+        self._maybe_add_error_listener()
 
     def close(self, exc_info=False):
         """Close this stream.
@@ -251,13 +364,9 @@ class BaseIOStream(object):
             if self._read_until_close:
                 if (self._streaming_callback is not None and
                         self._read_buffer_size):
-                    self._run_callback(self._streaming_callback,
-                                       self._consume(self._read_buffer_size))
-                callback = self._read_callback
-                self._read_callback = None
+                    self._run_read_callback(self._read_buffer_size, True)
                 self._read_until_close = False
-                self._run_callback(callback,
-                                   self._consume(self._read_buffer_size))
+                self._run_read_callback(self._read_buffer_size, False)
             if self._state is not None:
                 self.io_loop.remove_handler(self.fileno())
                 self._state = None
@@ -269,6 +378,25 @@ class BaseIOStream(object):
         # If there are pending callbacks, don't run the close callback
         # until they're done (see _maybe_add_error_handler)
         if self.closed() and self._pending_callbacks == 0:
+            futures = []
+            if self._read_future is not None:
+                futures.append(self._read_future)
+                self._read_future = None
+            if self._write_future is not None:
+                futures.append(self._write_future)
+                self._write_future = None
+            if self._connect_future is not None:
+                futures.append(self._connect_future)
+                self._connect_future = None
+            for future in futures:
+                if (isinstance(self.error, (socket.error, IOError)) and
+                        errno_from_exception(self.error) in _ERRNO_CONNRESET):
+                    # Treat connection resets as closed connections so
+                    # clients only have to catch one kind of exception
+                    # to avoid logging.
+                    future.set_exception(StreamClosedError())
+                else:
+                    future.set_exception(self.error or StreamClosedError())
             if self._close_callback is not None:
                 cb = self._close_callback
                 self._close_callback = None
@@ -282,7 +410,7 @@ class BaseIOStream(object):
 
     def reading(self):
         """Returns true if we are currently reading from the stream."""
-        return self._read_callback is not None
+        return self._read_callback is not None or self._read_future is not None
 
     def writing(self):
         """Returns true if we are currently writing to the stream."""
@@ -309,16 +437,22 @@ class BaseIOStream(object):
 
     def _handle_events(self, fd, events):
         if self.closed():
-            gen_log.warning("Got events for closed stream %d", fd)
+            gen_log.warning("Got events for closed stream %s", fd)
             return
         try:
+            if self._connecting:
+                # Most IOLoops will report a write failed connect
+                # with the WRITE event, but SelectIOLoop reports a
+                # READ as well so we must check for connecting before
+                # either.
+                self._handle_connect()
+            if self.closed():
+                return
             if events & self.io_loop.READ:
                 self._handle_read()
             if self.closed():
                 return
             if events & self.io_loop.WRITE:
-                if self._connecting:
-                    self._handle_connect()
                 self._handle_write()
             if self.closed():
                 return
@@ -334,13 +468,20 @@ class BaseIOStream(object):
                 state |= self.io_loop.READ
             if self.writing():
                 state |= self.io_loop.WRITE
-            if state == self.io_loop.ERROR:
+            if state == self.io_loop.ERROR and self._read_buffer_size == 0:
+                # If the connection is idle, listen for reads too so
+                # we can tell if the connection is closed.  If there is
+                # data in the read buffer we won't run the close callback
+                # yet anyway, so we don't need to listen in this case.
                 state |= self.io_loop.READ
             if state != self._state:
                 assert self._state is not None, \
                     "shouldn't happen: _handle_events without self._state"
                 self._state = state
                 self.io_loop.update_handler(self.fileno(), self._state)
+        except UnsatisfiableReadError as e:
+            gen_log.info("Unsatisfiable read, closing connection: %s" % e)
+            self.close(exc_info=True)
         except Exception:
             gen_log.error("Uncaught exception, closing connection.",
                           exc_info=True)
@@ -381,42 +522,108 @@ class BaseIOStream(object):
             self._pending_callbacks += 1
             self.io_loop.add_callback(wrapper)
 
-    def _handle_read(self):
+    def _read_to_buffer_loop(self):
+        # This method is called from _handle_read and _try_inline_read.
         try:
-            try:
-                # Pretend to have a pending callback so that an EOF in
-                # _read_to_buffer doesn't trigger an immediate close
-                # callback.  At the end of this method we'll either
-                # estabilsh a real pending callback via
-                # _read_from_buffer or run the close callback.
+            if self._read_bytes is not None:
+                target_bytes = self._read_bytes
+            elif self._read_max_bytes is not None:
+                target_bytes = self._read_max_bytes
+            elif self.reading():
+                # For read_until without max_bytes, or
+                # read_until_close, read as much as we can before
+                # scanning for the delimiter.
+                target_bytes = None
+            else:
+                target_bytes = 0
+            next_find_pos = 0
+            # Pretend to have a pending callback so that an EOF in
+            # _read_to_buffer doesn't trigger an immediate close
+            # callback.  At the end of this method we'll either
+            # estabilsh a real pending callback via
+            # _read_from_buffer or run the close callback.
+            #
+            # We need two try statements here so that
+            # pending_callbacks is decremented before the `except`
+            # clause below (which calls `close` and does need to
+            # trigger the callback)
+            self._pending_callbacks += 1
+            while not self.closed():
+                # Read from the socket until we get EWOULDBLOCK or equivalent.
+                # SSL sockets do some internal buffering, and if the data is
+                # sitting in the SSL object's buffer select() and friends
+                # can't see it; the only way to find out if it's there is to
+                # try to read it.
+                if self._read_to_buffer() == 0:
+                    break
+
+                self._run_streaming_callback()
+
+                # If we've read all the bytes we can use, break out of
+                # this loop.  We can't just call read_from_buffer here
+                # because of subtle interactions with the
+                # pending_callback and error_listener mechanisms.
                 #
-                # We need two try statements here so that
-                # pending_callbacks is decremented before the `except`
-                # clause below (which calls `close` and does need to
-                # trigger the callback)
-                self._pending_callbacks += 1
-                while not self.closed():
-                    # Read from the socket until we get EWOULDBLOCK or equivalent.
-                    # SSL sockets do some internal buffering, and if the data is
-                    # sitting in the SSL object's buffer select() and friends
-                    # can't see it; the only way to find out if it's there is to
-                    # try to read it.
-                    if self._read_to_buffer() == 0:
-                        break
-            finally:
-                self._pending_callbacks -= 1
+                # If we've reached target_bytes, we know we're done.
+                if (target_bytes is not None and
+                        self._read_buffer_size >= target_bytes):
+                    break
+
+                # Otherwise, we need to call the more expensive find_read_pos.
+                # It's inefficient to do this on every read, so instead
+                # do it on the first read and whenever the read buffer
+                # size has doubled.
+                if self._read_buffer_size >= next_find_pos:
+                    pos = self._find_read_pos()
+                    if pos is not None:
+                        return pos
+                    next_find_pos = self._read_buffer_size * 2
+            return self._find_read_pos()
+        finally:
+            self._pending_callbacks -= 1
+
+    def _handle_read(self):
+        try:
+            pos = self._read_to_buffer_loop()
+        except UnsatisfiableReadError:
+            raise
         except Exception:
             gen_log.warning("error on read", exc_info=True)
             self.close(exc_info=True)
             return
-        if self._read_from_buffer():
+        if pos is not None:
+            self._read_from_buffer(pos)
             return
         else:
             self._maybe_run_close_callback()
 
     def _set_read_callback(self, callback):
-        assert not self._read_callback, "Already reading"
-        self._read_callback = stack_context.wrap(callback)
+        assert self._read_callback is None, "Already reading"
+        assert self._read_future is None, "Already reading"
+        if callback is not None:
+            self._read_callback = stack_context.wrap(callback)
+        else:
+            self._read_future = TracebackFuture()
+        return self._read_future
+
+    def _run_read_callback(self, size, streaming):
+        if streaming:
+            callback = self._streaming_callback
+        else:
+            callback = self._read_callback
+            self._read_callback = self._streaming_callback = None
+        if self._read_future is not None:
+            assert callback is None
+            future = self._read_future
+            self._read_future = None
+            future.set_result(self._consume(size))
+        if callback is not None:
+            assert self._read_future is None
+            self._run_callback(callback, self._consume(size))
+        else:
+            # If we scheduled a callback, we will add the error listener
+            # afterwards.  If we didn't, we have to do it now.
+            self._maybe_add_error_listener()
 
     def _try_inline_read(self):
         """Attempt to complete the current read operation from buffered data.
@@ -426,18 +633,14 @@ class BaseIOStream(object):
         listening for reads on the socket.
         """
         # See if we've already got the data from a previous read
-        if self._read_from_buffer():
+        self._run_streaming_callback()
+        pos = self._find_read_pos()
+        if pos is not None:
+            self._read_from_buffer(pos)
             return
         self._check_closed()
         try:
-            try:
-                # See comments in _handle_read about incrementing _pending_callbacks
-                self._pending_callbacks += 1
-                while not self.closed():
-                    if self._read_to_buffer() == 0:
-                        break
-            finally:
-                self._pending_callbacks -= 1
+            pos = self._read_to_buffer_loop()
         except Exception:
             # If there was an in _read_to_buffer, we called close() already,
             # but couldn't run the close callback because of _pending_callbacks.
@@ -445,9 +648,15 @@ class BaseIOStream(object):
             # applicable.
             self._maybe_run_close_callback()
             raise
-        if self._read_from_buffer():
+        if pos is not None:
+            self._read_from_buffer(pos)
             return
-        self._maybe_add_error_listener()
+        # We couldn't satisfy the read inline, so either close the stream
+        # or listen for new data.
+        if self.closed():
+            self._maybe_run_close_callback()
+        else:
+            self._add_io_state(ioloop.IOLoop.READ)
 
     def _read_to_buffer(self):
         """Reads from the socket and appends the result to the read buffer.
@@ -472,32 +681,42 @@ class BaseIOStream(object):
             return 0
         self._read_buffer.append(chunk)
         self._read_buffer_size += len(chunk)
-        if self._read_buffer_size >= self.max_buffer_size:
+        if self._read_buffer_size > self.max_buffer_size:
             gen_log.error("Reached maximum read buffer size")
             self.close()
-            raise IOError("Reached maximum read buffer size")
+            raise StreamBufferFullError("Reached maximum read buffer size")
         return len(chunk)
 
-    def _read_from_buffer(self):
-        """Attempts to complete the currently-pending read from the buffer.
-
-        Returns True if the read was completed.
-        """
+    def _run_streaming_callback(self):
         if self._streaming_callback is not None and self._read_buffer_size:
             bytes_to_consume = self._read_buffer_size
             if self._read_bytes is not None:
                 bytes_to_consume = min(self._read_bytes, bytes_to_consume)
                 self._read_bytes -= bytes_to_consume
-            self._run_callback(self._streaming_callback,
-                               self._consume(bytes_to_consume))
-        if self._read_bytes is not None and self._read_buffer_size >= self._read_bytes:
-            num_bytes = self._read_bytes
-            callback = self._read_callback
-            self._read_callback = None
-            self._streaming_callback = None
-            self._read_bytes = None
-            self._run_callback(callback, self._consume(num_bytes))
-            return True
+            self._run_read_callback(bytes_to_consume, True)
+
+    def _read_from_buffer(self, pos):
+        """Attempts to complete the currently-pending read from the buffer.
+
+        The argument is either a position in the read buffer or None,
+        as returned by _find_read_pos.
+        """
+        self._read_bytes = self._read_delimiter = self._read_regex = None
+        self._read_partial = False
+        self._run_read_callback(pos, False)
+
+    def _find_read_pos(self):
+        """Attempts to find a position in the read buffer that satisfies
+        the currently-pending read.
+
+        Returns a position in the buffer if the current read can be satisfied,
+        or None if it cannot.
+        """
+        if (self._read_bytes is not None and
+            (self._read_buffer_size >= self._read_bytes or
+             (self._read_partial and self._read_buffer_size > 0))):
+            num_bytes = min(self._read_bytes, self._read_buffer_size)
+            return num_bytes
         elif self._read_delimiter is not None:
             # Multi-byte delimiters (e.g. '\r\n') may straddle two
             # chunks in the read buffer, so we can't easily find them
@@ -506,37 +725,40 @@ class BaseIOStream(object):
             # length) tend to be "line" oriented, the delimiter is likely
             # to be in the first few chunks.  Merge the buffer gradually
             # since large merges are relatively expensive and get undone in
-            # consume().
+            # _consume().
             if self._read_buffer:
                 while True:
                     loc = self._read_buffer[0].find(self._read_delimiter)
                     if loc != -1:
-                        callback = self._read_callback
                         delimiter_len = len(self._read_delimiter)
-                        self._read_callback = None
-                        self._streaming_callback = None
-                        self._read_delimiter = None
-                        self._run_callback(callback,
-                                           self._consume(loc + delimiter_len))
-                        return True
+                        self._check_max_bytes(self._read_delimiter,
+                                              loc + delimiter_len)
+                        return loc + delimiter_len
                     if len(self._read_buffer) == 1:
                         break
                     _double_prefix(self._read_buffer)
+                self._check_max_bytes(self._read_delimiter,
+                                      len(self._read_buffer[0]))
         elif self._read_regex is not None:
             if self._read_buffer:
                 while True:
                     m = self._read_regex.search(self._read_buffer[0])
                     if m is not None:
-                        callback = self._read_callback
-                        self._read_callback = None
-                        self._streaming_callback = None
-                        self._read_regex = None
-                        self._run_callback(callback, self._consume(m.end()))
-                        return True
+                        self._check_max_bytes(self._read_regex, m.end())
+                        return m.end()
                     if len(self._read_buffer) == 1:
                         break
                     _double_prefix(self._read_buffer)
-        return False
+                self._check_max_bytes(self._read_regex,
+                                      len(self._read_buffer[0]))
+        return None
+
+    def _check_max_bytes(self, delimiter, size):
+        if (self._read_max_bytes is not None and
+                size > self._read_max_bytes):
+            raise UnsatisfiableReadError(
+                "delimiter %r not found within %d bytes" % (
+                    delimiter, self._read_max_bytes))
 
     def _handle_write(self):
         while self._write_buffer:
@@ -563,6 +785,7 @@ class BaseIOStream(object):
                 self._write_buffer_frozen = False
                 _merge_prefix(self._write_buffer, num_bytes)
                 self._write_buffer.popleft()
+                self._write_buffer_size -= num_bytes
             except (socket.error, IOError, OSError) as e:
                 if e.args[0] in _ERRNO_WOULDBLOCK:
                     self._write_buffer_frozen = True
@@ -572,14 +795,19 @@ class BaseIOStream(object):
                         # Broken pipe errors are usually caused by connection
                         # reset, and its better to not log EPIPE errors to
                         # minimize log spam
-                        gen_log.warning("Write error on %d: %s",
+                        gen_log.warning("Write error on %s: %s",
                                         self.fileno(), e)
                     self.close(exc_info=True)
                     return
-        if not self._write_buffer and self._write_callback:
-            callback = self._write_callback
-            self._write_callback = None
-            self._run_callback(callback)
+        if not self._write_buffer:
+            if self._write_callback:
+                callback = self._write_callback
+                self._write_callback = None
+                self._run_callback(callback)
+            if self._write_future:
+                future = self._write_future
+                self._write_future = None
+                future.set_result(None)
 
     def _consume(self, loc):
         if loc == 0:
@@ -593,10 +821,19 @@ class BaseIOStream(object):
             raise StreamClosedError("Stream is closed")
 
     def _maybe_add_error_listener(self):
-        if self._state is None and self._pending_callbacks == 0:
+        # This method is part of an optimization: to detect a connection that
+        # is closed when we're not actively reading or writing, we must listen
+        # for read events.  However, it is inefficient to do this when the
+        # connection is first established because we are going to read or write
+        # immediately anyway.  Instead, we insert checks at various times to
+        # see if the connection is idle and add the read listener then.
+        if self._pending_callbacks != 0:
+            return
+        if self._state is None or self._state == ioloop.IOLoop.ERROR:
             if self.closed():
                 self._maybe_run_close_callback()
-            else:
+            elif (self._read_buffer_size == 0 and
+                  self._close_callback is not None):
                 self._add_io_state(ioloop.IOLoop.READ)
 
     def _add_io_state(self, state):
@@ -680,7 +917,7 @@ class IOStream(BaseIOStream):
         super(IOStream, self).__init__(*args, **kwargs)
 
     def fileno(self):
-        return self.socket.fileno()
+        return self.socket
 
     def close_fd(self):
         self.socket.close()
@@ -714,7 +951,9 @@ class IOStream(BaseIOStream):
         not previously connected.  The address parameter is in the
         same format as for `socket.connect <socket.socket.connect>`,
         i.e. a ``(host, port)`` tuple.  If ``callback`` is specified,
-        it will be called when the connection is completed.
+        it will be called with no arguments when the connection is
+        completed; if not this method returns a `.Future` (whose result
+        after a successful connection will be the stream itself).
 
         If specified, the ``server_hostname`` parameter will be used
         in SSL connections for certificate validation (if requested in
@@ -726,6 +965,9 @@ class IOStream(BaseIOStream):
         which case the data will be written as soon as the connection
         is ready.  Calling `IOStream` read methods before the socket is
         connected works on some platforms but is non-portable.
+
+        .. versionchanged:: 3.3
+            If no callback is given, returns a `.Future`.
         """
         self._connecting = True
         try:
@@ -738,14 +980,83 @@ class IOStream(BaseIOStream):
             # returned immediately when attempting to connect to
             # localhost, so handle them the same way as an error
             # reported later in _handle_connect.
-            if (e.args[0] != errno.EINPROGRESS and
-                    e.args[0] not in _ERRNO_WOULDBLOCK):
-                gen_log.warning("Connect error on fd %d: %s",
+            if (errno_from_exception(e) != errno.EINPROGRESS and
+                    errno_from_exception(e) not in _ERRNO_WOULDBLOCK):
+                gen_log.warning("Connect error on fd %s: %s",
                                 self.socket.fileno(), e)
                 self.close(exc_info=True)
                 return
-        self._connect_callback = stack_context.wrap(callback)
+        if callback is not None:
+            self._connect_callback = stack_context.wrap(callback)
+            future = None
+        else:
+            future = self._connect_future = TracebackFuture()
         self._add_io_state(self.io_loop.WRITE)
+        return future
+
+    def start_tls(self, server_side, ssl_options=None, server_hostname=None):
+        """Convert this `IOStream` to an `SSLIOStream`.
+
+        This enables protocols that begin in clear-text mode and
+        switch to SSL after some initial negotiation (such as the
+        ``STARTTLS`` extension to SMTP and IMAP).
+
+        This method cannot be used if there are outstanding reads
+        or writes on the stream, or if there is any data in the
+        IOStream's buffer (data in the operating system's socket
+        buffer is allowed).  This means it must generally be used
+        immediately after reading or writing the last clear-text
+        data.  It can also be used immediately after connecting,
+        before any reads or writes.
+
+        The ``ssl_options`` argument may be either a dictionary
+        of options or an `ssl.SSLContext`.  If a ``server_hostname``
+        is given, it will be used for certificate verification
+        (as configured in the ``ssl_options``).
+
+        This method returns a `.Future` whose result is the new
+        `SSLIOStream`.  After this method has been called,
+        any other operation on the original stream is undefined.
+
+        If a close callback is defined on this stream, it will be
+        transferred to the new stream.
+
+        .. versionadded:: 3.3
+        """
+        if (self._read_callback or self._read_future or
+                self._write_callback or self._write_future or
+                self._connect_callback or self._connect_future or
+                self._pending_callbacks or self._closed or
+                self._read_buffer or self._write_buffer):
+            raise ValueError("IOStream is not idle; cannot convert to SSL")
+        if ssl_options is None:
+            ssl_options = {}
+
+        socket = self.socket
+        self.io_loop.remove_handler(socket)
+        self.socket = None
+        socket = ssl_wrap_socket(socket, ssl_options, server_side=server_side,
+                                 do_handshake_on_connect=False)
+        orig_close_callback = self._close_callback
+        self._close_callback = None
+
+        future = TracebackFuture()
+        ssl_stream = SSLIOStream(socket, ssl_options=ssl_options,
+                                 io_loop=self.io_loop)
+        # Wrap the original close callback so we can fail our Future as well.
+        # If we had an "unwrap" counterpart to this method we would need
+        # to restore the original callback after our Future resolves
+        # so that repeated wrap/unwrap calls don't build up layers.
+        def close_callback():
+            if not future.done():
+                future.set_exception(ssl_stream.error or StreamClosedError())
+            if orig_close_callback is not None:
+                orig_close_callback()
+        ssl_stream.set_close_callback(close_callback)
+        ssl_stream._ssl_connect_callback = lambda: future.set_result(ssl_stream)
+        ssl_stream.max_buffer_size = self.max_buffer_size
+        ssl_stream.read_chunk_size = self.read_chunk_size
+        return future
 
     def _handle_connect(self):
         err = self.socket.getsockopt(socket.SOL_SOCKET, socket.SO_ERROR)
@@ -755,14 +1066,19 @@ class IOStream(BaseIOStream):
             # an error state before the socket becomes writable, so
             # in that case a connection failure would be handled by the
             # error path in _handle_events instead of here.
-            gen_log.warning("Connect error on fd %d: %s",
-                            self.socket.fileno(), errno.errorcode[err])
+            if self._connect_future is None:
+                gen_log.warning("Connect error on fd %s: %s",
+                                self.socket.fileno(), errno.errorcode[err])
             self.close()
             return
         if self._connect_callback is not None:
             callback = self._connect_callback
             self._connect_callback = None
             self._run_callback(callback)
+        if self._connect_future is not None:
+            future = self._connect_future
+            self._connect_future = None
+            future.set_result(self)
         self._connecting = False
 
     def set_nodelay(self, value):
@@ -841,7 +1157,7 @@ class SSLIOStream(IOStream):
                     peer = self.socket.getpeername()
                 except Exception:
                     peer = '(not connected)'
-                gen_log.warning("SSL Error on %d %s: %s",
+                gen_log.warning("SSL Error on %s %s: %s",
                                 self.socket.fileno(), peer, err)
                 return self.close(exc_info=True)
             raise
@@ -907,19 +1223,33 @@ class SSLIOStream(IOStream):
         # has completed.
         self._ssl_connect_callback = stack_context.wrap(callback)
         self._server_hostname = server_hostname
-        super(SSLIOStream, self).connect(address, callback=None)
+        # Note: Since we don't pass our callback argument along to
+        # super.connect(), this will always return a Future.
+        # This is harmless, but a bit less efficient than it could be.
+        return super(SSLIOStream, self).connect(address, callback=None)
 
     def _handle_connect(self):
+        # Call the superclass method to check for errors.
+        super(SSLIOStream, self)._handle_connect()
+        if self.closed():
+            return
         # When the connection is complete, wrap the socket for SSL
         # traffic.  Note that we do this by overriding _handle_connect
         # instead of by passing a callback to super().connect because
         # user callbacks are enqueued asynchronously on the IOLoop,
         # but since _handle_events calls _handle_connect immediately
         # followed by _handle_write we need this to be synchronous.
+        #
+        # The IOLoop will get confused if we swap out self.socket while the
+        # fd is registered, so remove it now and re-register after
+        # wrap_socket().
+        self.io_loop.remove_handler(self.socket)
+        old_state = self._state
+        self._state = None
         self.socket = ssl_wrap_socket(self.socket, self._ssl_options,
                                       server_hostname=self._server_hostname,
                                       do_handshake_on_connect=False)
-        super(SSLIOStream, self)._handle_connect()
+        self._add_io_state(old_state)
 
     def read_from_fd(self):
         if self._ssl_accepting:
@@ -978,9 +1308,9 @@ class PipeIOStream(BaseIOStream):
         try:
             chunk = os.read(self.fd, self.read_chunk_size)
         except (IOError, OSError) as e:
-            if e.args[0] in _ERRNO_WOULDBLOCK:
+            if errno_from_exception(e) in _ERRNO_WOULDBLOCK:
                 return None
-            elif e.args[0] == errno.EBADF:
+            elif errno_from_exception(e) == errno.EBADF:
                 # If the writing half of a pipe is closed, select will
                 # report it as readable but reads will fail with EBADF.
                 self.close(exc_info=True)

libs/tornado/log.py

@@ -83,10 +83,10 @@ class LogFormatter(logging.Formatter):
     DEFAULT_FORMAT = '%(color)s[%(levelname)1.1s %(asctime)s %(module)s:%(lineno)d]%(end_color)s %(message)s'
     DEFAULT_DATE_FORMAT = '%y%m%d %H:%M:%S'
     DEFAULT_COLORS = {
-        logging.DEBUG:      4,  # Blue
-        logging.INFO:       2,  # Green
-        logging.WARNING:    3,  # Yellow
-        logging.ERROR:      1,  # Red
+        logging.DEBUG: 4,  # Blue
+        logging.INFO: 2,  # Green
+        logging.WARNING: 3,  # Yellow
+        logging.ERROR: 1,  # Red
     }
 
     def __init__(self, color=True, fmt=DEFAULT_FORMAT,
@@ -184,7 +184,7 @@ def enable_pretty_logging(options=None, logger=None):
     """
     if options is None:
         from tornado.options import options
-    if options.logging == 'none':
+    if options.logging is None or options.logging.lower() == 'none':
         return
     if logger is None:
         logger = logging.getLogger()

libs/tornado/netutil.py

@@ -20,18 +20,26 @@ from __future__ import absolute_import, division, print_function, with_statement
 
 import errno
 import os
+import platform
 import socket
-import ssl
 import stat
 
 from tornado.concurrent import dummy_executor, run_on_executor
 from tornado.ioloop import IOLoop
 from tornado.platform.auto import set_close_exec
-from tornado.util import u, Configurable
+from tornado.util import u, Configurable, errno_from_exception
+
+try:
+    import ssl
+except ImportError:
+    # ssl is not available on Google App Engine
+    ssl = None
 
 if hasattr(ssl, 'match_hostname') and hasattr(ssl, 'CertificateError'):  # python 3.2+
     ssl_match_hostname = ssl.match_hostname
     SSLCertificateError = ssl.CertificateError
+elif ssl is None:
+    ssl_match_hostname = SSLCertificateError = None
 else:
     import backports.ssl_match_hostname
     ssl_match_hostname = backports.ssl_match_hostname.match_hostname
@@ -44,6 +52,11 @@ else:
 # thread now.
 u('foo').encode('idna')
 
+# These errnos indicate that a non-blocking operation must be retried
+# at a later time.  On most platforms they're the same value, but on
+# some they differ.
+_ERRNO_WOULDBLOCK = (errno.EWOULDBLOCK, errno.EAGAIN)
+
 
 def bind_sockets(port, address=None, family=socket.AF_UNSPEC, backlog=128, flags=None):
     """Creates listening sockets bound to the given port and address.
@@ -77,13 +90,23 @@ def bind_sockets(port, address=None, family=socket.AF_UNSPEC, backlog=128, flags
         family = socket.AF_INET
     if flags is None:
         flags = socket.AI_PASSIVE
+    bound_port = None
     for res in set(socket.getaddrinfo(address, port, family, socket.SOCK_STREAM,
                                       0, flags)):
         af, socktype, proto, canonname, sockaddr = res
+        if (platform.system() == 'Darwin' and address == 'localhost' and
+                af == socket.AF_INET6 and sockaddr[3] != 0):
+            # Mac OS X includes a link-local address fe80::1%lo0 in the
+            # getaddrinfo results for 'localhost'.  However, the firewall
+            # doesn't understand that this is a local address and will
+            # prompt for access (often repeatedly, due to an apparent
+            # bug in its ability to remember granting access to an
+            # application). Skip these addresses.
+            continue
         try:
             sock = socket.socket(af, socktype, proto)
         except socket.error as e:
-            if e.args[0] == errno.EAFNOSUPPORT:
+            if errno_from_exception(e) == errno.EAFNOSUPPORT:
                 continue
             raise
         set_close_exec(sock.fileno())
@@ -100,8 +123,16 @@ def bind_sockets(port, address=None, family=socket.AF_UNSPEC, backlog=128, flags
             # Python 2.x on windows doesn't have IPPROTO_IPV6.
             if hasattr(socket, "IPPROTO_IPV6"):
                 sock.setsockopt(socket.IPPROTO_IPV6, socket.IPV6_V6ONLY, 1)
+
+        # automatic port allocation with port=None
+        # should bind on the same port on IPv4 and IPv6
+        host, requested_port = sockaddr[:2]
+        if requested_port == 0 and bound_port is not None:
+            sockaddr = tuple([host, bound_port] + list(sockaddr[2:]))
+
         sock.setblocking(0)
         sock.bind(sockaddr)
+        bound_port = sock.getsockname()[1]
         sock.listen(backlog)
         sockets.append(sock)
     return sockets
@@ -124,7 +155,7 @@ if hasattr(socket, 'AF_UNIX'):
         try:
             st = os.stat(file)
         except OSError as err:
-            if err.errno != errno.ENOENT:
+            if errno_from_exception(err) != errno.ENOENT:
                 raise
         else:
             if stat.S_ISSOCK(st.st_mode):
@@ -154,18 +185,18 @@ def add_accept_handler(sock, callback, io_loop=None):
             try:
                 connection, address = sock.accept()
             except socket.error as e:
-                # EWOULDBLOCK and EAGAIN indicate we have accepted every
+                # _ERRNO_WOULDBLOCK indicate we have accepted every
                 # connection that is available.
-                if e.args[0] in (errno.EWOULDBLOCK, errno.EAGAIN):
+                if errno_from_exception(e) in _ERRNO_WOULDBLOCK:
                     return
                 # ECONNABORTED indicates that there was a connection
                 # but it was closed while still in the accept queue.
                 # (observed on FreeBSD).
-                if e.args[0] == errno.ECONNABORTED:
+                if errno_from_exception(e) == errno.ECONNABORTED:
                     continue
                 raise
             callback(connection, address)
-    io_loop.add_handler(sock.fileno(), accept_handler, IOLoop.READ)
+    io_loop.add_handler(sock, accept_handler, IOLoop.READ)
 
 
 def is_valid_ip(ip):
@@ -381,6 +412,10 @@ def ssl_options_to_context(ssl_options):
         context.load_verify_locations(ssl_options['ca_certs'])
     if 'ciphers' in ssl_options:
         context.set_ciphers(ssl_options['ciphers'])
+    if hasattr(ssl, 'OP_NO_COMPRESSION'):
+        # Disable TLS compression to avoid CRIME and related attacks.
+        # This constant wasn't added until python 3.3.
+        context.options |= ssl.OP_NO_COMPRESSION
     return context
 
 

libs/tornado/options.py

@@ -56,6 +56,18 @@ We support `datetimes <datetime.datetime>`, `timedeltas
 the top-level functions in this module (`define`, `parse_command_line`, etc)
 simply call methods on it.  You may create additional `OptionParser`
 instances to define isolated sets of options, such as for subcommands.
+
+.. note::
+
+   By default, several options are defined that will configure the
+   standard `logging` module when `parse_command_line` or `parse_config_file`
+   are called.  If you want Tornado to leave the logging configuration
+   alone so you can manage it yourself, either pass ``--logging=none``
+   on the command line or do the following to disable it in code::
+
+       from tornado.options import options, parse_command_line
+       options.logging = None
+       parse_command_line()
 """
 
 from __future__ import absolute_import, division, print_function, with_statement
@@ -360,6 +372,8 @@ class _Mockable(object):
 
 
 class _Option(object):
+    UNSET = object()
+
     def __init__(self, name, default=None, type=basestring_type, help=None,
                  metavar=None, multiple=False, file_name=None, group_name=None,
                  callback=None):
@@ -374,10 +388,10 @@ class _Option(object):
         self.group_name = group_name
         self.callback = callback
         self.default = default
-        self._value = None
+        self._value = _Option.UNSET
 
     def value(self):
-        return self.default if self._value is None else self._value
+        return self.default if self._value is _Option.UNSET else self._value
 
     def parse(self, value):
         _parse = {

libs/tornado/platform/asyncio.py

@@ -10,21 +10,31 @@ unfinished callbacks on the event loop that fail when it resumes)
 """
 
 from __future__ import absolute_import, division, print_function, with_statement
-import asyncio
 import datetime
 import functools
-import os
 
-from tornado.ioloop import IOLoop
+# _Timeout is used for its timedelta_to_seconds method for py26 compatibility.
+from tornado.ioloop import IOLoop, _Timeout
 from tornado import stack_context
 
+try:
+    # Import the real asyncio module for py33+ first.  Older versions of the
+    # trollius backport also use this name.
+    import asyncio
+except ImportError as e:
+    # Asyncio itself isn't available; see if trollius is (backport to py26+).
+    try:
+        import trollius as asyncio
+    except ImportError:
+        # Re-raise the original asyncio error, not the trollius one.
+        raise e
 
 class BaseAsyncIOLoop(IOLoop):
     def initialize(self, asyncio_loop, close_loop=False):
         self.asyncio_loop = asyncio_loop
         self.close_loop = close_loop
         self.asyncio_loop.call_soon(self.make_current)
-        # Maps fd to handler function (as in IOLoop.add_handler)
+        # Maps fd to (fileobj, handler function) pair (as in IOLoop.add_handler)
         self.handlers = {}
         # Set of fds listening for reads/writes
         self.readers = set()
@@ -34,19 +44,18 @@ class BaseAsyncIOLoop(IOLoop):
     def close(self, all_fds=False):
         self.closing = True
         for fd in list(self.handlers):
+            fileobj, handler_func = self.handlers[fd]
             self.remove_handler(fd)
             if all_fds:
-                try:
-                    os.close(fd)
-                except OSError:
-                    pass
+                self.close_fd(fileobj)
         if self.close_loop:
             self.asyncio_loop.close()
 
     def add_handler(self, fd, handler, events):
+        fd, fileobj = self.split_fd(fd)
         if fd in self.handlers:
-            raise ValueError("fd %d added twice" % fd)
-        self.handlers[fd] = stack_context.wrap(handler)
+            raise ValueError("fd %s added twice" % fd)
+        self.handlers[fd] = (fileobj, stack_context.wrap(handler))
         if events & IOLoop.READ:
             self.asyncio_loop.add_reader(
                 fd, self._handle_events, fd, IOLoop.READ)
@@ -57,6 +66,7 @@ class BaseAsyncIOLoop(IOLoop):
             self.writers.add(fd)
 
     def update_handler(self, fd, events):
+        fd, fileobj = self.split_fd(fd)
         if events & IOLoop.READ:
             if fd not in self.readers:
                 self.asyncio_loop.add_reader(
@@ -77,6 +87,7 @@ class BaseAsyncIOLoop(IOLoop):
                 self.writers.remove(fd)
 
     def remove_handler(self, fd):
+        fd, fileobj = self.split_fd(fd)
         if fd not in self.handlers:
             return
         if fd in self.readers:
@@ -88,7 +99,8 @@ class BaseAsyncIOLoop(IOLoop):
         del self.handlers[fd]
 
     def _handle_events(self, fd, events):
-        self.handlers[fd](fd, events)
+        fileobj, handler_func = self.handlers[fd]
+        handler_func(fileobj, events)
 
     def start(self):
         self._setup_logging()
@@ -107,7 +119,7 @@ class BaseAsyncIOLoop(IOLoop):
         if isinstance(deadline, (int, float)):
             delay = max(deadline - self.time(), 0)
         elif isinstance(deadline, datetime.timedelta):
-            delay = deadline.total_seconds()
+            delay = _Timeout.timedelta_to_seconds(deadline)
         else:
             raise TypeError("Unsupported deadline %r", deadline)
         return self.asyncio_loop.call_later(delay, self._run_callback,

libs/tornado/platform/auto.py

@@ -30,6 +30,10 @@ import os
 if os.name == 'nt':
     from tornado.platform.common import Waker
     from tornado.platform.windows import set_close_exec
+elif 'APPENGINE_RUNTIME' in os.environ:
+    from tornado.platform.common import Waker
+    def set_close_exec(fd):
+        pass
 else:
     from tornado.platform.posix import set_close_exec, Waker
 

libs/tornado/platform/common.py

@@ -15,7 +15,8 @@ class Waker(interface.Waker):
     and Jython.
     """
     def __init__(self):
-        # Based on Zope async.py: http://svn.zope.org/zc.ngi/trunk/src/zc/ngi/async.py
+        # Based on Zope select_trigger.py:
+        # https://github.com/zopefoundation/Zope/blob/master/src/ZServer/medusa/thread/select_trigger.py
 
         self.writer = socket.socket()
         # Disable buffering -- pulling the trigger sends 1 byte,

libs/tornado/platform/kqueue.py

@@ -37,7 +37,7 @@ class _KQueue(object):
 
     def register(self, fd, events):
         if fd in self._active:
-            raise IOError("fd %d already registered" % fd)
+            raise IOError("fd %s already registered" % fd)
         self._control(fd, events, select.KQ_EV_ADD)
         self._active[fd] = events
 

libs/tornado/platform/select.py

@@ -37,7 +37,7 @@ class _Select(object):
 
     def register(self, fd, events):
         if fd in self.read_fds or fd in self.write_fds or fd in self.error_fds:
-            raise IOError("fd %d already registered" % fd)
+            raise IOError("fd %s already registered" % fd)
         if events & IOLoop.READ:
             self.read_fds.add(fd)
         if events & IOLoop.WRITE:

libs/tornado/platform/twisted.py

@@ -91,6 +91,11 @@ from tornado.netutil import Resolver
 from tornado.stack_context import NullContext, wrap
 from tornado.ioloop import IOLoop
 
+try:
+    long  # py2
+except NameError:
+    long = int  # py3
+
 
 @implementer(IDelayedCall)
 class TornadoDelayedCall(object):
@@ -365,8 +370,9 @@ def install(io_loop=None):
 
 @implementer(IReadDescriptor, IWriteDescriptor)
 class _FD(object):
-    def __init__(self, fd, handler):
+    def __init__(self, fd, fileobj, handler):
         self.fd = fd
+        self.fileobj = fileobj
         self.handler = handler
         self.reading = False
         self.writing = False
@@ -377,15 +383,15 @@ class _FD(object):
 
     def doRead(self):
         if not self.lost:
-            self.handler(self.fd, tornado.ioloop.IOLoop.READ)
+            self.handler(self.fileobj, tornado.ioloop.IOLoop.READ)
 
     def doWrite(self):
         if not self.lost:
-            self.handler(self.fd, tornado.ioloop.IOLoop.WRITE)
+            self.handler(self.fileobj, tornado.ioloop.IOLoop.WRITE)
 
     def connectionLost(self, reason):
         if not self.lost:
-            self.handler(self.fd, tornado.ioloop.IOLoop.ERROR)
+            self.handler(self.fileobj, tornado.ioloop.IOLoop.ERROR)
             self.lost = True
 
     def logPrefix(self):
@@ -412,14 +418,19 @@ class TwistedIOLoop(tornado.ioloop.IOLoop):
         self.reactor.callWhenRunning(self.make_current)
 
     def close(self, all_fds=False):
+        fds = self.fds
         self.reactor.removeAll()
         for c in self.reactor.getDelayedCalls():
             c.cancel()
+        if all_fds:
+            for fd in fds.values():
+                self.close_fd(fd.fileobj)
 
     def add_handler(self, fd, handler, events):
         if fd in self.fds:
-            raise ValueError('fd %d added twice' % fd)
-        self.fds[fd] = _FD(fd, wrap(handler))
+            raise ValueError('fd %s added twice' % fd)
+        fd, fileobj = self.split_fd(fd)
+        self.fds[fd] = _FD(fd, fileobj, wrap(handler))
         if events & tornado.ioloop.IOLoop.READ:
             self.fds[fd].reading = True
             self.reactor.addReader(self.fds[fd])
@@ -428,6 +439,7 @@ class TwistedIOLoop(tornado.ioloop.IOLoop):
             self.reactor.addWriter(self.fds[fd])
 
     def update_handler(self, fd, events):
+        fd, fileobj = self.split_fd(fd)
         if events & tornado.ioloop.IOLoop.READ:
             if not self.fds[fd].reading:
                 self.fds[fd].reading = True
@@ -446,6 +458,7 @@ class TwistedIOLoop(tornado.ioloop.IOLoop):
                 self.reactor.removeWriter(self.fds[fd])
 
     def remove_handler(self, fd):
+        fd, fileobj = self.split_fd(fd)
         if fd not in self.fds:
             return
         self.fds[fd].lost = True

libs/tornado/process.py

@@ -21,7 +21,6 @@ the server into multiple processes and managing subprocesses.
 from __future__ import absolute_import, division, print_function, with_statement
 
 import errno
-import multiprocessing
 import os
 import signal
 import subprocess
@@ -35,6 +34,13 @@ from tornado.iostream import PipeIOStream
 from tornado.log import gen_log
 from tornado.platform.auto import set_close_exec
 from tornado import stack_context
+from tornado.util import errno_from_exception
+
+try:
+    import multiprocessing
+except ImportError:
+    # Multiprocessing is not availble on Google App Engine.
+    multiprocessing = None
 
 try:
     long  # py2
@@ -44,6 +50,8 @@ except NameError:
 
 def cpu_count():
     """Returns the number of processors on this machine."""
+    if multiprocessing is None:
+        return 1
     try:
         return multiprocessing.cpu_count()
     except NotImplementedError:
@@ -136,7 +144,7 @@ def fork_processes(num_processes, max_restarts=100):
         try:
             pid, status = os.wait()
         except OSError as e:
-            if e.errno == errno.EINTR:
+            if errno_from_exception(e) == errno.EINTR:
                 continue
             raise
         if pid not in children:
@@ -283,7 +291,7 @@ class Subprocess(object):
         try:
             ret_pid, status = os.waitpid(pid, os.WNOHANG)
         except OSError as e:
-            if e.args[0] == errno.ECHILD:
+            if errno_from_exception(e) == errno.ECHILD:
                 return
         if ret_pid == 0:
             return

libs/tornado/simple_httpclient.py

@@ -1,23 +1,23 @@
 #!/usr/bin/env python
 from __future__ import absolute_import, division, print_function, with_statement
 
-from tornado.escape import utf8, _unicode, native_str
+from tornado.concurrent import is_future
+from tornado.escape import utf8, _unicode
 from tornado.httpclient import HTTPResponse, HTTPError, AsyncHTTPClient, main, _RequestProxy
-from tornado.httputil import HTTPHeaders
-from tornado.iostream import IOStream, SSLIOStream
+from tornado import httputil
+from tornado.http1connection import HTTP1Connection, HTTP1ConnectionParameters
+from tornado.iostream import StreamClosedError
 from tornado.netutil import Resolver, OverrideResolver
 from tornado.log import gen_log
 from tornado import stack_context
-from tornado.util import GzipDecompressor
+from tornado.tcpclient import TCPClient
 
 import base64
 import collections
 import copy
 import functools
-import os.path
 import re
 import socket
-import ssl
 import sys
 
 try:
@@ -30,7 +30,23 @@ try:
 except ImportError:
     import urllib.parse as urlparse  # py3
 
-_DEFAULT_CA_CERTS = os.path.dirname(__file__) + '/ca-certificates.crt'
+try:
+    import ssl
+except ImportError:
+    # ssl is not available on Google App Engine.
+    ssl = None
+
+try:
+    import certifi
+except ImportError:
+    certifi = None
+
+
+def _default_ca_certs():
+    if certifi is None:
+        raise Exception("The 'certifi' package is required to use https "
+                        "in simple_httpclient")
+    return certifi.where()
 
 
 class SimpleAsyncHTTPClient(AsyncHTTPClient):
@@ -47,7 +63,7 @@ class SimpleAsyncHTTPClient(AsyncHTTPClient):
     """
     def initialize(self, io_loop, max_clients=10,
                    hostname_mapping=None, max_buffer_size=104857600,
-                   resolver=None, defaults=None):
+                   resolver=None, defaults=None, max_header_size=None):
         """Creates a AsyncHTTPClient.
 
         Only a single AsyncHTTPClient instance exists per IOLoop
@@ -74,6 +90,9 @@ class SimpleAsyncHTTPClient(AsyncHTTPClient):
         self.active = {}
         self.waiting = {}
         self.max_buffer_size = max_buffer_size
+        self.max_header_size = max_header_size
+        # TCPClient could create a Resolver for us, but we have to do it
+        # ourselves to support hostname_mapping.
         if resolver:
             self.resolver = resolver
             self.own_resolver = False
@@ -83,11 +102,13 @@ class SimpleAsyncHTTPClient(AsyncHTTPClient):
         if hostname_mapping is not None:
             self.resolver = OverrideResolver(resolver=self.resolver,
                                              mapping=hostname_mapping)
+        self.tcp_client = TCPClient(resolver=self.resolver, io_loop=io_loop)
 
     def close(self):
         super(SimpleAsyncHTTPClient, self).close()
         if self.own_resolver:
             self.resolver.close()
+        self.tcp_client.close()
 
     def fetch_impl(self, request, callback):
         key = object()
@@ -119,7 +140,8 @@ class SimpleAsyncHTTPClient(AsyncHTTPClient):
 
     def _handle_request(self, request, release_callback, final_callback):
         _HTTPConnection(self.io_loop, self, request, release_callback,
-                        final_callback, self.max_buffer_size, self.resolver)
+                        final_callback, self.max_buffer_size, self.tcp_client,
+                        self.max_header_size)
 
     def _release_fetch(self, key):
         del self.active[key]
@@ -142,11 +164,12 @@ class SimpleAsyncHTTPClient(AsyncHTTPClient):
         del self.waiting[key]
 
 
-class _HTTPConnection(object):
+class _HTTPConnection(httputil.HTTPMessageDelegate):
     _SUPPORTED_METHODS = set(["GET", "HEAD", "POST", "PUT", "DELETE", "PATCH", "OPTIONS"])
 
     def __init__(self, io_loop, client, request, release_callback,
-                 final_callback, max_buffer_size, resolver):
+                 final_callback, max_buffer_size, tcp_client,
+                 max_header_size):
         self.start_time = io_loop.time()
         self.io_loop = io_loop
         self.client = client
@@ -154,13 +177,15 @@ class _HTTPConnection(object):
         self.release_callback = release_callback
         self.final_callback = final_callback
         self.max_buffer_size = max_buffer_size
-        self.resolver = resolver
+        self.tcp_client = tcp_client
+        self.max_header_size = max_header_size
         self.code = None
         self.headers = None
-        self.chunks = None
+        self.chunks = []
         self._decompressor = None
         # Timeout handle returned by IOLoop.add_timeout
         self._timeout = None
+        self._sockaddr = None
         with stack_context.ExceptionStackContext(self._handle_exception):
             self.parsed = urlparse.urlsplit(_unicode(self.request.url))
             if self.parsed.scheme not in ("http", "https"):
@@ -183,42 +208,31 @@ class _HTTPConnection(object):
                 host = host[1:-1]
             self.parsed_hostname = host  # save final host for _on_connect
 
-            if request.allow_ipv6:
-                af = socket.AF_UNSPEC
-            else:
-                # We only try the first IP we get from getaddrinfo,
-                # so restrict to ipv4 by default.
+            if request.allow_ipv6 is False:
                 af = socket.AF_INET
+            else:
+                af = socket.AF_UNSPEC
+
+            ssl_options = self._get_ssl_options(self.parsed.scheme)
 
             timeout = min(self.request.connect_timeout, self.request.request_timeout)
             if timeout:
                 self._timeout = self.io_loop.add_timeout(
                     self.start_time + timeout,
                     stack_context.wrap(self._on_timeout))
-            self.resolver.resolve(host, port, af, callback=self._on_resolve)
+            self.tcp_client.connect(host, port, af=af,
+                                    ssl_options=ssl_options,
+                                    callback=self._on_connect)
 
-    def _on_resolve(self, addrinfo):
-        if self.final_callback is None:
-            # final_callback is cleared if we've hit our timeout
-            return
-        self.stream = self._create_stream(addrinfo)
-        self.stream.set_close_callback(self._on_close)
-        # ipv6 addresses are broken (in self.parsed.hostname) until
-        # 2.7, here is correctly parsed value calculated in __init__
-        sockaddr = addrinfo[0][1]
-        self.stream.connect(sockaddr, self._on_connect,
-                            server_hostname=self.parsed_hostname)
-
-    def _create_stream(self, addrinfo):
-        af = addrinfo[0][0]
-        if self.parsed.scheme == "https":
+    def _get_ssl_options(self, scheme):
+        if scheme == "https":
             ssl_options = {}
             if self.request.validate_cert:
                 ssl_options["cert_reqs"] = ssl.CERT_REQUIRED
             if self.request.ca_certs is not None:
                 ssl_options["ca_certs"] = self.request.ca_certs
             else:
-                ssl_options["ca_certs"] = _DEFAULT_CA_CERTS
+                ssl_options["ca_certs"] = _default_ca_certs()
             if self.request.client_key is not None:
                 ssl_options["keyfile"] = self.request.client_key
             if self.request.client_cert is not None:
@@ -236,21 +250,16 @@ class _HTTPConnection(object):
             # but nearly all servers support both SSLv3 and TLSv1:
             # http://blog.ivanristic.com/2011/09/ssl-survey-protocol-support.html
             if sys.version_info >= (2, 7):
-                ssl_options["ciphers"] = "DEFAULT:!SSLv2"
+                # In addition to disabling SSLv2, we also exclude certain
+                # classes of insecure ciphers.
+                ssl_options["ciphers"] = "DEFAULT:!SSLv2:!EXPORT:!DES"
             else:
                 # This is really only necessary for pre-1.0 versions
                 # of openssl, but python 2.6 doesn't expose version
                 # information.
                 ssl_options["ssl_version"] = ssl.PROTOCOL_TLSv1
-
-            return SSLIOStream(socket.socket(af),
-                               io_loop=self.io_loop,
-                               ssl_options=ssl_options,
-                               max_buffer_size=self.max_buffer_size)
-        else:
-            return IOStream(socket.socket(af),
-                            io_loop=self.io_loop,
-                            max_buffer_size=self.max_buffer_size)
+            return ssl_options
+        return None
 
     def _on_timeout(self):
         self._timeout = None
@@ -262,7 +271,13 @@ class _HTTPConnection(object):
             self.io_loop.remove_timeout(self._timeout)
             self._timeout = None
 
-    def _on_connect(self):
+    def _on_connect(self, stream):
+        if self.final_callback is None:
+            # final_callback is cleared if we've hit our timeout.
+            stream.close()
+            return
+        self.stream = stream
+        self.stream.set_close_callback(self._on_close)
         self._remove_timeout()
         if self.final_callback is None:
             return
@@ -302,16 +317,22 @@ class _HTTPConnection(object):
             self.request.headers["User-Agent"] = self.request.user_agent
         if not self.request.allow_nonstandard_methods:
             if self.request.method in ("POST", "PATCH", "PUT"):
-                if self.request.body is None:
+                if (self.request.body is None and
+                        self.request.body_producer is None):
                     raise AssertionError(
                         'Body must not be empty for "%s" request'
                         % self.request.method)
             else:
-                if self.request.body is not None:
+                if (self.request.body is not None or
+                        self.request.body_producer is not None):
                     raise AssertionError(
                         'Body must be empty for "%s" request'
                         % self.request.method)
+        if self.request.expect_100_continue:
+            self.request.headers["Expect"] = "100-continue"
         if self.request.body is not None:
+            # When body_producer is used the caller is responsible for
+            # setting Content-Length (or else chunked encoding will be used).
             self.request.headers["Content-Length"] = str(len(
                 self.request.body))
         if (self.request.method == "POST" and
@@ -320,20 +341,47 @@ class _HTTPConnection(object):
         if self.request.use_gzip:
             self.request.headers["Accept-Encoding"] = "gzip"
         req_path = ((self.parsed.path or '/') +
-                   (('?' + self.parsed.query) if self.parsed.query else ''))
-        request_lines = [utf8("%s %s HTTP/1.1" % (self.request.method,
-                                                  req_path))]
-        for k, v in self.request.headers.get_all():
-            line = utf8(k) + b": " + utf8(v)
-            if b'\n' in line:
-                raise ValueError('Newline in header: ' + repr(line))
-            request_lines.append(line)
-        request_str = b"\r\n".join(request_lines) + b"\r\n\r\n"
-        if self.request.body is not None:
-            request_str += self.request.body
+                    (('?' + self.parsed.query) if self.parsed.query else ''))
         self.stream.set_nodelay(True)
-        self.stream.write(request_str)
-        self.stream.read_until_regex(b"\r?\n\r?\n", self._on_headers)
+        self.connection = HTTP1Connection(
+            self.stream, True,
+            HTTP1ConnectionParameters(
+                no_keep_alive=True,
+                max_header_size=self.max_header_size,
+                use_gzip=self.request.use_gzip),
+            self._sockaddr)
+        start_line = httputil.RequestStartLine(self.request.method,
+                                               req_path, 'HTTP/1.1')
+        self.connection.write_headers(start_line, self.request.headers)
+        if self.request.expect_100_continue:
+            self._read_response()
+        else:
+            self._write_body(True)
+
+    def _write_body(self, start_read):
+        if self.request.body is not None:
+            self.connection.write(self.request.body)
+            self.connection.finish()
+        elif self.request.body_producer is not None:
+            fut = self.request.body_producer(self.connection.write)
+            if is_future(fut):
+                def on_body_written(fut):
+                    fut.result()
+                    self.connection.finish()
+                    if start_read:
+                        self._read_response()
+                self.io_loop.add_future(fut, on_body_written)
+                return
+            self.connection.finish()
+        if start_read:
+            self._read_response()
+
+    def _read_response(self):
+        # Ensure that any exception raised in read_response ends up in our
+        # stack context.
+        self.io_loop.add_future(
+            self.connection.read_response(self),
+            lambda f: f.result())
 
     def _release(self):
         if self.release_callback is not None:
@@ -351,43 +399,39 @@ class _HTTPConnection(object):
     def _handle_exception(self, typ, value, tb):
         if self.final_callback:
             self._remove_timeout()
+            if isinstance(value, StreamClosedError):
+                value = HTTPError(599, "Stream closed")
             self._run_callback(HTTPResponse(self.request, 599, error=value,
                                             request_time=self.io_loop.time() - self.start_time,
                                             ))
 
             if hasattr(self, "stream"):
+                # TODO: this may cause a StreamClosedError to be raised
+                # by the connection's Future.  Should we cancel the
+                # connection more gracefully?
                 self.stream.close()
             return True
         else:
             # If our callback has already been called, we are probably
             # catching an exception that is not caused by us but rather
             # some child of our callback. Rather than drop it on the floor,
-            # pass it along.
-            return False
+            # pass it along, unless it's just the stream being closed.
+            return isinstance(value, StreamClosedError)
 
     def _on_close(self):
         if self.final_callback is not None:
             message = "Connection closed"
             if self.stream.error:
-                message = str(self.stream.error)
+                raise self.stream.error
             raise HTTPError(599, message)
 
-    def _handle_1xx(self, code):
-        self.stream.read_until_regex(b"\r?\n\r?\n", self._on_headers)
-
-    def _on_headers(self, data):
-        data = native_str(data.decode("latin1"))
-        first_line, _, header_data = data.partition("\n")
-        match = re.match("HTTP/1.[01] ([0-9]+) ([^\r]*)", first_line)
-        assert match
-        code = int(match.group(1))
-        self.headers = HTTPHeaders.parse(header_data)
-        if 100 <= code < 200:
-            self._handle_1xx(code)
+    def headers_received(self, first_line, headers):
+        if self.request.expect_100_continue and first_line.code == 100:
+            self._write_body(False)
             return
-        else:
-            self.code = code
-            self.reason = match.group(2)
+        self.headers = headers
+        self.code = first_line.code
+        self.reason = first_line.reason
 
         if "Content-Length" in self.headers:
             if "," in self.headers["Content-Length"]:
@@ -404,17 +448,12 @@ class _HTTPConnection(object):
             content_length = None
 
         if self.request.header_callback is not None:
-            # re-attach the newline we split on earlier
-            self.request.header_callback(first_line + _)
+            # Reassemble the start line.
+            self.request.header_callback('%s %s %s\r\n' % first_line)
             for k, v in self.headers.get_all():
                 self.request.header_callback("%s: %s\r\n" % (k, v))
             self.request.header_callback('\r\n')
 
-        if self.request.method == "HEAD" or self.code == 304:
-            # HEAD requests and 304 responses never have content, even
-            # though they may have content-length headers
-            self._on_body(b"")
-            return
         if 100 <= self.code < 200 or self.code == 204:
             # These response codes never have bodies
             # http://www.w3.org/Protocols/rfc2616/rfc2616-sec4.html#sec4.3
@@ -422,21 +461,9 @@ class _HTTPConnection(object):
                     content_length not in (None, 0)):
                 raise ValueError("Response with code %d should not have body" %
                                  self.code)
-            self._on_body(b"")
-            return
 
-        if (self.request.use_gzip and
-                self.headers.get("Content-Encoding") == "gzip"):
-            self._decompressor = GzipDecompressor()
-        if self.headers.get("Transfer-Encoding") == "chunked":
-            self.chunks = []
-            self.stream.read_until(b"\r\n", self._on_chunk_length)
-        elif content_length is not None:
-            self.stream.read_bytes(content_length, self._on_body)
-        else:
-            self.stream.read_until_close(self._on_body)
-
-    def _on_body(self, data):
+    def finish(self):
+        data = b''.join(self.chunks)
         self._remove_timeout()
         original_request = getattr(self.request, "original_request",
                                    self.request)
@@ -472,19 +499,12 @@ class _HTTPConnection(object):
             self.client.fetch(new_request, final_callback)
             self._on_end_request()
             return
-        if self._decompressor:
-            data = (self._decompressor.decompress(data) +
-                    self._decompressor.flush())
         if self.request.streaming_callback:
-            if self.chunks is None:
-                # if chunks is not None, we already called streaming_callback
-                # in _on_chunk_data
-                self.request.streaming_callback(data)
             buffer = BytesIO()
         else:
             buffer = BytesIO(data)  # TODO: don't require one big string?
         response = HTTPResponse(original_request,
-                                self.code, reason=self.reason,
+                                self.code, reason=getattr(self, 'reason', None),
                                 headers=self.headers,
                                 request_time=self.io_loop.time() - self.start_time,
                                 buffer=buffer,
@@ -495,40 +515,11 @@ class _HTTPConnection(object):
     def _on_end_request(self):
         self.stream.close()
 
-    def _on_chunk_length(self, data):
-        # TODO: "chunk extensions" http://tools.ietf.org/html/rfc2616#section-3.6.1
-        length = int(data.strip(), 16)
-        if length == 0:
-            if self._decompressor is not None:
-                tail = self._decompressor.flush()
-                if tail:
-                    # I believe the tail will always be empty (i.e.
-                    # decompress will return all it can).  The purpose
-                    # of the flush call is to detect errors such
-                    # as truncated input.  But in case it ever returns
-                    # anything, treat it as an extra chunk
-                    if self.request.streaming_callback is not None:
-                        self.request.streaming_callback(tail)
-                    else:
-                        self.chunks.append(tail)
-                # all the data has been decompressed, so we don't need to
-                # decompress again in _on_body
-                self._decompressor = None
-            self._on_body(b''.join(self.chunks))
-        else:
-            self.stream.read_bytes(length + 2,  # chunk ends with \r\n
-                                   self._on_chunk_data)
-
-    def _on_chunk_data(self, data):
-        assert data[-2:] == b"\r\n"
-        chunk = data[:-2]
-        if self._decompressor:
-            chunk = self._decompressor.decompress(chunk)
+    def data_received(self, chunk):
         if self.request.streaming_callback is not None:
             self.request.streaming_callback(chunk)
         else:
             self.chunks.append(chunk)
-        self.stream.read_until(b"\r\n", self._on_chunk_length)
 
 
 if __name__ == "__main__":

libs/tornado/speedups.c

@@ -1,21 +1,24 @@
+#define PY_SSIZE_T_CLEAN
 #include <Python.h>
 
 static PyObject* websocket_mask(PyObject* self, PyObject* args) {
     const char* mask;
-    int mask_len;
+    Py_ssize_t mask_len;
     const char* data;
-    int data_len;
-    int i;
+    Py_ssize_t data_len;
+    Py_ssize_t i;
+    PyObject* result;
+    char* buf;
 
     if (!PyArg_ParseTuple(args, "s#s#", &mask, &mask_len, &data, &data_len)) {
         return NULL;
     }
 
-    PyObject* result = PyBytes_FromStringAndSize(NULL, data_len);
+    result = PyBytes_FromStringAndSize(NULL, data_len);
     if (!result) {
         return NULL;
     }
-    char* buf = PyBytes_AsString(result);
+    buf = PyBytes_AsString(result);
     for (i = 0; i < data_len; i++) {
         buf[i] = data[i] ^ mask[i % 4];
     }

libs/tornado/stack_context.py

@@ -266,6 +266,18 @@ def wrap(fn):
     # TODO: Any other better way to store contexts and update them in wrapped function?
     cap_contexts = [_state.contexts]
 
+    if not cap_contexts[0][0] and not cap_contexts[0][1]:
+        # Fast path when there are no active contexts.
+        def null_wrapper(*args, **kwargs):
+            try:
+                current_state = _state.contexts
+                _state.contexts = cap_contexts[0]
+                return fn(*args, **kwargs)
+            finally:
+                _state.contexts = current_state
+        null_wrapper._wrapped = True
+        return null_wrapper
+
     def wrapped(*args, **kwargs):
         ret = None
         try:

libs/tornado/tcpclient.py

@@ -0,0 +1,179 @@
+#!/usr/bin/env python
+#
+# Copyright 2014 Facebook
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+"""A non-blocking TCP connection factory.
+"""
+from __future__ import absolute_import, division, print_function, with_statement
+
+import functools
+import socket
+
+from tornado.concurrent import Future
+from tornado.ioloop import IOLoop
+from tornado.iostream import IOStream
+from tornado import gen
+from tornado.netutil import Resolver
+
+_INITIAL_CONNECT_TIMEOUT = 0.3
+
+
+class _Connector(object):
+    """A stateless implementation of the "Happy Eyeballs" algorithm.
+
+    "Happy Eyeballs" is documented in RFC6555 as the recommended practice
+    for when both IPv4 and IPv6 addresses are available.
+
+    In this implementation, we partition the addresses by family, and
+    make the first connection attempt to whichever address was
+    returned first by ``getaddrinfo``.  If that connection fails or
+    times out, we begin a connection in parallel to the first address
+    of the other family.  If there are additional failures we retry
+    with other addresses, keeping one connection attempt per family
+    in flight at a time.
+
+    http://tools.ietf.org/html/rfc6555
+
+    """
+    def __init__(self, addrinfo, io_loop, connect):
+        self.io_loop = io_loop
+        self.connect = connect
+
+        self.future = Future()
+        self.timeout = None
+        self.last_error = None
+        self.remaining = len(addrinfo)
+        self.primary_addrs, self.secondary_addrs = self.split(addrinfo)
+
+    @staticmethod
+    def split(addrinfo):
+        """Partition the ``addrinfo`` list by address family.
+
+        Returns two lists.  The first list contains the first entry from
+        ``addrinfo`` and all others with the same family, and the
+        second list contains all other addresses (normally one list will
+        be AF_INET and the other AF_INET6, although non-standard resolvers
+        may return additional families).
+        """
+        primary = []
+        secondary = []
+        primary_af = addrinfo[0][0]
+        for af, addr in addrinfo:
+            if af == primary_af:
+                primary.append((af, addr))
+            else:
+                secondary.append((af, addr))
+        return primary, secondary
+
+    def start(self, timeout=_INITIAL_CONNECT_TIMEOUT):
+        self.try_connect(iter(self.primary_addrs))
+        self.set_timout(timeout)
+        return self.future
+
+    def try_connect(self, addrs):
+        try:
+            af, addr = next(addrs)
+        except StopIteration:
+            # We've reached the end of our queue, but the other queue
+            # might still be working.  Send a final error on the future
+            # only when both queues are finished.
+            if self.remaining == 0 and not self.future.done():
+                self.future.set_exception(self.last_error or
+                                          IOError("connection failed"))
+            return
+        future = self.connect(af, addr)
+        future.add_done_callback(functools.partial(self.on_connect_done,
+                                                   addrs, af, addr))
+
+    def on_connect_done(self, addrs, af, addr, future):
+        self.remaining -= 1
+        try:
+            stream = future.result()
+        except Exception as e:
+            if self.future.done():
+                return
+            # Error: try again (but remember what happened so we have an
+            # error to raise in the end)
+            self.last_error = e
+            self.try_connect(addrs)
+            if self.timeout is not None:
+                # If the first attempt failed, don't wait for the
+                # timeout to try an address from the secondary queue.
+                self.on_timeout()
+            return
+        self.clear_timeout()
+        if self.future.done():
+            # This is a late arrival; just drop it.
+            stream.close()
+        else:
+            self.future.set_result((af, addr, stream))
+
+    def set_timout(self, timeout):
+        self.timeout = self.io_loop.add_timeout(self.io_loop.time() + timeout,
+                                                self.on_timeout)
+
+    def on_timeout(self):
+        self.timeout = None
+        self.try_connect(iter(self.secondary_addrs))
+
+    def clear_timeout(self):
+        if self.timeout is not None:
+            self.io_loop.remove_timeout(self.timeout)
+
+
+class TCPClient(object):
+    """A non-blocking TCP connection factory.
+    """
+    def __init__(self, resolver=None, io_loop=None):
+        self.io_loop = io_loop or IOLoop.current()
+        if resolver is not None:
+            self.resolver = resolver
+            self._own_resolver = False
+        else:
+            self.resolver = Resolver(io_loop=io_loop)
+            self._own_resolver = True
+
+    def close(self):
+        if self._own_resolver:
+            self.resolver.close()
+
+    @gen.coroutine
+    def connect(self, host, port, af=socket.AF_UNSPEC, ssl_options=None,
+                max_buffer_size=None):
+        """Connect to the given host and port.
+
+        Asynchronously returns an `.IOStream` (or `.SSLIOStream` if
+        ``ssl_options`` is not None).
+        """
+        addrinfo = yield self.resolver.resolve(host, port, af)
+        connector = _Connector(
+            addrinfo, self.io_loop,
+            functools.partial(self._create_stream, max_buffer_size))
+        af, addr, stream = yield connector.start()
+        # TODO: For better performance we could cache the (af, addr)
+        # information here and re-use it on sbusequent connections to
+        # the same host. (http://tools.ietf.org/html/rfc6555#section-4.2)
+        if ssl_options is not None:
+            stream = yield stream.start_tls(False, ssl_options=ssl_options,
+                                            server_hostname=host)
+        raise gen.Return(stream)
+
+    def _create_stream(self, max_buffer_size, af, addr):
+        # Always connect in plaintext; we'll convert to ssl if necessary
+        # after one connection has completed.
+        stream = IOStream(socket.socket(af),
+                          io_loop=self.io_loop,
+                          max_buffer_size=max_buffer_size)
+        return stream.connect(addr)

libs/tornado/tcpserver.py

@@ -20,13 +20,19 @@ from __future__ import absolute_import, division, print_function, with_statement
 import errno
 import os
 import socket
-import ssl
 
 from tornado.log import app_log
 from tornado.ioloop import IOLoop
 from tornado.iostream import IOStream, SSLIOStream
 from tornado.netutil import bind_sockets, add_accept_handler, ssl_wrap_socket
 from tornado import process
+from tornado.util import errno_from_exception
+
+try:
+    import ssl
+except ImportError:
+    # ssl is not available on Google App Engine.
+    ssl = None
 
 
 class TCPServer(object):
@@ -81,13 +87,15 @@ class TCPServer(object):
     .. versionadded:: 3.1
        The ``max_buffer_size`` argument.
     """
-    def __init__(self, io_loop=None, ssl_options=None, max_buffer_size=None):
+    def __init__(self, io_loop=None, ssl_options=None, max_buffer_size=None,
+                 read_chunk_size=None):
         self.io_loop = io_loop
         self.ssl_options = ssl_options
         self._sockets = {}  # fd -> socket object
         self._pending_sockets = []
         self._started = False
         self.max_buffer_size = max_buffer_size
+        self.read_chunk_size = None
 
         # Verify the SSL options. Otherwise we don't get errors until clients
         # connect. This doesn't verify that the keys are legitimate, but
@@ -230,16 +238,20 @@ class TCPServer(object):
                 # catch another error later on (AttributeError in
                 # SSLIOStream._do_ssl_handshake).
                 # To test this behavior, try nmap with the -sT flag.
-                # https://github.com/facebook/tornado/pull/750
-                if err.args[0] in (errno.ECONNABORTED, errno.EINVAL):
+                # https://github.com/tornadoweb/tornado/pull/750
+                if errno_from_exception(err) in (errno.ECONNABORTED, errno.EINVAL):
                     return connection.close()
                 else:
                     raise
         try:
             if self.ssl_options is not None:
-                stream = SSLIOStream(connection, io_loop=self.io_loop, max_buffer_size=self.max_buffer_size)
+                stream = SSLIOStream(connection, io_loop=self.io_loop,
+                                     max_buffer_size=self.max_buffer_size,
+                                     read_chunk_size=self.read_chunk_size)
             else:
-                stream = IOStream(connection, io_loop=self.io_loop, max_buffer_size=self.max_buffer_size)
+                stream = IOStream(connection, io_loop=self.io_loop,
+                                  max_buffer_size=self.max_buffer_size,
+                                  read_chunk_size=self.read_chunk_size)
             self.handle_stream(stream, address)
         except Exception:
             app_log.error("Error in connection callback", exc_info=True)

libs/tornado/template.py

@@ -180,7 +180,7 @@ with ``{# ... #}``.
 ``{% set *x* = *y* %}``
     Sets a local variable.
 
-``{% try %}...{% except %}...{% finally %}...{% else %}...{% end %}``
+``{% try %}...{% except %}...{% else %}...{% finally %}...{% end %}``
     Same as the python ``try`` statement.
 
 ``{% while *condition* %}... {% end %}``
@@ -367,10 +367,9 @@ class Loader(BaseLoader):
 
     def _create_template(self, name):
         path = os.path.join(self.root, name)
-        f = open(path, "rb")
-        template = Template(f.read(), name=name, loader=self)
-        f.close()
-        return template
+        with open(path, "rb") as f:
+            template = Template(f.read(), name=name, loader=self)
+            return template
 
 
 class DictLoader(BaseLoader):
@@ -785,7 +784,7 @@ def _parse(reader, template, in_block=None, in_loop=None):
         if allowed_parents is not None:
             if not in_block:
                 raise ParseError("%s outside %s block" %
-                                (operator, allowed_parents))
+                                 (operator, allowed_parents))
             if in_block not in allowed_parents:
                 raise ParseError("%s block cannot be attached to %s block" % (operator, in_block))
             body.chunks.append(_IntermediateControlBlock(contents, line))

libs/tornado/testing.py

@@ -17,7 +17,7 @@ try:
     from tornado.httpclient import AsyncHTTPClient
     from tornado.httpserver import HTTPServer
     from tornado.simple_httpclient import SimpleAsyncHTTPClient
-    from tornado.ioloop import IOLoop
+    from tornado.ioloop import IOLoop, TimeoutError
     from tornado import netutil
 except ImportError:
     # These modules are not importable on app engine.  Parts of this module
@@ -38,6 +38,7 @@ import re
 import signal
 import socket
 import sys
+import types
 
 try:
     from cStringIO import StringIO  # py2
@@ -48,10 +49,16 @@ except ImportError:
 # (either py27+ or unittest2) so tornado.test.util enforces
 # this requirement, but for other users of tornado.testing we want
 # to allow the older version if unitest2 is not available.
-try:
-    import unittest2 as unittest
-except ImportError:
+if sys.version_info >= (3,):
+    # On python 3, mixing unittest2 and unittest (including doctest)
+    # doesn't seem to work, so always use unittest.
     import unittest
+else:
+    # On python 2, prefer unittest2 when available.
+    try:
+        import unittest2 as unittest
+    except ImportError:
+        import unittest
 
 _next_port = 10000
 
@@ -95,6 +102,36 @@ def get_async_test_timeout():
         return 5
 
 
+class _TestMethodWrapper(object):
+    """Wraps a test method to raise an error if it returns a value.
+
+    This is mainly used to detect undecorated generators (if a test
+    method yields it must use a decorator to consume the generator),
+    but will also detect other kinds of return values (these are not
+    necessarily errors, but we alert anyway since there is no good
+    reason to return a value from a test.
+    """
+    def __init__(self, orig_method):
+        self.orig_method = orig_method
+
+    def __call__(self):
+        result = self.orig_method()
+        if isinstance(result, types.GeneratorType):
+            raise TypeError("Generator test methods should be decorated with "
+                            "tornado.testing.gen_test")
+        elif result is not None:
+            raise ValueError("Return value from test method ignored: %r" %
+                             result)
+
+    def __getattr__(self, name):
+        """Proxy all unknown attributes to the original method.
+
+        This is important for some of the decorators in the `unittest`
+        module, such as `unittest.skipIf`.
+        """
+        return getattr(self.orig_method, name)
+
+
 class AsyncTestCase(unittest.TestCase):
     """`~unittest.TestCase` subclass for testing `.IOLoop`-based
     asynchronous code.
@@ -157,14 +194,20 @@ class AsyncTestCase(unittest.TestCase):
                 self.assertIn("FriendFeed", response.body)
                 self.stop()
     """
-    def __init__(self, *args, **kwargs):
-        super(AsyncTestCase, self).__init__(*args, **kwargs)
+    def __init__(self, methodName='runTest', **kwargs):
+        super(AsyncTestCase, self).__init__(methodName, **kwargs)
         self.__stopped = False
         self.__running = False
         self.__failure = None
         self.__stop_args = None
         self.__timeout = None
 
+        # It's easy to forget the @gen_test decorator, but if you do
+        # the test will silently be ignored because nothing will consume
+        # the generator.  Replace the test method with a wrapper that will
+        # make sure it's not an undecorated generator.
+        setattr(self, methodName, _TestMethodWrapper(getattr(self, methodName)))
+
     def setUp(self):
         super(AsyncTestCase, self).setUp()
         self.io_loop = self.get_new_ioloop()
@@ -352,6 +395,7 @@ class AsyncHTTPTestCase(AsyncTestCase):
 
     def tearDown(self):
         self.http_server.stop()
+        self.io_loop.run_sync(self.http_server.close_all_connections)
         if (not IOLoop.initialized() or
                 self.http_client.io_loop is not IOLoop.instance()):
             self.http_client.close()
@@ -414,18 +458,50 @@ def gen_test(func=None, timeout=None):
     .. versionadded:: 3.1
        The ``timeout`` argument and ``ASYNC_TEST_TIMEOUT`` environment
        variable.
+
+    .. versionchanged:: 3.3
+       The wrapper now passes along ``*args, **kwargs`` so it can be used
+       on functions with arguments.
     """
     if timeout is None:
         timeout = get_async_test_timeout()
 
     def wrap(f):
-        f = gen.coroutine(f)
-
+        # Stack up several decorators to allow us to access the generator
+        # object itself.  In the innermost wrapper, we capture the generator
+        # and save it in an attribute of self.  Next, we run the wrapped
+        # function through @gen.coroutine.  Finally, the coroutine is
+        # wrapped again to make it synchronous with run_sync.
+        #
+        # This is a good case study arguing for either some sort of
+        # extensibility in the gen decorators or cancellation support.
         @functools.wraps(f)
-        def wrapper(self):
-            return self.io_loop.run_sync(
-                functools.partial(f, self), timeout=timeout)
-        return wrapper
+        def pre_coroutine(self, *args, **kwargs):
+            result = f(self, *args, **kwargs)
+            if isinstance(result, types.GeneratorType):
+                self._test_generator = result
+            else:
+                self._test_generator = None
+            return result
+
+        coro = gen.coroutine(pre_coroutine)
+
+        @functools.wraps(coro)
+        def post_coroutine(self, *args, **kwargs):
+            try:
+                return self.io_loop.run_sync(
+                    functools.partial(coro, self, *args, **kwargs),
+                    timeout=timeout)
+            except TimeoutError as e:
+                # run_sync raises an error with an unhelpful traceback.
+                # If we throw it back into the generator the stack trace
+                # will be replaced by the point where the test is stopped.
+                self._test_generator.throw(e)
+                # In case the test contains an overly broad except clause,
+                # we may get back here.  In this case re-raise the original
+                # exception, which is better than nothing.
+                raise
+        return post_coroutine
 
     if func is not None:
         # Used like:

libs/tornado/util.py

@@ -33,7 +33,7 @@ class ObjectDict(dict):
 class GzipDecompressor(object):
     """Streaming gzip decompressor.
 
-    The interface is like that of `zlib.decompressobj` (without the
+    The interface is like that of `zlib.decompressobj` (without some of the
     optional arguments, but it understands gzip headers and checksums.
     """
     def __init__(self):
@@ -42,14 +42,24 @@ class GzipDecompressor(object):
         # This works on cpython and pypy, but not jython.
         self.decompressobj = zlib.decompressobj(16 + zlib.MAX_WBITS)
 
-    def decompress(self, value):
+    def decompress(self, value, max_length=None):
         """Decompress a chunk, returning newly-available data.
 
         Some data may be buffered for later processing; `flush` must
         be called when there is no more input data to ensure that
         all data was processed.
+
+        If ``max_length`` is given, some input data may be left over
+        in ``unconsumed_tail``; you must retrieve this value and pass
+        it back to a future call to `decompress` if it is not empty.
+        """
+        return self.decompressobj.decompress(value, max_length)
+
+    @property
+    def unconsumed_tail(self):
+        """Returns the unconsumed portion left over
         """
-        return self.decompressobj.decompress(value)
+        return self.decompressobj.unconsumed_tail
 
     def flush(self):
         """Return any remaining buffered data not yet returned by decompress.
@@ -132,6 +142,24 @@ def exec_in(code, glob, loc=None):
 """)
 
 
+def errno_from_exception(e):
+    """Provides the errno from an Exception object.
+
+    There are cases that the errno attribute was not set so we pull
+    the errno out of the args but if someone instatiates an Exception
+    without any args you will get a tuple error. So this function
+    abstracts all that behavior to give you a safe way to get the
+    errno.
+    """
+
+    if hasattr(e, 'errno'):
+        return e.errno
+    elif e.args:
+        return e.args[0]
+    else:
+        return None
+
+
 class Configurable(object):
     """Base class for configurable interfaces.
 
@@ -243,6 +271,16 @@ class ArgReplacer(object):
             # Not a positional parameter
             self.arg_pos = None
 
+    def get_old_value(self, args, kwargs, default=None):
+        """Returns the old value of the named argument without replacing it.
+
+        Returns ``default`` if the argument is not present.
+        """
+        if self.arg_pos is not None and len(args) > self.arg_pos:
+            return args[self.arg_pos]
+        else:
+            return kwargs.get(self.name, default)
+
     def replace(self, new_value, args, kwargs):
         """Replace the named argument in ``args, kwargs`` with ``new_value``.
 

libs/tornado/web.py

@@ -74,9 +74,11 @@ import traceback
 import types
 import uuid
 
-from tornado.concurrent import Future
+from tornado.concurrent import Future, is_future
 from tornado import escape
+from tornado import gen
 from tornado import httputil
+from tornado import iostream
 from tornado import locale
 from tornado.log import access_log, app_log, gen_log
 from tornado import stack_context
@@ -105,6 +107,39 @@ except ImportError:
     from urllib.parse import urlencode  # py3
 
 
+MIN_SUPPORTED_SIGNED_VALUE_VERSION = 1
+"""The oldest signed value version supported by this version of Tornado.
+
+Signed values older than this version cannot be decoded.
+
+.. versionadded:: 3.2.1
+"""
+
+MAX_SUPPORTED_SIGNED_VALUE_VERSION = 2
+"""The newest signed value version supported by this version of Tornado.
+
+Signed values newer than this version cannot be decoded.
+
+.. versionadded:: 3.2.1
+"""
+
+DEFAULT_SIGNED_VALUE_VERSION = 2
+"""The signed value version produced by `.RequestHandler.create_signed_value`.
+
+May be overridden by passing a ``version`` keyword argument.
+
+.. versionadded:: 3.2.1
+"""
+
+DEFAULT_SIGNED_VALUE_MIN_VERSION = 1
+"""The oldest signed value accepted by `.RequestHandler.get_secure_cookie`.
+
+May be overrided by passing a ``min_version`` keyword argument.
+
+.. versionadded:: 3.2.1
+"""
+
+
 class RequestHandler(object):
     """Subclass this class and define `get()` or `post()` to make a handler.
 
@@ -128,6 +163,7 @@ class RequestHandler(object):
         self._finished = False
         self._auto_finish = True
         self._transforms = None  # will be set in _execute
+        self._prepared_future = None
         self.path_args = None
         self.path_kwargs = None
         self.ui = ObjectDict((n, self._ui_method(m)) for n, m in
@@ -141,10 +177,7 @@ class RequestHandler(object):
                                                     application.ui_modules)
         self.ui["modules"] = self.ui["_tt_modules"]
         self.clear()
-        # Check since connection is not available in WSGI
-        if getattr(self.request, "connection", None):
-            self.request.connection.set_close_callback(
-                self.on_connection_close)
+        self.request.connection.set_close_callback(self.on_connection_close)
         self.initialize(**kwargs)
 
     def initialize(self):
@@ -235,7 +268,9 @@ class RequestHandler(object):
         may not be called promptly after the end user closes their
         connection.
         """
-        pass
+        if _has_stream_request_body(self.__class__):
+            if not self.request.body.done():
+                self.request.body.set_exception(iostream.StreamClosedError())
 
     def clear(self):
         """Resets all headers and content for this response."""
@@ -245,12 +280,6 @@ class RequestHandler(object):
             "Date": httputil.format_timestamp(time.time()),
         })
         self.set_default_headers()
-        if (not self.request.supports_http_1_1() and
-            getattr(self.request, 'connection', None) and
-                not self.request.connection.no_keep_alive):
-            conn_header = self.request.headers.get("Connection")
-            if conn_header and (conn_header.lower() == "keep-alive"):
-                self._headers["Connection"] = "Keep-Alive"
         self._write_buffer = []
         self._status_code = 200
         self._reason = httputil.responses[200]
@@ -455,7 +484,7 @@ class RequestHandler(object):
 
     @property
     def cookies(self):
-        """An alias for `self.request.cookies <.httpserver.HTTPRequest.cookies>`."""
+        """An alias for `self.request.cookies <.httputil.HTTPServerRequest.cookies>`."""
         return self.request.cookies
 
     def get_cookie(self, name, default=None):
@@ -524,7 +553,8 @@ class RequestHandler(object):
         for name in self.request.cookies:
             self.clear_cookie(name, path=path, domain=domain)
 
-    def set_secure_cookie(self, name, value, expires_days=30, **kwargs):
+    def set_secure_cookie(self, name, value, expires_days=30, version=None,
+                          **kwargs):
         """Signs and timestamps a cookie so it cannot be forged.
 
         You must specify the ``cookie_secret`` setting in your Application
@@ -539,32 +569,50 @@ class RequestHandler(object):
 
         Secure cookies may contain arbitrary byte values, not just unicode
         strings (unlike regular cookies)
+
+        .. versionchanged:: 3.2.1
+
+           Added the ``version`` argument.  Introduced cookie version 2
+           and made it the default.
         """
-        self.set_cookie(name, self.create_signed_value(name, value),
+        self.set_cookie(name, self.create_signed_value(name, value,
+                                                       version=version),
                         expires_days=expires_days, **kwargs)
 
-    def create_signed_value(self, name, value):
+    def create_signed_value(self, name, value, version=None):
         """Signs and timestamps a string so it cannot be forged.
 
         Normally used via set_secure_cookie, but provided as a separate
         method for non-cookie uses.  To decode a value not stored
         as a cookie use the optional value argument to get_secure_cookie.
+
+        .. versionchanged:: 3.2.1
+
+           Added the ``version`` argument.  Introduced cookie version 2
+           and made it the default.
         """
         self.require_setting("cookie_secret", "secure cookies")
         return create_signed_value(self.application.settings["cookie_secret"],
-                                   name, value)
+                                   name, value, version=version)
 
-    def get_secure_cookie(self, name, value=None, max_age_days=31):
+    def get_secure_cookie(self, name, value=None, max_age_days=31,
+                          min_version=None):
         """Returns the given signed cookie if it validates, or None.
 
         The decoded cookie value is returned as a byte string (unlike
         `get_cookie`).
+
+        .. versionchanged:: 3.2.1
+
+           Added the ``min_version`` argument.  Introduced cookie version 2;
+           both versions 1 and 2 are accepted by default.
         """
         self.require_setting("cookie_secret", "secure cookies")
         if value is None:
             value = self.get_cookie(name)
         return decode_signed_value(self.application.settings["cookie_secret"],
-                                   name, value, max_age_days=max_age_days)
+                                   name, value, max_age_days=max_age_days,
+                                   min_version=min_version)
 
     def redirect(self, url, permanent=False, status=None):
         """Sends a redirect to the given (optionally relative) URL.
@@ -598,12 +646,15 @@ class RequestHandler(object):
         Note that lists are not converted to JSON because of a potential
         cross-site security vulnerability.  All JSON output should be
         wrapped in a dictionary.  More details at
-        http://haacked.com/archive/2008/11/20/anatomy-of-a-subtle-json-vulnerability.aspx
+        http://haacked.com/archive/2009/06/25/json-hijacking.aspx/ and
+        https://github.com/facebook/tornado/issues/1009
         """
         if self._finished:
             raise RuntimeError("Cannot write() after finish().  May be caused "
                                "by using async operations without the "
                                "@asynchronous decorator.")
+        if not isinstance(chunk, (bytes_type, unicode_type, dict)):
+            raise TypeError("write() only accepts bytes, unicode, and dict objects")
         if isinstance(chunk, dict):
             chunk = escape.json_encode(chunk)
             self.set_header("Content-Type", "application/json; charset=UTF-8")
@@ -769,14 +820,10 @@ class RequestHandler(object):
         Note that only one flush callback can be outstanding at a time;
         if another flush occurs before the previous flush's callback
         has been run, the previous callback will be discarded.
-        """
-        if self.application._wsgi:
-            # WSGI applications cannot usefully support flush, so just make
-            # it a no-op (and run the callback immediately).
-            if callback is not None:
-                callback()
-            return
 
+        .. versionchanged:: 3.3
+           Now returns a `.Future` if no callback is given.
+        """
         chunk = b"".join(self._write_buffer)
         self._write_buffer = []
         if not self._headers_written:
@@ -785,19 +832,32 @@ class RequestHandler(object):
                 self._status_code, self._headers, chunk = \
                     transform.transform_first_chunk(
                         self._status_code, self._headers, chunk, include_footers)
-            headers = self._generate_headers()
+            # Ignore the chunk and only write the headers for HEAD requests
+            if self.request.method == "HEAD":
+                chunk = None
+
+            # Finalize the cookie headers (which have been stored in a side
+            # object so an outgoing cookie could be overwritten before it
+            # is sent).
+            if hasattr(self, "_new_cookie"):
+                for cookie in self._new_cookie.values():
+                    self.add_header("Set-Cookie", cookie.OutputString(None))
+
+            start_line = httputil.ResponseStartLine(self.request.version,
+                                                    self._status_code,
+                                                    self._reason)
+            return self.request.connection.write_headers(
+                start_line, self._headers, chunk, callback=callback)
         else:
             for transform in self._transforms:
                 chunk = transform.transform_chunk(chunk, include_footers)
-            headers = b""
-
-        # Ignore the chunk and only write the headers for HEAD requests
-        if self.request.method == "HEAD":
-            if headers:
-                self.request.write(headers, callback=callback)
-            return
-
-        self.request.write(headers + chunk, callback=callback)
+            # Ignore the chunk and only write the headers for HEAD requests
+            if self.request.method != "HEAD":
+                return self.request.connection.write(chunk, callback=callback)
+            else:
+                future = Future()
+                future.set_result(None)
+                return future
 
     def finish(self, chunk=None):
         """Finishes this response, ending the HTTP request."""
@@ -833,10 +893,9 @@ class RequestHandler(object):
             # are keepalive connections)
             self.request.connection.set_close_callback(None)
 
-        if not self.application._wsgi:
-            self.flush(include_footers=True)
-            self.request.finish()
-            self._log()
+        self.flush(include_footers=True)
+        self.request.finish()
+        self._log()
         self._finished = True
         self.on_finish()
         # Break up a reference cycle between this handler and the
@@ -1194,6 +1253,7 @@ class RequestHandler(object):
             self._handle_request_exception(value)
         return True
 
+    @gen.coroutine
     def _execute(self, transforms, *args, **kwargs):
         """Executes this request with the given output transforms."""
         self._transforms = transforms
@@ -1208,52 +1268,52 @@ class RequestHandler(object):
             if self.request.method not in ("GET", "HEAD", "OPTIONS") and \
                     self.application.settings.get("xsrf_cookies"):
                 self.check_xsrf_cookie()
-            self._when_complete(self.prepare(), self._execute_method)
-        except Exception as e:
-            self._handle_request_exception(e)
 
-    def _when_complete(self, result, callback):
-        try:
-            if result is None:
-                callback()
-            elif isinstance(result, Future):
-                if result.done():
-                    if result.result() is not None:
-                        raise ValueError('Expected None, got %r' % result.result())
-                    callback()
-                else:
-                    # Delayed import of IOLoop because it's not available
-                    # on app engine
-                    from tornado.ioloop import IOLoop
-                    IOLoop.current().add_future(
-                        result, functools.partial(self._when_complete,
-                                                  callback=callback))
-            else:
-                raise ValueError("Expected Future or None, got %r" % result)
-        except Exception as e:
-            self._handle_request_exception(e)
+            result = self.prepare()
+            if is_future(result):
+                result = yield result
+            if result is not None:
+                raise TypeError("Expected None, got %r" % result)
+            if self._prepared_future is not None:
+                # Tell the Application we've finished with prepare()
+                # and are ready for the body to arrive.
+                self._prepared_future.set_result(None)
+            if self._finished:
+                return
 
-    def _execute_method(self):
-        if not self._finished:
-            method = getattr(self, self.request.method.lower())
-            self._when_complete(method(*self.path_args, **self.path_kwargs),
-                                self._execute_finish)
+            if _has_stream_request_body(self.__class__):
+                # In streaming mode request.body is a Future that signals
+                # the body has been completely received.  The Future has no
+                # result; the data has been passed to self.data_received
+                # instead.
+                try:
+                    yield self.request.body
+                except iostream.StreamClosedError:
+                    return
 
-    def _execute_finish(self):
-        if self._auto_finish and not self._finished:
-            self.finish()
+            method = getattr(self, self.request.method.lower())
+            result = method(*self.path_args, **self.path_kwargs)
+            if is_future(result):
+                result = yield result
+            if result is not None:
+                raise TypeError("Expected None, got %r" % result)
+            if self._auto_finish and not self._finished:
+                self.finish()
+        except Exception as e:
+            self._handle_request_exception(e)
+            if (self._prepared_future is not None and
+                    not self._prepared_future.done()):
+                # In case we failed before setting _prepared_future, do it
+                # now (to unblock the HTTP server).  Note that this is not
+                # in a finally block to avoid GC issues prior to Python 3.4.
+                self._prepared_future.set_result(None)
 
-    def _generate_headers(self):
-        reason = self._reason
-        lines = [utf8(self.request.version + " " +
-                      str(self._status_code) +
-                      " " + reason)]
-        lines.extend([utf8(n) + b": " + utf8(v) for n, v in self._headers.get_all()])
+    def data_received(self, chunk):
+        """Implement this method to handle streamed request data.
 
-        if hasattr(self, "_new_cookie"):
-            for cookie in self._new_cookie.values():
-                lines.append(utf8("Set-Cookie: " + cookie.OutputString(None)))
-        return b"\r\n".join(lines) + b"\r\n\r\n"
+        Requires the `.stream_request_body` decorator.
+        """
+        raise NotImplementedError()
 
     def _log(self):
         """Logs the current request.
@@ -1367,8 +1427,6 @@ def asynchronous(method):
     from tornado.ioloop import IOLoop
     @functools.wraps(method)
     def wrapper(self, *args, **kwargs):
-        if self.application._wsgi:
-            raise Exception("@asynchronous is not supported for WSGI apps")
         self._auto_finish = False
         with stack_context.ExceptionStackContext(
                 self._stack_context_handle_exception):
@@ -1395,6 +1453,40 @@ def asynchronous(method):
     return wrapper
 
 
+def stream_request_body(cls):
+    """Apply to `RequestHandler` subclasses to enable streaming body support.
+
+    This decorator implies the following changes:
+
+    * `.HTTPServerRequest.body` is undefined, and body arguments will not
+      be included in `RequestHandler.get_argument`.
+    * `RequestHandler.prepare` is called when the request headers have been
+      read instead of after the entire body has been read.
+    * The subclass must define a method ``data_received(self, data):``, which
+      will be called zero or more times as data is available.  Note that
+      if the request has an empty body, ``data_received`` may not be called.
+    * ``prepare`` and ``data_received`` may return Futures (such as via
+      ``@gen.coroutine``, in which case the next method will not be called
+      until those futures have completed.
+    * The regular HTTP method (``post``, ``put``, etc) will be called after
+      the entire body has been read.
+
+    There is a subtle interaction between ``data_received`` and asynchronous
+    ``prepare``: The first call to ``data_recieved`` may occur at any point
+    after the call to ``prepare`` has returned *or yielded*.
+    """
+    if not issubclass(cls, RequestHandler):
+        raise TypeError("expected subclass of RequestHandler, got %r", cls)
+    cls._stream_request_body = True
+    return cls
+
+
+def _has_stream_request_body(cls):
+    if not issubclass(cls, RequestHandler):
+        raise TypeError("expected subclass of RequestHandler, got %r", cls)
+    return getattr(cls, '_stream_request_body', False)
+
+
 def removeslash(method):
     """Use this decorator to remove trailing slashes from the request path.
 
@@ -1439,7 +1531,7 @@ def addslash(method):
     return wrapper
 
 
-class Application(object):
+class Application(httputil.HTTPServerConnectionDelegate):
     """A collection of request handlers that make up a web application.
 
     Instances of this class are callable and can be passed directly to
@@ -1491,12 +1583,11 @@ class Application(object):
 
     """
     def __init__(self, handlers=None, default_host="", transforms=None,
-                 wsgi=False, **settings):
+                 **settings):
         if transforms is None:
             self.transforms = []
             if settings.get("gzip"):
                 self.transforms.append(GZipContentEncoding)
-            self.transforms.append(ChunkedTransferEncoding)
         else:
             self.transforms = transforms
         self.handlers = []
@@ -1508,7 +1599,6 @@ class Application(object):
                            'Template': TemplateModule,
                            }
         self.ui_methods = {}
-        self._wsgi = wsgi
         self._load_ui_modules(settings.get("ui_modules", {}))
         self._load_ui_methods(settings.get("ui_methods", {}))
         if self.settings.get("static_path"):
@@ -1534,7 +1624,7 @@ class Application(object):
             self.settings.setdefault('serve_traceback', True)
 
         # Automatically reload modified modules
-        if self.settings.get('autoreload') and not wsgi:
+        if self.settings.get('autoreload'):
             from tornado import autoreload
             autoreload.start()
 
@@ -1634,64 +1724,15 @@ class Application(object):
                 except TypeError:
                     pass
 
-    def __call__(self, request):
-        """Called by HTTPServer to execute the request."""
-        transforms = [t(request) for t in self.transforms]
-        handler = None
-        args = []
-        kwargs = {}
-        handlers = self._get_host_handlers(request)
-        if not handlers:
-            handler = RedirectHandler(
-                self, request, url="http://" + self.default_host + "/")
-        else:
-            for spec in handlers:
-                match = spec.regex.match(request.path)
-                if match:
-                    handler = spec.handler_class(self, request, **spec.kwargs)
-                    if spec.regex.groups:
-                        # None-safe wrapper around url_unescape to handle
-                        # unmatched optional groups correctly
-                        def unquote(s):
-                            if s is None:
-                                return s
-                            return escape.url_unescape(s, encoding=None,
-                                                       plus=False)
-                        # Pass matched groups to the handler.  Since
-                        # match.groups() includes both named and unnamed groups,
-                        # we want to use either groups or groupdict but not both.
-                        # Note that args are passed as bytes so the handler can
-                        # decide what encoding to use.
-
-                        if spec.regex.groupindex:
-                            kwargs = dict(
-                                (str(k), unquote(v))
-                                for (k, v) in match.groupdict().items())
-                        else:
-                            args = [unquote(s) for s in match.groups()]
-                    break
-            if not handler:
-                if self.settings.get('default_handler_class'):
-                    handler_class = self.settings['default_handler_class']
-                    handler_args = self.settings.get(
-                        'default_handler_args', {})
-                else:
-                    handler_class = ErrorHandler
-                    handler_args = dict(status_code=404)
-                handler = handler_class(self, request, **handler_args)
-
-        # If template cache is disabled (usually in the debug mode),
-        # re-compile templates and reload static files on every
-        # request so you don't need to restart to see changes
-        if not self.settings.get("compiled_template_cache", True):
-            with RequestHandler._template_loader_lock:
-                for loader in RequestHandler._template_loaders.values():
-                    loader.reset()
-        if not self.settings.get('static_hash_cache', True):
-            StaticFileHandler.reset()
+    def start_request(self, connection):
+        # Modern HTTPServer interface
+        return _RequestDispatcher(self, connection)
 
-        handler._execute(transforms, *args, **kwargs)
-        return handler
+    def __call__(self, request):
+        # Legacy HTTPServer interface
+        dispatcher = _RequestDispatcher(self, None)
+        dispatcher.set_request(request)
+        return dispatcher.execute()
 
     def reverse_url(self, name, *args):
         """Returns a URL path for handler named ``name``
@@ -1728,6 +1769,113 @@ class Application(object):
                    handler._request_summary(), request_time)
 
 
+class _RequestDispatcher(httputil.HTTPMessageDelegate):
+    def __init__(self, application, connection):
+        self.application = application
+        self.connection = connection
+        self.request = None
+        self.chunks = []
+        self.handler_class = None
+        self.handler_kwargs = None
+        self.path_args = []
+        self.path_kwargs = {}
+
+    def headers_received(self, start_line, headers):
+        self.set_request(httputil.HTTPServerRequest(
+            connection=self.connection, start_line=start_line, headers=headers))
+        if self.stream_request_body:
+            self.request.body = Future()
+            return self.execute()
+
+    def set_request(self, request):
+        self.request = request
+        self._find_handler()
+        self.stream_request_body = _has_stream_request_body(self.handler_class)
+
+    def _find_handler(self):
+        # Identify the handler to use as soon as we have the request.
+        # Save url path arguments for later.
+        app = self.application
+        handlers = app._get_host_handlers(self.request)
+        if not handlers:
+            self.handler_class = RedirectHandler
+            self.handler_kwargs = dict(url="http://" + app.default_host + "/")
+            return
+        for spec in handlers:
+            match = spec.regex.match(self.request.path)
+            if match:
+                self.handler_class = spec.handler_class
+                self.handler_kwargs = spec.kwargs
+                if spec.regex.groups:
+                    # Pass matched groups to the handler.  Since
+                    # match.groups() includes both named and
+                    # unnamed groups, we want to use either groups
+                    # or groupdict but not both.
+                    if spec.regex.groupindex:
+                        self.path_kwargs = dict(
+                            (str(k), _unquote_or_none(v))
+                            for (k, v) in match.groupdict().items())
+                    else:
+                        self.path_args = [_unquote_or_none(s)
+                                          for s in match.groups()]
+                return
+        if app.settings.get('default_handler_class'):
+            self.handler_class = app.settings['default_handler_class']
+            self.handler_kwargs = app.settings.get(
+                'default_handler_args', {})
+        else:
+            self.handler_class = ErrorHandler
+            self.handler_kwargs = dict(status_code=404)
+
+    def data_received(self, data):
+        if self.stream_request_body:
+            return self.handler.data_received(data)
+        else:
+            self.chunks.append(data)
+
+    def finish(self):
+        if self.stream_request_body:
+            self.request.body.set_result(None)
+        else:
+            self.request.body = b''.join(self.chunks)
+            self.request._parse_body()
+            self.execute()
+
+    def on_connection_close(self):
+        if self.stream_request_body:
+            self.handler.on_connection_close()
+        else:
+            self.chunks = None
+
+    def execute(self):
+        # If template cache is disabled (usually in the debug mode),
+        # re-compile templates and reload static files on every
+        # request so you don't need to restart to see changes
+        if not self.application.settings.get("compiled_template_cache", True):
+            with RequestHandler._template_loader_lock:
+                for loader in RequestHandler._template_loaders.values():
+                    loader.reset()
+        if not self.application.settings.get('static_hash_cache', True):
+            StaticFileHandler.reset()
+
+        self.handler = self.handler_class(self.application, self.request,
+                                          **self.handler_kwargs)
+        transforms = [t(self.request) for t in self.application.transforms]
+
+        if self.stream_request_body:
+            self.handler._prepared_future = Future()
+        # Note that if an exception escapes handler._execute it will be
+        # trapped in the Future it returns (which we are ignoring here).
+        # However, that shouldn't happen because _execute has a blanket
+        # except handler, and we cannot easily access the IOLoop here to
+        # call add_future.
+        self.handler._execute(transforms, *self.path_args, **self.path_kwargs)
+        # If we are streaming the request body, then execute() is finished
+        # when the handler has prepared to receive the body.  If not,
+        # it doesn't matter when execute() finishes (so we return None)
+        return self.handler._prepared_future
+
+
 class HTTPError(Exception):
     """An exception that will turn into an HTTP error response.
 
@@ -1886,8 +2034,9 @@ class StaticFileHandler(RequestHandler):
             cls._static_hashes = {}
 
     def head(self, path):
-        self.get(path, include_body=False)
+        return self.get(path, include_body=False)
 
+    @gen.coroutine
     def get(self, path, include_body=True):
         # Set up our path instance variables.
         self.path = self.parse_url_path(path)
@@ -1912,9 +2061,9 @@ class StaticFileHandler(RequestHandler):
             # the request will be treated as if the header didn't exist.
             request_range = httputil._parse_request_range(range_header)
 
+        size = self.get_content_size()
         if request_range:
             start, end = request_range
-            size = self.get_content_size()
             if (start is not None and start >= size) or end == 0:
                 # As per RFC 2616 14.35.1, a range is not satisfiable only: if
                 # the first requested byte is equal to or greater than the
@@ -1939,18 +2088,26 @@ class StaticFileHandler(RequestHandler):
                                 httputil._get_content_range(start, end, size))
         else:
             start = end = None
-        content = self.get_content(self.absolute_path, start, end)
-        if isinstance(content, bytes_type):
-            content = [content]
-        content_length = 0
-        for chunk in content:
-            if include_body:
+
+        if start is not None and end is not None:
+            content_length = end - start
+        elif end is not None:
+            content_length = end
+        elif start is not None:
+            content_length = size - start
+        else:
+            content_length = size
+        self.set_header("Content-Length", content_length)
+
+        if include_body:
+            content = self.get_content(self.absolute_path, start, end)
+            if isinstance(content, bytes_type):
+                content = [content]
+            for chunk in content:
                 self.write(chunk)
-            else:
-                content_length += len(chunk)
-        if not include_body:
+                yield self.flush()
+        else:
             assert self.request.method == "HEAD"
-            self.set_header("Content-Length", content_length)
 
     def compute_etag(self):
         """Sets the ``Etag`` header based on static url version.
@@ -2130,10 +2287,13 @@ class StaticFileHandler(RequestHandler):
     def get_content_size(self):
         """Retrieve the total size of the resource at the given path.
 
-        This method may be overridden by subclasses. It will only
-        be called if a partial result is requested from `get_content`
+        This method may be overridden by subclasses.
 
         .. versionadded:: 3.1
+
+        .. versionchanged:: 3.3
+           This method is now always called, instead of only when
+           partial results are requested.
         """
         stat_result = self._stat()
         return stat_result[stat.ST_SIZE]
@@ -2255,7 +2415,7 @@ class FallbackHandler(RequestHandler):
     """A `RequestHandler` that wraps another HTTP server callback.
 
     The fallback is a callable object that accepts an
-    `~.httpserver.HTTPRequest`, such as an `Application` or
+    `~.httputil.HTTPServerRequest`, such as an `Application` or
     `tornado.wsgi.WSGIContainer`.  This is most useful to use both
     Tornado ``RequestHandlers`` and WSGI in the same server.  Typical
     usage::
@@ -2279,7 +2439,7 @@ class OutputTransform(object):
     """A transform modifies the result of an HTTP request (e.g., GZip encoding)
 
     A new transform instance is created for every request. See the
-    ChunkedTransferEncoding example below if you want to implement a
+    GZipContentEncoding example below if you want to implement a
     new Transform.
     """
     def __init__(self, request):
@@ -2296,16 +2456,24 @@ class GZipContentEncoding(OutputTransform):
     """Applies the gzip content encoding to the response.
 
     See http://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.11
+
+    .. versionchanged:: 3.3
+        Now compresses all mime types beginning with ``text/``, instead
+        of just a whitelist. (the whitelist is still used for certain
+        non-text mime types).
     """
-    CONTENT_TYPES = set([
-        "text/plain", "text/html", "text/css", "text/xml", "application/javascript",
-        "application/x-javascript", "application/xml", "application/atom+xml",
-        "text/javascript", "application/json", "application/xhtml+xml"])
+    # Whitelist of compressible mime types (in addition to any types
+    # beginning with "text/").
+    CONTENT_TYPES = set(["application/javascript", "application/x-javascript",
+                         "application/xml", "application/atom+xml",
+                         "application/json", "application/xhtml+xml"])
     MIN_LENGTH = 5
 
     def __init__(self, request):
-        self._gzipping = request.supports_http_1_1() and \
-            "gzip" in request.headers.get("Accept-Encoding", "")
+        self._gzipping = "gzip" in request.headers.get("Accept-Encoding", "")
+
+    def _compressible_type(self, ctype):
+        return ctype.startswith('text/') or ctype in self.CONTENT_TYPES
 
     def transform_first_chunk(self, status_code, headers, chunk, finishing):
         if 'Vary' in headers:
@@ -2314,7 +2482,7 @@ class GZipContentEncoding(OutputTransform):
             headers['Vary'] = b'Accept-Encoding'
         if self._gzipping:
             ctype = _unicode(headers.get("Content-Type", "")).split(";")[0]
-            self._gzipping = (ctype in self.CONTENT_TYPES) and \
+            self._gzipping = self._compressible_type(ctype) and \
                 (not finishing or len(chunk) >= self.MIN_LENGTH) and \
                 (finishing or "Content-Length" not in headers) and \
                 ("Content-Encoding" not in headers)
@@ -2340,42 +2508,16 @@ class GZipContentEncoding(OutputTransform):
         return chunk
 
 
-class ChunkedTransferEncoding(OutputTransform):
-    """Applies the chunked transfer encoding to the response.
-
-    See http://www.w3.org/Protocols/rfc2616/rfc2616-sec3.html#sec3.6.1
-    """
-    def __init__(self, request):
-        self._chunking = request.supports_http_1_1()
-
-    def transform_first_chunk(self, status_code, headers, chunk, finishing):
-        # 304 responses have no body (not even a zero-length body), and so
-        # should not have either Content-Length or Transfer-Encoding headers.
-        if self._chunking and status_code != 304:
-            # No need to chunk the output if a Content-Length is specified
-            if "Content-Length" in headers or "Transfer-Encoding" in headers:
-                self._chunking = False
-            else:
-                headers["Transfer-Encoding"] = "chunked"
-                chunk = self.transform_chunk(chunk, finishing)
-        return status_code, headers, chunk
-
-    def transform_chunk(self, block, finishing):
-        if self._chunking:
-            # Don't write out empty chunks because that means END-OF-STREAM
-            # with chunked encoding
-            if block:
-                block = utf8("%x" % len(block)) + b"\r\n" + block + b"\r\n"
-            if finishing:
-                block += b"0\r\n\r\n"
-        return block
-
-
 def authenticated(method):
     """Decorate methods with this to require that the user be logged in.
 
     If the user is not logged in, they will be redirected to the configured
     `login url <RequestHandler.get_login_url>`.
+
+    If you configure a login url with a query parameter, Tornado will
+    assume you know what you're doing and use it as-is.  If not, it
+    will add a `next` parameter so the login page knows where to send
+    you once you're logged in.
     """
     @functools.wraps(method)
     def wrapper(self, *args, **kwargs):
@@ -2640,29 +2782,103 @@ else:
         return result == 0
 
 
-def create_signed_value(secret, name, value):
-    timestamp = utf8(str(int(time.time())))
+def create_signed_value(secret, name, value, version=None, clock=None):
+    if version is None:
+        version = DEFAULT_SIGNED_VALUE_VERSION
+    if clock is None:
+        clock = time.time
+    timestamp = utf8(str(int(clock())))
     value = base64.b64encode(utf8(value))
-    signature = _create_signature(secret, name, value, timestamp)
-    value = b"|".join([value, timestamp, signature])
-    return value
-
-
-def decode_signed_value(secret, name, value, max_age_days=31):
+    if version == 1:
+        signature = _create_signature_v1(secret, name, value, timestamp)
+        value = b"|".join([value, timestamp, signature])
+        return value
+    elif version == 2:
+        # The v2 format consists of a version number and a series of
+        # length-prefixed fields "%d:%s", the last of which is a
+        # signature, all separated by pipes.  All numbers are in
+        # decimal format with no leading zeros.  The signature is an
+        # HMAC-SHA256 of the whole string up to that point, including
+        # the final pipe.
+        #
+        # The fields are:
+        # - format version (i.e. 2; no length prefix)
+        # - key version (currently 0; reserved for future key rotation features)
+        # - timestamp (integer seconds since epoch)
+        # - name (not encoded; assumed to be ~alphanumeric)
+        # - value (base64-encoded)
+        # - signature (hex-encoded; no length prefix)
+        def format_field(s):
+            return utf8("%d:" % len(s)) + utf8(s)
+        to_sign = b"|".join([
+            b"2|1:0",
+            format_field(timestamp),
+            format_field(name),
+            format_field(value),
+            b''])
+        signature = _create_signature_v2(secret, to_sign)
+        return to_sign + signature
+    else:
+        raise ValueError("Unsupported version %d" % version)
+
+# A leading version number in decimal with no leading zeros, followed by a pipe.
+_signed_value_version_re = re.compile(br"^([1-9][0-9]*)\|(.*)$")
+
+
+def decode_signed_value(secret, name, value, max_age_days=31, clock=None, min_version=None):
+    if clock is None:
+        clock = time.time
+    if min_version is None:
+        min_version = DEFAULT_SIGNED_VALUE_MIN_VERSION
+    if min_version > 2:
+        raise ValueError("Unsupported min_version %d" % min_version)
     if not value:
         return None
+
+    # Figure out what version this is.  Version 1 did not include an
+    # explicit version field and started with arbitrary base64 data,
+    # which makes this tricky.
+    value = utf8(value)
+    m = _signed_value_version_re.match(value)
+    if m is None:
+        version = 1
+    else:
+        try:
+            version = int(m.group(1))
+            if version > 999:
+                # Certain payloads from the version-less v1 format may
+                # be parsed as valid integers.  Due to base64 padding
+                # restrictions, this can only happen for numbers whose
+                # length is a multiple of 4, so we can treat all
+                # numbers up to 999 as versions, and for the rest we
+                # fall back to v1 format.
+                version = 1
+        except ValueError:
+            version = 1
+
+    if version < min_version:
+        return None
+    if version == 1:
+        return _decode_signed_value_v1(secret, name, value, max_age_days, clock)
+    elif version == 2:
+        return _decode_signed_value_v2(secret, name, value, max_age_days, clock)
+    else:
+        return None
+
+
+def _decode_signed_value_v1(secret, name, value, max_age_days, clock):
     parts = utf8(value).split(b"|")
     if len(parts) != 3:
         return None
-    signature = _create_signature(secret, name, parts[0], parts[1])
+    signature = _create_signature_v1(secret, name, parts[0], parts[1])
     if not _time_independent_equals(parts[2], signature):
         gen_log.warning("Invalid cookie signature %r", value)
         return None
     timestamp = int(parts[1])
-    if timestamp < time.time() - max_age_days * 86400:
+    if timestamp < clock() - max_age_days * 86400:
         gen_log.warning("Expired cookie %r", value)
         return None
-    if timestamp > time.time() + 31 * 86400:
+    if timestamp > clock() + 31 * 86400:
         # _cookie_signature does not hash a delimiter between the
         # parts of the cookie, so an attacker could transfer trailing
         # digits from the payload to the timestamp without altering the
@@ -2679,8 +2895,62 @@ def decode_signed_value(secret, name, value, max_age_days=31):
         return None
 
 
-def _create_signature(secret, *parts):
+def _decode_signed_value_v2(secret, name, value, max_age_days, clock):
+    def _consume_field(s):
+        length, _, rest = s.partition(b':')
+        n = int(length)
+        field_value = rest[:n]
+        # In python 3, indexing bytes returns small integers; we must
+        # use a slice to get a byte string as in python 2.
+        if rest[n:n + 1] != b'|':
+            raise ValueError("malformed v2 signed value field")
+        rest = rest[n + 1:]
+        return field_value, rest
+    rest = value[2:]  # remove version number
+    try:
+        key_version, rest = _consume_field(rest)
+        timestamp, rest = _consume_field(rest)
+        name_field, rest = _consume_field(rest)
+        value_field, rest = _consume_field(rest)
+    except ValueError:
+        return None
+    passed_sig = rest
+    signed_string = value[:-len(passed_sig)]
+    expected_sig = _create_signature_v2(secret, signed_string)
+    if not _time_independent_equals(passed_sig, expected_sig):
+        return None
+    if name_field != utf8(name):
+        return None
+    timestamp = int(timestamp)
+    if timestamp < clock() - max_age_days * 86400:
+        # The signature has expired.
+        return None
+    try:
+        return base64.b64decode(value_field)
+    except Exception:
+        return None
+
+
+def _create_signature_v1(secret, *parts):
     hash = hmac.new(utf8(secret), digestmod=hashlib.sha1)
     for part in parts:
         hash.update(utf8(part))
     return utf8(hash.hexdigest())
+
+
+def _create_signature_v2(secret, s):
+    hash = hmac.new(utf8(secret), digestmod=hashlib.sha256)
+    hash.update(utf8(s))
+    return utf8(hash.hexdigest())
+
+
+def _unquote_or_none(s):
+    """None-safe wrapper around url_unescape to handle unamteched optional
+    groups correctly.
+
+    Note that args are passed as bytes so the handler can decide what
+    encoding to use.
+    """
+    if s is None:
+        return s
+    return escape.url_unescape(s, encoding=None, plus=False)

libs/tornado/websocket.py

@@ -32,16 +32,21 @@ import tornado.escape
 import tornado.web
 
 from tornado.concurrent import TracebackFuture
-from tornado.escape import utf8, native_str
+from tornado.escape import utf8, native_str, to_unicode
 from tornado import httpclient, httputil
 from tornado.ioloop import IOLoop
 from tornado.iostream import StreamClosedError
 from tornado.log import gen_log, app_log
-from tornado.netutil import Resolver
 from tornado import simple_httpclient
+from tornado.tcpclient import TCPClient
 from tornado.util import bytes_type, unicode_type
 
 try:
+    from urllib.parse import urlparse # py2
+except ImportError:
+    from urlparse import urlparse # py3
+
+try:
     xrange  # py2
 except NameError:
     xrange = range  # py3
@@ -108,20 +113,17 @@ class WebSocketHandler(tornado.web.RequestHandler):
     def __init__(self, application, request, **kwargs):
         tornado.web.RequestHandler.__init__(self, application, request,
                                             **kwargs)
-        self.stream = request.connection.stream
         self.ws_connection = None
+        self.close_code = None
+        self.close_reason = None
 
-    def _execute(self, transforms, *args, **kwargs):
+    @tornado.web.asynchronous
+    def get(self, *args, **kwargs):
         self.open_args = args
         self.open_kwargs = kwargs
 
-        # Websocket only supports GET method
-        if self.request.method != 'GET':
-            self.stream.write(tornado.escape.utf8(
-                "HTTP/1.1 405 Method Not Allowed\r\n\r\n"
-            ))
-            self.stream.close()
-            return
+        self.stream = self.request.connection.detach()
+        self.stream.set_close_callback(self.on_connection_close)
 
         # Upgrade header should be present and should be equal to WebSocket
         if self.request.headers.get("Upgrade", "").lower() != 'websocket':
@@ -144,9 +146,26 @@ class WebSocketHandler(tornado.web.RequestHandler):
             self.stream.close()
             return
 
+        # Handle WebSocket Origin naming convention differences
         # The difference between version 8 and 13 is that in 8 the
         # client sends a "Sec-Websocket-Origin" header and in 13 it's
         # simply "Origin".
+        if "Origin" in self.request.headers:
+            origin = self.request.headers.get("Origin")
+        else:
+            origin = self.request.headers.get("Sec-Websocket-Origin", None)
+
+
+        # If there was an origin header, check to make sure it matches
+        # according to check_origin. When the origin is None, we assume it
+        # did not come from a browser and that it can be passed on.
+        if origin is not None and not self.check_origin(origin):
+            self.stream.write(tornado.escape.utf8(
+                "HTTP/1.1 403 Cross Origin Websockets Disabled\r\n\r\n"
+            ))
+            self.stream.close()
+            return
+
         if self.request.headers.get("Sec-WebSocket-Version") in ("7", "8", "13"):
             self.ws_connection = WebSocketProtocol13(self)
             self.ws_connection.accept_connection()
@@ -160,6 +179,7 @@ class WebSocketHandler(tornado.web.RequestHandler):
                 "Sec-WebSocket-Version: 8\r\n\r\n"))
             self.stream.close()
 
+
     def write_message(self, message, binary=False):
         """Sends the given message to the client of this Web Socket.
 
@@ -220,18 +240,70 @@ class WebSocketHandler(tornado.web.RequestHandler):
         pass
 
     def on_close(self):
-        """Invoked when the WebSocket is closed."""
+        """Invoked when the WebSocket is closed.
+
+        If the connection was closed cleanly and a status code or reason
+        phrase was supplied, these values will be available as the attributes
+        ``self.close_code`` and ``self.close_reason``.
+
+        .. versionchanged:: 3.3
+
+           Added ``close_code`` and ``close_reason`` attributes.
+        """
         pass
 
-    def close(self):
+    def close(self, code=None, reason=None):
         """Closes this Web Socket.
 
         Once the close handshake is successful the socket will be closed.
+
+        ``code`` may be a numeric status code, taken from the values
+        defined in `RFC 6455 section 7.4.1
+        <https://tools.ietf.org/html/rfc6455#section-7.4.1>`_.
+        ``reason`` may be a textual message about why the connection is
+        closing.  These values are made available to the client, but are
+        not otherwise interpreted by the websocket protocol.
+
+        The ``code`` and ``reason`` arguments are ignored in the "draft76"
+        protocol version.
+
+        .. versionchanged:: 3.3
+
+           Added the ``code`` and ``reason`` arguments.
         """
         if self.ws_connection:
-            self.ws_connection.close()
+            self.ws_connection.close(code, reason)
             self.ws_connection = None
 
+    def check_origin(self, origin):
+        """Override to enable support for allowing alternate origins.
+
+        The ``origin`` argument is the value of the ``Origin`` HTTP
+        header, the url responsible for initiating this request.  This
+        method is not called for clients that do not send this header;
+        such requests are always allowed (because all browsers that
+        implement WebSockets support this header, and non-browser
+        clients do not have the same cross-site security concerns).
+
+        Should return True to accept the request or False to reject it.
+        By default, rejects all requests with an origin on a host other
+        than this one.
+
+        This is a security protection against cross site scripting attacks on
+        browsers, since WebSockets are allowed to bypass the usual same-origin
+        policies and don't use CORS headers.
+
+        .. versionadded:: 3.3
+        """
+        parsed_origin = urlparse(origin)
+        origin = parsed_origin.netloc
+        origin = origin.lower()
+
+        host = self.request.headers.get("Host")
+
+        # Check to see that origin matches host directly, including ports
+        return origin == host
+
     def allow_draft76(self):
         """Override to enable support for the older "draft76" protocol.
 
@@ -489,7 +561,7 @@ class WebSocketProtocol76(WebSocketProtocol):
         """Send ping frame."""
         raise ValueError("Ping messages not supported by this version of websockets")
 
-    def close(self):
+    def close(self, code=None, reason=None):
         """Closes the WebSocket connection."""
         if not self.server_terminated:
             if not self.stream.closed():
@@ -739,6 +811,10 @@ class WebSocketProtocol13(WebSocketProtocol):
         elif opcode == 0x8:
             # Close
             self.client_terminated = True
+            if len(data) >= 2:
+                self.handler.close_code = struct.unpack('>H', data[:2])[0]
+            if len(data) > 2:
+                self.handler.close_reason = to_unicode(data[2:])
             self.close()
         elif opcode == 0x9:
             # Ping
@@ -749,11 +825,19 @@ class WebSocketProtocol13(WebSocketProtocol):
         else:
             self._abort()
 
-    def close(self):
+    def close(self, code=None, reason=None):
         """Closes the WebSocket connection."""
         if not self.server_terminated:
             if not self.stream.closed():
-                self._write_frame(True, 0x8, b"")
+                if code is None and reason is not None:
+                    code = 1000  # "normal closure" status code
+                if code is None:
+                    close_data = b''
+                else:
+                    close_data = struct.pack('>H', code)
+                if reason is not None:
+                    close_data += utf8(reason)
+                self._write_frame(True, 0x8, close_data)
             self.server_terminated = True
         if self.client_terminated:
             if self._waiting is not None:
@@ -789,18 +873,25 @@ class WebSocketClientConnection(simple_httpclient._HTTPConnection):
             'Sec-WebSocket-Version': '13',
         })
 
-        self.resolver = Resolver(io_loop=io_loop)
+        self.tcp_client = TCPClient(io_loop=io_loop)
         super(WebSocketClientConnection, self).__init__(
             io_loop, None, request, lambda: None, self._on_http_response,
-            104857600, self.resolver)
+            104857600, self.tcp_client, 65536)
 
-    def close(self):
+    def close(self, code=None, reason=None):
         """Closes the websocket connection.
 
+        ``code`` and ``reason`` are documented under
+        `WebSocketHandler.close`.
+
         .. versionadded:: 3.2
+
+        .. versionchanged:: 3.3
+
+           Added the ``code`` and ``reason`` arguments.
         """
         if self.protocol is not None:
-            self.protocol.close()
+            self.protocol.close(code, reason)
             self.protocol = None
 
     def _on_close(self):
@@ -816,8 +907,12 @@ class WebSocketClientConnection(simple_httpclient._HTTPConnection):
                 self.connect_future.set_exception(WebSocketError(
                     "Non-websocket response"))
 
-    def _handle_1xx(self, code):
-        assert code == 101
+    def headers_received(self, start_line, headers):
+        if start_line.code != 101:
+            return super(WebSocketClientConnection, self).headers_received(
+                start_line, headers)
+
+        self.headers = headers
         assert self.headers['Upgrade'].lower() == 'websocket'
         assert self.headers['Connection'].lower() == 'upgrade'
         accept = WebSocketProtocol13.compute_accept_value(self.key)
@@ -830,6 +925,9 @@ class WebSocketClientConnection(simple_httpclient._HTTPConnection):
             self.io_loop.remove_timeout(self._timeout)
             self._timeout = None
 
+        self.stream = self.connection.detach()
+        self.stream.set_close_callback(self._on_close)
+
         self.connect_future.set_result(self)
 
     def write_message(self, message, binary=False):
@@ -913,12 +1011,15 @@ def _websocket_mask_python(mask, data):
     else:
         return unmasked.tostring()
 
-if os.environ.get('TORNADO_NO_EXTENSION'):
-    # This environment variable exists to make it easier to do performance comparisons;
-    # it's not guaranteed to remain supported in the future.
+if (os.environ.get('TORNADO_NO_EXTENSION') or
+        os.environ.get('TORNADO_EXTENSION') == '0'):
+    # These environment variables exist to make it easier to do performance
+    # comparisons; they are not guaranteed to remain supported in the future.
     _websocket_mask = _websocket_mask_python
 else:
     try:
         from tornado.speedups import websocket_mask as _websocket_mask
     except ImportError:
+        if os.environ.get('TORNADO_EXTENSION') == '1':
+            raise
         _websocket_mask = _websocket_mask_python

libs/tornado/wsgi.py

@@ -20,9 +20,9 @@ WSGI is the Python standard for web servers, and allows for interoperability
 between Tornado and other Python web frameworks and servers.  This module
 provides WSGI support in two ways:
 
-* `WSGIApplication` is a version of `tornado.web.Application` that can run
-  inside a WSGI server.  This is useful for running a Tornado app on another
-  HTTP server, such as Google App Engine.  See the `WSGIApplication` class
+* `WSGIAdapter` converts a `tornado.web.Application` to the WSGI application
+  interface.  This is useful for running a Tornado app on another
+  HTTP server, such as Google App Engine.  See the `WSGIAdapter` class
   documentation for limitations that apply.
 * `WSGIContainer` lets you run other WSGI applications and frameworks on the
   Tornado HTTP server.  For example, with this class you can mix Django
@@ -32,15 +32,14 @@ provides WSGI support in two ways:
 from __future__ import absolute_import, division, print_function, with_statement
 
 import sys
-import time
-import copy
 import tornado
 
+from tornado.concurrent import Future
 from tornado import escape
 from tornado import httputil
 from tornado.log import access_log
 from tornado import web
-from tornado.escape import native_str, parse_qs_bytes
+from tornado.escape import native_str
 from tornado.util import bytes_type, unicode_type
 
 try:
@@ -49,11 +48,6 @@ except ImportError:
     from cStringIO import StringIO as BytesIO  # python 2
 
 try:
-    import Cookie  # py2
-except ImportError:
-    import http.cookies as Cookie  # py3
-
-try:
     import urllib.parse as urllib_parse  # py3
 except ImportError:
     import urllib as urllib_parse
@@ -83,11 +77,84 @@ else:
 class WSGIApplication(web.Application):
     """A WSGI equivalent of `tornado.web.Application`.
 
-    `WSGIApplication` is very similar to `tornado.web.Application`,
-    except no asynchronous methods are supported (since WSGI does not
-    support non-blocking requests properly). If you call
-    ``self.flush()`` or other asynchronous methods in your request
-    handlers running in a `WSGIApplication`, we throw an exception.
+    .. deprecated: 3.3::
+
+       Use a regular `.Application` and wrap it in `WSGIAdapter` instead.
+    """
+    def __call__(self, environ, start_response):
+        return WSGIAdapter(self)(environ, start_response)
+
+
+# WSGI has no facilities for flow control, so just return an already-done
+# Future when the interface requires it.
+_dummy_future = Future()
+_dummy_future.set_result(None)
+
+
+class _WSGIConnection(httputil.HTTPConnection):
+    def __init__(self, method, start_response, context):
+        self.method = method
+        self.start_response = start_response
+        self.context = context
+        self._write_buffer = []
+        self._finished = False
+        self._expected_content_remaining = None
+        self._error = None
+
+    def set_close_callback(self, callback):
+        # WSGI has no facility for detecting a closed connection mid-request,
+        # so we can simply ignore the callback.
+        pass
+
+    def write_headers(self, start_line, headers, chunk=None, callback=None):
+        if self.method == 'HEAD':
+            self._expected_content_remaining = 0
+        elif 'Content-Length' in headers:
+            self._expected_content_remaining = int(headers['Content-Length'])
+        else:
+            self._expected_content_remaining = None
+        self.start_response(
+            '%s %s' % (start_line.code, start_line.reason),
+            [(native_str(k), native_str(v)) for (k, v) in headers.get_all()])
+        if chunk is not None:
+            self.write(chunk, callback)
+        elif callback is not None:
+            callback()
+        return _dummy_future
+
+    def write(self, chunk, callback=None):
+        if self._expected_content_remaining is not None:
+            self._expected_content_remaining -= len(chunk)
+            if self._expected_content_remaining < 0:
+                self._error = httputil.HTTPOutputException(
+                    "Tried to write more data than Content-Length")
+                raise self._error
+        self._write_buffer.append(chunk)
+        if callback is not None:
+            callback()
+        return _dummy_future
+
+    def finish(self):
+        if (self._expected_content_remaining is not None and
+                self._expected_content_remaining != 0):
+            self._error = httputil.HTTPOutputException(
+                "Tried to write %d bytes less than Content-Length" %
+                self._expected_content_remaining)
+            raise self._error
+        self._finished = True
+
+
+class _WSGIRequestContext(object):
+    def __init__(self, remote_ip, protocol):
+        self.remote_ip = remote_ip
+        self.protocol = protocol
+
+    def __str__(self):
+        return self.remote_ip
+
+
+class WSGIAdapter(object):
+    """Converts a `tornado.web.Application` instance into a WSGI application.
 
     Example usage::
 
@@ -100,121 +167,83 @@ class WSGIApplication(web.Application):
                 self.write("Hello, world")
 
         if __name__ == "__main__":
-            application = tornado.wsgi.WSGIApplication([
+            application = tornado.web.Application([
                 (r"/", MainHandler),
             ])
-            server = wsgiref.simple_server.make_server('', 8888, application)
+            wsgi_app = tornado.wsgi.WSGIAdapter(application)
+            server = wsgiref.simple_server.make_server('', 8888, wsgi_app)
             server.serve_forever()
 
     See the `appengine demo
-    <https://github.com/facebook/tornado/tree/master/demos/appengine>`_
+    <https://github.com/tornadoweb/tornado/tree/stable/demos/appengine>`_
     for an example of using this module to run a Tornado app on Google
     App Engine.
 
-    WSGI applications use the same `.RequestHandler` class, but not
-    ``@asynchronous`` methods or ``flush()``.  This means that it is
-    not possible to use `.AsyncHTTPClient`, or the `tornado.auth` or
-    `tornado.websocket` modules.
+    In WSGI mode asynchronous methods are not supported.  This means
+    that it is not possible to use `.AsyncHTTPClient`, or the
+    `tornado.auth` or `tornado.websocket` modules.
+
+    .. versionadded:: 3.3
     """
-    def __init__(self, handlers=None, default_host="", **settings):
-        web.Application.__init__(self, handlers, default_host, transforms=[],
-                                 wsgi=True, **settings)
+    def __init__(self, application):
+        if isinstance(application, WSGIApplication):
+            self.application = lambda request: web.Application.__call__(
+                application, request)
+        else:
+            self.application = application
 
     def __call__(self, environ, start_response):
-        handler = web.Application.__call__(self, HTTPRequest(environ))
-        assert handler._finished
-        reason = handler._reason
-        status = str(handler._status_code) + " " + reason
-        headers = list(handler._headers.get_all())
-        if hasattr(handler, "_new_cookie"):
-            for cookie in handler._new_cookie.values():
-                headers.append(("Set-Cookie", cookie.OutputString(None)))
-        start_response(status,
-                       [(native_str(k), native_str(v)) for (k, v) in headers])
-        return handler._write_buffer
-
-
-class HTTPRequest(object):
-    """Mimics `tornado.httpserver.HTTPRequest` for WSGI applications."""
-    def __init__(self, environ):
-        """Parses the given WSGI environment to construct the request."""
-        self.method = environ["REQUEST_METHOD"]
-        self.path = urllib_parse.quote(from_wsgi_str(environ.get("SCRIPT_NAME", "")))
-        self.path += urllib_parse.quote(from_wsgi_str(environ.get("PATH_INFO", "")))
-        self.uri = self.path
-        self.arguments = {}
-        self.query_arguments = {}
-        self.body_arguments = {}
-        self.query = environ.get("QUERY_STRING", "")
-        if self.query:
-            self.uri += "?" + self.query
-            self.arguments = parse_qs_bytes(native_str(self.query),
-                                            keep_blank_values=True)
-            self.query_arguments = copy.deepcopy(self.arguments)
-        self.version = "HTTP/1.1"
-        self.headers = httputil.HTTPHeaders()
+        method = environ["REQUEST_METHOD"]
+        uri = urllib_parse.quote(from_wsgi_str(environ.get("SCRIPT_NAME", "")))
+        uri += urllib_parse.quote(from_wsgi_str(environ.get("PATH_INFO", "")))
+        if environ.get("QUERY_STRING"):
+            uri += "?" + environ["QUERY_STRING"]
+        headers = httputil.HTTPHeaders()
         if environ.get("CONTENT_TYPE"):
-            self.headers["Content-Type"] = environ["CONTENT_TYPE"]
+            headers["Content-Type"] = environ["CONTENT_TYPE"]
         if environ.get("CONTENT_LENGTH"):
-            self.headers["Content-Length"] = environ["CONTENT_LENGTH"]
+            headers["Content-Length"] = environ["CONTENT_LENGTH"]
         for key in environ:
             if key.startswith("HTTP_"):
-                self.headers[key[5:].replace("_", "-")] = environ[key]
-        if self.headers.get("Content-Length"):
-            self.body = environ["wsgi.input"].read(
-                int(self.headers["Content-Length"]))
+                headers[key[5:].replace("_", "-")] = environ[key]
+        if headers.get("Content-Length"):
+            body = environ["wsgi.input"].read(
+                int(headers["Content-Length"]))
         else:
-            self.body = ""
-        self.protocol = environ["wsgi.url_scheme"]
-        self.remote_ip = environ.get("REMOTE_ADDR", "")
+            body = ""
+        protocol = environ["wsgi.url_scheme"]
+        remote_ip = environ.get("REMOTE_ADDR", "")
         if environ.get("HTTP_HOST"):
-            self.host = environ["HTTP_HOST"]
+            host = environ["HTTP_HOST"]
         else:
-            self.host = environ["SERVER_NAME"]
-
-        # Parse request body
-        self.files = {}
-        httputil.parse_body_arguments(self.headers.get("Content-Type", ""),
-                                      self.body, self.body_arguments, self.files)
-
-        for k, v in self.body_arguments.items():
-            self.arguments.setdefault(k, []).extend(v)
-
-        self._start_time = time.time()
-        self._finish_time = None
-
-    def supports_http_1_1(self):
-        """Returns True if this request supports HTTP/1.1 semantics"""
-        return self.version == "HTTP/1.1"
-
-    @property
-    def cookies(self):
-        """A dictionary of Cookie.Morsel objects."""
-        if not hasattr(self, "_cookies"):
-            self._cookies = Cookie.SimpleCookie()
-            if "Cookie" in self.headers:
-                try:
-                    self._cookies.load(
-                        native_str(self.headers["Cookie"]))
-                except Exception:
-                    self._cookies = None
-        return self._cookies
-
-    def full_url(self):
-        """Reconstructs the full URL for this request."""
-        return self.protocol + "://" + self.host + self.uri
-
-    def request_time(self):
-        """Returns the amount of time it took for this request to execute."""
-        if self._finish_time is None:
-            return time.time() - self._start_time
-        else:
-            return self._finish_time - self._start_time
+            host = environ["SERVER_NAME"]
+        connection = _WSGIConnection(method, start_response,
+                                     _WSGIRequestContext(remote_ip, protocol))
+        request = httputil.HTTPServerRequest(
+            method, uri, "HTTP/1.1", headers=headers, body=body,
+            host=host, connection=connection)
+        request._parse_body()
+        self.application(request)
+        if connection._error:
+            raise connection._error
+        if not connection._finished:
+            raise Exception("request did not finish synchronously")
+        return connection._write_buffer
 
 
 class WSGIContainer(object):
     r"""Makes a WSGI-compatible function runnable on Tornado's HTTP server.
 
+    .. warning::
+
+       WSGI is a *synchronous* interface, while Tornado's concurrency model
+       is based on single-threaded asynchronous execution.  This means that
+       running a WSGI app with Tornado's `WSGIContainer` is *less scalable*
+       than running the same app in a multi-threaded WSGI server like
+       ``gunicorn`` or ``uwsgi``.  Use `WSGIContainer` only when there are
+       benefits to combining Tornado and WSGI in the same process that
+       outweigh the reduced scalability.
+
     Wrap a WSGI function in a `WSGIContainer` and pass it to `.HTTPServer` to
     run it. For example::
 
@@ -281,7 +310,7 @@ class WSGIContainer(object):
 
     @staticmethod
     def environ(request):
-        """Converts a `tornado.httpserver.HTTPRequest` to a WSGI environment.
+        """Converts a `tornado.httputil.HTTPServerRequest` to a WSGI environment.
         """
         hostport = request.host.split(":")
         if len(hostport) == 2:
@@ -327,3 +356,6 @@ class WSGIContainer(object):
         summary = request.method + " " + request.uri + " (" + \
             request.remote_ip + ")"
         log_method("%d %s %.2fms", status_code, summary, request_time)
+
+
+HTTPRequest = httputil.HTTPServerRequest