diff --git a/libs/tornado/auth.py b/libs/tornado/auth.py index 6892f91..f15413e 100755 --- a/libs/tornado/auth.py +++ b/libs/tornado/auth.py @@ -34,15 +34,29 @@ See the individual service classes below for complete documentation. Example usage for Google OpenID:: - class GoogleLoginHandler(tornado.web.RequestHandler, - tornado.auth.GoogleMixin): + class GoogleOAuth2LoginHandler(tornado.web.RequestHandler, + tornado.auth.GoogleOAuth2Mixin): @tornado.gen.coroutine def get(self): - if self.get_argument("openid.mode", None): - user = yield self.get_authenticated_user() - # Save the user with e.g. set_secure_cookie() + if self.get_argument('code', False): + user = yield self.get_authenticated_user( + redirect_uri='http://your.site.com/auth/google', + code=self.get_argument('code')) + # Save the user with e.g. set_secure_cookie else: - yield self.authenticate_redirect() + yield self.authorize_redirect( + redirect_uri='http://your.site.com/auth/google', + client_id=self.settings['google_oauth']['key'], + scope=['profile', 'email'], + response_type='code', + extra_params={'approval_prompt': 'auto'}) + +.. versionchanged:: 3.3 + All of the callback interfaces in this module are now guaranteed + to run their callback with an argument of ``None`` on error. + Previously some functions would do this while others would simply + terminate the request on their own. This change also ensures that + errors are more consistently reported through the ``Future`` interfaces. """ from __future__ import absolute_import, division, print_function, with_statement @@ -61,6 +75,7 @@ from tornado import httpclient from tornado import escape from tornado.httputil import url_concat from tornado.log import gen_log +from tornado.stack_context import ExceptionStackContext from tornado.util import bytes_type, u, unicode_type, ArgReplacer try: @@ -108,7 +123,14 @@ def _auth_return_future(f): if callback is not None: future.add_done_callback( functools.partial(_auth_future_to_callback, callback)) - f(*args, **kwargs) + def handle_exception(typ, value, tb): + if future.done(): + return False + else: + future.set_exc_info((typ, value, tb)) + return True + with ExceptionStackContext(handle_exception): + f(*args, **kwargs) return future return wrapper @@ -166,7 +188,7 @@ class OpenIdMixin(object): url = self._OPENID_ENDPOINT if http_client is None: http_client = self.get_auth_http_client() - http_client.fetch(url, self.async_callback( + http_client.fetch(url, functools.partial( self._on_authentication_verified, callback), method="POST", body=urllib_parse.urlencode(args)) @@ -338,7 +360,7 @@ class OAuthMixin(object): http_client.fetch( self._oauth_request_token_url(callback_uri=callback_uri, extra_params=extra_params), - self.async_callback( + functools.partial( self._on_request_token, self._OAUTH_AUTHORIZE_URL, callback_uri, @@ -346,7 +368,7 @@ class OAuthMixin(object): else: http_client.fetch( self._oauth_request_token_url(), - self.async_callback( + functools.partial( self._on_request_token, self._OAUTH_AUTHORIZE_URL, callback_uri, callback)) @@ -383,7 +405,7 @@ class OAuthMixin(object): if http_client is None: http_client = self.get_auth_http_client() http_client.fetch(self._oauth_access_token_url(token), - self.async_callback(self._on_access_token, callback)) + functools.partial(self._on_access_token, callback)) def _oauth_request_token_url(self, callback_uri=None, extra_params=None): consumer_token = self._oauth_consumer_token() @@ -460,7 +482,7 @@ class OAuthMixin(object): access_token = _oauth_parse_response(response.body) self._oauth_get_user_future(access_token).add_done_callback( - self.async_callback(self._on_oauth_get_user, access_token, future)) + functools.partial(self._on_oauth_get_user, access_token, future)) def _oauth_consumer_token(self): """Subclasses must override this to return their OAuth consumer keys. @@ -645,7 +667,7 @@ class TwitterMixin(OAuthMixin): """ http = self.get_auth_http_client() http.fetch(self._oauth_request_token_url(callback_uri=callback_uri), - self.async_callback( + functools.partial( self._on_request_token, self._OAUTH_AUTHENTICATE_URL, None, callback)) @@ -703,7 +725,7 @@ class TwitterMixin(OAuthMixin): if args: url += "?" + urllib_parse.urlencode(args) http = self.get_auth_http_client() - http_callback = self.async_callback(self._on_twitter_request, callback) + http_callback = functools.partial(self._on_twitter_request, callback) if post_args is not None: http.fetch(url, method="POST", body=urllib_parse.urlencode(post_args), callback=http_callback) @@ -820,7 +842,7 @@ class FriendFeedMixin(OAuthMixin): args.update(oauth) if args: url += "?" + urllib_parse.urlencode(args) - callback = self.async_callback(self._on_friendfeed_request, callback) + callback = functools.partial(self._on_friendfeed_request, callback) http = self.get_auth_http_client() if post_args is not None: http.fetch(url, method="POST", body=urllib_parse.urlencode(post_args), @@ -861,6 +883,10 @@ class FriendFeedMixin(OAuthMixin): class GoogleMixin(OpenIdMixin, OAuthMixin): """Google Open ID / OAuth authentication. + *Deprecated:* New applications should use `GoogleOAuth2Mixin` + below instead of this class. As of May 19, 2014, Google has stopped + supporting registration-free authentication. + No application registration is necessary to use Google for authentication or to access Google resources on behalf of a user. @@ -931,7 +957,7 @@ class GoogleMixin(OpenIdMixin, OAuthMixin): http = self.get_auth_http_client() token = dict(key=token, secret="") http.fetch(self._oauth_access_token_url(token), - self.async_callback(self._on_access_token, callback)) + functools.partial(self._on_access_token, callback)) else: chain_future(OpenIdMixin.get_authenticated_user(self), callback) @@ -950,6 +976,19 @@ class GoogleMixin(OpenIdMixin, OAuthMixin): class GoogleOAuth2Mixin(OAuth2Mixin): """Google authentication using OAuth2. + In order to use, register your application with Google and copy the + relevant parameters to your application settings. + + * Go to the Google Dev Console at http://console.developers.google.com + * Select a project, or create a new one. + * In the sidebar on the left, select APIs & Auth. + * In the list of APIs, find the Google+ API service and set it to ON. + * In the sidebar on the left, select Credentials. + * In the OAuth section of the page, select Create New Client ID. + * Set the Redirect URI to point to your auth handler + * Copy the "Client secret" and "Client ID" to the application settings as + {"google_oauth": {"key": CLIENT_ID, "secret": CLIENT_SECRET}} + .. versionadded:: 3.2 """ _OAUTH_AUTHORIZE_URL = "https://accounts.google.com/o/oauth2/auth" @@ -963,7 +1002,7 @@ class GoogleOAuth2Mixin(OAuth2Mixin): Example usage:: - class GoogleOAuth2LoginHandler(LoginHandler, + class GoogleOAuth2LoginHandler(tornado.web.RequestHandler, tornado.auth.GoogleOAuth2Mixin): @tornado.gen.coroutine def get(self): @@ -990,7 +1029,7 @@ class GoogleOAuth2Mixin(OAuth2Mixin): }) http.fetch(self._OAUTH_ACCESS_TOKEN_URL, - self.async_callback(self._on_access_token, callback), + functools.partial(self._on_access_token, callback), method="POST", headers={'Content-Type': 'application/x-www-form-urlencoded'}, body=body) def _on_access_token(self, future, response): @@ -1031,7 +1070,7 @@ class FacebookMixin(object): @tornado.web.asynchronous def get(self): if self.get_argument("session", None): - self.get_authenticated_user(self.async_callback(self._on_auth)) + self.get_authenticated_user(self._on_auth) return yield self.authenticate_redirect() @@ -1117,7 +1156,7 @@ class FacebookMixin(object): session = escape.json_decode(self.get_argument("session")) self.facebook_request( method="facebook.users.getInfo", - callback=self.async_callback( + callback=functools.partial( self._on_get_user_info, callback, session), session_key=session["session_key"], uids=session["uid"], @@ -1143,7 +1182,7 @@ class FacebookMixin(object): def get(self): self.facebook_request( method="stream.get", - callback=self.async_callback(self._on_stream), + callback=self._on_stream, session_key=self.current_user["session_key"]) def _on_stream(self, stream): @@ -1167,7 +1206,7 @@ class FacebookMixin(object): url = "http://api.facebook.com/restserver.php?" + \ urllib_parse.urlencode(args) http = self.get_auth_http_client() - http.fetch(url, callback=self.async_callback( + http.fetch(url, callback=functools.partial( self._parse_response, callback)) def _on_get_user_info(self, callback, session, users): @@ -1265,7 +1304,7 @@ class FacebookGraphMixin(OAuth2Mixin): fields.update(extra_fields) http.fetch(self._oauth_request_token_url(**args), - self.async_callback(self._on_access_token, redirect_uri, client_id, + functools.partial(self._on_access_token, redirect_uri, client_id, client_secret, callback, fields)) def _on_access_token(self, redirect_uri, client_id, client_secret, @@ -1282,7 +1321,7 @@ class FacebookGraphMixin(OAuth2Mixin): self.facebook_request( path="/me", - callback=self.async_callback( + callback=functools.partial( self._on_get_user_info, future, session, fields), access_token=session["access_token"], fields=",".join(fields) @@ -1349,7 +1388,7 @@ class FacebookGraphMixin(OAuth2Mixin): if all_args: url += "?" + urllib_parse.urlencode(all_args) - callback = self.async_callback(self._on_facebook_request, callback) + callback = functools.partial(self._on_facebook_request, callback) http = self.get_auth_http_client() if post_args is not None: http.fetch(url, method="POST", body=urllib_parse.urlencode(post_args), diff --git a/libs/tornado/http1connection.py b/libs/tornado/http1connection.py index 9f474e1..edaa5d9 100644 --- a/libs/tornado/http1connection.py +++ b/libs/tornado/http1connection.py @@ -31,6 +31,26 @@ from tornado import stack_context from tornado.util import GzipDecompressor +class _QuietException(Exception): + def __init__(self): + pass + +class _ExceptionLoggingContext(object): + """Used with the ``with`` statement when calling delegate methods to + log any exceptions with the given logger. Any exceptions caught are + converted to _QuietException + """ + def __init__(self, logger): + self.logger = logger + + def __enter__(self): + pass + + def __exit__(self, typ, value, tb): + if value is not None: + self.logger.error("Uncaught exception", exc_info=(typ, value, tb)) + raise _QuietException + class HTTP1ConnectionParameters(object): """Parameters for `.HTTP1Connection` and `.HTTP1ServerConnection`. """ @@ -155,9 +175,10 @@ class HTTP1Connection(httputil.HTTPConnection): self._disconnect_on_finish = not self._can_keep_alive( start_line, headers) need_delegate_close = True - header_future = delegate.headers_received(start_line, headers) - if header_future is not None: - yield header_future + with _ExceptionLoggingContext(app_log): + header_future = delegate.headers_received(start_line, headers) + if header_future is not None: + yield header_future if self.stream is None: # We've been detached. need_delegate_close = False @@ -196,7 +217,8 @@ class HTTP1Connection(httputil.HTTPConnection): self._read_finished = True if not self._write_finished or self.is_client: need_delegate_close = False - delegate.finish() + with _ExceptionLoggingContext(app_log): + delegate.finish() # If we're waiting for the application to produce an asynchronous # response, and we're not detached, register a close callback # on the stream (we didn't need one while we were reading) @@ -216,7 +238,8 @@ class HTTP1Connection(httputil.HTTPConnection): raise gen.Return(False) finally: if need_delegate_close: - delegate.on_connection_close() + with _ExceptionLoggingContext(app_log): + delegate.on_connection_close() self._clear_callbacks() raise gen.Return(True) @@ -478,7 +501,8 @@ class HTTP1Connection(httputil.HTTPConnection): min(self.params.chunk_size, content_length), partial=True) content_length -= len(body) if not self._write_finished or self.is_client: - yield gen.maybe_future(delegate.data_received(body)) + with _ExceptionLoggingContext(app_log): + yield gen.maybe_future(delegate.data_received(body)) @gen.coroutine def _read_chunked_body(self, delegate): @@ -498,8 +522,8 @@ class HTTP1Connection(httputil.HTTPConnection): min(bytes_to_read, self.params.chunk_size), partial=True) bytes_to_read -= len(chunk) if not self._write_finished or self.is_client: - yield gen.maybe_future( - delegate.data_received(chunk)) + with _ExceptionLoggingContext(app_log): + yield gen.maybe_future(delegate.data_received(chunk)) # chunk ends with \r\n crlf = yield self.stream.read_bytes(2) assert crlf == b"\r\n" @@ -508,7 +532,8 @@ class HTTP1Connection(httputil.HTTPConnection): def _read_body_until_close(self, delegate): body = yield self.stream.read_until_close() if not self._write_finished or self.is_client: - delegate.data_received(body) + with _ExceptionLoggingContext(app_log): + delegate.data_received(body) class _GzipMessageDelegate(httputil.HTTPMessageDelegate): @@ -610,11 +635,12 @@ class HTTP1ServerConnection(object): except (iostream.StreamClosedError, iostream.UnsatisfiableReadError): return + except _QuietException: + # This exception was already logged. + conn.close() + return except Exception: - # TODO: this is probably too broad; it would be better to - # wrap all delegate calls in something that writes to app_log, - # and then errors that reach this point can be gen_log. - app_log.error("Uncaught exception", exc_info=True) + gen_log.error("Uncaught exception", exc_info=True) conn.close() return if not ret: diff --git a/libs/tornado/ioloop.py b/libs/tornado/ioloop.py index 86c4a9f..cd59bfe 100755 --- a/libs/tornado/ioloop.py +++ b/libs/tornado/ioloop.py @@ -32,6 +32,7 @@ import datetime import errno import functools import heapq +import itertools import logging import numbers import os @@ -585,7 +586,8 @@ class PollIOLoop(IOLoop): self._closing = False self._thread_ident = None self._blocking_signal_threshold = None - + self._timeout_counter = itertools.count() + # Create a pipe that we send bogus data to when we want to wake # the I/O loop when it is idle self._waker = Waker() @@ -835,7 +837,7 @@ class _Timeout(object): """An IOLoop timeout, a UNIX timestamp and a callback""" # Reduce memory overhead when there are lots of pending callbacks - __slots__ = ['deadline', 'callback'] + __slots__ = ['deadline', 'callback', 'tiebreaker'] def __init__(self, deadline, callback, io_loop): if isinstance(deadline, numbers.Real): @@ -849,6 +851,7 @@ class _Timeout(object): else: raise TypeError("Unsupported deadline %r" % deadline) self.callback = callback + self.tiebreaker = next(io_loop._timeout_counter) @staticmethod def timedelta_to_seconds(td): @@ -860,12 +863,12 @@ class _Timeout(object): # in python2.5, and __lt__ in 2.6+ (sort() and most other comparisons # use __lt__). def __lt__(self, other): - return ((self.deadline, id(self)) < - (other.deadline, id(other))) + return ((self.deadline, self.tiebreaker) < + (other.deadline, other.tiebreaker)) def __le__(self, other): - return ((self.deadline, id(self)) <= - (other.deadline, id(other))) + return ((self.deadline, self.tiebreaker) <= + (other.deadline, other.tiebreaker)) class PeriodicCallback(object): diff --git a/libs/tornado/util.py b/libs/tornado/util.py index 7fe5b9b..49eea2c 100755 --- a/libs/tornado/util.py +++ b/libs/tornado/util.py @@ -12,11 +12,19 @@ and `.Resolver`. from __future__ import absolute_import, division, print_function, with_statement +import array import inspect +import os import sys import zlib +try: + xrange # py2 +except NameError: + xrange = range # py3 + + class ObjectDict(dict): """Makes a dictionary behave like an object, with attribute-style access. """ @@ -303,6 +311,41 @@ class ArgReplacer(object): return old_value, args, kwargs +def _websocket_mask_python(mask, data): + """Websocket masking function. + + `mask` is a `bytes` object of length 4; `data` is a `bytes` object of any length. + Returns a `bytes` object of the same length as `data` with the mask applied + as specified in section 5.3 of RFC 6455. + + This pure-python implementation may be replaced by an optimized version when available. + """ + mask = array.array("B", mask) + unmasked = array.array("B", data) + for i in xrange(len(data)): + unmasked[i] = unmasked[i] ^ mask[i % 4] + if hasattr(unmasked, 'tobytes'): + # tostring was deprecated in py32. It hasn't been removed, + # but since we turn on deprecation warnings in our tests + # we need to use the right one. + return unmasked.tobytes() + else: + return unmasked.tostring() + +if (os.environ.get('TORNADO_NO_EXTENSION') or + os.environ.get('TORNADO_EXTENSION') == '0'): + # These environment variables exist to make it easier to do performance + # comparisons; they are not guaranteed to remain supported in the future. + _websocket_mask = _websocket_mask_python +else: + try: + from tornado.speedups import websocket_mask as _websocket_mask + except ImportError: + if os.environ.get('TORNADO_EXTENSION') == '1': + raise + _websocket_mask = _websocket_mask_python + + def doctests(): import doctest return doctest.DocTestSuite() diff --git a/libs/tornado/web.py b/libs/tornado/web.py index 5a875fd..dd2b5ef 100755 --- a/libs/tornado/web.py +++ b/libs/tornado/web.py @@ -72,7 +72,6 @@ import time import tornado import traceback import types -import uuid from tornado.concurrent import Future, is_future from tornado import escape @@ -84,7 +83,7 @@ from tornado.log import access_log, app_log, gen_log from tornado import stack_context from tornado import template from tornado.escape import utf8, _unicode -from tornado.util import bytes_type, import_object, ObjectDict, raise_exc_info, unicode_type +from tornado.util import bytes_type, import_object, ObjectDict, raise_exc_info, unicode_type, _websocket_mask try: from io import BytesIO # python 3 @@ -1076,16 +1075,87 @@ class RequestHandler(object): as a potential forgery. See http://en.wikipedia.org/wiki/Cross-site_request_forgery + + .. versionchanged:: 3.2.2 + The xsrf token will now be have a random mask applied in every + request, which makes it safe to include the token in pages + that are compressed. See http://breachattack.com for more + information on the issue fixed by this change. Old (version 1) + cookies will be converted to version 2 when this method is called + unless the ``xsrf_cookie_version`` `Application` setting is + set to 1. """ if not hasattr(self, "_xsrf_token"): - token = self.get_cookie("_xsrf") - if not token: - token = binascii.b2a_hex(uuid.uuid4().bytes) + version, token, timestamp = self._get_raw_xsrf_token() + output_version = self.settings.get("xsrf_cookie_version", 2) + if output_version == 1: + self._xsrf_token = binascii.b2a_hex(token) + elif output_version == 2: + mask = os.urandom(4) + self._xsrf_token = b"|".join([ + b"2", + binascii.b2a_hex(mask), + binascii.b2a_hex(_websocket_mask(mask, token)), + utf8(str(int(timestamp)))]) + else: + raise ValueError("unknown xsrf cookie version %d", + output_version) + if version is None: expires_days = 30 if self.current_user else None - self.set_cookie("_xsrf", token, expires_days=expires_days) - self._xsrf_token = token + self.set_cookie("_xsrf", self._xsrf_token, + expires_days=expires_days) return self._xsrf_token + def _get_raw_xsrf_token(self): + """Read or generate the xsrf token in its raw form. + + The raw_xsrf_token is a tuple containing: + + * version: the version of the cookie from which this token was read, + or None if we generated a new token in this request. + * token: the raw token data; random (non-ascii) bytes. + * timestamp: the time this token was generated (will not be accurate + for version 1 cookies) + """ + if not hasattr(self, '_raw_xsrf_token'): + cookie = self.get_cookie("_xsrf") + if cookie: + version, token, timestamp = self._decode_xsrf_token(cookie) + else: + version, token, timestamp = None, None, None + if token is None: + version = None + token = os.urandom(16) + timestamp = time.time() + self._raw_xsrf_token = (version, token, timestamp) + return self._raw_xsrf_token + + def _decode_xsrf_token(self, cookie): + """Convert a cookie string into a the tuple form returned by + _get_raw_xsrf_token. + """ + m = _signed_value_version_re.match(utf8(cookie)) + if m: + version = int(m.group(1)) + if version == 2: + _, mask, masked_token, timestamp = cookie.split("|") + mask = binascii.a2b_hex(utf8(mask)) + token = _websocket_mask( + mask, binascii.a2b_hex(utf8(masked_token))) + timestamp = int(timestamp) + return version, token, timestamp + else: + # Treat unknown versions as not present instead of failing. + return None, None, None + elif len(cookie) == 32: + version = 1 + token = binascii.a2b_hex(utf8(cookie)) + # We don't have a usable timestamp in older versions. + timestamp = int(time.time()) + return (version, token, timestamp) + else: + return None, None, None + def check_xsrf_cookie(self): """Verifies that the ``_xsrf`` cookie matches the ``_xsrf`` argument. @@ -1106,13 +1176,19 @@ class RequestHandler(object): information please see http://www.djangoproject.com/weblog/2011/feb/08/security/ http://weblog.rubyonrails.org/2011/2/8/csrf-protection-bypass-in-ruby-on-rails + + .. versionchanged:: 3.2.2 + Added support for cookie version 2. Both versions 1 and 2 are + supported. """ token = (self.get_argument("_xsrf", None) or self.request.headers.get("X-Xsrftoken") or self.request.headers.get("X-Csrftoken")) if not token: raise HTTPError(403, "'_xsrf' argument missing from POST") - if self.xsrf_token != token: + _, token, _ = self._decode_xsrf_token(token) + _, expected_token, _ = self._get_raw_xsrf_token() + if not _time_independent_equals(utf8(token), utf8(expected_token)): raise HTTPError(403, "XSRF cookie does not match POST argument") def xsrf_form_html(self): diff --git a/libs/tornado/websocket.py b/libs/tornado/websocket.py index 3767a20..c0065c7 100755 --- a/libs/tornado/websocket.py +++ b/libs/tornado/websocket.py @@ -20,7 +20,6 @@ communication between the browser and server. from __future__ import absolute_import, division, print_function, with_statement # Author: Jacob Kristhammar, 2010 -import array import base64 import collections import functools @@ -39,7 +38,7 @@ from tornado.iostream import StreamClosedError from tornado.log import gen_log, app_log from tornado import simple_httpclient from tornado.tcpclient import TCPClient -from tornado.util import bytes_type, unicode_type +from tornado.util import bytes_type, unicode_type, _websocket_mask try: from urllib.parse import urlparse # py2 @@ -988,38 +987,3 @@ def websocket_connect(url, io_loop=None, callback=None, connect_timeout=None): if callback is not None: io_loop.add_future(conn.connect_future, callback) return conn.connect_future - - -def _websocket_mask_python(mask, data): - """Websocket masking function. - - `mask` is a `bytes` object of length 4; `data` is a `bytes` object of any length. - Returns a `bytes` object of the same length as `data` with the mask applied - as specified in section 5.3 of RFC 6455. - - This pure-python implementation may be replaced by an optimized version when available. - """ - mask = array.array("B", mask) - unmasked = array.array("B", data) - for i in xrange(len(data)): - unmasked[i] = unmasked[i] ^ mask[i % 4] - if hasattr(unmasked, 'tobytes'): - # tostring was deprecated in py32. It hasn't been removed, - # but since we turn on deprecation warnings in our tests - # we need to use the right one. - return unmasked.tobytes() - else: - return unmasked.tostring() - -if (os.environ.get('TORNADO_NO_EXTENSION') or - os.environ.get('TORNADO_EXTENSION') == '0'): - # These environment variables exist to make it easier to do performance - # comparisons; they are not guaranteed to remain supported in the future. - _websocket_mask = _websocket_mask_python -else: - try: - from tornado.speedups import websocket_mask as _websocket_mask - except ImportError: - if os.environ.get('TORNADO_EXTENSION') == '1': - raise - _websocket_mask = _websocket_mask_python