From 02e01fb2d6c453762b35a1bd16f8dbedf2154abd Mon Sep 17 00:00:00 2001 From: Ruud Date: Sat, 11 Feb 2012 16:28:06 +0100 Subject: [PATCH] Packages update --- .../core/providers/movie/themoviedb/main.py | 2 +- libs/argparse.py | 2353 ------------------ libs/decorator.py | 7 +- libs/flask/app.py | 40 +- libs/flask/blueprints.py | 44 +- libs/flask/ctx.py | 6 +- libs/flask/helpers.py | 86 +- libs/flask/views.py | 22 +- libs/jinja2/filters.py | 26 +- libs/jinja2/runtime.py | 17 +- libs/jinja2/utils.py | 19 + libs/migrate/__init__.py | 2 + libs/migrate/changeset/__init__.py | 2 +- libs/migrate/changeset/ansisql.py | 108 +- libs/migrate/changeset/constraint.py | 3 - libs/migrate/changeset/databases/firebird.py | 12 +- libs/migrate/changeset/databases/mysql.py | 43 +- libs/migrate/changeset/databases/oracle.py | 7 +- libs/migrate/changeset/databases/postgres.py | 12 +- libs/migrate/changeset/databases/sqlite.py | 10 +- libs/migrate/changeset/schema.py | 32 +- libs/migrate/versioning/api.py | 17 +- libs/migrate/versioning/genmodel.py | 183 +- libs/migrate/versioning/repository.py | 15 +- libs/migrate/versioning/schema.py | 21 +- libs/migrate/versioning/schemadiff.py | 77 +- libs/migrate/versioning/script/py.py | 16 +- libs/migrate/versioning/template.py | 1 - libs/migrate/versioning/templates/manage.py_tmpl | 5 - .../versioning/templates/manage/default.py_tmpl | 4 +- .../versioning/templates/manage/pylons.py_tmpl | 3 +- .../templates/repository/default/migrate.cfg | 5 + .../templates/repository/pylons/migrate.cfg | 5 + .../versioning/templates/script/default.py_tmpl | 6 +- .../versioning/templates/script/pylons.py_tmpl | 6 +- libs/migrate/versioning/version.py | 39 +- libs/pytwitter/__init__.py | 1542 +++++++++--- libs/requests/__init__.py | 4 +- libs/requests/api.py | 3 +- libs/requests/async.py | 30 +- libs/requests/auth.py | 40 +- libs/requests/compat.py | 102 + libs/requests/defaults.py | 6 +- libs/requests/hooks.py | 18 +- libs/requests/models.py | 218 +- libs/requests/packages/oreos/monkeys.py | 2 +- libs/requests/packages/urllib3/__init__.py | 4 +- libs/requests/packages/urllib3/_collections.py | 2 +- libs/requests/packages/urllib3/connectionpool.py | 124 +- libs/requests/packages/urllib3/exceptions.py | 41 +- libs/requests/packages/urllib3/filepost.py | 33 +- .../packages/mimetools_choose_boundary/__init__.py | 47 + libs/requests/packages/urllib3/packages/six.py | 372 +++ libs/requests/packages/urllib3/poolmanager.py | 28 +- libs/requests/packages/urllib3/request.py | 8 +- libs/requests/packages/urllib3/response.py | 83 +- libs/requests/sessions.py | 10 +- libs/requests/status_codes.py | 2 +- libs/requests/structures.py | 4 +- libs/requests/utils.py | 61 +- libs/simplejson/__init__.py | 438 ---- libs/simplejson/_speedups.c | 2602 -------------------- libs/simplejson/decoder.py | 421 ---- libs/simplejson/encoder.py | 501 ---- libs/simplejson/ordered_dict.py | 119 - libs/simplejson/scanner.py | 77 - libs/simplejson/tool.py | 39 - libs/sqlalchemy/__init__.py | 11 +- libs/sqlalchemy/cextension/processors.c | 64 +- libs/sqlalchemy/cextension/resultproxy.c | 6 +- libs/sqlalchemy/connectors/__init__.py | 2 +- libs/sqlalchemy/connectors/mxodbc.py | 11 +- libs/sqlalchemy/connectors/mysqldb.py | 150 ++ libs/sqlalchemy/connectors/pyodbc.py | 44 +- libs/sqlalchemy/connectors/zxJDBC.py | 4 +- libs/sqlalchemy/databases/__init__.py | 4 +- libs/sqlalchemy/dialects/__init__.py | 3 +- libs/sqlalchemy/dialects/access/base.py | 9 +- libs/sqlalchemy/dialects/drizzle/__init__.py | 18 + libs/sqlalchemy/dialects/drizzle/base.py | 582 +++++ libs/sqlalchemy/dialects/drizzle/mysqldb.py | 69 + libs/sqlalchemy/dialects/firebird/__init__.py | 2 +- libs/sqlalchemy/dialects/firebird/base.py | 34 +- libs/sqlalchemy/dialects/firebird/kinterbasdb.py | 25 +- libs/sqlalchemy/dialects/informix/__init__.py | 2 +- libs/sqlalchemy/dialects/informix/base.py | 125 +- libs/sqlalchemy/dialects/informix/informixdb.py | 4 +- libs/sqlalchemy/dialects/maxdb/__init__.py | 2 +- libs/sqlalchemy/dialects/maxdb/base.py | 77 +- libs/sqlalchemy/dialects/maxdb/sapdb.py | 2 +- libs/sqlalchemy/dialects/mssql/__init__.py | 2 +- libs/sqlalchemy/dialects/mssql/adodbapi.py | 5 +- libs/sqlalchemy/dialects/mssql/base.py | 213 +- .../dialects/mssql/information_schema.py | 4 +- libs/sqlalchemy/dialects/mssql/mxodbc.py | 8 +- libs/sqlalchemy/dialects/mssql/pymssql.py | 6 +- libs/sqlalchemy/dialects/mssql/pyodbc.py | 49 +- libs/sqlalchemy/dialects/mssql/zxjdbc.py | 2 +- libs/sqlalchemy/dialects/mysql/__init__.py | 4 +- libs/sqlalchemy/dialects/mysql/base.py | 317 ++- libs/sqlalchemy/dialects/mysql/mysqlconnector.py | 4 +- libs/sqlalchemy/dialects/mysql/mysqldb.py | 195 +- libs/sqlalchemy/dialects/mysql/oursql.py | 33 +- libs/sqlalchemy/dialects/mysql/pymysql.py | 39 + libs/sqlalchemy/dialects/mysql/pyodbc.py | 2 +- libs/sqlalchemy/dialects/mysql/zxjdbc.py | 2 +- libs/sqlalchemy/dialects/oracle/__init__.py | 2 +- libs/sqlalchemy/dialects/oracle/base.py | 125 +- libs/sqlalchemy/dialects/oracle/cx_oracle.py | 45 +- libs/sqlalchemy/dialects/oracle/zxjdbc.py | 7 +- libs/sqlalchemy/dialects/postgres.py | 2 +- libs/sqlalchemy/dialects/postgresql/__init__.py | 2 +- libs/sqlalchemy/dialects/postgresql/base.py | 559 ++++- libs/sqlalchemy/dialects/postgresql/pg8000.py | 17 +- libs/sqlalchemy/dialects/postgresql/psycopg2.py | 202 +- .../sqlalchemy/dialects/postgresql/pypostgresql.py | 8 +- libs/sqlalchemy/dialects/postgresql/zxjdbc.py | 21 +- libs/sqlalchemy/dialects/sqlite/__init__.py | 6 +- libs/sqlalchemy/dialects/sqlite/base.py | 255 +- libs/sqlalchemy/dialects/sqlite/pysqlite.py | 174 +- libs/sqlalchemy/dialects/sybase/__init__.py | 2 +- libs/sqlalchemy/dialects/sybase/base.py | 4 +- libs/sqlalchemy/dialects/sybase/mxodbc.py | 2 +- libs/sqlalchemy/dialects/sybase/pyodbc.py | 4 +- libs/sqlalchemy/dialects/sybase/pysybase.py | 2 +- libs/sqlalchemy/engine/__init__.py | 130 +- libs/sqlalchemy/engine/base.py | 1346 ++++++---- libs/sqlalchemy/engine/ddl.py | 124 +- libs/sqlalchemy/engine/default.py | 504 ++-- libs/sqlalchemy/engine/reflection.py | 63 +- libs/sqlalchemy/engine/strategies.py | 57 +- libs/sqlalchemy/engine/threadlocal.py | 14 +- libs/sqlalchemy/engine/url.py | 24 +- libs/sqlalchemy/event.py | 347 +++ libs/sqlalchemy/events.py | 433 ++++ libs/sqlalchemy/exc.py | 170 +- libs/sqlalchemy/ext/__init__.py | 2 +- libs/sqlalchemy/ext/associationproxy.py | 229 +- libs/sqlalchemy/ext/compiler.py | 177 +- libs/sqlalchemy/ext/declarative.py | 740 ++++-- libs/sqlalchemy/ext/horizontal_shard.py | 109 +- libs/sqlalchemy/ext/hybrid.py | 677 +++++ libs/sqlalchemy/ext/mutable.py | 563 +++++ libs/sqlalchemy/ext/orderinglist.py | 6 +- libs/sqlalchemy/ext/serializer.py | 2 +- libs/sqlalchemy/ext/sqlsoup.py | 33 +- libs/sqlalchemy/interfaces.py | 109 +- libs/sqlalchemy/log.py | 201 +- libs/sqlalchemy/orm/__init__.py | 1067 +++++--- libs/sqlalchemy/orm/attributes.py | 1465 ++++------- libs/sqlalchemy/orm/collections.py | 56 +- libs/sqlalchemy/orm/dependency.py | 224 +- libs/sqlalchemy/orm/deprecated_interfaces.py | 589 +++++ libs/sqlalchemy/orm/descriptor_props.py | 422 ++++ libs/sqlalchemy/orm/dynamic.py | 81 +- libs/sqlalchemy/orm/evaluator.py | 2 +- libs/sqlalchemy/orm/events.py | 1248 ++++++++++ libs/sqlalchemy/orm/exc.py | 46 +- libs/sqlalchemy/orm/identity.py | 162 +- libs/sqlalchemy/orm/instrumentation.py | 674 +++++ libs/sqlalchemy/orm/interfaces.py | 743 ++---- libs/sqlalchemy/orm/mapper.py | 1645 ++++++++----- libs/sqlalchemy/orm/properties.py | 1253 +++++----- libs/sqlalchemy/orm/query.py | 1269 ++++++---- libs/sqlalchemy/orm/scoping.py | 102 +- libs/sqlalchemy/orm/session.py | 882 ++++--- libs/sqlalchemy/orm/shard.py | 2 +- libs/sqlalchemy/orm/state.py | 300 +-- libs/sqlalchemy/orm/strategies.py | 1042 ++++---- libs/sqlalchemy/orm/sync.py | 28 +- libs/sqlalchemy/orm/unitofwork.py | 131 +- libs/sqlalchemy/orm/util.py | 352 +-- libs/sqlalchemy/pool.py | 344 ++- libs/sqlalchemy/processors.py | 60 +- libs/sqlalchemy/queue.py | 191 -- libs/sqlalchemy/schema.py | 1333 +++++++--- libs/sqlalchemy/sql/__init__.py | 6 +- libs/sqlalchemy/sql/compiler.py | 801 ++++-- libs/sqlalchemy/sql/expression.py | 2369 ++++++++++++------ libs/sqlalchemy/sql/functions.py | 28 +- libs/sqlalchemy/sql/operators.py | 448 +++- libs/sqlalchemy/sql/util.py | 136 +- libs/sqlalchemy/sql/visitors.py | 109 +- libs/sqlalchemy/test/__init__.py | 33 - libs/sqlalchemy/test/assertsql.py | 300 --- libs/sqlalchemy/test/engines.py | 311 --- libs/sqlalchemy/test/entities.py | 89 - libs/sqlalchemy/test/orm.py | 117 - libs/sqlalchemy/test/pickleable.py | 81 - libs/sqlalchemy/test/profiling.py | 228 -- libs/sqlalchemy/test/requires.py | 333 --- libs/sqlalchemy/test/schema.py | 85 - libs/sqlalchemy/test/testing.py | 803 ------ libs/sqlalchemy/test/util.py | 81 - libs/sqlalchemy/topological.py | 83 - libs/sqlalchemy/types.py | 1042 +++++--- libs/sqlalchemy/util.py | 1875 -------------- libs/sqlalchemy/util/__init__.py | 34 + libs/sqlalchemy/util/_collections.py | 905 +++++++ libs/sqlalchemy/util/compat.py | 231 ++ libs/sqlalchemy/util/deprecations.py | 118 + libs/sqlalchemy/util/langhelpers.py | 898 +++++++ libs/sqlalchemy/util/queue.py | 191 ++ libs/sqlalchemy/util/topological.py | 92 + libs/subliminal/core.py | 2 +- libs/themoviedb/tmdb.py | 244 +- libs/transmissionrpc/LICENSE | 22 - libs/transmissionrpc/__init__.py | 8 +- libs/transmissionrpc/client.py | 313 ++- libs/transmissionrpc/constants.py | 43 +- libs/transmissionrpc/error.py | 24 +- libs/transmissionrpc/httphandler.py | 12 +- libs/transmissionrpc/session.py | 8 +- libs/transmissionrpc/torrent.py | 112 +- libs/transmissionrpc/utils.py | 13 +- libs/werkzeug/contrib/cache.py | 3 +- libs/werkzeug/debug/console.py | 6 +- libs/werkzeug/urls.py | 2 +- libs/werkzeug/utils.py | 4 +- 219 files changed, 26714 insertions(+), 21513 deletions(-) delete mode 100644 libs/argparse.py delete mode 100644 libs/migrate/versioning/templates/manage.py_tmpl create mode 100644 libs/requests/compat.py create mode 100644 libs/requests/packages/urllib3/packages/mimetools_choose_boundary/__init__.py create mode 100644 libs/requests/packages/urllib3/packages/six.py delete mode 100644 libs/simplejson/__init__.py delete mode 100644 libs/simplejson/_speedups.c delete mode 100644 libs/simplejson/decoder.py delete mode 100644 libs/simplejson/encoder.py delete mode 100644 libs/simplejson/ordered_dict.py delete mode 100644 libs/simplejson/scanner.py delete mode 100644 libs/simplejson/tool.py create mode 100644 libs/sqlalchemy/connectors/mysqldb.py create mode 100644 libs/sqlalchemy/dialects/drizzle/__init__.py create mode 100644 libs/sqlalchemy/dialects/drizzle/base.py create mode 100644 libs/sqlalchemy/dialects/drizzle/mysqldb.py create mode 100644 libs/sqlalchemy/dialects/mysql/pymysql.py create mode 100644 libs/sqlalchemy/event.py create mode 100644 libs/sqlalchemy/events.py mode change 100644 => 100755 libs/sqlalchemy/ext/declarative.py create mode 100644 libs/sqlalchemy/ext/hybrid.py create mode 100644 libs/sqlalchemy/ext/mutable.py create mode 100644 libs/sqlalchemy/orm/deprecated_interfaces.py create mode 100644 libs/sqlalchemy/orm/descriptor_props.py create mode 100644 libs/sqlalchemy/orm/events.py create mode 100644 libs/sqlalchemy/orm/instrumentation.py delete mode 100644 libs/sqlalchemy/queue.py delete mode 100644 libs/sqlalchemy/test/__init__.py delete mode 100644 libs/sqlalchemy/test/assertsql.py delete mode 100644 libs/sqlalchemy/test/engines.py delete mode 100644 libs/sqlalchemy/test/entities.py delete mode 100644 libs/sqlalchemy/test/orm.py delete mode 100644 libs/sqlalchemy/test/pickleable.py delete mode 100644 libs/sqlalchemy/test/profiling.py delete mode 100644 libs/sqlalchemy/test/requires.py delete mode 100644 libs/sqlalchemy/test/schema.py delete mode 100644 libs/sqlalchemy/test/testing.py delete mode 100644 libs/sqlalchemy/test/util.py delete mode 100644 libs/sqlalchemy/topological.py delete mode 100644 libs/sqlalchemy/util.py create mode 100644 libs/sqlalchemy/util/__init__.py create mode 100644 libs/sqlalchemy/util/_collections.py create mode 100644 libs/sqlalchemy/util/compat.py create mode 100644 libs/sqlalchemy/util/deprecations.py create mode 100644 libs/sqlalchemy/util/langhelpers.py create mode 100644 libs/sqlalchemy/util/queue.py create mode 100644 libs/sqlalchemy/util/topological.py delete mode 100644 libs/transmissionrpc/LICENSE mode change 100644 => 100755 libs/transmissionrpc/__init__.py mode change 100644 => 100755 libs/transmissionrpc/client.py mode change 100644 => 100755 libs/transmissionrpc/constants.py mode change 100644 => 100755 libs/transmissionrpc/error.py mode change 100644 => 100755 libs/transmissionrpc/httphandler.py mode change 100644 => 100755 libs/transmissionrpc/session.py mode change 100644 => 100755 libs/transmissionrpc/torrent.py mode change 100644 => 100755 libs/transmissionrpc/utils.py diff --git a/couchpotato/core/providers/movie/themoviedb/main.py b/couchpotato/core/providers/movie/themoviedb/main.py index 8229841..ece79a6 100644 --- a/couchpotato/core/providers/movie/themoviedb/main.py +++ b/couchpotato/core/providers/movie/themoviedb/main.py @@ -16,7 +16,7 @@ class TheMovieDb(MovieProvider): addEvent('movie.info_by_tmdb', self.getInfoByTMDBId) # Use base wrapper - tmdb.Config.api_key = self.conf('api_key') + tmdb.configure(self.conf('api_key')) def byHash(self, file): ''' Find movie by hash ''' diff --git a/libs/argparse.py b/libs/argparse.py deleted file mode 100644 index a060129..0000000 --- a/libs/argparse.py +++ /dev/null @@ -1,2353 +0,0 @@ -# -*- coding: utf-8 -*- - -# Copyright © 2006-2009 Steven J. Bethard . -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may not -# use this file except in compliance with the License. You may obtain a copy -# of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -"""Command-line parsing library - -This module is an optparse-inspired command-line parsing library that: - - - handles both optional and positional arguments - - produces highly informative usage messages - - supports parsers that dispatch to sub-parsers - -The following is a simple usage example that sums integers from the -command-line and writes the result to a file:: - - parser = argparse.ArgumentParser( - description='sum the integers at the command line') - parser.add_argument( - 'integers', metavar='int', nargs='+', type=int, - help='an integer to be summed') - parser.add_argument( - '--log', default=sys.stdout, type=argparse.FileType('w'), - help='the file where the sum should be written') - args = parser.parse_args() - args.log.write('%s' % sum(args.integers)) - args.log.close() - -The module contains the following public classes: - - - ArgumentParser -- The main entry point for command-line parsing. As the - example above shows, the add_argument() method is used to populate - the parser with actions for optional and positional arguments. Then - the parse_args() method is invoked to convert the args at the - command-line into an object with attributes. - - - ArgumentError -- The exception raised by ArgumentParser objects when - there are errors with the parser's actions. Errors raised while - parsing the command-line are caught by ArgumentParser and emitted - as command-line messages. - - - FileType -- A factory for defining types of files to be created. As the - example above shows, instances of FileType are typically passed as - the type= argument of add_argument() calls. - - - Action -- The base class for parser actions. Typically actions are - selected by passing strings like 'store_true' or 'append_const' to - the action= argument of add_argument(). However, for greater - customization of ArgumentParser actions, subclasses of Action may - be defined and passed as the action= argument. - - - HelpFormatter, RawDescriptionHelpFormatter, RawTextHelpFormatter, - ArgumentDefaultsHelpFormatter -- Formatter classes which - may be passed as the formatter_class= argument to the - ArgumentParser constructor. HelpFormatter is the default, - RawDescriptionHelpFormatter and RawTextHelpFormatter tell the parser - not to change the formatting for help text, and - ArgumentDefaultsHelpFormatter adds information about argument defaults - to the help. - -All other classes in this module are considered implementation details. -(Also note that HelpFormatter and RawDescriptionHelpFormatter are only -considered public as object names -- the API of the formatter objects is -still considered an implementation detail.) -""" - -__version__ = '1.1' -__all__ = [ - 'ArgumentParser', - 'ArgumentError', - 'Namespace', - 'Action', - 'FileType', - 'HelpFormatter', - 'RawDescriptionHelpFormatter', - 'RawTextHelpFormatter', - 'ArgumentDefaultsHelpFormatter', -] - - -import copy as _copy -import os as _os -import re as _re -import sys as _sys -import textwrap as _textwrap - -from gettext import gettext as _ - -try: - _set = set -except NameError: - from sets import Set as _set - -try: - _basestring = basestring -except NameError: - _basestring = str - -try: - _sorted = sorted -except NameError: - - def _sorted(iterable, reverse=False): - result = list(iterable) - result.sort() - if reverse: - result.reverse() - return result - - -def _callable(obj): - return hasattr(obj, '__call__') or hasattr(obj, '__bases__') - -# silence Python 2.6 buggy warnings about Exception.message -if _sys.version_info[:2] == (2, 6): - import warnings - warnings.filterwarnings( - action='ignore', - message='BaseException.message has been deprecated as of Python 2.6', - category=DeprecationWarning, - module='argparse') - - -SUPPRESS = '==SUPPRESS==' - -OPTIONAL = '?' -ZERO_OR_MORE = '*' -ONE_OR_MORE = '+' -PARSER = 'A...' -REMAINDER = '...' - -# ============================= -# Utility functions and classes -# ============================= - -class _AttributeHolder(object): - """Abstract base class that provides __repr__. - - The __repr__ method returns a string in the format:: - ClassName(attr=name, attr=name, ...) - The attributes are determined either by a class-level attribute, - '_kwarg_names', or by inspecting the instance __dict__. - """ - - def __repr__(self): - type_name = type(self).__name__ - arg_strings = [] - for arg in self._get_args(): - arg_strings.append(repr(arg)) - for name, value in self._get_kwargs(): - arg_strings.append('%s=%r' % (name, value)) - return '%s(%s)' % (type_name, ', '.join(arg_strings)) - - def _get_kwargs(self): - return _sorted(self.__dict__.items()) - - def _get_args(self): - return [] - - -def _ensure_value(namespace, name, value): - if getattr(namespace, name, None) is None: - setattr(namespace, name, value) - return getattr(namespace, name) - - -# =============== -# Formatting Help -# =============== - -class HelpFormatter(object): - """Formatter for generating usage messages and argument help strings. - - Only the name of this class is considered a public API. All the methods - provided by the class are considered an implementation detail. - """ - - def __init__(self, - prog, - indent_increment=2, - max_help_position=24, - width=None): - - # default setting for width - if width is None: - try: - width = int(_os.environ['COLUMNS']) - except (KeyError, ValueError): - width = 80 - width -= 2 - - self._prog = prog - self._indent_increment = indent_increment - self._max_help_position = max_help_position - self._width = width - - self._current_indent = 0 - self._level = 0 - self._action_max_length = 0 - - self._root_section = self._Section(self, None) - self._current_section = self._root_section - - self._whitespace_matcher = _re.compile(r'\s+') - self._long_break_matcher = _re.compile(r'\n\n\n+') - - # =============================== - # Section and indentation methods - # =============================== - def _indent(self): - self._current_indent += self._indent_increment - self._level += 1 - - def _dedent(self): - self._current_indent -= self._indent_increment - assert self._current_indent >= 0, 'Indent decreased below 0.' - self._level -= 1 - - class _Section(object): - - def __init__(self, formatter, parent, heading=None): - self.formatter = formatter - self.parent = parent - self.heading = heading - self.items = [] - - def format_help(self): - # format the indented section - if self.parent is not None: - self.formatter._indent() - join = self.formatter._join_parts - for func, args in self.items: - func(*args) - item_help = join([func(*args) for func, args in self.items]) - if self.parent is not None: - self.formatter._dedent() - - # return nothing if the section was empty - if not item_help: - return '' - - # add the heading if the section was non-empty - if self.heading is not SUPPRESS and self.heading is not None: - current_indent = self.formatter._current_indent - heading = '%*s%s:\n' % (current_indent, '', self.heading) - else: - heading = '' - - # join the section-initial newline, the heading and the help - return join(['\n', heading, item_help, '\n']) - - def _add_item(self, func, args): - self._current_section.items.append((func, args)) - - # ======================== - # Message building methods - # ======================== - def start_section(self, heading): - self._indent() - section = self._Section(self, self._current_section, heading) - self._add_item(section.format_help, []) - self._current_section = section - - def end_section(self): - self._current_section = self._current_section.parent - self._dedent() - - def add_text(self, text): - if text is not SUPPRESS and text is not None: - self._add_item(self._format_text, [text]) - - def add_usage(self, usage, actions, groups, prefix=None): - if usage is not SUPPRESS: - args = usage, actions, groups, prefix - self._add_item(self._format_usage, args) - - def add_argument(self, action): - if action.help is not SUPPRESS: - - # find all invocations - get_invocation = self._format_action_invocation - invocations = [get_invocation(action)] - for subaction in self._iter_indented_subactions(action): - invocations.append(get_invocation(subaction)) - - # update the maximum item length - invocation_length = max([len(s) for s in invocations]) - action_length = invocation_length + self._current_indent - self._action_max_length = max(self._action_max_length, - action_length) - - # add the item to the list - self._add_item(self._format_action, [action]) - - def add_arguments(self, actions): - for action in actions: - self.add_argument(action) - - # ======================= - # Help-formatting methods - # ======================= - def format_help(self): - help = self._root_section.format_help() - if help: - help = self._long_break_matcher.sub('\n\n', help) - help = help.strip('\n') + '\n' - return help - - def _join_parts(self, part_strings): - return ''.join([part - for part in part_strings - if part and part is not SUPPRESS]) - - def _format_usage(self, usage, actions, groups, prefix): - if prefix is None: - prefix = _('usage: ') - - # if usage is specified, use that - if usage is not None: - usage = usage % dict(prog=self._prog) - - # if no optionals or positionals are available, usage is just prog - elif usage is None and not actions: - usage = '%(prog)s' % dict(prog=self._prog) - - # if optionals and positionals are available, calculate usage - elif usage is None: - prog = '%(prog)s' % dict(prog=self._prog) - - # split optionals from positionals - optionals = [] - positionals = [] - for action in actions: - if action.option_strings: - optionals.append(action) - else: - positionals.append(action) - - # build full usage string - format = self._format_actions_usage - action_usage = format(optionals + positionals, groups) - usage = ' '.join([s for s in [prog, action_usage] if s]) - - # wrap the usage parts if it's too long - text_width = self._width - self._current_indent - if len(prefix) + len(usage) > text_width: - - # break usage into wrappable parts - part_regexp = r'\(.*?\)+|\[.*?\]+|\S+' - opt_usage = format(optionals, groups) - pos_usage = format(positionals, groups) - opt_parts = _re.findall(part_regexp, opt_usage) - pos_parts = _re.findall(part_regexp, pos_usage) - assert ' '.join(opt_parts) == opt_usage - assert ' '.join(pos_parts) == pos_usage - - # helper for wrapping lines - def get_lines(parts, indent, prefix=None): - lines = [] - line = [] - if prefix is not None: - line_len = len(prefix) - 1 - else: - line_len = len(indent) - 1 - for part in parts: - if line_len + 1 + len(part) > text_width: - lines.append(indent + ' '.join(line)) - line = [] - line_len = len(indent) - 1 - line.append(part) - line_len += len(part) + 1 - if line: - lines.append(indent + ' '.join(line)) - if prefix is not None: - lines[0] = lines[0][len(indent):] - return lines - - # if prog is short, follow it with optionals or positionals - if len(prefix) + len(prog) <= 0.75 * text_width: - indent = ' ' * (len(prefix) + len(prog) + 1) - if opt_parts: - lines = get_lines([prog] + opt_parts, indent, prefix) - lines.extend(get_lines(pos_parts, indent)) - elif pos_parts: - lines = get_lines([prog] + pos_parts, indent, prefix) - else: - lines = [prog] - - # if prog is long, put it on its own line - else: - indent = ' ' * len(prefix) - parts = opt_parts + pos_parts - lines = get_lines(parts, indent) - if len(lines) > 1: - lines = [] - lines.extend(get_lines(opt_parts, indent)) - lines.extend(get_lines(pos_parts, indent)) - lines = [prog] + lines - - # join lines into usage - usage = '\n'.join(lines) - - # prefix with 'usage:' - return '%s%s\n\n' % (prefix, usage) - - def _format_actions_usage(self, actions, groups): - # find group indices and identify actions in groups - group_actions = _set() - inserts = {} - for group in groups: - try: - start = actions.index(group._group_actions[0]) - except ValueError: - continue - else: - end = start + len(group._group_actions) - if actions[start:end] == group._group_actions: - for action in group._group_actions: - group_actions.add(action) - if not group.required: - inserts[start] = '[' - inserts[end] = ']' - else: - inserts[start] = '(' - inserts[end] = ')' - for i in range(start + 1, end): - inserts[i] = '|' - - # collect all actions format strings - parts = [] - for i, action in enumerate(actions): - - # suppressed arguments are marked with None - # remove | separators for suppressed arguments - if action.help is SUPPRESS: - parts.append(None) - if inserts.get(i) == '|': - inserts.pop(i) - elif inserts.get(i + 1) == '|': - inserts.pop(i + 1) - - # produce all arg strings - elif not action.option_strings: - part = self._format_args(action, action.dest) - - # if it's in a group, strip the outer [] - if action in group_actions: - if part[0] == '[' and part[-1] == ']': - part = part[1:-1] - - # add the action string to the list - parts.append(part) - - # produce the first way to invoke the option in brackets - else: - option_string = action.option_strings[0] - - # if the Optional doesn't take a value, format is: - # -s or --long - if action.nargs == 0: - part = '%s' % option_string - - # if the Optional takes a value, format is: - # -s ARGS or --long ARGS - else: - default = action.dest.upper() - args_string = self._format_args(action, default) - part = '%s %s' % (option_string, args_string) - - # make it look optional if it's not required or in a group - if not action.required and action not in group_actions: - part = '[%s]' % part - - # add the action string to the list - parts.append(part) - - # insert things at the necessary indices - for i in _sorted(inserts, reverse=True): - parts[i:i] = [inserts[i]] - - # join all the action items with spaces - text = ' '.join([item for item in parts if item is not None]) - - # clean up separators for mutually exclusive groups - open = r'[\[(]' - close = r'[\])]' - text = _re.sub(r'(%s) ' % open, r'\1', text) - text = _re.sub(r' (%s)' % close, r'\1', text) - text = _re.sub(r'%s *%s' % (open, close), r'', text) - text = _re.sub(r'\(([^|]*)\)', r'\1', text) - text = text.strip() - - # return the text - return text - - def _format_text(self, text): - if '%(prog)' in text: - text = text % dict(prog=self._prog) - text_width = self._width - self._current_indent - indent = ' ' * self._current_indent - return self._fill_text(text, text_width, indent) + '\n\n' - - def _format_action(self, action): - # determine the required width and the entry label - help_position = min(self._action_max_length + 2, - self._max_help_position) - help_width = self._width - help_position - action_width = help_position - self._current_indent - 2 - action_header = self._format_action_invocation(action) - - # ho nelp; start on same line and add a final newline - if not action.help: - tup = self._current_indent, '', action_header - action_header = '%*s%s\n' % tup - - # short action name; start on the same line and pad two spaces - elif len(action_header) <= action_width: - tup = self._current_indent, '', action_width, action_header - action_header = '%*s%-*s ' % tup - indent_first = 0 - - # long action name; start on the next line - else: - tup = self._current_indent, '', action_header - action_header = '%*s%s\n' % tup - indent_first = help_position - - # collect the pieces of the action help - parts = [action_header] - - # if there was help for the action, add lines of help text - if action.help: - help_text = self._expand_help(action) - help_lines = self._split_lines(help_text, help_width) - parts.append('%*s%s\n' % (indent_first, '', help_lines[0])) - for line in help_lines[1:]: - parts.append('%*s%s\n' % (help_position, '', line)) - - # or add a newline if the description doesn't end with one - elif not action_header.endswith('\n'): - parts.append('\n') - - # if there are any sub-actions, add their help as well - for subaction in self._iter_indented_subactions(action): - parts.append(self._format_action(subaction)) - - # return a single string - return self._join_parts(parts) - - def _format_action_invocation(self, action): - if not action.option_strings: - metavar, = self._metavar_formatter(action, action.dest)(1) - return metavar - - else: - parts = [] - - # if the Optional doesn't take a value, format is: - # -s, --long - if action.nargs == 0: - parts.extend(action.option_strings) - - # if the Optional takes a value, format is: - # -s ARGS, --long ARGS - else: - default = action.dest.upper() - args_string = self._format_args(action, default) - for option_string in action.option_strings: - parts.append('%s %s' % (option_string, args_string)) - - return ', '.join(parts) - - def _metavar_formatter(self, action, default_metavar): - if action.metavar is not None: - result = action.metavar - elif action.choices is not None: - choice_strs = [str(choice) for choice in action.choices] - result = '{%s}' % ','.join(choice_strs) - else: - result = default_metavar - - def format(tuple_size): - if isinstance(result, tuple): - return result - else: - return (result, ) * tuple_size - return format - - def _format_args(self, action, default_metavar): - get_metavar = self._metavar_formatter(action, default_metavar) - if action.nargs is None: - result = '%s' % get_metavar(1) - elif action.nargs == OPTIONAL: - result = '[%s]' % get_metavar(1) - elif action.nargs == ZERO_OR_MORE: - result = '[%s [%s ...]]' % get_metavar(2) - elif action.nargs == ONE_OR_MORE: - result = '%s [%s ...]' % get_metavar(2) - elif action.nargs == REMAINDER: - result = '...' - elif action.nargs == PARSER: - result = '%s ...' % get_metavar(1) - else: - formats = ['%s' for _ in range(action.nargs)] - result = ' '.join(formats) % get_metavar(action.nargs) - return result - - def _expand_help(self, action): - params = dict(vars(action), prog=self._prog) - for name in list(params): - if params[name] is SUPPRESS: - del params[name] - for name in list(params): - if hasattr(params[name], '__name__'): - params[name] = params[name].__name__ - if params.get('choices') is not None: - choices_str = ', '.join([str(c) for c in params['choices']]) - params['choices'] = choices_str - return self._get_help_string(action) % params - - def _iter_indented_subactions(self, action): - try: - get_subactions = action._get_subactions - except AttributeError: - pass - else: - self._indent() - for subaction in get_subactions(): - yield subaction - self._dedent() - - def _split_lines(self, text, width): - text = self._whitespace_matcher.sub(' ', text).strip() - return _textwrap.wrap(text, width) - - def _fill_text(self, text, width, indent): - text = self._whitespace_matcher.sub(' ', text).strip() - return _textwrap.fill(text, width, initial_indent=indent, - subsequent_indent=indent) - - def _get_help_string(self, action): - return action.help - - -class RawDescriptionHelpFormatter(HelpFormatter): - """Help message formatter which retains any formatting in descriptions. - - Only the name of this class is considered a public API. All the methods - provided by the class are considered an implementation detail. - """ - - def _fill_text(self, text, width, indent): - return ''.join([indent + line for line in text.splitlines(True)]) - - -class RawTextHelpFormatter(RawDescriptionHelpFormatter): - """Help message formatter which retains formatting of all help text. - - Only the name of this class is considered a public API. All the methods - provided by the class are considered an implementation detail. - """ - - def _split_lines(self, text, width): - return text.splitlines() - - -class ArgumentDefaultsHelpFormatter(HelpFormatter): - """Help message formatter which adds default values to argument help. - - Only the name of this class is considered a public API. All the methods - provided by the class are considered an implementation detail. - """ - - def _get_help_string(self, action): - help = action.help - if '%(default)' not in action.help: - if action.default is not SUPPRESS: - defaulting_nargs = [OPTIONAL, ZERO_OR_MORE] - if action.option_strings or action.nargs in defaulting_nargs: - help += ' (default: %(default)s)' - return help - - -# ===================== -# Options and Arguments -# ===================== - -def _get_action_name(argument): - if argument is None: - return None - elif argument.option_strings: - return '/'.join(argument.option_strings) - elif argument.metavar not in (None, SUPPRESS): - return argument.metavar - elif argument.dest not in (None, SUPPRESS): - return argument.dest - else: - return None - - -class ArgumentError(Exception): - """An error from creating or using an argument (optional or positional). - - The string value of this exception is the message, augmented with - information about the argument that caused it. - """ - - def __init__(self, argument, message): - self.argument_name = _get_action_name(argument) - self.message = message - - def __str__(self): - if self.argument_name is None: - format = '%(message)s' - else: - format = 'argument %(argument_name)s: %(message)s' - return format % dict(message=self.message, - argument_name=self.argument_name) - - -class ArgumentTypeError(Exception): - """An error from trying to convert a command line string to a type.""" - pass - - -# ============== -# Action classes -# ============== - -class Action(_AttributeHolder): - """Information about how to convert command line strings to Python objects. - - Action objects are used by an ArgumentParser to represent the information - needed to parse a single argument from one or more strings from the - command line. The keyword arguments to the Action constructor are also - all attributes of Action instances. - - Keyword Arguments: - - - option_strings -- A list of command-line option strings which - should be associated with this action. - - - dest -- The name of the attribute to hold the created object(s) - - - nargs -- The number of command-line arguments that should be - consumed. By default, one argument will be consumed and a single - value will be produced. Other values include: - - N (an integer) consumes N arguments (and produces a list) - - '?' consumes zero or one arguments - - '*' consumes zero or more arguments (and produces a list) - - '+' consumes one or more arguments (and produces a list) - Note that the difference between the default and nargs=1 is that - with the default, a single value will be produced, while with - nargs=1, a list containing a single value will be produced. - - - const -- The value to be produced if the option is specified and the - option uses an action that takes no values. - - - default -- The value to be produced if the option is not specified. - - - type -- The type which the command-line arguments should be converted - to, should be one of 'string', 'int', 'float', 'complex' or a - callable object that accepts a single string argument. If None, - 'string' is assumed. - - - choices -- A container of values that should be allowed. If not None, - after a command-line argument has been converted to the appropriate - type, an exception will be raised if it is not a member of this - collection. - - - required -- True if the action must always be specified at the - command line. This is only meaningful for optional command-line - arguments. - - - help -- The help string describing the argument. - - - metavar -- The name to be used for the option's argument with the - help string. If None, the 'dest' value will be used as the name. - """ - - def __init__(self, - option_strings, - dest, - nargs=None, - const=None, - default=None, - type=None, - choices=None, - required=False, - help=None, - metavar=None): - self.option_strings = option_strings - self.dest = dest - self.nargs = nargs - self.const = const - self.default = default - self.type = type - self.choices = choices - self.required = required - self.help = help - self.metavar = metavar - - def _get_kwargs(self): - names = [ - 'option_strings', - 'dest', - 'nargs', - 'const', - 'default', - 'type', - 'choices', - 'help', - 'metavar', - ] - return [(name, getattr(self, name)) for name in names] - - def __call__(self, parser, namespace, values, option_string=None): - raise NotImplementedError(_('.__call__() not defined')) - - -class _StoreAction(Action): - - def __init__(self, - option_strings, - dest, - nargs=None, - const=None, - default=None, - type=None, - choices=None, - required=False, - help=None, - metavar=None): - if nargs == 0: - raise ValueError('nargs for store actions must be > 0; if you ' - 'have nothing to store, actions such as store ' - 'true or store const may be more appropriate') - if const is not None and nargs != OPTIONAL: - raise ValueError('nargs must be %r to supply const' % OPTIONAL) - super(_StoreAction, self).__init__( - option_strings=option_strings, - dest=dest, - nargs=nargs, - const=const, - default=default, - type=type, - choices=choices, - required=required, - help=help, - metavar=metavar) - - def __call__(self, parser, namespace, values, option_string=None): - setattr(namespace, self.dest, values) - - -class _StoreConstAction(Action): - - def __init__(self, - option_strings, - dest, - const, - default=None, - required=False, - help=None, - metavar=None): - super(_StoreConstAction, self).__init__( - option_strings=option_strings, - dest=dest, - nargs=0, - const=const, - default=default, - required=required, - help=help) - - def __call__(self, parser, namespace, values, option_string=None): - setattr(namespace, self.dest, self.const) - - -class _StoreTrueAction(_StoreConstAction): - - def __init__(self, - option_strings, - dest, - default=False, - required=False, - help=None): - super(_StoreTrueAction, self).__init__( - option_strings=option_strings, - dest=dest, - const=True, - default=default, - required=required, - help=help) - - -class _StoreFalseAction(_StoreConstAction): - - def __init__(self, - option_strings, - dest, - default=True, - required=False, - help=None): - super(_StoreFalseAction, self).__init__( - option_strings=option_strings, - dest=dest, - const=False, - default=default, - required=required, - help=help) - - -class _AppendAction(Action): - - def __init__(self, - option_strings, - dest, - nargs=None, - const=None, - default=None, - type=None, - choices=None, - required=False, - help=None, - metavar=None): - if nargs == 0: - raise ValueError('nargs for append actions must be > 0; if arg ' - 'strings are not supplying the value to append, ' - 'the append const action may be more appropriate') - if const is not None and nargs != OPTIONAL: - raise ValueError('nargs must be %r to supply const' % OPTIONAL) - super(_AppendAction, self).__init__( - option_strings=option_strings, - dest=dest, - nargs=nargs, - const=const, - default=default, - type=type, - choices=choices, - required=required, - help=help, - metavar=metavar) - - def __call__(self, parser, namespace, values, option_string=None): - items = _copy.copy(_ensure_value(namespace, self.dest, [])) - items.append(values) - setattr(namespace, self.dest, items) - - -class _AppendConstAction(Action): - - def __init__(self, - option_strings, - dest, - const, - default=None, - required=False, - help=None, - metavar=None): - super(_AppendConstAction, self).__init__( - option_strings=option_strings, - dest=dest, - nargs=0, - const=const, - default=default, - required=required, - help=help, - metavar=metavar) - - def __call__(self, parser, namespace, values, option_string=None): - items = _copy.copy(_ensure_value(namespace, self.dest, [])) - items.append(self.const) - setattr(namespace, self.dest, items) - - -class _CountAction(Action): - - def __init__(self, - option_strings, - dest, - default=None, - required=False, - help=None): - super(_CountAction, self).__init__( - option_strings=option_strings, - dest=dest, - nargs=0, - default=default, - required=required, - help=help) - - def __call__(self, parser, namespace, values, option_string=None): - new_count = _ensure_value(namespace, self.dest, 0) + 1 - setattr(namespace, self.dest, new_count) - - -class _HelpAction(Action): - - def __init__(self, - option_strings, - dest=SUPPRESS, - default=SUPPRESS, - help=None): - super(_HelpAction, self).__init__( - option_strings=option_strings, - dest=dest, - default=default, - nargs=0, - help=help) - - def __call__(self, parser, namespace, values, option_string=None): - parser.print_help() - parser.exit() - - -class _VersionAction(Action): - - def __init__(self, - option_strings, - version=None, - dest=SUPPRESS, - default=SUPPRESS, - help=None): - super(_VersionAction, self).__init__( - option_strings=option_strings, - dest=dest, - default=default, - nargs=0, - help=help) - self.version = version - - def __call__(self, parser, namespace, values, option_string=None): - version = self.version - if version is None: - version = parser.version - formatter = parser._get_formatter() - formatter.add_text(version) - parser.exit(message=formatter.format_help()) - - -class _SubParsersAction(Action): - - class _ChoicesPseudoAction(Action): - - def __init__(self, name, help): - sup = super(_SubParsersAction._ChoicesPseudoAction, self) - sup.__init__(option_strings=[], dest=name, help=help) - - def __init__(self, - option_strings, - prog, - parser_class, - dest=SUPPRESS, - help=None, - metavar=None): - - self._prog_prefix = prog - self._parser_class = parser_class - self._name_parser_map = {} - self._choices_actions = [] - - super(_SubParsersAction, self).__init__( - option_strings=option_strings, - dest=dest, - nargs=PARSER, - choices=self._name_parser_map, - help=help, - metavar=metavar) - - def add_parser(self, name, **kwargs): - # set prog from the existing prefix - if kwargs.get('prog') is None: - kwargs['prog'] = '%s %s' % (self._prog_prefix, name) - - # create a pseudo-action to hold the choice help - if 'help' in kwargs: - help = kwargs.pop('help') - choice_action = self._ChoicesPseudoAction(name, help) - self._choices_actions.append(choice_action) - - # create the parser and add it to the map - parser = self._parser_class(**kwargs) - self._name_parser_map[name] = parser - return parser - - def _get_subactions(self): - return self._choices_actions - - def __call__(self, parser, namespace, values, option_string=None): - parser_name = values[0] - arg_strings = values[1:] - - # set the parser name if requested - if self.dest is not SUPPRESS: - setattr(namespace, self.dest, parser_name) - - # select the parser - try: - parser = self._name_parser_map[parser_name] - except KeyError: - tup = parser_name, ', '.join(self._name_parser_map) - msg = _('unknown parser %r (choices: %s)' % tup) - raise ArgumentError(self, msg) - - # parse all the remaining options into the namespace - parser.parse_args(arg_strings, namespace) - - -# ============== -# Type classes -# ============== - -class FileType(object): - """Factory for creating file object types - - Instances of FileType are typically passed as type= arguments to the - ArgumentParser add_argument() method. - - Keyword Arguments: - - mode -- A string indicating how the file is to be opened. Accepts the - same values as the builtin open() function. - - bufsize -- The file's desired buffer size. Accepts the same values as - the builtin open() function. - """ - - def __init__(self, mode='r', bufsize=None): - self._mode = mode - self._bufsize = bufsize - - def __call__(self, string): - # the special argument "-" means sys.std{in,out} - if string == '-': - if 'r' in self._mode: - return _sys.stdin - elif 'w' in self._mode: - return _sys.stdout - else: - msg = _('argument "-" with mode %r' % self._mode) - raise ValueError(msg) - - # all other arguments are used as file names - if self._bufsize: - return open(string, self._mode, self._bufsize) - else: - return open(string, self._mode) - - def __repr__(self): - args = [self._mode, self._bufsize] - args_str = ', '.join([repr(arg) for arg in args if arg is not None]) - return '%s(%s)' % (type(self).__name__, args_str) - -# =========================== -# Optional and Positional Parsing -# =========================== - -class Namespace(_AttributeHolder): - """Simple object for storing attributes. - - Implements equality by attribute names and values, and provides a simple - string representation. - """ - - def __init__(self, **kwargs): - for name in kwargs: - setattr(self, name, kwargs[name]) - - def __eq__(self, other): - return vars(self) == vars(other) - - def __ne__(self, other): - return not (self == other) - - def __contains__(self, key): - return key in self.__dict__ - - -class _ActionsContainer(object): - - def __init__(self, - description, - prefix_chars, - argument_default, - conflict_handler): - super(_ActionsContainer, self).__init__() - - self.description = description - self.argument_default = argument_default - self.prefix_chars = prefix_chars - self.conflict_handler = conflict_handler - - # set up registries - self._registries = {} - - # register actions - self.register('action', None, _StoreAction) - self.register('action', 'store', _StoreAction) - self.register('action', 'store_const', _StoreConstAction) - self.register('action', 'store_true', _StoreTrueAction) - self.register('action', 'store_false', _StoreFalseAction) - self.register('action', 'append', _AppendAction) - self.register('action', 'append_const', _AppendConstAction) - self.register('action', 'count', _CountAction) - self.register('action', 'help', _HelpAction) - self.register('action', 'version', _VersionAction) - self.register('action', 'parsers', _SubParsersAction) - - # raise an exception if the conflict handler is invalid - self._get_handler() - - # action storage - self._actions = [] - self._option_string_actions = {} - - # groups - self._action_groups = [] - self._mutually_exclusive_groups = [] - - # defaults storage - self._defaults = {} - - # determines whether an "option" looks like a negative number - self._negative_number_matcher = _re.compile(r'^-\d+$|^-\d*\.\d+$') - - # whether or not there are any optionals that look like negative - # numbers -- uses a list so it can be shared and edited - self._has_negative_number_optionals = [] - - # ==================== - # Registration methods - # ==================== - def register(self, registry_name, value, object): - registry = self._registries.setdefault(registry_name, {}) - registry[value] = object - - def _registry_get(self, registry_name, value, default=None): - return self._registries[registry_name].get(value, default) - - # ================================== - # Namespace default accessor methods - # ================================== - def set_defaults(self, **kwargs): - self._defaults.update(kwargs) - - # if these defaults match any existing arguments, replace - # the previous default on the object with the new one - for action in self._actions: - if action.dest in kwargs: - action.default = kwargs[action.dest] - - def get_default(self, dest): - for action in self._actions: - if action.dest == dest and action.default is not None: - return action.default - return self._defaults.get(dest, None) - - - # ======================= - # Adding argument actions - # ======================= - def add_argument(self, *args, **kwargs): - """ - add_argument(dest, ..., name=value, ...) - add_argument(option_string, option_string, ..., name=value, ...) - """ - - # if no positional args are supplied or only one is supplied and - # it doesn't look like an option string, parse a positional - # argument - chars = self.prefix_chars - if not args or len(args) == 1 and args[0][0] not in chars: - if args and 'dest' in kwargs: - raise ValueError('dest supplied twice for positional argument') - kwargs = self._get_positional_kwargs(*args, **kwargs) - - # otherwise, we're adding an optional argument - else: - kwargs = self._get_optional_kwargs(*args, **kwargs) - - # if no default was supplied, use the parser-level default - if 'default' not in kwargs: - dest = kwargs['dest'] - if dest in self._defaults: - kwargs['default'] = self._defaults[dest] - elif self.argument_default is not None: - kwargs['default'] = self.argument_default - - # create the action object, and add it to the parser - action_class = self._pop_action_class(kwargs) - if not _callable(action_class): - raise ValueError('unknown action "%s"' % action_class) - action = action_class(**kwargs) - - # raise an error if the action type is not callable - type_func = self._registry_get('type', action.type, action.type) - if not _callable(type_func): - raise ValueError('%r is not callable' % type_func) - - return self._add_action(action) - - def add_argument_group(self, *args, **kwargs): - group = _ArgumentGroup(self, *args, **kwargs) - self._action_groups.append(group) - return group - - def add_mutually_exclusive_group(self, **kwargs): - group = _MutuallyExclusiveGroup(self, **kwargs) - self._mutually_exclusive_groups.append(group) - return group - - def _add_action(self, action): - # resolve any conflicts - self._check_conflict(action) - - # add to actions list - self._actions.append(action) - action.container = self - - # index the action by any option strings it has - for option_string in action.option_strings: - self._option_string_actions[option_string] = action - - # set the flag if any option strings look like negative numbers - for option_string in action.option_strings: - if self._negative_number_matcher.match(option_string): - if not self._has_negative_number_optionals: - self._has_negative_number_optionals.append(True) - - # return the created action - return action - - def _remove_action(self, action): - self._actions.remove(action) - - def _add_container_actions(self, container): - # collect groups by titles - title_group_map = {} - for group in self._action_groups: - if group.title in title_group_map: - msg = _('cannot merge actions - two groups are named %r') - raise ValueError(msg % (group.title)) - title_group_map[group.title] = group - - # map each action to its group - group_map = {} - for group in container._action_groups: - - # if a group with the title exists, use that, otherwise - # create a new group matching the container's group - if group.title not in title_group_map: - title_group_map[group.title] = self.add_argument_group( - title=group.title, - description=group.description, - conflict_handler=group.conflict_handler) - - # map the actions to their new group - for action in group._group_actions: - group_map[action] = title_group_map[group.title] - - # add container's mutually exclusive groups - # NOTE: if add_mutually_exclusive_group ever gains title= and - # description= then this code will need to be expanded as above - for group in container._mutually_exclusive_groups: - mutex_group = self.add_mutually_exclusive_group( - required=group.required) - - # map the actions to their new mutex group - for action in group._group_actions: - group_map[action] = mutex_group - - # add all actions to this container or their group - for action in container._actions: - group_map.get(action, self)._add_action(action) - - def _get_positional_kwargs(self, dest, **kwargs): - # make sure required is not specified - if 'required' in kwargs: - msg = _("'required' is an invalid argument for positionals") - raise TypeError(msg) - - # mark positional arguments as required if at least one is - # always required - if kwargs.get('nargs') not in [OPTIONAL, ZERO_OR_MORE]: - kwargs['required'] = True - if kwargs.get('nargs') == ZERO_OR_MORE and 'default' not in kwargs: - kwargs['required'] = True - - # return the keyword arguments with no option strings - return dict(kwargs, dest=dest, option_strings=[]) - - def _get_optional_kwargs(self, *args, **kwargs): - # determine short and long option strings - option_strings = [] - long_option_strings = [] - for option_string in args: - # error on strings that don't start with an appropriate prefix - if not option_string[0] in self.prefix_chars: - msg = _('invalid option string %r: ' - 'must start with a character %r') - tup = option_string, self.prefix_chars - raise ValueError(msg % tup) - - # strings starting with two prefix characters are long options - option_strings.append(option_string) - if option_string[0] in self.prefix_chars: - if len(option_string) > 1: - if option_string[1] in self.prefix_chars: - long_option_strings.append(option_string) - - # infer destination, '--foo-bar' -> 'foo_bar' and '-x' -> 'x' - dest = kwargs.pop('dest', None) - if dest is None: - if long_option_strings: - dest_option_string = long_option_strings[0] - else: - dest_option_string = option_strings[0] - dest = dest_option_string.lstrip(self.prefix_chars) - if not dest: - msg = _('dest= is required for options like %r') - raise ValueError(msg % option_string) - dest = dest.replace('-', '_') - - # return the updated keyword arguments - return dict(kwargs, dest=dest, option_strings=option_strings) - - def _pop_action_class(self, kwargs, default=None): - action = kwargs.pop('action', default) - return self._registry_get('action', action, action) - - def _get_handler(self): - # determine function from conflict handler string - handler_func_name = '_handle_conflict_%s' % self.conflict_handler - try: - return getattr(self, handler_func_name) - except AttributeError: - msg = _('invalid conflict_resolution value: %r') - raise ValueError(msg % self.conflict_handler) - - def _check_conflict(self, action): - - # find all options that conflict with this option - confl_optionals = [] - for option_string in action.option_strings: - if option_string in self._option_string_actions: - confl_optional = self._option_string_actions[option_string] - confl_optionals.append((option_string, confl_optional)) - - # resolve any conflicts - if confl_optionals: - conflict_handler = self._get_handler() - conflict_handler(action, confl_optionals) - - def _handle_conflict_error(self, action, conflicting_actions): - message = _('conflicting option string(s): %s') - conflict_string = ', '.join([option_string - for option_string, action - in conflicting_actions]) - raise ArgumentError(action, message % conflict_string) - - def _handle_conflict_resolve(self, action, conflicting_actions): - - # remove all conflicting options - for option_string, action in conflicting_actions: - - # remove the conflicting option - action.option_strings.remove(option_string) - self._option_string_actions.pop(option_string, None) - - # if the option now has no option string, remove it from the - # container holding it - if not action.option_strings: - action.container._remove_action(action) - - -class _ArgumentGroup(_ActionsContainer): - - def __init__(self, container, title=None, description=None, **kwargs): - # add any missing keyword arguments by checking the container - update = kwargs.setdefault - update('conflict_handler', container.conflict_handler) - update('prefix_chars', container.prefix_chars) - update('argument_default', container.argument_default) - super_init = super(_ArgumentGroup, self).__init__ - super_init(description=description, **kwargs) - - # group attributes - self.title = title - self._group_actions = [] - - # share most attributes with the container - self._registries = container._registries - self._actions = container._actions - self._option_string_actions = container._option_string_actions - self._defaults = container._defaults - self._has_negative_number_optionals = \ - container._has_negative_number_optionals - - def _add_action(self, action): - action = super(_ArgumentGroup, self)._add_action(action) - self._group_actions.append(action) - return action - - def _remove_action(self, action): - super(_ArgumentGroup, self)._remove_action(action) - self._group_actions.remove(action) - - -class _MutuallyExclusiveGroup(_ArgumentGroup): - - def __init__(self, container, required=False): - super(_MutuallyExclusiveGroup, self).__init__(container) - self.required = required - self._container = container - - def _add_action(self, action): - if action.required: - msg = _('mutually exclusive arguments must be optional') - raise ValueError(msg) - action = self._container._add_action(action) - self._group_actions.append(action) - return action - - def _remove_action(self, action): - self._container._remove_action(action) - self._group_actions.remove(action) - - -class ArgumentParser(_AttributeHolder, _ActionsContainer): - """Object for parsing command line strings into Python objects. - - Keyword Arguments: - - prog -- The name of the program (default: sys.argv[0]) - - usage -- A usage message (default: auto-generated from arguments) - - description -- A description of what the program does - - epilog -- Text following the argument descriptions - - parents -- Parsers whose arguments should be copied into this one - - formatter_class -- HelpFormatter class for printing help messages - - prefix_chars -- Characters that prefix optional arguments - - fromfile_prefix_chars -- Characters that prefix files containing - additional arguments - - argument_default -- The default value for all arguments - - conflict_handler -- String indicating how to handle conflicts - - add_help -- Add a -h/-help option - """ - - def __init__(self, - prog=None, - usage=None, - description=None, - epilog=None, - version=None, - parents=[], - formatter_class=HelpFormatter, - prefix_chars='-', - fromfile_prefix_chars=None, - argument_default=None, - conflict_handler='error', - add_help=True): - - if version is not None: - import warnings - warnings.warn( - """The "version" argument to ArgumentParser is deprecated. """ - """Please use """ - """"add_argument(..., action='version', version="N", ...)" """ - """instead""", DeprecationWarning) - - superinit = super(ArgumentParser, self).__init__ - superinit(description=description, - prefix_chars=prefix_chars, - argument_default=argument_default, - conflict_handler=conflict_handler) - - # default setting for prog - if prog is None: - prog = _os.path.basename(_sys.argv[0]) - - self.prog = prog - self.usage = usage - self.epilog = epilog - self.version = version - self.formatter_class = formatter_class - self.fromfile_prefix_chars = fromfile_prefix_chars - self.add_help = add_help - - add_group = self.add_argument_group - self._positionals = add_group(_('positional arguments')) - self._optionals = add_group(_('optional arguments')) - self._subparsers = None - - # register types - def identity(string): - return string - self.register('type', None, identity) - - # add help and version arguments if necessary - # (using explicit default to override global argument_default) - if self.add_help: - self.add_argument( - '-h', '--help', action='help', default=SUPPRESS, - help=_('show this help message and exit')) - if self.version: - self.add_argument( - '-v', '--version', action='version', default=SUPPRESS, - version=self.version, - help=_("show program's version number and exit")) - - # add parent arguments and defaults - for parent in parents: - self._add_container_actions(parent) - try: - defaults = parent._defaults - except AttributeError: - pass - else: - self._defaults.update(defaults) - - # ======================= - # Pretty __repr__ methods - # ======================= - def _get_kwargs(self): - names = [ - 'prog', - 'usage', - 'description', - 'version', - 'formatter_class', - 'conflict_handler', - 'add_help', - ] - return [(name, getattr(self, name)) for name in names] - - # ================================== - # Optional/Positional adding methods - # ================================== - def add_subparsers(self, **kwargs): - if self._subparsers is not None: - self.error(_('cannot have multiple subparser arguments')) - - # add the parser class to the arguments if it's not present - kwargs.setdefault('parser_class', type(self)) - - if 'title' in kwargs or 'description' in kwargs: - title = _(kwargs.pop('title', 'subcommands')) - description = _(kwargs.pop('description', None)) - self._subparsers = self.add_argument_group(title, description) - else: - self._subparsers = self._positionals - - # prog defaults to the usage message of this parser, skipping - # optional arguments and with no "usage:" prefix - if kwargs.get('prog') is None: - formatter = self._get_formatter() - positionals = self._get_positional_actions() - groups = self._mutually_exclusive_groups - formatter.add_usage(self.usage, positionals, groups, '') - kwargs['prog'] = formatter.format_help().strip() - - # create the parsers action and add it to the positionals list - parsers_class = self._pop_action_class(kwargs, 'parsers') - action = parsers_class(option_strings=[], **kwargs) - self._subparsers._add_action(action) - - # return the created parsers action - return action - - def _add_action(self, action): - if action.option_strings: - self._optionals._add_action(action) - else: - self._positionals._add_action(action) - return action - - def _get_optional_actions(self): - return [action - for action in self._actions - if action.option_strings] - - def _get_positional_actions(self): - return [action - for action in self._actions - if not action.option_strings] - - # ===================================== - # Command line argument parsing methods - # ===================================== - def parse_args(self, args=None, namespace=None): - args, argv = self.parse_known_args(args, namespace) - if argv: - msg = _('unrecognized arguments: %s') - self.error(msg % ' '.join(argv)) - return args - - def parse_known_args(self, args=None, namespace=None): - # args default to the system args - if args is None: - args = _sys.argv[1:] - - # default Namespace built from parser defaults - if namespace is None: - namespace = Namespace() - - # add any action defaults that aren't present - for action in self._actions: - if action.dest is not SUPPRESS: - if not hasattr(namespace, action.dest): - if action.default is not SUPPRESS: - default = action.default - if isinstance(action.default, _basestring): - default = self._get_value(action, default) - setattr(namespace, action.dest, default) - - # add any parser defaults that aren't present - for dest in self._defaults: - if not hasattr(namespace, dest): - setattr(namespace, dest, self._defaults[dest]) - - # parse the arguments and exit if there are any errors - try: - return self._parse_known_args(args, namespace) - except ArgumentError: - err = _sys.exc_info()[1] - self.error(str(err)) - - def _parse_known_args(self, arg_strings, namespace): - # replace arg strings that are file references - if self.fromfile_prefix_chars is not None: - arg_strings = self._read_args_from_files(arg_strings) - - # map all mutually exclusive arguments to the other arguments - # they can't occur with - action_conflicts = {} - for mutex_group in self._mutually_exclusive_groups: - group_actions = mutex_group._group_actions - for i, mutex_action in enumerate(mutex_group._group_actions): - conflicts = action_conflicts.setdefault(mutex_action, []) - conflicts.extend(group_actions[:i]) - conflicts.extend(group_actions[i + 1:]) - - # find all option indices, and determine the arg_string_pattern - # which has an 'O' if there is an option at an index, - # an 'A' if there is an argument, or a '-' if there is a '--' - option_string_indices = {} - arg_string_pattern_parts = [] - arg_strings_iter = iter(arg_strings) - for i, arg_string in enumerate(arg_strings_iter): - - # all args after -- are non-options - if arg_string == '--': - arg_string_pattern_parts.append('-') - for arg_string in arg_strings_iter: - arg_string_pattern_parts.append('A') - - # otherwise, add the arg to the arg strings - # and note the index if it was an option - else: - option_tuple = self._parse_optional(arg_string) - if option_tuple is None: - pattern = 'A' - else: - option_string_indices[i] = option_tuple - pattern = 'O' - arg_string_pattern_parts.append(pattern) - - # join the pieces together to form the pattern - arg_strings_pattern = ''.join(arg_string_pattern_parts) - - # converts arg strings to the appropriate and then takes the action - seen_actions = _set() - seen_non_default_actions = _set() - - def take_action(action, argument_strings, option_string=None): - seen_actions.add(action) - argument_values = self._get_values(action, argument_strings) - - # error if this argument is not allowed with other previously - # seen arguments, assuming that actions that use the default - # value don't really count as "present" - if argument_values is not action.default: - seen_non_default_actions.add(action) - for conflict_action in action_conflicts.get(action, []): - if conflict_action in seen_non_default_actions: - msg = _('not allowed with argument %s') - action_name = _get_action_name(conflict_action) - raise ArgumentError(action, msg % action_name) - - # take the action if we didn't receive a SUPPRESS value - # (e.g. from a default) - if argument_values is not SUPPRESS: - action(self, namespace, argument_values, option_string) - - # function to convert arg_strings into an optional action - def consume_optional(start_index): - - # get the optional identified at this index - option_tuple = option_string_indices[start_index] - action, option_string, explicit_arg = option_tuple - - # identify additional optionals in the same arg string - # (e.g. -xyz is the same as -x -y -z if no args are required) - match_argument = self._match_argument - action_tuples = [] - while True: - - # if we found no optional action, skip it - if action is None: - extras.append(arg_strings[start_index]) - return start_index + 1 - - # if there is an explicit argument, try to match the - # optional's string arguments to only this - if explicit_arg is not None: - arg_count = match_argument(action, 'A') - - # if the action is a single-dash option and takes no - # arguments, try to parse more single-dash options out - # of the tail of the option string - chars = self.prefix_chars - if arg_count == 0 and option_string[1] not in chars: - action_tuples.append((action, [], option_string)) - for char in self.prefix_chars: - option_string = char + explicit_arg[0] - explicit_arg = explicit_arg[1:] or None - optionals_map = self._option_string_actions - if option_string in optionals_map: - action = optionals_map[option_string] - break - else: - msg = _('ignored explicit argument %r') - raise ArgumentError(action, msg % explicit_arg) - - # if the action expect exactly one argument, we've - # successfully matched the option; exit the loop - elif arg_count == 1: - stop = start_index + 1 - args = [explicit_arg] - action_tuples.append((action, args, option_string)) - break - - # error if a double-dash option did not use the - # explicit argument - else: - msg = _('ignored explicit argument %r') - raise ArgumentError(action, msg % explicit_arg) - - # if there is no explicit argument, try to match the - # optional's string arguments with the following strings - # if successful, exit the loop - else: - start = start_index + 1 - selected_patterns = arg_strings_pattern[start:] - arg_count = match_argument(action, selected_patterns) - stop = start + arg_count - args = arg_strings[start:stop] - action_tuples.append((action, args, option_string)) - break - - # add the Optional to the list and return the index at which - # the Optional's string args stopped - assert action_tuples - for action, args, option_string in action_tuples: - take_action(action, args, option_string) - return stop - - # the list of Positionals left to be parsed; this is modified - # by consume_positionals() - positionals = self._get_positional_actions() - - # function to convert arg_strings into positional actions - def consume_positionals(start_index): - # match as many Positionals as possible - match_partial = self._match_arguments_partial - selected_pattern = arg_strings_pattern[start_index:] - arg_counts = match_partial(positionals, selected_pattern) - - # slice off the appropriate arg strings for each Positional - # and add the Positional and its args to the list - for action, arg_count in zip(positionals, arg_counts): - args = arg_strings[start_index: start_index + arg_count] - start_index += arg_count - take_action(action, args) - - # slice off the Positionals that we just parsed and return the - # index at which the Positionals' string args stopped - positionals[:] = positionals[len(arg_counts):] - return start_index - - # consume Positionals and Optionals alternately, until we have - # passed the last option string - extras = [] - start_index = 0 - if option_string_indices: - max_option_string_index = max(option_string_indices) - else: - max_option_string_index = -1 - while start_index <= max_option_string_index: - - # consume any Positionals preceding the next option - next_option_string_index = min([ - index - for index in option_string_indices - if index >= start_index]) - if start_index != next_option_string_index: - positionals_end_index = consume_positionals(start_index) - - # only try to parse the next optional if we didn't consume - # the option string during the positionals parsing - if positionals_end_index > start_index: - start_index = positionals_end_index - continue - else: - start_index = positionals_end_index - - # if we consumed all the positionals we could and we're not - # at the index of an option string, there were extra arguments - if start_index not in option_string_indices: - strings = arg_strings[start_index:next_option_string_index] - extras.extend(strings) - start_index = next_option_string_index - - # consume the next optional and any arguments for it - start_index = consume_optional(start_index) - - # consume any positionals following the last Optional - stop_index = consume_positionals(start_index) - - # if we didn't consume all the argument strings, there were extras - extras.extend(arg_strings[stop_index:]) - - # if we didn't use all the Positional objects, there were too few - # arg strings supplied. - if positionals: - self.error(_('too few arguments')) - - # make sure all required actions were present - for action in self._actions: - if action.required: - if action not in seen_actions: - name = _get_action_name(action) - self.error(_('argument %s is required') % name) - - # make sure all required groups had one option present - for group in self._mutually_exclusive_groups: - if group.required: - for action in group._group_actions: - if action in seen_non_default_actions: - break - - # if no actions were used, report the error - else: - names = [_get_action_name(action) - for action in group._group_actions - if action.help is not SUPPRESS] - msg = _('one of the arguments %s is required') - self.error(msg % ' '.join(names)) - - # return the updated namespace and the extra arguments - return namespace, extras - - def _read_args_from_files(self, arg_strings): - # expand arguments referencing files - new_arg_strings = [] - for arg_string in arg_strings: - - # for regular arguments, just add them back into the list - if arg_string[0] not in self.fromfile_prefix_chars: - new_arg_strings.append(arg_string) - - # replace arguments referencing files with the file content - else: - try: - args_file = open(arg_string[1:]) - try: - arg_strings = [] - for arg_line in args_file.read().splitlines(): - for arg in self.convert_arg_line_to_args(arg_line): - arg_strings.append(arg) - arg_strings = self._read_args_from_files(arg_strings) - new_arg_strings.extend(arg_strings) - finally: - args_file.close() - except IOError: - err = _sys.exc_info()[1] - self.error(str(err)) - - # return the modified argument list - return new_arg_strings - - def convert_arg_line_to_args(self, arg_line): - return [arg_line] - - def _match_argument(self, action, arg_strings_pattern): - # match the pattern for this action to the arg strings - nargs_pattern = self._get_nargs_pattern(action) - match = _re.match(nargs_pattern, arg_strings_pattern) - - # raise an exception if we weren't able to find a match - if match is None: - nargs_errors = { - None: _('expected one argument'), - OPTIONAL: _('expected at most one argument'), - ONE_OR_MORE: _('expected at least one argument'), - } - default = _('expected %s argument(s)') % action.nargs - msg = nargs_errors.get(action.nargs, default) - raise ArgumentError(action, msg) - - # return the number of arguments matched - return len(match.group(1)) - - def _match_arguments_partial(self, actions, arg_strings_pattern): - # progressively shorten the actions list by slicing off the - # final actions until we find a match - result = [] - for i in range(len(actions), 0, -1): - actions_slice = actions[:i] - pattern = ''.join([self._get_nargs_pattern(action) - for action in actions_slice]) - match = _re.match(pattern, arg_strings_pattern) - if match is not None: - result.extend([len(string) for string in match.groups()]) - break - - # return the list of arg string counts - return result - - def _parse_optional(self, arg_string): - # if it's an empty string, it was meant to be a positional - if not arg_string: - return None - - # if it doesn't start with a prefix, it was meant to be positional - if not arg_string[0] in self.prefix_chars: - return None - - # if the option string is present in the parser, return the action - if arg_string in self._option_string_actions: - action = self._option_string_actions[arg_string] - return action, arg_string, None - - # if it's just a single character, it was meant to be positional - if len(arg_string) == 1: - return None - - # if the option string before the "=" is present, return the action - if '=' in arg_string: - option_string, explicit_arg = arg_string.split('=', 1) - if option_string in self._option_string_actions: - action = self._option_string_actions[option_string] - return action, option_string, explicit_arg - - # search through all possible prefixes of the option string - # and all actions in the parser for possible interpretations - option_tuples = self._get_option_tuples(arg_string) - - # if multiple actions match, the option string was ambiguous - if len(option_tuples) > 1: - options = ', '.join([option_string - for action, option_string, explicit_arg in option_tuples]) - tup = arg_string, options - self.error(_('ambiguous option: %s could match %s') % tup) - - # if exactly one action matched, this segmentation is good, - # so return the parsed action - elif len(option_tuples) == 1: - option_tuple, = option_tuples - return option_tuple - - # if it was not found as an option, but it looks like a negative - # number, it was meant to be positional - # unless there are negative-number-like options - if self._negative_number_matcher.match(arg_string): - if not self._has_negative_number_optionals: - return None - - # if it contains a space, it was meant to be a positional - if ' ' in arg_string: - return None - - # it was meant to be an optional but there is no such option - # in this parser (though it might be a valid option in a subparser) - return None, arg_string, None - - def _get_option_tuples(self, option_string): - result = [] - - # option strings starting with two prefix characters are only - # split at the '=' - chars = self.prefix_chars - if option_string[0] in chars and option_string[1] in chars: - if '=' in option_string: - option_prefix, explicit_arg = option_string.split('=', 1) - else: - option_prefix = option_string - explicit_arg = None - for option_string in self._option_string_actions: - if option_string.startswith(option_prefix): - action = self._option_string_actions[option_string] - tup = action, option_string, explicit_arg - result.append(tup) - - # single character options can be concatenated with their arguments - # but multiple character options always have to have their argument - # separate - elif option_string[0] in chars and option_string[1] not in chars: - option_prefix = option_string - explicit_arg = None - short_option_prefix = option_string[:2] - short_explicit_arg = option_string[2:] - - for option_string in self._option_string_actions: - if option_string == short_option_prefix: - action = self._option_string_actions[option_string] - tup = action, option_string, short_explicit_arg - result.append(tup) - elif option_string.startswith(option_prefix): - action = self._option_string_actions[option_string] - tup = action, option_string, explicit_arg - result.append(tup) - - # shouldn't ever get here - else: - self.error(_('unexpected option string: %s') % option_string) - - # return the collected option tuples - return result - - def _get_nargs_pattern(self, action): - # in all examples below, we have to allow for '--' args - # which are represented as '-' in the pattern - nargs = action.nargs - - # the default (None) is assumed to be a single argument - if nargs is None: - nargs_pattern = '(-*A-*)' - - # allow zero or one arguments - elif nargs == OPTIONAL: - nargs_pattern = '(-*A?-*)' - - # allow zero or more arguments - elif nargs == ZERO_OR_MORE: - nargs_pattern = '(-*[A-]*)' - - # allow one or more arguments - elif nargs == ONE_OR_MORE: - nargs_pattern = '(-*A[A-]*)' - - # allow any number of options or arguments - elif nargs == REMAINDER: - nargs_pattern = '([-AO]*)' - - # allow one argument followed by any number of options or arguments - elif nargs == PARSER: - nargs_pattern = '(-*A[-AO]*)' - - # all others should be integers - else: - nargs_pattern = '(-*%s-*)' % '-*'.join('A' * nargs) - - # if this is an optional action, -- is not allowed - if action.option_strings: - nargs_pattern = nargs_pattern.replace('-*', '') - nargs_pattern = nargs_pattern.replace('-', '') - - # return the pattern - return nargs_pattern - - # ======================== - # Value conversion methods - # ======================== - def _get_values(self, action, arg_strings): - # for everything but PARSER args, strip out '--' - if action.nargs not in [PARSER, REMAINDER]: - arg_strings = [s for s in arg_strings if s != '--'] - - # optional argument produces a default when not present - if not arg_strings and action.nargs == OPTIONAL: - if action.option_strings: - value = action.const - else: - value = action.default - if isinstance(value, _basestring): - value = self._get_value(action, value) - self._check_value(action, value) - - # when nargs='*' on a positional, if there were no command-line - # args, use the default if it is anything other than None - elif (not arg_strings and action.nargs == ZERO_OR_MORE and - not action.option_strings): - if action.default is not None: - value = action.default - else: - value = arg_strings - self._check_value(action, value) - - # single argument or optional argument produces a single value - elif len(arg_strings) == 1 and action.nargs in [None, OPTIONAL]: - arg_string, = arg_strings - value = self._get_value(action, arg_string) - self._check_value(action, value) - - # REMAINDER arguments convert all values, checking none - elif action.nargs == REMAINDER: - value = [self._get_value(action, v) for v in arg_strings] - - # PARSER arguments convert all values, but check only the first - elif action.nargs == PARSER: - value = [self._get_value(action, v) for v in arg_strings] - self._check_value(action, value[0]) - - # all other types of nargs produce a list - else: - value = [self._get_value(action, v) for v in arg_strings] - for v in value: - self._check_value(action, v) - - # return the converted value - return value - - def _get_value(self, action, arg_string): - type_func = self._registry_get('type', action.type, action.type) - if not _callable(type_func): - msg = _('%r is not callable') - raise ArgumentError(action, msg % type_func) - - # convert the value to the appropriate type - try: - result = type_func(arg_string) - - # ArgumentTypeErrors indicate errors - except ArgumentTypeError: - name = getattr(action.type, '__name__', repr(action.type)) - msg = str(_sys.exc_info()[1]) - raise ArgumentError(action, msg) - - # TypeErrors or ValueErrors also indicate errors - except (TypeError, ValueError): - name = getattr(action.type, '__name__', repr(action.type)) - msg = _('invalid %s value: %r') - raise ArgumentError(action, msg % (name, arg_string)) - - # return the converted value - return result - - def _check_value(self, action, value): - # converted value must be one of the choices (if specified) - if action.choices is not None and value not in action.choices: - tup = value, ', '.join(map(repr, action.choices)) - msg = _('invalid choice: %r (choose from %s)') % tup - raise ArgumentError(action, msg) - - # ======================= - # Help-formatting methods - # ======================= - def format_usage(self): - formatter = self._get_formatter() - formatter.add_usage(self.usage, self._actions, - self._mutually_exclusive_groups) - return formatter.format_help() - - def format_help(self): - formatter = self._get_formatter() - - # usage - formatter.add_usage(self.usage, self._actions, - self._mutually_exclusive_groups) - - # description - formatter.add_text(self.description) - - # positionals, optionals and user-defined groups - for action_group in self._action_groups: - formatter.start_section(action_group.title) - formatter.add_text(action_group.description) - formatter.add_arguments(action_group._group_actions) - formatter.end_section() - - # epilog - formatter.add_text(self.epilog) - - # determine help from format above - return formatter.format_help() - - def format_version(self): - import warnings - warnings.warn( - 'The format_version method is deprecated -- the "version" ' - 'argument to ArgumentParser is no longer supported.', - DeprecationWarning) - formatter = self._get_formatter() - formatter.add_text(self.version) - return formatter.format_help() - - def _get_formatter(self): - return self.formatter_class(prog=self.prog) - - # ===================== - # Help-printing methods - # ===================== - def print_usage(self, file=None): - if file is None: - file = _sys.stdout - self._print_message(self.format_usage(), file) - - def print_help(self, file=None): - if file is None: - file = _sys.stdout - self._print_message(self.format_help(), file) - - def print_version(self, file=None): - import warnings - warnings.warn( - 'The print_version method is deprecated -- the "version" ' - 'argument to ArgumentParser is no longer supported.', - DeprecationWarning) - self._print_message(self.format_version(), file) - - def _print_message(self, message, file=None): - if message: - if file is None: - file = _sys.stderr - file.write(message) - - # =============== - # Exiting methods - # =============== - def exit(self, status=0, message=None): - if message: - self._print_message(message, _sys.stderr) - _sys.exit(status) - - def error(self, message): - """error(message: string) - - Prints a usage message incorporating the message to stderr and - exits. - - If you override this in a subclass, it should not return -- it - should either exit or raise an exception. - """ - self.print_usage(_sys.stderr) - self.exit(2, _('%s: error: %s\n') % (self.prog, message)) diff --git a/libs/decorator.py b/libs/decorator.py index a7815ab..ea7e990 100644 --- a/libs/decorator.py +++ b/libs/decorator.py @@ -28,7 +28,7 @@ Decorator module, see http://pypi.python.org/pypi/decorator for the documentation. """ -__version__ = '3.3.0' +__version__ = '3.3.2' __all__ = ["decorator", "FunctionMaker", "partial"] @@ -127,6 +127,7 @@ class FunctionMaker(object): func.__doc__ = getattr(self, 'doc', None) func.__dict__ = getattr(self, 'dict', {}) func.func_defaults = getattr(self, 'defaults', ()) + func.__kwdefaults__ = getattr(self, 'kwonlydefaults', None) callermodule = sys._getframe(3).f_globals.get('__name__', '?') func.__module__ = getattr(self, 'module', callermodule) func.__dict__.update(kw) @@ -193,7 +194,7 @@ def decorator(caller, func=None): evaldict['_func_'] = func return FunctionMaker.create( func, "return _call_(_func_, %(shortsignature)s)", - evaldict, undecorated=func) + evaldict, undecorated=func, __wrapped__=func) else: # returns a decorator if isinstance(caller, partial): return partial(decorator, caller) @@ -205,5 +206,5 @@ def decorator(caller, func=None): return FunctionMaker.create( '%s(%s)' % (caller.__name__, first), 'return decorator(_call_, %s)' % first, - evaldict, undecorated=caller, + evaldict, undecorated=caller, __wrapped__=caller, doc=caller.__doc__, module=caller.__module__) diff --git a/libs/flask/app.py b/libs/flask/app.py index 42ffea4..15e432d 100755 --- a/libs/flask/app.py +++ b/libs/flask/app.py @@ -664,7 +664,7 @@ class Flask(_PackageBoundObject): # existing views. context.update(orig_ctx) - def run(self, host='127.0.0.1', port=5000, debug=None, **options): + def run(self, host=None, port=None, debug=None, **options): """Runs the application on a local development server. If the :attr:`debug` flag is set the server will automatically reload for code changes and show a debugger in case an exception happened. @@ -684,9 +684,10 @@ class Flask(_PackageBoundObject): won't catch any exceptions because there won't be any to catch. - :param host: the hostname to listen on. set this to ``'0.0.0.0'`` - to have the server available externally as well. - :param port: the port of the webserver + :param host: the hostname to listen on. Set this to ``'0.0.0.0'`` to + have the server available externally as well. Defaults to + ``'127.0.0.1'``. + :param port: the port of the webserver. Defaults to ``5000``. :param debug: if given, enable or disable debug mode. See :attr:`debug`. :param options: the options to be forwarded to the underlying @@ -695,6 +696,10 @@ class Flask(_PackageBoundObject): information. """ from werkzeug.serving import run_simple + if host is None: + host = '127.0.0.1' + if port is None: + port = 5000 if debug is not None: self.debug = bool(debug) options.setdefault('use_reloader', self.debug) @@ -711,6 +716,17 @@ class Flask(_PackageBoundObject): """Creates a test client for this application. For information about unit testing head over to :ref:`testing`. + Note that if you are testing for assertions or exceptions in your + application code, you must set ``app.testing = True`` in order for the + exceptions to propagate to the test client. Otherwise, the exception + will be handled by the application (not visible to the test client) and + the only indication of an AssertionError or other exception will be a + 500 status code response to the test client. See the :attr:`testing` + attribute. For example:: + + app.testing = True + client = app.test_client() + The test client can be used in a `with` block to defer the closing down of the context until the end of the `with` block. This is useful if you want to access the context locals for testing:: @@ -1018,11 +1034,21 @@ class Flask(_PackageBoundObject): function name will be used. """ def decorator(f): - self.jinja_env.filters[name or f.__name__] = f + self.add_template_filter(f, name=name) return f return decorator @setupmethod + def add_template_filter(self, f, name=None): + """Register a custom template filter. Works exactly like the + :meth:`template_filter` decorator. + + :param name: the optional name of the filter, otherwise the + function name will be used. + """ + self.jinja_env.filters[name or f.__name__] = f + + @setupmethod def before_request(self, f): """Registers a function to run before each request.""" self.before_request_funcs.setdefault(None, []).append(f) @@ -1105,7 +1131,7 @@ class Flask(_PackageBoundObject): registered error handlers and fall back to returning the exception as response. - .. versionadded: 0.3 + .. versionadded:: 0.3 """ handlers = self.error_handler_spec.get(request.blueprint) if handlers and e.code in handlers: @@ -1174,7 +1200,7 @@ class Flask(_PackageBoundObject): for a 500 internal server error is used. If no such handler exists, a default 500 internal server error message is displayed. - .. versionadded: 0.3 + .. versionadded:: 0.3 """ exc_type, exc_value, tb = sys.exc_info() diff --git a/libs/flask/blueprints.py b/libs/flask/blueprints.py index fa93adc..d81d3c7 100755 --- a/libs/flask/blueprints.py +++ b/libs/flask/blueprints.py @@ -59,7 +59,7 @@ class BlueprintSetupState(object): self.url_defaults = dict(self.blueprint.url_values_defaults) self.url_defaults.update(self.options.get('url_defaults', ())) - def add_url_rule(self, rule, endpoint = None, view_func = None, **options): + def add_url_rule(self, rule, endpoint=None, view_func=None, **options): """A helper method to register a rule (and optionally a view function) to the application. The endpoint is automatically prefixed with the blueprint's name. @@ -73,7 +73,7 @@ class BlueprintSetupState(object): if 'defaults' in options: defaults = dict(defaults, **options.pop('defaults')) self.app.add_url_rule(rule, '%s.%s' % (self.blueprint.name, endpoint), - view_func, defaults = defaults, **options) + view_func, defaults=defaults, **options) class Blueprint(_PackageBoundObject): @@ -89,9 +89,9 @@ class Blueprint(_PackageBoundObject): warn_on_modifications = False _got_registered_once = False - def __init__(self, name, import_name, static_folder = None, - static_url_path = None, template_folder = None, - url_prefix = None, subdomain = None, url_defaults = None): + def __init__(self, name, import_name, static_folder=None, + static_url_path=None, template_folder=None, + url_prefix=None, subdomain=None, url_defaults=None): _PackageBoundObject.__init__(self, import_name, template_folder) self.name = name self.url_prefix = url_prefix @@ -128,14 +128,14 @@ class Blueprint(_PackageBoundObject): func(state) return self.record(update_wrapper(wrapper, func)) - def make_setup_state(self, app, options, first_registration = False): + def make_setup_state(self, app, options, first_registration=False): """Creates an instance of :meth:`~flask.blueprints.BlueprintSetupState` object that is later passed to the register callback functions. Subclasses can override this to return a subclass of the setup state. """ return BlueprintSetupState(self, app, options, first_registration) - def register(self, app, options, first_registration = False): + def register(self, app, options, first_registration=False): """Called by :meth:`Flask.register_blueprint` to register a blueprint on the application. This can be overridden to customize the register behavior. Keyword arguments from @@ -146,8 +146,8 @@ class Blueprint(_PackageBoundObject): state = self.make_setup_state(app, options, first_registration) if self.has_static_folder: state.add_url_rule(self.static_url_path + '/', - view_func = self.send_static_file, - endpoint = 'static') + view_func=self.send_static_file, + endpoint='static') for deferred in self.deferred_functions: deferred(state) @@ -162,7 +162,7 @@ class Blueprint(_PackageBoundObject): return f return decorator - def add_url_rule(self, rule, endpoint = None, view_func = None, **options): + def add_url_rule(self, rule, endpoint=None, view_func=None, **options): """Like :meth:`Flask.add_url_rule` but for a blueprint. The endpoint for the :func:`url_for` function is prefixed with the name of the blueprint. """ @@ -185,6 +185,30 @@ class Blueprint(_PackageBoundObject): return f return decorator + def app_template_filter(self, name=None): + """Register a custom template filter, available application wide. Like + :meth:`Flask.template_filter` but for a blueprint. + + :param name: the optional name of the filter, otherwise the + function name will be used. + """ + def decorator(f): + self.add_app_template_filter(f, name=name) + return f + return decorator + + def add_app_template_filter(self, f, name=None): + """Register a custom template filter, available application wide. Like + :meth:`Flask.add_template_filter` but for a blueprint. Works exactly + like the :meth:`app_template_filter` decorator. + + :param name: the optional name of the filter, otherwise the + function name will be used. + """ + def register_template(state): + state.app.jinja_env.filters[name or f.__name__] = f + self.record_once(register_template) + def before_request(self, f): """Like :meth:`Flask.before_request` but for a blueprint. This function is only executed before each request that is handled by a function of diff --git a/libs/flask/ctx.py b/libs/flask/ctx.py index 9a72d25..47ac0cc 100755 --- a/libs/flask/ctx.py +++ b/libs/flask/ctx.py @@ -21,9 +21,9 @@ class _RequestGlobals(object): def has_request_context(): """If you have code that wants to test if a request context is there or - not this function can be used. For instance if you want to take advantage - of request information is it's available but fail silently if the request - object is unavailable. + not this function can be used. For instance, you may want to take advantage + of request information if the request object is available, but fail + silently if it is unavailable. :: diff --git a/libs/flask/helpers.py b/libs/flask/helpers.py index 7295dc3..25250d2 100755 --- a/libs/flask/helpers.py +++ b/libs/flask/helpers.py @@ -11,8 +11,10 @@ from __future__ import with_statement +import imp import os import sys +import pkgutil import posixpath import mimetypes from time import time @@ -249,7 +251,7 @@ def flash(message, category='message'): flashed message from the session and to display it to the user, the template has to call :func:`get_flashed_messages`. - .. versionchanged: 0.3 + .. versionchanged:: 0.3 `category` parameter added. :param message: the message to be flashed. @@ -262,30 +264,40 @@ def flash(message, category='message'): session.setdefault('_flashes', []).append((category, message)) -def get_flashed_messages(with_categories=False): +def get_flashed_messages(with_categories=False, category_filter=[]): """Pulls all flashed messages from the session and returns them. Further calls in the same request to the function will return the same messages. By default just the messages are returned, but when `with_categories` is set to `True`, the return value will be a list of tuples in the form ``(category, message)`` instead. - Example usage: + Filter the flashed messages to one or more categories by providing those + categories in `category_filter`. This allows rendering categories in + separate html blocks. The `with_categories` and `category_filter` + arguments are distinct: - .. sourcecode:: html+jinja + * `with_categories` controls whether categories are returned with message + text (`True` gives a tuple, where `False` gives just the message text). + * `category_filter` filters the messages down to only those matching the + provided categories. - {% for category, msg in get_flashed_messages(with_categories=true) %} -

{{ msg }} - {% endfor %} + See :ref:`message-flashing-pattern` for examples. .. versionchanged:: 0.3 `with_categories` parameter added. + .. versionchanged:: 0.9 + `category_filter` parameter added. + :param with_categories: set to `True` to also receive categories. + :param category_filter: whitelist of categories to limit return values """ flashes = _request_ctx_stack.top.flashes if flashes is None: _request_ctx_stack.top.flashes = flashes = session.pop('_flashes') \ if '_flashes' in session else [] + if category_filter: + flashes = filter(lambda f: f[0] in category_filter, flashes) if not with_categories: return [x[1] for x in flashes] return flashes @@ -492,14 +504,19 @@ def get_root_path(import_name): Not to be confused with the package path returned by :func:`find_package`. """ - __import__(import_name) - try: - directory = os.path.dirname(sys.modules[import_name].__file__) - return os.path.abspath(directory) - except AttributeError: - # this is necessary in case we are running from the interactive - # python shell. It will never be used for production code however + loader = pkgutil.get_loader(import_name) + if loader is None or import_name == '__main__': + # import name is not found, or interactive/main module return os.getcwd() + # For .egg, zipimporter does not have get_filename until Python 2.7. + if hasattr(loader, 'get_filename'): + filepath = loader.get_filename(import_name) + else: + # Fall back to imports. + __import__(import_name) + filepath = sys.modules[import_name].__file__ + # filepath is import_name.py for a module, or __init__.py for a package. + return os.path.dirname(os.path.abspath(filepath)) def find_package(import_name): @@ -510,25 +527,34 @@ def find_package(import_name): import the module. The prefix is the path below which a UNIX like folder structure exists (lib, share etc.). """ - __import__(import_name) - root_mod = sys.modules[import_name.split('.')[0]] - package_path = getattr(root_mod, '__file__', None) - if package_path is None: - # support for the interactive python shell + root_mod_name = import_name.split('.')[0] + loader = pkgutil.get_loader(root_mod_name) + if loader is None or import_name == '__main__': + # import name is not found, or interactive/main module package_path = os.getcwd() else: - package_path = os.path.abspath(os.path.dirname(package_path)) - if hasattr(root_mod, '__path__'): - package_path = os.path.dirname(package_path) - - # leave the egg wrapper folder or the actual .egg on the filesystem - test_package_path = package_path - if os.path.basename(test_package_path).endswith('.egg'): - test_package_path = os.path.dirname(test_package_path) - - site_parent, site_folder = os.path.split(test_package_path) + # For .egg, zipimporter does not have get_filename until Python 2.7. + if hasattr(loader, 'get_filename'): + filename = loader.get_filename(root_mod_name) + elif hasattr(loader, 'archive'): + # zipimporter's loader.archive points to the .egg or .zip + # archive filename is dropped in call to dirname below. + filename = loader.archive + else: + # At least one loader is missing both get_filename and archive: + # Google App Engine's HardenedModulesHook + # + # Fall back to imports. + __import__(import_name) + filename = sys.modules[import_name].__file__ + package_path = os.path.abspath(os.path.dirname(filename)) + # package_path ends with __init__.py for a package + if loader.is_package(root_mod_name): + package_path = os.path.dirname(package_path) + + site_parent, site_folder = os.path.split(package_path) py_prefix = os.path.abspath(sys.prefix) - if test_package_path.startswith(py_prefix): + if package_path.startswith(py_prefix): return py_prefix, package_path elif site_folder.lower() == 'site-packages': parent, folder = os.path.split(site_parent) diff --git a/libs/flask/views.py b/libs/flask/views.py index f11c3dd..79d6299 100755 --- a/libs/flask/views.py +++ b/libs/flask/views.py @@ -3,7 +3,7 @@ flask.views ~~~~~~~~~~~ - This module provides class based views inspired by the ones in Django. + This module provides class-based views inspired by the ones in Django. :copyright: (c) 2011 by Armin Ronacher. :license: BSD, see LICENSE for more details. @@ -12,7 +12,7 @@ from .globals import request http_method_funcs = frozenset(['get', 'post', 'head', 'options', - 'delete', 'put', 'trace']) + 'delete', 'put', 'trace', 'patch']) class View(object): @@ -50,7 +50,7 @@ class View(object): #: A for which methods this pluggable view can handle. methods = None - #: The canonical way to decorate class based views is to decorate the + #: The canonical way to decorate class-based views is to decorate the #: return value of as_view(). However since this moves parts of the #: logic from the class declaration to the place where it's hooked #: into the routing system. @@ -70,10 +70,10 @@ class View(object): @classmethod def as_view(cls, name, *class_args, **class_kwargs): - """Converts the class into an actual view function that can be - used with the routing system. What it does internally is generating - a function on the fly that will instanciate the :class:`View` - on each request and call the :meth:`dispatch_request` method on it. + """Converts the class into an actual view function that can be used + with the routing system. Internally this generates a function on the + fly which will instantiate the :class:`View` on each request and call + the :meth:`dispatch_request` method on it. The arguments passed to :meth:`as_view` are forwarded to the constructor of the class. @@ -89,8 +89,8 @@ class View(object): view = decorator(view) # we attach the view class to the view function for two reasons: - # first of all it allows us to easily figure out what class based - # view this thing came from, secondly it's also used for instanciating + # first of all it allows us to easily figure out what class-based + # view this thing came from, secondly it's also used for instantiating # the view class so you can actually replace it with something else # for testing purposes and debugging. view.view_class = cls @@ -120,7 +120,7 @@ class MethodViewType(type): class MethodView(View): - """Like a regular class based view but that dispatches requests to + """Like a regular class-based view but that dispatches requests to particular methods. For instance if you implement a method called :meth:`get` it means you will response to ``'GET'`` requests and the :meth:`dispatch_request` implementation will automatically @@ -146,5 +146,5 @@ class MethodView(View): # retry with GET if meth is None and request.method == 'HEAD': meth = getattr(self, 'get', None) - assert meth is not None, 'Not implemented method %r' % request.method + assert meth is not None, 'Unimplemented method %r' % request.method return meth(*args, **kwargs) diff --git a/libs/jinja2/filters.py b/libs/jinja2/filters.py index 352b166..8dd6ff0 100755 --- a/libs/jinja2/filters.py +++ b/libs/jinja2/filters.py @@ -13,7 +13,8 @@ import math from random import choice from operator import itemgetter from itertools import imap, groupby -from jinja2.utils import Markup, escape, pformat, urlize, soft_unicode +from jinja2.utils import Markup, escape, pformat, urlize, soft_unicode, \ + unicode_urlencode from jinja2.runtime import Undefined from jinja2.exceptions import FilterArgumentError @@ -70,6 +71,26 @@ def do_forceescape(value): return escape(unicode(value)) +def do_urlencode(value): + """Escape strings for use in URLs (uses UTF-8 encoding). It accepts both + dictionaries and regular strings as well as pairwise iterables. + + .. versionadded:: 2.7 + """ + itemiter = None + if isinstance(value, dict): + itemiter = value.iteritems() + elif not isinstance(value, basestring): + try: + itemiter = iter(value) + except TypeError: + pass + if itemiter is None: + return unicode_urlencode(value) + return u'&'.join(unicode_urlencode(k) + '=' + + unicode_urlencode(v) for k, v in itemiter) + + @evalcontextfilter def do_replace(eval_ctx, s, old, new, count=None): """Return a copy of the value with all occurrences of a substring @@ -797,5 +818,6 @@ FILTERS = { 'round': do_round, 'groupby': do_groupby, 'safe': do_mark_safe, - 'xmlattr': do_xmlattr + 'xmlattr': do_xmlattr, + 'urlencode': do_urlencode } diff --git a/libs/jinja2/runtime.py b/libs/jinja2/runtime.py index a4a47a2..5c39984 100755 --- a/libs/jinja2/runtime.py +++ b/libs/jinja2/runtime.py @@ -30,6 +30,8 @@ to_string = unicode #: the identity function. Useful for certain things in the environment identity = lambda x: x +_last_iteration = object() + def markup_join(seq): """Concatenation that escapes if necessary and converts to unicode.""" @@ -270,6 +272,7 @@ class LoopContext(object): def __init__(self, iterable, recurse=None): self._iterator = iter(iterable) self._recurse = recurse + self._after = self._safe_next() self.index0 = -1 # try to get the length of the iterable early. This must be done @@ -288,7 +291,7 @@ class LoopContext(object): return args[self.index0 % len(args)] first = property(lambda x: x.index0 == 0) - last = property(lambda x: x.index0 + 1 == x.length) + last = property(lambda x: x._after is _last_iteration) index = property(lambda x: x.index0 + 1) revindex = property(lambda x: x.length - x.index0) revindex0 = property(lambda x: x.length - x.index) @@ -299,6 +302,12 @@ class LoopContext(object): def __iter__(self): return LoopContextIterator(self) + def _safe_next(self): + try: + return next(self._iterator) + except StopIteration: + return _last_iteration + @internalcode def loop(self, iterable): if self._recurse is None: @@ -344,7 +353,11 @@ class LoopContextIterator(object): def next(self): ctx = self.context ctx.index0 += 1 - return next(ctx._iterator), ctx + if ctx._after is _last_iteration: + raise StopIteration() + next_elem = ctx._after + ctx._after = ctx._safe_next() + return next_elem, ctx class Macro(object): diff --git a/libs/jinja2/utils.py b/libs/jinja2/utils.py index 49e9e9a..1e0bb81 100755 --- a/libs/jinja2/utils.py +++ b/libs/jinja2/utils.py @@ -12,6 +12,10 @@ import re import sys import errno try: + from urllib.parse import quote_from_bytes as url_quote +except ImportError: + from urllib import quote as url_quote +try: from thread import allocate_lock except ImportError: from dummy_thread import allocate_lock @@ -349,6 +353,21 @@ def generate_lorem_ipsum(n=5, html=True, min=20, max=100): return Markup(u'\n'.join(u'

%s

' % escape(x) for x in result)) +def unicode_urlencode(obj, charset='utf-8'): + """URL escapes a single bytestring or unicode string with the + given charset if applicable to URL safe quoting under all rules + that need to be considered under all supported Python versions. + + If non strings are provided they are converted to their unicode + representation first. + """ + if not isinstance(obj, basestring): + obj = unicode(obj) + if isinstance(obj, unicode): + obj = obj.encode(charset) + return unicode(url_quote(obj)) + + class LRUCache(object): """A simple LRU Cache implementation.""" diff --git a/libs/migrate/__init__.py b/libs/migrate/__init__.py index dd3c059..0cfdb72 100644 --- a/libs/migrate/__init__.py +++ b/libs/migrate/__init__.py @@ -7,3 +7,5 @@ from migrate.versioning import * from migrate.changeset import * + +__version__ = '0.7.2' diff --git a/libs/migrate/changeset/__init__.py b/libs/migrate/changeset/__init__.py index 7a12643..80ea152 100644 --- a/libs/migrate/changeset/__init__.py +++ b/libs/migrate/changeset/__init__.py @@ -13,7 +13,7 @@ from sqlalchemy import __version__ as _sa_version warnings.simplefilter('always', DeprecationWarning) _sa_version = tuple(int(re.match("\d+", x).group(0)) for x in _sa_version.split(".")) -SQLA_06 = _sa_version >= (0, 6) +SQLA_07 = _sa_version >= (0, 7) del re del _sa_version diff --git a/libs/migrate/changeset/ansisql.py b/libs/migrate/changeset/ansisql.py index e6cdd7c..9ded560 100644 --- a/libs/migrate/changeset/ansisql.py +++ b/libs/migrate/changeset/ansisql.py @@ -17,23 +17,19 @@ from sqlalchemy.schema import (ForeignKeyConstraint, Index) from migrate import exceptions -from migrate.changeset import constraint, SQLA_06 +from migrate.changeset import constraint -if not SQLA_06: - from sqlalchemy.sql.compiler import SchemaGenerator, SchemaDropper -else: - from sqlalchemy.schema import AddConstraint, DropConstraint - from sqlalchemy.sql.compiler import DDLCompiler - SchemaGenerator = SchemaDropper = DDLCompiler +from sqlalchemy.schema import AddConstraint, DropConstraint +from sqlalchemy.sql.compiler import DDLCompiler +SchemaGenerator = SchemaDropper = DDLCompiler class AlterTableVisitor(SchemaVisitor): """Common operations for ``ALTER TABLE`` statements.""" - if SQLA_06: - # engine.Compiler looks for .statement - # when it spawns off a new compiler - statement = ClauseElement() + # engine.Compiler looks for .statement + # when it spawns off a new compiler + statement = ClauseElement() def append(self, s): """Append content to the SchemaIterator's query buffer.""" @@ -116,16 +112,15 @@ class ANSIColumnGenerator(AlterTableVisitor, SchemaGenerator): # SA bounds FK constraints to table, add manually for fk in column.foreign_keys: self.add_foreignkey(fk.constraint) - + # add primary key constraint if needed if column.primary_key_name: cons = constraint.PrimaryKeyConstraint(column, name=column.primary_key_name) cons.create() - if SQLA_06: - def add_foreignkey(self, fk): - self.connection.execute(AddConstraint(fk)) + def add_foreignkey(self, fk): + self.connection.execute(AddConstraint(fk)) class ANSIColumnDropper(AlterTableVisitor, SchemaDropper): """Extends ANSI SQL dropper for column dropping (``ALTER TABLE @@ -232,10 +227,7 @@ class ANSISchemaChanger(AlterTableVisitor, SchemaGenerator): def _visit_column_type(self, table, column, delta): type_ = delta['type'] - if SQLA_06: - type_text = str(type_.compile(dialect=self.dialect)) - else: - type_text = type_.dialect_impl(self.dialect).get_col_spec() + type_text = str(type_.compile(dialect=self.dialect)) self.append("TYPE %s" % type_text) def _visit_column_name(self, table, column, delta): @@ -279,75 +271,17 @@ class ANSIConstraintCommon(AlterTableVisitor): def visit_migrate_unique_constraint(self, *p, **k): self._visit_constraint(*p, **k) -if SQLA_06: - class ANSIConstraintGenerator(ANSIConstraintCommon, SchemaGenerator): - def _visit_constraint(self, constraint): - constraint.name = self.get_constraint_name(constraint) - self.append(self.process(AddConstraint(constraint))) - self.execute() - - class ANSIConstraintDropper(ANSIConstraintCommon, SchemaDropper): - def _visit_constraint(self, constraint): - constraint.name = self.get_constraint_name(constraint) - self.append(self.process(DropConstraint(constraint, cascade=constraint.cascade))) - self.execute() - -else: - class ANSIConstraintGenerator(ANSIConstraintCommon, SchemaGenerator): +class ANSIConstraintGenerator(ANSIConstraintCommon, SchemaGenerator): + def _visit_constraint(self, constraint): + constraint.name = self.get_constraint_name(constraint) + self.append(self.process(AddConstraint(constraint))) + self.execute() - def get_constraint_specification(self, cons, **kwargs): - """Constaint SQL generators. - - We cannot use SA visitors because they append comma. - """ - - if isinstance(cons, PrimaryKeyConstraint): - if cons.name is not None: - self.append("CONSTRAINT %s " % self.preparer.format_constraint(cons)) - self.append("PRIMARY KEY ") - self.append("(%s)" % ', '.join(self.preparer.quote(c.name, c.quote) - for c in cons)) - self.define_constraint_deferrability(cons) - elif isinstance(cons, ForeignKeyConstraint): - self.define_foreign_key(cons) - elif isinstance(cons, CheckConstraint): - if cons.name is not None: - self.append("CONSTRAINT %s " % - self.preparer.format_constraint(cons)) - self.append("CHECK (%s)" % cons.sqltext) - self.define_constraint_deferrability(cons) - elif isinstance(cons, UniqueConstraint): - if cons.name is not None: - self.append("CONSTRAINT %s " % - self.preparer.format_constraint(cons)) - self.append("UNIQUE (%s)" % \ - (', '.join(self.preparer.quote(c.name, c.quote) for c in cons))) - self.define_constraint_deferrability(cons) - else: - raise exceptions.InvalidConstraintError(cons) - - def _visit_constraint(self, constraint): - - table = self.start_alter_table(constraint) - constraint.name = self.get_constraint_name(constraint) - self.append("ADD ") - self.get_constraint_specification(constraint) - self.execute() - - - class ANSIConstraintDropper(ANSIConstraintCommon, SchemaDropper): - - def _visit_constraint(self, constraint): - self.start_alter_table(constraint) - self.append("DROP CONSTRAINT ") - constraint.name = self.get_constraint_name(constraint) - self.append(self.preparer.format_constraint(constraint)) - if constraint.cascade: - self.cascade_constraint(constraint) - self.execute() - - def cascade_constraint(self, constraint): - self.append(" CASCADE") +class ANSIConstraintDropper(ANSIConstraintCommon, SchemaDropper): + def _visit_constraint(self, constraint): + constraint.name = self.get_constraint_name(constraint) + self.append(self.process(DropConstraint(constraint, cascade=constraint.cascade))) + self.execute() class ANSIDialect(DefaultDialect): diff --git a/libs/migrate/changeset/constraint.py b/libs/migrate/changeset/constraint.py index 476a456..96407bd 100644 --- a/libs/migrate/changeset/constraint.py +++ b/libs/migrate/changeset/constraint.py @@ -4,7 +4,6 @@ from sqlalchemy import schema from migrate.exceptions import * -from migrate.changeset import SQLA_06 class ConstraintChangeset(object): """Base class for Constraint classes.""" @@ -165,8 +164,6 @@ class CheckConstraint(ConstraintChangeset, schema.CheckConstraint): table = kwargs.pop('table', table) schema.CheckConstraint.__init__(self, sqltext, *args, **kwargs) if table is not None: - if not SQLA_06: - self.table = table self._set_parent(table) self.colnames = colnames diff --git a/libs/migrate/changeset/databases/firebird.py b/libs/migrate/changeset/databases/firebird.py index 675666c..226728b 100644 --- a/libs/migrate/changeset/databases/firebird.py +++ b/libs/migrate/changeset/databases/firebird.py @@ -4,13 +4,10 @@ from sqlalchemy.databases import firebird as sa_base from sqlalchemy.schema import PrimaryKeyConstraint from migrate import exceptions -from migrate.changeset import ansisql, SQLA_06 +from migrate.changeset import ansisql -if SQLA_06: - FBSchemaGenerator = sa_base.FBDDLCompiler -else: - FBSchemaGenerator = sa_base.FBSchemaGenerator +FBSchemaGenerator = sa_base.FBDDLCompiler class FBColumnGenerator(FBSchemaGenerator, ansisql.ANSIColumnGenerator): """Firebird column generator implementation.""" @@ -41,10 +38,7 @@ class FBColumnDropper(ansisql.ANSIColumnDropper): # is deleted! continue - if SQLA_06: - should_drop = column.name in cons.columns - else: - should_drop = cons.contains_column(column) and cons.name + should_drop = column.name in cons.columns if should_drop: self.start_alter_table(column) self.append("DROP CONSTRAINT ") diff --git a/libs/migrate/changeset/databases/mysql.py b/libs/migrate/changeset/databases/mysql.py index badd9fe..6987b4b 100644 --- a/libs/migrate/changeset/databases/mysql.py +++ b/libs/migrate/changeset/databases/mysql.py @@ -6,13 +6,10 @@ from sqlalchemy.databases import mysql as sa_base from sqlalchemy import types as sqltypes from migrate import exceptions -from migrate.changeset import ansisql, SQLA_06 +from migrate.changeset import ansisql -if not SQLA_06: - MySQLSchemaGenerator = sa_base.MySQLSchemaGenerator -else: - MySQLSchemaGenerator = sa_base.MySQLDDLCompiler +MySQLSchemaGenerator = sa_base.MySQLDDLCompiler class MySQLColumnGenerator(MySQLSchemaGenerator, ansisql.ANSIColumnGenerator): pass @@ -53,37 +50,11 @@ class MySQLSchemaChanger(MySQLSchemaGenerator, ansisql.ANSISchemaChanger): class MySQLConstraintGenerator(ansisql.ANSIConstraintGenerator): pass -if SQLA_06: - class MySQLConstraintDropper(MySQLSchemaGenerator, ansisql.ANSIConstraintDropper): - def visit_migrate_check_constraint(self, *p, **k): - raise exceptions.NotSupportedError("MySQL does not support CHECK" - " constraints, use triggers instead.") - -else: - class MySQLConstraintDropper(ansisql.ANSIConstraintDropper): - - def visit_migrate_primary_key_constraint(self, constraint): - self.start_alter_table(constraint) - self.append("DROP PRIMARY KEY") - self.execute() - - def visit_migrate_foreign_key_constraint(self, constraint): - self.start_alter_table(constraint) - self.append("DROP FOREIGN KEY ") - constraint.name = self.get_constraint_name(constraint) - self.append(self.preparer.format_constraint(constraint)) - self.execute() - - def visit_migrate_check_constraint(self, *p, **k): - raise exceptions.NotSupportedError("MySQL does not support CHECK" - " constraints, use triggers instead.") - - def visit_migrate_unique_constraint(self, constraint, *p, **k): - self.start_alter_table(constraint) - self.append('DROP INDEX ') - constraint.name = self.get_constraint_name(constraint) - self.append(self.preparer.format_constraint(constraint)) - self.execute() + +class MySQLConstraintDropper(MySQLSchemaGenerator, ansisql.ANSIConstraintDropper): + def visit_migrate_check_constraint(self, *p, **k): + raise exceptions.NotSupportedError("MySQL does not support CHECK" + " constraints, use triggers instead.") class MySQLDialect(ansisql.ANSIDialect): diff --git a/libs/migrate/changeset/databases/oracle.py b/libs/migrate/changeset/databases/oracle.py index bd761bc..2f16b5b 100644 --- a/libs/migrate/changeset/databases/oracle.py +++ b/libs/migrate/changeset/databases/oracle.py @@ -5,13 +5,10 @@ import sqlalchemy as sa from sqlalchemy.databases import oracle as sa_base from migrate import exceptions -from migrate.changeset import ansisql, SQLA_06 +from migrate.changeset import ansisql -if not SQLA_06: - OracleSchemaGenerator = sa_base.OracleSchemaGenerator -else: - OracleSchemaGenerator = sa_base.OracleDDLCompiler +OracleSchemaGenerator = sa_base.OracleDDLCompiler class OracleColumnGenerator(OracleSchemaGenerator, ansisql.ANSIColumnGenerator): diff --git a/libs/migrate/changeset/databases/postgres.py b/libs/migrate/changeset/databases/postgres.py index 015ad65..10ea094 100644 --- a/libs/migrate/changeset/databases/postgres.py +++ b/libs/migrate/changeset/databases/postgres.py @@ -3,14 +3,10 @@ .. _`PostgreSQL`: http://www.postgresql.org/ """ -from migrate.changeset import ansisql, SQLA_06 - -if not SQLA_06: - from sqlalchemy.databases import postgres as sa_base - PGSchemaGenerator = sa_base.PGSchemaGenerator -else: - from sqlalchemy.databases import postgresql as sa_base - PGSchemaGenerator = sa_base.PGDDLCompiler +from migrate.changeset import ansisql + +from sqlalchemy.databases import postgresql as sa_base +PGSchemaGenerator = sa_base.PGDDLCompiler class PGColumnGenerator(PGSchemaGenerator, ansisql.ANSIColumnGenerator): diff --git a/libs/migrate/changeset/databases/sqlite.py b/libs/migrate/changeset/databases/sqlite.py index 447412d..5ddd3f1 100644 --- a/libs/migrate/changeset/databases/sqlite.py +++ b/libs/migrate/changeset/databases/sqlite.py @@ -9,13 +9,11 @@ from copy import copy from sqlalchemy.databases import sqlite as sa_base from migrate import exceptions -from migrate.changeset import ansisql, SQLA_06 +from migrate.changeset import ansisql -if not SQLA_06: - SQLiteSchemaGenerator = sa_base.SQLiteSchemaGenerator -else: - SQLiteSchemaGenerator = sa_base.SQLiteDDLCompiler +SQLiteSchemaGenerator = sa_base.SQLiteDDLCompiler + class SQLiteCommon(object): @@ -39,7 +37,7 @@ class SQLiteHelper(SQLiteCommon): insertion_string = self._modify_table(table, column, delta) - table.create() + table.create(bind=self.connection) self.append(insertion_string % {'table_name': table_name}) self.execute() self.append('DROP TABLE migration_tmp') diff --git a/libs/migrate/changeset/schema.py b/libs/migrate/changeset/schema.py index 75cd3fd..c467cc5 100644 --- a/libs/migrate/changeset/schema.py +++ b/libs/migrate/changeset/schema.py @@ -11,7 +11,7 @@ from sqlalchemy.schema import ForeignKeyConstraint from sqlalchemy.schema import UniqueConstraint from migrate.exceptions import * -from migrate.changeset import SQLA_06 +from migrate.changeset import SQLA_07 from migrate.changeset.databases.visitor import (get_engine_visitor, run_single_visitor) @@ -349,10 +349,7 @@ class ColumnDelta(DictMixin, sqlalchemy.schema.SchemaItem): def process_column(self, column): """Processes default values for column""" # XXX: this is a snippet from SA processing of positional parameters - if not SQLA_06 and column.args: - toinit = list(column.args) - else: - toinit = list() + toinit = list() if column.server_default is not None: if isinstance(column.server_default, sqlalchemy.FetchedValue): @@ -367,9 +364,6 @@ class ColumnDelta(DictMixin, sqlalchemy.schema.SchemaItem): for_update=True)) if toinit: column._init_items(*toinit) - - if not SQLA_06: - column.args = [] def _get_table(self): return getattr(self, '_table', None) @@ -469,14 +463,18 @@ class ChangesetTable(object): self._set_parent(self.metadata) def _meta_key(self): + """Get the meta key for this table.""" return sqlalchemy.schema._get_table_key(self.name, self.schema) def deregister(self): """Remove this table from its metadata""" - key = self._meta_key() - meta = self.metadata - if key in meta.tables: - del meta.tables[key] + if SQLA_07: + self.metadata._remove_table(self.name, self.schema) + else: + key = self._meta_key() + meta = self.metadata + if key in meta.tables: + del meta.tables[key] class ChangesetColumn(object): @@ -555,7 +553,10 @@ populated with defaults def add_to_table(self, table): if table is not None and self.table is None: - self._set_parent(table) + if SQLA_07: + table.append_column(self) + else: + self._set_parent(table) def _col_name_in_constraint(self,cons,name): return False @@ -590,7 +591,10 @@ populated with defaults table.constraints = table.constraints - to_drop if table.c.contains_column(self): - table.c.remove(self) + if SQLA_07: + table._columns.remove(self) + else: + table.c.remove(self) # TODO: this is fixed in 0.6 def copy_fixed(self, **kw): diff --git a/libs/migrate/versioning/api.py b/libs/migrate/versioning/api.py index c159a49..570dc08 100644 --- a/libs/migrate/versioning/api.py +++ b/libs/migrate/versioning/api.py @@ -110,19 +110,19 @@ def script(description, repository, **opts): @catch_known_errors -def script_sql(database, repository, **opts): - """%prog script_sql DATABASE REPOSITORY_PATH +def script_sql(database, description, repository, **opts): + """%prog script_sql DATABASE DESCRIPTION REPOSITORY_PATH Create empty change SQL scripts for given DATABASE, where DATABASE - is either specific ('postgres', 'mysql', 'oracle', 'sqlite', etc.) + is either specific ('postgresql', 'mysql', 'oracle', 'sqlite', etc.) or generic ('default'). - For instance, manage.py script_sql postgres creates: - repository/versions/001_postgres_upgrade.sql and - repository/versions/001_postgres_postgres.sql + For instance, manage.py script_sql postgresql description creates: + repository/versions/001_description_postgresql_upgrade.sql and + repository/versions/001_description_postgresql_downgrade.sql """ repo = Repository(repository) - repo.create_script_sql(database, **opts) + repo.create_script_sql(database, description, **opts) def version(repository, **opts): @@ -212,14 +212,15 @@ def test(url, repository, **opts): """ engine = opts.pop('engine') repos = Repository(repository) - script = repos.version(None).script() # Upgrade log.info("Upgrading...") + script = repos.version(None).script(engine.name, 'upgrade') script.run(engine, 1) log.info("done") log.info("Downgrading...") + script = repos.version(None).script(engine.name, 'downgrade') script.run(engine, -1) log.info("done") log.info("Success") diff --git a/libs/migrate/versioning/genmodel.py b/libs/migrate/versioning/genmodel.py index b3449b6..85df627 100644 --- a/libs/migrate/versioning/genmodel.py +++ b/libs/migrate/versioning/genmodel.py @@ -1,9 +1,9 @@ """ - Code to generate a Python model from a database or differences - between a model and database. +Code to generate a Python model from a database or differences +between a model and database. - Some of this is borrowed heavily from the AutoCode project at: - http://code.google.com/p/sqlautocode/ +Some of this is borrowed heavily from the AutoCode project at: +http://code.google.com/p/sqlautocode/ """ import sys @@ -34,6 +34,13 @@ Base = declarative.declarative_base() class ModelGenerator(object): + """Various transformations from an A, B diff. + + In the implementation, A tends to be called the model and B + the database (although this is not true of all diffs). + The diff is directionless, but transformations apply the diff + in a particular direction, described in the method name. + """ def __init__(self, diff, engine, declarative=False): self.diff = diff @@ -59,7 +66,7 @@ class ModelGenerator(object): pass else: kwarg.append('default') - ks = ', '.join('%s=%r' % (k, getattr(col, k)) for k in kwarg) + args = ['%s=%r' % (k, getattr(col, k)) for k in kwarg] # crs: not sure if this is good idea, but it gets rid of extra # u'' @@ -73,43 +80,38 @@ class ModelGenerator(object): type_ = cls() break - data = { - 'name': name, - 'type': type_, - 'constraints': ', '.join([repr(cn) for cn in col.constraints]), - 'args': ks and ks or ''} + type_repr = repr(type_) + if type_repr.endswith('()'): + type_repr = type_repr[:-2] - if data['constraints']: - if data['args']: - data['args'] = ',' + data['args'] + constraints = [repr(cn) for cn in col.constraints] - if data['constraints'] or data['args']: - data['maybeComma'] = ',' - else: - data['maybeComma'] = '' + data = { + 'name': name, + 'commonStuff': ', '.join([type_repr] + constraints + args), + } - commonStuff = """ %(maybeComma)s %(constraints)s %(args)s)""" % data - commonStuff = commonStuff.strip() - data['commonStuff'] = commonStuff if self.declarative: - return """%(name)s = Column(%(type)r%(commonStuff)s""" % data + return """%(name)s = Column(%(commonStuff)s)""" % data else: - return """Column(%(name)r, %(type)r%(commonStuff)s""" % data + return """Column(%(name)r, %(commonStuff)s)""" % data - def getTableDefn(self, table): + def _getTableDefn(self, table, metaName='meta'): out = [] tableName = table.name if self.declarative: out.append("class %(table)s(Base):" % {'table': tableName}) - out.append(" __tablename__ = '%(table)s'" % {'table': tableName}) + out.append(" __tablename__ = '%(table)s'\n" % + {'table': tableName}) for col in table.columns: - out.append(" %s" % self.column_repr(col)) + out.append(" %s" % self.column_repr(col)) + out.append('\n') else: - out.append("%(table)s = Table('%(table)s', meta," % \ - {'table': tableName}) + out.append("%(table)s = Table('%(table)s', %(meta)s," % + {'table': tableName, 'meta': metaName}) for col in table.columns: - out.append(" %s," % self.column_repr(col)) - out.append(")") + out.append(" %s," % self.column_repr(col)) + out.append(")\n") return out def _get_tables(self,missingA=False,missingB=False,modified=False): @@ -122,9 +124,15 @@ class ModelGenerator(object): if bool_: for name in names: yield metadata.tables.get(name) - - def toPython(self): - """Assume database is current and model is empty.""" + + def genBDefinition(self): + """Generates the source code for a definition of B. + + Assumes a diff where A is empty. + + Was: toPython. Assume database (B) is current and model (A) is empty. + """ + out = [] if self.declarative: out.append(DECLARATIVE_HEADER) @@ -132,67 +140,89 @@ class ModelGenerator(object): out.append(HEADER) out.append("") for table in self._get_tables(missingA=True): - out.extend(self.getTableDefn(table)) - out.append("") + out.extend(self._getTableDefn(table)) return '\n'.join(out) - def toUpgradeDowngradePython(self, indent=' '): - ''' Assume model is most current and database is out-of-date. ''' + def genB2AMigration(self, indent=' '): + '''Generate a migration from B to A. + + Was: toUpgradeDowngradePython + Assume model (A) is most current and database (B) is out-of-date. + ''' + decls = ['from migrate.changeset import schema', - 'meta = MetaData()'] - for table in self._get_tables( - missingA=True,missingB=True,modified=True - ): - decls.extend(self.getTableDefn(table)) - - upgradeCommands, downgradeCommands = [], [] - for tableName in self.diff.tables_missing_from_A: - upgradeCommands.append("%(table)s.drop()" % {'table': tableName}) - downgradeCommands.append("%(table)s.create()" % \ - {'table': tableName}) - for tableName in self.diff.tables_missing_from_B: - upgradeCommands.append("%(table)s.create()" % {'table': tableName}) - downgradeCommands.append("%(table)s.drop()" % {'table': tableName}) - - for tableName in self.diff.tables_different: - dbTable = self.diff.metadataB.tables[tableName] - missingInDatabase, missingInModel, diffDecl = \ - self.diff.colDiffs[tableName] - for col in missingInDatabase: - upgradeCommands.append('%s.columns[%r].create()' % ( - modelTable, col.name)) - downgradeCommands.append('%s.columns[%r].drop()' % ( - modelTable, col.name)) - for col in missingInModel: - upgradeCommands.append('%s.columns[%r].drop()' % ( - modelTable, col.name)) - downgradeCommands.append('%s.columns[%r].create()' % ( - modelTable, col.name)) - for modelCol, databaseCol, modelDecl, databaseDecl in diffDecl: + 'pre_meta = MetaData()', + 'post_meta = MetaData()', + ] + upgradeCommands = ['pre_meta.bind = migrate_engine', + 'post_meta.bind = migrate_engine'] + downgradeCommands = list(upgradeCommands) + + for tn in self.diff.tables_missing_from_A: + pre_table = self.diff.metadataB.tables[tn] + decls.extend(self._getTableDefn(pre_table, metaName='pre_meta')) + upgradeCommands.append( + "pre_meta.tables[%(table)r].drop()" % {'table': tn}) + downgradeCommands.append( + "pre_meta.tables[%(table)r].create()" % {'table': tn}) + + for tn in self.diff.tables_missing_from_B: + post_table = self.diff.metadataA.tables[tn] + decls.extend(self._getTableDefn(post_table, metaName='post_meta')) + upgradeCommands.append( + "post_meta.tables[%(table)r].create()" % {'table': tn}) + downgradeCommands.append( + "post_meta.tables[%(table)r].drop()" % {'table': tn}) + + for (tn, td) in self.diff.tables_different.iteritems(): + if td.columns_missing_from_A or td.columns_different: + pre_table = self.diff.metadataB.tables[tn] + decls.extend(self._getTableDefn( + pre_table, metaName='pre_meta')) + if td.columns_missing_from_B or td.columns_different: + post_table = self.diff.metadataA.tables[tn] + decls.extend(self._getTableDefn( + post_table, metaName='post_meta')) + + for col in td.columns_missing_from_A: + upgradeCommands.append( + 'pre_meta.tables[%r].columns[%r].drop()' % (tn, col)) + downgradeCommands.append( + 'pre_meta.tables[%r].columns[%r].create()' % (tn, col)) + for col in td.columns_missing_from_B: + upgradeCommands.append( + 'post_meta.tables[%r].columns[%r].create()' % (tn, col)) + downgradeCommands.append( + 'post_meta.tables[%r].columns[%r].drop()' % (tn, col)) + for modelCol, databaseCol, modelDecl, databaseDecl in td.columns_different: upgradeCommands.append( 'assert False, "Can\'t alter columns: %s:%s=>%s"' % ( - modelTable, modelCol.name, databaseCol.name)) + tn, modelCol.name, databaseCol.name)) downgradeCommands.append( 'assert False, "Can\'t alter columns: %s:%s=>%s"' % ( - modelTable, modelCol.name, databaseCol.name)) - pre_command = ' meta.bind = migrate_engine' + tn, modelCol.name, databaseCol.name)) return ( '\n'.join(decls), - '\n'.join([pre_command] + ['%s%s' % (indent, line) for line in upgradeCommands]), - '\n'.join([pre_command] + ['%s%s' % (indent, line) for line in downgradeCommands])) + '\n'.join('%s%s' % (indent, line) for line in upgradeCommands), + '\n'.join('%s%s' % (indent, line) for line in downgradeCommands)) def _db_can_handle_this_change(self,td): + """Check if the database can handle going from B to A.""" + if (td.columns_missing_from_B and not td.columns_missing_from_A and not td.columns_different): - # Even sqlite can handle this. + # Even sqlite can handle column additions. return True else: return not self.engine.url.drivername.startswith('sqlite') - def applyModel(self): - """Apply model to current database.""" + def runB2A(self): + """Goes from B to A. + + Was: applyModel. Apply model (A) to current database (B). + """ meta = sqlalchemy.MetaData(self.engine) @@ -208,9 +238,9 @@ class ModelGenerator(object): dbTable = self.diff.metadataB.tables[tableName] td = self.diff.tables_different[tableName] - + if self._db_can_handle_this_change(td): - + for col in td.columns_missing_from_B: modelTable.columns[col].create() for col in td.columns_missing_from_A: @@ -252,3 +282,4 @@ class ModelGenerator(object): except: trans.rollback() raise + diff --git a/libs/migrate/versioning/repository.py b/libs/migrate/versioning/repository.py index 71f3912..6e2f678 100644 --- a/libs/migrate/versioning/repository.py +++ b/libs/migrate/versioning/repository.py @@ -115,6 +115,7 @@ class Repository(pathed.Pathed): options.setdefault('version_table', 'migrate_version') options.setdefault('repository_id', name) options.setdefault('required_dbs', []) + options.setdefault('use_timestamp_numbering', False) tmpl = open(os.path.join(tmpl_dir, cls._config)).read() ret = TempitaTemplate(tmpl).substitute(options) @@ -152,11 +153,14 @@ class Repository(pathed.Pathed): def create_script(self, description, **k): """API to :meth:`migrate.versioning.version.Collection.create_new_python_version`""" + + k['use_timestamp_numbering'] = self.use_timestamp_numbering self.versions.create_new_python_version(description, **k) - def create_script_sql(self, database, **k): + def create_script_sql(self, database, description, **k): """API to :meth:`migrate.versioning.version.Collection.create_new_sql_version`""" - self.versions.create_new_sql_version(database, **k) + k['use_timestamp_numbering'] = self.use_timestamp_numbering + self.versions.create_new_sql_version(database, description, **k) @property def latest(self): @@ -173,6 +177,13 @@ class Repository(pathed.Pathed): """Returns repository id specified in config""" return self.config.get('db_settings', 'repository_id') + @property + def use_timestamp_numbering(self): + """Returns use_timestamp_numbering specified in config""" + if self.config.has_option('db_settings', 'use_timestamp_numbering'): + return self.config.getboolean('db_settings', 'use_timestamp_numbering') + return False + def version(self, *p, **k): """API to :attr:`migrate.versioning.version.Collection.version`""" return self.versions.version(*p, **k) diff --git a/libs/migrate/versioning/schema.py b/libs/migrate/versioning/schema.py index 0aa6661..e4d9365 100644 --- a/libs/migrate/versioning/schema.py +++ b/libs/migrate/versioning/schema.py @@ -11,6 +11,7 @@ from sqlalchemy import exceptions as sa_exceptions from sqlalchemy.sql import bindparam from migrate import exceptions +from migrate.changeset import SQLA_07 from migrate.versioning import genmodel, schemadiff from migrate.versioning.repository import Repository from migrate.versioning.util import load_model @@ -57,14 +58,20 @@ class ControlledSchema(object): """ Remove version control from a database. """ - try: - self.table.drop() - except (sa_exceptions.SQLError): - raise exceptions.DatabaseNotControlledError(str(self.table)) + if SQLA_07: + try: + self.table.drop() + except sa_exceptions.DatabaseError: + raise exceptions.DatabaseNotControlledError(str(self.table)) + else: + try: + self.table.drop() + except (sa_exceptions.SQLError): + raise exceptions.DatabaseNotControlledError(str(self.table)) def changeset(self, version=None): """API to Changeset creation. - + Uses self.version for start version and engine.name to get database name. """ @@ -110,7 +117,7 @@ class ControlledSchema(object): diff = schemadiff.getDiffOfModelAgainstDatabase( model, self.engine, excludeTables=[self.repository.version_table] ) - genmodel.ModelGenerator(diff,self.engine).applyModel() + genmodel.ModelGenerator(diff,self.engine).runB2A() self.update_repository_table(self.version, int(self.repository.latest)) @@ -210,4 +217,4 @@ class ControlledSchema(object): diff = schemadiff.getDiffOfModelAgainstDatabase( MetaData(), engine, excludeTables=[repository.version_table] ) - return genmodel.ModelGenerator(diff, engine, declarative).toPython() + return genmodel.ModelGenerator(diff, engine, declarative).genBDefinition() diff --git a/libs/migrate/versioning/schemadiff.py b/libs/migrate/versioning/schemadiff.py index 17c2d8e..04cf83e 100644 --- a/libs/migrate/versioning/schemadiff.py +++ b/libs/migrate/versioning/schemadiff.py @@ -5,7 +5,6 @@ import logging import sqlalchemy -from migrate.changeset import SQLA_06 from sqlalchemy.types import Float log = logging.getLogger(__name__) @@ -17,8 +16,16 @@ def getDiffOfModelAgainstDatabase(metadata, engine, excludeTables=None): :return: object which will evaluate to :keyword:`True` if there \ are differences else :keyword:`False`. """ - return SchemaDiff(metadata, - sqlalchemy.MetaData(engine, reflect=True), + db_metadata = sqlalchemy.MetaData(engine, reflect=True) + + # sqlite will include a dynamically generated 'sqlite_sequence' table if + # there are autoincrement sequences in the database; this should not be + # compared. + if engine.dialect.name == 'sqlite': + if 'sqlite_sequence' in db_metadata.tables: + db_metadata.remove(db_metadata.tables['sqlite_sequence']) + + return SchemaDiff(metadata, db_metadata, labelA='model', labelB='database', excludeTables=excludeTables) @@ -39,11 +46,11 @@ class ColDiff(object): Container for differences in one :class:`~sqlalchemy.schema.Column` between two :class:`~sqlalchemy.schema.Table` instances, ``A`` and ``B``. - + .. attribute:: col_A The :class:`~sqlalchemy.schema.Column` object for A. - + .. attribute:: col_B The :class:`~sqlalchemy.schema.Column` object for B. @@ -51,15 +58,15 @@ class ColDiff(object): .. attribute:: type_A The most generic type of the :class:`~sqlalchemy.schema.Column` - object in A. - + object in A. + .. attribute:: type_B The most generic type of the :class:`~sqlalchemy.schema.Column` - object in A. - + object in A. + """ - + diff = False def __init__(self,col_A,col_B): @@ -87,10 +94,10 @@ class ColDiff(object): if not (A is None or B is None) and A!=B: self.diff=True return - + def __nonzero__(self): return self.diff - + class TableDiff(object): """ Container for differences in one :class:`~sqlalchemy.schema.Table` @@ -101,12 +108,12 @@ class TableDiff(object): A sequence of column names that were found in B but weren't in A. - + .. attribute:: columns_missing_from_B A sequence of column names that were found in A but weren't in B. - + .. attribute:: columns_different A dictionary containing information about columns that were @@ -126,7 +133,7 @@ class TableDiff(object): self.columns_missing_from_B or self.columns_different ) - + class SchemaDiff(object): """ Compute the difference between two :class:`~sqlalchemy.schema.MetaData` @@ -139,34 +146,34 @@ class SchemaDiff(object): The length of a :class:`SchemaDiff` will give the number of changes found, enabling it to be used much like a boolean in expressions. - + :param metadataA: First :class:`~sqlalchemy.schema.MetaData` to compare. - + :param metadataB: Second :class:`~sqlalchemy.schema.MetaData` to compare. - + :param labelA: The label to use in messages about the first - :class:`~sqlalchemy.schema.MetaData`. - - :param labelB: + :class:`~sqlalchemy.schema.MetaData`. + + :param labelB: The label to use in messages about the second - :class:`~sqlalchemy.schema.MetaData`. - + :class:`~sqlalchemy.schema.MetaData`. + :param excludeTables: A sequence of table names to exclude. - + .. attribute:: tables_missing_from_A A sequence of table names that were found in B but weren't in A. - + .. attribute:: tables_missing_from_B A sequence of table names that were found in A but weren't in B. - + .. attribute:: tables_different A dictionary containing information about tables that were found @@ -195,26 +202,26 @@ class SchemaDiff(object): self.tables_missing_from_B = sorted( A_table_names - B_table_names - excludeTables ) - + self.tables_different = {} for table_name in A_table_names.intersection(B_table_names): td = TableDiff() - + A_table = metadataA.tables[table_name] B_table = metadataB.tables[table_name] - + A_column_names = set(A_table.columns.keys()) B_column_names = set(B_table.columns.keys()) td.columns_missing_from_A = sorted( B_column_names - A_column_names ) - + td.columns_missing_from_B = sorted( A_column_names - B_column_names ) - + td.columns_different = {} for col_name in A_column_names.intersection(B_column_names): @@ -226,7 +233,7 @@ class SchemaDiff(object): if cd: td.columns_different[col_name]=cd - + # XXX - index and constraint differences should # be checked for here @@ -237,7 +244,7 @@ class SchemaDiff(object): ''' Summarize differences. ''' out = [] column_template =' %%%is: %%r' % self.label_width - + for names,label in ( (self.tables_missing_from_A,self.labelA), (self.tables_missing_from_B,self.labelB), @@ -248,7 +255,7 @@ class SchemaDiff(object): label,', '.join(sorted(names)) ) ) - + for name,td in sorted(self.tables_different.items()): out.append( ' table with differences: %s' % name @@ -267,7 +274,7 @@ class SchemaDiff(object): out.append(' column with differences: %s' % name) out.append(column_template % (self.labelA,cd.col_A)) out.append(column_template % (self.labelB,cd.col_B)) - + if out: out.insert(0, 'Schema diffs:') return '\n'.join(out) diff --git a/libs/migrate/versioning/script/py.py b/libs/migrate/versioning/script/py.py index 35fe4aa..3a090d4 100644 --- a/libs/migrate/versioning/script/py.py +++ b/libs/migrate/versioning/script/py.py @@ -25,7 +25,7 @@ class PythonScript(base.BaseScript): @classmethod def create(cls, path, **opts): """Create an empty migration script at specified path - + :returns: :class:`PythonScript instance `""" cls.require_notfound(path) @@ -38,7 +38,7 @@ class PythonScript(base.BaseScript): def make_update_script_for_model(cls, engine, oldmodel, model, repository, **opts): """Create a migration script based on difference between two SA models. - + :param repository: path to migrate repository :param oldmodel: dotted.module.name:SAClass or SAClass object :param model: dotted.module.name:SAClass or SAClass object @@ -50,7 +50,7 @@ class PythonScript(base.BaseScript): :returns: Upgrade / Downgrade script :rtype: string """ - + if isinstance(repository, basestring): # oh dear, an import cycle! from migrate.versioning.repository import Repository @@ -61,12 +61,12 @@ class PythonScript(base.BaseScript): # Compute differences. diff = schemadiff.getDiffOfModelAgainstModel( - oldmodel, model, + oldmodel, excludeTables=[repository.version_table]) # TODO: diff can be False (there is no difference?) decls, upgradeCommands, downgradeCommands = \ - genmodel.ModelGenerator(diff,engine).toUpgradeDowngradePython() + genmodel.ModelGenerator(diff,engine).genB2AMigration() # Store differences into file. src = Template(opts.pop('templates_path', None)).get_script(opts.pop('templates_theme', None)) @@ -86,7 +86,7 @@ class PythonScript(base.BaseScript): @classmethod def verify_module(cls, path): """Ensure path is a valid script - + :param path: Script location :type path: string :raises: :exc:`InvalidScriptError ` @@ -101,7 +101,7 @@ class PythonScript(base.BaseScript): return module def preview_sql(self, url, step, **args): - """Mocks SQLAlchemy Engine to store all executed calls in a string + """Mocks SQLAlchemy Engine to store all executed calls in a string and runs :meth:`PythonScript.run ` :returns: SQL file @@ -119,7 +119,7 @@ class PythonScript(base.BaseScript): return go(url, step, **args) def run(self, engine, step): - """Core method of Script file. + """Core method of Script file. Exectues :func:`update` or :func:`downgrade` functions :param engine: SQLAlchemy Engine diff --git a/libs/migrate/versioning/template.py b/libs/migrate/versioning/template.py index 0688934..182898a 100644 --- a/libs/migrate/versioning/template.py +++ b/libs/migrate/versioning/template.py @@ -38,7 +38,6 @@ class Template(pathed.Pathed): if `path` is not provided. """ pkg = 'migrate.versioning.templates' - _manage = 'manage.py_tmpl' def __new__(cls, path=None): if path is None: diff --git a/libs/migrate/versioning/templates/manage.py_tmpl b/libs/migrate/versioning/templates/manage.py_tmpl deleted file mode 100644 index e6fc4ba..0000000 --- a/libs/migrate/versioning/templates/manage.py_tmpl +++ /dev/null @@ -1,5 +0,0 @@ -#!/usr/bin/env python -from migrate.versioning.shell import main - -if __name__ == '__main__': - main(%(defaults)s) diff --git a/libs/migrate/versioning/templates/manage/default.py_tmpl b/libs/migrate/versioning/templates/manage/default.py_tmpl index cc4d0e8..f6d75c5 100644 --- a/libs/migrate/versioning/templates/manage/default.py_tmpl +++ b/libs/migrate/versioning/templates/manage/default.py_tmpl @@ -7,4 +7,6 @@ del _vars['__template_name__'] _vars.pop('repository_name', None) defaults = ", ".join(["%s='%s'" % var for var in _vars.iteritems()]) }} -main({{ defaults }}) + +if __name__ == '__main__': + main({{ defaults }}) diff --git a/libs/migrate/versioning/templates/manage/pylons.py_tmpl b/libs/migrate/versioning/templates/manage/pylons.py_tmpl index 475b8ce..cc2f788 100644 --- a/libs/migrate/versioning/templates/manage/pylons.py_tmpl +++ b/libs/migrate/versioning/templates/manage/pylons.py_tmpl @@ -26,4 +26,5 @@ conf_dict = ConfigLoader(conf_path).parser._sections['app:main'] # migrate supports passing url as an existing Engine instance (since 0.6.0) # usage: migrate -c path/to/config.ini COMMANDS -main(url=engine_from_config(conf_dict), repository=migrations.__path__[0],{{ defaults }}) +if __name__ == '__main__': + main(url=engine_from_config(conf_dict), repository=migrations.__path__[0],{{ defaults }}) diff --git a/libs/migrate/versioning/templates/repository/default/migrate.cfg b/libs/migrate/versioning/templates/repository/default/migrate.cfg index 1dc6ff6..dae0612 100644 --- a/libs/migrate/versioning/templates/repository/default/migrate.cfg +++ b/libs/migrate/versioning/templates/repository/default/migrate.cfg @@ -18,3 +18,8 @@ version_table={{ locals().pop('version_table') }} # be using to ensure your updates to that database work properly. # This must be a list; example: ['postgres','sqlite'] required_dbs={{ locals().pop('required_dbs') }} + +# When creating new change scripts, Migrate will stamp the new script with +# a version number. By default this is latest_version + 1. You can set this +# to 'true' to tell Migrate to use the UTC timestamp instead. +use_timestamp_numbering={{ locals().pop('use_timestamp_numbering') }} diff --git a/libs/migrate/versioning/templates/repository/pylons/migrate.cfg b/libs/migrate/versioning/templates/repository/pylons/migrate.cfg index 1dc6ff6..dae0612 100644 --- a/libs/migrate/versioning/templates/repository/pylons/migrate.cfg +++ b/libs/migrate/versioning/templates/repository/pylons/migrate.cfg @@ -18,3 +18,8 @@ version_table={{ locals().pop('version_table') }} # be using to ensure your updates to that database work properly. # This must be a list; example: ['postgres','sqlite'] required_dbs={{ locals().pop('required_dbs') }} + +# When creating new change scripts, Migrate will stamp the new script with +# a version number. By default this is latest_version + 1. You can set this +# to 'true' to tell Migrate to use the UTC timestamp instead. +use_timestamp_numbering={{ locals().pop('use_timestamp_numbering') }} diff --git a/libs/migrate/versioning/templates/script/default.py_tmpl b/libs/migrate/versioning/templates/script/default.py_tmpl index 711899c..58d874b 100644 --- a/libs/migrate/versioning/templates/script/default.py_tmpl +++ b/libs/migrate/versioning/templates/script/default.py_tmpl @@ -1,11 +1,13 @@ from sqlalchemy import * from migrate import * + def upgrade(migrate_engine): - # Upgrade operations go here. Don't create your own engine; bind migrate_engine - # to your metadata + # Upgrade operations go here. Don't create your own engine; bind + # migrate_engine to your metadata pass + def downgrade(migrate_engine): # Operations to reverse the above upgrade go here. pass diff --git a/libs/migrate/versioning/templates/script/pylons.py_tmpl b/libs/migrate/versioning/templates/script/pylons.py_tmpl index 711899c..58d874b 100644 --- a/libs/migrate/versioning/templates/script/pylons.py_tmpl +++ b/libs/migrate/versioning/templates/script/pylons.py_tmpl @@ -1,11 +1,13 @@ from sqlalchemy import * from migrate import * + def upgrade(migrate_engine): - # Upgrade operations go here. Don't create your own engine; bind migrate_engine - # to your metadata + # Upgrade operations go here. Don't create your own engine; bind + # migrate_engine to your metadata pass + def downgrade(migrate_engine): # Operations to reverse the above upgrade go here. pass diff --git a/libs/migrate/versioning/version.py b/libs/migrate/versioning/version.py index a09b8cd..d5a5be9 100644 --- a/libs/migrate/versioning/version.py +++ b/libs/migrate/versioning/version.py @@ -8,6 +8,7 @@ import logging from migrate import exceptions from migrate.versioning import pathed, script +from datetime import datetime log = logging.getLogger(__name__) @@ -88,9 +89,15 @@ class Collection(pathed.Pathed): """:returns: Latest version in Collection""" return max([VerNum(0)] + self.versions.keys()) + def _next_ver_num(self, use_timestamp_numbering): + if use_timestamp_numbering == True: + return VerNum(int(datetime.utcnow().strftime('%Y%m%d%H%M%S'))) + else: + return self.latest + 1 + def create_new_python_version(self, description, **k): """Create Python files for new version""" - ver = self.latest + 1 + ver = self._next_ver_num(k.pop('use_timestamp_numbering', False)) extra = str_to_filename(description) if extra: @@ -105,14 +112,22 @@ class Collection(pathed.Pathed): script.PythonScript.create(filepath, **k) self.versions[ver] = Version(ver, self.path, [filename]) - def create_new_sql_version(self, database, **k): + def create_new_sql_version(self, database, description, **k): """Create SQL files for new version""" - ver = self.latest + 1 + ver = self._next_ver_num(k.pop('use_timestamp_numbering', False)) self.versions[ver] = Version(ver, self.path, []) + extra = str_to_filename(description) + + if extra: + if extra == '_': + extra = '' + elif not extra.startswith('_'): + extra = '_%s' % extra + # Create new files. for op in ('upgrade', 'downgrade'): - filename = '%03d_%s_%s.sql' % (ver, database, op) + filename = '%03d%s_%s_%s.sql' % (ver, extra, database, op) filepath = self._version_path(filename) script.SqlScript.create(filepath, **k) self.versions[ver].add_script(filepath) @@ -176,18 +191,26 @@ class Version(object): elif path.endswith(Extensions.sql): self._add_script_sql(path) - SQL_FILENAME = re.compile(r'^(\d+)_([^_]+)_([^_]+).sql') + SQL_FILENAME = re.compile(r'^.*\.sql') def _add_script_sql(self, path): basename = os.path.basename(path) match = self.SQL_FILENAME.match(basename) - + if match: - version, dbms, op = match.group(1), match.group(2), match.group(3) + basename = basename.replace('.sql', '') + parts = basename.split('_') + if len(parts) < 3: + raise exceptions.ScriptError( + "Invalid SQL script name %s " % basename + \ + "(needs to be ###_description_database_operation.sql)") + version = parts[0] + op = parts[-1] + dbms = parts[-2] else: raise exceptions.ScriptError( "Invalid SQL script name %s " % basename + \ - "(needs to be ###_database_operation.sql)") + "(needs to be ###_description_database_operation.sql)") # File the script into a dictionary self.sql.setdefault(dbms, {})[op] = script.SqlScript(path) diff --git a/libs/pytwitter/__init__.py b/libs/pytwitter/__init__.py index 450c4ae..ce8a9d1 100644 --- a/libs/pytwitter/__init__.py +++ b/libs/pytwitter/__init__.py @@ -17,7 +17,7 @@ '''A library that provides a Python interface to the Twitter API''' __author__ = 'python-twitter@googlegroups.com' -__version__ = '0.8-devel' +__version__ = '0.8.3' import base64 @@ -30,6 +30,7 @@ import sys import tempfile import textwrap import time +import calendar import urllib import urllib2 import urlparse @@ -86,9 +87,9 @@ class TwitterError(Exception): class Status(object): '''A class representing the Status structure used by the twitter API. - + The Status structure exposes the following properties: - + status.created_at status.created_at_in_seconds # read only status.favorited @@ -102,6 +103,13 @@ class Status(object): status.location status.relative_created_at # read only status.user + status.urls + status.user_mentions + status.hashtags + status.geo + status.place + status.coordinates + status.contributors ''' def __init__(self, created_at = None, @@ -115,7 +123,17 @@ class Status(object): in_reply_to_status_id = None, truncated = None, source = None, - now = None): + now = None, + urls = None, + user_mentions = None, + hashtags = None, + geo = None, + place = None, + coordinates = None, + contributors = None, + retweeted = None, + retweeted_status = None, + retweet_count = None): '''An object to hold a Twitter status message. This class is normally instantiated by the twitter.Api class and @@ -124,18 +142,34 @@ class Status(object): Note: Dates are posted in the form "Sat Jan 27 04:17:38 +0000 2007" Args: - created_at: The time this status message was posted - favorited: Whether this is a favorite of the authenticated user - id: The unique id of this status message - text: The text of this status message - location: the geolocation string associated with this message + created_at: + The time this status message was posted. [Optional] + favorited: + Whether this is a favorite of the authenticated user. [Optional] + id: + The unique id of this status message. [Optional] + text: + The text of this status message. [Optional] + location: + the geolocation string associated with this message. [Optional] relative_created_at: - A human readable string representing the posting time + A human readable string representing the posting time. [Optional] user: - A twitter.User instance representing the person posting the message + A twitter.User instance representing the person posting the + message. [Optional] now: - The current time, if the client choses to set it. Defaults to the - wall clock time. + The current time, if the client choses to set it. + Defaults to the wall clock time. [Optional] + urls: + user_mentions: + hashtags: + geo: + place: + coordinates: + contributors: + retweeted: + retweeted_status: + retweet_count: ''' self.created_at = created_at self.favorited = favorited @@ -148,7 +182,17 @@ class Status(object): self.in_reply_to_user_id = in_reply_to_user_id self.in_reply_to_status_id = in_reply_to_status_id self.truncated = truncated + self.retweeted = retweeted self.source = source + self.urls = urls + self.user_mentions = user_mentions + self.hashtags = hashtags + self.geo = geo + self.place = place + self.coordinates = coordinates + self.contributors = contributors + self.retweeted_status = retweeted_status + self.retweet_count = retweet_count def GetCreatedAt(self): '''Get the time this status message was posted. @@ -162,7 +206,8 @@ class Status(object): '''Set the time this status message was posted. Args: - created_at: The time this status message was created + created_at: + The time this status message was created ''' self._created_at = created_at @@ -193,7 +238,8 @@ class Status(object): '''Set the favorited state of this status message. Args: - favorited: boolean True/False favorited state of this status message + favorited: + boolean True/False favorited state of this status message ''' self._favorited = favorited @@ -212,7 +258,8 @@ class Status(object): '''Set the unique id of this status message. Args: - id: The unique id of this status message + id: + The unique id of this status message ''' self._id = id @@ -226,7 +273,7 @@ class Status(object): self._in_reply_to_screen_name = in_reply_to_screen_name in_reply_to_screen_name = property(GetInReplyToScreenName, SetInReplyToScreenName, - doc = '') + doc = '') def GetInReplyToUserId(self): return self._in_reply_to_user_id @@ -235,7 +282,7 @@ class Status(object): self._in_reply_to_user_id = in_reply_to_user_id in_reply_to_user_id = property(GetInReplyToUserId, SetInReplyToUserId, - doc = '') + doc = '') def GetInReplyToStatusId(self): return self._in_reply_to_status_id @@ -244,7 +291,7 @@ class Status(object): self._in_reply_to_status_id = in_reply_to_status_id in_reply_to_status_id = property(GetInReplyToStatusId, SetInReplyToStatusId, - doc = '') + doc = '') def GetTruncated(self): return self._truncated @@ -253,7 +300,16 @@ class Status(object): self._truncated = truncated truncated = property(GetTruncated, SetTruncated, - doc = '') + doc = '') + + def GetRetweeted(self): + return self._retweeted + + def SetRetweeted(self, retweeted): + self._retweeted = retweeted + + retweeted = property(GetRetweeted, SetRetweeted, + doc = '') def GetSource(self): return self._source @@ -262,7 +318,7 @@ class Status(object): self._source = source source = property(GetSource, SetSource, - doc = '') + doc = '') def GetText(self): '''Get the text of this status message. @@ -276,7 +332,8 @@ class Status(object): '''Set the text of this status message. Args: - text: The text of this status message + text: + The text of this status message ''' self._text = text @@ -295,7 +352,8 @@ class Status(object): '''Set the geolocation associated with this status message Args: - location: The geolocation string of this status message + location: + The geolocation string of this status message ''' self._location = location @@ -329,7 +387,7 @@ class Status(object): return 'about %d days ago' % (delta / (60 * 60 * 24)) relative_created_at = property(GetRelativeCreatedAt, - doc = 'Get a human readable string representing' + doc = 'Get a human readable string representing ' 'the posting time') def GetUser(self): @@ -344,7 +402,8 @@ class Status(object): '''Set a twitter.User reprenting the entity posting this status message. Args: - user: A twitter.User reprenting the entity posting this status message + user: + A twitter.User reprenting the entity posting this status message ''' self._user = user @@ -373,13 +432,67 @@ class Status(object): the object was instantiated. Args: - now: The wallclock time for this instance. + now: + The wallclock time for this instance. ''' self._now = now now = property(GetNow, SetNow, doc = 'The wallclock time for this status instance.') + def GetGeo(self): + return self._geo + + def SetGeo(self, geo): + self._geo = geo + + geo = property(GetGeo, SetGeo, + doc = '') + + def GetPlace(self): + return self._place + + def SetPlace(self, place): + self._place = place + + place = property(GetPlace, SetPlace, + doc = '') + + def GetCoordinates(self): + return self._coordinates + + def SetCoordinates(self, coordinates): + self._coordinates = coordinates + + coordinates = property(GetCoordinates, SetCoordinates, + doc = '') + + def GetContributors(self): + return self._contributors + + def SetContributors(self, contributors): + self._contributors = contributors + + contributors = property(GetContributors, SetContributors, + doc = '') + + def GetRetweeted_status(self): + return self._retweeted_status + + def SetRetweeted_status(self, retweeted_status): + self._retweeted_status = retweeted_status + + retweeted_status = property(GetRetweeted_status, SetRetweeted_status, + doc = '') + + def GetRetweetCount(self): + return self._retweet_count + + def SetRetweetCount(self, retweet_count): + self._retweet_count = retweet_count + + retweet_count = property(GetRetweetCount, SetRetweetCount, + doc = '') def __ne__(self, other): return not self.__eq__(other) @@ -396,8 +509,15 @@ class Status(object): self.in_reply_to_user_id == other.in_reply_to_user_id and \ self.in_reply_to_status_id == other.in_reply_to_status_id and \ self.truncated == other.truncated and \ + self.retweeted == other.retweeted and \ self.favorited == other.favorited and \ - self.source == other.source + self.source == other.source and \ + self.geo == other.geo and \ + self.place == other.place and \ + self.coordinates == other.coordinates and \ + self.contributors == other.contributors and \ + self.retweeted_status == other.retweeted_status and \ + self.retweet_count == other.retweet_count except AttributeError: return False @@ -448,10 +568,30 @@ class Status(object): data['in_reply_to_status_id'] = self.in_reply_to_status_id if self.truncated is not None: data['truncated'] = self.truncated + if self.retweeted is not None: + data['retweeted'] = self.retweeted if self.favorited is not None: data['favorited'] = self.favorited if self.source: data['source'] = self.source + if self.geo: + data['geo'] = self.geo + if self.place: + data['place'] = self.place + if self.coordinates: + data['coordinates'] = self.coordinates + if self.contributors: + data['contributors'] = self.contributors + if self.hashtags: + data['hashtags'] = [h.text for h in self.hashtags] + if self.retweeted_status: + data['retweeted_status'] = self.retweeted_status.AsDict() + if self.retweet_count: + data['retweet_count'] = self.retweet_count + if self.urls: + data['urls'] = dict([(url.url, url.expanded_url) for url in self.urls]) + if self.user_mentions: + data['user_mentions'] = [um.AsDict() for um in self.user_mentions] return data @staticmethod @@ -467,6 +607,20 @@ class Status(object): user = User.NewFromJsonDict(data['user']) else: user = None + if 'retweeted_status' in data: + retweeted_status = Status.NewFromJsonDict(data['retweeted_status']) + else: + retweeted_status = None + urls = None + user_mentions = None + hashtags = None + if 'entities' in data: + if 'urls' in data['entities']: + urls = [Url.NewFromJsonDict(u) for u in data['entities']['urls']] + if 'user_mentions' in data['entities']: + user_mentions = [User.NewFromJsonDict(u) for u in data['entities']['user_mentions']] + if 'hashtags' in data['entities']: + hashtags = [Hashtag.NewFromJsonDict(h) for h in data['entities']['hashtags']] return Status(created_at = data.get('created_at', None), favorited = data.get('favorited', None), id = data.get('id', None), @@ -476,8 +630,18 @@ class Status(object): in_reply_to_user_id = data.get('in_reply_to_user_id', None), in_reply_to_status_id = data.get('in_reply_to_status_id', None), truncated = data.get('truncated', None), + retweeted = data.get('retweeted', None), source = data.get('source', None), - user = user) + user = user, + urls = urls, + user_mentions = user_mentions, + hashtags = hashtags, + geo = data.get('geo', None), + place = data.get('place', None), + coordinates = data.get('coordinates', None), + contributors = data.get('contributors', None), + retweeted_status = retweeted_status, + retweet_count = data.get('retweet_count', None)) class User(object): @@ -506,6 +670,13 @@ class User(object): user.followers_count user.friends_count user.favourites_count + user.geo_enabled + user.verified + user.lang + user.notifications + user.contributors_enabled + user.created_at + user.listed_count ''' def __init__(self, id = None, @@ -528,7 +699,14 @@ class User(object): statuses_count = None, favourites_count = None, url = None, - status = None): + status = None, + geo_enabled = None, + verified = None, + lang = None, + notifications = None, + contributors_enabled = None, + created_at = None, + listed_count = None): self.id = id self.name = name self.screen_name = screen_name @@ -550,7 +728,13 @@ class User(object): self.favourites_count = favourites_count self.url = url self.status = status - + self.geo_enabled = geo_enabled + self.verified = verified + self.lang = lang + self.notifications = notifications + self.contributors_enabled = contributors_enabled + self.created_at = created_at + self.listed_count = listed_count def GetId(self): '''Get the unique id of this user. @@ -591,23 +775,23 @@ class User(object): doc = 'The real name of this user.') def GetScreenName(self): - '''Get the short username of this user. + '''Get the short twitter name of this user. Returns: - The short username of this user + The short twitter name of this user ''' return self._screen_name def SetScreenName(self, screen_name): - '''Set the short username of this user. + '''Set the short twitter name of this user. Args: - screen_name: the short username of this user + screen_name: the short twitter name of this user ''' self._screen_name = screen_name screen_name = property(GetScreenName, SetScreenName, - doc = 'The short username of this user.') + doc = 'The short twitter name of this user.') def GetLocation(self): '''Get the geographic location of this user. @@ -773,7 +957,8 @@ class User(object): '''Sets the user's time zone string. Args: - time_zone: The descriptive time zone to assign for the user. + time_zone: + The descriptive time zone to assign for the user. ''' self._time_zone = time_zone @@ -791,16 +976,17 @@ class User(object): '''Set the latest twitter.Status of this user. Args: - status: The latest twitter.Status of this user + status: + The latest twitter.Status of this user ''' self._status = status status = property(GetStatus, SetStatus, - doc = 'The latest twitter.Status of this user.') + doc = 'The latest twitter.Status of this user.') def GetFriendsCount(self): '''Get the friend count for this user. - + Returns: The number of users this user has befriended. ''' @@ -810,16 +996,37 @@ class User(object): '''Set the friend count for this user. Args: - count: The number of users this user has befriended. + count: + The number of users this user has befriended. ''' self._friends_count = count friends_count = property(GetFriendsCount, SetFriendsCount, - doc = 'The number of friends for this user.') + doc = 'The number of friends for this user.') + + def GetListedCount(self): + '''Get the listed count for this user. + + Returns: + The number of lists this user belongs to. + ''' + return self._listed_count + + def SetListedCount(self, count): + '''Set the listed count for this user. + + Args: + count: + The number of lists this user belongs to. + ''' + self._listed_count = count + + listed_count = property(GetListedCount, SetListedCount, + doc = 'The number of lists this user belongs to.') def GetFollowersCount(self): '''Get the follower count for this user. - + Returns: The number of users following this user. ''' @@ -829,16 +1036,17 @@ class User(object): '''Set the follower count for this user. Args: - count: The number of users following this user. + count: + The number of users following this user. ''' self._followers_count = count followers_count = property(GetFollowersCount, SetFollowersCount, - doc = 'The number of users following this user.') + doc = 'The number of users following this user.') def GetStatusesCount(self): '''Get the number of status updates for this user. - + Returns: The number of status updates for this user. ''' @@ -848,16 +1056,17 @@ class User(object): '''Set the status update count for this user. Args: - count: The number of updates for this user. + count: + The number of updates for this user. ''' self._statuses_count = count statuses_count = property(GetStatusesCount, SetStatusesCount, - doc = 'The number of updates for this user.') + doc = 'The number of updates for this user.') def GetFavouritesCount(self): '''Get the number of favourites for this user. - + Returns: The number of favourites for this user. ''' @@ -867,12 +1076,133 @@ class User(object): '''Set the favourite count for this user. Args: - count: The number of favourites for this user. + count: + The number of favourites for this user. ''' self._favourites_count = count favourites_count = property(GetFavouritesCount, SetFavouritesCount, - doc = 'The number of favourites for this user.') + doc = 'The number of favourites for this user.') + + def GetGeoEnabled(self): + '''Get the setting of geo_enabled for this user. + + Returns: + True/False if Geo tagging is enabled + ''' + return self._geo_enabled + + def SetGeoEnabled(self, geo_enabled): + '''Set the latest twitter.geo_enabled of this user. + + Args: + geo_enabled: + True/False if Geo tagging is to be enabled + ''' + self._geo_enabled = geo_enabled + + geo_enabled = property(GetGeoEnabled, SetGeoEnabled, + doc = 'The value of twitter.geo_enabled for this user.') + + def GetVerified(self): + '''Get the setting of verified for this user. + + Returns: + True/False if user is a verified account + ''' + return self._verified + + def SetVerified(self, verified): + '''Set twitter.verified for this user. + + Args: + verified: + True/False if user is a verified account + ''' + self._verified = verified + + verified = property(GetVerified, SetVerified, + doc = 'The value of twitter.verified for this user.') + + def GetLang(self): + '''Get the setting of lang for this user. + + Returns: + language code of the user + ''' + return self._lang + + def SetLang(self, lang): + '''Set twitter.lang for this user. + + Args: + lang: + language code for the user + ''' + self._lang = lang + + lang = property(GetLang, SetLang, + doc = 'The value of twitter.lang for this user.') + + def GetNotifications(self): + '''Get the setting of notifications for this user. + + Returns: + True/False for the notifications setting of the user + ''' + return self._notifications + + def SetNotifications(self, notifications): + '''Set twitter.notifications for this user. + + Args: + notifications: + True/False notifications setting for the user + ''' + self._notifications = notifications + + notifications = property(GetNotifications, SetNotifications, + doc = 'The value of twitter.notifications for this user.') + + def GetContributorsEnabled(self): + '''Get the setting of contributors_enabled for this user. + + Returns: + True/False contributors_enabled of the user + ''' + return self._contributors_enabled + + def SetContributorsEnabled(self, contributors_enabled): + '''Set twitter.contributors_enabled for this user. + + Args: + contributors_enabled: + True/False contributors_enabled setting for the user + ''' + self._contributors_enabled = contributors_enabled + + contributors_enabled = property(GetContributorsEnabled, SetContributorsEnabled, + doc = 'The value of twitter.contributors_enabled for this user.') + + def GetCreatedAt(self): + '''Get the setting of created_at for this user. + + Returns: + created_at value of the user + ''' + return self._created_at + + def SetCreatedAt(self, created_at): + '''Set twitter.created_at for this user. + + Args: + created_at: + created_at value for the user + ''' + self._created_at = created_at + + created_at = property(GetCreatedAt, SetCreatedAt, + doc = 'The value of twitter.created_at for this user.') def __ne__(self, other): return not self.__eq__(other) @@ -900,7 +1230,15 @@ class User(object): self.followers_count == other.followers_count and \ self.favourites_count == other.favourites_count and \ self.friends_count == other.friends_count and \ - self.status == other.status + self.status == other.status and \ + self.geo_enabled == other.geo_enabled and \ + self.verified == other.verified and \ + self.lang == other.lang and \ + self.notifications == other.notifications and \ + self.contributors_enabled == other.contributors_enabled and \ + self.created_at == other.created_at and \ + self.listed_count == other.listed_count + except AttributeError: return False @@ -971,6 +1309,21 @@ class User(object): data['statuses_count'] = self.statuses_count if self.favourites_count: data['favourites_count'] = self.favourites_count + if self.geo_enabled: + data['geo_enabled'] = self.geo_enabled + if self.verified: + data['verified'] = self.verified + if self.lang: + data['lang'] = self.lang + if self.notifications: + data['notifications'] = self.notifications + if self.contributors_enabled: + data['contributors_enabled'] = self.contributors_enabled + if self.created_at: + data['created_at'] = self.created_at + if self.listed_count: + data['listed_count'] = self.listed_count + return data @staticmethod @@ -978,7 +1331,9 @@ class User(object): '''Create a new instance based on a JSON dict. Args: - data: A JSON dict, as converted from the JSON in the twitter API + data: + A JSON dict, as converted from the JSON in the twitter API + Returns: A twitter.User instance ''' @@ -1006,13 +1361,20 @@ class User(object): utc_offset = data.get('utc_offset', None), time_zone = data.get('time_zone', None), url = data.get('url', None), - status = status) + status = status, + geo_enabled = data.get('geo_enabled', None), + verified = data.get('verified', None), + lang = data.get('lang', None), + notifications = data.get('notifications', None), + contributors_enabled = data.get('contributors_enabled', None), + created_at = data.get('created_at', None), + listed_count = data.get('listed_count', None)) class List(object): '''A class representing the List structure used by the twitter API. - + The List structure exposes the following properties: - + list.id list.name list.slug @@ -1060,7 +1422,8 @@ class List(object): '''Set the unique id of this list. Args: - id: The unique id of this list. + id: + The unique id of this list. ''' self._id = id @@ -1079,7 +1442,8 @@ class List(object): '''Set the real name of this list. Args: - name: The real name of this list + name: + The real name of this list ''' self._name = name @@ -1098,7 +1462,8 @@ class List(object): '''Set the slug of this list. Args: - slug: The slug of this list. + slug: + The slug of this list. ''' self._slug = slug @@ -1117,7 +1482,8 @@ class List(object): '''Set the description of this list. Args: - description: The description of this list. + description: + The description of this list. ''' self._description = description @@ -1136,7 +1502,8 @@ class List(object): '''Set the full_name of this list. Args: - full_name: The full_name of this list. + full_name: + The full_name of this list. ''' self._full_name = full_name @@ -1155,7 +1522,8 @@ class List(object): '''Set the mode of this list. Args: - mode: The mode of this list. + mode: + The mode of this list. ''' self._mode = mode @@ -1174,7 +1542,8 @@ class List(object): '''Set the uri of this list. Args: - uri: The uri of this list. + uri: + The uri of this list. ''' self._uri = uri @@ -1193,7 +1562,8 @@ class List(object): '''Set the member_count of this list. Args: - member_count: The member_count of this list. + member_count: + The member_count of this list. ''' self._member_count = member_count @@ -1212,7 +1582,8 @@ class List(object): '''Set the subscriber_count of this list. Args: - subscriber_count: The subscriber_count of this list. + subscriber_count: + The subscriber_count of this list. ''' self._subscriber_count = subscriber_count @@ -1231,7 +1602,8 @@ class List(object): '''Set the following status of this list. Args: - following: The following of this list. + following: + The following of this list. ''' self._following = following @@ -1250,7 +1622,8 @@ class List(object): '''Set the user of this list. Args: - user: The owner of this list. + user: + The owner of this list. ''' self._user = user @@ -1334,7 +1707,9 @@ class List(object): '''Create a new instance based on a JSON dict. Args: - data: A JSON dict, as converted from the JSON in the twitter API + data: + A JSON dict, as converted from the JSON in the twitter API + Returns: A twitter.List instance ''' @@ -1356,9 +1731,9 @@ class List(object): class DirectMessage(object): '''A class representing the DirectMessage structure used by the twitter API. - + The DirectMessage structure exposes the following properties: - + direct_message.id direct_message.created_at direct_message.created_at_in_seconds # read only @@ -1385,13 +1760,20 @@ class DirectMessage(object): Note: Dates are posted in the form "Sat Jan 27 04:17:38 +0000 2007" Args: - id: The unique id of this direct message - created_at: The time this direct message was posted - sender_id: The id of the twitter user that sent this message - sender_screen_name: The name of the twitter user that sent this message - recipient_id: The id of the twitter that received this message - recipient_screen_name: The name of the twitter that received this message - text: The text of this direct message + id: + The unique id of this direct message. [Optional] + created_at: + The time this direct message was posted. [Optional] + sender_id: + The id of the twitter user that sent this message. [Optional] + sender_screen_name: + The name of the twitter user that sent this message. [Optional] + recipient_id: + The id of the twitter that received this message. [Optional] + recipient_screen_name: + The name of the twitter that received this message. [Optional] + text: + The text of this direct message. [Optional] ''' self.id = id self.created_at = created_at @@ -1413,7 +1795,8 @@ class DirectMessage(object): '''Set the unique id of this direct message. Args: - id: The unique id of this direct message + id: + The unique id of this direct message ''' self._id = id @@ -1432,7 +1815,8 @@ class DirectMessage(object): '''Set the time this direct message was posted. Args: - created_at: The time this direct message was created + created_at: + The time this direct message was created ''' self._created_at = created_at @@ -1463,7 +1847,8 @@ class DirectMessage(object): '''Set the unique sender id of this direct message. Args: - sender id: The unique sender id of this direct message + sender_id: + The unique sender id of this direct message ''' self._sender_id = sender_id @@ -1482,7 +1867,8 @@ class DirectMessage(object): '''Set the unique sender screen name of this direct message. Args: - sender_screen_name: The unique sender screen name of this direct message + sender_screen_name: + The unique sender screen name of this direct message ''' self._sender_screen_name = sender_screen_name @@ -1501,7 +1887,8 @@ class DirectMessage(object): '''Set the unique recipient id of this direct message. Args: - recipient id: The unique recipient id of this direct message + recipient_id: + The unique recipient id of this direct message ''' self._recipient_id = recipient_id @@ -1520,7 +1907,8 @@ class DirectMessage(object): '''Set the unique recipient screen name of this direct message. Args: - recipient_screen_name: The unique recipient screen name of this direct message + recipient_screen_name: + The unique recipient screen name of this direct message ''' self._recipient_screen_name = recipient_screen_name @@ -1539,7 +1927,8 @@ class DirectMessage(object): '''Set the text of this direct message. Args: - text: The text of this direct message + text: + The text of this direct message ''' self._text = text @@ -1610,7 +1999,9 @@ class DirectMessage(object): '''Create a new instance based on a JSON dict. Args: - data: A JSON dict, as converted from the JSON in the twitter API + data: + A JSON dict, as converted from the JSON in the twitter API + Returns: A twitter.DirectMessage instance ''' @@ -1622,6 +2013,88 @@ class DirectMessage(object): id = data.get('id', None), recipient_screen_name = data.get('recipient_screen_name', None)) +class Hashtag(object): + ''' A class represeinting a twitter hashtag + ''' + def __init__(self, + text = None): + self.text = text + + @staticmethod + def NewFromJsonDict(data): + '''Create a new instance based on a JSON dict. + + Args: + data: + A JSON dict, as converted from the JSON in the twitter API + + Returns: + A twitter.Hashtag instance + ''' + return Hashtag(text = data.get('text', None)) + +class Trend(object): + ''' A class representing a trending topic + ''' + def __init__(self, name = None, query = None, timestamp = None): + self.name = name + self.query = query + self.timestamp = timestamp + + def __str__(self): + return 'Name: %s\nQuery: %s\nTimestamp: %s\n' % (self.name, self.query, self.timestamp) + + def __ne__(self, other): + return not self.__eq__(other) + + def __eq__(self, other): + try: + return other and \ + self.name == other.name and \ + self.query == other.query and \ + self.timestamp == other.timestamp + except AttributeError: + return False + + @staticmethod + def NewFromJsonDict(data, timestamp = None): + '''Create a new instance based on a JSON dict + + Args: + data: + A JSON dict + timestamp: + Gets set as the timestamp property of the new object + + Returns: + A twitter.Trend object + ''' + return Trend(name = data.get('name', None), + query = data.get('query', None), + timestamp = timestamp) + +class Url(object): + '''A class representing an URL contained in a tweet''' + def __init__(self, + url = None, + expanded_url = None): + self.url = url + self.expanded_url = expanded_url + + @staticmethod + def NewFromJsonDict(data): + '''Create a new instance based on a JSON dict. + + Args: + data: + A JSON dict, as converted from the JSON in the twitter API + + Returns: + A twitter.Url instance + ''' + return Url(url = data.get('url', None), + expanded_url = data.get('expanded_url', None)) + class Api(object): '''A python interface into the Twitter API @@ -1647,9 +2120,10 @@ class Api(object): >>> print [s.text for s in statuses] To use authentication, instantiate the twitter.Api class with a - username, password and the oAuth key and secret: + consumer key and secret; and the oAuth key and secret: - >>> api = twitter.Api(username='twitter user', password='twitter pass', + >>> api = twitter.Api(consumer_key='twitter consumer key', + consumer_secret='twitter consumer secret', access_token_key='the_key_given', access_token_secret='the_key_secret') @@ -1690,8 +2164,8 @@ class Api(object): _API_REALM = 'Twitter API' def __init__(self, - username = None, - password = None, + consumer_key = None, + consumer_secret = None, access_token_key = None, access_token_secret = None, input_encoding = None, @@ -1699,21 +2173,15 @@ class Api(object): cache = DEFAULT_CACHE, shortner = None, base_url = None, - use_gzip_compression = False): + use_gzip_compression = False, + debugHTTP = False): '''Instantiate a new twitter.Api object. Args: - username: - The username of the twitter account. [optional] - NOTE: for oAuth based authentication, this is not - optional and the value is the Twitter - Consumer Key value *not* your Twitter ID - password: - The password for the twitter account. [optional] - NOTE: for oAuth based authentication, this is not - optional and the value is the Twitter - Consumer Secret value *not* your Twitter - password + consumer_key: + Your Twitter user's consumer_key. + consumer_secret: + Your Twitter user's consumer_secret. access_token_key: The oAuth access token key value you retrieved from running get_access_token.py. @@ -1721,28 +2189,33 @@ class Api(object): The oAuth access token's secret, also retrieved from the get_access_token.py run. input_encoding: - The encoding used to encode input strings. [optional] + The encoding used to encode input strings. [Optional] request_header: - A dictionary of additional HTTP request headers. [optional] + A dictionary of additional HTTP request headers. [Optional] cache: The cache instance to use. Defaults to DEFAULT_CACHE. - Use None to disable caching. [optional] + Use None to disable caching. [Optional] shortner: The shortner instance to use. Defaults to None. - See shorten_url.py for an example shortner. [optional] + See shorten_url.py for an example shortner. [Optional] base_url: The base URL to use to contact the Twitter API. - Defaults to https://twitter.com. [optional] + Defaults to https://api.twitter.com. [Optional] use_gzip_compression: Set to True to tell enable gzip compression for any call - made to Twitter. Defaults to False. [optional] + made to Twitter. Defaults to False. [Optional] + debugHTTP: + Set to True to enable debug output from urllib2 when performing + any HTTP requests. Defaults to False. [Optional] ''' self.SetCache(cache) self._urllib = urllib2 self._cache_timeout = Api.DEFAULT_CACHE_TIMEOUT self._input_encoding = input_encoding self._use_gzip = use_gzip_compression + self._debugHTTP = debugHTTP self._oauth_consumer = None + self._shortlink_size = 19 self._InitializeRequestHeaders(request_headers) self._InitializeUserAgent() @@ -1753,28 +2226,28 @@ class Api(object): else: self.base_url = base_url - if username is not None and (access_token_key is None or - access_token_secret is None): + if consumer_key is not None and (access_token_key is None or + access_token_secret is None): print >> sys.stderr, 'Twitter now requires an oAuth Access Token for API calls.' print >> sys.stderr, 'If your using this library from a command line utility, please' print >> sys.stderr, 'run the the included get_access_token.py tool to generate one.' raise TwitterError('Twitter requires oAuth Access Token for all API access') - self.SetCredentials(username, password, access_token_key, access_token_secret) + self.SetCredentials(consumer_key, consumer_secret, access_token_key, access_token_secret) def SetCredentials(self, - username, - password, + consumer_key, + consumer_secret, access_token_key = None, access_token_secret = None): - '''Set the username and password for this instance + '''Set the consumer_key and consumer_secret for this instance Args: - username: - The username of the twitter account. - password: - The password for the twitter account. + consumer_key: + The consumer_key of the twitter account. + consumer_secret: + The consumer_secret for the twitter account. access_token_key: The oAuth access token key value you retrieved from running get_access_token.py. @@ -1782,37 +2255,50 @@ class Api(object): The oAuth access token's secret, also retrieved from the get_access_token.py run. ''' - self._username = username - self._password = password + self._consumer_key = consumer_key + self._consumer_secret = consumer_secret self._access_token_key = access_token_key self._access_token_secret = access_token_secret self._oauth_consumer = None - if username is not None and password is not None and \ + if consumer_key is not None and consumer_secret is not None and \ access_token_key is not None and access_token_secret is not None: self._signature_method_plaintext = oauth.SignatureMethod_PLAINTEXT() self._signature_method_hmac_sha1 = oauth.SignatureMethod_HMAC_SHA1() self._oauth_token = oauth.Token(key = access_token_key, secret = access_token_secret) - self._oauth_consumer = oauth.Consumer(key = username, secret = password) + self._oauth_consumer = oauth.Consumer(key = consumer_key, secret = consumer_secret) def ClearCredentials(self): '''Clear the any credentials for this instance ''' - self._username = None - self._password = None + self._consumer_key = None + self._consumer_secret = None self._access_token_key = None self._access_token_secret = None self._oauth_consumer = None def GetPublicTimeline(self, - since_id = None): + since_id = None, + include_rts = None, + include_entities = None): '''Fetch the sequence of public twitter.Status message for all users. Args: since_id: - Returns only public statuses with an ID greater than - (that is, more recent than) the specified ID. [optional] + Returns results with an ID greater than (that is, more recent + than) the specified ID. There are limits to the number of + Tweets which can be accessed through the API. If the limit of + Tweets has occured since the since_id, the since_id will be + forced to the oldest ID available. [Optional] + include_rts: + If True, the timeline will contain native retweets (if they + exist) in addition to the standard stream of tweets. [Optional] + include_entities: + If True, each tweet will include a node called "entities,". + This node offers a variety of metadata about the tweet in a + discreet structure, including: user_mentions, urls, and + hashtags. [Optional] Returns: An sequence of twitter.Status instances, one for each message @@ -1821,13 +2307,14 @@ class Api(object): if since_id: parameters['since_id'] = since_id + if include_rts: + parameters['include_rts'] = 1 + if include_entities: + parameters['include_entities'] = 1 url = '%s/statuses/public_timeline.json' % self.base_url json = self._FetchUrl(url, parameters = parameters) - data = simplejson.loads(json) - - self._CheckForTwitterError(data) - + data = self._ParseAndCheckTwitter(json) return [Status.NewFromJsonDict(x) for x in data] def FilterPublicTimeline(self, @@ -1840,8 +2327,11 @@ class Api(object): term: term to search by. since_id: - Returns only public statuses with an ID greater than - (that is, more recent than) the specified ID. [optional] + Returns results with an ID greater than (that is, more recent + than) the specified ID. There are limits to the number of + Tweets which can be accessed through the API. If the limit of + Tweets has occured since the since_id, the since_id will be + forced to the oldest ID available. [Optional] Returns: A sequence of twitter.Status instances, one for each message @@ -1857,7 +2347,7 @@ class Api(object): return results def GetSearch(self, - term, + term = None, geocode = None, since_id = None, per_page = 15, @@ -1869,19 +2359,23 @@ class Api(object): Args: term: - term to search by. + term to search by. Optional if you include geocode. since_id: - Returns only public statuses with an ID greater than - (that is, more recent than) the specified ID. [optional] + Returns results with an ID greater than (that is, more recent + than) the specified ID. There are limits to the number of + Tweets which can be accessed through the API. If the limit of + Tweets has occured since the since_id, the since_id will be + forced to the oldest ID available. [Optional] geocode: geolocation information in the form (latitude, longitude, radius) - [optional] + [Optional] per_page: - number of results to return. Default is 15 [optional] + number of results to return. Default is 15 [Optional] page: - which page of search results to return + Specifies the page of results to retrieve. + Note: there are pagination limits. [Optional] lang: - language for results. Default is English [optional] + language for results. Default is English [Optional] show_user: prefixes screen name in status query_users: @@ -1889,6 +2383,7 @@ class Api(object): profile_image_url available. If set to True, all information of users are available, but it uses lots of request quota, one per status. + Returns: A sequence of twitter.Status instances, one for each message containing the term @@ -1899,24 +2394,24 @@ class Api(object): if since_id: parameters['since_id'] = since_id - if not term: + if term is None and geocode is None: return [] - parameters['q'] = urllib.quote_plus(term) + if term is not None: + parameters['q'] = term + + if geocode is not None: + parameters['geocode'] = ','.join(map(str, geocode)) + parameters['show_user'] = show_user parameters['lang'] = lang parameters['rpp'] = per_page parameters['page'] = page - if geocode is not None: - parameters['geocode'] = ','.join(map(str, geocode)) - # Make and send requests url = 'http://search.twitter.com/search.json' json = self._FetchUrl(url, parameters = parameters) - data = simplejson.loads(json) - - self._CheckForTwitterError(data) + data = self._ParseAndCheckTwitter(json) results = [] @@ -1934,12 +2429,140 @@ class Api(object): # Return built list of statuses return results # [Status.NewFromJsonDict(x) for x in data['results']] + def GetTrendsCurrent(self, exclude = None): + '''Get the current top trending topics + + Args: + exclude: + Appends the exclude parameter as a request parameter. + Currently only exclude=hashtags is supported. [Optional] + + Returns: + A list with 10 entries. Each entry contains the twitter. + ''' + parameters = {} + if exclude: + parameters['exclude'] = exclude + url = '%s/trends/current.json' % self.base_url + json = self._FetchUrl(url, parameters = parameters) + data = self._ParseAndCheckTwitter(json) + + trends = [] + + for t in data['trends']: + for item in data['trends'][t]: + trends.append(Trend.NewFromJsonDict(item, timestamp = t)) + return trends + + def GetTrendsWoeid(self, woeid, exclude = None): + '''Return the top 10 trending topics for a specific WOEID, if trending + information is available for it. + + Args: + woeid: + the Yahoo! Where On Earth ID for a location. + exclude: + Appends the exclude parameter as a request parameter. + Currently only exclude=hashtags is supported. [Optional] + + Returns: + A list with 10 entries. Each entry contains a Trend. + ''' + parameters = {} + if exclude: + parameters['exclude'] = exclude + url = '%s/trends/%s.json' % (self.base_url, woeid) + json = self._FetchUrl(url, parameters = parameters) + data = self._ParseAndCheckTwitter(json) + + trends = [] + timestamp = data[0]['as_of'] + + for trend in data[0]['trends']: + trends.append(Trend.NewFromJsonDict(trend, timestamp = timestamp)) + return trends + + def GetTrendsDaily(self, exclude = None, startdate = None): + '''Get the current top trending topics for each hour in a given day + + Args: + startdate: + The start date for the report. + Should be in the format YYYY-MM-DD. [Optional] + exclude: + Appends the exclude parameter as a request parameter. + Currently only exclude=hashtags is supported. [Optional] + + Returns: + A list with 24 entries. Each entry contains the twitter. + Trend elements that were trending at the corresponding hour of the day. + ''' + parameters = {} + if exclude: + parameters['exclude'] = exclude + if not startdate: + startdate = time.strftime('%Y-%m-%d', time.gmtime()) + parameters['date'] = startdate + url = '%s/trends/daily.json' % self.base_url + json = self._FetchUrl(url, parameters = parameters) + data = self._ParseAndCheckTwitter(json) + + trends = [] + + for i in xrange(24): + trends.append(None) + for t in data['trends']: + idx = int(time.strftime('%H', time.strptime(t, '%Y-%m-%d %H:%M'))) + trends[idx] = [Trend.NewFromJsonDict(x, timestamp = t) + for x in data['trends'][t]] + return trends + + def GetTrendsWeekly(self, exclude = None, startdate = None): + '''Get the top 30 trending topics for each day in a given week. + + Args: + startdate: + The start date for the report. + Should be in the format YYYY-MM-DD. [Optional] + exclude: + Appends the exclude parameter as a request parameter. + Currently only exclude=hashtags is supported. [Optional] + Returns: + A list with each entry contains the twitter. + Trend elements of trending topics for the corrsponding day of the week + ''' + parameters = {} + if exclude: + parameters['exclude'] = exclude + if not startdate: + startdate = time.strftime('%Y-%m-%d', time.gmtime()) + parameters['date'] = startdate + url = '%s/trends/weekly.json' % self.base_url + json = self._FetchUrl(url, parameters = parameters) + data = self._ParseAndCheckTwitter(json) + + trends = [] + + for i in xrange(7): + trends.append(None) + # use the epochs of the dates as keys for a dictionary + times = dict([(calendar.timegm(time.strptime(t, '%Y-%m-%d')), t) + for t in data['trends']]) + cnt = 0 + # create the resulting structure ordered by the epochs of the dates + for e in sorted(times.keys()): + trends[cnt] = [Trend.NewFromJsonDict(x, timestamp = times[e]) + for x in data['trends'][times[e]]] + cnt += 1 + return trends + def GetFriendsTimeline(self, user = None, count = None, - since = None, + page = None, since_id = None, - retweets = False): + retweets = None, + include_entities = None): '''Fetch the sequence of twitter.Status messages for a user's friends The twitter.Api instance must be authenticated if the user is private. @@ -1947,47 +2570,59 @@ class Api(object): Args: user: Specifies the ID or screen name of the user for whom to return - the friends_timeline. If unspecified, the username and password - must be set in the twitter.Api instance. [Optional] - count: + the friends_timeline. If not specified then the authenticated + user set in the twitter.Api instance will be used. [Optional] + count: Specifies the number of statuses to retrieve. May not be - greater than 200. [Optional] - since: - Narrows the returned results to just those statuses created - after the specified HTTP-formatted date. [Optional] + greater than 100. [Optional] + page: + Specifies the page of results to retrieve. + Note: there are pagination limits. [Optional] since_id: - Returns only public statuses with an ID greater than (that is, - more recent than) the specified ID. [Optional] + Returns results with an ID greater than (that is, more recent + than) the specified ID. There are limits to the number of + Tweets which can be accessed through the API. If the limit of + Tweets has occured since the since_id, the since_id will be + forced to the oldest ID available. [Optional] + retweets: + If True, the timeline will contain native retweets. [Optional] + include_entities: + If True, each tweet will include a node called "entities,". + This node offers a variety of metadata about the tweet in a + discreet structure, including: user_mentions, urls, and + hashtags. [Optional] Returns: A sequence of twitter.Status instances, one for each message ''' if not user and not self._oauth_consumer: raise TwitterError("User must be specified if API is not authenticated.") - url = '%s/statuses' % self.base_url - if retweets: - src = 'home_timeline' - else: - src = 'friends_timeline' + url = '%s/statuses/friends_timeline' % self.base_url if user: - url = '%s/%s/%s.json' % (url, src, user) + url = '%s/%s.json' % (url, user) else: - url = '%s/%s.json' % (url, src) + url = '%s.json' % url parameters = {} if count is not None: try: - if int(count) > 200: - raise TwitterError("'count' may not be greater than 200") + if int(count) > 100: + raise TwitterError("'count' may not be greater than 100") except ValueError: raise TwitterError("'count' must be an integer") parameters['count'] = count - if since: - parameters['since'] = since + if page is not None: + try: + parameters['page'] = int(page) + except ValueError: + raise TwitterError("'page' must be an integer") if since_id: parameters['since_id'] = since_id + if retweets: + parameters['include_rts'] = True + if include_entities: + parameters['include_entities'] = True json = self._FetchUrl(url, parameters = parameters) - data = simplejson.loads(json) - self._CheckForTwitterError(data) + data = self._ParseAndCheckTwitter(json) return [Status.NewFromJsonDict(x) for x in data] def GetUserTimeline(self, @@ -1997,7 +2632,9 @@ class Api(object): since_id = None, max_id = None, count = None, - page = None): + page = None, + include_rts = None, + include_entities = None): '''Fetch the sequence of public Status messages for a single user. The twitter.Api instance must be authenticated if the user is private. @@ -2005,27 +2642,38 @@ class Api(object): Args: id: Specifies the ID or screen name of the user for whom to return - the user_timeline. [optional] + the user_timeline. [Optional] user_id: Specfies the ID of the user for whom to return the user_timeline. Helpful for disambiguating when a valid user ID - is also a valid screen name. [optional] + is also a valid screen name. [Optional] screen_name: Specfies the screen name of the user for whom to return the user_timeline. Helpful for disambiguating when a valid screen - name is also a user ID. [optional] + name is also a user ID. [Optional] since_id: - Returns only public statuses with an ID greater than (that is, - more recent than) the specified ID. [optional] + Returns results with an ID greater than (that is, more recent + than) the specified ID. There are limits to the number of + Tweets which can be accessed through the API. If the limit of + Tweets has occured since the since_id, the since_id will be + forced to the oldest ID available. [Optional] max_id: Returns only statuses with an ID less than (that is, older - than) or equal to the specified ID. [optional] + than) or equal to the specified ID. [Optional] count: Specifies the number of statuses to retrieve. May not be - greater than 200. [optional] + greater than 200. [Optional] page: - Specifies the page of results to retrieve. Note: there are - pagination limits. [optional] + Specifies the page of results to retrieve. + Note: there are pagination limits. [Optional] + include_rts: + If True, the timeline will contain native retweets (if they + exist) in addition to the standard stream of tweets. [Optional] + include_entities: + If True, each tweet will include a node called "entities,". + This node offers a variety of metadata about the tweet in a + discreet structure, including: user_mentions, urls, and + hashtags. [Optional] Returns: A sequence of Status instances, one for each message up to count @@ -2068,19 +2716,30 @@ class Api(object): except: raise TwitterError("page must be an integer") + if include_rts: + parameters['include_rts'] = 1 + + if include_entities: + parameters['include_entities'] = 1 + json = self._FetchUrl(url, parameters = parameters) - data = simplejson.loads(json) - self._CheckForTwitterError(data) + data = self._ParseAndCheckTwitter(json) return [Status.NewFromJsonDict(x) for x in data] - def GetStatus(self, id): + def GetStatus(self, id, include_entities = None): '''Returns a single status message. - The twitter.Api instance must be authenticated if the status message is private. + The twitter.Api instance must be authenticated if the + status message is private. Args: - id: The numerical ID of the status you're trying to retrieve. - + id: + The numeric ID of the status you are trying to retrieve. + include_entities: + If True, each tweet will include a node called "entities". + This node offers a variety of metadata about the tweet in a + discreet structure, including: user_mentions, urls, and + hashtags. [Optional] Returns: A twitter.Status instance representing that status message ''' @@ -2089,20 +2748,25 @@ class Api(object): long(id) except: raise TwitterError("id must be an long integer") + + parameters = {} + if include_entities: + parameters['include_entities'] = 1 + url = '%s/statuses/show/%s.json' % (self.base_url, id) - json = self._FetchUrl(url) - data = simplejson.loads(json) - self._CheckForTwitterError(data) + json = self._FetchUrl(url, parameters = parameters) + data = self._ParseAndCheckTwitter(json) return Status.NewFromJsonDict(data) def DestroyStatus(self, id): '''Destroys the status specified by the required ID parameter. - The twitter.Api instance must be authenticated and thee + The twitter.Api instance must be authenticated and the authenticating user must be the author of the specified status. Args: - id: The numerical ID of the status you're trying to destroy. + id: + The numerical ID of the status you're trying to destroy. Returns: A twitter.Status instance representing the destroyed status message @@ -2114,10 +2778,19 @@ class Api(object): raise TwitterError("id must be an integer") url = '%s/statuses/destroy/%s.json' % (self.base_url, id) json = self._FetchUrl(url, post_data = {'id': id}) - data = simplejson.loads(json) - self._CheckForTwitterError(data) + data = self._ParseAndCheckTwitter(json) return Status.NewFromJsonDict(data) + @classmethod + def _calculate_status_length(cls, status, linksize = 19): + dummy_link_replacement = 'https://-%d-chars%s/' % (linksize, '-' * (linksize - 18)) + shortened = ' '.join([x if not (x.startswith('http://') or + x.startswith('https://')) + else + dummy_link_replacement + for x in status.split(' ')]) + return len(shortened) + def PostUpdate(self, status, in_reply_to_status_id = None): '''Post a twitter status message from the authenticated user. @@ -2125,8 +2798,8 @@ class Api(object): Args: status: - The message text to be posted. Must be less than or equal to - 140 characters. + The message text to be posted. + Must be less than or equal to 140 characters. in_reply_to_status_id: The ID of an existing status that the status to be posted is in reply to. This implicitly sets the in_reply_to_user_id @@ -2146,7 +2819,7 @@ class Api(object): else: u_status = unicode(status, self._input_encoding) - if len(u_status) > CHARACTER_LIMIT: + if self._calculate_status_length(u_status, self._shortlink_size) > CHARACTER_LIMIT: raise TwitterError("Text must be less than or equal to %d characters. " "Consider using PostUpdates." % CHARACTER_LIMIT) @@ -2154,8 +2827,7 @@ class Api(object): if in_reply_to_status_id: data['in_reply_to_status_id'] = in_reply_to_status_id json = self._FetchUrl(url, post_data = data) - data = simplejson.loads(json) - self._CheckForTwitterError(data) + data = self._ParseAndCheckTwitter(json) return Status.NewFromJsonDict(data) def PostUpdates(self, status, continuation = None, **kwargs): @@ -2168,7 +2840,8 @@ class Api(object): Args: status: - The message text to be posted. May be longer than 140 characters. + The message text to be posted. + May be longer than 140 characters. continuation: The character string, if any, to be appended to all but the last message. Note that Twitter strips trailing '...' strings @@ -2176,6 +2849,7 @@ class Api(object): (horizontal ellipsis) instead. [Defaults to None] **kwargs: See api.PostUpdate for a list of accepted parameters. + Returns: A of list twitter.Status instance representing the messages posted. ''' @@ -2189,19 +2863,73 @@ class Api(object): results.append(self.PostUpdate(lines[-1], **kwargs)) return results + def GetUserRetweets(self, count = None, since_id = None, max_id = None, include_entities = False): + '''Fetch the sequence of retweets made by a single user. + + The twitter.Api instance must be authenticated. + + Args: + count: + The number of status messages to retrieve. [Optional] + since_id: + Returns results with an ID greater than (that is, more recent + than) the specified ID. There are limits to the number of + Tweets which can be accessed through the API. If the limit of + Tweets has occured since the since_id, the since_id will be + forced to the oldest ID available. [Optional] + max_id: + Returns results with an ID less than (that is, older than) or + equal to the specified ID. [Optional] + include_entities: + If True, each tweet will include a node called "entities,". + This node offers a variety of metadata about the tweet in a + discreet structure, including: user_mentions, urls, and + hashtags. [Optional] + + Returns: + A sequence of twitter.Status instances, one for each message up to count + ''' + url = '%s/statuses/retweeted_by_me.json' % self.base_url + if not self._oauth_consumer: + raise TwitterError("The twitter.Api instance must be authenticated.") + parameters = {} + if count is not None: + try: + if int(count) > 100: + raise TwitterError("'count' may not be greater than 100") + except ValueError: + raise TwitterError("'count' must be an integer") + if count: + parameters['count'] = count + if since_id: + parameters['since_id'] = since_id + if include_entities: + parameters['include_entities'] = True + if max_id: + try: + parameters['max_id'] = long(max_id) + except: + raise TwitterError("max_id must be an integer") + json = self._FetchUrl(url, parameters = parameters) + data = self._ParseAndCheckTwitter(json) + return [Status.NewFromJsonDict(x) for x in data] + def GetReplies(self, since = None, since_id = None, page = None): - '''Get a sequence of status messages representing the 20 most recent - replies (status updates prefixed with @username) to the authenticating - user. + '''Get a sequence of status messages representing the 20 most + recent replies (status updates prefixed with @twitterID) to the + authenticating user. Args: - page: - since: - Narrows the returned results to just those statuses created - after the specified HTTP-formatted date. [optional] since_id: - Returns only public statuses with an ID greater than (that is, - more recent than) the specified ID. [Optional] + Returns results with an ID greater than (that is, more recent + than) the specified ID. There are limits to the number of + Tweets which can be accessed through the API. If the limit of + Tweets has occured since the since_id, the since_id will be + forced to the oldest ID available. [Optional] + page: + Specifies the page of results to retrieve. + Note: there are pagination limits. [Optional] + since: Returns: A sequence of twitter.Status instances, one for each reply to the user. @@ -2217,19 +2945,38 @@ class Api(object): if page: parameters['page'] = page json = self._FetchUrl(url, parameters = parameters) - data = simplejson.loads(json) - self._CheckForTwitterError(data) + data = self._ParseAndCheckTwitter(json) return [Status.NewFromJsonDict(x) for x in data] - def GetFriends(self, user = None, cursor = -1): - '''Fetch the sequence of twitter.User instances, one for each friend. + def GetRetweets(self, statusid): + '''Returns up to 100 of the first retweets of the tweet identified + by statusid Args: - user: the username or id of the user whose friends you are fetching. If - not specified, defaults to the authenticated user. [optional] + statusid: + The ID of the tweet for which retweets should be searched for + + Returns: + A list of twitter.Status instances, which are retweets of statusid + ''' + if not self._oauth_consumer: + raise TwitterError("The twitter.Api instsance must be authenticated.") + url = '%s/statuses/retweets/%s.json?include_entities=true&include_rts=true' % (self.base_url, statusid) + parameters = {} + json = self._FetchUrl(url, parameters = parameters) + data = self._ParseAndCheckTwitter(json) + return [Status.NewFromJsonDict(s) for s in data] + + def GetFriends(self, user = None, cursor = -1): + '''Fetch the sequence of twitter.User instances, one for each friend. The twitter.Api instance must be authenticated. + Args: + user: + The twitter name or id of the user whose friends you are fetching. + If not specified, defaults to the authenticated user. [Optional] + Returns: A sequence of twitter.User instances, one for each friend ''' @@ -2242,8 +2989,7 @@ class Api(object): parameters = {} parameters['cursor'] = cursor json = self._FetchUrl(url, parameters = parameters) - data = simplejson.loads(json) - self._CheckForTwitterError(data) + data = self._ParseAndCheckTwitter(json) return [User.NewFromJsonDict(x) for x in data['users']] def GetFriendIDs(self, user = None, cursor = -1): @@ -2253,7 +2999,7 @@ class Api(object): Args: user: The id or screen_name of the user to retrieve the id list for - [optional] + [Optional] Returns: A list of integers, one for each user id. @@ -2267,8 +3013,7 @@ class Api(object): parameters = {} parameters['cursor'] = cursor json = self._FetchUrl(url, parameters = parameters) - data = simplejson.loads(json) - self._CheckForTwitterError(data) + data = self._ParseAndCheckTwitter(json) return data def GetFollowerIDs(self, userid = None, cursor = -1): @@ -2279,34 +3024,43 @@ class Api(object): Returns: A sequence of twitter.User instances, one for each follower ''' - url = 'http://twitter.com/followers/ids.json' + url = '%s/followers/ids.json' % self.base_url parameters = {} parameters['cursor'] = cursor if userid: parameters['user_id'] = userid json = self._FetchUrl(url, parameters = parameters) - data = simplejson.loads(json) - self._CheckForTwitterError(data) + data = self._ParseAndCheckTwitter(json) return data - def GetFollowers(self, page = None): + def GetFollowers(self, cursor = -1): '''Fetch the sequence of twitter.User instances, one for each follower The twitter.Api instance must be authenticated. + Args: + cursor: + Specifies the Twitter API Cursor location to start at. [Optional] + Note: there are pagination limits. + Returns: A sequence of twitter.User instances, one for each follower ''' if not self._oauth_consumer: raise TwitterError("twitter.Api instance must be authenticated") url = '%s/statuses/followers.json' % self.base_url - parameters = {} - if page: - parameters['page'] = page - json = self._FetchUrl(url, parameters = parameters) - data = simplejson.loads(json) - self._CheckForTwitterError(data) - return [User.NewFromJsonDict(x) for x in data] + result = [] + while True: + parameters = { 'cursor': cursor } + json = self._FetchUrl(url, parameters = parameters) + data = self._ParseAndCheckTwitter(json) + result += [User.NewFromJsonDict(x) for x in data['users']] + if 'next_cursor' in data: + if data['next_cursor'] == 0 or data['next_cursor'] == data['previous_cursor']: + break + else: + break + return result def GetFeatured(self): '''Fetch the sequence of twitter.User instances featured on twitter.com @@ -2318,25 +3072,66 @@ class Api(object): ''' url = '%s/statuses/featured.json' % self.base_url json = self._FetchUrl(url) - data = simplejson.loads(json) - self._CheckForTwitterError(data) + data = self._ParseAndCheckTwitter(json) return [User.NewFromJsonDict(x) for x in data] + def UsersLookup(self, user_id = None, screen_name = None, users = None): + '''Fetch extended information for the specified users. + + Users may be specified either as lists of either user_ids, + screen_names, or twitter.User objects. The list of users that + are queried is the union of all specified parameters. + + The twitter.Api instance must be authenticated. + + Args: + user_id: + A list of user_ids to retrieve extended information. + [Optional] + screen_name: + A list of screen_names to retrieve extended information. + [Optional] + users: + A list of twitter.User objects to retrieve extended information. + [Optional] + + Returns: + A list of twitter.User objects for the requested users + ''' + + if not self._oauth_consumer: + raise TwitterError("The twitter.Api instance must be authenticated.") + if not user_id and not screen_name and not users: + raise TwitterError("Specify at least on of user_id, screen_name, or users.") + url = '%s/users/lookup.json' % self.base_url + parameters = {} + uids = list() + if user_id: + uids.extend(user_id) + if users: + uids.extend([u.id for u in users]) + if len(uids): + parameters['user_id'] = ','.join(["%s" % u for u in uids]) + if screen_name: + parameters['screen_name'] = ','.join(screen_name) + json = self._FetchUrl(url, parameters = parameters) + data = self._ParseAndCheckTwitter(json) + return [User.NewFromJsonDict(u) for u in data] + def GetUser(self, user): '''Returns a single user. The twitter.Api instance must be authenticated. Args: - user: The username or id of the user to retrieve. + user: The twitter name or id of the user to retrieve. Returns: A twitter.User instance representing that user ''' url = '%s/users/show/%s.json' % (self.base_url, user) json = self._FetchUrl(url) - data = simplejson.loads(json) - self._CheckForTwitterError(data) + data = self._ParseAndCheckTwitter(json) return User.NewFromJsonDict(data) def GetDirectMessages(self, since = None, since_id = None, page = None): @@ -2347,10 +3142,16 @@ class Api(object): Args: since: Narrows the returned results to just those statuses created - after the specified HTTP-formatted date. [optional] + after the specified HTTP-formatted date. [Optional] since_id: - Returns only public statuses with an ID greater than (that is, - more recent than) the specified ID. [Optional] + Returns results with an ID greater than (that is, more recent + than) the specified ID. There are limits to the number of + Tweets which can be accessed through the API. If the limit of + Tweets has occured since the since_id, the since_id will be + forced to the oldest ID available. [Optional] + page: + Specifies the page of results to retrieve. + Note: there are pagination limits. [Optional] Returns: A sequence of twitter.DirectMessage instances @@ -2366,8 +3167,7 @@ class Api(object): if page: parameters['page'] = page json = self._FetchUrl(url, parameters = parameters) - data = simplejson.loads(json) - self._CheckForTwitterError(data) + data = self._ParseAndCheckTwitter(json) return [DirectMessage.NewFromJsonDict(x) for x in data] def PostDirectMessage(self, user, text): @@ -2387,8 +3187,7 @@ class Api(object): url = '%s/direct_messages/new.json' % self.base_url data = {'text': text, 'user': user} json = self._FetchUrl(url, post_data = data) - data = simplejson.loads(json) - self._CheckForTwitterError(data) + data = self._ParseAndCheckTwitter(json) return DirectMessage.NewFromJsonDict(data) def DestroyDirectMessage(self, id): @@ -2406,8 +3205,7 @@ class Api(object): ''' url = '%s/direct_messages/destroy/%s.json' % (self.base_url, id) json = self._FetchUrl(url, post_data = {'id': id}) - data = simplejson.loads(json) - self._CheckForTwitterError(data) + data = self._ParseAndCheckTwitter(json) return DirectMessage.NewFromJsonDict(data) def CreateFriendship(self, user): @@ -2422,8 +3220,7 @@ class Api(object): ''' url = '%s/friendships/create/%s.json' % (self.base_url, user) json = self._FetchUrl(url, post_data = {'user': user}) - data = simplejson.loads(json) - self._CheckForTwitterError(data) + data = self._ParseAndCheckTwitter(json) return User.NewFromJsonDict(data) def DestroyFriendship(self, user): @@ -2438,8 +3235,7 @@ class Api(object): ''' url = '%s/friendships/destroy/%s.json' % (self.base_url, user) json = self._FetchUrl(url, post_data = {'user': user}) - data = simplejson.loads(json) - self._CheckForTwitterError(data) + data = self._ParseAndCheckTwitter(json) return User.NewFromJsonDict(data) def CreateFavorite(self, status): @@ -2455,8 +3251,7 @@ class Api(object): ''' url = '%s/favorites/create/%s.json' % (self.base_url, status.id) json = self._FetchUrl(url, post_data = {'id': status.id}) - data = simplejson.loads(json) - self._CheckForTwitterError(data) + data = self._ParseAndCheckTwitter(json) return Status.NewFromJsonDict(data) def DestroyFavorite(self, status): @@ -2472,8 +3267,7 @@ class Api(object): ''' url = '%s/favorites/destroy/%s.json' % (self.base_url, status.id) json = self._FetchUrl(url, post_data = {'id': status.id}) - data = simplejson.loads(json) - self._CheckForTwitterError(data) + data = self._ParseAndCheckTwitter(json) return Status.NewFromJsonDict(data) def GetFavorites(self, @@ -2482,14 +3276,14 @@ class Api(object): '''Return a list of Status objects representing favorited tweets. By default, returns the (up to) 20 most recent tweets for the authenticated user. - + Args: user: - The username or id of the user whose favorites you are fetching. - If not specified, defaults to the authenticated user. [optional] - + The twitter name or id of the user whose favorites you are fetching. + If not specified, defaults to the authenticated user. [Optional] page: - Retrieves the 20 next most recent favorite statuses. [optional] + Specifies the page of results to retrieve. + Note: there are pagination limits. [Optional] ''' parameters = {} @@ -2504,34 +3298,32 @@ class Api(object): url = '%s/favorites.json' % self.base_url json = self._FetchUrl(url, parameters = parameters) - data = simplejson.loads(json) - - self._CheckForTwitterError(data) - + data = self._ParseAndCheckTwitter(json) return [Status.NewFromJsonDict(x) for x in data] def GetMentions(self, since_id = None, max_id = None, page = None): - '''Returns the 20 most recent mentions (status containing @username) + '''Returns the 20 most recent mentions (status containing @twitterID) for the authenticating user. - + Args: since_id: - Returns only public statuses with an ID greater than - (that is, more recent than) the specified ID. [optional] - + Returns results with an ID greater than (that is, more recent + than) the specified ID. There are limits to the number of + Tweets which can be accessed through the API. If the limit of + Tweets has occured since the since_id, the since_id will be + forced to the oldest ID available. [Optional] max_id: Returns only statuses with an ID less than - (that is, older than) the specified ID. [optional] - + (that is, older than) the specified ID. [Optional] page: - Retrieves the 20 next most recent replies. [optional] - + Specifies the page of results to retrieve. + Note: there are pagination limits. [Optional] + Returns: A sequence of twitter.Status instances, one for each mention of the user. - see: http://apiwiki.twitter.com/REST-API-Documentation#statuses/mentions ''' url = '%s/statuses/mentions.json' % self.base_url @@ -2544,15 +3336,15 @@ class Api(object): if since_id: parameters['since_id'] = since_id if max_id: - parameters['max_id'] = max_id + try: + parameters['max_id'] = long(max_id) + except: + raise TwitterError("max_id must be an integer") if page: parameters['page'] = page json = self._FetchUrl(url, parameters = parameters) - data = simplejson.loads(json) - - self._CheckForTwitterError(data) - + data = self._ParseAndCheckTwitter(json) return [Status.NewFromJsonDict(x) for x in data] def CreateList(self, user, name, mode = None, description = None): @@ -2561,12 +3353,15 @@ class Api(object): The twitter.Api instance must be authenticated. Args: - user: username to create the list for - name: new name for the list - mode: 'public' or 'private'. defaults to 'public' if not given - [optional] - description: description of the list - [optional] + user: + Twitter name to create the list for + name: + New name for the list + mode: + 'public' or 'private'. + Defaults to 'public'. [Optional] + description: + Description of the list. [Optional] Returns: A twitter.List instance representing the new list @@ -2578,8 +3373,7 @@ class Api(object): if description is not None: parameters['description'] = description json = self._FetchUrl(url, post_data = parameters) - data = simplejson.loads(json) - self._CheckForTwitterError(data) + data = self._ParseAndCheckTwitter(json) return List.NewFromJsonDict(data) def DestroyList(self, user, id): @@ -2588,15 +3382,16 @@ class Api(object): The twitter.Api instance must be authenticated. Args: - user: the user to remove the list form - id: the slug or id of the list to remove + user: + The user to remove the list from. + id: + The slug or id of the list to remove. Returns: A twitter.List instance representing the removed list. ''' url = '%s/%s/lists/%s.json' % (self.base_url, user, id) json = self._FetchUrl(url, post_data = {'_method': 'DELETE'}) - data = simplejson.loads(json) - self._CheckForTwitterError(data) + data = self._ParseAndCheckTwitter(json) return List.NewFromJsonDict(data) def CreateSubscription(self, owner, list): @@ -2605,16 +3400,17 @@ class Api(object): The twitter.Api instance must be authenticated. Args: - owner: user name or id of the owner of the list being subscribed to - list: the slug or list id to subscribe the user to + owner: + User name or id of the owner of the list being subscribed to. + list: + The slug or list id to subscribe the user to Returns: A twitter.List instance representing the list subscribed to ''' url = '%s/%s/%s/subscribers.json' % (self.base_url, owner, list) json = self._FetchUrl(url, post_data = {'list_id': list}) - data = simplejson.loads(json) - self._CheckForTwitterError(data) + data = self._ParseAndCheckTwitter(json) return List.NewFromJsonDict(data) def DestroySubscription(self, owner, list): @@ -2623,31 +3419,34 @@ class Api(object): The twitter.Api instance must be authenticated. Args: - owner: the user id or screen name of the user that owns the - list that is to be unsubscribed from - list: the slug or list id of the list to unsubscribe from + owner: + The user id or screen name of the user that owns the + list that is to be unsubscribed from + list: + The slug or list id of the list to unsubscribe from + Returns: A twitter.List instance representing the removed list. ''' url = '%s/%s/%s/subscribers.json' % (self.base_url, owner, list) json = self._FetchUrl(url, post_data = {'_method': 'DELETE', 'list_id': list}) - data = simplejson.loads(json) - self._CheckForTwitterError(data) + data = self._ParseAndCheckTwitter(json) return List.NewFromJsonDict(data) def GetSubscriptions(self, user, cursor = -1): '''Fetch the sequence of Lists that the given user is subscribed to - Args: - user: the username or id of the user - cursor: "page" value that Twitter will use to start building the - list sequence from. -1 to start at the beginning. - Twitter will return in the result the values for next_cursor - and previous_cursor - [optional] - The twitter.Api instance must be authenticated. + Args: + user: + The twitter name or id of the user + cursor: + "page" value that Twitter will use to start building the + list sequence from. -1 to start at the beginning. + Twitter will return in the result the values for next_cursor + and previous_cursor. [Optional] + Returns: A sequence of twitter.List instances, one for each list ''' @@ -2659,27 +3458,25 @@ class Api(object): parameters['cursor'] = cursor json = self._FetchUrl(url, parameters = parameters) - data = simplejson.loads(json) - self._CheckForTwitterError(data) - print data + data = self._ParseAndCheckTwitter(json) return [List.NewFromJsonDict(x) for x in data['lists']] def GetLists(self, user, cursor = -1): '''Fetch the sequence of lists for a user. - Args: - user: the username or id of the user whose friends you are fetching. - If the passed in user is the same as the authenticated user - then you will also receive private list data. - - cursor: "page" value that Twitter will use to start building the - list sequence from. -1 to start at the beginning. - Twitter will return in the result the values for next_cursor - and previous_cursor - [optional] - The twitter.Api instance must be authenticated. + Args: + user: + The twitter name or id of the user whose friends you are fetching. + If the passed in user is the same as the authenticated user + then you will also receive private list data. + cursor: + "page" value that Twitter will use to start building the + list sequence from. -1 to start at the beginning. + Twitter will return in the result the values for next_cursor + and previous_cursor. [Optional] + Returns: A sequence of twitter.List instances, one for each list ''' @@ -2691,28 +3488,28 @@ class Api(object): parameters['cursor'] = cursor json = self._FetchUrl(url, parameters = parameters) - data = simplejson.loads(json) - self._CheckForTwitterError(data) + data = self._ParseAndCheckTwitter(json) return [List.NewFromJsonDict(x) for x in data['lists']] def GetUserByEmail(self, email): '''Returns a single user by email address. Args: - email: The email of the user to retrieve. + email: + The email of the user to retrieve. + Returns: A twitter.User instance representing that user ''' url = '%s/users/show.json?email=%s' % (self.base_url, email) json = self._FetchUrl(url) - data = simplejson.loads(json) - self._CheckForTwitterError(data) + data = self._ParseAndCheckTwitter(json) return User.NewFromJsonDict(data) def VerifyCredentials(self): '''Returns a twitter.User instance if the authenticating user is valid. - Returns: + Returns: A twitter.User instance representing that user if the credentials are valid, None otherwise. ''' @@ -2726,15 +3523,15 @@ class Api(object): return None else: raise http_error - data = simplejson.loads(json) - self._CheckForTwitterError(data) + data = self._ParseAndCheckTwitter(json) return User.NewFromJsonDict(data) def SetCache(self, cache): '''Override the default cache. Set to None to prevent caching. Args: - cache: an instance that supports the same API as the twitter._FileCache + cache: + An instance that supports the same API as the twitter._FileCache ''' if cache == DEFAULT_CACHE: self._cache = _FileCache() @@ -2745,7 +3542,8 @@ class Api(object): '''Override the default urllib implementation. Args: - urllib: an instance that supports the same API as the urllib2 module + urllib: + An instance that supports the same API as the urllib2 module ''' self._urllib = urllib @@ -2753,7 +3551,8 @@ class Api(object): '''Override the default cache timeout. Args: - cache_timeout: time, in seconds, that responses should be reused. + cache_timeout: + Time, in seconds, that responses should be reused. ''' self._cache_timeout = cache_timeout @@ -2761,7 +3560,8 @@ class Api(object): '''Override the default user agent Args: - user_agent: a string that should be send to the server as the User-agent + user_agent: + A string that should be send to the server as the User-agent ''' self._request_headers['User-Agent'] = user_agent @@ -2799,48 +3599,48 @@ class Api(object): def GetRateLimitStatus(self): '''Fetch the rate limit status for the currently authorized user. - + Returns: A dictionary containing the time the limit will reset (reset_time), the number of remaining hits allowed before the reset (remaining_hits), - the number of hits allowed in a 60-minute period (hourly_limit), and the - time of the reset in seconds since The Epoch (reset_time_in_seconds). + the number of hits allowed in a 60-minute period (hourly_limit), and + the time of the reset in seconds since The Epoch (reset_time_in_seconds). ''' url = '%s/account/rate_limit_status.json' % self.base_url json = self._FetchUrl(url, no_cache = True) - data = simplejson.loads(json) - - self._CheckForTwitterError(data) - + data = self._ParseAndCheckTwitter(json) return data def MaximumHitFrequency(self): - '''Determines the minimum number of seconds that a program must wait before - hitting the server again without exceeding the rate_limit imposed for the - currently authenticated user. - + '''Determines the minimum number of seconds that a program must wait + before hitting the server again without exceeding the rate_limit + imposed for the currently authenticated user. + Returns: - The minimum second interval that a program must use so as to not exceed - the rate_limit imposed for the user. + The minimum second interval that a program must use so as to not + exceed the rate_limit imposed for the user. ''' rate_status = self.GetRateLimitStatus() reset_time = rate_status.get('reset_time', None) limit = rate_status.get('remaining_hits', None) - if reset_time and limit: + if reset_time: # put the reset time into a datetime object reset = datetime.datetime(*rfc822.parsedate(reset_time)[:7]) # find the difference in time between now and the reset time + 1 hour delta = reset + datetime.timedelta(hours = 1) - datetime.datetime.utcnow() + if not limit: + return int(delta.seconds) + # determine the minimum number of seconds allowed as a regular interval - max_frequency = int(delta.seconds / limit) + max_frequency = int(delta.seconds / limit) + 1 # return the number of seconds return max_frequency - return 0 + return 60 def _BuildUrl(self, url, path_elements = None, extra_params = None): # Break url into consituent parts @@ -2903,6 +3703,7 @@ class Api(object): parameters: A dict of (key, value) tuples, where value is encoded as specified by self._encoding + Returns: A URL-encoded string in "key=value&key=value" form ''' @@ -2921,6 +3722,7 @@ class Api(object): post_data: A dict of (key, value) tuples, where value is encoded as specified by self._encoding + Returns: A URL-encoded string in "key=value&key=value" form ''' @@ -2929,11 +3731,30 @@ class Api(object): else: return urllib.urlencode(dict([(k, self._Encode(v)) for k, v in post_data.items()])) + def _ParseAndCheckTwitter(self, json): + """Try and parse the JSON returned from Twitter and return + an empty dictionary if there is any error. This is a purely + defensive check because during some Twitter network outages + it will return an HTML failwhale page.""" + try: + data = simplejson.loads(json) + self._CheckForTwitterError(data) + except ValueError: + if "Twitter / Over capacity" in json: + raise TwitterError("Capacity Error") + if "Twitter / Error" in json: + raise TwitterError("Technical Error") + raise TwitterError("json decoding") + + return data + def _CheckForTwitterError(self, data): """Raises a TwitterError if twitter returns an error message. Args: - data: A python dict created from the Twitter json response + data: + A python dict created from the Twitter json response + Raises: TwitterError wrapping the twitter error message if one exists. """ @@ -2941,6 +3762,8 @@ class Api(object): # to check first, rather than try and catch the exception if 'error' in data: raise TwitterError(data['error']) + if 'errors' in data: + raise TwitterError(data['errors']) def _FetchUrl(self, url, @@ -2958,14 +3781,14 @@ class Api(object): If set, POST will be used. parameters: A dict whose key/value pairs should encoded and added - to the query string. [optional] + to the query string. [Optional] no_cache: If true, overrides the cache on the current request use_gzip_compression: If True, tells the server to gzip-compress the response. It does not apply to POST requests. Defaults to None, which will get the value to use from - the instance variable self._use_gzip [optional] + the instance variable self._use_gzip [Optional] Returns: A string containing the body of the response. @@ -2982,8 +3805,13 @@ class Api(object): else: http_method = "GET" - http_handler = self._urllib.HTTPHandler() - https_handler = self._urllib.HTTPSHandler() + if self._debugHTTP: + _debug = 1 + else: + _debug = 0 + + http_handler = self._urllib.HTTPHandler(debuglevel = _debug) + https_handler = self._urllib.HTTPSHandler(debuglevel = _debug) opener = self._urllib.OpenerDirector() opener.add_handler(http_handler) @@ -3027,8 +3855,8 @@ class Api(object): opener.close() else: # Unique keys are a combination of the url and the oAuth Consumer Key - if self._username: - key = self._username + ':' + url + if self._consumer_key: + key = self._consumer_key + ':' + url else: key = url @@ -3040,10 +3868,10 @@ class Api(object): try: response = opener.open(url, encoded_post_data) url_data = self._DecompressGzippedResponse(response) + self._cache.Set(key, url_data) except urllib2.HTTPError, e: print e opener.close() - self._cache.Set(key, url_data) else: url_data = self._cache.Get(key) @@ -3108,7 +3936,7 @@ class _FileCache(object): os.getenv('USERNAME') or \ os.getlogin() or \ 'nobody' - except (IOError, OSError), e: + except (AttributeError, IOError, OSError), e: return 'nobody' def _GetTmpCachePath(self): diff --git a/libs/requests/__init__.py b/libs/requests/__init__.py index f2708fd..48fb389 100644 --- a/libs/requests/__init__.py +++ b/libs/requests/__init__.py @@ -15,8 +15,8 @@ requests """ __title__ = 'requests' -__version__ = '0.9.1' -__build__ = 0x000901 +__version__ = '0.10.1' +__build__ = 0x001001 __author__ = 'Kenneth Reitz' __license__ = 'ISC' __copyright__ = 'Copyright 2012 Kenneth Reitz' diff --git a/libs/requests/api.py b/libs/requests/api.py index 8ff22e7..b7d4158 100644 --- a/libs/requests/api.py +++ b/libs/requests/api.py @@ -32,9 +32,10 @@ def request(method, url, **kwargs): :param session: (optional) A :class:`Session` object to be used for the request. :param config: (optional) A configuration dictionary. :param verify: (optional) if ``True``, the SSL cert will be verified. A CA_BUNDLE path can also be provided. + :param prefetch: (optional) if ``True``, the response content will be immediately downloaded. """ - s = kwargs.get('session') or sessions.session() + s = kwargs.pop('session') if 'session' in kwargs else sessions.session() return s.request(method=method, url=url, **kwargs) diff --git a/libs/requests/async.py b/libs/requests/async.py index 6f8de38..9488447 100644 --- a/libs/requests/async.py +++ b/libs/requests/async.py @@ -46,15 +46,15 @@ def patched(f): return wrapped -def send(r, pools=None): - """Sends a given Request object.""" +def send(r, pool=None): + """Sends the request object using the specified pool. If a pool isn't + specified this method blocks. Pools are useful because you can specify size + and can hence limit concurrency.""" - if pools: - r._pools = pools + if pool != None: + return pool.spawn(r.send) - r.send() - - return r.response + return gevent.spawn(r.send) # Patched requests.api functions. @@ -78,19 +78,11 @@ def map(requests, prefetch=True, size=None): requests = list(requests) - if size: - pool = Pool(size) - pool.map(send, requests) - pool.join() - else: - jobs = [gevent.spawn(send, r) for r in requests] - gevent.joinall(jobs) + pool = Pool(size) if size else None + jobs = [send(r, pool) for r in requests] + gevent.joinall(jobs) if prefetch: [r.response.content for r in requests] - return [r.response for r in requests] - - - - + return [r.response for r in requests] \ No newline at end of file diff --git a/libs/requests/auth.py b/libs/requests/auth.py index 4af3d6d..183731b 100644 --- a/libs/requests/auth.py +++ b/libs/requests/auth.py @@ -7,19 +7,21 @@ requests.auth This module contains the authentication handlers for Requests. """ +from __future__ import unicode_literals + import time import hashlib from base64 import b64encode -from urlparse import urlparse - +from .compat import urlparse, str, bytes from .utils import randombytes, parse_dict_header def _basic_auth_str(username, password): """Returns a Basic Auth string.""" - return 'Basic %s' % b64encode('%s:%s' % (username, password)) + + return 'Basic ' + b64encode(("%s:%s" % (username, password)).encode('utf-8')).strip().decode('utf-8') class AuthBase(object): @@ -32,8 +34,8 @@ class AuthBase(object): class HTTPBasicAuth(AuthBase): """Attaches HTTP Basic Authentication to the given Request object.""" def __init__(self, username, password): - self.username = str(username) - self.password = str(password) + self.username = username + self.password = password def __call__(self, r): r.headers['Authorization'] = _basic_auth_str(self.username, self.password) @@ -74,9 +76,17 @@ class HTTPDigestAuth(AuthBase): algorithm = algorithm.upper() # lambdas assume digest modules are imported at the top level if algorithm == 'MD5': - H = lambda x: hashlib.md5(x).hexdigest() + def h(x): + if isinstance(x, str): + x = x.encode('utf-8') + return hashlib.md5(x).hexdigest() + H = h elif algorithm == 'SHA': - H = lambda x: hashlib.sha1(x).hexdigest() + def h(x): + if isinstance(x, str): + x = x.encode('utf-8') + return hashlib.sha1(x).hexdigest() + H = h # XXX MD5-sess KD = lambda s, d: H("%s:%s" % (s, d)) @@ -86,7 +96,9 @@ class HTTPDigestAuth(AuthBase): # XXX not implemented yet entdig = None p_parsed = urlparse(r.request.url) - path = p_parsed.path + p_parsed.query + path = p_parsed.path + if p_parsed.query: + path += '?' + p_parsed.query A1 = '%s:%s:%s' % (self.username, realm, self.password) A2 = '%s:%s' % (r.request.method, path) @@ -99,10 +111,12 @@ class HTTPDigestAuth(AuthBase): last_nonce = nonce ncvalue = '%08x' % nonce_count - cnonce = (hashlib.sha1("%s:%s:%s:%s" % ( - nonce_count, nonce, time.ctime(), randombytes(8))) - .hexdigest()[:16] - ) + s = str(nonce_count).encode('utf-8') + s += nonce.encode('utf-8') + s += time.ctime().encode('utf-8') + s += randombytes(8) + + cnonce = (hashlib.sha1(s).hexdigest()[:16]) noncebit = "%s:%s:%s:%s:%s" % (nonce, ncvalue, cnonce, qop, H(A2)) respdig = KD(H(A1), noncebit) elif qop is None: @@ -132,5 +146,5 @@ class HTTPDigestAuth(AuthBase): return r def __call__(self, r): - r.hooks['response'] = self.handle_401 + r.register_hook('response', self.handle_401) return r diff --git a/libs/requests/compat.py b/libs/requests/compat.py new file mode 100644 index 0000000..224bfd0 --- /dev/null +++ b/libs/requests/compat.py @@ -0,0 +1,102 @@ +# -*- coding: utf-8 -*- + +""" +pythoncompat +""" + + +import sys + +# ------- +# Pythons +# ------- + +# Syntax sugar. +_ver = sys.version_info + +#: Python 2.x? +is_py2 = (_ver[0] == 2) + +#: Python 3.x? +is_py3 = (_ver[0] == 3) + +#: Python 3.0.x +is_py30 = (is_py3 and _ver[1] == 0) + +#: Python 3.1.x +is_py31 = (is_py3 and _ver[1] == 1) + +#: Python 3.2.x +is_py32 = (is_py3 and _ver[1] == 2) + +#: Python 3.3.x +is_py33 = (is_py3 and _ver[1] == 3) + +#: Python 3.4.x +is_py34 = (is_py3 and _ver[1] == 4) + +#: Python 2.7.x +is_py27 = (is_py2 and _ver[1] == 7) + +#: Python 2.6.x +is_py26 = (is_py2 and _ver[1] == 6) + +#: Python 2.5.x +is_py25 = (is_py2 and _ver[1] == 5) + +#: Python 2.4.x +is_py24 = (is_py2 and _ver[1] == 4) # I'm assuming this is not by choice. + + +# --------- +# Platforms +# --------- + + +# Syntax sugar. +_ver = sys.version.lower() + +is_pypy = ('pypy' in _ver) +is_jython = ('jython' in _ver) +is_ironpython = ('iron' in _ver) + +# Assume CPython, if nothing else. +is_cpython = not any((is_pypy, is_jython, is_ironpython)) + +# Windows-based system. +is_windows = 'win32' in str(sys.platform).lower() + +# Standard Linux 2+ system. +is_linux = ('linux' in str(sys.platform).lower()) +is_osx = ('darwin' in str(sys.platform).lower()) +is_hpux = ('hpux' in str(sys.platform).lower()) # Complete guess. +is_solaris = ('solar==' in str(sys.platform).lower()) # Complete guess. + + +# --------- +# Specifics +# --------- + + +if is_py2: + from urllib import quote, unquote, urlencode + from urlparse import urlparse, urlunparse, urljoin, urlsplit + from urllib2 import parse_http_list + import cookielib + from .packages.oreos.monkeys import SimpleCookie + from StringIO import StringIO + + str = unicode + bytes = str + + +elif is_py3: + from urllib.parse import urlparse, urlunparse, urljoin, urlsplit, urlencode, quote, unquote + from urllib.request import parse_http_list + from http import cookiejar as cookielib + from http.cookies import SimpleCookie + from io import StringIO + + str = str + bytes = bytes + diff --git a/libs/requests/defaults.py b/libs/requests/defaults.py index 1d6716d..424d373 100644 --- a/libs/requests/defaults.py +++ b/libs/requests/defaults.py @@ -10,16 +10,13 @@ Configurations: :base_headers: Default HTTP headers. :verbose: Stream to write request logging to. -:timeout: Seconds until request timeout. -:max_redirects: Maximum number of redirects allowed within a request. -:decode_unicode: Decode unicode responses automatically? +:max_redirects: Maximum number of redirects allowed within a request.s :keep_alive: Reuse HTTP Connections? :max_retries: The number of times a request should be retried in the event of a connection failure. :danger_mode: If true, Requests will raise errors immediately. :safe_mode: If true, Requests will catch all errors. :pool_maxsize: The maximium size of an HTTP connection pool. :pool_connections: The number of active HTTP connection pools to use. - """ from . import __version__ @@ -35,7 +32,6 @@ defaults['base_headers'] = { defaults['verbose'] = None defaults['max_redirects'] = 30 -defaults['decode_unicode'] = True defaults['pool_connections'] = 10 defaults['pool_maxsize'] = 10 defaults['max_retries'] = 0 diff --git a/libs/requests/hooks.py b/libs/requests/hooks.py index f9cf480..3560b89 100644 --- a/libs/requests/hooks.py +++ b/libs/requests/hooks.py @@ -22,7 +22,10 @@ Available hooks: """ -import warnings +import traceback + + +HOOKS = ('args', 'pre_request', 'post_request', 'response') def dispatch_hook(key, hooks, hook_data): @@ -31,10 +34,15 @@ def dispatch_hook(key, hooks, hook_data): hooks = hooks or dict() if key in hooks: - try: - return hooks.get(key).__call__(hook_data) or hook_data + hooks = hooks.get(key) + + if hasattr(hooks, '__call__'): + hooks = [hooks] - except Exception, why: - warnings.warn(str(why)) + for hook in hooks: + try: + hook_data = hook(hook_data) or hook_data + except Exception: + traceback.print_exc() return hook_data diff --git a/libs/requests/models.py b/libs/requests/models.py index a16bcf2..c200896 100644 --- a/libs/requests/models.py +++ b/libs/requests/models.py @@ -8,15 +8,12 @@ This module contains the primary objects that power Requests. """ import os -import urllib - -from urlparse import urlparse, urlunparse, urljoin, urlsplit from datetime import datetime -from .hooks import dispatch_hook +from .hooks import dispatch_hook, HOOKS from .structures import CaseInsensitiveDict from .status_codes import codes -from .packages import oreos + from .auth import HTTPBasicAuth, HTTPProxyAuth from .packages.urllib3.response import HTTPResponse from .packages.urllib3.exceptions import MaxRetryError @@ -29,8 +26,15 @@ from .exceptions import ( URLRequired, SSLError) from .utils import ( get_encoding_from_headers, stream_decode_response_unicode, - stream_decompress, guess_filename, requote_path) + stream_decompress, guess_filename, requote_path, dict_from_string) +from .compat import urlparse, urlunparse, urljoin, urlsplit, urlencode, quote, unquote, str, bytes, SimpleCookie, is_py3, is_py2 + +# Import chardet if it is available. +try: + import chardet +except ImportError: + pass REDIRECT_STATI = (codes.moved, codes.found, codes.other, codes.temporary_moved) @@ -57,13 +61,19 @@ class Request(object): hooks=None, config=None, _poolmanager=None, - verify=None): + verify=None, + session=None): #: Float describes the timeout of the request. # (Use socket.setdefaulttimeout() as fallback) self.timeout = timeout #: Request URL. + + # if isinstance(url, str): + # url = url.encode('utf-8') + # print(dir(url)) + self.url = url #: Dictionary of HTTP Headers to attach to the :class:`Request `. @@ -82,7 +92,6 @@ class Request(object): #: Dictionary or byte of querystring data to attach to the #: :class:`Request `. self.params = None - self.params = dict(params or []) #: True if :class:`Request ` is part of a redirect chain (disables history #: and HTTPError storage). @@ -114,10 +123,18 @@ class Request(object): self.sent = False #: Event-handling hooks. - self.hooks = hooks + self.hooks = {} + + for event in HOOKS: + self.hooks[event] = [] + + hooks = hooks or {} + + for (k, v) in list(hooks.items()): + self.register_hook(event=k, hook=v) #: Session. - self.session = None + self.session = session #: SSL Verification. self.verify = verify @@ -128,7 +145,7 @@ class Request(object): headers = CaseInsensitiveDict() # Add configured base headers. - for (k, v) in self.config.get('base_headers', {}).items(): + for (k, v) in list(self.config.get('base_headers', {}).items()): if k not in headers: headers[k] = v @@ -144,7 +161,7 @@ class Request(object): return '' % (self.method) - def _build_response(self, resp, is_error=False): + def _build_response(self, resp): """Build internal :class:`Response ` object from given response. """ @@ -173,7 +190,7 @@ class Request(object): # Add new cookies from the server. if 'set-cookie' in response.headers: cookie_header = response.headers['set-cookie'] - cookies = oreos.dict_from_string(cookie_header) + cookies = dict_from_string(cookie_header) # Save cookies in Response. response.cookies = cookies @@ -183,10 +200,6 @@ class Request(object): # Save original response for later. response.raw = resp - - if is_error: - response.error = resp - response.url = self.full_url return response @@ -247,7 +260,8 @@ class Request(object): timeout=self.timeout, _poolmanager=self._poolmanager, proxies = self.proxies, - verify = self.verify + verify = self.verify, + session = self.session ) request.send() @@ -274,16 +288,17 @@ class Request(object): returns it twice. """ - if hasattr(data, '__iter__'): + if hasattr(data, '__iter__') and not isinstance(data, str): data = dict(data) + if hasattr(data, 'items'): result = [] - for k, vs in data.items(): + for k, vs in list(data.items()): for v in isinstance(vs, list) and vs or [vs]: - result.append((k.encode('utf-8') if isinstance(k, unicode) else k, - v.encode('utf-8') if isinstance(v, unicode) else v)) - return result, urllib.urlencode(result, doseq=True) + result.append((k.encode('utf-8') if isinstance(k, str) else k, + v.encode('utf-8') if isinstance(v, str) else v)) + return result, urlencode(result, doseq=True) else: return data, data @@ -294,20 +309,27 @@ class Request(object): if not self.url: raise URLRequired() + url = self.url + # Support for unicode domain names and paths. - scheme, netloc, path, params, query, fragment = urlparse(self.url) + scheme, netloc, path, params, query, fragment = urlparse(url) + if not scheme: - raise ValueError() + raise ValueError("Invalid URL %r: No schema supplied" % url) - netloc = netloc.encode('idna') + netloc = netloc.encode('idna').decode('utf-8') - if isinstance(path, unicode): - path = path.encode('utf-8') + if is_py2: + if isinstance(path, str): + path = path.encode('utf-8') - path = requote_path(path) + path = requote_path(path) - url = str(urlunparse([ scheme, netloc, path, params, query, fragment ])) + # print([ scheme, netloc, path, params, query, fragment ]) + # print('---------------------') + + url = (urlunparse([ scheme, netloc, path, params, query, fragment ])) if self._enc_params: if urlparse(url).query: @@ -332,6 +354,10 @@ class Request(object): path = p.path if not path: path = '/' + + # if is_py3: + path = quote(path.encode('utf-8')) + url.append(path) query = p.query @@ -339,9 +365,16 @@ class Request(object): url.append('?') url.append(query) + # print(url) + return ''.join(url) + def register_hook(self, event, hook): + """Properly register a hook.""" + + return self.hooks[event].append(hook) + def send(self, anyway=False, prefetch=False): """Sends the request. Returns True of successful, false if not. @@ -369,14 +402,14 @@ class Request(object): # Multi-part file uploads. if self.files: - if not isinstance(self.data, basestring): + if not isinstance(self.data, str): try: fields = self.data.copy() except AttributeError: fields = dict(self.data) - for (k, v) in self.files.items(): + for (k, v) in list(self.files.items()): # support for explicit filename if isinstance(v, (tuple, list)): fn, fp = v @@ -393,7 +426,7 @@ class Request(object): if self.data: body = self._enc_data - if isinstance(self.data, basestring): + if isinstance(self.data, str): content_type = None else: content_type = 'application/x-www-form-urlencoded' @@ -454,6 +487,9 @@ class Request(object): conn.cert_reqs = 'CERT_REQUIRED' conn.ca_certs = cert_loc + else: + conn.cert_reqs = 'CERT_NONE' + conn.ca_certs = None if not self.sent or anyway: @@ -463,8 +499,8 @@ class Request(object): if 'cookie' not in self.headers: # Simple cookie with our dict. - c = oreos.monkeys.SimpleCookie() - for (k, v) in self.cookies.items(): + c = SimpleCookie() + for (k, v) in list(self.cookies.items()): c[k] = v # Turn it into a header. @@ -493,16 +529,16 @@ class Request(object): ) self.sent = True - except MaxRetryError, e: + except MaxRetryError as e: raise ConnectionError(e) - except (_SSLError, _HTTPError), e: + except (_SSLError, _HTTPError) as e: if self.verify and isinstance(e, _SSLError): raise SSLError(e) raise Timeout('Request timed out.') - except RequestException, e: + except RequestException as e: if self.config.get('safe_mode', False): # In safe mode, catch the exception and attach it to # a blank urllib3.HTTPResponse object. @@ -524,7 +560,7 @@ class Request(object): if prefetch: # Save the response. self.response.content - + if self.config.get('danger_mode'): self.response.raise_for_status() @@ -581,6 +617,10 @@ class Response(object): def __repr__(self): return '' % (self.status_code) + def __bool__(self): + """Returns true if :attr:`status_code` is 'OK'.""" + return self.ok + def __nonzero__(self): """Returns true if :attr:`status_code` is 'OK'.""" return self.ok @@ -594,7 +634,7 @@ class Response(object): return True - def iter_content(self, chunk_size=10 * 1024, decode_unicode=None): + def iter_content(self, chunk_size=10 * 1024, decode_unicode=False): """Iterates over the response data. This avoids reading the content at once into memory for large responses. The chunk size is the number of bytes it should read into memory. This is not necessarily the @@ -613,16 +653,41 @@ class Response(object): yield chunk self._content_consumed = True - gen = generate() + def generate_chunked(): + resp = self.raw._original_response + fp = resp.fp + if resp.chunk_left is not None: + pending_bytes = resp.chunk_left + while pending_bytes: + chunk = fp.read(min(chunk_size, pending_bytes)) + pending_bytes-=len(chunk) + yield chunk + fp.read(2) # throw away crlf + while 1: + #XXX correct line size? (httplib has 64kb, seems insane) + pending_bytes = fp.readline(40).strip() + pending_bytes = int(pending_bytes, 16) + if pending_bytes == 0: + break + while pending_bytes: + chunk = fp.read(min(chunk_size, pending_bytes)) + pending_bytes-=len(chunk) + yield chunk + fp.read(2) # throw away crlf + self._content_consumed = True + fp.close() + + + if getattr(getattr(self.raw, '_original_response', None), 'chunked', False): + gen = generate_chunked() + else: + gen = generate() if 'gzip' in self.headers.get('content-encoding', ''): gen = stream_decompress(gen, mode='gzip') elif 'deflate' in self.headers.get('content-encoding', ''): gen = stream_decompress(gen, mode='deflate') - if decode_unicode is None: - decode_unicode = self.config.get('decode_unicode') - if decode_unicode: gen = stream_decode_response_unicode(gen, self) @@ -635,15 +700,29 @@ class Response(object): responses. """ + #TODO: why rstrip by default pending = None + for chunk in self.iter_content(chunk_size, decode_unicode=decode_unicode): + if pending is not None: chunk = pending + chunk lines = chunk.splitlines(True) + for line in lines[:-1]: yield line.rstrip() + # Save the last part of the chunk for next iteration, to keep full line together - pending = lines[-1] + # lines may be empty for the last chunk of a chunked response + + if lines: + pending = lines[-1] + #if pending is a complete line, give it baack + if pending[-1] == '\n': + yield pending.rstrip() + pending = None + else: + pending = None # Yield the last line if pending is not None: @@ -652,9 +731,7 @@ class Response(object): @property def content(self): - """Content of the response, in bytes or unicode - (if available). - """ + """Content of the response, in bytes.""" if self._content is None: # Read the contents. @@ -667,26 +744,45 @@ class Response(object): except AttributeError: self._content = None - content = self._content + self._content_consumed = True + return self._content - # Decode unicode content. - if self.config.get('decode_unicode'): - # Try charset from content-type + @property + def text(self): + """Content of the response, in unicode. - if self.encoding: - try: - content = unicode(content, self.encoding) - except UnicodeError: - pass + if Response.encoding is None and chardet module is available, encoding + will be guessed. + """ + + # Try charset from content-type + content = None + encoding = self.encoding + + # Fallback to auto-detected encoding if chardet is available. + if self.encoding is None: + try: + detected = chardet.detect(self.content) or {} + encoding = detected.get('encoding') + + # Trust that chardet isn't available or something went terribly wrong. + except Exception: + pass - # Fall back: + # Decode unicode from given encoding. + try: + content = str(self.content, encoding) + except (UnicodeError, TypeError): + pass + + # Try to fall back: + if not content: try: - content = unicode(content, self.encoding, errors='replace') - except TypeError: + content = str(content, encoding, errors='replace') + except (UnicodeError, TypeError): pass - self._content_consumed = True return content diff --git a/libs/requests/packages/oreos/monkeys.py b/libs/requests/packages/oreos/monkeys.py index 6be3074..2269e30 100644 --- a/libs/requests/packages/oreos/monkeys.py +++ b/libs/requests/packages/oreos/monkeys.py @@ -318,7 +318,7 @@ _Translator = { '\375' : '\\375', '\376' : '\\376', '\377' : '\\377' } -_idmap = ''.join(chr(x) for x in xrange(256)) +_idmap = ''.join(chr(x) for x in range(256)) def _quote(str, LegalChars=_LegalChars, idmap=_idmap, translate=string.translate): diff --git a/libs/requests/packages/urllib3/__init__.py b/libs/requests/packages/urllib3/__init__.py index 20b1fb4..5f70c56 100644 --- a/libs/requests/packages/urllib3/__init__.py +++ b/libs/requests/packages/urllib3/__init__.py @@ -1,5 +1,5 @@ # urllib3/__init__.py -# Copyright 2008-2011 Andrey Petrov and contributors (see CONTRIBUTORS.txt) +# Copyright 2008-2012 Andrey Petrov and contributors (see CONTRIBUTORS.txt) # # This module is part of urllib3 and is released under # the MIT License: http://www.opensource.org/licenses/mit-license.php @@ -10,7 +10,7 @@ urllib3 - Thread-safe connection pooling and re-using. __author__ = 'Andrey Petrov (andrey.petrov@shazow.net)' __license__ = 'MIT' -__version__ = '1.0.2' +__version__ = '1.1' from .connectionpool import ( diff --git a/libs/requests/packages/urllib3/_collections.py b/libs/requests/packages/urllib3/_collections.py index 00b2cd5..3cef081 100644 --- a/libs/requests/packages/urllib3/_collections.py +++ b/libs/requests/packages/urllib3/_collections.py @@ -1,5 +1,5 @@ # urllib3/_collections.py -# Copyright 2008-2011 Andrey Petrov and contributors (see CONTRIBUTORS.txt) +# Copyright 2008-2012 Andrey Petrov and contributors (see CONTRIBUTORS.txt) # # This module is part of urllib3 and is released under # the MIT License: http://www.opensource.org/licenses/mit-license.php diff --git a/libs/requests/packages/urllib3/connectionpool.py b/libs/requests/packages/urllib3/connectionpool.py index c1ebed4..52b1802 100644 --- a/libs/requests/packages/urllib3/connectionpool.py +++ b/libs/requests/packages/urllib3/connectionpool.py @@ -1,5 +1,5 @@ # urllib3/connectionpool.py -# Copyright 2008-2011 Andrey Petrov and contributors (see CONTRIBUTORS.txt) +# Copyright 2008-2012 Andrey Petrov and contributors (see CONTRIBUTORS.txt) # # This module is part of urllib3 and is released under # the MIT License: http://www.opensource.org/licenses/mit-license.php @@ -7,15 +7,27 @@ import logging import socket - -from httplib import HTTPConnection, HTTPSConnection, HTTPException -from Queue import Queue, Empty, Full -from select import select from socket import error as SocketError, timeout as SocketTimeout -from .packages.ssl_match_hostname import match_hostname, CertificateError - try: + from select import poll, POLLIN +except ImportError: # Doesn't exist on OSX and other platforms + from select import select + poll = False + +try: # Python 3 + from http.client import HTTPConnection, HTTPSConnection, HTTPException + from http.client import HTTP_PORT, HTTPS_PORT +except ImportError: + from httplib import HTTPConnection, HTTPSConnection, HTTPException + from httplib import HTTP_PORT, HTTPS_PORT + +try: # Python 3 + from queue import Queue, Empty, Full +except ImportError: + from Queue import Queue, Empty, Full + +try: # Compiled with SSL? import ssl BaseSSLError = ssl.SSLError except ImportError: @@ -23,21 +35,29 @@ except ImportError: BaseSSLError = None +from .packages.ssl_match_hostname import match_hostname, CertificateError from .request import RequestMethods from .response import HTTPResponse -from .exceptions import ( - SSLError, +from .exceptions import (SSLError, MaxRetryError, TimeoutError, HostChangedError, EmptyPoolError, ) +from .packages.ssl_match_hostname import match_hostname, CertificateError +from .packages import six + +xrange = six.moves.xrange log = logging.getLogger(__name__) _Default = object() +port_by_scheme = { + 'http': HTTP_PORT, + 'https': HTTPS_PORT, +} ## Connection objects (extension of httplib) @@ -81,7 +101,16 @@ class ConnectionPool(object): Base class for all connection pools, such as :class:`.HTTPConnectionPool` and :class:`.HTTPSConnectionPool`. """ - pass + + scheme = None + + def __init__(self, host, port=None): + self.host = host + self.port = port + + def __str__(self): + return '%s(host=%r, port=%r)' % (type(self).__name__, + self.host, self.port) class HTTPConnectionPool(ConnectionPool, RequestMethods): @@ -169,14 +198,14 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods): conn = self.pool.get(block=self.block, timeout=timeout) # If this is a persistent connection, check if it got disconnected - if conn and conn.sock and select([conn.sock], [], [], 0.0)[0]: - # Either data is buffered (bad), or the connection is dropped. + if conn and conn.sock and is_connection_dropped(conn): log.info("Resetting dropped connection: %s" % self.host) conn.close() except Empty: if self.block: - raise EmptyPoolError("Pool reached maximum size and no more " + raise EmptyPoolError(self, + "Pool reached maximum size and no more " "connections are allowed.") pass # Oh well, we'll create a new connection then @@ -229,11 +258,17 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods): def is_same_host(self, url): """ Check if the given ``url`` is a member of the same host as this - conncetion pool. + connection pool. """ # TODO: Add optional support for socket.gethostbyname checking. + scheme, host, port = get_host(url) + + if self.port and not port: + # Use explicit default port for comparison when none is given. + port = port_by_scheme.get(scheme) + return (url.startswith('/') or - get_host(url) == (self.scheme, self.host, self.port)) + (scheme, host, port) == (self.scheme, self.host, self.port)) def urlopen(self, method, url, body=None, headers=None, retries=3, redirect=True, assert_same_host=True, timeout=_Default, @@ -306,7 +341,7 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods): headers = self.headers if retries < 0: - raise MaxRetryError("Max retries exceeded for url: %s" % url) + raise MaxRetryError(self, url) if timeout is _Default: timeout = self.timeout @@ -320,8 +355,7 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods): if self.port: host = "%s:%d" % (host, self.port) - raise HostChangedError("Connection pool with host '%s' tried to " - "open a foreign host: %s" % (host, url)) + raise HostChangedError(self, url, retries - 1) conn = None @@ -352,27 +386,29 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods): # ``response.release_conn()`` is called (implicitly by # ``response.read()``) - except (Empty), e: + except Empty as e: # Timed out by queue - raise TimeoutError("Request timed out. (pool_timeout=%s)" % + raise TimeoutError(self, "Request timed out. (pool_timeout=%s)" % pool_timeout) - except (SocketTimeout), e: + except SocketTimeout as e: # Timed out by socket - raise TimeoutError("Request timed out. (timeout=%s)" % + raise TimeoutError(self, "Request timed out. (timeout=%s)" % timeout) - except (BaseSSLError), e: + except BaseSSLError as e: # SSL certificate error raise SSLError(e) - except (CertificateError), e: + except CertificateError as e: # Name mismatch raise SSLError(e) - - except (HTTPException, SocketError), e: + + except (HTTPException, SocketError) as e: # Connection broken, discard. It will be replaced next _get_conn(). conn = None + # This is necessary so we can access e below + err = e finally: if conn and release_conn: @@ -381,19 +417,16 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods): if not conn: log.warn("Retrying (%d attempts remain) after connection " - "broken by '%r': %s" % (retries, e, url)) + "broken by '%r': %s" % (retries, err, url)) return self.urlopen(method, url, body, headers, retries - 1, redirect, assert_same_host) # Try again - # Handle redirection - if (redirect and - response.status in [301, 302, 303, 307] and - 'location' in response.headers): # Redirect, retry - log.info("Redirecting %s -> %s" % - (url, response.headers.get('location'))) - return self.urlopen(method, response.headers.get('location'), body, - headers, retries - 1, redirect, - assert_same_host) + # Handle redirect? + redirect_location = redirect and response.get_redirect_location() + if redirect_location: + log.info("Redirecting %s -> %s" % (url, redirect_location)) + return self.urlopen(method, redirect_location, body, headers, + retries - 1, redirect, assert_same_host) return response @@ -550,3 +583,22 @@ def connection_from_url(url, **kw): return HTTPSConnectionPool(host, port=port, **kw) else: return HTTPConnectionPool(host, port=port, **kw) + + +def is_connection_dropped(conn): + """ + Returns True if the connection is dropped and should be closed. + + :param conn: + ``HTTPConnection`` object. + """ + if not poll: + return select([conn.sock], [], [], 0.0)[0] + + # This version is better on platforms that support it. + p = poll() + p.register(conn.sock, POLLIN) + for (fno, ev) in p.poll(0.0): + if fno == conn.sock.fileno(): + # Either data is buffered (bad), or the connection is dropped. + return True diff --git a/libs/requests/packages/urllib3/exceptions.py b/libs/requests/packages/urllib3/exceptions.py index 69f459b..0bffeb4 100644 --- a/libs/requests/packages/urllib3/exceptions.py +++ b/libs/requests/packages/urllib3/exceptions.py @@ -1,35 +1,54 @@ # urllib3/exceptions.py -# Copyright 2008-2011 Andrey Petrov and contributors (see CONTRIBUTORS.txt) +# Copyright 2008-2012 Andrey Petrov and contributors (see CONTRIBUTORS.txt) # # This module is part of urllib3 and is released under # the MIT License: http://www.opensource.org/licenses/mit-license.php -## Exceptions +## Base Exceptions class HTTPError(Exception): "Base exception used by this module." pass -class SSLError(Exception): +class PoolError(HTTPError): + "Base exception for errors caused within a pool." + def __init__(self, pool, message): + self.pool = pool + HTTPError.__init__(self, "%s: %s" % (pool, message)) + + +class SSLError(HTTPError): "Raised when SSL certificate fails in an HTTPS connection." pass -class MaxRetryError(HTTPError): - "Raised when the maximum number of retries is exceeded." - pass +## Leaf Exceptions +class MaxRetryError(PoolError): + "Raised when the maximum number of retries is exceeded." + def __init__(self, pool, url): + PoolError.__init__(self, pool, + "Max retries exceeded with url: %s" % url) -class TimeoutError(HTTPError): - "Raised when a socket timeout occurs." - pass + self.url = url -class HostChangedError(HTTPError): +class HostChangedError(PoolError): "Raised when an existing pool gets a request for a foreign host." + def __init__(self, pool, url, retries=3): + PoolError.__init__(self, pool, + "Tried to open a foreign host with url: %s" % url) + + self.url = url + self.retries = retries + + +class TimeoutError(PoolError): + "Raised when a socket timeout occurs." pass -class EmptyPoolError(HTTPError): + +class EmptyPoolError(PoolError): "Raised when a pool runs out of connections and no more are allowed." pass diff --git a/libs/requests/packages/urllib3/filepost.py b/libs/requests/packages/urllib3/filepost.py index 2ffea8b..e1ec8af 100644 --- a/libs/requests/packages/urllib3/filepost.py +++ b/libs/requests/packages/urllib3/filepost.py @@ -1,18 +1,21 @@ # urllib3/filepost.py -# Copyright 2008-2011 Andrey Petrov and contributors (see CONTRIBUTORS.txt) +# Copyright 2008-2012 Andrey Petrov and contributors (see CONTRIBUTORS.txt) # # This module is part of urllib3 and is released under # the MIT License: http://www.opensource.org/licenses/mit-license.php import codecs -import mimetools import mimetypes try: - from cStringIO import StringIO + from mimetools import choose_boundary except ImportError: - from StringIO import StringIO # pylint: disable-msg=W0404 + from .packages.mimetools_choose_boundary import choose_boundary +from io import BytesIO + +from .packages import six +from .packages.six import b writer = codecs.lookup('utf-8')[3] @@ -35,37 +38,37 @@ def encode_multipart_formdata(fields, boundary=None): If not specified, then a random boundary will be generated using :func:`mimetools.choose_boundary`. """ - body = StringIO() + body = BytesIO() if boundary is None: - boundary = mimetools.choose_boundary() + boundary = choose_boundary() - for fieldname, value in fields.iteritems(): - body.write('--%s\r\n' % (boundary)) + for fieldname, value in six.iteritems(fields): + body.write(b('--%s\r\n' % (boundary))) if isinstance(value, tuple): filename, data = value writer(body).write('Content-Disposition: form-data; name="%s"; ' 'filename="%s"\r\n' % (fieldname, filename)) - body.write('Content-Type: %s\r\n\r\n' % - (get_content_type(filename))) + body.write(b('Content-Type: %s\r\n\r\n' % + (get_content_type(filename)))) else: data = value writer(body).write('Content-Disposition: form-data; name="%s"\r\n' % (fieldname)) - body.write('Content-Type: text/plain\r\n\r\n') + body.write(b'Content-Type: text/plain\r\n\r\n') if isinstance(data, int): data = str(data) # Backwards compatibility - if isinstance(data, unicode): + if isinstance(data, six.text_type): writer(body).write(data) else: body.write(data) - body.write('\r\n') + body.write(b'\r\n') - body.write('--%s--\r\n' % (boundary)) + body.write(b('--%s--\r\n' % (boundary))) - content_type = 'multipart/form-data; boundary=%s' % boundary + content_type = b('multipart/form-data; boundary=%s' % boundary) return body.getvalue(), content_type diff --git a/libs/requests/packages/urllib3/packages/mimetools_choose_boundary/__init__.py b/libs/requests/packages/urllib3/packages/mimetools_choose_boundary/__init__.py new file mode 100644 index 0000000..a0109ab --- /dev/null +++ b/libs/requests/packages/urllib3/packages/mimetools_choose_boundary/__init__.py @@ -0,0 +1,47 @@ +"""The function mimetools.choose_boundary() from Python 2.7, which seems to +have disappeared in Python 3 (although email.generator._make_boundary() might +work as a replacement?). + +Tweaked to use lock from threading rather than thread. +""" +import os +from threading import Lock +_counter_lock = Lock() + +_counter = 0 +def _get_next_counter(): + global _counter + with _counter_lock: + _counter += 1 + return _counter + +_prefix = None + +def choose_boundary(): + """Return a string usable as a multipart boundary. + + The string chosen is unique within a single program run, and + incorporates the user id (if available), process id (if available), + and current time. So it's very unlikely the returned string appears + in message text, but there's no guarantee. + + The boundary contains dots so you have to quote it in the header.""" + + global _prefix + import time + if _prefix is None: + import socket + try: + hostid = socket.gethostbyname(socket.gethostname()) + except socket.gaierror: + hostid = '127.0.0.1' + try: + uid = repr(os.getuid()) + except AttributeError: + uid = '1' + try: + pid = repr(os.getpid()) + except AttributeError: + pid = '1' + _prefix = hostid + '.' + uid + '.' + pid + return "%s.%.3f.%d" % (_prefix, time.time(), _get_next_counter()) diff --git a/libs/requests/packages/urllib3/packages/six.py b/libs/requests/packages/urllib3/packages/six.py new file mode 100644 index 0000000..a64f6fb --- /dev/null +++ b/libs/requests/packages/urllib3/packages/six.py @@ -0,0 +1,372 @@ +"""Utilities for writing code that runs on Python 2 and 3""" + +#Copyright (c) 2010-2011 Benjamin Peterson + +#Permission is hereby granted, free of charge, to any person obtaining a copy of +#this software and associated documentation files (the "Software"), to deal in +#the Software without restriction, including without limitation the rights to +#use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of +#the Software, and to permit persons to whom the Software is furnished to do so, +#subject to the following conditions: + +#The above copyright notice and this permission notice shall be included in all +#copies or substantial portions of the Software. + +#THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +#IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +#FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR +#COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER +#IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN +#CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +import operator +import sys +import types + +__author__ = "Benjamin Peterson " +__version__ = "1.1.0" + + +# True if we are running on Python 3. +PY3 = sys.version_info[0] == 3 + +if PY3: + string_types = str, + integer_types = int, + class_types = type, + text_type = str + binary_type = bytes + + MAXSIZE = sys.maxsize +else: + string_types = basestring, + integer_types = (int, long) + class_types = (type, types.ClassType) + text_type = unicode + binary_type = str + + # It's possible to have sizeof(long) != sizeof(Py_ssize_t). + class X(object): + def __len__(self): + return 1 << 31 + try: + len(X()) + except OverflowError: + # 32-bit + MAXSIZE = int((1 << 31) - 1) + else: + # 64-bit + MAXSIZE = int((1 << 63) - 1) + del X + + +def _add_doc(func, doc): + """Add documentation to a function.""" + func.__doc__ = doc + + +def _import_module(name): + """Import module, returning the module after the last dot.""" + __import__(name) + return sys.modules[name] + + +class _LazyDescr(object): + + def __init__(self, name): + self.name = name + + def __get__(self, obj, tp): + result = self._resolve() + setattr(obj, self.name, result) + # This is a bit ugly, but it avoids running this again. + delattr(tp, self.name) + return result + + +class MovedModule(_LazyDescr): + + def __init__(self, name, old, new=None): + super(MovedModule, self).__init__(name) + if PY3: + if new is None: + new = name + self.mod = new + else: + self.mod = old + + def _resolve(self): + return _import_module(self.mod) + + +class MovedAttribute(_LazyDescr): + + def __init__(self, name, old_mod, new_mod, old_attr=None, new_attr=None): + super(MovedAttribute, self).__init__(name) + if PY3: + if new_mod is None: + new_mod = name + self.mod = new_mod + if new_attr is None: + if old_attr is None: + new_attr = name + else: + new_attr = old_attr + self.attr = new_attr + else: + self.mod = old_mod + if old_attr is None: + old_attr = name + self.attr = old_attr + + def _resolve(self): + module = _import_module(self.mod) + return getattr(module, self.attr) + + + +class _MovedItems(types.ModuleType): + """Lazy loading of moved objects""" + + +_moved_attributes = [ + MovedAttribute("cStringIO", "cStringIO", "io", "StringIO"), + MovedAttribute("filter", "itertools", "builtins", "ifilter", "filter"), + MovedAttribute("map", "itertools", "builtins", "imap", "map"), + MovedAttribute("reload_module", "__builtin__", "imp", "reload"), + MovedAttribute("reduce", "__builtin__", "functools"), + MovedAttribute("StringIO", "StringIO", "io"), + MovedAttribute("xrange", "__builtin__", "builtins", "xrange", "range"), + MovedAttribute("zip", "itertools", "builtins", "izip", "zip"), + + MovedModule("builtins", "__builtin__"), + MovedModule("configparser", "ConfigParser"), + MovedModule("copyreg", "copy_reg"), + MovedModule("http_cookiejar", "cookielib", "http.cookiejar"), + MovedModule("http_cookies", "Cookie", "http.cookies"), + MovedModule("html_entities", "htmlentitydefs", "html.entities"), + MovedModule("html_parser", "HTMLParser", "html.parser"), + MovedModule("http_client", "httplib", "http.client"), + MovedModule("BaseHTTPServer", "BaseHTTPServer", "http.server"), + MovedModule("CGIHTTPServer", "CGIHTTPServer", "http.server"), + MovedModule("SimpleHTTPServer", "SimpleHTTPServer", "http.server"), + MovedModule("cPickle", "cPickle", "pickle"), + MovedModule("queue", "Queue"), + MovedModule("reprlib", "repr"), + MovedModule("socketserver", "SocketServer"), + MovedModule("tkinter", "Tkinter"), + MovedModule("tkinter_dialog", "Dialog", "tkinter.dialog"), + MovedModule("tkinter_filedialog", "FileDialog", "tkinter.filedialog"), + MovedModule("tkinter_scrolledtext", "ScrolledText", "tkinter.scrolledtext"), + MovedModule("tkinter_simpledialog", "SimpleDialog", "tkinter.simpledialog"), + MovedModule("tkinter_tix", "Tix", "tkinter.tix"), + MovedModule("tkinter_constants", "Tkconstants", "tkinter.constants"), + MovedModule("tkinter_dnd", "Tkdnd", "tkinter.dnd"), + MovedModule("tkinter_colorchooser", "tkColorChooser", + "tkinter.colorchooser"), + MovedModule("tkinter_commondialog", "tkCommonDialog", + "tkinter.commondialog"), + MovedModule("tkinter_tkfiledialog", "tkFileDialog", "tkinter.filedialog"), + MovedModule("tkinter_font", "tkFont", "tkinter.font"), + MovedModule("tkinter_messagebox", "tkMessageBox", "tkinter.messagebox"), + MovedModule("tkinter_tksimpledialog", "tkSimpleDialog", + "tkinter.simpledialog"), + MovedModule("urllib_robotparser", "robotparser", "urllib.robotparser"), + MovedModule("winreg", "_winreg"), +] +for attr in _moved_attributes: + setattr(_MovedItems, attr.name, attr) +del attr + +moves = sys.modules["six.moves"] = _MovedItems("moves") + + +def add_move(move): + """Add an item to six.moves.""" + setattr(_MovedItems, move.name, move) + + +def remove_move(name): + """Remove item from six.moves.""" + try: + delattr(_MovedItems, name) + except AttributeError: + try: + del moves.__dict__[name] + except KeyError: + raise AttributeError("no such move, %r" % (name,)) + + +if PY3: + _meth_func = "__func__" + _meth_self = "__self__" + + _func_code = "__code__" + _func_defaults = "__defaults__" + + _iterkeys = "keys" + _itervalues = "values" + _iteritems = "items" +else: + _meth_func = "im_func" + _meth_self = "im_self" + + _func_code = "func_code" + _func_defaults = "func_defaults" + + _iterkeys = "iterkeys" + _itervalues = "itervalues" + _iteritems = "iteritems" + + +if PY3: + def get_unbound_function(unbound): + return unbound + + + advance_iterator = next + + def callable(obj): + return any("__call__" in klass.__dict__ for klass in type(obj).__mro__) +else: + def get_unbound_function(unbound): + return unbound.im_func + + + def advance_iterator(it): + return it.next() + + callable = callable +_add_doc(get_unbound_function, + """Get the function out of a possibly unbound function""") + + +get_method_function = operator.attrgetter(_meth_func) +get_method_self = operator.attrgetter(_meth_self) +get_function_code = operator.attrgetter(_func_code) +get_function_defaults = operator.attrgetter(_func_defaults) + + +def iterkeys(d): + """Return an iterator over the keys of a dictionary.""" + return getattr(d, _iterkeys)() + +def itervalues(d): + """Return an iterator over the values of a dictionary.""" + return getattr(d, _itervalues)() + +def iteritems(d): + """Return an iterator over the (key, value) pairs of a dictionary.""" + return getattr(d, _iteritems)() + + +if PY3: + def b(s): + return s.encode("latin-1") + def u(s): + return s + if sys.version_info[1] <= 1: + def int2byte(i): + return bytes((i,)) + else: + # This is about 2x faster than the implementation above on 3.2+ + int2byte = operator.methodcaller("to_bytes", 1, "big") + import io + StringIO = io.StringIO + BytesIO = io.BytesIO +else: + def b(s): + return s + def u(s): + return unicode(s, "unicode_escape") + int2byte = chr + import StringIO + StringIO = BytesIO = StringIO.StringIO +_add_doc(b, """Byte literal""") +_add_doc(u, """Text literal""") + + +if PY3: + import builtins + exec_ = getattr(builtins, "exec") + + + def reraise(tp, value, tb=None): + if value.__traceback__ is not tb: + raise value.with_traceback(tb) + raise value + + + print_ = getattr(builtins, "print") + del builtins + +else: + def exec_(code, globs=None, locs=None): + """Execute code in a namespace.""" + if globs is None: + frame = sys._getframe(1) + globs = frame.f_globals + if locs is None: + locs = frame.f_locals + del frame + elif locs is None: + locs = globs + exec("""exec code in globs, locs""") + + + exec_("""def reraise(tp, value, tb=None): + raise tp, value, tb +""") + + + def print_(*args, **kwargs): + """The new-style print function.""" + fp = kwargs.pop("file", sys.stdout) + if fp is None: + return + def write(data): + if not isinstance(data, basestring): + data = str(data) + fp.write(data) + want_unicode = False + sep = kwargs.pop("sep", None) + if sep is not None: + if isinstance(sep, unicode): + want_unicode = True + elif not isinstance(sep, str): + raise TypeError("sep must be None or a string") + end = kwargs.pop("end", None) + if end is not None: + if isinstance(end, unicode): + want_unicode = True + elif not isinstance(end, str): + raise TypeError("end must be None or a string") + if kwargs: + raise TypeError("invalid keyword arguments to print()") + if not want_unicode: + for arg in args: + if isinstance(arg, unicode): + want_unicode = True + break + if want_unicode: + newline = unicode("\n") + space = unicode(" ") + else: + newline = "\n" + space = " " + if sep is None: + sep = space + if end is None: + end = newline + for i, arg in enumerate(args): + if i: + write(sep) + write(arg) + write(end) + +_add_doc(reraise, """Reraise an exception.""") + + +def with_metaclass(meta, base=object): + """Create a base class with a metaclass.""" + return meta("NewBase", (base,), {}) diff --git a/libs/requests/packages/urllib3/poolmanager.py b/libs/requests/packages/urllib3/poolmanager.py index c08e327..f194b2e 100644 --- a/libs/requests/packages/urllib3/poolmanager.py +++ b/libs/requests/packages/urllib3/poolmanager.py @@ -1,32 +1,27 @@ # urllib3/poolmanager.py -# Copyright 2008-2011 Andrey Petrov and contributors (see CONTRIBUTORS.txt) +# Copyright 2008-2012 Andrey Petrov and contributors (see CONTRIBUTORS.txt) # # This module is part of urllib3 and is released under # the MIT License: http://www.opensource.org/licenses/mit-license.php +import logging + from ._collections import RecentlyUsedContainer -from .connectionpool import ( - HTTPConnectionPool, HTTPSConnectionPool, - get_host, connection_from_url, -) +from .connectionpool import HTTPConnectionPool, HTTPSConnectionPool +from .connectionpool import get_host, connection_from_url, port_by_scheme +from .exceptions import HostChangedError +from .request import RequestMethods __all__ = ['PoolManager', 'ProxyManager', 'proxy_from_url'] -from .request import RequestMethods -from .connectionpool import HTTPConnectionPool, HTTPSConnectionPool - - pool_classes_by_scheme = { 'http': HTTPConnectionPool, 'https': HTTPSConnectionPool, } -port_by_scheme = { - 'http': 80, - 'https': 443, -} +log = logging.getLogger(__name__) class PoolManager(RequestMethods): @@ -105,7 +100,12 @@ class PoolManager(RequestMethods): :class:`urllib3.connectionpool.ConnectionPool` can be chosen for it. """ conn = self.connection_from_url(url) - return conn.urlopen(method, url, assert_same_host=False, **kw) + try: + return conn.urlopen(method, url, **kw) + + except HostChangedError as e: + kw['retries'] = e.retries # Persist retries countdown + return self.urlopen(method, e.url, **kw) class ProxyManager(RequestMethods): diff --git a/libs/requests/packages/urllib3/request.py b/libs/requests/packages/urllib3/request.py index a7e0b5d..5ea26a0 100644 --- a/libs/requests/packages/urllib3/request.py +++ b/libs/requests/packages/urllib3/request.py @@ -1,11 +1,13 @@ # urllib3/request.py -# Copyright 2008-2011 Andrey Petrov and contributors (see CONTRIBUTORS.txt) +# Copyright 2008-2012 Andrey Petrov and contributors (see CONTRIBUTORS.txt) # # This module is part of urllib3 and is released under # the MIT License: http://www.opensource.org/licenses/mit-license.php - -from urllib import urlencode +try: + from urllib.parse import urlencode +except ImportError: + from urllib import urlencode from .filepost import encode_multipart_formdata diff --git a/libs/requests/packages/urllib3/response.py b/libs/requests/packages/urllib3/response.py index 4cd15c1..e023970 100644 --- a/libs/requests/packages/urllib3/response.py +++ b/libs/requests/packages/urllib3/response.py @@ -1,5 +1,5 @@ # urllib3/response.py -# Copyright 2008-2011 Andrey Petrov and contributors (see CONTRIBUTORS.txt) +# Copyright 2008-2012 Andrey Petrov and contributors (see CONTRIBUTORS.txt) # # This module is part of urllib3 and is released under # the MIT License: http://www.opensource.org/licenses/mit-license.php @@ -8,21 +8,22 @@ import gzip import logging import zlib +from io import BytesIO -try: - from cStringIO import StringIO -except ImportError: - from StringIO import StringIO # pylint: disable-msg=W0404 +from .exceptions import HTTPError -from .exceptions import HTTPError +try: + basestring = basestring +except NameError: # Python 3 + basestring = (str, bytes) log = logging.getLogger(__name__) def decode_gzip(data): - gzipper = gzip.GzipFile(fileobj=StringIO(data)) + gzipper = gzip.GzipFile(fileobj=BytesIO(data)) return gzipper.read() @@ -71,7 +72,7 @@ class HTTPResponse(object): self.strict = strict self._decode_content = decode_content - self._body = None + self._body = body if body and isinstance(body, basestring) else None self._fp = None self._original_response = original_response @@ -81,9 +82,22 @@ class HTTPResponse(object): if hasattr(body, 'read'): self._fp = body - if preload_content: + if preload_content and not self._body: self._body = self.read(decode_content=decode_content) + def get_redirect_location(self): + """ + Should we redirect and where to? + + :returns: Truthy redirect location string if we got a redirect status + code and valid location. ``None`` if redirect status and no + location. ``False`` if not a redirect status code. + """ + if self.status in [301, 302, 303, 307]: + return self.headers.get('location') + + return False + def release_conn(self): if not self._pool or not self._connection: return @@ -98,10 +112,9 @@ class HTTPResponse(object): return self._body if self._fp: - return self.read(decode_content=self._decode_content, - cache_content=True) + return self.read(cache_content=True) - def read(self, amt=None, decode_content=True, cache_content=False): + def read(self, amt=None, decode_content=None, cache_content=False): """ Similar to :meth:`httplib.HTTPResponse.read`, but with two additional parameters: ``decode_content`` and ``cache_content``. @@ -124,22 +137,22 @@ class HTTPResponse(object): """ content_encoding = self.headers.get('content-encoding') decoder = self.CONTENT_DECODERS.get(content_encoding) + if decode_content is None: + decode_content = self._decode_content - data = self._fp and self._fp.read(amt) + if self._fp is None: + return try: - - if amt: - return data - - if not decode_content or not decoder: - if cache_content: - self._body = data - - return data + if amt is None: + # cStringIO doesn't like amt=None + data = self._fp.read() + else: + return self._fp.read(amt) try: - data = decoder(data) + if decode_content and decoder: + data = decoder(data) except IOError: raise HTTPError("Received response with content-encoding: %s, but " "failed to decode it." % content_encoding) @@ -150,12 +163,11 @@ class HTTPResponse(object): return data finally: - if self._original_response and self._original_response.isclosed(): self.release_conn() - @staticmethod - def from_httplib(r, **response_kw): + @classmethod + def from_httplib(ResponseCls, r, **response_kw): """ Given an :class:`httplib.HTTPResponse` instance ``r``, return a corresponding :class:`urllib3.response.HTTPResponse` object. @@ -164,14 +176,17 @@ class HTTPResponse(object): with ``original_response=r``. """ - return HTTPResponse(body=r, - headers=dict(r.getheaders()), - status=r.status, - version=r.version, - reason=r.reason, - strict=r.strict, - original_response=r, - **response_kw) + # HTTPResponse objects in Python 3 don't have a .strict attribute + strict = getattr(r, 'strict', 0) + return ResponseCls(body=r, + # In Python 3, the header keys are returned capitalised + headers=dict((k.lower(), v) for k,v in r.getheaders()), + status=r.status, + version=r.version, + reason=r.reason, + strict=strict, + original_response=r, + **response_kw) # Backwards-compatibility methods for httplib.HTTPResponse def getheaders(self): diff --git a/libs/requests/sessions.py b/libs/requests/sessions.py index af9f9c7..d9683b0 100644 --- a/libs/requests/sessions.py +++ b/libs/requests/sessions.py @@ -25,7 +25,7 @@ def merge_kwargs(local_kwarg, default_kwarg): if default_kwarg is None: return local_kwarg - if isinstance(local_kwarg, basestring): + if isinstance(local_kwarg, str): return local_kwarg if local_kwarg is None: @@ -40,7 +40,7 @@ def merge_kwargs(local_kwarg, default_kwarg): kwargs.update(local_kwarg) # Remove keys that are set to None. - for (k,v) in local_kwarg.items(): + for (k,v) in list(local_kwarg.items()): if v is None: del kwargs[k] @@ -76,7 +76,7 @@ class Session(object): self.config = config or {} self.verify = verify - for (k, v) in defaults.items(): + for (k, v) in list(defaults.items()): self.config.setdefault(k, v) self.poolmanager = PoolManager( @@ -150,12 +150,12 @@ class Session(object): verify = self.verify # use session's hooks as defaults - for key, cb in self.hooks.iteritems(): + for key, cb in list(self.hooks.items()): hooks.setdefault(key, cb) # Expand header values. if headers: - for k, v in headers.items() or {}: + for k, v in list(headers.items()) or {}: headers[k] = header_expand(v) args = dict( diff --git a/libs/requests/status_codes.py b/libs/requests/status_codes.py index fab8e95..da74286 100644 --- a/libs/requests/status_codes.py +++ b/libs/requests/status_codes.py @@ -79,7 +79,7 @@ _codes = { codes = LookupDict(name='status_codes') -for (code, titles) in _codes.items(): +for (code, titles) in list(_codes.items()): for title in titles: setattr(codes, title, code) if not title.startswith('\\'): diff --git a/libs/requests/structures.py b/libs/requests/structures.py index 35a903f..3746754 100644 --- a/libs/requests/structures.py +++ b/libs/requests/structures.py @@ -18,7 +18,7 @@ class CaseInsensitiveDict(dict): @property def lower_keys(self): if not hasattr(self, '_lower_keys') or not self._lower_keys: - self._lower_keys = dict((k.lower(), k) for k in self.iterkeys()) + self._lower_keys = dict((k.lower(), k) for k in list(self.keys())) return self._lower_keys def _clear_lower_keys(self): @@ -63,4 +63,4 @@ class LookupDict(dict): return self.__dict__.get(key, None) def get(self, key, default=None): - return self.__dict__.get(key, default) \ No newline at end of file + return self.__dict__.get(key, default) diff --git a/libs/requests/utils.py b/libs/requests/utils.py index c7ab0a4..0e0f69e 100644 --- a/libs/requests/utils.py +++ b/libs/requests/utils.py @@ -11,16 +11,28 @@ that are also useful for external consumption. import cgi import codecs -import cookielib import os import random import re import zlib -import urllib -from urllib2 import parse_http_list as _parse_list_header +from .compat import parse_http_list as _parse_list_header +from .compat import quote, unquote, cookielib, SimpleCookie, is_py2 +def dict_from_string(s): + """Returns a MultiDict with Cookies.""" + + cookies = dict() + + c = SimpleCookie() + c.load(s) + + for k,v in list(c.items()): + cookies.update({k: v.value}) + + return cookies + def guess_filename(obj): """Tries to guess the filename of the given object.""" name = getattr(obj, 'name', None) @@ -132,16 +144,16 @@ def header_expand(headers): collector = [] if isinstance(headers, dict): - headers = headers.items() + headers = list(headers.items()) - elif isinstance(headers, basestring): + elif isinstance(headers, str): return headers for i, (value, params) in enumerate(headers): _params = [] - for (p_k, p_v) in params.items(): + for (p_k, p_v) in list(params.items()): _params.append('%s=%s' % (p_k, p_v)) @@ -166,17 +178,11 @@ def header_expand(headers): def randombytes(n): """Return n random bytes.""" - # Use /dev/urandom if it is available. Fall back to random module - # if not. It might be worthwhile to extend this function to use - # other platform-specific mechanisms for getting random bytes. - if os.path.exists("/dev/urandom"): - f = open("/dev/urandom") - s = f.read(n) - f.close() - return s - else: + if is_py2: L = [chr(random.randrange(0, 256)) for i in range(n)] - return "".join(L) + else: + L = [chr(random.randrange(0, 256)).encode('utf-8') for i in range(n)] + return b"".join(L) def dict_from_cookiejar(cj): @@ -187,9 +193,9 @@ def dict_from_cookiejar(cj): cookie_dict = {} - for _, cookies in cj._cookies.items(): - for _, cookies in cookies.items(): - for cookie in cookies.values(): + for _, cookies in list(cj._cookies.items()): + for _, cookies in list(cookies.items()): + for cookie in list(cookies.values()): # print cookie cookie_dict[cookie.name] = cookie.value @@ -221,7 +227,7 @@ def add_dict_to_cookiejar(cj, cookie_dict): :param cookie_dict: Dict of key/values to insert into CookieJar. """ - for k, v in cookie_dict.items(): + for k, v in list(cookie_dict.items()): cookie = cookielib.Cookie( version=0, @@ -276,6 +282,9 @@ def get_encoding_from_headers(headers): if 'charset' in params: return params['charset'].strip("'\"") + if 'text' in content_type: + return 'ISO-8859-1' + def unicode_from_html(content): """Attempts to decode an HTML string into unicode. @@ -287,7 +296,7 @@ def unicode_from_html(content): for encoding in encodings: try: - return unicode(content, encoding) + return str(content, encoding) except (UnicodeError, TypeError): pass @@ -334,13 +343,13 @@ def get_unicode_from_response(r): if encoding: try: - return unicode(r.content, encoding) + return str(r.content, encoding) except UnicodeError: tried_encodings.append(encoding) # Fall back: try: - return unicode(r.content, encoding, errors='replace') + return str(r.content, encoding, errors='replace') except TypeError: return r.content @@ -393,6 +402,6 @@ def requote_path(path): This function passes the given path through an unquote/quote cycle to ensure that it is fully and consistently quoted. """ - parts = path.split("/") - parts = (urllib.quote(urllib.unquote(part), safe="") for part in parts) - return "/".join(parts) + parts = path.split(b"/") + parts = (quote(unquote(part), safe=b"") for part in parts) + return b"/".join(parts) diff --git a/libs/simplejson/__init__.py b/libs/simplejson/__init__.py deleted file mode 100644 index c36be23..0000000 --- a/libs/simplejson/__init__.py +++ /dev/null @@ -1,438 +0,0 @@ -r"""JSON (JavaScript Object Notation) is a subset of -JavaScript syntax (ECMA-262 3rd edition) used as a lightweight data -interchange format. - -:mod:`simplejson` exposes an API familiar to users of the standard library -:mod:`marshal` and :mod:`pickle` modules. It is the externally maintained -version of the :mod:`json` library contained in Python 2.6, but maintains -compatibility with Python 2.4 and Python 2.5 and (currently) has -significant performance advantages, even without using the optional C -extension for speedups. - -Encoding basic Python object hierarchies:: - - >>> import simplejson as json - >>> json.dumps(['foo', {'bar': ('baz', None, 1.0, 2)}]) - '["foo", {"bar": ["baz", null, 1.0, 2]}]' - >>> print json.dumps("\"foo\bar") - "\"foo\bar" - >>> print json.dumps(u'\u1234') - "\u1234" - >>> print json.dumps('\\') - "\\" - >>> print json.dumps({"c": 0, "b": 0, "a": 0}, sort_keys=True) - {"a": 0, "b": 0, "c": 0} - >>> from StringIO import StringIO - >>> io = StringIO() - >>> json.dump(['streaming API'], io) - >>> io.getvalue() - '["streaming API"]' - -Compact encoding:: - - >>> import simplejson as json - >>> json.dumps([1,2,3,{'4': 5, '6': 7}], separators=(',',':')) - '[1,2,3,{"4":5,"6":7}]' - -Pretty printing:: - - >>> import simplejson as json - >>> s = json.dumps({'4': 5, '6': 7}, sort_keys=True, indent=' ') - >>> print '\n'.join([l.rstrip() for l in s.splitlines()]) - { - "4": 5, - "6": 7 - } - -Decoding JSON:: - - >>> import simplejson as json - >>> obj = [u'foo', {u'bar': [u'baz', None, 1.0, 2]}] - >>> json.loads('["foo", {"bar":["baz", null, 1.0, 2]}]') == obj - True - >>> json.loads('"\\"foo\\bar"') == u'"foo\x08ar' - True - >>> from StringIO import StringIO - >>> io = StringIO('["streaming API"]') - >>> json.load(io)[0] == 'streaming API' - True - -Specializing JSON object decoding:: - - >>> import simplejson as json - >>> def as_complex(dct): - ... if '__complex__' in dct: - ... return complex(dct['real'], dct['imag']) - ... return dct - ... - >>> json.loads('{"__complex__": true, "real": 1, "imag": 2}', - ... object_hook=as_complex) - (1+2j) - >>> from decimal import Decimal - >>> json.loads('1.1', parse_float=Decimal) == Decimal('1.1') - True - -Specializing JSON object encoding:: - - >>> import simplejson as json - >>> def encode_complex(obj): - ... if isinstance(obj, complex): - ... return [obj.real, obj.imag] - ... raise TypeError(repr(o) + " is not JSON serializable") - ... - >>> json.dumps(2 + 1j, default=encode_complex) - '[2.0, 1.0]' - >>> json.JSONEncoder(default=encode_complex).encode(2 + 1j) - '[2.0, 1.0]' - >>> ''.join(json.JSONEncoder(default=encode_complex).iterencode(2 + 1j)) - '[2.0, 1.0]' - - -Using simplejson.tool from the shell to validate and pretty-print:: - - $ echo '{"json":"obj"}' | python -m simplejson.tool - { - "json": "obj" - } - $ echo '{ 1.2:3.4}' | python -m simplejson.tool - Expecting property name: line 1 column 2 (char 2) -""" -__version__ = '2.1.3' -__all__ = [ - 'dump', 'dumps', 'load', 'loads', - 'JSONDecoder', 'JSONDecodeError', 'JSONEncoder', - 'OrderedDict', -] - -__author__ = 'Bob Ippolito ' - -from decimal import Decimal - -from decoder import JSONDecoder, JSONDecodeError -from encoder import JSONEncoder -def _import_OrderedDict(): - import collections - try: - return collections.OrderedDict - except AttributeError: - import ordered_dict - return ordered_dict.OrderedDict -OrderedDict = _import_OrderedDict() - -def _import_c_make_encoder(): - try: - from simplejson._speedups import make_encoder - return make_encoder - except ImportError: - return None - -_default_encoder = JSONEncoder( - skipkeys=False, - ensure_ascii=True, - check_circular=True, - allow_nan=True, - indent=None, - separators=None, - encoding='utf-8', - default=None, - use_decimal=False, -) - -def dump(obj, fp, skipkeys=False, ensure_ascii=True, check_circular=True, - allow_nan=True, cls=None, indent=None, separators=None, - encoding='utf-8', default=None, use_decimal=False, **kw): - """Serialize ``obj`` as a JSON formatted stream to ``fp`` (a - ``.write()``-supporting file-like object). - - If ``skipkeys`` is true then ``dict`` keys that are not basic types - (``str``, ``unicode``, ``int``, ``long``, ``float``, ``bool``, ``None``) - will be skipped instead of raising a ``TypeError``. - - If ``ensure_ascii`` is false, then the some chunks written to ``fp`` - may be ``unicode`` instances, subject to normal Python ``str`` to - ``unicode`` coercion rules. Unless ``fp.write()`` explicitly - understands ``unicode`` (as in ``codecs.getwriter()``) this is likely - to cause an error. - - If ``check_circular`` is false, then the circular reference check - for container types will be skipped and a circular reference will - result in an ``OverflowError`` (or worse). - - If ``allow_nan`` is false, then it will be a ``ValueError`` to - serialize out of range ``float`` values (``nan``, ``inf``, ``-inf``) - in strict compliance of the JSON specification, instead of using the - JavaScript equivalents (``NaN``, ``Infinity``, ``-Infinity``). - - If *indent* is a string, then JSON array elements and object members - will be pretty-printed with a newline followed by that string repeated - for each level of nesting. ``None`` (the default) selects the most compact - representation without any newlines. For backwards compatibility with - versions of simplejson earlier than 2.1.0, an integer is also accepted - and is converted to a string with that many spaces. - - If ``separators`` is an ``(item_separator, dict_separator)`` tuple - then it will be used instead of the default ``(', ', ': ')`` separators. - ``(',', ':')`` is the most compact JSON representation. - - ``encoding`` is the character encoding for str instances, default is UTF-8. - - ``default(obj)`` is a function that should return a serializable version - of obj or raise TypeError. The default simply raises TypeError. - - If *use_decimal* is true (default: ``False``) then decimal.Decimal - will be natively serialized to JSON with full precision. - - To use a custom ``JSONEncoder`` subclass (e.g. one that overrides the - ``.default()`` method to serialize additional types), specify it with - the ``cls`` kwarg. - - """ - # cached encoder - if (not skipkeys and ensure_ascii and - check_circular and allow_nan and - cls is None and indent is None and separators is None and - encoding == 'utf-8' and default is None and not use_decimal - and not kw): - iterable = _default_encoder.iterencode(obj) - else: - if cls is None: - cls = JSONEncoder - iterable = cls(skipkeys=skipkeys, ensure_ascii=ensure_ascii, - check_circular=check_circular, allow_nan=allow_nan, indent=indent, - separators=separators, encoding=encoding, - default=default, use_decimal=use_decimal, **kw).iterencode(obj) - # could accelerate with writelines in some versions of Python, at - # a debuggability cost - for chunk in iterable: - fp.write(chunk) - - -def dumps(obj, skipkeys=False, ensure_ascii=True, check_circular=True, - allow_nan=True, cls=None, indent=None, separators=None, - encoding='utf-8', default=None, use_decimal=False, **kw): - """Serialize ``obj`` to a JSON formatted ``str``. - - If ``skipkeys`` is false then ``dict`` keys that are not basic types - (``str``, ``unicode``, ``int``, ``long``, ``float``, ``bool``, ``None``) - will be skipped instead of raising a ``TypeError``. - - If ``ensure_ascii`` is false, then the return value will be a - ``unicode`` instance subject to normal Python ``str`` to ``unicode`` - coercion rules instead of being escaped to an ASCII ``str``. - - If ``check_circular`` is false, then the circular reference check - for container types will be skipped and a circular reference will - result in an ``OverflowError`` (or worse). - - If ``allow_nan`` is false, then it will be a ``ValueError`` to - serialize out of range ``float`` values (``nan``, ``inf``, ``-inf``) in - strict compliance of the JSON specification, instead of using the - JavaScript equivalents (``NaN``, ``Infinity``, ``-Infinity``). - - If ``indent`` is a string, then JSON array elements and object members - will be pretty-printed with a newline followed by that string repeated - for each level of nesting. ``None`` (the default) selects the most compact - representation without any newlines. For backwards compatibility with - versions of simplejson earlier than 2.1.0, an integer is also accepted - and is converted to a string with that many spaces. - - If ``separators`` is an ``(item_separator, dict_separator)`` tuple - then it will be used instead of the default ``(', ', ': ')`` separators. - ``(',', ':')`` is the most compact JSON representation. - - ``encoding`` is the character encoding for str instances, default is UTF-8. - - ``default(obj)`` is a function that should return a serializable version - of obj or raise TypeError. The default simply raises TypeError. - - If *use_decimal* is true (default: ``False``) then decimal.Decimal - will be natively serialized to JSON with full precision. - - To use a custom ``JSONEncoder`` subclass (e.g. one that overrides the - ``.default()`` method to serialize additional types), specify it with - the ``cls`` kwarg. - - """ - # cached encoder - if (not skipkeys and ensure_ascii and - check_circular and allow_nan and - cls is None and indent is None and separators is None and - encoding == 'utf-8' and default is None and not use_decimal - and not kw): - return _default_encoder.encode(obj) - if cls is None: - cls = JSONEncoder - return cls( - skipkeys=skipkeys, ensure_ascii=ensure_ascii, - check_circular=check_circular, allow_nan=allow_nan, indent=indent, - separators=separators, encoding=encoding, default=default, - use_decimal=use_decimal, **kw).encode(obj) - - -_default_decoder = JSONDecoder(encoding=None, object_hook=None, - object_pairs_hook=None) - - -def load(fp, encoding=None, cls=None, object_hook=None, parse_float=None, - parse_int=None, parse_constant=None, object_pairs_hook=None, - use_decimal=False, **kw): - """Deserialize ``fp`` (a ``.read()``-supporting file-like object containing - a JSON document) to a Python object. - - *encoding* determines the encoding used to interpret any - :class:`str` objects decoded by this instance (``'utf-8'`` by - default). It has no effect when decoding :class:`unicode` objects. - - Note that currently only encodings that are a superset of ASCII work, - strings of other encodings should be passed in as :class:`unicode`. - - *object_hook*, if specified, will be called with the result of every - JSON object decoded and its return value will be used in place of the - given :class:`dict`. This can be used to provide custom - deserializations (e.g. to support JSON-RPC class hinting). - - *object_pairs_hook* is an optional function that will be called with - the result of any object literal decode with an ordered list of pairs. - The return value of *object_pairs_hook* will be used instead of the - :class:`dict`. This feature can be used to implement custom decoders - that rely on the order that the key and value pairs are decoded (for - example, :func:`collections.OrderedDict` will remember the order of - insertion). If *object_hook* is also defined, the *object_pairs_hook* - takes priority. - - *parse_float*, if specified, will be called with the string of every - JSON float to be decoded. By default, this is equivalent to - ``float(num_str)``. This can be used to use another datatype or parser - for JSON floats (e.g. :class:`decimal.Decimal`). - - *parse_int*, if specified, will be called with the string of every - JSON int to be decoded. By default, this is equivalent to - ``int(num_str)``. This can be used to use another datatype or parser - for JSON integers (e.g. :class:`float`). - - *parse_constant*, if specified, will be called with one of the - following strings: ``'-Infinity'``, ``'Infinity'``, ``'NaN'``. This - can be used to raise an exception if invalid JSON numbers are - encountered. - - If *use_decimal* is true (default: ``False``) then it implies - parse_float=decimal.Decimal for parity with ``dump``. - - To use a custom ``JSONDecoder`` subclass, specify it with the ``cls`` - kwarg. - - """ - return loads(fp.read(), - encoding=encoding, cls=cls, object_hook=object_hook, - parse_float=parse_float, parse_int=parse_int, - parse_constant=parse_constant, object_pairs_hook=object_pairs_hook, - use_decimal=use_decimal, **kw) - - -def loads(s, encoding=None, cls=None, object_hook=None, parse_float=None, - parse_int=None, parse_constant=None, object_pairs_hook=None, - use_decimal=False, **kw): - """Deserialize ``s`` (a ``str`` or ``unicode`` instance containing a JSON - document) to a Python object. - - *encoding* determines the encoding used to interpret any - :class:`str` objects decoded by this instance (``'utf-8'`` by - default). It has no effect when decoding :class:`unicode` objects. - - Note that currently only encodings that are a superset of ASCII work, - strings of other encodings should be passed in as :class:`unicode`. - - *object_hook*, if specified, will be called with the result of every - JSON object decoded and its return value will be used in place of the - given :class:`dict`. This can be used to provide custom - deserializations (e.g. to support JSON-RPC class hinting). - - *object_pairs_hook* is an optional function that will be called with - the result of any object literal decode with an ordered list of pairs. - The return value of *object_pairs_hook* will be used instead of the - :class:`dict`. This feature can be used to implement custom decoders - that rely on the order that the key and value pairs are decoded (for - example, :func:`collections.OrderedDict` will remember the order of - insertion). If *object_hook* is also defined, the *object_pairs_hook* - takes priority. - - *parse_float*, if specified, will be called with the string of every - JSON float to be decoded. By default, this is equivalent to - ``float(num_str)``. This can be used to use another datatype or parser - for JSON floats (e.g. :class:`decimal.Decimal`). - - *parse_int*, if specified, will be called with the string of every - JSON int to be decoded. By default, this is equivalent to - ``int(num_str)``. This can be used to use another datatype or parser - for JSON integers (e.g. :class:`float`). - - *parse_constant*, if specified, will be called with one of the - following strings: ``'-Infinity'``, ``'Infinity'``, ``'NaN'``. This - can be used to raise an exception if invalid JSON numbers are - encountered. - - If *use_decimal* is true (default: ``False``) then it implies - parse_float=decimal.Decimal for parity with ``dump``. - - To use a custom ``JSONDecoder`` subclass, specify it with the ``cls`` - kwarg. - - """ - if (cls is None and encoding is None and object_hook is None and - parse_int is None and parse_float is None and - parse_constant is None and object_pairs_hook is None - and not use_decimal and not kw): - return _default_decoder.decode(s) - if cls is None: - cls = JSONDecoder - if object_hook is not None: - kw['object_hook'] = object_hook - if object_pairs_hook is not None: - kw['object_pairs_hook'] = object_pairs_hook - if parse_float is not None: - kw['parse_float'] = parse_float - if parse_int is not None: - kw['parse_int'] = parse_int - if parse_constant is not None: - kw['parse_constant'] = parse_constant - if use_decimal: - if parse_float is not None: - raise TypeError("use_decimal=True implies parse_float=Decimal") - kw['parse_float'] = Decimal - return cls(encoding=encoding, **kw).decode(s) - - -def _toggle_speedups(enabled): - import simplejson.decoder as dec - import simplejson.encoder as enc - import simplejson.scanner as scan - c_make_encoder = _import_c_make_encoder() - if enabled: - dec.scanstring = dec.c_scanstring or dec.py_scanstring - enc.c_make_encoder = c_make_encoder - enc.encode_basestring_ascii = (enc.c_encode_basestring_ascii or - enc.py_encode_basestring_ascii) - scan.make_scanner = scan.c_make_scanner or scan.py_make_scanner - else: - dec.scanstring = dec.py_scanstring - enc.c_make_encoder = None - enc.encode_basestring_ascii = enc.py_encode_basestring_ascii - scan.make_scanner = scan.py_make_scanner - dec.make_scanner = scan.make_scanner - global _default_decoder - _default_decoder = JSONDecoder( - encoding=None, - object_hook=None, - object_pairs_hook=None, - ) - global _default_encoder - _default_encoder = JSONEncoder( - skipkeys=False, - ensure_ascii=True, - check_circular=True, - allow_nan=True, - indent=None, - separators=None, - encoding='utf-8', - default=None, - ) diff --git a/libs/simplejson/_speedups.c b/libs/simplejson/_speedups.c deleted file mode 100644 index e43ae7c..0000000 --- a/libs/simplejson/_speedups.c +++ /dev/null @@ -1,2602 +0,0 @@ -#include "Python.h" -#include "structmember.h" -#if PY_VERSION_HEX < 0x02070000 && !defined(PyOS_string_to_double) -#define PyOS_string_to_double json_PyOS_string_to_double -static double -json_PyOS_string_to_double(const char *s, char **endptr, PyObject *overflow_exception); -static double -json_PyOS_string_to_double(const char *s, char **endptr, PyObject *overflow_exception) { - double x; - assert(endptr == NULL); - assert(overflow_exception == NULL); - PyFPE_START_PROTECT("json_PyOS_string_to_double", return -1.0;) - x = PyOS_ascii_atof(s); - PyFPE_END_PROTECT(x) - return x; -} -#endif -#if PY_VERSION_HEX < 0x02060000 && !defined(Py_TYPE) -#define Py_TYPE(ob) (((PyObject*)(ob))->ob_type) -#endif -#if PY_VERSION_HEX < 0x02050000 && !defined(PY_SSIZE_T_MIN) -typedef int Py_ssize_t; -#define PY_SSIZE_T_MAX INT_MAX -#define PY_SSIZE_T_MIN INT_MIN -#define PyInt_FromSsize_t PyInt_FromLong -#define PyInt_AsSsize_t PyInt_AsLong -#endif -#ifndef Py_IS_FINITE -#define Py_IS_FINITE(X) (!Py_IS_INFINITY(X) && !Py_IS_NAN(X)) -#endif - -#ifdef __GNUC__ -#define UNUSED __attribute__((__unused__)) -#else -#define UNUSED -#endif - -#define DEFAULT_ENCODING "utf-8" - -#define PyScanner_Check(op) PyObject_TypeCheck(op, &PyScannerType) -#define PyScanner_CheckExact(op) (Py_TYPE(op) == &PyScannerType) -#define PyEncoder_Check(op) PyObject_TypeCheck(op, &PyEncoderType) -#define PyEncoder_CheckExact(op) (Py_TYPE(op) == &PyEncoderType) -#define Decimal_Check(op) (PyObject_TypeCheck(op, DecimalTypePtr)) - -static PyTypeObject PyScannerType; -static PyTypeObject PyEncoderType; -static PyTypeObject *DecimalTypePtr; - -typedef struct _PyScannerObject { - PyObject_HEAD - PyObject *encoding; - PyObject *strict; - PyObject *object_hook; - PyObject *pairs_hook; - PyObject *parse_float; - PyObject *parse_int; - PyObject *parse_constant; - PyObject *memo; -} PyScannerObject; - -static PyMemberDef scanner_members[] = { - {"encoding", T_OBJECT, offsetof(PyScannerObject, encoding), READONLY, "encoding"}, - {"strict", T_OBJECT, offsetof(PyScannerObject, strict), READONLY, "strict"}, - {"object_hook", T_OBJECT, offsetof(PyScannerObject, object_hook), READONLY, "object_hook"}, - {"object_pairs_hook", T_OBJECT, offsetof(PyScannerObject, pairs_hook), READONLY, "object_pairs_hook"}, - {"parse_float", T_OBJECT, offsetof(PyScannerObject, parse_float), READONLY, "parse_float"}, - {"parse_int", T_OBJECT, offsetof(PyScannerObject, parse_int), READONLY, "parse_int"}, - {"parse_constant", T_OBJECT, offsetof(PyScannerObject, parse_constant), READONLY, "parse_constant"}, - {NULL} -}; - -typedef struct _PyEncoderObject { - PyObject_HEAD - PyObject *markers; - PyObject *defaultfn; - PyObject *encoder; - PyObject *indent; - PyObject *key_separator; - PyObject *item_separator; - PyObject *sort_keys; - PyObject *skipkeys; - PyObject *key_memo; - int fast_encode; - int allow_nan; - int use_decimal; -} PyEncoderObject; - -static PyMemberDef encoder_members[] = { - {"markers", T_OBJECT, offsetof(PyEncoderObject, markers), READONLY, "markers"}, - {"default", T_OBJECT, offsetof(PyEncoderObject, defaultfn), READONLY, "default"}, - {"encoder", T_OBJECT, offsetof(PyEncoderObject, encoder), READONLY, "encoder"}, - {"indent", T_OBJECT, offsetof(PyEncoderObject, indent), READONLY, "indent"}, - {"key_separator", T_OBJECT, offsetof(PyEncoderObject, key_separator), READONLY, "key_separator"}, - {"item_separator", T_OBJECT, offsetof(PyEncoderObject, item_separator), READONLY, "item_separator"}, - {"sort_keys", T_OBJECT, offsetof(PyEncoderObject, sort_keys), READONLY, "sort_keys"}, - {"skipkeys", T_OBJECT, offsetof(PyEncoderObject, skipkeys), READONLY, "skipkeys"}, - {"key_memo", T_OBJECT, offsetof(PyEncoderObject, key_memo), READONLY, "key_memo"}, - {NULL} -}; - -static Py_ssize_t -ascii_escape_char(Py_UNICODE c, char *output, Py_ssize_t chars); -static PyObject * -ascii_escape_unicode(PyObject *pystr); -static PyObject * -ascii_escape_str(PyObject *pystr); -static PyObject * -py_encode_basestring_ascii(PyObject* self UNUSED, PyObject *pystr); -void init_speedups(void); -static PyObject * -scan_once_str(PyScannerObject *s, PyObject *pystr, Py_ssize_t idx, Py_ssize_t *next_idx_ptr); -static PyObject * -scan_once_unicode(PyScannerObject *s, PyObject *pystr, Py_ssize_t idx, Py_ssize_t *next_idx_ptr); -static PyObject * -_build_rval_index_tuple(PyObject *rval, Py_ssize_t idx); -static PyObject * -scanner_new(PyTypeObject *type, PyObject *args, PyObject *kwds); -static int -scanner_init(PyObject *self, PyObject *args, PyObject *kwds); -static void -scanner_dealloc(PyObject *self); -static int -scanner_clear(PyObject *self); -static PyObject * -encoder_new(PyTypeObject *type, PyObject *args, PyObject *kwds); -static int -encoder_init(PyObject *self, PyObject *args, PyObject *kwds); -static void -encoder_dealloc(PyObject *self); -static int -encoder_clear(PyObject *self); -static int -encoder_listencode_list(PyEncoderObject *s, PyObject *rval, PyObject *seq, Py_ssize_t indent_level); -static int -encoder_listencode_obj(PyEncoderObject *s, PyObject *rval, PyObject *obj, Py_ssize_t indent_level); -static int -encoder_listencode_dict(PyEncoderObject *s, PyObject *rval, PyObject *dct, Py_ssize_t indent_level); -static PyObject * -_encoded_const(PyObject *obj); -static void -raise_errmsg(char *msg, PyObject *s, Py_ssize_t end); -static PyObject * -encoder_encode_string(PyEncoderObject *s, PyObject *obj); -static int -_convertPyInt_AsSsize_t(PyObject *o, Py_ssize_t *size_ptr); -static PyObject * -_convertPyInt_FromSsize_t(Py_ssize_t *size_ptr); -static PyObject * -encoder_encode_float(PyEncoderObject *s, PyObject *obj); - -#define S_CHAR(c) (c >= ' ' && c <= '~' && c != '\\' && c != '"') -#define IS_WHITESPACE(c) (((c) == ' ') || ((c) == '\t') || ((c) == '\n') || ((c) == '\r')) - -#define MIN_EXPANSION 6 -#ifdef Py_UNICODE_WIDE -#define MAX_EXPANSION (2 * MIN_EXPANSION) -#else -#define MAX_EXPANSION MIN_EXPANSION -#endif - -static int -_convertPyInt_AsSsize_t(PyObject *o, Py_ssize_t *size_ptr) -{ - /* PyObject to Py_ssize_t converter */ - *size_ptr = PyInt_AsSsize_t(o); - if (*size_ptr == -1 && PyErr_Occurred()) - return 0; - return 1; -} - -static PyObject * -_convertPyInt_FromSsize_t(Py_ssize_t *size_ptr) -{ - /* Py_ssize_t to PyObject converter */ - return PyInt_FromSsize_t(*size_ptr); -} - -static Py_ssize_t -ascii_escape_char(Py_UNICODE c, char *output, Py_ssize_t chars) -{ - /* Escape unicode code point c to ASCII escape sequences - in char *output. output must have at least 12 bytes unused to - accommodate an escaped surrogate pair "\uXXXX\uXXXX" */ - output[chars++] = '\\'; - switch (c) { - case '\\': output[chars++] = (char)c; break; - case '"': output[chars++] = (char)c; break; - case '\b': output[chars++] = 'b'; break; - case '\f': output[chars++] = 'f'; break; - case '\n': output[chars++] = 'n'; break; - case '\r': output[chars++] = 'r'; break; - case '\t': output[chars++] = 't'; break; - default: -#ifdef Py_UNICODE_WIDE - if (c >= 0x10000) { - /* UTF-16 surrogate pair */ - Py_UNICODE v = c - 0x10000; - c = 0xd800 | ((v >> 10) & 0x3ff); - output[chars++] = 'u'; - output[chars++] = "0123456789abcdef"[(c >> 12) & 0xf]; - output[chars++] = "0123456789abcdef"[(c >> 8) & 0xf]; - output[chars++] = "0123456789abcdef"[(c >> 4) & 0xf]; - output[chars++] = "0123456789abcdef"[(c ) & 0xf]; - c = 0xdc00 | (v & 0x3ff); - output[chars++] = '\\'; - } -#endif - output[chars++] = 'u'; - output[chars++] = "0123456789abcdef"[(c >> 12) & 0xf]; - output[chars++] = "0123456789abcdef"[(c >> 8) & 0xf]; - output[chars++] = "0123456789abcdef"[(c >> 4) & 0xf]; - output[chars++] = "0123456789abcdef"[(c ) & 0xf]; - } - return chars; -} - -static PyObject * -ascii_escape_unicode(PyObject *pystr) -{ - /* Take a PyUnicode pystr and return a new ASCII-only escaped PyString */ - Py_ssize_t i; - Py_ssize_t input_chars; - Py_ssize_t output_size; - Py_ssize_t max_output_size; - Py_ssize_t chars; - PyObject *rval; - char *output; - Py_UNICODE *input_unicode; - - input_chars = PyUnicode_GET_SIZE(pystr); - input_unicode = PyUnicode_AS_UNICODE(pystr); - - /* One char input can be up to 6 chars output, estimate 4 of these */ - output_size = 2 + (MIN_EXPANSION * 4) + input_chars; - max_output_size = 2 + (input_chars * MAX_EXPANSION); - rval = PyString_FromStringAndSize(NULL, output_size); - if (rval == NULL) { - return NULL; - } - output = PyString_AS_STRING(rval); - chars = 0; - output[chars++] = '"'; - for (i = 0; i < input_chars; i++) { - Py_UNICODE c = input_unicode[i]; - if (S_CHAR(c)) { - output[chars++] = (char)c; - } - else { - chars = ascii_escape_char(c, output, chars); - } - if (output_size - chars < (1 + MAX_EXPANSION)) { - /* There's more than four, so let's resize by a lot */ - Py_ssize_t new_output_size = output_size * 2; - /* This is an upper bound */ - if (new_output_size > max_output_size) { - new_output_size = max_output_size; - } - /* Make sure that the output size changed before resizing */ - if (new_output_size != output_size) { - output_size = new_output_size; - if (_PyString_Resize(&rval, output_size) == -1) { - return NULL; - } - output = PyString_AS_STRING(rval); - } - } - } - output[chars++] = '"'; - if (_PyString_Resize(&rval, chars) == -1) { - return NULL; - } - return rval; -} - -static PyObject * -ascii_escape_str(PyObject *pystr) -{ - /* Take a PyString pystr and return a new ASCII-only escaped PyString */ - Py_ssize_t i; - Py_ssize_t input_chars; - Py_ssize_t output_size; - Py_ssize_t chars; - PyObject *rval; - char *output; - char *input_str; - - input_chars = PyString_GET_SIZE(pystr); - input_str = PyString_AS_STRING(pystr); - - /* Fast path for a string that's already ASCII */ - for (i = 0; i < input_chars; i++) { - Py_UNICODE c = (Py_UNICODE)(unsigned char)input_str[i]; - if (!S_CHAR(c)) { - /* If we have to escape something, scan the string for unicode */ - Py_ssize_t j; - for (j = i; j < input_chars; j++) { - c = (Py_UNICODE)(unsigned char)input_str[j]; - if (c > 0x7f) { - /* We hit a non-ASCII character, bail to unicode mode */ - PyObject *uni; - uni = PyUnicode_DecodeUTF8(input_str, input_chars, "strict"); - if (uni == NULL) { - return NULL; - } - rval = ascii_escape_unicode(uni); - Py_DECREF(uni); - return rval; - } - } - break; - } - } - - if (i == input_chars) { - /* Input is already ASCII */ - output_size = 2 + input_chars; - } - else { - /* One char input can be up to 6 chars output, estimate 4 of these */ - output_size = 2 + (MIN_EXPANSION * 4) + input_chars; - } - rval = PyString_FromStringAndSize(NULL, output_size); - if (rval == NULL) { - return NULL; - } - output = PyString_AS_STRING(rval); - output[0] = '"'; - - /* We know that everything up to i is ASCII already */ - chars = i + 1; - memcpy(&output[1], input_str, i); - - for (; i < input_chars; i++) { - Py_UNICODE c = (Py_UNICODE)(unsigned char)input_str[i]; - if (S_CHAR(c)) { - output[chars++] = (char)c; - } - else { - chars = ascii_escape_char(c, output, chars); - } - /* An ASCII char can't possibly expand to a surrogate! */ - if (output_size - chars < (1 + MIN_EXPANSION)) { - /* There's more than four, so let's resize by a lot */ - output_size *= 2; - if (output_size > 2 + (input_chars * MIN_EXPANSION)) { - output_size = 2 + (input_chars * MIN_EXPANSION); - } - if (_PyString_Resize(&rval, output_size) == -1) { - return NULL; - } - output = PyString_AS_STRING(rval); - } - } - output[chars++] = '"'; - if (_PyString_Resize(&rval, chars) == -1) { - return NULL; - } - return rval; -} - -static void -raise_errmsg(char *msg, PyObject *s, Py_ssize_t end) -{ - /* Use the Python function simplejson.decoder.errmsg to raise a nice - looking ValueError exception */ - static PyObject *JSONDecodeError = NULL; - PyObject *exc; - if (JSONDecodeError == NULL) { - PyObject *decoder = PyImport_ImportModule("simplejson.decoder"); - if (decoder == NULL) - return; - JSONDecodeError = PyObject_GetAttrString(decoder, "JSONDecodeError"); - Py_DECREF(decoder); - if (JSONDecodeError == NULL) - return; - } - exc = PyObject_CallFunction(JSONDecodeError, "(zOO&)", msg, s, _convertPyInt_FromSsize_t, &end); - if (exc) { - PyErr_SetObject(JSONDecodeError, exc); - Py_DECREF(exc); - } -} - -static PyObject * -join_list_unicode(PyObject *lst) -{ - /* return u''.join(lst) */ - static PyObject *joinfn = NULL; - if (joinfn == NULL) { - PyObject *ustr = PyUnicode_FromUnicode(NULL, 0); - if (ustr == NULL) - return NULL; - - joinfn = PyObject_GetAttrString(ustr, "join"); - Py_DECREF(ustr); - if (joinfn == NULL) - return NULL; - } - return PyObject_CallFunctionObjArgs(joinfn, lst, NULL); -} - -static PyObject * -join_list_string(PyObject *lst) -{ - /* return ''.join(lst) */ - static PyObject *joinfn = NULL; - if (joinfn == NULL) { - PyObject *ustr = PyString_FromStringAndSize(NULL, 0); - if (ustr == NULL) - return NULL; - - joinfn = PyObject_GetAttrString(ustr, "join"); - Py_DECREF(ustr); - if (joinfn == NULL) - return NULL; - } - return PyObject_CallFunctionObjArgs(joinfn, lst, NULL); -} - -static PyObject * -_build_rval_index_tuple(PyObject *rval, Py_ssize_t idx) { - /* return (rval, idx) tuple, stealing reference to rval */ - PyObject *tpl; - PyObject *pyidx; - /* - steal a reference to rval, returns (rval, idx) - */ - if (rval == NULL) { - return NULL; - } - pyidx = PyInt_FromSsize_t(idx); - if (pyidx == NULL) { - Py_DECREF(rval); - return NULL; - } - tpl = PyTuple_New(2); - if (tpl == NULL) { - Py_DECREF(pyidx); - Py_DECREF(rval); - return NULL; - } - PyTuple_SET_ITEM(tpl, 0, rval); - PyTuple_SET_ITEM(tpl, 1, pyidx); - return tpl; -} - -#define APPEND_OLD_CHUNK \ - if (chunk != NULL) { \ - if (chunks == NULL) { \ - chunks = PyList_New(0); \ - if (chunks == NULL) { \ - goto bail; \ - } \ - } \ - if (PyList_Append(chunks, chunk)) { \ - goto bail; \ - } \ - Py_CLEAR(chunk); \ - } - -static PyObject * -scanstring_str(PyObject *pystr, Py_ssize_t end, char *encoding, int strict, Py_ssize_t *next_end_ptr) -{ - /* Read the JSON string from PyString pystr. - end is the index of the first character after the quote. - encoding is the encoding of pystr (must be an ASCII superset) - if strict is zero then literal control characters are allowed - *next_end_ptr is a return-by-reference index of the character - after the end quote - - Return value is a new PyString (if ASCII-only) or PyUnicode - */ - PyObject *rval; - Py_ssize_t len = PyString_GET_SIZE(pystr); - Py_ssize_t begin = end - 1; - Py_ssize_t next = begin; - int has_unicode = 0; - char *buf = PyString_AS_STRING(pystr); - PyObject *chunks = NULL; - PyObject *chunk = NULL; - - if (end < 0 || len <= end) { - PyErr_SetString(PyExc_ValueError, "end is out of bounds"); - goto bail; - } - while (1) { - /* Find the end of the string or the next escape */ - Py_UNICODE c = 0; - for (next = end; next < len; next++) { - c = (unsigned char)buf[next]; - if (c == '"' || c == '\\') { - break; - } - else if (strict && c <= 0x1f) { - raise_errmsg("Invalid control character at", pystr, next); - goto bail; - } - else if (c > 0x7f) { - has_unicode = 1; - } - } - if (!(c == '"' || c == '\\')) { - raise_errmsg("Unterminated string starting at", pystr, begin); - goto bail; - } - /* Pick up this chunk if it's not zero length */ - if (next != end) { - PyObject *strchunk; - APPEND_OLD_CHUNK - strchunk = PyString_FromStringAndSize(&buf[end], next - end); - if (strchunk == NULL) { - goto bail; - } - if (has_unicode) { - chunk = PyUnicode_FromEncodedObject(strchunk, encoding, NULL); - Py_DECREF(strchunk); - if (chunk == NULL) { - goto bail; - } - } - else { - chunk = strchunk; - } - } - next++; - if (c == '"') { - end = next; - break; - } - if (next == len) { - raise_errmsg("Unterminated string starting at", pystr, begin); - goto bail; - } - c = buf[next]; - if (c != 'u') { - /* Non-unicode backslash escapes */ - end = next + 1; - switch (c) { - case '"': break; - case '\\': break; - case '/': break; - case 'b': c = '\b'; break; - case 'f': c = '\f'; break; - case 'n': c = '\n'; break; - case 'r': c = '\r'; break; - case 't': c = '\t'; break; - default: c = 0; - } - if (c == 0) { - raise_errmsg("Invalid \\escape", pystr, end - 2); - goto bail; - } - } - else { - c = 0; - next++; - end = next + 4; - if (end >= len) { - raise_errmsg("Invalid \\uXXXX escape", pystr, next - 1); - goto bail; - } - /* Decode 4 hex digits */ - for (; next < end; next++) { - Py_UNICODE digit = buf[next]; - c <<= 4; - switch (digit) { - case '0': case '1': case '2': case '3': case '4': - case '5': case '6': case '7': case '8': case '9': - c |= (digit - '0'); break; - case 'a': case 'b': case 'c': case 'd': case 'e': - case 'f': - c |= (digit - 'a' + 10); break; - case 'A': case 'B': case 'C': case 'D': case 'E': - case 'F': - c |= (digit - 'A' + 10); break; - default: - raise_errmsg("Invalid \\uXXXX escape", pystr, end - 5); - goto bail; - } - } -#ifdef Py_UNICODE_WIDE - /* Surrogate pair */ - if ((c & 0xfc00) == 0xd800) { - Py_UNICODE c2 = 0; - if (end + 6 >= len) { - raise_errmsg("Unpaired high surrogate", pystr, end - 5); - goto bail; - } - if (buf[next++] != '\\' || buf[next++] != 'u') { - raise_errmsg("Unpaired high surrogate", pystr, end - 5); - goto bail; - } - end += 6; - /* Decode 4 hex digits */ - for (; next < end; next++) { - c2 <<= 4; - Py_UNICODE digit = buf[next]; - switch (digit) { - case '0': case '1': case '2': case '3': case '4': - case '5': case '6': case '7': case '8': case '9': - c2 |= (digit - '0'); break; - case 'a': case 'b': case 'c': case 'd': case 'e': - case 'f': - c2 |= (digit - 'a' + 10); break; - case 'A': case 'B': case 'C': case 'D': case 'E': - case 'F': - c2 |= (digit - 'A' + 10); break; - default: - raise_errmsg("Invalid \\uXXXX escape", pystr, end - 5); - goto bail; - } - } - if ((c2 & 0xfc00) != 0xdc00) { - raise_errmsg("Unpaired high surrogate", pystr, end - 5); - goto bail; - } - c = 0x10000 + (((c - 0xd800) << 10) | (c2 - 0xdc00)); - } - else if ((c & 0xfc00) == 0xdc00) { - raise_errmsg("Unpaired low surrogate", pystr, end - 5); - goto bail; - } -#endif - } - if (c > 0x7f) { - has_unicode = 1; - } - APPEND_OLD_CHUNK - if (has_unicode) { - chunk = PyUnicode_FromUnicode(&c, 1); - if (chunk == NULL) { - goto bail; - } - } - else { - char c_char = Py_CHARMASK(c); - chunk = PyString_FromStringAndSize(&c_char, 1); - if (chunk == NULL) { - goto bail; - } - } - } - - if (chunks == NULL) { - if (chunk != NULL) - rval = chunk; - else - rval = PyString_FromStringAndSize("", 0); - } - else { - APPEND_OLD_CHUNK - rval = join_list_string(chunks); - if (rval == NULL) { - goto bail; - } - Py_CLEAR(chunks); - } - - *next_end_ptr = end; - return rval; -bail: - *next_end_ptr = -1; - Py_XDECREF(chunk); - Py_XDECREF(chunks); - return NULL; -} - - -static PyObject * -scanstring_unicode(PyObject *pystr, Py_ssize_t end, int strict, Py_ssize_t *next_end_ptr) -{ - /* Read the JSON string from PyUnicode pystr. - end is the index of the first character after the quote. - if strict is zero then literal control characters are allowed - *next_end_ptr is a return-by-reference index of the character - after the end quote - - Return value is a new PyUnicode - */ - PyObject *rval; - Py_ssize_t len = PyUnicode_GET_SIZE(pystr); - Py_ssize_t begin = end - 1; - Py_ssize_t next = begin; - const Py_UNICODE *buf = PyUnicode_AS_UNICODE(pystr); - PyObject *chunks = NULL; - PyObject *chunk = NULL; - - if (end < 0 || len <= end) { - PyErr_SetString(PyExc_ValueError, "end is out of bounds"); - goto bail; - } - while (1) { - /* Find the end of the string or the next escape */ - Py_UNICODE c = 0; - for (next = end; next < len; next++) { - c = buf[next]; - if (c == '"' || c == '\\') { - break; - } - else if (strict && c <= 0x1f) { - raise_errmsg("Invalid control character at", pystr, next); - goto bail; - } - } - if (!(c == '"' || c == '\\')) { - raise_errmsg("Unterminated string starting at", pystr, begin); - goto bail; - } - /* Pick up this chunk if it's not zero length */ - if (next != end) { - APPEND_OLD_CHUNK - chunk = PyUnicode_FromUnicode(&buf[end], next - end); - if (chunk == NULL) { - goto bail; - } - } - next++; - if (c == '"') { - end = next; - break; - } - if (next == len) { - raise_errmsg("Unterminated string starting at", pystr, begin); - goto bail; - } - c = buf[next]; - if (c != 'u') { - /* Non-unicode backslash escapes */ - end = next + 1; - switch (c) { - case '"': break; - case '\\': break; - case '/': break; - case 'b': c = '\b'; break; - case 'f': c = '\f'; break; - case 'n': c = '\n'; break; - case 'r': c = '\r'; break; - case 't': c = '\t'; break; - default: c = 0; - } - if (c == 0) { - raise_errmsg("Invalid \\escape", pystr, end - 2); - goto bail; - } - } - else { - c = 0; - next++; - end = next + 4; - if (end >= len) { - raise_errmsg("Invalid \\uXXXX escape", pystr, next - 1); - goto bail; - } - /* Decode 4 hex digits */ - for (; next < end; next++) { - Py_UNICODE digit = buf[next]; - c <<= 4; - switch (digit) { - case '0': case '1': case '2': case '3': case '4': - case '5': case '6': case '7': case '8': case '9': - c |= (digit - '0'); break; - case 'a': case 'b': case 'c': case 'd': case 'e': - case 'f': - c |= (digit - 'a' + 10); break; - case 'A': case 'B': case 'C': case 'D': case 'E': - case 'F': - c |= (digit - 'A' + 10); break; - default: - raise_errmsg("Invalid \\uXXXX escape", pystr, end - 5); - goto bail; - } - } -#ifdef Py_UNICODE_WIDE - /* Surrogate pair */ - if ((c & 0xfc00) == 0xd800) { - Py_UNICODE c2 = 0; - if (end + 6 >= len) { - raise_errmsg("Unpaired high surrogate", pystr, end - 5); - goto bail; - } - if (buf[next++] != '\\' || buf[next++] != 'u') { - raise_errmsg("Unpaired high surrogate", pystr, end - 5); - goto bail; - } - end += 6; - /* Decode 4 hex digits */ - for (; next < end; next++) { - c2 <<= 4; - Py_UNICODE digit = buf[next]; - switch (digit) { - case '0': case '1': case '2': case '3': case '4': - case '5': case '6': case '7': case '8': case '9': - c2 |= (digit - '0'); break; - case 'a': case 'b': case 'c': case 'd': case 'e': - case 'f': - c2 |= (digit - 'a' + 10); break; - case 'A': case 'B': case 'C': case 'D': case 'E': - case 'F': - c2 |= (digit - 'A' + 10); break; - default: - raise_errmsg("Invalid \\uXXXX escape", pystr, end - 5); - goto bail; - } - } - if ((c2 & 0xfc00) != 0xdc00) { - raise_errmsg("Unpaired high surrogate", pystr, end - 5); - goto bail; - } - c = 0x10000 + (((c - 0xd800) << 10) | (c2 - 0xdc00)); - } - else if ((c & 0xfc00) == 0xdc00) { - raise_errmsg("Unpaired low surrogate", pystr, end - 5); - goto bail; - } -#endif - } - APPEND_OLD_CHUNK - chunk = PyUnicode_FromUnicode(&c, 1); - if (chunk == NULL) { - goto bail; - } - } - - if (chunks == NULL) { - if (chunk != NULL) - rval = chunk; - else - rval = PyUnicode_FromUnicode(NULL, 0); - } - else { - APPEND_OLD_CHUNK - rval = join_list_unicode(chunks); - if (rval == NULL) { - goto bail; - } - Py_CLEAR(chunks); - } - *next_end_ptr = end; - return rval; -bail: - *next_end_ptr = -1; - Py_XDECREF(chunk); - Py_XDECREF(chunks); - return NULL; -} - -PyDoc_STRVAR(pydoc_scanstring, - "scanstring(basestring, end, encoding, strict=True) -> (str, end)\n" - "\n" - "Scan the string s for a JSON string. End is the index of the\n" - "character in s after the quote that started the JSON string.\n" - "Unescapes all valid JSON string escape sequences and raises ValueError\n" - "on attempt to decode an invalid string. If strict is False then literal\n" - "control characters are allowed in the string.\n" - "\n" - "Returns a tuple of the decoded string and the index of the character in s\n" - "after the end quote." -); - -static PyObject * -py_scanstring(PyObject* self UNUSED, PyObject *args) -{ - PyObject *pystr; - PyObject *rval; - Py_ssize_t end; - Py_ssize_t next_end = -1; - char *encoding = NULL; - int strict = 1; - if (!PyArg_ParseTuple(args, "OO&|zi:scanstring", &pystr, _convertPyInt_AsSsize_t, &end, &encoding, &strict)) { - return NULL; - } - if (encoding == NULL) { - encoding = DEFAULT_ENCODING; - } - if (PyString_Check(pystr)) { - rval = scanstring_str(pystr, end, encoding, strict, &next_end); - } - else if (PyUnicode_Check(pystr)) { - rval = scanstring_unicode(pystr, end, strict, &next_end); - } - else { - PyErr_Format(PyExc_TypeError, - "first argument must be a string, not %.80s", - Py_TYPE(pystr)->tp_name); - return NULL; - } - return _build_rval_index_tuple(rval, next_end); -} - -PyDoc_STRVAR(pydoc_encode_basestring_ascii, - "encode_basestring_ascii(basestring) -> str\n" - "\n" - "Return an ASCII-only JSON representation of a Python string" -); - -static PyObject * -py_encode_basestring_ascii(PyObject* self UNUSED, PyObject *pystr) -{ - /* Return an ASCII-only JSON representation of a Python string */ - /* METH_O */ - if (PyString_Check(pystr)) { - return ascii_escape_str(pystr); - } - else if (PyUnicode_Check(pystr)) { - return ascii_escape_unicode(pystr); - } - else { - PyErr_Format(PyExc_TypeError, - "first argument must be a string, not %.80s", - Py_TYPE(pystr)->tp_name); - return NULL; - } -} - -static void -scanner_dealloc(PyObject *self) -{ - /* Deallocate scanner object */ - scanner_clear(self); - Py_TYPE(self)->tp_free(self); -} - -static int -scanner_traverse(PyObject *self, visitproc visit, void *arg) -{ - PyScannerObject *s; - assert(PyScanner_Check(self)); - s = (PyScannerObject *)self; - Py_VISIT(s->encoding); - Py_VISIT(s->strict); - Py_VISIT(s->object_hook); - Py_VISIT(s->pairs_hook); - Py_VISIT(s->parse_float); - Py_VISIT(s->parse_int); - Py_VISIT(s->parse_constant); - Py_VISIT(s->memo); - return 0; -} - -static int -scanner_clear(PyObject *self) -{ - PyScannerObject *s; - assert(PyScanner_Check(self)); - s = (PyScannerObject *)self; - Py_CLEAR(s->encoding); - Py_CLEAR(s->strict); - Py_CLEAR(s->object_hook); - Py_CLEAR(s->pairs_hook); - Py_CLEAR(s->parse_float); - Py_CLEAR(s->parse_int); - Py_CLEAR(s->parse_constant); - Py_CLEAR(s->memo); - return 0; -} - -static PyObject * -_parse_object_str(PyScannerObject *s, PyObject *pystr, Py_ssize_t idx, Py_ssize_t *next_idx_ptr) { - /* Read a JSON object from PyString pystr. - idx is the index of the first character after the opening curly brace. - *next_idx_ptr is a return-by-reference index to the first character after - the closing curly brace. - - Returns a new PyObject (usually a dict, but object_hook or - object_pairs_hook can change that) - */ - char *str = PyString_AS_STRING(pystr); - Py_ssize_t end_idx = PyString_GET_SIZE(pystr) - 1; - PyObject *rval = NULL; - PyObject *pairs = NULL; - PyObject *item; - PyObject *key = NULL; - PyObject *val = NULL; - char *encoding = PyString_AS_STRING(s->encoding); - int strict = PyObject_IsTrue(s->strict); - int has_pairs_hook = (s->pairs_hook != Py_None); - Py_ssize_t next_idx; - if (has_pairs_hook) { - pairs = PyList_New(0); - if (pairs == NULL) - return NULL; - } - else { - rval = PyDict_New(); - if (rval == NULL) - return NULL; - } - - /* skip whitespace after { */ - while (idx <= end_idx && IS_WHITESPACE(str[idx])) idx++; - - /* only loop if the object is non-empty */ - if (idx <= end_idx && str[idx] != '}') { - while (idx <= end_idx) { - PyObject *memokey; - - /* read key */ - if (str[idx] != '"') { - raise_errmsg("Expecting property name", pystr, idx); - goto bail; - } - key = scanstring_str(pystr, idx + 1, encoding, strict, &next_idx); - if (key == NULL) - goto bail; - memokey = PyDict_GetItem(s->memo, key); - if (memokey != NULL) { - Py_INCREF(memokey); - Py_DECREF(key); - key = memokey; - } - else { - if (PyDict_SetItem(s->memo, key, key) < 0) - goto bail; - } - idx = next_idx; - - /* skip whitespace between key and : delimiter, read :, skip whitespace */ - while (idx <= end_idx && IS_WHITESPACE(str[idx])) idx++; - if (idx > end_idx || str[idx] != ':') { - raise_errmsg("Expecting : delimiter", pystr, idx); - goto bail; - } - idx++; - while (idx <= end_idx && IS_WHITESPACE(str[idx])) idx++; - - /* read any JSON data type */ - val = scan_once_str(s, pystr, idx, &next_idx); - if (val == NULL) - goto bail; - - if (has_pairs_hook) { - item = PyTuple_Pack(2, key, val); - if (item == NULL) - goto bail; - Py_CLEAR(key); - Py_CLEAR(val); - if (PyList_Append(pairs, item) == -1) { - Py_DECREF(item); - goto bail; - } - Py_DECREF(item); - } - else { - if (PyDict_SetItem(rval, key, val) < 0) - goto bail; - Py_CLEAR(key); - Py_CLEAR(val); - } - idx = next_idx; - - /* skip whitespace before } or , */ - while (idx <= end_idx && IS_WHITESPACE(str[idx])) idx++; - - /* bail if the object is closed or we didn't get the , delimiter */ - if (idx > end_idx) break; - if (str[idx] == '}') { - break; - } - else if (str[idx] != ',') { - raise_errmsg("Expecting , delimiter", pystr, idx); - goto bail; - } - idx++; - - /* skip whitespace after , delimiter */ - while (idx <= end_idx && IS_WHITESPACE(str[idx])) idx++; - } - } - /* verify that idx < end_idx, str[idx] should be '}' */ - if (idx > end_idx || str[idx] != '}') { - raise_errmsg("Expecting object", pystr, end_idx); - goto bail; - } - - /* if pairs_hook is not None: rval = object_pairs_hook(pairs) */ - if (s->pairs_hook != Py_None) { - val = PyObject_CallFunctionObjArgs(s->pairs_hook, pairs, NULL); - if (val == NULL) - goto bail; - Py_DECREF(pairs); - *next_idx_ptr = idx + 1; - return val; - } - - /* if object_hook is not None: rval = object_hook(rval) */ - if (s->object_hook != Py_None) { - val = PyObject_CallFunctionObjArgs(s->object_hook, rval, NULL); - if (val == NULL) - goto bail; - Py_DECREF(rval); - rval = val; - val = NULL; - } - *next_idx_ptr = idx + 1; - return rval; -bail: - Py_XDECREF(rval); - Py_XDECREF(key); - Py_XDECREF(val); - Py_XDECREF(pairs); - return NULL; -} - -static PyObject * -_parse_object_unicode(PyScannerObject *s, PyObject *pystr, Py_ssize_t idx, Py_ssize_t *next_idx_ptr) { - /* Read a JSON object from PyUnicode pystr. - idx is the index of the first character after the opening curly brace. - *next_idx_ptr is a return-by-reference index to the first character after - the closing curly brace. - - Returns a new PyObject (usually a dict, but object_hook can change that) - */ - Py_UNICODE *str = PyUnicode_AS_UNICODE(pystr); - Py_ssize_t end_idx = PyUnicode_GET_SIZE(pystr) - 1; - PyObject *rval = NULL; - PyObject *pairs = NULL; - PyObject *item; - PyObject *key = NULL; - PyObject *val = NULL; - int strict = PyObject_IsTrue(s->strict); - int has_pairs_hook = (s->pairs_hook != Py_None); - Py_ssize_t next_idx; - - if (has_pairs_hook) { - pairs = PyList_New(0); - if (pairs == NULL) - return NULL; - } - else { - rval = PyDict_New(); - if (rval == NULL) - return NULL; - } - - /* skip whitespace after { */ - while (idx <= end_idx && IS_WHITESPACE(str[idx])) idx++; - - /* only loop if the object is non-empty */ - if (idx <= end_idx && str[idx] != '}') { - while (idx <= end_idx) { - PyObject *memokey; - - /* read key */ - if (str[idx] != '"') { - raise_errmsg("Expecting property name", pystr, idx); - goto bail; - } - key = scanstring_unicode(pystr, idx + 1, strict, &next_idx); - if (key == NULL) - goto bail; - memokey = PyDict_GetItem(s->memo, key); - if (memokey != NULL) { - Py_INCREF(memokey); - Py_DECREF(key); - key = memokey; - } - else { - if (PyDict_SetItem(s->memo, key, key) < 0) - goto bail; - } - idx = next_idx; - - /* skip whitespace between key and : delimiter, read :, skip whitespace */ - while (idx <= end_idx && IS_WHITESPACE(str[idx])) idx++; - if (idx > end_idx || str[idx] != ':') { - raise_errmsg("Expecting : delimiter", pystr, idx); - goto bail; - } - idx++; - while (idx <= end_idx && IS_WHITESPACE(str[idx])) idx++; - - /* read any JSON term */ - val = scan_once_unicode(s, pystr, idx, &next_idx); - if (val == NULL) - goto bail; - - if (has_pairs_hook) { - item = PyTuple_Pack(2, key, val); - if (item == NULL) - goto bail; - Py_CLEAR(key); - Py_CLEAR(val); - if (PyList_Append(pairs, item) == -1) { - Py_DECREF(item); - goto bail; - } - Py_DECREF(item); - } - else { - if (PyDict_SetItem(rval, key, val) < 0) - goto bail; - Py_CLEAR(key); - Py_CLEAR(val); - } - idx = next_idx; - - /* skip whitespace before } or , */ - while (idx <= end_idx && IS_WHITESPACE(str[idx])) idx++; - - /* bail if the object is closed or we didn't get the , delimiter */ - if (idx > end_idx) break; - if (str[idx] == '}') { - break; - } - else if (str[idx] != ',') { - raise_errmsg("Expecting , delimiter", pystr, idx); - goto bail; - } - idx++; - - /* skip whitespace after , delimiter */ - while (idx <= end_idx && IS_WHITESPACE(str[idx])) idx++; - } - } - - /* verify that idx < end_idx, str[idx] should be '}' */ - if (idx > end_idx || str[idx] != '}') { - raise_errmsg("Expecting object", pystr, end_idx); - goto bail; - } - - /* if pairs_hook is not None: rval = object_pairs_hook(pairs) */ - if (s->pairs_hook != Py_None) { - val = PyObject_CallFunctionObjArgs(s->pairs_hook, pairs, NULL); - if (val == NULL) - goto bail; - Py_DECREF(pairs); - *next_idx_ptr = idx + 1; - return val; - } - - /* if object_hook is not None: rval = object_hook(rval) */ - if (s->object_hook != Py_None) { - val = PyObject_CallFunctionObjArgs(s->object_hook, rval, NULL); - if (val == NULL) - goto bail; - Py_DECREF(rval); - rval = val; - val = NULL; - } - *next_idx_ptr = idx + 1; - return rval; -bail: - Py_XDECREF(rval); - Py_XDECREF(key); - Py_XDECREF(val); - Py_XDECREF(pairs); - return NULL; -} - -static PyObject * -_parse_array_str(PyScannerObject *s, PyObject *pystr, Py_ssize_t idx, Py_ssize_t *next_idx_ptr) { - /* Read a JSON array from PyString pystr. - idx is the index of the first character after the opening brace. - *next_idx_ptr is a return-by-reference index to the first character after - the closing brace. - - Returns a new PyList - */ - char *str = PyString_AS_STRING(pystr); - Py_ssize_t end_idx = PyString_GET_SIZE(pystr) - 1; - PyObject *val = NULL; - PyObject *rval = PyList_New(0); - Py_ssize_t next_idx; - if (rval == NULL) - return NULL; - - /* skip whitespace after [ */ - while (idx <= end_idx && IS_WHITESPACE(str[idx])) idx++; - - /* only loop if the array is non-empty */ - if (idx <= end_idx && str[idx] != ']') { - while (idx <= end_idx) { - - /* read any JSON term and de-tuplefy the (rval, idx) */ - val = scan_once_str(s, pystr, idx, &next_idx); - if (val == NULL) { - if (PyErr_ExceptionMatches(PyExc_StopIteration)) { - PyErr_Clear(); - raise_errmsg("Expecting object", pystr, idx); - } - goto bail; - } - - if (PyList_Append(rval, val) == -1) - goto bail; - - Py_CLEAR(val); - idx = next_idx; - - /* skip whitespace between term and , */ - while (idx <= end_idx && IS_WHITESPACE(str[idx])) idx++; - - /* bail if the array is closed or we didn't get the , delimiter */ - if (idx > end_idx) break; - if (str[idx] == ']') { - break; - } - else if (str[idx] != ',') { - raise_errmsg("Expecting , delimiter", pystr, idx); - goto bail; - } - idx++; - - /* skip whitespace after , */ - while (idx <= end_idx && IS_WHITESPACE(str[idx])) idx++; - } - } - - /* verify that idx < end_idx, str[idx] should be ']' */ - if (idx > end_idx || str[idx] != ']') { - raise_errmsg("Expecting object", pystr, end_idx); - goto bail; - } - *next_idx_ptr = idx + 1; - return rval; -bail: - Py_XDECREF(val); - Py_DECREF(rval); - return NULL; -} - -static PyObject * -_parse_array_unicode(PyScannerObject *s, PyObject *pystr, Py_ssize_t idx, Py_ssize_t *next_idx_ptr) { - /* Read a JSON array from PyString pystr. - idx is the index of the first character after the opening brace. - *next_idx_ptr is a return-by-reference index to the first character after - the closing brace. - - Returns a new PyList - */ - Py_UNICODE *str = PyUnicode_AS_UNICODE(pystr); - Py_ssize_t end_idx = PyUnicode_GET_SIZE(pystr) - 1; - PyObject *val = NULL; - PyObject *rval = PyList_New(0); - Py_ssize_t next_idx; - if (rval == NULL) - return NULL; - - /* skip whitespace after [ */ - while (idx <= end_idx && IS_WHITESPACE(str[idx])) idx++; - - /* only loop if the array is non-empty */ - if (idx <= end_idx && str[idx] != ']') { - while (idx <= end_idx) { - - /* read any JSON term */ - val = scan_once_unicode(s, pystr, idx, &next_idx); - if (val == NULL) { - if (PyErr_ExceptionMatches(PyExc_StopIteration)) { - PyErr_Clear(); - raise_errmsg("Expecting object", pystr, idx); - } - goto bail; - } - - if (PyList_Append(rval, val) == -1) - goto bail; - - Py_CLEAR(val); - idx = next_idx; - - /* skip whitespace between term and , */ - while (idx <= end_idx && IS_WHITESPACE(str[idx])) idx++; - - /* bail if the array is closed or we didn't get the , delimiter */ - if (idx > end_idx) break; - if (str[idx] == ']') { - break; - } - else if (str[idx] != ',') { - raise_errmsg("Expecting , delimiter", pystr, idx); - goto bail; - } - idx++; - - /* skip whitespace after , */ - while (idx <= end_idx && IS_WHITESPACE(str[idx])) idx++; - } - } - - /* verify that idx < end_idx, str[idx] should be ']' */ - if (idx > end_idx || str[idx] != ']') { - raise_errmsg("Expecting object", pystr, end_idx); - goto bail; - } - *next_idx_ptr = idx + 1; - return rval; -bail: - Py_XDECREF(val); - Py_DECREF(rval); - return NULL; -} - -static PyObject * -_parse_constant(PyScannerObject *s, char *constant, Py_ssize_t idx, Py_ssize_t *next_idx_ptr) { - /* Read a JSON constant from PyString pystr. - constant is the constant string that was found - ("NaN", "Infinity", "-Infinity"). - idx is the index of the first character of the constant - *next_idx_ptr is a return-by-reference index to the first character after - the constant. - - Returns the result of parse_constant - */ - PyObject *cstr; - PyObject *rval; - /* constant is "NaN", "Infinity", or "-Infinity" */ - cstr = PyString_InternFromString(constant); - if (cstr == NULL) - return NULL; - - /* rval = parse_constant(constant) */ - rval = PyObject_CallFunctionObjArgs(s->parse_constant, cstr, NULL); - idx += PyString_GET_SIZE(cstr); - Py_DECREF(cstr); - *next_idx_ptr = idx; - return rval; -} - -static PyObject * -_match_number_str(PyScannerObject *s, PyObject *pystr, Py_ssize_t start, Py_ssize_t *next_idx_ptr) { - /* Read a JSON number from PyString pystr. - idx is the index of the first character of the number - *next_idx_ptr is a return-by-reference index to the first character after - the number. - - Returns a new PyObject representation of that number: - PyInt, PyLong, or PyFloat. - May return other types if parse_int or parse_float are set - */ - char *str = PyString_AS_STRING(pystr); - Py_ssize_t end_idx = PyString_GET_SIZE(pystr) - 1; - Py_ssize_t idx = start; - int is_float = 0; - PyObject *rval; - PyObject *numstr; - - /* read a sign if it's there, make sure it's not the end of the string */ - if (str[idx] == '-') { - idx++; - if (idx > end_idx) { - PyErr_SetNone(PyExc_StopIteration); - return NULL; - } - } - - /* read as many integer digits as we find as long as it doesn't start with 0 */ - if (str[idx] >= '1' && str[idx] <= '9') { - idx++; - while (idx <= end_idx && str[idx] >= '0' && str[idx] <= '9') idx++; - } - /* if it starts with 0 we only expect one integer digit */ - else if (str[idx] == '0') { - idx++; - } - /* no integer digits, error */ - else { - PyErr_SetNone(PyExc_StopIteration); - return NULL; - } - - /* if the next char is '.' followed by a digit then read all float digits */ - if (idx < end_idx && str[idx] == '.' && str[idx + 1] >= '0' && str[idx + 1] <= '9') { - is_float = 1; - idx += 2; - while (idx <= end_idx && str[idx] >= '0' && str[idx] <= '9') idx++; - } - - /* if the next char is 'e' or 'E' then maybe read the exponent (or backtrack) */ - if (idx < end_idx && (str[idx] == 'e' || str[idx] == 'E')) { - - /* save the index of the 'e' or 'E' just in case we need to backtrack */ - Py_ssize_t e_start = idx; - idx++; - - /* read an exponent sign if present */ - if (idx < end_idx && (str[idx] == '-' || str[idx] == '+')) idx++; - - /* read all digits */ - while (idx <= end_idx && str[idx] >= '0' && str[idx] <= '9') idx++; - - /* if we got a digit, then parse as float. if not, backtrack */ - if (str[idx - 1] >= '0' && str[idx - 1] <= '9') { - is_float = 1; - } - else { - idx = e_start; - } - } - - /* copy the section we determined to be a number */ - numstr = PyString_FromStringAndSize(&str[start], idx - start); - if (numstr == NULL) - return NULL; - if (is_float) { - /* parse as a float using a fast path if available, otherwise call user defined method */ - if (s->parse_float != (PyObject *)&PyFloat_Type) { - rval = PyObject_CallFunctionObjArgs(s->parse_float, numstr, NULL); - } - else { - /* rval = PyFloat_FromDouble(PyOS_ascii_atof(PyString_AS_STRING(numstr))); */ - double d = PyOS_string_to_double(PyString_AS_STRING(numstr), - NULL, NULL); - if (d == -1.0 && PyErr_Occurred()) - return NULL; - rval = PyFloat_FromDouble(d); - } - } - else { - /* parse as an int using a fast path if available, otherwise call user defined method */ - if (s->parse_int != (PyObject *)&PyInt_Type) { - rval = PyObject_CallFunctionObjArgs(s->parse_int, numstr, NULL); - } - else { - rval = PyInt_FromString(PyString_AS_STRING(numstr), NULL, 10); - } - } - Py_DECREF(numstr); - *next_idx_ptr = idx; - return rval; -} - -static PyObject * -_match_number_unicode(PyScannerObject *s, PyObject *pystr, Py_ssize_t start, Py_ssize_t *next_idx_ptr) { - /* Read a JSON number from PyUnicode pystr. - idx is the index of the first character of the number - *next_idx_ptr is a return-by-reference index to the first character after - the number. - - Returns a new PyObject representation of that number: - PyInt, PyLong, or PyFloat. - May return other types if parse_int or parse_float are set - */ - Py_UNICODE *str = PyUnicode_AS_UNICODE(pystr); - Py_ssize_t end_idx = PyUnicode_GET_SIZE(pystr) - 1; - Py_ssize_t idx = start; - int is_float = 0; - PyObject *rval; - PyObject *numstr; - - /* read a sign if it's there, make sure it's not the end of the string */ - if (str[idx] == '-') { - idx++; - if (idx > end_idx) { - PyErr_SetNone(PyExc_StopIteration); - return NULL; - } - } - - /* read as many integer digits as we find as long as it doesn't start with 0 */ - if (str[idx] >= '1' && str[idx] <= '9') { - idx++; - while (idx <= end_idx && str[idx] >= '0' && str[idx] <= '9') idx++; - } - /* if it starts with 0 we only expect one integer digit */ - else if (str[idx] == '0') { - idx++; - } - /* no integer digits, error */ - else { - PyErr_SetNone(PyExc_StopIteration); - return NULL; - } - - /* if the next char is '.' followed by a digit then read all float digits */ - if (idx < end_idx && str[idx] == '.' && str[idx + 1] >= '0' && str[idx + 1] <= '9') { - is_float = 1; - idx += 2; - while (idx <= end_idx && str[idx] >= '0' && str[idx] <= '9') idx++; - } - - /* if the next char is 'e' or 'E' then maybe read the exponent (or backtrack) */ - if (idx < end_idx && (str[idx] == 'e' || str[idx] == 'E')) { - Py_ssize_t e_start = idx; - idx++; - - /* read an exponent sign if present */ - if (idx < end_idx && (str[idx] == '-' || str[idx] == '+')) idx++; - - /* read all digits */ - while (idx <= end_idx && str[idx] >= '0' && str[idx] <= '9') idx++; - - /* if we got a digit, then parse as float. if not, backtrack */ - if (str[idx - 1] >= '0' && str[idx - 1] <= '9') { - is_float = 1; - } - else { - idx = e_start; - } - } - - /* copy the section we determined to be a number */ - numstr = PyUnicode_FromUnicode(&str[start], idx - start); - if (numstr == NULL) - return NULL; - if (is_float) { - /* parse as a float using a fast path if available, otherwise call user defined method */ - if (s->parse_float != (PyObject *)&PyFloat_Type) { - rval = PyObject_CallFunctionObjArgs(s->parse_float, numstr, NULL); - } - else { - rval = PyFloat_FromString(numstr, NULL); - } - } - else { - /* no fast path for unicode -> int, just call */ - rval = PyObject_CallFunctionObjArgs(s->parse_int, numstr, NULL); - } - Py_DECREF(numstr); - *next_idx_ptr = idx; - return rval; -} - -static PyObject * -scan_once_str(PyScannerObject *s, PyObject *pystr, Py_ssize_t idx, Py_ssize_t *next_idx_ptr) -{ - /* Read one JSON term (of any kind) from PyString pystr. - idx is the index of the first character of the term - *next_idx_ptr is a return-by-reference index to the first character after - the number. - - Returns a new PyObject representation of the term. - */ - char *str = PyString_AS_STRING(pystr); - Py_ssize_t length = PyString_GET_SIZE(pystr); - if (idx >= length) { - PyErr_SetNone(PyExc_StopIteration); - return NULL; - } - switch (str[idx]) { - case '"': - /* string */ - return scanstring_str(pystr, idx + 1, - PyString_AS_STRING(s->encoding), - PyObject_IsTrue(s->strict), - next_idx_ptr); - case '{': - /* object */ - return _parse_object_str(s, pystr, idx + 1, next_idx_ptr); - case '[': - /* array */ - return _parse_array_str(s, pystr, idx + 1, next_idx_ptr); - case 'n': - /* null */ - if ((idx + 3 < length) && str[idx + 1] == 'u' && str[idx + 2] == 'l' && str[idx + 3] == 'l') { - Py_INCREF(Py_None); - *next_idx_ptr = idx + 4; - return Py_None; - } - break; - case 't': - /* true */ - if ((idx + 3 < length) && str[idx + 1] == 'r' && str[idx + 2] == 'u' && str[idx + 3] == 'e') { - Py_INCREF(Py_True); - *next_idx_ptr = idx + 4; - return Py_True; - } - break; - case 'f': - /* false */ - if ((idx + 4 < length) && str[idx + 1] == 'a' && str[idx + 2] == 'l' && str[idx + 3] == 's' && str[idx + 4] == 'e') { - Py_INCREF(Py_False); - *next_idx_ptr = idx + 5; - return Py_False; - } - break; - case 'N': - /* NaN */ - if ((idx + 2 < length) && str[idx + 1] == 'a' && str[idx + 2] == 'N') { - return _parse_constant(s, "NaN", idx, next_idx_ptr); - } - break; - case 'I': - /* Infinity */ - if ((idx + 7 < length) && str[idx + 1] == 'n' && str[idx + 2] == 'f' && str[idx + 3] == 'i' && str[idx + 4] == 'n' && str[idx + 5] == 'i' && str[idx + 6] == 't' && str[idx + 7] == 'y') { - return _parse_constant(s, "Infinity", idx, next_idx_ptr); - } - break; - case '-': - /* -Infinity */ - if ((idx + 8 < length) && str[idx + 1] == 'I' && str[idx + 2] == 'n' && str[idx + 3] == 'f' && str[idx + 4] == 'i' && str[idx + 5] == 'n' && str[idx + 6] == 'i' && str[idx + 7] == 't' && str[idx + 8] == 'y') { - return _parse_constant(s, "-Infinity", idx, next_idx_ptr); - } - break; - } - /* Didn't find a string, object, array, or named constant. Look for a number. */ - return _match_number_str(s, pystr, idx, next_idx_ptr); -} - -static PyObject * -scan_once_unicode(PyScannerObject *s, PyObject *pystr, Py_ssize_t idx, Py_ssize_t *next_idx_ptr) -{ - /* Read one JSON term (of any kind) from PyUnicode pystr. - idx is the index of the first character of the term - *next_idx_ptr is a return-by-reference index to the first character after - the number. - - Returns a new PyObject representation of the term. - */ - Py_UNICODE *str = PyUnicode_AS_UNICODE(pystr); - Py_ssize_t length = PyUnicode_GET_SIZE(pystr); - if (idx >= length) { - PyErr_SetNone(PyExc_StopIteration); - return NULL; - } - switch (str[idx]) { - case '"': - /* string */ - return scanstring_unicode(pystr, idx + 1, - PyObject_IsTrue(s->strict), - next_idx_ptr); - case '{': - /* object */ - return _parse_object_unicode(s, pystr, idx + 1, next_idx_ptr); - case '[': - /* array */ - return _parse_array_unicode(s, pystr, idx + 1, next_idx_ptr); - case 'n': - /* null */ - if ((idx + 3 < length) && str[idx + 1] == 'u' && str[idx + 2] == 'l' && str[idx + 3] == 'l') { - Py_INCREF(Py_None); - *next_idx_ptr = idx + 4; - return Py_None; - } - break; - case 't': - /* true */ - if ((idx + 3 < length) && str[idx + 1] == 'r' && str[idx + 2] == 'u' && str[idx + 3] == 'e') { - Py_INCREF(Py_True); - *next_idx_ptr = idx + 4; - return Py_True; - } - break; - case 'f': - /* false */ - if ((idx + 4 < length) && str[idx + 1] == 'a' && str[idx + 2] == 'l' && str[idx + 3] == 's' && str[idx + 4] == 'e') { - Py_INCREF(Py_False); - *next_idx_ptr = idx + 5; - return Py_False; - } - break; - case 'N': - /* NaN */ - if ((idx + 2 < length) && str[idx + 1] == 'a' && str[idx + 2] == 'N') { - return _parse_constant(s, "NaN", idx, next_idx_ptr); - } - break; - case 'I': - /* Infinity */ - if ((idx + 7 < length) && str[idx + 1] == 'n' && str[idx + 2] == 'f' && str[idx + 3] == 'i' && str[idx + 4] == 'n' && str[idx + 5] == 'i' && str[idx + 6] == 't' && str[idx + 7] == 'y') { - return _parse_constant(s, "Infinity", idx, next_idx_ptr); - } - break; - case '-': - /* -Infinity */ - if ((idx + 8 < length) && str[idx + 1] == 'I' && str[idx + 2] == 'n' && str[idx + 3] == 'f' && str[idx + 4] == 'i' && str[idx + 5] == 'n' && str[idx + 6] == 'i' && str[idx + 7] == 't' && str[idx + 8] == 'y') { - return _parse_constant(s, "-Infinity", idx, next_idx_ptr); - } - break; - } - /* Didn't find a string, object, array, or named constant. Look for a number. */ - return _match_number_unicode(s, pystr, idx, next_idx_ptr); -} - -static PyObject * -scanner_call(PyObject *self, PyObject *args, PyObject *kwds) -{ - /* Python callable interface to scan_once_{str,unicode} */ - PyObject *pystr; - PyObject *rval; - Py_ssize_t idx; - Py_ssize_t next_idx = -1; - static char *kwlist[] = {"string", "idx", NULL}; - PyScannerObject *s; - assert(PyScanner_Check(self)); - s = (PyScannerObject *)self; - if (!PyArg_ParseTupleAndKeywords(args, kwds, "OO&:scan_once", kwlist, &pystr, _convertPyInt_AsSsize_t, &idx)) - return NULL; - - if (PyString_Check(pystr)) { - rval = scan_once_str(s, pystr, idx, &next_idx); - } - else if (PyUnicode_Check(pystr)) { - rval = scan_once_unicode(s, pystr, idx, &next_idx); - } - else { - PyErr_Format(PyExc_TypeError, - "first argument must be a string, not %.80s", - Py_TYPE(pystr)->tp_name); - return NULL; - } - PyDict_Clear(s->memo); - return _build_rval_index_tuple(rval, next_idx); -} - -static PyObject * -scanner_new(PyTypeObject *type, PyObject *args, PyObject *kwds) -{ - PyScannerObject *s; - s = (PyScannerObject *)type->tp_alloc(type, 0); - if (s != NULL) { - s->encoding = NULL; - s->strict = NULL; - s->object_hook = NULL; - s->pairs_hook = NULL; - s->parse_float = NULL; - s->parse_int = NULL; - s->parse_constant = NULL; - } - return (PyObject *)s; -} - -static int -scanner_init(PyObject *self, PyObject *args, PyObject *kwds) -{ - /* Initialize Scanner object */ - PyObject *ctx; - static char *kwlist[] = {"context", NULL}; - PyScannerObject *s; - - assert(PyScanner_Check(self)); - s = (PyScannerObject *)self; - - if (!PyArg_ParseTupleAndKeywords(args, kwds, "O:make_scanner", kwlist, &ctx)) - return -1; - - if (s->memo == NULL) { - s->memo = PyDict_New(); - if (s->memo == NULL) - goto bail; - } - - /* PyString_AS_STRING is used on encoding */ - s->encoding = PyObject_GetAttrString(ctx, "encoding"); - if (s->encoding == NULL) - goto bail; - if (s->encoding == Py_None) { - Py_DECREF(Py_None); - s->encoding = PyString_InternFromString(DEFAULT_ENCODING); - } - else if (PyUnicode_Check(s->encoding)) { - PyObject *tmp = PyUnicode_AsEncodedString(s->encoding, NULL, NULL); - Py_DECREF(s->encoding); - s->encoding = tmp; - } - if (s->encoding == NULL || !PyString_Check(s->encoding)) - goto bail; - - /* All of these will fail "gracefully" so we don't need to verify them */ - s->strict = PyObject_GetAttrString(ctx, "strict"); - if (s->strict == NULL) - goto bail; - s->object_hook = PyObject_GetAttrString(ctx, "object_hook"); - if (s->object_hook == NULL) - goto bail; - s->pairs_hook = PyObject_GetAttrString(ctx, "object_pairs_hook"); - if (s->pairs_hook == NULL) - goto bail; - s->parse_float = PyObject_GetAttrString(ctx, "parse_float"); - if (s->parse_float == NULL) - goto bail; - s->parse_int = PyObject_GetAttrString(ctx, "parse_int"); - if (s->parse_int == NULL) - goto bail; - s->parse_constant = PyObject_GetAttrString(ctx, "parse_constant"); - if (s->parse_constant == NULL) - goto bail; - - return 0; - -bail: - Py_CLEAR(s->encoding); - Py_CLEAR(s->strict); - Py_CLEAR(s->object_hook); - Py_CLEAR(s->pairs_hook); - Py_CLEAR(s->parse_float); - Py_CLEAR(s->parse_int); - Py_CLEAR(s->parse_constant); - return -1; -} - -PyDoc_STRVAR(scanner_doc, "JSON scanner object"); - -static -PyTypeObject PyScannerType = { - PyObject_HEAD_INIT(NULL) - 0, /* tp_internal */ - "simplejson._speedups.Scanner", /* tp_name */ - sizeof(PyScannerObject), /* tp_basicsize */ - 0, /* tp_itemsize */ - scanner_dealloc, /* tp_dealloc */ - 0, /* tp_print */ - 0, /* tp_getattr */ - 0, /* tp_setattr */ - 0, /* tp_compare */ - 0, /* tp_repr */ - 0, /* tp_as_number */ - 0, /* tp_as_sequence */ - 0, /* tp_as_mapping */ - 0, /* tp_hash */ - scanner_call, /* tp_call */ - 0, /* tp_str */ - 0,/* PyObject_GenericGetAttr, */ /* tp_getattro */ - 0,/* PyObject_GenericSetAttr, */ /* tp_setattro */ - 0, /* tp_as_buffer */ - Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC, /* tp_flags */ - scanner_doc, /* tp_doc */ - scanner_traverse, /* tp_traverse */ - scanner_clear, /* tp_clear */ - 0, /* tp_richcompare */ - 0, /* tp_weaklistoffset */ - 0, /* tp_iter */ - 0, /* tp_iternext */ - 0, /* tp_methods */ - scanner_members, /* tp_members */ - 0, /* tp_getset */ - 0, /* tp_base */ - 0, /* tp_dict */ - 0, /* tp_descr_get */ - 0, /* tp_descr_set */ - 0, /* tp_dictoffset */ - scanner_init, /* tp_init */ - 0,/* PyType_GenericAlloc, */ /* tp_alloc */ - scanner_new, /* tp_new */ - 0,/* PyObject_GC_Del, */ /* tp_free */ -}; - -static PyObject * -encoder_new(PyTypeObject *type, PyObject *args, PyObject *kwds) -{ - PyEncoderObject *s; - s = (PyEncoderObject *)type->tp_alloc(type, 0); - if (s != NULL) { - s->markers = NULL; - s->defaultfn = NULL; - s->encoder = NULL; - s->indent = NULL; - s->key_separator = NULL; - s->item_separator = NULL; - s->sort_keys = NULL; - s->skipkeys = NULL; - s->key_memo = NULL; - } - return (PyObject *)s; -} - -static int -encoder_init(PyObject *self, PyObject *args, PyObject *kwds) -{ - /* initialize Encoder object */ - static char *kwlist[] = {"markers", "default", "encoder", "indent", "key_separator", "item_separator", "sort_keys", "skipkeys", "allow_nan", "key_memo", "use_decimal", NULL}; - - PyEncoderObject *s; - PyObject *markers, *defaultfn, *encoder, *indent, *key_separator; - PyObject *item_separator, *sort_keys, *skipkeys, *allow_nan, *key_memo, *use_decimal; - - assert(PyEncoder_Check(self)); - s = (PyEncoderObject *)self; - - if (!PyArg_ParseTupleAndKeywords(args, kwds, "OOOOOOOOOOO:make_encoder", kwlist, - &markers, &defaultfn, &encoder, &indent, &key_separator, &item_separator, - &sort_keys, &skipkeys, &allow_nan, &key_memo, &use_decimal)) - return -1; - - s->markers = markers; - s->defaultfn = defaultfn; - s->encoder = encoder; - s->indent = indent; - s->key_separator = key_separator; - s->item_separator = item_separator; - s->sort_keys = sort_keys; - s->skipkeys = skipkeys; - s->key_memo = key_memo; - s->fast_encode = (PyCFunction_Check(s->encoder) && PyCFunction_GetFunction(s->encoder) == (PyCFunction)py_encode_basestring_ascii); - s->allow_nan = PyObject_IsTrue(allow_nan); - s->use_decimal = PyObject_IsTrue(use_decimal); - - Py_INCREF(s->markers); - Py_INCREF(s->defaultfn); - Py_INCREF(s->encoder); - Py_INCREF(s->indent); - Py_INCREF(s->key_separator); - Py_INCREF(s->item_separator); - Py_INCREF(s->sort_keys); - Py_INCREF(s->skipkeys); - Py_INCREF(s->key_memo); - return 0; -} - -static PyObject * -encoder_call(PyObject *self, PyObject *args, PyObject *kwds) -{ - /* Python callable interface to encode_listencode_obj */ - static char *kwlist[] = {"obj", "_current_indent_level", NULL}; - PyObject *obj; - PyObject *rval; - Py_ssize_t indent_level; - PyEncoderObject *s; - assert(PyEncoder_Check(self)); - s = (PyEncoderObject *)self; - if (!PyArg_ParseTupleAndKeywords(args, kwds, "OO&:_iterencode", kwlist, - &obj, _convertPyInt_AsSsize_t, &indent_level)) - return NULL; - rval = PyList_New(0); - if (rval == NULL) - return NULL; - if (encoder_listencode_obj(s, rval, obj, indent_level)) { - Py_DECREF(rval); - return NULL; - } - return rval; -} - -static PyObject * -_encoded_const(PyObject *obj) -{ - /* Return the JSON string representation of None, True, False */ - if (obj == Py_None) { - static PyObject *s_null = NULL; - if (s_null == NULL) { - s_null = PyString_InternFromString("null"); - } - Py_INCREF(s_null); - return s_null; - } - else if (obj == Py_True) { - static PyObject *s_true = NULL; - if (s_true == NULL) { - s_true = PyString_InternFromString("true"); - } - Py_INCREF(s_true); - return s_true; - } - else if (obj == Py_False) { - static PyObject *s_false = NULL; - if (s_false == NULL) { - s_false = PyString_InternFromString("false"); - } - Py_INCREF(s_false); - return s_false; - } - else { - PyErr_SetString(PyExc_ValueError, "not a const"); - return NULL; - } -} - -static PyObject * -encoder_encode_float(PyEncoderObject *s, PyObject *obj) -{ - /* Return the JSON representation of a PyFloat */ - double i = PyFloat_AS_DOUBLE(obj); - if (!Py_IS_FINITE(i)) { - if (!s->allow_nan) { - PyErr_SetString(PyExc_ValueError, "Out of range float values are not JSON compliant"); - return NULL; - } - if (i > 0) { - return PyString_FromString("Infinity"); - } - else if (i < 0) { - return PyString_FromString("-Infinity"); - } - else { - return PyString_FromString("NaN"); - } - } - /* Use a better float format here? */ - return PyObject_Repr(obj); -} - -static PyObject * -encoder_encode_string(PyEncoderObject *s, PyObject *obj) -{ - /* Return the JSON representation of a string */ - if (s->fast_encode) - return py_encode_basestring_ascii(NULL, obj); - else - return PyObject_CallFunctionObjArgs(s->encoder, obj, NULL); -} - -static int -_steal_list_append(PyObject *lst, PyObject *stolen) -{ - /* Append stolen and then decrement its reference count */ - int rval = PyList_Append(lst, stolen); - Py_DECREF(stolen); - return rval; -} - -static int -encoder_listencode_obj(PyEncoderObject *s, PyObject *rval, PyObject *obj, Py_ssize_t indent_level) -{ - /* Encode Python object obj to a JSON term, rval is a PyList */ - PyObject *newobj; - int rv; - - if (obj == Py_None || obj == Py_True || obj == Py_False) { - PyObject *cstr = _encoded_const(obj); - if (cstr == NULL) - return -1; - return _steal_list_append(rval, cstr); - } - else if (PyString_Check(obj) || PyUnicode_Check(obj)) - { - PyObject *encoded = encoder_encode_string(s, obj); - if (encoded == NULL) - return -1; - return _steal_list_append(rval, encoded); - } - else if (PyInt_Check(obj) || PyLong_Check(obj)) { - PyObject *encoded = PyObject_Str(obj); - if (encoded == NULL) - return -1; - return _steal_list_append(rval, encoded); - } - else if (PyFloat_Check(obj)) { - PyObject *encoded = encoder_encode_float(s, obj); - if (encoded == NULL) - return -1; - return _steal_list_append(rval, encoded); - } - else if (PyList_Check(obj) || PyTuple_Check(obj)) { - return encoder_listencode_list(s, rval, obj, indent_level); - } - else if (PyDict_Check(obj)) { - return encoder_listencode_dict(s, rval, obj, indent_level); - } - else if (s->use_decimal && Decimal_Check(obj)) { - PyObject *encoded = PyObject_Str(obj); - if (encoded == NULL) - return -1; - return _steal_list_append(rval, encoded); - } - else { - PyObject *ident = NULL; - if (s->markers != Py_None) { - int has_key; - ident = PyLong_FromVoidPtr(obj); - if (ident == NULL) - return -1; - has_key = PyDict_Contains(s->markers, ident); - if (has_key) { - if (has_key != -1) - PyErr_SetString(PyExc_ValueError, "Circular reference detected"); - Py_DECREF(ident); - return -1; - } - if (PyDict_SetItem(s->markers, ident, obj)) { - Py_DECREF(ident); - return -1; - } - } - newobj = PyObject_CallFunctionObjArgs(s->defaultfn, obj, NULL); - if (newobj == NULL) { - Py_XDECREF(ident); - return -1; - } - rv = encoder_listencode_obj(s, rval, newobj, indent_level); - Py_DECREF(newobj); - if (rv) { - Py_XDECREF(ident); - return -1; - } - if (ident != NULL) { - if (PyDict_DelItem(s->markers, ident)) { - Py_XDECREF(ident); - return -1; - } - Py_XDECREF(ident); - } - return rv; - } -} - -static int -encoder_listencode_dict(PyEncoderObject *s, PyObject *rval, PyObject *dct, Py_ssize_t indent_level) -{ - /* Encode Python dict dct a JSON term, rval is a PyList */ - static PyObject *open_dict = NULL; - static PyObject *close_dict = NULL; - static PyObject *empty_dict = NULL; - static PyObject *iteritems = NULL; - PyObject *kstr = NULL; - PyObject *ident = NULL; - PyObject *iter = NULL; - PyObject *item = NULL; - PyObject *items = NULL; - PyObject *encoded = NULL; - int skipkeys; - Py_ssize_t idx; - - if (open_dict == NULL || close_dict == NULL || empty_dict == NULL || iteritems == NULL) { - open_dict = PyString_InternFromString("{"); - close_dict = PyString_InternFromString("}"); - empty_dict = PyString_InternFromString("{}"); - iteritems = PyString_InternFromString("iteritems"); - if (open_dict == NULL || close_dict == NULL || empty_dict == NULL || iteritems == NULL) - return -1; - } - if (PyDict_Size(dct) == 0) - return PyList_Append(rval, empty_dict); - - if (s->markers != Py_None) { - int has_key; - ident = PyLong_FromVoidPtr(dct); - if (ident == NULL) - goto bail; - has_key = PyDict_Contains(s->markers, ident); - if (has_key) { - if (has_key != -1) - PyErr_SetString(PyExc_ValueError, "Circular reference detected"); - goto bail; - } - if (PyDict_SetItem(s->markers, ident, dct)) { - goto bail; - } - } - - if (PyList_Append(rval, open_dict)) - goto bail; - - if (s->indent != Py_None) { - /* TODO: DOES NOT RUN */ - indent_level += 1; - /* - newline_indent = '\n' + (_indent * _current_indent_level) - separator = _item_separator + newline_indent - buf += newline_indent - */ - } - - if (PyObject_IsTrue(s->sort_keys)) { - /* First sort the keys then replace them with (key, value) tuples. */ - Py_ssize_t i, nitems; - if (PyDict_CheckExact(dct)) - items = PyDict_Keys(dct); - else - items = PyMapping_Keys(dct); - if (items == NULL) - goto bail; - if (!PyList_Check(items)) { - PyErr_SetString(PyExc_ValueError, "keys must return list"); - goto bail; - } - if (PyList_Sort(items) < 0) - goto bail; - nitems = PyList_GET_SIZE(items); - for (i = 0; i < nitems; i++) { - PyObject *key, *value; - key = PyList_GET_ITEM(items, i); - value = PyDict_GetItem(dct, key); - item = PyTuple_Pack(2, key, value); - if (item == NULL) - goto bail; - PyList_SET_ITEM(items, i, item); - Py_DECREF(key); - } - } - else { - if (PyDict_CheckExact(dct)) - items = PyDict_Items(dct); - else - items = PyMapping_Items(dct); - } - if (items == NULL) - goto bail; - iter = PyObject_GetIter(items); - Py_DECREF(items); - if (iter == NULL) - goto bail; - - skipkeys = PyObject_IsTrue(s->skipkeys); - idx = 0; - while ((item = PyIter_Next(iter))) { - PyObject *encoded, *key, *value; - if (!PyTuple_Check(item) || Py_SIZE(item) != 2) { - PyErr_SetString(PyExc_ValueError, "items must return 2-tuples"); - goto bail; - } - key = PyTuple_GET_ITEM(item, 0); - if (key == NULL) - goto bail; - value = PyTuple_GET_ITEM(item, 1); - if (value == NULL) - goto bail; - - encoded = PyDict_GetItem(s->key_memo, key); - if (encoded != NULL) { - Py_INCREF(encoded); - } - else if (PyString_Check(key) || PyUnicode_Check(key)) { - Py_INCREF(key); - kstr = key; - } - else if (PyFloat_Check(key)) { - kstr = encoder_encode_float(s, key); - if (kstr == NULL) - goto bail; - } - else if (key == Py_True || key == Py_False || key == Py_None) { - /* This must come before the PyInt_Check because - True and False are also 1 and 0.*/ - kstr = _encoded_const(key); - if (kstr == NULL) - goto bail; - } - else if (PyInt_Check(key) || PyLong_Check(key)) { - kstr = PyObject_Str(key); - if (kstr == NULL) - goto bail; - } - else if (skipkeys) { - Py_DECREF(item); - continue; - } - else { - /* TODO: include repr of key */ - PyErr_SetString(PyExc_TypeError, "keys must be a string"); - goto bail; - } - - if (idx) { - if (PyList_Append(rval, s->item_separator)) - goto bail; - } - - if (encoded == NULL) { - encoded = encoder_encode_string(s, kstr); - Py_CLEAR(kstr); - if (encoded == NULL) - goto bail; - if (PyDict_SetItem(s->key_memo, key, encoded)) - goto bail; - } - if (PyList_Append(rval, encoded)) { - goto bail; - } - Py_CLEAR(encoded); - if (PyList_Append(rval, s->key_separator)) - goto bail; - if (encoder_listencode_obj(s, rval, value, indent_level)) - goto bail; - Py_CLEAR(item); - idx += 1; - } - Py_CLEAR(iter); - if (PyErr_Occurred()) - goto bail; - if (ident != NULL) { - if (PyDict_DelItem(s->markers, ident)) - goto bail; - Py_CLEAR(ident); - } - if (s->indent != Py_None) { - /* TODO: DOES NOT RUN */ - indent_level -= 1; - /* - yield '\n' + (_indent * _current_indent_level) - */ - } - if (PyList_Append(rval, close_dict)) - goto bail; - return 0; - -bail: - Py_XDECREF(encoded); - Py_XDECREF(items); - Py_XDECREF(iter); - Py_XDECREF(kstr); - Py_XDECREF(ident); - return -1; -} - - -static int -encoder_listencode_list(PyEncoderObject *s, PyObject *rval, PyObject *seq, Py_ssize_t indent_level) -{ - /* Encode Python list seq to a JSON term, rval is a PyList */ - static PyObject *open_array = NULL; - static PyObject *close_array = NULL; - static PyObject *empty_array = NULL; - PyObject *ident = NULL; - PyObject *iter = NULL; - PyObject *obj = NULL; - int is_true; - int i = 0; - - if (open_array == NULL || close_array == NULL || empty_array == NULL) { - open_array = PyString_InternFromString("["); - close_array = PyString_InternFromString("]"); - empty_array = PyString_InternFromString("[]"); - if (open_array == NULL || close_array == NULL || empty_array == NULL) - return -1; - } - ident = NULL; - is_true = PyObject_IsTrue(seq); - if (is_true == -1) - return -1; - else if (is_true == 0) - return PyList_Append(rval, empty_array); - - if (s->markers != Py_None) { - int has_key; - ident = PyLong_FromVoidPtr(seq); - if (ident == NULL) - goto bail; - has_key = PyDict_Contains(s->markers, ident); - if (has_key) { - if (has_key != -1) - PyErr_SetString(PyExc_ValueError, "Circular reference detected"); - goto bail; - } - if (PyDict_SetItem(s->markers, ident, seq)) { - goto bail; - } - } - - iter = PyObject_GetIter(seq); - if (iter == NULL) - goto bail; - - if (PyList_Append(rval, open_array)) - goto bail; - if (s->indent != Py_None) { - /* TODO: DOES NOT RUN */ - indent_level += 1; - /* - newline_indent = '\n' + (_indent * _current_indent_level) - separator = _item_separator + newline_indent - buf += newline_indent - */ - } - while ((obj = PyIter_Next(iter))) { - if (i) { - if (PyList_Append(rval, s->item_separator)) - goto bail; - } - if (encoder_listencode_obj(s, rval, obj, indent_level)) - goto bail; - i++; - Py_CLEAR(obj); - } - Py_CLEAR(iter); - if (PyErr_Occurred()) - goto bail; - if (ident != NULL) { - if (PyDict_DelItem(s->markers, ident)) - goto bail; - Py_CLEAR(ident); - } - if (s->indent != Py_None) { - /* TODO: DOES NOT RUN */ - indent_level -= 1; - /* - yield '\n' + (_indent * _current_indent_level) - */ - } - if (PyList_Append(rval, close_array)) - goto bail; - return 0; - -bail: - Py_XDECREF(obj); - Py_XDECREF(iter); - Py_XDECREF(ident); - return -1; -} - -static void -encoder_dealloc(PyObject *self) -{ - /* Deallocate Encoder */ - encoder_clear(self); - Py_TYPE(self)->tp_free(self); -} - -static int -encoder_traverse(PyObject *self, visitproc visit, void *arg) -{ - PyEncoderObject *s; - assert(PyEncoder_Check(self)); - s = (PyEncoderObject *)self; - Py_VISIT(s->markers); - Py_VISIT(s->defaultfn); - Py_VISIT(s->encoder); - Py_VISIT(s->indent); - Py_VISIT(s->key_separator); - Py_VISIT(s->item_separator); - Py_VISIT(s->sort_keys); - Py_VISIT(s->skipkeys); - Py_VISIT(s->key_memo); - return 0; -} - -static int -encoder_clear(PyObject *self) -{ - /* Deallocate Encoder */ - PyEncoderObject *s; - assert(PyEncoder_Check(self)); - s = (PyEncoderObject *)self; - Py_CLEAR(s->markers); - Py_CLEAR(s->defaultfn); - Py_CLEAR(s->encoder); - Py_CLEAR(s->indent); - Py_CLEAR(s->key_separator); - Py_CLEAR(s->item_separator); - Py_CLEAR(s->sort_keys); - Py_CLEAR(s->skipkeys); - Py_CLEAR(s->key_memo); - return 0; -} - -PyDoc_STRVAR(encoder_doc, "_iterencode(obj, _current_indent_level) -> iterable"); - -static -PyTypeObject PyEncoderType = { - PyObject_HEAD_INIT(NULL) - 0, /* tp_internal */ - "simplejson._speedups.Encoder", /* tp_name */ - sizeof(PyEncoderObject), /* tp_basicsize */ - 0, /* tp_itemsize */ - encoder_dealloc, /* tp_dealloc */ - 0, /* tp_print */ - 0, /* tp_getattr */ - 0, /* tp_setattr */ - 0, /* tp_compare */ - 0, /* tp_repr */ - 0, /* tp_as_number */ - 0, /* tp_as_sequence */ - 0, /* tp_as_mapping */ - 0, /* tp_hash */ - encoder_call, /* tp_call */ - 0, /* tp_str */ - 0, /* tp_getattro */ - 0, /* tp_setattro */ - 0, /* tp_as_buffer */ - Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC, /* tp_flags */ - encoder_doc, /* tp_doc */ - encoder_traverse, /* tp_traverse */ - encoder_clear, /* tp_clear */ - 0, /* tp_richcompare */ - 0, /* tp_weaklistoffset */ - 0, /* tp_iter */ - 0, /* tp_iternext */ - 0, /* tp_methods */ - encoder_members, /* tp_members */ - 0, /* tp_getset */ - 0, /* tp_base */ - 0, /* tp_dict */ - 0, /* tp_descr_get */ - 0, /* tp_descr_set */ - 0, /* tp_dictoffset */ - encoder_init, /* tp_init */ - 0, /* tp_alloc */ - encoder_new, /* tp_new */ - 0, /* tp_free */ -}; - -static PyMethodDef speedups_methods[] = { - {"encode_basestring_ascii", - (PyCFunction)py_encode_basestring_ascii, - METH_O, - pydoc_encode_basestring_ascii}, - {"scanstring", - (PyCFunction)py_scanstring, - METH_VARARGS, - pydoc_scanstring}, - {NULL, NULL, 0, NULL} -}; - -PyDoc_STRVAR(module_doc, -"simplejson speedups\n"); - -void -init_speedups(void) -{ - PyObject *m, *decimal; - PyScannerType.tp_new = PyType_GenericNew; - if (PyType_Ready(&PyScannerType) < 0) - return; - PyEncoderType.tp_new = PyType_GenericNew; - if (PyType_Ready(&PyEncoderType) < 0) - return; - - decimal = PyImport_ImportModule("decimal"); - if (decimal == NULL) - return; - DecimalTypePtr = (PyTypeObject*)PyObject_GetAttrString(decimal, "Decimal"); - Py_DECREF(decimal); - if (DecimalTypePtr == NULL) - return; - - m = Py_InitModule3("_speedups", speedups_methods, module_doc); - Py_INCREF((PyObject*)&PyScannerType); - PyModule_AddObject(m, "make_scanner", (PyObject*)&PyScannerType); - Py_INCREF((PyObject*)&PyEncoderType); - PyModule_AddObject(m, "make_encoder", (PyObject*)&PyEncoderType); -} diff --git a/libs/simplejson/decoder.py b/libs/simplejson/decoder.py deleted file mode 100644 index e5496d6..0000000 --- a/libs/simplejson/decoder.py +++ /dev/null @@ -1,421 +0,0 @@ -"""Implementation of JSONDecoder -""" -import re -import sys -import struct - -from simplejson.scanner import make_scanner -def _import_c_scanstring(): - try: - from simplejson._speedups import scanstring - return scanstring - except ImportError: - return None -c_scanstring = _import_c_scanstring() - -__all__ = ['JSONDecoder'] - -FLAGS = re.VERBOSE | re.MULTILINE | re.DOTALL - -def _floatconstants(): - _BYTES = '7FF80000000000007FF0000000000000'.decode('hex') - # The struct module in Python 2.4 would get frexp() out of range here - # when an endian is specified in the format string. Fixed in Python 2.5+ - if sys.byteorder != 'big': - _BYTES = _BYTES[:8][::-1] + _BYTES[8:][::-1] - nan, inf = struct.unpack('dd', _BYTES) - return nan, inf, -inf - -NaN, PosInf, NegInf = _floatconstants() - - -class JSONDecodeError(ValueError): - """Subclass of ValueError with the following additional properties: - - msg: The unformatted error message - doc: The JSON document being parsed - pos: The start index of doc where parsing failed - end: The end index of doc where parsing failed (may be None) - lineno: The line corresponding to pos - colno: The column corresponding to pos - endlineno: The line corresponding to end (may be None) - endcolno: The column corresponding to end (may be None) - - """ - def __init__(self, msg, doc, pos, end=None): - ValueError.__init__(self, errmsg(msg, doc, pos, end=end)) - self.msg = msg - self.doc = doc - self.pos = pos - self.end = end - self.lineno, self.colno = linecol(doc, pos) - if end is not None: - self.endlineno, self.endcolno = linecol(doc, end) - else: - self.endlineno, self.endcolno = None, None - - -def linecol(doc, pos): - lineno = doc.count('\n', 0, pos) + 1 - if lineno == 1: - colno = pos - else: - colno = pos - doc.rindex('\n', 0, pos) - return lineno, colno - - -def errmsg(msg, doc, pos, end=None): - # Note that this function is called from _speedups - lineno, colno = linecol(doc, pos) - if end is None: - #fmt = '{0}: line {1} column {2} (char {3})' - #return fmt.format(msg, lineno, colno, pos) - fmt = '%s: line %d column %d (char %d)' - return fmt % (msg, lineno, colno, pos) - endlineno, endcolno = linecol(doc, end) - #fmt = '{0}: line {1} column {2} - line {3} column {4} (char {5} - {6})' - #return fmt.format(msg, lineno, colno, endlineno, endcolno, pos, end) - fmt = '%s: line %d column %d - line %d column %d (char %d - %d)' - return fmt % (msg, lineno, colno, endlineno, endcolno, pos, end) - - -_CONSTANTS = { - '-Infinity': NegInf, - 'Infinity': PosInf, - 'NaN': NaN, -} - -STRINGCHUNK = re.compile(r'(.*?)(["\\\x00-\x1f])', FLAGS) -BACKSLASH = { - '"': u'"', '\\': u'\\', '/': u'/', - 'b': u'\b', 'f': u'\f', 'n': u'\n', 'r': u'\r', 't': u'\t', -} - -DEFAULT_ENCODING = "utf-8" - -def py_scanstring(s, end, encoding=None, strict=True, - _b=BACKSLASH, _m=STRINGCHUNK.match): - """Scan the string s for a JSON string. End is the index of the - character in s after the quote that started the JSON string. - Unescapes all valid JSON string escape sequences and raises ValueError - on attempt to decode an invalid string. If strict is False then literal - control characters are allowed in the string. - - Returns a tuple of the decoded string and the index of the character in s - after the end quote.""" - if encoding is None: - encoding = DEFAULT_ENCODING - chunks = [] - _append = chunks.append - begin = end - 1 - while 1: - chunk = _m(s, end) - if chunk is None: - raise JSONDecodeError( - "Unterminated string starting at", s, begin) - end = chunk.end() - content, terminator = chunk.groups() - # Content is contains zero or more unescaped string characters - if content: - if not isinstance(content, unicode): - content = unicode(content, encoding) - _append(content) - # Terminator is the end of string, a literal control character, - # or a backslash denoting that an escape sequence follows - if terminator == '"': - break - elif terminator != '\\': - if strict: - msg = "Invalid control character %r at" % (terminator,) - #msg = "Invalid control character {0!r} at".format(terminator) - raise JSONDecodeError(msg, s, end) - else: - _append(terminator) - continue - try: - esc = s[end] - except IndexError: - raise JSONDecodeError( - "Unterminated string starting at", s, begin) - # If not a unicode escape sequence, must be in the lookup table - if esc != 'u': - try: - char = _b[esc] - except KeyError: - msg = "Invalid \\escape: " + repr(esc) - raise JSONDecodeError(msg, s, end) - end += 1 - else: - # Unicode escape sequence - esc = s[end + 1:end + 5] - next_end = end + 5 - if len(esc) != 4: - msg = "Invalid \\uXXXX escape" - raise JSONDecodeError(msg, s, end) - uni = int(esc, 16) - # Check for surrogate pair on UCS-4 systems - if 0xd800 <= uni <= 0xdbff and sys.maxunicode > 65535: - msg = "Invalid \\uXXXX\\uXXXX surrogate pair" - if not s[end + 5:end + 7] == '\\u': - raise JSONDecodeError(msg, s, end) - esc2 = s[end + 7:end + 11] - if len(esc2) != 4: - raise JSONDecodeError(msg, s, end) - uni2 = int(esc2, 16) - uni = 0x10000 + (((uni - 0xd800) << 10) | (uni2 - 0xdc00)) - next_end += 6 - char = unichr(uni) - end = next_end - # Append the unescaped character - _append(char) - return u''.join(chunks), end - - -# Use speedup if available -scanstring = c_scanstring or py_scanstring - -WHITESPACE = re.compile(r'[ \t\n\r]*', FLAGS) -WHITESPACE_STR = ' \t\n\r' - -def JSONObject((s, end), encoding, strict, scan_once, object_hook, - object_pairs_hook, memo=None, - _w=WHITESPACE.match, _ws=WHITESPACE_STR): - # Backwards compatibility - if memo is None: - memo = {} - memo_get = memo.setdefault - pairs = [] - # Use a slice to prevent IndexError from being raised, the following - # check will raise a more specific ValueError if the string is empty - nextchar = s[end:end + 1] - # Normally we expect nextchar == '"' - if nextchar != '"': - if nextchar in _ws: - end = _w(s, end).end() - nextchar = s[end:end + 1] - # Trivial empty object - if nextchar == '}': - if object_pairs_hook is not None: - result = object_pairs_hook(pairs) - return result, end + 1 - pairs = {} - if object_hook is not None: - pairs = object_hook(pairs) - return pairs, end + 1 - elif nextchar != '"': - raise JSONDecodeError("Expecting property name", s, end) - end += 1 - while True: - key, end = scanstring(s, end, encoding, strict) - key = memo_get(key, key) - - # To skip some function call overhead we optimize the fast paths where - # the JSON key separator is ": " or just ":". - if s[end:end + 1] != ':': - end = _w(s, end).end() - if s[end:end + 1] != ':': - raise JSONDecodeError("Expecting : delimiter", s, end) - - end += 1 - - try: - if s[end] in _ws: - end += 1 - if s[end] in _ws: - end = _w(s, end + 1).end() - except IndexError: - pass - - try: - value, end = scan_once(s, end) - except StopIteration: - raise JSONDecodeError("Expecting object", s, end) - pairs.append((key, value)) - - try: - nextchar = s[end] - if nextchar in _ws: - end = _w(s, end + 1).end() - nextchar = s[end] - except IndexError: - nextchar = '' - end += 1 - - if nextchar == '}': - break - elif nextchar != ',': - raise JSONDecodeError("Expecting , delimiter", s, end - 1) - - try: - nextchar = s[end] - if nextchar in _ws: - end += 1 - nextchar = s[end] - if nextchar in _ws: - end = _w(s, end + 1).end() - nextchar = s[end] - except IndexError: - nextchar = '' - - end += 1 - if nextchar != '"': - raise JSONDecodeError("Expecting property name", s, end - 1) - - if object_pairs_hook is not None: - result = object_pairs_hook(pairs) - return result, end - pairs = dict(pairs) - if object_hook is not None: - pairs = object_hook(pairs) - return pairs, end - -def JSONArray((s, end), scan_once, _w=WHITESPACE.match, _ws=WHITESPACE_STR): - values = [] - nextchar = s[end:end + 1] - if nextchar in _ws: - end = _w(s, end + 1).end() - nextchar = s[end:end + 1] - # Look-ahead for trivial empty array - if nextchar == ']': - return values, end + 1 - _append = values.append - while True: - try: - value, end = scan_once(s, end) - except StopIteration: - raise JSONDecodeError("Expecting object", s, end) - _append(value) - nextchar = s[end:end + 1] - if nextchar in _ws: - end = _w(s, end + 1).end() - nextchar = s[end:end + 1] - end += 1 - if nextchar == ']': - break - elif nextchar != ',': - raise JSONDecodeError("Expecting , delimiter", s, end) - - try: - if s[end] in _ws: - end += 1 - if s[end] in _ws: - end = _w(s, end + 1).end() - except IndexError: - pass - - return values, end - -class JSONDecoder(object): - """Simple JSON decoder - - Performs the following translations in decoding by default: - - +---------------+-------------------+ - | JSON | Python | - +===============+===================+ - | object | dict | - +---------------+-------------------+ - | array | list | - +---------------+-------------------+ - | string | unicode | - +---------------+-------------------+ - | number (int) | int, long | - +---------------+-------------------+ - | number (real) | float | - +---------------+-------------------+ - | true | True | - +---------------+-------------------+ - | false | False | - +---------------+-------------------+ - | null | None | - +---------------+-------------------+ - - It also understands ``NaN``, ``Infinity``, and ``-Infinity`` as - their corresponding ``float`` values, which is outside the JSON spec. - - """ - - def __init__(self, encoding=None, object_hook=None, parse_float=None, - parse_int=None, parse_constant=None, strict=True, - object_pairs_hook=None): - """ - *encoding* determines the encoding used to interpret any - :class:`str` objects decoded by this instance (``'utf-8'`` by - default). It has no effect when decoding :class:`unicode` objects. - - Note that currently only encodings that are a superset of ASCII work, - strings of other encodings should be passed in as :class:`unicode`. - - *object_hook*, if specified, will be called with the result of every - JSON object decoded and its return value will be used in place of the - given :class:`dict`. This can be used to provide custom - deserializations (e.g. to support JSON-RPC class hinting). - - *object_pairs_hook* is an optional function that will be called with - the result of any object literal decode with an ordered list of pairs. - The return value of *object_pairs_hook* will be used instead of the - :class:`dict`. This feature can be used to implement custom decoders - that rely on the order that the key and value pairs are decoded (for - example, :func:`collections.OrderedDict` will remember the order of - insertion). If *object_hook* is also defined, the *object_pairs_hook* - takes priority. - - *parse_float*, if specified, will be called with the string of every - JSON float to be decoded. By default, this is equivalent to - ``float(num_str)``. This can be used to use another datatype or parser - for JSON floats (e.g. :class:`decimal.Decimal`). - - *parse_int*, if specified, will be called with the string of every - JSON int to be decoded. By default, this is equivalent to - ``int(num_str)``. This can be used to use another datatype or parser - for JSON integers (e.g. :class:`float`). - - *parse_constant*, if specified, will be called with one of the - following strings: ``'-Infinity'``, ``'Infinity'``, ``'NaN'``. This - can be used to raise an exception if invalid JSON numbers are - encountered. - - *strict* controls the parser's behavior when it encounters an - invalid control character in a string. The default setting of - ``True`` means that unescaped control characters are parse errors, if - ``False`` then control characters will be allowed in strings. - - """ - self.encoding = encoding - self.object_hook = object_hook - self.object_pairs_hook = object_pairs_hook - self.parse_float = parse_float or float - self.parse_int = parse_int or int - self.parse_constant = parse_constant or _CONSTANTS.__getitem__ - self.strict = strict - self.parse_object = JSONObject - self.parse_array = JSONArray - self.parse_string = scanstring - self.memo = {} - self.scan_once = make_scanner(self) - - def decode(self, s, _w=WHITESPACE.match): - """Return the Python representation of ``s`` (a ``str`` or ``unicode`` - instance containing a JSON document) - - """ - obj, end = self.raw_decode(s, idx=_w(s, 0).end()) - end = _w(s, end).end() - if end != len(s): - raise JSONDecodeError("Extra data", s, end, len(s)) - return obj - - def raw_decode(self, s, idx=0): - """Decode a JSON document from ``s`` (a ``str`` or ``unicode`` - beginning with a JSON document) and return a 2-tuple of the Python - representation and the index in ``s`` where the document ended. - - This can be used to decode a JSON document from a string that may - have extraneous data at the end. - - """ - try: - obj, end = self.scan_once(s, idx) - except StopIteration: - raise JSONDecodeError("No JSON object could be decoded", s, idx) - return obj, end diff --git a/libs/simplejson/encoder.py b/libs/simplejson/encoder.py deleted file mode 100644 index 468d1bd..0000000 --- a/libs/simplejson/encoder.py +++ /dev/null @@ -1,501 +0,0 @@ -"""Implementation of JSONEncoder -""" -import re -from decimal import Decimal - -def _import_speedups(): - try: - from simplejson import _speedups - return _speedups.encode_basestring_ascii, _speedups.make_encoder - except ImportError: - return None, None -c_encode_basestring_ascii, c_make_encoder = _import_speedups() - -from simplejson.decoder import PosInf - -ESCAPE = re.compile(r'[\x00-\x1f\\"\b\f\n\r\t]') -ESCAPE_ASCII = re.compile(r'([\\"]|[^\ -~])') -HAS_UTF8 = re.compile(r'[\x80-\xff]') -ESCAPE_DCT = { - '\\': '\\\\', - '"': '\\"', - '\b': '\\b', - '\f': '\\f', - '\n': '\\n', - '\r': '\\r', - '\t': '\\t', -} -for i in range(0x20): - #ESCAPE_DCT.setdefault(chr(i), '\\u{0:04x}'.format(i)) - ESCAPE_DCT.setdefault(chr(i), '\\u%04x' % (i,)) - -FLOAT_REPR = repr - -def encode_basestring(s): - """Return a JSON representation of a Python string - - """ - if isinstance(s, str) and HAS_UTF8.search(s) is not None: - s = s.decode('utf-8') - def replace(match): - return ESCAPE_DCT[match.group(0)] - return u'"' + ESCAPE.sub(replace, s) + u'"' - - -def py_encode_basestring_ascii(s): - """Return an ASCII-only JSON representation of a Python string - - """ - if isinstance(s, str) and HAS_UTF8.search(s) is not None: - s = s.decode('utf-8') - def replace(match): - s = match.group(0) - try: - return ESCAPE_DCT[s] - except KeyError: - n = ord(s) - if n < 0x10000: - #return '\\u{0:04x}'.format(n) - return '\\u%04x' % (n,) - else: - # surrogate pair - n -= 0x10000 - s1 = 0xd800 | ((n >> 10) & 0x3ff) - s2 = 0xdc00 | (n & 0x3ff) - #return '\\u{0:04x}\\u{1:04x}'.format(s1, s2) - return '\\u%04x\\u%04x' % (s1, s2) - return '"' + str(ESCAPE_ASCII.sub(replace, s)) + '"' - - -encode_basestring_ascii = ( - c_encode_basestring_ascii or py_encode_basestring_ascii) - -class JSONEncoder(object): - """Extensible JSON encoder for Python data structures. - - Supports the following objects and types by default: - - +-------------------+---------------+ - | Python | JSON | - +===================+===============+ - | dict | object | - +-------------------+---------------+ - | list, tuple | array | - +-------------------+---------------+ - | str, unicode | string | - +-------------------+---------------+ - | int, long, float | number | - +-------------------+---------------+ - | True | true | - +-------------------+---------------+ - | False | false | - +-------------------+---------------+ - | None | null | - +-------------------+---------------+ - - To extend this to recognize other objects, subclass and implement a - ``.default()`` method with another method that returns a serializable - object for ``o`` if possible, otherwise it should call the superclass - implementation (to raise ``TypeError``). - - """ - item_separator = ', ' - key_separator = ': ' - def __init__(self, skipkeys=False, ensure_ascii=True, - check_circular=True, allow_nan=True, sort_keys=False, - indent=None, separators=None, encoding='utf-8', default=None, - use_decimal=False): - """Constructor for JSONEncoder, with sensible defaults. - - If skipkeys is false, then it is a TypeError to attempt - encoding of keys that are not str, int, long, float or None. If - skipkeys is True, such items are simply skipped. - - If ensure_ascii is true, the output is guaranteed to be str - objects with all incoming unicode characters escaped. If - ensure_ascii is false, the output will be unicode object. - - If check_circular is true, then lists, dicts, and custom encoded - objects will be checked for circular references during encoding to - prevent an infinite recursion (which would cause an OverflowError). - Otherwise, no such check takes place. - - If allow_nan is true, then NaN, Infinity, and -Infinity will be - encoded as such. This behavior is not JSON specification compliant, - but is consistent with most JavaScript based encoders and decoders. - Otherwise, it will be a ValueError to encode such floats. - - If sort_keys is true, then the output of dictionaries will be - sorted by key; this is useful for regression tests to ensure - that JSON serializations can be compared on a day-to-day basis. - - If indent is a string, then JSON array elements and object members - will be pretty-printed with a newline followed by that string repeated - for each level of nesting. ``None`` (the default) selects the most compact - representation without any newlines. For backwards compatibility with - versions of simplejson earlier than 2.1.0, an integer is also accepted - and is converted to a string with that many spaces. - - If specified, separators should be a (item_separator, key_separator) - tuple. The default is (', ', ': '). To get the most compact JSON - representation you should specify (',', ':') to eliminate whitespace. - - If specified, default is a function that gets called for objects - that can't otherwise be serialized. It should return a JSON encodable - version of the object or raise a ``TypeError``. - - If encoding is not None, then all input strings will be - transformed into unicode using that encoding prior to JSON-encoding. - The default is UTF-8. - - If use_decimal is true (not the default), ``decimal.Decimal`` will - be supported directly by the encoder. For the inverse, decode JSON - with ``parse_float=decimal.Decimal``. - - """ - - self.skipkeys = skipkeys - self.ensure_ascii = ensure_ascii - self.check_circular = check_circular - self.allow_nan = allow_nan - self.sort_keys = sort_keys - self.use_decimal = use_decimal - if isinstance(indent, (int, long)): - indent = ' ' * indent - self.indent = indent - if separators is not None: - self.item_separator, self.key_separator = separators - if default is not None: - self.default = default - self.encoding = encoding - - def default(self, o): - """Implement this method in a subclass such that it returns - a serializable object for ``o``, or calls the base implementation - (to raise a ``TypeError``). - - For example, to support arbitrary iterators, you could - implement default like this:: - - def default(self, o): - try: - iterable = iter(o) - except TypeError: - pass - else: - return list(iterable) - return JSONEncoder.default(self, o) - - """ - raise TypeError(repr(o) + " is not JSON serializable") - - def encode(self, o): - """Return a JSON string representation of a Python data structure. - - >>> from simplejson import JSONEncoder - >>> JSONEncoder().encode({"foo": ["bar", "baz"]}) - '{"foo": ["bar", "baz"]}' - - """ - # This is for extremely simple cases and benchmarks. - if isinstance(o, basestring): - if isinstance(o, str): - _encoding = self.encoding - if (_encoding is not None - and not (_encoding == 'utf-8')): - o = o.decode(_encoding) - if self.ensure_ascii: - return encode_basestring_ascii(o) - else: - return encode_basestring(o) - # This doesn't pass the iterator directly to ''.join() because the - # exceptions aren't as detailed. The list call should be roughly - # equivalent to the PySequence_Fast that ''.join() would do. - chunks = self.iterencode(o, _one_shot=True) - if not isinstance(chunks, (list, tuple)): - chunks = list(chunks) - if self.ensure_ascii: - return ''.join(chunks) - else: - return u''.join(chunks) - - def iterencode(self, o, _one_shot=False): - """Encode the given object and yield each string - representation as available. - - For example:: - - for chunk in JSONEncoder().iterencode(bigobject): - mysocket.write(chunk) - - """ - if self.check_circular: - markers = {} - else: - markers = None - if self.ensure_ascii: - _encoder = encode_basestring_ascii - else: - _encoder = encode_basestring - if self.encoding != 'utf-8': - def _encoder(o, _orig_encoder=_encoder, _encoding=self.encoding): - if isinstance(o, str): - o = o.decode(_encoding) - return _orig_encoder(o) - - def floatstr(o, allow_nan=self.allow_nan, - _repr=FLOAT_REPR, _inf=PosInf, _neginf=-PosInf): - # Check for specials. Note that this type of test is processor - # and/or platform-specific, so do tests which don't depend on - # the internals. - - if o != o: - text = 'NaN' - elif o == _inf: - text = 'Infinity' - elif o == _neginf: - text = '-Infinity' - else: - return _repr(o) - - if not allow_nan: - raise ValueError( - "Out of range float values are not JSON compliant: " + - repr(o)) - - return text - - - key_memo = {} - if (_one_shot and c_make_encoder is not None - and self.indent is None): - _iterencode = c_make_encoder( - markers, self.default, _encoder, self.indent, - self.key_separator, self.item_separator, self.sort_keys, - self.skipkeys, self.allow_nan, key_memo, self.use_decimal) - else: - _iterencode = _make_iterencode( - markers, self.default, _encoder, self.indent, floatstr, - self.key_separator, self.item_separator, self.sort_keys, - self.skipkeys, _one_shot, self.use_decimal) - try: - return _iterencode(o, 0) - finally: - key_memo.clear() - - -class JSONEncoderForHTML(JSONEncoder): - """An encoder that produces JSON safe to embed in HTML. - - To embed JSON content in, say, a script tag on a web page, the - characters &, < and > should be escaped. They cannot be escaped - with the usual entities (e.g. &) because they are not expanded - within