diff --git a/SABnzbd.py b/SABnzbd.py index 7543780..b2f5dbe 100755 --- a/SABnzbd.py +++ b/SABnzbd.py @@ -52,8 +52,8 @@ except: sys.exit(1) import cherrypy -if [int(n) for n in cherrypy.__version__.split('.')] < [6, 0, 2]: - print 'Sorry, requires Python module Cherrypy 6.0.2+ (use the included version)' +if [int(n) for n in cherrypy.__version__.split('.')] < [8, 1, 2]: + print 'Sorry, requires Python module Cherrypy 8.1.0+ (use the included version)' sys.exit(1) from cherrypy import _cpserver diff --git a/cherrypy/LICENSE.txt b/cherrypy/LICENSE.txt index 34d1505..9848d32 100644 --- a/cherrypy/LICENSE.txt +++ b/cherrypy/LICENSE.txt @@ -1,25 +1,25 @@ -Copyright (c) 2004-2016, CherryPy Team (team@cherrypy.org) -All rights reserved. - -Redistribution and use in source and binary forms, with or without modification, -are permitted provided that the following conditions are met: - - * Redistributions of source code must retain the above copyright notice, - this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above copyright notice, - this list of conditions and the following disclaimer in the documentation - and/or other materials provided with the distribution. - * Neither the name of the CherryPy Team nor the names of its contributors - may be used to endorse or promote products derived from this software - without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND -ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE -FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +Copyright (c) 2004-2016, CherryPy Team (team@cherrypy.org) +All rights reserved. + +Redistribution and use in source and binary forms, with or without modification, +are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + * Neither the name of the CherryPy Team nor the names of its contributors + may be used to endorse or promote products derived from this software + without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/cherrypy/VERSION.txt b/cherrypy/VERSION.txt index 499deb1..621da11 100644 --- a/cherrypy/VERSION.txt +++ b/cherrypy/VERSION.txt @@ -1,4 +1,5 @@ -CherryPy 6.0.2 Official distribution: https://pypi.python.org/packages/source/C/CherryPy/CherryPy-6.0.2.tar.gz +CherryPy 8.1.0 +Official distribution: https://github.com/cherrypy/cherrypy/releases The folders 'tutorial', 'test' and 'scaffold' have been removed. This file has been added. diff --git a/cherrypy/__init__.py b/cherrypy/__init__.py index ae54838..c85a8cf 100644 --- a/cherrypy/__init__.py +++ b/cherrypy/__init__.py @@ -61,23 +61,26 @@ try: except ImportError: pass -from cherrypy._cperror import HTTPError, HTTPRedirect, InternalRedirect -from cherrypy._cperror import NotFound, CherryPyException, TimeoutError +from threading import local as _local -from cherrypy import _cpdispatch as dispatch +from cherrypy._cperror import HTTPError, HTTPRedirect, InternalRedirect # noqa +from cherrypy._cperror import NotFound, CherryPyException, TimeoutError # noqa + +from cherrypy import _cplogging + +from cherrypy import _cpdispatch as dispatch # noqa from cherrypy import _cptools -tools = _cptools.default_toolbox -Tool = _cptools.Tool +from cherrypy._cptools import default_toolbox as tools, Tool from cherrypy import _cprequest from cherrypy.lib import httputil as _httputil from cherrypy import _cptree -tree = _cptree.Tree() -from cherrypy._cptree import Application -from cherrypy import _cpwsgi as wsgi +from cherrypy._cptree import Application # noqa +from cherrypy import _cpwsgi as wsgi # noqa +from cherrypy import _cpserver from cherrypy import process try: from cherrypy.process import win32 @@ -88,11 +91,11 @@ except ImportError: engine = process.bus -try: - __version__ = pkg_resources.require('cherrypy')[0].version -except Exception: - __version__ = 'unknown' -__version__ = '6.0.2' +tree = _cptree.Tree() + + +__version__ = '8.1.2' + # Timeout monitor. We add two channels to the engine # to which cherrypy.Application will publish. @@ -141,20 +144,19 @@ class _HandleSignalsPlugin(object): def subscribe(self): """Add the handlers based on the platform""" - if hasattr(self.bus, "signal_handler"): + if hasattr(self.bus, 'signal_handler'): self.bus.signal_handler.subscribe() - if hasattr(self.bus, "console_control_handler"): + if hasattr(self.bus, 'console_control_handler'): self.bus.console_control_handler.subscribe() engine.signals = _HandleSignalsPlugin(engine) -from cherrypy import _cpserver server = _cpserver.Server() server.subscribe() -def quickstart(root=None, script_name="", config=None): +def quickstart(root=None, script_name='', config=None): """Mount the given root, start the builtin server (and engine), then block. root: an instance of a "controller class" (a collection of page handler @@ -181,9 +183,6 @@ def quickstart(root=None, script_name="", config=None): engine.block() -from cherrypy._cpcompat import threadlocal as _local - - class _Serving(_local): """An interface for registering request and response objects. @@ -196,8 +195,8 @@ class _Serving(_local): thread-safe way. """ - request = _cprequest.Request(_httputil.Host("127.0.0.1", 80), - _httputil.Host("127.0.0.1", 1111)) + request = _cprequest.Request(_httputil.Host('127.0.0.1', 80), + _httputil.Host('127.0.0.1', 1111)) """ The request object for the current thread. In the main thread, and any threads which are not receiving HTTP requests, this is None.""" @@ -230,7 +229,7 @@ class _ThreadLocalProxy(object): return getattr(child, name) def __setattr__(self, name, value): - if name in ("__attrname__", ): + if name in ('__attrname__', ): object.__setattr__(self, name, value) else: child = getattr(serving, self.__attrname__) @@ -306,9 +305,6 @@ except ImportError: pass -from cherrypy import _cplogging - - class _GlobalLogManager(_cplogging.LogManager): """A site-wide LogManager; routes to app.log or global log as appropriate. @@ -352,10 +348,10 @@ def _buslog(msg, level): log.error(msg, 'ENGINE', severity=level) engine.subscribe('log', _buslog) -from cherrypy._helper import expose, popargs, url +from cherrypy._helper import expose, popargs, url # noqa # import _cpconfig last so it can reference other top-level objects -from cherrypy import _cpconfig +from cherrypy import _cpconfig # noqa # Use _global_conf_alias so quickstart can use 'config' as an arg # without shadowing cherrypy.config. config = _global_conf_alias = _cpconfig.Config() @@ -365,11 +361,11 @@ config.defaults = { 'tools.trailing_slash.on': True, 'tools.encode.on': True } -config.namespaces["log"] = lambda k, v: setattr(log, k, v) -config.namespaces["checker"] = lambda k, v: setattr(checker, k, v) +config.namespaces['log'] = lambda k, v: setattr(log, k, v) +config.namespaces['checker'] = lambda k, v: setattr(checker, k, v) # Must reset to get our defaults applied. config.reset() -from cherrypy import _cpchecker +from cherrypy import _cpchecker # noqa checker = _cpchecker.Checker() engine.subscribe('start', checker) diff --git a/cherrypy/_cpchecker.py b/cherrypy/_cpchecker.py index 4ef8259..d67f9ad 100644 --- a/cherrypy/_cpchecker.py +++ b/cherrypy/_cpchecker.py @@ -33,7 +33,7 @@ class Checker(object): warnings.formatwarning = self.formatwarning try: for name in dir(self): - if name.startswith("check_"): + if name.startswith('check_'): method = getattr(self, name) if method and hasattr(method, '__call__'): method() @@ -42,7 +42,7 @@ class Checker(object): def formatwarning(self, message, category, filename, lineno, line=None): """Function to format a warning.""" - return "CherryPy Checker:\n%s\n\n" % message + return 'CherryPy Checker:\n%s\n\n' % message # This value should be set inside _cpconfig. global_config_contained_paths = False @@ -57,13 +57,13 @@ class Checker(object): continue if sn == '': continue - sn_atoms = sn.strip("/").split("/") + sn_atoms = sn.strip('/').split('/') for key in app.config.keys(): - key_atoms = key.strip("/").split("/") + key_atoms = key.strip('/').split('/') if key_atoms[:len(sn_atoms)] == sn_atoms: warnings.warn( - "The application mounted at %r has config " - "entries that start with its script name: %r" % (sn, + 'The application mounted at %r has config ' + 'entries that start with its script name: %r' % (sn, key)) def check_site_config_entries_in_app_config(self): @@ -76,17 +76,17 @@ class Checker(object): for section, entries in iteritems(app.config): if section.startswith('/'): for key, value in iteritems(entries): - for n in ("engine.", "server.", "tree.", "checker."): + for n in ('engine.', 'server.', 'tree.', 'checker.'): if key.startswith(n): - msg.append("[%s] %s = %s" % + msg.append('[%s] %s = %s' % (section, key, value)) if msg: msg.insert(0, - "The application mounted at %r contains the " - "following config entries, which are only allowed " - "in site-wide config. Move them to a [global] " - "section and pass them to cherrypy.config.update() " - "instead of tree.mount()." % sn) + 'The application mounted at %r contains the ' + 'following config entries, which are only allowed ' + 'in site-wide config. Move them to a [global] ' + 'section and pass them to cherrypy.config.update() ' + 'instead of tree.mount().' % sn) warnings.warn(os.linesep.join(msg)) def check_skipped_app_config(self): @@ -95,13 +95,13 @@ class Checker(object): if not isinstance(app, cherrypy.Application): continue if not app.config: - msg = "The Application mounted at %r has an empty config." % sn + msg = 'The Application mounted at %r has an empty config.' % sn if self.global_config_contained_paths: - msg += (" It looks like the config you passed to " - "cherrypy.config.update() contains application-" - "specific sections. You must explicitly pass " - "application config via " - "cherrypy.tree.mount(..., config=app_config)") + msg += (' It looks like the config you passed to ' + 'cherrypy.config.update() contains application-' + 'specific sections. You must explicitly pass ' + 'application config via ' + 'cherrypy.tree.mount(..., config=app_config)') warnings.warn(msg) return @@ -115,12 +115,12 @@ class Checker(object): if not app.config: continue for key in app.config.keys(): - if key.startswith("[") or key.endswith("]"): + if key.startswith('[') or key.endswith(']'): warnings.warn( - "The application mounted at %r has config " - "section names with extraneous brackets: %r. " - "Config *files* need brackets; config *dicts* " - "(e.g. passed to tree.mount) do not." % (sn, key)) + 'The application mounted at %r has config ' + 'section names with extraneous brackets: %r. ' + 'Config *files* need brackets; config *dicts* ' + '(e.g. passed to tree.mount) do not.' % (sn, key)) def check_static_paths(self): """Check Application config for incorrect static paths.""" @@ -132,47 +132,47 @@ class Checker(object): request.app = app for section in app.config: # get_resource will populate request.config - request.get_resource(section + "/dummy.html") + request.get_resource(section + '/dummy.html') conf = request.config.get - if conf("tools.staticdir.on", False): - msg = "" - root = conf("tools.staticdir.root") - dir = conf("tools.staticdir.dir") + if conf('tools.staticdir.on', False): + msg = '' + root = conf('tools.staticdir.root') + dir = conf('tools.staticdir.dir') if dir is None: - msg = "tools.staticdir.dir is not set." + msg = 'tools.staticdir.dir is not set.' else: - fulldir = "" + fulldir = '' if os.path.isabs(dir): fulldir = dir if root: - msg = ("dir is an absolute path, even " - "though a root is provided.") + msg = ('dir is an absolute path, even ' + 'though a root is provided.') testdir = os.path.join(root, dir[1:]) if os.path.exists(testdir): msg += ( - "\nIf you meant to serve the " - "filesystem folder at %r, remove the " - "leading slash from dir." % (testdir,)) + '\nIf you meant to serve the ' + 'filesystem folder at %r, remove the ' + 'leading slash from dir.' % (testdir,)) else: if not root: msg = ( - "dir is a relative path and " - "no root provided.") + 'dir is a relative path and ' + 'no root provided.') else: fulldir = os.path.join(root, dir) if not os.path.isabs(fulldir): - msg = ("%r is not an absolute path." % ( + msg = ('%r is not an absolute path.' % ( fulldir,)) if fulldir and not os.path.exists(fulldir): if msg: - msg += "\n" - msg += ("%r (root + dir) is not an existing " - "filesystem path." % fulldir) + msg += '\n' + msg += ('%r (root + dir) is not an existing ' + 'filesystem path.' % fulldir) if msg: - warnings.warn("%s\nsection: [%s]\nroot: %r\ndir: %r" + warnings.warn('%s\nsection: [%s]\nroot: %r\ndir: %r' % (msg, section, root, dir)) # -------------------------- Compatibility -------------------------- # @@ -198,19 +198,19 @@ class Checker(object): if isinstance(conf, dict): for k, v in conf.items(): if k in self.obsolete: - warnings.warn("%r is obsolete. Use %r instead.\n" - "section: [%s]" % + warnings.warn('%r is obsolete. Use %r instead.\n' + 'section: [%s]' % (k, self.obsolete[k], section)) elif k in self.deprecated: - warnings.warn("%r is deprecated. Use %r instead.\n" - "section: [%s]" % + warnings.warn('%r is deprecated. Use %r instead.\n' + 'section: [%s]' % (k, self.deprecated[k], section)) else: if section in self.obsolete: - warnings.warn("%r is obsolete. Use %r instead." + warnings.warn('%r is obsolete. Use %r instead.' % (section, self.obsolete[section])) elif section in self.deprecated: - warnings.warn("%r is deprecated. Use %r instead." + warnings.warn('%r is deprecated. Use %r instead.' % (section, self.deprecated[section])) def check_compatibility(self): @@ -225,7 +225,7 @@ class Checker(object): extra_config_namespaces = [] def _known_ns(self, app): - ns = ["wsgi"] + ns = ['wsgi'] ns.extend(copykeys(app.toolboxes)) ns.extend(copykeys(app.namespaces)) ns.extend(copykeys(app.request_class.namespaces)) @@ -233,32 +233,32 @@ class Checker(object): ns += self.extra_config_namespaces for section, conf in app.config.items(): - is_path_section = section.startswith("/") + is_path_section = section.startswith('/') if is_path_section and isinstance(conf, dict): for k, v in conf.items(): - atoms = k.split(".") + atoms = k.split('.') if len(atoms) > 1: if atoms[0] not in ns: # Spit out a special warning if a known # namespace is preceded by "cherrypy." - if atoms[0] == "cherrypy" and atoms[1] in ns: + if atoms[0] == 'cherrypy' and atoms[1] in ns: msg = ( - "The config entry %r is invalid; " - "try %r instead.\nsection: [%s]" - % (k, ".".join(atoms[1:]), section)) + 'The config entry %r is invalid; ' + 'try %r instead.\nsection: [%s]' + % (k, '.'.join(atoms[1:]), section)) else: msg = ( - "The config entry %r is invalid, " - "because the %r config namespace " - "is unknown.\n" - "section: [%s]" % (k, atoms[0], section)) + 'The config entry %r is invalid, ' + 'because the %r config namespace ' + 'is unknown.\n' + 'section: [%s]' % (k, atoms[0], section)) warnings.warn(msg) - elif atoms[0] == "tools": + elif atoms[0] == 'tools': if atoms[1] not in dir(cherrypy.tools): msg = ( - "The config entry %r may be invalid, " - "because the %r tool was not found.\n" - "section: [%s]" % (k, atoms[1], section)) + 'The config entry %r may be invalid, ' + 'because the %r tool was not found.\n' + 'section: [%s]' % (k, atoms[1], section)) warnings.warn(msg) def check_config_namespaces(self): @@ -282,17 +282,17 @@ class Checker(object): continue vtype = type(getattr(obj, name, None)) if vtype in b: - self.known_config_types[namespace + "." + name] = vtype + self.known_config_types[namespace + '.' + name] = vtype - traverse(cherrypy.request, "request") - traverse(cherrypy.response, "response") - traverse(cherrypy.server, "server") - traverse(cherrypy.engine, "engine") - traverse(cherrypy.log, "log") + traverse(cherrypy.request, 'request') + traverse(cherrypy.response, 'response') + traverse(cherrypy.server, 'server') + traverse(cherrypy.engine, 'engine') + traverse(cherrypy.log, 'log') def _known_types(self, config): - msg = ("The config entry %r in section %r is of type %r, " - "which does not match the expected type %r.") + msg = ('The config entry %r in section %r is of type %r, ' + 'which does not match the expected type %r.') for section, conf in config.items(): if isinstance(conf, dict): @@ -326,7 +326,7 @@ class Checker(object): for k, v in cherrypy.config.items(): if k == 'server.socket_host' and v == 'localhost': warnings.warn("The use of 'localhost' as a socket host can " - "cause problems on newer systems, since " + 'cause problems on newer systems, since ' "'localhost' can map to either an IPv4 or an " "IPv6 address. You should use '127.0.0.1' " "or '[::1]' instead.") diff --git a/cherrypy/_cpcompat.py b/cherrypy/_cpcompat.py index dc03331..9dd1a2f 100644 --- a/cherrypy/_cpcompat.py +++ b/cherrypy/_cpcompat.py @@ -1,6 +1,6 @@ """Compatibility code for using CherryPy with various versions of Python. -CherryPy 3.2 is compatible with Python versions 2.3+. This module provides a +CherryPy 3.2 is compatible with Python versions 2.6+. This module provides a useful abstraction over the differences between Python versions, sometimes by preferring a newer idiom, sometimes an older one, and sometimes a custom one. @@ -15,6 +15,8 @@ specifically with bytes, and a 'StringIO' name for dealing with native strings. It also provides a 'base64_decode' function with native strings as input and output. """ + +import binascii import os import re import sys @@ -23,8 +25,6 @@ import threading import six if six.PY3: - basestring = (bytes, str) - def ntob(n, encoding='ISO-8859-1'): """Return the given native string as a byte string in the given encoding. @@ -49,8 +49,6 @@ if six.PY3: return n else: # Python 2 - basestring = basestring - def ntob(n, encoding='ISO-8859-1'): """Return the given native string as a byte string in the given encoding. @@ -90,7 +88,7 @@ else: def assert_native(n): if not isinstance(n, str): - raise TypeError("n must be a native str (got %s)" % type(n).__name__) + raise TypeError('n must be a native str (got %s)' % type(n).__name__) try: # Python 3.1+ @@ -140,16 +138,11 @@ try: from urllib.request import parse_http_list, parse_keqv_list except ImportError: # Python 2 - from urlparse import urljoin - from urllib import urlencode, urlopen - from urllib import quote, quote_plus - from urllib import unquote - from urllib2 import parse_http_list, parse_keqv_list - -try: - from threading import local as threadlocal -except ImportError: - from cherrypy._cpthreadinglocal import local as threadlocal + from urlparse import urljoin # noqa + from urllib import urlencode, urlopen # noqa + from urllib import quote, quote_plus # noqa + from urllib import unquote # noqa + from urllib2 import parse_http_list, parse_keqv_list # noqa try: dict.iteritems @@ -186,7 +179,7 @@ try: import builtins except ImportError: # Python 2 - import __builtin__ as builtins + import __builtin__ as builtins # noqa try: # Python 2. We try Python 2 first clients on Python 2 @@ -197,10 +190,10 @@ try: from BaseHTTPServer import BaseHTTPRequestHandler except ImportError: # Python 3 - from http.cookies import SimpleCookie, CookieError - from http.client import BadStatusLine, HTTPConnection, IncompleteRead - from http.client import NotConnected - from http.server import BaseHTTPRequestHandler + from http.cookies import SimpleCookie, CookieError # noqa + from http.client import BadStatusLine, HTTPConnection, IncompleteRead # noqa + from http.client import NotConnected # noqa + from http.server import BaseHTTPRequestHandler # noqa # Some platforms don't expose HTTPSConnection, so handle it separately if six.PY3: @@ -222,29 +215,6 @@ except NameError: # Python 3 xrange = range -import threading -if hasattr(threading.Thread, "daemon"): - # Python 2.6+ - def get_daemon(t): - return t.daemon - - def set_daemon(t, val): - t.daemon = val -else: - def get_daemon(t): - return t.isDaemon() - - def set_daemon(t, val): - t.setDaemon(val) - -try: - from email.utils import formatdate - - def HTTPDate(timeval=None): - return formatdate(timeval, usegmt=True) -except ImportError: - from rfc822 import formatdate as HTTPDate - try: # Python 3 from urllib.parse import unquote as parse_unquote @@ -291,15 +261,14 @@ finally: else: json_encode = _json_encode +text_or_bytes = six.text_type, six.binary_type try: import cPickle as pickle except ImportError: # In Python 2, pickle is a Python version. # In Python 3, pickle is the sped-up C version. - import pickle - -import binascii + import pickle # noqa def random20(): return binascii.hexlify(os.urandom(20)).decode('ascii') @@ -307,7 +276,7 @@ def random20(): try: from _thread import get_ident as get_thread_ident except ImportError: - from thread import get_ident as get_thread_ident + from thread import get_ident as get_thread_ident # noqa try: # Python 3 @@ -325,17 +294,41 @@ else: Timer = threading._Timer Event = threading._Event -# Prior to Python 2.6, the Thread class did not have a .daemon property. -# This mix-in adds that property. - - -class SetDaemonProperty: - - def __get_daemon(self): - return self.isDaemon() - - def __set_daemon(self, daemon): - self.setDaemon(daemon) +try: + # Python 2.7+ + from subprocess import _args_from_interpreter_flags +except ImportError: + def _args_from_interpreter_flags(): + """Tries to reconstruct original interpreter args from sys.flags for Python 2.6 - if sys.version_info < (2, 6): - daemon = property(__get_daemon, __set_daemon) + Backported from Python 3.5. Aims to return a list of + command-line arguments reproducing the current + settings in sys.flags and sys.warnoptions. + """ + flag_opt_map = { + 'debug': 'd', + # 'inspect': 'i', + # 'interactive': 'i', + 'optimize': 'O', + 'dont_write_bytecode': 'B', + 'no_user_site': 's', + 'no_site': 'S', + 'ignore_environment': 'E', + 'verbose': 'v', + 'bytes_warning': 'b', + 'quiet': 'q', + 'hash_randomization': 'R', + 'py3k_warning': '3', + } + + args = [] + for flag, opt in flag_opt_map.items(): + v = getattr(sys.flags, flag) + if v > 0: + if flag == 'hash_randomization': + v = 1 # Handle specification of an exact seed + args.append('-' + opt * v) + for opt in sys.warnoptions: + args.append('-W' + opt) + + return args diff --git a/cherrypy/_cpcompat_subprocess.py b/cherrypy/_cpcompat_subprocess.py deleted file mode 100644 index 517b8d4..0000000 --- a/cherrypy/_cpcompat_subprocess.py +++ /dev/null @@ -1,1544 +0,0 @@ -# subprocess - Subprocesses with accessible I/O streams -# -# For more information about this module, see PEP 324. -# -# This module should remain compatible with Python 2.2, see PEP 291. -# -# Copyright (c) 2003-2005 by Peter Astrand -# -# Licensed to PSF under a Contributor Agreement. -# See http://www.python.org/2.4/license for licensing details. - -r"""subprocess - Subprocesses with accessible I/O streams - -This module allows you to spawn processes, connect to their -input/output/error pipes, and obtain their return codes. This module -intends to replace several other, older modules and functions, like: - -os.system -os.spawn* -os.popen* -popen2.* -commands.* - -Information about how the subprocess module can be used to replace these -modules and functions can be found below. - - - -Using the subprocess module -=========================== -This module defines one class called Popen: - -class Popen(args, bufsize=0, executable=None, - stdin=None, stdout=None, stderr=None, - preexec_fn=None, close_fds=False, shell=False, - cwd=None, env=None, universal_newlines=False, - startupinfo=None, creationflags=0): - - -Arguments are: - -args should be a string, or a sequence of program arguments. The -program to execute is normally the first item in the args sequence or -string, but can be explicitly set by using the executable argument. - -On UNIX, with shell=False (default): In this case, the Popen class -uses os.execvp() to execute the child program. args should normally -be a sequence. A string will be treated as a sequence with the string -as the only item (the program to execute). - -On UNIX, with shell=True: If args is a string, it specifies the -command string to execute through the shell. If args is a sequence, -the first item specifies the command string, and any additional items -will be treated as additional shell arguments. - -On Windows: the Popen class uses CreateProcess() to execute the child -program, which operates on strings. If args is a sequence, it will be -converted to a string using the list2cmdline method. Please note that -not all MS Windows applications interpret the command line the same -way: The list2cmdline is designed for applications using the same -rules as the MS C runtime. - -bufsize, if given, has the same meaning as the corresponding argument -to the built-in open() function: 0 means unbuffered, 1 means line -buffered, any other positive value means use a buffer of -(approximately) that size. A negative bufsize means to use the system -default, which usually means fully buffered. The default value for -bufsize is 0 (unbuffered). - -stdin, stdout and stderr specify the executed programs' standard -input, standard output and standard error file handles, respectively. -Valid values are PIPE, an existing file descriptor (a positive -integer), an existing file object, and None. PIPE indicates that a -new pipe to the child should be created. With None, no redirection -will occur; the child's file handles will be inherited from the -parent. Additionally, stderr can be STDOUT, which indicates that the -stderr data from the applications should be captured into the same -file handle as for stdout. - -If preexec_fn is set to a callable object, this object will be called -in the child process just before the child is executed. - -If close_fds is true, all file descriptors except 0, 1 and 2 will be -closed before the child process is executed. - -if shell is true, the specified command will be executed through the -shell. - -If cwd is not None, the current directory will be changed to cwd -before the child is executed. - -If env is not None, it defines the environment variables for the new -process. - -If universal_newlines is true, the file objects stdout and stderr are -opened as a text files, but lines may be terminated by any of '\n', -the Unix end-of-line convention, '\r', the Macintosh convention or -'\r\n', the Windows convention. All of these external representations -are seen as '\n' by the Python program. Note: This feature is only -available if Python is built with universal newline support (the -default). Also, the newlines attribute of the file objects stdout, -stdin and stderr are not updated by the communicate() method. - -The startupinfo and creationflags, if given, will be passed to the -underlying CreateProcess() function. They can specify things such as -appearance of the main window and priority for the new process. -(Windows only) - - -This module also defines some shortcut functions: - -call(*popenargs, **kwargs): - Run command with arguments. Wait for command to complete, then - return the returncode attribute. - - The arguments are the same as for the Popen constructor. Example: - - retcode = call(["ls", "-l"]) - -check_call(*popenargs, **kwargs): - Run command with arguments. Wait for command to complete. If the - exit code was zero then return, otherwise raise - CalledProcessError. The CalledProcessError object will have the - return code in the returncode attribute. - - The arguments are the same as for the Popen constructor. Example: - - check_call(["ls", "-l"]) - -check_output(*popenargs, **kwargs): - Run command with arguments and return its output as a byte string. - - If the exit code was non-zero it raises a CalledProcessError. The - CalledProcessError object will have the return code in the returncode - attribute and output in the output attribute. - - The arguments are the same as for the Popen constructor. Example: - - output = check_output(["ls", "-l", "/dev/null"]) - - -Exceptions ----------- -Exceptions raised in the child process, before the new program has -started to execute, will be re-raised in the parent. Additionally, -the exception object will have one extra attribute called -'child_traceback', which is a string containing traceback information -from the childs point of view. - -The most common exception raised is OSError. This occurs, for -example, when trying to execute a non-existent file. Applications -should prepare for OSErrors. - -A ValueError will be raised if Popen is called with invalid arguments. - -check_call() and check_output() will raise CalledProcessError, if the -called process returns a non-zero return code. - - -Security --------- -Unlike some other popen functions, this implementation will never call -/bin/sh implicitly. This means that all characters, including shell -metacharacters, can safely be passed to child processes. - - -Popen objects -============= -Instances of the Popen class have the following methods: - -poll() - Check if child process has terminated. Returns returncode - attribute. - -wait() - Wait for child process to terminate. Returns returncode attribute. - -communicate(input=None) - Interact with process: Send data to stdin. Read data from stdout - and stderr, until end-of-file is reached. Wait for process to - terminate. The optional input argument should be a string to be - sent to the child process, or None, if no data should be sent to - the child. - - communicate() returns a tuple (stdout, stderr). - - Note: The data read is buffered in memory, so do not use this - method if the data size is large or unlimited. - -The following attributes are also available: - -stdin - If the stdin argument is PIPE, this attribute is a file object - that provides input to the child process. Otherwise, it is None. - -stdout - If the stdout argument is PIPE, this attribute is a file object - that provides output from the child process. Otherwise, it is - None. - -stderr - If the stderr argument is PIPE, this attribute is file object that - provides error output from the child process. Otherwise, it is - None. - -pid - The process ID of the child process. - -returncode - The child return code. A None value indicates that the process - hasn't terminated yet. A negative value -N indicates that the - child was terminated by signal N (UNIX only). - - -Replacing older functions with the subprocess module -==================================================== -In this section, "a ==> b" means that b can be used as a replacement -for a. - -Note: All functions in this section fail (more or less) silently if -the executed program cannot be found; this module raises an OSError -exception. - -In the following examples, we assume that the subprocess module is -imported with "from subprocess import *". - - -Replacing /bin/sh shell backquote ---------------------------------- -output=`mycmd myarg` -==> -output = Popen(["mycmd", "myarg"], stdout=PIPE).communicate()[0] - - -Replacing shell pipe line -------------------------- -output=`dmesg | grep hda` -==> -p1 = Popen(["dmesg"], stdout=PIPE) -p2 = Popen(["grep", "hda"], stdin=p1.stdout, stdout=PIPE) -output = p2.communicate()[0] - - -Replacing os.system() ---------------------- -sts = os.system("mycmd" + " myarg") -==> -p = Popen("mycmd" + " myarg", shell=True) -pid, sts = os.waitpid(p.pid, 0) - -Note: - -* Calling the program through the shell is usually not required. - -* It's easier to look at the returncode attribute than the - exitstatus. - -A more real-world example would look like this: - -try: - retcode = call("mycmd" + " myarg", shell=True) - if retcode < 0: - print >>sys.stderr, "Child was terminated by signal", -retcode - else: - print >>sys.stderr, "Child returned", retcode -except OSError, e: - print >>sys.stderr, "Execution failed:", e - - -Replacing os.spawn* -------------------- -P_NOWAIT example: - -pid = os.spawnlp(os.P_NOWAIT, "/bin/mycmd", "mycmd", "myarg") -==> -pid = Popen(["/bin/mycmd", "myarg"]).pid - - -P_WAIT example: - -retcode = os.spawnlp(os.P_WAIT, "/bin/mycmd", "mycmd", "myarg") -==> -retcode = call(["/bin/mycmd", "myarg"]) - - -Vector example: - -os.spawnvp(os.P_NOWAIT, path, args) -==> -Popen([path] + args[1:]) - - -Environment example: - -os.spawnlpe(os.P_NOWAIT, "/bin/mycmd", "mycmd", "myarg", env) -==> -Popen(["/bin/mycmd", "myarg"], env={"PATH": "/usr/bin"}) - - -Replacing os.popen* -------------------- -pipe = os.popen("cmd", mode='r', bufsize) -==> -pipe = Popen("cmd", shell=True, bufsize=bufsize, stdout=PIPE).stdout - -pipe = os.popen("cmd", mode='w', bufsize) -==> -pipe = Popen("cmd", shell=True, bufsize=bufsize, stdin=PIPE).stdin - - -(child_stdin, child_stdout) = os.popen2("cmd", mode, bufsize) -==> -p = Popen("cmd", shell=True, bufsize=bufsize, - stdin=PIPE, stdout=PIPE, close_fds=True) -(child_stdin, child_stdout) = (p.stdin, p.stdout) - - -(child_stdin, - child_stdout, - child_stderr) = os.popen3("cmd", mode, bufsize) -==> -p = Popen("cmd", shell=True, bufsize=bufsize, - stdin=PIPE, stdout=PIPE, stderr=PIPE, close_fds=True) -(child_stdin, - child_stdout, - child_stderr) = (p.stdin, p.stdout, p.stderr) - - -(child_stdin, child_stdout_and_stderr) = os.popen4("cmd", mode, - bufsize) -==> -p = Popen("cmd", shell=True, bufsize=bufsize, - stdin=PIPE, stdout=PIPE, stderr=STDOUT, close_fds=True) -(child_stdin, child_stdout_and_stderr) = (p.stdin, p.stdout) - -On Unix, os.popen2, os.popen3 and os.popen4 also accept a sequence as -the command to execute, in which case arguments will be passed -directly to the program without shell intervention. This usage can be -replaced as follows: - -(child_stdin, child_stdout) = os.popen2(["/bin/ls", "-l"], mode, - bufsize) -==> -p = Popen(["/bin/ls", "-l"], bufsize=bufsize, stdin=PIPE, stdout=PIPE) -(child_stdin, child_stdout) = (p.stdin, p.stdout) - -Return code handling translates as follows: - -pipe = os.popen("cmd", 'w') -... -rc = pipe.close() -if rc is not None and rc % 256: - print "There were some errors" -==> -process = Popen("cmd", 'w', shell=True, stdin=PIPE) -... -process.stdin.close() -if process.wait() != 0: - print "There were some errors" - - -Replacing popen2.* ------------------- -(child_stdout, child_stdin) = popen2.popen2("somestring", bufsize, mode) -==> -p = Popen(["somestring"], shell=True, bufsize=bufsize - stdin=PIPE, stdout=PIPE, close_fds=True) -(child_stdout, child_stdin) = (p.stdout, p.stdin) - -On Unix, popen2 also accepts a sequence as the command to execute, in -which case arguments will be passed directly to the program without -shell intervention. This usage can be replaced as follows: - -(child_stdout, child_stdin) = popen2.popen2(["mycmd", "myarg"], bufsize, - mode) -==> -p = Popen(["mycmd", "myarg"], bufsize=bufsize, - stdin=PIPE, stdout=PIPE, close_fds=True) -(child_stdout, child_stdin) = (p.stdout, p.stdin) - -The popen2.Popen3 and popen2.Popen4 basically works as subprocess.Popen, -except that: - -* subprocess.Popen raises an exception if the execution fails -* the capturestderr argument is replaced with the stderr argument. -* stdin=PIPE and stdout=PIPE must be specified. -* popen2 closes all filedescriptors by default, but you have to specify - close_fds=True with subprocess.Popen. -""" - -import sys -mswindows = (sys.platform == "win32") - -import os -import types -import traceback -import gc -import signal -import errno - -try: - set -except NameError: - from sets import Set as set - -# Exception classes used by this module. - - -class CalledProcessError(Exception): - - """This exception is raised when a process run by check_call() or - check_output() returns a non-zero exit status. - The exit status will be stored in the returncode attribute; - check_output() will also store the output in the output attribute. - """ - - def __init__(self, returncode, cmd, output=None): - self.returncode = returncode - self.cmd = cmd - self.output = output - - def __str__(self): - return "Command '%s' returned non-zero exit status %d" % ( - self.cmd, self.returncode) - - -if mswindows: - import threading - import msvcrt - import _subprocess - - class STARTUPINFO: - dwFlags = 0 - hStdInput = None - hStdOutput = None - hStdError = None - wShowWindow = 0 - - class pywintypes: - error = IOError -else: - import select - _has_poll = hasattr(select, 'poll') - import fcntl - import pickle - - # When select or poll has indicated that the file is writable, - # we can write up to _PIPE_BUF bytes without risk of blocking. - # POSIX defines PIPE_BUF as >= 512. - _PIPE_BUF = getattr(select, 'PIPE_BUF', 512) - - -__all__ = ["Popen", "PIPE", "STDOUT", "call", "check_call", - "check_output", "CalledProcessError"] - -if mswindows: - from _subprocess import CREATE_NEW_CONSOLE, CREATE_NEW_PROCESS_GROUP, \ - STD_INPUT_HANDLE, STD_OUTPUT_HANDLE, \ - STD_ERROR_HANDLE, SW_HIDE, \ - STARTF_USESTDHANDLES, STARTF_USESHOWWINDOW - - __all__.extend(["CREATE_NEW_CONSOLE", "CREATE_NEW_PROCESS_GROUP", - "STD_INPUT_HANDLE", "STD_OUTPUT_HANDLE", - "STD_ERROR_HANDLE", "SW_HIDE", - "STARTF_USESTDHANDLES", "STARTF_USESHOWWINDOW"]) -try: - MAXFD = os.sysconf("SC_OPEN_MAX") -except: - MAXFD = 256 - -_active = [] - - -def _cleanup(): - for inst in _active[:]: - res = inst._internal_poll(_deadstate=sys.maxint) - if res is not None: - try: - _active.remove(inst) - except ValueError: - # This can happen if two threads create a new Popen instance. - # It's harmless that it was already removed, so ignore. - pass - -PIPE = -1 -STDOUT = -2 - - -def _eintr_retry_call(func, *args): - while True: - try: - return func(*args) - except (OSError, IOError), e: - if e.errno == errno.EINTR: - continue - raise - - -def call(*popenargs, **kwargs): - """Run command with arguments. Wait for command to complete, then - return the returncode attribute. - - The arguments are the same as for the Popen constructor. Example: - - retcode = call(["ls", "-l"]) - """ - return Popen(*popenargs, **kwargs).wait() - - -def check_call(*popenargs, **kwargs): - """Run command with arguments. Wait for command to complete. If - the exit code was zero then return, otherwise raise - CalledProcessError. The CalledProcessError object will have the - return code in the returncode attribute. - - The arguments are the same as for the Popen constructor. Example: - - check_call(["ls", "-l"]) - """ - retcode = call(*popenargs, **kwargs) - if retcode: - cmd = kwargs.get("args") - if cmd is None: - cmd = popenargs[0] - raise CalledProcessError(retcode, cmd) - return 0 - - -def check_output(*popenargs, **kwargs): - r"""Run command with arguments and return its output as a byte string. - - If the exit code was non-zero it raises a CalledProcessError. The - CalledProcessError object will have the return code in the returncode - attribute and output in the output attribute. - - The arguments are the same as for the Popen constructor. Example: - - >>> check_output(["ls", "-l", "/dev/null"]) - 'crw-rw-rw- 1 root root 1, 3 Oct 18 2007 /dev/null\n' - - The stdout argument is not allowed as it is used internally. - To capture standard error in the result, use stderr=STDOUT. - - >>> check_output(["/bin/sh", "-c", - ... "ls -l non_existent_file ; exit 0"], - ... stderr=STDOUT) - 'ls: non_existent_file: No such file or directory\n' - """ - if 'stdout' in kwargs: - raise ValueError('stdout argument not allowed, it will be overridden.') - process = Popen(stdout=PIPE, *popenargs, **kwargs) - output, unused_err = process.communicate() - retcode = process.poll() - if retcode: - cmd = kwargs.get("args") - if cmd is None: - cmd = popenargs[0] - raise CalledProcessError(retcode, cmd, output=output) - return output - - -def list2cmdline(seq): - """ - Translate a sequence of arguments into a command line - string, using the same rules as the MS C runtime: - - 1) Arguments are delimited by white space, which is either a - space or a tab. - - 2) A string surrounded by double quotation marks is - interpreted as a single argument, regardless of white space - contained within. A quoted string can be embedded in an - argument. - - 3) A double quotation mark preceded by a backslash is - interpreted as a literal double quotation mark. - - 4) Backslashes are interpreted literally, unless they - immediately precede a double quotation mark. - - 5) If backslashes immediately precede a double quotation mark, - every pair of backslashes is interpreted as a literal - backslash. If the number of backslashes is odd, the last - backslash escapes the next double quotation mark as - described in rule 3. - """ - - # See - # http://msdn.microsoft.com/en-us/library/17w5ykft.aspx - # or search http://msdn.microsoft.com for - # "Parsing C++ Command-Line Arguments" - result = [] - needquote = False - for arg in seq: - bs_buf = [] - - # Add a space to separate this argument from the others - if result: - result.append(' ') - - needquote = (" " in arg) or ("\t" in arg) or not arg - if needquote: - result.append('"') - - for c in arg: - if c == '\\': - # Don't know if we need to double yet. - bs_buf.append(c) - elif c == '"': - # Double backslashes. - result.append('\\' * len(bs_buf) * 2) - bs_buf = [] - result.append('\\"') - else: - # Normal char - if bs_buf: - result.extend(bs_buf) - bs_buf = [] - result.append(c) - - # Add remaining backslashes, if any. - if bs_buf: - result.extend(bs_buf) - - if needquote: - result.extend(bs_buf) - result.append('"') - - return ''.join(result) - - -class Popen(object): - - def __init__(self, args, bufsize=0, executable=None, - stdin=None, stdout=None, stderr=None, - preexec_fn=None, close_fds=False, shell=False, - cwd=None, env=None, universal_newlines=False, - startupinfo=None, creationflags=0): - """Create new Popen instance.""" - _cleanup() - - self._child_created = False - if not isinstance(bufsize, (int, long)): - raise TypeError("bufsize must be an integer") - - if mswindows: - if preexec_fn is not None: - raise ValueError("preexec_fn is not supported on Windows " - "platforms") - if close_fds and (stdin is not None or stdout is not None or - stderr is not None): - raise ValueError("close_fds is not supported on Windows " - "platforms if you redirect " - "stdin/stdout/stderr") - else: - # POSIX - if startupinfo is not None: - raise ValueError("startupinfo is only supported on Windows " - "platforms") - if creationflags != 0: - raise ValueError("creationflags is only supported on Windows " - "platforms") - - self.stdin = None - self.stdout = None - self.stderr = None - self.pid = None - self.returncode = None - self.universal_newlines = universal_newlines - - # Input and output objects. The general principle is like - # this: - # - # Parent Child - # ------ ----- - # p2cwrite ---stdin---> p2cread - # c2pread <--stdout--- c2pwrite - # errread <--stderr--- errwrite - # - # On POSIX, the child objects are file descriptors. On - # Windows, these are Windows file handles. The parent objects - # are file descriptors on both platforms. The parent objects - # are None when not using PIPEs. The child objects are None - # when not redirecting. - - (p2cread, p2cwrite, - c2pread, c2pwrite, - errread, errwrite) = self._get_handles(stdin, stdout, stderr) - - self._execute_child(args, executable, preexec_fn, close_fds, - cwd, env, universal_newlines, - startupinfo, creationflags, shell, - p2cread, p2cwrite, - c2pread, c2pwrite, - errread, errwrite) - - if mswindows: - if p2cwrite is not None: - p2cwrite = msvcrt.open_osfhandle(p2cwrite.Detach(), 0) - if c2pread is not None: - c2pread = msvcrt.open_osfhandle(c2pread.Detach(), 0) - if errread is not None: - errread = msvcrt.open_osfhandle(errread.Detach(), 0) - - if p2cwrite is not None: - self.stdin = os.fdopen(p2cwrite, 'wb', bufsize) - if c2pread is not None: - if universal_newlines: - self.stdout = os.fdopen(c2pread, 'rU', bufsize) - else: - self.stdout = os.fdopen(c2pread, 'rb', bufsize) - if errread is not None: - if universal_newlines: - self.stderr = os.fdopen(errread, 'rU', bufsize) - else: - self.stderr = os.fdopen(errread, 'rb', bufsize) - - def _translate_newlines(self, data): - data = data.replace("\r\n", "\n") - data = data.replace("\r", "\n") - return data - - def __del__(self, _maxint=sys.maxint, _active=_active): - # If __init__ hasn't had a chance to execute (e.g. if it - # was passed an undeclared keyword argument), we don't - # have a _child_created attribute at all. - if not getattr(self, '_child_created', False): - # We didn't get to successfully create a child process. - return - # In case the child hasn't been waited on, check if it's done. - self._internal_poll(_deadstate=_maxint) - if self.returncode is None and _active is not None: - # Child is still running, keep us alive until we can wait on it. - _active.append(self) - - def communicate(self, input=None): - """Interact with process: Send data to stdin. Read data from - stdout and stderr, until end-of-file is reached. Wait for - process to terminate. The optional input argument should be a - string to be sent to the child process, or None, if no data - should be sent to the child. - - communicate() returns a tuple (stdout, stderr).""" - - # Optimization: If we are only using one pipe, or no pipe at - # all, using select() or threads is unnecessary. - if [self.stdin, self.stdout, self.stderr].count(None) >= 2: - stdout = None - stderr = None - if self.stdin: - if input: - try: - self.stdin.write(input) - except IOError, e: - if e.errno != errno.EPIPE and e.errno != errno.EINVAL: - raise - self.stdin.close() - elif self.stdout: - stdout = _eintr_retry_call(self.stdout.read) - self.stdout.close() - elif self.stderr: - stderr = _eintr_retry_call(self.stderr.read) - self.stderr.close() - self.wait() - return (stdout, stderr) - - return self._communicate(input) - - def poll(self): - return self._internal_poll() - - if mswindows: - # - # Windows methods - # - def _get_handles(self, stdin, stdout, stderr): - """Construct and return tuple with IO objects: - p2cread, p2cwrite, c2pread, c2pwrite, errread, errwrite - """ - if stdin is None and stdout is None and stderr is None: - return (None, None, None, None, None, None) - - p2cread, p2cwrite = None, None - c2pread, c2pwrite = None, None - errread, errwrite = None, None - - if stdin is None: - p2cread = _subprocess.GetStdHandle( - _subprocess.STD_INPUT_HANDLE) - if p2cread is None: - p2cread, _ = _subprocess.CreatePipe(None, 0) - elif stdin == PIPE: - p2cread, p2cwrite = _subprocess.CreatePipe(None, 0) - elif isinstance(stdin, int): - p2cread = msvcrt.get_osfhandle(stdin) - else: - # Assuming file-like object - p2cread = msvcrt.get_osfhandle(stdin.fileno()) - p2cread = self._make_inheritable(p2cread) - - if stdout is None: - c2pwrite = _subprocess.GetStdHandle( - _subprocess.STD_OUTPUT_HANDLE) - if c2pwrite is None: - _, c2pwrite = _subprocess.CreatePipe(None, 0) - elif stdout == PIPE: - c2pread, c2pwrite = _subprocess.CreatePipe(None, 0) - elif isinstance(stdout, int): - c2pwrite = msvcrt.get_osfhandle(stdout) - else: - # Assuming file-like object - c2pwrite = msvcrt.get_osfhandle(stdout.fileno()) - c2pwrite = self._make_inheritable(c2pwrite) - - if stderr is None: - errwrite = _subprocess.GetStdHandle( - _subprocess.STD_ERROR_HANDLE) - if errwrite is None: - _, errwrite = _subprocess.CreatePipe(None, 0) - elif stderr == PIPE: - errread, errwrite = _subprocess.CreatePipe(None, 0) - elif stderr == STDOUT: - errwrite = c2pwrite - elif isinstance(stderr, int): - errwrite = msvcrt.get_osfhandle(stderr) - else: - # Assuming file-like object - errwrite = msvcrt.get_osfhandle(stderr.fileno()) - errwrite = self._make_inheritable(errwrite) - - return (p2cread, p2cwrite, - c2pread, c2pwrite, - errread, errwrite) - - def _make_inheritable(self, handle): - """Return a duplicate of handle, which is inheritable""" - return _subprocess.DuplicateHandle( - _subprocess.GetCurrentProcess(), - handle, - _subprocess.GetCurrentProcess(), - 0, - 1, - _subprocess.DUPLICATE_SAME_ACCESS - ) - - def _find_w9xpopen(self): - """Find and return absolut path to w9xpopen.exe""" - w9xpopen = os.path.join( - os.path.dirname(_subprocess.GetModuleFileName(0)), - "w9xpopen.exe") - if not os.path.exists(w9xpopen): - # Eeek - file-not-found - possibly an embedding - # situation - see if we can locate it in sys.exec_prefix - w9xpopen = os.path.join(os.path.dirname(sys.exec_prefix), - "w9xpopen.exe") - if not os.path.exists(w9xpopen): - raise RuntimeError("Cannot locate w9xpopen.exe, which is " - "needed for Popen to work with your " - "shell or platform.") - return w9xpopen - - def _execute_child(self, args, executable, preexec_fn, close_fds, - cwd, env, universal_newlines, - startupinfo, creationflags, shell, - p2cread, p2cwrite, - c2pread, c2pwrite, - errread, errwrite): - """Execute program (MS Windows version)""" - - if not isinstance(args, types.StringTypes): - args = list2cmdline(args) - - # Process startup details - if startupinfo is None: - startupinfo = STARTUPINFO() - if None not in (p2cread, c2pwrite, errwrite): - startupinfo.dwFlags |= _subprocess.STARTF_USESTDHANDLES - startupinfo.hStdInput = p2cread - startupinfo.hStdOutput = c2pwrite - startupinfo.hStdError = errwrite - - if shell: - startupinfo.dwFlags |= _subprocess.STARTF_USESHOWWINDOW - startupinfo.wShowWindow = _subprocess.SW_HIDE - comspec = os.environ.get("COMSPEC", "cmd.exe") - args = '{0} /c "{1}"'.format(comspec, args) - if (_subprocess.GetVersion() >= 0x80000000 or - os.path.basename(comspec).lower() == "command.com"): - # Win9x, or using command.com on NT. We need to - # use the w9xpopen intermediate program. For more - # information, see KB Q150956 - # (http://web.archive.org/web/20011105084002/http://support.microsoft.com/support/kb/articles/Q150/9/56.asp) - w9xpopen = self._find_w9xpopen() - args = '"%s" %s' % (w9xpopen, args) - # Not passing CREATE_NEW_CONSOLE has been known to - # cause random failures on win9x. Specifically a - # dialog: "Your program accessed mem currently in - # use at xxx" and a hopeful warning about the - # stability of your system. Cost is Ctrl+C wont - # kill children. - creationflags |= _subprocess.CREATE_NEW_CONSOLE - - # Start the process - try: - try: - hp, ht, pid, tid = _subprocess.CreateProcess( - executable, args, - # no special - # security - None, None, - int(not close_fds), - creationflags, - env, - cwd, - startupinfo) - except pywintypes.error, e: - # Translate pywintypes.error to WindowsError, which is - # a subclass of OSError. FIXME: We should really - # translate errno using _sys_errlist (or similar), but - # how can this be done from Python? - raise WindowsError(*e.args) - finally: - # Child is launched. Close the parent's copy of those pipe - # handles that only the child should have open. You need - # to make sure that no handles to the write end of the - # output pipe are maintained in this process or else the - # pipe will not close when the child process exits and the - # ReadFile will hang. - if p2cread is not None: - p2cread.Close() - if c2pwrite is not None: - c2pwrite.Close() - if errwrite is not None: - errwrite.Close() - - # Retain the process handle, but close the thread handle - self._child_created = True - self._handle = hp - self.pid = pid - ht.Close() - - def _internal_poll( - self, _deadstate=None, - _WaitForSingleObject=_subprocess.WaitForSingleObject, - _WAIT_OBJECT_0=_subprocess.WAIT_OBJECT_0, - _GetExitCodeProcess=_subprocess.GetExitCodeProcess - ): - """Check if child process has terminated. Returns returncode - attribute. - - This method is called by __del__, so it can only refer to objects - in its local scope. - - """ - if self.returncode is None: - if _WaitForSingleObject(self._handle, 0) == _WAIT_OBJECT_0: - self.returncode = _GetExitCodeProcess(self._handle) - return self.returncode - - def wait(self): - """Wait for child process to terminate. Returns returncode - attribute.""" - if self.returncode is None: - _subprocess.WaitForSingleObject(self._handle, - _subprocess.INFINITE) - self.returncode = _subprocess.GetExitCodeProcess(self._handle) - return self.returncode - - def _readerthread(self, fh, buffer): - buffer.append(fh.read()) - - def _communicate(self, input): - stdout = None # Return - stderr = None # Return - - if self.stdout: - stdout = [] - stdout_thread = threading.Thread(target=self._readerthread, - args=(self.stdout, stdout)) - stdout_thread.setDaemon(True) - stdout_thread.start() - if self.stderr: - stderr = [] - stderr_thread = threading.Thread(target=self._readerthread, - args=(self.stderr, stderr)) - stderr_thread.setDaemon(True) - stderr_thread.start() - - if self.stdin: - if input is not None: - try: - self.stdin.write(input) - except IOError, e: - if e.errno != errno.EPIPE: - raise - self.stdin.close() - - if self.stdout: - stdout_thread.join() - if self.stderr: - stderr_thread.join() - - # All data exchanged. Translate lists into strings. - if stdout is not None: - stdout = stdout[0] - if stderr is not None: - stderr = stderr[0] - - # Translate newlines, if requested. We cannot let the file - # object do the translation: It is based on stdio, which is - # impossible to combine with select (unless forcing no - # buffering). - if self.universal_newlines and hasattr(file, 'newlines'): - if stdout: - stdout = self._translate_newlines(stdout) - if stderr: - stderr = self._translate_newlines(stderr) - - self.wait() - return (stdout, stderr) - - def send_signal(self, sig): - """Send a signal to the process - """ - if sig == signal.SIGTERM: - self.terminate() - elif sig == signal.CTRL_C_EVENT: - os.kill(self.pid, signal.CTRL_C_EVENT) - elif sig == signal.CTRL_BREAK_EVENT: - os.kill(self.pid, signal.CTRL_BREAK_EVENT) - else: - raise ValueError("Unsupported signal: {0}".format(sig)) - - def terminate(self): - """Terminates the process - """ - _subprocess.TerminateProcess(self._handle, 1) - - kill = terminate - - else: - # - # POSIX methods - # - def _get_handles(self, stdin, stdout, stderr): - """Construct and return tuple with IO objects: - p2cread, p2cwrite, c2pread, c2pwrite, errread, errwrite - """ - p2cread, p2cwrite = None, None - c2pread, c2pwrite = None, None - errread, errwrite = None, None - - if stdin is None: - pass - elif stdin == PIPE: - p2cread, p2cwrite = self.pipe_cloexec() - elif isinstance(stdin, int): - p2cread = stdin - else: - # Assuming file-like object - p2cread = stdin.fileno() - - if stdout is None: - pass - elif stdout == PIPE: - c2pread, c2pwrite = self.pipe_cloexec() - elif isinstance(stdout, int): - c2pwrite = stdout - else: - # Assuming file-like object - c2pwrite = stdout.fileno() - - if stderr is None: - pass - elif stderr == PIPE: - errread, errwrite = self.pipe_cloexec() - elif stderr == STDOUT: - errwrite = c2pwrite - elif isinstance(stderr, int): - errwrite = stderr - else: - # Assuming file-like object - errwrite = stderr.fileno() - - return (p2cread, p2cwrite, - c2pread, c2pwrite, - errread, errwrite) - - def _set_cloexec_flag(self, fd, cloexec=True): - try: - cloexec_flag = fcntl.FD_CLOEXEC - except AttributeError: - cloexec_flag = 1 - - old = fcntl.fcntl(fd, fcntl.F_GETFD) - if cloexec: - fcntl.fcntl(fd, fcntl.F_SETFD, old | cloexec_flag) - else: - fcntl.fcntl(fd, fcntl.F_SETFD, old & ~cloexec_flag) - - def pipe_cloexec(self): - """Create a pipe with FDs set CLOEXEC.""" - # Pipes' FDs are set CLOEXEC by default because we don't want them - # to be inherited by other subprocesses: the CLOEXEC flag is - # removed from the child's FDs by _dup2(), between fork() and - # exec(). - # This is not atomic: we would need the pipe2() syscall for that. - r, w = os.pipe() - self._set_cloexec_flag(r) - self._set_cloexec_flag(w) - return r, w - - def _close_fds(self, but): - if hasattr(os, 'closerange'): - os.closerange(3, but) - os.closerange(but + 1, MAXFD) - else: - for i in xrange(3, MAXFD): - if i == but: - continue - try: - os.close(i) - except: - pass - - def _execute_child(self, args, executable, preexec_fn, close_fds, - cwd, env, universal_newlines, - startupinfo, creationflags, shell, - p2cread, p2cwrite, - c2pread, c2pwrite, - errread, errwrite): - """Execute program (POSIX version)""" - - if isinstance(args, types.StringTypes): - args = [args] - else: - args = list(args) - - if shell: - args = ["/bin/sh", "-c"] + args - if executable: - args[0] = executable - - if executable is None: - executable = args[0] - - # For transferring possible exec failure from child to parent - # The first char specifies the exception type: 0 means - # OSError, 1 means some other error. - errpipe_read, errpipe_write = self.pipe_cloexec() - try: - try: - gc_was_enabled = gc.isenabled() - # Disable gc to avoid bug where gc -> file_dealloc -> - # write to stderr -> hang. - # http://bugs.python.org/issue1336 - gc.disable() - try: - self.pid = os.fork() - except: - if gc_was_enabled: - gc.enable() - raise - self._child_created = True - if self.pid == 0: - # Child - try: - # Close parent's pipe ends - if p2cwrite is not None: - os.close(p2cwrite) - if c2pread is not None: - os.close(c2pread) - if errread is not None: - os.close(errread) - os.close(errpipe_read) - - # When duping fds, if there arises a situation - # where one of the fds is either 0, 1 or 2, it - # is possible that it is overwritten (#12607). - if c2pwrite == 0: - c2pwrite = os.dup(c2pwrite) - if errwrite == 0 or errwrite == 1: - errwrite = os.dup(errwrite) - - # Dup fds for child - def _dup2(a, b): - # dup2() removes the CLOEXEC flag but - # we must do it ourselves if dup2() - # would be a no-op (issue #10806). - if a == b: - self._set_cloexec_flag(a, False) - elif a is not None: - os.dup2(a, b) - _dup2(p2cread, 0) - _dup2(c2pwrite, 1) - _dup2(errwrite, 2) - - # Close pipe fds. Make sure we don't close the - # same fd more than once, or standard fds. - closed = set([None]) - for fd in [p2cread, c2pwrite, errwrite]: - if fd not in closed and fd > 2: - os.close(fd) - closed.add(fd) - - # Close all other fds, if asked for - if close_fds: - self._close_fds(but=errpipe_write) - - if cwd is not None: - os.chdir(cwd) - - if preexec_fn: - preexec_fn() - - if env is None: - os.execvp(executable, args) - else: - os.execvpe(executable, args, env) - - except: - exc_type, exc_value, tb = sys.exc_info() - # Save the traceback and attach it to the exception - # object - exc_lines = traceback.format_exception(exc_type, - exc_value, - tb) - exc_value.child_traceback = ''.join(exc_lines) - os.write(errpipe_write, pickle.dumps(exc_value)) - - # This exitcode won't be reported to applications, - # so it really doesn't matter what we return. - os._exit(255) - - # Parent - if gc_was_enabled: - gc.enable() - finally: - # be sure the FD is closed no matter what - os.close(errpipe_write) - - if p2cread is not None and p2cwrite is not None: - os.close(p2cread) - if c2pwrite is not None and c2pread is not None: - os.close(c2pwrite) - if errwrite is not None and errread is not None: - os.close(errwrite) - - # Wait for exec to fail or succeed; possibly raising exception - # Exception limited to 1M - data = _eintr_retry_call(os.read, errpipe_read, 1048576) - finally: - # be sure the FD is closed no matter what - os.close(errpipe_read) - - if data != "": - try: - _eintr_retry_call(os.waitpid, self.pid, 0) - except OSError, e: - if e.errno != errno.ECHILD: - raise - child_exception = pickle.loads(data) - for fd in (p2cwrite, c2pread, errread): - if fd is not None: - os.close(fd) - raise child_exception - - def _handle_exitstatus(self, sts, _WIFSIGNALED=os.WIFSIGNALED, - _WTERMSIG=os.WTERMSIG, _WIFEXITED=os.WIFEXITED, - _WEXITSTATUS=os.WEXITSTATUS): - # This method is called (indirectly) by __del__, so it cannot - # refer to anything outside of its local scope.""" - if _WIFSIGNALED(sts): - self.returncode = -_WTERMSIG(sts) - elif _WIFEXITED(sts): - self.returncode = _WEXITSTATUS(sts) - else: - # Should never happen - raise RuntimeError("Unknown child exit status!") - - def _internal_poll(self, _deadstate=None, _waitpid=os.waitpid, - _WNOHANG=os.WNOHANG, _os_error=os.error): - """Check if child process has terminated. Returns returncode - attribute. - - This method is called by __del__, so it cannot reference anything - outside of the local scope (nor can any methods it calls). - - """ - if self.returncode is None: - try: - pid, sts = _waitpid(self.pid, _WNOHANG) - if pid == self.pid: - self._handle_exitstatus(sts) - except _os_error: - if _deadstate is not None: - self.returncode = _deadstate - return self.returncode - - def wait(self): - """Wait for child process to terminate. Returns returncode - attribute.""" - if self.returncode is None: - try: - pid, sts = _eintr_retry_call(os.waitpid, self.pid, 0) - except OSError, e: - if e.errno != errno.ECHILD: - raise - # This happens if SIGCLD is set to be ignored or waiting - # for child processes has otherwise been disabled for our - # process. This child is dead, we can't get the status. - sts = 0 - self._handle_exitstatus(sts) - return self.returncode - - def _communicate(self, input): - if self.stdin: - # Flush stdio buffer. This might block, if the user has - # been writing to .stdin in an uncontrolled fashion. - self.stdin.flush() - if not input: - self.stdin.close() - - if _has_poll: - stdout, stderr = self._communicate_with_poll(input) - else: - stdout, stderr = self._communicate_with_select(input) - - # All data exchanged. Translate lists into strings. - if stdout is not None: - stdout = ''.join(stdout) - if stderr is not None: - stderr = ''.join(stderr) - - # Translate newlines, if requested. We cannot let the file - # object do the translation: It is based on stdio, which is - # impossible to combine with select (unless forcing no - # buffering). - if self.universal_newlines and hasattr(file, 'newlines'): - if stdout: - stdout = self._translate_newlines(stdout) - if stderr: - stderr = self._translate_newlines(stderr) - - self.wait() - return (stdout, stderr) - - def _communicate_with_poll(self, input): - stdout = None # Return - stderr = None # Return - fd2file = {} - fd2output = {} - - poller = select.poll() - - def register_and_append(file_obj, eventmask): - poller.register(file_obj.fileno(), eventmask) - fd2file[file_obj.fileno()] = file_obj - - def close_unregister_and_remove(fd): - poller.unregister(fd) - fd2file[fd].close() - fd2file.pop(fd) - - if self.stdin and input: - register_and_append(self.stdin, select.POLLOUT) - - select_POLLIN_POLLPRI = select.POLLIN | select.POLLPRI - if self.stdout: - register_and_append(self.stdout, select_POLLIN_POLLPRI) - fd2output[self.stdout.fileno()] = stdout = [] - if self.stderr: - register_and_append(self.stderr, select_POLLIN_POLLPRI) - fd2output[self.stderr.fileno()] = stderr = [] - - input_offset = 0 - while fd2file: - try: - ready = poller.poll() - except select.error, e: - if e.args[0] == errno.EINTR: - continue - raise - - for fd, mode in ready: - if mode & select.POLLOUT: - chunk = input[input_offset: input_offset + _PIPE_BUF] - try: - input_offset += os.write(fd, chunk) - except OSError, e: - if e.errno == errno.EPIPE: - close_unregister_and_remove(fd) - else: - raise - else: - if input_offset >= len(input): - close_unregister_and_remove(fd) - elif mode & select_POLLIN_POLLPRI: - data = os.read(fd, 4096) - if not data: - close_unregister_and_remove(fd) - fd2output[fd].append(data) - else: - # Ignore hang up or errors. - close_unregister_and_remove(fd) - - return (stdout, stderr) - - def _communicate_with_select(self, input): - read_set = [] - write_set = [] - stdout = None # Return - stderr = None # Return - - if self.stdin and input: - write_set.append(self.stdin) - if self.stdout: - read_set.append(self.stdout) - stdout = [] - if self.stderr: - read_set.append(self.stderr) - stderr = [] - - input_offset = 0 - while read_set or write_set: - try: - rlist, wlist, xlist = select.select( - read_set, write_set, []) - except select.error, e: - if e.args[0] == errno.EINTR: - continue - raise - - if self.stdin in wlist: - chunk = input[input_offset: input_offset + _PIPE_BUF] - try: - bytes_written = os.write(self.stdin.fileno(), chunk) - except OSError, e: - if e.errno == errno.EPIPE: - self.stdin.close() - write_set.remove(self.stdin) - else: - raise - else: - input_offset += bytes_written - if input_offset >= len(input): - self.stdin.close() - write_set.remove(self.stdin) - - if self.stdout in rlist: - data = os.read(self.stdout.fileno(), 1024) - if data == "": - self.stdout.close() - read_set.remove(self.stdout) - stdout.append(data) - - if self.stderr in rlist: - data = os.read(self.stderr.fileno(), 1024) - if data == "": - self.stderr.close() - read_set.remove(self.stderr) - stderr.append(data) - - return (stdout, stderr) - - def send_signal(self, sig): - """Send a signal to the process - """ - os.kill(self.pid, sig) - - def terminate(self): - """Terminate the process with SIGTERM - """ - self.send_signal(signal.SIGTERM) - - def kill(self): - """Kill the process with SIGKILL - """ - self.send_signal(signal.SIGKILL) - - -def _demo_posix(): - # - # Example 1: Simple redirection: Get process list - # - plist = Popen(["ps"], stdout=PIPE).communicate()[0] - print "Process list:" - print plist - - # - # Example 2: Change uid before executing child - # - if os.getuid() == 0: - p = Popen(["id"], preexec_fn=lambda: os.setuid(100)) - p.wait() - - # - # Example 3: Connecting several subprocesses - # - print "Looking for 'hda'..." - p1 = Popen(["dmesg"], stdout=PIPE) - p2 = Popen(["grep", "hda"], stdin=p1.stdout, stdout=PIPE) - print repr(p2.communicate()[0]) - - # - # Example 4: Catch execution error - # - print - print "Trying a weird file..." - try: - print Popen(["/this/path/does/not/exist"]).communicate() - except OSError, e: - if e.errno == errno.ENOENT: - print "The file didn't exist. I thought so..." - print "Child traceback:" - print e.child_traceback - else: - print "Error", e.errno - else: - print >>sys.stderr, "Gosh. No error." - - -def _demo_windows(): - # - # Example 1: Connecting several subprocesses - # - print "Looking for 'PROMPT' in set output..." - p1 = Popen("set", stdout=PIPE, shell=True) - p2 = Popen('find "PROMPT"', stdin=p1.stdout, stdout=PIPE) - print repr(p2.communicate()[0]) - - # - # Example 2: Simple execution of program - # - print "Executing calc..." - p = Popen("calc") - p.wait() - - -if __name__ == "__main__": - if mswindows: - _demo_windows() - else: - _demo_posix() diff --git a/cherrypy/_cpconfig.py b/cherrypy/_cpconfig.py index 129ddcb..674199f 100644 --- a/cherrypy/_cpconfig.py +++ b/cherrypy/_cpconfig.py @@ -119,7 +119,7 @@ style) context manager. """ import cherrypy -from cherrypy._cpcompat import basestring +from cherrypy._cpcompat import text_or_bytes from cherrypy.lib import reprconf # Deprecated in CherryPy 3.2--remove in 3.3 @@ -132,16 +132,16 @@ def merge(base, other): If the given config is a filename, it will be appended to the list of files to monitor for "autoreload" changes. """ - if isinstance(other, basestring): + if isinstance(other, text_or_bytes): cherrypy.engine.autoreload.files.add(other) # Load other into base for section, value_map in reprconf.as_dict(other).items(): if not isinstance(value_map, dict): raise ValueError( - "Application config must include section headers, but the " + 'Application config must include section headers, but the ' "config you tried to merge doesn't have any sections. " - "Wrap your config in another dict with paths as section " + 'Wrap your config in another dict with paths as section ' "headers, for example: {'/': config}.") base.setdefault(section, {}).update(value_map) @@ -152,19 +152,19 @@ class Config(reprconf.Config): def update(self, config): """Update self from a dict, file or filename.""" - if isinstance(config, basestring): + if isinstance(config, text_or_bytes): # Filename cherrypy.engine.autoreload.files.add(config) reprconf.Config.update(self, config) def _apply(self, config): """Update self from a dict.""" - if isinstance(config.get("global"), dict): + if isinstance(config.get('global'), dict): if len(config) > 1: cherrypy.checker.global_config_contained_paths = True - config = config["global"] + config = config['global'] if 'tools.staticdir.dir' in config: - config['tools.staticdir.section'] = "global" + config['tools.staticdir.section'] = 'global' reprconf.Config._apply(self, config) @staticmethod @@ -172,8 +172,8 @@ class Config(reprconf.Config): """Decorator for page handlers to set _cp_config.""" if args: raise TypeError( - "The cherrypy.config decorator does not accept positional " - "arguments; you must use keyword arguments.") + 'The cherrypy.config decorator does not accept positional ' + 'arguments; you must use keyword arguments.') def tool_decorator(f): _Vars(f).setdefault('_cp_config', {}).update(kwargs) @@ -197,14 +197,14 @@ class _Vars(object): # Sphinx begin config.environments Config.environments = environments = { - "staging": { + 'staging': { 'engine.autoreload.on': False, 'checker.on': False, 'tools.log_headers.on': False, 'request.show_tracebacks': False, 'request.show_mismatched_params': False, }, - "production": { + 'production': { 'engine.autoreload.on': False, 'checker.on': False, 'tools.log_headers.on': False, @@ -212,7 +212,7 @@ Config.environments = environments = { 'request.show_mismatched_params': False, 'log.screen': False, }, - "embedded": { + 'embedded': { # For use with CherryPy embedded in another deployment stack. 'engine.autoreload.on': False, 'checker.on': False, @@ -223,7 +223,7 @@ Config.environments = environments = { 'engine.SIGHUP': None, 'engine.SIGTERM': None, }, - "test_suite": { + 'test_suite': { 'engine.autoreload.on': False, 'checker.on': False, 'tools.log_headers.on': False, @@ -237,11 +237,11 @@ Config.environments = environments = { def _server_namespace_handler(k, v): """Config handler for the "server" namespace.""" - atoms = k.split(".", 1) + atoms = k.split('.', 1) if len(atoms) > 1: # Special-case config keys of the form 'server.servername.socket_port' # to configure additional HTTP servers. - if not hasattr(cherrypy, "servers"): + if not hasattr(cherrypy, 'servers'): cherrypy.servers = {} servername, k = atoms @@ -260,45 +260,19 @@ def _server_namespace_handler(k, v): setattr(cherrypy.servers[servername], k, v) else: setattr(cherrypy.server, k, v) -Config.namespaces["server"] = _server_namespace_handler +Config.namespaces['server'] = _server_namespace_handler def _engine_namespace_handler(k, v): - """Backward compatibility handler for the "engine" namespace.""" + """Config handler for the "engine" namespace.""" engine = cherrypy.engine - deprecated = { - 'autoreload_on': 'autoreload.on', - 'autoreload_frequency': 'autoreload.frequency', - 'autoreload_match': 'autoreload.match', - 'reload_files': 'autoreload.files', - 'deadlock_poll_freq': 'timeout_monitor.frequency' - } - - if k in deprecated: - engine.log( - 'WARNING: Use of engine.%s is deprecated and will be removed in a ' - 'future version. Use engine.%s instead.' % (k, deprecated[k])) - - if k == 'autoreload_on': - if v: - engine.autoreload.subscribe() - else: - engine.autoreload.unsubscribe() - elif k == 'autoreload_frequency': - engine.autoreload.frequency = v - elif k == 'autoreload_match': - engine.autoreload.match = v - elif k == 'reload_files': - engine.autoreload.files = set(v) - elif k == 'deadlock_poll_freq': - engine.timeout_monitor.frequency = v - elif k == 'SIGHUP': - engine.listeners['SIGHUP'] = set([v]) + if k == 'SIGHUP': + engine.subscribe('SIGHUP', v) elif k == 'SIGTERM': - engine.listeners['SIGTERM'] = set([v]) - elif "." in k: - plugin, attrname = k.split(".", 1) + engine.subscribe('SIGTERM', v) + elif '.' in k: + plugin, attrname = k.split('.', 1) plugin = getattr(engine, plugin) if attrname == 'on': if v and hasattr(getattr(plugin, 'subscribe', None), '__call__'): @@ -313,7 +287,7 @@ def _engine_namespace_handler(k, v): setattr(plugin, attrname, v) else: setattr(engine, k, v) -Config.namespaces["engine"] = _engine_namespace_handler +Config.namespaces['engine'] = _engine_namespace_handler def _tree_namespace_handler(k, v): @@ -321,9 +295,9 @@ def _tree_namespace_handler(k, v): if isinstance(v, dict): for script_name, app in v.items(): cherrypy.tree.graft(app, script_name) - msg = "Mounted: %s on %s" % (app, script_name or "/") + msg = 'Mounted: %s on %s' % (app, script_name or '/') cherrypy.engine.log(msg) else: cherrypy.tree.graft(v, v.script_name) - cherrypy.engine.log("Mounted: %s on %s" % (v, v.script_name or "/")) -Config.namespaces["tree"] = _tree_namespace_handler + cherrypy.engine.log('Mounted: %s on %s' % (v, v.script_name or '/')) +Config.namespaces['tree'] = _tree_namespace_handler diff --git a/cherrypy/_cpdispatch.py b/cherrypy/_cpdispatch.py index 2cb03c7..eb5484c 100644 --- a/cherrypy/_cpdispatch.py +++ b/cherrypy/_cpdispatch.py @@ -39,7 +39,7 @@ class PageHandler(object): args = property( get_args, set_args, - doc="The ordered args should be accessible from post dispatch hooks" + doc='The ordered args should be accessible from post dispatch hooks' ) def get_kwargs(self): @@ -52,7 +52,7 @@ class PageHandler(object): kwargs = property( get_kwargs, set_kwargs, - doc="The named kwargs should be accessible from post dispatch hooks" + doc='The named kwargs should be accessible from post dispatch hooks' ) def __call__(self): @@ -153,7 +153,7 @@ def test_callable_spec(callable, callable_args, callable_kwargs): # arguments it's definitely a 404. message = None if show_mismatched_params: - message = "Missing parameters: %s" % ",".join(missing_args) + message = 'Missing parameters: %s' % ','.join(missing_args) raise cherrypy.HTTPError(404, message=message) # the extra positional arguments come from the path - 404 Not Found @@ -175,8 +175,8 @@ def test_callable_spec(callable, callable_args, callable_kwargs): message = None if show_mismatched_params: - message = "Multiple values for parameters: "\ - "%s" % ",".join(multiple_args) + message = 'Multiple values for parameters: '\ + '%s' % ','.join(multiple_args) raise cherrypy.HTTPError(error, message=message) if not varkw and varkw_usage > 0: @@ -186,8 +186,8 @@ def test_callable_spec(callable, callable_args, callable_kwargs): if extra_qs_params: message = None if show_mismatched_params: - message = "Unexpected query string "\ - "parameters: %s" % ", ".join(extra_qs_params) + message = 'Unexpected query string '\ + 'parameters: %s' % ', '.join(extra_qs_params) raise cherrypy.HTTPError(404, message=message) # If there were any extra body parameters, it's a 400 Not Found @@ -195,8 +195,8 @@ def test_callable_spec(callable, callable_args, callable_kwargs): if extra_body_params: message = None if show_mismatched_params: - message = "Unexpected body parameters: "\ - "%s" % ", ".join(extra_body_params) + message = 'Unexpected body parameters: '\ + '%s' % ', '.join(extra_body_params) raise cherrypy.HTTPError(400, message=message) @@ -244,14 +244,14 @@ if sys.version_info < (3, 0): def validate_translator(t): if not isinstance(t, str) or len(t) != 256: raise ValueError( - "The translate argument must be a str of len 256.") + 'The translate argument must be a str of len 256.') else: punctuation_to_underscores = str.maketrans( string.punctuation, '_' * len(string.punctuation)) def validate_translator(t): if not isinstance(t, dict): - raise ValueError("The translate argument must be a dict.") + raise ValueError('The translate argument must be a dict.') class Dispatcher(object): @@ -289,7 +289,7 @@ class Dispatcher(object): if func: # Decode any leftover %2F in the virtual_path atoms. - vpath = [x.replace("%2F", "/") for x in vpath] + vpath = [x.replace('%2F', '/') for x in vpath] request.handler = LateParamPageHandler(func, *vpath) else: request.handler = cherrypy.NotFound() @@ -323,10 +323,10 @@ class Dispatcher(object): fullpath_len = len(fullpath) segleft = fullpath_len nodeconf = {} - if hasattr(root, "_cp_config"): + if hasattr(root, '_cp_config'): nodeconf.update(root._cp_config) - if "/" in app.config: - nodeconf.update(app.config["/"]) + if '/' in app.config: + nodeconf.update(app.config['/']) object_trail = [['root', root, nodeconf, segleft]] node = root @@ -361,9 +361,9 @@ class Dispatcher(object): if segleft > pre_len: # No path segment was removed. Raise an error. raise cherrypy.CherryPyException( - "A vpath segment was added. Custom dispatchers may only " - + "remove elements. While trying to process " - + "{0} in {1}".format(name, fullpath) + 'A vpath segment was added. Custom dispatchers may only ' + + 'remove elements. While trying to process ' + + '{0} in {1}'.format(name, fullpath) ) elif segleft == pre_len: # Assume that the handler used the current path segment, but @@ -375,7 +375,7 @@ class Dispatcher(object): if node is not None: # Get _cp_config attached to this node. - if hasattr(node, "_cp_config"): + if hasattr(node, '_cp_config'): nodeconf.update(node._cp_config) # Mix in values from app.config for this path. @@ -414,16 +414,16 @@ class Dispatcher(object): continue # Try a "default" method on the current leaf. - if hasattr(candidate, "default"): + if hasattr(candidate, 'default'): defhandler = candidate.default if getattr(defhandler, 'exposed', False): # Insert any extra _cp_config from the default handler. - conf = getattr(defhandler, "_cp_config", {}) + conf = getattr(defhandler, '_cp_config', {}) object_trail.insert( - i + 1, ["default", defhandler, conf, segleft]) + i + 1, ['default', defhandler, conf, segleft]) request.config = set_conf() # See https://github.com/cherrypy/cherrypy/issues/613 - request.is_index = path.endswith("/") + request.is_index = path.endswith('/') return defhandler, fullpath[fullpath_len - segleft:-1] # Uncomment the next line to restrict positional params to @@ -470,23 +470,23 @@ class MethodDispatcher(Dispatcher): if resource: # Set Allow header avail = [m for m in dir(resource) if m.isupper()] - if "GET" in avail and "HEAD" not in avail: - avail.append("HEAD") + if 'GET' in avail and 'HEAD' not in avail: + avail.append('HEAD') avail.sort() - cherrypy.serving.response.headers['Allow'] = ", ".join(avail) + cherrypy.serving.response.headers['Allow'] = ', '.join(avail) # Find the subhandler meth = request.method.upper() func = getattr(resource, meth, None) - if func is None and meth == "HEAD": - func = getattr(resource, "GET", None) + if func is None and meth == 'HEAD': + func = getattr(resource, 'GET', None) if func: # Grab any _cp_config on the subhandler. - if hasattr(func, "_cp_config"): + if hasattr(func, '_cp_config'): request.config.update(func._cp_config) # Decode any leftover %2F in the virtual_path atoms. - vpath = [x.replace("%2F", "/") for x in vpath] + vpath = [x.replace('%2F', '/') for x in vpath] request.handler = LateParamPageHandler(func, *vpath) else: request.handler = cherrypy.HTTPError(405) @@ -554,28 +554,28 @@ class RoutesDispatcher(object): # Get config for the root object/path. request.config = base = cherrypy.config.copy() - curpath = "" + curpath = '' def merge(nodeconf): if 'tools.staticdir.dir' in nodeconf: - nodeconf['tools.staticdir.section'] = curpath or "/" + nodeconf['tools.staticdir.section'] = curpath or '/' base.update(nodeconf) app = request.app root = app.root - if hasattr(root, "_cp_config"): + if hasattr(root, '_cp_config'): merge(root._cp_config) - if "/" in app.config: - merge(app.config["/"]) + if '/' in app.config: + merge(app.config['/']) # Mix in values from app.config. - atoms = [x for x in path_info.split("/") if x] + atoms = [x for x in path_info.split('/') if x] if atoms: last = atoms.pop() else: last = None for atom in atoms: - curpath = "/".join((curpath, atom)) + curpath = '/'.join((curpath, atom)) if curpath in app.config: merge(app.config[curpath]) @@ -587,14 +587,14 @@ class RoutesDispatcher(object): if isinstance(controller, classtype): controller = controller() # Get config from the controller. - if hasattr(controller, "_cp_config"): + if hasattr(controller, '_cp_config'): merge(controller._cp_config) action = result.get('action') if action is not None: handler = getattr(controller, action, None) # Get config from the handler - if hasattr(handler, "_cp_config"): + if hasattr(handler, '_cp_config'): merge(handler._cp_config) else: handler = controller @@ -602,7 +602,7 @@ class RoutesDispatcher(object): # Do the last path atom here so it can # override the controller's _cp_config. if last: - curpath = "/".join((curpath, last)) + curpath = '/'.join((curpath, last)) if curpath in app.config: merge(app.config[curpath]) @@ -666,9 +666,9 @@ def VirtualHost(next_dispatcher=Dispatcher(), use_x_forwarded_host=True, domain = header('Host', '') if use_x_forwarded_host: - domain = header("X-Forwarded-Host", domain) + domain = header('X-Forwarded-Host', domain) - prefix = domains.get(domain, "") + prefix = domains.get(domain, '') if prefix: path_info = httputil.urljoin(prefix, path_info) diff --git a/cherrypy/_cperror.py b/cherrypy/_cperror.py index 64cf3aa..205aa33 100644 --- a/cherrypy/_cperror.py +++ b/cherrypy/_cperror.py @@ -115,13 +115,15 @@ Note that you have to explicitly set and not simply return an error message as a result. """ +import contextlib from cgi import escape as _escape from sys import exc_info as _exc_info from traceback import format_exception as _format_exception +from xml.sax import saxutils import six -from cherrypy._cpcompat import basestring, iteritems, ntob +from cherrypy._cpcompat import text_or_bytes, iteritems, ntob from cherrypy._cpcompat import tonative, urljoin as _urljoin from cherrypy.lib import httputil as _httputil @@ -148,14 +150,14 @@ class InternalRedirect(CherryPyException): URL. """ - def __init__(self, path, query_string=""): + def __init__(self, path, query_string=''): import cherrypy self.request = cherrypy.serving.request self.query_string = query_string - if "?" in path: + if '?' in path: # Separate any params included in the path - path, self.query_string = path.split("?", 1) + path, self.query_string = path.split('?', 1) # Note that urljoin will "do the right thing" whether url is: # 1. a URL relative to root (e.g. "/dummy") @@ -209,7 +211,7 @@ class HTTPRedirect(CherryPyException): import cherrypy request = cherrypy.serving.request - if isinstance(urls, basestring): + if isinstance(urls, text_or_bytes): urls = [urls] abs_urls = [] @@ -236,7 +238,7 @@ class HTTPRedirect(CherryPyException): else: status = int(status) if status < 300 or status > 399: - raise ValueError("status must be between 300 and 399.") + raise ValueError('status must be between 300 and 399.') self.status = status CherryPyException.__init__(self, abs_urls, status) @@ -253,7 +255,7 @@ class HTTPRedirect(CherryPyException): response.status = status = self.status if status in (300, 301, 302, 303, 307): - response.headers['Content-Type'] = "text/html;charset=utf-8" + response.headers['Content-Type'] = 'text/html;charset=utf-8' # "The ... URI SHOULD be given by the Location field # in the response." response.headers['Location'] = self.urls[0] @@ -262,16 +264,15 @@ class HTTPRedirect(CherryPyException): # SHOULD contain a short hypertext note with a hyperlink to the # new URI(s)." msg = { - 300: "This resource can be found at ", - 301: "This resource has permanently moved to ", - 302: "This resource resides temporarily at ", - 303: "This resource can be found at ", - 307: "This resource has moved temporarily to ", + 300: 'This resource can be found at ', + 301: 'This resource has permanently moved to ', + 302: 'This resource resides temporarily at ', + 303: 'This resource can be found at ', + 307: 'This resource has moved temporarily to ', }[status] msg += '%s.' - from xml.sax import saxutils msgs = [msg % (saxutils.quoteattr(u), u) for u in self.urls] - response.body = ntob("
\n".join(msgs), 'utf-8') + response.body = ntob('
\n'.join(msgs), 'utf-8') # Previous code may have set C-L, so we have to reset it # (allow finalize to set it). response.headers.pop('Content-Length', None) @@ -301,7 +302,7 @@ class HTTPRedirect(CherryPyException): # Previous code may have set C-L, so we have to reset it. response.headers.pop('Content-Length', None) else: - raise ValueError("The %s status code is unknown." % status) + raise ValueError('The %s status code is unknown.' % status) def __call__(self): """Use this exception as a request.handler (raise self).""" @@ -317,9 +318,9 @@ def clean_headers(status): # Remove headers which applied to the original content, # but do not apply to the error page. respheaders = response.headers - for key in ["Accept-Ranges", "Age", "ETag", "Location", "Retry-After", - "Vary", "Content-Encoding", "Content-Length", "Expires", - "Content-Location", "Content-MD5", "Last-Modified"]: + for key in ['Accept-Ranges', 'Age', 'ETag', 'Location', 'Retry-After', + 'Vary', 'Content-Encoding', 'Content-Length', 'Expires', + 'Content-Location', 'Content-MD5', 'Last-Modified']: if key in respheaders: del respheaders[key] @@ -330,8 +331,8 @@ def clean_headers(status): # specifies the current length of the selected resource. # A response with status code 206 (Partial Content) MUST NOT # include a Content-Range field with a byte-range- resp-spec of "*". - if "Content-Range" in respheaders: - del respheaders["Content-Range"] + if 'Content-Range' in respheaders: + del respheaders['Content-Range'] class HTTPError(CherryPyException): @@ -371,7 +372,7 @@ class HTTPError(CherryPyException): raise self.__class__(500, _exc_info()[1].args[0]) if self.code < 400 or self.code > 599: - raise ValueError("status must be between 400 and 599.") + raise ValueError('status must be between 400 and 599.') # See http://www.python.org/dev/peps/pep-0352/ # self.message = message @@ -413,6 +414,15 @@ class HTTPError(CherryPyException): """Use this exception as a request.handler (raise self).""" raise self + @classmethod + @contextlib.contextmanager + def handle(cls, exception, status=500, message=''): + """Translate exception into an HTTPError.""" + try: + yield + except exception as exc: + raise cls(status, message or str(exc)) + class NotFound(HTTPError): @@ -480,7 +490,7 @@ def get_error_page(status, **kwargs): # We can't use setdefault here, because some # callers send None for kwarg values. if kwargs.get('status') is None: - kwargs['status'] = "%s %s" % (code, reason) + kwargs['status'] = '%s %s' % (code, reason) if kwargs.get('message') is None: kwargs['message'] = message if kwargs.get('traceback') is None: @@ -490,7 +500,7 @@ def get_error_page(status, **kwargs): for k, v in iteritems(kwargs): if v is None: - kwargs[k] = "" + kwargs[k] = '' else: kwargs[k] = _escape(kwargs[k]) @@ -528,12 +538,12 @@ def get_error_page(status, **kwargs): e = _format_exception(*_exc_info())[-1] m = kwargs['message'] if m: - m += "
" - m += "In addition, the custom error page failed:\n
%s" % e + m += '
' + m += 'In addition, the custom error page failed:\n
%s' % e kwargs['message'] = m response = cherrypy.serving.response - response.headers['Content-Type'] = "text/html;charset=utf-8" + response.headers['Content-Type'] = 'text/html;charset=utf-8' result = template % kwargs return result.encode('utf-8') @@ -565,7 +575,7 @@ def _be_ie_unfriendly(status): if l and l < s: # IN ADDITION: the response must be written to IE # in one chunk or it will still get replaced! Bah. - content = content + (ntob(" ") * (s - l)) + content = content + (ntob(' ') * (s - l)) response.body = content response.headers['Content-Length'] = str(len(content)) @@ -576,9 +586,9 @@ def format_exc(exc=None): if exc is None: exc = _exc_info() if exc == (None, None, None): - return "" + return '' import traceback - return "".join(traceback.format_exception(*exc)) + return ''.join(traceback.format_exception(*exc)) finally: del exc @@ -600,13 +610,13 @@ def bare_error(extrabody=None): # it cannot be allowed to fail. Therefore, don't add to it! # In particular, don't call any other CP functions. - body = ntob("Unrecoverable error in the server.") + body = ntob('Unrecoverable error in the server.') if extrabody is not None: if not isinstance(extrabody, bytes): extrabody = extrabody.encode('utf-8') - body += ntob("\n") + extrabody + body += ntob('\n') + extrabody - return (ntob("500 Internal Server Error"), + return (ntob('500 Internal Server Error'), [(ntob('Content-Type'), ntob('text/plain')), (ntob('Content-Length'), ntob(str(len(body)), 'ISO-8859-1'))], [body]) diff --git a/cherrypy/_cplogging.py b/cherrypy/_cplogging.py index b13b49e..79fe5a8 100644 --- a/cherrypy/_cplogging.py +++ b/cherrypy/_cplogging.py @@ -109,9 +109,6 @@ the "log.error_file" config entry, for example). import datetime import logging -# Silence the no-handlers "warning" (stderr write!) in stdlib logging -logging.Logger.manager.emittedNoHandlerWarning = 1 -logfmt = logging.Formatter("%(message)s") import os import sys @@ -122,6 +119,11 @@ from cherrypy import _cperror from cherrypy._cpcompat import ntob +# Silence the no-handlers "warning" (stderr write!) in stdlib logging +logging.Logger.manager.emittedNoHandlerWarning = 1 +logfmt = logging.Formatter('%(message)s') + + class NullHandler(logging.Handler): """A no-op logging handler to silence the logging.lastResort handler.""" @@ -170,17 +172,17 @@ class LogManager(object): cherrypy.access. """ - def __init__(self, appid=None, logger_root="cherrypy"): + def __init__(self, appid=None, logger_root='cherrypy'): self.logger_root = logger_root self.appid = appid if appid is None: - self.error_log = logging.getLogger("%s.error" % logger_root) - self.access_log = logging.getLogger("%s.access" % logger_root) + self.error_log = logging.getLogger('%s.error' % logger_root) + self.access_log = logging.getLogger('%s.access' % logger_root) else: self.error_log = logging.getLogger( - "%s.error.%s" % (logger_root, appid)) + '%s.error.%s' % (logger_root, appid)) self.access_log = logging.getLogger( - "%s.access.%s" % (logger_root, appid)) + '%s.access.%s' % (logger_root, appid)) self.error_log.setLevel(logging.INFO) self.access_log.setLevel(logging.INFO) @@ -244,19 +246,19 @@ class LogManager(object): outheaders = response.headers inheaders = request.headers if response.output_status is None: - status = "-" + status = '-' else: - status = response.output_status.split(ntob(" "), 1)[0] + status = response.output_status.split(ntob(' '), 1)[0] if six.PY3: status = status.decode('ISO-8859-1') atoms = {'h': remote.name or remote.ip, 'l': '-', - 'u': getattr(request, "login", None) or "-", + 'u': getattr(request, 'login', None) or '-', 't': self.time(), 'r': request.request_line, 's': status, - 'b': dict.get(outheaders, 'Content-Length', '') or "-", + 'b': dict.get(outheaders, 'Content-Length', '') or '-', 'f': dict.get(inheaders, 'Referer', ''), 'a': dict.get(inheaders, 'User-Agent', ''), 'o': dict.get(inheaders, 'Host', '-'), @@ -312,26 +314,26 @@ class LogManager(object): def _get_builtin_handler(self, log, key): for h in log.handlers: - if getattr(h, "_cpbuiltin", None) == key: + if getattr(h, '_cpbuiltin', None) == key: return h # ------------------------- Screen handlers ------------------------- # def _set_screen_handler(self, log, enable, stream=None): - h = self._get_builtin_handler(log, "screen") + h = self._get_builtin_handler(log, 'screen') if enable: if not h: if stream is None: stream = sys.stderr h = logging.StreamHandler(stream) h.setFormatter(logfmt) - h._cpbuiltin = "screen" + h._cpbuiltin = 'screen' log.addHandler(h) elif h: log.handlers.remove(h) def _get_screen(self): h = self._get_builtin_handler - has_h = h(self.error_log, "screen") or h(self.access_log, "screen") + has_h = h(self.error_log, 'screen') or h(self.access_log, 'screen') return bool(has_h) def _set_screen(self, newvalue): @@ -349,11 +351,11 @@ class LogManager(object): def _add_builtin_file_handler(self, log, fname): h = logging.FileHandler(fname) h.setFormatter(logfmt) - h._cpbuiltin = "file" + h._cpbuiltin = 'file' log.addHandler(h) def _set_file_handler(self, log, filename): - h = self._get_builtin_handler(log, "file") + h = self._get_builtin_handler(log, 'file') if filename: if h: if h.baseFilename != os.path.abspath(filename): @@ -368,7 +370,7 @@ class LogManager(object): log.handlers.remove(h) def _get_error_file(self): - h = self._get_builtin_handler(self.error_log, "file") + h = self._get_builtin_handler(self.error_log, 'file') if h: return h.baseFilename return '' @@ -383,7 +385,7 @@ class LogManager(object): """) def _get_access_file(self): - h = self._get_builtin_handler(self.access_log, "file") + h = self._get_builtin_handler(self.access_log, 'file') if h: return h.baseFilename return '' @@ -400,18 +402,18 @@ class LogManager(object): # ------------------------- WSGI handlers ------------------------- # def _set_wsgi_handler(self, log, enable): - h = self._get_builtin_handler(log, "wsgi") + h = self._get_builtin_handler(log, 'wsgi') if enable: if not h: h = WSGIErrorHandler() h.setFormatter(logfmt) - h._cpbuiltin = "wsgi" + h._cpbuiltin = 'wsgi' log.addHandler(h) elif h: log.handlers.remove(h) def _get_wsgi(self): - return bool(self._get_builtin_handler(self.error_log, "wsgi")) + return bool(self._get_builtin_handler(self.error_log, 'wsgi')) def _set_wsgi(self, newvalue): self._set_wsgi_handler(self.error_log, newvalue) @@ -447,16 +449,16 @@ class WSGIErrorHandler(logging.Handler): else: try: msg = self.format(record) - fs = "%s\n" + fs = '%s\n' import types # if no unicode support... - if not hasattr(types, "UnicodeType"): + if not hasattr(types, 'UnicodeType'): stream.write(fs % msg) else: try: stream.write(fs % msg) except UnicodeError: - stream.write(fs % msg.encode("UTF-8")) + stream.write(fs % msg.encode('UTF-8')) self.flush() except: self.handleError(record) diff --git a/cherrypy/_cpmodpy.py b/cherrypy/_cpmodpy.py index 82f6ab3..5f0a467 100644 --- a/cherrypy/_cpmodpy.py +++ b/cherrypy/_cpmodpy.py @@ -55,9 +55,11 @@ resides in the global site-package this won't be needed. Then restart apache2 and access http://127.0.0.1:8080 """ +import io import logging +import os +import re import sys -import io import cherrypy from cherrypy._cpcompat import copyitems, ntob @@ -86,14 +88,14 @@ def setup(req): func() cherrypy.config.update({'log.screen': False, - "tools.ignore_headers.on": True, - "tools.ignore_headers.headers": ['Range'], + 'tools.ignore_headers.on': True, + 'tools.ignore_headers.headers': ['Range'], }) engine = cherrypy.engine - if hasattr(engine, "signal_handler"): + if hasattr(engine, 'signal_handler'): engine.signal_handler.unsubscribe() - if hasattr(engine, "console_control_handler"): + if hasattr(engine, 'console_control_handler'): engine.console_control_handler.unsubscribe() engine.autoreload.unsubscribe() cherrypy.server.unsubscribe() @@ -147,10 +149,10 @@ def handler(req): # Obtain a Request object from CherryPy local = req.connection.local_addr local = httputil.Host( - local[0], local[1], req.connection.local_host or "") + local[0], local[1], req.connection.local_host or '') remote = req.connection.remote_addr remote = httputil.Host( - remote[0], remote[1], req.connection.remote_host or "") + remote[0], remote[1], req.connection.remote_host or '') scheme = req.parsed_uri[0] or 'http' req.get_basic_auth_pw() @@ -163,7 +165,7 @@ def handler(req): except AttributeError: bad_value = ("You must provide a PythonOption '%s', " "either 'on' or 'off', when running a version " - "of mod_python < 3.1") + 'of mod_python < 3.1') threaded = options.get('multithread', '').lower() if threaded == 'on': @@ -171,7 +173,7 @@ def handler(req): elif threaded == 'off': threaded = False else: - raise ValueError(bad_value % "multithread") + raise ValueError(bad_value % 'multithread') forked = options.get('multiprocess', '').lower() if forked == 'on': @@ -179,16 +181,16 @@ def handler(req): elif forked == 'off': forked = False else: - raise ValueError(bad_value % "multiprocess") + raise ValueError(bad_value % 'multiprocess') - sn = cherrypy.tree.script_name(req.uri or "/") + sn = cherrypy.tree.script_name(req.uri or '/') if sn is None: send_response(req, '404 Not Found', [], '') else: app = cherrypy.tree.apps[sn] method = req.method path = req.uri - qs = req.args or "" + qs = req.args or '' reqproto = req.protocol headers = copyitems(req.headers_in) rfile = _ReadOnlyRequest(req) @@ -198,7 +200,7 @@ def handler(req): redirections = [] while True: request, response = app.get_serving(local, remote, scheme, - "HTTP/1.1") + 'HTTP/1.1') request.login = req.user request.multithread = bool(threaded) request.multiprocess = bool(forked) @@ -217,17 +219,17 @@ def handler(req): if not recursive: if ir.path in redirections: raise RuntimeError( - "InternalRedirector visited the same URL " - "twice: %r" % ir.path) + 'InternalRedirector visited the same URL ' + 'twice: %r' % ir.path) else: # Add the *previous* path_info + qs to # redirections. if qs: - qs = "?" + qs + qs = '?' + qs redirections.append(sn + path + qs) # Munge environment and try again. - method = "GET" + method = 'GET' path = ir.path qs = ir.query_string rfile = io.BytesIO() @@ -250,7 +252,7 @@ def send_response(req, status, headers, body, stream=False): req.status = int(status[:3]) # Set response headers - req.content_type = "text/plain" + req.content_type = 'text/plain' for header, value in headers: if header.lower() == 'content-type': req.content_type = value @@ -262,7 +264,7 @@ def send_response(req, status, headers, body, stream=False): req.flush() # Set response body - if isinstance(body, basestring): + if isinstance(body, text_or_bytes): req.write(body) else: for seg in body: @@ -270,8 +272,6 @@ def send_response(req, status, headers, body, stream=False): # --------------- Startup tools for CherryPy + mod_python --------------- # -import os -import re try: import subprocess @@ -286,13 +286,13 @@ except ImportError: return pipeout -def read_process(cmd, args=""): - fullcmd = "%s %s" % (cmd, args) +def read_process(cmd, args=''): + fullcmd = '%s %s' % (cmd, args) pipeout = popen(fullcmd) try: firstline = pipeout.readline() cmd_not_found = re.search( - ntob("(not recognized|No such file|not found)"), + ntob('(not recognized|No such file|not found)'), firstline, re.IGNORECASE ) @@ -321,8 +321,8 @@ LoadModule python_module modules/mod_python.so """ - def __init__(self, loc="/", port=80, opts=None, apache_path="apache", - handler="cherrypy._cpmodpy::handler"): + def __init__(self, loc='/', port=80, opts=None, apache_path='apache', + handler='cherrypy._cpmodpy::handler'): self.loc = loc self.port = port self.opts = opts @@ -330,25 +330,25 @@ LoadModule python_module modules/mod_python.so self.handler = handler def start(self): - opts = "".join([" PythonOption %s %s\n" % (k, v) + opts = ''.join([' PythonOption %s %s\n' % (k, v) for k, v in self.opts]) - conf_data = self.template % {"port": self.port, - "loc": self.loc, - "opts": opts, - "handler": self.handler, + conf_data = self.template % {'port': self.port, + 'loc': self.loc, + 'opts': opts, + 'handler': self.handler, } - mpconf = os.path.join(os.path.dirname(__file__), "cpmodpy.conf") + mpconf = os.path.join(os.path.dirname(__file__), 'cpmodpy.conf') f = open(mpconf, 'wb') try: f.write(conf_data) finally: f.close() - response = read_process(self.apache_path, "-k start -f %s" % mpconf) + response = read_process(self.apache_path, '-k start -f %s' % mpconf) self.ready = True return response def stop(self): - os.popen("apache -k stop") + os.popen('apache -k stop') self.ready = False diff --git a/cherrypy/_cpnative_server.py b/cherrypy/_cpnative_server.py index 8edb89c..026b95c 100644 --- a/cherrypy/_cpnative_server.py +++ b/cherrypy/_cpnative_server.py @@ -19,19 +19,19 @@ class NativeGateway(wsgiserver.Gateway): try: # Obtain a Request object from CherryPy local = req.server.bind_addr - local = httputil.Host(local[0], local[1], "") + local = httputil.Host(local[0], local[1], '') remote = req.conn.remote_addr, req.conn.remote_port - remote = httputil.Host(remote[0], remote[1], "") + remote = httputil.Host(remote[0], remote[1], '') scheme = req.scheme - sn = cherrypy.tree.script_name(req.uri or "/") + sn = cherrypy.tree.script_name(req.uri or '/') if sn is None: self.send_response('404 Not Found', [], ['']) else: app = cherrypy.tree.apps[sn] method = req.method path = req.path - qs = req.qs or "" + qs = req.qs or '' headers = req.inheaders.items() rfile = req.rfile prev = None @@ -40,7 +40,7 @@ class NativeGateway(wsgiserver.Gateway): redirections = [] while True: request, response = app.get_serving( - local, remote, scheme, "HTTP/1.1") + local, remote, scheme, 'HTTP/1.1') request.multithread = True request.multiprocess = False request.app = app @@ -60,17 +60,17 @@ class NativeGateway(wsgiserver.Gateway): if not self.recursive: if ir.path in redirections: raise RuntimeError( - "InternalRedirector visited the same " - "URL twice: %r" % ir.path) + 'InternalRedirector visited the same ' + 'URL twice: %r' % ir.path) else: # Add the *previous* path_info + qs to # redirections. if qs: - qs = "?" + qs + qs = '?' + qs redirections.append(sn + path + qs) # Munge environment and try again. - method = "GET" + method = 'GET' path = ir.path qs = ir.query_string rfile = io.BytesIO() @@ -91,7 +91,7 @@ class NativeGateway(wsgiserver.Gateway): req = self.req # Set response status - req.status = str(status or "500 Server Error") + req.status = str(status or '500 Server Error') # Set response headers for header, value in headers: diff --git a/cherrypy/_cpreqbody.py b/cherrypy/_cpreqbody.py index 7b46e00..1d21509 100644 --- a/cherrypy/_cpreqbody.py +++ b/cherrypy/_cpreqbody.py @@ -132,7 +132,7 @@ except ImportError: return ntob('').join(atoms) import cherrypy -from cherrypy._cpcompat import basestring, ntob, ntou +from cherrypy._cpcompat import text_or_bytes, ntob, ntou from cherrypy.lib import httputil @@ -169,8 +169,8 @@ def process_urlencoded(entity): break else: raise cherrypy.HTTPError( - 400, "The request entity could not be decoded. The following " - "charsets were attempted: %s" % repr(entity.attempt_charsets)) + 400, 'The request entity could not be decoded. The following ' + 'charsets were attempted: %s' % repr(entity.attempt_charsets)) # Now that all values have been successfully parsed and decoded, # apply them to the entity.params dict. @@ -185,7 +185,7 @@ def process_urlencoded(entity): def process_multipart(entity): """Read all multipart parts into entity.parts.""" - ib = "" + ib = '' if 'boundary' in entity.content_type.params: # http://tools.ietf.org/html/rfc2046#section-5.1.1 # "The grammar for parameters on the Content-type field is such that it @@ -193,7 +193,7 @@ def process_multipart(entity): # on the Content-type line" ib = entity.content_type.params['boundary'].strip('"') - if not re.match("^[ -~]{0,200}[!-~]$", ib): + if not re.match('^[ -~]{0,200}[!-~]$', ib): raise ValueError('Invalid boundary in multipart form: %r' % (ib,)) ib = ('--' + ib).encode('ascii') @@ -428,7 +428,7 @@ class Entity(object): # Copy the class 'attempt_charsets', prepending any Content-Type # charset - dec = self.content_type.params.get("charset", None) + dec = self.content_type.params.get('charset', None) if dec: self.attempt_charsets = [dec] + [c for c in self.attempt_charsets if c != dec] @@ -469,8 +469,8 @@ class Entity(object): # The 'type' attribute is deprecated in 3.2; remove it in 3.3. type = property( lambda self: self.content_type, - doc="A deprecated alias for " - ":attr:`content_type`." + doc='A deprecated alias for ' + ':attr:`content_type`.' ) def read(self, size=None, fp_out=None): @@ -536,8 +536,8 @@ class Entity(object): else: raise cherrypy.HTTPError( 400, - "The request entity could not be decoded. The following " - "charsets were attempted: %s" % repr(self.attempt_charsets) + 'The request entity could not be decoded. The following ' + 'charsets were attempted: %s' % repr(self.attempt_charsets) ) def process(self): @@ -613,40 +613,40 @@ class Part(Entity): self.file = None self.value = None + @classmethod def from_fp(cls, fp, boundary): headers = cls.read_headers(fp) return cls(fp, headers, boundary) - from_fp = classmethod(from_fp) + @classmethod def read_headers(cls, fp): headers = httputil.HeaderMap() while True: line = fp.readline() if not line: # No more data--illegal end of headers - raise EOFError("Illegal end of headers.") + raise EOFError('Illegal end of headers.') - if line == ntob('\r\n') or line == ntob('\n'): + if line == ntob('\r\n'): # Normal end of headers break - if not line.endswith(ntob('\n')): - raise ValueError("MIME requires CRLF terminators: %r" % line) + if not line.endswith(ntob('\r\n')): + raise ValueError('MIME requires CRLF terminators: %r' % line) if line[0] in ntob(' \t'): # It's a continuation line. v = line.strip().decode('ISO-8859-1') else: - k, v = line.split(ntob(":"), 1) + k, v = line.split(ntob(':'), 1) k = k.strip().decode('ISO-8859-1') v = v.strip().decode('ISO-8859-1') existing = headers.get(k) if existing: - v = ", ".join((existing, v)) + v = ', '.join((existing, v)) headers[k] = v return headers - read_headers = classmethod(read_headers) def read_lines_to_boundary(self, fp_out=None): """Read bytes from self.fp and return or write them to a file. @@ -658,16 +658,16 @@ class Part(Entity): object that supports the 'write' method; all bytes read will be written to the fp, and that fp is returned. """ - endmarker = self.boundary + ntob("--") - delim = ntob("") + endmarker = self.boundary + ntob('--') + delim = ntob('') prev_lf = True lines = [] seen = 0 while True: line = self.fp.readline(1 << 16) if not line: - raise EOFError("Illegal end of multipart body.") - if line.startswith(ntob("--")) and prev_lf: + raise EOFError('Illegal end of multipart body.') + if line.startswith(ntob('--')) and prev_lf: strippedline = line.strip() if strippedline == self.boundary: break @@ -677,16 +677,16 @@ class Part(Entity): line = delim + line - if line.endswith(ntob("\r\n")): - delim = ntob("\r\n") + if line.endswith(ntob('\r\n')): + delim = ntob('\r\n') line = line[:-2] prev_lf = True - elif line.endswith(ntob("\n")): - delim = ntob("\n") + elif line.endswith(ntob('\n')): + delim = ntob('\n') line = line[:-1] prev_lf = True else: - delim = ntob("") + delim = ntob('') prev_lf = False if fp_out is None: @@ -715,7 +715,7 @@ class Part(Entity): self.file = self.read_into_file() else: result = self.read_lines_to_boundary() - if isinstance(result, basestring): + if isinstance(result, text_or_bytes): self.value = result else: self.file = result @@ -732,19 +732,7 @@ class Part(Entity): Entity.part_class = Part -try: - inf = float('inf') -except ValueError: - # Python 2.4 and lower - class Infinity(object): - - def __cmp__(self, other): - return 1 - - def __sub__(self, other): - return self - inf = Infinity() - +inf = float('inf') comma_separated_headers = [ 'Accept', 'Accept-Charset', 'Accept-Encoding', @@ -839,7 +827,7 @@ class SizedReader: if e.__class__.__name__ == 'MaxSizeExceeded': # Post data is too big raise cherrypy.HTTPError( - 413, "Maximum request length: %r" % e.args[1]) + 413, 'Maximum request length: %r' % e.args[1]) else: raise if not data: @@ -915,23 +903,23 @@ class SizedReader: v = line.strip() else: try: - k, v = line.split(ntob(":"), 1) + k, v = line.split(ntob(':'), 1) except ValueError: - raise ValueError("Illegal header line.") + raise ValueError('Illegal header line.') k = k.strip().title() v = v.strip() if k in comma_separated_headers: existing = self.trailers.get(envname) if existing: - v = ntob(", ").join((existing, v)) + v = ntob(', ').join((existing, v)) self.trailers[k] = v except Exception: e = sys.exc_info()[1] if e.__class__.__name__ == 'MaxSizeExceeded': # Post data is too big raise cherrypy.HTTPError( - 413, "Maximum request length: %r" % e.args[1]) + 413, 'Maximum request length: %r' % e.args[1]) else: raise diff --git a/cherrypy/_cprequest.py b/cherrypy/_cprequest.py index a40e51b..267f210 100644 --- a/cherrypy/_cprequest.py +++ b/cherrypy/_cprequest.py @@ -5,7 +5,7 @@ import warnings import six import cherrypy -from cherrypy._cpcompat import basestring, copykeys, ntob +from cherrypy._cpcompat import text_or_bytes, copykeys, ntob from cherrypy._cpcompat import SimpleCookie, CookieError from cherrypy import _cpreqbody, _cpconfig from cherrypy._cperror import format_exc, bare_error @@ -41,11 +41,11 @@ class Hook(object): self.callback = callback if failsafe is None: - failsafe = getattr(callback, "failsafe", False) + failsafe = getattr(callback, 'failsafe', False) self.failsafe = failsafe if priority is None: - priority = getattr(callback, "priority", 50) + priority = getattr(callback, 'priority', 50) self.priority = priority self.kwargs = kwargs @@ -64,10 +64,10 @@ class Hook(object): def __repr__(self): cls = self.__class__ - return ("%s.%s(callback=%r, failsafe=%r, priority=%r, %s)" + return ('%s.%s(callback=%r, failsafe=%r, priority=%r, %s)' % (cls.__module__, cls.__name__, self.callback, self.failsafe, self.priority, - ", ".join(['%s=%r' % (k, v) + ', '.join(['%s=%r' % (k, v) for k, v in self.kwargs.items()]))) @@ -124,7 +124,7 @@ class HookMap(dict): def __repr__(self): cls = self.__class__ - return "%s.%s(points=%r)" % ( + return '%s.%s(points=%r)' % ( cls.__module__, cls.__name__, copykeys(self) @@ -138,8 +138,8 @@ def hooks_namespace(k, v): # Use split again to allow multiple hooks for a single # hookpoint per path (e.g. "hooks.before_handler.1"). # Little-known fact you only get from reading source ;) - hookpoint = k.split(".", 1)[0] - if isinstance(v, basestring): + hookpoint = k.split('.', 1)[0] + if isinstance(v, text_or_bytes): v = cherrypy.lib.attributes(v) if not isinstance(v, Hook): v = Hook(v) @@ -199,23 +199,23 @@ class Request(object): unless we are processing an InternalRedirect.""" # Conversation/connection attributes - local = httputil.Host("127.0.0.1", 80) - "An httputil.Host(ip, port, hostname) object for the server socket." + local = httputil.Host('127.0.0.1', 80) + 'An httputil.Host(ip, port, hostname) object for the server socket.' - remote = httputil.Host("127.0.0.1", 1111) - "An httputil.Host(ip, port, hostname) object for the client socket." + remote = httputil.Host('127.0.0.1', 1111) + 'An httputil.Host(ip, port, hostname) object for the client socket.' - scheme = "http" + scheme = 'http' """ The protocol used between client and server. In most cases, this will be either 'http' or 'https'.""" - server_protocol = "HTTP/1.1" + server_protocol = 'HTTP/1.1' """ The HTTP version for which the HTTP server is at least conditionally compliant.""" - base = "" + base = '' """The (scheme://host) portion of the requested URL. In some cases (e.g. when proxying via mod_rewrite), this may contain path segments which cherrypy.url uses when constructing url's, but @@ -223,13 +223,13 @@ class Request(object): MUST NOT end in a slash.""" # Request-Line attributes - request_line = "" + request_line = '' """ The complete Request-Line received from the client. This is a single string consisting of the request method, URI, and protocol version (joined by spaces). Any final CRLF is removed.""" - method = "GET" + method = 'GET' """ Indicates the HTTP method to be performed on the resource identified by the Request-URI. Common methods include GET, HEAD, POST, PUT, and @@ -237,7 +237,7 @@ class Request(object): servers and gateways may restrict the set of allowable methods. CherryPy applications SHOULD restrict the set (on a per-URI basis).""" - query_string = "" + query_string = '' """ The query component of the Request-URI, a string of information to be interpreted by the resource. The query portion of a URI follows the @@ -312,7 +312,7 @@ class Request(object): If True, the rfile (if any) is automatically read and parsed, and the result placed into request.params or request.body.""" - methods_with_bodies = ("POST", "PUT") + methods_with_bodies = ('POST', 'PUT') """ A sequence of HTTP methods for which CherryPy will automatically attempt to read a body from the rfile. If you are going to change @@ -341,7 +341,7 @@ class Request(object): to a hierarchical arrangement of objects, starting at request.app.root. See help(cherrypy.dispatch) for more information.""" - script_name = "" + script_name = '' """ The 'mount point' of the application which is handling this request. @@ -349,7 +349,7 @@ class Request(object): the root of the URI, it MUST be an empty string (not "/"). """ - path_info = "/" + path_info = '/' """ The 'relative path' portion of the Request-URI. This is relative to the script_name ('mount point') of the application which is @@ -468,15 +468,15 @@ class Request(object): This is useful when debugging a live server with hung requests.""" namespaces = _cpconfig.NamespaceSet( - **{"hooks": hooks_namespace, - "request": request_namespace, - "response": response_namespace, - "error_page": error_page_namespace, - "tools": cherrypy.tools, + **{'hooks': hooks_namespace, + 'request': request_namespace, + 'response': response_namespace, + 'error_page': error_page_namespace, + 'tools': cherrypy.tools, }) - def __init__(self, local_host, remote_host, scheme="http", - server_protocol="HTTP/1.1"): + def __init__(self, local_host, remote_host, scheme='http', + server_protocol='HTTP/1.1'): """Populate a new Request object. local_host should be an httputil.Host object with the server info. @@ -544,7 +544,7 @@ class Request(object): self.error_response = cherrypy.HTTPError(500).set_response self.method = method - path = path or "/" + path = path or '/' self.query_string = query_string or '' self.params = {} @@ -600,11 +600,11 @@ class Request(object): if self.show_tracebacks: body = format_exc() else: - body = "" + body = '' r = bare_error(body) response.output_status, response.header_list, response.body = r - if self.method == "HEAD": + if self.method == 'HEAD': # HEAD requests MUST NOT return a message-body in the response. response.body = [] @@ -696,8 +696,8 @@ class Request(object): self.query_string, encoding=self.query_string_encoding) except UnicodeDecodeError: raise cherrypy.HTTPError( - 404, "The given query string could not be processed. Query " - "strings for this resource must be encoded with %r." % + 404, 'The given query string could not be processed. Query ' + 'strings for this resource must be encoded with %r.' % self.query_string_encoding) # Python 2 only: keyword arguments must be byte strings (type 'str'). @@ -722,7 +722,7 @@ class Request(object): # (AFAIK, only Konqueror does that), only the last one will # remain in headers (but they will be correctly stored in # request.cookie). - if "=?" in value: + if '=?' in value: dict.__setitem__(headers, name, httputil.decode_TEXT(value)) else: dict.__setitem__(headers, name, value) @@ -733,7 +733,7 @@ class Request(object): try: self.cookie.load(value) except CookieError: - msg = "Illegal cookie name %s" % value.split('=')[0] + msg = 'Illegal cookie name %s' % value.split('=')[0] raise cherrypy.HTTPError(400, msg) if not dict.__contains__(headers, 'Host'): @@ -746,7 +746,7 @@ class Request(object): host = dict.get(headers, 'Host') if not host: host = self.local.name or self.local.ip - self.base = "%s://%s" % (self.scheme, host) + self.base = '%s://%s' % (self.scheme, host) def get_resource(self, path): """Call a dispatcher (which sets self.handler and .config). (Core)""" @@ -754,7 +754,7 @@ class Request(object): # dispatchers can only be specified in app.config, not in _cp_config # (since custom dispatchers may not even have an app.root). dispatch = self.app.find_config( - path, "request.dispatch", self.dispatch) + path, 'request.dispatch', self.dispatch) # dispatch() should set self.handler and self.config dispatch(path) @@ -762,10 +762,10 @@ class Request(object): def handle_error(self): """Handle the last unanticipated exception. (Core)""" try: - self.hooks.run("before_error_response") + self.hooks.run('before_error_response') if self.error_response: self.error_response() - self.hooks.run("after_error_response") + self.hooks.run('after_error_response') cherrypy.serving.response.finalize() except cherrypy.HTTPRedirect: inst = sys.exc_info()[1] @@ -776,8 +776,8 @@ class Request(object): def _get_body_params(self): warnings.warn( - "body_params is deprecated in CherryPy 3.2, will be removed in " - "CherryPy 3.3.", + 'body_params is deprecated in CherryPy 3.2, will be removed in ' + 'CherryPy 3.3.', DeprecationWarning ) return self.body.params @@ -800,8 +800,8 @@ class ResponseBody(object): """The body of the HTTP response (the response entity).""" if six.PY3: - unicode_err = ("Page handlers MUST return bytes. Use tools.encode " - "if you wish to return unicode.") + unicode_err = ('Page handlers MUST return bytes. Use tools.encode ' + 'if you wish to return unicode.') def __get__(self, obj, objclass=None): if obj is None: @@ -815,7 +815,7 @@ class ResponseBody(object): if six.PY3 and isinstance(value, str): raise ValueError(self.unicode_err) - if isinstance(value, basestring): + if isinstance(value, text_or_bytes): # strings get wrapped in a list because iterating over a single # item list is much faster than iterating over every character # in a long string. @@ -842,7 +842,7 @@ class Response(object): """An HTTP Response, including status, headers, and body.""" - status = "" + status = '' """The HTTP Status-Code and Reason-Phrase.""" header_list = [] @@ -893,15 +893,15 @@ class Response(object): # Since we know all our keys are titled strings, we can # bypass HeaderMap.update and get a big speed boost. dict.update(self.headers, { - "Content-Type": 'text/html', - "Server": "CherryPy/" + cherrypy.__version__, - "Date": httputil.HTTPDate(self.time), + 'Content-Type': 'text/html', + 'Server': 'CherryPy/' + cherrypy.__version__, + 'Date': httputil.HTTPDate(self.time), }) self.cookie = SimpleCookie() def collapse_body(self): """Collapse self.body to a single string; replace it and return it.""" - if isinstance(self.body, basestring): + if isinstance(self.body, text_or_bytes): return self.body newbody = [] @@ -924,9 +924,9 @@ class Response(object): headers = self.headers - self.status = "%s %s" % (code, reason) + self.status = '%s %s' % (code, reason) self.output_status = ntob(str(code), 'ascii') + \ - ntob(" ") + headers.encode(reason) + ntob(' ') + headers.encode(reason) if self.stream: # The upshot: wsgiserver will chunk the response if @@ -939,7 +939,7 @@ class Response(object): # and 304 (not modified) responses MUST NOT # include a message-body." dict.pop(headers, 'Content-Length', None) - self.body = ntob("") + self.body = ntob('') else: # Responses which are not streamed should have a Content-Length, # but allow user code to set Content-Length if desired. @@ -952,13 +952,10 @@ class Response(object): cookie = self.cookie.output() if cookie: - for line in cookie.split("\n"): - if line.endswith("\r"): - # Python 2.4 emits cookies joined by LF but 2.5+ by CRLF. - line = line[:-1] - name, value = line.split(": ", 1) + for line in cookie.split('\r\n'): + name, value = line.split(': ', 1) if isinstance(name, six.text_type): - name = name.encode("ISO-8859-1") + name = name.encode('ISO-8859-1') if isinstance(value, six.text_type): value = headers.encode(value) h.append((name, value)) diff --git a/cherrypy/_cpserver.py b/cherrypy/_cpserver.py index e47085c..9fed02f 100644 --- a/cherrypy/_cpserver.py +++ b/cherrypy/_cpserver.py @@ -3,8 +3,8 @@ import six import cherrypy -from cherrypy.lib import attributes -from cherrypy._cpcompat import basestring +from cherrypy.lib.reprconf import attributes +from cherrypy._cpcompat import text_or_bytes # We import * because we want to export check_port # et al as attributes of this module. @@ -35,7 +35,7 @@ class Server(ServerAdapter): if value == '': raise ValueError("The empty string ('') is not an allowed value. " "Use '0.0.0.0' instead to listen on all active " - "interfaces (INADDR_ANY).") + 'interfaces (INADDR_ANY).') self._socket_host = value socket_host = property( _get_socket_host, @@ -156,7 +156,7 @@ class Server(ServerAdapter): if httpserver is None: from cherrypy import _cpwsgi_server httpserver = _cpwsgi_server.CPWSGIServer(self) - if isinstance(httpserver, basestring): + if isinstance(httpserver, text_or_bytes): # Is anyone using this? Can I add an arg? httpserver = attributes(httpserver)(self) return httpserver, self.bind_addr @@ -180,7 +180,7 @@ class Server(ServerAdapter): self.socket_file = None self.socket_host = None self.socket_port = None - elif isinstance(value, basestring): + elif isinstance(value, text_or_bytes): self.socket_file = value self.socket_host = None self.socket_port = None @@ -189,9 +189,9 @@ class Server(ServerAdapter): self.socket_host, self.socket_port = value self.socket_file = None except ValueError: - raise ValueError("bind_addr must be a (host, port) tuple " - "(for TCP sockets) or a string (for Unix " - "domain sockets), not %r" % value) + raise ValueError('bind_addr must be a (host, port) tuple ' + '(for TCP sockets) or a string (for Unix ' + 'domain sockets), not %r' % value) bind_addr = property( _get_bind_addr, _set_bind_addr, @@ -215,12 +215,12 @@ class Server(ServerAdapter): port = self.socket_port if self.ssl_certificate: - scheme = "https" + scheme = 'https' if port != 443: - host += ":%s" % port + host += ':%s' % port else: - scheme = "http" + scheme = 'http' if port != 80: - host += ":%s" % port + host += ':%s' % port - return "%s://%s" % (scheme, host) + return '%s://%s' % (scheme, host) diff --git a/cherrypy/_cpthreadinglocal.py b/cherrypy/_cpthreadinglocal.py deleted file mode 100644 index 238c322..0000000 --- a/cherrypy/_cpthreadinglocal.py +++ /dev/null @@ -1,241 +0,0 @@ -# This is a backport of Python-2.4's threading.local() implementation - -"""Thread-local objects - -(Note that this module provides a Python version of thread - threading.local class. Depending on the version of Python you're - using, there may be a faster one available. You should always import - the local class from threading.) - -Thread-local objects support the management of thread-local data. -If you have data that you want to be local to a thread, simply create -a thread-local object and use its attributes: - - >>> mydata = local() - >>> mydata.number = 42 - >>> mydata.number - 42 - -You can also access the local-object's dictionary: - - >>> mydata.__dict__ - {'number': 42} - >>> mydata.__dict__.setdefault('widgets', []) - [] - >>> mydata.widgets - [] - -What's important about thread-local objects is that their data are -local to a thread. If we access the data in a different thread: - - >>> log = [] - >>> def f(): - ... items = mydata.__dict__.items() - ... items.sort() - ... log.append(items) - ... mydata.number = 11 - ... log.append(mydata.number) - - >>> import threading - >>> thread = threading.Thread(target=f) - >>> thread.start() - >>> thread.join() - >>> log - [[], 11] - -we get different data. Furthermore, changes made in the other thread -don't affect data seen in this thread: - - >>> mydata.number - 42 - -Of course, values you get from a local object, including a __dict__ -attribute, are for whatever thread was current at the time the -attribute was read. For that reason, you generally don't want to save -these values across threads, as they apply only to the thread they -came from. - -You can create custom local objects by subclassing the local class: - - >>> class MyLocal(local): - ... number = 2 - ... initialized = False - ... def __init__(self, **kw): - ... if self.initialized: - ... raise SystemError('__init__ called too many times') - ... self.initialized = True - ... self.__dict__.update(kw) - ... def squared(self): - ... return self.number ** 2 - -This can be useful to support default values, methods and -initialization. Note that if you define an __init__ method, it will be -called each time the local object is used in a separate thread. This -is necessary to initialize each thread's dictionary. - -Now if we create a local object: - - >>> mydata = MyLocal(color='red') - -Now we have a default number: - - >>> mydata.number - 2 - -an initial color: - - >>> mydata.color - 'red' - >>> del mydata.color - -And a method that operates on the data: - - >>> mydata.squared() - 4 - -As before, we can access the data in a separate thread: - - >>> log = [] - >>> thread = threading.Thread(target=f) - >>> thread.start() - >>> thread.join() - >>> log - [[('color', 'red'), ('initialized', True)], 11] - -without affecting this thread's data: - - >>> mydata.number - 2 - >>> mydata.color - Traceback (most recent call last): - ... - AttributeError: 'MyLocal' object has no attribute 'color' - -Note that subclasses can define slots, but they are not thread -local. They are shared across threads: - - >>> class MyLocal(local): - ... __slots__ = 'number' - - >>> mydata = MyLocal() - >>> mydata.number = 42 - >>> mydata.color = 'red' - -So, the separate thread: - - >>> thread = threading.Thread(target=f) - >>> thread.start() - >>> thread.join() - -affects what we see: - - >>> mydata.number - 11 - ->>> del mydata -""" - -# Threading import is at end - - -class _localbase(object): - __slots__ = '_local__key', '_local__args', '_local__lock' - - def __new__(cls, *args, **kw): - self = object.__new__(cls) - key = 'thread.local.' + str(id(self)) - object.__setattr__(self, '_local__key', key) - object.__setattr__(self, '_local__args', (args, kw)) - object.__setattr__(self, '_local__lock', RLock()) - - if args or kw and (cls.__init__ is object.__init__): - raise TypeError("Initialization arguments are not supported") - - # We need to create the thread dict in anticipation of - # __init__ being called, to make sure we don't call it - # again ourselves. - dict = object.__getattribute__(self, '__dict__') - currentThread().__dict__[key] = dict - - return self - - -def _patch(self): - key = object.__getattribute__(self, '_local__key') - d = currentThread().__dict__.get(key) - if d is None: - d = {} - currentThread().__dict__[key] = d - object.__setattr__(self, '__dict__', d) - - # we have a new instance dict, so call out __init__ if we have - # one - cls = type(self) - if cls.__init__ is not object.__init__: - args, kw = object.__getattribute__(self, '_local__args') - cls.__init__(self, *args, **kw) - else: - object.__setattr__(self, '__dict__', d) - - -class local(_localbase): - - def __getattribute__(self, name): - lock = object.__getattribute__(self, '_local__lock') - lock.acquire() - try: - _patch(self) - return object.__getattribute__(self, name) - finally: - lock.release() - - def __setattr__(self, name, value): - lock = object.__getattribute__(self, '_local__lock') - lock.acquire() - try: - _patch(self) - return object.__setattr__(self, name, value) - finally: - lock.release() - - def __delattr__(self, name): - lock = object.__getattribute__(self, '_local__lock') - lock.acquire() - try: - _patch(self) - return object.__delattr__(self, name) - finally: - lock.release() - - def __del__(): - threading_enumerate = enumerate - __getattribute__ = object.__getattribute__ - - def __del__(self): - key = __getattribute__(self, '_local__key') - - try: - threads = list(threading_enumerate()) - except: - # if enumerate fails, as it seems to do during - # shutdown, we'll skip cleanup under the assumption - # that there is nothing to clean up - return - - for thread in threads: - try: - __dict__ = thread.__dict__ - except AttributeError: - # Thread is dying, rest in peace - continue - - if key in __dict__: - try: - del __dict__[key] - except KeyError: - pass # didn't have anything in this thread - - return __del__ - __del__ = __del__() - -from threading import currentThread, enumerate, RLock diff --git a/cherrypy/_cptools.py b/cherrypy/_cptools.py index 54c6373..98d3b21 100644 --- a/cherrypy/_cptools.py +++ b/cherrypy/_cptools.py @@ -28,6 +28,11 @@ import warnings import cherrypy from cherrypy._helper import expose +from cherrypy.lib import cptools, encoding, auth, static, jsontools +from cherrypy.lib import sessions as _sessions, xmlrpcutil as _xmlrpc +from cherrypy.lib import caching as _caching +from cherrypy.lib import auth_basic, auth_digest + def _getargs(func): """Return the names of all static arguments to the given function.""" @@ -45,8 +50,8 @@ def _getargs(func): _attr_error = ( - "CherryPy Tools cannot be turned on directly. Instead, turn them " - "on via config, or use them as decorators on your page handlers." + 'CherryPy Tools cannot be turned on directly. Instead, turn them ' + 'on via config, or use them as decorators on your page handlers.' ) @@ -57,7 +62,7 @@ class Tool(object): help(tool.callable) should give you more information about this Tool. """ - namespace = "tools" + namespace = 'tools' def __init__(self, point, callable, name=None, priority=50): self._point = point @@ -80,7 +85,7 @@ class Tool(object): for arg in _getargs(self.callable): setattr(self, arg, None) except (TypeError, AttributeError): - if hasattr(self.callable, "__call__"): + if hasattr(self.callable, '__call__'): for arg in _getargs(self.callable.__call__): setattr(self, arg, None) # IronPython 1.0 raises NotImplementedError because @@ -104,8 +109,8 @@ class Tool(object): if self._name in tm: conf.update(tm[self._name]) - if "on" in conf: - del conf["on"] + if 'on' in conf: + del conf['on'] return conf @@ -120,15 +125,15 @@ class Tool(object): return cherrypy.request.base """ if args: - raise TypeError("The %r Tool does not accept positional " - "arguments; you must use keyword arguments." + raise TypeError('The %r Tool does not accept positional ' + 'arguments; you must use keyword arguments.' % self._name) def tool_decorator(f): - if not hasattr(f, "_cp_config"): + if not hasattr(f, '_cp_config'): f._cp_config = {} - subspace = self.namespace + "." + self._name + "." - f._cp_config[subspace + "on"] = True + subspace = self.namespace + '.' + self._name + '.' + f._cp_config[subspace + 'on'] = True for k, v in kwargs.items(): f._cp_config[subspace + k] = v return f @@ -141,9 +146,9 @@ class Tool(object): method when the tool is "turned on" in config. """ conf = self._merged_args() - p = conf.pop("priority", None) + p = conf.pop('priority', None) if p is None: - p = getattr(self.callable, "priority", self._priority) + p = getattr(self.callable, 'priority', self._priority) cherrypy.serving.request.hooks.attach(self._point, self.callable, priority=p, **conf) @@ -191,9 +196,9 @@ class HandlerTool(Tool): method when the tool is "turned on" in config. """ conf = self._merged_args() - p = conf.pop("priority", None) + p = conf.pop('priority', None) if p is None: - p = getattr(self.callable, "priority", self._priority) + p = getattr(self.callable, 'priority', self._priority) cherrypy.serving.request.hooks.attach(self._point, self._wrapper, priority=p, **conf) @@ -254,11 +259,6 @@ class ErrorTool(Tool): # Builtin tools # -from cherrypy.lib import cptools, encoding, auth, static, jsontools -from cherrypy.lib import sessions as _sessions, xmlrpcutil as _xmlrpc -from cherrypy.lib import caching as _caching -from cherrypy.lib import auth_basic, auth_digest - class SessionTool(Tool): @@ -296,9 +296,9 @@ class SessionTool(Tool): conf = self._merged_args() - p = conf.pop("priority", None) + p = conf.pop('priority', None) if p is None: - p = getattr(self.callable, "priority", self._priority) + p = getattr(self.callable, 'priority', self._priority) hooks.attach(self._point, self.callable, priority=p, **conf) @@ -374,7 +374,7 @@ class XMLRPCController(object): for attr in str(rpcmethod).split('.'): subhandler = getattr(subhandler, attr, None) - if subhandler and getattr(subhandler, "exposed", False): + if subhandler and getattr(subhandler, 'exposed', False): body = subhandler(*(vpath + rpcparams), **params) else: @@ -384,7 +384,7 @@ class XMLRPCController(object): # cherrypy.lib.xmlrpcutil.on_error raise Exception('method "%s" is not supported' % attr) - conf = cherrypy.serving.request.toolmaps['tools'].get("xmlrpc", {}) + conf = cherrypy.serving.request.toolmaps['tools'].get('xmlrpc', {}) _xmlrpc.respond(body, conf.get('encoding', 'utf-8'), conf.get('allow_none', 0)) @@ -395,7 +395,7 @@ class SessionAuthTool(HandlerTool): def _setargs(self): for name in dir(cptools.SessionAuth): - if not name.startswith("__"): + if not name.startswith('__'): setattr(self, name, None) @@ -418,7 +418,7 @@ class CachingTool(Tool): """Hook caching into cherrypy.request.""" conf = self._merged_args() - p = conf.pop("priority", None) + p = conf.pop('priority', None) cherrypy.serving.request.hooks.attach('before_handler', self._wrapper, priority=p, **conf) @@ -447,7 +447,7 @@ class Toolbox(object): cherrypy.serving.request.toolmaps[self.namespace] = map = {} def populate(k, v): - toolname, arg = k.split(".", 1) + toolname, arg = k.split('.', 1) bucket = map.setdefault(toolname, {}) bucket[arg] = v return populate @@ -457,7 +457,7 @@ class Toolbox(object): map = cherrypy.serving.request.toolmaps.get(self.namespace) if map: for name, settings in map.items(): - if settings.get("on", False): + if settings.get('on', False): tool = getattr(self, name) tool._setup() @@ -472,7 +472,7 @@ class Toolbox(object): class DeprecatedTool(Tool): _name = None - warnmsg = "This Tool is deprecated." + warnmsg = 'This Tool is deprecated.' def __init__(self, point, warnmsg=None): self.point = point @@ -490,7 +490,7 @@ class DeprecatedTool(Tool): warnings.warn(self.warnmsg) -default_toolbox = _d = Toolbox("tools") +default_toolbox = _d = Toolbox('tools') _d.session_auth = SessionAuthTool(cptools.session_auth) _d.allow = Tool('on_start_resource', cptools.allow) _d.proxy = Tool('before_request_body', cptools.proxy, priority=30) @@ -512,14 +512,14 @@ _d.caching = CachingTool('before_handler', _caching.get, 'caching') _d.expires = Tool('before_finalize', _caching.expires) _d.tidy = DeprecatedTool( 'before_finalize', - "The tidy tool has been removed from the standard distribution of " - "CherryPy. The most recent version can be found at " - "http://tools.cherrypy.org/browser.") + 'The tidy tool has been removed from the standard distribution of ' + 'CherryPy. The most recent version can be found at ' + 'http://tools.cherrypy.org/browser.') _d.nsgmls = DeprecatedTool( 'before_finalize', - "The nsgmls tool has been removed from the standard distribution of " - "CherryPy. The most recent version can be found at " - "http://tools.cherrypy.org/browser.") + 'The nsgmls tool has been removed from the standard distribution of ' + 'CherryPy. The most recent version can be found at ' + 'http://tools.cherrypy.org/browser.') _d.ignore_headers = Tool('before_request_body', cptools.ignore_headers) _d.referer = Tool('before_request_body', cptools.referer) _d.basic_auth = Tool('on_start_resource', auth.basic_auth) @@ -533,5 +533,6 @@ _d.json_in = Tool('before_request_body', jsontools.json_in, priority=30) _d.json_out = Tool('before_handler', jsontools.json_out, priority=30) _d.auth_basic = Tool('before_handler', auth_basic.basic_auth, priority=1) _d.auth_digest = Tool('before_handler', auth_digest.digest_auth, priority=1) +_d.params = Tool('before_handler', cptools.convert_params) del _d, cptools, encoding, auth, static diff --git a/cherrypy/_cptree.py b/cherrypy/_cptree.py index c40e3b3..93ad101 100644 --- a/cherrypy/_cptree.py +++ b/cherrypy/_cptree.py @@ -46,22 +46,22 @@ class Application(object): relative_urls = False - def __init__(self, root, script_name="", config=None): + def __init__(self, root, script_name='', config=None): self.log = _cplogging.LogManager(id(self), cherrypy.log.logger_root) self.root = root self.script_name = script_name self.wsgiapp = _cpwsgi.CPWSGIApp(self) self.namespaces = self.namespaces.copy() - self.namespaces["log"] = lambda k, v: setattr(self.log, k, v) - self.namespaces["wsgi"] = self.wsgiapp.namespace_handler + self.namespaces['log'] = lambda k, v: setattr(self.log, k, v) + self.namespaces['wsgi'] = self.wsgiapp.namespace_handler self.config = self.__class__.config.copy() if config: self.merge(config) def __repr__(self): - return "%s.%s(%r, %r)" % (self.__module__, self.__class__.__name__, + return '%s.%s(%r, %r)' % (self.__module__, self.__class__.__name__, self.root, self.script_name) script_name_doc = """The URI "mount point" for this app. A mount point @@ -86,11 +86,11 @@ class Application(object): # A `_script_name` with a value of None signals that the script name # should be pulled from WSGI environ. - return cherrypy.serving.request.wsgi_environ['SCRIPT_NAME'].rstrip("/") + return cherrypy.serving.request.wsgi_environ['SCRIPT_NAME'].rstrip('/') def _set_script_name(self, value): if value: - value = value.rstrip("/") + value = value.rstrip('/') self._script_name = value script_name = property(fget=_get_script_name, fset=_set_script_name, doc=script_name_doc) @@ -100,22 +100,22 @@ class Application(object): _cpconfig.merge(self.config, config) # Handle namespaces specified in config. - self.namespaces(self.config.get("/", {})) + self.namespaces(self.config.get('/', {})) def find_config(self, path, key, default=None): """Return the most-specific value for key along path, or default.""" - trail = path or "/" + trail = path or '/' while trail: nodeconf = self.config.get(trail, {}) if key in nodeconf: return nodeconf[key] - lastslash = trail.rfind("/") + lastslash = trail.rfind('/') if lastslash == -1: break - elif lastslash == 0 and trail != "/": - trail = "/" + elif lastslash == 0 and trail != '/': + trail = '/' else: trail = trail[:lastslash] @@ -172,7 +172,7 @@ class Tree(object): def __init__(self): self.apps = {} - def mount(self, root, script_name="", config=None): + def mount(self, root, script_name='', config=None): """Mount a new app from a root object, script_name, and config. root @@ -197,29 +197,29 @@ class Tree(object): if script_name is None: raise TypeError( "The 'script_name' argument may not be None. Application " - "objects may, however, possess a script_name of None (in " - "order to inpect the WSGI environ for SCRIPT_NAME upon each " - "request). You cannot mount such Applications on this Tree; " - "you must pass them to a WSGI server interface directly.") + 'objects may, however, possess a script_name of None (in ' + 'order to inpect the WSGI environ for SCRIPT_NAME upon each ' + 'request). You cannot mount such Applications on this Tree; ' + 'you must pass them to a WSGI server interface directly.') # Next line both 1) strips trailing slash and 2) maps "/" -> "". - script_name = script_name.rstrip("/") + script_name = script_name.rstrip('/') if isinstance(root, Application): app = root - if script_name != "" and script_name != app.script_name: + if script_name != '' and script_name != app.script_name: raise ValueError( - "Cannot specify a different script name and pass an " - "Application instance to cherrypy.mount") + 'Cannot specify a different script name and pass an ' + 'Application instance to cherrypy.mount') script_name = app.script_name else: app = Application(root, script_name) # If mounted at "", add favicon.ico - if (script_name == "" and root is not None - and not hasattr(root, "favicon_ico")): + if (script_name == '' and root is not None + and not hasattr(root, 'favicon_ico')): favicon = os.path.join(os.getcwd(), os.path.dirname(__file__), - "favicon.ico") + 'favicon.ico') root.favicon_ico = tools.staticfile.handler(favicon) if config: @@ -229,10 +229,10 @@ class Tree(object): return app - def graft(self, wsgi_callable, script_name=""): + def graft(self, wsgi_callable, script_name=''): """Mount a wsgi callable at the given script_name.""" # Next line both 1) strips trailing slash and 2) maps "/" -> "". - script_name = script_name.rstrip("/") + script_name = script_name.rstrip('/') self.apps[script_name] = wsgi_callable def script_name(self, path=None): @@ -252,11 +252,11 @@ class Tree(object): if path in self.apps: return path - if path == "": + if path == '': return None # Move one node up the tree and try again. - path = path[:path.rfind("/")] + path = path[:path.rfind('/')] def __call__(self, environ, start_response): # If you're calling this, then you're probably setting SCRIPT_NAME @@ -267,7 +267,7 @@ class Tree(object): env1x = _cpwsgi.downgrade_wsgi_ux_to_1x(environ) path = httputil.urljoin(env1x.get('SCRIPT_NAME', ''), env1x.get('PATH_INFO', '')) - sn = self.script_name(path or "/") + sn = self.script_name(path or '/') if sn is None: start_response('404 Not Found', []) return [] @@ -280,8 +280,8 @@ class Tree(object): # Python 2/WSGI u.0: all strings MUST be of type unicode enc = environ[ntou('wsgi.url_encoding')] environ[ntou('SCRIPT_NAME')] = sn.decode(enc) - environ[ntou('PATH_INFO')] = path[len(sn.rstrip("/")):].decode(enc) + environ[ntou('PATH_INFO')] = path[len(sn.rstrip('/')):].decode(enc) else: environ['SCRIPT_NAME'] = sn - environ['PATH_INFO'] = path[len(sn.rstrip("/")):] + environ['PATH_INFO'] = path[len(sn.rstrip('/')):] return app(environ, start_response) diff --git a/cherrypy/_cpwsgi.py b/cherrypy/_cpwsgi.py index ec72f2b..fe650f0 100644 --- a/cherrypy/_cpwsgi.py +++ b/cherrypy/_cpwsgi.py @@ -46,10 +46,13 @@ class VirtualHost(object): Domain2App = cherrypy.Application(root) SecureApp = cherrypy.Application(Secure()) - vhost = cherrypy._cpwsgi.VirtualHost(RootApp, - domains={'www.domain2.example': Domain2App, - 'www.domain2.example:443': SecureApp, - }) + vhost = cherrypy._cpwsgi.VirtualHost( + RootApp, + domains={ + 'www.domain2.example': Domain2App, + 'www.domain2.example:443': SecureApp, + }, + ) cherrypy.tree.graft(vhost) """ @@ -78,7 +81,7 @@ class VirtualHost(object): def __call__(self, environ, start_response): domain = environ.get('HTTP_HOST', '') if self.use_x_forwarded_host: - domain = environ.get("HTTP_X_FORWARDED_HOST", domain) + domain = environ.get('HTTP_X_FORWARDED_HOST', domain) nextapp = self.domains.get(domain) if nextapp is None: @@ -109,7 +112,7 @@ class InternalRedirector(object): # Add the *previous* path_info + qs to redirections. old_uri = sn + path if qs: - old_uri += "?" + qs + old_uri += '?' + qs redirections.append(old_uri) if not self.recursive: @@ -117,18 +120,20 @@ class InternalRedirector(object): # already new_uri = sn + ir.path if ir.query_string: - new_uri += "?" + ir.query_string + new_uri += '?' + ir.query_string if new_uri in redirections: ir.request.close() - raise RuntimeError("InternalRedirector visited the " - "same URL twice: %r" % new_uri) + tmpl = ( + 'InternalRedirector visited the same URL twice: %r' + ) + raise RuntimeError(tmpl % new_uri) # Munge the environment and try again. - environ['REQUEST_METHOD'] = "GET" + environ['REQUEST_METHOD'] = 'GET' environ['PATH_INFO'] = ir.path environ['QUERY_STRING'] = ir.query_string environ['wsgi.input'] = io.BytesIO() - environ['CONTENT_LENGTH'] = "0" + environ['CONTENT_LENGTH'] = '0' environ['cherrypy.previous_request'] = ir.request @@ -160,7 +165,8 @@ class _TrappedResponse(object): self.throws = throws self.started_response = False self.response = self.trap( - self.nextapp, self.environ, self.start_response) + self.nextapp, self.environ, self.start_response, + ) self.iter_response = iter(self.response) def __iter__(self): @@ -187,16 +193,17 @@ class _TrappedResponse(object): raise except: tb = _cperror.format_exc() - #print('trapped (started %s):' % self.started_response, tb) _cherrypy.log(tb, severity=40) if not _cherrypy.request.show_tracebacks: - tb = "" + tb = '' s, h, b = _cperror.bare_error(tb) if six.PY3: # What fun. s = s.decode('ISO-8859-1') - h = [(k.decode('ISO-8859-1'), v.decode('ISO-8859-1')) - for k, v in h] + h = [ + (k.decode('ISO-8859-1'), v.decode('ISO-8859-1')) + for k, v in h + ] if self.started_response: # Empty our iterable (so future calls raise StopIteration) self.iter_response = iter([]) @@ -215,7 +222,7 @@ class _TrappedResponse(object): raise if self.started_response: - return ntob("").join(b) + return ntob('').join(b) else: return b @@ -240,18 +247,18 @@ class AppResponse(object): outstatus = r.output_status if not isinstance(outstatus, bytes): - raise TypeError("response.output_status is not a byte string.") + raise TypeError('response.output_status is not a byte string.') outheaders = [] for k, v in r.header_list: if not isinstance(k, bytes): - raise TypeError( - "response.header_list key %r is not a byte string." % - k) + tmpl = 'response.header_list key %r is not a byte string.' + raise TypeError(tmpl % k) if not isinstance(v, bytes): - raise TypeError( - "response.header_list value %r is not a byte string." % - v) + tmpl = ( + 'response.header_list value %r is not a byte string.' + ) + raise TypeError(tmpl % v) outheaders.append((k, v)) if six.PY3: @@ -260,8 +267,10 @@ class AppResponse(object): # that is, they must be of type "str" but are restricted to # code points in the "latin-1" set. outstatus = outstatus.decode('ISO-8859-1') - outheaders = [(k.decode('ISO-8859-1'), v.decode('ISO-8859-1')) - for k, v in outheaders] + outheaders = [ + (k.decode('ISO-8859-1'), v.decode('ISO-8859-1')) + for k, v in outheaders + ] self.iter_response = iter(r.body) self.write = start_response(outstatus, outheaders) @@ -299,14 +308,18 @@ class AppResponse(object): """Create a Request object using environ.""" env = self.environ.get - local = httputil.Host('', - int(env('SERVER_PORT', 80) or -1), - env('SERVER_NAME', '')) - remote = httputil.Host(env('REMOTE_ADDR', ''), - int(env('REMOTE_PORT', -1) or -1), - env('REMOTE_HOST', '')) + local = httputil.Host( + '', + int(env('SERVER_PORT', 80) or -1), + env('SERVER_NAME', ''), + ) + remote = httputil.Host( + env('REMOTE_ADDR', ''), + int(env('REMOTE_PORT', -1) or -1), + env('REMOTE_HOST', ''), + ) scheme = env('wsgi.url_scheme') - sproto = env('ACTUAL_SERVER_PROTOCOL', "HTTP/1.1") + sproto = env('ACTUAL_SERVER_PROTOCOL', 'HTTP/1.1') request, resp = self.cpapp.get_serving(local, remote, scheme, sproto) # LOGON_USER is served by IIS, and is the name of the @@ -320,44 +333,54 @@ class AppResponse(object): meth = self.environ['REQUEST_METHOD'] - path = httputil.urljoin(self.environ.get('SCRIPT_NAME', ''), - self.environ.get('PATH_INFO', '')) + path = httputil.urljoin( + self.environ.get('SCRIPT_NAME', ''), + self.environ.get('PATH_INFO', ''), + ) qs = self.environ.get('QUERY_STRING', '') - if six.PY3: - # This isn't perfect; if the given PATH_INFO is in the - # wrong encoding, it may fail to match the appropriate config - # section URI. But meh. - old_enc = self.environ.get('wsgi.url_encoding', 'ISO-8859-1') - new_enc = self.cpapp.find_config(self.environ.get('PATH_INFO', ''), - "request.uri_encoding", 'utf-8') - if new_enc.lower() != old_enc.lower(): - # Even though the path and qs are unicode, the WSGI server - # is required by PEP 3333 to coerce them to ISO-8859-1 - # masquerading as unicode. So we have to encode back to - # bytes and then decode again using the "correct" encoding. - try: - u_path = path.encode(old_enc).decode(new_enc) - u_qs = qs.encode(old_enc).decode(new_enc) - except (UnicodeEncodeError, UnicodeDecodeError): - # Just pass them through without transcoding and hope. - pass - else: - # Only set transcoded values if they both succeed. - path = u_path - qs = u_qs + path, qs = self.recode_path_qs(path, qs) or (path, qs) rproto = self.environ.get('SERVER_PROTOCOL') headers = self.translate_headers(self.environ) rfile = self.environ['wsgi.input'] request.run(meth, path, qs, rproto, headers, rfile) - headerNames = {'HTTP_CGI_AUTHORIZATION': 'Authorization', - 'CONTENT_LENGTH': 'Content-Length', - 'CONTENT_TYPE': 'Content-Type', - 'REMOTE_HOST': 'Remote-Host', - 'REMOTE_ADDR': 'Remote-Addr', - } + headerNames = { + 'HTTP_CGI_AUTHORIZATION': 'Authorization', + 'CONTENT_LENGTH': 'Content-Length', + 'CONTENT_TYPE': 'Content-Type', + 'REMOTE_HOST': 'Remote-Host', + 'REMOTE_ADDR': 'Remote-Addr', + } + + def recode_path_qs(self, path, qs): + if not six.PY3: + return + + # This isn't perfect; if the given PATH_INFO is in the + # wrong encoding, it may fail to match the appropriate config + # section URI. But meh. + old_enc = self.environ.get('wsgi.url_encoding', 'ISO-8859-1') + new_enc = self.cpapp.find_config( + self.environ.get('PATH_INFO', ''), + 'request.uri_encoding', 'utf-8', + ) + if new_enc.lower() == old_enc.lower(): + return + + # Even though the path and qs are unicode, the WSGI server + # is required by PEP 3333 to coerce them to ISO-8859-1 + # masquerading as unicode. So we have to encode back to + # bytes and then decode again using the "correct" encoding. + try: + return ( + path.encode(old_enc).decode(new_enc), + qs.encode(old_enc).decode(new_enc), + ) + except (UnicodeEncodeError, UnicodeDecodeError): + # Just pass them through without transcoding and hope. + pass def translate_headers(self, environ): """Translate CGI-environ header names to HTTP header names.""" @@ -365,9 +388,9 @@ class AppResponse(object): # We assume all incoming header keys are uppercase already. if cgiName in self.headerNames: yield self.headerNames[cgiName], environ[cgiName] - elif cgiName[:5] == "HTTP_": + elif cgiName[:5] == 'HTTP_': # Hackish attempt at recovering original header names. - translatedHeader = cgiName[5:].replace("_", "-") + translatedHeader = cgiName[5:].replace('_', '-') yield translatedHeader, environ[cgiName] @@ -375,9 +398,10 @@ class CPWSGIApp(object): """A WSGI application object for a CherryPy Application.""" - pipeline = [('ExceptionTrapper', ExceptionTrapper), - ('InternalRedirector', InternalRedirector), - ] + pipeline = [ + ('ExceptionTrapper', ExceptionTrapper), + ('InternalRedirector', InternalRedirector), + ] """A list of (name, wsgiapp) pairs. Each 'wsgiapp' MUST be a constructor that takes an initial, positional 'nextapp' argument, plus optional keyword arguments, and returns a WSGI application @@ -427,16 +451,16 @@ class CPWSGIApp(object): def namespace_handler(self, k, v): """Config handler for the 'wsgi' namespace.""" - if k == "pipeline": + if k == 'pipeline': # Note this allows multiple 'wsgi.pipeline' config entries # (but each entry will be processed in a 'random' order). # It should also allow developers to set default middleware # in code (passed to self.__init__) that deployers can add to # (but not remove) via config. self.pipeline.extend(v) - elif k == "response_class": + elif k == 'response_class': self.response_class = v else: - name, arg = k.split(".", 1) + name, arg = k.split('.', 1) bucket = self.config.setdefault(name, {}) bucket[arg] = v diff --git a/cherrypy/_cpwsgi_server.py b/cherrypy/_cpwsgi_server.py index 874e2e9..d03a077 100644 --- a/cherrypy/_cpwsgi_server.py +++ b/cherrypy/_cpwsgi_server.py @@ -66,5 +66,5 @@ class CPWSGIServer(wsgiserver.CherryPyWSGIServer): self.stats['Enabled'] = getattr( self.server_adapter, 'statistics', False) - def error_log(self, msg="", level=20, traceback=False): + def error_log(self, msg='', level=20, traceback=False): cherrypy.engine.log(msg, level, traceback) diff --git a/cherrypy/_helper.py b/cherrypy/_helper.py index 970e2a4..5875ec0 100644 --- a/cherrypy/_helper.py +++ b/cherrypy/_helper.py @@ -5,7 +5,7 @@ Helper functions for CP apps import six from cherrypy._cpcompat import urljoin as _urljoin, urlencode as _urlencode -from cherrypy._cpcompat import basestring +from cherrypy._cpcompat import text_or_bytes import cherrypy @@ -17,11 +17,11 @@ def expose(func=None, alias=None): def expose_(func): func.exposed = True if alias is not None: - if isinstance(alias, basestring): - parents[alias.replace(".", "_")] = func + if isinstance(alias, text_or_bytes): + parents[alias.replace('.', '_')] = func else: for a in alias: - parents[a.replace(".", "_")] = func + parents[a.replace('.', '_')] = func return func import sys @@ -191,7 +191,7 @@ def popargs(*args, **kwargs): return decorated -def url(path="", qs="", script_name=None, base=None, relative=None): +def url(path='', qs='', script_name=None, base=None, relative=None): """Create an absolute URL for the given path. If 'path' starts with a slash ('/'), this will return @@ -224,7 +224,7 @@ def url(path="", qs="", script_name=None, base=None, relative=None): qs = '?' + qs if cherrypy.request.app: - if not path.startswith("/"): + if not path.startswith('/'): # Append/remove trailing slash from path_info as needed # (this is to support mistyped URL's without redirecting; # if you want to redirect, use tools.trailing_slash). @@ -236,7 +236,7 @@ def url(path="", qs="", script_name=None, base=None, relative=None): if pi.endswith('/') and pi != '/': pi = pi[:-1] - if path == "": + if path == '': path = pi else: path = _urljoin(pi, path) @@ -255,7 +255,7 @@ def url(path="", qs="", script_name=None, base=None, relative=None): if base is None: base = cherrypy.server.base() - path = (script_name or "") + path + path = (script_name or '') + path newurl = base + path + qs if './' in newurl: @@ -273,7 +273,7 @@ def url(path="", qs="", script_name=None, base=None, relative=None): # At this point, we should have a fully-qualified absolute URL. if relative is None: - relative = getattr(cherrypy.request.app, "relative_urls", False) + relative = getattr(cherrypy.request.app, 'relative_urls', False) # See http://www.ietf.org/rfc/rfc2396.txt if relative == 'server': diff --git a/cherrypy/daemon.py b/cherrypy/daemon.py index 395a2e6..772b2fc 100755 --- a/cherrypy/daemon.py +++ b/cherrypy/daemon.py @@ -13,7 +13,7 @@ def start(configfiles=None, daemonize=False, environment=None, """Subscribe all engine plugins and start the engine.""" sys.path = [''] + sys.path for i in imports or []: - exec("import %s" % i) + exec('import %s' % i) for c in configfiles or []: cherrypy.config.update(c) @@ -37,18 +37,18 @@ def start(configfiles=None, daemonize=False, environment=None, if pidfile: plugins.PIDFile(engine, pidfile).subscribe() - if hasattr(engine, "signal_handler"): + if hasattr(engine, 'signal_handler'): engine.signal_handler.subscribe() - if hasattr(engine, "console_control_handler"): + if hasattr(engine, 'console_control_handler'): engine.console_control_handler.subscribe() if (fastcgi and (scgi or cgi)) or (scgi and cgi): - cherrypy.log.error("You may only specify one of the cgi, fastcgi, and " - "scgi options.", 'ENGINE') + cherrypy.log.error('You may only specify one of the cgi, fastcgi, and ' + 'scgi options.', 'ENGINE') sys.exit(1) elif fastcgi or scgi or cgi: # Turn off autoreload when using *cgi. - cherrypy.config.update({'engine.autoreload_on': False}) + cherrypy.config.update({'engine.autoreload.on': False}) # Turn off the default HTTP server (which is subscribed by default). cherrypy.server.unsubscribe() @@ -76,25 +76,25 @@ def run(): from optparse import OptionParser p = OptionParser() - p.add_option('-c', '--config', action="append", dest='config', - help="specify config file(s)") - p.add_option('-d', action="store_true", dest='daemonize', - help="run the server as a daemon") + p.add_option('-c', '--config', action='append', dest='config', + help='specify config file(s)') + p.add_option('-d', action='store_true', dest='daemonize', + help='run the server as a daemon') p.add_option('-e', '--environment', dest='environment', default=None, - help="apply the given config environment") - p.add_option('-f', action="store_true", dest='fastcgi', - help="start a fastcgi server instead of the default HTTP " - "server") - p.add_option('-s', action="store_true", dest='scgi', - help="start a scgi server instead of the default HTTP server") - p.add_option('-x', action="store_true", dest='cgi', - help="start a cgi server instead of the default HTTP server") - p.add_option('-i', '--import', action="append", dest='imports', - help="specify modules to import") + help='apply the given config environment') + p.add_option('-f', action='store_true', dest='fastcgi', + help='start a fastcgi server instead of the default HTTP ' + 'server') + p.add_option('-s', action='store_true', dest='scgi', + help='start a scgi server instead of the default HTTP server') + p.add_option('-x', action='store_true', dest='cgi', + help='start a cgi server instead of the default HTTP server') + p.add_option('-i', '--import', action='append', dest='imports', + help='specify modules to import') p.add_option('-p', '--pidfile', dest='pidfile', default=None, - help="store the process id in the given file") - p.add_option('-P', '--Path', action="append", dest='Path', - help="add the given paths to sys.path") + help='store the process id in the given file') + p.add_option('-P', '--Path', action='append', dest='Path', + help='add the given paths to sys.path') options, args = p.parse_args() if options.Path: diff --git a/cherrypy/lib/__init__.py b/cherrypy/lib/__init__.py index a75a53d..af08028 100644 --- a/cherrypy/lib/__init__.py +++ b/cherrypy/lib/__init__.py @@ -1,7 +1,5 @@ """CherryPy Library""" -# Deprecated in CherryPy 3.2 -- remove in CherryPy 3.3 -from cherrypy.lib.reprconf import unrepr, modules, attributes def is_iterator(obj): '''Returns a boolean indicating if the object provided implements @@ -16,22 +14,23 @@ def is_iterator(obj): # Types which implement the protocol must return themselves when # invoking 'iter' upon them. return iter(obj) is obj - + + def is_closable_iterator(obj): - + # Not an iterator. if not is_iterator(obj): return False - + # A generator - the easiest thing to deal with. import inspect if inspect.isgenerator(obj): return True - + # A custom iterator. Look for a close method... if not (hasattr(obj, 'close') and callable(obj.close)): return False - + # ... which doesn't require any arguments. try: inspect.getcallargs(obj.close) @@ -40,6 +39,7 @@ def is_closable_iterator(obj): else: return True + class file_generator(object): """Yield the given input (a file object) in chunks (default 64k). (Core)""" @@ -77,9 +77,9 @@ def file_generator_limited(fileobj, count, chunk_size=65536): def set_vary_header(response, header_name): - "Add a Vary header to a response" - varies = response.headers.get("Vary", "") - varies = [x.strip() for x in varies.split(",") if x.strip()] + 'Add a Vary header to a response' + varies = response.headers.get('Vary', '') + varies = [x.strip() for x in varies.split(',') if x.strip()] if header_name not in varies: varies.append(header_name) - response.headers['Vary'] = ", ".join(varies) + response.headers['Vary'] = ', '.join(varies) diff --git a/cherrypy/lib/auth.py b/cherrypy/lib/auth.py index 71591aa..34ad688 100644 --- a/cherrypy/lib/auth.py +++ b/cherrypy/lib/auth.py @@ -22,25 +22,25 @@ def check_auth(users, encrypt=None, realm=None): if not isinstance(users, dict): raise ValueError( - "Authentication users must be a dictionary") + 'Authentication users must be a dictionary') # fetch the user password - password = users.get(ah["username"], None) + password = users.get(ah['username'], None) except TypeError: # returns a password (encrypted or clear text) - password = users(ah["username"]) + password = users(ah['username']) else: if not isinstance(users, dict): - raise ValueError("Authentication users must be a dictionary") + raise ValueError('Authentication users must be a dictionary') # fetch the user password - password = users.get(ah["username"], None) + password = users.get(ah['username'], None) # validate the authorization by re-computing it here # and compare it with what the user-agent provided if httpauth.checkResponse(ah, password, method=request.method, encrypt=encrypt, realm=realm): - request.login = ah["username"] + request.login = ah['username'] return True request.login = False @@ -72,7 +72,7 @@ def basic_auth(realm, users, encrypt=None, debug=False): 'www-authenticate'] = httpauth.basicAuth(realm) raise cherrypy.HTTPError( - 401, "You are not authorized to access that resource") + 401, 'You are not authorized to access that resource') def digest_auth(realm, users, debug=False): @@ -94,4 +94,4 @@ def digest_auth(realm, users, debug=False): 'www-authenticate'] = httpauth.digestAuth(realm) raise cherrypy.HTTPError( - 401, "You are not authorized to access that resource") + 401, 'You are not authorized to access that resource') diff --git a/cherrypy/lib/auth_basic.py b/cherrypy/lib/auth_basic.py index 5ba16f7..c9c9bd5 100644 --- a/cherrypy/lib/auth_basic.py +++ b/cherrypy/lib/auth_basic.py @@ -2,6 +2,12 @@ # -*- coding: utf-8 -*- # vim:ts=4:sw=4:expandtab:fileencoding=utf-8 +import binascii + +import cherrypy +from cherrypy._cpcompat import base64_decode + + __doc__ = """This module provides a CherryPy 3.x tool which implements the server-side of HTTP Basic Access Authentication, as described in :rfc:`2617`. @@ -22,10 +28,6 @@ as the credentials store:: __author__ = 'visteya' __date__ = 'April 2009' -import binascii -from cherrypy._cpcompat import base64_decode -import cherrypy - def checkpassword_dict(user_password_dict): """Returns a checkpassword function which checks credentials @@ -70,7 +72,8 @@ def basic_auth(realm, checkpassword, debug=False): auth_header = request.headers.get('authorization') if auth_header is not None: - try: + # split() error, base64.decodestring() error + with cherrypy.HTTPError.handle((ValueError, binascii.Error), 400, 'Bad Request'): scheme, params = auth_header.split(' ', 1) if scheme.lower() == 'basic': username, password = base64_decode(params).split(':', 1) @@ -79,12 +82,9 @@ def basic_auth(realm, checkpassword, debug=False): cherrypy.log('Auth succeeded', 'TOOLS.AUTH_BASIC') request.login = username return # successful authentication - # split() error, base64.decodestring() error - except (ValueError, binascii.Error): - raise cherrypy.HTTPError(400, 'Bad Request') # Respond with 401 status and a WWW-Authenticate header cherrypy.serving.response.headers[ 'www-authenticate'] = 'Basic realm="%s"' % realm raise cherrypy.HTTPError( - 401, "You are not authorized to access that resource") + 401, 'You are not authorized to access that resource') diff --git a/cherrypy/lib/auth_digest.py b/cherrypy/lib/auth_digest.py index e833ff7..0e767e9 100644 --- a/cherrypy/lib/auth_digest.py +++ b/cherrypy/lib/auth_digest.py @@ -2,6 +2,13 @@ # -*- coding: utf-8 -*- # vim:ts=4:sw=4:expandtab:fileencoding=utf-8 +import time +from hashlib import md5 + +import cherrypy +from cherrypy._cpcompat import ntob, parse_http_list, parse_keqv_list + + __doc__ = """An implementation of the server-side of HTTP Digest Access Authentication, which is described in :rfc:`2617`. @@ -22,12 +29,6 @@ __author__ = 'visteya' __date__ = 'April 2009' -import time -from hashlib import md5 -from cherrypy._cpcompat import parse_http_list, parse_keqv_list - -import cherrypy -from cherrypy._cpcompat import ntob md5_hex = lambda s: md5(ntob(s)).hexdigest() qop_auth = 'auth' @@ -142,7 +143,7 @@ class HttpDigestAuthorization (object): def __init__(self, auth_header, http_method, debug=False): self.http_method = http_method self.debug = debug - scheme, params = auth_header.split(" ", 1) + scheme, params = auth_header.split(' ', 1) self.scheme = scheme.lower() if self.scheme != 'digest': raise ValueError('Authorization scheme is not "Digest"') @@ -180,7 +181,7 @@ class HttpDigestAuthorization (object): ) if not has_reqd: raise ValueError( - self.errmsg("Not all required parameters are present.")) + self.errmsg('Not all required parameters are present.')) if self.qop: if self.qop not in valid_qops: @@ -188,13 +189,13 @@ class HttpDigestAuthorization (object): self.errmsg("Unsupported value for qop: '%s'" % self.qop)) if not (self.cnonce and self.nc): raise ValueError( - self.errmsg("If qop is sent then " - "cnonce and nc MUST be present")) + self.errmsg('If qop is sent then ' + 'cnonce and nc MUST be present')) else: if self.cnonce or self.nc: raise ValueError( - self.errmsg("If qop is not sent, " - "neither cnonce nor nc can be present")) + self.errmsg('If qop is not sent, ' + 'neither cnonce nor nc can be present')) def __str__(self): return 'authorization : %s' % self.auth_header @@ -239,7 +240,7 @@ class HttpDigestAuthorization (object): except ValueError: # int() error pass if self.debug: - TRACE("nonce is stale") + TRACE('nonce is stale') return True def HA2(self, entity_body=''): @@ -251,14 +252,14 @@ class HttpDigestAuthorization (object): # # If the "qop" value is "auth-int", then A2 is: # A2 = method ":" digest-uri-value ":" H(entity-body) - if self.qop is None or self.qop == "auth": + if self.qop is None or self.qop == 'auth': a2 = '%s:%s' % (self.http_method, self.uri) - elif self.qop == "auth-int": - a2 = "%s:%s:%s" % (self.http_method, self.uri, H(entity_body)) + elif self.qop == 'auth-int': + a2 = '%s:%s:%s' % (self.http_method, self.uri, H(entity_body)) else: # in theory, this should never happen, since I validate qop in # __init__() - raise ValueError(self.errmsg("Unrecognized value for qop!")) + raise ValueError(self.errmsg('Unrecognized value for qop!')) return H(a2) def request_digest(self, ha1, entity_body=''): @@ -279,10 +280,10 @@ class HttpDigestAuthorization (object): ha2 = self.HA2(entity_body) # Request-Digest -- RFC 2617 3.2.2.1 if self.qop: - req = "%s:%s:%s:%s:%s" % ( + req = '%s:%s:%s:%s:%s' % ( self.nonce, self.nc, self.cnonce, self.qop, ha2) else: - req = "%s:%s" % (self.nonce, ha2) + req = '%s:%s' % (self.nonce, ha2) # RFC 2617 3.2.2.2 # @@ -351,12 +352,10 @@ def digest_auth(realm, get_ha1, key, debug=False): auth_header = request.headers.get('authorization') nonce_is_stale = False if auth_header is not None: - try: + with cherrypy.HTTPError.handle(ValueError, 400, + 'The Authorization header could not be parsed.'): auth = HttpDigestAuthorization( auth_header, request.method, debug=debug) - except ValueError: - raise cherrypy.HTTPError( - 400, "The Authorization header could not be parsed.") if debug: TRACE(str(auth)) @@ -370,7 +369,7 @@ def digest_auth(realm, get_ha1, key, debug=False): digest = auth.request_digest(ha1, entity_body=request.body) if digest == auth.response: # authenticated if debug: - TRACE("digest matches auth.response") + TRACE('digest matches auth.response') # Now check if nonce is stale. # The choice of ten minutes' lifetime for nonce is somewhat # arbitrary @@ -378,7 +377,7 @@ def digest_auth(realm, get_ha1, key, debug=False): if not nonce_is_stale: request.login = auth.username if debug: - TRACE("authentication of %s successful" % + TRACE('authentication of %s successful' % auth.username) return @@ -388,4 +387,4 @@ def digest_auth(realm, get_ha1, key, debug=False): TRACE(header) cherrypy.serving.response.headers['WWW-Authenticate'] = header raise cherrypy.HTTPError( - 401, "You are not authorized to access that resource") + 401, 'You are not authorized to access that resource') diff --git a/cherrypy/lib/caching.py b/cherrypy/lib/caching.py index 375d5f0..f363a67 100644 --- a/cherrypy/lib/caching.py +++ b/cherrypy/lib/caching.py @@ -39,7 +39,7 @@ import time import cherrypy from cherrypy.lib import cptools, httputil -from cherrypy._cpcompat import copyitems, ntob, set_daemon, sorted, Event +from cherrypy._cpcompat import copyitems, ntob, sorted, Event class Cache(object): @@ -170,7 +170,7 @@ class MemoryCache(Cache): # Run self.expire_cache in a separate daemon thread. t = threading.Thread(target=self.expire_cache, name='expire_cache') self.expiration_thread = t - set_daemon(t, True) + t.daemon = True t.start() def clear(self): @@ -265,7 +265,7 @@ class MemoryCache(Cache): self.store.pop(uri, None) -def get(invalid_methods=("POST", "PUT", "DELETE"), debug=False, **kwargs): +def get(invalid_methods=('POST', 'PUT', 'DELETE'), debug=False, **kwargs): """Try to obtain cached output. If fresh enough, raise HTTPError(304). If POST, PUT, or DELETE: @@ -291,9 +291,9 @@ def get(invalid_methods=("POST", "PUT", "DELETE"), debug=False, **kwargs): request = cherrypy.serving.request response = cherrypy.serving.response - if not hasattr(cherrypy, "_cache"): + if not hasattr(cherrypy, '_cache'): # Make a process-wide Cache object. - cherrypy._cache = kwargs.pop("cache_class", MemoryCache)() + cherrypy._cache = kwargs.pop('cache_class', MemoryCache)() # Take all remaining kwargs and set them on the Cache object. for k, v in kwargs.items(): @@ -328,7 +328,7 @@ def get(invalid_methods=("POST", "PUT", "DELETE"), debug=False, **kwargs): if directive == 'max-age': if len(atoms) != 1 or not atoms[0].isdigit(): raise cherrypy.HTTPError( - 400, "Invalid Cache-Control header") + 400, 'Invalid Cache-Control header') max_age = int(atoms[0]) break elif directive == 'no-cache': @@ -359,7 +359,7 @@ def get(invalid_methods=("POST", "PUT", "DELETE"), debug=False, **kwargs): dict.__setitem__(rh, k, dict.__getitem__(h, k)) # Add the required Age header - response.headers["Age"] = str(age) + response.headers['Age'] = str(age) try: # Note that validate_since depends on a Last-Modified header; @@ -457,14 +457,14 @@ def expires(secs=0, force=False, debug=False): secs = (86400 * secs.days) + secs.seconds if secs == 0: - if force or ("Pragma" not in headers): - headers["Pragma"] = "no-cache" + if force or ('Pragma' not in headers): + headers['Pragma'] = 'no-cache' if cherrypy.serving.request.protocol >= (1, 1): - if force or "Cache-Control" not in headers: - headers["Cache-Control"] = "no-cache, must-revalidate" + if force or 'Cache-Control' not in headers: + headers['Cache-Control'] = 'no-cache, must-revalidate' # Set an explicit Expires date in the past. expiry = httputil.HTTPDate(1169942400.0) else: expiry = httputil.HTTPDate(response.time + secs) - if force or "Expires" not in headers: - headers["Expires"] = expiry + if force or 'Expires' not in headers: + headers['Expires'] = expiry diff --git a/cherrypy/lib/covercp.py b/cherrypy/lib/covercp.py index dd58a0c..cf1743c 100644 --- a/cherrypy/lib/covercp.py +++ b/cherrypy/lib/covercp.py @@ -25,11 +25,12 @@ import sys import cgi import os import os.path -localFile = os.path.join(os.path.dirname(__file__), "coverage.cache") +import cherrypy from cherrypy._cpcompat import quote_plus -import cherrypy + +localFile = os.path.join(os.path.dirname(__file__), 'coverage.cache') the_coverage = None try: @@ -45,8 +46,8 @@ except ImportError: import warnings warnings.warn( - "No code coverage will be performed; " - "coverage.py could not be imported.") + 'No code coverage will be performed; ' + 'coverage.py could not be imported.') def start(): pass @@ -196,7 +197,7 @@ def _percent(statements, missing): return 0 -def _show_branch(root, base, path, pct=0, showpct=False, exclude="", +def _show_branch(root, base, path, pct=0, showpct=False, exclude='', coverage=the_coverage): # Show the directory name and any of our children @@ -207,7 +208,7 @@ def _show_branch(root, base, path, pct=0, showpct=False, exclude="", if newpath.lower().startswith(base): relpath = newpath[len(base):] - yield "| " * relpath.count(os.sep) + yield '| ' * relpath.count(os.sep) yield ( "%s\n" % @@ -228,7 +229,7 @@ def _show_branch(root, base, path, pct=0, showpct=False, exclude="", for name in files: newpath = os.path.join(path, name) - pc_str = "" + pc_str = '' if showpct: try: _, statements, _, missing, _ = coverage.analysis2(newpath) @@ -237,13 +238,13 @@ def _show_branch(root, base, path, pct=0, showpct=False, exclude="", pass else: pc = _percent(statements, missing) - pc_str = ("%3d%% " % pc).replace(' ', ' ') + pc_str = ('%3d%% ' % pc).replace(' ', ' ') if pc < float(pct) or pc == -1: pc_str = "%s" % pc_str else: pc_str = "%s" % pc_str - yield TEMPLATE_ITEM % ("| " * (relpath.count(os.sep) + 1), + yield TEMPLATE_ITEM % ('| ' * (relpath.count(os.sep) + 1), pc_str, newpath, name) @@ -263,8 +264,8 @@ def _graft(path, tree): break atoms.append(tail) atoms.append(p) - if p != "/": - atoms.append("/") + if p != '/': + atoms.append('/') atoms.reverse() for node in atoms: @@ -298,7 +299,7 @@ class CoverStats(object): return TEMPLATE_FRAMESET % self.root.lower() @cherrypy.expose - def menu(self, base="/", pct="50", showpct="", + def menu(self, base='/', pct='50', showpct='', exclude=r'python\d\.\d|test|tut\d|tutorial'): # The coverage module uses all-lower-case names. @@ -309,36 +310,36 @@ class CoverStats(object): # Start by showing links for parent paths yield "
" - path = "" + path = '' atoms = base.split(os.sep) atoms.pop() for atom in atoms: path += atom + os.sep yield ("%s %s" % (path, quote_plus(exclude), atom, os.sep)) - yield "
" + yield '' yield "
" # Then display the tree tree = get_tree(base, exclude, self.coverage) if not tree: - yield "

No modules covered.

" + yield '

No modules covered.

' else: - for chunk in _show_branch(tree, base, "/", pct, + for chunk in _show_branch(tree, base, '/', pct, showpct == 'checked', exclude, coverage=self.coverage): yield chunk - yield "
" - yield "" + yield '' + yield '' def annotated_file(self, filename, statements, excluded, missing): source = open(filename, 'r') buffer = [] for lineno, line in enumerate(source.readlines()): lineno += 1 - line = line.strip("\n\r") + line = line.strip('\n\r') empty_the_buffer = True if lineno in excluded: template = TEMPLATE_LOC_EXCLUDED @@ -374,7 +375,7 @@ class CoverStats(object): def serve(path=localFile, port=8080, root=None): if coverage is None: - raise ImportError("The coverage module could not be imported.") + raise ImportError('The coverage module could not be imported.') from coverage import coverage cov = coverage(data_file=path) cov.load() @@ -382,9 +383,9 @@ def serve(path=localFile, port=8080, root=None): import cherrypy cherrypy.config.update({'server.socket_port': int(port), 'server.thread_pool': 10, - 'environment': "production", + 'environment': 'production', }) cherrypy.quickstart(CoverStats(cov, root)) -if __name__ == "__main__": +if __name__ == '__main__': serve(*tuple(sys.argv[1:])) diff --git a/cherrypy/lib/cpstats.py b/cherrypy/lib/cpstats.py index 49d1320..734d9dd 100644 --- a/cherrypy/lib/cpstats.py +++ b/cherrypy/lib/cpstats.py @@ -187,9 +187,17 @@ To format statistics reports:: """ +import logging +import os +import sys +import threading +import time + +import cherrypy +from cherrypy._cpcompat import json + # ------------------------------- Statistics -------------------------------- # -import logging if not hasattr(logging, 'statistics'): logging.statistics = {} @@ -210,12 +218,6 @@ def extrapolate_statistics(scope): # -------------------- CherryPy Applications Statistics --------------------- # -import sys -import threading -import time - -import cherrypy - appstats = logging.statistics.setdefault('CherryPy Applications', {}) appstats.update({ 'Enabled': True, @@ -390,24 +392,13 @@ class StatsTool(cherrypy.Tool): sq.pop(0) -import cherrypy cherrypy.tools.cpstats = StatsTool() # ---------------------- CherryPy Statistics Reporting ---------------------- # -import os thisdir = os.path.abspath(os.path.dirname(__file__)) -try: - import json -except ImportError: - try: - import simplejson as json - except ImportError: - json = None - - missing = object() locale_date = lambda v: time.strftime('%c', time.gmtime(v)) diff --git a/cherrypy/lib/cptools.py b/cherrypy/lib/cptools.py index fbed0e2..c99bc7c 100644 --- a/cherrypy/lib/cptools.py +++ b/cherrypy/lib/cptools.py @@ -7,7 +7,7 @@ from hashlib import md5 import six import cherrypy -from cherrypy._cpcompat import basestring +from cherrypy._cpcompat import text_or_bytes from cherrypy.lib import httputil as _httputil from cherrypy.lib import is_iterator @@ -33,7 +33,7 @@ def validate_etags(autotags=False, debug=False): response = cherrypy.serving.response # Guard against being run twice. - if hasattr(response, "ETag"): + if hasattr(response, 'ETag'): return status, reason, msg = _httputil.valid_status(response.status) @@ -72,24 +72,24 @@ def validate_etags(autotags=False, debug=False): if debug: cherrypy.log('If-Match conditions: %s' % repr(conditions), 'TOOLS.ETAGS') - if conditions and not (conditions == ["*"] or etag in conditions): - raise cherrypy.HTTPError(412, "If-Match failed: ETag %r did " - "not match %r" % (etag, conditions)) + if conditions and not (conditions == ['*'] or etag in conditions): + raise cherrypy.HTTPError(412, 'If-Match failed: ETag %r did ' + 'not match %r' % (etag, conditions)) conditions = request.headers.elements('If-None-Match') or [] conditions = [str(x) for x in conditions] if debug: cherrypy.log('If-None-Match conditions: %s' % repr(conditions), 'TOOLS.ETAGS') - if conditions == ["*"] or etag in conditions: + if conditions == ['*'] or etag in conditions: if debug: cherrypy.log('request.method: %s' % request.method, 'TOOLS.ETAGS') - if request.method in ("GET", "HEAD"): + if request.method in ('GET', 'HEAD'): raise cherrypy.HTTPRedirect([], 304) else: - raise cherrypy.HTTPError(412, "If-None-Match failed: ETag %r " - "matched %r" % (etag, conditions)) + raise cherrypy.HTTPError(412, 'If-None-Match failed: ETag %r ' + 'matched %r' % (etag, conditions)) def validate_since(): @@ -113,7 +113,7 @@ def validate_since(): since = request.headers.get('If-Modified-Since') if since and since == lastmod: if (status >= 200 and status <= 299) or status == 304: - if request.method in ("GET", "HEAD"): + if request.method in ('GET', 'HEAD'): raise cherrypy.HTTPRedirect([], 304) else: raise cherrypy.HTTPError(412) @@ -186,7 +186,7 @@ def proxy(base=None, local='X-Forwarded-Host', remote='X-Forwarded-For', # This is for lighttpd/pound/Mongrel's 'X-Forwarded-Proto: https' scheme = s if not scheme: - scheme = request.base[:request.base.find("://")] + scheme = request.base[:request.base.find('://')] if local: lbase = request.headers.get(local, None) @@ -200,9 +200,9 @@ def proxy(base=None, local='X-Forwarded-Host', remote='X-Forwarded-For', if port != 80: base += ':%s' % port - if base.find("://") == -1: + if base.find('://') == -1: # add http:// or https:// if needed - base = scheme + "://" + base + base = scheme + '://' + base request.base = base @@ -212,7 +212,7 @@ def proxy(base=None, local='X-Forwarded-Host', remote='X-Forwarded-For', cherrypy.log('Testing remote %r:%r' % (remote, xff), 'TOOLS.PROXY') if xff: if remote == 'X-Forwarded-For': - #Bug #1268 + # Bug #1268 xff = xff.split(',')[0].strip() request.remote.ip = xff @@ -285,7 +285,7 @@ class SessionAuth(object): """Assert that the user is logged in.""" - session_key = "username" + session_key = 'username' debug = False def check_username_and_password(self, username, password): @@ -317,7 +317,7 @@ Message: %(error_msg)s
-""") % vars()).encode("utf-8") +""") % vars()).encode('utf-8') def do_login(self, username, password, from_page='..', **kwargs): """Login. May raise redirect, or return True if request handled.""" @@ -326,15 +326,15 @@ Message: %(error_msg)s if error_msg: body = self.login_screen(from_page, username, error_msg) response.body = body - if "Content-Length" in response.headers: + if 'Content-Length' in response.headers: # Delete Content-Length header so finalize() recalcs it. - del response.headers["Content-Length"] + del response.headers['Content-Length'] return True else: cherrypy.serving.request.login = username cherrypy.session[self.session_key] = username self.on_login(username) - raise cherrypy.HTTPRedirect(from_page or "/") + raise cherrypy.HTTPRedirect(from_page or '/') def do_logout(self, from_page='..', **kwargs): """Logout. May raise redirect, or return True if request handled.""" @@ -364,9 +364,9 @@ Message: %(error_msg)s locals(), ) response.body = self.login_screen(url) - if "Content-Length" in response.headers: + if 'Content-Length' in response.headers: # Delete Content-Length header so finalize() recalcs it. - del response.headers["Content-Length"] + del response.headers['Content-Length'] return True self._debug_message('Setting request.login to %(username)r', locals()) request.login = username @@ -388,14 +388,14 @@ Message: %(error_msg)s return True elif path.endswith('do_login'): if request.method != 'POST': - response.headers['Allow'] = "POST" + response.headers['Allow'] = 'POST' self._debug_message('do_login requires POST') raise cherrypy.HTTPError(405) self._debug_message('routing %(path)r to do_login', locals()) return self.do_login(**request.params) elif path.endswith('do_logout'): if request.method != 'POST': - response.headers['Allow'] = "POST" + response.headers['Allow'] = 'POST' raise cherrypy.HTTPError(405) self._debug_message('routing %(path)r to do_logout', locals()) return self.do_logout(**request.params) @@ -414,19 +414,19 @@ session_auth.__doc__ = """Session authentication hook. Any attribute of the SessionAuth class may be overridden via a keyword arg to this function: -""" + "\n".join(["%s: %s" % (k, type(getattr(SessionAuth, k)).__name__) - for k in dir(SessionAuth) if not k.startswith("__")]) +""" + '\n'.join(['%s: %s' % (k, type(getattr(SessionAuth, k)).__name__) + for k in dir(SessionAuth) if not k.startswith('__')]) def log_traceback(severity=logging.ERROR, debug=False): """Write the last error's traceback to the cherrypy error log.""" - cherrypy.log("", "HTTP", severity=severity, traceback=True) + cherrypy.log('', 'HTTP', severity=severity, traceback=True) def log_request_headers(debug=False): """Write request headers to the cherrypy error log.""" - h = [" %s: %s" % (k, v) for k, v in cherrypy.serving.request.header_list] - cherrypy.log('\nRequest Headers:\n' + '\n'.join(h), "HTTP") + h = [' %s: %s' % (k, v) for k, v in cherrypy.serving.request.header_list] + cherrypy.log('\nRequest Headers:\n' + '\n'.join(h), 'HTTP') def log_hooks(debug=False): @@ -442,13 +442,13 @@ def log_hooks(debug=False): points.append(k) for k in points: - msg.append(" %s:" % k) + msg.append(' %s:' % k) v = request.hooks.get(k, []) v.sort() for h in v: - msg.append(" %r" % h) + msg.append(' %r' % h) cherrypy.log('\nRequest Hooks for ' + cherrypy.url() + - ':\n' + '\n'.join(msg), "HTTP") + ':\n' + '\n'.join(msg), 'HTTP') def redirect(url='', internal=True, debug=False): @@ -533,7 +533,7 @@ def accept(media=None, debug=False): """ if not media: return - if isinstance(media, basestring): + if isinstance(media, text_or_bytes): media = [media] request = cherrypy.serving.request @@ -549,12 +549,12 @@ def accept(media=None, debug=False): # Note that 'ranges' is sorted in order of preference for element in ranges: if element.qvalue > 0: - if element.value == "*/*": + if element.value == '*/*': # Matches any type or subtype if debug: cherrypy.log('Match due to */*', 'TOOLS.ACCEPT') return media[0] - elif element.value.endswith("/*"): + elif element.value.endswith('/*'): # Matches any subtype mtype = element.value[:-1] # Keep the slash for m in media: @@ -574,11 +574,11 @@ def accept(media=None, debug=False): # No suitable media-range found. ah = request.headers.get('Accept') if ah is None: - msg = "Your client did not send an Accept header." + msg = 'Your client did not send an Accept header.' else: - msg = "Your client sent this Accept header: %s." % ah - msg += (" But this resource only emits these media types: %s." % - ", ".join(media)) + msg = 'Your client sent this Accept header: %s.' % ah + msg += (' But this resource only emits these media types: %s.' % + ', '.join(media)) raise cherrypy.HTTPError(406, msg) @@ -630,3 +630,19 @@ def autovary(ignore=None, debug=False): v.sort() resp_h['Vary'] = ', '.join(v) request.hooks.attach('before_finalize', set_response_header, 95) + + +def convert_params(exception=ValueError, error=400): + """Convert request params based on function annotations, with error handling. + + exception + Exception class to catch. + + status + The HTTP error code to return to the client on failure. + """ + request = cherrypy.serving.request + types = request.handler.callable.__annotations__ + with cherrypy.HTTPError.handle(exception, error): + for key in set(types).intersection(request.params): + request.params[key] = types[key](request.params[key]) diff --git a/cherrypy/lib/encoding.py b/cherrypy/lib/encoding.py index 88f2563..dc93d99 100644 --- a/cherrypy/lib/encoding.py +++ b/cherrypy/lib/encoding.py @@ -5,7 +5,7 @@ import io import six import cherrypy -from cherrypy._cpcompat import basestring, ntob +from cherrypy._cpcompat import text_or_bytes, ntob from cherrypy.lib import file_generator from cherrypy.lib import is_closable_iterator from cherrypy.lib import set_vary_header @@ -66,7 +66,7 @@ class UTF8StreamEncoder: class ResponseEncoder: default_encoding = 'utf-8' - failmsg = "Response body could not be encoded with %r." + failmsg = 'Response body could not be encoded with %r.' encoding = None errors = 'strict' text_only = True @@ -131,7 +131,7 @@ class ResponseEncoder: encoder = self.encode_stream else: encoder = self.encode_string - if "Content-Length" in response.headers: + if 'Content-Length' in response.headers: # Delete Content-Length header so finalize() recalcs it. # Encoded strings may be of different lengths from their # unicode equivalents, and even from each other. For example: @@ -142,7 +142,7 @@ class ResponseEncoder: # 6 # >>> len(t.encode("utf7")) # 8 - del response.headers["Content-Length"] + del response.headers['Content-Length'] # Parse the Accept-Charset request header, and try to provide one # of the requested charsets (in order of user preference). @@ -157,7 +157,7 @@ class ResponseEncoder: if self.debug: cherrypy.log('Specified encoding %r' % encoding, 'TOOLS.ENCODE') - if (not charsets) or "*" in charsets or encoding in charsets: + if (not charsets) or '*' in charsets or encoding in charsets: if self.debug: cherrypy.log('Attempting encoding %r' % encoding, 'TOOLS.ENCODE') @@ -177,7 +177,7 @@ class ResponseEncoder: else: for element in encs: if element.qvalue > 0: - if element.value == "*": + if element.value == '*': # Matches any charset. Try our default. if self.debug: cherrypy.log('Attempting default encoding due ' @@ -192,7 +192,7 @@ class ResponseEncoder: if encoder(encoding): return encoding - if "*" not in charsets: + if '*' not in charsets: # If no "*" is present in an Accept-Charset field, then all # character sets not explicitly mentioned get a quality # value of 0, except for ISO-8859-1, which gets a quality @@ -208,18 +208,18 @@ class ResponseEncoder: # No suitable encoding found. ac = request.headers.get('Accept-Charset') if ac is None: - msg = "Your client did not send an Accept-Charset header." + msg = 'Your client did not send an Accept-Charset header.' else: - msg = "Your client sent this Accept-Charset header: %s." % ac - _charsets = ", ".join(sorted(self.attempted_charsets)) - msg += " We tried these charsets: %s." % (_charsets,) + msg = 'Your client sent this Accept-Charset header: %s.' % ac + _charsets = ', '.join(sorted(self.attempted_charsets)) + msg += ' We tried these charsets: %s.' % (_charsets,) raise cherrypy.HTTPError(406, msg) def __call__(self, *args, **kwargs): response = cherrypy.serving.response self.body = self.oldhandler(*args, **kwargs) - if isinstance(self.body, basestring): + if isinstance(self.body, text_or_bytes): # strings get wrapped in a list because iterating over a single # item list is much faster than iterating over every character # in a long string. @@ -233,14 +233,14 @@ class ResponseEncoder: elif self.body is None: self.body = [] - ct = response.headers.elements("Content-Type") + ct = response.headers.elements('Content-Type') if self.debug: cherrypy.log('Content-Type: %r' % [str(h) for h in ct], 'TOOLS.ENCODE') if ct and self.add_charset: ct = ct[0] if self.text_only: - if ct.value.lower().startswith("text/"): + if ct.value.lower().startswith('text/'): if self.debug: cherrypy.log( 'Content-Type %s starts with "text/"' % ct, @@ -264,7 +264,7 @@ class ResponseEncoder: if self.debug: cherrypy.log('Setting Content-Type %s' % ct, 'TOOLS.ENCODE') - response.headers["Content-Type"] = str(ct) + response.headers['Content-Type'] = str(ct) return self.body @@ -280,11 +280,11 @@ def compress(body, compress_level): yield ntob('\x08') # CM: compression method yield ntob('\x00') # FLG: none set # MTIME: 4 bytes - yield struct.pack(" self.maxparents: - return [("[%s referrers]" % len(refs), [])] + return [('[%s referrers]' % len(refs), [])] try: ascendcode = self.ascend.__code__ @@ -72,20 +71,20 @@ class ReferrerTree(object): return self.peek(repr(obj)) if isinstance(obj, dict): - return "{" + ", ".join(["%s: %s" % (self._format(k, descend=False), + return '{' + ', '.join(['%s: %s' % (self._format(k, descend=False), self._format(v, descend=False)) - for k, v in obj.items()]) + "}" + for k, v in obj.items()]) + '}' elif isinstance(obj, list): - return "[" + ", ".join([self._format(item, descend=False) - for item in obj]) + "]" + return '[' + ', '.join([self._format(item, descend=False) + for item in obj]) + ']' elif isinstance(obj, tuple): - return "(" + ", ".join([self._format(item, descend=False) - for item in obj]) + ")" + return '(' + ', '.join([self._format(item, descend=False) + for item in obj]) + ')' r = self.peek(repr(obj)) if isinstance(obj, (str, int, float)): return r - return "%s: %s" % (type(obj), r) + return '%s: %s' % (type(obj), r) def format(self, tree): """Return a list of string reprs from a nested list of referrers.""" @@ -93,7 +92,7 @@ class ReferrerTree(object): def ascend(branch, depth=1): for parent, grandparents in branch: - output.append((" " * depth) + self._format(parent)) + output.append((' ' * depth) + self._format(parent)) if grandparents: ascend(grandparents, depth + 1) ascend(tree) @@ -120,14 +119,14 @@ request_counter.subscribe() def get_context(obj): if isinstance(obj, _cprequest.Request): - return "path=%s;stage=%s" % (obj.path_info, obj.stage) + return 'path=%s;stage=%s' % (obj.path_info, obj.stage) elif isinstance(obj, _cprequest.Response): - return "status=%s" % obj.status + return 'status=%s' % obj.status elif isinstance(obj, _cpwsgi.AppResponse): - return "PATH_INFO=%s" % obj.environ.get('PATH_INFO', '') - elif hasattr(obj, "tb_lineno"): - return "tb_lineno=%s" % obj.tb_lineno - return "" + return 'PATH_INFO=%s' % obj.environ.get('PATH_INFO', '') + elif hasattr(obj, 'tb_lineno'): + return 'tb_lineno=%s' % obj.tb_lineno + return '' class GCRoot(object): @@ -136,27 +135,27 @@ class GCRoot(object): classes = [ (_cprequest.Request, 2, 2, - "Should be 1 in this request thread and 1 in the main thread."), + 'Should be 1 in this request thread and 1 in the main thread.'), (_cprequest.Response, 2, 2, - "Should be 1 in this request thread and 1 in the main thread."), + 'Should be 1 in this request thread and 1 in the main thread.'), (_cpwsgi.AppResponse, 1, 1, - "Should be 1 in this request thread only."), + 'Should be 1 in this request thread only.'), ] @cherrypy.expose def index(self): - return "Hello, world!" + return 'Hello, world!' @cherrypy.expose def stats(self): - output = ["Statistics:"] + output = ['Statistics:'] for trial in range(10): if request_counter.count > 0: break time.sleep(0.5) else: - output.append("\nNot all requests closed properly.") + output.append('\nNot all requests closed properly.') # gc_collect isn't perfectly synchronous, because it may # break reference cycles that then take time to fully @@ -174,11 +173,11 @@ class GCRoot(object): for x in gc.garbage: trash[type(x)] = trash.get(type(x), 0) + 1 if trash: - output.insert(0, "\n%s unreachable objects:" % unreachable) + output.insert(0, '\n%s unreachable objects:' % unreachable) trash = [(v, k) for k, v in trash.items()] trash.sort() for pair in trash: - output.append(" " + repr(pair)) + output.append(' ' + repr(pair)) # Check declared classes to verify uncollected instances. # These don't have to be part of a cycle; they can be @@ -194,24 +193,24 @@ class GCRoot(object): if lenobj < minobj or lenobj > maxobj: if minobj == maxobj: output.append( - "\nExpected %s %r references, got %s." % + '\nExpected %s %r references, got %s.' % (minobj, cls, lenobj)) else: output.append( - "\nExpected %s to %s %r references, got %s." % + '\nExpected %s to %s %r references, got %s.' % (minobj, maxobj, cls, lenobj)) for obj in objs: if objgraph is not None: ig = [id(objs), id(inspect.currentframe())] - fname = "graph_%s_%s.png" % (cls.__name__, id(obj)) + fname = 'graph_%s_%s.png' % (cls.__name__, id(obj)) objgraph.show_backrefs( obj, extra_ignore=ig, max_depth=4, too_many=20, filename=fname, extra_info=get_context) - output.append("\nReferrers for %s (refcount=%s):" % + output.append('\nReferrers for %s (refcount=%s):' % (repr(obj), sys.getrefcount(obj))) t = ReferrerTree(ignore=[objs], maxdepth=3) tree = t.ascend(obj) output.extend(t.format(tree)) - return "\n".join(output) + return '\n'.join(output) diff --git a/cherrypy/lib/http.py b/cherrypy/lib/http.py deleted file mode 100644 index 12043ad..0000000 --- a/cherrypy/lib/http.py +++ /dev/null @@ -1,6 +0,0 @@ -import warnings -warnings.warn('cherrypy.lib.http has been deprecated and will be removed ' - 'in CherryPy 3.3 use cherrypy.lib.httputil instead.', - DeprecationWarning) - -from cherrypy.lib.httputil import * diff --git a/cherrypy/lib/httpauth.py b/cherrypy/lib/httpauth.py index 6d51990..55dd2f8 100644 --- a/cherrypy/lib/httpauth.py +++ b/cherrypy/lib/httpauth.py @@ -20,8 +20,18 @@ Usage: SUPPORTED_ALGORITHM - list of supported 'Digest' algorithms SUPPORTED_QOP - list of supported 'Digest' 'qop'. """ + +import time +from hashlib import md5 + +from cherrypy._cpcompat import ( + base64_decode, ntob, + parse_http_list, parse_keqv_list +) + + __version__ = 1, 0, 1 -__author__ = "Tiago Cogumbreiro " +__author__ = 'Tiago Cogumbreiro ' __credits__ = """ Peter van Kampen for its recipe which implement most of Digest authentication: @@ -56,21 +66,16 @@ OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """ -__all__ = ("digestAuth", "basicAuth", "doAuth", "checkResponse", - "parseAuthorization", "SUPPORTED_ALGORITHM", "md5SessionKey", - "calculateNonce", "SUPPORTED_QOP") +__all__ = ('digestAuth', 'basicAuth', 'doAuth', 'checkResponse', + 'parseAuthorization', 'SUPPORTED_ALGORITHM', 'md5SessionKey', + 'calculateNonce', 'SUPPORTED_QOP') ########################################################################## -import time -from hashlib import md5 - -from cherrypy._cpcompat import base64_decode, ntob -from cherrypy._cpcompat import parse_http_list, parse_keqv_list -MD5 = "MD5" -MD5_SESS = "MD5-sess" -AUTH = "auth" -AUTH_INT = "auth-int" +MD5 = 'MD5' +MD5_SESS = 'MD5-sess' +AUTH = 'auth' +AUTH_INT = 'auth-int' SUPPORTED_ALGORITHM = (MD5, MD5_SESS) SUPPORTED_QOP = (AUTH, AUTH_INT) @@ -95,10 +100,10 @@ def calculateNonce(realm, algorithm=MD5): try: encoder = DIGEST_AUTH_ENCODERS[algorithm] except KeyError: - raise NotImplementedError("The chosen algorithm (%s) does not have " - "an implementation yet" % algorithm) + raise NotImplementedError('The chosen algorithm (%s) does not have ' + 'an implementation yet' % algorithm) - return encoder("%d:%s" % (time.time(), realm)) + return encoder('%d:%s' % (time.time(), realm)) def digestAuth(realm, algorithm=MD5, nonce=None, qop=AUTH): @@ -129,7 +134,7 @@ def doAuth(realm): This should be set in the HTTP header under the key 'WWW-Authenticate'.""" - return digestAuth(realm) + " " + basicAuth(realm) + return digestAuth(realm) + ' ' + basicAuth(realm) ########################################################################## @@ -143,31 +148,31 @@ def _parseDigestAuthorization(auth_params): # Now validate the params # Check for required parameters - required = ["username", "realm", "nonce", "uri", "response"] + required = ['username', 'realm', 'nonce', 'uri', 'response'] for k in required: if k not in params: return None # If qop is sent then cnonce and nc MUST be present - if "qop" in params and not ("cnonce" in params - and "nc" in params): + if 'qop' in params and not ('cnonce' in params + and 'nc' in params): return None # If qop is not sent, neither cnonce nor nc can be present - if ("cnonce" in params or "nc" in params) and \ - "qop" not in params: + if ('cnonce' in params or 'nc' in params) and \ + 'qop' not in params: return None return params def _parseBasicAuthorization(auth_params): - username, password = base64_decode(auth_params).split(":", 1) - return {"username": username, "password": password} + username, password = base64_decode(auth_params).split(':', 1) + return {'username': username, 'password': password} AUTH_SCHEMES = { - "basic": _parseBasicAuthorization, - "digest": _parseDigestAuthorization, + 'basic': _parseBasicAuthorization, + 'digest': _parseDigestAuthorization, } @@ -178,7 +183,7 @@ def parseAuthorization(credentials): global AUTH_SCHEMES - auth_scheme, auth_params = credentials.split(" ", 1) + auth_scheme, auth_params = credentials.split(' ', 1) auth_scheme = auth_scheme.lower() parser = AUTH_SCHEMES[auth_scheme] @@ -187,8 +192,8 @@ def parseAuthorization(credentials): if params is None: return - assert "auth_scheme" not in params - params["auth_scheme"] = auth_scheme + assert 'auth_scheme' not in params + params['auth_scheme'] = auth_scheme return params @@ -214,50 +219,50 @@ def md5SessionKey(params, password): specification. """ - keys = ("username", "realm", "nonce", "cnonce") + keys = ('username', 'realm', 'nonce', 'cnonce') params_copy = {} for key in keys: params_copy[key] = params[key] - params_copy["algorithm"] = MD5_SESS + params_copy['algorithm'] = MD5_SESS return _A1(params_copy, password) def _A1(params, password): - algorithm = params.get("algorithm", MD5) + algorithm = params.get('algorithm', MD5) H = DIGEST_AUTH_ENCODERS[algorithm] if algorithm == MD5: # If the "algorithm" directive's value is "MD5" or is # unspecified, then A1 is: # A1 = unq(username-value) ":" unq(realm-value) ":" passwd - return "%s:%s:%s" % (params["username"], params["realm"], password) + return '%s:%s:%s' % (params['username'], params['realm'], password) elif algorithm == MD5_SESS: # This is A1 if qop is set # A1 = H( unq(username-value) ":" unq(realm-value) ":" passwd ) # ":" unq(nonce-value) ":" unq(cnonce-value) - h_a1 = H("%s:%s:%s" % (params["username"], params["realm"], password)) - return "%s:%s:%s" % (h_a1, params["nonce"], params["cnonce"]) + h_a1 = H('%s:%s:%s' % (params['username'], params['realm'], password)) + return '%s:%s:%s' % (h_a1, params['nonce'], params['cnonce']) def _A2(params, method, kwargs): # If the "qop" directive's value is "auth" or is unspecified, then A2 is: # A2 = Method ":" digest-uri-value - qop = params.get("qop", "auth") - if qop == "auth": - return method + ":" + params["uri"] - elif qop == "auth-int": + qop = params.get('qop', 'auth') + if qop == 'auth': + return method + ':' + params['uri'] + elif qop == 'auth-int': # If the "qop" value is "auth-int", then A2 is: # A2 = Method ":" digest-uri-value ":" H(entity-body) - entity_body = kwargs.get("entity_body", "") - H = kwargs["H"] + entity_body = kwargs.get('entity_body', '') + H = kwargs['H'] - return "%s:%s:%s" % ( + return '%s:%s:%s' % ( method, - params["uri"], + params['uri'], H(entity_body) ) @@ -265,19 +270,19 @@ def _A2(params, method, kwargs): raise NotImplementedError("The 'qop' method is unknown: %s" % qop) -def _computeDigestResponse(auth_map, password, method="GET", A1=None, +def _computeDigestResponse(auth_map, password, method='GET', A1=None, **kwargs): """ Generates a response respecting the algorithm defined in RFC 2617 """ params = auth_map - algorithm = params.get("algorithm", MD5) + algorithm = params.get('algorithm', MD5) H = DIGEST_AUTH_ENCODERS[algorithm] - KD = lambda secret, data: H(secret + ":" + data) + KD = lambda secret, data: H(secret + ':' + data) - qop = params.get("qop", None) + qop = params.get('qop', None) H_A2 = H(_A2(params, method, kwargs)) @@ -286,7 +291,7 @@ def _computeDigestResponse(auth_map, password, method="GET", A1=None, else: H_A1 = H(_A1(params, password)) - if qop in ("auth", "auth-int"): + if qop in ('auth', 'auth-int'): # If the "qop" value is "auth" or "auth-int": # request-digest = <"> < KD ( H(A1), unq(nonce-value) # ":" nc-value @@ -294,11 +299,11 @@ def _computeDigestResponse(auth_map, password, method="GET", A1=None, # ":" unq(qop-value) # ":" H(A2) # ) <"> - request = "%s:%s:%s:%s:%s" % ( - params["nonce"], - params["nc"], - params["cnonce"], - params["qop"], + request = '%s:%s:%s:%s:%s' % ( + params['nonce'], + params['nc'], + params['cnonce'], + params['qop'], H_A2, ) elif qop is None: @@ -306,12 +311,12 @@ def _computeDigestResponse(auth_map, password, method="GET", A1=None, # for compatibility with RFC 2069): # request-digest = # <"> < KD ( H(A1), unq(nonce-value) ":" H(A2) ) > <"> - request = "%s:%s" % (params["nonce"], H_A2) + request = '%s:%s' % (params['nonce'], H_A2) return KD(H_A1, request) -def _checkDigestResponse(auth_map, password, method="GET", A1=None, **kwargs): +def _checkDigestResponse(auth_map, password, method='GET', A1=None, **kwargs): """This function is used to verify the response given by the client when he tries to authenticate. Optional arguments: @@ -329,7 +334,7 @@ def _checkDigestResponse(auth_map, password, method="GET", A1=None, **kwargs): response = _computeDigestResponse( auth_map, password, method, A1, **kwargs) - return response == auth_map["response"] + return response == auth_map['response'] def _checkBasicResponse(auth_map, password, method='GET', encrypt=None, @@ -339,19 +344,19 @@ def _checkBasicResponse(auth_map, password, method='GET', encrypt=None, pass_through = lambda password, username=None: password encrypt = encrypt or pass_through try: - candidate = encrypt(auth_map["password"], auth_map["username"]) + candidate = encrypt(auth_map['password'], auth_map['username']) except TypeError: # if encrypt only takes one parameter, it's the password - candidate = encrypt(auth_map["password"]) + candidate = encrypt(auth_map['password']) return candidate == password AUTH_RESPONSES = { - "basic": _checkBasicResponse, - "digest": _checkDigestResponse, + 'basic': _checkBasicResponse, + 'digest': _checkDigestResponse, } -def checkResponse(auth_map, password, method="GET", encrypt=None, **kwargs): +def checkResponse(auth_map, password, method='GET', encrypt=None, **kwargs): """'checkResponse' compares the auth_map with the password and optionally other arguments that each implementation might need. @@ -368,6 +373,6 @@ def checkResponse(auth_map, password, method="GET", encrypt=None, **kwargs): The 'A1' argument is only used in MD5_SESS algorithm based responses. Check md5SessionKey() for more info. """ - checker = AUTH_RESPONSES[auth_map["auth_scheme"]] + checker = AUTH_RESPONSES[auth_map['auth_scheme']] return checker(auth_map, password, method=method, encrypt=encrypt, **kwargs) diff --git a/cherrypy/lib/httputil.py b/cherrypy/lib/httputil.py index c0af01e..105f9a3 100644 --- a/cherrypy/lib/httputil.py +++ b/cherrypy/lib/httputil.py @@ -7,13 +7,23 @@ FuManChu will personally hang you up by your thumbs and submit you to a public caning. """ +import functools +import email.utils +import re from binascii import b2a_base64 +from cgi import parse_header +try: + # Python 3 + from email.header import decode_header +except ImportError: + from email.Header import decode_header import six -from cherrypy._cpcompat import BaseHTTPRequestHandler, HTTPDate, ntob, ntou -from cherrypy._cpcompat import basestring, iteritems +from cherrypy._cpcompat import BaseHTTPRequestHandler, ntob, ntou +from cherrypy._cpcompat import text_or_bytes, iteritems from cherrypy._cpcompat import reversed, sorted, unquote_qs + response_codes = BaseHTTPRequestHandler.responses.copy() # From https://github.com/cherrypy/cherrypy/issues/361 @@ -25,8 +35,8 @@ response_codes[503] = ('Service Unavailable', 'request due to a temporary overloading or ' 'maintenance of the server.') -import re -from cgi import parse_header + +HTTPDate = functools.partial(email.utils.formatdate, usegmt=True) def urljoin(*atoms): @@ -35,11 +45,11 @@ def urljoin(*atoms): This will correctly join a SCRIPT_NAME and PATH_INFO into the original URL, even if either atom is blank. """ - url = "/".join([x for x in atoms if x]) - while "//" in url: - url = url.replace("//", "/") + url = '/'.join([x for x in atoms if x]) + while '//' in url: + url = url.replace('//', '/') # Special-case the final url of "", and return "/" instead. - return url or "/" + return url or '/' def urljoin_bytes(*atoms): @@ -48,11 +58,11 @@ def urljoin_bytes(*atoms): This will correctly join a SCRIPT_NAME and PATH_INFO into the original URL, even if either atom is blank. """ - url = ntob("/").join([x for x in atoms if x]) - while ntob("//") in url: - url = url.replace(ntob("//"), ntob("/")) + url = ntob('/').join([x for x in atoms if x]) + while ntob('//') in url: + url = url.replace(ntob('//'), ntob('/')) # Special-case the final url of "", and return "/" instead. - return url or ntob("/") + return url or ntob('/') def protocol_from_http(protocol_str): @@ -75,9 +85,9 @@ def get_ranges(headervalue, content_length): return None result = [] - bytesunit, byteranges = headervalue.split("=", 1) - for brange in byteranges.split(","): - start, stop = [x.strip() for x in brange.split("-", 1)] + bytesunit, byteranges = headervalue.split('=', 1) + for brange in byteranges.split(','): + start, stop = [x.strip() for x in brange.split('-', 1)] if start: if not stop: stop = content_length - 1 @@ -135,8 +145,8 @@ class HeaderElement(object): return self.value < other.value def __str__(self): - p = [";%s=%s" % (k, v) for k, v in iteritems(self.params)] - return str("%s%s" % (self.value, "".join(p))) + p = [';%s=%s' % (k, v) for k, v in iteritems(self.params)] + return str('%s%s' % (self.value, ''.join(p))) def __bytes__(self): return ntob(self.__str__()) @@ -144,17 +154,17 @@ class HeaderElement(object): def __unicode__(self): return ntou(self.__str__()) + @staticmethod def parse(elementstr): """Transform 'token;key=val' to ('token', {'key': 'val'}).""" initial_value, params = parse_header(elementstr) return initial_value, params - parse = staticmethod(parse) + @classmethod def from_str(cls, elementstr): """Construct an instance from a string of the form 'token;key=val'.""" ival, params = cls.parse(elementstr) return cls(ival, params) - from_str = classmethod(from_str) q_separator = re.compile(r'; *q *=') @@ -171,6 +181,7 @@ class AcceptElement(HeaderElement): have been the other way around, but it's too late to fix now. """ + @classmethod def from_str(cls, elementstr): qvalue = None # The first "q" parameter (if any) separates the initial @@ -184,16 +195,16 @@ class AcceptElement(HeaderElement): media_type, params = cls.parse(media_range) if qvalue is not None: - params["q"] = qvalue + params['q'] = qvalue return cls(media_type, params) - from_str = classmethod(from_str) + @property def qvalue(self): - val = self.params.get("q", "1") + 'The qvalue, or priority, of this value.' + val = self.params.get('q', '1') if isinstance(val, HeaderElement): val = val.value return float(val) - qvalue = property(qvalue, doc="The qvalue, or priority, of this value.") def __cmp__(self, other): diff = cmp(self.qvalue, other.qvalue) @@ -216,7 +227,7 @@ def header_elements(fieldname, fieldvalue): result = [] for element in RE_HEADER_SPLIT.split(fieldvalue): - if fieldname.startswith("Accept") or fieldname == 'TE': + if fieldname.startswith('Accept') or fieldname == 'TE': hv = AcceptElement.from_str(element) else: hv = HeaderElement.from_str(element) @@ -227,13 +238,8 @@ def header_elements(fieldname, fieldvalue): def decode_TEXT(value): r"""Decode :rfc:`2047` TEXT (e.g. "=?utf-8?q?f=C3=BCr?=" -> "f\xfcr").""" - try: - # Python 3 - from email.header import decode_header - except ImportError: - from email.Header import decode_header atoms = decode_header(value) - decodedvalue = "" + decodedvalue = '' for atom, charset in atoms: if charset is not None: atom = atom.decode(charset) @@ -254,7 +260,7 @@ def valid_status(status): status = 200 status = str(status) - parts = status.split(" ", 1) + parts = status.split(' ', 1) if len(parts) == 1: # No reason supplied. code, = parts @@ -266,16 +272,16 @@ def valid_status(status): try: code = int(code) except ValueError: - raise ValueError("Illegal response status from server " - "(%s is non-numeric)." % repr(code)) + raise ValueError('Illegal response status from server ' + '(%s is non-numeric).' % repr(code)) if code < 100 or code > 599: - raise ValueError("Illegal response status from server " - "(%s is out of range)." % repr(code)) + raise ValueError('Illegal response status from server ' + '(%s is out of range).' % repr(code)) if code not in response_codes: # code is unknown but not illegal - default_reason, message = "", "" + default_reason, message = '', '' else: default_reason, message = response_codes[code] @@ -316,7 +322,7 @@ def _parse_qs(qs, keep_blank_values=0, strict_parsing=0, encoding='utf-8'): nv = name_value.split('=', 1) if len(nv) != 2: if strict_parsing: - raise ValueError("bad query field: %r" % (name_value,)) + raise ValueError('bad query field: %r' % (name_value,)) # Handle case of a control-name with no equal sign if keep_blank_values: nv.append('') @@ -334,7 +340,7 @@ def _parse_qs(qs, keep_blank_values=0, strict_parsing=0, encoding='utf-8'): return d -image_map_pattern = re.compile(r"[0-9]+,[0-9]+") +image_map_pattern = re.compile(r'[0-9]+,[0-9]+') def parse_query_string(query_string, keep_blank_values=True, encoding='utf-8'): @@ -347,7 +353,7 @@ def parse_query_string(query_string, keep_blank_values=True, encoding='utf-8'): if image_map_pattern.match(query_string): # Server-side image map. Map the coords to 'x' and 'y' # (like CGI::Request does). - pm = query_string.split(",") + pm = query_string.split(',') pm = {'x': int(pm[0]), 'y': int(pm[1])} else: pm = _parse_qs(query_string, keep_blank_values, encoding=encoding) @@ -384,12 +390,12 @@ class CaseInsensitiveDict(dict): for k in E.keys(): self[str(k).title()] = E[k] + @classmethod def fromkeys(cls, seq, value=None): newdict = cls() for k in seq: newdict[str(k).title()] = value return newdict - fromkeys = classmethod(fromkeys) def setdefault(self, key, x=None): key = str(key).title() @@ -428,7 +434,7 @@ class HeaderMap(CaseInsensitiveDict): """ protocol = (1, 1) - encodings = ["ISO-8859-1"] + encodings = ['ISO-8859-1'] # Someday, when http-bis is done, this will probably get dropped # since few servers, clients, or intermediaries do it. But until then, @@ -451,6 +457,7 @@ class HeaderMap(CaseInsensitiveDict): """Transform self into a list of (name, value) tuples.""" return list(self.encode_header_items(self.items())) + @classmethod def encode_header_items(cls, header_items): """ Prepare the sequence of name, value tuples into a form suitable for @@ -460,7 +467,7 @@ class HeaderMap(CaseInsensitiveDict): if isinstance(k, six.text_type): k = cls.encode(k) - if not isinstance(v, basestring): + if not isinstance(v, text_or_bytes): v = str(v) if isinstance(v, six.text_type): @@ -474,8 +481,8 @@ class HeaderMap(CaseInsensitiveDict): header_translate_deletechars) yield (k, v) - encode_header_items = classmethod(encode_header_items) + @classmethod def encode(cls, v): """Return the given header name or value, encoded for HTTP output.""" for enc in cls.encodings: @@ -493,10 +500,9 @@ class HeaderMap(CaseInsensitiveDict): v = b2a_base64(v.encode('utf-8')) return (ntob('=?utf-8?b?') + v.strip(ntob('\n')) + ntob('?=')) - raise ValueError("Could not encode header part %r using " - "any of the encodings %r." % + raise ValueError('Could not encode header part %r using ' + 'any of the encodings %r.' % (v, cls.encodings)) - encode = classmethod(encode) class Host(object): @@ -509,9 +515,9 @@ class Host(object): """ - ip = "0.0.0.0" + ip = '0.0.0.0' port = 80 - name = "unknown.tld" + name = 'unknown.tld' def __init__(self, ip, port, name=None): self.ip = ip @@ -521,4 +527,4 @@ class Host(object): self.name = name def __repr__(self): - return "httputil.Host(%r, %r, %r)" % (self.ip, self.port, self.name) + return 'httputil.Host(%r, %r, %r)' % (self.ip, self.port, self.name) diff --git a/cherrypy/lib/jsontools.py b/cherrypy/lib/jsontools.py index 90b3ff8..91ea74e 100644 --- a/cherrypy/lib/jsontools.py +++ b/cherrypy/lib/jsontools.py @@ -1,17 +1,15 @@ import cherrypy -from cherrypy._cpcompat import basestring, ntou, json_encode, json_decode +from cherrypy._cpcompat import text_or_bytes, ntou, json_encode, json_decode def json_processor(entity): """Read application/json data into request.json.""" - if not entity.headers.get(ntou("Content-Length"), ntou("")): + if not entity.headers.get(ntou('Content-Length'), ntou('')): raise cherrypy.HTTPError(411) body = entity.fp.read() - try: + with cherrypy.HTTPError.handle(ValueError, 400, 'Invalid JSON document'): cherrypy.serving.request.json = json_decode(body.decode('utf-8')) - except ValueError: - raise cherrypy.HTTPError(400, 'Invalid JSON document') def json_in(content_type=[ntou('application/json'), ntou('text/javascript')], @@ -41,7 +39,7 @@ def json_in(content_type=[ntou('application/json'), ntou('text/javascript')], package importable; otherwise, ValueError is raised during processing. """ request = cherrypy.serving.request - if isinstance(content_type, basestring): + if isinstance(content_type, text_or_bytes): content_type = [content_type] if force: diff --git a/cherrypy/lib/lockfile.py b/cherrypy/lib/lockfile.py index b9d8e02..bb1b5bb 100644 --- a/cherrypy/lib/lockfile.py +++ b/cherrypy/lib/lockfile.py @@ -1,147 +1,142 @@ -""" -Platform-independent file locking. Inspired by and modeled after zc.lockfile. -""" - -import os - -try: - import msvcrt -except ImportError: - pass - -try: - import fcntl -except ImportError: - pass - - -class LockError(Exception): - - "Could not obtain a lock" - - msg = "Unable to lock %r" - - def __init__(self, path): - super(LockError, self).__init__(self.msg % path) - - -class UnlockError(LockError): - - "Could not release a lock" - - msg = "Unable to unlock %r" - - -# first, a default, naive locking implementation -class LockFile(object): - - """ - A default, naive locking implementation. Always fails if the file - already exists. - """ - - def __init__(self, path): - self.path = path - try: - fd = os.open(path, os.O_CREAT | os.O_WRONLY | os.O_EXCL) - except OSError: - raise LockError(self.path) - os.close(fd) - - def release(self): - os.remove(self.path) - - def remove(self): - pass - - -class SystemLockFile(object): - - """ - An abstract base class for platform-specific locking. - """ - - def __init__(self, path): - self.path = path - - try: - # Open lockfile for writing without truncation: - self.fp = open(path, 'r+') - except IOError: - # If the file doesn't exist, IOError is raised; Use a+ instead. - # Note that there may be a race here. Multiple processes - # could fail on the r+ open and open the file a+, but only - # one will get the the lock and write a pid. - self.fp = open(path, 'a+') - - try: - self._lock_file() - except: - self.fp.seek(1) - self.fp.close() - del self.fp - raise - - self.fp.write(" %s\n" % os.getpid()) - self.fp.truncate() - self.fp.flush() - - def release(self): - if not hasattr(self, 'fp'): - return - self._unlock_file() - self.fp.close() - del self.fp - - def remove(self): - """ - Attempt to remove the file - """ - try: - os.remove(self.path) - except: - pass - - #@abc.abstract_method - # def _lock_file(self): - # """Attempt to obtain the lock on self.fp. Raise LockError if not - # acquired.""" - - def _unlock_file(self): - """Attempt to obtain the lock on self.fp. Raise UnlockError if not - released.""" - - -class WindowsLockFile(SystemLockFile): - - def _lock_file(self): - # Lock just the first byte - try: - msvcrt.locking(self.fp.fileno(), msvcrt.LK_NBLCK, 1) - except IOError: - raise LockError(self.fp.name) - - def _unlock_file(self): - try: - self.fp.seek(0) - msvcrt.locking(self.fp.fileno(), msvcrt.LK_UNLCK, 1) - except IOError: - raise UnlockError(self.fp.name) - -if 'msvcrt' in globals(): - LockFile = WindowsLockFile - - -class UnixLockFile(SystemLockFile): - - def _lock_file(self): - flags = fcntl.LOCK_EX | fcntl.LOCK_NB - try: - fcntl.flock(self.fp.fileno(), flags) - except IOError: - raise LockError(self.fp.name) - - # no need to implement _unlock_file, it will be unlocked on close() - -if 'fcntl' in globals(): - LockFile = UnixLockFile +""" +Platform-independent file locking. Inspired by and modeled after zc.lockfile. +""" + +import os + +try: + import msvcrt +except ImportError: + pass + +try: + import fcntl +except ImportError: + pass + + +class LockError(Exception): + + 'Could not obtain a lock' + + msg = 'Unable to lock %r' + + def __init__(self, path): + super(LockError, self).__init__(self.msg % path) + + +class UnlockError(LockError): + + 'Could not release a lock' + + msg = 'Unable to unlock %r' + + +# first, a default, naive locking implementation +class LockFile(object): + + """ + A default, naive locking implementation. Always fails if the file + already exists. + """ + + def __init__(self, path): + self.path = path + try: + fd = os.open(path, os.O_CREAT | os.O_WRONLY | os.O_EXCL) + except OSError: + raise LockError(self.path) + os.close(fd) + + def release(self): + os.remove(self.path) + + def remove(self): + pass + + +class SystemLockFile(object): + + """ + An abstract base class for platform-specific locking. + """ + + def __init__(self, path): + self.path = path + + try: + # Open lockfile for writing without truncation: + self.fp = open(path, 'r+') + except IOError: + # If the file doesn't exist, IOError is raised; Use a+ instead. + # Note that there may be a race here. Multiple processes + # could fail on the r+ open and open the file a+, but only + # one will get the the lock and write a pid. + self.fp = open(path, 'a+') + + try: + self._lock_file() + except: + self.fp.seek(1) + self.fp.close() + del self.fp + raise + + self.fp.write(' %s\n' % os.getpid()) + self.fp.truncate() + self.fp.flush() + + def release(self): + if not hasattr(self, 'fp'): + return + self._unlock_file() + self.fp.close() + del self.fp + + def remove(self): + """ + Attempt to remove the file + """ + try: + os.remove(self.path) + except: + pass + + def _unlock_file(self): + """Attempt to obtain the lock on self.fp. Raise UnlockError if not + released.""" + + +class WindowsLockFile(SystemLockFile): + + def _lock_file(self): + # Lock just the first byte + try: + msvcrt.locking(self.fp.fileno(), msvcrt.LK_NBLCK, 1) + except IOError: + raise LockError(self.fp.name) + + def _unlock_file(self): + try: + self.fp.seek(0) + msvcrt.locking(self.fp.fileno(), msvcrt.LK_UNLCK, 1) + except IOError: + raise UnlockError(self.fp.name) + +if 'msvcrt' in globals(): + LockFile = WindowsLockFile + + +class UnixLockFile(SystemLockFile): + + def _lock_file(self): + flags = fcntl.LOCK_EX | fcntl.LOCK_NB + try: + fcntl.flock(self.fp.fileno(), flags) + except IOError: + raise LockError(self.fp.name) + + # no need to implement _unlock_file, it will be unlocked on close() + +if 'fcntl' in globals(): + LockFile = UnixLockFile diff --git a/cherrypy/lib/locking.py b/cherrypy/lib/locking.py index 72dda9b..317fb58 100644 --- a/cherrypy/lib/locking.py +++ b/cherrypy/lib/locking.py @@ -11,7 +11,7 @@ class Timer(object): A simple timer that will indicate when an expiration time has passed. """ def __init__(self, expiration): - "Create a timer that expires at `expiration` (UTC datetime)" + 'Create a timer that expires at `expiration` (UTC datetime)' self.expiration = expiration @classmethod @@ -26,7 +26,7 @@ class Timer(object): class LockTimeout(Exception): - "An exception when a lock could not be acquired before a timeout period" + 'An exception when a lock could not be acquired before a timeout period' class LockChecker(object): @@ -43,5 +43,5 @@ class LockChecker(object): def expired(self): if self.timer.expired(): raise LockTimeout( - "Timeout acquiring lock for %(session_id)s" % vars(self)) + 'Timeout acquiring lock for %(session_id)s' % vars(self)) return False diff --git a/cherrypy/lib/profiler.py b/cherrypy/lib/profiler.py index 53cb83a..94b8798 100644 --- a/cherrypy/lib/profiler.py +++ b/cherrypy/lib/profiler.py @@ -34,30 +34,31 @@ module from the command line, it will call ``serve()`` for you. """ import io +import os +import os.path +import sys +import warnings import cherrypy -def new_func_strip_path(func_name): - """Make profiler output more readable by adding `__init__` modules' parents - """ - filename, line, name = func_name - if filename.endswith("__init__.py"): - return os.path.basename(filename[:-12]) + filename[-12:], line, name - return os.path.basename(filename), line, name - try: import profile import pstats + + def new_func_strip_path(func_name): + """Make profiler output more readable by adding `__init__` modules' parents + """ + filename, line, name = func_name + if filename.endswith('__init__.py'): + return os.path.basename(filename[:-12]) + filename[-12:], line, name + return os.path.basename(filename), line, name + pstats.func_strip_path = new_func_strip_path except ImportError: profile = None pstats = None -import os -import os.path -import sys -import warnings _count = 0 @@ -66,7 +67,7 @@ class Profiler(object): def __init__(self, path=None): if not path: - path = os.path.join(os.path.dirname(__file__), "profile") + path = os.path.join(os.path.dirname(__file__), 'profile') self.path = path if not os.path.exists(path): os.makedirs(path) @@ -75,7 +76,7 @@ class Profiler(object): """Dump profile data into self.path.""" global _count c = _count = _count + 1 - path = os.path.join(self.path, "cp_%04d.prof" % c) + path = os.path.join(self.path, 'cp_%04d.prof' % c) prof = profile.Profile() result = prof.runcall(func, *args, **params) prof.dump_stats(path) @@ -85,7 +86,7 @@ class Profiler(object): """:rtype: list of available profiles. """ return [f for f in os.listdir(self.path) - if f.startswith("cp_") and f.endswith(".prof")] + if f.startswith('cp_') and f.endswith('.prof')] def stats(self, filename, sortby='cumulative'): """:rtype stats(index): output of print_stats() for the given profile. @@ -125,8 +126,8 @@ class Profiler(object): @cherrypy.expose def menu(self): - yield "

Profiling runs

" - yield "

Click on one of the runs below to see profiling data.

" + yield '

Profiling runs

' + yield '

Click on one of the runs below to see profiling data.

' runs = self.statfiles() runs.sort() for i in runs: @@ -135,7 +136,6 @@ class Profiler(object): @cherrypy.expose def report(self, filename): - import cherrypy cherrypy.response.headers['Content-Type'] = 'text/plain' return self.stats(filename) @@ -149,7 +149,7 @@ class ProfileAggregator(Profiler): self.profiler = profile.Profile() def run(self, func, *args, **params): - path = os.path.join(self.path, "cp_%04d.prof" % self.count) + path = os.path.join(self.path, 'cp_%04d.prof' % self.count) result = self.profiler.runcall(func, *args, **params) self.profiler.dump_stats(path) return result @@ -174,11 +174,11 @@ class make_app: """ if profile is None or pstats is None: - msg = ("Your installation of Python does not have a profile " + msg = ('Your installation of Python does not have a profile ' "module. If you're on Debian, try " - "`sudo apt-get install python-profiler`. " - "See http://www.cherrypy.org/wiki/ProfilingOnDebian " - "for details.") + '`sudo apt-get install python-profiler`. ' + 'See http://www.cherrypy.org/wiki/ProfilingOnDebian ' + 'for details.') warnings.warn(msg) self.nextapp = nextapp @@ -199,20 +199,19 @@ class make_app: def serve(path=None, port=8080): if profile is None or pstats is None: - msg = ("Your installation of Python does not have a profile module. " + msg = ('Your installation of Python does not have a profile module. ' "If you're on Debian, try " - "`sudo apt-get install python-profiler`. " - "See http://www.cherrypy.org/wiki/ProfilingOnDebian " - "for details.") + '`sudo apt-get install python-profiler`. ' + 'See http://www.cherrypy.org/wiki/ProfilingOnDebian ' + 'for details.') warnings.warn(msg) - import cherrypy cherrypy.config.update({'server.socket_port': int(port), 'server.thread_pool': 10, - 'environment': "production", + 'environment': 'production', }) cherrypy.quickstart(Profiler(path)) -if __name__ == "__main__": +if __name__ == '__main__': serve(*tuple(sys.argv[1:])) diff --git a/cherrypy/lib/reprconf.py b/cherrypy/lib/reprconf.py index 8af1f77..2553292 100644 --- a/cherrypy/lib/reprconf.py +++ b/cherrypy/lib/reprconf.py @@ -25,14 +25,9 @@ except ImportError: from ConfigParser import ConfigParser try: - set + text_or_bytes except NameError: - from sets import Set as set - -try: - basestring -except NameError: - basestring = str + text_or_bytes = str try: # Python 3 @@ -47,7 +42,7 @@ import sys def as_dict(config): """Return a dict from 'config' whether it is a dict, file, or filename.""" - if isinstance(config, basestring): + if isinstance(config, text_or_bytes): config = Parser().dict_from_file(config) elif hasattr(config, 'read'): config = Parser().dict_from_file(config) @@ -83,8 +78,8 @@ class NamespaceSet(dict): # Separate the given config into namespaces ns_confs = {} for k in config: - if "." in k: - ns, name = k.split(".", 1) + if '.' in k: + ns, name = k.split('.', 1) bucket = ns_confs.setdefault(ns, {}) bucket[name] = config[k] @@ -95,7 +90,7 @@ class NamespaceSet(dict): # for k, v in ns_confs.get(ns, {}).iteritems(): # callable(k, v) for ns, handler in self.items(): - exit = getattr(handler, "__exit__", None) + exit = getattr(handler, '__exit__', None) if exit: callable = handler.__enter__() no_exc = True @@ -120,7 +115,7 @@ class NamespaceSet(dict): handler(k, v) def __repr__(self): - return "%s.%s(%s)" % (self.__module__, self.__class__.__name__, + return '%s.%s(%s)' % (self.__module__, self.__class__.__name__, dict.__repr__(self)) def __copy__(self): @@ -155,7 +150,7 @@ class Config(dict): def update(self, config): """Update self from a dict, file or filename.""" - if isinstance(config, basestring): + if isinstance(config, text_or_bytes): # Filename config = Parser().dict_from_file(config) elif hasattr(config, 'read'): @@ -192,7 +187,7 @@ class Parser(ConfigParser): return optionstr def read(self, filenames): - if isinstance(filenames, basestring): + if isinstance(filenames, text_or_bytes): filenames = [filenames] for filename in filenames: # try: @@ -218,8 +213,8 @@ class Parser(ConfigParser): value = unrepr(value) except Exception: x = sys.exc_info()[1] - msg = ("Config error in section: %r, option: %r, " - "value: %r. Config values must be valid Python." % + msg = ('Config error in section: %r, option: %r, ' + 'value: %r. Config values must be valid Python.' % (section, option, value)) raise ValueError(msg, x.__class__.__name__, x.args) result[section][option] = value @@ -241,7 +236,7 @@ class _Builder2: def build(self, o): m = getattr(self, 'build_' + o.__class__.__name__, None) if m is None: - raise TypeError("unrepr does not recognize %s" % + raise TypeError('unrepr does not recognize %s' % repr(o.__class__.__name__)) return m(o) @@ -254,7 +249,7 @@ class _Builder2: # e.g. IronPython 1.0. return eval(s) - p = compiler.parse("__tempvalue__ = " + s) + p = compiler.parse('__tempvalue__ = ' + s) return p.getChildren()[1].getChildren()[0].getChildren()[1] def build_Subscript(self, o): @@ -327,7 +322,7 @@ class _Builder2: except AttributeError: pass - raise TypeError("unrepr could not resolve the name %s" % repr(name)) + raise TypeError('unrepr could not resolve the name %s' % repr(name)) def build_Add(self, o): left, right = map(self.build, o.getChildren()) @@ -356,7 +351,7 @@ class _Builder3: def build(self, o): m = getattr(self, 'build_' + o.__class__.__name__, None) if m is None: - raise TypeError("unrepr does not recognize %s" % + raise TypeError('unrepr does not recognize %s' % repr(o.__class__.__name__)) return m(o) @@ -369,7 +364,7 @@ class _Builder3: # e.g. IronPython 1.0. return eval(s) - p = ast.parse("__tempvalue__ = " + s) + p = ast.parse('__tempvalue__ = ' + s) return p.body[0].value def build_Subscript(self, o): @@ -397,8 +392,8 @@ class _Builder3: if kw.arg is None: # double asterix `**` rst = self.build(kw.value) if not isinstance(rst, dict): - raise TypeError("Invalid argument for call." - "Must be a mapping object.") + raise TypeError('Invalid argument for call.' + 'Must be a mapping object.') # give preference to the keys set directly from arg=value for k, v in rst.items(): if k not in kwargs: @@ -471,7 +466,7 @@ class _Builder3: except AttributeError: pass - raise TypeError("unrepr could not resolve the name %s" % repr(name)) + raise TypeError('unrepr could not resolve the name %s' % repr(name)) def build_NameConstant(self, o): return o.value @@ -523,7 +518,7 @@ def attributes(full_attribute_name): """Load a module and retrieve an attribute of that module.""" # Parse out the path, module, and attribute - last_dot = full_attribute_name.rfind(".") + last_dot = full_attribute_name.rfind('.') attr_name = full_attribute_name[last_dot + 1:] mod_path = full_attribute_name[:last_dot] diff --git a/cherrypy/lib/sessions.py b/cherrypy/lib/sessions.py index cc0ca4f..e28cbcc 100644 --- a/cherrypy/lib/sessions.py +++ b/cherrypy/lib/sessions.py @@ -4,13 +4,13 @@ You need to edit your config file to use sessions. Here's an example:: [/] tools.sessions.on = True - tools.sessions.storage_type = "file" + tools.sessions.storage_class = cherrypy.lib.sessions.FileSession tools.sessions.storage_path = "/home/site/sessions" tools.sessions.timeout = 60 This sets the session to be stored in files in the directory /home/site/sessions, and the session timeout to 60 minutes. If you omit -``storage_type`` the sessions will be saved in RAM. +``storage_class``, the sessions will be saved in RAM. ``tools.sessions.on`` is the only required line for working sessions, the rest are optional. @@ -95,8 +95,6 @@ import os import time import threading -import six - import cherrypy from cherrypy._cpcompat import copyitems, pickle, random20 from cherrypy.lib import httputil @@ -123,10 +121,10 @@ class Session(object): self._id = value for o in self.id_observers: o(value) - id = property(_get_id, _set_id, doc="The current session ID.") + id = property(_get_id, _set_id, doc='The current session ID.') timeout = 60 - "Number of minutes after which to delete session data." + 'Number of minutes after which to delete session data.' locked = False """ @@ -139,16 +137,16 @@ class Session(object): automatically on the first attempt to access session data.""" clean_thread = None - "Class-level Monitor which calls self.clean_up." + 'Class-level Monitor which calls self.clean_up.' clean_freq = 5 - "The poll rate for expired session cleanup in minutes." + 'The poll rate for expired session cleanup in minutes.' originalid = None - "The session id passed by the client. May be missing or unsafe." + 'The session id passed by the client. May be missing or unsafe.' missing = False - "True if the session requested by the client did not exist." + 'True if the session requested by the client did not exist.' regenerated = False """ @@ -156,7 +154,7 @@ class Session(object): internal calls to regenerate the session id.""" debug = False - "If True, log debug information." + 'If True, log debug information.' # --------------------- Session management methods --------------------- # @@ -472,9 +470,10 @@ class FileSession(Session): if isinstance(self.lock_timeout, (int, float)): self.lock_timeout = datetime.timedelta(seconds=self.lock_timeout) if not isinstance(self.lock_timeout, (datetime.timedelta, type(None))): - raise ValueError("Lock timeout must be numeric seconds or " - "a timedelta instance.") + raise ValueError('Lock timeout must be numeric seconds or ' + 'a timedelta instance.') + @classmethod def setup(cls, **kwargs): """Set up the storage system for file-based sessions. @@ -486,12 +485,11 @@ class FileSession(Session): for k, v in kwargs.items(): setattr(cls, k, v) - setup = classmethod(setup) def _get_file_path(self): f = os.path.join(self.storage_path, self.SESSION_PREFIX + self.id) if not os.path.abspath(f).startswith(self.storage_path): - raise cherrypy.HTTPError(400, "Invalid session id in cookie.") + raise cherrypy.HTTPError(400, 'Invalid session id in cookie.') return f def _exists(self): @@ -499,12 +497,12 @@ class FileSession(Session): return os.path.exists(path) def _load(self, path=None): - assert self.locked, ("The session load without being locked. " + assert self.locked, ('The session load without being locked. ' "Check your tools' priority levels.") if path is None: path = self._get_file_path() try: - f = open(path, "rb") + f = open(path, 'rb') try: return pickle.load(f) finally: @@ -512,21 +510,21 @@ class FileSession(Session): except (IOError, EOFError): e = sys.exc_info()[1] if self.debug: - cherrypy.log("Error loading the session pickle: %s" % + cherrypy.log('Error loading the session pickle: %s' % e, 'TOOLS.SESSIONS') return None def _save(self, expiration_time): - assert self.locked, ("The session was saved without being locked. " + assert self.locked, ('The session was saved without being locked. ' "Check your tools' priority levels.") - f = open(self._get_file_path(), "wb") + f = open(self._get_file_path(), 'wb') try: pickle.dump((self._data, expiration_time), f, self.pickle_protocol) finally: f.close() def _delete(self): - assert self.locked, ("The session deletion without being locked. " + assert self.locked, ('The session deletion without being locked. ' "Check your tools' priority levels.") try: os.unlink(self._get_file_path()) @@ -603,6 +601,7 @@ class MemcachedSession(Session): servers = ['127.0.0.1:11211'] + @classmethod def setup(cls, **kwargs): """Set up the storage system for memcached-based sessions. @@ -614,21 +613,6 @@ class MemcachedSession(Session): import memcache cls.cache = memcache.Client(cls.servers) - setup = classmethod(setup) - - def _get_id(self): - return self._id - - def _set_id(self, value): - # This encode() call is where we differ from the superclass. - # Memcache keys MUST be byte strings, not unicode. - if isinstance(value, six.text_type): - value = value.encode('utf-8') - - self._id = value - for o in self.id_observers: - o(value) - id = property(_get_id, _set_id, doc="The current session ID.") def _exists(self): self.mc_lock.acquire() @@ -651,7 +635,7 @@ class MemcachedSession(Session): try: if not self.cache.set(self.id, (self._data, expiration_time), td): raise AssertionError( - "Session data for id %r not set." % self.id) + 'Session data for id %r not set.' % self.id) finally: self.mc_lock.release() @@ -680,13 +664,13 @@ class MemcachedSession(Session): def save(): """Save any changed session data.""" - if not hasattr(cherrypy.serving, "session"): + if not hasattr(cherrypy.serving, 'session'): return request = cherrypy.serving.request response = cherrypy.serving.response # Guard against running twice - if hasattr(request, "_sessionsaved"): + if hasattr(request, '_sessionsaved'): return request._sessionsaved = True @@ -705,8 +689,8 @@ save.failsafe = True def close(): """Close the session object for this request.""" - sess = getattr(cherrypy.serving, "session", None) - if getattr(sess, "locked", False): + sess = getattr(cherrypy.serving, 'session', None) + if getattr(sess, 'locked', False): # If the session is still locked we release the lock sess.release_lock() if sess.debug: @@ -715,12 +699,19 @@ close.failsafe = True close.priority = 90 -def init(storage_type='ram', path=None, path_header=None, name='session_id', +def init(storage_type=None, path=None, path_header=None, name='session_id', timeout=60, domain=None, secure=False, clean_freq=5, - persistent=True, httponly=False, debug=False, **kwargs): + persistent=True, httponly=False, debug=False, + # Py27 compat + # *, storage_class=RamSession, + **kwargs): """Initialize session object (using cookies). + storage_class + The Session subclass to use. Defaults to RamSession. + storage_type + (deprecated) One of 'ram', 'file', memcached'. This will be used to look up the corresponding class in cherrypy.lib.sessions globals. For example, 'file' will use the FileSession class. @@ -765,10 +756,13 @@ def init(storage_type='ram', path=None, path_header=None, name='session_id', you're using for more information. """ + # Py27 compat + storage_class = kwargs.pop('storage_class', RamSession) + request = cherrypy.serving.request # Guard against running twice - if hasattr(request, "_session_init_flag"): + if hasattr(request, '_session_init_flag'): return request._session_init_flag = True @@ -780,11 +774,18 @@ def init(storage_type='ram', path=None, path_header=None, name='session_id', cherrypy.log('ID obtained from request.cookie: %r' % id, 'TOOLS.SESSIONS') - # Find the storage class and call setup (first time only). - storage_class = storage_type.title() + 'Session' - storage_class = globals()[storage_class] - if not hasattr(cherrypy, "session"): - if hasattr(storage_class, "setup"): + first_time = not hasattr(cherrypy, 'session') + + if storage_type: + if first_time: + msg = 'storage_type is deprecated. Supply storage_class instead' + cherrypy.log(msg) + storage_class = storage_type.title() + 'Session' + storage_class = globals()[storage_class] + + # call setup first time only + if first_time: + if hasattr(storage_class, 'setup'): storage_class.setup(**kwargs) # Create and attach a new Session instance to cherrypy.serving. @@ -801,7 +802,7 @@ def init(storage_type='ram', path=None, path_header=None, name='session_id', sess.id_observers.append(update_cookie) # Create cherrypy.session which will proxy to cherrypy.serving.session - if not hasattr(cherrypy, "session"): + if not hasattr(cherrypy, 'session'): cherrypy.session = cherrypy._ThreadLocalProxy('session') if persistent: @@ -869,7 +870,7 @@ def set_response_cookie(path=None, path_header=None, name='session_id', cookie[name]['secure'] = 1 if httponly: if not cookie[name].isReservedKey('httponly'): - raise ValueError("The httponly cookie token is not supported.") + raise ValueError('The httponly cookie token is not supported.') cookie[name]['httponly'] = 1 diff --git a/cherrypy/lib/static.py b/cherrypy/lib/static.py index 6a78fc1..4159c86 100644 --- a/cherrypy/lib/static.py +++ b/cherrypy/lib/static.py @@ -71,7 +71,7 @@ def serve_file(path, content_type=None, disposition=None, name=None, if content_type is None: # Set content-type based on filename extension - ext = "" + ext = '' i = path.rfind('.') if i != -1: ext = path[i:].lower() @@ -86,7 +86,7 @@ def serve_file(path, content_type=None, disposition=None, name=None, if name is None: name = os.path.basename(path) cd = '%s; filename="%s"' % (disposition, name) - response.headers["Content-Disposition"] = cd + response.headers['Content-Disposition'] = cd if debug: cherrypy.log('Content-Disposition: %r' % cd, 'TOOLS.STATIC') @@ -144,7 +144,7 @@ def serve_fileobj(fileobj, content_type=None, disposition=None, name=None, cd = disposition else: cd = '%s; filename="%s"' % (disposition, name) - response.headers["Content-Disposition"] = cd + response.headers['Content-Disposition'] = cd if debug: cherrypy.log('Content-Disposition: %r' % cd, 'TOOLS.STATIC') @@ -158,12 +158,12 @@ def _serve_fileobj(fileobj, content_type, content_length, debug=False): # HTTP/1.0 didn't have Range/Accept-Ranges headers, or the 206 code request = cherrypy.serving.request if request.protocol >= (1, 1): - response.headers["Accept-Ranges"] = "bytes" + response.headers['Accept-Ranges'] = 'bytes' r = httputil.get_ranges(request.headers.get('Range'), content_length) if r == []: - response.headers['Content-Range'] = "bytes */%s" % content_length - message = ("Invalid Range (first-byte-pos greater than " - "Content-Length)") + response.headers['Content-Range'] = 'bytes */%s' % content_length + message = ('Invalid Range (first-byte-pos greater than ' + 'Content-Length)') if debug: cherrypy.log(message, 'TOOLS.STATIC') raise cherrypy.HTTPError(416, message) @@ -179,15 +179,15 @@ def _serve_fileobj(fileobj, content_type, content_length, debug=False): cherrypy.log( 'Single part; start: %r, stop: %r' % (start, stop), 'TOOLS.STATIC') - response.status = "206 Partial Content" + response.status = '206 Partial Content' response.headers['Content-Range'] = ( - "bytes %s-%s/%s" % (start, stop - 1, content_length)) + 'bytes %s-%s/%s' % (start, stop - 1, content_length)) response.headers['Content-Length'] = r_len fileobj.seek(start) response.body = file_generator_limited(fileobj, r_len) else: # Return a multipart/byteranges response. - response.status = "206 Partial Content" + response.status = '206 Partial Content' try: # Python 3 from email.generator import _make_boundary as make_boundary @@ -195,15 +195,15 @@ def _serve_fileobj(fileobj, content_type, content_length, debug=False): # Python 2 from mimetools import choose_boundary as make_boundary boundary = make_boundary() - ct = "multipart/byteranges; boundary=%s" % boundary + ct = 'multipart/byteranges; boundary=%s' % boundary response.headers['Content-Type'] = ct - if "Content-Length" in response.headers: + if 'Content-Length' in response.headers: # Delete Content-Length header so finalize() recalcs it. - del response.headers["Content-Length"] + del response.headers['Content-Length'] def file_ranges(): # Apache compatibility: - yield ntob("\r\n") + yield ntob('\r\n') for start, stop in r: if debug: @@ -211,23 +211,23 @@ def _serve_fileobj(fileobj, content_type, content_length, debug=False): 'Multipart; start: %r, stop: %r' % ( start, stop), 'TOOLS.STATIC') - yield ntob("--" + boundary, 'ascii') - yield ntob("\r\nContent-type: %s" % content_type, + yield ntob('--' + boundary, 'ascii') + yield ntob('\r\nContent-type: %s' % content_type, 'ascii') yield ntob( - "\r\nContent-range: bytes %s-%s/%s\r\n\r\n" % ( + '\r\nContent-range: bytes %s-%s/%s\r\n\r\n' % ( start, stop - 1, content_length), 'ascii') fileobj.seek(start) gen = file_generator_limited(fileobj, stop - start) for chunk in gen: yield chunk - yield ntob("\r\n") + yield ntob('\r\n') # Final boundary - yield ntob("--" + boundary + "--", 'ascii') + yield ntob('--' + boundary + '--', 'ascii') # Apache compatibility: - yield ntob("\r\n") + yield ntob('\r\n') response.body = file_ranges() return response.body else: @@ -244,7 +244,7 @@ def _serve_fileobj(fileobj, content_type, content_length, debug=False): def serve_download(path, name=None): """Serve 'path' as an application/x-download attachment.""" # This is such a common idiom I felt it deserved its own wrapper. - return serve_file(path, "application/x-download", "attachment", name) + return serve_file(path, 'application/x-download', 'attachment', name) def _attempt(filename, content_types, debug=False): @@ -268,7 +268,7 @@ def _attempt(filename, content_types, debug=False): return False -def staticdir(section, dir, root="", match="", content_types=None, index="", +def staticdir(section, dir, root='', match='', content_types=None, index='', debug=False): """Serve a static resource from the given (root +) dir. @@ -306,7 +306,7 @@ def staticdir(section, dir, root="", match="", content_types=None, index="", # If dir is relative, make absolute using "root". if not os.path.isabs(dir): if not root: - msg = "Static dir requires an absolute dir (or root)." + msg = 'Static dir requires an absolute dir (or root).' if debug: cherrypy.log(msg, 'TOOLS.STATICDIR') raise ValueError(msg) @@ -315,10 +315,10 @@ def staticdir(section, dir, root="", match="", content_types=None, index="", # Determine where we are in the object tree relative to 'section' # (where the static tool was defined). if section == 'global': - section = "/" - section = section.rstrip(r"\/") + section = '/' + section = section.rstrip(r'\/') branch = request.path_info[len(section) + 1:] - branch = unquote(branch.lstrip(r"\/")) + branch = unquote(branch.lstrip(r'\/')) # If branch is "", filename will end in a slash filename = os.path.join(dir, branch) @@ -338,11 +338,11 @@ def staticdir(section, dir, root="", match="", content_types=None, index="", if index: handled = _attempt(os.path.join(filename, index), content_types) if handled: - request.is_index = filename[-1] in (r"\/") + request.is_index = filename[-1] in (r'\/') return handled -def staticfile(filename, root=None, match="", content_types=None, debug=False): +def staticfile(filename, root=None, match='', content_types=None, debug=False): """Serve a static resource from the given (root +) filename. match diff --git a/cherrypy/process/__init__.py b/cherrypy/process/__init__.py index f15b123..97f91ce 100644 --- a/cherrypy/process/__init__.py +++ b/cherrypy/process/__init__.py @@ -10,5 +10,5 @@ use with the bus. Some use tool-specific channels; see the documentation for each class. """ -from cherrypy.process.wspbus import bus -from cherrypy.process import plugins, servers +from cherrypy.process.wspbus import bus # noqa +from cherrypy.process import plugins, servers # noqa diff --git a/cherrypy/process/plugins.py b/cherrypy/process/plugins.py index 0ec585c..31e7d76 100644 --- a/cherrypy/process/plugins.py +++ b/cherrypy/process/plugins.py @@ -7,8 +7,8 @@ import sys import time import threading -from cherrypy._cpcompat import basestring, get_daemon, get_thread_ident -from cherrypy._cpcompat import ntob, Timer, SetDaemonProperty +from cherrypy._cpcompat import text_or_bytes, get_thread_ident +from cherrypy._cpcompat import ntob, Timer # _module__file__base is used by Autoreload to make # absolute any filenames retrieved from sys.modules which are not @@ -104,8 +104,8 @@ class SignalHandler(object): if sys.platform[:4] == 'java': del self.handlers['SIGUSR1'] self.handlers['SIGUSR2'] = self.bus.graceful - self.bus.log("SIGUSR1 cannot be set on the JVM platform. " - "Using SIGUSR2 instead.") + self.bus.log('SIGUSR1 cannot be set on the JVM platform. ' + 'Using SIGUSR2 instead.') self.handlers['SIGINT'] = self._jython_SIGINT_handler self._previous_handlers = {} @@ -152,19 +152,19 @@ class SignalHandler(object): signame = self.signals[signum] if handler is None: - self.bus.log("Restoring %s handler to SIG_DFL." % signame) + self.bus.log('Restoring %s handler to SIG_DFL.' % signame) handler = _signal.SIG_DFL else: - self.bus.log("Restoring %s handler %r." % (signame, handler)) + self.bus.log('Restoring %s handler %r.' % (signame, handler)) try: our_handler = _signal.signal(signum, handler) if our_handler is None: - self.bus.log("Restored old %s handler %r, but our " - "handler was not registered." % + self.bus.log('Restored old %s handler %r, but our ' + 'handler was not registered.' % (signame, handler), level=30) except ValueError: - self.bus.log("Unable to restore %s handler %r." % + self.bus.log('Unable to restore %s handler %r.' % (signame, handler), level=40, traceback=True) def set_handler(self, signal, listener=None): @@ -176,39 +176,39 @@ class SignalHandler(object): If the given signal name or number is not available on the current platform, ValueError is raised. """ - if isinstance(signal, basestring): + if isinstance(signal, text_or_bytes): signum = getattr(_signal, signal, None) if signum is None: - raise ValueError("No such signal: %r" % signal) + raise ValueError('No such signal: %r' % signal) signame = signal else: try: signame = self.signals[signal] except KeyError: - raise ValueError("No such signal: %r" % signal) + raise ValueError('No such signal: %r' % signal) signum = signal prev = _signal.signal(signum, self._handle_signal) self._previous_handlers[signum] = prev if listener is not None: - self.bus.log("Listening for %s." % signame) + self.bus.log('Listening for %s.' % signame) self.bus.subscribe(signame, listener) def _handle_signal(self, signum=None, frame=None): """Python signal handler (self.set_handler subscribes it for you).""" signame = self.signals[signum] - self.bus.log("Caught signal %s." % signame) + self.bus.log('Caught signal %s.' % signame) self.bus.publish(signame) def handle_SIGHUP(self): """Restart if daemonized, else exit.""" if self._is_daemonized(): - self.bus.log("SIGHUP caught while daemonized. Restarting.") + self.bus.log('SIGHUP caught while daemonized. Restarting.') self.bus.restart() else: # not daemonized (may be foreground or background) - self.bus.log("SIGHUP caught but not daemonized. Exiting.") + self.bus.log('SIGHUP caught but not daemonized. Exiting.') self.bus.exit() @@ -239,14 +239,14 @@ class DropPrivileges(SimplePlugin): def _set_uid(self, val): if val is not None: if pwd is None: - self.bus.log("pwd module not available; ignoring uid.", + self.bus.log('pwd module not available; ignoring uid.', level=30) val = None - elif isinstance(val, basestring): + elif isinstance(val, text_or_bytes): val = pwd.getpwnam(val)[2] self._uid = val uid = property(_get_uid, _set_uid, - doc="The uid under which to run. Availability: Unix.") + doc='The uid under which to run. Availability: Unix.') def _get_gid(self): return self._gid @@ -254,14 +254,14 @@ class DropPrivileges(SimplePlugin): def _set_gid(self, val): if val is not None: if grp is None: - self.bus.log("grp module not available; ignoring gid.", + self.bus.log('grp module not available; ignoring gid.', level=30) val = None - elif isinstance(val, basestring): + elif isinstance(val, text_or_bytes): val = grp.getgrnam(val)[2] self._gid = val gid = property(_get_gid, _set_gid, - doc="The gid under which to run. Availability: Unix.") + doc='The gid under which to run. Availability: Unix.') def _get_umask(self): return self._umask @@ -271,7 +271,7 @@ class DropPrivileges(SimplePlugin): try: os.umask except AttributeError: - self.bus.log("umask function not available; ignoring umask.", + self.bus.log('umask function not available; ignoring umask.', level=30) val = None self._umask = val @@ -393,7 +393,7 @@ class Daemonizer(SimplePlugin): except OSError: # Python raises OSError rather than returning negative numbers. exc = sys.exc_info()[1] - sys.exit("%s: fork #1 failed: (%d) %s\n" + sys.exit('%s: fork #1 failed: (%d) %s\n' % (sys.argv[0], exc.errno, exc.strerror)) os.setsid() @@ -406,15 +406,15 @@ class Daemonizer(SimplePlugin): os._exit(0) # Exit second parent except OSError: exc = sys.exc_info()[1] - sys.exit("%s: fork #2 failed: (%d) %s\n" + sys.exit('%s: fork #2 failed: (%d) %s\n' % (sys.argv[0], exc.errno, exc.strerror)) - os.chdir("/") + os.chdir('/') os.umask(0) - si = open(self.stdin, "r") - so = open(self.stdout, "a+") - se = open(self.stderr, "a+") + si = open(self.stdin, 'r') + so = open(self.stdout, 'a+') + se = open(self.stderr, 'a+') # os.dup2(fd, fd2) will close fd2 if necessary, # so we don't explicitly close stdin/out/err. @@ -442,7 +442,7 @@ class PIDFile(SimplePlugin): if self.finalized: self.bus.log('PID %r already written to %r.' % (pid, self.pidfile)) else: - open(self.pidfile, "wb").write(ntob("%s\n" % pid, 'utf8')) + open(self.pidfile, 'wb').write(ntob('%s\n' % pid, 'utf8')) self.bus.log('PID %r written to %r.' % (pid, self.pidfile)) self.finalized = True start.priority = 70 @@ -481,13 +481,13 @@ class PerpetualTimer(Timer): except Exception: if self.bus: self.bus.log( - "Error in perpetual timer thread function %r." % + 'Error in perpetual timer thread function %r.' % self.function, level=40, traceback=True) # Quit on first error to avoid massive logs. raise -class BackgroundTask(SetDaemonProperty, threading.Thread): +class BackgroundTask(threading.Thread): """A subclass of threading.Thread whose run() method repeats. @@ -499,7 +499,7 @@ class BackgroundTask(SetDaemonProperty, threading.Thread): """ def __init__(self, interval, function, args=[], kwargs={}, bus=None): - threading.Thread.__init__(self) + super(BackgroundTask, self).__init__() self.interval = interval self.function = function self.args = args @@ -523,7 +523,7 @@ class BackgroundTask(SetDaemonProperty, threading.Thread): self.function(*self.args, **self.kwargs) except Exception: if self.bus: - self.bus.log("Error in background task thread function %r." + self.bus.log('Error in background task thread function %r.' % self.function, level=40, traceback=True) # Quit on first error to avoid massive logs. raise @@ -560,24 +560,24 @@ class Monitor(SimplePlugin): bus=self.bus) self.thread.setName(threadname) self.thread.start() - self.bus.log("Started monitor thread %r." % threadname) + self.bus.log('Started monitor thread %r.' % threadname) else: - self.bus.log("Monitor thread %r already started." % threadname) + self.bus.log('Monitor thread %r already started.' % threadname) start.priority = 70 def stop(self): """Stop our callback's background task thread.""" if self.thread is None: - self.bus.log("No thread running for %s." % + self.bus.log('No thread running for %s.' % self.name or self.__class__.__name__) else: if self.thread is not threading.currentThread(): name = self.thread.getName() self.thread.cancel() - if not get_daemon(self.thread): - self.bus.log("Joining %r" % name) + if not self.thread.daemon: + self.bus.log('Joining %r' % name) self.thread.join() - self.bus.log("Stopped thread %r." % name) + self.bus.log('Stopped thread %r.' % name) self.thread = None def graceful(self): @@ -674,10 +674,10 @@ class Autoreloader(Monitor): else: if mtime is None or mtime > oldtime: # The file has been deleted or modified. - self.bus.log("Restarting because %s changed." % + self.bus.log('Restarting because %s changed.' % filename) self.thread.cancel() - self.bus.log("Stopped thread %r." % + self.bus.log('Stopped thread %r.' % self.thread.getName()) self.bus.restart() return diff --git a/cherrypy/process/servers.py b/cherrypy/process/servers.py index d340796..7d6ec6c 100644 --- a/cherrypy/process/servers.py +++ b/cherrypy/process/servers.py @@ -113,6 +113,7 @@ Please see `Lighttpd FastCGI Docs an explanation of the possible configuration options. """ +import os import sys import time import warnings @@ -151,32 +152,33 @@ class ServerAdapter(object): def start(self): """Start the HTTP server.""" if self.bind_addr is None: - on_what = "unknown interface (dynamic?)" + on_what = 'unknown interface (dynamic?)' elif isinstance(self.bind_addr, tuple): on_what = self._get_base() else: - on_what = "socket file: %s" % self.bind_addr + on_what = 'socket file: %s' % self.bind_addr if self.running: - self.bus.log("Already serving on %s" % on_what) + self.bus.log('Already serving on %s' % on_what) return self.interrupt = None if not self.httpserver: - raise ValueError("No HTTP server has been created.") + raise ValueError('No HTTP server has been created.') - # Start the httpserver in a new thread. - if isinstance(self.bind_addr, tuple): - wait_for_free_port(*self.bind_addr) + if not os.environ.get('LISTEN_PID', None): + # Start the httpserver in a new thread. + if isinstance(self.bind_addr, tuple): + wait_for_free_port(*self.bind_addr) import threading t = threading.Thread(target=self._start_http_thread) - t.setName("HTTPServer " + t.getName()) + t.setName('HTTPServer ' + t.getName()) t.start() self.wait() self.running = True - self.bus.log("Serving on %s" % on_what) + self.bus.log('Serving on %s' % on_what) start.priority = 75 def _get_base(self): @@ -184,15 +186,15 @@ class ServerAdapter(object): return '' host, port = self.bind_addr if getattr(self.httpserver, 'ssl_adapter', None): - scheme = "https" + scheme = 'https' if port != 443: - host += ":%s" % port + host += ':%s' % port else: - scheme = "http" + scheme = 'http' if port != 80: - host += ":%s" % port + host += ':%s' % port - return "%s://%s" % (scheme, host) + return '%s://%s' % (scheme, host) def _start_http_thread(self): """HTTP servers MUST be running in new threads, so that the @@ -204,32 +206,35 @@ class ServerAdapter(object): try: self.httpserver.start() except KeyboardInterrupt: - self.bus.log(" hit: shutting down HTTP server") + self.bus.log(' hit: shutting down HTTP server') self.interrupt = sys.exc_info()[1] self.bus.exit() except SystemExit: - self.bus.log("SystemExit raised: shutting down HTTP server") + self.bus.log('SystemExit raised: shutting down HTTP server') self.interrupt = sys.exc_info()[1] self.bus.exit() raise except: self.interrupt = sys.exc_info()[1] - self.bus.log("Error in HTTP server: shutting down", + self.bus.log('Error in HTTP server: shutting down', traceback=True, level=40) self.bus.exit() raise def wait(self): """Wait until the HTTP server is ready to receive requests.""" - while not getattr(self.httpserver, "ready", False): + while not getattr(self.httpserver, 'ready', False): if self.interrupt: raise self.interrupt time.sleep(.1) # Wait for port to be occupied - if isinstance(self.bind_addr, tuple): - host, port = self.bind_addr - wait_for_occupied_port(host, port) + if not os.environ.get('LISTEN_PID', None): + # Wait for port to be occupied if not running via socket-activation + # (for socket-activation the port will be managed by systemd ) + if isinstance(self.bind_addr, tuple): + host, port = self.bind_addr + wait_for_occupied_port(host, port) def stop(self): """Stop the HTTP server.""" @@ -240,9 +245,9 @@ class ServerAdapter(object): if isinstance(self.bind_addr, tuple): wait_for_free_port(*self.bind_addr) self.running = False - self.bus.log("HTTP Server %s shut down" % self.httpserver) + self.bus.log('HTTP Server %s shut down' % self.httpserver) else: - self.bus.log("HTTP Server %s already shut down" % self.httpserver) + self.bus.log('HTTP Server %s already shut down' % self.httpserver) stop.priority = 25 def restart(self): @@ -389,10 +394,10 @@ def check_port(host, port, timeout=1.0): except socket.gaierror: if ':' in host: info = [( - socket.AF_INET6, socket.SOCK_STREAM, 0, "", (host, port, 0, 0) + socket.AF_INET6, socket.SOCK_STREAM, 0, '', (host, port, 0, 0) )] else: - info = [(socket.AF_INET, socket.SOCK_STREAM, 0, "", (host, port))] + info = [(socket.AF_INET, socket.SOCK_STREAM, 0, '', (host, port))] for res in info: af, socktype, proto, canonname, sa = res @@ -408,8 +413,8 @@ def check_port(host, port, timeout=1.0): if s: s.close() else: - raise IOError("Port %s is in use on %s; perhaps the previous " - "httpserver did not shut down properly." % + raise IOError('Port %s is in use on %s; perhaps the previous ' + 'httpserver did not shut down properly.' % (repr(port), repr(host))) @@ -435,7 +440,7 @@ def wait_for_free_port(host, port, timeout=None): else: return - raise IOError("Port %r not free on %r" % (port, host)) + raise IOError('Port %r not free on %r' % (port, host)) def wait_for_occupied_port(host, port, timeout=None): @@ -455,11 +460,11 @@ def wait_for_occupied_port(host, port, timeout=None): time.sleep(timeout) if host == client_host(host): - raise IOError("Port %r not bound on %r" % (port, host)) + raise IOError('Port %r not bound on %r' % (port, host)) # On systems where a loopback interface is not available and the # server is bound to all interfaces, it's difficult to determine # whether the server is in fact occupying the port. In this case, # just issue a warning and move on. See issue #1100. - msg = "Unable to verify that the server is bound on %r" % port + msg = 'Unable to verify that the server is bound on %r' % port warnings.warn(msg) diff --git a/cherrypy/process/win32.py b/cherrypy/process/win32.py index 4afd3f1..edaf48b 100644 --- a/cherrypy/process/win32.py +++ b/cherrypy/process/win32.py @@ -85,7 +85,7 @@ class Win32Bus(wspbus.Bus): return self.events[state] except KeyError: event = win32event.CreateEvent(None, 0, 0, - "WSPBus %s Event (pid=%r)" % + 'WSPBus %s Event (pid=%r)' % (state.name, os.getpid())) self.events[state] = event return event @@ -135,7 +135,7 @@ class _ControlCodes(dict): for key, val in self.items(): if val is obj: return key - raise ValueError("The given object could not be found: %r" % obj) + raise ValueError('The given object could not be found: %r' % obj) control_codes = _ControlCodes({'graceful': 138}) @@ -153,14 +153,14 @@ class PyWebService(win32serviceutil.ServiceFramework): """Python Web Service.""" - _svc_name_ = "Python Web Service" - _svc_display_name_ = "Python Web Service" + _svc_name_ = 'Python Web Service' + _svc_display_name_ = 'Python Web Service' _svc_deps_ = None # sequence of service names on which this depends - _exe_name_ = "pywebsvc" + _exe_name_ = 'pywebsvc' _exe_args_ = None # Default to no arguments # Only exists on Windows 2000 or later, ignored on windows NT - _svc_description_ = "Python Web Service" + _svc_description_ = 'Python Web Service' def SvcDoRun(self): from cherrypy import process diff --git a/cherrypy/process/wspbus.py b/cherrypy/process/wspbus.py index aa637ab..372c9dc 100644 --- a/cherrypy/process/wspbus.py +++ b/cherrypy/process/wspbus.py @@ -61,13 +61,20 @@ the new state.:: """ import atexit +import ctypes +import operator import os +import subprocess import sys import threading import time import traceback as _traceback import warnings -import operator + +import six + +from cherrypy._cpcompat import _args_from_interpreter_flags + # Here I save the value of os.getcwd(), which, if I am imported early enough, # will be the directory from which the startup script was run. This is needed @@ -85,9 +92,7 @@ class ChannelFailures(Exception): delimiter = '\n' def __init__(self, *args, **kwargs): - # Don't use 'super' here; Exceptions are old-style in Py2.4 - # See https://github.com/cherrypy/cherrypy/issues/959 - Exception.__init__(self, *args, **kwargs) + super(Exception, self).__init__(*args, **kwargs) self._exceptions = list() def handle_exception(self): @@ -117,7 +122,7 @@ class _StateEnum(object): name = None def __repr__(self): - return "states.%s" % self.name + return 'states.%s' % self.name def __setattr__(self, key, value): if isinstance(value, self.State): @@ -170,9 +175,8 @@ class Bus(object): def subscribe(self, channel, callback, priority=None): """Add the given callback at the given channel (if not present).""" - if channel not in self.listeners: - self.listeners[channel] = set() - self.listeners[channel].add(callback) + ch_listeners = self.listeners.setdefault(channel, set()) + ch_listeners.add(callback) if priority is None: priority = getattr(callback, 'priority', 50) @@ -215,7 +219,7 @@ class Bus(object): # Assume any further messages to 'log' will fail. pass else: - self.log("Error in %r listener %r" % (channel, listener), + self.log('Error in %r listener %r' % (channel, listener), level=40, traceback=True) if exc: raise exc @@ -225,10 +229,10 @@ class Bus(object): """An atexit handler which asserts the Bus is not running.""" if self.state != states.EXITING: warnings.warn( - "The main thread is exiting, but the Bus is in the %r state; " - "shutting it down automatically now. You must either call " - "bus.block() after start(), or call bus.exit() before the " - "main thread exits." % self.state, RuntimeWarning) + 'The main thread is exiting, but the Bus is in the %r state; ' + 'shutting it down automatically now. You must either call ' + 'bus.block() after start(), or call bus.exit() before the ' + 'main thread exits.' % self.state, RuntimeWarning) self.exit() def start(self): @@ -244,7 +248,7 @@ class Bus(object): except (KeyboardInterrupt, SystemExit): raise except: - self.log("Shutting down due to error in start listener:", + self.log('Shutting down due to error in start listener:', level=40, traceback=True) e_info = sys.exc_info()[1] try: @@ -321,7 +325,7 @@ class Bus(object): # It's also good to let them all shut down before allowing # the main thread to call atexit handlers. # See https://github.com/cherrypy/cherrypy/issues/751. - self.log("Waiting for child threads to terminate...") + self.log('Waiting for child threads to terminate...') for t in threading.enumerate(): # Validate the we're not trying to join the MainThread # that will cause a deadlock and the case exist when @@ -333,13 +337,13 @@ class Bus(object): not isinstance(t, threading._MainThread) ): # Note that any dummy (external) threads are always daemonic. - if hasattr(threading.Thread, "daemon"): + if hasattr(threading.Thread, 'daemon'): # Python 2.6+ d = t.daemon else: d = t.isDaemon() if not d: - self.log("Waiting for thread %s." % t.getName()) + self.log('Waiting for thread %s.' % t.getName()) t.join() if self.execv: @@ -376,14 +380,20 @@ class Bus(object): This must be called from the main thread, because certain platforms (OS X) don't allow execv to be called in a child thread very well. """ - args = sys.argv[:] + try: + args = self._get_true_argv() + except NotImplementedError: + """It's probably win32""" + args = [sys.executable] + _args_from_interpreter_flags() + sys.argv + self.log('Re-spawning %s' % ' '.join(args)) + self._extend_pythonpath(os.environ) + if sys.platform[:4] == 'java': from _systemrestart import SystemRestart raise SystemRestart else: - args.insert(0, sys.executable) if sys.platform == 'win32': args = ['"%s"' % arg for arg in args] @@ -392,6 +402,58 @@ class Bus(object): self._set_cloexec() os.execv(sys.executable, args) + @staticmethod + def _get_true_argv(): + """Retrieves all real arguments of the python interpreter + + ...even those not listed in ``sys.argv`` + + :seealso: http://stackoverflow.com/a/28338254/595220 + :seealso: http://stackoverflow.com/a/6683222/595220 + :seealso: http://stackoverflow.com/a/28414807/595220 + """ + + try: + char_p = ctypes.c_char_p if six.PY2 else ctypes.c_wchar_p + + argv = ctypes.POINTER(char_p)() + argc = ctypes.c_int() + + ctypes.pythonapi.Py_GetArgcArgv(ctypes.byref(argc), ctypes.byref(argv)) + except AttributeError: + """It looks Py_GetArgcArgv is completely absent in MS Windows + + :seealso: https://github.com/cherrypy/cherrypy/issues/1506 + :ref: https://chromium.googlesource.com/infra/infra/+/69eb0279c12bcede5937ce9298020dd4581e38dd%5E!/ + """ + raise NotImplementedError + else: + return argv[:argc.value] + + @staticmethod + def _extend_pythonpath(env): + """ + If sys.path[0] is an empty string, the interpreter was likely + invoked with -m and the effective path is about to change on + re-exec. Add the current directory to $PYTHONPATH to ensure + that the new process sees the same path. + + This issue cannot be addressed in the general case because + Python cannot reliably reconstruct the + original command line (http://bugs.python.org/issue14208). + + (This idea filched from tornado.autoreload) + """ + path_prefix = '.' + os.pathsep + existing_path = env.get('PYTHONPATH', '') + needs_patch = ( + sys.path[0] == '' and + not existing_path.startswith(path_prefix) + ) + + if needs_patch: + env['PYTHONPATH'] = path_prefix + existing_path + def _set_cloexec(self): """Set the CLOEXEC flag on all open files (except stdin/out/err). @@ -437,16 +499,10 @@ class Bus(object): return t - def log(self, msg="", level=20, traceback=False): + def log(self, msg='', level=20, traceback=False): """Log the given message. Append the last traceback if requested.""" if traceback: - # Work-around for bug in Python's traceback implementation - # which crashes when the error message contains %1, %2 etc. - errors = sys.exc_info() - if '%' in errors[1].message: - errors[1].message = errors[1].message.replace('%', '#') - errors[1].args = [item.replace('%', '#') for item in errors[1].args] - msg += "\n" + "".join(_traceback.format_exception(*errors)) + msg += '\n' + ''.join(_traceback.format_exception(*sys.exc_info())) self.publish('log', msg, level) bus = Bus() diff --git a/cherrypy/wsgiserver/__init__.py b/cherrypy/wsgiserver/__init__.py index ee6190f..3b347cc 100644 --- a/cherrypy/wsgiserver/__init__.py +++ b/cherrypy/wsgiserver/__init__.py @@ -1,14 +1,2573 @@ +"""A high-speed, production ready, thread pooled, generic HTTP server. + +Simplest example on how to use this module directly +(without using CherryPy's application machinery):: + + from cherrypy import wsgiserver + + def my_crazy_app(environ, start_response): + status = '200 OK' + response_headers = [('Content-type','text/plain')] + start_response(status, response_headers) + return ['Hello world!'] + + server = wsgiserver.CherryPyWSGIServer( + ('0.0.0.0', 8070), my_crazy_app, + server_name='www.cherrypy.example') + server.start() + +The CherryPy WSGI server can serve as many WSGI applications +as you want in one instance by using a WSGIPathInfoDispatcher:: + + d = WSGIPathInfoDispatcher({'/': my_crazy_app, '/blog': my_blog_app}) + server = wsgiserver.CherryPyWSGIServer(('0.0.0.0', 80), d) + +Want SSL support? Just set server.ssl_adapter to an SSLAdapter instance. + +This won't call the CherryPy engine (application side) at all, only the +HTTP server, which is independent from the rest of CherryPy. Don't +let the name "CherryPyWSGIServer" throw you; the name merely reflects +its origin, not its coupling. + +For those of you wanting to understand internals of this module, here's the +basic call flow. The server's listening thread runs a very tight loop, +sticking incoming connections onto a Queue:: + + server = CherryPyWSGIServer(...) + server.start() + while True: + tick() + # This blocks until a request comes in: + child = socket.accept() + conn = HTTPConnection(child, ...) + server.requests.put(conn) + +Worker threads are kept in a pool and poll the Queue, popping off and then +handling each connection in turn. Each connection can consist of an arbitrary +number of requests and their responses, so we run a nested loop:: + + while True: + conn = server.requests.get() + conn.communicate() + -> while True: + req = HTTPRequest(...) + req.parse_request() + -> # Read the Request-Line, e.g. "GET /page HTTP/1.1" + req.rfile.readline() + read_headers(req.rfile, req.inheaders) + req.respond() + -> response = app(...) + try: + for chunk in response: + if chunk: + req.write(chunk) + finally: + if hasattr(response, "close"): + response.close() + if req.close_connection: + return +""" + +import os +try: + import queue +except: + import Queue as queue +import re +import email.utils +import socket +import sys +import threading +import time +import traceback as traceback_ +try: + from urllib.parse import unquote_to_bytes, urlparse +except ImportError: + from urlparse import unquote as unquote_to_bytes + from urlparse import urlparse +import errno +import logging + +import six +from six.moves import filter + +try: + # prefer slower Python-based io module + import _pyio as io +except ImportError: + # Python 2.6 + import io + +try: + import pkg_resources +except ImportError: + pass + + __all__ = ['HTTPRequest', 'HTTPConnection', 'HTTPServer', 'SizeCheckWrapper', 'KnownLengthRFile', 'ChunkedRFile', + 'CP_makefile', 'MaxSizeExceeded', 'NoSSLError', 'FatalSSLAlert', 'WorkerThread', 'ThreadPool', 'SSLAdapter', 'CherryPyWSGIServer', 'Gateway', 'WSGIGateway', 'WSGIGateway_10', 'WSGIGateway_u0', - 'WSGIPathInfoDispatcher', 'get_ssl_adapter_class'] + 'WSGIPathInfoDispatcher', 'get_ssl_adapter_class', + 'socket_errors_to_ignore'] -import sys -if sys.version_info < (3, 0): - from wsgiserver2 import * + +if 'win' in sys.platform and hasattr(socket, 'AF_INET6'): + if not hasattr(socket, 'IPPROTO_IPV6'): + socket.IPPROTO_IPV6 = 41 + if not hasattr(socket, 'IPV6_V6ONLY'): + socket.IPV6_V6ONLY = 27 + + +DEFAULT_BUFFER_SIZE = io.DEFAULT_BUFFER_SIZE + + +try: + cp_version = pkg_resources.require('cherrypy')[0].version +except Exception: + cp_version = 'unknown' + + +if six.PY3: + def ntob(n, encoding='ISO-8859-1'): + """Return the given native string as a byte string in the given + encoding. + """ + # In Python 3, the native string type is unicode + return n.encode(encoding) + + def bton(b, encoding='ISO-8859-1'): + return b.decode(encoding) else: - # Le sigh. Boo for backward-incompatible syntax. - exec('from .wsgiserver3 import *') + def ntob(n, encoding='ISO-8859-1'): + """Return the given native string as a byte string in the given + encoding. + """ + # In Python 2, the native string type is bytes. Assume it's already + # in the given encoding, which for ISO-8859-1 is almost always what + # was intended. + return n + + def bton(b, encoding='ISO-8859-1'): + return b + + +LF = ntob('\n') +CRLF = ntob('\r\n') +TAB = ntob('\t') +SPACE = ntob(' ') +COLON = ntob(':') +SEMICOLON = ntob(';') +EMPTY = ntob('') +NUMBER_SIGN = ntob('#') +QUESTION_MARK = ntob('?') +ASTERISK = ntob('*') +FORWARD_SLASH = ntob('/') +quoted_slash = re.compile(ntob('(?i)%2F')) + + +def plat_specific_errors(*errnames): + """Return error numbers for all errors in errnames on this platform. + + The 'errno' module contains different global constants depending on + the specific platform (OS). This function will return the list of + numeric values for a given list of potential names. + """ + errno_names = dir(errno) + nums = [getattr(errno, k) for k in errnames if k in errno_names] + # de-dupe the list + return list(dict.fromkeys(nums).keys()) + +socket_error_eintr = plat_specific_errors('EINTR', 'WSAEINTR') + +socket_errors_to_ignore = plat_specific_errors( + 'EPIPE', + 'EBADF', 'WSAEBADF', + 'ENOTSOCK', 'WSAENOTSOCK', + 'ETIMEDOUT', 'WSAETIMEDOUT', + 'ECONNREFUSED', 'WSAECONNREFUSED', + 'ECONNRESET', 'WSAECONNRESET', + 'ECONNABORTED', 'WSAECONNABORTED', + 'ENETRESET', 'WSAENETRESET', + 'EHOSTDOWN', 'EHOSTUNREACH', +) +socket_errors_to_ignore.append('timed out') +socket_errors_to_ignore.append('The read operation timed out') +if sys.platform == 'darwin': + socket_errors_to_ignore.append(plat_specific_errors('EPROTOTYPE')) + +socket_errors_nonblocking = plat_specific_errors( + 'EAGAIN', 'EWOULDBLOCK', 'WSAEWOULDBLOCK') + +comma_separated_headers = [ + ntob(h) for h in + ['Accept', 'Accept-Charset', 'Accept-Encoding', + 'Accept-Language', 'Accept-Ranges', 'Allow', 'Cache-Control', + 'Connection', 'Content-Encoding', 'Content-Language', 'Expect', + 'If-Match', 'If-None-Match', 'Pragma', 'Proxy-Authenticate', 'TE', + 'Trailer', 'Transfer-Encoding', 'Upgrade', 'Vary', 'Via', 'Warning', + 'WWW-Authenticate'] +] + + +if not hasattr(logging, 'statistics'): + logging.statistics = {} + + +def read_headers(rfile, hdict=None): + """Read headers from the given stream into the given header dict. + + If hdict is None, a new header dict is created. Returns the populated + header dict. + + Headers which are repeated are folded together using a comma if their + specification so dictates. + + This function raises ValueError when the read bytes violate the HTTP spec. + You should probably return "400 Bad Request" if this happens. + """ + if hdict is None: + hdict = {} + + while True: + line = rfile.readline() + if not line: + # No more data--illegal end of headers + raise ValueError('Illegal end of headers.') + + if line == CRLF: + # Normal end of headers + break + if not line.endswith(CRLF): + raise ValueError('HTTP requires CRLF terminators') + + if line[0] in (SPACE, TAB): + # It's a continuation line. + v = line.strip() + else: + try: + k, v = line.split(COLON, 1) + except ValueError: + raise ValueError('Illegal header line.') + # TODO: what about TE and WWW-Authenticate? + k = k.strip().title() + v = v.strip() + hname = k + + if k in comma_separated_headers: + existing = hdict.get(hname) + if existing: + v = b', '.join((existing, v)) + hdict[hname] = v + + return hdict + + +class MaxSizeExceeded(Exception): + pass + + +class SizeCheckWrapper(object): + + """Wraps a file-like object, raising MaxSizeExceeded if too large.""" + + def __init__(self, rfile, maxlen): + self.rfile = rfile + self.maxlen = maxlen + self.bytes_read = 0 + + def _check_length(self): + if self.maxlen and self.bytes_read > self.maxlen: + raise MaxSizeExceeded() + + def read(self, size=None): + data = self.rfile.read(size) + self.bytes_read += len(data) + self._check_length() + return data + + def readline(self, size=None): + if size is not None: + data = self.rfile.readline(size) + self.bytes_read += len(data) + self._check_length() + return data + + # User didn't specify a size ... + # We read the line in chunks to make sure it's not a 100MB line ! + res = [] + while True: + data = self.rfile.readline(256) + self.bytes_read += len(data) + self._check_length() + res.append(data) + # See https://github.com/cherrypy/cherrypy/issues/421 + if len(data) < 256 or data[-1:] == LF: + return EMPTY.join(res) + + def readlines(self, sizehint=0): + # Shamelessly stolen from StringIO + total = 0 + lines = [] + line = self.readline() + while line: + lines.append(line) + total += len(line) + if 0 < sizehint <= total: + break + line = self.readline() + return lines + + def close(self): + self.rfile.close() + + def __iter__(self): + return self + + def __next__(self): + data = next(self.rfile) + self.bytes_read += len(data) + self._check_length() + return data + + def next(self): + data = self.rfile.next() + self.bytes_read += len(data) + self._check_length() + return data + + +class KnownLengthRFile(object): + + """Wraps a file-like object, returning an empty string when exhausted.""" + + def __init__(self, rfile, content_length): + self.rfile = rfile + self.remaining = content_length + + def read(self, size=None): + if self.remaining == 0: + return b'' + if size is None: + size = self.remaining + else: + size = min(size, self.remaining) + + data = self.rfile.read(size) + self.remaining -= len(data) + return data + + def readline(self, size=None): + if self.remaining == 0: + return b'' + if size is None: + size = self.remaining + else: + size = min(size, self.remaining) + + data = self.rfile.readline(size) + self.remaining -= len(data) + return data + + def readlines(self, sizehint=0): + # Shamelessly stolen from StringIO + total = 0 + lines = [] + line = self.readline(sizehint) + while line: + lines.append(line) + total += len(line) + if 0 < sizehint <= total: + break + line = self.readline(sizehint) + return lines + + def close(self): + self.rfile.close() + + def __iter__(self): + return self + + def __next__(self): + data = next(self.rfile) + self.remaining -= len(data) + return data + + +class ChunkedRFile(object): + + """Wraps a file-like object, returning an empty string when exhausted. + + This class is intended to provide a conforming wsgi.input value for + request entities that have been encoded with the 'chunked' transfer + encoding. + """ + + def __init__(self, rfile, maxlen, bufsize=8192): + self.rfile = rfile + self.maxlen = maxlen + self.bytes_read = 0 + self.buffer = EMPTY + self.bufsize = bufsize + self.closed = False + + def _fetch(self): + if self.closed: + return + + line = self.rfile.readline() + self.bytes_read += len(line) + + if self.maxlen and self.bytes_read > self.maxlen: + raise MaxSizeExceeded('Request Entity Too Large', self.maxlen) + + line = line.strip().split(SEMICOLON, 1) + + try: + chunk_size = line.pop(0) + chunk_size = int(chunk_size, 16) + except ValueError: + raise ValueError('Bad chunked transfer size: ' + repr(chunk_size)) + + if chunk_size <= 0: + self.closed = True + return + +## if line: chunk_extension = line[0] + + if self.maxlen and self.bytes_read + chunk_size > self.maxlen: + raise IOError('Request Entity Too Large') + + chunk = self.rfile.read(chunk_size) + self.bytes_read += len(chunk) + self.buffer += chunk + + crlf = self.rfile.read(2) + if crlf != CRLF: + raise ValueError( + "Bad chunked transfer coding (expected '\\r\\n', " + 'got ' + repr(crlf) + ')') + + def read(self, size=None): + data = EMPTY + while True: + if size and len(data) >= size: + return data + + if not self.buffer: + self._fetch() + if not self.buffer: + # EOF + return data + + if size: + remaining = size - len(data) + data += self.buffer[:remaining] + self.buffer = self.buffer[remaining:] + else: + data += self.buffer + + def readline(self, size=None): + data = EMPTY + while True: + if size and len(data) >= size: + return data + + if not self.buffer: + self._fetch() + if not self.buffer: + # EOF + return data + + newline_pos = self.buffer.find(LF) + if size: + if newline_pos == -1: + remaining = size - len(data) + data += self.buffer[:remaining] + self.buffer = self.buffer[remaining:] + else: + remaining = min(size - len(data), newline_pos) + data += self.buffer[:remaining] + self.buffer = self.buffer[remaining:] + else: + if newline_pos == -1: + data += self.buffer + else: + data += self.buffer[:newline_pos] + self.buffer = self.buffer[newline_pos:] + + def readlines(self, sizehint=0): + # Shamelessly stolen from StringIO + total = 0 + lines = [] + line = self.readline(sizehint) + while line: + lines.append(line) + total += len(line) + if 0 < sizehint <= total: + break + line = self.readline(sizehint) + return lines + + def read_trailer_lines(self): + if not self.closed: + raise ValueError( + 'Cannot read trailers until the request body has been read.') + + while True: + line = self.rfile.readline() + if not line: + # No more data--illegal end of headers + raise ValueError('Illegal end of headers.') + + self.bytes_read += len(line) + if self.maxlen and self.bytes_read > self.maxlen: + raise IOError('Request Entity Too Large') + + if line == CRLF: + # Normal end of headers + break + if not line.endswith(CRLF): + raise ValueError('HTTP requires CRLF terminators') + + yield line + + def close(self): + self.rfile.close() + + +class HTTPRequest(object): + + """An HTTP Request (and response). + + A single HTTP connection may consist of multiple request/response pairs. + """ + + server = None + """The HTTPServer object which is receiving this request.""" + + conn = None + """The HTTPConnection object on which this request connected.""" + + inheaders = {} + """A dict of request headers.""" + + outheaders = [] + """A list of header tuples to write in the response.""" + + ready = False + """When True, the request has been parsed and is ready to begin generating + the response. When False, signals the calling Connection that the response + should not be generated and the connection should close.""" + + close_connection = False + """Signals the calling Connection that the request should close. This does + not imply an error! The client and/or server may each request that the + connection be closed.""" + + chunked_write = False + """If True, output will be encoded with the "chunked" transfer-coding. + + This value is set automatically inside send_headers.""" + + def __init__(self, server, conn): + self.server = server + self.conn = conn + + self.ready = False + self.started_request = False + self.scheme = ntob('http') + if self.server.ssl_adapter is not None: + self.scheme = ntob('https') + # Use the lowest-common protocol in case read_request_line errors. + self.response_protocol = 'HTTP/1.0' + self.inheaders = {} + + self.status = '' + self.outheaders = [] + self.sent_headers = False + self.close_connection = self.__class__.close_connection + self.chunked_read = False + self.chunked_write = self.__class__.chunked_write + + def parse_request(self): + """Parse the next HTTP request start-line and message-headers.""" + self.rfile = SizeCheckWrapper(self.conn.rfile, + self.server.max_request_header_size) + try: + success = self.read_request_line() + except MaxSizeExceeded: + self.simple_response( + '414 Request-URI Too Long', + 'The Request-URI sent with the request exceeds the maximum ' + 'allowed bytes.') + return + else: + if not success: + return + + try: + success = self.read_request_headers() + except MaxSizeExceeded: + self.simple_response( + '413 Request Entity Too Large', + 'The headers sent with the request exceed the maximum ' + 'allowed bytes.') + return + else: + if not success: + return + + self.ready = True + + def read_request_line(self): + # HTTP/1.1 connections are persistent by default. If a client + # requests a page, then idles (leaves the connection open), + # then rfile.readline() will raise socket.error("timed out"). + # Note that it does this based on the value given to settimeout(), + # and doesn't need the client to request or acknowledge the close + # (although your TCP stack might suffer for it: cf Apache's history + # with FIN_WAIT_2). + request_line = self.rfile.readline() + + # Set started_request to True so communicate() knows to send 408 + # from here on out. + self.started_request = True + if not request_line: + return False + + if request_line == CRLF: + # RFC 2616 sec 4.1: "...if the server is reading the protocol + # stream at the beginning of a message and receives a CRLF + # first, it should ignore the CRLF." + # But only ignore one leading line! else we enable a DoS. + request_line = self.rfile.readline() + if not request_line: + return False + + if not request_line.endswith(CRLF): + self.simple_response( + '400 Bad Request', 'HTTP requires CRLF terminators') + return False + + try: + method, uri, req_protocol = request_line.strip().split(SPACE, 2) + req_protocol_str = req_protocol.decode('ascii') + rp = int(req_protocol_str[5]), int(req_protocol_str[7]) + except (ValueError, IndexError): + self.simple_response('400 Bad Request', 'Malformed Request-Line') + return False + + self.uri = uri + self.method = method + + # uri may be an abs_path (including "http://host.domain.tld"); + scheme, authority, path = self.parse_request_uri(uri) + if path is None: + self.simple_response('400 Bad Request', + 'Invalid path in Request-URI.') + return False + if NUMBER_SIGN in path: + self.simple_response('400 Bad Request', + 'Illegal #fragment in Request-URI.') + return False + + if scheme: + self.scheme = scheme + + qs = EMPTY + if QUESTION_MARK in path: + path, qs = path.split(QUESTION_MARK, 1) + + # Unquote the path+params (e.g. "/this%20path" -> "/this path"). + # http://www.w3.org/Protocols/rfc2616/rfc2616-sec5.html#sec5.1.2 + # + # But note that "...a URI must be separated into its components + # before the escaped characters within those components can be + # safely decoded." http://www.ietf.org/rfc/rfc2396.txt, sec 2.4.2 + # Therefore, "/this%2Fpath" becomes "/this%2Fpath", not "/this/path". + try: + atoms = [unquote_to_bytes(x) for x in quoted_slash.split(path)] + except ValueError: + ex = sys.exc_info()[1] + self.simple_response('400 Bad Request', ex.args[0]) + return False + path = b'%2F'.join(atoms) + self.path = path + + # Note that, like wsgiref and most other HTTP servers, + # we "% HEX HEX"-unquote the path but not the query string. + self.qs = qs + + # Compare request and server HTTP protocol versions, in case our + # server does not support the requested protocol. Limit our output + # to min(req, server). We want the following output: + # request server actual written supported response + # protocol protocol response protocol feature set + # a 1.0 1.0 1.0 1.0 + # b 1.0 1.1 1.1 1.0 + # c 1.1 1.0 1.0 1.0 + # d 1.1 1.1 1.1 1.1 + # Notice that, in (b), the response will be "HTTP/1.1" even though + # the client only understands 1.0. RFC 2616 10.5.6 says we should + # only return 505 if the _major_ version is different. + sp = int(self.server.protocol[5]), int(self.server.protocol[7]) + + if sp[0] != rp[0]: + self.simple_response('505 HTTP Version Not Supported') + return False + + self.request_protocol = req_protocol + self.response_protocol = 'HTTP/%s.%s' % min(rp, sp) + + return True + + def read_request_headers(self): + """Read self.rfile into self.inheaders. Return success.""" + + # then all the http headers + try: + read_headers(self.rfile, self.inheaders) + except ValueError: + ex = sys.exc_info()[1] + self.simple_response('400 Bad Request', ex.args[0]) + return False + + mrbs = self.server.max_request_body_size + if mrbs and int(self.inheaders.get(b'Content-Length', 0)) > mrbs: + self.simple_response( + '413 Request Entity Too Large', + 'The entity sent with the request exceeds the maximum ' + 'allowed bytes.') + return False + + # Persistent connection support + if self.response_protocol == 'HTTP/1.1': + # Both server and client are HTTP/1.1 + if self.inheaders.get(b'Connection', b'') == b'close': + self.close_connection = True + else: + # Either the server or client (or both) are HTTP/1.0 + if self.inheaders.get(b'Connection', b'') != b'Keep-Alive': + self.close_connection = True + + # Transfer-Encoding support + te = None + if self.response_protocol == 'HTTP/1.1': + te = self.inheaders.get(b'Transfer-Encoding') + if te: + te = [x.strip().lower() for x in te.split(b',') if x.strip()] + + self.chunked_read = False + + if te: + for enc in te: + if enc == b'chunked': + self.chunked_read = True + else: + # Note that, even if we see "chunked", we must reject + # if there is an extension we don't recognize. + self.simple_response('501 Unimplemented') + self.close_connection = True + return False + + # From PEP 333: + # "Servers and gateways that implement HTTP 1.1 must provide + # transparent support for HTTP 1.1's "expect/continue" mechanism. + # This may be done in any of several ways: + # 1. Respond to requests containing an Expect: 100-continue request + # with an immediate "100 Continue" response, and proceed normally. + # 2. Proceed with the request normally, but provide the application + # with a wsgi.input stream that will send the "100 Continue" + # response if/when the application first attempts to read from + # the input stream. The read request must then remain blocked + # until the client responds. + # 3. Wait until the client decides that the server does not support + # expect/continue, and sends the request body on its own. + # (This is suboptimal, and is not recommended.) + # + # We used to do 3, but are now doing 1. Maybe we'll do 2 someday, + # but it seems like it would be a big slowdown for such a rare case. + if self.inheaders.get(b'Expect', b'') == b'100-continue': + # Don't use simple_response here, because it emits headers + # we don't want. See + # https://github.com/cherrypy/cherrypy/issues/951 + msg = self.server.protocol.encode('ascii') + msg += b' 100 Continue\r\n\r\n' + try: + self.conn.wfile.write(msg) + except socket.error: + x = sys.exc_info()[1] + if x.args[0] not in socket_errors_to_ignore: + raise + return True + + def parse_request_uri(self, uri): + """Parse a Request-URI into (scheme, authority, path). + + Note that Request-URI's must be one of:: + + Request-URI = "*" | absoluteURI | abs_path | authority + + Therefore, a Request-URI which starts with a double forward-slash + cannot be a "net_path":: + + net_path = "//" authority [ abs_path ] + + Instead, it must be interpreted as an "abs_path" with an empty first + path segment:: + + abs_path = "/" path_segments + path_segments = segment *( "/" segment ) + segment = *pchar *( ";" param ) + param = *pchar + """ + if uri == ASTERISK: + return None, None, uri + + scheme, authority, path, params, query, fragment = urlparse(uri) + if scheme and QUESTION_MARK not in scheme: + # An absoluteURI. + # If there's a scheme (and it must be http or https), then: + # http_URL = "http:" "//" host [ ":" port ] [ abs_path [ "?" query + # ]] + return scheme, authority, path + + if uri.startswith(FORWARD_SLASH): + # An abs_path. + return None, None, uri + else: + # An authority. + return None, uri, None + + def respond(self): + """Call the gateway and write its iterable output.""" + mrbs = self.server.max_request_body_size + if self.chunked_read: + self.rfile = ChunkedRFile(self.conn.rfile, mrbs) + else: + cl = int(self.inheaders.get(b'Content-Length', 0)) + if mrbs and mrbs < cl: + if not self.sent_headers: + self.simple_response( + '413 Request Entity Too Large', + 'The entity sent with the request exceeds the ' + 'maximum allowed bytes.') + return + self.rfile = KnownLengthRFile(self.conn.rfile, cl) + + self.server.gateway(self).respond() + + if (self.ready and not self.sent_headers): + self.sent_headers = True + self.send_headers() + if self.chunked_write: + self.conn.wfile.write(b'0\r\n\r\n') + + def simple_response(self, status, msg=''): + """Write a simple response back to the client.""" + status = str(status) + proto_status = '%s %s\r\n' % (self.server.protocol, status) + content_length = 'Content-Length: %s\r\n' % len(msg) + content_type = 'Content-Type: text/plain\r\n' + buf = [ + proto_status.encode('ISO-8859-1'), + content_length.encode('ISO-8859-1'), + content_type.encode('ISO-8859-1'), + ] + + if status[:3] in ('413', '414'): + # Request Entity Too Large / Request-URI Too Long + self.close_connection = True + if self.response_protocol == 'HTTP/1.1': + # This will not be true for 414, since read_request_line + # usually raises 414 before reading the whole line, and we + # therefore cannot know the proper response_protocol. + buf.append(b'Connection: close\r\n') + else: + # HTTP/1.0 had no 413/414 status nor Connection header. + # Emit 400 instead and trust the message body is enough. + status = '400 Bad Request' + + buf.append(CRLF) + if msg: + if isinstance(msg, six.text_type): + msg = msg.encode('ISO-8859-1') + buf.append(msg) + + try: + self.conn.wfile.write(EMPTY.join(buf)) + except socket.error: + x = sys.exc_info()[1] + if x.args[0] not in socket_errors_to_ignore: + raise + + def write(self, chunk): + """Write unbuffered data to the client.""" + if self.chunked_write and chunk: + chunk_size_hex = hex(len(chunk))[2:].encode('ascii') + buf = [chunk_size_hex, CRLF, chunk, CRLF] + self.conn.wfile.write(EMPTY.join(buf)) + else: + self.conn.wfile.write(chunk) + + def send_headers(self): + """Assert, process, and send the HTTP response message-headers. + + You must set self.status, and self.outheaders before calling this. + """ + hkeys = [key.lower() for key, value in self.outheaders] + status = int(self.status[:3]) + + if status == 413: + # Request Entity Too Large. Close conn to avoid garbage. + self.close_connection = True + elif b'content-length' not in hkeys: + # "All 1xx (informational), 204 (no content), + # and 304 (not modified) responses MUST NOT + # include a message-body." So no point chunking. + if status < 200 or status in (204, 205, 304): + pass + else: + if (self.response_protocol == 'HTTP/1.1' + and self.method != b'HEAD'): + # Use the chunked transfer-coding + self.chunked_write = True + self.outheaders.append((b'Transfer-Encoding', b'chunked')) + else: + # Closing the conn is the only way to determine len. + self.close_connection = True + + if b'connection' not in hkeys: + if self.response_protocol == 'HTTP/1.1': + # Both server and client are HTTP/1.1 or better + if self.close_connection: + self.outheaders.append((b'Connection', b'close')) + else: + # Server and/or client are HTTP/1.0 + if not self.close_connection: + self.outheaders.append((b'Connection', b'Keep-Alive')) + + if (not self.close_connection) and (not self.chunked_read): + # Read any remaining request body data on the socket. + # "If an origin server receives a request that does not include an + # Expect request-header field with the "100-continue" expectation, + # the request includes a request body, and the server responds + # with a final status code before reading the entire request body + # from the transport connection, then the server SHOULD NOT close + # the transport connection until it has read the entire request, + # or until the client closes the connection. Otherwise, the client + # might not reliably receive the response message. However, this + # requirement is not be construed as preventing a server from + # defending itself against denial-of-service attacks, or from + # badly broken client implementations." + remaining = getattr(self.rfile, 'remaining', 0) + if remaining > 0: + self.rfile.read(remaining) + + if b'date' not in hkeys: + self.outheaders.append(( + b'Date', + email.utils.formatdate(usegmt=True).encode('ISO-8859-1'), + )) + + if b'server' not in hkeys: + self.outheaders.append(( + b'Server', + self.server.server_name.encode('ISO-8859-1'), + )) + + proto = self.server.protocol.encode('ascii') + buf = [proto + SPACE + self.status + CRLF] + for k, v in self.outheaders: + buf.append(k + COLON + SPACE + v + CRLF) + buf.append(CRLF) + self.conn.wfile.write(EMPTY.join(buf)) + + +class NoSSLError(Exception): + + """Exception raised when a client speaks HTTP to an HTTPS socket.""" + pass + + +class FatalSSLAlert(Exception): + + """Exception raised when the SSL implementation signals a fatal alert.""" + pass + + +class CP_BufferedWriter(io.BufferedWriter): + + """Faux file object attached to a socket object.""" + + def write(self, b): + self._checkClosed() + if isinstance(b, str): + raise TypeError("can't write str to binary stream") + + with self._write_lock: + self._write_buf.extend(b) + self._flush_unlocked() + return len(b) + + def _flush_unlocked(self): + self._checkClosed('flush of closed file') + while self._write_buf: + try: + # ssl sockets only except 'bytes', not bytearrays + # so perhaps we should conditionally wrap this for perf? + n = self.raw.write(bytes(self._write_buf)) + except io.BlockingIOError as e: + n = e.characters_written + del self._write_buf[:n] + + +def CP_makefile_PY3(sock, mode='r', bufsize=DEFAULT_BUFFER_SIZE): + if 'r' in mode: + return io.BufferedReader(socket.SocketIO(sock, mode), bufsize) + else: + return CP_BufferedWriter(socket.SocketIO(sock, mode), bufsize) + + +class CP_makefile_PY2(getattr(socket, '_fileobject', object)): + + """Faux file object attached to a socket object.""" + + def __init__(self, *args, **kwargs): + self.bytes_read = 0 + self.bytes_written = 0 + socket._fileobject.__init__(self, *args, **kwargs) + + def write(self, data): + """Sendall for non-blocking sockets.""" + while data: + try: + bytes_sent = self.send(data) + data = data[bytes_sent:] + except socket.error as e: + if e.args[0] not in socket_errors_nonblocking: + raise + + def send(self, data): + bytes_sent = self._sock.send(data) + self.bytes_written += bytes_sent + return bytes_sent + + def flush(self): + if self._wbuf: + buffer = ''.join(self._wbuf) + self._wbuf = [] + self.write(buffer) + + def recv(self, size): + while True: + try: + data = self._sock.recv(size) + self.bytes_read += len(data) + return data + except socket.error as e: + if (e.args[0] not in socket_errors_nonblocking + and e.args[0] not in socket_error_eintr): + raise + + class FauxSocket(object): + + """Faux socket with the minimal interface required by pypy""" + + def _reuse(self): + pass + + _fileobject_uses_str_type = six.PY2 and isinstance( + socket._fileobject(FauxSocket())._rbuf, six.string_types) + + # FauxSocket is no longer needed + del FauxSocket + + if not _fileobject_uses_str_type: + def read(self, size=-1): + # Use max, disallow tiny reads in a loop as they are very + # inefficient. + # We never leave read() with any leftover data from a new recv() + # call in our internal buffer. + rbufsize = max(self._rbufsize, self.default_bufsize) + # Our use of StringIO rather than lists of string objects returned + # by recv() minimizes memory usage and fragmentation that occurs + # when rbufsize is large compared to the typical return value of + # recv(). + buf = self._rbuf + buf.seek(0, 2) # seek end + if size < 0: + # Read until EOF + # reset _rbuf. we consume it via buf. + self._rbuf = io.BytesIO() + while True: + data = self.recv(rbufsize) + if not data: + break + buf.write(data) + return buf.getvalue() + else: + # Read until size bytes or EOF seen, whichever comes first + buf_len = buf.tell() + if buf_len >= size: + # Already have size bytes in our buffer? Extract and + # return. + buf.seek(0) + rv = buf.read(size) + self._rbuf = io.BytesIO() + self._rbuf.write(buf.read()) + return rv + + # reset _rbuf. we consume it via buf. + self._rbuf = io.BytesIO() + while True: + left = size - buf_len + # recv() will malloc the amount of memory given as its + # parameter even though it often returns much less data + # than that. The returned data string is short lived + # as we copy it into a StringIO and free it. This avoids + # fragmentation issues on many platforms. + data = self.recv(left) + if not data: + break + n = len(data) + if n == size and not buf_len: + # Shortcut. Avoid buffer data copies when: + # - We have no data in our buffer. + # AND + # - Our call to recv returned exactly the + # number of bytes we were asked to read. + return data + if n == left: + buf.write(data) + del data # explicit free + break + assert n <= left, 'recv(%d) returned %d bytes' % (left, n) + buf.write(data) + buf_len += n + del data # explicit free + # assert buf_len == buf.tell() + return buf.getvalue() + + def readline(self, size=-1): + buf = self._rbuf + buf.seek(0, 2) # seek end + if buf.tell() > 0: + # check if we already have it in our buffer + buf.seek(0) + bline = buf.readline(size) + if bline.endswith('\n') or len(bline) == size: + self._rbuf = io.BytesIO() + self._rbuf.write(buf.read()) + return bline + del bline + if size < 0: + # Read until \n or EOF, whichever comes first + if self._rbufsize <= 1: + # Speed up unbuffered case + buf.seek(0) + buffers = [buf.read()] + # reset _rbuf. we consume it via buf. + self._rbuf = io.BytesIO() + data = None + recv = self.recv + while data != '\n': + data = recv(1) + if not data: + break + buffers.append(data) + return ''.join(buffers) + + buf.seek(0, 2) # seek end + # reset _rbuf. we consume it via buf. + self._rbuf = io.BytesIO() + while True: + data = self.recv(self._rbufsize) + if not data: + break + nl = data.find('\n') + if nl >= 0: + nl += 1 + buf.write(data[:nl]) + self._rbuf.write(data[nl:]) + del data + break + buf.write(data) + return buf.getvalue() + else: + # Read until size bytes or \n or EOF seen, whichever comes + # first + buf.seek(0, 2) # seek end + buf_len = buf.tell() + if buf_len >= size: + buf.seek(0) + rv = buf.read(size) + self._rbuf = io.BytesIO() + self._rbuf.write(buf.read()) + return rv + # reset _rbuf. we consume it via buf. + self._rbuf = io.BytesIO() + while True: + data = self.recv(self._rbufsize) + if not data: + break + left = size - buf_len + # did we just receive a newline? + nl = data.find('\n', 0, left) + if nl >= 0: + nl += 1 + # save the excess data to _rbuf + self._rbuf.write(data[nl:]) + if buf_len: + buf.write(data[:nl]) + break + else: + # Shortcut. Avoid data copy through buf when + # returning a substring of our first recv(). + return data[:nl] + n = len(data) + if n == size and not buf_len: + # Shortcut. Avoid data copy through buf when + # returning exactly all of our first recv(). + return data + if n >= left: + buf.write(data[:left]) + self._rbuf.write(data[left:]) + break + buf.write(data) + buf_len += n + # assert buf_len == buf.tell() + return buf.getvalue() + else: + def read(self, size=-1): + if size < 0: + # Read until EOF + buffers = [self._rbuf] + self._rbuf = '' + if self._rbufsize <= 1: + recv_size = self.default_bufsize + else: + recv_size = self._rbufsize + + while True: + data = self.recv(recv_size) + if not data: + break + buffers.append(data) + return ''.join(buffers) + else: + # Read until size bytes or EOF seen, whichever comes first + data = self._rbuf + buf_len = len(data) + if buf_len >= size: + self._rbuf = data[size:] + return data[:size] + buffers = [] + if data: + buffers.append(data) + self._rbuf = '' + while True: + left = size - buf_len + recv_size = max(self._rbufsize, left) + data = self.recv(recv_size) + if not data: + break + buffers.append(data) + n = len(data) + if n >= left: + self._rbuf = data[left:] + buffers[-1] = data[:left] + break + buf_len += n + return ''.join(buffers) + + def readline(self, size=-1): + data = self._rbuf + if size < 0: + # Read until \n or EOF, whichever comes first + if self._rbufsize <= 1: + # Speed up unbuffered case + assert data == '' + buffers = [] + while data != '\n': + data = self.recv(1) + if not data: + break + buffers.append(data) + return ''.join(buffers) + nl = data.find('\n') + if nl >= 0: + nl += 1 + self._rbuf = data[nl:] + return data[:nl] + buffers = [] + if data: + buffers.append(data) + self._rbuf = '' + while True: + data = self.recv(self._rbufsize) + if not data: + break + buffers.append(data) + nl = data.find('\n') + if nl >= 0: + nl += 1 + self._rbuf = data[nl:] + buffers[-1] = data[:nl] + break + return ''.join(buffers) + else: + # Read until size bytes or \n or EOF seen, whichever comes + # first + nl = data.find('\n', 0, size) + if nl >= 0: + nl += 1 + self._rbuf = data[nl:] + return data[:nl] + buf_len = len(data) + if buf_len >= size: + self._rbuf = data[size:] + return data[:size] + buffers = [] + if data: + buffers.append(data) + self._rbuf = '' + while True: + data = self.recv(self._rbufsize) + if not data: + break + buffers.append(data) + left = size - buf_len + nl = data.find('\n', 0, left) + if nl >= 0: + nl += 1 + self._rbuf = data[nl:] + buffers[-1] = data[:nl] + break + n = len(data) + if n >= left: + self._rbuf = data[left:] + buffers[-1] = data[:left] + break + buf_len += n + return ''.join(buffers) + + +CP_makefile = CP_makefile_PY2 if six.PY2 else CP_makefile_PY3 + + +class HTTPConnection(object): + + """An HTTP connection (active socket). + + server: the Server object which received this connection. + socket: the raw socket object (usually TCP) for this connection. + makefile: a fileobject class for reading from the socket. + """ + + remote_addr = None + remote_port = None + ssl_env = None + rbufsize = DEFAULT_BUFFER_SIZE + wbufsize = DEFAULT_BUFFER_SIZE + RequestHandlerClass = HTTPRequest + + def __init__(self, server, sock, makefile=CP_makefile): + self.server = server + self.socket = sock + self.rfile = makefile(sock, 'rb', self.rbufsize) + self.wfile = makefile(sock, 'wb', self.wbufsize) + self.requests_seen = 0 + + def communicate(self): + """Read each request and respond appropriately.""" + request_seen = False + try: + while True: + # (re)set req to None so that if something goes wrong in + # the RequestHandlerClass constructor, the error doesn't + # get written to the previous request. + req = None + req = self.RequestHandlerClass(self.server, self) + + # This order of operations should guarantee correct pipelining. + req.parse_request() + if self.server.stats['Enabled']: + self.requests_seen += 1 + if not req.ready: + # Something went wrong in the parsing (and the server has + # probably already made a simple_response). Return and + # let the conn close. + return + + request_seen = True + req.respond() + if req.close_connection: + return + except socket.error: + e = sys.exc_info()[1] + errnum = e.args[0] + # sadly SSL sockets return a different (longer) time out string + if ( + errnum == 'timed out' or + errnum == 'The read operation timed out' + ): + # Don't error if we're between requests; only error + # if 1) no request has been started at all, or 2) we're + # in the middle of a request. + # See https://github.com/cherrypy/cherrypy/issues/853 + if (not request_seen) or (req and req.started_request): + # Don't bother writing the 408 if the response + # has already started being written. + if req and not req.sent_headers: + try: + req.simple_response('408 Request Timeout') + except FatalSSLAlert: + # Close the connection. + return + except NoSSLError: + self._handle_no_ssl() + elif errnum not in socket_errors_to_ignore: + self.server.error_log('socket.error %s' % repr(errnum), + level=logging.WARNING, traceback=True) + if req and not req.sent_headers: + try: + req.simple_response('500 Internal Server Error') + except FatalSSLAlert: + # Close the connection. + return + except NoSSLError: + self._handle_no_ssl() + return + except (KeyboardInterrupt, SystemExit): + raise + except FatalSSLAlert: + # Close the connection. + return + except NoSSLError: + self._handle_no_ssl(req) + except Exception: + e = sys.exc_info()[1] + self.server.error_log(repr(e), level=logging.ERROR, traceback=True) + if req and not req.sent_headers: + try: + req.simple_response('500 Internal Server Error') + except FatalSSLAlert: + # Close the connection. + return + + linger = False + + def _handle_no_ssl(self, req): + if not req or req.sent_headers: + return + # Unwrap wfile + self.wfile = CP_makefile(self.socket._sock, 'wb', self.wbufsize) + msg = ( + 'The client sent a plain HTTP request, but ' + 'this server only speaks HTTPS on this port.' + ) + req.simple_response('400 Bad Request', msg) + self.linger = True + + def close(self): + """Close the socket underlying this connection.""" + self.rfile.close() + + if not self.linger: + self._close_kernel_socket() + self.socket.close() + else: + # On the other hand, sometimes we want to hang around for a bit + # to make sure the client has a chance to read our entire + # response. Skipping the close() calls here delays the FIN + # packet until the socket object is garbage-collected later. + # Someday, perhaps, we'll do the full lingering_close that + # Apache does, but not today. + pass + + def _close_kernel_socket(self): + """ + On old Python versions, + Python's socket module does NOT call close on the kernel + socket when you call socket.close(). We do so manually here + because we want this server to send a FIN TCP segment + immediately. Note this must be called *before* calling + socket.close(), because the latter drops its reference to + the kernel socket. + """ + if six.PY2 and hasattr(self.socket, '_sock'): + self.socket._sock.close() + + +class TrueyZero(object): + + """An object which equals and does math like the integer 0 but evals True. + """ + + def __add__(self, other): + return other + + def __radd__(self, other): + return other +trueyzero = TrueyZero() + + +_SHUTDOWNREQUEST = None + + +class WorkerThread(threading.Thread): + + """Thread which continuously polls a Queue for Connection objects. + + Due to the timing issues of polling a Queue, a WorkerThread does not + check its own 'ready' flag after it has started. To stop the thread, + it is necessary to stick a _SHUTDOWNREQUEST object onto the Queue + (one for each running WorkerThread). + """ + + conn = None + """The current connection pulled off the Queue, or None.""" + + server = None + """The HTTP Server which spawned this thread, and which owns the + Queue and is placing active connections into it.""" + + ready = False + """A simple flag for the calling server to know when this thread + has begun polling the Queue.""" + + def __init__(self, server): + self.ready = False + self.server = server + + self.requests_seen = 0 + self.bytes_read = 0 + self.bytes_written = 0 + self.start_time = None + self.work_time = 0 + self.stats = { + 'Requests': lambda s: self.requests_seen + ( + (self.start_time is None) and + trueyzero or + self.conn.requests_seen + ), + 'Bytes Read': lambda s: self.bytes_read + ( + (self.start_time is None) and + trueyzero or + self.conn.rfile.bytes_read + ), + 'Bytes Written': lambda s: self.bytes_written + ( + (self.start_time is None) and + trueyzero or + self.conn.wfile.bytes_written + ), + 'Work Time': lambda s: self.work_time + ( + (self.start_time is None) and + trueyzero or + time.time() - self.start_time + ), + 'Read Throughput': lambda s: s['Bytes Read'](s) / ( + s['Work Time'](s) or 1e-6), + 'Write Throughput': lambda s: s['Bytes Written'](s) / ( + s['Work Time'](s) or 1e-6), + } + threading.Thread.__init__(self) + + def run(self): + self.server.stats['Worker Threads'][self.getName()] = self.stats + try: + self.ready = True + while True: + conn = self.server.requests.get() + if conn is _SHUTDOWNREQUEST: + return + + self.conn = conn + if self.server.stats['Enabled']: + self.start_time = time.time() + try: + conn.communicate() + finally: + conn.close() + if self.server.stats['Enabled']: + self.requests_seen += self.conn.requests_seen + self.bytes_read += self.conn.rfile.bytes_read + self.bytes_written += self.conn.wfile.bytes_written + self.work_time += time.time() - self.start_time + self.start_time = None + self.conn = None + except (KeyboardInterrupt, SystemExit): + exc = sys.exc_info()[1] + self.server.interrupt = exc + + +class ThreadPool(object): + + """A Request Queue for an HTTPServer which pools threads. + + ThreadPool objects must provide min, get(), put(obj), start() + and stop(timeout) attributes. + """ + + def __init__(self, server, min=10, max=-1, + accepted_queue_size=-1, accepted_queue_timeout=10): + self.server = server + self.min = min + self.max = max + self._threads = [] + self._queue = queue.Queue(maxsize=accepted_queue_size) + self._queue_put_timeout = accepted_queue_timeout + self.get = self._queue.get + + def start(self): + """Start the pool of threads.""" + for i in range(self.min): + self._threads.append(WorkerThread(self.server)) + for worker in self._threads: + worker.setName('CP Server ' + worker.getName()) + worker.start() + for worker in self._threads: + while not worker.ready: + time.sleep(.1) + + def _get_idle(self): + """Number of worker threads which are idle. Read-only.""" + return len([t for t in self._threads if t.conn is None]) + idle = property(_get_idle, doc=_get_idle.__doc__) + + def put(self, obj): + self._queue.put(obj, block=True, timeout=self._queue_put_timeout) + if obj is _SHUTDOWNREQUEST: + return + + def grow(self, amount): + """Spawn new worker threads (not above self.max).""" + if self.max > 0: + budget = max(self.max - len(self._threads), 0) + else: + # self.max <= 0 indicates no maximum + budget = float('inf') + + n_new = min(amount, budget) + + workers = [self._spawn_worker() for i in range(n_new)] + while not all(worker.ready for worker in workers): + time.sleep(.1) + self._threads.extend(workers) + + def _spawn_worker(self): + worker = WorkerThread(self.server) + worker.setName('CP Server ' + worker.getName()) + worker.start() + return worker + + def shrink(self, amount): + """Kill off worker threads (not below self.min).""" + # Grow/shrink the pool if necessary. + # Remove any dead threads from our list + for t in self._threads: + if not t.isAlive(): + self._threads.remove(t) + amount -= 1 + + # calculate the number of threads above the minimum + n_extra = max(len(self._threads) - self.min, 0) + + # don't remove more than amount + n_to_remove = min(amount, n_extra) + + # put shutdown requests on the queue equal to the number of threads + # to remove. As each request is processed by a worker, that worker + # will terminate and be culled from the list. + for n in range(n_to_remove): + self._queue.put(_SHUTDOWNREQUEST) + + def stop(self, timeout=5): + # Must shut down threads here so the code that calls + # this method can know when all threads are stopped. + for worker in self._threads: + self._queue.put(_SHUTDOWNREQUEST) + + # Don't join currentThread (when stop is called inside a request). + current = threading.currentThread() + if timeout and timeout >= 0: + endtime = time.time() + timeout + while self._threads: + worker = self._threads.pop() + if worker is not current and worker.isAlive(): + try: + if timeout is None or timeout < 0: + worker.join() + else: + remaining_time = endtime - time.time() + if remaining_time > 0: + worker.join(remaining_time) + if worker.isAlive(): + # We exhausted the timeout. + # Forcibly shut down the socket. + c = worker.conn + if c and not c.rfile.closed: + try: + c.socket.shutdown(socket.SHUT_RD) + except TypeError: + # pyOpenSSL sockets don't take an arg + c.socket.shutdown() + worker.join() + except (AssertionError, + # Ignore repeated Ctrl-C. + # See + # https://github.com/cherrypy/cherrypy/issues/691. + KeyboardInterrupt): + pass + + def _get_qsize(self): + return self._queue.qsize() + qsize = property(_get_qsize) + + +try: + import fcntl +except ImportError: + try: + from ctypes import windll, WinError + import ctypes.wintypes + _SetHandleInformation = windll.kernel32.SetHandleInformation + _SetHandleInformation.argtypes = [ + ctypes.wintypes.HANDLE, + ctypes.wintypes.DWORD, + ctypes.wintypes.DWORD, + ] + _SetHandleInformation.restype = ctypes.wintypes.BOOL + except ImportError: + def prevent_socket_inheritance(sock): + """Dummy function, since neither fcntl nor ctypes are available.""" + pass + else: + def prevent_socket_inheritance(sock): + """Mark the given socket fd as non-inheritable (Windows).""" + if not _SetHandleInformation(sock.fileno(), 1, 0): + raise WinError() +else: + def prevent_socket_inheritance(sock): + """Mark the given socket fd as non-inheritable (POSIX).""" + fd = sock.fileno() + old_flags = fcntl.fcntl(fd, fcntl.F_GETFD) + fcntl.fcntl(fd, fcntl.F_SETFD, old_flags | fcntl.FD_CLOEXEC) + + +class SSLAdapter(object): + + """Base class for SSL driver library adapters. + + Required methods: + + * ``wrap(sock) -> (wrapped socket, ssl environ dict)`` + * ``makefile(sock, mode='r', bufsize=DEFAULT_BUFFER_SIZE) -> + socket file object`` + """ + + def __init__(self, certificate, private_key, certificate_chain=None): + self.certificate = certificate + self.private_key = private_key + self.certificate_chain = certificate_chain + + def wrap(self, sock): + raise NotImplemented + + def makefile(self, sock, mode='r', bufsize=DEFAULT_BUFFER_SIZE): + raise NotImplemented + + +class HTTPServer(object): + + """An HTTP server.""" + + _bind_addr = '127.0.0.1' + _interrupt = None + + gateway = None + """A Gateway instance.""" + + minthreads = None + """The minimum number of worker threads to create (default 10).""" + + maxthreads = None + """The maximum number of worker threads to create (default -1 = no limit). + """ + + server_name = None + """The name of the server; defaults to socket.gethostname().""" + + protocol = 'HTTP/1.1' + """The version string to write in the Status-Line of all HTTP responses. + + For example, "HTTP/1.1" is the default. This also limits the supported + features used in the response.""" + + request_queue_size = 5 + """The 'backlog' arg to socket.listen(); max queued connections + (default 5). + """ + + shutdown_timeout = 5 + """The total time, in seconds, to wait for worker threads to cleanly exit. + """ + + timeout = 10 + """The timeout in seconds for accepted connections (default 10).""" + + version = 'CherryPy/' + cp_version + """A version string for the HTTPServer.""" + + software = None + """The value to set for the SERVER_SOFTWARE entry in the WSGI environ. + + If None, this defaults to ``'%s Server' % self.version``.""" + + ready = False + """An internal flag which marks whether the socket is accepting + connections. + """ + + max_request_header_size = 0 + """The maximum size, in bytes, for request headers, or 0 for no limit.""" + + max_request_body_size = 0 + """The maximum size, in bytes, for request bodies, or 0 for no limit.""" + + nodelay = True + """If True (the default since 3.1), sets the TCP_NODELAY socket option.""" + + ConnectionClass = HTTPConnection + """The class to use for handling HTTP connections.""" + + ssl_adapter = None + """An instance of SSLAdapter (or a subclass). + + You must have the corresponding SSL driver library installed.""" + + def __init__(self, bind_addr, gateway, minthreads=10, maxthreads=-1, + server_name=None): + self.bind_addr = bind_addr + self.gateway = gateway + + self.requests = ThreadPool(self, min=minthreads or 1, max=maxthreads) + + if not server_name: + server_name = socket.gethostname() + self.server_name = server_name + self.clear_stats() + + def clear_stats(self): + self._start_time = None + self._run_time = 0 + self.stats = { + 'Enabled': False, + 'Bind Address': lambda s: repr(self.bind_addr), + 'Run time': lambda s: (not s['Enabled']) and -1 or self.runtime(), + 'Accepts': 0, + 'Accepts/sec': lambda s: s['Accepts'] / self.runtime(), + 'Queue': lambda s: getattr(self.requests, 'qsize', None), + 'Threads': lambda s: len(getattr(self.requests, '_threads', [])), + 'Threads Idle': lambda s: getattr(self.requests, 'idle', None), + 'Socket Errors': 0, + 'Requests': lambda s: (not s['Enabled']) and -1 or sum( + [w['Requests'](w) for w in s['Worker Threads'].values()], 0), + 'Bytes Read': lambda s: (not s['Enabled']) and -1 or sum( + [w['Bytes Read'](w) for w in s['Worker Threads'].values()], 0), + 'Bytes Written': lambda s: (not s['Enabled']) and -1 or sum( + [w['Bytes Written'](w) for w in s['Worker Threads'].values()], + 0), + 'Work Time': lambda s: (not s['Enabled']) and -1 or sum( + [w['Work Time'](w) for w in s['Worker Threads'].values()], 0), + 'Read Throughput': lambda s: (not s['Enabled']) and -1 or sum( + [w['Bytes Read'](w) / (w['Work Time'](w) or 1e-6) + for w in s['Worker Threads'].values()], 0), + 'Write Throughput': lambda s: (not s['Enabled']) and -1 or sum( + [w['Bytes Written'](w) / (w['Work Time'](w) or 1e-6) + for w in s['Worker Threads'].values()], 0), + 'Worker Threads': {}, + } + logging.statistics['CherryPy HTTPServer %d' % id(self)] = self.stats + + def runtime(self): + if self._start_time is None: + return self._run_time + else: + return self._run_time + (time.time() - self._start_time) + + def __str__(self): + return '%s.%s(%r)' % (self.__module__, self.__class__.__name__, + self.bind_addr) + + def _get_bind_addr(self): + return self._bind_addr + + def _set_bind_addr(self, value): + if isinstance(value, tuple) and value[0] in ('', None): + # Despite the socket module docs, using '' does not + # allow AI_PASSIVE to work. Passing None instead + # returns '0.0.0.0' like we want. In other words: + # host AI_PASSIVE result + # '' Y 192.168.x.y + # '' N 192.168.x.y + # None Y 0.0.0.0 + # None N 127.0.0.1 + # But since you can get the same effect with an explicit + # '0.0.0.0', we deny both the empty string and None as values. + raise ValueError("Host values of '' or None are not allowed. " + "Use '0.0.0.0' (IPv4) or '::' (IPv6) instead " + 'to listen on all active interfaces.') + self._bind_addr = value + bind_addr = property( + _get_bind_addr, + _set_bind_addr, + doc="""The interface on which to listen for connections. + + For TCP sockets, a (host, port) tuple. Host values may be any IPv4 + or IPv6 address, or any valid hostname. The string 'localhost' is a + synonym for '127.0.0.1' (or '::1', if your hosts file prefers IPv6). + The string '0.0.0.0' is a special IPv4 entry meaning "any active + interface" (INADDR_ANY), and '::' is the similar IN6ADDR_ANY for + IPv6. The empty string or None are not allowed. + + For UNIX sockets, supply the filename as a string. + + Systemd socket activation is automatic and doesn't require tempering + with this variable""") + + def start(self): + """Run the server forever.""" + # We don't have to trap KeyboardInterrupt or SystemExit here, + # because cherrpy.server already does so, calling self.stop() for us. + # If you're using this server with another framework, you should + # trap those exceptions in whatever code block calls start(). + self._interrupt = None + + if self.software is None: + self.software = '%s Server' % self.version + + # Select the appropriate socket + self.socket = None + if os.getenv('LISTEN_PID', None): + # systemd socket activation + self.socket = socket.fromfd(3, socket.AF_INET, socket.SOCK_STREAM) + elif isinstance(self.bind_addr, six.string_types): + # AF_UNIX socket + + # So we can reuse the socket... + try: + os.unlink(self.bind_addr) + except: + pass + + # So everyone can access the socket... + try: + os.chmod(self.bind_addr, 0o777) + except: + pass + + info = [ + (socket.AF_UNIX, socket.SOCK_STREAM, 0, '', self.bind_addr)] + else: + # AF_INET or AF_INET6 socket + # Get the correct address family for our host (allows IPv6 + # addresses) + host, port = self.bind_addr + try: + info = socket.getaddrinfo( + host, port, socket.AF_UNSPEC, + socket.SOCK_STREAM, 0, socket.AI_PASSIVE) + except socket.gaierror: + if ':' in self.bind_addr[0]: + info = [(socket.AF_INET6, socket.SOCK_STREAM, + 0, '', self.bind_addr + (0, 0))] + else: + info = [(socket.AF_INET, socket.SOCK_STREAM, + 0, '', self.bind_addr)] + + if not self.socket: + msg = 'No socket could be created' + for res in info: + af, socktype, proto, canonname, sa = res + try: + self.bind(af, socktype, proto) + break + except socket.error as serr: + msg = '%s -- (%s: %s)' % (msg, sa, serr) + if self.socket: + self.socket.close() + self.socket = None + + if not self.socket: + raise socket.error(msg) + + # Timeout so KeyboardInterrupt can be caught on Win32 + self.socket.settimeout(1) + self.socket.listen(self.request_queue_size) + + # Create worker threads + self.requests.start() + + self.ready = True + self._start_time = time.time() + while self.ready: + try: + self.tick() + except (KeyboardInterrupt, SystemExit): + raise + except: + self.error_log('Error in HTTPServer.tick', level=logging.ERROR, + traceback=True) + + if self.interrupt: + while self.interrupt is True: + # Wait for self.stop() to complete. See _set_interrupt. + time.sleep(0.1) + if self.interrupt: + raise self.interrupt + + def error_log(self, msg='', level=20, traceback=False): + # Override this in subclasses as desired + sys.stderr.write(msg + '\n') + sys.stderr.flush() + if traceback: + tblines = traceback_.format_exc() + sys.stderr.write(tblines) + sys.stderr.flush() + + def bind(self, family, type, proto=0): + """Create (or recreate) the actual socket object.""" + self.socket = socket.socket(family, type, proto) + prevent_socket_inheritance(self.socket) + self.socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + if self.nodelay and not isinstance(self.bind_addr, str): + self.socket.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1) + + if self.ssl_adapter is not None: + self.socket = self.ssl_adapter.bind(self.socket) + + # If listening on the IPV6 any address ('::' = IN6ADDR_ANY), + # activate dual-stack. See + # https://github.com/cherrypy/cherrypy/issues/871. + if (hasattr(socket, 'AF_INET6') and family == socket.AF_INET6 + and self.bind_addr[0] in ('::', '::0', '::0.0.0.0')): + try: + self.socket.setsockopt( + socket.IPPROTO_IPV6, socket.IPV6_V6ONLY, 0) + except (AttributeError, socket.error): + # Apparently, the socket option is not available in + # this machine's TCP stack + pass + + self.socket.bind(self.bind_addr) + + def tick(self): + """Accept a new connection and put it on the Queue.""" + try: + s, addr = self.socket.accept() + if self.stats['Enabled']: + self.stats['Accepts'] += 1 + if not self.ready: + return + + prevent_socket_inheritance(s) + if hasattr(s, 'settimeout'): + s.settimeout(self.timeout) + + makefile = CP_makefile + ssl_env = {} + # if ssl cert and key are set, we try to be a secure HTTP server + if self.ssl_adapter is not None: + try: + s, ssl_env = self.ssl_adapter.wrap(s) + except NoSSLError: + msg = ('The client sent a plain HTTP request, but ' + 'this server only speaks HTTPS on this port.') + buf = ['%s 400 Bad Request\r\n' % self.protocol, + 'Content-Length: %s\r\n' % len(msg), + 'Content-Type: text/plain\r\n\r\n', + msg] + + sock_to_make = s if six.PY3 else s._sock + wfile = makefile(sock_to_make, 'wb', DEFAULT_BUFFER_SIZE) + try: + wfile.write(ntob(''.join(buf))) + except socket.error: + x = sys.exc_info()[1] + if x.args[0] not in socket_errors_to_ignore: + raise + return + if not s: + return + makefile = self.ssl_adapter.makefile + # Re-apply our timeout since we may have a new socket object + if hasattr(s, 'settimeout'): + s.settimeout(self.timeout) + + conn = self.ConnectionClass(self, s, makefile) + + if not isinstance(self.bind_addr, six.string_types): + # optional values + # Until we do DNS lookups, omit REMOTE_HOST + if addr is None: # sometimes this can happen + # figure out if AF_INET or AF_INET6. + if len(s.getsockname()) == 2: + # AF_INET + addr = ('0.0.0.0', 0) + else: + # AF_INET6 + addr = ('::', 0) + conn.remote_addr = addr[0] + conn.remote_port = addr[1] + + conn.ssl_env = ssl_env + + try: + self.requests.put(conn) + except queue.Full: + # Just drop the conn. TODO: write 503 back? + conn.close() + return + except socket.timeout: + # The only reason for the timeout in start() is so we can + # notice keyboard interrupts on Win32, which don't interrupt + # accept() by default + return + except socket.error: + x = sys.exc_info()[1] + if self.stats['Enabled']: + self.stats['Socket Errors'] += 1 + if x.args[0] in socket_error_eintr: + # I *think* this is right. EINTR should occur when a signal + # is received during the accept() call; all docs say retry + # the call, and I *think* I'm reading it right that Python + # will then go ahead and poll for and handle the signal + # elsewhere. See + # https://github.com/cherrypy/cherrypy/issues/707. + return + if x.args[0] in socket_errors_nonblocking: + # Just try again. See + # https://github.com/cherrypy/cherrypy/issues/479. + return + if x.args[0] in socket_errors_to_ignore: + # Our socket was closed. + # See https://github.com/cherrypy/cherrypy/issues/686. + return + raise + + def _get_interrupt(self): + return self._interrupt + + def _set_interrupt(self, interrupt): + self._interrupt = True + self.stop() + self._interrupt = interrupt + interrupt = property(_get_interrupt, _set_interrupt, + doc='Set this to an Exception instance to ' + 'interrupt the server.') + + def stop(self): + """Gracefully shutdown a server that is serving forever.""" + self.ready = False + if self._start_time is not None: + self._run_time += (time.time() - self._start_time) + self._start_time = None + + sock = getattr(self, 'socket', None) + if sock: + if not isinstance(self.bind_addr, six.string_types): + # Touch our own socket to make accept() return immediately. + try: + host, port = sock.getsockname()[:2] + except socket.error: + x = sys.exc_info()[1] + if x.args[0] not in socket_errors_to_ignore: + # Changed to use error code and not message + # See + # https://github.com/cherrypy/cherrypy/issues/860. + raise + else: + # Note that we're explicitly NOT using AI_PASSIVE, + # here, because we want an actual IP to touch. + # localhost won't work if we've bound to a public IP, + # but it will if we bound to '0.0.0.0' (INADDR_ANY). + for res in socket.getaddrinfo(host, port, socket.AF_UNSPEC, + socket.SOCK_STREAM): + af, socktype, proto, canonname, sa = res + s = None + try: + s = socket.socket(af, socktype, proto) + # See + # http://groups.google.com/group/cherrypy-users/ + # browse_frm/thread/bbfe5eb39c904fe0 + s.settimeout(1.0) + s.connect((host, port)) + s.close() + except socket.error: + if s: + s.close() + if hasattr(sock, 'close'): + sock.close() + self.socket = None + + self.requests.stop(self.shutdown_timeout) + + +class Gateway(object): + + """A base class to interface HTTPServer with other systems, such as WSGI. + """ + + def __init__(self, req): + self.req = req + + def respond(self): + """Process the current request. Must be overridden in a subclass.""" + raise NotImplemented + + +# These may either be wsgiserver.SSLAdapter subclasses or the string names +# of such classes (in which case they will be lazily loaded). +ssl_adapters = { + 'builtin': 'cherrypy.wsgiserver.ssl_builtin.BuiltinSSLAdapter', + 'pyopenssl': 'cherrypy.wsgiserver.ssl_pyopenssl.pyOpenSSLAdapter', +} + + +def get_ssl_adapter_class(name='builtin'): + """Return an SSL adapter class for the given name.""" + adapter = ssl_adapters[name.lower()] + if isinstance(adapter, six.string_types): + last_dot = adapter.rfind('.') + attr_name = adapter[last_dot + 1:] + mod_path = adapter[:last_dot] + + try: + mod = sys.modules[mod_path] + if mod is None: + raise KeyError() + except KeyError: + # The last [''] is important. + mod = __import__(mod_path, globals(), locals(), ['']) + + # Let an AttributeError propagate outward. + try: + adapter = getattr(mod, attr_name) + except AttributeError: + raise AttributeError("'%s' object has no attribute '%s'" + % (mod_path, attr_name)) + + return adapter + +# ------------------------------- WSGI Stuff -------------------------------- # + + +class CherryPyWSGIServer(HTTPServer): + + """A subclass of HTTPServer which calls a WSGI application.""" + + wsgi_version = (1, 0) + """The version of WSGI to produce.""" + + def __init__(self, bind_addr, wsgi_app, numthreads=10, server_name=None, + max=-1, request_queue_size=5, timeout=10, shutdown_timeout=5, + accepted_queue_size=-1, accepted_queue_timeout=10): + self.requests = ThreadPool(self, min=numthreads or 1, max=max, + accepted_queue_size=accepted_queue_size, + accepted_queue_timeout=accepted_queue_timeout) + self.wsgi_app = wsgi_app + self.gateway = wsgi_gateways[self.wsgi_version] + + self.bind_addr = bind_addr + if not server_name: + server_name = socket.gethostname() + self.server_name = server_name + self.request_queue_size = request_queue_size + + self.timeout = timeout + self.shutdown_timeout = shutdown_timeout + self.clear_stats() + + def _get_numthreads(self): + return self.requests.min + + def _set_numthreads(self, value): + self.requests.min = value + numthreads = property(_get_numthreads, _set_numthreads) + + +class WSGIGateway(Gateway): + + """A base class to interface HTTPServer with WSGI.""" + + def __init__(self, req): + self.req = req + self.started_response = False + self.env = self.get_environ() + self.remaining_bytes_out = None + + def get_environ(self): + """Return a new environ dict targeting the given wsgi.version""" + raise NotImplemented + + def respond(self): + """Process the current request.""" + + """ + From PEP 333: + + The start_response callable must not actually transmit + the response headers. Instead, it must store them for the + server or gateway to transmit only after the first + iteration of the application return value that yields + a NON-EMPTY string, or upon the application's first + invocation of the write() callable. + """ + + response = self.req.server.wsgi_app(self.env, self.start_response) + try: + for chunk in filter(None, response): + if not isinstance(chunk, six.binary_type): + raise ValueError('WSGI Applications must yield bytes') + self.write(chunk) + finally: + if hasattr(response, 'close'): + response.close() + + def start_response(self, status, headers, exc_info=None): + """ + WSGI callable to begin the HTTP response. + """ + # "The application may call start_response more than once, + # if and only if the exc_info argument is provided." + if self.started_response and not exc_info: + raise AssertionError('WSGI start_response called a second ' + 'time with no exc_info.') + self.started_response = True + + # "if exc_info is provided, and the HTTP headers have already been + # sent, start_response must raise an error, and should raise the + # exc_info tuple." + if self.req.sent_headers: + try: + six.reraise(*exc_info) + finally: + exc_info = None + + self.req.status = self._encode_status(status) + + for k, v in headers: + if not isinstance(k, str): + raise TypeError( + 'WSGI response header key %r is not of type str.' % k) + if not isinstance(v, str): + raise TypeError( + 'WSGI response header value %r is not of type str.' % v) + if k.lower() == 'content-length': + self.remaining_bytes_out = int(v) + out_header = ntob(k), ntob(v) + self.req.outheaders.append(out_header) + + return self.write + + @staticmethod + def _encode_status(status): + """ + According to PEP 3333, when using Python 3, the response status + and headers must be bytes masquerading as unicode; that is, they + must be of type "str" but are restricted to code points in the + "latin-1" set. + """ + if six.PY2: + return status + if not isinstance(status, str): + raise TypeError('WSGI response status is not of type str.') + return status.encode('ISO-8859-1') + + def write(self, chunk): + """WSGI callable to write unbuffered data to the client. + + This method is also used internally by start_response (to write + data from the iterable returned by the WSGI application). + """ + if not self.started_response: + raise AssertionError('WSGI write called before start_response.') + + chunklen = len(chunk) + rbo = self.remaining_bytes_out + if rbo is not None and chunklen > rbo: + if not self.req.sent_headers: + # Whew. We can send a 500 to the client. + self.req.simple_response( + '500 Internal Server Error', + 'The requested resource returned more bytes than the ' + 'declared Content-Length.') + else: + # Dang. We have probably already sent data. Truncate the chunk + # to fit (so the client doesn't hang) and raise an error later. + chunk = chunk[:rbo] + + if not self.req.sent_headers: + self.req.sent_headers = True + self.req.send_headers() + + self.req.write(chunk) + + if rbo is not None: + rbo -= chunklen + if rbo < 0: + raise ValueError( + 'Response body exceeds the declared Content-Length.') + + +class WSGIGateway_10(WSGIGateway): + + """A Gateway class to interface HTTPServer with WSGI 1.0.x.""" + + def get_environ(self): + """Return a new environ dict targeting the given wsgi.version""" + req = self.req + env = { + # set a non-standard environ entry so the WSGI app can know what + # the *real* server protocol is (and what features to support). + # See http://www.faqs.org/rfcs/rfc2145.html. + 'ACTUAL_SERVER_PROTOCOL': req.server.protocol, + 'PATH_INFO': bton(req.path), + 'QUERY_STRING': bton(req.qs), + 'REMOTE_ADDR': req.conn.remote_addr or '', + 'REMOTE_PORT': str(req.conn.remote_port or ''), + 'REQUEST_METHOD': bton(req.method), + 'REQUEST_URI': bton(req.uri), + 'SCRIPT_NAME': '', + 'SERVER_NAME': req.server.server_name, + # Bah. "SERVER_PROTOCOL" is actually the REQUEST protocol. + 'SERVER_PROTOCOL': bton(req.request_protocol), + 'SERVER_SOFTWARE': req.server.software, + 'wsgi.errors': sys.stderr, + 'wsgi.input': req.rfile, + 'wsgi.multiprocess': False, + 'wsgi.multithread': True, + 'wsgi.run_once': False, + 'wsgi.url_scheme': bton(req.scheme), + 'wsgi.version': (1, 0), + } + + if isinstance(req.server.bind_addr, six.string_types): + # AF_UNIX. This isn't really allowed by WSGI, which doesn't + # address unix domain sockets. But it's better than nothing. + env['SERVER_PORT'] = '' + else: + env['SERVER_PORT'] = str(req.server.bind_addr[1]) + + # Request headers + env.update( + ('HTTP_' + bton(k).upper().replace('-', '_'), bton(v)) + for k, v in req.inheaders.items() + ) + + # CONTENT_TYPE/CONTENT_LENGTH + ct = env.pop('HTTP_CONTENT_TYPE', None) + if ct is not None: + env['CONTENT_TYPE'] = ct + cl = env.pop('HTTP_CONTENT_LENGTH', None) + if cl is not None: + env['CONTENT_LENGTH'] = cl + + if req.conn.ssl_env: + env.update(req.conn.ssl_env) + + return env + + +class WSGIGateway_u0(WSGIGateway_10): + + """A Gateway class to interface HTTPServer with WSGI u.0. + + WSGI u.0 is an experimental protocol, which uses unicode for keys + and values in both Python 2 and Python 3. + """ + + def get_environ(self): + """Return a new environ dict targeting the given wsgi.version""" + req = self.req + env_10 = WSGIGateway_10.get_environ(self) + env = dict(map(self._decode_key, env_10.items())) + env[six.u('wsgi.version')] = ('u', 0) + + # Request-URI + enc = env.setdefault(six.u('wsgi.url_encoding'), six.u('utf-8')) + try: + env['PATH_INFO'] = req.path.decode(enc) + env['QUERY_STRING'] = req.qs.decode(enc) + except UnicodeDecodeError: + # Fall back to latin 1 so apps can transcode if needed. + env['wsgi.url_encoding'] = 'ISO-8859-1' + env['PATH_INFO'] = env_10['PATH_INFO'] + env['QUERY_STRING'] = env_10['QUERY_STRING'] + + env.update(map(self._decode_value, env.items())) + + return env + + @staticmethod + def _decode_key(item): + k, v = item + if six.PY2: + k = k.decode('ISO-8859-1') + return k, v + + @staticmethod + def _decode_value(item): + k, v = item + skip_keys = 'REQUEST_URI', 'wsgi.input' + if six.PY3 or not isinstance(v, bytes) or k in skip_keys: + return k, v + return k, v.decode('ISO-8859-1') + + +wsgi_gateways = { + (1, 0): WSGIGateway_10, + ('u', 0): WSGIGateway_u0, +} + + +class WSGIPathInfoDispatcher(object): + + """A WSGI dispatcher for dispatch based on the PATH_INFO. + + apps: a dict or list of (path_prefix, app) pairs. + """ + + def __init__(self, apps): + try: + apps = list(apps.items()) + except AttributeError: + pass + + # Sort the apps by len(path), descending + by_path_len = lambda app: len(app[0]) + apps.sort(key=by_path_len, reverse=True) + + # The path_prefix strings must start, but not end, with a slash. + # Use "" instead of "/". + self.apps = [(p.rstrip('/'), a) for p, a in apps] + + def __call__(self, environ, start_response): + path = environ['PATH_INFO'] or '/' + for p, app in self.apps: + # The apps list should be sorted by length, descending. + if path.startswith(p + '/') or path == p: + environ = environ.copy() + environ['SCRIPT_NAME'] = environ['SCRIPT_NAME'] + p + environ['PATH_INFO'] = path[len(p):] + return app(environ, start_response) + + start_response('404 Not Found', [('Content-Type', 'text/plain'), + ('Content-Length', '0')]) + return [''] diff --git a/cherrypy/wsgiserver/ssl_builtin.py b/cherrypy/wsgiserver/ssl_builtin.py index 3faf703..a538c19 100644 --- a/cherrypy/wsgiserver/ssl_builtin.py +++ b/cherrypy/wsgiserver/ssl_builtin.py @@ -33,10 +33,10 @@ class BuiltinSSLAdapter(wsgiserver.SSLAdapter): private_key = None """The filename of the server's private key file.""" - + certificate_chain = None """The filename of the certificate chain file.""" - + """The ssl.SSLContext that will be used to wrap sockets where available (on Python > 2.7.9 / 3.3) """ @@ -44,7 +44,7 @@ class BuiltinSSLAdapter(wsgiserver.SSLAdapter): def __init__(self, certificate, private_key, certificate_chain=None): if ssl is None: - raise ImportError("You must install the ssl module to use HTTPS.") + raise ImportError('You must install the ssl module to use HTTPS.') self.certificate = certificate self.private_key = private_key self.certificate_chain = certificate_chain @@ -79,13 +79,18 @@ class BuiltinSSLAdapter(wsgiserver.SSLAdapter): # the 'ping' isn't SSL. return None, {} elif e.errno == ssl.SSL_ERROR_SSL: - if e.args[1].endswith('http request'): + if 'http request' in e.args[1]: # The client is speaking HTTP to an HTTPS server. raise wsgiserver.NoSSLError - elif e.args[1].endswith('unknown protocol'): + elif 'unknown protocol' in e.args[1]: # The client is speaking some non-HTTP protocol. # Drop the conn. return None, {} + elif 'handshake operation timed out' in e.args[0]: + # This error is thrown by builtin SSL after a timeout + # when client is speaking HTTP to an HTTPS server. + # The connection can safely be dropped. + return None, {} raise return s, self.get_environ(s) @@ -94,8 +99,8 @@ class BuiltinSSLAdapter(wsgiserver.SSLAdapter): """Create WSGI environ entries to be merged into each request.""" cipher = sock.cipher() ssl_environ = { - "wsgi.url_scheme": "https", - "HTTPS": "on", + 'wsgi.url_scheme': 'https', + 'HTTPS': 'on', 'SSL_PROTOCOL': cipher[1], 'SSL_CIPHER': cipher[0] # SSL_VERSION_INTERFACE string The mod_ssl program version @@ -103,9 +108,5 @@ class BuiltinSSLAdapter(wsgiserver.SSLAdapter): } return ssl_environ - if sys.version_info >= (3, 0): - def makefile(self, sock, mode='r', bufsize=DEFAULT_BUFFER_SIZE): - return wsgiserver.CP_makefile(sock, mode, bufsize) - else: - def makefile(self, sock, mode='r', bufsize=DEFAULT_BUFFER_SIZE): - return wsgiserver.CP_fileobject(sock, mode, bufsize) + def makefile(self, sock, mode='r', bufsize=DEFAULT_BUFFER_SIZE): + return wsgiserver.CP_makefile(sock, mode, bufsize) diff --git a/cherrypy/wsgiserver/ssl_pyopenssl.py b/cherrypy/wsgiserver/ssl_pyopenssl.py index ebace19..a1c09d8 100644 --- a/cherrypy/wsgiserver/ssl_pyopenssl.py +++ b/cherrypy/wsgiserver/ssl_pyopenssl.py @@ -43,7 +43,7 @@ except ImportError: SSL = None -class SSL_fileobject(wsgiserver.CP_fileobject): +class SSL_fileobject(wsgiserver.CP_makefile): """SSL file object attached to a socket object.""" @@ -70,15 +70,15 @@ class SSL_fileobject(wsgiserver.CP_fileobject): time.sleep(self.ssl_retry) except SSL.SysCallError as e: if is_reader and e.args == (-1, 'Unexpected EOF'): - return "" + return '' errnum = e.args[0] if is_reader and errnum in wsgiserver.socket_errors_to_ignore: - return "" + return '' raise socket.error(errnum) except SSL.Error as e: if is_reader and e.args == (-1, 'Unexpected EOF'): - return "" + return '' thirdarg = None try: @@ -95,7 +95,7 @@ class SSL_fileobject(wsgiserver.CP_fileobject): raise if time.time() - start > self.ssl_timeout: - raise socket.timeout("timed out") + raise socket.timeout('timed out') def recv(self, size): return self._safe_call(True, super(SSL_fileobject, self).recv, size) @@ -166,7 +166,7 @@ class pyOpenSSLAdapter(wsgiserver.SSLAdapter): def __init__(self, certificate, private_key, certificate_chain=None): if SSL is None: - raise ImportError("You must install pyOpenSSL to use HTTPS.") + raise ImportError('You must install pyOpenSSL to use HTTPS.') self.context = None self.certificate = certificate @@ -192,18 +192,14 @@ class pyOpenSSLAdapter(wsgiserver.SSLAdapter): c = SSL.Context(SSL.SSLv23_METHOD) c.use_privatekey_file(self.private_key) if self.certificate_chain: - if isinstance(self.certificate_chain, unicode) and self.certificate_chain.encode('cp1252', 'ignore') == self.certificate_chain.encode('cp1252', 'replace'): - # Support buggy PyOpenSSL 0.14, which cannot handle Unicode names - c.load_verify_locations(self.certificate_chain.encode('cp1252', 'ignore')) - else: - c.load_verify_locations(self.certificate_chain) + c.load_verify_locations(self.certificate_chain) c.use_certificate_file(self.certificate) return c def get_environ(self): """Return WSGI environ entries to be merged into each request.""" ssl_environ = { - "HTTPS": "on", + 'HTTPS': 'on', # pyOpenSSL doesn't provide access to any of these AFAICT # 'SSL_PROTOCOL': 'SSLv2', # SSL_CIPHER string The cipher specification name @@ -224,8 +220,8 @@ class pyOpenSSLAdapter(wsgiserver.SSLAdapter): # Validity of server's certificate (end time), }) - for prefix, dn in [("I", cert.get_issuer()), - ("S", cert.get_subject())]: + for prefix, dn in [('I', cert.get_issuer()), + ('S', cert.get_subject())]: # X509Name objects don't seem to have a way to get the # complete DN string. Use str() and slice it instead, # because str(dn) == "" @@ -237,9 +233,9 @@ class pyOpenSSLAdapter(wsgiserver.SSLAdapter): # The DN should be of the form: /k1=v1/k2=v2, but we must allow # for any value to contain slashes itself (in a URL). while dnstr: - pos = dnstr.rfind("=") + pos = dnstr.rfind('=') dnstr, value = dnstr[:pos], dnstr[pos + 1:] - pos = dnstr.rfind("/") + pos = dnstr.rfind('/') dnstr, key = dnstr[:pos], dnstr[pos + 1:] if key and value: wsgikey = 'SSL_SERVER_%s_DN_%s' % (prefix, key) diff --git a/cherrypy/wsgiserver/test_wsgiserver.py b/cherrypy/wsgiserver/test_wsgiserver.py new file mode 100644 index 0000000..72c08ba --- /dev/null +++ b/cherrypy/wsgiserver/test_wsgiserver.py @@ -0,0 +1,16 @@ +import six + +import mock + +from cherrypy import wsgiserver + + +class TestWSGIGateway_u0: + @mock.patch('cherrypy.wsgiserver.WSGIGateway_10.get_environ', + lambda self: {'foo': 'bar'}) + def test_decodes_items(self): + req = mock.MagicMock(path=b'/', qs=b'') + gw = wsgiserver.WSGIGateway_u0(req=req) + env = gw.get_environ() + assert env['foo'] == 'bar' + assert isinstance(env['foo'], six.text_type) diff --git a/cherrypy/wsgiserver/wsgiserver2.py b/cherrypy/wsgiserver/wsgiserver2.py deleted file mode 100644 index 1492d0e..0000000 --- a/cherrypy/wsgiserver/wsgiserver2.py +++ /dev/null @@ -1,2483 +0,0 @@ -"""A high-speed, production ready, thread pooled, generic HTTP server. - -Simplest example on how to use this module directly -(without using CherryPy's application machinery):: - - from cherrypy import wsgiserver - - def my_crazy_app(environ, start_response): - status = '200 OK' - response_headers = [('Content-type','text/plain')] - start_response(status, response_headers) - return ['Hello world!'] - - server = wsgiserver.CherryPyWSGIServer( - ('0.0.0.0', 8070), my_crazy_app, - server_name='www.cherrypy.example') - server.start() - -The CherryPy WSGI server can serve as many WSGI applications -as you want in one instance by using a WSGIPathInfoDispatcher:: - - d = WSGIPathInfoDispatcher({'/': my_crazy_app, '/blog': my_blog_app}) - server = wsgiserver.CherryPyWSGIServer(('0.0.0.0', 80), d) - -Want SSL support? Just set server.ssl_adapter to an SSLAdapter instance. - -This won't call the CherryPy engine (application side) at all, only the -HTTP server, which is independent from the rest of CherryPy. Don't -let the name "CherryPyWSGIServer" throw you; the name merely reflects -its origin, not its coupling. - -For those of you wanting to understand internals of this module, here's the -basic call flow. The server's listening thread runs a very tight loop, -sticking incoming connections onto a Queue:: - - server = CherryPyWSGIServer(...) - server.start() - while True: - tick() - # This blocks until a request comes in: - child = socket.accept() - conn = HTTPConnection(child, ...) - server.requests.put(conn) - -Worker threads are kept in a pool and poll the Queue, popping off and then -handling each connection in turn. Each connection can consist of an arbitrary -number of requests and their responses, so we run a nested loop:: - - while True: - conn = server.requests.get() - conn.communicate() - -> while True: - req = HTTPRequest(...) - req.parse_request() - -> # Read the Request-Line, e.g. "GET /page HTTP/1.1" - req.rfile.readline() - read_headers(req.rfile, req.inheaders) - req.respond() - -> response = app(...) - try: - for chunk in response: - if chunk: - req.write(chunk) - finally: - if hasattr(response, "close"): - response.close() - if req.close_connection: - return -""" - -__all__ = ['HTTPRequest', 'HTTPConnection', 'HTTPServer', - 'SizeCheckWrapper', 'KnownLengthRFile', 'ChunkedRFile', - 'CP_fileobject', - 'MaxSizeExceeded', 'NoSSLError', 'FatalSSLAlert', - 'WorkerThread', 'ThreadPool', 'SSLAdapter', - 'CherryPyWSGIServer', - 'Gateway', 'WSGIGateway', 'WSGIGateway_10', 'WSGIGateway_u0', - 'WSGIPathInfoDispatcher', 'get_ssl_adapter_class', - 'socket_errors_to_ignore', 'redirect_url'] - -import os -try: - import queue -except: - import Queue as queue -import re -import email.utils -import socket -import sys -import threading -import time -import traceback as traceback_ -import operator -from urllib import unquote -from urlparse import urlparse -import warnings -import errno -import logging -try: - # prefer slower Python-based io module - import _pyio as io -except ImportError: - # Python 2.6 - import io - -try: - import pkg_resources -except ImportError: - pass - -if 'win' in sys.platform and hasattr(socket, "AF_INET6"): - if not hasattr(socket, 'IPPROTO_IPV6'): - socket.IPPROTO_IPV6 = 41 - if not hasattr(socket, 'IPV6_V6ONLY'): - socket.IPV6_V6ONLY = 27 - - -DEFAULT_BUFFER_SIZE = io.DEFAULT_BUFFER_SIZE - -REDIRECT_URL = None # Application can write its HTTP-->HTTPS redirection URL here - -try: - cp_version = pkg_resources.require('cherrypy')[0].version -except Exception: - cp_version = 'unknown' - - -class FauxSocket(object): - - """Faux socket with the minimal interface required by pypy""" - - def _reuse(self): - pass - -_fileobject_uses_str_type = isinstance( - socket._fileobject(FauxSocket())._rbuf, basestring) -del FauxSocket # this class is not longer required for anything. - - -if sys.version_info >= (3, 0): - unicodestr = str - basestring = (bytes, str) - - def ntob(n, encoding='ISO-8859-1'): - """Return the given native string as a byte string in the given - encoding. - """ - # In Python 3, the native string type is unicode - return n.encode(encoding) -else: - unicodestr = unicode - basestring = basestring - - def ntob(n, encoding='ISO-8859-1'): - """Return the given native string as a byte string in the given - encoding. - """ - # In Python 2, the native string type is bytes. Assume it's already - # in the given encoding, which for ISO-8859-1 is almost always what - # was intended. - return n - -LF = ntob('\n') -CRLF = ntob('\r\n') -TAB = ntob('\t') -SPACE = ntob(' ') -COLON = ntob(':') -SEMICOLON = ntob(';') -EMPTY = ntob('') -NUMBER_SIGN = ntob('#') -QUESTION_MARK = ntob('?') -ASTERISK = ntob('*') -FORWARD_SLASH = ntob('/') -quoted_slash = re.compile(ntob("(?i)%2F")) - - -def redirect_url(url=None): - global REDIRECT_URL - if url and '%s' in url: - REDIRECT_URL = url - return REDIRECT_URL - - -def plat_specific_errors(*errnames): - """Return error numbers for all errors in errnames on this platform. - - The 'errno' module contains different global constants depending on - the specific platform (OS). This function will return the list of - numeric values for a given list of potential names. - """ - errno_names = dir(errno) - nums = [getattr(errno, k) for k in errnames if k in errno_names] - # de-dupe the list - return list(dict.fromkeys(nums).keys()) - -socket_error_eintr = plat_specific_errors("EINTR", "WSAEINTR") - -socket_errors_to_ignore = plat_specific_errors( - "EPIPE", - "EBADF", "WSAEBADF", - "ENOTSOCK", "WSAENOTSOCK", - "ETIMEDOUT", "WSAETIMEDOUT", - "ECONNREFUSED", "WSAECONNREFUSED", - "ECONNRESET", "WSAECONNRESET", - "ECONNABORTED", "WSAECONNABORTED", - "ENETRESET", "WSAENETRESET", - "EHOSTDOWN", "EHOSTUNREACH", -) -socket_errors_to_ignore.append("timed out") -socket_errors_to_ignore.append("The read operation timed out") -if sys.platform == 'darwin': - socket_errors_to_ignore.append(plat_specific_errors("EPROTOTYPE")) - -socket_errors_nonblocking = plat_specific_errors( - 'EAGAIN', 'EWOULDBLOCK', 'WSAEWOULDBLOCK') - -comma_separated_headers = [ - ntob(h) for h in - ['Accept', 'Accept-Charset', 'Accept-Encoding', - 'Accept-Language', 'Accept-Ranges', 'Allow', 'Cache-Control', - 'Connection', 'Content-Encoding', 'Content-Language', 'Expect', - 'If-Match', 'If-None-Match', 'Pragma', 'Proxy-Authenticate', 'TE', - 'Trailer', 'Transfer-Encoding', 'Upgrade', 'Vary', 'Via', 'Warning', - 'WWW-Authenticate'] -] - - -if not hasattr(logging, 'statistics'): - logging.statistics = {} - - -def read_headers(rfile, hdict=None): - """Read headers from the given stream into the given header dict. - - If hdict is None, a new header dict is created. Returns the populated - header dict. - - Headers which are repeated are folded together using a comma if their - specification so dictates. - - This function raises ValueError when the read bytes violate the HTTP spec. - You should probably return "400 Bad Request" if this happens. - """ - if hdict is None: - hdict = {} - - while True: - line = rfile.readline() - if not line: - # No more data--illegal end of headers - raise ValueError("Illegal end of headers.") - - if line == CRLF: - # Normal end of headers - break - if not line.endswith(CRLF): - raise ValueError("HTTP requires CRLF terminators") - - if line[0] in (SPACE, TAB): - # It's a continuation line. - v = line.strip() - else: - try: - k, v = line.split(COLON, 1) - except ValueError: - raise ValueError("Illegal header line.") - # TODO: what about TE and WWW-Authenticate? - k = k.strip().title() - v = v.strip() - hname = k - - if k in comma_separated_headers: - existing = hdict.get(hname) - if existing: - v = ", ".join((existing, v)) - hdict[hname] = v - - return hdict - - -class MaxSizeExceeded(Exception): - pass - - -class SizeCheckWrapper(object): - - """Wraps a file-like object, raising MaxSizeExceeded if too large.""" - - def __init__(self, rfile, maxlen): - self.rfile = rfile - self.maxlen = maxlen - self.bytes_read = 0 - - def _check_length(self): - if self.maxlen and self.bytes_read > self.maxlen: - raise MaxSizeExceeded() - - def read(self, size=None): - data = self.rfile.read(size) - self.bytes_read += len(data) - self._check_length() - return data - - def readline(self, size=None): - if size is not None: - data = self.rfile.readline(size) - self.bytes_read += len(data) - self._check_length() - return data - - # User didn't specify a size ... - # We read the line in chunks to make sure it's not a 100MB line ! - res = [] - while True: - data = self.rfile.readline(256) - self.bytes_read += len(data) - self._check_length() - res.append(data) - # See https://github.com/cherrypy/cherrypy/issues/421 - if len(data) < 256 or data[-1:] == LF: - return EMPTY.join(res) - - def readlines(self, sizehint=0): - # Shamelessly stolen from StringIO - total = 0 - lines = [] - line = self.readline() - while line: - lines.append(line) - total += len(line) - if 0 < sizehint <= total: - break - line = self.readline() - return lines - - def close(self): - self.rfile.close() - - def __iter__(self): - return self - - def __next__(self): - data = next(self.rfile) - self.bytes_read += len(data) - self._check_length() - return data - - def next(self): - data = self.rfile.next() - self.bytes_read += len(data) - self._check_length() - return data - - -class KnownLengthRFile(object): - - """Wraps a file-like object, returning an empty string when exhausted.""" - - def __init__(self, rfile, content_length): - self.rfile = rfile - self.remaining = content_length - - def read(self, size=None): - if self.remaining == 0: - return '' - if size is None: - size = self.remaining - else: - size = min(size, self.remaining) - - data = self.rfile.read(size) - self.remaining -= len(data) - return data - - def readline(self, size=None): - if self.remaining == 0: - return '' - if size is None: - size = self.remaining - else: - size = min(size, self.remaining) - - data = self.rfile.readline(size) - self.remaining -= len(data) - return data - - def readlines(self, sizehint=0): - # Shamelessly stolen from StringIO - total = 0 - lines = [] - line = self.readline(sizehint) - while line: - lines.append(line) - total += len(line) - if 0 < sizehint <= total: - break - line = self.readline(sizehint) - return lines - - def close(self): - self.rfile.close() - - def __iter__(self): - return self - - def __next__(self): - data = next(self.rfile) - self.remaining -= len(data) - return data - - -class ChunkedRFile(object): - - """Wraps a file-like object, returning an empty string when exhausted. - - This class is intended to provide a conforming wsgi.input value for - request entities that have been encoded with the 'chunked' transfer - encoding. - """ - - def __init__(self, rfile, maxlen, bufsize=8192): - self.rfile = rfile - self.maxlen = maxlen - self.bytes_read = 0 - self.buffer = EMPTY - self.bufsize = bufsize - self.closed = False - - def _fetch(self): - if self.closed: - return - - line = self.rfile.readline() - self.bytes_read += len(line) - - if self.maxlen and self.bytes_read > self.maxlen: - raise MaxSizeExceeded("Request Entity Too Large", self.maxlen) - - line = line.strip().split(SEMICOLON, 1) - - try: - chunk_size = line.pop(0) - chunk_size = int(chunk_size, 16) - except ValueError: - raise ValueError("Bad chunked transfer size: " + repr(chunk_size)) - - if chunk_size <= 0: - self.closed = True - return - -## if line: chunk_extension = line[0] - - if self.maxlen and self.bytes_read + chunk_size > self.maxlen: - raise IOError("Request Entity Too Large") - - chunk = self.rfile.read(chunk_size) - self.bytes_read += len(chunk) - self.buffer += chunk - - crlf = self.rfile.read(2) - if crlf != CRLF: - raise ValueError( - "Bad chunked transfer coding (expected '\\r\\n', " - "got " + repr(crlf) + ")") - - def read(self, size=None): - data = EMPTY - while True: - if size and len(data) >= size: - return data - - if not self.buffer: - self._fetch() - if not self.buffer: - # EOF - return data - - if size: - remaining = size - len(data) - data += self.buffer[:remaining] - self.buffer = self.buffer[remaining:] - else: - data += self.buffer - - def readline(self, size=None): - data = EMPTY - while True: - if size and len(data) >= size: - return data - - if not self.buffer: - self._fetch() - if not self.buffer: - # EOF - return data - - newline_pos = self.buffer.find(LF) - if size: - if newline_pos == -1: - remaining = size - len(data) - data += self.buffer[:remaining] - self.buffer = self.buffer[remaining:] - else: - remaining = min(size - len(data), newline_pos) - data += self.buffer[:remaining] - self.buffer = self.buffer[remaining:] - else: - if newline_pos == -1: - data += self.buffer - else: - data += self.buffer[:newline_pos] - self.buffer = self.buffer[newline_pos:] - - def readlines(self, sizehint=0): - # Shamelessly stolen from StringIO - total = 0 - lines = [] - line = self.readline(sizehint) - while line: - lines.append(line) - total += len(line) - if 0 < sizehint <= total: - break - line = self.readline(sizehint) - return lines - - def read_trailer_lines(self): - if not self.closed: - raise ValueError( - "Cannot read trailers until the request body has been read.") - - while True: - line = self.rfile.readline() - if not line: - # No more data--illegal end of headers - raise ValueError("Illegal end of headers.") - - self.bytes_read += len(line) - if self.maxlen and self.bytes_read > self.maxlen: - raise IOError("Request Entity Too Large") - - if line == CRLF: - # Normal end of headers - break - if not line.endswith(CRLF): - raise ValueError("HTTP requires CRLF terminators") - - yield line - - def close(self): - self.rfile.close() - - def __iter__(self): - # Shamelessly stolen from StringIO - total = 0 - line = self.readline(sizehint) - while line: - yield line - total += len(line) - if 0 < sizehint <= total: - break - line = self.readline(sizehint) - - -class HTTPRequest(object): - - """An HTTP Request (and response). - - A single HTTP connection may consist of multiple request/response pairs. - """ - - server = None - """The HTTPServer object which is receiving this request.""" - - conn = None - """The HTTPConnection object on which this request connected.""" - - inheaders = {} - """A dict of request headers.""" - - outheaders = [] - """A list of header tuples to write in the response.""" - - ready = False - """When True, the request has been parsed and is ready to begin generating - the response. When False, signals the calling Connection that the response - should not be generated and the connection should close.""" - - close_connection = False - """Signals the calling Connection that the request should close. This does - not imply an error! The client and/or server may each request that the - connection be closed.""" - - chunked_write = False - """If True, output will be encoded with the "chunked" transfer-coding. - - This value is set automatically inside send_headers.""" - - def __init__(self, server, conn): - self.server = server - self.conn = conn - - self.ready = False - self.started_request = False - self.scheme = ntob("http") - if self.server.ssl_adapter is not None: - self.scheme = ntob("https") - # Use the lowest-common protocol in case read_request_line errors. - self.response_protocol = 'HTTP/1.0' - self.inheaders = {} - - self.status = "" - self.outheaders = [] - self.sent_headers = False - self.close_connection = self.__class__.close_connection - self.chunked_read = False - self.chunked_write = self.__class__.chunked_write - - def parse_request(self): - """Parse the next HTTP request start-line and message-headers.""" - self.rfile = SizeCheckWrapper(self.conn.rfile, - self.server.max_request_header_size) - try: - success = self.read_request_line() - except MaxSizeExceeded: - self.simple_response( - "414 Request-URI Too Long", - "The Request-URI sent with the request exceeds the maximum " - "allowed bytes.") - return - else: - if not success: - return - - try: - success = self.read_request_headers() - except MaxSizeExceeded: - self.simple_response( - "413 Request Entity Too Large", - "The headers sent with the request exceed the maximum " - "allowed bytes.") - return - else: - if not success: - return - - self.ready = True - - def read_request_line(self): - # HTTP/1.1 connections are persistent by default. If a client - # requests a page, then idles (leaves the connection open), - # then rfile.readline() will raise socket.error("timed out"). - # Note that it does this based on the value given to settimeout(), - # and doesn't need the client to request or acknowledge the close - # (although your TCP stack might suffer for it: cf Apache's history - # with FIN_WAIT_2). - request_line = self.rfile.readline() - - # Set started_request to True so communicate() knows to send 408 - # from here on out. - self.started_request = True - if not request_line: - return False - - if request_line == CRLF: - # RFC 2616 sec 4.1: "...if the server is reading the protocol - # stream at the beginning of a message and receives a CRLF - # first, it should ignore the CRLF." - # But only ignore one leading line! else we enable a DoS. - request_line = self.rfile.readline() - if not request_line: - return False - - if not request_line.endswith(CRLF): - self.simple_response( - "400 Bad Request", "HTTP requires CRLF terminators") - return False - - try: - method, uri, req_protocol = request_line.strip().split(SPACE, 2) - rp = int(req_protocol[5]), int(req_protocol[7]) - except (ValueError, IndexError): - self.simple_response("400 Bad Request", "Malformed Request-Line") - return False - - self.uri = uri - self.method = method - - # uri may be an abs_path (including "http://host.domain.tld"); - scheme, authority, path = self.parse_request_uri(uri) - if path is None: - self.simple_response("400 Bad Request", - "Invalid path in Request-URI.") - return False - if path and NUMBER_SIGN in path: - self.simple_response("400 Bad Request", - "Illegal #fragment in Request-URI.") - return False - - if scheme: - self.scheme = scheme - - qs = EMPTY - if path and QUESTION_MARK in path: - path, qs = path.split(QUESTION_MARK, 1) - - # Unquote the path+params (e.g. "/this%20path" -> "/this path"). - # http://www.w3.org/Protocols/rfc2616/rfc2616-sec5.html#sec5.1.2 - # - # But note that "...a URI must be separated into its components - # before the escaped characters within those components can be - # safely decoded." http://www.ietf.org/rfc/rfc2396.txt, sec 2.4.2 - # Therefore, "/this%2Fpath" becomes "/this%2Fpath", not "/this/path". - try: - atoms = [unquote(x) for x in quoted_slash.split(path)] - except ValueError: - ex = sys.exc_info()[1] - self.simple_response("400 Bad Request", ex.args[0]) - return False - path = "%2F".join(atoms) - self.path = path - - # Note that, like wsgiref and most other HTTP servers, - # we "% HEX HEX"-unquote the path but not the query string. - self.qs = qs - - # Compare request and server HTTP protocol versions, in case our - # server does not support the requested protocol. Limit our output - # to min(req, server). We want the following output: - # request server actual written supported response - # protocol protocol response protocol feature set - # a 1.0 1.0 1.0 1.0 - # b 1.0 1.1 1.1 1.0 - # c 1.1 1.0 1.0 1.0 - # d 1.1 1.1 1.1 1.1 - # Notice that, in (b), the response will be "HTTP/1.1" even though - # the client only understands 1.0. RFC 2616 10.5.6 says we should - # only return 505 if the _major_ version is different. - sp = int(self.server.protocol[5]), int(self.server.protocol[7]) - - if sp[0] != rp[0]: - self.simple_response("505 HTTP Version Not Supported") - return False - - self.request_protocol = req_protocol - self.response_protocol = "HTTP/%s.%s" % min(rp, sp) - - return True - - def read_request_headers(self): - """Read self.rfile into self.inheaders. Return success.""" - - # then all the http headers - try: - read_headers(self.rfile, self.inheaders) - except ValueError: - ex = sys.exc_info()[1] - self.simple_response("400 Bad Request", ex.args[0]) - return False - - mrbs = self.server.max_request_body_size - if mrbs and int(self.inheaders.get("Content-Length", 0)) > mrbs: - self.simple_response( - "413 Request Entity Too Large", - "The entity sent with the request exceeds the maximum " - "allowed bytes.") - return False - - # Persistent connection support - if self.response_protocol == "HTTP/1.1": - # Both server and client are HTTP/1.1 - if self.inheaders.get("Connection", "") == "close": - self.close_connection = True - else: - # Either the server or client (or both) are HTTP/1.0 - if self.inheaders.get("Connection", "") != "Keep-Alive": - self.close_connection = True - - # Transfer-Encoding support - te = None - if self.response_protocol == "HTTP/1.1": - te = self.inheaders.get("Transfer-Encoding") - if te: - te = [x.strip().lower() for x in te.split(",") if x.strip()] - - self.chunked_read = False - - if te: - for enc in te: - if enc == "chunked": - self.chunked_read = True - else: - # Note that, even if we see "chunked", we must reject - # if there is an extension we don't recognize. - self.simple_response("501 Unimplemented") - self.close_connection = True - return False - - # From PEP 333: - # "Servers and gateways that implement HTTP 1.1 must provide - # transparent support for HTTP 1.1's "expect/continue" mechanism. - # This may be done in any of several ways: - # 1. Respond to requests containing an Expect: 100-continue request - # with an immediate "100 Continue" response, and proceed normally. - # 2. Proceed with the request normally, but provide the application - # with a wsgi.input stream that will send the "100 Continue" - # response if/when the application first attempts to read from - # the input stream. The read request must then remain blocked - # until the client responds. - # 3. Wait until the client decides that the server does not support - # expect/continue, and sends the request body on its own. - # (This is suboptimal, and is not recommended.) - # - # We used to do 3, but are now doing 1. Maybe we'll do 2 someday, - # but it seems like it would be a big slowdown for such a rare case. - if self.inheaders.get("Expect", "") == "100-continue": - # Don't use simple_response here, because it emits headers - # we don't want. See - # https://github.com/cherrypy/cherrypy/issues/951 - msg = self.server.protocol + " 100 Continue\r\n\r\n" - try: - self.conn.wfile.sendall(msg) - except socket.error: - x = sys.exc_info()[1] - if x.args[0] not in socket_errors_to_ignore: - raise - return True - - def parse_request_uri(self, uri): - """Parse a Request-URI into (scheme, authority, path). - - Note that Request-URI's must be one of:: - - Request-URI = "*" | absoluteURI | abs_path | authority - - Therefore, a Request-URI which starts with a double forward-slash - cannot be a "net_path":: - - net_path = "//" authority [ abs_path ] - - Instead, it must be interpreted as an "abs_path" with an empty first - path segment:: - - abs_path = "/" path_segments - path_segments = segment *( "/" segment ) - segment = *pchar *( ";" param ) - param = *pchar - """ - if uri == ASTERISK: - return None, None, uri - - scheme, authority, path, params, query, fragment = urlparse(uri) - if scheme and QUESTION_MARK not in scheme: - # An absoluteURI. - # If there's a scheme (and it must be http or https), then: - # http_URL = "http:" "//" host [ ":" port ] [ abs_path [ "?" query - # ]] - return scheme, authority, path - - if uri.startswith(FORWARD_SLASH): - # An abs_path. - return None, None, uri - else: - # An authority. - return None, uri, None - - def respond(self): - """Call the gateway and write its iterable output.""" - mrbs = self.server.max_request_body_size - if self.chunked_read: - self.rfile = ChunkedRFile(self.conn.rfile, mrbs) - else: - cl = int(self.inheaders.get("Content-Length", 0)) - if mrbs and mrbs < cl: - if not self.sent_headers: - self.simple_response( - "413 Request Entity Too Large", - "The entity sent with the request exceeds the maximum " - "allowed bytes.") - return - self.rfile = KnownLengthRFile(self.conn.rfile, cl) - - self.server.gateway(self).respond() - - if (self.ready and not self.sent_headers): - self.sent_headers = True - self.send_headers() - if self.chunked_write: - self.conn.wfile.sendall("0\r\n\r\n") - - def simple_response(self, status, msg=""): - """Write a simple response back to the client.""" - status = str(status) - buf = [self.server.protocol + SPACE + - status + CRLF, - "Content-Length: %s\r\n" % len(msg), - "Content-Type: text/html\r\n" if status[:3] == '301' else "Content-Type: text/plain\r\n"] - - if status[:3] in ("413", "414"): - # Request Entity Too Large / Request-URI Too Long - self.close_connection = True - if self.response_protocol == 'HTTP/1.1': - # This will not be true for 414, since read_request_line - # usually raises 414 before reading the whole line, and we - # therefore cannot know the proper response_protocol. - buf.append("Connection: close\r\n") - else: - # HTTP/1.0 had no 413/414 status nor Connection header. - # Emit 400 instead and trust the message body is enough. - status = "400 Bad Request" - - buf.append(CRLF) - if msg: - if isinstance(msg, unicodestr): - msg = msg.encode("ISO-8859-1") - buf.append(msg) - - try: - self.conn.wfile.sendall("".join(buf)) - except socket.error: - x = sys.exc_info()[1] - if x.args[0] not in socket_errors_to_ignore: - raise - - def write(self, chunk): - """Write unbuffered data to the client.""" - if self.chunked_write and chunk: - buf = [hex(len(chunk))[2:], CRLF, chunk, CRLF] - self.conn.wfile.sendall(EMPTY.join(buf)) - else: - self.conn.wfile.sendall(chunk) - - def send_headers(self): - """Assert, process, and send the HTTP response message-headers. - - You must set self.status, and self.outheaders before calling this. - """ - hkeys = [key.lower() for key, value in self.outheaders] - status = int(self.status[:3]) - - if status == 413: - # Request Entity Too Large. Close conn to avoid garbage. - self.close_connection = True - elif "content-length" not in hkeys: - # "All 1xx (informational), 204 (no content), - # and 304 (not modified) responses MUST NOT - # include a message-body." So no point chunking. - if status < 200 or status in (204, 205, 304): - pass - else: - if (self.response_protocol == 'HTTP/1.1' - and self.method != 'HEAD'): - # Use the chunked transfer-coding - self.chunked_write = True - self.outheaders.append(("Transfer-Encoding", "chunked")) - else: - # Closing the conn is the only way to determine len. - self.close_connection = True - - if "connection" not in hkeys: - if self.response_protocol == 'HTTP/1.1': - # Both server and client are HTTP/1.1 or better - if self.close_connection: - self.outheaders.append(("Connection", "close")) - else: - # Server and/or client are HTTP/1.0 - if not self.close_connection: - self.outheaders.append(("Connection", "Keep-Alive")) - - if (not self.close_connection) and (not self.chunked_read): - # Read any remaining request body data on the socket. - # "If an origin server receives a request that does not include an - # Expect request-header field with the "100-continue" expectation, - # the request includes a request body, and the server responds - # with a final status code before reading the entire request body - # from the transport connection, then the server SHOULD NOT close - # the transport connection until it has read the entire request, - # or until the client closes the connection. Otherwise, the client - # might not reliably receive the response message. However, this - # requirement is not be construed as preventing a server from - # defending itself against denial-of-service attacks, or from - # badly broken client implementations." - remaining = getattr(self.rfile, 'remaining', 0) - if remaining > 0: - self.rfile.read(remaining) - - if "date" not in hkeys: - self.outheaders.append(("Date", email.utils.formatdate())) - - if "server" not in hkeys: - self.outheaders.append(("Server", self.server.server_name)) - - buf = [self.server.protocol + SPACE + self.status + CRLF] - for k, v in self.outheaders: - buf.append(k + COLON + SPACE + v + CRLF) - buf.append(CRLF) - self.conn.wfile.sendall(EMPTY.join(buf)) - - -class NoSSLError(Exception): - - """Exception raised when a client speaks HTTP to an HTTPS socket.""" - pass - - -class FatalSSLAlert(Exception): - - """Exception raised when the SSL implementation signals a fatal alert.""" - pass - - -class CP_fileobject(socket._fileobject): - - """Faux file object attached to a socket object.""" - - def __init__(self, *args, **kwargs): - self.bytes_read = 0 - self.bytes_written = 0 - socket._fileobject.__init__(self, *args, **kwargs) - - def sendall(self, data): - """Sendall for non-blocking sockets.""" - while data: - try: - bytes_sent = self.send(data) - data = data[bytes_sent:] - except socket.error, e: - if e.args[0] not in socket_errors_nonblocking: - raise - - def send(self, data): - bytes_sent = self._sock.send(data) - self.bytes_written += bytes_sent - return bytes_sent - - def flush(self): - if self._wbuf: - buffer = "".join(self._wbuf) - self._wbuf = [] - self.sendall(buffer) - - def recv(self, size): - while True: - try: - data = self._sock.recv(size) - self.bytes_read += len(data) - return data - except socket.error, e: - if (e.args[0] not in socket_errors_nonblocking - and e.args[0] not in socket_error_eintr): - raise - - if not _fileobject_uses_str_type: - def read(self, size=-1): - # Use max, disallow tiny reads in a loop as they are very - # inefficient. - # We never leave read() with any leftover data from a new recv() - # call in our internal buffer. - rbufsize = max(self._rbufsize, self.default_bufsize) - # Our use of StringIO rather than lists of string objects returned - # by recv() minimizes memory usage and fragmentation that occurs - # when rbufsize is large compared to the typical return value of - # recv(). - buf = self._rbuf - buf.seek(0, 2) # seek end - if size < 0: - # Read until EOF - # reset _rbuf. we consume it via buf. - self._rbuf = io.BytesIO() - while True: - data = self.recv(rbufsize) - if not data: - break - buf.write(data) - return buf.getvalue() - else: - # Read until size bytes or EOF seen, whichever comes first - buf_len = buf.tell() - if buf_len >= size: - # Already have size bytes in our buffer? Extract and - # return. - buf.seek(0) - rv = buf.read(size) - self._rbuf = io.BytesIO() - self._rbuf.write(buf.read()) - return rv - - # reset _rbuf. we consume it via buf. - self._rbuf = io.BytesIO() - while True: - left = size - buf_len - # recv() will malloc the amount of memory given as its - # parameter even though it often returns much less data - # than that. The returned data string is short lived - # as we copy it into a StringIO and free it. This avoids - # fragmentation issues on many platforms. - data = self.recv(left) - if not data: - break - n = len(data) - if n == size and not buf_len: - # Shortcut. Avoid buffer data copies when: - # - We have no data in our buffer. - # AND - # - Our call to recv returned exactly the - # number of bytes we were asked to read. - return data - if n == left: - buf.write(data) - del data # explicit free - break - assert n <= left, "recv(%d) returned %d bytes" % (left, n) - buf.write(data) - buf_len += n - del data # explicit free - #assert buf_len == buf.tell() - return buf.getvalue() - - def readline(self, size=-1): - buf = self._rbuf - buf.seek(0, 2) # seek end - if buf.tell() > 0: - # check if we already have it in our buffer - buf.seek(0) - bline = buf.readline(size) - if bline.endswith('\n') or len(bline) == size: - self._rbuf = io.BytesIO() - self._rbuf.write(buf.read()) - return bline - del bline - if size < 0: - # Read until \n or EOF, whichever comes first - if self._rbufsize <= 1: - # Speed up unbuffered case - buf.seek(0) - buffers = [buf.read()] - # reset _rbuf. we consume it via buf. - self._rbuf = io.BytesIO() - data = None - recv = self.recv - while data != "\n": - data = recv(1) - if not data: - break - buffers.append(data) - return "".join(buffers) - - buf.seek(0, 2) # seek end - # reset _rbuf. we consume it via buf. - self._rbuf = io.BytesIO() - while True: - data = self.recv(self._rbufsize) - if not data: - break - nl = data.find('\n') - if nl >= 0: - nl += 1 - buf.write(data[:nl]) - self._rbuf.write(data[nl:]) - del data - break - buf.write(data) - return buf.getvalue() - else: - # Read until size bytes or \n or EOF seen, whichever comes - # first - buf.seek(0, 2) # seek end - buf_len = buf.tell() - if buf_len >= size: - buf.seek(0) - rv = buf.read(size) - self._rbuf = io.BytesIO() - self._rbuf.write(buf.read()) - return rv - # reset _rbuf. we consume it via buf. - self._rbuf = io.BytesIO() - while True: - data = self.recv(self._rbufsize) - if not data: - break - left = size - buf_len - # did we just receive a newline? - nl = data.find('\n', 0, left) - if nl >= 0: - nl += 1 - # save the excess data to _rbuf - self._rbuf.write(data[nl:]) - if buf_len: - buf.write(data[:nl]) - break - else: - # Shortcut. Avoid data copy through buf when - # returning a substring of our first recv(). - return data[:nl] - n = len(data) - if n == size and not buf_len: - # Shortcut. Avoid data copy through buf when - # returning exactly all of our first recv(). - return data - if n >= left: - buf.write(data[:left]) - self._rbuf.write(data[left:]) - break - buf.write(data) - buf_len += n - #assert buf_len == buf.tell() - return buf.getvalue() - else: - def read(self, size=-1): - if size < 0: - # Read until EOF - buffers = [self._rbuf] - self._rbuf = "" - if self._rbufsize <= 1: - recv_size = self.default_bufsize - else: - recv_size = self._rbufsize - - while True: - data = self.recv(recv_size) - if not data: - break - buffers.append(data) - return "".join(buffers) - else: - # Read until size bytes or EOF seen, whichever comes first - data = self._rbuf - buf_len = len(data) - if buf_len >= size: - self._rbuf = data[size:] - return data[:size] - buffers = [] - if data: - buffers.append(data) - self._rbuf = "" - while True: - left = size - buf_len - recv_size = max(self._rbufsize, left) - data = self.recv(recv_size) - if not data: - break - buffers.append(data) - n = len(data) - if n >= left: - self._rbuf = data[left:] - buffers[-1] = data[:left] - break - buf_len += n - return "".join(buffers) - - def readline(self, size=-1): - data = self._rbuf - if size < 0: - # Read until \n or EOF, whichever comes first - if self._rbufsize <= 1: - # Speed up unbuffered case - assert data == "" - buffers = [] - while data != "\n": - data = self.recv(1) - if not data: - break - buffers.append(data) - return "".join(buffers) - nl = data.find('\n') - if nl >= 0: - nl += 1 - self._rbuf = data[nl:] - return data[:nl] - buffers = [] - if data: - buffers.append(data) - self._rbuf = "" - while True: - data = self.recv(self._rbufsize) - if not data: - break - buffers.append(data) - nl = data.find('\n') - if nl >= 0: - nl += 1 - self._rbuf = data[nl:] - buffers[-1] = data[:nl] - break - return "".join(buffers) - else: - # Read until size bytes or \n or EOF seen, whichever comes - # first - nl = data.find('\n', 0, size) - if nl >= 0: - nl += 1 - self._rbuf = data[nl:] - return data[:nl] - buf_len = len(data) - if buf_len >= size: - self._rbuf = data[size:] - return data[:size] - buffers = [] - if data: - buffers.append(data) - self._rbuf = "" - while True: - data = self.recv(self._rbufsize) - if not data: - break - buffers.append(data) - left = size - buf_len - nl = data.find('\n', 0, left) - if nl >= 0: - nl += 1 - self._rbuf = data[nl:] - buffers[-1] = data[:nl] - break - n = len(data) - if n >= left: - self._rbuf = data[left:] - buffers[-1] = data[:left] - break - buf_len += n - return "".join(buffers) - - -class HTTPConnection(object): - - """An HTTP connection (active socket). - - server: the Server object which received this connection. - socket: the raw socket object (usually TCP) for this connection. - makefile: a fileobject class for reading from the socket. - """ - - remote_addr = None - remote_port = None - ssl_env = None - rbufsize = DEFAULT_BUFFER_SIZE - wbufsize = DEFAULT_BUFFER_SIZE - RequestHandlerClass = HTTPRequest - - def __init__(self, server, sock, makefile=CP_fileobject): - self.server = server - self.socket = sock - self.rfile = makefile(sock, "rb", self.rbufsize) - self.wfile = makefile(sock, "wb", self.wbufsize) - self.requests_seen = 0 - - def communicate(self): - """Read each request and respond appropriately.""" - request_seen = False - try: - while True: - # (re)set req to None so that if something goes wrong in - # the RequestHandlerClass constructor, the error doesn't - # get written to the previous request. - req = None - req = self.RequestHandlerClass(self.server, self) - - # This order of operations should guarantee correct pipelining. - req.parse_request() - if self.server.stats['Enabled']: - self.requests_seen += 1 - if not req.ready: - # Something went wrong in the parsing (and the server has - # probably already made a simple_response). Return and - # let the conn close. - return - - request_seen = True - req.respond() - if req.close_connection: - return - except socket.error: - e = sys.exc_info()[1] - errnum = e.args[0] - # sadly SSL sockets return a different (longer) time out string - if ( - errnum == 'timed out' or - errnum == 'The read operation timed out' - ): - # Don't error if we're between requests; only error - # if 1) no request has been started at all, or 2) we're - # in the middle of a request. - # See https://github.com/cherrypy/cherrypy/issues/853 - if (not request_seen) or (req and req.started_request): - # Don't bother writing the 408 if the response - # has already started being written. - if req and not req.sent_headers: - try: - req.simple_response("408 Request Timeout") - except FatalSSLAlert: - # Close the connection. - return - elif errnum not in socket_errors_to_ignore: - self.server.error_log("socket.error %s" % repr(errnum), - level=logging.WARNING, traceback=True) - if req and not req.sent_headers: - try: - req.simple_response("500 Internal Server Error") - except FatalSSLAlert: - # Close the connection. - return - return - except (KeyboardInterrupt, SystemExit): - raise - except FatalSSLAlert: - # Close the connection. - return - except NoSSLError: - if req and not req.sent_headers: - # Unwrap our wfile - self.wfile = CP_fileobject( - self.socket._sock, "wb", self.wbufsize) - if REDIRECT_URL: - msg = '' \ - '' \ - '' % (REDIRECT_URL % self.remote_addr) - req.simple_response("301 Moved Permanently", msg) - else: - req.simple_response( - "400 Bad Request", - "The client sent a plain HTTP request, but " - "this server only speaks HTTPS on this port.") - self.linger = True - except Exception: - e = sys.exc_info()[1] - self.server.error_log(repr(e), level=logging.ERROR, traceback=True) - if req and not req.sent_headers: - try: - req.simple_response("500 Internal Server Error") - except FatalSSLAlert: - # Close the connection. - return - - linger = False - - def close(self): - """Close the socket underlying this connection.""" - self.rfile.close() - - if not self.linger: - # Python's socket module does NOT call close on the kernel - # socket when you call socket.close(). We do so manually here - # because we want this server to send a FIN TCP segment - # immediately. Note this must be called *before* calling - # socket.close(), because the latter drops its reference to - # the kernel socket. - if hasattr(self.socket, '_sock'): - self.socket._sock.close() - self.socket.close() - else: - # On the other hand, sometimes we want to hang around for a bit - # to make sure the client has a chance to read our entire - # response. Skipping the close() calls here delays the FIN - # packet until the socket object is garbage-collected later. - # Someday, perhaps, we'll do the full lingering_close that - # Apache does, but not today. - pass - - -class TrueyZero(object): - - """An object which equals and does math like the integer 0 but evals True. - """ - - def __add__(self, other): - return other - - def __radd__(self, other): - return other -trueyzero = TrueyZero() - - -_SHUTDOWNREQUEST = None - - -class WorkerThread(threading.Thread): - - """Thread which continuously polls a Queue for Connection objects. - - Due to the timing issues of polling a Queue, a WorkerThread does not - check its own 'ready' flag after it has started. To stop the thread, - it is necessary to stick a _SHUTDOWNREQUEST object onto the Queue - (one for each running WorkerThread). - """ - - conn = None - """The current connection pulled off the Queue, or None.""" - - server = None - """The HTTP Server which spawned this thread, and which owns the - Queue and is placing active connections into it.""" - - ready = False - """A simple flag for the calling server to know when this thread - has begun polling the Queue.""" - - def __init__(self, server): - self.ready = False - self.server = server - - self.requests_seen = 0 - self.bytes_read = 0 - self.bytes_written = 0 - self.start_time = None - self.work_time = 0 - self.stats = { - 'Requests': lambda s: self.requests_seen + ( - (self.start_time is None) and - trueyzero or - self.conn.requests_seen - ), - 'Bytes Read': lambda s: self.bytes_read + ( - (self.start_time is None) and - trueyzero or - self.conn.rfile.bytes_read - ), - 'Bytes Written': lambda s: self.bytes_written + ( - (self.start_time is None) and - trueyzero or - self.conn.wfile.bytes_written - ), - 'Work Time': lambda s: self.work_time + ( - (self.start_time is None) and - trueyzero or - time.time() - self.start_time - ), - 'Read Throughput': lambda s: s['Bytes Read'](s) / ( - s['Work Time'](s) or 1e-6), - 'Write Throughput': lambda s: s['Bytes Written'](s) / ( - s['Work Time'](s) or 1e-6), - } - threading.Thread.__init__(self) - - def run(self): - self.server.stats['Worker Threads'][self.getName()] = self.stats - try: - self.ready = True - while True: - conn = self.server.requests.get() - if conn is _SHUTDOWNREQUEST: - return - - self.conn = conn - if self.server.stats['Enabled']: - self.start_time = time.time() - try: - conn.communicate() - finally: - conn.close() - if self.server.stats['Enabled']: - self.requests_seen += self.conn.requests_seen - self.bytes_read += self.conn.rfile.bytes_read - self.bytes_written += self.conn.wfile.bytes_written - self.work_time += time.time() - self.start_time - self.start_time = None - self.conn = None - except (KeyboardInterrupt, SystemExit): - exc = sys.exc_info()[1] - self.server.interrupt = exc - - -class ThreadPool(object): - - """A Request Queue for an HTTPServer which pools threads. - - ThreadPool objects must provide min, get(), put(obj), start() - and stop(timeout) attributes. - """ - - def __init__(self, server, min=10, max=-1, - accepted_queue_size=-1, accepted_queue_timeout=10): - self.server = server - self.min = min - self.max = max - self._threads = [] - self._queue = queue.Queue(maxsize=accepted_queue_size) - self._queue_put_timeout = accepted_queue_timeout - self.get = self._queue.get - - def start(self): - """Start the pool of threads.""" - for i in range(self.min): - self._threads.append(WorkerThread(self.server)) - for worker in self._threads: - worker.setName("CP Server " + worker.getName()) - worker.start() - for worker in self._threads: - while not worker.ready: - time.sleep(.1) - - def _get_idle(self): - """Number of worker threads which are idle. Read-only.""" - return len([t for t in self._threads if t.conn is None]) - idle = property(_get_idle, doc=_get_idle.__doc__) - - def put(self, obj): - self._queue.put(obj, block=True, timeout=self._queue_put_timeout) - if obj is _SHUTDOWNREQUEST: - return - - def grow(self, amount): - """Spawn new worker threads (not above self.max).""" - if self.max > 0: - budget = max(self.max - len(self._threads), 0) - else: - # self.max <= 0 indicates no maximum - budget = float('inf') - - n_new = min(amount, budget) - - workers = [self._spawn_worker() for i in range(n_new)] - while not self._all(operator.attrgetter('ready'), workers): - time.sleep(.1) - self._threads.extend(workers) - - def _spawn_worker(self): - worker = WorkerThread(self.server) - worker.setName("CP Server " + worker.getName()) - worker.start() - return worker - - def _all(func, items): - results = [func(item) for item in items] - return reduce(operator.and_, results, True) - _all = staticmethod(_all) - - def shrink(self, amount): - """Kill off worker threads (not below self.min).""" - # Grow/shrink the pool if necessary. - # Remove any dead threads from our list - for t in self._threads: - if not t.isAlive(): - self._threads.remove(t) - amount -= 1 - - # calculate the number of threads above the minimum - n_extra = max(len(self._threads) - self.min, 0) - - # don't remove more than amount - n_to_remove = min(amount, n_extra) - - # put shutdown requests on the queue equal to the number of threads - # to remove. As each request is processed by a worker, that worker - # will terminate and be culled from the list. - for n in range(n_to_remove): - self._queue.put(_SHUTDOWNREQUEST) - - def stop(self, timeout=5): - # Must shut down threads here so the code that calls - # this method can know when all threads are stopped. - for worker in self._threads: - self._queue.put(_SHUTDOWNREQUEST) - - # Don't join currentThread (when stop is called inside a request). - current = threading.currentThread() - if timeout and timeout >= 0: - endtime = time.time() + timeout - while self._threads: - worker = self._threads.pop() - if worker is not current and worker.isAlive(): - try: - if timeout is None or timeout < 0: - worker.join() - else: - remaining_time = endtime - time.time() - if remaining_time > 0: - worker.join(remaining_time) - if worker.isAlive(): - # We exhausted the timeout. - # Forcibly shut down the socket. - c = worker.conn - if c and not c.rfile.closed: - try: - c.socket.shutdown(socket.SHUT_RD) - except TypeError: - # pyOpenSSL sockets don't take an arg - c.socket.shutdown() - worker.join() - except (AssertionError, - # Ignore repeated Ctrl-C. - # See - # https://github.com/cherrypy/cherrypy/issues/691. - KeyboardInterrupt): - pass - - def _get_qsize(self): - return self._queue.qsize() - qsize = property(_get_qsize) - - -try: - import fcntl -except ImportError: - try: - from ctypes import windll, WinError - import ctypes.wintypes - _SetHandleInformation = windll.kernel32.SetHandleInformation - _SetHandleInformation.argtypes = [ - ctypes.wintypes.HANDLE, - ctypes.wintypes.DWORD, - ctypes.wintypes.DWORD, - ] - _SetHandleInformation.restype = ctypes.wintypes.BOOL - except ImportError: - def prevent_socket_inheritance(sock): - """Dummy function, since neither fcntl nor ctypes are available.""" - pass - else: - def prevent_socket_inheritance(sock): - """Mark the given socket fd as non-inheritable (Windows).""" - if not _SetHandleInformation(sock.fileno(), 1, 0): - raise WinError() -else: - def prevent_socket_inheritance(sock): - """Mark the given socket fd as non-inheritable (POSIX).""" - fd = sock.fileno() - old_flags = fcntl.fcntl(fd, fcntl.F_GETFD) - fcntl.fcntl(fd, fcntl.F_SETFD, old_flags | fcntl.FD_CLOEXEC) - - -class SSLAdapter(object): - - """Base class for SSL driver library adapters. - - Required methods: - - * ``wrap(sock) -> (wrapped socket, ssl environ dict)`` - * ``makefile(sock, mode='r', bufsize=DEFAULT_BUFFER_SIZE) -> - socket file object`` - """ - - def __init__(self, certificate, private_key, certificate_chain=None): - self.certificate = certificate - self.private_key = private_key - self.certificate_chain = certificate_chain - - def wrap(self, sock): - raise NotImplemented - - def makefile(self, sock, mode='r', bufsize=DEFAULT_BUFFER_SIZE): - raise NotImplemented - - -class HTTPServer(object): - - """An HTTP server.""" - - _bind_addr = "127.0.0.1" - _interrupt = None - - gateway = None - """A Gateway instance.""" - - minthreads = None - """The minimum number of worker threads to create (default 10).""" - - maxthreads = None - """The maximum number of worker threads to create (default -1 = no limit). - """ - - server_name = None - """The name of the server; defaults to socket.gethostname().""" - - protocol = "HTTP/1.1" - """The version string to write in the Status-Line of all HTTP responses. - - For example, "HTTP/1.1" is the default. This also limits the supported - features used in the response.""" - - request_queue_size = 5 - """The 'backlog' arg to socket.listen(); max queued connections - (default 5). - """ - - shutdown_timeout = 5 - """The total time, in seconds, to wait for worker threads to cleanly exit. - """ - - timeout = 10 - """The timeout in seconds for accepted connections (default 10).""" - - version = "CherryPy/" + cp_version - """A version string for the HTTPServer.""" - - software = None - """The value to set for the SERVER_SOFTWARE entry in the WSGI environ. - - If None, this defaults to ``'%s Server' % self.version``.""" - - ready = False - """An internal flag which marks whether the socket is accepting connections - """ - - max_request_header_size = 0 - """The maximum size, in bytes, for request headers, or 0 for no limit.""" - - max_request_body_size = 0 - """The maximum size, in bytes, for request bodies, or 0 for no limit.""" - - nodelay = True - """If True (the default since 3.1), sets the TCP_NODELAY socket option.""" - - ConnectionClass = HTTPConnection - """The class to use for handling HTTP connections.""" - - ssl_adapter = None - """An instance of SSLAdapter (or a subclass). - - You must have the corresponding SSL driver library installed.""" - - def __init__(self, bind_addr, gateway, minthreads=10, maxthreads=-1, - server_name=None): - self.bind_addr = bind_addr - self.gateway = gateway - - self.requests = ThreadPool(self, min=minthreads or 1, max=maxthreads) - - if not server_name: - server_name = socket.gethostname() - self.server_name = server_name - self.clear_stats() - - def clear_stats(self): - self._start_time = None - self._run_time = 0 - self.stats = { - 'Enabled': False, - 'Bind Address': lambda s: repr(self.bind_addr), - 'Run time': lambda s: (not s['Enabled']) and -1 or self.runtime(), - 'Accepts': 0, - 'Accepts/sec': lambda s: s['Accepts'] / self.runtime(), - 'Queue': lambda s: getattr(self.requests, "qsize", None), - 'Threads': lambda s: len(getattr(self.requests, "_threads", [])), - 'Threads Idle': lambda s: getattr(self.requests, "idle", None), - 'Socket Errors': 0, - 'Requests': lambda s: (not s['Enabled']) and -1 or sum( - [w['Requests'](w) for w in s['Worker Threads'].values()], 0), - 'Bytes Read': lambda s: (not s['Enabled']) and -1 or sum( - [w['Bytes Read'](w) for w in s['Worker Threads'].values()], 0), - 'Bytes Written': lambda s: (not s['Enabled']) and -1 or sum( - [w['Bytes Written'](w) for w in s['Worker Threads'].values()], - 0), - 'Work Time': lambda s: (not s['Enabled']) and -1 or sum( - [w['Work Time'](w) for w in s['Worker Threads'].values()], 0), - 'Read Throughput': lambda s: (not s['Enabled']) and -1 or sum( - [w['Bytes Read'](w) / (w['Work Time'](w) or 1e-6) - for w in s['Worker Threads'].values()], 0), - 'Write Throughput': lambda s: (not s['Enabled']) and -1 or sum( - [w['Bytes Written'](w) / (w['Work Time'](w) or 1e-6) - for w in s['Worker Threads'].values()], 0), - 'Worker Threads': {}, - } - logging.statistics["CherryPy HTTPServer %d" % id(self)] = self.stats - - def runtime(self): - if self._start_time is None: - return self._run_time - else: - return self._run_time + (time.time() - self._start_time) - - def __str__(self): - return "%s.%s(%r)" % (self.__module__, self.__class__.__name__, - self.bind_addr) - - def _get_bind_addr(self): - return self._bind_addr - - def _set_bind_addr(self, value): - if isinstance(value, tuple) and value[0] in ('', None): - # Despite the socket module docs, using '' does not - # allow AI_PASSIVE to work. Passing None instead - # returns '0.0.0.0' like we want. In other words: - # host AI_PASSIVE result - # '' Y 192.168.x.y - # '' N 192.168.x.y - # None Y 0.0.0.0 - # None N 127.0.0.1 - # But since you can get the same effect with an explicit - # '0.0.0.0', we deny both the empty string and None as values. - raise ValueError("Host values of '' or None are not allowed. " - "Use '0.0.0.0' (IPv4) or '::' (IPv6) instead " - "to listen on all active interfaces.") - self._bind_addr = value - bind_addr = property( - _get_bind_addr, - _set_bind_addr, - doc="""The interface on which to listen for connections. - - For TCP sockets, a (host, port) tuple. Host values may be any IPv4 - or IPv6 address, or any valid hostname. The string 'localhost' is a - synonym for '127.0.0.1' (or '::1', if your hosts file prefers IPv6). - The string '0.0.0.0' is a special IPv4 entry meaning "any active - interface" (INADDR_ANY), and '::' is the similar IN6ADDR_ANY for - IPv6. The empty string or None are not allowed. - - For UNIX sockets, supply the filename as a string.""") - - def start(self): - """Run the server forever.""" - # We don't have to trap KeyboardInterrupt or SystemExit here, - # because cherrpy.server already does so, calling self.stop() for us. - # If you're using this server with another framework, you should - # trap those exceptions in whatever code block calls start(). - self._interrupt = None - - if self.software is None: - self.software = "%s Server" % self.version - - # Select the appropriate socket - if isinstance(self.bind_addr, basestring): - # AF_UNIX socket - - # So we can reuse the socket... - try: - os.unlink(self.bind_addr) - except: - pass - - # So everyone can access the socket... - try: - os.chmod(self.bind_addr, 0o777) - except: - pass - - info = [ - (socket.AF_UNIX, socket.SOCK_STREAM, 0, "", self.bind_addr)] - else: - # AF_INET or AF_INET6 socket - # Get the correct address family for our host (allows IPv6 - # addresses) - host, port = self.bind_addr - try: - info = socket.getaddrinfo( - host, port, socket.AF_UNSPEC, - socket.SOCK_STREAM, 0, socket.AI_PASSIVE) - except socket.gaierror: - if ':' in self.bind_addr[0]: - info = [(socket.AF_INET6, socket.SOCK_STREAM, - 0, "", self.bind_addr + (0, 0))] - else: - info = [(socket.AF_INET, socket.SOCK_STREAM, - 0, "", self.bind_addr)] - - self.socket = None - msg = "No socket could be created" - for res in info: - af, socktype, proto, canonname, sa = res - try: - self.bind(af, socktype, proto) - except socket.error, serr: - msg = "%s -- (%s: %s)" % (msg, sa, serr) - if self.socket: - self.socket.close() - self.socket = None - continue - break - if not self.socket: - raise socket.error(msg) - - # Timeout so KeyboardInterrupt can be caught on Win32 - self.socket.settimeout(1) - self.socket.listen(self.request_queue_size) - - # Create worker threads - self.requests.start() - - self.ready = True - self._start_time = time.time() - while self.ready: - try: - self.tick() - except (KeyboardInterrupt, SystemExit): - raise - except: - self.error_log("Error in HTTPServer.tick", level=logging.ERROR, - traceback=True) - - if self.interrupt: - while self.interrupt is True: - # Wait for self.stop() to complete. See _set_interrupt. - time.sleep(0.1) - if self.interrupt: - raise self.interrupt - - def error_log(self, msg="", level=20, traceback=False): - # Override this in subclasses as desired - sys.stderr.write(msg + '\n') - sys.stderr.flush() - if traceback: - tblines = traceback_.format_exc() - sys.stderr.write(tblines) - sys.stderr.flush() - - def bind(self, family, type, proto=0): - """Create (or recreate) the actual socket object.""" - self.socket = socket.socket(family, type, proto) - prevent_socket_inheritance(self.socket) - self.socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) - if self.nodelay and not isinstance(self.bind_addr, str): - self.socket.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1) - - if self.ssl_adapter is not None: - self.socket = self.ssl_adapter.bind(self.socket) - - # If listening on the IPV6 any address ('::' = IN6ADDR_ANY), - # activate dual-stack. See - # https://github.com/cherrypy/cherrypy/issues/871. - if (hasattr(socket, 'AF_INET6') and family == socket.AF_INET6 - and self.bind_addr[0] in ('::', '::0', '::0.0.0.0')): - try: - self.socket.setsockopt( - socket.IPPROTO_IPV6, socket.IPV6_V6ONLY, 0) - except (AttributeError, socket.error): - # Apparently, the socket option is not available in - # this machine's TCP stack - pass - - self.socket.bind(self.bind_addr) - - def tick(self): - """Accept a new connection and put it on the Queue.""" - try: - s, addr = self.socket.accept() - if self.stats['Enabled']: - self.stats['Accepts'] += 1 - if not self.ready: - return - - prevent_socket_inheritance(s) - if hasattr(s, 'settimeout'): - s.settimeout(self.timeout) - - makefile = CP_fileobject - ssl_env = {} - # if ssl cert and key are set, we try to be a secure HTTP server - if self.ssl_adapter is not None: - try: - s, ssl_env = self.ssl_adapter.wrap(s) - except NoSSLError: - msg = ("The client sent a plain HTTP request, but " - "this server only speaks HTTPS on this port.") - buf = ["%s 400 Bad Request\r\n" % self.protocol, - "Content-Length: %s\r\n" % len(msg), - "Content-Type: text/plain\r\n\r\n", - msg] - - wfile = makefile(s._sock, "wb", DEFAULT_BUFFER_SIZE) - try: - wfile.sendall("".join(buf)) - except socket.error: - x = sys.exc_info()[1] - if x.args[0] not in socket_errors_to_ignore: - raise - return - if not s: - return - makefile = self.ssl_adapter.makefile - # Re-apply our timeout since we may have a new socket object - if hasattr(s, 'settimeout'): - s.settimeout(self.timeout) - - conn = self.ConnectionClass(self, s, makefile) - - if not isinstance(self.bind_addr, basestring): - # optional values - # Until we do DNS lookups, omit REMOTE_HOST - if addr is None: # sometimes this can happen - # figure out if AF_INET or AF_INET6. - if len(s.getsockname()) == 2: - # AF_INET - addr = ('0.0.0.0', 0) - else: - # AF_INET6 - addr = ('::', 0) - conn.remote_addr = addr[0] - conn.remote_port = addr[1] - - conn.ssl_env = ssl_env - - try: - self.requests.put(conn) - except queue.Full: - # Just drop the conn. TODO: write 503 back? - conn.close() - return - except socket.timeout: - # The only reason for the timeout in start() is so we can - # notice keyboard interrupts on Win32, which don't interrupt - # accept() by default - return - except socket.error: - x = sys.exc_info()[1] - if self.stats['Enabled']: - self.stats['Socket Errors'] += 1 - if x.args[0] in socket_error_eintr: - # I *think* this is right. EINTR should occur when a signal - # is received during the accept() call; all docs say retry - # the call, and I *think* I'm reading it right that Python - # will then go ahead and poll for and handle the signal - # elsewhere. See - # https://github.com/cherrypy/cherrypy/issues/707. - return - if x.args[0] in socket_errors_nonblocking: - # Just try again. See - # https://github.com/cherrypy/cherrypy/issues/479. - return - if x.args[0] in socket_errors_to_ignore: - # Our socket was closed. - # See https://github.com/cherrypy/cherrypy/issues/686. - return - raise - - def _get_interrupt(self): - return self._interrupt - - def _set_interrupt(self, interrupt): - self._interrupt = True - self.stop() - self._interrupt = interrupt - interrupt = property(_get_interrupt, _set_interrupt, - doc="Set this to an Exception instance to " - "interrupt the server.") - - def stop(self): - """Gracefully shutdown a server that is serving forever.""" - self.ready = False - if self._start_time is not None: - self._run_time += (time.time() - self._start_time) - self._start_time = None - - sock = getattr(self, "socket", None) - if sock: - if not isinstance(self.bind_addr, basestring): - # Touch our own socket to make accept() return immediately. - try: - host, port = sock.getsockname()[:2] - except socket.error: - x = sys.exc_info()[1] - if x.args[0] not in socket_errors_to_ignore: - # Changed to use error code and not message - # See - # https://github.com/cherrypy/cherrypy/issues/860. - raise - else: - # Note that we're explicitly NOT using AI_PASSIVE, - # here, because we want an actual IP to touch. - # localhost won't work if we've bound to a public IP, - # but it will if we bound to '0.0.0.0' (INADDR_ANY). - for res in socket.getaddrinfo(host, port, socket.AF_UNSPEC, - socket.SOCK_STREAM): - af, socktype, proto, canonname, sa = res - s = None - try: - s = socket.socket(af, socktype, proto) - # See - # http://groups.google.com/group/cherrypy-users/ - # browse_frm/thread/bbfe5eb39c904fe0 - s.settimeout(1.0) - s.connect((host, port)) - s.close() - except socket.error: - if s: - s.close() - if hasattr(sock, "close"): - sock.close() - self.socket = None - - self.requests.stop(self.shutdown_timeout) - - -class Gateway(object): - - """A base class to interface HTTPServer with other systems, such as WSGI. - """ - - def __init__(self, req): - self.req = req - - def respond(self): - """Process the current request. Must be overridden in a subclass.""" - raise NotImplemented - - -# These may either be wsgiserver.SSLAdapter subclasses or the string names -# of such classes (in which case they will be lazily loaded). -ssl_adapters = { - 'builtin': 'cherrypy.wsgiserver.ssl_builtin.BuiltinSSLAdapter', - 'pyopenssl': 'cherrypy.wsgiserver.ssl_pyopenssl.pyOpenSSLAdapter', -} - - -def get_ssl_adapter_class(name='builtin'): - """Return an SSL adapter class for the given name.""" - adapter = ssl_adapters[name.lower()] - if isinstance(adapter, basestring): - last_dot = adapter.rfind(".") - attr_name = adapter[last_dot + 1:] - mod_path = adapter[:last_dot] - - try: - mod = sys.modules[mod_path] - if mod is None: - raise KeyError() - except KeyError: - # The last [''] is important. - mod = __import__(mod_path, globals(), locals(), ['']) - - # Let an AttributeError propagate outward. - try: - adapter = getattr(mod, attr_name) - except AttributeError: - raise AttributeError("'%s' object has no attribute '%s'" - % (mod_path, attr_name)) - - return adapter - -# ------------------------------- WSGI Stuff -------------------------------- # - - -class CherryPyWSGIServer(HTTPServer): - - """A subclass of HTTPServer which calls a WSGI application.""" - - wsgi_version = (1, 0) - """The version of WSGI to produce.""" - - def __init__(self, bind_addr, wsgi_app, numthreads=10, server_name=None, - max=-1, request_queue_size=5, timeout=10, shutdown_timeout=5, - accepted_queue_size=-1, accepted_queue_timeout=10): - self.requests = ThreadPool(self, min=numthreads or 1, max=max, - accepted_queue_size=accepted_queue_size, - accepted_queue_timeout=accepted_queue_timeout) - self.wsgi_app = wsgi_app - self.gateway = wsgi_gateways[self.wsgi_version] - - self.bind_addr = bind_addr - if not server_name: - server_name = socket.gethostname() - self.server_name = server_name - self.request_queue_size = request_queue_size - - self.timeout = timeout - self.shutdown_timeout = shutdown_timeout - self.clear_stats() - - def _get_numthreads(self): - return self.requests.min - - def _set_numthreads(self, value): - self.requests.min = value - numthreads = property(_get_numthreads, _set_numthreads) - - -class WSGIGateway(Gateway): - - """A base class to interface HTTPServer with WSGI.""" - - def __init__(self, req): - self.req = req - self.started_response = False - self.env = self.get_environ() - self.remaining_bytes_out = None - - def get_environ(self): - """Return a new environ dict targeting the given wsgi.version""" - raise NotImplemented - - def respond(self): - """Process the current request.""" - response = self.req.server.wsgi_app(self.env, self.start_response) - try: - for chunk in response: - # "The start_response callable must not actually transmit - # the response headers. Instead, it must store them for the - # server or gateway to transmit only after the first - # iteration of the application return value that yields - # a NON-EMPTY string, or upon the application's first - # invocation of the write() callable." (PEP 333) - if chunk: - if isinstance(chunk, unicodestr): - chunk = chunk.encode('ISO-8859-1') - self.write(chunk) - finally: - if hasattr(response, "close"): - response.close() - - def start_response(self, status, headers, exc_info=None): - """WSGI callable to begin the HTTP response.""" - # "The application may call start_response more than once, - # if and only if the exc_info argument is provided." - if self.started_response and not exc_info: - raise AssertionError("WSGI start_response called a second " - "time with no exc_info.") - self.started_response = True - - # "if exc_info is provided, and the HTTP headers have already been - # sent, start_response must raise an error, and should raise the - # exc_info tuple." - if self.req.sent_headers: - try: - raise exc_info[0], exc_info[1], exc_info[2] - finally: - exc_info = None - - self.req.status = status - for k, v in headers: - if not isinstance(k, str): - raise TypeError( - "WSGI response header key %r is not of type str." % k) - if not isinstance(v, str): - raise TypeError( - "WSGI response header value %r is not of type str." % v) - if k.lower() == 'content-length': - self.remaining_bytes_out = int(v) - self.req.outheaders.extend(headers) - - return self.write - - def write(self, chunk): - """WSGI callable to write unbuffered data to the client. - - This method is also used internally by start_response (to write - data from the iterable returned by the WSGI application). - """ - if not self.started_response: - raise AssertionError("WSGI write called before start_response.") - - chunklen = len(chunk) - rbo = self.remaining_bytes_out - if rbo is not None and chunklen > rbo: - if not self.req.sent_headers: - # Whew. We can send a 500 to the client. - self.req.simple_response( - "500 Internal Server Error", - "The requested resource returned more bytes than the " - "declared Content-Length.") - else: - # Dang. We have probably already sent data. Truncate the chunk - # to fit (so the client doesn't hang) and raise an error later. - chunk = chunk[:rbo] - - if not self.req.sent_headers: - self.req.sent_headers = True - self.req.send_headers() - - self.req.write(chunk) - - if rbo is not None: - rbo -= chunklen - if rbo < 0: - raise ValueError( - "Response body exceeds the declared Content-Length.") - - -class WSGIGateway_10(WSGIGateway): - - """A Gateway class to interface HTTPServer with WSGI 1.0.x.""" - - def get_environ(self): - """Return a new environ dict targeting the given wsgi.version""" - req = self.req - env = { - # set a non-standard environ entry so the WSGI app can know what - # the *real* server protocol is (and what features to support). - # See http://www.faqs.org/rfcs/rfc2145.html. - 'ACTUAL_SERVER_PROTOCOL': req.server.protocol, - 'PATH_INFO': req.path, - 'QUERY_STRING': req.qs, - 'REMOTE_ADDR': req.conn.remote_addr or '', - 'REMOTE_PORT': str(req.conn.remote_port or ''), - 'REQUEST_METHOD': req.method, - 'REQUEST_URI': req.uri, - 'SCRIPT_NAME': '', - 'SERVER_NAME': req.server.server_name, - # Bah. "SERVER_PROTOCOL" is actually the REQUEST protocol. - 'SERVER_PROTOCOL': req.request_protocol, - 'SERVER_SOFTWARE': req.server.software, - 'wsgi.errors': sys.stderr, - 'wsgi.input': req.rfile, - 'wsgi.multiprocess': False, - 'wsgi.multithread': True, - 'wsgi.run_once': False, - 'wsgi.url_scheme': req.scheme, - 'wsgi.version': (1, 0), - } - - if isinstance(req.server.bind_addr, basestring): - # AF_UNIX. This isn't really allowed by WSGI, which doesn't - # address unix domain sockets. But it's better than nothing. - env["SERVER_PORT"] = "" - else: - env["SERVER_PORT"] = str(req.server.bind_addr[1]) - - # Request headers - for k, v in req.inheaders.iteritems(): - env["HTTP_" + k.upper().replace("-", "_")] = v - - # CONTENT_TYPE/CONTENT_LENGTH - ct = env.pop("HTTP_CONTENT_TYPE", None) - if ct is not None: - env["CONTENT_TYPE"] = ct - cl = env.pop("HTTP_CONTENT_LENGTH", None) - if cl is not None: - env["CONTENT_LENGTH"] = cl - - if req.conn.ssl_env: - env.update(req.conn.ssl_env) - - return env - - -class WSGIGateway_u0(WSGIGateway_10): - - """A Gateway class to interface HTTPServer with WSGI u.0. - - WSGI u.0 is an experimental protocol, which uses unicode for keys and - values in both Python 2 and Python 3. - """ - - def get_environ(self): - """Return a new environ dict targeting the given wsgi.version""" - req = self.req - env_10 = WSGIGateway_10.get_environ(self) - env = dict([(k.decode('ISO-8859-1'), v) - for k, v in env_10.iteritems()]) - env[u'wsgi.version'] = ('u', 0) - - # Request-URI - env.setdefault(u'wsgi.url_encoding', u'utf-8') - try: - for key in [u"PATH_INFO", u"SCRIPT_NAME", u"QUERY_STRING"]: - env[key] = env_10[str(key)].decode(env[u'wsgi.url_encoding']) - except UnicodeDecodeError: - # Fall back to latin 1 so apps can transcode if needed. - env[u'wsgi.url_encoding'] = u'ISO-8859-1' - for key in [u"PATH_INFO", u"SCRIPT_NAME", u"QUERY_STRING"]: - env[key] = env_10[str(key)].decode(env[u'wsgi.url_encoding']) - - for k, v in sorted(env.items()): - if isinstance(v, str) and k not in ('REQUEST_URI', 'wsgi.input'): - env[k] = v.decode('ISO-8859-1') - - return env - -wsgi_gateways = { - (1, 0): WSGIGateway_10, - ('u', 0): WSGIGateway_u0, -} - - -class WSGIPathInfoDispatcher(object): - - """A WSGI dispatcher for dispatch based on the PATH_INFO. - - apps: a dict or list of (path_prefix, app) pairs. - """ - - def __init__(self, apps): - try: - apps = list(apps.items()) - except AttributeError: - pass - - # Sort the apps by len(path), descending - apps.sort(cmp=lambda x, y: cmp(len(x[0]), len(y[0]))) - apps.reverse() - - # The path_prefix strings must start, but not end, with a slash. - # Use "" instead of "/". - self.apps = [(p.rstrip("/"), a) for p, a in apps] - - def __call__(self, environ, start_response): - path = environ["PATH_INFO"] or "/" - for p, app in self.apps: - # The apps list should be sorted by length, descending. - if path.startswith(p + "/") or path == p: - environ = environ.copy() - environ["SCRIPT_NAME"] = environ["SCRIPT_NAME"] + p - environ["PATH_INFO"] = path[len(p):] - return app(environ, start_response) - - start_response('404 Not Found', [('Content-Type', 'text/plain'), - ('Content-Length', '0')]) - return [''] diff --git a/cherrypy/wsgiserver/wsgiserver3.py b/cherrypy/wsgiserver/wsgiserver3.py deleted file mode 100644 index e69eb0d..0000000 --- a/cherrypy/wsgiserver/wsgiserver3.py +++ /dev/null @@ -1,2198 +0,0 @@ -"""A high-speed, production ready, thread pooled, generic HTTP server. - -Simplest example on how to use this module directly -(without using CherryPy's application machinery):: - - from cherrypy import wsgiserver - - def my_crazy_app(environ, start_response): - status = '200 OK' - response_headers = [('Content-type','text/plain')] - start_response(status, response_headers) - return ['Hello world!'] - - server = wsgiserver.CherryPyWSGIServer( - ('0.0.0.0', 8070), my_crazy_app, - server_name='www.cherrypy.example') - server.start() - -The CherryPy WSGI server can serve as many WSGI applications -as you want in one instance by using a WSGIPathInfoDispatcher:: - - d = WSGIPathInfoDispatcher({'/': my_crazy_app, '/blog': my_blog_app}) - server = wsgiserver.CherryPyWSGIServer(('0.0.0.0', 80), d) - -Want SSL support? Just set server.ssl_adapter to an SSLAdapter instance. - -This won't call the CherryPy engine (application side) at all, only the -HTTP server, which is independent from the rest of CherryPy. Don't -let the name "CherryPyWSGIServer" throw you; the name merely reflects -its origin, not its coupling. - -For those of you wanting to understand internals of this module, here's the -basic call flow. The server's listening thread runs a very tight loop, -sticking incoming connections onto a Queue:: - - server = CherryPyWSGIServer(...) - server.start() - while True: - tick() - # This blocks until a request comes in: - child = socket.accept() - conn = HTTPConnection(child, ...) - server.requests.put(conn) - -Worker threads are kept in a pool and poll the Queue, popping off and then -handling each connection in turn. Each connection can consist of an arbitrary -number of requests and their responses, so we run a nested loop:: - - while True: - conn = server.requests.get() - conn.communicate() - -> while True: - req = HTTPRequest(...) - req.parse_request() - -> # Read the Request-Line, e.g. "GET /page HTTP/1.1" - req.rfile.readline() - read_headers(req.rfile, req.inheaders) - req.respond() - -> response = app(...) - try: - for chunk in response: - if chunk: - req.write(chunk) - finally: - if hasattr(response, "close"): - response.close() - if req.close_connection: - return -""" - -__all__ = ['HTTPRequest', 'HTTPConnection', 'HTTPServer', - 'SizeCheckWrapper', 'KnownLengthRFile', 'ChunkedRFile', - 'CP_makefile', - 'MaxSizeExceeded', 'NoSSLError', 'FatalSSLAlert', - 'WorkerThread', 'ThreadPool', 'SSLAdapter', - 'CherryPyWSGIServer', - 'Gateway', 'WSGIGateway', 'WSGIGateway_10', 'WSGIGateway_u0', - 'WSGIPathInfoDispatcher', 'get_ssl_adapter_class', - 'socket_errors_to_ignore'] - -import os -try: - import queue -except: - import Queue as queue -import re -import email.utils -import socket -import sys -import threading -import time -import traceback as traceback_ -import errno -import logging -from urllib.parse import urlparse - -try: - # prefer slower Python-based io module - import _pyio as io -except ImportError: - # Python 2.6 - import io - -try: - import pkg_resources -except ImportError: - pass - -if 'win' in sys.platform and hasattr(socket, "AF_INET6"): - if not hasattr(socket, 'IPPROTO_IPV6'): - socket.IPPROTO_IPV6 = 41 - if not hasattr(socket, 'IPV6_V6ONLY'): - socket.IPV6_V6ONLY = 27 - - -DEFAULT_BUFFER_SIZE = io.DEFAULT_BUFFER_SIZE - - -try: - cp_version = pkg_resources.require('cherrypy')[0].version -except Exception: - cp_version = 'unknown' - - -if sys.version_info >= (3, 0): - unicodestr = str - basestring = (bytes, str) - - def ntob(n, encoding='ISO-8859-1'): - """Return the given native string as a byte string in the given - encoding. - """ - # In Python 3, the native string type is unicode - return n.encode(encoding) -else: - unicodestr = unicode - basestring = basestring - - def ntob(n, encoding='ISO-8859-1'): - """Return the given native string as a byte string in the given - encoding. - """ - # In Python 2, the native string type is bytes. Assume it's already - # in the given encoding, which for ISO-8859-1 is almost always what - # was intended. - return n - -LF = ntob('\n') -CRLF = ntob('\r\n') -TAB = ntob('\t') -SPACE = ntob(' ') -COLON = ntob(':') -SEMICOLON = ntob(';') -EMPTY = ntob('') -NUMBER_SIGN = ntob('#') -QUESTION_MARK = ntob('?') -ASTERISK = ntob('*') -FORWARD_SLASH = ntob('/') -quoted_slash = re.compile(ntob("(?i)%2F")) - - -def plat_specific_errors(*errnames): - """Return error numbers for all errors in errnames on this platform. - - The 'errno' module contains different global constants depending on - the specific platform (OS). This function will return the list of - numeric values for a given list of potential names. - """ - errno_names = dir(errno) - nums = [getattr(errno, k) for k in errnames if k in errno_names] - # de-dupe the list - return list(dict.fromkeys(nums).keys()) - -socket_error_eintr = plat_specific_errors("EINTR", "WSAEINTR") - -socket_errors_to_ignore = plat_specific_errors( - "EPIPE", - "EBADF", "WSAEBADF", - "ENOTSOCK", "WSAENOTSOCK", - "ETIMEDOUT", "WSAETIMEDOUT", - "ECONNREFUSED", "WSAECONNREFUSED", - "ECONNRESET", "WSAECONNRESET", - "ECONNABORTED", "WSAECONNABORTED", - "ENETRESET", "WSAENETRESET", - "EHOSTDOWN", "EHOSTUNREACH", -) -socket_errors_to_ignore.append("timed out") -socket_errors_to_ignore.append("The read operation timed out") -if sys.platform == 'darwin': - socket_errors_to_ignore.append(plat_specific_errors("EPROTOTYPE")) - -socket_errors_nonblocking = plat_specific_errors( - 'EAGAIN', 'EWOULDBLOCK', 'WSAEWOULDBLOCK') - -comma_separated_headers = [ - ntob(h) for h in - ['Accept', 'Accept-Charset', 'Accept-Encoding', - 'Accept-Language', 'Accept-Ranges', 'Allow', 'Cache-Control', - 'Connection', 'Content-Encoding', 'Content-Language', 'Expect', - 'If-Match', 'If-None-Match', 'Pragma', 'Proxy-Authenticate', 'TE', - 'Trailer', 'Transfer-Encoding', 'Upgrade', 'Vary', 'Via', 'Warning', - 'WWW-Authenticate'] -] - - -if not hasattr(logging, 'statistics'): - logging.statistics = {} - - -def read_headers(rfile, hdict=None): - """Read headers from the given stream into the given header dict. - - If hdict is None, a new header dict is created. Returns the populated - header dict. - - Headers which are repeated are folded together using a comma if their - specification so dictates. - - This function raises ValueError when the read bytes violate the HTTP spec. - You should probably return "400 Bad Request" if this happens. - """ - if hdict is None: - hdict = {} - - while True: - line = rfile.readline() - if not line: - # No more data--illegal end of headers - raise ValueError("Illegal end of headers.") - - if line == CRLF: - # Normal end of headers - break - if not line.endswith(CRLF): - raise ValueError("HTTP requires CRLF terminators") - - if line[0] in (SPACE, TAB): - # It's a continuation line. - v = line.strip() - else: - try: - k, v = line.split(COLON, 1) - except ValueError: - raise ValueError("Illegal header line.") - # TODO: what about TE and WWW-Authenticate? - k = k.strip().title() - v = v.strip() - hname = k - - if k in comma_separated_headers: - existing = hdict.get(hname) - if existing: - v = b", ".join((existing, v)) - hdict[hname] = v - - return hdict - - -class MaxSizeExceeded(Exception): - pass - - -class SizeCheckWrapper(object): - - """Wraps a file-like object, raising MaxSizeExceeded if too large.""" - - def __init__(self, rfile, maxlen): - self.rfile = rfile - self.maxlen = maxlen - self.bytes_read = 0 - - def _check_length(self): - if self.maxlen and self.bytes_read > self.maxlen: - raise MaxSizeExceeded() - - def read(self, size=None): - data = self.rfile.read(size) - self.bytes_read += len(data) - self._check_length() - return data - - def readline(self, size=None): - if size is not None: - data = self.rfile.readline(size) - self.bytes_read += len(data) - self._check_length() - return data - - # User didn't specify a size ... - # We read the line in chunks to make sure it's not a 100MB line ! - res = [] - while True: - data = self.rfile.readline(256) - self.bytes_read += len(data) - self._check_length() - res.append(data) - # See https://github.com/cherrypy/cherrypy/issues/421 - if len(data) < 256 or data[-1:] == LF: - return EMPTY.join(res) - - def readlines(self, sizehint=0): - # Shamelessly stolen from StringIO - total = 0 - lines = [] - line = self.readline() - while line: - lines.append(line) - total += len(line) - if 0 < sizehint <= total: - break - line = self.readline() - return lines - - def close(self): - self.rfile.close() - - def __iter__(self): - return self - - def __next__(self): - data = next(self.rfile) - self.bytes_read += len(data) - self._check_length() - return data - - def next(self): - data = self.rfile.next() - self.bytes_read += len(data) - self._check_length() - return data - - -class KnownLengthRFile(object): - - """Wraps a file-like object, returning an empty string when exhausted.""" - - def __init__(self, rfile, content_length): - self.rfile = rfile - self.remaining = content_length - - def read(self, size=None): - if self.remaining == 0: - return b'' - if size is None: - size = self.remaining - else: - size = min(size, self.remaining) - - data = self.rfile.read(size) - self.remaining -= len(data) - return data - - def readline(self, size=None): - if self.remaining == 0: - return b'' - if size is None: - size = self.remaining - else: - size = min(size, self.remaining) - - data = self.rfile.readline(size) - self.remaining -= len(data) - return data - - def readlines(self, sizehint=0): - # Shamelessly stolen from StringIO - total = 0 - lines = [] - line = self.readline(sizehint) - while line: - lines.append(line) - total += len(line) - if 0 < sizehint <= total: - break - line = self.readline(sizehint) - return lines - - def close(self): - self.rfile.close() - - def __iter__(self): - return self - - def __next__(self): - data = next(self.rfile) - self.remaining -= len(data) - return data - - -class ChunkedRFile(object): - - """Wraps a file-like object, returning an empty string when exhausted. - - This class is intended to provide a conforming wsgi.input value for - request entities that have been encoded with the 'chunked' transfer - encoding. - """ - - def __init__(self, rfile, maxlen, bufsize=8192): - self.rfile = rfile - self.maxlen = maxlen - self.bytes_read = 0 - self.buffer = EMPTY - self.bufsize = bufsize - self.closed = False - - def _fetch(self): - if self.closed: - return - - line = self.rfile.readline() - self.bytes_read += len(line) - - if self.maxlen and self.bytes_read > self.maxlen: - raise MaxSizeExceeded("Request Entity Too Large", self.maxlen) - - line = line.strip().split(SEMICOLON, 1) - - try: - chunk_size = line.pop(0) - chunk_size = int(chunk_size, 16) - except ValueError: - raise ValueError("Bad chunked transfer size: " + repr(chunk_size)) - - if chunk_size <= 0: - self.closed = True - return - -## if line: chunk_extension = line[0] - - if self.maxlen and self.bytes_read + chunk_size > self.maxlen: - raise IOError("Request Entity Too Large") - - chunk = self.rfile.read(chunk_size) - self.bytes_read += len(chunk) - self.buffer += chunk - - crlf = self.rfile.read(2) - if crlf != CRLF: - raise ValueError( - "Bad chunked transfer coding (expected '\\r\\n', " - "got " + repr(crlf) + ")") - - def read(self, size=None): - data = EMPTY - while True: - if size and len(data) >= size: - return data - - if not self.buffer: - self._fetch() - if not self.buffer: - # EOF - return data - - if size: - remaining = size - len(data) - data += self.buffer[:remaining] - self.buffer = self.buffer[remaining:] - else: - data += self.buffer - - def readline(self, size=None): - data = EMPTY - while True: - if size and len(data) >= size: - return data - - if not self.buffer: - self._fetch() - if not self.buffer: - # EOF - return data - - newline_pos = self.buffer.find(LF) - if size: - if newline_pos == -1: - remaining = size - len(data) - data += self.buffer[:remaining] - self.buffer = self.buffer[remaining:] - else: - remaining = min(size - len(data), newline_pos) - data += self.buffer[:remaining] - self.buffer = self.buffer[remaining:] - else: - if newline_pos == -1: - data += self.buffer - else: - data += self.buffer[:newline_pos] - self.buffer = self.buffer[newline_pos:] - - def readlines(self, sizehint=0): - # Shamelessly stolen from StringIO - total = 0 - lines = [] - line = self.readline(sizehint) - while line: - lines.append(line) - total += len(line) - if 0 < sizehint <= total: - break - line = self.readline(sizehint) - return lines - - def read_trailer_lines(self): - if not self.closed: - raise ValueError( - "Cannot read trailers until the request body has been read.") - - while True: - line = self.rfile.readline() - if not line: - # No more data--illegal end of headers - raise ValueError("Illegal end of headers.") - - self.bytes_read += len(line) - if self.maxlen and self.bytes_read > self.maxlen: - raise IOError("Request Entity Too Large") - - if line == CRLF: - # Normal end of headers - break - if not line.endswith(CRLF): - raise ValueError("HTTP requires CRLF terminators") - - yield line - - def close(self): - self.rfile.close() - - def __iter__(self): - # Shamelessly stolen from StringIO - total = 0 - line = self.readline(sizehint) - while line: - yield line - total += len(line) - if 0 < sizehint <= total: - break - line = self.readline(sizehint) - - -class HTTPRequest(object): - - """An HTTP Request (and response). - - A single HTTP connection may consist of multiple request/response pairs. - """ - - server = None - """The HTTPServer object which is receiving this request.""" - - conn = None - """The HTTPConnection object on which this request connected.""" - - inheaders = {} - """A dict of request headers.""" - - outheaders = [] - """A list of header tuples to write in the response.""" - - ready = False - """When True, the request has been parsed and is ready to begin generating - the response. When False, signals the calling Connection that the response - should not be generated and the connection should close.""" - - close_connection = False - """Signals the calling Connection that the request should close. This does - not imply an error! The client and/or server may each request that the - connection be closed.""" - - chunked_write = False - """If True, output will be encoded with the "chunked" transfer-coding. - - This value is set automatically inside send_headers.""" - - def __init__(self, server, conn): - self.server = server - self.conn = conn - - self.ready = False - self.started_request = False - self.scheme = ntob("http") - if self.server.ssl_adapter is not None: - self.scheme = ntob("https") - # Use the lowest-common protocol in case read_request_line errors. - self.response_protocol = 'HTTP/1.0' - self.inheaders = {} - - self.status = "" - self.outheaders = [] - self.sent_headers = False - self.close_connection = self.__class__.close_connection - self.chunked_read = False - self.chunked_write = self.__class__.chunked_write - - def parse_request(self): - """Parse the next HTTP request start-line and message-headers.""" - self.rfile = SizeCheckWrapper(self.conn.rfile, - self.server.max_request_header_size) - try: - success = self.read_request_line() - except MaxSizeExceeded: - self.simple_response( - "414 Request-URI Too Long", - "The Request-URI sent with the request exceeds the maximum " - "allowed bytes.") - return - else: - if not success: - return - - try: - success = self.read_request_headers() - except MaxSizeExceeded: - self.simple_response( - "413 Request Entity Too Large", - "The headers sent with the request exceed the maximum " - "allowed bytes.") - return - else: - if not success: - return - - self.ready = True - - def read_request_line(self): - # HTTP/1.1 connections are persistent by default. If a client - # requests a page, then idles (leaves the connection open), - # then rfile.readline() will raise socket.error("timed out"). - # Note that it does this based on the value given to settimeout(), - # and doesn't need the client to request or acknowledge the close - # (although your TCP stack might suffer for it: cf Apache's history - # with FIN_WAIT_2). - request_line = self.rfile.readline() - - # Set started_request to True so communicate() knows to send 408 - # from here on out. - self.started_request = True - if not request_line: - return False - - if request_line == CRLF: - # RFC 2616 sec 4.1: "...if the server is reading the protocol - # stream at the beginning of a message and receives a CRLF - # first, it should ignore the CRLF." - # But only ignore one leading line! else we enable a DoS. - request_line = self.rfile.readline() - if not request_line: - return False - - if not request_line.endswith(CRLF): - self.simple_response( - "400 Bad Request", "HTTP requires CRLF terminators") - return False - - try: - method, uri, req_protocol = request_line.strip().split(SPACE, 2) - # The [x:y] slicing is necessary for byte strings to avoid getting - # ord's - rp = int(req_protocol[5:6]), int(req_protocol[7:8]) - except ValueError: - self.simple_response("400 Bad Request", "Malformed Request-Line") - return False - - self.uri = uri - self.method = method - - # uri may be an abs_path (including "http://host.domain.tld"); - scheme, authority, path = self.parse_request_uri(uri) - if path is None: - self.simple_response("400 Bad Request", - "Invalid path in Request-URI.") - return False - if NUMBER_SIGN in path: - self.simple_response("400 Bad Request", - "Illegal #fragment in Request-URI.") - return False - - if scheme: - self.scheme = scheme - - qs = EMPTY - if QUESTION_MARK in path: - path, qs = path.split(QUESTION_MARK, 1) - - # Unquote the path+params (e.g. "/this%20path" -> "/this path"). - # http://www.w3.org/Protocols/rfc2616/rfc2616-sec5.html#sec5.1.2 - # - # But note that "...a URI must be separated into its components - # before the escaped characters within those components can be - # safely decoded." http://www.ietf.org/rfc/rfc2396.txt, sec 2.4.2 - # Therefore, "/this%2Fpath" becomes "/this%2Fpath", not "/this/path". - try: - atoms = [self.unquote_bytes(x) for x in quoted_slash.split(path)] - except ValueError: - ex = sys.exc_info()[1] - self.simple_response("400 Bad Request", ex.args[0]) - return False - path = b"%2F".join(atoms) - self.path = path - - # Note that, like wsgiref and most other HTTP servers, - # we "% HEX HEX"-unquote the path but not the query string. - self.qs = qs - - # Compare request and server HTTP protocol versions, in case our - # server does not support the requested protocol. Limit our output - # to min(req, server). We want the following output: - # request server actual written supported response - # protocol protocol response protocol feature set - # a 1.0 1.0 1.0 1.0 - # b 1.0 1.1 1.1 1.0 - # c 1.1 1.0 1.0 1.0 - # d 1.1 1.1 1.1 1.1 - # Notice that, in (b), the response will be "HTTP/1.1" even though - # the client only understands 1.0. RFC 2616 10.5.6 says we should - # only return 505 if the _major_ version is different. - # The [x:y] slicing is necessary for byte strings to avoid getting - # ord's - sp = int(self.server.protocol[5:6]), int(self.server.protocol[7:8]) - - if sp[0] != rp[0]: - self.simple_response("505 HTTP Version Not Supported") - return False - - self.request_protocol = req_protocol - self.response_protocol = "HTTP/%s.%s" % min(rp, sp) - return True - - def read_request_headers(self): - """Read self.rfile into self.inheaders. Return success.""" - - # then all the http headers - try: - read_headers(self.rfile, self.inheaders) - except ValueError: - ex = sys.exc_info()[1] - self.simple_response("400 Bad Request", ex.args[0]) - return False - - mrbs = self.server.max_request_body_size - if mrbs and int(self.inheaders.get(b"Content-Length", 0)) > mrbs: - self.simple_response( - "413 Request Entity Too Large", - "The entity sent with the request exceeds the maximum " - "allowed bytes.") - return False - - # Persistent connection support - if self.response_protocol == "HTTP/1.1": - # Both server and client are HTTP/1.1 - if self.inheaders.get(b"Connection", b"") == b"close": - self.close_connection = True - else: - # Either the server or client (or both) are HTTP/1.0 - if self.inheaders.get(b"Connection", b"") != b"Keep-Alive": - self.close_connection = True - - # Transfer-Encoding support - te = None - if self.response_protocol == "HTTP/1.1": - te = self.inheaders.get(b"Transfer-Encoding") - if te: - te = [x.strip().lower() for x in te.split(b",") if x.strip()] - - self.chunked_read = False - - if te: - for enc in te: - if enc == b"chunked": - self.chunked_read = True - else: - # Note that, even if we see "chunked", we must reject - # if there is an extension we don't recognize. - self.simple_response("501 Unimplemented") - self.close_connection = True - return False - - # From PEP 333: - # "Servers and gateways that implement HTTP 1.1 must provide - # transparent support for HTTP 1.1's "expect/continue" mechanism. - # This may be done in any of several ways: - # 1. Respond to requests containing an Expect: 100-continue request - # with an immediate "100 Continue" response, and proceed normally. - # 2. Proceed with the request normally, but provide the application - # with a wsgi.input stream that will send the "100 Continue" - # response if/when the application first attempts to read from - # the input stream. The read request must then remain blocked - # until the client responds. - # 3. Wait until the client decides that the server does not support - # expect/continue, and sends the request body on its own. - # (This is suboptimal, and is not recommended.) - # - # We used to do 3, but are now doing 1. Maybe we'll do 2 someday, - # but it seems like it would be a big slowdown for such a rare case. - if self.inheaders.get(b"Expect", b"") == b"100-continue": - # Don't use simple_response here, because it emits headers - # we don't want. See - # https://github.com/cherrypy/cherrypy/issues/951 - msg = self.server.protocol.encode( - 'ascii') + b" 100 Continue\r\n\r\n" - try: - self.conn.wfile.write(msg) - except socket.error: - x = sys.exc_info()[1] - if x.args[0] not in socket_errors_to_ignore: - raise - return True - - def parse_request_uri(self, uri): - """Parse a Request-URI into (scheme, authority, path). - - Note that Request-URI's must be one of:: - - Request-URI = "*" | absoluteURI | abs_path | authority - - Therefore, a Request-URI which starts with a double forward-slash - cannot be a "net_path":: - - net_path = "//" authority [ abs_path ] - - Instead, it must be interpreted as an "abs_path" with an empty first - path segment:: - - abs_path = "/" path_segments - path_segments = segment *( "/" segment ) - segment = *pchar *( ";" param ) - param = *pchar - """ - if uri == ASTERISK: - return None, None, uri - - scheme, authority, path, params, query, fragment = urlparse(uri) - if scheme and QUESTION_MARK not in scheme: - # An absoluteURI. - # If there's a scheme (and it must be http or https), then: - # http_URL = "http:" "//" host [ ":" port ] [ abs_path [ "?" query - # ]] - return scheme, authority, path - - if uri.startswith(FORWARD_SLASH): - # An abs_path. - return None, None, uri - else: - # An authority. - return None, uri, None - - def unquote_bytes(self, path): - """takes quoted string and unquotes % encoded values""" - res = path.split(b'%') - - for i in range(1, len(res)): - item = res[i] - try: - res[i] = bytes([int(item[:2], 16)]) + item[2:] - except ValueError: - raise - return b''.join(res) - - def respond(self): - """Call the gateway and write its iterable output.""" - mrbs = self.server.max_request_body_size - if self.chunked_read: - self.rfile = ChunkedRFile(self.conn.rfile, mrbs) - else: - cl = int(self.inheaders.get(b"Content-Length", 0)) - if mrbs and mrbs < cl: - if not self.sent_headers: - self.simple_response( - "413 Request Entity Too Large", - "The entity sent with the request exceeds the " - "maximum allowed bytes.") - return - self.rfile = KnownLengthRFile(self.conn.rfile, cl) - - self.server.gateway(self).respond() - - if (self.ready and not self.sent_headers): - self.sent_headers = True - self.send_headers() - if self.chunked_write: - self.conn.wfile.write(b"0\r\n\r\n") - - def simple_response(self, status, msg=""): - """Write a simple response back to the client.""" - status = str(status) - buf = [bytes(self.server.protocol, "ascii") + SPACE + - bytes(status, "ISO-8859-1") + CRLF, - bytes("Content-Length: %s\r\n" % len(msg), "ISO-8859-1"), - b"Content-Type: text/plain\r\n"] - - if status[:3] in ("413", "414"): - # Request Entity Too Large / Request-URI Too Long - self.close_connection = True - if self.response_protocol == 'HTTP/1.1': - # This will not be true for 414, since read_request_line - # usually raises 414 before reading the whole line, and we - # therefore cannot know the proper response_protocol. - buf.append(b"Connection: close\r\n") - else: - # HTTP/1.0 had no 413/414 status nor Connection header. - # Emit 400 instead and trust the message body is enough. - status = "400 Bad Request" - - buf.append(CRLF) - if msg: - if isinstance(msg, unicodestr): - msg = msg.encode("ISO-8859-1") - buf.append(msg) - - try: - self.conn.wfile.write(b"".join(buf)) - except socket.error: - x = sys.exc_info()[1] - if x.args[0] not in socket_errors_to_ignore: - raise - - def write(self, chunk): - """Write unbuffered data to the client.""" - if self.chunked_write and chunk: - buf = [bytes(hex(len(chunk)), 'ASCII')[2:], CRLF, chunk, CRLF] - self.conn.wfile.write(EMPTY.join(buf)) - else: - self.conn.wfile.write(chunk) - - def send_headers(self): - """Assert, process, and send the HTTP response message-headers. - - You must set self.status, and self.outheaders before calling this. - """ - hkeys = [key.lower() for key, value in self.outheaders] - status = int(self.status[:3]) - - if status == 413: - # Request Entity Too Large. Close conn to avoid garbage. - self.close_connection = True - elif b"content-length" not in hkeys: - # "All 1xx (informational), 204 (no content), - # and 304 (not modified) responses MUST NOT - # include a message-body." So no point chunking. - if status < 200 or status in (204, 205, 304): - pass - else: - if (self.response_protocol == 'HTTP/1.1' - and self.method != b'HEAD'): - # Use the chunked transfer-coding - self.chunked_write = True - self.outheaders.append((b"Transfer-Encoding", b"chunked")) - else: - # Closing the conn is the only way to determine len. - self.close_connection = True - - if b"connection" not in hkeys: - if self.response_protocol == 'HTTP/1.1': - # Both server and client are HTTP/1.1 or better - if self.close_connection: - self.outheaders.append((b"Connection", b"close")) - else: - # Server and/or client are HTTP/1.0 - if not self.close_connection: - self.outheaders.append((b"Connection", b"Keep-Alive")) - - if (not self.close_connection) and (not self.chunked_read): - # Read any remaining request body data on the socket. - # "If an origin server receives a request that does not include an - # Expect request-header field with the "100-continue" expectation, - # the request includes a request body, and the server responds - # with a final status code before reading the entire request body - # from the transport connection, then the server SHOULD NOT close - # the transport connection until it has read the entire request, - # or until the client closes the connection. Otherwise, the client - # might not reliably receive the response message. However, this - # requirement is not be construed as preventing a server from - # defending itself against denial-of-service attacks, or from - # badly broken client implementations." - remaining = getattr(self.rfile, 'remaining', 0) - if remaining > 0: - self.rfile.read(remaining) - - if b"date" not in hkeys: - self.outheaders.append(( - b"Date", - email.utils.formatdate(usegmt=True).encode('ISO-8859-1') - )) - - if b"server" not in hkeys: - self.outheaders.append( - (b"Server", self.server.server_name.encode('ISO-8859-1'))) - - buf = [self.server.protocol.encode( - 'ascii') + SPACE + self.status + CRLF] - for k, v in self.outheaders: - buf.append(k + COLON + SPACE + v + CRLF) - buf.append(CRLF) - self.conn.wfile.write(EMPTY.join(buf)) - - -class NoSSLError(Exception): - - """Exception raised when a client speaks HTTP to an HTTPS socket.""" - pass - - -class FatalSSLAlert(Exception): - - """Exception raised when the SSL implementation signals a fatal alert.""" - pass - - -class CP_BufferedWriter(io.BufferedWriter): - - """Faux file object attached to a socket object.""" - - def write(self, b): - self._checkClosed() - if isinstance(b, str): - raise TypeError("can't write str to binary stream") - - with self._write_lock: - self._write_buf.extend(b) - self._flush_unlocked() - return len(b) - - def _flush_unlocked(self): - self._checkClosed("flush of closed file") - while self._write_buf: - try: - # ssl sockets only except 'bytes', not bytearrays - # so perhaps we should conditionally wrap this for perf? - n = self.raw.write(bytes(self._write_buf)) - except io.BlockingIOError as e: - n = e.characters_written - del self._write_buf[:n] - - -def CP_makefile(sock, mode='r', bufsize=DEFAULT_BUFFER_SIZE): - if 'r' in mode: - return io.BufferedReader(socket.SocketIO(sock, mode), bufsize) - else: - return CP_BufferedWriter(socket.SocketIO(sock, mode), bufsize) - - -class HTTPConnection(object): - - """An HTTP connection (active socket). - - server: the Server object which received this connection. - socket: the raw socket object (usually TCP) for this connection. - makefile: a fileobject class for reading from the socket. - """ - - remote_addr = None - remote_port = None - ssl_env = None - rbufsize = DEFAULT_BUFFER_SIZE - wbufsize = DEFAULT_BUFFER_SIZE - RequestHandlerClass = HTTPRequest - - def __init__(self, server, sock, makefile=CP_makefile): - self.server = server - self.socket = sock - self.rfile = makefile(sock, "rb", self.rbufsize) - self.wfile = makefile(sock, "wb", self.wbufsize) - self.requests_seen = 0 - - def communicate(self): - """Read each request and respond appropriately.""" - request_seen = False - try: - while True: - # (re)set req to None so that if something goes wrong in - # the RequestHandlerClass constructor, the error doesn't - # get written to the previous request. - req = None - req = self.RequestHandlerClass(self.server, self) - - # This order of operations should guarantee correct pipelining. - req.parse_request() - if self.server.stats['Enabled']: - self.requests_seen += 1 - if not req.ready: - # Something went wrong in the parsing (and the server has - # probably already made a simple_response). Return and - # let the conn close. - return - - request_seen = True - req.respond() - if req.close_connection: - return - except socket.error: - e = sys.exc_info()[1] - errnum = e.args[0] - # sadly SSL sockets return a different (longer) time out string - if ( - errnum == 'timed out' or - errnum == 'The read operation timed out' - ): - # Don't error if we're between requests; only error - # if 1) no request has been started at all, or 2) we're - # in the middle of a request. - # See https://github.com/cherrypy/cherrypy/issues/853 - if (not request_seen) or (req and req.started_request): - # Don't bother writing the 408 if the response - # has already started being written. - if req and not req.sent_headers: - try: - req.simple_response("408 Request Timeout") - except FatalSSLAlert: - # Close the connection. - return - elif errnum not in socket_errors_to_ignore: - self.server.error_log("socket.error %s" % repr(errnum), - level=logging.WARNING, traceback=True) - if req and not req.sent_headers: - try: - req.simple_response("500 Internal Server Error") - except FatalSSLAlert: - # Close the connection. - return - return - except (KeyboardInterrupt, SystemExit): - raise - except FatalSSLAlert: - # Close the connection. - return - except NoSSLError: - if req and not req.sent_headers: - # Unwrap our wfile - self.wfile = CP_makefile( - self.socket._sock, "wb", self.wbufsize) - req.simple_response( - "400 Bad Request", - "The client sent a plain HTTP request, but this server " - "only speaks HTTPS on this port.") - self.linger = True - except Exception: - e = sys.exc_info()[1] - self.server.error_log(repr(e), level=logging.ERROR, traceback=True) - if req and not req.sent_headers: - try: - req.simple_response("500 Internal Server Error") - except FatalSSLAlert: - # Close the connection. - return - - linger = False - - def close(self): - """Close the socket underlying this connection.""" - self.rfile.close() - - if not self.linger: - # Python's socket module does NOT call close on the kernel - # socket when you call socket.close(). We do so manually here - # because we want this server to send a FIN TCP segment - # immediately. Note this must be called *before* calling - # socket.close(), because the latter drops its reference to - # the kernel socket. - # Python 3 *probably* fixed this with socket._real_close; - # hard to tell. -# self.socket._sock.close() - self.socket.close() - else: - # On the other hand, sometimes we want to hang around for a bit - # to make sure the client has a chance to read our entire - # response. Skipping the close() calls here delays the FIN - # packet until the socket object is garbage-collected later. - # Someday, perhaps, we'll do the full lingering_close that - # Apache does, but not today. - pass - - -class TrueyZero(object): - - """An object which equals and does math like the integer 0 but evals True. - """ - - def __add__(self, other): - return other - - def __radd__(self, other): - return other -trueyzero = TrueyZero() - - -_SHUTDOWNREQUEST = None - - -class WorkerThread(threading.Thread): - - """Thread which continuously polls a Queue for Connection objects. - - Due to the timing issues of polling a Queue, a WorkerThread does not - check its own 'ready' flag after it has started. To stop the thread, - it is necessary to stick a _SHUTDOWNREQUEST object onto the Queue - (one for each running WorkerThread). - """ - - conn = None - """The current connection pulled off the Queue, or None.""" - - server = None - """The HTTP Server which spawned this thread, and which owns the - Queue and is placing active connections into it.""" - - ready = False - """A simple flag for the calling server to know when this thread - has begun polling the Queue.""" - - def __init__(self, server): - self.ready = False - self.server = server - - self.requests_seen = 0 - self.bytes_read = 0 - self.bytes_written = 0 - self.start_time = None - self.work_time = 0 - self.stats = { - 'Requests': lambda s: self.requests_seen + ( - (self.start_time is None) and - trueyzero or - self.conn.requests_seen - ), - 'Bytes Read': lambda s: self.bytes_read + ( - (self.start_time is None) and - trueyzero or - self.conn.rfile.bytes_read - ), - 'Bytes Written': lambda s: self.bytes_written + ( - (self.start_time is None) and - trueyzero or - self.conn.wfile.bytes_written - ), - 'Work Time': lambda s: self.work_time + ( - (self.start_time is None) and - trueyzero or - time.time() - self.start_time - ), - 'Read Throughput': lambda s: s['Bytes Read'](s) / ( - s['Work Time'](s) or 1e-6), - 'Write Throughput': lambda s: s['Bytes Written'](s) / ( - s['Work Time'](s) or 1e-6), - } - threading.Thread.__init__(self) - - def run(self): - self.server.stats['Worker Threads'][self.getName()] = self.stats - try: - self.ready = True - while True: - conn = self.server.requests.get() - if conn is _SHUTDOWNREQUEST: - return - - self.conn = conn - if self.server.stats['Enabled']: - self.start_time = time.time() - try: - conn.communicate() - finally: - conn.close() - if self.server.stats['Enabled']: - self.requests_seen += self.conn.requests_seen - self.bytes_read += self.conn.rfile.bytes_read - self.bytes_written += self.conn.wfile.bytes_written - self.work_time += time.time() - self.start_time - self.start_time = None - self.conn = None - except (KeyboardInterrupt, SystemExit): - exc = sys.exc_info()[1] - self.server.interrupt = exc - - -class ThreadPool(object): - - """A Request Queue for an HTTPServer which pools threads. - - ThreadPool objects must provide min, get(), put(obj), start() - and stop(timeout) attributes. - """ - - def __init__(self, server, min=10, max=-1, - accepted_queue_size=-1, accepted_queue_timeout=10): - self.server = server - self.min = min - self.max = max - self._threads = [] - self._queue = queue.Queue(maxsize=accepted_queue_size) - self._queue_put_timeout = accepted_queue_timeout - self.get = self._queue.get - - def start(self): - """Start the pool of threads.""" - for i in range(self.min): - self._threads.append(WorkerThread(self.server)) - for worker in self._threads: - worker.setName("CP Server " + worker.getName()) - worker.start() - for worker in self._threads: - while not worker.ready: - time.sleep(.1) - - def _get_idle(self): - """Number of worker threads which are idle. Read-only.""" - return len([t for t in self._threads if t.conn is None]) - idle = property(_get_idle, doc=_get_idle.__doc__) - - def put(self, obj): - self._queue.put(obj, block=True, timeout=self._queue_put_timeout) - if obj is _SHUTDOWNREQUEST: - return - - def grow(self, amount): - """Spawn new worker threads (not above self.max).""" - if self.max > 0: - budget = max(self.max - len(self._threads), 0) - else: - # self.max <= 0 indicates no maximum - budget = float('inf') - - n_new = min(amount, budget) - - workers = [self._spawn_worker() for i in range(n_new)] - while not all(worker.ready for worker in workers): - time.sleep(.1) - self._threads.extend(workers) - - def _spawn_worker(self): - worker = WorkerThread(self.server) - worker.setName("CP Server " + worker.getName()) - worker.start() - return worker - - def shrink(self, amount): - """Kill off worker threads (not below self.min).""" - # Grow/shrink the pool if necessary. - # Remove any dead threads from our list - for t in self._threads: - if not t.isAlive(): - self._threads.remove(t) - amount -= 1 - - # calculate the number of threads above the minimum - n_extra = max(len(self._threads) - self.min, 0) - - # don't remove more than amount - n_to_remove = min(amount, n_extra) - - # put shutdown requests on the queue equal to the number of threads - # to remove. As each request is processed by a worker, that worker - # will terminate and be culled from the list. - for n in range(n_to_remove): - self._queue.put(_SHUTDOWNREQUEST) - - def stop(self, timeout=5): - # Must shut down threads here so the code that calls - # this method can know when all threads are stopped. - for worker in self._threads: - self._queue.put(_SHUTDOWNREQUEST) - - # Don't join currentThread (when stop is called inside a request). - current = threading.currentThread() - if timeout and timeout >= 0: - endtime = time.time() + timeout - while self._threads: - worker = self._threads.pop() - if worker is not current and worker.isAlive(): - try: - if timeout is None or timeout < 0: - worker.join() - else: - remaining_time = endtime - time.time() - if remaining_time > 0: - worker.join(remaining_time) - if worker.isAlive(): - # We exhausted the timeout. - # Forcibly shut down the socket. - c = worker.conn - if c and not c.rfile.closed: - try: - c.socket.shutdown(socket.SHUT_RD) - except TypeError: - # pyOpenSSL sockets don't take an arg - c.socket.shutdown() - worker.join() - except (AssertionError, - # Ignore repeated Ctrl-C. - # See - # https://github.com/cherrypy/cherrypy/issues/691. - KeyboardInterrupt): - pass - - def _get_qsize(self): - return self._queue.qsize() - qsize = property(_get_qsize) - - -try: - import fcntl -except ImportError: - try: - from ctypes import windll, WinError - import ctypes.wintypes - _SetHandleInformation = windll.kernel32.SetHandleInformation - _SetHandleInformation.argtypes = [ - ctypes.wintypes.HANDLE, - ctypes.wintypes.DWORD, - ctypes.wintypes.DWORD, - ] - _SetHandleInformation.restype = ctypes.wintypes.BOOL - except ImportError: - def prevent_socket_inheritance(sock): - """Dummy function, since neither fcntl nor ctypes are available.""" - pass - else: - def prevent_socket_inheritance(sock): - """Mark the given socket fd as non-inheritable (Windows).""" - if not _SetHandleInformation(sock.fileno(), 1, 0): - raise WinError() -else: - def prevent_socket_inheritance(sock): - """Mark the given socket fd as non-inheritable (POSIX).""" - fd = sock.fileno() - old_flags = fcntl.fcntl(fd, fcntl.F_GETFD) - fcntl.fcntl(fd, fcntl.F_SETFD, old_flags | fcntl.FD_CLOEXEC) - - -class SSLAdapter(object): - - """Base class for SSL driver library adapters. - - Required methods: - - * ``wrap(sock) -> (wrapped socket, ssl environ dict)`` - * ``makefile(sock, mode='r', bufsize=DEFAULT_BUFFER_SIZE) -> - socket file object`` - """ - - def __init__(self, certificate, private_key, certificate_chain=None): - self.certificate = certificate - self.private_key = private_key - self.certificate_chain = certificate_chain - - def wrap(self, sock): - raise NotImplemented - - def makefile(self, sock, mode='r', bufsize=DEFAULT_BUFFER_SIZE): - raise NotImplemented - - -class HTTPServer(object): - - """An HTTP server.""" - - _bind_addr = "127.0.0.1" - _interrupt = None - - gateway = None - """A Gateway instance.""" - - minthreads = None - """The minimum number of worker threads to create (default 10).""" - - maxthreads = None - """The maximum number of worker threads to create (default -1 = no limit). - """ - - server_name = None - """The name of the server; defaults to socket.gethostname().""" - - protocol = "HTTP/1.1" - """The version string to write in the Status-Line of all HTTP responses. - - For example, "HTTP/1.1" is the default. This also limits the supported - features used in the response.""" - - request_queue_size = 5 - """The 'backlog' arg to socket.listen(); max queued connections - (default 5). - """ - - shutdown_timeout = 5 - """The total time, in seconds, to wait for worker threads to cleanly exit. - """ - - timeout = 10 - """The timeout in seconds for accepted connections (default 10).""" - - version = "CherryPy/" + cp_version - """A version string for the HTTPServer.""" - - software = None - """The value to set for the SERVER_SOFTWARE entry in the WSGI environ. - - If None, this defaults to ``'%s Server' % self.version``.""" - - ready = False - """An internal flag which marks whether the socket is accepting - connections. - """ - - max_request_header_size = 0 - """The maximum size, in bytes, for request headers, or 0 for no limit.""" - - max_request_body_size = 0 - """The maximum size, in bytes, for request bodies, or 0 for no limit.""" - - nodelay = True - """If True (the default since 3.1), sets the TCP_NODELAY socket option.""" - - ConnectionClass = HTTPConnection - """The class to use for handling HTTP connections.""" - - ssl_adapter = None - """An instance of SSLAdapter (or a subclass). - - You must have the corresponding SSL driver library installed.""" - - def __init__(self, bind_addr, gateway, minthreads=10, maxthreads=-1, - server_name=None): - self.bind_addr = bind_addr - self.gateway = gateway - - self.requests = ThreadPool(self, min=minthreads or 1, max=maxthreads) - - if not server_name: - server_name = socket.gethostname() - self.server_name = server_name - self.clear_stats() - - def clear_stats(self): - self._start_time = None - self._run_time = 0 - self.stats = { - 'Enabled': False, - 'Bind Address': lambda s: repr(self.bind_addr), - 'Run time': lambda s: (not s['Enabled']) and -1 or self.runtime(), - 'Accepts': 0, - 'Accepts/sec': lambda s: s['Accepts'] / self.runtime(), - 'Queue': lambda s: getattr(self.requests, "qsize", None), - 'Threads': lambda s: len(getattr(self.requests, "_threads", [])), - 'Threads Idle': lambda s: getattr(self.requests, "idle", None), - 'Socket Errors': 0, - 'Requests': lambda s: (not s['Enabled']) and -1 or sum( - [w['Requests'](w) for w in s['Worker Threads'].values()], 0), - 'Bytes Read': lambda s: (not s['Enabled']) and -1 or sum( - [w['Bytes Read'](w) for w in s['Worker Threads'].values()], 0), - 'Bytes Written': lambda s: (not s['Enabled']) and -1 or sum( - [w['Bytes Written'](w) for w in s['Worker Threads'].values()], - 0), - 'Work Time': lambda s: (not s['Enabled']) and -1 or sum( - [w['Work Time'](w) for w in s['Worker Threads'].values()], 0), - 'Read Throughput': lambda s: (not s['Enabled']) and -1 or sum( - [w['Bytes Read'](w) / (w['Work Time'](w) or 1e-6) - for w in s['Worker Threads'].values()], 0), - 'Write Throughput': lambda s: (not s['Enabled']) and -1 or sum( - [w['Bytes Written'](w) / (w['Work Time'](w) or 1e-6) - for w in s['Worker Threads'].values()], 0), - 'Worker Threads': {}, - } - logging.statistics["CherryPy HTTPServer %d" % id(self)] = self.stats - - def runtime(self): - if self._start_time is None: - return self._run_time - else: - return self._run_time + (time.time() - self._start_time) - - def __str__(self): - return "%s.%s(%r)" % (self.__module__, self.__class__.__name__, - self.bind_addr) - - def _get_bind_addr(self): - return self._bind_addr - - def _set_bind_addr(self, value): - if isinstance(value, tuple) and value[0] in ('', None): - # Despite the socket module docs, using '' does not - # allow AI_PASSIVE to work. Passing None instead - # returns '0.0.0.0' like we want. In other words: - # host AI_PASSIVE result - # '' Y 192.168.x.y - # '' N 192.168.x.y - # None Y 0.0.0.0 - # None N 127.0.0.1 - # But since you can get the same effect with an explicit - # '0.0.0.0', we deny both the empty string and None as values. - raise ValueError("Host values of '' or None are not allowed. " - "Use '0.0.0.0' (IPv4) or '::' (IPv6) instead " - "to listen on all active interfaces.") - self._bind_addr = value - bind_addr = property( - _get_bind_addr, - _set_bind_addr, - doc="""The interface on which to listen for connections. - - For TCP sockets, a (host, port) tuple. Host values may be any IPv4 - or IPv6 address, or any valid hostname. The string 'localhost' is a - synonym for '127.0.0.1' (or '::1', if your hosts file prefers IPv6). - The string '0.0.0.0' is a special IPv4 entry meaning "any active - interface" (INADDR_ANY), and '::' is the similar IN6ADDR_ANY for - IPv6. The empty string or None are not allowed. - - For UNIX sockets, supply the filename as a string.""") - - def start(self): - """Run the server forever.""" - # We don't have to trap KeyboardInterrupt or SystemExit here, - # because cherrpy.server already does so, calling self.stop() for us. - # If you're using this server with another framework, you should - # trap those exceptions in whatever code block calls start(). - self._interrupt = None - - if self.software is None: - self.software = "%s Server" % self.version - - # Select the appropriate socket - if isinstance(self.bind_addr, basestring): - # AF_UNIX socket - - # So we can reuse the socket... - try: - os.unlink(self.bind_addr) - except: - pass - - # So everyone can access the socket... - try: - os.chmod(self.bind_addr, 0o777) - except: - pass - - info = [ - (socket.AF_UNIX, socket.SOCK_STREAM, 0, "", self.bind_addr)] - else: - # AF_INET or AF_INET6 socket - # Get the correct address family for our host (allows IPv6 - # addresses) - host, port = self.bind_addr - try: - info = socket.getaddrinfo(host, port, socket.AF_UNSPEC, - socket.SOCK_STREAM, 0, - socket.AI_PASSIVE) - except socket.gaierror: - if ':' in self.bind_addr[0]: - info = [(socket.AF_INET6, socket.SOCK_STREAM, - 0, "", self.bind_addr + (0, 0))] - else: - info = [(socket.AF_INET, socket.SOCK_STREAM, - 0, "", self.bind_addr)] - - self.socket = None - msg = "No socket could be created" - for res in info: - af, socktype, proto, canonname, sa = res - try: - self.bind(af, socktype, proto) - except socket.error as serr: - msg = "%s -- (%s: %s)" % (msg, sa, serr) - if self.socket: - self.socket.close() - self.socket = None - continue - break - if not self.socket: - raise socket.error(msg) - - # Timeout so KeyboardInterrupt can be caught on Win32 - self.socket.settimeout(1) - self.socket.listen(self.request_queue_size) - - # Create worker threads - self.requests.start() - - self.ready = True - self._start_time = time.time() - while self.ready: - try: - self.tick() - except (KeyboardInterrupt, SystemExit): - raise - except: - self.error_log("Error in HTTPServer.tick", level=logging.ERROR, - traceback=True) - if self.interrupt: - while self.interrupt is True: - # Wait for self.stop() to complete. See _set_interrupt. - time.sleep(0.1) - if self.interrupt: - raise self.interrupt - - def error_log(self, msg="", level=20, traceback=False): - # Override this in subclasses as desired - sys.stderr.write(msg + '\n') - sys.stderr.flush() - if traceback: - tblines = traceback_.format_exc() - sys.stderr.write(tblines) - sys.stderr.flush() - - def bind(self, family, type, proto=0): - """Create (or recreate) the actual socket object.""" - self.socket = socket.socket(family, type, proto) - prevent_socket_inheritance(self.socket) - self.socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) - if self.nodelay and not isinstance(self.bind_addr, str): - self.socket.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1) - - if self.ssl_adapter is not None: - self.socket = self.ssl_adapter.bind(self.socket) - - # If listening on the IPV6 any address ('::' = IN6ADDR_ANY), - # activate dual-stack. See - # https://github.com/cherrypy/cherrypy/issues/871. - if (hasattr(socket, 'AF_INET6') and family == socket.AF_INET6 - and self.bind_addr[0] in ('::', '::0', '::0.0.0.0')): - try: - self.socket.setsockopt( - socket.IPPROTO_IPV6, socket.IPV6_V6ONLY, 0) - except (AttributeError, socket.error): - # Apparently, the socket option is not available in - # this machine's TCP stack - pass - - self.socket.bind(self.bind_addr) - - def tick(self): - """Accept a new connection and put it on the Queue.""" - try: - s, addr = self.socket.accept() - if self.stats['Enabled']: - self.stats['Accepts'] += 1 - if not self.ready: - return - - prevent_socket_inheritance(s) - if hasattr(s, 'settimeout'): - s.settimeout(self.timeout) - - makefile = CP_makefile - ssl_env = {} - # if ssl cert and key are set, we try to be a secure HTTP server - if self.ssl_adapter is not None: - try: - s, ssl_env = self.ssl_adapter.wrap(s) - except NoSSLError: - msg = ("The client sent a plain HTTP request, but " - "this server only speaks HTTPS on this port.") - buf = ["%s 400 Bad Request\r\n" % self.protocol, - "Content-Length: %s\r\n" % len(msg), - "Content-Type: text/plain\r\n\r\n", - msg] - - wfile = makefile(s, "wb", DEFAULT_BUFFER_SIZE) - try: - wfile.write("".join(buf).encode('ISO-8859-1')) - except socket.error: - x = sys.exc_info()[1] - if x.args[0] not in socket_errors_to_ignore: - raise - return - if not s: - return - makefile = self.ssl_adapter.makefile - # Re-apply our timeout since we may have a new socket object - if hasattr(s, 'settimeout'): - s.settimeout(self.timeout) - - conn = self.ConnectionClass(self, s, makefile) - - if not isinstance(self.bind_addr, basestring): - # optional values - # Until we do DNS lookups, omit REMOTE_HOST - if addr is None: # sometimes this can happen - # figure out if AF_INET or AF_INET6. - if len(s.getsockname()) == 2: - # AF_INET - addr = ('0.0.0.0', 0) - else: - # AF_INET6 - addr = ('::', 0) - conn.remote_addr = addr[0] - conn.remote_port = addr[1] - - conn.ssl_env = ssl_env - - try: - self.requests.put(conn) - except queue.Full: - # Just drop the conn. TODO: write 503 back? - conn.close() - return - except socket.timeout: - # The only reason for the timeout in start() is so we can - # notice keyboard interrupts on Win32, which don't interrupt - # accept() by default - return - except socket.error: - x = sys.exc_info()[1] - if self.stats['Enabled']: - self.stats['Socket Errors'] += 1 - if x.args[0] in socket_error_eintr: - # I *think* this is right. EINTR should occur when a signal - # is received during the accept() call; all docs say retry - # the call, and I *think* I'm reading it right that Python - # will then go ahead and poll for and handle the signal - # elsewhere. See - # https://github.com/cherrypy/cherrypy/issues/707. - return - if x.args[0] in socket_errors_nonblocking: - # Just try again. See - # https://github.com/cherrypy/cherrypy/issues/479. - return - if x.args[0] in socket_errors_to_ignore: - # Our socket was closed. - # See https://github.com/cherrypy/cherrypy/issues/686. - return - raise - - def _get_interrupt(self): - return self._interrupt - - def _set_interrupt(self, interrupt): - self._interrupt = True - self.stop() - self._interrupt = interrupt - interrupt = property(_get_interrupt, _set_interrupt, - doc="Set this to an Exception instance to " - "interrupt the server.") - - def stop(self): - """Gracefully shutdown a server that is serving forever.""" - self.ready = False - if self._start_time is not None: - self._run_time += (time.time() - self._start_time) - self._start_time = None - - sock = getattr(self, "socket", None) - if sock: - if not isinstance(self.bind_addr, basestring): - # Touch our own socket to make accept() return immediately. - try: - host, port = sock.getsockname()[:2] - except socket.error: - x = sys.exc_info()[1] - if x.args[0] not in socket_errors_to_ignore: - # Changed to use error code and not message - # See - # https://github.com/cherrypy/cherrypy/issues/860. - raise - else: - # Note that we're explicitly NOT using AI_PASSIVE, - # here, because we want an actual IP to touch. - # localhost won't work if we've bound to a public IP, - # but it will if we bound to '0.0.0.0' (INADDR_ANY). - for res in socket.getaddrinfo(host, port, socket.AF_UNSPEC, - socket.SOCK_STREAM): - af, socktype, proto, canonname, sa = res - s = None - try: - s = socket.socket(af, socktype, proto) - # See - # http://groups.google.com/group/cherrypy-users/ - # browse_frm/thread/bbfe5eb39c904fe0 - s.settimeout(1.0) - s.connect((host, port)) - s.close() - except socket.error: - if s: - s.close() - if hasattr(sock, "close"): - sock.close() - self.socket = None - - self.requests.stop(self.shutdown_timeout) - - -class Gateway(object): - - """A base class to interface HTTPServer with other systems, such as WSGI. - """ - - def __init__(self, req): - self.req = req - - def respond(self): - """Process the current request. Must be overridden in a subclass.""" - raise NotImplemented - - -# These may either be wsgiserver.SSLAdapter subclasses or the string names -# of such classes (in which case they will be lazily loaded). -ssl_adapters = { - 'builtin': 'cherrypy.wsgiserver.ssl_builtin.BuiltinSSLAdapter', - 'pyopenssl': 'cherrypy.wsgiserver.ssl_pyopenssl.pyOpenSSLAdapter', -} - - -def get_ssl_adapter_class(name='builtin'): - """Return an SSL adapter class for the given name.""" - adapter = ssl_adapters[name.lower()] - if isinstance(adapter, basestring): - last_dot = adapter.rfind(".") - attr_name = adapter[last_dot + 1:] - mod_path = adapter[:last_dot] - - try: - mod = sys.modules[mod_path] - if mod is None: - raise KeyError() - except KeyError: - # The last [''] is important. - mod = __import__(mod_path, globals(), locals(), ['']) - - # Let an AttributeError propagate outward. - try: - adapter = getattr(mod, attr_name) - except AttributeError: - raise AttributeError("'%s' object has no attribute '%s'" - % (mod_path, attr_name)) - - return adapter - -# ------------------------------- WSGI Stuff -------------------------------- # - - -class CherryPyWSGIServer(HTTPServer): - - """A subclass of HTTPServer which calls a WSGI application.""" - - wsgi_version = (1, 0) - """The version of WSGI to produce.""" - - def __init__(self, bind_addr, wsgi_app, numthreads=10, server_name=None, - max=-1, request_queue_size=5, timeout=10, shutdown_timeout=5, - accepted_queue_size=-1, accepted_queue_timeout=10): - self.requests = ThreadPool(self, min=numthreads or 1, max=max, - accepted_queue_size=accepted_queue_size, - accepted_queue_timeout=accepted_queue_timeout) - self.wsgi_app = wsgi_app - self.gateway = wsgi_gateways[self.wsgi_version] - - self.bind_addr = bind_addr - if not server_name: - server_name = socket.gethostname() - self.server_name = server_name - self.request_queue_size = request_queue_size - - self.timeout = timeout - self.shutdown_timeout = shutdown_timeout - self.clear_stats() - - def _get_numthreads(self): - return self.requests.min - - def _set_numthreads(self, value): - self.requests.min = value - numthreads = property(_get_numthreads, _set_numthreads) - - -class WSGIGateway(Gateway): - - """A base class to interface HTTPServer with WSGI.""" - - def __init__(self, req): - self.req = req - self.started_response = False - self.env = self.get_environ() - self.remaining_bytes_out = None - - def get_environ(self): - """Return a new environ dict targeting the given wsgi.version""" - raise NotImplemented - - def respond(self): - """Process the current request.""" - response = self.req.server.wsgi_app(self.env, self.start_response) - try: - for chunk in response: - # "The start_response callable must not actually transmit - # the response headers. Instead, it must store them for the - # server or gateway to transmit only after the first - # iteration of the application return value that yields - # a NON-EMPTY string, or upon the application's first - # invocation of the write() callable." (PEP 333) - if chunk: - if isinstance(chunk, unicodestr): - chunk = chunk.encode('ISO-8859-1') - self.write(chunk) - finally: - if hasattr(response, "close"): - response.close() - - def start_response(self, status, headers, exc_info=None): - """WSGI callable to begin the HTTP response.""" - # "The application may call start_response more than once, - # if and only if the exc_info argument is provided." - if self.started_response and not exc_info: - raise AssertionError("WSGI start_response called a second " - "time with no exc_info.") - self.started_response = True - - # "if exc_info is provided, and the HTTP headers have already been - # sent, start_response must raise an error, and should raise the - # exc_info tuple." - if self.req.sent_headers: - try: - raise exc_info[0](exc_info[1]).with_traceback(exc_info[2]) - finally: - exc_info = None - - # According to PEP 3333, when using Python 3, the response status - # and headers must be bytes masquerading as unicode; that is, they - # must be of type "str" but are restricted to code points in the - # "latin-1" set. - if not isinstance(status, str): - raise TypeError("WSGI response status is not of type str.") - self.req.status = status.encode('ISO-8859-1') - - for k, v in headers: - if not isinstance(k, str): - raise TypeError( - "WSGI response header key %r is not of type str." % k) - if not isinstance(v, str): - raise TypeError( - "WSGI response header value %r is not of type str." % v) - if k.lower() == 'content-length': - self.remaining_bytes_out = int(v) - self.req.outheaders.append( - (k.encode('ISO-8859-1'), v.encode('ISO-8859-1'))) - - return self.write - - def write(self, chunk): - """WSGI callable to write unbuffered data to the client. - - This method is also used internally by start_response (to write - data from the iterable returned by the WSGI application). - """ - if not self.started_response: - raise AssertionError("WSGI write called before start_response.") - - chunklen = len(chunk) - rbo = self.remaining_bytes_out - if rbo is not None and chunklen > rbo: - if not self.req.sent_headers: - # Whew. We can send a 500 to the client. - self.req.simple_response("500 Internal Server Error", - "The requested resource returned " - "more bytes than the declared " - "Content-Length.") - else: - # Dang. We have probably already sent data. Truncate the chunk - # to fit (so the client doesn't hang) and raise an error later. - chunk = chunk[:rbo] - - if not self.req.sent_headers: - self.req.sent_headers = True - self.req.send_headers() - - self.req.write(chunk) - - if rbo is not None: - rbo -= chunklen - if rbo < 0: - raise ValueError( - "Response body exceeds the declared Content-Length.") - - -class WSGIGateway_10(WSGIGateway): - - """A Gateway class to interface HTTPServer with WSGI 1.0.x.""" - - def get_environ(self): - """Return a new environ dict targeting the given wsgi.version""" - req = self.req - env = { - # set a non-standard environ entry so the WSGI app can know what - # the *real* server protocol is (and what features to support). - # See http://www.faqs.org/rfcs/rfc2145.html. - 'ACTUAL_SERVER_PROTOCOL': req.server.protocol, - 'PATH_INFO': req.path.decode('ISO-8859-1'), - 'QUERY_STRING': req.qs.decode('ISO-8859-1'), - 'REMOTE_ADDR': req.conn.remote_addr or '', - 'REMOTE_PORT': str(req.conn.remote_port or ''), - 'REQUEST_METHOD': req.method.decode('ISO-8859-1'), - 'REQUEST_URI': req.uri.decode('ISO-8859-1'), - 'SCRIPT_NAME': '', - 'SERVER_NAME': req.server.server_name, - # Bah. "SERVER_PROTOCOL" is actually the REQUEST protocol. - 'SERVER_PROTOCOL': req.request_protocol.decode('ISO-8859-1'), - 'SERVER_SOFTWARE': req.server.software, - 'wsgi.errors': sys.stderr, - 'wsgi.input': req.rfile, - 'wsgi.multiprocess': False, - 'wsgi.multithread': True, - 'wsgi.run_once': False, - 'wsgi.url_scheme': req.scheme.decode('ISO-8859-1'), - 'wsgi.version': (1, 0), - } - if isinstance(req.server.bind_addr, basestring): - # AF_UNIX. This isn't really allowed by WSGI, which doesn't - # address unix domain sockets. But it's better than nothing. - env["SERVER_PORT"] = "" - else: - env["SERVER_PORT"] = str(req.server.bind_addr[1]) - - # Request headers - for k, v in req.inheaders.items(): - k = k.decode('ISO-8859-1').upper().replace("-", "_") - env["HTTP_" + k] = v.decode('ISO-8859-1') - - # CONTENT_TYPE/CONTENT_LENGTH - ct = env.pop("HTTP_CONTENT_TYPE", None) - if ct is not None: - env["CONTENT_TYPE"] = ct - cl = env.pop("HTTP_CONTENT_LENGTH", None) - if cl is not None: - env["CONTENT_LENGTH"] = cl - - if req.conn.ssl_env: - env.update(req.conn.ssl_env) - - return env - - -class WSGIGateway_u0(WSGIGateway_10): - - """A Gateway class to interface HTTPServer with WSGI u.0. - - WSGI u.0 is an experimental protocol, which uses unicode for keys - and values in both Python 2 and Python 3. - """ - - def get_environ(self): - """Return a new environ dict targeting the given wsgi.version""" - req = self.req - env_10 = WSGIGateway_10.get_environ(self) - env = env_10.copy() - env['wsgi.version'] = ('u', 0) - - # Request-URI - env.setdefault('wsgi.url_encoding', 'utf-8') - try: - # SCRIPT_NAME is the empty string, who cares what encoding it is? - env["PATH_INFO"] = req.path.decode(env['wsgi.url_encoding']) - env["QUERY_STRING"] = req.qs.decode(env['wsgi.url_encoding']) - except UnicodeDecodeError: - # Fall back to latin 1 so apps can transcode if needed. - env['wsgi.url_encoding'] = 'ISO-8859-1' - env["PATH_INFO"] = env_10["PATH_INFO"] - env["QUERY_STRING"] = env_10["QUERY_STRING"] - - return env - -wsgi_gateways = { - (1, 0): WSGIGateway_10, - ('u', 0): WSGIGateway_u0, -} - - -class WSGIPathInfoDispatcher(object): - - """A WSGI dispatcher for dispatch based on the PATH_INFO. - - apps: a dict or list of (path_prefix, app) pairs. - """ - - def __init__(self, apps): - try: - apps = list(apps.items()) - except AttributeError: - pass - - # Sort the apps by len(path), descending - apps.sort() - apps.reverse() - - # The path_prefix strings must start, but not end, with a slash. - # Use "" instead of "/". - self.apps = [(p.rstrip("/"), a) for p, a in apps] - - def __call__(self, environ, start_response): - path = environ["PATH_INFO"] or "/" - for p, app in self.apps: - # The apps list should be sorted by length, descending. - if path.startswith(p + "/") or path == p: - environ = environ.copy() - environ["SCRIPT_NAME"] = environ["SCRIPT_NAME"] + p - environ["PATH_INFO"] = path[len(p):] - return app(environ, start_response) - - start_response('404 Not Found', [('Content-Type', 'text/plain'), - ('Content-Length', '0')]) - return ['']