Ruud
10 years ago
9 changed files with 1421 additions and 0 deletions
@ -0,0 +1,19 @@ |
|||
"""ndg_httpsclient - PyOpenSSL utility to make a httplib-like interface suitable |
|||
for use with urllib2 |
|||
|
|||
This is a setuptools namespace_package. DO NOT place any other |
|||
code in this file! There is no guarantee that it will be installed |
|||
with easy_install. See: |
|||
|
|||
http://peak.telecommunity.com/DevCenter/setuptools#namespace-packages |
|||
|
|||
... for details. |
|||
""" |
|||
__author__ = "P J Kershaw" |
|||
__date__ = "06/01/12" |
|||
__copyright__ = "(C) 2012 Science and Technology Facilities Council" |
|||
__license__ = "BSD - see LICENSE file in top-level directory" |
|||
__contact__ = "Philip.Kershaw@stfc.ac.uk" |
|||
__revision__ = '$Id$' |
|||
|
|||
__import__('pkg_resources').declare_namespace(__name__) |
|||
@ -0,0 +1,9 @@ |
|||
"""ndg_httpsclient - PyOpenSSL utility to make a httplib-like interface suitable |
|||
for use with urllib2 |
|||
""" |
|||
__author__ = "P J Kershaw (STFC) and Richard Wilkinson (Tessella)" |
|||
__date__ = "09/12/11" |
|||
__copyright__ = "(C) 2011 Science and Technology Facilities Council" |
|||
__license__ = "BSD - see LICENSE file in top-level directory" |
|||
__contact__ = "Philip.Kershaw@stfc.ac.uk" |
|||
__revision__ = '$Id$' |
|||
@ -0,0 +1,131 @@ |
|||
"""ndg_httpsclient HTTPS module containing PyOpenSSL implementation of |
|||
httplib.HTTPSConnection |
|||
|
|||
PyOpenSSL utility to make a httplib-like interface suitable for use with |
|||
urllib2 |
|||
""" |
|||
__author__ = "P J Kershaw (STFC)" |
|||
__date__ = "09/12/11" |
|||
__copyright__ = "(C) 2012 Science and Technology Facilities Council" |
|||
__license__ = "BSD - see LICENSE file in top-level directory" |
|||
__contact__ = "Philip.Kershaw@stfc.ac.uk" |
|||
__revision__ = '$Id$' |
|||
import logging |
|||
import socket |
|||
import sys |
|||
|
|||
if sys.version_info[0] > 2: |
|||
from http.client import HTTPS_PORT |
|||
from http.client import HTTPConnection |
|||
|
|||
from urllib.request import AbstractHTTPHandler |
|||
else: |
|||
from httplib import HTTPS_PORT |
|||
from httplib import HTTPConnection |
|||
|
|||
from urllib2 import AbstractHTTPHandler |
|||
|
|||
|
|||
from OpenSSL import SSL |
|||
|
|||
from ndg.httpsclient.ssl_socket import SSLSocket |
|||
|
|||
log = logging.getLogger(__name__) |
|||
|
|||
|
|||
class HTTPSConnection(HTTPConnection): |
|||
"""This class allows communication via SSL using PyOpenSSL. |
|||
It is based on httplib.HTTPSConnection, modified to use PyOpenSSL. |
|||
|
|||
Note: This uses the constructor inherited from HTTPConnection to allow it to |
|||
be used with httplib and HTTPSContextHandler. To use the class directly with |
|||
an SSL context set ssl_context after construction. |
|||
|
|||
@cvar default_port: default port for this class (443) |
|||
@type default_port: int |
|||
@cvar default_ssl_method: default SSL method used if no SSL context is |
|||
explicitly set - defaults to version 2/3. |
|||
@type default_ssl_method: int |
|||
""" |
|||
default_port = HTTPS_PORT |
|||
default_ssl_method = SSL.SSLv23_METHOD |
|||
|
|||
def __init__(self, host, port=None, strict=None, |
|||
timeout=socket._GLOBAL_DEFAULT_TIMEOUT, ssl_context=None): |
|||
HTTPConnection.__init__(self, host, port, strict, timeout) |
|||
if not hasattr(self, 'ssl_context'): |
|||
self.ssl_context = None |
|||
|
|||
if ssl_context is not None: |
|||
if not isinstance(ssl_context, SSL.Context): |
|||
raise TypeError('Expecting OpenSSL.SSL.Context type for "' |
|||
'ssl_context" keyword; got %r instead' % |
|||
ssl_context) |
|||
|
|||
self.ssl_context = ssl_context |
|||
|
|||
def connect(self): |
|||
"""Create SSL socket and connect to peer |
|||
""" |
|||
if getattr(self, 'ssl_context', None): |
|||
if not isinstance(self.ssl_context, SSL.Context): |
|||
raise TypeError('Expecting OpenSSL.SSL.Context type for "' |
|||
'ssl_context" attribute; got %r instead' % |
|||
self.ssl_context) |
|||
ssl_context = self.ssl_context |
|||
else: |
|||
ssl_context = SSL.Context(self.__class__.default_ssl_method) |
|||
|
|||
sock = socket.create_connection((self.host, self.port), self.timeout) |
|||
|
|||
# Tunnel if using a proxy - ONLY available for Python 2.6.2 and above |
|||
if getattr(self, '_tunnel_host', None): |
|||
self.sock = sock |
|||
self._tunnel() |
|||
|
|||
self.sock = SSLSocket(ssl_context, sock) |
|||
|
|||
# Go to client mode. |
|||
self.sock.set_connect_state() |
|||
|
|||
def close(self): |
|||
"""Close socket and shut down SSL connection""" |
|||
if hasattr(self.sock, "close"): |
|||
self.sock.close() |
|||
|
|||
|
|||
class HTTPSContextHandler(AbstractHTTPHandler): |
|||
'''HTTPS handler that allows a SSL context to be set for the SSL |
|||
connections. |
|||
''' |
|||
https_request = AbstractHTTPHandler.do_request_ |
|||
|
|||
def __init__(self, ssl_context, debuglevel=0): |
|||
""" |
|||
@param ssl_context:SSL context |
|||
@type ssl_context: OpenSSL.SSL.Context |
|||
@param debuglevel: debug level for HTTPSHandler |
|||
@type debuglevel: int |
|||
""" |
|||
AbstractHTTPHandler.__init__(self, debuglevel) |
|||
|
|||
if ssl_context is not None: |
|||
if not isinstance(ssl_context, SSL.Context): |
|||
raise TypeError('Expecting OpenSSL.SSL.Context type for "' |
|||
'ssl_context" keyword; got %r instead' % |
|||
ssl_context) |
|||
self.ssl_context = ssl_context |
|||
else: |
|||
self.ssl_context = SSL.Context(SSL.TLSv1_METHOD) |
|||
|
|||
def https_open(self, req): |
|||
"""Opens HTTPS request |
|||
@param req: HTTP request |
|||
@return: HTTP Response object |
|||
""" |
|||
# Make a custom class extending HTTPSConnection, with the SSL context |
|||
# set as a class variable so that it is available to the connect method. |
|||
customHTTPSContextConnection = type('CustomHTTPSContextConnection', |
|||
(HTTPSConnection, object), |
|||
{'ssl_context': self.ssl_context}) |
|||
return self.do_open(customHTTPSContextConnection, req) |
|||
@ -0,0 +1,98 @@ |
|||
"""ndg_httpsclient SSL Context utilities module containing convenience routines |
|||
for setting SSL context configuration. |
|||
|
|||
""" |
|||
__author__ = "P J Kershaw (STFC)" |
|||
__date__ = "09/12/11" |
|||
__copyright__ = "(C) 2012 Science and Technology Facilities Council" |
|||
__license__ = "BSD - see LICENSE file in top-level directory" |
|||
__contact__ = "Philip.Kershaw@stfc.ac.uk" |
|||
__revision__ = '$Id$' |
|||
import sys |
|||
|
|||
if sys.version_info[0] > 2: |
|||
import urllib.parse as urlparse_ |
|||
else: |
|||
import urlparse as urlparse_ |
|||
|
|||
from OpenSSL import SSL |
|||
|
|||
from ndg.httpsclient.ssl_peer_verification import ServerSSLCertVerification |
|||
|
|||
|
|||
class SSlContextConfig(object): |
|||
""" |
|||
Holds configuration options for creating a SSL context. This is used as a |
|||
template to create the contexts with specific verification callbacks. |
|||
""" |
|||
def __init__(self, key_file=None, cert_file=None, pem_file=None, ca_dir=None, |
|||
verify_peer=False): |
|||
self.key_file = key_file |
|||
self.cert_file = cert_file |
|||
self.pem_file = pem_file |
|||
self.ca_dir = ca_dir |
|||
self.verify_peer = verify_peer |
|||
|
|||
|
|||
def make_ssl_context_from_config(ssl_config=False, url=None): |
|||
return make_ssl_context(ssl_config.key_file, ssl_config.cert_file, |
|||
ssl_config.pem_file, ssl_config.ca_dir, |
|||
ssl_config.verify_peer, url) |
|||
|
|||
|
|||
def make_ssl_context(key_file=None, cert_file=None, pem_file=None, ca_dir=None, |
|||
verify_peer=False, url=None, method=SSL.TLSv1_METHOD, |
|||
key_file_passphrase=None): |
|||
""" |
|||
Creates SSL context containing certificate and key file locations. |
|||
""" |
|||
ssl_context = SSL.Context(method) |
|||
|
|||
# Key file defaults to certificate file if present. |
|||
if cert_file: |
|||
ssl_context.use_certificate_file(cert_file) |
|||
|
|||
if key_file_passphrase: |
|||
passwd_cb = lambda max_passphrase_len, set_prompt, userdata: \ |
|||
key_file_passphrase |
|||
ssl_context.set_passwd_cb(passwd_cb) |
|||
|
|||
if key_file: |
|||
ssl_context.use_privatekey_file(key_file) |
|||
elif cert_file: |
|||
ssl_context.use_privatekey_file(cert_file) |
|||
|
|||
if pem_file or ca_dir: |
|||
ssl_context.load_verify_locations(pem_file, ca_dir) |
|||
|
|||
def _callback(conn, x509, errnum, errdepth, preverify_ok): |
|||
"""Default certification verification callback. |
|||
Performs no checks and returns the status passed in. |
|||
""" |
|||
return preverify_ok |
|||
|
|||
verify_callback = _callback |
|||
|
|||
if verify_peer: |
|||
ssl_context.set_verify_depth(9) |
|||
if url: |
|||
set_peer_verification_for_url_hostname(ssl_context, url) |
|||
else: |
|||
ssl_context.set_verify(SSL.VERIFY_PEER, verify_callback) |
|||
else: |
|||
ssl_context.set_verify(SSL.VERIFY_NONE, verify_callback) |
|||
|
|||
return ssl_context |
|||
|
|||
|
|||
def set_peer_verification_for_url_hostname(ssl_context, url, |
|||
if_verify_enabled=False): |
|||
'''Convenience routine to set peer verification callback based on |
|||
ServerSSLCertVerification class''' |
|||
if not if_verify_enabled or (ssl_context.get_verify_mode() & SSL.VERIFY_PEER): |
|||
urlObj = urlparse_.urlparse(url) |
|||
hostname = urlObj.hostname |
|||
server_ssl_cert_verif = ServerSSLCertVerification(hostname=hostname) |
|||
verify_callback_ = server_ssl_cert_verif.get_verify_server_cert_func() |
|||
ssl_context.set_verify(SSL.VERIFY_PEER, verify_callback_) |
|||
|
|||
@ -0,0 +1,237 @@ |
|||
"""ndg_httpsclient - module containing SSL peer verification class. |
|||
""" |
|||
__author__ = "P J Kershaw (STFC)" |
|||
__date__ = "09/12/11" |
|||
__copyright__ = "(C) 2012 Science and Technology Facilities Council" |
|||
__license__ = "BSD - see LICENSE file in top-level directory" |
|||
__contact__ = "Philip.Kershaw@stfc.ac.uk" |
|||
__revision__ = '$Id$' |
|||
import re |
|||
import logging |
|||
log = logging.getLogger(__name__) |
|||
|
|||
try: |
|||
from ndg.httpsclient.subj_alt_name import SubjectAltName |
|||
from pyasn1.codec.der import decoder as der_decoder |
|||
SUBJ_ALT_NAME_SUPPORT = True |
|||
|
|||
except ImportError as e: |
|||
SUBJ_ALT_NAME_SUPPORT = False |
|||
SUBJ_ALT_NAME_SUPPORT_MSG = ( |
|||
'SubjectAltName support is disabled - check pyasn1 package ' |
|||
'installation to enable' |
|||
) |
|||
import warnings |
|||
warnings.warn(SUBJ_ALT_NAME_SUPPORT_MSG) |
|||
|
|||
|
|||
class ServerSSLCertVerification(object): |
|||
"""Check server identity. If hostname doesn't match, allow match of |
|||
host's Distinguished Name against server DN setting""" |
|||
DN_LUT = { |
|||
'commonName': 'CN', |
|||
'organisationalUnitName': 'OU', |
|||
'organisation': 'O', |
|||
'countryName': 'C', |
|||
'emailAddress': 'EMAILADDRESS', |
|||
'localityName': 'L', |
|||
'stateOrProvinceName': 'ST', |
|||
'streetAddress': 'STREET', |
|||
'domainComponent': 'DC', |
|||
'userid': 'UID' |
|||
} |
|||
SUBJ_ALT_NAME_EXT_NAME = 'subjectAltName' |
|||
PARSER_RE_STR = '/(%s)=' % '|'.join(list(DN_LUT.keys()) + list(DN_LUT.values())) |
|||
PARSER_RE = re.compile(PARSER_RE_STR) |
|||
|
|||
__slots__ = ('__hostname', '__certDN', '__subj_alt_name_match') |
|||
|
|||
def __init__(self, certDN=None, hostname=None, subj_alt_name_match=True): |
|||
"""Override parent class __init__ to enable setting of certDN |
|||
setting |
|||
|
|||
@type certDN: string |
|||
@param certDN: Set the expected Distinguished Name of the |
|||
server to avoid errors matching hostnames. This is useful |
|||
where the hostname is not fully qualified |
|||
@type hostname: string |
|||
@param hostname: hostname to match against peer certificate |
|||
subjectAltNames or subject common name |
|||
@type subj_alt_name_match: bool |
|||
@param subj_alt_name_match: flag to enable/disable matching of hostname |
|||
against peer certificate subjectAltNames. Nb. A setting of True will |
|||
be ignored if the pyasn1 package is not installed |
|||
""" |
|||
self.__certDN = None |
|||
self.__hostname = None |
|||
|
|||
if certDN is not None: |
|||
self.certDN = certDN |
|||
|
|||
if hostname is not None: |
|||
self.hostname = hostname |
|||
|
|||
if subj_alt_name_match: |
|||
if not SUBJ_ALT_NAME_SUPPORT: |
|||
log.warning('Overriding "subj_alt_name_match" keyword setting: ' |
|||
'peer verification with subjectAltNames is disabled') |
|||
self.__subj_alt_name_match = False |
|||
else: |
|||
self.__subj_alt_name_match = True |
|||
else: |
|||
log.debug('Disabling peer verification with subject ' |
|||
'subjectAltNames!') |
|||
self.__subj_alt_name_match = False |
|||
|
|||
def __call__(self, connection, peerCert, errorStatus, errorDepth, |
|||
preverifyOK): |
|||
"""Verify server certificate |
|||
|
|||
@type connection: OpenSSL.SSL.Connection |
|||
@param connection: SSL connection object |
|||
@type peerCert: basestring |
|||
@param peerCert: server host certificate as OpenSSL.crypto.X509 |
|||
instance |
|||
@type errorStatus: int |
|||
@param errorStatus: error status passed from caller. This is the value |
|||
returned by the OpenSSL C function X509_STORE_CTX_get_error(). Look-up |
|||
x509_vfy.h in the OpenSSL source to get the meanings of the different |
|||
codes. PyOpenSSL doesn't help you! |
|||
@type errorDepth: int |
|||
@param errorDepth: a non-negative integer representing where in the |
|||
certificate chain the error occurred. If it is zero it occured in the |
|||
end entity certificate, one if it is the certificate which signed the |
|||
end entity certificate and so on. |
|||
|
|||
@type preverifyOK: int |
|||
@param preverifyOK: the error status - 0 = Error, 1 = OK of the current |
|||
SSL context irrespective of any verification checks done here. If this |
|||
function yields an OK status, it should enforce the preverifyOK value |
|||
so that any error set upstream overrides and is honoured. |
|||
@rtype: int |
|||
@return: status code - 0/False = Error, 1/True = OK |
|||
""" |
|||
if peerCert.has_expired(): |
|||
# Any expired certificate in the chain should result in an error |
|||
log.error('Certificate %r in peer certificate chain has expired', |
|||
peerCert.get_subject()) |
|||
|
|||
return False |
|||
|
|||
elif errorDepth == 0: |
|||
# Only interested in DN of last certificate in the chain - this must |
|||
# match the expected Server DN setting |
|||
peerCertSubj = peerCert.get_subject() |
|||
peerCertDN = peerCertSubj.get_components() |
|||
peerCertDN.sort() |
|||
|
|||
if self.certDN is None: |
|||
# Check hostname against peer certificate CN field instead: |
|||
if self.hostname is None: |
|||
log.error('No "hostname" or "certDN" set to check peer ' |
|||
'certificate against') |
|||
return False |
|||
|
|||
# Check for subject alternative names |
|||
if self.__subj_alt_name_match: |
|||
dns_names = self._get_subj_alt_name(peerCert) |
|||
if self.hostname in dns_names: |
|||
return preverifyOK |
|||
|
|||
# If no subjectAltNames, default to check of subject Common Name |
|||
if peerCertSubj.commonName == self.hostname: |
|||
return preverifyOK |
|||
else: |
|||
log.error('Peer certificate CN %r doesn\'t match the ' |
|||
'expected CN %r', peerCertSubj.commonName, |
|||
self.hostname) |
|||
return False |
|||
else: |
|||
if peerCertDN == self.certDN: |
|||
return preverifyOK |
|||
else: |
|||
log.error('Peer certificate DN %r doesn\'t match the ' |
|||
'expected DN %r', peerCertDN, self.certDN) |
|||
return False |
|||
else: |
|||
return preverifyOK |
|||
|
|||
def get_verify_server_cert_func(self): |
|||
def verify_server_cert(connection, peerCert, errorStatus, errorDepth, |
|||
preverifyOK): |
|||
return self.__call__(connection, peerCert, errorStatus, |
|||
errorDepth, preverifyOK) |
|||
|
|||
return verify_server_cert |
|||
|
|||
@classmethod |
|||
def _get_subj_alt_name(cls, peer_cert): |
|||
'''Extract subjectAltName DNS name settings from certificate extensions |
|||
|
|||
@param peer_cert: peer certificate in SSL connection. subjectAltName |
|||
settings if any will be extracted from this |
|||
@type peer_cert: OpenSSL.crypto.X509 |
|||
''' |
|||
# Search through extensions |
|||
dns_name = [] |
|||
general_names = SubjectAltName() |
|||
for i in range(peer_cert.get_extension_count()): |
|||
ext = peer_cert.get_extension(i) |
|||
ext_name = ext.get_short_name() |
|||
if ext_name == cls.SUBJ_ALT_NAME_EXT_NAME: |
|||
# PyOpenSSL returns extension data in ASN.1 encoded form |
|||
ext_dat = ext.get_data() |
|||
decoded_dat = der_decoder.decode(ext_dat, |
|||
asn1Spec=general_names) |
|||
|
|||
for name in decoded_dat: |
|||
if isinstance(name, SubjectAltName): |
|||
for entry in range(len(name)): |
|||
component = name.getComponentByPosition(entry) |
|||
dns_name.append(str(component.getComponent())) |
|||
|
|||
return dns_name |
|||
|
|||
def _getCertDN(self): |
|||
return self.__certDN |
|||
|
|||
def _setCertDN(self, val): |
|||
if isinstance(val, str): |
|||
# Allow for quoted DN |
|||
certDN = val.strip('"') |
|||
|
|||
dnFields = self.__class__.PARSER_RE.split(certDN) |
|||
if len(dnFields) < 2: |
|||
raise TypeError('Error parsing DN string: "%s"' % certDN) |
|||
|
|||
self.__certDN = list(zip(dnFields[1::2], dnFields[2::2])) |
|||
self.__certDN.sort() |
|||
|
|||
elif not isinstance(val, list): |
|||
for i in val: |
|||
if not len(i) == 2: |
|||
raise TypeError('Expecting list of two element DN field, ' |
|||
'DN field value pairs for "certDN" ' |
|||
'attribute') |
|||
self.__certDN = val |
|||
else: |
|||
raise TypeError('Expecting list or string type for "certDN" ' |
|||
'attribute') |
|||
|
|||
certDN = property(fget=_getCertDN, |
|||
fset=_setCertDN, |
|||
doc="Distinguished Name for Server Certificate") |
|||
|
|||
# Get/Set Property methods |
|||
def _getHostname(self): |
|||
return self.__hostname |
|||
|
|||
def _setHostname(self, val): |
|||
if not isinstance(val, str): |
|||
raise TypeError("Expecting string type for hostname " |
|||
"attribute") |
|||
self.__hostname = val |
|||
|
|||
hostname = property(fget=_getHostname, |
|||
fset=_setHostname, |
|||
doc="hostname of server") |
|||
@ -0,0 +1,282 @@ |
|||
"""PyOpenSSL utilities including HTTPSSocket class which wraps PyOpenSSL |
|||
SSL connection into a httplib-like interface suitable for use with urllib2 |
|||
|
|||
""" |
|||
__author__ = "P J Kershaw" |
|||
__date__ = "21/12/10" |
|||
__copyright__ = "(C) 2012 Science and Technology Facilities Council" |
|||
__license__ = "BSD - see LICENSE file in top-level directory" |
|||
__contact__ = "Philip.Kershaw@stfc.ac.uk" |
|||
__revision__ = '$Id$' |
|||
|
|||
from datetime import datetime |
|||
import logging |
|||
import socket |
|||
from io import BytesIO |
|||
|
|||
from OpenSSL import SSL |
|||
|
|||
log = logging.getLogger(__name__) |
|||
|
|||
|
|||
class SSLSocket(object): |
|||
"""SSL Socket class wraps pyOpenSSL's SSL.Connection class implementing |
|||
the makefile method so that it is compatible with the standard socket |
|||
interface and usable with httplib. |
|||
|
|||
@cvar default_buf_size: default buffer size for recv operations in the |
|||
makefile method |
|||
@type default_buf_size: int |
|||
""" |
|||
default_buf_size = 8192 |
|||
|
|||
def __init__(self, ctx, sock=None): |
|||
"""Create SSL socket object |
|||
|
|||
@param ctx: SSL context |
|||
@type ctx: OpenSSL.SSL.Context |
|||
@param sock: underlying socket object |
|||
@type sock: socket.socket |
|||
""" |
|||
if sock is not None: |
|||
self.socket = sock |
|||
else: |
|||
self.socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) |
|||
self.socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) |
|||
|
|||
self.__ssl_conn = SSL.Connection(ctx, self.socket) |
|||
self.buf_size = self.__class__.default_buf_size |
|||
self._makefile_refs = 0 |
|||
|
|||
def __del__(self): |
|||
"""Close underlying socket when this object goes out of scope |
|||
""" |
|||
self.close() |
|||
|
|||
@property |
|||
def buf_size(self): |
|||
"""Buffer size for makefile method recv() operations""" |
|||
return self.__buf_size |
|||
|
|||
@buf_size.setter |
|||
def buf_size(self, value): |
|||
"""Buffer size for makefile method recv() operations""" |
|||
if not isinstance(value, int): |
|||
raise TypeError('Expecting int type for "buf_size"; ' |
|||
'got %r instead' % type(value)) |
|||
self.__buf_size = value |
|||
|
|||
def close(self): |
|||
"""Shutdown the SSL connection and call the close method of the |
|||
underlying socket""" |
|||
if self._makefile_refs < 1: |
|||
try: |
|||
self.__ssl_conn.shutdown() |
|||
except (SSL.Error, SSL.SysCallError): |
|||
# Make errors on shutdown non-fatal |
|||
pass |
|||
else: |
|||
self._makefile_refs -= 1 |
|||
|
|||
def set_shutdown(self, mode): |
|||
"""Set the shutdown state of the Connection. |
|||
@param mode: bit vector of either or both of SENT_SHUTDOWN and |
|||
RECEIVED_SHUTDOWN |
|||
""" |
|||
self.__ssl_conn.set_shutdown(mode) |
|||
|
|||
def get_shutdown(self): |
|||
"""Get the shutdown state of the Connection. |
|||
@return: bit vector of either or both of SENT_SHUTDOWN and |
|||
RECEIVED_SHUTDOWN |
|||
""" |
|||
return self.__ssl_conn.get_shutdown() |
|||
|
|||
def bind(self, addr): |
|||
"""bind to the given address - calls method of the underlying socket |
|||
@param addr: address/port number tuple |
|||
@type addr: tuple""" |
|||
self.__ssl_conn.bind(addr) |
|||
|
|||
def listen(self, backlog): |
|||
"""Listen for connections made to the socket. |
|||
|
|||
@param backlog: specifies the maximum number of queued connections and |
|||
should be at least 1; the maximum value is system-dependent (usually 5). |
|||
@param backlog: int |
|||
""" |
|||
self.__ssl_conn.listen(backlog) |
|||
|
|||
def set_accept_state(self): |
|||
"""Set the connection to work in server mode. The handshake will be |
|||
handled automatically by read/write""" |
|||
self.__ssl_conn.set_accept_state() |
|||
|
|||
def accept(self): |
|||
"""Accept an SSL connection. |
|||
|
|||
@return: pair (ssl, addr) where ssl is a new SSL connection object and |
|||
addr is the address bound to the other end of the SSL connection. |
|||
@rtype: tuple |
|||
""" |
|||
return self.__ssl_conn.accept() |
|||
|
|||
def set_connect_state(self): |
|||
"""Set the connection to work in client mode. The handshake will be |
|||
handled automatically by read/write""" |
|||
self.__ssl_conn.set_connect_state() |
|||
|
|||
def connect(self, addr): |
|||
"""Call the connect method of the underlying socket and set up SSL on |
|||
the socket, using the Context object supplied to this Connection object |
|||
at creation. |
|||
|
|||
@param addr: address/port number pair |
|||
@type addr: tuple |
|||
""" |
|||
self.__ssl_conn.connect(addr) |
|||
|
|||
def shutdown(self, how): |
|||
"""Send the shutdown message to the Connection. |
|||
|
|||
@param how: for socket.socket this flag determines whether read, write |
|||
or both type operations are supported. OpenSSL.SSL.Connection doesn't |
|||
support this so this parameter is IGNORED |
|||
@return: true if the shutdown message exchange is completed and false |
|||
otherwise (in which case you call recv() or send() when the connection |
|||
becomes readable/writeable. |
|||
@rtype: bool |
|||
""" |
|||
return self.__ssl_conn.shutdown() |
|||
|
|||
def renegotiate(self): |
|||
"""Renegotiate this connection's SSL parameters.""" |
|||
return self.__ssl_conn.renegotiate() |
|||
|
|||
def pending(self): |
|||
"""@return: numbers of bytes that can be safely read from the SSL |
|||
buffer. |
|||
@rtype: int |
|||
""" |
|||
return self.__ssl_conn.pending() |
|||
|
|||
def send(self, data, *flags_arg): |
|||
"""Send data to the socket. Nb. The optional flags argument is ignored. |
|||
- retained for compatibility with socket.socket interface |
|||
|
|||
@param data: data to send down the socket |
|||
@type data: string |
|||
""" |
|||
return self.__ssl_conn.send(data) |
|||
|
|||
def sendall(self, data): |
|||
self.__ssl_conn.sendall(data) |
|||
|
|||
def recv(self, size=default_buf_size): |
|||
"""Receive data from the Connection. |
|||
|
|||
@param size: The maximum amount of data to be received at once |
|||
@type size: int |
|||
@return: data received. |
|||
@rtype: string |
|||
""" |
|||
return self.__ssl_conn.recv(size) |
|||
|
|||
def setblocking(self, mode): |
|||
"""Set this connection's underlying socket blocking _mode_. |
|||
|
|||
@param mode: blocking mode |
|||
@type mode: int |
|||
""" |
|||
self.__ssl_conn.setblocking(mode) |
|||
|
|||
def fileno(self): |
|||
""" |
|||
@return: file descriptor number for the underlying socket |
|||
@rtype: int |
|||
""" |
|||
return self.__ssl_conn.fileno() |
|||
|
|||
def getsockopt(self, *args): |
|||
"""See socket.socket.getsockopt |
|||
""" |
|||
return self.__ssl_conn.getsockopt(*args) |
|||
|
|||
def setsockopt(self, *args): |
|||
"""See socket.socket.setsockopt |
|||
|
|||
@return: value of the given socket option |
|||
@rtype: int/string |
|||
""" |
|||
return self.__ssl_conn.setsockopt(*args) |
|||
|
|||
def state_string(self): |
|||
"""Return the SSL state of this connection.""" |
|||
return self.__ssl_conn.state_string() |
|||
|
|||
def makefile(self, *args): |
|||
"""Specific to Python socket API and required by httplib: convert |
|||
response into a file-like object. This implementation reads using recv |
|||
and copies the output into a StringIO buffer to simulate a file object |
|||
for consumption by httplib |
|||
|
|||
Nb. Ignoring optional file open mode (StringIO is generic and will |
|||
open for read and write unless a string is passed to the constructor) |
|||
and buffer size - httplib set a zero buffer size which results in recv |
|||
reading nothing |
|||
|
|||
@return: file object for data returned from socket |
|||
@rtype: cStringIO.StringO |
|||
""" |
|||
self._makefile_refs += 1 |
|||
|
|||
# Optimisation |
|||
_buf_size = self.buf_size |
|||
|
|||
i=0 |
|||
stream = BytesIO() |
|||
startTime = datetime.utcnow() |
|||
try: |
|||
dat = self.__ssl_conn.recv(_buf_size) |
|||
while dat: |
|||
i+=1 |
|||
stream.write(dat) |
|||
dat = self.__ssl_conn.recv(_buf_size) |
|||
|
|||
except (SSL.ZeroReturnError, SSL.SysCallError): |
|||
# Connection is closed - assuming here that all is well and full |
|||
# response has been received. httplib will catch an error in |
|||
# incomplete content since it checks the content-length header |
|||
# against the actual length of data received |
|||
pass |
|||
|
|||
if log.getEffectiveLevel() <= logging.DEBUG: |
|||
log.debug("Socket.makefile %d recv calls completed in %s", i, |
|||
datetime.utcnow() - startTime) |
|||
|
|||
# Make sure to rewind the buffer otherwise consumers of the content will |
|||
# read from the end of the buffer |
|||
stream.seek(0) |
|||
|
|||
return stream |
|||
|
|||
def getsockname(self): |
|||
""" |
|||
@return: the socket's own address |
|||
@rtype: |
|||
""" |
|||
return self.__ssl_conn.getsockname() |
|||
|
|||
def getpeername(self): |
|||
""" |
|||
@return: remote address to which the socket is connected |
|||
""" |
|||
return self.__ssl_conn.getpeername() |
|||
|
|||
def get_context(self): |
|||
'''Retrieve the Context object associated with this Connection. ''' |
|||
return self.__ssl_conn.get_context() |
|||
|
|||
def get_peer_certificate(self): |
|||
'''Retrieve the other side's certificate (if any) ''' |
|||
return self.__ssl_conn.get_peer_certificate() |
|||
@ -0,0 +1,153 @@ |
|||
"""NDG HTTPS Client package |
|||
|
|||
Use pyasn1 to provide support for parsing ASN.1 formatted subjectAltName |
|||
content for SSL peer verification. Code based on: |
|||
|
|||
http://stackoverflow.com/questions/5519958/how-do-i-parse-subjectaltname-extension-data-using-pyasn1 |
|||
""" |
|||
__author__ = "P J Kershaw" |
|||
__date__ = "01/02/12" |
|||
__copyright__ = "(C) 2012 Science and Technology Facilities Council" |
|||
__license__ = "BSD - see LICENSE file in top-level directory" |
|||
__contact__ = "Philip.Kershaw@stfc.ac.uk" |
|||
__revision__ = '$Id$' |
|||
try: |
|||
from pyasn1.type import univ, constraint, char, namedtype, tag |
|||
|
|||
except ImportError as e: |
|||
import_error_msg = ('Error importing pyasn1, subjectAltName check for SSL ' |
|||
'peer verification will be disabled. Import error ' |
|||
'is: %s' % e) |
|||
import warnings |
|||
warnings.warn(import_error_msg) |
|||
class Pyasn1ImportError(ImportError): |
|||
"Raise for pyasn1 import error" |
|||
raise Pyasn1ImportError(import_error_msg) |
|||
|
|||
|
|||
MAX = 64 |
|||
|
|||
|
|||
class DirectoryString(univ.Choice): |
|||
"""ASN.1 Directory string class""" |
|||
componentType = namedtype.NamedTypes( |
|||
namedtype.NamedType( |
|||
'teletexString', char.TeletexString().subtype( |
|||
subtypeSpec=constraint.ValueSizeConstraint(1, MAX))), |
|||
namedtype.NamedType( |
|||
'printableString', char.PrintableString().subtype( |
|||
subtypeSpec=constraint.ValueSizeConstraint(1, MAX))), |
|||
namedtype.NamedType( |
|||
'universalString', char.UniversalString().subtype( |
|||
subtypeSpec=constraint.ValueSizeConstraint(1, MAX))), |
|||
namedtype.NamedType( |
|||
'utf8String', char.UTF8String().subtype( |
|||
subtypeSpec=constraint.ValueSizeConstraint(1, MAX))), |
|||
namedtype.NamedType( |
|||
'bmpString', char.BMPString().subtype( |
|||
subtypeSpec=constraint.ValueSizeConstraint(1, MAX))), |
|||
namedtype.NamedType( |
|||
'ia5String', char.IA5String().subtype( |
|||
subtypeSpec=constraint.ValueSizeConstraint(1, MAX))), |
|||
) |
|||
|
|||
|
|||
class AttributeValue(DirectoryString): |
|||
"""ASN.1 Attribute value""" |
|||
|
|||
|
|||
class AttributeType(univ.ObjectIdentifier): |
|||
"""ASN.1 Attribute type""" |
|||
|
|||
|
|||
class AttributeTypeAndValue(univ.Sequence): |
|||
"""ASN.1 Attribute type and value class""" |
|||
componentType = namedtype.NamedTypes( |
|||
namedtype.NamedType('type', AttributeType()), |
|||
namedtype.NamedType('value', AttributeValue()), |
|||
) |
|||
|
|||
|
|||
class RelativeDistinguishedName(univ.SetOf): |
|||
'''ASN.1 Realtive distinguished name''' |
|||
componentType = AttributeTypeAndValue() |
|||
|
|||
class RDNSequence(univ.SequenceOf): |
|||
'''ASN.1 RDN sequence class''' |
|||
componentType = RelativeDistinguishedName() |
|||
|
|||
|
|||
class Name(univ.Choice): |
|||
'''ASN.1 name class''' |
|||
componentType = namedtype.NamedTypes( |
|||
namedtype.NamedType('', RDNSequence()), |
|||
) |
|||
|
|||
|
|||
class Extension(univ.Sequence): |
|||
'''ASN.1 extension class''' |
|||
componentType = namedtype.NamedTypes( |
|||
namedtype.NamedType('extnID', univ.ObjectIdentifier()), |
|||
namedtype.DefaultedNamedType('critical', univ.Boolean('False')), |
|||
namedtype.NamedType('extnValue', univ.OctetString()), |
|||
) |
|||
|
|||
|
|||
class Extensions(univ.SequenceOf): |
|||
'''ASN.1 extensions class''' |
|||
componentType = Extension() |
|||
sizeSpec = univ.SequenceOf.sizeSpec + constraint.ValueSizeConstraint(1, MAX) |
|||
|
|||
|
|||
class AnotherName(univ.Sequence): |
|||
componentType = namedtype.NamedTypes( |
|||
namedtype.NamedType('type-id', univ.ObjectIdentifier()), |
|||
namedtype.NamedType('value', univ.Any().subtype( |
|||
explicitTag=tag.Tag(tag.tagClassContext, |
|||
tag.tagFormatSimple, 0))) |
|||
) |
|||
|
|||
|
|||
class GeneralName(univ.Choice): |
|||
'''ASN.1 configuration for X.509 certificate subjectAltNames fields''' |
|||
componentType = namedtype.NamedTypes( |
|||
namedtype.NamedType('otherName', AnotherName().subtype( |
|||
implicitTag=tag.Tag(tag.tagClassContext, |
|||
tag.tagFormatSimple, 0))), |
|||
namedtype.NamedType('rfc822Name', char.IA5String().subtype( |
|||
implicitTag=tag.Tag(tag.tagClassContext, |
|||
tag.tagFormatSimple, 1))), |
|||
namedtype.NamedType('dNSName', char.IA5String().subtype( |
|||
implicitTag=tag.Tag(tag.tagClassContext, |
|||
tag.tagFormatSimple, 2))), |
|||
# namedtype.NamedType('x400Address', ORAddress().subtype( |
|||
# implicitTag=tag.Tag(tag.tagClassContext, |
|||
# tag.tagFormatSimple, 3))), |
|||
namedtype.NamedType('directoryName', Name().subtype( |
|||
implicitTag=tag.Tag(tag.tagClassContext, |
|||
tag.tagFormatSimple, 4))), |
|||
# namedtype.NamedType('ediPartyName', EDIPartyName().subtype( |
|||
# implicitTag=tag.Tag(tag.tagClassContext, |
|||
# tag.tagFormatSimple, 5))), |
|||
namedtype.NamedType('uniformResourceIdentifier', char.IA5String().subtype( |
|||
implicitTag=tag.Tag(tag.tagClassContext, |
|||
tag.tagFormatSimple, 6))), |
|||
namedtype.NamedType('iPAddress', univ.OctetString().subtype( |
|||
implicitTag=tag.Tag(tag.tagClassContext, |
|||
tag.tagFormatSimple, 7))), |
|||
namedtype.NamedType('registeredID', univ.ObjectIdentifier().subtype( |
|||
implicitTag=tag.Tag(tag.tagClassContext, |
|||
tag.tagFormatSimple, 8))), |
|||
) |
|||
|
|||
|
|||
class GeneralNames(univ.SequenceOf): |
|||
'''Sequence of names for ASN.1 subjectAltNames settings''' |
|||
componentType = GeneralName() |
|||
sizeSpec = univ.SequenceOf.sizeSpec + constraint.ValueSizeConstraint(1, MAX) |
|||
|
|||
|
|||
class SubjectAltName(GeneralNames): |
|||
'''ASN.1 implementation for subjectAltNames support''' |
|||
|
|||
|
|||
@ -0,0 +1,78 @@ |
|||
"""urllib2 style build opener integrates with HTTPSConnection class from this |
|||
package. |
|||
""" |
|||
__author__ = "P J Kershaw" |
|||
__date__ = "21/12/10" |
|||
__copyright__ = "(C) 2011 Science and Technology Facilities Council" |
|||
__license__ = "BSD - see LICENSE file in top-level directory" |
|||
__contact__ = "Philip.Kershaw@stfc.ac.uk" |
|||
__revision__ = '$Id$' |
|||
import logging |
|||
import sys |
|||
|
|||
# Py 2 <=> 3 compatibility for class type checking |
|||
if sys.version_info[0] > 2: |
|||
class_type_ = type |
|||
from urllib.request import (ProxyHandler, UnknownHandler, |
|||
HTTPDefaultErrorHandler, FTPHandler, |
|||
FileHandler, HTTPErrorProcessor, |
|||
HTTPHandler, OpenerDirector, |
|||
HTTPRedirectHandler) |
|||
else: |
|||
import types |
|||
class_type_ = types.ClassType |
|||
|
|||
from urllib2 import (ProxyHandler, UnknownHandler, HTTPDefaultErrorHandler, |
|||
FTPHandler, FileHandler, HTTPErrorProcessor, |
|||
HTTPHandler, OpenerDirector, HTTPRedirectHandler) |
|||
|
|||
from ndg.httpsclient.https import HTTPSContextHandler |
|||
|
|||
log = logging.getLogger(__name__) |
|||
|
|||
|
|||
# Copied from urllib2 with modifications for ssl |
|||
def build_opener(*handlers, **kw): |
|||
"""Create an opener object from a list of handlers. |
|||
|
|||
The opener will use several default handlers, including support |
|||
for HTTP and FTP. |
|||
|
|||
If any of the handlers passed as arguments are subclasses of the |
|||
default handlers, the default handlers will not be used. |
|||
""" |
|||
def isclass(obj): |
|||
return isinstance(obj, class_type_) or hasattr(obj, "__bases__") |
|||
|
|||
opener = OpenerDirector() |
|||
default_classes = [ProxyHandler, UnknownHandler, HTTPHandler, |
|||
HTTPDefaultErrorHandler, HTTPRedirectHandler, |
|||
FTPHandler, FileHandler, HTTPErrorProcessor] |
|||
check_classes = list(default_classes) |
|||
check_classes.append(HTTPSContextHandler) |
|||
skip = [] |
|||
for klass in check_classes: |
|||
for check in handlers: |
|||
if isclass(check): |
|||
if issubclass(check, klass): |
|||
skip.append(klass) |
|||
elif isinstance(check, klass): |
|||
skip.append(klass) |
|||
|
|||
for klass in default_classes: |
|||
if klass not in skip: |
|||
opener.add_handler(klass()) |
|||
|
|||
# Pick up SSL context from keyword settings |
|||
ssl_context = kw.get('ssl_context') |
|||
|
|||
# Add the HTTPS handler with ssl_context |
|||
if HTTPSContextHandler not in skip: |
|||
opener.add_handler(HTTPSContextHandler(ssl_context)) |
|||
|
|||
for h in handlers: |
|||
if isclass(h): |
|||
h = h() |
|||
opener.add_handler(h) |
|||
|
|||
return opener |
|||
@ -0,0 +1,414 @@ |
|||
"""Utilities using NDG HTTPS Client, including a main module that can be used to |
|||
fetch from a URL. |
|||
""" |
|||
__author__ = "R B Wilkinson" |
|||
__date__ = "09/12/11" |
|||
__copyright__ = "(C) 2011 Science and Technology Facilities Council" |
|||
__license__ = "BSD - see LICENSE file in top-level directory" |
|||
__contact__ = "Philip.Kershaw@stfc.ac.uk" |
|||
__revision__ = '$Id$' |
|||
|
|||
import logging |
|||
from optparse import OptionParser |
|||
import os |
|||
import sys |
|||
|
|||
if sys.version_info[0] > 2: |
|||
import http.cookiejar as cookiejar_ |
|||
import http.client as http_client_ |
|||
from urllib.request import Request as Request_ |
|||
from urllib.request import HTTPHandler as HTTPHandler_ |
|||
from urllib.request import HTTPCookieProcessor as HTTPCookieProcessor_ |
|||
from urllib.request import HTTPBasicAuthHandler as HTTPBasicAuthHandler_ |
|||
from urllib.request import HTTPPasswordMgrWithDefaultRealm as \ |
|||
HTTPPasswordMgrWithDefaultRealm_ |
|||
from urllib.request import ProxyHandler as ProxyHandler_ |
|||
from urllib.error import HTTPError as HTTPError_ |
|||
import urllib.parse as urlparse_ |
|||
else: |
|||
import cookielib as cookiejar_ |
|||
import httplib as http_client_ |
|||
from urllib2 import Request as Request_ |
|||
from urllib2 import HTTPHandler as HTTPHandler_ |
|||
from urllib2 import HTTPCookieProcessor as HTTPCookieProcessor_ |
|||
from urllib2 import HTTPBasicAuthHandler as HTTPBasicAuthHandler_ |
|||
from urllib2 import HTTPPasswordMgrWithDefaultRealm as \ |
|||
HTTPPasswordMgrWithDefaultRealm_ |
|||
from urllib2 import ProxyHandler as ProxyHandler_ |
|||
from urllib2 import HTTPError as HTTPError_ |
|||
import urlparse as urlparse_ |
|||
|
|||
from ndg.httpsclient.urllib2_build_opener import build_opener |
|||
from ndg.httpsclient.https import HTTPSContextHandler |
|||
from ndg.httpsclient import ssl_context_util |
|||
|
|||
log = logging.getLogger(__name__) |
|||
|
|||
class AccumulatingHTTPCookieProcessor(HTTPCookieProcessor_): |
|||
"""Cookie processor that adds new cookies (instead of replacing the existing |
|||
ones as HTTPCookieProcessor does) |
|||
""" |
|||
def http_request(self, request): |
|||
"""Processes cookies for a HTTP request. |
|||
@param request: request to process |
|||
@type request: urllib2.Request |
|||
@return: request |
|||
@rtype: urllib2.Request |
|||
""" |
|||
COOKIE_HEADER_NAME = "Cookie" |
|||
tmp_request = Request_(request.get_full_url(), request.data, {}, |
|||
request.origin_req_host, |
|||
request.unverifiable) |
|||
self.cookiejar.add_cookie_header(tmp_request) |
|||
# Combine existing and new cookies. |
|||
new_cookies = tmp_request.get_header(COOKIE_HEADER_NAME) |
|||
if new_cookies: |
|||
if request.has_header(COOKIE_HEADER_NAME): |
|||
# Merge new cookies with existing ones. |
|||
old_cookies = request.get_header(COOKIE_HEADER_NAME) |
|||
merged_cookies = '; '.join([old_cookies, new_cookies]) |
|||
request.add_unredirected_header(COOKIE_HEADER_NAME, |
|||
merged_cookies) |
|||
else: |
|||
# No existing cookies so just set new ones. |
|||
request.add_unredirected_header(COOKIE_HEADER_NAME, new_cookies) |
|||
return request |
|||
|
|||
# Process cookies for HTTPS in the same way. |
|||
https_request = http_request |
|||
|
|||
|
|||
class URLFetchError(Exception): |
|||
"""Error fetching content from URL""" |
|||
|
|||
|
|||
def fetch_from_url(url, config, data=None, handlers=None): |
|||
"""Returns data retrieved from a URL. |
|||
@param url: URL to attempt to open |
|||
@type url: basestring |
|||
@param config: SSL context configuration |
|||
@type config: Configuration |
|||
@return data retrieved from URL or None |
|||
""" |
|||
return_code, return_message, response = open_url(url, config, data=data, |
|||
handlers=handlers) |
|||
if return_code and return_code == http_client_.OK: |
|||
return_data = response.read() |
|||
response.close() |
|||
return return_data |
|||
else: |
|||
raise URLFetchError(return_message) |
|||
|
|||
def fetch_from_url_to_file(url, config, output_file, data=None, handlers=None): |
|||
"""Writes data retrieved from a URL to a file. |
|||
@param url: URL to attempt to open |
|||
@type url: basestring |
|||
@param config: SSL context configuration |
|||
@type config: Configuration |
|||
@param output_file: output file |
|||
@type output_file: basestring |
|||
@return: tuple ( |
|||
returned HTTP status code or 0 if an error occurred |
|||
returned message |
|||
boolean indicating whether access was successful) |
|||
""" |
|||
return_code, return_message, response = open_url(url, config, data=data, |
|||
handlers=handlers) |
|||
if return_code == http_client_.OK: |
|||
return_data = response.read() |
|||
response.close() |
|||
outfile = open(output_file, "w") |
|||
outfile.write(return_data) |
|||
outfile.close() |
|||
|
|||
return return_code, return_message, return_code == http_client_.OK |
|||
|
|||
|
|||
def fetch_stream_from_url(url, config, data=None, handlers=None): |
|||
"""Returns data retrieved from a URL. |
|||
@param url: URL to attempt to open |
|||
@type url: basestring |
|||
@param config: SSL context configuration |
|||
@type config: Configuration |
|||
@param data: HTTP POST data |
|||
@type data: str |
|||
@param handlers: list of custom urllib2 handlers to add to the request |
|||
@type handlers: iterable |
|||
@return: data retrieved from URL or None |
|||
@rtype: file derived type |
|||
""" |
|||
return_code, return_message, response = open_url(url, config, data=data, |
|||
handlers=handlers) |
|||
if return_code and return_code == http_client_.OK: |
|||
return response |
|||
else: |
|||
raise URLFetchError(return_message) |
|||
|
|||
|
|||
def open_url(url, config, data=None, handlers=None): |
|||
"""Attempts to open a connection to a specified URL. |
|||
@param url: URL to attempt to open |
|||
@param config: SSL context configuration |
|||
@type config: Configuration |
|||
@param data: HTTP POST data |
|||
@type data: str |
|||
@param handlers: list of custom urllib2 handlers to add to the request |
|||
@type handlers: iterable |
|||
@return: tuple ( |
|||
returned HTTP status code or 0 if an error occurred |
|||
returned message or error description |
|||
response object) |
|||
""" |
|||
debuglevel = 1 if config.debug else 0 |
|||
|
|||
# Set up handlers for URL opener. |
|||
if config.cookie: |
|||
cj = config.cookie |
|||
else: |
|||
cj = cookiejar_.CookieJar() |
|||
|
|||
# Use a cookie processor that accumulates cookies when redirects occur so |
|||
# that an application can redirect for authentication and retain both any |
|||
# cookies for the application and the security system (c.f., |
|||
# urllib2.HTTPCookieProcessor which replaces cookies). |
|||
cookie_handler = AccumulatingHTTPCookieProcessor(cj) |
|||
|
|||
if not handlers: |
|||
handlers = [] |
|||
|
|||
handlers.append(cookie_handler) |
|||
|
|||
if config.debug: |
|||
http_handler = HTTPHandler_(debuglevel=debuglevel) |
|||
https_handler = HTTPSContextHandler(config.ssl_context, |
|||
debuglevel=debuglevel) |
|||
handlers.extend([http_handler, https_handler]) |
|||
|
|||
if config.http_basicauth: |
|||
# currently only supports http basic auth |
|||
auth_handler = HTTPBasicAuthHandler_(HTTPPasswordMgrWithDefaultRealm_()) |
|||
auth_handler.add_password(realm=None, uri=url, |
|||
user=config.httpauth[0], |
|||
passwd=config.httpauth[1]) |
|||
handlers.append(auth_handler) |
|||
|
|||
|
|||
# Explicitly remove proxy handling if the host is one listed in the value of |
|||
# the no_proxy environment variable because urllib2 does use proxy settings |
|||
# set via http_proxy and https_proxy, but does not take the no_proxy value |
|||
# into account. |
|||
if not _should_use_proxy(url, config.no_proxy): |
|||
handlers.append(ProxyHandler_({})) |
|||
log.debug("Not using proxy") |
|||
elif config.proxies: |
|||
handlers.append(ProxyHandler_(config.proxies)) |
|||
log.debug("Configuring proxies: %s" % config.proxies) |
|||
|
|||
opener = build_opener(*handlers, ssl_context=config.ssl_context) |
|||
|
|||
headers = config.headers |
|||
if headers is None: |
|||
headers = {} |
|||
|
|||
request = Request_(url, data, headers) |
|||
|
|||
# Open the URL and check the response. |
|||
return_code = 0 |
|||
return_message = '' |
|||
response = None |
|||
|
|||
# FIXME |
|||
response = opener.open(request) |
|||
|
|||
try: |
|||
response = opener.open(request) |
|||
return_message = response.msg |
|||
return_code = response.code |
|||
if log.isEnabledFor(logging.DEBUG): |
|||
for index, cookie in enumerate(cj): |
|||
log.debug("%s : %s", index, cookie) |
|||
|
|||
except HTTPError_ as exc: |
|||
return_code = exc.code |
|||
return_message = "Error: %s" % exc.msg |
|||
if log.isEnabledFor(logging.DEBUG): |
|||
log.debug("%s %s", exc.code, exc.msg) |
|||
|
|||
except Exception as exc: |
|||
return_message = "Error: %s" % exc.__str__() |
|||
if log.isEnabledFor(logging.DEBUG): |
|||
import traceback |
|||
log.debug(traceback.format_exc()) |
|||
|
|||
return (return_code, return_message, response) |
|||
|
|||
|
|||
def _should_use_proxy(url, no_proxy=None): |
|||
"""Determines whether a proxy should be used to open a connection to the |
|||
specified URL, based on the value of the no_proxy environment variable. |
|||
@param url: URL |
|||
@type url: basestring or urllib2.Request |
|||
""" |
|||
if no_proxy is None: |
|||
no_proxy_effective = os.environ.get('no_proxy', '') |
|||
else: |
|||
no_proxy_effective = no_proxy |
|||
|
|||
urlObj = urlparse_.urlparse(_url_as_string(url)) |
|||
for np in [h.strip() for h in no_proxy_effective.split(',')]: |
|||
if urlObj.hostname == np: |
|||
return False |
|||
|
|||
return True |
|||
|
|||
def _url_as_string(url): |
|||
"""Returns the URL string from a URL value that is either a string or |
|||
urllib2.Request.. |
|||
@param url: URL |
|||
@type url: basestring or urllib2.Request |
|||
@return: URL string |
|||
@rtype: basestring |
|||
""" |
|||
if isinstance(url, Request_): |
|||
return url.get_full_url() |
|||
elif isinstance(url, str): |
|||
return url |
|||
else: |
|||
raise TypeError("Expected type %r or %r" % |
|||
(str, Request_)) |
|||
|
|||
|
|||
class Configuration(object): |
|||
"""Connection configuration. |
|||
""" |
|||
def __init__(self, ssl_context, debug=False, proxies=None, no_proxy=None, |
|||
cookie=None, http_basicauth=None, headers=None): |
|||
""" |
|||
@param ssl_context: SSL context to use with this configuration |
|||
@type ssl_context: OpenSSL.SSL.Context |
|||
@param debug: if True, output debugging information |
|||
@type debug: bool |
|||
@param proxies: proxies to use for |
|||
@type proxies: dict with basestring keys and values |
|||
@param no_proxy: hosts for which a proxy should not be used |
|||
@type no_proxy: basestring |
|||
@param cookie: cookies to set for request |
|||
@type cookie: cookielib.CookieJar (python 3 - http.cookiejar) |
|||
@param http_basicauth: http authentication, or None |
|||
@type http_basicauth: tuple of (username,password) |
|||
@param headers: http headers |
|||
@type headers: dict |
|||
""" |
|||
self.ssl_context = ssl_context |
|||
self.debug = debug |
|||
self.proxies = proxies |
|||
self.no_proxy = no_proxy |
|||
self.cookie = cookie |
|||
self.http_basicauth = http_basicauth |
|||
self.headers = headers |
|||
|
|||
|
|||
def main(): |
|||
'''Utility to fetch data using HTTP or HTTPS GET from a specified URL. |
|||
''' |
|||
parser = OptionParser(usage="%prog [options] url") |
|||
parser.add_option("-c", "--certificate", dest="cert_file", metavar="FILE", |
|||
default=os.path.expanduser("~/credentials.pem"), |
|||
help="Certificate file - defaults to $HOME/credentials.pem") |
|||
parser.add_option("-k", "--private-key", dest="key_file", metavar="FILE", |
|||
default=None, |
|||
help="Private key file - defaults to the certificate file") |
|||
parser.add_option("-t", "--ca-certificate-dir", dest="ca_dir", |
|||
metavar="PATH", |
|||
default=None, |
|||
help="Trusted CA certificate file directory") |
|||
parser.add_option("-d", "--debug", action="store_true", dest="debug", |
|||
default=False, |
|||
help="Print debug information.") |
|||
parser.add_option("-p", "--post-data-file", dest="data_file", |
|||
metavar="FILE", default=None, |
|||
help="POST data file") |
|||
parser.add_option("-f", "--fetch", dest="output_file", metavar="FILE", |
|||
default=None, help="Output file") |
|||
parser.add_option("-n", "--no-verify-peer", action="store_true", |
|||
dest="no_verify_peer", default=False, |
|||
help="Skip verification of peer certificate.") |
|||
parser.add_option("-a", "--basicauth", dest="basicauth", |
|||
metavar="USER:PASSWD", |
|||
default=None, |
|||
help="HTTP authentication credentials") |
|||
parser.add_option("--header", action="append", dest="headers", |
|||
metavar="HEADER: VALUE", |
|||
help="Add HTTP header to request") |
|||
(options, args) = parser.parse_args() |
|||
if len(args) != 1: |
|||
parser.error("Incorrect number of arguments") |
|||
|
|||
url = args[0] |
|||
|
|||
if options.debug: |
|||
logging.getLogger().setLevel(logging.DEBUG) |
|||
|
|||
if options.key_file and os.path.exists(options.key_file): |
|||
key_file = options.key_file |
|||
else: |
|||
key_file = None |
|||
|
|||
if options.cert_file and os.path.exists(options.cert_file): |
|||
cert_file = options.cert_file |
|||
else: |
|||
cert_file = None |
|||
|
|||
if options.ca_dir and os.path.exists(options.ca_dir): |
|||
ca_dir = options.ca_dir |
|||
else: |
|||
ca_dir = None |
|||
|
|||
verify_peer = not options.no_verify_peer |
|||
|
|||
if options.data_file and os.path.exists(options.data_file): |
|||
data_file = open(options.data_file) |
|||
data = data_file.read() |
|||
data_file.close() |
|||
else: |
|||
data = None |
|||
|
|||
if options.basicauth: |
|||
http_basicauth = options.basicauth.split(':', 1) |
|||
else: |
|||
http_basicauth = None |
|||
|
|||
headers = {} |
|||
if options.headers: |
|||
for h in options.headers: |
|||
key, val = h.split(':', 1) |
|||
headers[key.strip()] = val.lstrip() |
|||
|
|||
# If a private key file is not specified, the key is assumed to be stored in |
|||
# the certificate file. |
|||
ssl_context = ssl_context_util.make_ssl_context(key_file, |
|||
cert_file, |
|||
None, |
|||
ca_dir, |
|||
verify_peer, |
|||
url) |
|||
|
|||
config = Configuration(ssl_context, |
|||
options.debug, |
|||
http_basicauth=http_basicauth, |
|||
headers=headers) |
|||
if options.output_file: |
|||
return_code, return_message = fetch_from_url_to_file( |
|||
url, |
|||
config, |
|||
options.output_file, |
|||
data)[:2] |
|||
raise SystemExit(return_code, return_message) |
|||
else: |
|||
data = fetch_from_url(url, config) |
|||
print(data) |
|||
|
|||
|
|||
if __name__=='__main__': |
|||
logging.basicConfig() |
|||
main() |
|||
Loading…
Reference in new issue