diff --git a/libs/ndg/__init__.py b/libs/ndg/__init__.py new file mode 100644 index 0000000..3b01e15 --- /dev/null +++ b/libs/ndg/__init__.py @@ -0,0 +1,19 @@ +"""ndg_httpsclient - PyOpenSSL utility to make a httplib-like interface suitable +for use with urllib2 + +This is a setuptools namespace_package. DO NOT place any other +code in this file! There is no guarantee that it will be installed +with easy_install. See: + +http://peak.telecommunity.com/DevCenter/setuptools#namespace-packages + +... for details. +""" +__author__ = "P J Kershaw" +__date__ = "06/01/12" +__copyright__ = "(C) 2012 Science and Technology Facilities Council" +__license__ = "BSD - see LICENSE file in top-level directory" +__contact__ = "Philip.Kershaw@stfc.ac.uk" +__revision__ = '$Id$' + +__import__('pkg_resources').declare_namespace(__name__) \ No newline at end of file diff --git a/libs/ndg/httpsclient/__init__.py b/libs/ndg/httpsclient/__init__.py new file mode 100644 index 0000000..83f2087 --- /dev/null +++ b/libs/ndg/httpsclient/__init__.py @@ -0,0 +1,9 @@ +"""ndg_httpsclient - PyOpenSSL utility to make a httplib-like interface suitable +for use with urllib2 +""" +__author__ = "P J Kershaw (STFC) and Richard Wilkinson (Tessella)" +__date__ = "09/12/11" +__copyright__ = "(C) 2011 Science and Technology Facilities Council" +__license__ = "BSD - see LICENSE file in top-level directory" +__contact__ = "Philip.Kershaw@stfc.ac.uk" +__revision__ = '$Id$' diff --git a/libs/ndg/httpsclient/https.py b/libs/ndg/httpsclient/https.py new file mode 100644 index 0000000..41ad1c6 --- /dev/null +++ b/libs/ndg/httpsclient/https.py @@ -0,0 +1,131 @@ +"""ndg_httpsclient HTTPS module containing PyOpenSSL implementation of +httplib.HTTPSConnection + +PyOpenSSL utility to make a httplib-like interface suitable for use with +urllib2 +""" +__author__ = "P J Kershaw (STFC)" +__date__ = "09/12/11" +__copyright__ = "(C) 2012 Science and Technology Facilities Council" +__license__ = "BSD - see LICENSE file in top-level directory" +__contact__ = "Philip.Kershaw@stfc.ac.uk" +__revision__ = '$Id$' +import logging +import socket +import sys + +if sys.version_info[0] > 2: + from http.client import HTTPS_PORT + from http.client import HTTPConnection + + from urllib.request import AbstractHTTPHandler +else: + from httplib import HTTPS_PORT + from httplib import HTTPConnection + + from urllib2 import AbstractHTTPHandler + + +from OpenSSL import SSL + +from ndg.httpsclient.ssl_socket import SSLSocket + +log = logging.getLogger(__name__) + + +class HTTPSConnection(HTTPConnection): + """This class allows communication via SSL using PyOpenSSL. + It is based on httplib.HTTPSConnection, modified to use PyOpenSSL. + + Note: This uses the constructor inherited from HTTPConnection to allow it to + be used with httplib and HTTPSContextHandler. To use the class directly with + an SSL context set ssl_context after construction. + + @cvar default_port: default port for this class (443) + @type default_port: int + @cvar default_ssl_method: default SSL method used if no SSL context is + explicitly set - defaults to version 2/3. + @type default_ssl_method: int + """ + default_port = HTTPS_PORT + default_ssl_method = SSL.SSLv23_METHOD + + def __init__(self, host, port=None, strict=None, + timeout=socket._GLOBAL_DEFAULT_TIMEOUT, ssl_context=None): + HTTPConnection.__init__(self, host, port, strict, timeout) + if not hasattr(self, 'ssl_context'): + self.ssl_context = None + + if ssl_context is not None: + if not isinstance(ssl_context, SSL.Context): + raise TypeError('Expecting OpenSSL.SSL.Context type for "' + 'ssl_context" keyword; got %r instead' % + ssl_context) + + self.ssl_context = ssl_context + + def connect(self): + """Create SSL socket and connect to peer + """ + if getattr(self, 'ssl_context', None): + if not isinstance(self.ssl_context, SSL.Context): + raise TypeError('Expecting OpenSSL.SSL.Context type for "' + 'ssl_context" attribute; got %r instead' % + self.ssl_context) + ssl_context = self.ssl_context + else: + ssl_context = SSL.Context(self.__class__.default_ssl_method) + + sock = socket.create_connection((self.host, self.port), self.timeout) + + # Tunnel if using a proxy - ONLY available for Python 2.6.2 and above + if getattr(self, '_tunnel_host', None): + self.sock = sock + self._tunnel() + + self.sock = SSLSocket(ssl_context, sock) + + # Go to client mode. + self.sock.set_connect_state() + + def close(self): + """Close socket and shut down SSL connection""" + if hasattr(self.sock, "close"): + self.sock.close() + + +class HTTPSContextHandler(AbstractHTTPHandler): + '''HTTPS handler that allows a SSL context to be set for the SSL + connections. + ''' + https_request = AbstractHTTPHandler.do_request_ + + def __init__(self, ssl_context, debuglevel=0): + """ + @param ssl_context:SSL context + @type ssl_context: OpenSSL.SSL.Context + @param debuglevel: debug level for HTTPSHandler + @type debuglevel: int + """ + AbstractHTTPHandler.__init__(self, debuglevel) + + if ssl_context is not None: + if not isinstance(ssl_context, SSL.Context): + raise TypeError('Expecting OpenSSL.SSL.Context type for "' + 'ssl_context" keyword; got %r instead' % + ssl_context) + self.ssl_context = ssl_context + else: + self.ssl_context = SSL.Context(SSL.TLSv1_METHOD) + + def https_open(self, req): + """Opens HTTPS request + @param req: HTTP request + @return: HTTP Response object + """ + # Make a custom class extending HTTPSConnection, with the SSL context + # set as a class variable so that it is available to the connect method. + customHTTPSContextConnection = type('CustomHTTPSContextConnection', + (HTTPSConnection, object), + {'ssl_context': self.ssl_context}) + return self.do_open(customHTTPSContextConnection, req) diff --git a/libs/ndg/httpsclient/ssl_context_util.py b/libs/ndg/httpsclient/ssl_context_util.py new file mode 100644 index 0000000..0ed1d32 --- /dev/null +++ b/libs/ndg/httpsclient/ssl_context_util.py @@ -0,0 +1,98 @@ +"""ndg_httpsclient SSL Context utilities module containing convenience routines +for setting SSL context configuration. + +""" +__author__ = "P J Kershaw (STFC)" +__date__ = "09/12/11" +__copyright__ = "(C) 2012 Science and Technology Facilities Council" +__license__ = "BSD - see LICENSE file in top-level directory" +__contact__ = "Philip.Kershaw@stfc.ac.uk" +__revision__ = '$Id$' +import sys + +if sys.version_info[0] > 2: + import urllib.parse as urlparse_ +else: + import urlparse as urlparse_ + +from OpenSSL import SSL + +from ndg.httpsclient.ssl_peer_verification import ServerSSLCertVerification + + +class SSlContextConfig(object): + """ + Holds configuration options for creating a SSL context. This is used as a + template to create the contexts with specific verification callbacks. + """ + def __init__(self, key_file=None, cert_file=None, pem_file=None, ca_dir=None, + verify_peer=False): + self.key_file = key_file + self.cert_file = cert_file + self.pem_file = pem_file + self.ca_dir = ca_dir + self.verify_peer = verify_peer + + +def make_ssl_context_from_config(ssl_config=False, url=None): + return make_ssl_context(ssl_config.key_file, ssl_config.cert_file, + ssl_config.pem_file, ssl_config.ca_dir, + ssl_config.verify_peer, url) + + +def make_ssl_context(key_file=None, cert_file=None, pem_file=None, ca_dir=None, + verify_peer=False, url=None, method=SSL.TLSv1_METHOD, + key_file_passphrase=None): + """ + Creates SSL context containing certificate and key file locations. + """ + ssl_context = SSL.Context(method) + + # Key file defaults to certificate file if present. + if cert_file: + ssl_context.use_certificate_file(cert_file) + + if key_file_passphrase: + passwd_cb = lambda max_passphrase_len, set_prompt, userdata: \ + key_file_passphrase + ssl_context.set_passwd_cb(passwd_cb) + + if key_file: + ssl_context.use_privatekey_file(key_file) + elif cert_file: + ssl_context.use_privatekey_file(cert_file) + + if pem_file or ca_dir: + ssl_context.load_verify_locations(pem_file, ca_dir) + + def _callback(conn, x509, errnum, errdepth, preverify_ok): + """Default certification verification callback. + Performs no checks and returns the status passed in. + """ + return preverify_ok + + verify_callback = _callback + + if verify_peer: + ssl_context.set_verify_depth(9) + if url: + set_peer_verification_for_url_hostname(ssl_context, url) + else: + ssl_context.set_verify(SSL.VERIFY_PEER, verify_callback) + else: + ssl_context.set_verify(SSL.VERIFY_NONE, verify_callback) + + return ssl_context + + +def set_peer_verification_for_url_hostname(ssl_context, url, + if_verify_enabled=False): + '''Convenience routine to set peer verification callback based on + ServerSSLCertVerification class''' + if not if_verify_enabled or (ssl_context.get_verify_mode() & SSL.VERIFY_PEER): + urlObj = urlparse_.urlparse(url) + hostname = urlObj.hostname + server_ssl_cert_verif = ServerSSLCertVerification(hostname=hostname) + verify_callback_ = server_ssl_cert_verif.get_verify_server_cert_func() + ssl_context.set_verify(SSL.VERIFY_PEER, verify_callback_) + diff --git a/libs/ndg/httpsclient/ssl_peer_verification.py b/libs/ndg/httpsclient/ssl_peer_verification.py new file mode 100644 index 0000000..5e82dae --- /dev/null +++ b/libs/ndg/httpsclient/ssl_peer_verification.py @@ -0,0 +1,237 @@ +"""ndg_httpsclient - module containing SSL peer verification class. +""" +__author__ = "P J Kershaw (STFC)" +__date__ = "09/12/11" +__copyright__ = "(C) 2012 Science and Technology Facilities Council" +__license__ = "BSD - see LICENSE file in top-level directory" +__contact__ = "Philip.Kershaw@stfc.ac.uk" +__revision__ = '$Id$' +import re +import logging +log = logging.getLogger(__name__) + +try: + from ndg.httpsclient.subj_alt_name import SubjectAltName + from pyasn1.codec.der import decoder as der_decoder + SUBJ_ALT_NAME_SUPPORT = True + +except ImportError as e: + SUBJ_ALT_NAME_SUPPORT = False + SUBJ_ALT_NAME_SUPPORT_MSG = ( + 'SubjectAltName support is disabled - check pyasn1 package ' + 'installation to enable' + ) + import warnings + warnings.warn(SUBJ_ALT_NAME_SUPPORT_MSG) + + +class ServerSSLCertVerification(object): + """Check server identity. If hostname doesn't match, allow match of + host's Distinguished Name against server DN setting""" + DN_LUT = { + 'commonName': 'CN', + 'organisationalUnitName': 'OU', + 'organisation': 'O', + 'countryName': 'C', + 'emailAddress': 'EMAILADDRESS', + 'localityName': 'L', + 'stateOrProvinceName': 'ST', + 'streetAddress': 'STREET', + 'domainComponent': 'DC', + 'userid': 'UID' + } + SUBJ_ALT_NAME_EXT_NAME = 'subjectAltName' + PARSER_RE_STR = '/(%s)=' % '|'.join(list(DN_LUT.keys()) + list(DN_LUT.values())) + PARSER_RE = re.compile(PARSER_RE_STR) + + __slots__ = ('__hostname', '__certDN', '__subj_alt_name_match') + + def __init__(self, certDN=None, hostname=None, subj_alt_name_match=True): + """Override parent class __init__ to enable setting of certDN + setting + + @type certDN: string + @param certDN: Set the expected Distinguished Name of the + server to avoid errors matching hostnames. This is useful + where the hostname is not fully qualified + @type hostname: string + @param hostname: hostname to match against peer certificate + subjectAltNames or subject common name + @type subj_alt_name_match: bool + @param subj_alt_name_match: flag to enable/disable matching of hostname + against peer certificate subjectAltNames. Nb. A setting of True will + be ignored if the pyasn1 package is not installed + """ + self.__certDN = None + self.__hostname = None + + if certDN is not None: + self.certDN = certDN + + if hostname is not None: + self.hostname = hostname + + if subj_alt_name_match: + if not SUBJ_ALT_NAME_SUPPORT: + log.warning('Overriding "subj_alt_name_match" keyword setting: ' + 'peer verification with subjectAltNames is disabled') + self.__subj_alt_name_match = False + else: + self.__subj_alt_name_match = True + else: + log.debug('Disabling peer verification with subject ' + 'subjectAltNames!') + self.__subj_alt_name_match = False + + def __call__(self, connection, peerCert, errorStatus, errorDepth, + preverifyOK): + """Verify server certificate + + @type connection: OpenSSL.SSL.Connection + @param connection: SSL connection object + @type peerCert: basestring + @param peerCert: server host certificate as OpenSSL.crypto.X509 + instance + @type errorStatus: int + @param errorStatus: error status passed from caller. This is the value + returned by the OpenSSL C function X509_STORE_CTX_get_error(). Look-up + x509_vfy.h in the OpenSSL source to get the meanings of the different + codes. PyOpenSSL doesn't help you! + @type errorDepth: int + @param errorDepth: a non-negative integer representing where in the + certificate chain the error occurred. If it is zero it occured in the + end entity certificate, one if it is the certificate which signed the + end entity certificate and so on. + + @type preverifyOK: int + @param preverifyOK: the error status - 0 = Error, 1 = OK of the current + SSL context irrespective of any verification checks done here. If this + function yields an OK status, it should enforce the preverifyOK value + so that any error set upstream overrides and is honoured. + @rtype: int + @return: status code - 0/False = Error, 1/True = OK + """ + if peerCert.has_expired(): + # Any expired certificate in the chain should result in an error + log.error('Certificate %r in peer certificate chain has expired', + peerCert.get_subject()) + + return False + + elif errorDepth == 0: + # Only interested in DN of last certificate in the chain - this must + # match the expected Server DN setting + peerCertSubj = peerCert.get_subject() + peerCertDN = peerCertSubj.get_components() + peerCertDN.sort() + + if self.certDN is None: + # Check hostname against peer certificate CN field instead: + if self.hostname is None: + log.error('No "hostname" or "certDN" set to check peer ' + 'certificate against') + return False + + # Check for subject alternative names + if self.__subj_alt_name_match: + dns_names = self._get_subj_alt_name(peerCert) + if self.hostname in dns_names: + return preverifyOK + + # If no subjectAltNames, default to check of subject Common Name + if peerCertSubj.commonName == self.hostname: + return preverifyOK + else: + log.error('Peer certificate CN %r doesn\'t match the ' + 'expected CN %r', peerCertSubj.commonName, + self.hostname) + return False + else: + if peerCertDN == self.certDN: + return preverifyOK + else: + log.error('Peer certificate DN %r doesn\'t match the ' + 'expected DN %r', peerCertDN, self.certDN) + return False + else: + return preverifyOK + + def get_verify_server_cert_func(self): + def verify_server_cert(connection, peerCert, errorStatus, errorDepth, + preverifyOK): + return self.__call__(connection, peerCert, errorStatus, + errorDepth, preverifyOK) + + return verify_server_cert + + @classmethod + def _get_subj_alt_name(cls, peer_cert): + '''Extract subjectAltName DNS name settings from certificate extensions + + @param peer_cert: peer certificate in SSL connection. subjectAltName + settings if any will be extracted from this + @type peer_cert: OpenSSL.crypto.X509 + ''' + # Search through extensions + dns_name = [] + general_names = SubjectAltName() + for i in range(peer_cert.get_extension_count()): + ext = peer_cert.get_extension(i) + ext_name = ext.get_short_name() + if ext_name == cls.SUBJ_ALT_NAME_EXT_NAME: + # PyOpenSSL returns extension data in ASN.1 encoded form + ext_dat = ext.get_data() + decoded_dat = der_decoder.decode(ext_dat, + asn1Spec=general_names) + + for name in decoded_dat: + if isinstance(name, SubjectAltName): + for entry in range(len(name)): + component = name.getComponentByPosition(entry) + dns_name.append(str(component.getComponent())) + + return dns_name + + def _getCertDN(self): + return self.__certDN + + def _setCertDN(self, val): + if isinstance(val, str): + # Allow for quoted DN + certDN = val.strip('"') + + dnFields = self.__class__.PARSER_RE.split(certDN) + if len(dnFields) < 2: + raise TypeError('Error parsing DN string: "%s"' % certDN) + + self.__certDN = list(zip(dnFields[1::2], dnFields[2::2])) + self.__certDN.sort() + + elif not isinstance(val, list): + for i in val: + if not len(i) == 2: + raise TypeError('Expecting list of two element DN field, ' + 'DN field value pairs for "certDN" ' + 'attribute') + self.__certDN = val + else: + raise TypeError('Expecting list or string type for "certDN" ' + 'attribute') + + certDN = property(fget=_getCertDN, + fset=_setCertDN, + doc="Distinguished Name for Server Certificate") + + # Get/Set Property methods + def _getHostname(self): + return self.__hostname + + def _setHostname(self, val): + if not isinstance(val, str): + raise TypeError("Expecting string type for hostname " + "attribute") + self.__hostname = val + + hostname = property(fget=_getHostname, + fset=_setHostname, + doc="hostname of server") diff --git a/libs/ndg/httpsclient/ssl_socket.py b/libs/ndg/httpsclient/ssl_socket.py new file mode 100644 index 0000000..7780314 --- /dev/null +++ b/libs/ndg/httpsclient/ssl_socket.py @@ -0,0 +1,282 @@ +"""PyOpenSSL utilities including HTTPSSocket class which wraps PyOpenSSL +SSL connection into a httplib-like interface suitable for use with urllib2 + +""" +__author__ = "P J Kershaw" +__date__ = "21/12/10" +__copyright__ = "(C) 2012 Science and Technology Facilities Council" +__license__ = "BSD - see LICENSE file in top-level directory" +__contact__ = "Philip.Kershaw@stfc.ac.uk" +__revision__ = '$Id$' + +from datetime import datetime +import logging +import socket +from io import BytesIO + +from OpenSSL import SSL + +log = logging.getLogger(__name__) + + +class SSLSocket(object): + """SSL Socket class wraps pyOpenSSL's SSL.Connection class implementing + the makefile method so that it is compatible with the standard socket + interface and usable with httplib. + + @cvar default_buf_size: default buffer size for recv operations in the + makefile method + @type default_buf_size: int + """ + default_buf_size = 8192 + + def __init__(self, ctx, sock=None): + """Create SSL socket object + + @param ctx: SSL context + @type ctx: OpenSSL.SSL.Context + @param sock: underlying socket object + @type sock: socket.socket + """ + if sock is not None: + self.socket = sock + else: + self.socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + self.socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + + self.__ssl_conn = SSL.Connection(ctx, self.socket) + self.buf_size = self.__class__.default_buf_size + self._makefile_refs = 0 + + def __del__(self): + """Close underlying socket when this object goes out of scope + """ + self.close() + + @property + def buf_size(self): + """Buffer size for makefile method recv() operations""" + return self.__buf_size + + @buf_size.setter + def buf_size(self, value): + """Buffer size for makefile method recv() operations""" + if not isinstance(value, int): + raise TypeError('Expecting int type for "buf_size"; ' + 'got %r instead' % type(value)) + self.__buf_size = value + + def close(self): + """Shutdown the SSL connection and call the close method of the + underlying socket""" + if self._makefile_refs < 1: + try: + self.__ssl_conn.shutdown() + except (SSL.Error, SSL.SysCallError): + # Make errors on shutdown non-fatal + pass + else: + self._makefile_refs -= 1 + + def set_shutdown(self, mode): + """Set the shutdown state of the Connection. + @param mode: bit vector of either or both of SENT_SHUTDOWN and + RECEIVED_SHUTDOWN + """ + self.__ssl_conn.set_shutdown(mode) + + def get_shutdown(self): + """Get the shutdown state of the Connection. + @return: bit vector of either or both of SENT_SHUTDOWN and + RECEIVED_SHUTDOWN + """ + return self.__ssl_conn.get_shutdown() + + def bind(self, addr): + """bind to the given address - calls method of the underlying socket + @param addr: address/port number tuple + @type addr: tuple""" + self.__ssl_conn.bind(addr) + + def listen(self, backlog): + """Listen for connections made to the socket. + + @param backlog: specifies the maximum number of queued connections and + should be at least 1; the maximum value is system-dependent (usually 5). + @param backlog: int + """ + self.__ssl_conn.listen(backlog) + + def set_accept_state(self): + """Set the connection to work in server mode. The handshake will be + handled automatically by read/write""" + self.__ssl_conn.set_accept_state() + + def accept(self): + """Accept an SSL connection. + + @return: pair (ssl, addr) where ssl is a new SSL connection object and + addr is the address bound to the other end of the SSL connection. + @rtype: tuple + """ + return self.__ssl_conn.accept() + + def set_connect_state(self): + """Set the connection to work in client mode. The handshake will be + handled automatically by read/write""" + self.__ssl_conn.set_connect_state() + + def connect(self, addr): + """Call the connect method of the underlying socket and set up SSL on + the socket, using the Context object supplied to this Connection object + at creation. + + @param addr: address/port number pair + @type addr: tuple + """ + self.__ssl_conn.connect(addr) + + def shutdown(self, how): + """Send the shutdown message to the Connection. + + @param how: for socket.socket this flag determines whether read, write + or both type operations are supported. OpenSSL.SSL.Connection doesn't + support this so this parameter is IGNORED + @return: true if the shutdown message exchange is completed and false + otherwise (in which case you call recv() or send() when the connection + becomes readable/writeable. + @rtype: bool + """ + return self.__ssl_conn.shutdown() + + def renegotiate(self): + """Renegotiate this connection's SSL parameters.""" + return self.__ssl_conn.renegotiate() + + def pending(self): + """@return: numbers of bytes that can be safely read from the SSL + buffer. + @rtype: int + """ + return self.__ssl_conn.pending() + + def send(self, data, *flags_arg): + """Send data to the socket. Nb. The optional flags argument is ignored. + - retained for compatibility with socket.socket interface + + @param data: data to send down the socket + @type data: string + """ + return self.__ssl_conn.send(data) + + def sendall(self, data): + self.__ssl_conn.sendall(data) + + def recv(self, size=default_buf_size): + """Receive data from the Connection. + + @param size: The maximum amount of data to be received at once + @type size: int + @return: data received. + @rtype: string + """ + return self.__ssl_conn.recv(size) + + def setblocking(self, mode): + """Set this connection's underlying socket blocking _mode_. + + @param mode: blocking mode + @type mode: int + """ + self.__ssl_conn.setblocking(mode) + + def fileno(self): + """ + @return: file descriptor number for the underlying socket + @rtype: int + """ + return self.__ssl_conn.fileno() + + def getsockopt(self, *args): + """See socket.socket.getsockopt + """ + return self.__ssl_conn.getsockopt(*args) + + def setsockopt(self, *args): + """See socket.socket.setsockopt + + @return: value of the given socket option + @rtype: int/string + """ + return self.__ssl_conn.setsockopt(*args) + + def state_string(self): + """Return the SSL state of this connection.""" + return self.__ssl_conn.state_string() + + def makefile(self, *args): + """Specific to Python socket API and required by httplib: convert + response into a file-like object. This implementation reads using recv + and copies the output into a StringIO buffer to simulate a file object + for consumption by httplib + + Nb. Ignoring optional file open mode (StringIO is generic and will + open for read and write unless a string is passed to the constructor) + and buffer size - httplib set a zero buffer size which results in recv + reading nothing + + @return: file object for data returned from socket + @rtype: cStringIO.StringO + """ + self._makefile_refs += 1 + + # Optimisation + _buf_size = self.buf_size + + i=0 + stream = BytesIO() + startTime = datetime.utcnow() + try: + dat = self.__ssl_conn.recv(_buf_size) + while dat: + i+=1 + stream.write(dat) + dat = self.__ssl_conn.recv(_buf_size) + + except (SSL.ZeroReturnError, SSL.SysCallError): + # Connection is closed - assuming here that all is well and full + # response has been received. httplib will catch an error in + # incomplete content since it checks the content-length header + # against the actual length of data received + pass + + if log.getEffectiveLevel() <= logging.DEBUG: + log.debug("Socket.makefile %d recv calls completed in %s", i, + datetime.utcnow() - startTime) + + # Make sure to rewind the buffer otherwise consumers of the content will + # read from the end of the buffer + stream.seek(0) + + return stream + + def getsockname(self): + """ + @return: the socket's own address + @rtype: + """ + return self.__ssl_conn.getsockname() + + def getpeername(self): + """ + @return: remote address to which the socket is connected + """ + return self.__ssl_conn.getpeername() + + def get_context(self): + '''Retrieve the Context object associated with this Connection. ''' + return self.__ssl_conn.get_context() + + def get_peer_certificate(self): + '''Retrieve the other side's certificate (if any) ''' + return self.__ssl_conn.get_peer_certificate() diff --git a/libs/ndg/httpsclient/subj_alt_name.py b/libs/ndg/httpsclient/subj_alt_name.py new file mode 100644 index 0000000..b2c1918 --- /dev/null +++ b/libs/ndg/httpsclient/subj_alt_name.py @@ -0,0 +1,153 @@ +"""NDG HTTPS Client package + +Use pyasn1 to provide support for parsing ASN.1 formatted subjectAltName +content for SSL peer verification. Code based on: + +http://stackoverflow.com/questions/5519958/how-do-i-parse-subjectaltname-extension-data-using-pyasn1 +""" +__author__ = "P J Kershaw" +__date__ = "01/02/12" +__copyright__ = "(C) 2012 Science and Technology Facilities Council" +__license__ = "BSD - see LICENSE file in top-level directory" +__contact__ = "Philip.Kershaw@stfc.ac.uk" +__revision__ = '$Id$' +try: + from pyasn1.type import univ, constraint, char, namedtype, tag + +except ImportError as e: + import_error_msg = ('Error importing pyasn1, subjectAltName check for SSL ' + 'peer verification will be disabled. Import error ' + 'is: %s' % e) + import warnings + warnings.warn(import_error_msg) + class Pyasn1ImportError(ImportError): + "Raise for pyasn1 import error" + raise Pyasn1ImportError(import_error_msg) + + +MAX = 64 + + +class DirectoryString(univ.Choice): + """ASN.1 Directory string class""" + componentType = namedtype.NamedTypes( + namedtype.NamedType( + 'teletexString', char.TeletexString().subtype( + subtypeSpec=constraint.ValueSizeConstraint(1, MAX))), + namedtype.NamedType( + 'printableString', char.PrintableString().subtype( + subtypeSpec=constraint.ValueSizeConstraint(1, MAX))), + namedtype.NamedType( + 'universalString', char.UniversalString().subtype( + subtypeSpec=constraint.ValueSizeConstraint(1, MAX))), + namedtype.NamedType( + 'utf8String', char.UTF8String().subtype( + subtypeSpec=constraint.ValueSizeConstraint(1, MAX))), + namedtype.NamedType( + 'bmpString', char.BMPString().subtype( + subtypeSpec=constraint.ValueSizeConstraint(1, MAX))), + namedtype.NamedType( + 'ia5String', char.IA5String().subtype( + subtypeSpec=constraint.ValueSizeConstraint(1, MAX))), + ) + + +class AttributeValue(DirectoryString): + """ASN.1 Attribute value""" + + +class AttributeType(univ.ObjectIdentifier): + """ASN.1 Attribute type""" + + +class AttributeTypeAndValue(univ.Sequence): + """ASN.1 Attribute type and value class""" + componentType = namedtype.NamedTypes( + namedtype.NamedType('type', AttributeType()), + namedtype.NamedType('value', AttributeValue()), + ) + + +class RelativeDistinguishedName(univ.SetOf): + '''ASN.1 Realtive distinguished name''' + componentType = AttributeTypeAndValue() + +class RDNSequence(univ.SequenceOf): + '''ASN.1 RDN sequence class''' + componentType = RelativeDistinguishedName() + + +class Name(univ.Choice): + '''ASN.1 name class''' + componentType = namedtype.NamedTypes( + namedtype.NamedType('', RDNSequence()), + ) + + +class Extension(univ.Sequence): + '''ASN.1 extension class''' + componentType = namedtype.NamedTypes( + namedtype.NamedType('extnID', univ.ObjectIdentifier()), + namedtype.DefaultedNamedType('critical', univ.Boolean('False')), + namedtype.NamedType('extnValue', univ.OctetString()), + ) + + +class Extensions(univ.SequenceOf): + '''ASN.1 extensions class''' + componentType = Extension() + sizeSpec = univ.SequenceOf.sizeSpec + constraint.ValueSizeConstraint(1, MAX) + + +class AnotherName(univ.Sequence): + componentType = namedtype.NamedTypes( + namedtype.NamedType('type-id', univ.ObjectIdentifier()), + namedtype.NamedType('value', univ.Any().subtype( + explicitTag=tag.Tag(tag.tagClassContext, + tag.tagFormatSimple, 0))) + ) + + +class GeneralName(univ.Choice): + '''ASN.1 configuration for X.509 certificate subjectAltNames fields''' + componentType = namedtype.NamedTypes( + namedtype.NamedType('otherName', AnotherName().subtype( + implicitTag=tag.Tag(tag.tagClassContext, + tag.tagFormatSimple, 0))), + namedtype.NamedType('rfc822Name', char.IA5String().subtype( + implicitTag=tag.Tag(tag.tagClassContext, + tag.tagFormatSimple, 1))), + namedtype.NamedType('dNSName', char.IA5String().subtype( + implicitTag=tag.Tag(tag.tagClassContext, + tag.tagFormatSimple, 2))), +# namedtype.NamedType('x400Address', ORAddress().subtype( +# implicitTag=tag.Tag(tag.tagClassContext, +# tag.tagFormatSimple, 3))), + namedtype.NamedType('directoryName', Name().subtype( + implicitTag=tag.Tag(tag.tagClassContext, + tag.tagFormatSimple, 4))), +# namedtype.NamedType('ediPartyName', EDIPartyName().subtype( +# implicitTag=tag.Tag(tag.tagClassContext, +# tag.tagFormatSimple, 5))), + namedtype.NamedType('uniformResourceIdentifier', char.IA5String().subtype( + implicitTag=tag.Tag(tag.tagClassContext, + tag.tagFormatSimple, 6))), + namedtype.NamedType('iPAddress', univ.OctetString().subtype( + implicitTag=tag.Tag(tag.tagClassContext, + tag.tagFormatSimple, 7))), + namedtype.NamedType('registeredID', univ.ObjectIdentifier().subtype( + implicitTag=tag.Tag(tag.tagClassContext, + tag.tagFormatSimple, 8))), + ) + + +class GeneralNames(univ.SequenceOf): + '''Sequence of names for ASN.1 subjectAltNames settings''' + componentType = GeneralName() + sizeSpec = univ.SequenceOf.sizeSpec + constraint.ValueSizeConstraint(1, MAX) + + +class SubjectAltName(GeneralNames): + '''ASN.1 implementation for subjectAltNames support''' + + diff --git a/libs/ndg/httpsclient/urllib2_build_opener.py b/libs/ndg/httpsclient/urllib2_build_opener.py new file mode 100644 index 0000000..55d8632 --- /dev/null +++ b/libs/ndg/httpsclient/urllib2_build_opener.py @@ -0,0 +1,78 @@ +"""urllib2 style build opener integrates with HTTPSConnection class from this +package. +""" +__author__ = "P J Kershaw" +__date__ = "21/12/10" +__copyright__ = "(C) 2011 Science and Technology Facilities Council" +__license__ = "BSD - see LICENSE file in top-level directory" +__contact__ = "Philip.Kershaw@stfc.ac.uk" +__revision__ = '$Id$' +import logging +import sys + +# Py 2 <=> 3 compatibility for class type checking +if sys.version_info[0] > 2: + class_type_ = type + from urllib.request import (ProxyHandler, UnknownHandler, + HTTPDefaultErrorHandler, FTPHandler, + FileHandler, HTTPErrorProcessor, + HTTPHandler, OpenerDirector, + HTTPRedirectHandler) +else: + import types + class_type_ = types.ClassType + + from urllib2 import (ProxyHandler, UnknownHandler, HTTPDefaultErrorHandler, + FTPHandler, FileHandler, HTTPErrorProcessor, + HTTPHandler, OpenerDirector, HTTPRedirectHandler) + +from ndg.httpsclient.https import HTTPSContextHandler + +log = logging.getLogger(__name__) + + +# Copied from urllib2 with modifications for ssl +def build_opener(*handlers, **kw): + """Create an opener object from a list of handlers. + + The opener will use several default handlers, including support + for HTTP and FTP. + + If any of the handlers passed as arguments are subclasses of the + default handlers, the default handlers will not be used. + """ + def isclass(obj): + return isinstance(obj, class_type_) or hasattr(obj, "__bases__") + + opener = OpenerDirector() + default_classes = [ProxyHandler, UnknownHandler, HTTPHandler, + HTTPDefaultErrorHandler, HTTPRedirectHandler, + FTPHandler, FileHandler, HTTPErrorProcessor] + check_classes = list(default_classes) + check_classes.append(HTTPSContextHandler) + skip = [] + for klass in check_classes: + for check in handlers: + if isclass(check): + if issubclass(check, klass): + skip.append(klass) + elif isinstance(check, klass): + skip.append(klass) + + for klass in default_classes: + if klass not in skip: + opener.add_handler(klass()) + + # Pick up SSL context from keyword settings + ssl_context = kw.get('ssl_context') + + # Add the HTTPS handler with ssl_context + if HTTPSContextHandler not in skip: + opener.add_handler(HTTPSContextHandler(ssl_context)) + + for h in handlers: + if isclass(h): + h = h() + opener.add_handler(h) + + return opener diff --git a/libs/ndg/httpsclient/utils.py b/libs/ndg/httpsclient/utils.py new file mode 100644 index 0000000..a2b0ed3 --- /dev/null +++ b/libs/ndg/httpsclient/utils.py @@ -0,0 +1,414 @@ +"""Utilities using NDG HTTPS Client, including a main module that can be used to +fetch from a URL. +""" +__author__ = "R B Wilkinson" +__date__ = "09/12/11" +__copyright__ = "(C) 2011 Science and Technology Facilities Council" +__license__ = "BSD - see LICENSE file in top-level directory" +__contact__ = "Philip.Kershaw@stfc.ac.uk" +__revision__ = '$Id$' + +import logging +from optparse import OptionParser +import os +import sys + +if sys.version_info[0] > 2: + import http.cookiejar as cookiejar_ + import http.client as http_client_ + from urllib.request import Request as Request_ + from urllib.request import HTTPHandler as HTTPHandler_ + from urllib.request import HTTPCookieProcessor as HTTPCookieProcessor_ + from urllib.request import HTTPBasicAuthHandler as HTTPBasicAuthHandler_ + from urllib.request import HTTPPasswordMgrWithDefaultRealm as \ + HTTPPasswordMgrWithDefaultRealm_ + from urllib.request import ProxyHandler as ProxyHandler_ + from urllib.error import HTTPError as HTTPError_ + import urllib.parse as urlparse_ +else: + import cookielib as cookiejar_ + import httplib as http_client_ + from urllib2 import Request as Request_ + from urllib2 import HTTPHandler as HTTPHandler_ + from urllib2 import HTTPCookieProcessor as HTTPCookieProcessor_ + from urllib2 import HTTPBasicAuthHandler as HTTPBasicAuthHandler_ + from urllib2 import HTTPPasswordMgrWithDefaultRealm as \ + HTTPPasswordMgrWithDefaultRealm_ + from urllib2 import ProxyHandler as ProxyHandler_ + from urllib2 import HTTPError as HTTPError_ + import urlparse as urlparse_ + +from ndg.httpsclient.urllib2_build_opener import build_opener +from ndg.httpsclient.https import HTTPSContextHandler +from ndg.httpsclient import ssl_context_util + +log = logging.getLogger(__name__) + +class AccumulatingHTTPCookieProcessor(HTTPCookieProcessor_): + """Cookie processor that adds new cookies (instead of replacing the existing + ones as HTTPCookieProcessor does) + """ + def http_request(self, request): + """Processes cookies for a HTTP request. + @param request: request to process + @type request: urllib2.Request + @return: request + @rtype: urllib2.Request + """ + COOKIE_HEADER_NAME = "Cookie" + tmp_request = Request_(request.get_full_url(), request.data, {}, + request.origin_req_host, + request.unverifiable) + self.cookiejar.add_cookie_header(tmp_request) + # Combine existing and new cookies. + new_cookies = tmp_request.get_header(COOKIE_HEADER_NAME) + if new_cookies: + if request.has_header(COOKIE_HEADER_NAME): + # Merge new cookies with existing ones. + old_cookies = request.get_header(COOKIE_HEADER_NAME) + merged_cookies = '; '.join([old_cookies, new_cookies]) + request.add_unredirected_header(COOKIE_HEADER_NAME, + merged_cookies) + else: + # No existing cookies so just set new ones. + request.add_unredirected_header(COOKIE_HEADER_NAME, new_cookies) + return request + + # Process cookies for HTTPS in the same way. + https_request = http_request + + +class URLFetchError(Exception): + """Error fetching content from URL""" + + +def fetch_from_url(url, config, data=None, handlers=None): + """Returns data retrieved from a URL. + @param url: URL to attempt to open + @type url: basestring + @param config: SSL context configuration + @type config: Configuration + @return data retrieved from URL or None + """ + return_code, return_message, response = open_url(url, config, data=data, + handlers=handlers) + if return_code and return_code == http_client_.OK: + return_data = response.read() + response.close() + return return_data + else: + raise URLFetchError(return_message) + +def fetch_from_url_to_file(url, config, output_file, data=None, handlers=None): + """Writes data retrieved from a URL to a file. + @param url: URL to attempt to open + @type url: basestring + @param config: SSL context configuration + @type config: Configuration + @param output_file: output file + @type output_file: basestring + @return: tuple ( + returned HTTP status code or 0 if an error occurred + returned message + boolean indicating whether access was successful) + """ + return_code, return_message, response = open_url(url, config, data=data, + handlers=handlers) + if return_code == http_client_.OK: + return_data = response.read() + response.close() + outfile = open(output_file, "w") + outfile.write(return_data) + outfile.close() + + return return_code, return_message, return_code == http_client_.OK + + +def fetch_stream_from_url(url, config, data=None, handlers=None): + """Returns data retrieved from a URL. + @param url: URL to attempt to open + @type url: basestring + @param config: SSL context configuration + @type config: Configuration + @param data: HTTP POST data + @type data: str + @param handlers: list of custom urllib2 handlers to add to the request + @type handlers: iterable + @return: data retrieved from URL or None + @rtype: file derived type + """ + return_code, return_message, response = open_url(url, config, data=data, + handlers=handlers) + if return_code and return_code == http_client_.OK: + return response + else: + raise URLFetchError(return_message) + + +def open_url(url, config, data=None, handlers=None): + """Attempts to open a connection to a specified URL. + @param url: URL to attempt to open + @param config: SSL context configuration + @type config: Configuration + @param data: HTTP POST data + @type data: str + @param handlers: list of custom urllib2 handlers to add to the request + @type handlers: iterable + @return: tuple ( + returned HTTP status code or 0 if an error occurred + returned message or error description + response object) + """ + debuglevel = 1 if config.debug else 0 + + # Set up handlers for URL opener. + if config.cookie: + cj = config.cookie + else: + cj = cookiejar_.CookieJar() + + # Use a cookie processor that accumulates cookies when redirects occur so + # that an application can redirect for authentication and retain both any + # cookies for the application and the security system (c.f., + # urllib2.HTTPCookieProcessor which replaces cookies). + cookie_handler = AccumulatingHTTPCookieProcessor(cj) + + if not handlers: + handlers = [] + + handlers.append(cookie_handler) + + if config.debug: + http_handler = HTTPHandler_(debuglevel=debuglevel) + https_handler = HTTPSContextHandler(config.ssl_context, + debuglevel=debuglevel) + handlers.extend([http_handler, https_handler]) + + if config.http_basicauth: + # currently only supports http basic auth + auth_handler = HTTPBasicAuthHandler_(HTTPPasswordMgrWithDefaultRealm_()) + auth_handler.add_password(realm=None, uri=url, + user=config.httpauth[0], + passwd=config.httpauth[1]) + handlers.append(auth_handler) + + + # Explicitly remove proxy handling if the host is one listed in the value of + # the no_proxy environment variable because urllib2 does use proxy settings + # set via http_proxy and https_proxy, but does not take the no_proxy value + # into account. + if not _should_use_proxy(url, config.no_proxy): + handlers.append(ProxyHandler_({})) + log.debug("Not using proxy") + elif config.proxies: + handlers.append(ProxyHandler_(config.proxies)) + log.debug("Configuring proxies: %s" % config.proxies) + + opener = build_opener(*handlers, ssl_context=config.ssl_context) + + headers = config.headers + if headers is None: + headers = {} + + request = Request_(url, data, headers) + + # Open the URL and check the response. + return_code = 0 + return_message = '' + response = None + + # FIXME + response = opener.open(request) + + try: + response = opener.open(request) + return_message = response.msg + return_code = response.code + if log.isEnabledFor(logging.DEBUG): + for index, cookie in enumerate(cj): + log.debug("%s : %s", index, cookie) + + except HTTPError_ as exc: + return_code = exc.code + return_message = "Error: %s" % exc.msg + if log.isEnabledFor(logging.DEBUG): + log.debug("%s %s", exc.code, exc.msg) + + except Exception as exc: + return_message = "Error: %s" % exc.__str__() + if log.isEnabledFor(logging.DEBUG): + import traceback + log.debug(traceback.format_exc()) + + return (return_code, return_message, response) + + +def _should_use_proxy(url, no_proxy=None): + """Determines whether a proxy should be used to open a connection to the + specified URL, based on the value of the no_proxy environment variable. + @param url: URL + @type url: basestring or urllib2.Request + """ + if no_proxy is None: + no_proxy_effective = os.environ.get('no_proxy', '') + else: + no_proxy_effective = no_proxy + + urlObj = urlparse_.urlparse(_url_as_string(url)) + for np in [h.strip() for h in no_proxy_effective.split(',')]: + if urlObj.hostname == np: + return False + + return True + +def _url_as_string(url): + """Returns the URL string from a URL value that is either a string or + urllib2.Request.. + @param url: URL + @type url: basestring or urllib2.Request + @return: URL string + @rtype: basestring + """ + if isinstance(url, Request_): + return url.get_full_url() + elif isinstance(url, str): + return url + else: + raise TypeError("Expected type %r or %r" % + (str, Request_)) + + +class Configuration(object): + """Connection configuration. + """ + def __init__(self, ssl_context, debug=False, proxies=None, no_proxy=None, + cookie=None, http_basicauth=None, headers=None): + """ + @param ssl_context: SSL context to use with this configuration + @type ssl_context: OpenSSL.SSL.Context + @param debug: if True, output debugging information + @type debug: bool + @param proxies: proxies to use for + @type proxies: dict with basestring keys and values + @param no_proxy: hosts for which a proxy should not be used + @type no_proxy: basestring + @param cookie: cookies to set for request + @type cookie: cookielib.CookieJar (python 3 - http.cookiejar) + @param http_basicauth: http authentication, or None + @type http_basicauth: tuple of (username,password) + @param headers: http headers + @type headers: dict + """ + self.ssl_context = ssl_context + self.debug = debug + self.proxies = proxies + self.no_proxy = no_proxy + self.cookie = cookie + self.http_basicauth = http_basicauth + self.headers = headers + + +def main(): + '''Utility to fetch data using HTTP or HTTPS GET from a specified URL. + ''' + parser = OptionParser(usage="%prog [options] url") + parser.add_option("-c", "--certificate", dest="cert_file", metavar="FILE", + default=os.path.expanduser("~/credentials.pem"), + help="Certificate file - defaults to $HOME/credentials.pem") + parser.add_option("-k", "--private-key", dest="key_file", metavar="FILE", + default=None, + help="Private key file - defaults to the certificate file") + parser.add_option("-t", "--ca-certificate-dir", dest="ca_dir", + metavar="PATH", + default=None, + help="Trusted CA certificate file directory") + parser.add_option("-d", "--debug", action="store_true", dest="debug", + default=False, + help="Print debug information.") + parser.add_option("-p", "--post-data-file", dest="data_file", + metavar="FILE", default=None, + help="POST data file") + parser.add_option("-f", "--fetch", dest="output_file", metavar="FILE", + default=None, help="Output file") + parser.add_option("-n", "--no-verify-peer", action="store_true", + dest="no_verify_peer", default=False, + help="Skip verification of peer certificate.") + parser.add_option("-a", "--basicauth", dest="basicauth", + metavar="USER:PASSWD", + default=None, + help="HTTP authentication credentials") + parser.add_option("--header", action="append", dest="headers", + metavar="HEADER: VALUE", + help="Add HTTP header to request") + (options, args) = parser.parse_args() + if len(args) != 1: + parser.error("Incorrect number of arguments") + + url = args[0] + + if options.debug: + logging.getLogger().setLevel(logging.DEBUG) + + if options.key_file and os.path.exists(options.key_file): + key_file = options.key_file + else: + key_file = None + + if options.cert_file and os.path.exists(options.cert_file): + cert_file = options.cert_file + else: + cert_file = None + + if options.ca_dir and os.path.exists(options.ca_dir): + ca_dir = options.ca_dir + else: + ca_dir = None + + verify_peer = not options.no_verify_peer + + if options.data_file and os.path.exists(options.data_file): + data_file = open(options.data_file) + data = data_file.read() + data_file.close() + else: + data = None + + if options.basicauth: + http_basicauth = options.basicauth.split(':', 1) + else: + http_basicauth = None + + headers = {} + if options.headers: + for h in options.headers: + key, val = h.split(':', 1) + headers[key.strip()] = val.lstrip() + + # If a private key file is not specified, the key is assumed to be stored in + # the certificate file. + ssl_context = ssl_context_util.make_ssl_context(key_file, + cert_file, + None, + ca_dir, + verify_peer, + url) + + config = Configuration(ssl_context, + options.debug, + http_basicauth=http_basicauth, + headers=headers) + if options.output_file: + return_code, return_message = fetch_from_url_to_file( + url, + config, + options.output_file, + data)[:2] + raise SystemExit(return_code, return_message) + else: + data = fetch_from_url(url, config) + print(data) + + +if __name__=='__main__': + logging.basicConfig() + main()