ndg httpsclient lib

libs/ndg/__init__.py

@@ -0,0 +1,19 @@
+"""ndg_httpsclient - PyOpenSSL utility to make a httplib-like interface suitable
+for use with urllib2
+
+This is a setuptools namespace_package.  DO NOT place any other
+code in this file!  There is no guarantee that it will be installed
+with easy_install.  See:
+
+http://peak.telecommunity.com/DevCenter/setuptools#namespace-packages
+
+... for details.
+"""
+__author__ = "P J Kershaw"
+__date__ = "06/01/12"
+__copyright__ = "(C) 2012 Science and Technology Facilities Council"
+__license__ = "BSD - see LICENSE file in top-level directory"
+__contact__ = "Philip.Kershaw@stfc.ac.uk"
+__revision__ = '$Id$'
+
+__import__('pkg_resources').declare_namespace(__name__)

libs/ndg/httpsclient/__init__.py

@@ -0,0 +1,9 @@
+"""ndg_httpsclient - PyOpenSSL utility to make a httplib-like interface suitable
+for use with urllib2
+"""
+__author__ = "P J Kershaw (STFC) and Richard Wilkinson (Tessella)"
+__date__ = "09/12/11"
+__copyright__ = "(C) 2011 Science and Technology Facilities Council"
+__license__ = "BSD - see LICENSE file in top-level directory"
+__contact__ = "Philip.Kershaw@stfc.ac.uk"
+__revision__ = '$Id$'

libs/ndg/httpsclient/https.py

@@ -0,0 +1,131 @@
+"""ndg_httpsclient HTTPS module containing PyOpenSSL implementation of
+httplib.HTTPSConnection
+
+PyOpenSSL utility to make a httplib-like interface suitable for use with 
+urllib2
+"""
+__author__ = "P J Kershaw (STFC)"
+__date__ = "09/12/11"
+__copyright__ = "(C) 2012 Science and Technology Facilities Council"
+__license__ = "BSD - see LICENSE file in top-level directory"
+__contact__ = "Philip.Kershaw@stfc.ac.uk"
+__revision__ = '$Id$'
+import logging
+import socket
+import sys
+
+if sys.version_info[0] > 2:
+    from http.client import HTTPS_PORT
+    from http.client import HTTPConnection
+
+    from urllib.request import AbstractHTTPHandler
+else:
+    from httplib import HTTPS_PORT
+    from httplib import HTTPConnection
+
+    from urllib2 import AbstractHTTPHandler
+
+
+from OpenSSL import SSL
+
+from ndg.httpsclient.ssl_socket import SSLSocket
+
+log = logging.getLogger(__name__)
+
+
+class HTTPSConnection(HTTPConnection):
+    """This class allows communication via SSL using PyOpenSSL.
+    It is based on httplib.HTTPSConnection, modified to use PyOpenSSL.
+
+    Note: This uses the constructor inherited from HTTPConnection to allow it to
+    be used with httplib and HTTPSContextHandler. To use the class directly with
+    an SSL context set ssl_context after construction.
+    
+    @cvar default_port: default port for this class (443)
+    @type default_port: int
+    @cvar default_ssl_method: default SSL method used if no SSL context is
+    explicitly set - defaults to version 2/3.
+    @type default_ssl_method: int
+    """
+    default_port = HTTPS_PORT
+    default_ssl_method = SSL.SSLv23_METHOD
+    
+    def __init__(self, host, port=None, strict=None,
+                 timeout=socket._GLOBAL_DEFAULT_TIMEOUT, ssl_context=None):
+        HTTPConnection.__init__(self, host, port, strict, timeout)
+        if not hasattr(self, 'ssl_context'):
+            self.ssl_context = None
+
+        if ssl_context is not None:
+            if not isinstance(ssl_context, SSL.Context):
+                raise TypeError('Expecting OpenSSL.SSL.Context type for "'
+                                'ssl_context" keyword; got %r instead' %
+                                ssl_context)
+                
+            self.ssl_context = ssl_context
+            
+    def connect(self):
+        """Create SSL socket and connect to peer
+        """
+        if getattr(self, 'ssl_context', None):
+            if not isinstance(self.ssl_context, SSL.Context):
+                raise TypeError('Expecting OpenSSL.SSL.Context type for "'
+                                'ssl_context" attribute; got %r instead' %
+                                self.ssl_context)
+            ssl_context = self.ssl_context
+        else:
+            ssl_context = SSL.Context(self.__class__.default_ssl_method)
+
+        sock = socket.create_connection((self.host, self.port), self.timeout)
+        
+        # Tunnel if using a proxy - ONLY available for Python 2.6.2 and above      
+        if getattr(self, '_tunnel_host', None):
+            self.sock = sock
+            self._tunnel()
+            
+        self.sock = SSLSocket(ssl_context, sock)
+        
+        # Go to client mode.
+        self.sock.set_connect_state()
+
+    def close(self):
+        """Close socket and shut down SSL connection"""
+        if hasattr(self.sock, "close"):
+            self.sock.close()
+        
+        
+class HTTPSContextHandler(AbstractHTTPHandler):
+    '''HTTPS handler that allows a SSL context to be set for the SSL
+    connections.
+    '''
+    https_request = AbstractHTTPHandler.do_request_
+
+    def __init__(self, ssl_context, debuglevel=0):
+        """
+        @param ssl_context:SSL context
+        @type ssl_context: OpenSSL.SSL.Context
+        @param debuglevel: debug level for HTTPSHandler
+        @type debuglevel: int
+        """
+        AbstractHTTPHandler.__init__(self, debuglevel)
+
+        if ssl_context is not None:
+            if not isinstance(ssl_context, SSL.Context):
+                raise TypeError('Expecting OpenSSL.SSL.Context type for "'
+                                'ssl_context" keyword; got %r instead' %
+                                ssl_context)
+            self.ssl_context = ssl_context
+        else:
+            self.ssl_context = SSL.Context(SSL.TLSv1_METHOD)
+
+    def https_open(self, req):
+        """Opens HTTPS request
+        @param req: HTTP request
+        @return: HTTP Response object
+        """
+        # Make a custom class extending HTTPSConnection, with the SSL context
+        # set as a class variable so that it is available to the connect method.
+        customHTTPSContextConnection = type('CustomHTTPSContextConnection',
+                                            (HTTPSConnection, object),
+                                            {'ssl_context': self.ssl_context})
+        return self.do_open(customHTTPSContextConnection, req)

libs/ndg/httpsclient/ssl_context_util.py

@@ -0,0 +1,98 @@
+"""ndg_httpsclient SSL Context utilities module containing convenience routines
+for setting SSL context configuration.
+
+"""
+__author__ = "P J Kershaw (STFC)"
+__date__ = "09/12/11"
+__copyright__ = "(C) 2012 Science and Technology Facilities Council"
+__license__ = "BSD - see LICENSE file in top-level directory"
+__contact__ = "Philip.Kershaw@stfc.ac.uk"
+__revision__ = '$Id$'
+import sys
+
+if sys.version_info[0] > 2:
+    import urllib.parse as urlparse_
+else:
+    import urlparse as urlparse_
+
+from OpenSSL import SSL
+
+from ndg.httpsclient.ssl_peer_verification import ServerSSLCertVerification
+
+
+class SSlContextConfig(object):
+    """
+    Holds configuration options for creating a SSL context. This is used as a
+    template to create the contexts with specific verification callbacks.
+    """
+    def __init__(self, key_file=None, cert_file=None, pem_file=None, ca_dir=None,
+                 verify_peer=False):
+        self.key_file = key_file
+        self.cert_file = cert_file
+        self.pem_file = pem_file
+        self.ca_dir = ca_dir
+        self.verify_peer = verify_peer
+
+
+def make_ssl_context_from_config(ssl_config=False, url=None):
+    return make_ssl_context(ssl_config.key_file, ssl_config.cert_file,
+                            ssl_config.pem_file, ssl_config.ca_dir,
+                            ssl_config.verify_peer, url)
+
+
+def make_ssl_context(key_file=None, cert_file=None, pem_file=None, ca_dir=None,
+                     verify_peer=False, url=None, method=SSL.TLSv1_METHOD,
+                     key_file_passphrase=None):
+    """
+    Creates SSL context containing certificate and key file locations.
+    """
+    ssl_context = SSL.Context(method)
+    
+    # Key file defaults to certificate file if present.
+    if cert_file:
+        ssl_context.use_certificate_file(cert_file)
+        
+    if key_file_passphrase:
+        passwd_cb = lambda max_passphrase_len, set_prompt, userdata: \
+                           key_file_passphrase 
+        ssl_context.set_passwd_cb(passwd_cb)
+        
+    if key_file:
+        ssl_context.use_privatekey_file(key_file)
+    elif cert_file:
+        ssl_context.use_privatekey_file(cert_file)
+
+    if pem_file or ca_dir:
+        ssl_context.load_verify_locations(pem_file, ca_dir)
+
+    def _callback(conn, x509, errnum, errdepth, preverify_ok):
+        """Default certification verification callback.
+        Performs no checks and returns the status passed in.
+        """
+        return preverify_ok
+    
+    verify_callback = _callback
+
+    if verify_peer:
+        ssl_context.set_verify_depth(9)
+        if url:
+            set_peer_verification_for_url_hostname(ssl_context, url)
+        else:
+            ssl_context.set_verify(SSL.VERIFY_PEER, verify_callback)
+    else:
+        ssl_context.set_verify(SSL.VERIFY_NONE, verify_callback)
+        
+    return ssl_context
+
+
+def set_peer_verification_for_url_hostname(ssl_context, url, 
+                                           if_verify_enabled=False):
+    '''Convenience routine to set peer verification callback based on
+    ServerSSLCertVerification class'''
+    if not if_verify_enabled or (ssl_context.get_verify_mode() & SSL.VERIFY_PEER):
+        urlObj = urlparse_.urlparse(url)
+        hostname = urlObj.hostname
+        server_ssl_cert_verif = ServerSSLCertVerification(hostname=hostname)
+        verify_callback_ = server_ssl_cert_verif.get_verify_server_cert_func()
+        ssl_context.set_verify(SSL.VERIFY_PEER, verify_callback_)
+

libs/ndg/httpsclient/ssl_peer_verification.py

@@ -0,0 +1,237 @@
+"""ndg_httpsclient - module containing SSL peer verification class.
+"""
+__author__ = "P J Kershaw (STFC)"
+__date__ = "09/12/11"
+__copyright__ = "(C) 2012 Science and Technology Facilities Council"
+__license__ = "BSD - see LICENSE file in top-level directory"
+__contact__ = "Philip.Kershaw@stfc.ac.uk"
+__revision__ = '$Id$'
+import re
+import logging
+log = logging.getLogger(__name__)
+
+try:
+    from ndg.httpsclient.subj_alt_name import SubjectAltName
+    from pyasn1.codec.der import decoder as der_decoder
+    SUBJ_ALT_NAME_SUPPORT = True
+    
+except ImportError as e:
+    SUBJ_ALT_NAME_SUPPORT = False
+    SUBJ_ALT_NAME_SUPPORT_MSG = (
+        'SubjectAltName support is disabled - check pyasn1 package '
+        'installation to enable'
+    )
+    import warnings
+    warnings.warn(SUBJ_ALT_NAME_SUPPORT_MSG)
+
+
+class ServerSSLCertVerification(object):
+    """Check server identity.  If hostname doesn't match, allow match of
+    host's Distinguished Name against server DN setting"""
+    DN_LUT = {
+        'commonName':               'CN',
+        'organisationalUnitName':   'OU',
+        'organisation':             'O',
+        'countryName':              'C',
+        'emailAddress':             'EMAILADDRESS',
+        'localityName':             'L',
+        'stateOrProvinceName':      'ST',
+        'streetAddress':            'STREET',
+        'domainComponent':          'DC',
+        'userid':                   'UID'
+    }
+    SUBJ_ALT_NAME_EXT_NAME = 'subjectAltName'
+    PARSER_RE_STR = '/(%s)=' % '|'.join(list(DN_LUT.keys()) + list(DN_LUT.values()))
+    PARSER_RE = re.compile(PARSER_RE_STR)
+
+    __slots__ = ('__hostname', '__certDN', '__subj_alt_name_match')
+
+    def __init__(self, certDN=None, hostname=None, subj_alt_name_match=True):
+        """Override parent class __init__ to enable setting of certDN
+        setting
+
+        @type certDN: string
+        @param certDN: Set the expected Distinguished Name of the
+        server to avoid errors matching hostnames.  This is useful
+        where the hostname is not fully qualified
+        @type hostname: string
+        @param hostname: hostname to match against peer certificate 
+        subjectAltNames or subject common name
+        @type subj_alt_name_match: bool
+        @param subj_alt_name_match: flag to enable/disable matching of hostname
+        against peer certificate subjectAltNames.  Nb. A setting of True will 
+        be ignored if the pyasn1 package is not installed
+        """
+        self.__certDN = None
+        self.__hostname = None
+
+        if certDN is not None:
+            self.certDN = certDN
+
+        if hostname is not None:
+            self.hostname = hostname
+        
+        if subj_alt_name_match:
+            if not SUBJ_ALT_NAME_SUPPORT:
+                log.warning('Overriding "subj_alt_name_match" keyword setting: '
+                            'peer verification with subjectAltNames is disabled')
+                self.__subj_alt_name_match = False
+            else:    
+                self.__subj_alt_name_match = True
+        else:
+            log.debug('Disabling peer verification with subject '
+                      'subjectAltNames!')
+            self.__subj_alt_name_match = False
+
+    def __call__(self, connection, peerCert, errorStatus, errorDepth,
+                 preverifyOK):
+        """Verify server certificate
+
+        @type connection: OpenSSL.SSL.Connection
+        @param connection: SSL connection object
+        @type peerCert: basestring
+        @param peerCert: server host certificate as OpenSSL.crypto.X509
+        instance
+        @type errorStatus: int
+        @param errorStatus: error status passed from caller.  This is the value
+        returned by the OpenSSL C function X509_STORE_CTX_get_error().  Look-up
+        x509_vfy.h in the OpenSSL source to get the meanings of the different
+        codes.  PyOpenSSL doesn't help you!
+        @type errorDepth: int
+        @param errorDepth: a non-negative integer representing where in the
+        certificate chain the error occurred. If it is zero it occured in the
+        end entity certificate, one if it is the certificate which signed the
+        end entity certificate and so on.
+
+        @type preverifyOK: int
+        @param preverifyOK: the error status - 0 = Error, 1 = OK of the current
+        SSL context irrespective of any verification checks done here.  If this
+        function yields an OK status, it should enforce the preverifyOK value
+        so that any error set upstream overrides and is honoured.
+        @rtype: int
+        @return: status code - 0/False = Error, 1/True = OK
+        """
+        if peerCert.has_expired():
+            # Any expired certificate in the chain should result in an error
+            log.error('Certificate %r in peer certificate chain has expired',
+                      peerCert.get_subject())
+
+            return False
+
+        elif errorDepth == 0:
+            # Only interested in DN of last certificate in the chain - this must
+            # match the expected Server DN setting
+            peerCertSubj = peerCert.get_subject()
+            peerCertDN = peerCertSubj.get_components()
+            peerCertDN.sort()
+
+            if self.certDN is None:
+                # Check hostname against peer certificate CN field instead:
+                if self.hostname is None:
+                    log.error('No "hostname" or "certDN" set to check peer '
+                              'certificate against')
+                    return False
+
+                # Check for subject alternative names
+                if self.__subj_alt_name_match:
+                    dns_names = self._get_subj_alt_name(peerCert)
+                    if self.hostname in dns_names:
+                        return preverifyOK
+                
+                # If no subjectAltNames, default to check of subject Common Name   
+                if peerCertSubj.commonName == self.hostname:
+                    return preverifyOK
+                else:
+                    log.error('Peer certificate CN %r doesn\'t match the '
+                              'expected CN %r', peerCertSubj.commonName,
+                              self.hostname)
+                    return False
+            else:
+                if peerCertDN == self.certDN:
+                    return preverifyOK
+                else:
+                    log.error('Peer certificate DN %r doesn\'t match the '
+                              'expected DN %r', peerCertDN, self.certDN)
+                    return False
+        else:
+            return preverifyOK
+
+    def get_verify_server_cert_func(self):
+        def verify_server_cert(connection, peerCert, errorStatus, errorDepth,
+                preverifyOK):
+            return self.__call__(connection, peerCert, errorStatus,
+                                 errorDepth, preverifyOK)
+        
+        return verify_server_cert
+
+    @classmethod
+    def _get_subj_alt_name(cls, peer_cert):
+        '''Extract subjectAltName DNS name settings from certificate extensions
+        
+        @param peer_cert: peer certificate in SSL connection.  subjectAltName
+        settings if any will be extracted from this
+        @type peer_cert: OpenSSL.crypto.X509
+        '''
+        # Search through extensions
+        dns_name = []
+        general_names = SubjectAltName()
+        for i in range(peer_cert.get_extension_count()):
+            ext = peer_cert.get_extension(i)
+            ext_name = ext.get_short_name()
+            if ext_name == cls.SUBJ_ALT_NAME_EXT_NAME:
+                # PyOpenSSL returns extension data in ASN.1 encoded form
+                ext_dat = ext.get_data()
+                decoded_dat = der_decoder.decode(ext_dat, 
+                                                 asn1Spec=general_names)
+                
+                for name in decoded_dat:
+                    if isinstance(name, SubjectAltName):
+                        for entry in range(len(name)):
+                            component = name.getComponentByPosition(entry)
+                            dns_name.append(str(component.getComponent()))
+                            
+        return dns_name
+        
+    def _getCertDN(self):
+        return self.__certDN
+
+    def _setCertDN(self, val):
+        if isinstance(val, str):
+            # Allow for quoted DN
+            certDN = val.strip('"')
+
+            dnFields = self.__class__.PARSER_RE.split(certDN)
+            if len(dnFields) < 2:
+                raise TypeError('Error parsing DN string: "%s"' % certDN)
+
+            self.__certDN = list(zip(dnFields[1::2], dnFields[2::2]))
+            self.__certDN.sort()
+
+        elif not isinstance(val, list):
+            for i in val:
+                if not len(i) == 2:
+                    raise TypeError('Expecting list of two element DN field, '
+                                    'DN field value pairs for "certDN" '
+                                    'attribute')
+            self.__certDN = val
+        else:
+            raise TypeError('Expecting list or string type for "certDN" '
+                            'attribute')
+
+    certDN = property(fget=_getCertDN,
+                      fset=_setCertDN,
+                      doc="Distinguished Name for Server Certificate")
+
+    # Get/Set Property methods
+    def _getHostname(self):
+        return self.__hostname
+
+    def _setHostname(self, val):
+        if not isinstance(val, str):
+            raise TypeError("Expecting string type for hostname "
+                                 "attribute")
+        self.__hostname = val
+
+    hostname = property(fget=_getHostname,
+                        fset=_setHostname,
+                        doc="hostname of server")

libs/ndg/httpsclient/ssl_socket.py

@@ -0,0 +1,282 @@
+"""PyOpenSSL utilities including HTTPSSocket class which wraps PyOpenSSL
+SSL connection into a httplib-like interface suitable for use with urllib2
+
+"""
+__author__ = "P J Kershaw"
+__date__ = "21/12/10"
+__copyright__ = "(C) 2012 Science and Technology Facilities Council"
+__license__ = "BSD - see LICENSE file in top-level directory"
+__contact__ = "Philip.Kershaw@stfc.ac.uk"
+__revision__ = '$Id$'
+
+from datetime import datetime
+import logging
+import socket
+from io import BytesIO
+
+from OpenSSL import SSL
+
+log = logging.getLogger(__name__)
+
+
+class SSLSocket(object):
+    """SSL Socket class wraps pyOpenSSL's SSL.Connection class implementing
+    the makefile method so that it is compatible with the standard socket
+    interface and usable with httplib.
+
+    @cvar default_buf_size: default buffer size for recv operations in the
+    makefile method
+    @type default_buf_size: int
+    """
+    default_buf_size = 8192
+
+    def __init__(self, ctx, sock=None):
+        """Create SSL socket object
+
+        @param ctx: SSL context
+        @type ctx: OpenSSL.SSL.Context
+        @param sock: underlying socket object
+        @type sock: socket.socket
+        """
+        if sock is not None:
+            self.socket = sock
+        else:
+            self.socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+            self.socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
+
+        self.__ssl_conn = SSL.Connection(ctx, self.socket)
+        self.buf_size = self.__class__.default_buf_size
+        self._makefile_refs = 0
+
+    def __del__(self):
+        """Close underlying socket when this object goes out of scope
+        """
+        self.close()
+
+    @property
+    def buf_size(self):
+        """Buffer size for makefile method recv() operations"""
+        return self.__buf_size
+
+    @buf_size.setter
+    def buf_size(self, value):
+        """Buffer size for makefile method recv() operations"""
+        if not isinstance(value, int):
+            raise TypeError('Expecting int type for "buf_size"; '
+                            'got %r instead' % type(value))
+        self.__buf_size = value
+
+    def close(self):
+        """Shutdown the SSL connection and call the close method of the
+        underlying socket"""
+        if self._makefile_refs < 1:        
+            try:
+                self.__ssl_conn.shutdown()
+            except (SSL.Error, SSL.SysCallError):
+                # Make errors on shutdown non-fatal
+                pass
+        else:
+            self._makefile_refs -= 1
+
+    def set_shutdown(self, mode):
+        """Set the shutdown state of the Connection.
+        @param mode: bit vector of either or both of SENT_SHUTDOWN and
+        RECEIVED_SHUTDOWN
+        """
+        self.__ssl_conn.set_shutdown(mode)
+
+    def get_shutdown(self):
+        """Get the shutdown state of the Connection.
+        @return: bit vector of either or both of SENT_SHUTDOWN and
+        RECEIVED_SHUTDOWN
+        """
+        return self.__ssl_conn.get_shutdown()
+
+    def bind(self, addr):
+        """bind to the given address - calls method of the underlying socket
+        @param addr: address/port number tuple
+        @type addr: tuple"""
+        self.__ssl_conn.bind(addr)
+
+    def listen(self, backlog):
+        """Listen for connections made to the socket.
+
+        @param backlog: specifies the maximum number of queued connections and
+        should be at least 1; the maximum value is system-dependent (usually 5).
+        @param backlog: int
+        """
+        self.__ssl_conn.listen(backlog)
+
+    def set_accept_state(self):
+        """Set the connection to work in server mode. The handshake will be
+        handled automatically by read/write"""
+        self.__ssl_conn.set_accept_state()
+
+    def accept(self):
+        """Accept an SSL connection.
+
+        @return: pair (ssl, addr) where ssl is a new SSL connection object and
+        addr is the address bound to the other end of the SSL connection.
+        @rtype: tuple
+        """
+        return self.__ssl_conn.accept()
+
+    def set_connect_state(self):
+        """Set the connection to work in client mode. The handshake will be
+        handled automatically by read/write"""
+        self.__ssl_conn.set_connect_state()
+
+    def connect(self, addr):
+        """Call the connect method of the underlying socket and set up SSL on
+        the socket, using the Context object supplied to this Connection object
+        at creation.
+
+        @param addr: address/port number pair
+        @type addr: tuple
+        """
+        self.__ssl_conn.connect(addr)
+
+    def shutdown(self, how):
+        """Send the shutdown message to the Connection.
+
+        @param how: for socket.socket this flag determines whether read, write
+        or both type operations are supported.  OpenSSL.SSL.Connection doesn't
+        support this so this parameter is IGNORED
+        @return: true if the shutdown message exchange is completed and false
+        otherwise (in which case you call recv() or send() when the connection
+        becomes readable/writeable.
+        @rtype: bool
+        """
+        return self.__ssl_conn.shutdown()
+
+    def renegotiate(self):
+        """Renegotiate this connection's SSL parameters."""
+        return self.__ssl_conn.renegotiate()
+
+    def pending(self):
+        """@return: numbers of bytes that can be safely read from the SSL
+        buffer.
+        @rtype: int
+        """
+        return self.__ssl_conn.pending()
+
+    def send(self, data, *flags_arg):
+        """Send data to the socket. Nb. The optional flags argument is ignored.
+        - retained for compatibility with socket.socket interface
+
+        @param data: data to send down the socket
+        @type data: string
+        """
+        return self.__ssl_conn.send(data)
+
+    def sendall(self, data):
+        self.__ssl_conn.sendall(data)
+
+    def recv(self, size=default_buf_size):
+        """Receive data from the Connection.
+
+        @param size: The maximum amount of data to be received at once
+        @type size: int
+        @return: data received.
+        @rtype: string
+        """
+        return self.__ssl_conn.recv(size)
+
+    def setblocking(self, mode):
+        """Set this connection's underlying socket blocking _mode_.
+
+        @param mode: blocking mode
+        @type mode: int
+        """
+        self.__ssl_conn.setblocking(mode)
+
+    def fileno(self):
+        """
+        @return: file descriptor number for the underlying socket
+        @rtype: int
+        """
+        return self.__ssl_conn.fileno()
+
+    def getsockopt(self, *args):
+        """See socket.socket.getsockopt
+        """
+        return self.__ssl_conn.getsockopt(*args)
+
+    def setsockopt(self, *args):
+        """See socket.socket.setsockopt
+
+        @return: value of the given socket option
+        @rtype: int/string
+        """
+        return self.__ssl_conn.setsockopt(*args)
+
+    def state_string(self):
+        """Return the SSL state of this connection."""
+        return self.__ssl_conn.state_string()
+
+    def makefile(self, *args):
+        """Specific to Python socket API and required by httplib: convert
+        response into a file-like object.  This implementation reads using recv
+        and copies the output into a StringIO buffer to simulate a file object
+        for consumption by httplib
+
+        Nb. Ignoring optional file open mode (StringIO is generic and will
+        open for read and write unless a string is passed to the constructor)
+        and buffer size - httplib set a zero buffer size which results in recv
+        reading nothing
+
+        @return: file object for data returned from socket
+        @rtype: cStringIO.StringO
+        """
+        self._makefile_refs += 1
+        
+        # Optimisation
+        _buf_size = self.buf_size
+
+        i=0
+        stream = BytesIO()
+        startTime = datetime.utcnow()
+        try:
+            dat = self.__ssl_conn.recv(_buf_size)
+            while dat:
+                i+=1
+                stream.write(dat)
+                dat = self.__ssl_conn.recv(_buf_size)
+
+        except (SSL.ZeroReturnError, SSL.SysCallError):
+            # Connection is closed - assuming here that all is well and full
+            # response has been received.  httplib will catch an error in
+            # incomplete content since it checks the content-length header
+            # against the actual length of data received
+            pass
+
+        if log.getEffectiveLevel() <= logging.DEBUG:
+            log.debug("Socket.makefile %d recv calls completed in %s", i,
+                      datetime.utcnow() - startTime)
+
+        # Make sure to rewind the buffer otherwise consumers of the content will
+        # read from the end of the buffer
+        stream.seek(0)
+
+        return stream
+    
+    def getsockname(self):
+        """
+        @return: the socket's own address
+        @rtype:
+        """
+        return self.__ssl_conn.getsockname()
+
+    def getpeername(self):
+        """
+        @return: remote address to which the socket is connected
+        """
+        return self.__ssl_conn.getpeername()
+
+    def get_context(self):
+        '''Retrieve the Context object associated with this Connection. '''
+        return self.__ssl_conn.get_context()
+
+    def get_peer_certificate(self):
+        '''Retrieve the other side's certificate (if any)  '''
+        return self.__ssl_conn.get_peer_certificate()

libs/ndg/httpsclient/subj_alt_name.py

@@ -0,0 +1,153 @@
+"""NDG HTTPS Client package
+
+Use pyasn1 to provide support for parsing ASN.1 formatted subjectAltName
+content for SSL peer verification.  Code based on:
+
+http://stackoverflow.com/questions/5519958/how-do-i-parse-subjectaltname-extension-data-using-pyasn1
+"""
+__author__ = "P J Kershaw"
+__date__ = "01/02/12"
+__copyright__ = "(C) 2012 Science and Technology Facilities Council"
+__license__ = "BSD - see LICENSE file in top-level directory"
+__contact__ = "Philip.Kershaw@stfc.ac.uk"
+__revision__ = '$Id$'
+try:
+    from pyasn1.type import univ, constraint, char, namedtype, tag
+    
+except ImportError as e:
+    import_error_msg = ('Error importing pyasn1, subjectAltName check for SSL '
+                        'peer verification will be disabled.  Import error '
+                        'is: %s' % e)
+    import warnings
+    warnings.warn(import_error_msg)
+    class Pyasn1ImportError(ImportError):
+        "Raise for pyasn1 import error"
+    raise Pyasn1ImportError(import_error_msg)
+    
+    
+MAX = 64
+
+
+class DirectoryString(univ.Choice):
+    """ASN.1 Directory string class"""
+    componentType = namedtype.NamedTypes(
+        namedtype.NamedType(
+            'teletexString', char.TeletexString().subtype(
+                subtypeSpec=constraint.ValueSizeConstraint(1, MAX))),
+        namedtype.NamedType(
+            'printableString', char.PrintableString().subtype(
+                subtypeSpec=constraint.ValueSizeConstraint(1, MAX))),
+        namedtype.NamedType(
+            'universalString', char.UniversalString().subtype(
+                subtypeSpec=constraint.ValueSizeConstraint(1, MAX))),
+        namedtype.NamedType(
+            'utf8String', char.UTF8String().subtype(
+                subtypeSpec=constraint.ValueSizeConstraint(1, MAX))),
+        namedtype.NamedType(
+            'bmpString', char.BMPString().subtype(
+                subtypeSpec=constraint.ValueSizeConstraint(1, MAX))),
+        namedtype.NamedType(
+            'ia5String', char.IA5String().subtype(
+                subtypeSpec=constraint.ValueSizeConstraint(1, MAX))),
+        )
+
+
+class AttributeValue(DirectoryString):
+    """ASN.1 Attribute value"""
+
+
+class AttributeType(univ.ObjectIdentifier):
+    """ASN.1 Attribute type"""
+
+
+class AttributeTypeAndValue(univ.Sequence):
+    """ASN.1 Attribute type and value class"""
+    componentType = namedtype.NamedTypes(
+        namedtype.NamedType('type', AttributeType()),
+        namedtype.NamedType('value', AttributeValue()),
+        )
+
+
+class RelativeDistinguishedName(univ.SetOf):
+    '''ASN.1 Realtive distinguished name'''
+    componentType = AttributeTypeAndValue()
+
+class RDNSequence(univ.SequenceOf):
+    '''ASN.1 RDN sequence class'''
+    componentType = RelativeDistinguishedName()
+
+
+class Name(univ.Choice):
+    '''ASN.1 name class'''
+    componentType = namedtype.NamedTypes(
+        namedtype.NamedType('', RDNSequence()),
+        )
+
+
+class Extension(univ.Sequence):
+    '''ASN.1 extension class'''
+    componentType = namedtype.NamedTypes(
+        namedtype.NamedType('extnID', univ.ObjectIdentifier()),
+        namedtype.DefaultedNamedType('critical', univ.Boolean('False')),
+        namedtype.NamedType('extnValue', univ.OctetString()),
+        )
+
+
+class Extensions(univ.SequenceOf):
+    '''ASN.1 extensions class'''
+    componentType = Extension()
+    sizeSpec = univ.SequenceOf.sizeSpec + constraint.ValueSizeConstraint(1, MAX)
+
+
+class AnotherName(univ.Sequence):
+    componentType = namedtype.NamedTypes(
+        namedtype.NamedType('type-id', univ.ObjectIdentifier()),
+        namedtype.NamedType('value', univ.Any().subtype(
+                            explicitTag=tag.Tag(tag.tagClassContext,
+                                                tag.tagFormatSimple, 0)))
+        )
+
+
+class GeneralName(univ.Choice):
+    '''ASN.1 configuration for X.509 certificate subjectAltNames fields'''
+    componentType = namedtype.NamedTypes(
+        namedtype.NamedType('otherName', AnotherName().subtype(
+                            implicitTag=tag.Tag(tag.tagClassContext,
+                                                tag.tagFormatSimple, 0))),
+        namedtype.NamedType('rfc822Name', char.IA5String().subtype(
+                            implicitTag=tag.Tag(tag.tagClassContext,
+                                                tag.tagFormatSimple, 1))),
+        namedtype.NamedType('dNSName', char.IA5String().subtype(
+                            implicitTag=tag.Tag(tag.tagClassContext,
+                                                tag.tagFormatSimple, 2))),
+#        namedtype.NamedType('x400Address', ORAddress().subtype(
+#                            implicitTag=tag.Tag(tag.tagClassContext,
+#                                                tag.tagFormatSimple, 3))),
+        namedtype.NamedType('directoryName', Name().subtype(
+                            implicitTag=tag.Tag(tag.tagClassContext,
+                                                tag.tagFormatSimple, 4))),
+#        namedtype.NamedType('ediPartyName', EDIPartyName().subtype(
+#                            implicitTag=tag.Tag(tag.tagClassContext,
+#                                                tag.tagFormatSimple, 5))),
+        namedtype.NamedType('uniformResourceIdentifier', char.IA5String().subtype(
+                            implicitTag=tag.Tag(tag.tagClassContext,
+                                                tag.tagFormatSimple, 6))),
+        namedtype.NamedType('iPAddress', univ.OctetString().subtype(
+                            implicitTag=tag.Tag(tag.tagClassContext,
+                                                tag.tagFormatSimple, 7))),
+        namedtype.NamedType('registeredID', univ.ObjectIdentifier().subtype(
+                            implicitTag=tag.Tag(tag.tagClassContext,
+                                                tag.tagFormatSimple, 8))),
+        )
+
+
+class GeneralNames(univ.SequenceOf):
+    '''Sequence of names for ASN.1 subjectAltNames settings'''
+    componentType = GeneralName()
+    sizeSpec = univ.SequenceOf.sizeSpec + constraint.ValueSizeConstraint(1, MAX)
+
+
+class SubjectAltName(GeneralNames):
+    '''ASN.1 implementation for subjectAltNames support'''
+
+

libs/ndg/httpsclient/urllib2_build_opener.py

@@ -0,0 +1,78 @@
+"""urllib2 style build opener integrates with HTTPSConnection class from this
+package.
+"""
+__author__ = "P J Kershaw"
+__date__ = "21/12/10"
+__copyright__ = "(C) 2011 Science and Technology Facilities Council"
+__license__ = "BSD - see LICENSE file in top-level directory"
+__contact__ = "Philip.Kershaw@stfc.ac.uk"
+__revision__ = '$Id$'
+import logging
+import sys
+
+# Py 2 <=> 3 compatibility for class type checking
+if sys.version_info[0] > 2:
+    class_type_ = type
+    from urllib.request import (ProxyHandler, UnknownHandler, 
+                                HTTPDefaultErrorHandler, FTPHandler, 
+                                FileHandler, HTTPErrorProcessor, 
+                                HTTPHandler, OpenerDirector, 
+                                HTTPRedirectHandler)
+else:
+    import types
+    class_type_ = types.ClassType
+    
+    from urllib2 import (ProxyHandler, UnknownHandler, HTTPDefaultErrorHandler, 
+                         FTPHandler, FileHandler, HTTPErrorProcessor, 
+                         HTTPHandler, OpenerDirector, HTTPRedirectHandler)
+
+from ndg.httpsclient.https import HTTPSContextHandler
+
+log = logging.getLogger(__name__)
+
+
+# Copied from urllib2 with modifications for ssl
+def build_opener(*handlers, **kw):
+    """Create an opener object from a list of handlers.
+
+    The opener will use several default handlers, including support
+    for HTTP and FTP.
+
+    If any of the handlers passed as arguments are subclasses of the
+    default handlers, the default handlers will not be used.
+    """
+    def isclass(obj):
+        return isinstance(obj, class_type_) or hasattr(obj, "__bases__")
+
+    opener = OpenerDirector()
+    default_classes = [ProxyHandler, UnknownHandler, HTTPHandler,
+                       HTTPDefaultErrorHandler, HTTPRedirectHandler,
+                       FTPHandler, FileHandler, HTTPErrorProcessor]
+    check_classes = list(default_classes)
+    check_classes.append(HTTPSContextHandler)
+    skip = []
+    for klass in check_classes:
+        for check in handlers:
+            if isclass(check):
+                if issubclass(check, klass):
+                    skip.append(klass)
+            elif isinstance(check, klass):
+                skip.append(klass)
+
+    for klass in default_classes:
+        if klass not in skip:
+            opener.add_handler(klass())
+            
+    # Pick up SSL context from keyword settings
+    ssl_context = kw.get('ssl_context')
+    
+    # Add the HTTPS handler with ssl_context
+    if HTTPSContextHandler not in skip:
+        opener.add_handler(HTTPSContextHandler(ssl_context))
+
+    for h in handlers:
+        if isclass(h):
+            h = h()
+        opener.add_handler(h)
+
+    return opener

libs/ndg/httpsclient/utils.py

@@ -0,0 +1,414 @@
+"""Utilities using NDG HTTPS Client, including a main module that can be used to
+fetch from a URL.
+"""
+__author__ = "R B Wilkinson"
+__date__ = "09/12/11"
+__copyright__ = "(C) 2011 Science and Technology Facilities Council"
+__license__ = "BSD - see LICENSE file in top-level directory"
+__contact__ = "Philip.Kershaw@stfc.ac.uk"
+__revision__ = '$Id$'
+
+import logging
+from optparse import OptionParser
+import os
+import sys
+
+if sys.version_info[0] > 2:
+    import http.cookiejar as cookiejar_
+    import http.client as http_client_
+    from urllib.request import Request as Request_
+    from urllib.request import HTTPHandler as HTTPHandler_
+    from urllib.request import HTTPCookieProcessor as HTTPCookieProcessor_
+    from urllib.request import HTTPBasicAuthHandler as HTTPBasicAuthHandler_
+    from urllib.request import HTTPPasswordMgrWithDefaultRealm as \
+                                            HTTPPasswordMgrWithDefaultRealm_
+    from urllib.request import ProxyHandler as ProxyHandler_
+    from urllib.error import HTTPError as HTTPError_
+    import urllib.parse as urlparse_
+else:
+    import cookielib as cookiejar_
+    import httplib as http_client_
+    from urllib2 import Request as Request_
+    from urllib2 import HTTPHandler as HTTPHandler_
+    from urllib2 import HTTPCookieProcessor as HTTPCookieProcessor_
+    from urllib2 import HTTPBasicAuthHandler as HTTPBasicAuthHandler_
+    from urllib2 import HTTPPasswordMgrWithDefaultRealm as \
+                                            HTTPPasswordMgrWithDefaultRealm_
+    from urllib2 import ProxyHandler as ProxyHandler_
+    from urllib2 import HTTPError as HTTPError_
+    import urlparse as urlparse_
+
+from ndg.httpsclient.urllib2_build_opener import build_opener
+from ndg.httpsclient.https import HTTPSContextHandler
+from ndg.httpsclient import ssl_context_util
+
+log = logging.getLogger(__name__)
+
+class AccumulatingHTTPCookieProcessor(HTTPCookieProcessor_):
+    """Cookie processor that adds new cookies (instead of replacing the existing
+    ones as HTTPCookieProcessor does)
+    """
+    def http_request(self, request):
+        """Processes cookies for a HTTP request.
+        @param request: request to process
+        @type request: urllib2.Request
+        @return: request
+        @rtype: urllib2.Request
+        """
+        COOKIE_HEADER_NAME = "Cookie"
+        tmp_request = Request_(request.get_full_url(), request.data, {},
+                                      request.origin_req_host,
+                                      request.unverifiable)
+        self.cookiejar.add_cookie_header(tmp_request)
+        # Combine existing and new cookies.
+        new_cookies = tmp_request.get_header(COOKIE_HEADER_NAME)
+        if new_cookies:
+            if request.has_header(COOKIE_HEADER_NAME):
+                # Merge new cookies with existing ones.
+                old_cookies = request.get_header(COOKIE_HEADER_NAME)
+                merged_cookies = '; '.join([old_cookies, new_cookies])
+                request.add_unredirected_header(COOKIE_HEADER_NAME,
+                                                merged_cookies)
+            else:
+                # No existing cookies so just set new ones.
+                request.add_unredirected_header(COOKIE_HEADER_NAME, new_cookies)
+        return request
+
+    # Process cookies for HTTPS in the same way.
+    https_request = http_request
+
+
+class URLFetchError(Exception):
+    """Error fetching content from URL"""
+    
+
+def fetch_from_url(url, config, data=None, handlers=None):
+    """Returns data retrieved from a URL.
+    @param url: URL to attempt to open
+    @type url: basestring
+    @param config: SSL context configuration
+    @type config: Configuration
+    @return data retrieved from URL or None
+    """
+    return_code, return_message, response = open_url(url, config, data=data,
+                                                     handlers=handlers)
+    if return_code and return_code == http_client_.OK:
+        return_data = response.read()
+        response.close()
+        return return_data
+    else:
+        raise URLFetchError(return_message)
+
+def fetch_from_url_to_file(url, config, output_file, data=None, handlers=None):
+    """Writes data retrieved from a URL to a file.
+    @param url: URL to attempt to open
+    @type url: basestring
+    @param config: SSL context configuration
+    @type config: Configuration
+    @param output_file: output file
+    @type output_file: basestring
+    @return: tuple (
+        returned HTTP status code or 0 if an error occurred
+        returned message
+        boolean indicating whether access was successful)
+    """
+    return_code, return_message, response = open_url(url, config, data=data,
+                                                     handlers=handlers)
+    if return_code == http_client_.OK:
+        return_data = response.read()
+        response.close()
+        outfile = open(output_file, "w")
+        outfile.write(return_data)
+        outfile.close()
+        
+    return return_code, return_message, return_code == http_client_.OK
+
+
+def fetch_stream_from_url(url, config, data=None, handlers=None):
+    """Returns data retrieved from a URL.
+    @param url: URL to attempt to open
+    @type url: basestring
+    @param config: SSL context configuration
+    @type config: Configuration
+    @param data: HTTP POST data
+    @type data: str
+    @param handlers: list of custom urllib2 handlers to add to the request
+    @type handlers: iterable
+    @return: data retrieved from URL or None
+    @rtype: file derived type
+    """
+    return_code, return_message, response = open_url(url, config, data=data,
+                                                     handlers=handlers)
+    if return_code and return_code == http_client_.OK:
+        return response
+    else:
+        raise URLFetchError(return_message)
+
+
+def open_url(url, config, data=None, handlers=None):
+    """Attempts to open a connection to a specified URL.
+    @param url: URL to attempt to open
+    @param config: SSL context configuration
+    @type config: Configuration
+    @param data: HTTP POST data
+    @type data: str
+    @param handlers: list of custom urllib2 handlers to add to the request
+    @type handlers: iterable
+    @return: tuple (
+        returned HTTP status code or 0 if an error occurred
+        returned message or error description
+        response object)
+    """
+    debuglevel = 1 if config.debug else 0
+
+    # Set up handlers for URL opener.
+    if config.cookie:
+        cj = config.cookie
+    else:
+        cj = cookiejar_.CookieJar()
+        
+    # Use a cookie processor that accumulates cookies when redirects occur so
+    # that an application can redirect for authentication and retain both any
+    # cookies for the application and the security system (c.f.,
+    # urllib2.HTTPCookieProcessor which replaces cookies).
+    cookie_handler = AccumulatingHTTPCookieProcessor(cj)
+
+    if not handlers:
+        handlers = []
+        
+    handlers.append(cookie_handler)
+
+    if config.debug:
+        http_handler = HTTPHandler_(debuglevel=debuglevel)
+        https_handler = HTTPSContextHandler(config.ssl_context, 
+                                            debuglevel=debuglevel)
+        handlers.extend([http_handler, https_handler])
+        
+    if config.http_basicauth:
+        # currently only supports http basic auth
+        auth_handler = HTTPBasicAuthHandler_(HTTPPasswordMgrWithDefaultRealm_())
+        auth_handler.add_password(realm=None, uri=url,
+                                  user=config.httpauth[0],
+                                  passwd=config.httpauth[1])
+        handlers.append(auth_handler)
+
+
+    # Explicitly remove proxy handling if the host is one listed in the value of
+    # the no_proxy environment variable because urllib2 does use proxy settings 
+    # set via http_proxy and https_proxy, but does not take the no_proxy value 
+    # into account.
+    if not _should_use_proxy(url, config.no_proxy):
+        handlers.append(ProxyHandler_({}))
+        log.debug("Not using proxy")
+    elif config.proxies:
+        handlers.append(ProxyHandler_(config.proxies))
+        log.debug("Configuring proxies: %s" % config.proxies)
+
+    opener = build_opener(*handlers, ssl_context=config.ssl_context)
+    
+    headers = config.headers
+    if headers is None: 
+        headers = {}
+        
+    request = Request_(url, data, headers)
+
+    # Open the URL and check the response.
+    return_code = 0
+    return_message = ''
+    response = None
+    
+    # FIXME
+    response = opener.open(request)
+
+    try:
+        response = opener.open(request)
+        return_message = response.msg
+        return_code = response.code
+        if log.isEnabledFor(logging.DEBUG):
+            for index, cookie in enumerate(cj):
+                log.debug("%s  :  %s", index, cookie)
+                
+    except HTTPError_ as exc:
+        return_code = exc.code
+        return_message = "Error: %s" % exc.msg
+        if log.isEnabledFor(logging.DEBUG):
+            log.debug("%s %s", exc.code, exc.msg)
+            
+    except Exception as exc:
+        return_message = "Error: %s" % exc.__str__()
+        if log.isEnabledFor(logging.DEBUG):
+            import traceback
+            log.debug(traceback.format_exc())
+            
+    return (return_code, return_message, response)
+
+
+def _should_use_proxy(url, no_proxy=None):
+    """Determines whether a proxy should be used to open a connection to the 
+    specified URL, based on the value of the no_proxy environment variable.
+    @param url: URL
+    @type url: basestring or urllib2.Request
+    """
+    if no_proxy is None:
+        no_proxy_effective = os.environ.get('no_proxy', '')
+    else:
+        no_proxy_effective = no_proxy
+
+    urlObj = urlparse_.urlparse(_url_as_string(url))
+    for np in [h.strip() for h in no_proxy_effective.split(',')]:
+        if urlObj.hostname == np:
+            return False
+
+    return True
+
+def _url_as_string(url):
+    """Returns the URL string from a URL value that is either a string or
+    urllib2.Request..
+    @param url: URL
+    @type url: basestring or urllib2.Request
+    @return: URL string
+    @rtype: basestring
+    """
+    if isinstance(url, Request_):
+        return url.get_full_url()
+    elif isinstance(url, str):
+        return url
+    else:
+        raise TypeError("Expected type %r or %r" %
+                        (str, Request_))
+
+
+class Configuration(object):
+    """Connection configuration.
+    """
+    def __init__(self, ssl_context, debug=False, proxies=None, no_proxy=None,
+                 cookie=None, http_basicauth=None, headers=None):
+        """
+        @param ssl_context: SSL context to use with this configuration
+        @type ssl_context: OpenSSL.SSL.Context
+        @param debug: if True, output debugging information
+        @type debug: bool
+        @param proxies: proxies to use for 
+        @type proxies: dict with basestring keys and values
+        @param no_proxy: hosts for which a proxy should not be used
+        @type no_proxy: basestring
+        @param cookie: cookies to set for request
+        @type cookie: cookielib.CookieJar (python 3 - http.cookiejar)
+        @param http_basicauth: http authentication, or None
+        @type http_basicauth: tuple of (username,password)
+        @param headers: http headers
+        @type headers: dict
+        """
+        self.ssl_context = ssl_context
+        self.debug = debug
+        self.proxies = proxies
+        self.no_proxy = no_proxy
+        self.cookie = cookie
+        self.http_basicauth = http_basicauth
+        self.headers = headers
+
+
+def main():
+    '''Utility to fetch data using HTTP or HTTPS GET from a specified URL.
+    '''
+    parser = OptionParser(usage="%prog [options] url")
+    parser.add_option("-c", "--certificate", dest="cert_file", metavar="FILE",
+                      default=os.path.expanduser("~/credentials.pem"),
+                      help="Certificate file - defaults to $HOME/credentials.pem")
+    parser.add_option("-k", "--private-key", dest="key_file", metavar="FILE",
+                      default=None,
+                      help="Private key file - defaults to the certificate file")
+    parser.add_option("-t", "--ca-certificate-dir", dest="ca_dir", 
+                      metavar="PATH",
+                      default=None,
+                      help="Trusted CA certificate file directory")
+    parser.add_option("-d", "--debug", action="store_true", dest="debug", 
+                      default=False,
+                      help="Print debug information.")
+    parser.add_option("-p", "--post-data-file", dest="data_file",
+                      metavar="FILE", default=None,
+                      help="POST data file")
+    parser.add_option("-f", "--fetch", dest="output_file", metavar="FILE",
+                      default=None, help="Output file")
+    parser.add_option("-n", "--no-verify-peer", action="store_true", 
+                      dest="no_verify_peer", default=False,
+                      help="Skip verification of peer certificate.")
+    parser.add_option("-a", "--basicauth", dest="basicauth", 
+                      metavar="USER:PASSWD",
+                      default=None,
+                      help="HTTP authentication credentials")
+    parser.add_option("--header", action="append", dest="headers", 
+                      metavar="HEADER: VALUE",
+                      help="Add HTTP header to request")
+    (options, args) = parser.parse_args()
+    if len(args) != 1:
+        parser.error("Incorrect number of arguments")
+
+    url = args[0]
+
+    if options.debug:
+        logging.getLogger().setLevel(logging.DEBUG)
+
+    if options.key_file and os.path.exists(options.key_file):
+        key_file = options.key_file
+    else:
+        key_file = None
+    
+    if options.cert_file and os.path.exists(options.cert_file):
+        cert_file = options.cert_file
+    else:
+        cert_file = None
+    
+    if options.ca_dir and os.path.exists(options.ca_dir):
+        ca_dir = options.ca_dir 
+    else:
+        ca_dir = None
+        
+    verify_peer = not options.no_verify_peer
+
+    if options.data_file and os.path.exists(options.data_file):
+        data_file = open(options.data_file)
+        data = data_file.read()
+        data_file.close()
+    else:
+        data = None
+    
+    if options.basicauth:
+        http_basicauth = options.basicauth.split(':', 1)
+    else:
+        http_basicauth = None
+
+    headers = {}
+    if options.headers:
+        for h in options.headers:
+            key, val = h.split(':', 1)
+            headers[key.strip()] = val.lstrip()
+            
+    # If a private key file is not specified, the key is assumed to be stored in 
+    # the certificate file.
+    ssl_context = ssl_context_util.make_ssl_context(key_file,
+                                                    cert_file,
+                                                    None,
+                                                    ca_dir,
+                                                    verify_peer, 
+                                                    url)
+
+    config = Configuration(ssl_context, 
+                           options.debug,
+                           http_basicauth=http_basicauth,
+                           headers=headers)
+    if options.output_file:
+        return_code, return_message = fetch_from_url_to_file(
+                                                      url, 
+                                                      config,
+                                                      options.output_file,
+                                                      data)[:2]
+        raise SystemExit(return_code, return_message)
+    else:
+        data = fetch_from_url(url, config)
+        print(data)
+
+
+if __name__=='__main__':
+    logging.basicConfig()
+    main()