def _request (self, method, url, *a, **kw) :
        if not url or (not url.startswith("http://") and not url.startswith("https://")) :
            if not self._base_url :
                return defer.maybeDeferred(lambda x : Response(url, ), )

            url = urllib.basejoin(self._base_url, (url and url or ""), )

        _scheme, _host, _port, _path = client_http._parse(url, )

        kw["method"] = method
        _factory = self._client_factory(url, *a, **kw)

        if _scheme == "https" :
            from twisted.internet import ssl
            #_contextFactory = kw.get("contextFactory")
            #if _contextFactory is None :
            #    _contextFactory = ssl.ClientContextFactory()
            _contextFactory = ssl.ClientContextFactory()

            reactor.connectSSL(_host, _port, _factory, _contextFactory, )
        else:
            reactor.connectTCP(_host, _port, _factory)

        return _factory.deferred.addCallback(
            self._cb_request, _factory, url,
        ).addCallback(
            self._cb_request_debug,
        )

 def handleStatus_301(self):
     l = self.headers.get('location')
     if not l:
         self.handleStatusDefault()
         return
     url = l[0]
     if self.followRedirect:
         scheme, host, port, path = \
             _parse(url, defaultPort=self.transport.getPeer().port)
         self.factory.setURL(url)
 
         if self.factory.scheme == 'https':
             from twisted.internet import ssl
             contextFactory = ssl.ClientContextFactory()
             reactor.connectSSL(self.factory.host, self.factory.port, 
                                self.factory, contextFactory)
         else:
             reactor.connectTCP(self.factory.host, self.factory.port, 
                                self.factory)
     else:
         self.handleStatusDefault()
         self.factory.noPage(
             failure.Failure(
                 error.PageRedirect(
                     self.status, self.message, location = url)))
     self.quietLoss = 1
     self.transport.loseConnection()

 def write(self, notifications):
   "Connect to the APNS service and send notifications"
   if not self.factory:
     log.msg('APNSService write (connecting)')
     server, port = ((APNS_SERVER_SANDBOX_HOSTNAME 
                     if self.environment == 'sandbox'
                     else APNS_SERVER_HOSTNAME), APNS_SERVER_PORT)
     self.factory = self.clientProtocolFactory()
     context = self.getContextFactory()
     reactor.connectSSL(server, port, self.factory, context)
   
   client = self.factory.clientProtocol
   if client:
     return client.sendMessage(notifications)
   else:      
     d = self.factory.deferred
     timeout = reactor.callLater(self.timeout, 
       lambda: d.called or d.errback(
         Exception('Notification timed out after %i seconds' % self.timeout)))
     def cancel_timeout(r):
       try: timeout.cancel()
       except: pass
       return r
     
     d.addCallback(lambda p: p.sendMessage(notifications))
     d.addErrback(log_errback('apns-service-write'))
     d.addBoth(cancel_timeout)
     return d

    def connect(self):
        """connect to the jabber server"""
        self.dprint('Starting to connect')
        self.dprint('Building context factory for jid %s' % self.jidString(self.userId))
        jidStr = self.jidString(self.userId)
        self._fact = factory = client.basicClientFactory(jid.JID(jidStr), self.userPassword)
        factory.addBootstrap('//event/stream/authd', self.authenticate)
        factory.addBootstrap(client.BasicAuthenticator.INVALID_USER_EVENT, self.invalidUser)
        factory.addBootstrap(client.BasicAuthenticator.AUTH_FAILED_EVENT, self.fatalError)
        factory.addBootstrap(client.BasicAuthenticator.REGISTER_FAILED_EVENT, self.fatalError)

        self.dprint('connecting to server %s using id %s...' % (self.server, self.userId))
        if self.wants_ssl:
            class contextFactory:
                isClient = 1
                method = ssl.SSL.SSLv3_METHOD

                def getContext(self):
                    context = ssl.SSL.Context(self.method)
                    return context

            self.dprint('Connecting with ssl...')
            ctxFactory = contextFactory()
            reactor.connectSSL(self.host, self.port, factory, ctxFactory)
        else:
            reactor.connectTCP(self.host, self.port, factory)
        return reactor

def _makeGetterFactory(url, factoryFactory, contextFactory=None, *args, **kwargs):
    """
    Create and connect an HTTP page getting factory.

    Any additional positional or keyword arguments are used when calling
    C{factoryFactory}.

    @param factoryFactory: Factory factory that is called with C{url}, C{args}
        and C{kwargs} to produce the getter

    @param contextFactory: Context factory to use when creating a secure
        connection, defaulting to C{None}

    @return: The factory created by C{factoryFactory}
    """
    scheme, host, port, path = _parse(url)
    factory = factoryFactory(url, *args, **kwargs)
    if scheme == "https":
        from twisted.internet import ssl

        if contextFactory is None:
            contextFactory = ssl.ClientContextFactory()
        reactor.connectSSL(host, port, factory, contextFactory)
    else:
        reactor.connectTCP(host, port, factory)
    return factory

    def get_page(self, contextFactory=None, description=None, *args, **kwargs):
        if description is None:
            description = self.url

        log.debug("Requesting %s", description)

        scheme, _, _, _ = twisted_web_client_parse(self.url)
        factory = txwebclient.HTTPClientFactory(self.url, *args, **kwargs)
        if scheme == 'https':
            from twisted.internet import ssl
            if contextFactory is None:
                contextFactory = ssl.ClientContextFactory()
            if self.use_proxy:
                ip = yield self.resolve(self.proxy_host)
                reactor.connectSSL(ip, self.proxy_port,
                                   factory, contextFactory)
            else:
                ip = yield self.resolve(self.host)
                reactor.connectSSL(ip, self.port,
                                   factory, contextFactory)
        else:
            if self.use_proxy:
                ip = yield self.resolve(self.proxy_host)
                reactor.connectTCP(ip, self.proxy_port, factory)
            else:
                ip = yield self.resolve(self.host)
                reactor.connectTCP(ip, self.port, factory)

        result = yield factory.deferred
        defer.returnValue(result)

	def __init__(self, url, contextFactory=None, retries=0):

		url = stripNoPrint(url)
		if retries > 0:
			print "Retrying: ", url
		else:
			print "Get: ", url
		self.retries = retries
		self.url = url
		self.charset = None
		scheme, host, port, path = _parse(url)
		HTTPClientFactory.__init__(self, url,
			method='GET', postdata=None, headers=None,
			agent='Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US;' + 
				' rv:1.9.2.10) Gecko/20100914 Firefox/3.6.10')
		if scheme == 'https':
			from twisted.internet import ssl
			if contextFactory is None:
				contextFactory = ssl.ClientContextFactory()
			reactor.connectSSL(host, port, self, contextFactory)
		else:
			reactor.connectTCP(host, port, self)
		
		self.deferred.addCallbacks(self.getCharset, self.Err)
		self.deferred.addCallbacks(self.getTitle, self.Err)

 def __init__(self, version):
  self.version = version
  self.stopped = False
  self.tc = 0
  self.th = {}
  self.sjid = u'%[email protected]%s/%s' % (config.USER, config.SERVER, config.RESOURCE)
  self.jid = jid.JID(self.sjid)
  self.onauthd = None
  self.a = XMPPAuthenticator(self.jid, config.PASSWD)
  self.c = ClientFactory(self.a, self)
  self.c.maxRetries = 0
  self.c.addBootstrap(xmlstream.STREAM_AUTHD_EVENT, self.authd)
  self.c.addBootstrap(xmlstream.INIT_FAILED_EVENT, self.initfailed)
  self.c.addBootstrap(xmlstream.STREAM_CONNECTED_EVENT, self.onConnected)
  self.c.addBootstrap(xmlstream.STREAM_END_EVENT, self.onDisconnected)
  self.x = None
  self.log = log.logger()
  self.msghandlers = []
  port = config.PORT
  if config.CONNECT_SERVER: server = config.CONNECT_SERVER
  else: server = config.SERVER
  if config.USE_SSL:
   from twisted.internet import ssl
   reactor.connectSSL(server, port, self.c, ssl.ClientContextFactory())
  else: reactor.connectTCP(server, port, self.c)

 def connectSSL(self, server, port, vhost):
     """Connect to the server using an SSL socket."""
     drivers.log.info("Attempting an SSL connection.")
     if SSL:
         reactor.connectSSL(server, port, self, ssl.ClientContextFactory(), bindAddress=(vhost, 0))
     else:
         drivers.log.error("PyOpenSSL is not available. Not connecting.")

    def do_connect():
        """Connect and authenticate."""
        result_deferred = Deferred()
        context_factory = None
        if use_tls:
            from twisted.internet import ssl as twisted_ssl
            context_factory = twisted_ssl.ClientContextFactory()

        body = MIMEText(message)
        body['Subject'] = subject
        factory = ESMTPSenderFactory(
            username,
            password,
            from_address,
            to_address,
            StringIO(body.as_string()),
            result_deferred,
            contextFactory=context_factory,
            requireTransportSecurity=use_tls,
            requireAuthentication=True,
            heloFallback=helo_fallback)

        if use_tls:
            reactor.connectSSL(host, port, factory, context_factory)
        else:
            reactor.connectTCP(host, port, factory)
        result = yield result_deferred

        if result[0] == 0:
            raise RuntimeError("failed to send email via smtp")

 def connect_to_server(self):
     """
     Connect to media server
     """
     reactor.connectSSL(sys.argv[1], int(sys.argv[2]),
                        TheaterFactory(self), ssl.ClientContextFactory())
     reactor.run()

    def testFailedVerify(self):
        org = "twisted.test.test_ssl"
        self.setupServerAndClient(
            (org, org + ", client"), {},
            (org, org + ", server"), {})

        def verify(*a):
            return False
        self.clientCtxFactory.getContext().set_verify(SSL.VERIFY_PEER, verify)

        serverConnLost = defer.Deferred()
        serverProtocol = protocol.Protocol()
        serverProtocol.connectionLost = serverConnLost.callback
        serverProtocolFactory = protocol.ServerFactory()
        serverProtocolFactory.protocol = lambda: serverProtocol
        self.serverPort = serverPort = reactor.listenSSL(0,
            serverProtocolFactory, self.serverCtxFactory)

        clientConnLost = defer.Deferred()
        clientProtocol = protocol.Protocol()
        clientProtocol.connectionLost = clientConnLost.callback
        clientProtocolFactory = protocol.ClientFactory()
        clientProtocolFactory.protocol = lambda: clientProtocol
        reactor.connectSSL('127.0.0.1',
            serverPort.getHost().port, clientProtocolFactory, self.clientCtxFactory)

        dl = defer.DeferredList([serverConnLost, clientConnLost], consumeErrors=True)
        return dl.addCallback(self._cbLostConns)

    def run(self):
        """ main loop - pretty simple """
        while True:
            try:
                clientFactory = self.parent.request_queue.get(True,1)
                if type(clientFactory) == type("Quit"): return
            except Queue.Empty:
                if self.finished: return
                continue

            self.parent.ppc.getToken()
            if self.parent.ssl:

                with open(self.parent.keyfile) as keyFile:
                    with open(self.parent.crtfile) as certFile:
                        clientCert = ssl.PrivateCertificate.loadPEM(
                        keyFile.read() + certFile.read())

                ctx = clientCert.options()
                #contextFactory = ssl.ClientContextFactory()
                reactor.connectSSL(self.parent.host, self.parent.port, clientFactory, ctx)
            else:
                reactor.connectTCP(self.parent.host, self.parent.port, clientFactory)

            time_taken = time()-clientFactory.Qtime
            stats = self.parent.statistics
            stats.avg_time_taken = (
                stats.total_serviced * stats.avg_time_taken + time_taken) \
                / (stats.total_serviced+1)
            stats.total_serviced += 1

def historical_pull(start_date, end_date, account, api_key, topic_id, filename, time_zone='edt'):

    
    url = 'wss://pipeline.attensity.com/account/' + account + '/feed?api_key=' + api_key + '&topic_id[]=' + topic_id

    # need to generalize and make accessible as a tool
    if time_zone.lower() == 'edt':        
        time_shift = datetime.timedelta(hours=4)
    else:
        # I think, should double check
        time_shift = datetime.timedelta(hours=5)

    start_t = datetime.datetime(start_date['year'], start_date['month'], start_date['day'], start_date['hour'], start_date['minute']) + time_shift    
    start_timestamp = (start_t - datetime.datetime(1970, 1, 1)).total_seconds()

    end_t = datetime.datetime(end_date['year'], end_date['month'], end_date['day'], end_date['hour'], end_date['minute']) + time_shift
    end_timestamp = (end_t - datetime.datetime(1970, 1, 1)).total_seconds()

    if start_timestamp > end_timestamp:
        print 'Start Time is after End Time.'
        print 'Exiting...'
        sys.exit(1)

    url += '&starttime=' + str(int(start_timestamp)) + '&endtime=' + str(int(end_timestamp)) + '&stream_mode=historical'

    print url

    f = open(filename, 'w')

    factory = ws.AttensityFactory(url, file_handle=f)
    reactor.connectSSL(factory.host, factory.port, factory, ssl.ClientContextFactory())

    def getProxyPage(url, contextFactory=None, host=None, port=None,
                     status=None, *args, **kwargs):
        """Download a web page as a string. (modified from twisted.web.client.getPage)

        Download a page. Return a deferred, which will callback with a
        page (as a string) or errback with a description of the error.

        See HTTPClientFactory to see what extra args can be passed.
        """
        if status > 300 and status < 304:
            factory = RedirHTTPClientFactory(url, *args, **kwargs)
        else:
            factory = client.HTTPClientFactory(url, *args, **kwargs)

        host = host or factory.host
        port = port or factory.port

        if factory.scheme == 'https':
            from twisted.internet import ssl
            if contextFactory is None:
                contextFactory = ssl.ClientContextFactory()
            reactor.connectSSL(host, port, factory, contextFactory)
        else:
            reactor.connectTCP(host, port, factory)
        return factory.deferred

 def execute(self):
     if self.scheme == "https":
         from twisted.internet import ssl
         reactor.connectSSL(self.host, self.port, self, ssl.ClientContextFactory())
     else:
         reactor.connectTCP(self.host, self.port, self)
     return self.deferred

 def testFactoryInfo(self):
     url = self.getURL("file")
     uri = client.URI.fromBytes(url)
     factory = client.HTTPClientFactory(url)
     reactor.connectSSL(nativeString(uri.host), uri.port, factory, ssl.ClientContextFactory())
     # The base class defines _cbFactoryInfo correctly for this
     return factory.deferred.addCallback(self._cbFactoryInfo, factory)

def httpRequest(url, payload, headers, method='POST', timeout=DEFAULT_TIMEOUT, ctx_factory=None):
    # copied from twisted.web.client in order to get access to the
    # factory (which contains response codes, headers, etc)

    if type(url) is not str:
        e = HTTPRequestError('URL must be string, not %s' % type(url))
        return defer.fail(e)

    if not url.startswith('http'):
        e = HTTPRequestError('URL does not start with http (URL %s)' % (url))
        return defer.fail(e)

    log.msg(" -- Sending Payload to %s --\n%s\n -- END. Sending Payload --" % (url, payload), system=LOG_SYSTEM, payload=True)

    scheme, netloc, _ , _, _, _ = twhttp.urlparse(url)
    if not ':' in netloc:
        host = netloc
        port = 80 if scheme == 'http' else 443
    else:
        host, s_port = netloc.split(':',1)
        port = int(s_port)

    factory = twclient.HTTPClientFactory(url, method, postdata=payload, timeout=timeout)
    factory.noisy = False # stop spewing about factory start/stop
    factory.protocol.handleStatus_204 = lambda _ : None # 204 is an ok reply, needed by NCS VPN backend

    # fix missing port in header (bug in twisted.web.client)
    factory.headers['host'] = host + ':' + str(port)
    factory.headers['User-Agent'] = 'OpenNSA/Twisted'

    for header, value in headers.items():
        factory.headers[header] = value

    if scheme == 'https':
        if ctx_factory is None:
            return defer.fail(HTTPRequestError('Cannot perform https request without context factory'))
        reactor.connectSSL(host, port, factory, ctx_factory)
    else:
        reactor.connectTCP(host, port, factory)

    def invocationError(err):
        if isinstance(err.value, ConnectionClosed): # note: this also includes ConnectionDone and ConnectionLost
            pass # these are pretty common when the remote shuts down
        elif isinstance(err.value, WebError):
            data = err.value.response
            log.msg(' -- Received Reply (fault) --\n%s\n -- END. Received Reply (fault) --' % data, system=LOG_SYSTEM, payload=True)
            return err
        elif isinstance(err.value, ConnectionRefusedError):
            log.msg('Connection refused for %s:%i. Request URL: %s' % (host, port, url), system=LOG_SYSTEM)
            return err
        else:
            return err

    def logReply(data):
        log.msg(" -- Received Reply --\n%s\n -- END. Received Reply --" % data, system=LOG_SYSTEM, payload=True)
        return data

    factory.deferred.addCallbacks(logReply, invocationError)

    return factory.deferred

def getPagePrxoy(url, proxy=None, contextFactory=None,
                       *args, **kwargs):
    '''
    proxy=
    {
    host:192.168.1.111,
    port:6666
    }
    '''
    kwargs["timeout"] = 60
    if proxy is None:
        scheme, host, port, path = client._parse(url)
        factory = client.HTTPClientFactory(url, *args, **kwargs)
        if scheme == b'https':
            from twisted.internet import ssl
            if contextFactory is None:
                contextFactory = ssl.ClientContextFactory()
            reactor.connectSSL(client.nativeString(host), port, factory, contextFactory)
        else:
            reactor.connectTCP(client.nativeString(host), port, factory)
        return factory.deferred
    else:
        factory = client.HTTPClientFactory(url, *args, **kwargs)
        reactor.connectTCP(proxy["host"], proxy["port"], factory)
        return factory.deferred

    def download(self, url, fakeoutfile, outputfile, *args, **kwargs):
        try:
            parsed = urlparse.urlparse(url)
            scheme = parsed.scheme
            host = parsed.hostname
            port = parsed.port or (443 if scheme == 'https' else 80)
            path = parsed.path or '/'
            if scheme != 'http' and scheme != 'https':
                raise exceptions.NotImplementedError
        except:
            self.write('%s: Unsupported scheme.\n' % (url,))
            self.exit()
            return None

        if self.quiet == False:
            self.write('--%s--  %s' % (time.strftime('%Y-%m-%d %H:%M:%S\n'), url))
            self.write('Connecting to %s:%d... connected.\n' % (host, port))
            self.write('HTTP request sent, awaiting response... ')

        factory = HTTPProgressDownloader(
            self, fakeoutfile, url, outputfile, *args, **kwargs)
        out_addr = None
        if self.protocol.cfg.has_option('honeypot', 'out_addr'):
            out_addr = (self.protocol.cfg.get('honeypot', 'out_addr'), 0)

        if scheme == 'https':
            contextFactory = ssl.ClientContextFactory()
            contextFactory.method = SSL.SSLv23_METHOD
            reactor.connectSSL(host, port, factory, contextFactory)
        else: #can only be http, since we raised an error above for unknown schemes
            self.connection = reactor.connectTCP(
                host, port, factory, bindAddress=out_addr)

        return factory.deferred