def ooni_main(self, cmd):
        # We don't have the Command object so cheating for now.
        url = cmd.hostname

        # FIXME: validate that url is on the form scheme://host[:port]/path
        scheme, host, port, path = client._parse(url)
        
        ctrl_dest = self.endpoint(scheme, host, port)
        if not ctrl_dest:
            raise Exception('unsupported scheme %s in %s' % (scheme, url))
        if cmd.controlproxy:
            assert scheme != 'https', "no support for proxied https atm, sorry"
            _, proxy_host, proxy_port, _ = client._parse(cmd.controlproxy)
            control = SOCKSWrapper(reactor, proxy_host, proxy_port, ctrl_dest)
            print "proxy: ", proxy_host, proxy_port
        else:
            control = ctrl_dest
        f = client.HTTPClientFactory(url)
        f.deferred.addCallback(lambda x: self.cb('control', x))
        control.connect(f)

        exp_dest = self.endpoint(scheme, host, port)
        if not exp_dest:
            raise Exception('unsupported scheme %s in %s' % (scheme, url))
        # FIXME: use the experiment proxy if there is one
        experiment = exp_dest
        f = client.HTTPClientFactory(url)
        f.deferred.addCallback(lambda x: self.cb('experiment', x))
        experiment.connect(f)
        
        reactor.run()

    def test_parse(self):
        """
        L{client._parse} correctly parses a URL into its various components.
        """
        # The default port for HTTP is 80.
        self.assertEqual(
            client._parse('http://127.0.0.1/'),
            ('http', '127.0.0.1', 80, '/'))

        # The default port for HTTPS is 443.
        self.assertEqual(
            client._parse('https://127.0.0.1/'),
            ('https', '127.0.0.1', 443, '/'))

        # Specifying a port.
        self.assertEqual(
            client._parse('http://spam:12345/'),
            ('http', 'spam', 12345, '/'))

        # Weird (but commonly accepted) structure uses default port.
        self.assertEqual(
            client._parse('http://spam:/'),
            ('http', 'spam', 80, '/'))

        # Spaces in the hostname are trimmed, the default path is /.
        self.assertEqual(
            client._parse('http://foo '),
            ('http', 'foo', 80, '/'))

    def ooni_main(self):
        # We don't have the Command object so cheating for now.
        url = 'http://check.torproject.org/'
        self.controlproxy = 'socks4a://127.0.0.1:9050'
        self.experimentalproxy = ''

        if not re.match("[a-zA-Z0-9]+\:\/\/[a-zA-Z0-9]+", url):
          return None
        scheme, host, port, path = client._parse(url)

        ctrl_dest = self.endpoint(scheme, host, port)
        if not ctrl_dest:
            raise Exception('unsupported scheme %s in %s' % (scheme, url))
        if self.controlproxy:
            _, proxy_host, proxy_port, _ = client._parse(self.controlproxy)
            control = SOCKSWrapper(reactor, proxy_host, proxy_port, ctrl_dest)
        else:
            control = ctrl_dest
        f = client.HTTPClientFactory(url)
        f.deferred.addCallback(lambda x: self.cb('control', x))
        control.connect(f)

        exp_dest = self.endpoint(scheme, host, port)
        if not exp_dest:
            raise Exception('unsupported scheme %s in %s' % (scheme, url))
        # FIXME: use the experiment proxy if there is one
        experiment = exp_dest
        f = client.HTTPClientFactory(url)
        f.deferred.addCallback(lambda x: self.cb('experiment', x))
        experiment.connect(f)

        reactor.run()

def _httpRequest(url, soap_action, soap_envelope, timeout=DEFAULT_TIMEOUT, ctx_factory=None):
    # copied from twisted.web.client in order to get access to the
    # factory (which contains response codes, headers, etc)

    if type(url) is not str:
        e = RequestError('URL must be string, not %s' % type(url))
        return defer.fail(e), None

    log.msg(" -- Sending payload --\n%s\n -- End of payload --" % soap_envelope, system=LOG_SYSTEM, payload=True)

    scheme, host, port, _ = twclient._parse(url)

    factory = twclient.HTTPClientFactory(url, method='POST', postdata=soap_envelope, timeout=timeout)
    factory.noisy = False # stop spewing about factory start/stop

    # fix missing port in header (bug in twisted.web.client)
    if port:
        factory.headers['host'] = host + ':' + str(port)

    factory.headers['Content-Type'] = 'text/xml' # CXF will complain if this is not set
    factory.headers['soapaction'] = soap_action
    factory.headers['Authorization'] = 'Basic bnNpZGVtbzpSaW9QbHVnLUZlc3QyMDExIQ==' # base64.b64encode('nsidemo:RioPlug-Fest2011!')

    if scheme == 'https':
        if ctx_factory is None:
            return defer.fail(RequestError('Cannot perform https request without context factory')), None
        reactor.connectSSL(host, port, factory, ctx_factory)
    else:
        reactor.connectTCP(host, port, factory)

    return factory.deferred, factory

def downloadToFile(url, file, contextFactory=None, *args, **kwargs):
    """Download a web page to a file.

    @param file: path to file on filesystem, or file-like object.

    See HTTPDownloader to see what extra args can be passed.
    
    @note: This function taken from downloadToPage function in Twisted source code (twisted/web/client.py) and modified
    so that it wouldn't be noisy. Twisted source code is BSD licensed.
    """
    log.msg(
        'Making HTTP request to "%s" -- downloading returned data to "%s"...' % (url, file), lvl="d2", ss="ss_webreq"
    )

    scheme, host, port, path = http_client._parse(url)
    factory = http_client.HTTPDownloader(url, file, *args, **kwargs)
    # CASTDOT-CUSTOM: make it so this function is not noisy
    factory.noisy = False
    if scheme == "https":
        from twisted.internet import ssl

        if contextFactory is None:
            contextFactory = ssl.ClientContextFactory()
        reactor.connectSSL(host, port, factory, contextFactory)  # IGNORE:E1101
    else:
        reactor.connectTCP(host, port, factory)  # IGNORE:E1101
    return factory.deferred

	def __init__(self, url, contextFactory=None, retries=0):

		url = stripNoPrint(url)
		if retries > 0:
			print "Retrying: ", url
		else:
			print "Get: ", url
		self.retries = retries
		self.url = url
		self.charset = None
		scheme, host, port, path = _parse(url)
		HTTPClientFactory.__init__(self, url,
			method='GET', postdata=None, headers=None,
			agent='Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US;' + 
				' rv:1.9.2.10) Gecko/20100914 Firefox/3.6.10')
		if scheme == 'https':
			from twisted.internet import ssl
			if contextFactory is None:
				contextFactory = ssl.ClientContextFactory()
			reactor.connectSSL(host, port, self, contextFactory)
		else:
			reactor.connectTCP(host, port, self)
		
		self.deferred.addCallbacks(self.getCharset, self.Err)
		self.deferred.addCallbacks(self.getTitle, self.Err)

def getPageFactory(url,
                   agent="BitTorrent client",
                   bindAddress=None,
                   contextFactory=None,
                   proxy=None,
                   timeout=120):
    """Download a web page as a string.

    Download a page. Return a deferred, which will callback with a
    page (as a string) or errback with a description of the error.

    See HTTPClientFactory to see what extra args can be passed.
    """
    scheme, host, port, path = client._parse(url)
    if proxy:
        host, port = proxy.split(':')
        port = int(port)
    factory = HTTPProxyUnGzipClientFactory(url, agent=agent, proxy=proxy)
    if scheme == 'https':
        from twisted.internet import ssl
        if contextFactory is None:
            contextFactory = ssl.ClientContextFactory()
        reactor.connectSSL(host, port, factory, contextFactory,
                           bindAddress=bindAddress,
                           timeout=timeout)
    else:
        reactor.connectTCP(host, port, factory,
                           bindAddress=bindAddress,
                           timeout=timeout)
    return factory

    def _request (self, method, url, *a, **kw) :
        if not url or (not url.startswith("http://") and not url.startswith("https://")) :
            if not self._base_url :
                return defer.maybeDeferred(lambda x : Response(url, ), )

            url = urllib.basejoin(self._base_url, (url and url or ""), )

        _scheme, _host, _port, _path = client_http._parse(url, )

        kw["method"] = method
        _factory = self._client_factory(url, *a, **kw)

        if _scheme == "https" :
            from twisted.internet import ssl
            #_contextFactory = kw.get("contextFactory")
            #if _contextFactory is None :
            #    _contextFactory = ssl.ClientContextFactory()
            _contextFactory = ssl.ClientContextFactory()

            reactor.connectSSL(_host, _port, _factory, _contextFactory, )
        else:
            reactor.connectTCP(_host, _port, _factory)

        return _factory.deferred.addCallback(
            self._cb_request, _factory, url,
        ).addCallback(
            self._cb_request_debug,
        )

	def __init__(self, url, outputfile, contextFactory=None, *args, **kwargs):
		scheme, host, port, path = client._parse(url)
		self.factory = HTTPProgressDownloader(url, outputfile, *args, **kwargs)
		if scheme == "https":
			self.connection = reactor.connectSSL(host, port, self.factory, ssl.ClientContextFactory())
		else:
			self.connection = reactor.connectTCP(host, port, self.factory)

def getPageFactory(url, contextFactory=None, *args, **kwargs):

    def failedConnect(reason, factory):
        try:
            i = factory.status
            return reason
        except:
            pass
        #logger.warn("couldn't connect to %s:%d in getPageFactory: %s" 
        #       % (factory.host, factory.port, reason))
        #logger.warn("state of factory is %s" % factory)
        #logger.warn("dir() of factory is %s" % dir(factory))
        return reason

    if len(url) >= 16384:
        raise ValueError(
                "Too much data sent: twisted server doesn't appear to"
                " support urls longer than 16384")
    scheme, host, port, path = client._parse(url)
    factory = client.HTTPClientFactory(url, *args, **kwargs)
    factory.deferred.addErrback(failedConnect, factory)
    to = CONNECT_TO+random.randrange(2+CONNECT_TO_VAR)-CONNECT_TO_VAR
    if scheme == 'https':
        from twisted.internet import ssl
        if contextFactory is None:
            contextFactory = ssl.ClientContextFactory()
        reactor.connectSSL(host, port, factory, contextFactory)
    else:
        reactor.connectTCP(host, port, factory, timeout=to)
    return factory

def getPage(url, contextFactory=None, *args, **kwargs):
    log.msg('Method: %s' % kwargs.get('method', 'GET'))
    log.msg('URI: %s' % url)
    try:
        log.msg('Headers: %r' % kwargs['headers'])
    except KeyError:
        pass
    try:
        log.msg('Payload: %r' % kwargs['postdata'])
    except KeyError:
        pass
    scheme, host, port, path = client._parse(url)
    factory = HTTPClientFactory(url, *args, **kwargs)
    if scheme == 'https':
        from twisted.internet import ssl
        if contextFactory is None:
            contextFactory = ssl.ClientContextFactory()
        reactor.connectSSL(host, port, factory, contextFactory)
    else:
        reactor.connectTCP(host, port, factory)

    def _eb(failure):
        log.msg('Failed.')
        log.msg(failure)
        return failure

    return factory.deferred.addCallback(_checkCacheControl).addErrback(_eb)

    def request(self, method, uri, headers=None, bodyProducer=None):
        parsedURI = client._parse(uri)
        
        host_addr = address.IPv4Address('TCP', parsedURI.host, parsedURI.port)
        
        
        # ripped from _AgentBase._requestWithEndpoint
        if headers is None:
            headers = Headers()
        if not headers.hasHeader('host'):
            headers = headers.copy()
            headers.addRawHeader(
                'host', self._computeHostValue(parsedURI.scheme, parsedURI.host,
                                               parsedURI.port))
        request = client.Request(method, parsedURI.path, headers, bodyProducer,
                                 persistent=False)

        c = ClientProtocol(request)
        
        # ouch
        self.root.putChild('', self.root)
        
        server = Site(self.root).buildProtocol(self.addr)
        loopbackAsync(server, c, host_addr, self.addr)
        return c.response.addBoth(self._done, c)

def getPage(url, factoryFactory = client.HTTPClientFactory, proxyFactoryFactory = ProxyHTTPClientFactory, *args, **kwargs):
    if not proxy_host or not proxy_port:
        #logging.debug('No proxy information - default behaviour')
        return client.getPage(url, *args, **kwargs)
    scheme, host, port, path = client._parse(url)
    if scheme == 'https':
        #logging.debug('Proxy and HTTPS - connect via new class')
        http_factory = factoryFactory(url, followRedirect = 0, *args, **kwargs)
        https_factory = https.ProxyHTTPSConnectionFactory(http_factory, host, port, True, proxy_user, proxy_pass)
        reactor.connectTCP(proxy_host, proxy_port, https_factory)
        return http_factory.deferred

    if 'headers' in kwargs:
        headers = kwargs['headers']
    else:
        headers = {}
    if proxy_user and proxy_pass:
        auth = base64.encodestring("%s:%s" %(proxy_user, proxy_pass))
        headers['Proxy-Authorization'] = 'Basic %s' % (auth.strip())
        #logging.debug('Adding header: %s', headers['Proxy-Authorization'])
        kwargs['headers'] = headers
    #Cleanup proxy params
    factory = proxyFactoryFactory(url, proxy_host, proxy_port, followRedirect = 0, *args, **kwargs)
    #logging.debug('Do proxy %s %i', proxy_host, proxy_port)
    reactor.connectTCP(proxy_host, proxy_port, factory)
    return factory.deferred

 def _getPage(self, req): 
     scheme, host, port = _parse(req['url'])[0:3]
     factory = HTTPClientFactory(
         req['url'],
         method=req['method'],
         postdata=req['postdata'],
         headers=req['headers'],
         agent=req['agent'],
         timeout=req['timeout'],
         cookies=req['cookies'],
         followRedirect=req['follow_redirect']
     )
     if scheme == 'https':
         reactor.connectSSL(
                                 host, 
                                 port, 
                                 factory, 
                                 AllCipherSSLClientContextFactory(), 
                                 timeout=req['timeout']
                             )
     else:
         reactor.connectTCP(host, port, factory, timeout=req['timeout'])
     factory.deferred.addCallback(self._getPageComplete, factory)
     factory.deferred.addErrback(self._getPageError, factory)
     return factory.deferred

 def __init__(self, jobId, jobSpec):
     self.jobId = jobId
     self.jobSpec = jobSpec
     
     # attributes which configure the engine
     self.clientFunction = lambda self, t: 1
     self.requests = {"":{}}
     self.userAgent = str("thundercloud client/%s" % constants.VERSION)
     self.iterator = lambda: True
     self.httpClientRequestQueue = Queue()
     self.jobState = JobState.NEW
     self.timeout = 10
 
     # attributes for time management
     self.duration = float("inf") #60
     self.startTime = None
     self.endTime = None
     self.elapsedTime = 0.00000001    # so clientFunction(0) != 0
     self.pausedTime = 0.0
     self._timeAtPause = 0.0
     
     # attributes for data management
     self.bytesTransferred = 0
     self.transferLimit = float("inf")
     
     # attributes for statistics generation    
     self.iterations = 0
     self.requestsCompleted = 0
     self.requestsFailed = 0
     self.errors = copy.deepcopy(JobResults().results_errors)
     self.statisticsByTime = copy.deepcopy(JobResults().results_byTime)
     self._averageTimeToConnect = 0
     self._averageTimeToFirstByte = 0
     self._averageResponseTime = 0
     self.statsInterval = 60
     self._statsBookmark = 0          # shortcut to last time stats were generated.
                                      # avoids listing/sorting statisticsByTime keys
     
     # read the job spec and update attributes
     self.requests = jobSpec.requests
     self.transferLimit = jobSpec.transferLimit
     self.duration = jobSpec.duration
     self.userAgent = jobSpec.userAgent
     self.statsInterval = jobSpec.statsInterval
     self.timeout = jobSpec.timeout
     self.clientFunction = lambda t: eval(jobSpec.clientFunction)
     
     # dump the host/port/URLs to be fetched into a queue
     for url in self.requests.keys():
         scheme, host, port, path = _parse(str(url))
         self.httpClientRequestQueue.put([host, port, 
                                          str(self.requests[url]["method"]), 
                                          str(url), 
                                          self.requests[url]["postdata"],
                                          self.requests[url]["cookies"]])
     
     db.execute("INSERT INTO jobs (id, startTime, spec) VALUES (?, ?, ?)", 
                 (self.jobId, datetime.datetime.now(), self.jobSpec))
     db.execute("INSERT INTO accounting (job, elapsedTime, bytesTransferred) VALUES (?, ?, ?)", 
                 (self.jobId, 0, 0))

 def testParse(self):
     scheme, host, port, path = client._parse("http://127.0.0.1/")
     self.assertEquals(path, "/")
     self.assertEquals(port, 80)
     scheme, host, port, path = client._parse("https://127.0.0.1/")
     self.assertEquals(path, "/")
     self.assertEquals(port, 443)
     scheme, host, port, path = client._parse("http://spam:12345/")
     self.assertEquals(port, 12345)
     scheme, host, port, path = client._parse("http://foo ")
     self.assertEquals(host, "foo")
     self.assertEquals(path, "/")
     scheme, host, port, path = client._parse("http://egg:7890")
     self.assertEquals(port, 7890)
     self.assertEquals(host, "egg")
     self.assertEquals(path, "/")

    def request(self, method, uri, headers=None, bodyProducer=None):
        """
        Issue a new request via the configured proxy.
        """

        if version >= Version('twisted', 13, 1, 0):
            parsed_uri = _URI.getFromBytes(uri)
            scheme = parsed_uri.scheme
            host = parsed_uri.host
            port = parsed_uri.port
        else:
            scheme, host, port, path = _parse(uri)
        request_path = uri

        d = self._connect(scheme, host, port)

        if headers is None:
            headers = Headers()
        if not headers.hasHeader('host'):
            # This is a lot of copying.  It might be nice if there were a bit
            # less.
            headers = Headers(dict(headers.getAllRawHeaders()))
            headers.addRawHeader(
                'host', self._computeHostValue(scheme, host, port))

        def cbConnected(proto):
            # NOTE: For the proxy case the path should be the full URI.
            return proto.request(
                Request(method, request_path, headers, bodyProducer))

        d.addCallback(cbConnected)
        return d

def getPageWithHeaders(url, *args, **kwargs):
	# basically a clone of client.getPage, but with a handle on the factory
	# so we can pull the headers later
	scheme, host, port, path = client._parse(url)
	factory = client.HTTPClientFactory(url, *args, **kwargs)
	reactor.connectTCP(host, port, factory)
	return factory

def _RestApiClient__request(cls, url, method, postdata=None, cookies={}, timeout=None, credentials=None):

    extraHeaders = {}
    
    if postdata is not None:
        postdata = json.dumps(postdata)

    if credentials is not None:
        cred = "%s:%s" % (credentials[0], credentials[1])
        extraHeaders["Authorization"] = "Basic " + base64.encodestring(cred).replace('\012','')

   
    scheme, host, port, path = _parse(str(url))
       
    # replace multiple slashes in the url to a single slash
    # in the name of genericism this might be a bad idea but
    # whatever
    while "//" in path:
        path = path.replace("//", "/")
    if path[0] == "/":
        path = path[1:]

    url = str("%s://%s:%s/%s" % (scheme, host, port, path))
    log.debug("REST API Client request: %s %s" % (method, url))

    factory = HTTPClientFactory(url,
                         method=method,
                         postdata=postdata,
                         cookies=cookies, 
                         timeout=timeout,
                         headers=extraHeaders)

    
    reactor.connectTCP(host, port, factory)
    return factory

def request(rq):
    headers = Headers(dict((k, [v]) for k, v in rq.headers.iteritems()))
    scheme, host, port, path = _parse(rq.url)
    headers.setRawHeaders('Host', [host])
    return (rq.method, rq.url,
            headers,
            StringProducer(rq.data) if rq.data else None)