def __init__(self, base_url='http://localhost:8888', quiet_requests=True, **kwargs):
        self.client_id = str(uuid4())
        self.base_url = base_url
        pool = HTTPConnectionPool(reactor, persistent=True)
        agent = ContentDecoderAgent(RedirectAgent(Agent(reactor, pool=pool)), [('gzip', GzipDecoder)])

        if quiet_requests:
            pool._factory = QuieterHTTP11ClientFactory

        auth_url = kwargs.get('auth_url')
        if auth_url:
            username = kwargs.get('username')
            password = kwargs.get('password')
            api_key = kwargs.get('api_key')

            if not username:
                raise RuntimeError('Marconi "auth_url" specified with no username')

            if api_key:
                cred = api_key
                auth_type = 'api_key'
            elif password:
                cred = password
                auth_type = 'password'
            else:
                raise RuntimeError('Marconi "auth_url" specified with no "password" or "api_key"')

            agent = KeystoneAgent(agent, auth_url, (username, cred), auth_type=auth_type)

        self.http_client = HTTPClient(agent)

    def requestAvatarId(self, c):
        creds = credentials.IUsernamePassword(c, None)

        if creds is not None:
            locks = []
            pool = HTTPConnectionPool(reactor, persistent=False)
            pool.cachedConnectionTimeout = self.timeout
            if self.max_concurrency:
                pool.persistent = True
                pool.maxPersistentPerHost = self.max_concurrency
                locks.append(
                    defer.DeferredSemaphore(self.max_concurrency))

            if self.global_max_concurrency:
                locks.append(
                    defer.DeferredSemaphore(self.global_max_concurrency))

            conn = ThrottledSwiftConnection(
                locks, self.auth_url, creds.username, creds.password,
                pool=pool,
                extra_headers=self.extra_headers,
                verbose=self.verbose,
                ceph_compatible=self.ceph_compatible
            )
            conn.user_agent = USER_AGENT

            d = conn.authenticate()
            d.addCallback(self._after_auth, conn)
            d.addErrback(eb_failed_auth)
            return d
        return defer.fail(error.UnauthorizedLogin())

def makeService(options):
    """
    Makes a new swftp-ftp service. The only option is the config file
    location. The config file has the following options:
     - host
     - port
     - auth_url
     - num_persistent_connections
     - connection_timeout
     - welcome_message
    """
    from twisted.protocols.ftp import FTPFactory
    from twisted.web.client import HTTPConnectionPool
    from twisted.cred.portal import Portal

    from swftp.ftp.server import SwiftFTPRealm
    from swftp.auth import SwiftBasedAuthDB
    from swftp.utils import print_runtime_info

    print('Starting SwFTP-ftp %s' % VERSION)

    c = get_config(options['config_file'], options)
    ftp_service = service.MultiService()

    # Add statsd service
    if c.get('ftp', 'log_statsd_host'):
        try:
            from swftp.statsd import makeService as makeStatsdService
            makeStatsdService(
                c.get('ftp', 'log_statsd_host'),
                c.getint('ftp', 'log_statsd_port'),
                sample_rate=c.getfloat('ftp', 'log_statsd_sample_rate'),
                prefix=c.get('ftp', 'log_statsd_metric_prefix')
            ).setServiceParent(ftp_service)
        except ImportError:
            log.err('Missing Statsd Module. Requires "txstatsd"')

    pool = HTTPConnectionPool(reactor, persistent=True)
    pool.maxPersistentPerHost = c.getint('ftp', 'num_persistent_connections')
    pool.cachedConnectionTimeout = c.getint('ftp', 'connection_timeout')

    authdb = SwiftBasedAuthDB(auth_url=c.get('ftp', 'auth_url'),
                              verbose=c.getboolean('ftp', 'verbose'))

    ftpportal = Portal(SwiftFTPRealm())
    ftpportal.registerChecker(authdb)
    ftpfactory = FTPFactory(ftpportal)
    ftpfactory.welcomeMessage = c.get('ftp', 'welcome_message')
    ftpfactory.allowAnonymous = False

    signal.signal(signal.SIGUSR1, print_runtime_info)
    signal.signal(signal.SIGUSR2, print_runtime_info)

    internet.TCPServer(
        c.getint('ftp', 'port'),
        ftpfactory,
        interface=c.get('ftp', 'host')).setServiceParent(ftp_service)
    return ftp_service

 def __init__(self, hs):
     self.hs = hs
     self.signing_key = hs.config.signing_key[0]
     self.server_name = hs.hostname
     pool = HTTPConnectionPool(reactor)
     pool.maxPersistentPerHost = 10
     self.agent = MatrixFederationHttpAgent(reactor, pool=pool)
     self.clock = hs.get_clock()
     self.version_string = hs.version_string

 def __init__(self, hs):
     self.hs = hs
     # The default context factory in Twisted 14.0.0 (which we require) is
     # BrowserLikePolicyForHTTPS which will do regular cert validation
     # 'like a browser'
     pool = HTTPConnectionPool(reactor)
     pool.maxPersistentPerHost = 10
     self.agent = Agent(reactor, pool=pool)
     self.version_string = hs.version_string

    def _getConnectionPool(self):
        pool = HTTPConnectionPool(reactor, self._persistent)

        if self._persistent:
            pool.maxPersistentPerHost = self._maxPersistentPerHost
            pool.cachedConnectionTimeout = self._cachedConnectionTimeout
            pool.retryAutomatically = self._retryAutomatically

        return pool

 def __init__(self, hs):
     self.hs = hs
     self.signing_key = hs.config.signing_key[0]
     self.server_name = hs.hostname
     pool = HTTPConnectionPool(reactor)
     pool.maxPersistentPerHost = 10
     self.agent = Agent.usingEndpointFactory(reactor, MatrixFederationEndpointFactory(hs), pool=pool)
     self.clock = hs.get_clock()
     self.version_string = hs.version_string
     self._next_id = 1

 def __init__(self, debug=False):
     self.gpg = gnupg.GPG()
     self.mpex_url = 'http://polimedia.us/bitcoin/mpex.php'
     self.mpex_fingerprint = 'F1B69921'
     self.passphrase = None
     self.debug = debug
     if(self.debug) :
         self.df = open("mpex_%d.txt" % time.time(),'w')
     pool = HTTPConnectionPool(reactor)
     #close connections at same time as server to prevent ResponseNeverReceived error
     #timeout can be determined automatically from Keep-Alive header
     pool.cachedConnectionTimeout = 4
     self.agent = Agent(reactor, pool=pool)

def _get_agent():
    context_factory = MyWebClientContextFactory()
    try:
        # HTTPConnectionPool has been present since Twisted version 12.1
        from twisted.web.client import HTTPConnectionPool
        pool = HTTPConnectionPool(reactor, persistent=True)
        pool.maxPersistentPerHost = _MAX_PERSISTENT_PER_HOST
        pool.cachedConnectionTimeout = _CACHED_CONNECTION_TIMEOUT
        agent = Agent(reactor, context_factory,
                       connectTimeout=_CONNECT_TIMEOUT, pool=pool)
    except ImportError:
        from _zenclient import ZenAgent
        agent = ZenAgent(reactor, context_factory, persistent=True, maxConnectionsPerHostName=1)
    return agent

class StateTestCase(unittest.TestCase):
    def setUp(self):
        pnconf_uuid_set = copy(pnconf)
        pnconf_uuid_set.uuid = 'someuuid'
        self.pool = HTTPConnectionPool(reactor, persistent=False)
        self.pubnub = PubNubTwisted(pnconf_uuid_set, reactor=reactor, pool=self.pool)

    def tearDown(self):
        return self.pool.closeCachedConnections()

    def assert_valid_state_envelope(self, envelope):
        self.assertIsInstance(envelope, TwistedEnvelope)
        self.assertIsInstance(envelope.result, PNSetStateResult)
        self.assertEqual(envelope.result.state, state)

    @inlineCallbacks
    @pn_vcr.use_cassette(
        'tests/integrational/fixtures/twisted/state/single_channel.yaml',
        filter_query_parameters=['uuid'])
    def test_state_single_channel(self):
        envelope = yield self.pubnub.set_state().channels(channel).state(state).deferred()
        self.assert_valid_state_envelope(envelope)
        returnValue(envelope)

    @inlineCallbacks
    @pn_vcr.use_cassette(
        'tests/integrational/fixtures/twisted/state/multiple_channels.yaml',
        filter_query_parameters=['uuid'])
    def test_state_multiple_channels(self):
        envelope = yield self.pubnub.set_state().channels(channels).state(state).deferred()
        self.assert_valid_state_envelope(envelope)
        returnValue(envelope)

 def setUp(self):
     self.active_connections = []
     self.pool = HTTPConnectionPool(reactor, persistent=True)
     self.swift = get_swift_client(CONFIG, pool=self.pool)
     self.tmpdir = tempfile.mkdtemp()
     _, self.sftp = self.get_client()
     yield clean_swift(self.swift)

class SFTPFuncTest(unittest.TestCase):
    @defer.inlineCallbacks
    def setUp(self):
        self.active_connections = []
        self.pool = HTTPConnectionPool(reactor, persistent=True)
        self.swift = get_swift_client(CONFIG, pool=self.pool)
        self.tmpdir = tempfile.mkdtemp()
        _, self.sftp = self.get_client()
        yield clean_swift(self.swift)

    @defer.inlineCallbacks
    def tearDown(self):
        shutil.rmtree(self.tmpdir)
        for (transport, conn) in self.active_connections:
            try:
                conn.close()
            except:
                pass
            try:
                transport.close()
            except:
                pass

        yield clean_swift(self.swift)
        yield self.pool.closeCachedConnections()

    def get_client(self):
        transport, conn = get_sftp_client_with_transport(CONFIG)
        self.active_connections.append((transport, conn))
        return transport, conn

class HTTP11DownloadHandler(object):
    def __init__(self, settings):
        self.settings = settings
        self.ssl_methods = settings.get('DOWNLOAD_HANDLER_SSL_METHODS')
        self.context_factories = [CrawlmiClientContextFactory(method) for method in self.ssl_methods]
        self.pool = HTTPConnectionPool(reactor, persistent=True)
        self.pool.maxPersistentPerHost = settings.get_int('CONCURRENT_REQUESTS_PER_DOMAIN')
        self.pool._factory.noisy = False

    def download_request(self, request):
        '''Return a deferred for the HTTP download.'''
        dfd = None
        for context_factory in self.context_factories:
            if dfd is None:
                dfd = self._download(request, context_factory)
            else:
                def _failure(failure):
                    failure.trap(SSL.Error)
                    return self._download(request, context_factory)
                dfd.addErrback(_failure)
        return dfd

    def _download(self, request, context_factory):
        agent = CrawlmiAgent(
            context_factory,
            self.settings.get_float('DOWNLOAD_TIMEOUT', 180, request),
            self.settings.get_int('DOWNLOAD_SIZE_LIMIT', 0, request),
            request.meta.get('bind_address'),
            self.pool)
        return agent.download_request(request)

    def close(self):
        return self.pool.closeCachedConnections()

 def __init__(self, settings):
     self.settings = settings
     self.ssl_methods = settings.get('DOWNLOAD_HANDLER_SSL_METHODS')
     self.context_factories = [CrawlmiClientContextFactory(method) for method in self.ssl_methods]
     self.pool = HTTPConnectionPool(reactor, persistent=True)
     self.pool.maxPersistentPerHost = settings.get_int('CONCURRENT_REQUESTS_PER_DOMAIN')
     self.pool._factory.noisy = False

    def __init__(self, reactor, node=('localhost', 4001), ca=None, cert=None):
        self.reactor = reactor
        self.node = node
        self.scheme = 'http'
        self.ca = ca
        self.cert = cert
        context = None
        if ca:
            self.scheme = 'https'
            context = PolicyForHTTPS(ca, cert)

        quietPool = HTTPConnectionPool(reactor, persistent = True)
        quietPool.maxPersistentPerHost = 2
        quietPool._factory = QuietHTTP11ClientFactory

        self.agent = Agent(self.reactor, contextFactory=context, pool=quietPool)

 def setUp(self):
     description = yield self._httpbin_process.server_description(
         reactor)
     self.baseurl = URL(scheme=u"http",
                        host=description.host,
                        port=description.port).asText()
     self.agent = Agent(reactor)
     self.pool = HTTPConnectionPool(reactor, False)

def _get_agent():
    global _AGENT
    if _AGENT is None:
        try:
            # HTTPConnectionPool has been present since Twisted version 12.1
            from twisted.web.client import HTTPConnectionPool
            pool = HTTPConnectionPool(reactor, persistent=True)
            pool.maxPersistentPerHost = _MAX_PERSISTENT_PER_HOST
            pool.cachedConnectionTimeout = _CACHED_CONNECTION_TIMEOUT
            _AGENT = Agent(
                reactor, connectTimeout=_CONNECT_TIMEOUT, pool=pool)
        except ImportError:
            try:
                # connectTimeout first showed up in Twisted version 11.1
                _AGENT = Agent(reactor, connectTimeout=_CONNECT_TIMEOUT)
            except TypeError:
                _AGENT = Agent(reactor)
    return _AGENT

    def __init__(self, reactor):
        self.reactor = reactor
        pool = HTTPConnectionPool(reactor, persistent=True)
        pool.maxPersistentPerHost = 1
        pool.cachedConnectionTimeout = 600
        self.agent = RedirectAgent(Agent(reactor, pool=pool))
        self.reqQ = HttpReqQ(self.agent, self.reactor)
        self.clientPlaylist = HlsPlaylist()
        self.verbose = False
        self.download = False
        self.outDir = ""
        self.encryptionHandled=False

        # required for the dump durations functionality
        self.dur_dump_file = None
        self.dur_avproble_acc = 0
        self.dur_vt_acc = 0
        self.dur_playlist_acc = 0

	def parallelFetchAllProblems(self):
			pool = HTTPConnectionPool(reactor)
			pool.maxPersistentPerHost = self.concurrency
			agent = Agent(reactor, pool=pool)
			sem = DeferredSemaphore(self.concurrency)
			self.done = 0

			def assign():
				self.query.execute('BEGIN')
				for id in range(1001, self.getProblemMax()+1):
					sem.acquire().addCallback(requestFactory, id)
			
			def requestFactory(token, id):
				deferred = agent.request('GET', self.baseUrl + self.problemPath + str(id))
				deferred.addCallback(onHeader, id)
				deferred.addErrback(errorHandler, id)
				return deferred
			
			def onHeader(response, id):
				deferred = readBody(response)
				deferred.addCallback(onBody, id)
				deferred.addErrback(errorHandler, id)
				return deferred

			def onBody(html, id):
				sem.release()
				d = pyq(html)
				title = d('#content_body > center:nth-child(1) > span').text(),
				body = d('#content_body').text()
				print('Fetched ProblemID: %s, Title: %s, done: %s' % (id, title[0], self.done))
				self.storeProblem(id, title[0], body)
				self.done += 1
				if(self.done == self.problemCount):
					print('Fetch data used %s s' % (reactor.seconds() - startTimeStamp))
					print('Fetch data end, writing to database')
					self.query.execute('COMMIT')
					reactor.stop()

			def errorHandler(err, id):
				print('[%s] id %s: %s' % (reactor.seconds() - startTimeStamp, id, err))

			startTimeStamp = reactor.seconds()
			reactor.callWhenRunning(assign)
			reactor.run()

 def __init__(self, common_headers = None, hang_up = True, use_cookies = True, pool = True, dns = True, max_hops = 5, connection_timeout = 10, verbose = False):
     if pool:
         self.connection_pool = HTTPConnectionPool(reactor, persistent=True)            
     else:
         self.connection_pool = HTTPConnectionPool(reactor, persistent=False)
         
     if use_cookies:
         cookieJar = CookieJar()
         self.agent = CookieAgent(Agent(reactor, pool = self.connection_pool), cookieJar)
     else:
         self.agent = Agent(reactor, pool = self.connection_pool)
     
     if verbose:
         log.startLogging(sys.stdout)
     
     self.hang_up = hang_up
     
     self.common_headers = common_headers
     self.max_hops = max_hops
     self.connection_timeout = connection_timeout