def __init__(self,
              userAgent      = 'Mozilla/4.0 (compatible; MSIE 8.0)',
              followLocation = 1,            # follow redirects?
              autoReferer    = 1,            # allow 'referer' to be set normally?
              verifySSL      = 0,            # tell SSL to verify IDs?
              useCookies     = True,         # will hold all pycurl cookies
              useSOCKS       = False,        # use SOCKS5 proxy?
              proxy          = 'localhost',  # SOCKS host
              proxyPort      = 8080,         # SOCKS port
              proxyType      = 5,            # SOCKS protocol
              verbose        = False,
              debug          = False,
              ) :
     self.followLocation = followLocation
     self.autoReferer    = autoReferer
     self.verifySSL      = verifySSL
     self.useCookies     = useCookies
     self.useSOCKS       = useSOCKS
     self.proxy          = proxy
     self.proxyPort      = proxyPort
     self.proxyType      = proxyType
     self.pco = pycurl.Curl()
     self.pco.setopt(pycurl.CAINFO, os.path.join('.', 'linode', 'cloud-cacerts.pem'))
     self.pco.setopt(pycurl.USERAGENT,      userAgent)
     self.pco.setopt(pycurl.FOLLOWLOCATION, followLocation)
     self.pco.setopt(pycurl.MAXREDIRS,      20)
     self.pco.setopt(pycurl.CONNECTTIMEOUT, 30)
     self.pco.setopt(pycurl.AUTOREFERER,    autoReferer)
     # SSL verification (True/False)
     self.pco.setopt(pycurl.SSL_VERIFYPEER, verifySSL)
     self.pco.setopt(pycurl.SSL_VERIFYHOST, verifySSL)
     if useCookies == True :
         cjf = os.tmpfile() # potential security risk here; see python documentation
         self.pco.setopt(pycurl.COOKIEFILE, cjf.name)
         self.pco.setopt(pycurl.COOKIEJAR,  cjf.name)
     if useSOCKS :
         # if you wish to use SOCKS, it is configured through these parms
         self.pco.setopt(pycurl.PROXY,     proxy)
         self.pco.setopt(pycurl.PROXYPORT, proxyPort)
         self.pco.setopt(pycurl.PROXYTYPE, proxyType)
     if verbose :
         self.pco.setopt(pycurl.VERBOSE, 1)
     if debug :
         print 'PyCurl version info:'
         print pycurl.version_info()
         print
         self.pco.setopt(pycurl.DEBUGFUNCTION, self.debug)
     return

    def __init__(self, base_url="", fakeheaders=()):
        super().__init__(base_url, fakeheaders)
        self.set_option(pycurl.SSL_VERIFYPEER, True)
        self.set_option(pycurl.ENCODING, "")  # accept all encodings

        # workaround buggy pycurl versions before Dec 2013
        self.payload = None
        self.payload_io = BytesIO()
        self.set_option(pycurl.WRITEFUNCTION, self.payload_io.write)

        def header_callback(x):
            if isinstance(x, str):
                # workaround buggy pycurl versions
                self.hdr += x
            else:
                self.hdr += x.decode("ascii")
        self.set_option(pycurl.HEADERFUNCTION, header_callback)

        ssl_library = pycurl.version_info()[5]
        # use the only one secure cipher that Sina supports
        if "OpenSSL" in ssl_library or "LibreSSL" in ssl_library:
            self.set_option(pycurl.SSL_CIPHER_LIST, "ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384")
        elif "GnuTLS".lower() in ssl_library.lower():  # not sure about the capitalization, use lower case
            self.set_option(pycurl.SSL_CIPHER_LIST, "PFS")
        else:
            raise NotImplemented("Unsupported SSL/TLS library (%s)!" % ssl_library)

    def __init__(self, base_url="", fakeheaders=()):
        super().__init__(base_url, fakeheaders)
        self.set_option(pycurl.SSL_VERIFYPEER, True)
        self.set_option(pycurl.ENCODING, "")  # accept all encodings

        # workaround buggy pycurl versions before Dec 2013
        self.payload = None
        self.payload_io = BytesIO()
        self.set_option(pycurl.WRITEFUNCTION, self.payload_io.write)

        def header_callback(x):
            if isinstance(x, str):
                # workaround buggy pycurl versions
                self.hdr += x
            else:
                self.hdr += x.decode("ascii")

        self.set_option(pycurl.HEADERFUNCTION, header_callback)

        # use the only one secure cipher that Sina supports
        if "OpenSSL" in pycurl.version_info()[5]:
            self.set_option(pycurl.SSL_CIPHER_LIST, "ECDHE-RSA-AES256-SHA")
        else:
            # Assume GnuTLS. what? You've built libcurl with NSS? Hum...
            self.set_option(pycurl.SSL_CIPHER_LIST, "PFS")

    def init_handle(self):
        """Sets common options to curl handle."""
        self.setopt(pycurl.FOLLOWLOCATION, 1)
        self.setopt(pycurl.MAXREDIRS, 5)
        self.setopt(pycurl.CONNECTTIMEOUT, 30)
        self.setopt(pycurl.NOSIGNAL, 1)
        self.setopt(pycurl.NOPROGRESS, 1)
        if hasattr(pycurl, 'AUTOREFERER'):
            self.setopt(pycurl.AUTOREFERER, 1)
        self.setopt(pycurl.SSL_VERIFYPEER, 0)
        # Interval for low speed, detects connection loss, but can abort dl if
        # hoster stalls the download
        self.setopt(pycurl.LOW_SPEED_TIME, 45)
        self.setopt(pycurl.LOW_SPEED_LIMIT, 5)

        # do not save the cookies
        self.setopt(pycurl.COOKIEFILE, '')
        self.setopt(pycurl.COOKIEJAR, '')

        # self.setopt(pycurl.VERBOSE, 1)

        self.setopt(
            pycurl.USERAGENT,
            'Mozilla/5.0 (Windows NT 10.0; Win64; rv:53.0) '
            'Gecko/20100101 Firefox/53.0')
        if pycurl.version_info()[7]:
            self.setopt(pycurl.ENCODING, 'gzip,deflate')

        self.headers.update(
            {'Accept': '*/*',
             'Accept-Language': 'en-US,en',
             'Accept-Charset': 'ISO-8859-1,utf-8;q=0.7,*;q=0.7',
             'Connection': 'keep-alive',
             'Keep-Alive': '300',
             'Expect': ''})

    def initHandle(self):
        """ sets common options to curl handle """
        self.c.setopt(pycurl.FOLLOWLOCATION, 1)
        self.c.setopt(pycurl.MAXREDIRS, 10)
        self.c.setopt(pycurl.CONNECTTIMEOUT, 30)
        self.c.setopt(pycurl.NOSIGNAL, 1)
        self.c.setopt(pycurl.NOPROGRESS, 1)
        if hasattr(pycurl, "AUTOREFERER"):
            self.c.setopt(pycurl.AUTOREFERER, 1)
        self.c.setopt(pycurl.SSL_VERIFYPEER, 0)
        self.c.setopt(pycurl.LOW_SPEED_TIME, 60)
        self.c.setopt(pycurl.LOW_SPEED_LIMIT, 5)
        if hasattr(pycurl, "USE_SSL"):
            self.c.setopt(pycurl.USE_SSL, pycurl.CURLUSESSL_TRY)

        # self.c.setopt(pycurl.VERBOSE, 1)

        self.c.setopt(pycurl.USERAGENT,
                      "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:37.0) Gecko/20100101 Firefox/37.0")

        if pycurl.version_info()[7]:
            self.c.setopt(pycurl.ENCODING, "gzip, deflate")
        self.c.setopt(pycurl.HTTPHEADER, ["Accept: */*",
                                          "Accept-Language: en-US, en",
                                          "Accept-Charset: ISO-8859-1, utf-8;q=0.7,*;q=0.7",
                                          "Connection: keep-alive",
                                          "Keep-Alive: 300",
                                          "Expect:"])

    def _process_queue(self):
        with stack_context.NullContext():
            while True:
                started = 0
                while self._free_list and self._requests:
                    started += 1
                    curl = self._free_list.pop()
                    (request, callback) = self._requests.popleft()
                    curl.info = {
                        "headers": httputil.HTTPHeaders(),
                        "buffer": cStringIO.StringIO(),
                        "request": request,
                        "callback": callback,
                        "curl_start_time": monotime(),
                    }
                    # Disable IPv6 to mitigate the effects of this bug
                    # on curl versions <= 7.21.0
                    # http://sourceforge.net/tracker/?func=detail&aid=3017819&group_id=976&atid=100976
                    if pycurl.version_info()[2] <= 0x71500:  # 7.21.0
                        curl.setopt(pycurl.IPRESOLVE, pycurl.IPRESOLVE_V4)
                    _curl_setup_request(curl, request, curl.info["buffer"],
                                        curl.info["headers"])
                    self._multi.add_handle(curl)

                if not started:
                    break

    def initHandle(self):
        """ sets common options to curl handle """

        self.c.setopt(pycurl.FOLLOWLOCATION, 1)
        self.c.setopt(pycurl.MAXREDIRS, 5)
        self.c.setopt(pycurl.CONNECTTIMEOUT, 30)
        self.c.setopt(pycurl.NOSIGNAL, 1)
        self.c.setopt(pycurl.NOPROGRESS, 1)
        if hasattr(pycurl, "AUTOREFERER"):
            self.c.setopt(pycurl.AUTOREFERER, 1)
        self.c.setopt(pycurl.SSL_VERIFYPEER, 0)
        # Interval for low speed, detects connection loss, but can abort dl if hoster stalls the download
        self.c.setopt(pycurl.LOW_SPEED_TIME, 45)
        self.c.setopt(pycurl.LOW_SPEED_LIMIT, 5)

        # don't save the cookies
        self.c.setopt(pycurl.COOKIEFILE, "")
        self.c.setopt(pycurl.COOKIEJAR, "")

        #self.c.setopt(pycurl.VERBOSE, 1)

        self.c.setopt(pycurl.USERAGENT,
                      "Mozilla/5.0 (Windows NT 6.1; Win64; x64;en; rv:5.0) Gecko/20110619 Firefox/5.0")
        if pycurl.version_info()[7]:
            self.c.setopt(pycurl.ENCODING, "gzip, deflate")
        self.c.setopt(pycurl.HTTPHEADER, ["Accept: */*",
                                          "Accept-Language: en-US,en",
                                          "Accept-Charset: ISO-8859-1,utf-8;q=0.7,*;q=0.7",
                                          "Connection: keep-alive",
                                          "Keep-Alive: 300",
                                          "Expect:"])

def _curl_request(curl, timeout=30, redirect=True, verbose=False):
    curl.setopt(pycurl.FOLLOWLOCATION, int(redirect))
    curl.setopt(pycurl.MAXREDIRS, 5)

    if timeout:
        curl.setopt(pycurl.CONNECTTIMEOUT, timeout)

    curl.setopt(pycurl.NOSIGNAL, 1)
    curl.setopt(pycurl.NOPROGRESS, 1)

    if hasattr(pycurl, 'AUTOREFERER'):
        curl.setopt(pycurl.AUTOREFERER, 1)

    curl.setopt(pycurl.SSL_VERIFYPEER, 0)

    curl.setopt(pycurl.LOW_SPEED_TIME, 30)
    curl.setopt(pycurl.LOW_SPEED_LIMIT, 5)

    if verbose:
        curl.setopt(pycurl.VERBOSE, 1)

    curl.setopt(pycurl.USERAGENT, user_agent)

    if pycurl.version_info()[7]:
        curl.setopt(pycurl.ENCODING, 'gzip, deflate')

def is_libcurl_compiled_with_async_dns_resolver():
    """Per this (http://tornado.readthedocs.org/en/latest/httpclient.html),
    if you've configured Tornado to use async curl_httpclient, you'll want
    to make sure that libcurl has been compiled with async DNS resolver.
    The programmatic approach to checking for libcurl being compiled
    with async DNS resolve is a mess of gory details. It was this mess
    that drove the need for this function. Specifically, this function implements
    all the gory details so the caller doesn't have to worry about them!

    This function is intended to be used in an application's mainline
    in the following manner:

        #!/usr/bin/env python

        import logging

        from tor_async_util import is_libcurl_compiled_with_async_dns_resolver

        _logger = logging.getLogger(__name__)

        if __name__ == "__main__":

            if not is_libcurl_compiled_with_async_dns_resolver():
                msg = (
                    "libcurl does not appear to have been "
                    "compiled with aysnc dns resolve which "
                    "may result in timeouts on async requests"
                )
                _logger.warning(msg)

    If you really want to understand the details start with the following
    article:

        http://stackoverflow.com/questions/25998063/how-can-i-tell-if-the-libcurl-installed-has-asynchronous-dns-enabled

    Other references that you'll find useful on your question for understanding:

        http://curl.haxx.se/libcurl/
        https://github.com/bagder/curl/blob/master/include/curl/curl.h#L2286

    If you don't care about the implementation details just know that
    this function returns True if libcurl has been compiled with async DNS
    resolver otherwise this functions returns False.
    """
    try:
        version_info = pycurl.version_info()
        features = version_info[4]
        # to understand CURL_VERSION_ASYNCHDNS see
        # https://github.com/bagder/curl/blob/master/include/curl/curl.h#L2286
        CURL_VERSION_ASYNCHDNS = 1 << 7
        return (features & CURL_VERSION_ASYNCHDNS) == CURL_VERSION_ASYNCHDNS
    except Exception as ex:
        fmt = (
            "Error trying to figure out if libcurl is complied with "
            "async DNS resolver - %s"
        )
        msg = fmt % ex
        _logger.debug(msg)
        return False

    def decorated(*args, **kwargs):
        # easier to check that pycurl supports https, although
        # theoretically it is not the same test.
        # pycurl.version_info()[8] is a tuple of protocols supported by libcurl
        if "https" not in pycurl.version_info()[8]:
            raise nose.plugins.skip.SkipTest("libcurl does not support ssl")

        return fn(*args, **kwargs)

 def __init__(self, url):
     self.host = url
     self.resource = ''
     self.caller = pycurl.Curl()
     self.validateHostName(url)
     header = 'PopDB API/1.0 (CMS) %s/%s %s/%s (%s)' % (pycurl.__name__, pycurl.version_info()[1], platform.system(), platform.release(), platform.processor())
     self.caller.setopt(pycurl.HTTPHEADER, ['User-agent: %s' % header])
     #self.caller.setopt(self.caller.URL, url)
     self.caller.setopt(self.caller.VERBOSE, True)
     self.caller.setopt(self.caller.SSL_VERIFYPEER, 0)

 def __init__(self, base, _from_transport=None):
     super(PyCurlTransport, self).__init__(base,
                                           _from_transport=_from_transport)
     if base.startswith('https'):
         # Check availability of https into pycurl supported
         # protocols
         supported = pycurl.version_info()[8]
         if 'https' not in supported:
             raise errors.DependencyNotPresent('pycurl', 'no https support')
     self.cabundle = ca_bundle.get_ca_path()

 def _queryTier0DataSvc( self, url ):
     """
     Queries Tier0DataSvc.
     url: Tier0DataSvc URL.
     @returns: dictionary, from whence the required information must be retrieved according to the API call.
     Raises if connection error, bad response, or timeout after retries occur.
     """
     cHandle = pycurl.Curl()
     cHandle.setopt( cHandle.SSL_VERIFYPEER, 0 )
     cHandle.setopt( cHandle.SSL_VERIFYHOST, 0 )
     cHandle.setopt( cHandle.URL, url )
     cHandle.setopt( cHandle.HTTPHEADER, [ "User-Agent: ConditionWebServices/1.0 python/%d.%d.%d PycURL/%s" %
                                           ( sys.version_info[ :3 ] + ( pycurl.version_info()[ 1 ], ) )
                                         , ] )
     cHandle.setopt( cHandle.TIMEOUT, self._timeOut )
     if self._proxy:
         cHandle.setopt( cHandle.PROXY, self._proxy )
     if self._debug:
         cHandle.setopt( cHandle.VERBOSE, 1 )
     retry = 0
     while retry < self._retries:
         try:
             jsonCall = cStringIO.StringIO()
             cHandle.setopt( cHandle.WRITEFUNCTION, jsonCall.write )
             cHandle.perform()
             if cHandle.getinfo( cHandle.RESPONSE_CODE ) != 200:
                 _raise_http_error( cHandle, jsonCall.getvalue(), self._proxy )
             return json.loads( jsonCall.getvalue().replace("'", '"') )
         except pycurl.error as pyCURLerror:
             errorCode, errorMessage = pyCURLerror
             if self._debug:
                 errStr = """Unable to establish connection to Tier0DataSvc from URL \"%s\"""" %( url, )
                 if self._proxy:
                     errStr += """ using proxy \"%s\"""" %( str( self._proxy ), )
                 errStr += """ with timeout \"%d\".\nThe reason is: \"%s\" (error code \"%d\").""" %( self._timeOut, errorMessage, errorCode )
                 logging.error("pycurl.error: %s", errStr)
             retry += 1
             if retry < self._retries: # no sleep in last iteration
                 time.sleep( self._retryPeriod )
         except ResponseError as r:
             if self._debug:
                 logging.error("ResponseError: %s", r)
             retry += 1
             if retry < self._retries: # no sleep in last iteration
                 time.sleep( self._retryPeriod )
         finally:
             jsonCall.close()
     errStr = """Unable to get Tier0DataSvc data from URL \"%s\"""" %( url, )
     if self._proxy:
         errStr += """ using proxy \"%s\"""" %( str( self._proxy ), )
     errStr += """ with timeout \"%d\" since maximum number of retries \"%d\" with retry period \"%d\" was reached.""" % ( self._timeOut, self._retries, self._retryPeriod )
     raise Tier0Error( errStr )

def _useragent():
    vi = pycurl.version_info()
    ua = "pycurl_wrapper: libcurl/%s %s %s" % (vi[1], vi[5], vi[3])
    try:
        apt_ua = file("/etc/apt/apt.conf.d/01turnkey").read()
        m = re.search(r" \((.*?)\)", apt_ua)
        if m:
            ua += " (%s)" % m.groups(1)

    except:
        pass

    return ua

 def test_socket_open_bad(self):
     self.curl.setopt(pycurl.OPENSOCKETFUNCTION, socket_open_bad)
     self.curl.setopt(self.curl.URL, 'http://%s:8380/success' % localhost)
     try:
         self.curl.perform()
     except pycurl.error as e:
         # libcurl 7.38.0 for some reason fails with a timeout
         # (and spends 5 minutes on this test)
         if pycurl.version_info()[1].split('.') == ['7', '38', '0']:
             self.assertEqual(pycurl.E_OPERATION_TIMEDOUT, e.args[0])
         else:
             self.assertEqual(pycurl.E_COULDNT_CONNECT, e.args[0])
     else:
         self.fail('Should have raised')

        def decorated(*args, **kwargs):
            # easier to check that pycurl supports https, although
            # theoretically it is not the same test.
            # pycurl.version_info()[8] is a tuple of protocols supported by libcurl
            if 'https' not in pycurl.version_info()[8]:
                raise nose.plugins.skip.SkipTest('libcurl does not support ssl')

            # XXX move to pycurl library
            if 'OpenSSL/' in pycurl.version:
                current_backend = 'openssl'
            elif 'GnuTLS/' in pycurl.version:
                current_backend = 'gnutls'
            elif 'NSS/' in pycurl.version:
                current_backend = 'nss'
            else:
                current_backend = 'none'
            if current_backend not in backends:
                raise nose.plugins.skip.SkipTest('SSL backend is %s' % current_backend)

            return fn(*args, **kwargs)

    def init_handle(self):
        """
        sets common options to curl handle.
        """
        self.c.setopt(pycurl.FOLLOWLOCATION, 1)
        self.c.setopt(pycurl.MAXREDIRS, 10)
        self.c.setopt(pycurl.CONNECTTIMEOUT, 30)
        self.c.setopt(pycurl.NOSIGNAL, 1)
        self.c.setopt(pycurl.NOPROGRESS, 1)
        if hasattr(pycurl, "AUTOREFERER"):
            self.c.setopt(pycurl.AUTOREFERER, 1)
        self.c.setopt(pycurl.SSL_VERIFYPEER, 0)
        self.c.setopt(pycurl.LOW_SPEED_TIME, 60)
        self.c.setopt(pycurl.LOW_SPEED_LIMIT, 5)
        if hasattr(pycurl, "USE_SSL"):
            self.c.setopt(pycurl.USE_SSL, pycurl.USESSL_TRY)

        # self.c.setopt(pycurl.VERBOSE, 1)

        self.c.setopt(
            pycurl.USERAGENT,
            b"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.113 Safari/537.36",
        )
        if pycurl.version_info()[7]:
            self.c.setopt(pycurl.ENCODING, b"gzip, deflate")
        self.c.setopt(
            pycurl.HTTPHEADER,
            [
                b"Accept: */*",
                b"Accept-Language: en-US,en",
                b"Accept-Charset: ISO-8859-1,utf-8;q=0.7,*;q=0.7",
                b"Connection: keep-alive",
                b"Keep-Alive: 300",
                b"Expect:",
            ],
        )

    def _queryTier0DataSvc( self, url ):
        """
        Queries Tier0DataSvc.
        url: Tier0DataSvc URL.
        @returns: dictionary, from whence the required information must be retrieved according to the API call.
        Raises if connection error, bad response, or timeout after retries occur.
        """

        userAgent = "User-Agent: DQMIntegration/2.0 python/%d.%d.%d PycURL/%s" % ( sys.version_info[ :3 ] + ( pycurl.version_info()[ 1 ], ) )

        proxy = ""
        if self._proxy: proxy = ' --proxy %s ' % self._proxy
        
        debug = " -s -S "
        if self._debug: debug = " -v "
        
        cmd = '/usr/bin/curl -k -L --user-agent "%s" %s --connect-timeout %i --retry %i %s %s ' % (userAgent, proxy, self._timeOut, self._retries, debug, url)

        process = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
        (stdoutdata, stderrdata) =  process.communicate()
        retcode = process.returncode

        if retcode != 0 or stderrdata:
           msg = "looks like curl returned an error: retcode=%s" % (retcode,)
           msg += ' msg = "'+str(stderrdata)+'"'
           raise Tier0Error(msg)

        return json.loads( ''.join(stdoutdata).replace( "'", '"').replace(' None', ' "None"') )

import asyncurl
import utils
import version
import cStringIO

WEBVID_USER_AGENT = 'libwebvi/%s %s' % (version.VERSION, pycurl.version)
MOZILLA_USER_AGENT = 'Mozilla/5.0 (X11; U; Linux i686 (x86_64); en-US; rv:1.9.1.5) Gecko/20091102 Firefox/3.5.5'

try:
    from libmimms import libmms
except ImportError, exc:
    pass

# Mapping from curl error codes to webvi errors. The error constants
# are defined only in pycurl 7.16.1 and newer.
if pycurl.version_info()[2] >= 0x071001:
    CURL_ERROR_CODE_MAPPING = \
             {pycurl.E_OK:                     0,
              pycurl.E_OPERATION_TIMEOUTED:    408,
              pycurl.E_OUT_OF_MEMORY:          500,
              pycurl.E_PARTIAL_FILE:           504,
              pycurl.E_READ_ERROR:             504,
              pycurl.E_RECV_ERROR:             504,
              pycurl.E_REMOTE_FILE_NOT_FOUND:  404,
              pycurl.E_TOO_MANY_REDIRECTS:     404,
              pycurl.E_UNSUPPORTED_PROTOCOL:   500,
              pycurl.E_URL_MALFORMAT:          400,
              pycurl.E_COULDNT_CONNECT:        506,
              pycurl.E_COULDNT_RESOLVE_HOST:   506,
              pycurl.E_COULDNT_RESOLVE_PROXY:  506,
              pycurl.E_FILE_COULDNT_READ_FILE: 404,

#!/usr/bin/env python

import pycurl

print(pycurl.version_info())