def test_extract_macaroons_from_request(self):
        def encode_macaroon(m):
            macaroons = '[' + utils.macaroon_to_json_string(m) + ']'
            return base64.urlsafe_b64encode(utils.to_bytes(macaroons)).decode('ascii')

        req = Request('http://example.com')
        m1 = pymacaroons.Macaroon(version=pymacaroons.MACAROON_V2, identifier='one')
        req.add_header('Macaroons', encode_macaroon(m1))
        m2 = pymacaroons.Macaroon(version=pymacaroons.MACAROON_V2, identifier='two')
        jar = requests.cookies.RequestsCookieJar()
        jar.set_cookie(utils.cookie(
            name='macaroon-auth',
            value=encode_macaroon(m2),
            url='http://example.com',
        ))
        jar.set_cookie(utils.cookie(
            name='macaroon-empty',
            value='',
            url='http://example.com',
        ))
        jar.add_cookie_header(req)

        macaroons = httpbakery.extract_macaroons(req)
        self.assertEquals(len(macaroons), 2)
        macaroons.sort(key=lambda ms: ms[0].identifier)
        self.assertEquals(macaroons[0][0].identifier, m1.identifier)
        self.assertEquals(macaroons[1][0].identifier, m2.identifier)

def download(baseurl, parameters={}, headers={}):
    """Download Data from an url and returns it as a String
    @param baseurl Url to download from (e.g. http://www.google.com)
    @param parameters Parameter dict to be encoded with url
    @param headers Headers dict to pass with Request
    @returns String of data from URL
    """
    url = "?".join([baseurl, urlencode(parameters)])
    log.debug("Downloading: " + url)
    data = ""
    for _ in range(MAX_RETRIES):
        try:
            req = Request(url, headers=headers)
            req.add_header(USER_AGENT, USER_AGENT_STRING)
            response = urlopen(req)
            if six.PY2:
                data = response.read()
            else:
                data = response.read().decode("utf-8")
            response.close()
            break
        except Exception as err:
            if not isinstance(err, URLError):
                log.debug("Error %s during HTTP Request, abort", repr(err))
                raise  # propagate non-URLError
            log.debug("Error %s during HTTP Request, retrying", repr(err))
    else:
        raise
    return data

    def _pd_api(self, url, data=None, method='GET'):
        url = '%s/%s' % (PD_API_BASE, url)
        request_args = {
            'headers': dict(self._pd_headers)
        }
        if six.PY3:  # pragma: no cover
            request_args['method'] = method

        if data is not None:
            request_args['data'] = json.dumps(data).encode('utf-8')
            request_args['headers']['Content-Type'] = APPLICATION_JSON

        request = Request(url, **request_args)
        if six.PY2:  # pragma: no cover
            request.get_method = lambda: method

        try:
            response = urlopen(request)
            return json.loads(response.read().decode('utf-8'))
        except HTTPError as e:
            response = e.read().decode('utf-8')
            logger.warning("API error: %s", response)
            if method == 'GET' and e.code == 404:
                return None
            else:
                raise e

 def _query_api(self, clip):
     # Adapted from SpeechRecognition source code, modified to get text onsets
     flac_data = clip.get_flac_data(
         convert_rate = None if clip.sample_rate >= 16000 else 16000,
         convert_width = None if clip.sample_width >= 2 else 2
     )
     model = "{0}_BroadbandModel".format("en-US")
     url = "https://stream.watsonplatform.net/speech-to-text/api/v1/recognize?{0}".format(urlencode({
         "profanity_filter": "false",
         "continuous": "true",
         "model": model,
         "timestamps": "true",
     }))
     request = Request(url, data = flac_data, headers = {
         "Content-Type": "audio/x-flac",
         "X-Watson-Learning-Opt-Out": "true",
     })
     
     if hasattr("", "encode"): # Python 2.6 compatibility
         authorization_value = base64.standard_b64encode("{0}:{1}".format(self.username, self.password).encode("utf-8")).decode("utf-8")
     else:
         authorization_value = base64.standard_b64encode("{0}:{1}".format(self.username, self.password))
     request.add_header("Authorization", "Basic {0}".format(authorization_value))
     
     try:
         response = urlopen(request, timeout=None)
     except HTTPError as e:
         raise Exception("recognition request failed: {0}".format(getattr(e, "reason", "status {0}".format(e.code))))
     except URLError as e:
         raise Exception("recognition connection failed: {0}".format(e.reason))
     
     response_text = response.read().decode("utf-8")
     result = json.loads(response_text)
     return result

    def _notify_emby(self, message, host=None, emby_apikey=None):
        """Handles notifying Emby host via HTTP API

        Returns:
            Returns True for no issue or False if there was an error

        """

        # fill in omitted parameters
        if not host:
            host = sickbeard.EMBY_HOST
        if not emby_apikey:
            emby_apikey = sickbeard.EMBY_APIKEY

        url = 'http://%s/emby/Notifications/Admin' % host
        values = {'Name': 'Medusa', 'Description': message, 'ImageUrl': sickbeard.LOGO_URL}
        data = json.dumps(values)
        try:
            req = Request(url, data)
            req.add_header('X-MediaBrowser-Token', emby_apikey)
            req.add_header('Content-Type', 'application/json')

            response = urlopen(req)
            result = response.read()
            response.close()

            logger.log(u'EMBY: HTTP response: ' + result.replace('\n', ''), logger.DEBUG)
            return True

        except (URLError, IOError) as e:
            logger.log(u'EMBY: Warning: Couldn\'t contact Emby at ' + url + ' ' + ex(e), logger.WARNING)
            return False

def set_genomespace_format_identifiers( url_opener, dm_site ):
    gs_request = Request( "%s/%s/dataformat/list" % ( dm_site, GENOMESPACE_API_VERSION_STRING ) )
    gs_request.get_method = lambda: 'GET'
    opened_gs_request = url_opener.open( gs_request )
    genomespace_formats = json.loads( opened_gs_request.read() )
    for format in genomespace_formats:
        GENOMESPACE_FORMAT_IDENTIFIER_TO_GENOMESPACE_EXT[ format['url'] ] = format['name']

    def download_from_repository(self, repo_source, target):
        """
        Download given source file from the repository and store
        it as target file

        The repo_source location is used relative to the repository
        location and will be part of a mime type source like:
        file://repo_path/repo_source

        :param string source: source file in the repo
        :param string target: file path
        """
        try:
            request = Request(
                os.sep.join([self._get_mime_typed_uri(), repo_source])
            )
            if self.user and self.secret:
                credentials = b64encode(
                    format(':'.join([self.user, self.secret])).encode()
                )
                request.add_header(
                    'Authorization', b'Basic ' + credentials
                )
            location = urlopen(request)
        except Exception as e:
            raise KiwiUriOpenError(
                '{0}: {1}'.format(type(e).__name__, format(e))
            )
        with open(target, 'wb') as target_file:
            target_file.write(location.read())

def get_genome_space_launch_apps( atm_url, url_opener, file_url, file_type ):
    gs_request = Request( "%s/%s/webtool/descriptor" % ( atm_url, GENOMESPACE_API_VERSION_STRING ) )
    gs_request.get_method = lambda: 'GET'
    opened_gs_request = url_opener.open( gs_request )
    webtool_descriptors = json.loads( opened_gs_request.read() )
    webtools = []
    for webtool in webtool_descriptors:
        webtool_name = webtool.get( 'name' )
        base_url = webtool.get( 'baseUrl' )
        use_tool = False
        for param in webtool.get( 'fileParameters', [] ):
            for format in param.get( 'formats', [] ):
                if format.get( 'name' ) == file_type:
                    use_tool = True
                    break
            if use_tool:
                file_param_name = param.get( 'name' )
                # file_name_delimiters = param.get( 'nameDelimiters' )
                if '?' in base_url:
                    url_delimiter = "&"
                else:
                    url_delimiter = "?"
                launch_url = "%s%s%s" % ( base_url, url_delimiter, urlencode( [ ( file_param_name, file_url ) ] ) )
                webtools.append( ( launch_url, webtool_name ) )
                break
    return webtools

    def get_version(self):
        """Get the version of this Master.

        :returns: This master's version number ``str``

        Example::

            >>> j = Jenkins()
            >>> info = j.get_version()
            >>> print info
            >>> 1.541

        """
        try:
            request = Request(self.server + "/login")
            request.add_header('X-Jenkins', '0.0')
            response = urlopen(request, timeout=self.timeout)
            if response is None:
                raise EmptyResponseException(
                    "Error communicating with server[%s]: "
                    "empty response" % self.server)

            if six.PY2:
                return response.info().getheader('X-Jenkins')

            if six.PY3:
                return response.getheader('X-Jenkins')

        except (HTTPError, BadStatusLine):
            raise BadHTTPException("Error communicating with server[%s]"
                                   % self.server)

    def _request(self, url, data=None, headers=None, checker=None, method=None):
        if not headers:
            headers = {}
        if self.token:
            headers["X-API-Key"] = self.token
        self.log.debug("Request: %s %s %s", method if method else 'GET', url,
                       data[:self.logger_limit] if data else None)
        # .encode("utf-8") is probably better
        data = data.encode() if isinstance(data, six.text_type) else data
        request = Request(url, data, headers)
        if method:
            request.get_method = lambda: method

        response = urlopen(request, timeout=self.timeout)

        if checker:
            checker(response)

        resp = response.read()

        if not isinstance(resp, str):
            resp = resp.decode()

        self.log.debug("Response: %s", resp[:self.logger_limit] if resp else None)
        return json.loads(resp) if len(resp) else {}

def _request_with_auth(url, username, password):
    request = Request(url)
    base64string = base64.b64encode(
        username.encode('ascii') + b':' + password.encode('ascii')
    )
    request.add_header(b"Authorization", b"Basic " + base64string)
    return urlopen(request)

    def _send_to_kodi(command, host=None, username=None, password=None, dest_app="KODI"):  # pylint: disable=too-many-arguments
        """Handles communication to KODI servers via HTTP API

        Args:
            command: Dictionary of field/data pairs, encoded via urllib and passed to the KODI API via HTTP
            host: KODI webserver host:port
            username: KODI webserver username
            password: KODI webserver password

        Returns:
            Returns response.result for successful commands or False if there was an error

        """

        # fill in omitted parameters
        if not username:
            username = sickbeard.KODI_USERNAME
        if not password:
            password = sickbeard.KODI_PASSWORD

        if not host:
            logger.log(u'No %s host passed, aborting update' % dest_app, logger.WARNING)
            return False

        for key in command:
            if isinstance(command[key], text_type):
                command[key] = command[key].encode('utf-8')

        enc_command = urlencode(command)
        logger.log(u"%s encoded API command: %r" % (dest_app, enc_command), logger.DEBUG)

        # url = 'http://%s/xbmcCmds/xbmcHttp/?%s' % (host, enc_command)  # maybe need for old plex?
        url = 'http://%s/kodiCmds/kodiHttp/?%s' % (host, enc_command)
        try:
            req = Request(url)
            # if we have a password, use authentication
            if password:
                base64string = base64.encodestring('%s:%s' % (username, password))[:-1]
                authheader = "Basic %s" % base64string
                req.add_header("Authorization", authheader)
                logger.log(u"Contacting %s (with auth header) via url: %s" % (dest_app, ss(url)), logger.DEBUG)
            else:
                logger.log(u"Contacting %s via url: %s" % (dest_app, ss(url)), logger.DEBUG)

            try:
                response = urlopen(req)
            except (BadStatusLine, URLError) as e:
                logger.log(u"Couldn't contact %s HTTP at %r : %r" % (dest_app, url, ex(e)), logger.DEBUG)
                return False

            result = response.read().decode(sickbeard.SYS_ENCODING)
            response.close()

            logger.log(u"%s HTTP response: %s" % (dest_app, result.replace('\n', '')), logger.DEBUG)
            return result

        except Exception as e:
            logger.log(u"Couldn't contact %s HTTP at %r : %r" % (dest_app, url, ex(e)), logger.DEBUG)
            return False

    def __init__(self, url, body=b'', headers={}, method='PUT'):
        normalized_headers = {
            str(key): str(value)
            for key, value in six.iteritems(headers)
        }

        URLRequest.__init__(self, str(url), body, normalized_headers)
        self.method = str(method)

def http_request(method, url, request=None, timeout=30):
    """Perform HTTP request"""
    if method == 'POST':
        return http_post(url, request, timeout=timeout)
    else:  # GET
        request = Request(url)
        request.add_header('User-Agent', 'pycsw (http://pycsw.org/)')
        return urlopen(request, timeout=timeout).read()

 def post_soap(self, url, xml, soapaction=None):
     url = self.opener.relative(url)
     request = Request(url, etree.tostring(soap_body(xml)))
     request.add_header('Content-type', 'text/xml; charset=utf-8')
     if soapaction:
         request.add_header('Soapaction', soapaction)
     response = self.opener.open(request, timeout=self.timeout)
     return etree.parse(response).xpath('/soap:Envelope/soap:Body/*', namespaces=namespaces)[0]

def __del(api_key, url, data):
    """
    Do the actual DELETE
    """
    url = make_url(api_key, url)
    req = Request(url, headers={'Content-Type': 'application/json'}, data=json.dumps(data))
    req.get_method = lambda: 'DELETE'
    return json.loads(urlopen(req).read())

    def execute(cls, uri, http_verb, extra_headers=None, batch=False, body=None, **kw):
        """
        if batch == False, execute a command with the given parameters and
        return the response JSON.
        If batch == True, return the dictionary that would be used in a batch
        command.
        """
        if batch:
            ret = {"method": http_verb, "path": uri.split("parse.com", 1)[1]}
            if kw:
                ret["body"] = kw
            return ret

        if not ('app_id' in ACCESS_KEYS and 'rest_key' in ACCESS_KEYS):
            raise core.ParseError('Missing connection credentials')

        app_id = ACCESS_KEYS.get('app_id')
        rest_key = ACCESS_KEYS.get('rest_key')
        master_key = ACCESS_KEYS.get('master_key')

        url = uri if uri.startswith(API_ROOT) else cls.ENDPOINT_ROOT + uri
        if body is None:
            data = kw and json.dumps(kw, default=date_handler) or "{}"
        else:
            data = body
        if http_verb == 'GET' and data:
            url += '?%s' % urlencode(kw)
            data = None
        else:
            data = data.encode('utf-8')

        headers = {
            'Content-type': 'application/json',
            'X-Parse-Application-Id': app_id,
            'X-Parse-REST-API-Key': rest_key
        }
        headers.update(extra_headers or {})

        request = Request(url, data, headers)

        if master_key and 'X-Parse-Session-Token' not in headers.keys():
            request.add_header('X-Parse-Master-Key', master_key)

        request.get_method = lambda: http_verb

        try:
            response = urlopen(request, timeout=CONNECTION_TIMEOUT)
        except HTTPError as e:
            exc = {
                400: core.ResourceRequestBadRequest,
                401: core.ResourceRequestLoginRequired,
                403: core.ResourceRequestForbidden,
                404: core.ResourceRequestNotFound
                }.get(e.code, core.ParseError)
            raise exc(e.read())

        return json.loads(response.read().decode('utf-8'))

 def __init__(self, **kwargs):
     self.site_url = kwargs.get('site_url')
     self.site_domain = Request(self.site_url).get_host()
     self.src_url = kwargs.get('src_url')
     self.src_domain = Request(self.src_url).get_host()
     self.p = kwargs.get('p')
     self.replace_dict = {}
     self.total_count = 0
     self.broken_links = {}

def test_filter_json_post_data(tmpdir):
    data = json.dumps({"id": "secret", "foo": "bar"}).encode("utf-8")
    request = Request("http://httpbin.org/post", data=data)
    request.add_header("Content-Type", "application/json")

    cass_file = str(tmpdir.join("filter_jpd.yaml"))
    with vcr.use_cassette(cass_file, filter_post_data_parameters=["id"]):
        urlopen(request)
    with vcr.use_cassette(cass_file, filter_post_data_parameters=["id"]) as cass:
        assert b'"id": "secret"' not in cass.requests[0].body

def create_directory( url_opener, directory_dict, new_dir, dm_url ):
    payload = { "isDirectory": True }
    for dir_slice in new_dir:
        if dir_slice in ( '', '/', None ):
            continue
        url = '/'.join( ( directory_dict['url'], quote( dir_slice.replace( '/', '_' ), safe='' ) ) )
        new_dir_request = Request( url, headers={ 'Content-Type': 'application/json', 'Accept': 'application/json' }, data=json.dumps( payload ) )
        new_dir_request.get_method = lambda: 'PUT'
        directory_dict = json.loads( url_opener.open( new_dir_request ).read() )
    return directory_dict