def url_distance(preprocessor, url1, url2):
    url1 = urlparse(url1)
    url2 = urlparse(url2)

    process_fn = lambda s: preprocessor(unquote(s))
    path1 = map(process_fn, url1.path.strip('/').split('/'))
    path2 = map(process_fn, url2.path.strip('/').split('/'))
    path_distance = levenshtein_array(path1, path2)

    query_distance = dict_distance(preprocessor,
        parse_qs(url1.query, True),
        parse_qs(url2.query, True)
    )

    domain_distance = 4 * levenshtein_array(
        (url1.hostname or '').split('.'),
        (url2.hostname or '').split('.')
    )

    return (
        domain_distance +
        path_distance +
        query_distance +
        (url1.fragment != url2.fragment)
    )

def assert_url_equal(url, expected, compare_host=False):
    """Compare url paths and query strings."""
    parsed = urlparse(six.text_type(url))
    parsed_expected = urlparse(six.text_type(expected))
    compare_url_part(parsed.path, parsed_expected.path)
    compare_url_part(parse_qs(parsed.query), parse_qs(parsed_expected.query))
    if compare_host:
        compare_url_part(parsed.netloc, parsed_expected.netloc)

 def _build_url(self):
     url = list(urlsplit(self._path))
     qs = parse_qs(url[3])
     qs["count"] = self._count
     qs["page"] = self._page
     url[3] = urlencode(qs, doseq=True)
     return urlunsplit(url)

    def on_request(self, env, start_response):
        method = env['REQUEST_METHOD'].upper()
        path = env['PATH_INFO'].lower().strip('/')

        # If the API request is proxied, the path comes as a full url for some reason
        if '://' in path:
            path = urlparse.urlparse(path).path.strip('/')

        if method not in self.method_handlers or path not in self.method_handlers[method]:
            start_response('404 Not Found', [])
            return

        if method == 'GET':
            request = urlparse.parse_qs(env.get('QUERY_STRING', ''), True)
            for query in request:
                if len(request[query]) == 1:
                    request[query] = request[query][0]
        else:
            inp = env.get('wsgi.input', None)
            request = json.loads(inp.read()) if inp is not None else {}

        response = self.method_handlers[method][path](request)

        if type(response) == tuple:
            response, options = response
        else:
            options = ResponseOptions()

        start_response('%d %s' % (options.status, options.reason), [('Content-Type', options.content_type)])

        if options.send_json:
            return [json.dumps(response)]
        else:
            return [response]

def depaginate(api, result):
    """Depaginate the first (or only) page of a paginated result"""
    meta = result['meta']
    if meta['next'] is None:
        # No pages means we're done
        return result

    # make a copy of meta that we'll mess with and eventually return
    # since we'll be chewing on the 'meta' object with every new GET
    # same thing for objects, since we'll just be appending to it
    # while we pull more records
    ret_meta = meta.copy()
    ret_objects = result['objects']
    while meta['next']:
        # parse out url bits for constructing the new api req
        next_url = urlparse(meta['next'])
        # ugh...need to find the word after 'api/' in the next URL to
        # get the resource endpoint name; not sure how to make this better
        next_endpoint = next_url.path.strip('/').split('/')[-1]
        next_params = {k: v[0] for k, v in parse_qs(next_url.query).items()}
        result = getattr(api, next_endpoint).get(**next_params)
        ret_objects.extend(result['objects'])
        meta = result['meta']

    # fix meta up to not tell lies
    ret_meta['total_count'] = len(ret_objects)
    ret_meta['next'] = None
    ret_meta['limit'] = ret_meta['total_count']
    return {
        'meta': ret_meta,
        'objects': ret_objects
    }

    def _parse_season(self, row, download_url, title):
        """Parse the torrent's detail page and return the season pack title."""
        details_url = row.find('span').find_next(title='View torrent').get('href')
        torrent_id = parse_qs(download_url).get('id')
        if not all([details_url, torrent_id]):
            log.debug("Couldn't parse season pack details page for title: {0}", title)
            return title

        # Take a break before querying the provider again
        time.sleep(0.5)
        response = self.session.get(urljoin(self.url, details_url))
        if not response or not response.text:
            log.debug("Couldn't open season pack details page for title: {0}", title)
            return title

        with BS4Parser(response.text, 'html5lib') as html:
            torrent_table = html.find('table', class_='torrent_table')
            torrent_row = torrent_table.find('tr', id='torrent_{0}'.format(torrent_id[0]))
            if not torrent_row:
                log.debug("Couldn't find season pack details for title: {0}", title)
                return title

            # Strip leading and trailing slash
            season_title = torrent_row.find('div', class_='filelist_path')
            if not season_title or not season_title.get_text():
                log.debug("Couldn't parse season pack title for: {0}", title)
                return title
            return season_title.get_text(strip=True).strip('/')

    def _parse_request(self):
        self._buffer.seek(0)
        request_line = self._buffer.readline()  # e.g. GET /v?v='http://www.nbc.com' HTTP/1.1

        match = re.match("(GET|POST|PUT|DELETE|HEAD) (.+) .+", request_line)
        if match is None:
            raise ValueError("Invalid HTTP request %s" % request_line)

        self.method = match.group(1)  # e.g. GET

        whole_url = match.group(2)  # e.g. /?v='http://www.nbc.com'

        parsed_whole_url = urlparse.urlparse(whole_url)
        params = urlparse.parse_qs(parsed_whole_url.query)

        url_param = params.get('v', None)  # e.g. http://www.nbc.com
        if url_param is None or len(url_param) == 0:
            logger.warning("Skipping %s" % whole_url.strip())
            return
            # raise ValueError("No request url received in HTTP request")
        self.url = url_param[0]

        for line in self._buffer.readlines():
            match = re.match("(.+):\\s*(.+)", line.strip())
            if match is None:
                continue
            self.headers[match.group(1)] = match.group(2)

        return True

def test_fxa_login_url_requiring_two_factor_auth():
    path = u'/en-US/addons/abp/?source=ddg'
    request = RequestFactory().get(path)
    request.session = {'fxa_state': 'myfxastate'}

    raw_url = utils.fxa_login_url(
        config=FXA_CONFIG['default'],
        state=request.session['fxa_state'], next_path=path, action='signin',
        force_two_factor=True)

    url = urlparse(raw_url)
    base = u'{scheme}://{netloc}{path}'.format(
        scheme=url.scheme, netloc=url.netloc, path=url.path)
    assert base == 'https://accounts.firefox.com/oauth/authorization'
    query = parse_qs(url.query)
    next_path = urlsafe_b64encode(path.encode('utf-8')).rstrip(b'=')
    assert query == {
        'acr_values': ['AAL2'],
        'action': ['signin'],
        'client_id': ['foo'],
        'redirect_url': ['https://testserver/fxa'],
        'scope': ['profile'],
        'state': ['myfxastate:{next_path}'.format(
            next_path=force_text(next_path))],
    }

    def on_request(self, env, start_response):
        method = env['REQUEST_METHOD'].upper()
        path = env['PATH_INFO'].lower().strip('/')

        if method not in self.method_handlers or path not in self.method_handlers[method]:
            start_response('404 Not Found', [])
            return

        if method == 'GET':
            request = urlparse.parse_qs(env.get('QUERY_STRING', ''), True)
            for query in request:
                if len(request[query]) == 1:
                    request[query] = request[query][0]
        else:
            request = json.loads(env.get('wsgi.input', '{}'))

        response = self.method_handlers[method][path](request)

        if type(response) == tuple:
            response, options = response
        else:
            options = ResponseOptions()

        start_response('%d %s' % (options.status, options.reason), [('Content-Type', options.content_type)])

        if options.send_json:
            yield json.dumps(response)
        else:
            yield response

    def test_submit_service_result(self):
        with requests_mock.Mocker() as m:
            m.register_uri(requests_mock.POST,
                           self.ws_arbiter_url + "/push_check_result")

            check_result = {
                "return_code": "0",
                "output": "TEST OUTPUT",
                "time_stamp": "1409149234"
            }

            response = self.app.post_json(
                "/v1/hosts/bogus-router/services/service-example/results",
                params=check_result
            )

            self.assertEqual(response.status_int, 204)
            self.assertEqual(
                urllib_parse.parse_qs(m.last_request.body),
                {
                    u'output': [u'TEST OUTPUT'],
                    u'return_code': [u'0'],
                    u'service_description': [u'service-example'],
                    u'host_name': [u'bogus-router'],
                    u'time_stamp': [u'1409149234']
                }
            )

    def request(self, url, method="GET", body=None, headers={}):
        fail_state = {
            'status':'400'
        }, "{}"

        parsed = urlparse(url)
        options = parse_qs(parsed.query)

        fn_name = str(active_call)
        if fn_name == 'get_authorize_login_url':
            return {
                'status': '200',
                'content-location':'http://example.com/redirect/login'
            }, None

        if not 'access_token' in options and not 'client_id' in options:
            fn_name += '_unauthorized'
        if 'self' in url and not 'access_token' in options:
            fn_name += '_no_auth_user'

        fl = open('fixtures/%s.json' % fn_name)
        content = fl.read()
        json_content = simplejson.loads(content)
        status = json_content['meta']['code']
        fl.close()
        return {
            'status': status
        }, content

    def test_token_refresh_store_expired(self):
        expiration = datetime.datetime.utcnow() - datetime.timedelta(minutes=15)
        credentials = self._create_test_credentials(expiration=expiration)

        storage = file_module.Storage(FILENAME)
        storage.put(credentials)
        credentials = storage.get()
        new_cred = copy.copy(credentials)
        new_cred.access_token = "bar"
        storage.put(new_cred)

        access_token = "1/3w"
        token_response = {"access_token": access_token, "expires_in": 3600}
        response_content = json.dumps(token_response).encode("utf-8")
        http = http_mock.HttpMock(data=response_content)

        credentials._refresh(http)
        self.assertEquals(credentials.access_token, access_token)

        # Verify mocks.
        self.assertEqual(http.requests, 1)
        self.assertEqual(http.uri, credentials.token_uri)
        self.assertEqual(http.method, "POST")
        expected_body = {
            "grant_type": ["refresh_token"],
            "client_id": [credentials.client_id],
            "client_secret": [credentials.client_secret],
            "refresh_token": [credentials.refresh_token],
        }
        self.assertEqual(urllib_parse.parse_qs(http.body), expected_body)
        expected_headers = {"content-type": "application/x-www-form-urlencoded", "user-agent": credentials.user_agent}
        self.assertEqual(http.headers, expected_headers)

def parse_grip_uri(uri):
	parsed = urlparse(uri)
	# HACK: work around '+' character in base64-encoded values
	query = parsed.query.replace('+', '%2B')
	params = parse_qs(query)
	iss = None
	key = None
	if 'iss' in params:
		iss = params['iss'][0]
		del params['iss']
	if 'key' in params:
		key = params['key'][0]
		del params['key']
	if key is not None and key.startswith('base64:'):
		key = b64decode(key[7:])
	qs = urlencode(params, True)
	path = parsed.path
	if path.endswith('/'):
		path = path[:-1]
	control_uri = parsed.scheme + '://' + parsed.netloc + path
	if qs:
		control_uri += '?' + qs
	out = {'control_uri': control_uri}
	if iss:
		out['control_iss'] = iss
	if key:
		out['key'] = key
	return out

    def assert_url(self, expected_url, url):
        """Check if two URL are same

        Assertions are called inside this this method. If anything is
        different, it will fail immediately.

        :param str expected_url: expected URL compare.
        :param str url: the URL to check if it is same as the expected URL.
        """
        url = urlparse(url)
        expected_url = urlparse(expected_url)

        self.assertEqual(expected_url.scheme, url.scheme)
        self.assertEqual(expected_url.netloc, url.netloc)
        self.assertEqual(expected_url.path, url.path)
        self.assertEqual(parse_qs(expected_url.query), parse_qs(url.query))

    def execute(self, subscription, messages, **kwargs):
        subscriber = urllib_parse.urlparse(subscription['subscriber'])
        params = urllib_parse.parse_qs(subscriber.query)
        params = dict((k.lower(), v) for k, v in params.items())
        conf_n = kwargs.get('conf').notification
        try:
            for message in messages:
                # Send confirmation email to subscriber.
                if (message.get('Message_Type') ==
                        MessageType.SubscriptionConfirmation.name):
                    content = conf_n.subscription_confirmation_email_template
                    msg = self._make_confirmation_email(content['body'],
                                                        subscription,
                                                        message, conf_n)
                    msg["to"] = subscriber.path
                    msg["from"] = content['sender']
                    msg["subject"] = content['topic']
                elif (message.get('Message_Type') ==
                        MessageType.UnsubscribeConfirmation.name):
                    content = conf_n.unsubscribe_confirmation_email_template
                    msg = self._make_confirmation_email(content['body'],
                                                        subscription,
                                                        message, conf_n)
                    msg["to"] = subscriber.path
                    msg["from"] = content['sender']
                    msg["subject"] = content['topic']
                else:
                    # NOTE(Eva-i): Unfortunately this will add 'queue_name' key
                    # to our original messages(dicts) which will be later
                    # consumed in the storage controller. It seems safe though.
                    message['queue_name'] = subscription['source']
                    msg = text.MIMEText(json.dumps(message))
                    msg["to"] = subscriber.path
                    msg["from"] = subscription['options'].get('from', '')
                    subject_opt = subscription['options'].get('subject', '')
                    msg["subject"] = params.get('subject', subject_opt)
                if conf_n.smtp_mode == 'third_part':
                    p = subprocess.Popen(conf_n.smtp_command.split(' '),
                                         stdin=subprocess.PIPE)
                    p.communicate(msg.as_string())
                elif conf_n.smtp_mode == 'self_local':
                    sender = smtplib.SMTP_SSL(conf_n.smtp_host,
                                              conf_n.smtp_port)
                    sender.set_debuglevel(1)

                    sender.ehlo(conf_n.smtp_host)
                    try:
                        sender.login(conf_n.smtp_user_name,
                                     conf_n.smtp_user_password)
                    except smtplib.SMTPException:
                        LOG.error("Failed to connect to the SMTP service")
                        continue
                    sender.sendmail(msg['from'], msg['to'], msg.as_string())
                LOG.debug("Send mail successfully: %s", msg.as_string())
        except OSError as err:
            LOG.exception('Failed to create process for sendmail, '
                          'because %s.', str(err))
        except Exception as exc:
            LOG.exception('Failed to send email because %s.', str(exc))

def _get_params_from_url(url):
    """
    Given a URL, return a dict of parameters and their values.  If a
    parameter appears more than once all but the first value will be lost.
    """
    parsed = urlparse.urlparse(url)
    params = urlparse.parse_qs(parsed.query, keep_blank_values=True)
    return dict((key, vals[0]) for key, vals in params.iteritems())

 def test_url_simple(self):
     connection = Mock()
     connection.call.return_value = (None, {'start': 0, 'total_size': 0})
     page = Page(connection, '/some-path', None)
     built_qs = parse_qs(urlsplit(page._build_url()).query)
     self.assertEqual(built_qs, dict(
         count=[str(DEFAULT_PAGE_ITEM_COUNT)],
         page=["1"]))

def test_unicode_next_path():
    path = u'/en-US/føø/bãr'
    request = RequestFactory().get(path)
    request.session = {}
    url = utils.default_fxa_login_url(request)
    state = parse_qs(urlparse(url).query)['state'][0]
    next_path = urlsafe_b64decode(state.split(':')[1] + '===')
    assert next_path.decode('utf-8') == path

    def __call__(self, request):
        if request.method == "POST":
            data = urllib_parse.parse_qs(request.body)
            data["username"] = [self.username]
            data["password"] = [self.password]

            request.prepare_body(data, None)

        return request

 def testQueryRemapping(self):
     method_config = base_api.ApiMethodInfo(
         request_type_name="MessageWithRemappings", query_params=["remapped_field", "enum_field"]
     )
     request = MessageWithRemappings(str_field="foo", enum_field=MessageWithRemappings.AnEnum.value_one)
     http_request = FakeService().PrepareHttpRequest(method_config, request)
     result_params = urllib_parse.parse_qs(urllib_parse.urlparse(http_request.url).query)
     expected_params = {"enum_field": "ONE%2FTWO", "remapped_field": "foo"}
     self.assertTrue(expected_params, result_params)