def testLoginIsPassive(self):
        """
        Tests the login method of the OneLogin_Saml2_Auth class
        Case Logout with no parameters. A AuthN Request is built with IsPassive and redirect executed
        """
        settings_info = self.loadSettingsJSON()
        return_to = u'http://example.com/returnto'
        settings_info['idp']['singleSignOnService']['url']

        auth = OneLogin_Saml2_Auth(self.get_request(), old_settings=settings_info)
        target_url = auth.login(return_to)
        parsed_query = parse_qs(urlparse(target_url)[4])
        sso_url = settings_info['idp']['singleSignOnService']['url']
        self.assertIn(sso_url, target_url)
        self.assertIn('SAMLRequest', parsed_query)
        request = compat.to_string(OneLogin_Saml2_Utils.decode_base64_and_inflate(parsed_query['SAMLRequest'][0]))
        self.assertNotIn('IsPassive="true"', request)

        auth_2 = OneLogin_Saml2_Auth(self.get_request(), old_settings=settings_info)
        target_url_2 = auth_2.login(return_to, False, False)
        parsed_query_2 = parse_qs(urlparse(target_url_2)[4])
        self.assertIn(sso_url, target_url_2)
        self.assertIn('SAMLRequest', parsed_query_2)
        request_2 = compat.to_string(OneLogin_Saml2_Utils.decode_base64_and_inflate(parsed_query_2['SAMLRequest'][0]))
        self.assertNotIn('IsPassive="true"', request_2)

        auth_3 = OneLogin_Saml2_Auth(self.get_request(), old_settings=settings_info)
        target_url_3 = auth_3.login(return_to, False, True)
        parsed_query_3 = parse_qs(urlparse(target_url_3)[4])
        self.assertIn(sso_url, target_url_3)
        self.assertIn('SAMLRequest', parsed_query_3)
        request_3 = compat.to_string(OneLogin_Saml2_Utils.decode_base64_and_inflate(parsed_query_3['SAMLRequest'][0]))
        self.assertIn('IsPassive="true"', request_3)

    def check(self, domain):
        import requests
        from urllib.parse import urlparse

        score = 0
        reason = ""

        nakedreq = requests.get("http://%s" % domain)
        nakedreqURL = urlparse(nakedreq.url)
        wwwreq = requests.get("http://www.%s" % domain)
        wwwreqURL = urlparse(wwwreq.url)


        if nakedreqURL.netloc == domain:
            score = 0
            reason = "Naked domain (%s) does not redirect to www (but www does not redirect to naked)" % domain
            if wwwreqURL.netloc == domain:
                score = 1
                reason = "www.%s redirects to %s" % (domain, domain)
        elif nakedreqURL.netloc == "www.%s" % domain:
            score = -1
            reason = "Naked domain redirects to www."

        return {
            "score": score,
            "reason": reason
        }

 def _isswe(self, url):
     if re.search(r".se$", urlparse(url).netloc):
         return "sasong"
     elif re.search(r".dk$", urlparse(url).netloc):
         return "saeson"
     else:
         return "sesong"

def test_suomifi_logout_sp_request(django_client, django_user_model, fixed_saml_id):
    '''Suomi.fi use case #3: sending logout request'''
    oidc_client = create_oidc_client()
    user = create_user(django_user_model)
    create_social_user(user)
    create_oidc_token(user, oidc_client)
    django_client.login(username=TEST_USER, password=TEST_PASSWORD)
    args = {
        'id_token_hint': ID_TOKEN_JWT,
        'post_logout_redirect_uri': REDIRECT_URI,
    }
    logout_page_url = reverse('end-session') + '?{}'.format(urlencode(args))
    logout_page_response = django_client.get(logout_page_url)

    # Logout request results in redirect to Suomi.fi with a SAML message in
    # query parameters. The OIDC token for the user is deleted.
    assert Token.objects.count() == 0
    assert logout_page_response.status_code == 302
    suomifi_redirect = urlparse(logout_page_response.url)
    suomifi_query_params = parse_qs(suomifi_redirect.query)
    suomifi_saml_request = SAMLUtils.decode_base64_and_inflate(suomifi_query_params['SAMLRequest'][0])
    expected_slo_url = urlparse(getattr(settings, 'SOCIAL_AUTH_SUOMIFI_ENABLED_IDPS')['suomifi']['logout_url'])
    expected_logout_request = load_file('suomifi_logout_request.xml')
    expected_logout_signature = load_file('suomifi_logout_signature.b64').decode()
    assert suomifi_redirect[:3] == expected_slo_url[:3]
    assert suomifi_saml_request == expected_logout_request
    assert suomifi_query_params['RelayState'][0] == '{"cli": "test_client", "idx": 0}'
    assert suomifi_query_params['Signature'][0] == expected_logout_signature

def test_suomifi_idp_logout(django_client, fixed_saml_id):
    '''Suomi.fi use cases #4: receiving logout request, and #6: sending logout response'''
    create_oidc_client()
    args = {
        'SAMLRequest': load_file('suomifi_idp_logout_request_encoded.b64').decode(),
        'RelayState': RELAY_STATE,
        'SigAlg': 'http://www.w3.org/2001/04/xmldsig-more#rsa-sha256',
        'Signature': load_file('suomifi_idp_logout_signature.b64').decode()
    }
    callback_url = reverse('auth_backends:suomifi_logout_callback') + '?{}'.format(urlencode(args))
    callback_response = django_client.get(callback_url)

    # IdP initiated logout request results in redirect to Suomi.fi SLO URL with
    # SAML response and RelayState from request.
    assert callback_response.status_code == 302
    suomifi_redirect = urlparse(callback_response.url)
    suomifi_query_params = parse_qs(suomifi_redirect.query)
    suomifi_saml_response = SAMLUtils.decode_base64_and_inflate(suomifi_query_params['SAMLResponse'][0])
    expected_slo_url = urlparse(getattr(settings, 'SOCIAL_AUTH_SUOMIFI_ENABLED_IDPS')['suomifi']['logout_url'])
    expected_logout_response = load_file('suomifi_idp_logout_response.xml')
    expected_logout_signature = load_file('suomifi_idp_logout_response_signature.b64').decode()
    assert suomifi_redirect[:3] == expected_slo_url[:3]
    assert suomifi_saml_response == expected_logout_response
    assert suomifi_query_params['RelayState'][0] == RELAY_STATE
    assert suomifi_query_params['Signature'][0] == expected_logout_signature

    def find_local_path(self, uri, default_destination=None, netloc=False, infix=""):
        base_uri = uri
        destination = None
        path = None
        postfix = None
        local_path = None
        while destination is None:
            try:
                destination = self.mappings[base_uri]
                postfix = uri[len(base_uri) + 1:]
            except KeyError:
                if path == "":
                    break
                else:
                    (base_uri, path) = DestinationMap.shorten(base_uri)

        if destination is None:
            destination = default_destination

        if destination is None and netloc:
            destination = urlparse(uri).netloc
            l = len(urlparse(uri).scheme) + len(destination)
            postfix = uri[l + 4:]

        if destination is not None and not os.path.isabs(destination):
            destination = os.path.join(self.root_folder, destination)

        if destination is not None and postfix is not None:
            local_path = os.path.join(destination, infix, postfix)

        return base_uri, local_path

def validate_url(url: str):
    try:
        urlparse(url)
    except ValueError as ex:
        raise AssertionError(
            "Couldn't parse given value `{}` as an URL".format(url)
        ) from ex

    def match_allowed_origin(self, parsed_origin, pattern):
        """
        Returns ``True`` if the origin is either an exact match or a match
        to the wildcard pattern. Compares scheme, domain, port of origin and pattern.

        Any pattern can be begins with a scheme. After the scheme must be a domain,
        or just domain without scheme.
        Any domain beginning with a period corresponds to the domain and all
        its subdomains (for example, ``.example.com`` ``example.com``
        and any subdomain). Also with scheme (for example, ``http://.example.com``
        ``http://exapmple.com``). After the domain there must be a port,
        but it can be omitted.

        Note. This function assumes that the given origin has a schema, domain and port,
        or only domain. Port is optional.
        """
        # Get ResultParse object
        parsed_pattern = urlparse(pattern.lower(), scheme=None)
        if parsed_pattern.scheme is None:
            pattern_hostname = urlparse("//" + pattern).hostname or pattern
            return is_same_domain(parsed_origin.hostname, pattern_hostname)
        # Get origin.port or default ports for origin or None
        origin_port = self.get_origin_port(parsed_origin)
        # Get pattern.port or default ports for pattern or None
        pattern_port = self.get_origin_port(parsed_pattern)
        # Compares hostname, scheme, ports of pattern and origin
        if parsed_pattern.scheme == parsed_origin.scheme and origin_port == pattern_port \
                and is_same_domain(parsed_origin.hostname, parsed_pattern.hostname):
            return True
        return False

 def __init__(self,
              bucket_key,
              bucket_name=None,
              wildcard_match=False,
              aws_conn_id='aws_default',
              verify=None,
              *args,
              **kwargs):
     super().__init__(*args, **kwargs)
     # Parse
     if bucket_name is None:
         parsed_url = urlparse(bucket_key)
         if parsed_url.netloc == '':
             raise AirflowException('Please provide a bucket_name')
         else:
             bucket_name = parsed_url.netloc
             bucket_key = parsed_url.path.lstrip('/')
     else:
         parsed_url = urlparse(bucket_key)
         if parsed_url.scheme != '' or parsed_url.netloc != '':
             raise AirflowException('If bucket_name is provided, bucket_key' +
                                    ' should be relative path from root' +
                                    ' level, rather than a full s3:// url')
     self.bucket_name = bucket_name
     self.bucket_key = bucket_key
     self.wildcard_match = wildcard_match
     self.aws_conn_id = aws_conn_id
     self.verify = verify

def scrape(tracker, hashes):
    """
	Returns the list of seeds, peers and downloads a torrent info_hash has, according to the specified tracker

	Args:
		tracker (str): The announce url for a tracker, usually taken directly from the torrent metadata
		hashes (list): A list of torrent info_hash's to query the tracker for

	Returns:
		A dict of dicts. The key is the torrent info_hash's from the 'hashes' parameter,
		and the value is a dict containing "seeds", "peers" and "complete".
		Eg:
		{
			"2d88e693eda7edf3c1fd0c48e8b99b8fd5a820b2" : { "seeds" : "34", "peers" : "189", "complete" : "10" },
			"8929b29b83736ae650ee8152789559355275bd5c" : { "seeds" : "12", "peers" : "0", "complete" : "290" }
		}
	"""
    tracker = tracker.lower()
    parsed = urlparse(tracker)
    if parsed.scheme == "udp":
        return scrape_udp(parsed, hashes)

    if parsed.scheme in ["http", "https"]:
        if "announce" not in tracker:
            raise RuntimeError("%s doesnt support scrape" % tracker)
        parsed = urlparse(tracker.replace("announce", "scrape"))
        return scrape_http(parsed, hashes)

    raise RuntimeError("Unknown tracker scheme: %s" % parsed.scheme)

 def check_url_redirection(self, url_record):
     """ Checks redirection in URL and saves new domains, if any """
     current_url = url_record['url']
     recursive, url = self._check_redirection(current_url)
     url_record['final_url'] = url
     if recursive:
         print('    This is a recursive redirect. Action: Skipped!')
         url_record['redirected'] = True
         url_record['recursive'] = True
     else: 
         url_record['recursive'] = False
         if url == current_url:
             url_record['redirected'] = False
         else:
             url_record['redirected'] = True
             domain1 = _extract_domain(current_url)
             domain2 = _extract_domain(url)
             if urlparse(domain1).netloc == urlparse(domain2).netloc:
                 url_record['same_domain'] = True
             else:
                 url_record['same_domain'] = False
                 if domain2 not in self.new_domains:
                     # Make sure the domain is not in the blacklist
                     if urlparse(domain2).netloc not in self.blacklist:
                         # Make sure that the URL is that of a web archive snapshot
                         if '://web.archive.org/web/' in url:
                             print('    New domain found: %s' % domain2)
                             self.new_domains.append(domain2)
                 
     return url_record

def fix_content(content):
    """
    Parse article content to rewrite it for a better reader experience
    """
    parsed_content = BeautifulSoup(content, "html.parser")
    CAMO_KEY = settings.CAMO_KEY
    for img in parsed_content.find_all('img'):
        if img.get('src') and CAMO_KEY and urlparse(img['src']).scheme != 'https':
            img['src'] = get_camo_url(img['src'])

        del img['srcset']
        del img['sizes']
        img['class'] = img.get('class', []) + ['img-responsive']

    for div in parsed_content.find_all('div'):
        del div['style']

    for table in parsed_content.find_all('table'):
        table['class'] = table.get('class', []) + ['table-responsive']

    for a in parsed_content.find_all('a'):
        a['target'] = '_blank'

    for iframe in parsed_content.find_all('iframe'):
        url = urlparse(iframe['src'])
        if url.scheme != 'https' and url.netloc in SSLIFY_HOST_LIST:
            iframe['src'] = url._replace(scheme='https').geturl()

    return str(parsed_content)

def create_graphml(dists_triu, data, time_max=164, sim_min=0.8):
    size = dists_triu.shape[0]
    G = nx.DiGraph()
    G.add_node(0, step=0, date=0,domain=urlparse(data[0]['link']).netloc)
    date_init = data[0]['published']
    outs = []
    for i in range(1, size):
        pub_i = data[i]['published']
        column = list(dists_triu[:,i])
        pos = get_pos(data, pub_i, column, time_max, sim_min, outs)

        if pos != None:
            if pos not in G.nodes():
                domain_1 = urlparse(data[pos]['link']).netloc
                date_1 = create_date(date_init, data[pos]['published'], 5)
                G.add_node(pos, date=date_1, domain=domain_1)
            if i not in G.nodes():
                domain_2 = urlparse(data[i]['link']).netloc
                date_2 = create_date(date_init, pub_i, 5)
                G.add_node(i, date=date_2, domain=domain_2)

            G.add_edge(pos, i)
        else:
            outs.append(i)
    return G

def validate_url(request):
    assert request.method == "GET"
    assert request.is_ajax()
    url = request.GET.get('url')
    assert url
    try:
        URLValidator(url)
    except ValidationError:
        raise AssertionError()
    assert 'HTTP_REFERER' in request.META
    referer = urlparse(request.META.get('HTTP_REFERER'))
    toproxy = urlparse(url)
    local = urlparse(settings.SITE_URL)
    assert toproxy.hostname
    assert referer.hostname == local.hostname
    assert toproxy.hostname != "localhost"
    assert toproxy.netloc != local.netloc
    try:
        # clean this when in python 3.4
        ipaddress = socket.gethostbyname(toproxy.hostname)
    except:
        raise AssertionError()
    assert not ipaddress.startswith('127.')
    assert not ipaddress.startswith('192.168.')
    return url

def url(value):
    try:
        url_str = str(value)
        urlparse(url_str)
        return url_str
    except Exception as err:
        raise ValueError("Invalid URL `{0}`: {1}".format(value, err))

def parse(
          target, max_level, stay_local, search_string,
          level=1, parsed_links={}):
    if (level > max_level):
        return parsed_links

    print("Parsing [level %d] - %s" % (level, target))

    try:
        hostname = urlparse(target)[1]
        tree = lxml.html.parse(target)
        root = tree.getroot()

        text = ''.join(root.itertext())
        parsed_links[target] = True if search_string.search(text) else False

        root.make_links_absolute(root.base)
        for link in root.iterlinks():
            href = link[2]
            if (stay_local and urlparse(href)[1] != hostname):
                continue
            if (href not in parsed_links):
                parse(href, max_level, stay_local, search_string,
                      level + 1, parsed_links)
    except Exception as e:
        print("Couldn't parse the target {}: {!s}".format(target, e))

    return parsed_links

def relative_uri(source, target):
    """
    Make a relative URI from source to target.
    """
    su = urlparse.urlparse(source)
    tu = urlparse.urlparse(target)
    extra = list(tu[3:])
    relative = None
    if tu[0] == '' and tu[1] == '':
        if tu[2] == su[2]:
            relative = ''
        elif not tu[2].startswith('/'):
            relative = tu[2]
    elif su[0:2] != tu[0:2]:
        return target

    if relative is None:
        if tu[2] == su[2]:
            relative = ''
        else:
            relative = os.path.relpath(tu[2], os.path.dirname(su[2]))
    if relative == '.':
        relative = ''
    relative = urlparse.urlunparse(["", "", relative] + extra)
    return relative

def _is_latest(js_option, request):
    """
    Return whether we should serve latest untranspiled code.

    Set according to user's preference and URL override.
    """
    import hass_frontend

    if request is None:
        return js_option == 'latest'

    # latest in query
    if 'latest' in request.query or (
            request.headers.get('Referer') and
            'latest' in urlparse(request.headers['Referer']).query):
        return True

    # es5 in query
    if 'es5' in request.query or (
            request.headers.get('Referer') and
            'es5' in urlparse(request.headers['Referer']).query):
        return False

    # non-auto option in config
    if js_option != 'auto':
        return js_option == 'latest'

    useragent = request.headers.get('User-Agent')

    return useragent and hass_frontend.version(useragent)

    def get_referrer(self, oldurl, newurl):
        """
        Get the referrer to send when doing a request.
        If we should not send a referrer, it will return None.

        Reference: https://en.wikipedia.org/wiki/HTTP_referer

        :param oldurl: Current absolute URL
        :type oldurl: str or None

        :param newurl: Target absolute URL
        :type newurl: str

        :rtype: str or None
        """
        if oldurl is None:
            return None
        old = urlparse(oldurl)
        new = urlparse(newurl)
        # Do not leak secure URLs to insecure URLs
        if old.scheme == 'https' and new.scheme != 'https':
            return None
        # Reloading the page. Usually no referrer.
        if oldurl == newurl:
            return None
        # TODO maybe implement some *optional* privacy features:
        # * do not leak referrer to other domains (often breaks websites)
        # * send a fake referrer (root of the current domain)
        # * never send the referrer
        # Inspired by the RefControl Firefox addon.
        return oldurl

def test_spawn_redirect(app, io_loop):
    name = 'wash'
    cookies = app.login_user(name)
    u = app.users[orm.User.find(app.db, name)]
    
    # ensure wash's server isn't running:
    r = api_request(app, 'users', name, 'server', method='delete', cookies=cookies)
    r.raise_for_status()
    status = io_loop.run_sync(u.spawner.poll)
    assert status is not None
    
    # test spawn page when no server is running
    r = get_page('spawn', app, cookies=cookies)
    r.raise_for_status()
    print(urlparse(r.url))
    path = urlparse(r.url).path
    assert path == ujoin(app.base_url, 'user/%s' % name)
    
    # should have started server
    status = io_loop.run_sync(u.spawner.poll)
    assert status is None
    
    # test spawn page when server is already running (just redirect)
    r = get_page('spawn', app, cookies=cookies)
    r.raise_for_status()
    print(urlparse(r.url))
    path = urlparse(r.url).path
    assert path == ujoin(app.base_url, '/user/%s' % name)