def getProxiedURIT(uriT):
    tmurl = list(urlsplit(uriT))
    if app.proxy is not None:
        # urlsplit put domain in path for "example.com"
        tmurl[1] = app.proxy  # Set replay host/port if no scheme
        proxyuri = urlsplit(app.proxy)
        if proxyuri.scheme != '':
            tmurl[0] = proxyuri.scheme
            tmurl[1] = proxyuri.netloc + proxyuri.path

    return tmurl

def remove_user_from_uri(apps, schema_editor):
    Formula = apps.get_model('formulas', 'Formula')

    for formula in Formula.objects.all():
        url_bits = urlsplit(formula.uri)

        # don't do it if it's an ssh formula
        if 'ssh' in url_bits.scheme:
            continue

        if url_bits.username:
            formula.git_username = url_bits.username

            if url_bits.port:
                new_netloc = '{}:{}'.format(url_bits.hostname, url_bits.port)
            else:
                new_netloc = url_bits.hostname

            formula.uri = urljoin((
                url_bits.scheme,
                new_netloc,
                url_bits.path,
                url_bits.query,
                url_bits.fragment,
            ))

            formula.save()

def update_archive(thermoml_path=None):
    """Use RSS feeds to find and download any missing ThermoML XML files
    from the ThermoML archive.  
    
    Parameters
    ----------
    thermoml_path : str, optional, default=None
        If specified, use this path to store ThermoML XML files.  If None,
        use the THERMOML_PATH environment variable.
    """
    if thermoml_path is None:
        if "THERMOML_PATH" in os.environ:
            thermoml_path = os.environ["THERMOML_PATH"]
        else:
            raise(KeyError("You must either specify thermoml_path or the THERMOML_PATH environment variable."))

    for key, url in THERMOML_FEEDS.items():
        feed = feedparser.parse(url)
        for entry in feed["entries"]:
            link = entry["link"]
            base_filename = urllib_parse.urlsplit(link).path
            base_filename = base_filename[1:]  # Strip off preceeding backslash character so os.path.join will work
            filename = os.path.join(thermoml_path, base_filename)
            make_path(filename)
            if os.path.exists(filename):
                print("Already downloaded %s from %s" % (filename, link))
            else:
                print("Fetching %s from %s" % (filename, link))
                urllib.request.urlretrieve (link, filename)

def collect_hosts(hosts):
    """
       Collect a set of hosts and an optional chroot from
       a string or a list of strings.
    """
    if isinstance(hosts, list):
        if hosts[-1].strip().startswith('/'):
            host_ports, chroot = hosts[:-1], hosts[-1]
        else:
            host_ports, chroot = hosts, None
    else:
        host_ports, chroot = hosts.partition("/")[::2]
        host_ports = host_ports.split(",")
        chroot = "/" + chroot if chroot else None

    result = []
    for host_port in host_ports:
        # put all complexity of dealing with
        # IPv4 & IPv6 address:port on the urlsplit
        res = urllib_parse.urlsplit("xxx://" + host_port)
        host = res.hostname
        if host is None:
            raise ValueError("bad hostname")
        port = int(res.port) if res.port else 2181
        result.append((host.strip(), port))

    return result, chroot

def get_connection(url, tls=None):
    """This convenience functions returns a :class:`HTTPConnection` or
    :class:`HTTPSConnection` based on the information contained in URL.

    :param url: URL string to create a connection for. Alternatively, passing
                in the results of :py:func:`urlparse.urlsplit` works as well.
    :param tls: When the URL scheme is ``https``, this is passed in as the
                ``tls`` parameter to :class:`HTTPSConnection`.

    """
    if isinstance(url, six.string_types):
        url = urllib_parse.urlsplit(url, 'http')
    host = url.netloc or 'localhost'
    host = host.rsplit(':', 1)[0]
    port = url.port
    kwargs = {}

    if six.PY2:
        # strict is deprecated on Python3
        # https://docs.python.org/3.2/library/http.client.html#httpconnection-objects
        kwargs['strict'] = True

    if url.scheme == 'https':
        conn = HTTPSConnection(host, tls=tls, **kwargs)
    else:
        conn = HTTPConnection(host, **kwargs)
    return conn

def authorize(user):
    if user.orcid is None:
        proofing_state = current_app.proofing_statedb.get_state_by_eppn(user.eppn, raise_on_missing=False)
        if not proofing_state:
            current_app.logger.debug('No proofing state found for user {!s}. Initializing new proofing state.'.format(
                user))
            proofing_state = OrcidProofingState({'eduPersonPrincipalName': user.eppn, 'state': get_unique_hash(),
                                                 'nonce': get_unique_hash()})
            current_app.proofing_statedb.save(proofing_state)

        claims_request = ClaimsRequest(userinfo=Claims(id=None))
        oidc_args = {
            'client_id': current_app.oidc_client.client_id,
            'response_type': 'code',
            'scope': 'openid',
            'claims': claims_request.to_json(),
            'redirect_uri': url_for('orcid.authorization_response', _external=True),
            'state': proofing_state.state,
            'nonce': proofing_state.nonce,
        }
        authorization_url = '{}?{}'.format(current_app.oidc_client.authorization_endpoint, urlencode(oidc_args))
        current_app.logger.debug('Authorization url: {!s}'.format(authorization_url))
        current_app.stats.count(name='authn_request')
        return redirect(authorization_url)
    # Orcid already connected to user
    url = urlappend(current_app.config['DASHBOARD_URL'], 'accountlinking')
    scheme, netloc, path, query_string, fragment = urlsplit(url)
    new_query_string = urlencode({'msg': ':ERROR:orc.already_connected'})
    url = urlunsplit((scheme, netloc, path, new_query_string, fragment))
    return redirect(url)

 def _build_url(self):
     url = list(urlsplit(self._path))
     qs = parse_qs(url[3])
     qs["count"] = self._count
     qs["page"] = self._page
     url[3] = urlencode(qs, doseq=True)
     return urlunsplit(url)

 def on_headers_complete(parser):
     # http-parser callback: the HTTP header is complete. This is the point
     # where we hand off the message to our consumer. Going forward,
     # on_body() will continue to write chunks of the body to message.body.
     self = ffi.from_handle(parser.data)
     self._complete_header_value(b'')
     m = self._message
     m.message_type = lib.http_message_type(parser)
     m.version = '{}.{}'.format(parser.http_major, parser.http_minor)
     if self._server_side:
         m.method = _http_methods.get(parser.method, '<unknown>')
         m.url = _ba2s(self._url)
         try:
             m.parsed_url = urlsplit(m.url)
         except ValueError as e:
             self._error = HttpError('urlsplit(): {!s}'.format(e))
             return 2  # error
         m.is_upgrade = lib.http_is_upgrade(parser)
     else:
         m.status_code = parser.status_code
     m.should_keep_alive = lib.http_should_keep_alive(parser)
     m.body = StreamReader(self._update_body_size)
     # Make the message available. There is no need to call
     # read_buffer_size_change() here as the changes sum up to 0.
     self._queue.put_nowait(m, self._header_size)
     self._header_size = 0
     # Return 1 if this is a HEAD request, 0 otherwise. This instructs the
     # parser whether or not a body follows.
     if not self._requests:
         return 0
     return 1 if self._requests.pop(0) == 'HEAD' else 0

    def validate_url(cls, url):
        """
        Return a boolean indicating whether the URL has a protocol and hostname.
        If a port is specified, ensure it is an integer.

        Arguments:
            url (str): The URL to check.

        Returns:
            Boolean indicating whether the URL has a protocol and hostname.
        """
        result = urllib_parse.urlsplit(url)

        # Check that we have a protocol and hostname
        if not result.scheme or not result.netloc:
            return False

        # Check that the port is an integer
        try:
            if result.port is not None:
                int(result.port)
            elif result.netloc.endswith(':'):
                # Valid URLs do not end with colons.
                return False
        except ValueError:
            return False
        else:
            return True

    def rename(self, new_job_name):
        """Changes the name of this job

        :param str new_job_name:
            new name to assign to this job
        """
        args = {
            "params": {
                "newName": new_job_name
            }
        }
        self._api.post(self._api.url + "doRename", args=args)

        # NOTE: In order to properly support jobs that may contain nested
        #       jobs we have to do some URL manipulations to extrapolate the
        #       REST API endpoint for the parent object to which the cloned
        #       view is to be contained.
        parts = urllib_parse.urlsplit(self._api.url).path.split("/")
        parts = [cur_part for cur_part in parts if cur_part.strip()]
        assert len(parts) >= 2
        assert parts[-2] == "job"
        new_url = urllib_parse.urljoin(
            self._api.url, "/" + "/".join(parts[:-2]))
        new_url += "/job/" + new_job_name
        self._api = self._api.clone(new_url)

        assert self.name == new_job_name

 def get_user_details(self, response):
     """Generate username from identity url"""
     values = super(LiveJournalOpenId, self).get_user_details(response)
     values['username'] = values.get('username') or \
                          urlsplit(response.identity_url)\
                             .netloc.split('.', 1)[0]
     return values

 def __init__(self, url, pool_size=None, tls=None, ehlo_as=None,
              timeout=None, idle_timeout=None):
     super(HttpRelay, self).__init__(pool_size)
     self.url = urllib_parse.urlsplit(url, 'http')
     self.tls = tls
     self.ehlo_as = ehlo_as or getfqdn()
     self.timeout = timeout
     self.idle_timeout = idle_timeout

def redirect_action():
    # Setup a redirect url to action app root
    scheme, netloc, path, query_string, fragment = urlsplit(request.url)
    path = url_for('actions.authn')
    return_url = urlunsplit((scheme, netloc, path, query_string, fragment))
    # TODO: Look in ret to figure out if we need to add a query string with a user message
    ret = _do_action()
    return redirect(return_url)

 def test_url_simple(self):
     connection = Mock()
     connection.call.return_value = (None, {'start': 0, 'total_size': 0})
     page = Page(connection, '/some-path', None)
     built_qs = parse_qs(urlsplit(page._build_url()).query)
     self.assertEqual(built_qs, dict(
         count=[str(DEFAULT_PAGE_ITEM_COUNT)],
         page=["1"]))

    def build_absolute_uri(self, location=None):
        if location is None:
            return None

        bits = urlsplit(location)
        if not (bits.scheme and bits.netloc):
            location = urljoin(self.prod_url, location)
        return iri_to_uri(location)

 def clean_contributions(self):
     if self.cleaned_data['contributions']:
         hostname = urlsplit(self.cleaned_data['contributions']).hostname
         if not hostname.endswith(amo.VALID_CONTRIBUTION_DOMAINS):
             raise forms.ValidationError(ugettext(
                 'URL domain must be one of [%s], or a subdomain.'
             ) % ', '.join(amo.VALID_CONTRIBUTION_DOMAINS))
     return self.cleaned_data['contributions']

 def get_user_details(self, response):
     """Generate username from identity url"""
     values = super(YandexOpenId, self).get_user_details(response)
     values['username'] = values.get('username') or\
                          urlsplit(response.identity_url)\
                                 .path.strip('/')
     values['email'] = values.get('email', '')
     return values

def forward(apps, schema_editor):
    FormulaComponent = apps.get_model('formulas', 'FormulaComponent')
    Formula = apps.get_model('formulas', 'Formula')

    formulas = {}

    for formula in Formula.objects.all():
        # Get the username out of the URI if it's a private formula
        if formula.git_username:
            parse_res = urlsplit(formula.uri)
            if '@' in parse_res.netloc:
                new_netloc = parse_res.netloc.split('@')[-1]
                formula.uri = urlunsplit((
                    parse_res.scheme,
                    new_netloc,
                    parse_res.path,
                    parse_res.query,
                    parse_res.fragment
                ))
                formula.save()

        if formula.uri not in formulas:
            formulas[formula.uri] = formula
            continue

        # Otherwise we need to delete the formula and everything associated with it
        for component in FormulaComponent.objects.filter(formula=formula):
            for bhfc in component.blueprinthostformulacomponent_set.all():
                try:
                    bhfc.component = FormulaComponent.objects.get(sls_path=bhfc.component.sls_path,
                                                                  formula=formulas[formula.uri])
                    bhfc.save()
                except FormulaComponent.DoesNotExist:
                    bhfc.component.formula = formulas[formula.uri]
                    bhfc.component.save()

            component.delete()

        formula.delete()

    # re-import all the formulas
    for formula in Formula.objects.all():
        # there's nothing we can do about private repos without having the password :(
        if not formula.git_username:
            try:
                import_formula(formula.id, '')
            except FormulaTaskException as e:
                if 'SPECFILE' in e.message:
                    print('Skipping import of formula: {0}'.format(formula.uri))
                else:
                    raise
        else:
            print('Please manually update this formula via the API: {0}'.format(formula.uri))

    # remove the old ones
    old_formula_dir = os.path.join(settings.STACKDIO_CONFIG['storage_root'], 'user_states')
    if os.path.isdir(old_formula_dir):
        shutil.rmtree(old_formula_dir)

def relurl(url, starturl):
    """Works like :py:func:`os.path.relpath`, but for urls

    >>> relurl("http://example.org/other/index.html", "http://example.org/main/index.html") == '../other/index.html'
    True
    >>> relurl("http://other.org/foo.html", "http://example.org/bar.html") == 'http://other.org/foo.html'
    True

    """
    urlseg = urlsplit(url)
    startseg = urlsplit(starturl)
    urldomain = urlunsplit(urlseg[:2] + tuple('' for i in range(3)))
    startdomain = urlunsplit(startseg[:2] + tuple('' for i in range(3)))
    if urldomain != startdomain:  # different domain, no relative url possible
        return url

    relpath = posixpath.relpath(urlseg.path, posixpath.dirname(startseg.path))
    res = urlunsplit(('', '', relpath, urlseg.query, urlseg.fragment))
    return res

 def test_url_with_qs(self):
     connection = Mock()
     connection.call.return_value = (None, {'start': 0, 'total_size': 0})
     page = Page(connection, '/some-path?with=a&query=string', None)
     built_qs = parse_qs(urlsplit(page._build_url()).query)
     self.assertEqual(built_qs, {
         "with": ["a"],
         "query": ["string"],
         "count": [str(DEFAULT_PAGE_ITEM_COUNT)],
         "page": ["1"],
         })