def get_url(self):
        """Return the base URL of the Foreman deployment being tested.

        The following values from the config file are used to build the URL:

        * ``[server] scheme`` (default: https)
        * ``[server] hostname`` (required)
        * ``[server] port`` (default: none)

        Setting ``port`` to 80 does *not* imply that ``scheme`` is 'https'. If
        ``port`` is 80 and ``scheme`` is unset, ``scheme`` will still default
        to 'https'.

        :return: A URL.
        :rtype: str

        """
        if not self.scheme:
            scheme = "https"
        else:
            scheme = self.scheme
        # All anticipated error cases have been handled at this point.
        if not self.port:
            return urlunsplit((scheme, self.hostname, "", "", ""))
        else:
            return urlunsplit((scheme, "{0}:{1}".format(self.hostname, self.port), "", "", ""))

    def __init__(self, address, name=""):
        """Bind the publisher class to a port.
        """
        # pylint: disable=E1103
        self.name = name
        self.destination = address
        self.publish = get_context().socket(zmq.PUB)

        # Check for port 0 (random port)
        u__ = urlsplit(self.destination)
        port = u__.port

        if port == 0:
            dest = urlunsplit((u__.scheme, u__.hostname,
                               u__.path, u__.query, u__.fragment))
            self.port_number = self.publish.bind_to_random_port(dest)
            netloc = u__.hostname + ":" + str(self.port_number)
            self.destination = urlunsplit((u__.scheme, netloc, u__.path,
                                           u__.query, u__.fragment))
        else:
            self.publish.bind(self.destination)
            self.port_number = port

        LOGGER.info("publisher started on port %s", str(self.port_number))

        # Initialize no heartbeat
        self._heartbeat = None
        self._pub_lock = Lock()

    def __init__(self, url, application=None, session=None, output_grid=True):
        # download DDS/DAS
        scheme, netloc, path, query, fragment = urlsplit(url)

        ddsurl = urlunsplit((scheme, netloc, path + ".dds", query, fragment))
        r = GET(ddsurl, application, session)
        raise_for_status(r)
        dds = r.text

        dasurl = urlunsplit((scheme, netloc, path + ".das", query, fragment))
        r = GET(dasurl, application, session)
        raise_for_status(r)
        das = r.text

        # build the dataset from the DDS and add attributes from the DAS
        self.dataset = build_dataset(dds)
        add_attributes(self.dataset, parse_das(das))

        # remove any projection from the url, leaving selections
        projection, selection = parse_ce(query)
        url = urlunsplit((scheme, netloc, path, "&".join(selection), fragment))

        # now add data proxies
        for var in walk(self.dataset, BaseType):
            var.data = BaseProxy(url, var.id, var.dtype, var.shape, application=application, session=session)
        for var in walk(self.dataset, SequenceType):
            template = copy.copy(var)
            var.data = SequenceProxy(url, template, application=application, session=session)

        # apply projections
        for var in projection:
            target = self.dataset
            while var:
                token, index = var.pop(0)
                target = target[token]
                if isinstance(target, BaseType):
                    target.data.slice = fix_slice(index, target.shape)
                elif isinstance(target, GridType):
                    index = fix_slice(index, target.array.shape)
                    target.array.data.slice = index
                    for s, child in zip(index, target.maps):
                        target[child].data.slice = (s,)
                elif isinstance(target, SequenceType):
                    target.data.slice = index

        # retrieve only main variable for grid types:
        for var in walk(self.dataset, GridType):
            var.set_output_grid(output_grid)

def _send_new_pending_email(instance):
    """ Send an email to settings.API_ACCESS_MANAGER_EMAIL with the contents of this API access request. """
    context = {
        'approval_url': urlunsplit(
            (
                'https' if settings.HTTPS == 'on' else 'http',
                instance.site.domain,
                reverse('admin:api_admin_apiaccessrequest_change', args=(instance.id,)),
                '',
                '',
            )
        ),
        'api_request': instance
    }

    message = render_to_string('api_admin/api_access_request_email_new_request.txt', context)
    try:
        send_mail(
            _(u'API access request from {company}').format(company=instance.company_name),
            message,
            settings.API_ACCESS_FROM_EMAIL,
            [settings.API_ACCESS_MANAGER_EMAIL],
            fail_silently=False
        )
    except SMTPException:
        log.exception(u'Error sending API user notification email for request [%s].', instance.id)

    def _get_heat_signal_url(self, project_id=None):
        """Return a heat-api signal URL for this resource.

        This URL is not pre-signed, valid user credentials are required.
        If a project_id is provided, it is used in place of the original
        project_id. This is useful to generate a signal URL that uses
        the heat stack user project instead of the user's.
        """
        stored = self.data().get('heat_signal_url')
        if stored is not None:
            return stored

        if self.id is None:
            # it is too early
            return

        url = self.client_plugin('heat').get_heat_url()
        host_url = urlparse.urlparse(url)
        path = self.identifier().url_path()
        if project_id is not None:
            path = project_id + path[path.find('/'):]

        url = urlparse.urlunsplit(
            (host_url.scheme, host_url.netloc, 'v1/%s/signal' % path, '', ''))

        self.data_set('heat_signal_url', url)
        return url

def _build_api_url(url, query):
    scheme, netloc, path, base_query, fragment = urlsplit(url)

    if base_query:
        query = '%s&%s' % (base_query, query)

    return urlunsplit((scheme, netloc, path, query, fragment))

def _get_safe_url(url):
    """Gets version of *url* with basic auth passwords obscured. This function
    returns results suitable for printing and logging.

    E.g.: https://user:[email protected] => https://user:********@example.com

    .. note::

       The number of astrisks is invariant in the length of the basic auth
       password, so minimal information is leaked.

    :param url: a url
    :type url: ``str``

    :return: *url* with password obscured
    :rtype: ``str``
    """
    safe_url = url
    url, username, _ = _strip_basic_auth(url)
    if username is not None:
        # case: url contained basic auth creds; obscure password
        url_parts = parse.urlsplit(url)
        safe_netloc = '{0}@{1}'.format(username, url_parts.hostname)
        # replace original netloc w/ obscured version
        frags = list(url_parts)
        frags[1] = safe_netloc
        safe_url = parse.urlunsplit(frags)

    return safe_url

    def authenticate(self):
        magic_tuple = netutils.urlsplit(self.auth_url)
        scheme, netloc, path, query, frag = magic_tuple
        port = magic_tuple.port
        if port is None:
            port = 80
        path_parts = path.split('/')
        for part in path_parts:
            if len(part) > 0 and part[0] == 'v':
                self.version = part
                break

        if self.auth_token and self.management_url:
            self._save_keys()
            return

        # TODO(sandy): Assume admin endpoint is 35357 for now.
        # Ideally this is going to have to be provided by the service catalog.
        new_netloc = netloc.replace(':%d' % port, ':%d' % (35357,))
        admin_url = parse.urlunsplit(
            (scheme, new_netloc, path, query, frag))

        auth_url = self.auth_url
        if self.version == "v2.0":  # FIXME(chris): This should be better.
            while auth_url:
                if not self.auth_system or self.auth_system == 'keystone':
                    auth_url = self._v2_auth(auth_url)
                else:
                    auth_url = self._plugin_auth(auth_url)

            # Are we acting on behalf of another user via an
            # existing token? If so, our actual endpoints may
            # be different than that of the admin token.
            if self.proxy_token:
                if self.bypass_url:
                    self.set_management_url(self.bypass_url)
                else:
                    self._fetch_endpoints_from_auth(admin_url)
                # Since keystone no longer returns the user token
                # with the endpoints any more, we need to replace
                # our service account token with the user token.
                self.auth_token = self.proxy_token
        else:
            try:
                while auth_url:
                    auth_url = self._v1_auth(auth_url)
            # In some configurations nova makes redirection to
            # v2.0 keystone endpoint. Also, new location does not contain
            # real endpoint, only hostname and port.
            except exceptions.AuthorizationFailure:
                if auth_url.find('v2.0') < 0:
                    auth_url = auth_url + '/v2.0'
                self._v2_auth(auth_url)

        if self.bypass_url:
            self.set_management_url(self.bypass_url)
        elif not self.management_url:
            raise exceptions.Unauthorized('Nova Client')

        self._save_keys()

def remove_version_from_href(href):
    """Removes the first api version from the href.

    Given: 'http://manila.example.com/v1.1/123'
    Returns: 'http://manila.example.com/123'

    Given: 'http://www.manila.com/v1.1'
    Returns: 'http://www.manila.com'

    Given: 'http://manila.example.com/share/v1.1/123'
    Returns: 'http://manila.example.com/share/123'

    """
    parsed_url = parse.urlsplit(href)
    url_parts = parsed_url.path.split('/')

    # NOTE: this should match vX.X or vX
    expression = re.compile(r'^v([0-9]+|[0-9]+\.[0-9]+)(/.*|$)')
    for x in range(len(url_parts)):
        if expression.match(url_parts[x]):
            del url_parts[x]
            break

    new_path = '/'.join(url_parts)

    if new_path == parsed_url.path:
        msg = 'href %s does not contain version' % href
        LOG.debug(msg)
        raise ValueError(msg)

    parsed_url = list(parsed_url)
    parsed_url[2] = new_path
    return parse.urlunsplit(parsed_url)

    def _parse_url(self, url, ssl=False):
        """Create a url from test data.

        If provided with a full URL, just return that. If SSL is requested
        set the scheme appropriately.

        Scheme and netloc are saved for later use in comparisons.
        """
        parsed_url = urlparse.urlsplit(url)
        url_scheme = parsed_url[0]
        scheme = 'http'
        netloc = self.host

        if not url_scheme:
            if self.port:
                netloc = '%s:%s' % (self.host, self.port)
            if ssl:
                scheme = 'https'
            path = parsed_url[2]
            if self.prefix:
                path = '%s%s' % (self.prefix, path)
            full_url = urlparse.urlunsplit((scheme, netloc, path,
                                            parsed_url[3], ''))
            self.scheme = scheme
            self.netloc = netloc
        else:
            full_url = url
            self.scheme = url_scheme
            self.netloc = parsed_url[1]

        return full_url

    def query_lookupd(self):
        """
        Trigger a query of the configured ``nsq_lookupd_http_addresses``.
        """
        endpoint = self.lookupd_http_addresses[self.lookupd_query_index]
        self.lookupd_query_index = (self.lookupd_query_index + 1) % len(self.lookupd_http_addresses)

        # urlsplit() is faulty if scheme not present
        if '://' not in endpoint:
            endpoint = 'http://' + endpoint

        scheme, netloc, path, query, fragment = urlparse.urlsplit(endpoint)

        if not path or path == "/":
            path = "/lookup"

        params = cgi.parse_qs(query)
        params['topic'] = self.topic
        query = urlparse.urlencode(_utf8_params(params), doseq=1)
        lookupd_url = urlparse.urlunsplit((scheme, netloc, path, query, fragment))

        req = tornado.httpclient.HTTPRequest(
            lookupd_url, method='GET',
            connect_timeout=self.lookupd_connect_timeout,
            request_timeout=self.lookupd_request_timeout)
        callback = functools.partial(self._finish_query_lookupd, lookupd_url=lookupd_url)
        self.http_client.fetch(req, callback=callback)

    def get_authorization_url(self, redirect_url):
        """
        Get the authorization url based on the client id and the redirect url passed in

        :param redirect_url:
            An HTTPS URI or custom URL scheme where the response will be redirected. Optional if the redirect URI is
            registered with Box already.
        :type redirect_url:
            `unicode` or None
        :return:
            A tuple of the URL of Box's authorization page and the CSRF token.
            This is the URL that your application should forward the user to in first leg of OAuth 2.
        :rtype:
            (`unicode`, `unicode`)
        """
        csrf_token = self._get_state_csrf_token()
        # For the query string parameters, use a sequence of two-element
        # tuples, rather than a dictionary, in order to get a consistent and
        # predictable order of parameters in the output of `urlencode()`.
        params = [
            ('state', csrf_token),
            ('response_type', 'code'),
            ('client_id', self._client_id),
        ]
        if redirect_url:
            params.append(('redirect_uri', redirect_url))
        # `urlencode()` doesn't work with non-ASCII unicode characters, so
        # encode the parameters as ASCII bytes.
        params = [(key.encode('utf-8'), value.encode('utf-8')) for (key, value) in params]
        query_string = urlencode(params)
        return urlunsplit(('', '', API.OAUTH2_AUTHORIZE_URL, query_string, '')), csrf_token

    def set_url(self, url):

        path = url.lstrip('/')

        if url.startswith("http://") or url.startswith("https://"):
            u = urlparse.urlsplit(url)
            if u.username is not None:
                password = u.password or ""
                encode = base64.b64encode("%s:%s" % (u.username, password))
                self.headers['Authorization'] = 'Basic %s' %  encode

            self.scheme = u.scheme,
            self.host = u.netloc.split("@")[-1]
            self.path_info = u.path or "/"
            self.query_string = u.query
            url = urlparse.urlunsplit((u.scheme, u.netloc.split("@")[-1],
                u.path, u.query, u.fragment))
        else:

            if '?' in path:
                path, self.query_string = path.split('?', 1)
            self.path_info = '/' + path


            url = self.url
        self.scheme, self.host, self.path_info = urlparse.urlparse(url)[0:3]

    def _proxy_request(self, instance_id, tenant_id, req):
        headers = {
            'X-Forwarded-For': req.headers.get('X-Forwarded-For'),
            'X-Instance-ID': instance_id,
            'X-Tenant-ID': tenant_id,
            'X-Instance-ID-Signature': self._sign_instance_id(instance_id)
        }

        nova_host_port = '%s:%s' % (self.conf.nova_metadata_host,
                                    self.conf.nova_metadata_port)
        url = urlparse.urlunsplit((
            self.conf.nova_metadata_protocol,
            nova_host_port,
            req.path_info,
            req.query_string,
            ''))

        disable_ssl_certificate_validation = self.conf.nova_metadata_insecure
        if self.conf.auth_ca_cert and not disable_ssl_certificate_validation:
            verify_cert = self.conf.auth_ca_cert
        else:
            verify_cert = not disable_ssl_certificate_validation

        client_cert = None
        if self.conf.nova_client_cert and self.conf.nova_client_priv_key:
            client_cert = (self.conf.nova_client_cert,
                           self.conf.nova_client_priv_key)

        resp = requests.request(method=req.method, url=url,
                                headers=headers,
                                data=req.body,
                                cert=client_cert,
                                verify=verify_cert)

        if resp.status_code == 200:
            req.response.content_type = resp.headers['content-type']
            req.response.body = resp.content
            LOG.debug(str(resp))
            return req.response
        elif resp.status_code == 403:
            LOG.warning(
                'The remote metadata server responded with Forbidden. This '
                'response usually occurs when shared secrets do not match.'
            )
            return webob.exc.HTTPForbidden()
        elif resp.status_code == 400:
            return webob.exc.HTTPBadRequest()
        elif resp.status_code == 404:
            return webob.exc.HTTPNotFound()
        elif resp.status_code == 409:
            return webob.exc.HTTPConflict()
        elif resp.status_code == 500:
            msg = _(
                'Remote metadata server experienced an internal server error.'
            )
            LOG.warning(msg)
            explanation = six.text_type(msg)
            return webob.exc.HTTPInternalServerError(explanation=explanation)
        else:
            raise Exception(_('Unexpected response code: %s') % resp.status)

def download_attachments(output_path, urls):
    """Downloads WordPress attachments and returns a list of paths to
    attachments that can be associated with a post (relative path to output
    directory). Files that fail to download, will not be added to posts"""
    locations = {}
    for url in urls:
        path = urlparse(url).path
        # teardown path and rebuild to negate any errors with
        # os.path.join and leading /'s
        path = path.split('/')
        filename = path.pop(-1)
        localpath = ''
        for item in path:
            if sys.platform != 'win32' or ':' not in item:
                localpath = os.path.join(localpath, item)
        full_path = os.path.join(output_path, localpath)

        # Generate percent-encoded URL
        scheme, netloc, path, query, fragment = urlsplit(url)
        path = quote(path)
        url = urlunsplit((scheme, netloc, path, query, fragment))

        if not os.path.exists(full_path):
            os.makedirs(full_path)
        print('downloading {}'.format(filename))
        try:
            urlretrieve(url, os.path.join(full_path, filename))
            locations[url] = os.path.join(localpath, filename)
        except (URLError, IOError) as e:
            # Python 2.7 throws an IOError rather Than URLError
            logger.warning("No file could be downloaded from %s\n%s", url, e)
    return locations

 def update_actions(self, old_alarm=None):
     trustor_user_id = pecan.request.headers.get("X-User-Id")
     trustor_project_id = pecan.request.headers.get("X-Project-Id")
     roles = pecan.request.headers.get("X-Roles", "")
     if roles:
         roles = roles.split(",")
     else:
         roles = []
     auth_plugin = pecan.request.environ.get("keystone.token_auth")
     for actions in (self.ok_actions, self.alarm_actions, self.insufficient_data_actions):
         if actions is not None:
             for index, action in enumerate(actions[:]):
                 url = netutils.urlsplit(action)
                 if self._is_trust_url(url):
                     if "@" not in url.netloc:
                         # We have a trust action without a trust ID,
                         # create it
                         trust_id = keystone_client.create_trust_id(
                             pecan.request.cfg, trustor_user_id, trustor_project_id, roles, auth_plugin
                         )
                         netloc = "%s:[email protected]%s" % (trust_id, url.netloc)
                         url = list(url)
                         url[1] = netloc
                         actions[index] = urlparse.urlunsplit(url)
     if old_alarm:
         new_actions = list(
             itertools.chain(self.ok_actions or [], self.alarm_actions or [], self.insufficient_data_actions or [])
         )
         for action in itertools.chain(
             old_alarm.ok_actions or [], old_alarm.alarm_actions or [], old_alarm.insufficient_data_actions or []
         ):
             if action not in new_actions:
                 self.delete_trust(action)

 def update_actions(self, old_alarm=None):
     trustor_user_id = pecan.request.headers.get('X-User-Id')
     trustor_project_id = pecan.request.headers.get('X-Project-Id')
     roles = pecan.request.headers.get('X-Roles', '')
     if roles:
         roles = roles.split(',')
     else:
         roles = []
     auth_plugin = pecan.request.environ.get('keystone.token_auth')
     for actions in (self.ok_actions, self.alarm_actions,
                     self.insufficient_data_actions):
         if actions is not None:
             for index, action in enumerate(actions[:]):
                 url = netutils.urlsplit(action)
                 if self._is_trust_url(url):
                     if '@' not in url.netloc:
                         # We have a trust action without a trust ID,
                         # create it
                         trust_id = keystone_client.create_trust_id(
                             trustor_user_id, trustor_project_id, roles,
                             auth_plugin)
                         netloc = '%s:[email protected]%s' % (trust_id, url.netloc)
                         url = list(url)
                         url[1] = netloc
                         actions[index] = urlparse.urlunsplit(url)
     if old_alarm:
         for key in ('ok_actions', 'alarm_actions',
                     'insufficient_data_actions'):
             for action in getattr(old_alarm, key):
                 url = netutils.urlsplit(action)
                 if (self._is_trust_url(url) and url.password and
                         action not in getattr(self, key)):
                     keystone_client.delete_trust_id(
                         url.username, auth_plugin)

 def __init__(self, base, relative=None):
     self._has_token = False
     self._url = None
     self._url_parts = None
     self._loaded = False
     self._xml = None
     self._url_parts = None
     self._headers = None
     self._config = None
     if isinstance(base, six.string_types):
         base_url = base
         self._url_parts = list(parse.urlsplit(base_url))
     elif isinstance(base, RequestBase):
         base_url = base.url
         self._has_token = base.has_token
         self._url_parts = base._url_parts[:]
         self._headers = base._headers
         self._config = base.config
     if relative:
         scheme, netloc, path, qs, fragment = parse.urlsplit(relative)
         if path:
             self._url_parts[2] = _join_plex(self._url_parts[2], path)
         if qs:
             data = parse.parse_qsl(self._url_parts[3]) + parse.parse_qsl(qs)
             self._url_parts[3] = parse.urlencode(data)
         else:
             # Strip of all non-token parts
             data = parse.parse_qsl(self._url_parts[3])
             self._url_parts[3] = parse.urlencode([(x, y) for x, y in data if x == 'X-Plex-Token'])
     if not self._has_token:
         self._has_token = 'X-Plex-Token' in parse.parse_qs(self._url_parts[3])
     self._url = parse.urlunsplit(self._url_parts)

def _deprecated_amqp_url():
    """Allow for deprecating amqp_url setting over time.
    This warns and attempts to translate an amqp_url to something
    oslo_messaging can use to load a driver.
    """
    url = cfg.CONF.amqp_url
    if not url:
        return
    LOG.warning(_LW(
        'Use of amqp_url is deprecated. Please instead use options defined in '
        'oslo_messaging_rabbit to declare your AMQP connection.'))
    url = urlparse.urlsplit(url)
    if url.scheme == 'amqp':
        scheme = 'rabbit'
    else:
        scheme = url.scheme
    port = str(url.port or 5672)
    netloc = url.netloc
    if netloc.endswith(':'):
        netloc = netloc[:-1]
    out = urlparse.urlunsplit((
        scheme,
        '%s:%s' % (netloc, port),
        url.path,
        '', ''
    ))
    return out

def _send_decision_email(instance):
    """ Send an email to requesting user with the decision made about their request. """
    context = {
        'name': instance.user.username,
        'api_management_url': urlunsplit(
            (
                'https' if settings.HTTPS == 'on' else 'http',
                instance.site.domain,
                reverse('api_admin:api-status'),
                '',
                '',
            )
        ),
        'authentication_docs_url': settings.AUTH_DOCUMENTATION_URL,
        'api_docs_url': settings.API_DOCUMENTATION_URL,
        'support_email_address': settings.API_ACCESS_FROM_EMAIL,
        'platform_name': configuration_helpers.get_value('PLATFORM_NAME', settings.PLATFORM_NAME)
    }

    message = render_to_string(
        'api_admin/api_access_request_email_{status}.txt'.format(status=instance.status),
        context
    )
    try:
        send_mail(
            _('API access request'),
            message,
            settings.API_ACCESS_FROM_EMAIL,
            [instance.user.email],
            fail_silently=False
        )
        instance.contacted = True
    except SMTPException:
        log.exception(u'Error sending API user notification email for request [%s].', instance.id)