def base_request(self, method, container=None, name=None, prefix=None,
                     headers={}, proxy=None, contents=None, full_listing=None):
        # Common request method
        url = self.url

        if self.token:
            headers['X-Auth-Token'] = self.token

        if container:
            url = '%s/%s' % (url.rstrip('/'), quote(container))

        if name:
            url = '%s/%s' % (url.rstrip('/'), quote(name))

        url += '?format=json'

        if prefix:
            url += '&prefix=%s' % prefix

        if proxy:
            proxy = urlparse.urlparse(proxy)
            proxy = urllib2.ProxyHandler({proxy.scheme: proxy.netloc})
            opener = urllib2.build_opener(proxy)
            urllib2.install_opener(opener)

        req = urllib2.Request(url, headers=headers, data=contents)
        req.get_method = lambda: method
        urllib2.urlopen(req)
        conn = urllib2.urlopen(req)
        body = conn.read()
        try:
            body_data = json.loads(body)
        except ValueError:
            body_data = None
        return [None, body_data]

def _get_direct_account_container(path, stype, node, part,
                                  marker=None, limit=None,
                                  prefix=None, delimiter=None, conn_timeout=5,
                                  response_timeout=15):
    """Base class for get direct account and container.

    Do not use directly use the get_direct_account or
    get_direct_container instead.
    """
    qs = 'format=json'
    if marker:
        qs += '&marker=%s' % quote(marker)
    if limit:
        qs += '&limit=%d' % limit
    if prefix:
        qs += '&prefix=%s' % quote(prefix)
    if delimiter:
        qs += '&delimiter=%s' % quote(delimiter)
    with Timeout(conn_timeout):
        conn = http_connect(node['ip'], node['port'], node['device'], part,
                            'GET', path, query_string=qs,
                            headers=gen_headers())
    with Timeout(response_timeout):
        resp = conn.getresponse()
    if not is_success(resp.status):
        resp.read()
        raise DirectClientException(stype, 'GET', node, part, path, resp)

    resp_headers = HeaderKeyDict()
    for header, value in resp.getheaders():
        resp_headers[header] = value
    if resp.status == HTTP_NO_CONTENT:
        resp.read()
        return resp_headers, []
    return resp_headers, json.loads(resp.read())

    def _get_container_listing(self, req, version, account, container,
                               prefix, marker=''):
        '''
        :param version: whatever
        :param account: native
        :param container: native
        :param prefix: native
        :param marker: native
        '''
        con_req = make_subrequest(
            req.environ,
            path=wsgi_quote('/'.join([
                '', str_to_wsgi(version),
                str_to_wsgi(account), str_to_wsgi(container)])),
            method='GET',
            headers={'x-auth-token': req.headers.get('x-auth-token')},
            agent=('%(orig)s ' + 'DLO MultipartGET'), swift_source='DLO')
        con_req.query_string = 'prefix=%s' % quote(prefix)
        if marker:
            con_req.query_string += '&marker=%s' % quote(marker)

        con_resp = con_req.get_response(self.dlo.app)
        if not is_success(con_resp.status_int):
            if req.method == 'HEAD':
                con_resp.body = b''
            return con_resp, None
        with closing_if_possible(con_resp.app_iter):
            return None, json.loads(b''.join(con_resp.app_iter))

    def _iter_items(
            self, path, marker='', end_marker='',
            acceptable_statuses=(2, HTTP_NOT_FOUND)):
        """
        Returns an iterator of items from a json listing.  Assumes listing has
        'name' key defined and uses markers.

        :param path: Path to do GET on.
        :param marker: Prefix of first desired item, defaults to ''.
        :param end_marker: Last item returned will be 'less' than this,
                           defaults to ''.
        :param acceptable_statuses: List of status for valid responses,
                                    defaults to (2, HTTP_NOT_FOUND).

        :raises UnexpectedResponse: Exception raised when requests fail
                                    to get a response with an acceptable status
        :raises Exception: Exception is raised when code fails in an
                           unexpected way.
        """

        while True:
            resp = self.make_request(
                'GET', '%s?format=json&marker=%s&end_marker=%s' %
                (path, quote(marker), quote(end_marker)),
                {}, acceptable_statuses)
            if not resp.status_int == 200:
                if resp.status_int >= HTTP_MULTIPLE_CHOICES:
                    ''.join(resp.app_iter)
                break
            data = json.loads(resp.body)
            if not data:
                break
            for item in data:
                yield item
            marker = data[-1]['name'].encode('utf8')

    def _test_redirect_slash_remap_acct(self, anonymous):
        host = self.domain_remap_acct
        path = '/%s' % quote(self.env.container.name)
        self._test_redirect_with_slash(host, path, anonymous=anonymous)

        path = '/%s/%s' % (quote(self.env.container.name),
                           self.env.objects['dir/'].name)
        self._test_redirect_with_slash(host, path, anonymous=anonymous)

        def validate_seg_dict(seg_dict, head_seg_resp):
            if not head_seg_resp.is_success:
                problem_segments.append([quote(obj_name),
                                         head_seg_resp.status])
                return 0, None

            segment_length = head_seg_resp.content_length
            if seg_dict.get('range'):
                # Since we now know the length, we can normalize the
                # range. We know that there is exactly one range
                # requested since we checked that earlier in
                # parse_and_validate_input().
                ranges = seg_dict['range'].ranges_for_length(
                    head_seg_resp.content_length)

                if not ranges:
                    problem_segments.append([quote(obj_name),
                                             'Unsatisfiable Range'])
                elif ranges == [(0, head_seg_resp.content_length)]:
                    # Just one range, and it exactly matches the object.
                    # Why'd we do this again?
                    del seg_dict['range']
                    segment_length = head_seg_resp.content_length
                else:
                    rng = ranges[0]
                    seg_dict['range'] = '%d-%d' % (rng[0], rng[1] - 1)
                    segment_length = rng[1] - rng[0]

            if segment_length < 1:
                problem_segments.append(
                    [quote(obj_name),
                     'Too small; each segment must be at least 1 byte.'])
            if seg_dict.get('size_bytes') is not None and \
                    seg_dict['size_bytes'] != head_seg_resp.content_length:
                problem_segments.append([quote(obj_name), 'Size Mismatch'])
            if seg_dict.get('etag') is not None and \
                    seg_dict['etag'] != head_seg_resp.etag:
                problem_segments.append([quote(obj_name), 'Etag Mismatch'])
            if head_seg_resp.last_modified:
                last_modified = head_seg_resp.last_modified
            else:
                # shouldn't happen
                last_modified = datetime.now()

            last_modified_formatted = \
                last_modified.strftime('%Y-%m-%dT%H:%M:%S.%f')
            seg_data = {'name': '/' + seg_dict['path'].lstrip('/'),
                        'bytes': head_seg_resp.content_length,
                        'hash': head_seg_resp.etag,
                        'content_type': head_seg_resp.content_type,
                        'last_modified': last_modified_formatted}
            if seg_dict.get('range'):
                seg_data['range'] = seg_dict['range']
            if config_true_value(
                    head_seg_resp.headers.get('X-Static-Large-Object')):
                seg_data['sub_slo'] = True
            return segment_length, seg_data

def disposition_format(disposition_type, filename):
    # Content-Disposition in HTTP is defined in
    # https://tools.ietf.org/html/rfc6266 and references
    # https://tools.ietf.org/html/rfc5987#section-3.2
    # to explain the filename*= encoding format. The summary
    # is that it's the charset, then an optional (and empty) language
    # then the filename. Looks funny, but it's right.
    return '''%s; filename="%s"; filename*=UTF-8''%s''' % (
        disposition_type, quote(filename, safe=' /'), quote(filename))

    def _test_redirect_slash_direct(self, anonymous):
        host = self.env.account.conn.storage_netloc
        path = '%s/%s' % (self.env.account.conn.storage_path,
                          quote(self.env.container.name))
        self._test_redirect_with_slash(host, path, anonymous=anonymous)

        path = '%s/%s/%s' % (self.env.account.conn.storage_path,
                             quote(self.env.container.name),
                             quote(self.env.objects['dir/'].name))
        self._test_redirect_with_slash(host, path, anonymous=anonymous)

    def _test_index_direct(self, anonymous):
        objects = self.env.objects
        host = self.env.account.conn.storage_netloc
        path = '%s/%s/' % (self.env.account.conn.storage_path,
                           quote(self.env.container.name))
        self._test_index(host, path, anonymous=anonymous)

        path = '%s/%s/%s/' % (self.env.account.conn.storage_path,
                              quote(self.env.container.name),
                              quote(objects['dir/'].name))
        self._test_index(host, path, anonymous=anonymous, expected_status=404)

    def _build_css_path(self, prefix=''):
        """
        Constructs a relative path from a given prefix within the container.
        URLs and paths starting with '/' are not modified.

        :param prefix: The prefix for the container listing.
        """
        if self._listings_css.startswith(('/', 'http://', 'https://')):
            css_path = quote(self._listings_css, ':/')
        else:
            css_path = '../' * prefix.count('/') + quote(self._listings_css)
        return css_path

    def _build_css_path(self, prefix=""):
        """
        Constructs a relative path from a given prefix within the container.
        URLs and paths starting with '/' are not modified.

        :param prefix: The prefix for the container listing.
        """
        if self._listings_css.startswith(("/", "http://", "https://")):
            css_path = quote(self._listings_css, ":/")
        else:
            css_path = "../" * prefix.count("/") + quote(self._listings_css)
        return css_path

    def _get_container_listing(self, req, version, account, container,
                               prefix, marker=''):
        con_req = make_request(
            req.environ, path='/'.join(['', version, account, container]),
            method='GET',
            headers={'x-auth-token': req.headers.get('x-auth-token')},
            agent=('%(orig)s ' + 'DLO MultipartGET'), swift_source='DLO')
        con_req.query_string = 'format=json&prefix=%s' % quote(prefix)
        if marker:
            con_req.query_string += '&marker=%s' % quote(marker)

        con_resp = con_req.get_response(self.dlo.app)
        if not is_success(con_resp.status_int):
            return con_resp, None
        return None, json.loads(''.join(con_resp.app_iter))

    def _get_container_listing(self, req, version, account, container,
                               prefix, marker=''):
        con_req = req.copy_get()
        con_req.script_name = ''
        con_req.range = None
        con_req.path_info = '/'.join(['', version, account, container])
        con_req.query_string = 'format=json&prefix=%s' % quote(prefix)
        con_req.user_agent = '%s DLO MultipartGET' % con_req.user_agent
        if marker:
            con_req.query_string += '&marker=%s' % quote(marker)

        con_resp = con_req.get_response(self.dlo.app)
        if not is_success(con_resp.status_int):
            return con_resp, None
        return None, json.loads(''.join(con_resp.app_iter))

    def _update_sync_to_headers(self, name, sync_to, user_key,
                                realm, realm_key, method, headers):
        """
        Updates container sync headers

        :param name: The name of the object
        :param sync_to: The URL to the remote container.
        :param user_key: The X-Container-Sync-Key to use when sending requests
                         to the other container.
        :param realm: The realm from self.realms_conf, if there is one.
            If None, fallback to using the older allowed_sync_hosts
            way of syncing.
        :param realm_key: The realm key from self.realms_conf, if there
            is one. If None, fallback to using the older
            allowed_sync_hosts way of syncing.
        :param method: HTTP method to create sig with
        :param headers: headers to update with container sync headers
        """
        if realm and realm_key:
            nonce = uuid.uuid4().hex
            path = urlparse(sync_to).path + '/' + quote(name)
            sig = self.realms_conf.get_sig(method, path,
                                           headers.get('x-timestamp', 0),
                                           nonce, realm_key,
                                           user_key)
            headers['x-container-sync-auth'] = '%s %s %s' % (realm,
                                                             nonce,
                                                             sig)
        else:
            headers['x-container-sync-key'] = user_key

 def _test_listing(self, host, path, title=None, links=[], notins=[],
                   css=None, anonymous=False):
     self._set_staticweb_headers(listings=True,
                                 listings_css=(css is not None))
     if title is None:
         title = unquote(path)
     expected_in = ['Listing of %s' % title] + [
         '<a href="{0}">{1}</a>'.format(quote(link), link)
         for link in links]
     expected_not_in = notins
     if css:
         expected_in.append('<link rel="stylesheet" type="text/css" '
                            'href="%s" />' % quote(css))
     self._test_get_path(host, path, anonymous=anonymous,
                         expected_in=expected_in,
                         expected_not_in=expected_not_in)

    def test_retry_client_exception(self):
        logger = debug_logger('direct-client-test')

        with mock.patch('swift.common.direct_client.sleep') as mock_sleep, \
                mocked_http_conn(500) as conn:
            with self.assertRaises(direct_client.ClientException) as err_ctx:
                direct_client.retry(direct_client.direct_delete_object,
                                    self.node, self.part,
                                    self.account, self.container, self.obj,
                                    retries=2, error_log=logger.error)
        self.assertEqual('DELETE', conn.method)
        self.assertEqual(err_ctx.exception.http_status, 500)
        self.assertIn('DELETE', err_ctx.exception.message)
        self.assertIn(quote('/%s/%s/%s/%s/%s'
                            % (self.node['device'], self.part, self.account,
                               self.container, self.obj)),
                      err_ctx.exception.message)
        self.assertIn(self.node['ip'], err_ctx.exception.message)
        self.assertIn(self.node['port'], err_ctx.exception.message)
        self.assertEqual(self.node['ip'], err_ctx.exception.http_host)
        self.assertEqual(self.node['port'], err_ctx.exception.http_port)
        self.assertEqual(self.node['device'], err_ctx.exception.http_device)
        self.assertEqual(500, err_ctx.exception.http_status)
        self.assertEqual([mock.call(1), mock.call(2)],
                         mock_sleep.call_args_list)
        error_lines = logger.get_lines_for_level('error')
        self.assertEqual(3, len(error_lines))
        for line in error_lines:
            self.assertIn('500 Internal Error', line)

    def _get_container_listing(self, req, version, account, container, prefix, marker=""):
        con_req = make_subrequest(
            req.environ,
            path="/".join(["", version, account, container]),
            method="GET",
            headers={"x-auth-token": req.headers.get("x-auth-token")},
            agent=("%(orig)s " + "DLO MultipartGET"),
            swift_source="DLO",
        )
        con_req.query_string = "format=json&prefix=%s" % quote(prefix)
        if marker:
            con_req.query_string += "&marker=%s" % quote(marker)

        con_resp = con_req.get_response(self.dlo.app)
        if not is_success(con_resp.status_int):
            return con_resp, None
        return None, json.loads("".join(con_resp.app_iter))

 def __init__(self, stype, method, node, part, path, resp):
     full_path = quote('/%s/%s%s' % (node['device'], part, path))
     msg = '%s server %s:%s direct %s %r gave status %s' % (
         stype, node['ip'], node['port'], method, full_path, resp.status)
     headers = HeaderKeyDict(resp.getheaders())
     super(DirectClientException, self).__init__(
         msg, http_host=node['ip'], http_port=node['port'],
         http_device=node['device'], http_status=resp.status,
         http_reason=resp.reason, http_headers=headers)

def validate_device_partition(device, partition):
    """
    Validate that a device and a partition are valid and won't lead to
    directory traversal when used.

    :param device: device to validate
    :param partition: partition to validate
    :raises: ValueError if given an invalid device or partition
    """
    invalid_device = False
    invalid_partition = False
    if not device or '/' in device or device in ['.', '..']:
        invalid_device = True
    if not partition or '/' in partition or partition in ['.', '..']:
        invalid_partition = True

    if invalid_device:
        raise ValueError('Invalid device: %s' % quote(device or ''))
    elif invalid_partition:
        raise ValueError('Invalid partition: %s' % quote(partition or ''))

def _get_direct_account_container(path, stype, node, part,
                                  marker=None, limit=None,
                                  prefix=None, delimiter=None,
                                  conn_timeout=5, response_timeout=15,
                                  end_marker=None, reverse=None, headers=None):
    """Base class for get direct account and container.

    Do not use directly use the get_direct_account or
    get_direct_container instead.
    """
    params = ['format=json']
    if marker:
        params.append('marker=%s' % quote(marker))
    if limit:
        params.append('limit=%d' % limit)
    if prefix:
        params.append('prefix=%s' % quote(prefix))
    if delimiter:
        params.append('delimiter=%s' % quote(delimiter))
    if end_marker:
        params.append('end_marker=%s' % quote(end_marker))
    if reverse:
        params.append('reverse=%s' % quote(reverse))
    qs = '&'.join(params)
    with Timeout(conn_timeout):
        conn = http_connect(node['ip'], node['port'], node['device'], part,
                            'GET', path, query_string=qs,
                            headers=gen_headers(hdrs_in=headers))
    with Timeout(response_timeout):
        resp = conn.getresponse()
    if not is_success(resp.status):
        resp.read()
        raise DirectClientException(stype, 'GET', node, part, path, resp)

    resp_headers = HeaderKeyDict()
    for header, value in resp.getheaders():
        resp_headers[header] = value
    if resp.status == HTTP_NO_CONTENT:
        resp.read()
        return resp_headers, []
    return resp_headers, json.loads(resp.read())