def handle_request(self, env, req):
        """
        Handles the POST /origin/.prep call for preparing the backing store
        Swift cluster for use with the origin subsystem. Can only be called by
        .origin_admin

        :param req: The webob.Request to process.
        :returns: webob.Response, 204 on success
        """
        if not self.is_origin_admin(req):
            return HTTPForbidden(request=req)
        try:
            vsn, account = split_path(req.path, 2, 2)
        except ValueError:
            return HTTPBadRequest(request=req)
        if account == '.prep':
            path = '/v1/%s' % self.origin_account
            resp = make_pre_authed_request(req.environ, 'PUT',
                path, agent='SwiftOrigin').get_response(self.app)
            if resp.status_int // 100 != 2:
                raise Exception(
                    'Could not create the main origin account: %s %s' %
                    (path, resp.status))
            for i in xrange(self.num_hash_cont):
                cont_name = '.hash_%d' % i
                path = '/v1/%s/%s' % (self.origin_account, cont_name)
                resp = make_pre_authed_request(req.environ, 'PUT',
                    path, agent='SwiftOrigin').get_response(self.app)
                if resp.status_int // 100 != 2:
                    raise Exception('Could not create %s container: %s %s' %
                                    (cont_name, path, resp.status))
            return HTTPNoContent(request=req)
        return HTTPNotFound(request=req)

 def test_pre_auth_req_drops_query(self):
     r = wsgi.make_pre_authed_request({"QUERY_STRING": "original"}, "GET", "path")
     self.assertEquals(r.query_string, "original")
     r = wsgi.make_pre_authed_request({"QUERY_STRING": "original"}, "GET", "path?replacement")
     self.assertEquals(r.query_string, "replacement")
     r = wsgi.make_pre_authed_request({"QUERY_STRING": "original"}, "GET", "path?")
     self.assertEquals(r.query_string, "")

    def _set_hash_data(self, env, cdn_obj_path, new_hash_data,
                       update_listings=True):
        """
        Actually sets the data in the .origin account. If not successful on
        any of the several updates this has to do, will raise a OriginDbFailure
        """
        cdn_obj_data = new_hash_data.get_json_str()
        cdn_obj_etag = md5(cdn_obj_data).hexdigest()
        # this is always a PUT because a POST needs to update the file
        cdn_obj_resp = make_pre_authed_request(
            env, 'PUT', cdn_obj_path, body=cdn_obj_data,
            headers={'Etag': cdn_obj_etag},
            agent='SwiftOrigin', swift_source='SOS').get_response(self.app)

        if cdn_obj_resp.status_int // 100 != 2:
            raise OriginDbFailure(
                'Could not PUT .hash obj in origin '
                'db: %s %s' % (cdn_obj_path, cdn_obj_resp.status_int))

        memcache_client = utils.cache_from_env(env)
        if memcache_client:
            memcache_key = self.cdn_data_memcache_key(cdn_obj_path)
            memcache_client.delete(memcache_key)

        if not update_listings:
            return

        listing_cont_path = quote('/v1/%s/%s' % (self.origin_account,
                                                 new_hash_data.account))
        resp = make_pre_authed_request(
            env, 'HEAD', listing_cont_path,
            agent='SwiftOrigin', swift_source='SOS').get_response(self.app)
        if resp.status_int == 404:
            # create new container for listings
            resp = make_pre_authed_request(
                env, 'PUT', listing_cont_path,
                agent='SwiftOrigin', swift_source='SOS').get_response(self.app)
            if resp.status_int // 100 != 2:
                raise OriginDbFailure(
                    'Could not create listing container '
                    'in origin db: %s %s' % (listing_cont_path, resp.status))

        cdn_list_path = quote('/v1/%s/%s/%s' % (
            self.origin_account, new_hash_data.account.encode('utf-8'),
            new_hash_data.container.encode('utf-8')))

        listing_content_type = new_hash_data.gen_listing_content_type()

        cdn_list_resp = make_pre_authed_request(
            env, 'PUT', cdn_list_path,
            headers={'Content-Type': listing_content_type,
                     'Content-Length': 0},
            agent='SwiftOrigin', swift_source='SOS').get_response(self.app)

        if cdn_list_resp.status_int // 100 != 2:
            raise OriginDbFailure(
                'Could not PUT/POST to cdn listing in '
                'origin db: %s %s' % (cdn_list_path, cdn_list_resp.status_int))

 def test_pre_auth_req_drops_query(self):
     r = wsgi.make_pre_authed_request(
         {'QUERY_STRING': 'original'}, 'GET', 'path')
     self.assertEquals(r.query_string, 'original')
     r = wsgi.make_pre_authed_request(
         {'QUERY_STRING': 'original'}, 'GET', 'path?replacement')
     self.assertEquals(r.query_string, 'replacement')
     r = wsgi.make_pre_authed_request(
         {'QUERY_STRING': 'original'}, 'GET', 'path?')
     self.assertEquals(r.query_string, '')

    def test_pre_auth_req(self):
        class FakeReq(object):
            @classmethod
            def fake_blank(cls, path, environ={}, body="", headers={}):
                self.assertEquals(environ["swift.authorize"]("test"), None)
                self.assertFalse("HTTP_X_TRANS_ID" in environ)

        was_blank = Request.blank
        Request.blank = FakeReq.fake_blank
        wsgi.make_pre_authed_request({"HTTP_X_TRANS_ID": "1234"}, "PUT", "/", body="tester", headers={})
        wsgi.make_pre_authed_request({"HTTP_X_TRANS_ID": "1234"}, "PUT", "/", headers={})
        Request.blank = was_blank

 def test_pre_auth_req(self):
     class FakeReq(object):
         @classmethod
         def fake_blank(cls, path, environ={}, body='', headers={}):
             self.assertEquals(environ['swift.authorize']('test'), None)
             self.assertFalse('HTTP_X_TRANS_ID' in environ)
     was_blank = Request.blank
     Request.blank = FakeReq.fake_blank
     wsgi.make_pre_authed_request({'HTTP_X_TRANS_ID': '1234'},
                                  'PUT', '/', body='tester', headers={})
     wsgi.make_pre_authed_request({'HTTP_X_TRANS_ID': '1234'},
                                  'PUT', '/', headers={})
     Request.blank = was_blank

 def test_pre_auth_req_with_env_path_and_script(self):
     env = {"PATH_INFO": "/unquoted path with %20", "SCRIPT_NAME": "/script"}
     r = wsgi.make_pre_authed_request(env, "GET")
     expected_path = quote(env["SCRIPT_NAME"] + env["PATH_INFO"])
     self.assertEquals(r.path, expected_path)
     env = {"PATH_INFO": "", "SCRIPT_NAME": "/script"}
     r = wsgi.make_pre_authed_request(env, "GET")
     self.assertEquals(r.path, "/script")
     env = {"PATH_INFO": "/path", "SCRIPT_NAME": ""}
     r = wsgi.make_pre_authed_request(env, "GET")
     self.assertEquals(r.path, "/path")
     env = {"PATH_INFO": "", "SCRIPT_NAME": ""}
     r = wsgi.make_pre_authed_request(env, "GET")
     self.assertEquals(r.path, "")

 def test_pre_auth_req_with_env_path_and_script(self):
     env = {'PATH_INFO': '/unquoted path with %20',
            'SCRIPT_NAME': '/script'}
     r = wsgi.make_pre_authed_request(env, 'GET')
     expected_path = quote(env['SCRIPT_NAME'] + env['PATH_INFO'])
     self.assertEquals(r.path, expected_path)
     env = {'PATH_INFO': '', 'SCRIPT_NAME': '/script'}
     r = wsgi.make_pre_authed_request(env, 'GET')
     self.assertEquals(r.path, '/script')
     env = {'PATH_INFO': '/path', 'SCRIPT_NAME': ''}
     r = wsgi.make_pre_authed_request(env, 'GET')
     self.assertEquals(r.path, '/path')
     env = {'PATH_INFO': '', 'SCRIPT_NAME': ''}
     r = wsgi.make_pre_authed_request(env, 'GET')
     self.assertEquals(r.path, '')

    def remove_chunks(self, env, chunks_names, container):
        error = False
        self.app.logger.info('StackSync API: internal remove uploaded chunks: container: %s', str(container))
        for chunk_name in chunks_names:

            env_aux = env.copy()
            new_path = "/v1/" + env['stacksync_user_account'] + "/" + container + "/" + str(chunk_name)
            del env_aux['HTTP_STACKSYNC_API']
            seg_req = make_pre_authed_request(env_aux, method='DELETE', path=new_path,
                                              agent=str(container))

            seg_resp = seg_req.get_response(self.app)

            if not is_valid_status(seg_resp.status_int):
                self.app.logger.error('StackSync API: remove_chunks: error deleting uploaded chunks %s', str(chunk_name))
                error = True
                break

        if error:
            self.app.logger.error(
                'StackSync API: upload_file_chunks: status: %s description: Error uploading chunks to storage backend',
                seg_resp.status)
            return False

        return True

    def upload_file_chunks(self, env, chunked_file, container):
        error = False
        self.app.logger.info('StackSync API: upload_file_chunks: container: %s', str(container))
        upload_chunks = []
        for i in range(len(chunked_file.chunks)):
            chunk_name = chunked_file.name_list[i-1]
            chunk_content = chunked_file.chunks[i-1]

            env_aux = env.copy()
            new_path = "/v1/" + env['stacksync_user_account'] + "/" + container + "/" + chunk_name
            del env_aux['HTTP_STACKSYNC_API']
            seg_req = make_pre_authed_request(env_aux, method='PUT', path=new_path, body=chunk_content,
                                              agent=str(container))

            seg_resp = seg_req.get_response(self.app)

            if not is_valid_status(seg_resp.status_int):
                self.app.logger.error('StackSync API: upload_file_chunks: error uploading chunk %s', chunk_name)
                error = True
                break
            upload_chunks.append(chunk_name)

        if error:
            self.app.logger.error(
                'StackSync API: upload_file_chunks: status: %s description: Error uploading chunks to storage backend',
                seg_resp.status)
            response = create_error_response(500, "Error uploading chunks to storage backend")
            self.remove_chunks(env, upload_chunks, container)

        else:
            response = HTTPCreated()

        return response

    def _listing_pages_iter(self, account_name, lcontainer, lprefix, env):
        marker = ''
        while True:
            lreq = make_pre_authed_request(
                env, method='GET', swift_source='VW',
                path='/v1/%s/%s' % (account_name, lcontainer))
            lreq.environ['QUERY_STRING'] = \
                'format=json&prefix=%s&marker=%s' % (quote(lprefix),
                                                     quote(marker))
            lresp = lreq.get_response(self.app)
            if not is_success(lresp.status_int):
                if lresp.status_int == HTTP_NOT_FOUND:
                    raise ListingIterNotFound()
                elif is_client_error(lresp.status_int):
                    raise HTTPPreconditionFailed()
                else:
                    raise ListingIterError()

            if not lresp.body:
                break

            sublisting = json.loads(lresp.body)
            if not sublisting:
                break
            marker = sublisting[-1]['name'].encode('utf-8')
            yield sublisting

    def origin_db_get(self, env, req):
        '''
        Handles GETs to the Origin database
        The only part of the path this pays attention to is the account.
        '''
        #TODO: this does not return transfer-encoding: chunked
        try:
            account = req.path.split('/')[2]
        except IndexError:
            return HTTPBadRequest('Invalid request. '
                                  'URI format: /<api version>/<account>')
        #TODO: make sure to test with unicode container names
        marker = get_param(req, 'marker', default='')
        list_format = get_param(req, 'format')
        enabled_only = get_param(req, 'enabled',
                                 default='false') in TRUE_VALUES
        limit = get_param(req, 'limit')
        if limit:
            try:
                limit = int(limit)
            except ValueError:
                return HTTPBadRequest('Invalid limit, must be an integer')
        listing_path = '/v1/%s/%s?format=json&marker=%s' % \
                       (self.origin_account, account, marker)
        # no limit in request because may have to filter on cdn_enabled
        resp = make_pre_authed_request(env, 'GET',
            listing_path, agent='SwiftOrigin').get_response(self.app)
        resp_headers = {}
        # {'Transfer-Encoding': 'chunked'}
        #TODO is this right? was chunked in old one
        if resp.status_int // 100 == 2:
            cont_listing = json.loads(resp.body)
            # TODO: is it ok to load the whole thing? do i have a choice?
            listing_formatted = []
            for listing_dict in cont_listing:
                if limit is None or len(listing_formatted) < limit:
                    try:
                        formatted_data = self._parse_container_listing(
                            account, listing_dict, list_format,
                            only_cdn_enabled=enabled_only)
                        if formatted_data:
                            listing_formatted.append(formatted_data)
                    except InvalidContentType, e:
                        self.logger.exception(e)
                        continue
                else:
                    break
            if list_format == 'xml':
                resp_headers['Content-Type'] = 'application/xml'
                response_body = ('<?xml version="1.0" encoding="UTF-8"?>\n'
                    '<account name="%s">\n%s\n</account>') % (account,
                        '\n'.join(listing_formatted))

            elif list_format == 'json':
                resp_headers['Content-Type'] = 'application/json'
                response_body = json.dumps(listing_formatted)
            else:
                resp_headers['Content-Type'] = 'text/plain; charset=UTF-8'
                response_body = '\n'.join(listing_formatted)
            return Response(body=response_body, headers=resp_headers)

def get_account_info(env, app):
    """
    Get the info structure for an account, based on env and app.
    This is useful to middlewares.
    Note: This call bypasses auth. Success does not imply that the
          request has authorization to the account_info.
    """
    
    (version, account, container, _) = \
        split_path(env['PATH_INFO'], 2, 4, True)
    
    new_env = env.copy()
    obj_path = '/%s/%s' % (version, account)
    if isinstance(obj_path, unicode):
        obj_path = obj_path.encode('utf-8')
    new_env['PATH_INFO'] = obj_path
    if new_env.has_key('wsgi.input'):
        del(new_env['wsgi.input'])
    if new_env.has_key('QUERY_STRING'):
        del(new_env['QUERY_STRING'])
    new_env['CONTENT_LENGTH'] = 0
                
    resp = make_pre_authed_request(new_env, 'HEAD', '/%s/%s' % (version, account)).get_response(app)
    account_info = headers_to_account_info(
        resp.headers, resp.status_int)
        
    return account_info

def iter_objects_by_prefix(account, container, prefix, swift_client=None,
                           app=None):
    marker = ''
    while True:
        param = 'format=json&marker=%s' % marker
        if marker == '':
            param = '%s&prefix=%s' % (param, prefix)

        if swift_client:
            path = swift_client.make_path(account, container)
            resp = swift_client.make_request('GET', '%s?%s' % (path, param),
                                             {}, (2, 4))
        elif app:
            path = '/v1/%s/%s' % (account, container)
            env = make_pre_authed_env({}, method='GET', path=path,
                                      query_string=param)
            req = make_pre_authed_request(env)
            resp = req.get_response(app)

        if not resp.status_int == 200:
            break

        data = json.loads(resp.body)
        if not data:
            break
        for item in data:
            yield item
        marker = data[-1]['name'].encode('utf8')

 def _put_versioned_obj(self, req, put_path_info, source_resp):
     # Create a new Request object to PUT to the versions container, copying
     # all headers from the source object apart from x-timestamp.
     put_req = make_pre_authed_request(req.environ, path=put_path_info, method="PUT", swift_source="VW")
     copy_header_subset(source_resp, put_req, lambda k: k.lower() != "x-timestamp")
     put_req.environ["wsgi.input"] = FileLikeIter(source_resp.app_iter)
     return put_req.get_response(self.app)

    def _get_container_info(self, env):
        """
        Retrieves x-container-meta-web-index, x-container-meta-web-error,
        x-container-meta-web-listings, and x-container-meta-web-listings-css
        from memcache or from the cluster and stores the result in memcache and
        in self._index, self._error, self._listings, and self._listings_css.

        :param env: The WSGI environment dict.
        """
        self._index = self._error = self._listings = self._listings_css = None
        memcache_client = cache_from_env(env)
        if memcache_client:
            memcache_key = "/staticweb/%s/%s/%s" % (self.version, self.account, self.container)
            cached_data = memcache_client.get(memcache_key)
            if cached_data:
                (self._index, self._error, self._listings, self._listings_css) = cached_data
                return
        resp = make_pre_authed_request(
            env, "HEAD", "/%s/%s/%s" % (self.version, self.account, self.container), agent=self.agent
        ).get_response(self.app)
        if is_success(resp.status_int):
            self._index = resp.headers.get("x-container-meta-web-index", "").strip()
            self._error = resp.headers.get("x-container-meta-web-error", "").strip()
            self._listings = resp.headers.get("x-container-meta-web-listings", "").strip()
            self._listings_css = resp.headers.get("x-container-meta-web-listings-css", "").strip()
            if memcache_client:
                memcache_client.set(
                    memcache_key,
                    (self._index, self._error, self._listings, self._listings_css),
                    timeout=self.cache_timeout,
                )

def get_account_info(env, app, swift_source=None):
    """
    Get the info structure for an account, based on env and app.
    This is useful to middlewares.
    Note: This call bypasses auth. Success does not imply that the
          request has authorization to the account_info.
    """
    cache = cache_from_env(env)
    if not cache:
        return None
    (version, account, _junk, _junk) = \
        split_path(env['PATH_INFO'], 2, 4, True)
    cache_key = get_account_memcache_key(account)
    # Use a unique environment cache key per account.  If you copy this env
    # to make a new request, it won't accidentally reuse the old account info
    env_key = 'swift.%s' % cache_key
    if env_key not in env:
        account_info = cache.get(cache_key)
        if not account_info:
            resp = make_pre_authed_request(
                env, 'HEAD', '/%s/%s' % (version, account),
                swift_source=swift_source,
            ).get_response(app)
            account_info = headers_to_account_info(
                resp.headers, resp.status_int)
        env[env_key] = account_info
    return env[env_key]

    def iter_objects(self, env, path, prefix, marker, end, count):
        path_with_params = '%s?format=json&prefix=%s' % (path, prefix)
        seg = ''
        force_break = False
        while count > 0:
            l = 1000 if count > 1000 else count
            count -= 1000
            rpath = path_with_params + ('&marker=%s' % marker) + (
                '&limit=%d' % l)
            req = make_pre_authed_request(env, 'GET', rpath)
            req.environ['swift.proxy_access_log_made'] = True
            resp = req.get_response(self.app)
            segments = json.loads(resp.body)
            for seg in segments:
                name = seg['name']
                record_ts = int(name.split('/')[1])
                if record_ts > end:
                    force_break = True
                    break
                yield name

            if force_break:
                break

            if len(segments) != l:
                break

            if segments:
                marker = seg['name']
            else:
                break

 def get_listings(marker):
     listing_path = quote('/v1/%s/%s' % (self.origin_account, account))
     listing_path += '?format=json&marker=' + quote(marker)
     # no limit in request because may have to filter on cdn_enabled
     resp = make_pre_authed_request(
         env, 'GET', listing_path,
         agent='SwiftOrigin', swift_source='SOS').get_response(self.app)
     resp_headers = {}
     listing_formatted = []
     if resp.status_int // 100 == 2:
         cont_listing = json.loads(resp.body)
         for listing_dict in cont_listing:
             if limit is None or len(listing_formatted) < limit:
                 try:
                     formatted_data = self._parse_container_listing(
                         account, listing_dict, list_format,
                         only_cdn_enabled=enabled_only)
                     if formatted_data:
                         listing_formatted.append(formatted_data)
                 except InvalidContentType, e:
                     self.logger.exception(e)
                     continue
             else:
                 break
         if cont_listing and not listing_formatted:
             # there were rows returned but none matched enabled_only-
             # requery with new marker
             new_marker = cont_listing[-1]['name']
             if isinstance(new_marker, unicode):
                 new_marker = new_marker.encode('utf-8')
             return get_listings(new_marker)

def get_container_info(env, app, swift_source=None):
    """
    Get the info structure for a container, based on env and app.
    This is useful to middlewares.
    """
    cache = cache_from_env(env)
    if not cache:
        return None
    (version, account, container, obj) = \
        split_path(env['PATH_INFO'], 2, 4, True)
    cache_key = get_container_memcache_key(account, container)
    # Use a unique environment cache key per container.  If you copy this env
    # to make a new request, it won't accidentally reuse the old container info
    env_key = 'swift.%s' % cache_key
    if env_key not in env:
        container_info = cache.get(cache_key)
        if not container_info:
            resp = make_pre_authed_request(
                env, 'HEAD', '/%s/%s/%s' % (version, account, container),
                swift_source=swift_source,
            ).get_response(app)
            container_info = headers_to_container_info(
                resp.headers, resp.status_int)
        env[env_key] = container_info
    return env[env_key]