def test_db_migration(self, tempdir, mock_recon):
        db_path = os.path.join(tempdir, 'sda', 'containers', '0', '0', '0',
                               'test.db')
        with test_backend.TestContainerBrokerBeforeSPI.old_broker() as \
                old_ContainerBroker:
            broker = old_ContainerBroker(db_path, account='a', container='c')
            broker.initialize(normalize_timestamp(0), -1)

        with broker.get() as conn:
            try:
                conn.execute('SELECT storage_policy_index '
                             'FROM container_stat')
            except Exception as err:
                self.assertTrue('no such column: storage_policy_index' in
                                str(err))
            else:
                self.fail('TestContainerBrokerBeforeSPI broker class '
                          'was already migrated')

        conf = {'devices': tempdir, 'mount_check': False}
        test_auditor = auditor.ContainerAuditor(conf, logger=debug_logger())
        test_auditor.run_once()

        broker = auditor.ContainerBroker(db_path, account='a', container='c')
        info = broker.get_info()
        expected = {
            'account': 'a',
            'container': 'c',
            'object_count': 0,
            'bytes_used': 0,
            'storage_policy_index': 0,
        }
        for k, v in expected.items():
            self.assertEqual(info[k], v)

    def setUp(self):
        # SOF
        self._orig_hash_suffix = utils.HASH_PATH_SUFFIX
        self._orig_hash_prefix = utils.HASH_PATH_PREFIX
        utils.HASH_PATH_SUFFIX = "endcap"
        utils.HASH_PATH_PREFIX = ""
        self.logger = debug_logger()
        self.container_ring = FakeRing(max_more_nodes=9)
        self.app = proxy_server.Application(
            None, FakeMemcache(), logger=self.logger, account_ring=FakeRing(), container_ring=self.container_ring
        )

        self.account_info = {
            "status": 200,
            "container_count": "10",
            "total_object_count": "100",
            "bytes": "1000",
            "meta": {},
            "sysmeta": {},
        }

        class FakeAccountInfoContainerController(proxy_server.ContainerController):
            def account_info(controller, *args, **kwargs):
                patch_path = "swift.proxy.controllers.base.get_info"
                with mock.patch(patch_path) as mock_get_info:
                    mock_get_info.return_value = dict(self.account_info)
                    return super(FakeAccountInfoContainerController, controller).account_info(*args, **kwargs)

        _orig_get_controller = self.app.get_controller

        def wrapped_get_controller(*args, **kwargs):
            with mock.patch("swift.proxy.server.ContainerController", new=FakeAccountInfoContainerController):
                return _orig_get_controller(*args, **kwargs)

        self.app.get_controller = wrapped_get_controller

 def setUp(self):
     utils.HASH_PATH_SUFFIX = 'endcap'
     utils.HASH_PATH_PREFIX = ''
     self.testdir = mkdtemp()
     ring_file = os.path.join(self.testdir, 'container.ring.gz')
     with closing(GzipFile(ring_file, 'wb')) as f:
         pickle.dump(
             RingData([[0, 1, 2, 0, 1, 2],
                       [1, 2, 0, 1, 2, 0],
                       [2, 3, 1, 2, 3, 1]],
                      [{'id': 0, 'ip': '127.0.0.1', 'port': 1,
                        'device': 'sda1', 'zone': 0},
                       {'id': 1, 'ip': '127.0.0.1', 'port': 1,
                        'device': 'sda1', 'zone': 2},
                       {'id': 2, 'ip': '127.0.0.1', 'port': 1,
                        'device': 'sda1', 'zone': 4},
                       {'id': 3, 'ip': '127.0.0.1', 'port': 1,
                        'device': 'sda1', 'zone': 6}], 30),
             f)
     self.devices_dir = os.path.join(self.testdir, 'devices')
     os.mkdir(self.devices_dir)
     self.sda1 = os.path.join(self.devices_dir, 'sda1')
     os.mkdir(self.sda1)
     for policy in POLICIES:
         os.mkdir(os.path.join(self.sda1, get_tmp_dir(policy)))
     self.logger = debug_logger()
     self.ts_iter = make_timestamp_iter()

    def setUp(self):
        # setup fake rings with handoffs
        self.obj_ring = FakeRing(max_more_nodes=3)
        for policy in POLICIES:
            policy.object_ring = self.obj_ring

        logger = debug_logger('proxy-server')
        logger.thread_locals = ('txn1', '127.0.0.2')
        self.app = PatchedObjControllerApp(
            None, FakeMemcache(), account_ring=FakeRing(),
            container_ring=FakeRing(), logger=logger)

        class FakeContainerInfoObjController(proxy_server.ObjectController):

            def container_info(controller, *args, **kwargs):
                patch_path = 'swift.proxy.controllers.base.get_info'
                with mock.patch(patch_path) as mock_get_info:
                    mock_get_info.return_value = dict(self.container_info)
                    return super(FakeContainerInfoObjController,
                                 controller).container_info(*args, **kwargs)

        # this is taking advantage of the fact that self.app is a
        # PachedObjControllerApp, so handle_response will route into an
        # instance of our FakeContainerInfoObjController just by
        # overriding the class attribute for object_controller
        self.app.object_controller = FakeContainerInfoObjController

 def setUp(self):
     utils.HASH_PATH_SUFFIX = "endcap"
     utils.HASH_PATH_PREFIX = ""
     self.testdir = mkdtemp()
     ring_file = os.path.join(self.testdir, "container.ring.gz")
     with closing(GzipFile(ring_file, "wb")) as f:
         pickle.dump(
             RingData(
                 [[0, 1, 2, 0, 1, 2], [1, 2, 0, 1, 2, 0], [2, 3, 1, 2, 3, 1]],
                 [
                     {"id": 0, "ip": "127.0.0.1", "port": 1, "device": "sda1", "zone": 0},
                     {"id": 1, "ip": "127.0.0.1", "port": 1, "device": "sda1", "zone": 2},
                     {"id": 2, "ip": "127.0.0.1", "port": 1, "device": "sda1", "zone": 4},
                 ],
                 30,
             ),
             f,
         )
     self.devices_dir = os.path.join(self.testdir, "devices")
     os.mkdir(self.devices_dir)
     self.sda1 = os.path.join(self.devices_dir, "sda1")
     os.mkdir(self.sda1)
     for policy in POLICIES:
         os.mkdir(os.path.join(self.sda1, get_tmp_dir(int(policy))))
     self.logger = debug_logger()

    def test_run_once(self):
        def prepare_data_dir():
            devices_path = tempfile.mkdtemp()
            # will be deleted by teardown
            self.to_delete.append(devices_path)
            path = os.path.join(devices_path, 'sda1', DATADIR)
            os.makedirs(path)
            return devices_path

        def init_reaper(devices):
            r = reaper.AccountReaper({'devices': devices})
            return r

        devices = prepare_data_dir()
        r = init_reaper(devices)

        with patch('swift.account.reaper.AccountReaper.reap_device') as foo, \
                unit.mock_check_drive(ismount=True):
            r.run_once()
        self.assertEqual(foo.called, 1)

        with patch('swift.account.reaper.AccountReaper.reap_device') as foo, \
                unit.mock_check_drive(ismount=False):
            r.run_once()
        self.assertFalse(foo.called)

        with patch('swift.account.reaper.AccountReaper.reap_device') as foo:
            r.logger = unit.debug_logger('test-reaper')
            r.devices = 'thisdeviceisbad'
            r.run_once()
        self.assertTrue(r.logger.get_lines_for_level(
            'error')[-1].startswith('Exception in top-level account reaper'))

 def test_reap_object(self):
     conf = {
         'mount_check': 'false',
     }
     r = reaper.AccountReaper(conf, logger=unit.debug_logger())
     ring = unit.FakeRing()
     mock_path = 'swift.account.reaper.direct_delete_object'
     for policy in POLICIES:
         r.reset_stats()
         with patch(mock_path) as fake_direct_delete:
             r.reap_object('a', 'c', 'partition', cont_nodes, 'o',
                           policy.idx)
             for i, call_args in enumerate(
                     fake_direct_delete.call_args_list):
                 cnode = cont_nodes[i]
                 host = '%(ip)s:%(port)s' % cnode
                 device = cnode['device']
                 headers = {
                     'X-Container-Host': host,
                     'X-Container-Partition': 'partition',
                     'X-Container-Device': device,
                     'X-Backend-Storage-Policy-Index': policy.idx
                 }
                 ring = r.get_object_ring(policy.idx)
                 expected = call(ring.devs[i], 1, 'a', 'c', 'o',
                                 headers=headers, conn_timeout=0.5,
                                 response_timeout=10)
                 self.assertEqual(call_args, expected)
         self.assertEqual(r.stats_objects_deleted, 3)

    def test_retry_client_exception(self):
        logger = debug_logger('direct-client-test')

        with mock.patch('swift.common.direct_client.sleep') as mock_sleep, \
                mocked_http_conn(500) as conn:
            with self.assertRaises(direct_client.ClientException) as err_ctx:
                direct_client.retry(direct_client.direct_delete_object,
                                    self.node, self.part,
                                    self.account, self.container, self.obj,
                                    retries=2, error_log=logger.error)
        self.assertEqual('DELETE', conn.method)
        self.assertEqual(err_ctx.exception.http_status, 500)
        self.assertIn('DELETE', err_ctx.exception.message)
        self.assertIn(quote('/%s/%s/%s/%s/%s'
                            % (self.node['device'], self.part, self.account,
                               self.container, self.obj)),
                      err_ctx.exception.message)
        self.assertIn(self.node['ip'], err_ctx.exception.message)
        self.assertIn(self.node['port'], err_ctx.exception.message)
        self.assertEqual(self.node['ip'], err_ctx.exception.http_host)
        self.assertEqual(self.node['port'], err_ctx.exception.http_port)
        self.assertEqual(self.node['device'], err_ctx.exception.http_device)
        self.assertEqual(500, err_ctx.exception.http_status)
        self.assertEqual([mock.call(1), mock.call(2)],
                         mock_sleep.call_args_list)
        error_lines = logger.get_lines_for_level('error')
        self.assertEqual(3, len(error_lines))
        for line in error_lines:
            self.assertIn('500 Internal Error', line)

 def setUp(self):
     self.logger = debug_logger('test-container-sync-store')
     self.logger.level = logging.DEBUG
     self.test_dir_prefix = tempfile.mkdtemp()
     self.devices_dir = os.path.join(self.test_dir_prefix, 'srv/node/')
     os.makedirs(self.devices_dir)
     # Create dummy container dbs
     self.devices = ['sdax', 'sdb', 'sdc']
     self.partitions = ['21765', '38965', '13234']
     self.suffixes = ['312', '435']
     self.hashes = ['f19ed', '53ef', '0ab5', '9c3a']
     for device in self.devices:
         data_dir_path = os.path.join(self.devices_dir,
                                      device,
                                      DATADIR)
         os.makedirs(data_dir_path)
         for part in self.partitions:
             for suffix in self.suffixes:
                 for hsh in self.hashes:
                     db_dir = os.path.join(data_dir_path,
                                           part,
                                           suffix,
                                           hsh)
                     os.makedirs(db_dir)
                     db_file = os.path.join(db_dir, '%s.db' % hsh)
                     with open(db_file, 'w') as outfile:
                         outfile.write('%s' % db_file)

    def test_incr_failed_connection_mid_request(self):
        memcache_client = memcached.MemcacheRing(['1.2.3.4:11211'])
        mock = MockMemcached()
        memcache_client._client_cache['1.2.3.4:11211'] = MockedMemcachePool(
            [(mock, mock)] * 2)
        self.assertEqual(memcache_client.incr('some_key', delta=5), 5)
        self.assertEqual(memcache_client.get('some_key'), '5')
        self.assertEqual(memcache_client.incr('some_key', delta=5), 10)
        self.assertEqual(memcache_client.get('some_key'), '10')

        # Now lets return an empty string, and make sure we aren't logging
        # the error.
        fake_stdout = six.StringIO()
        # force the logging through the DebugLogger instead of the nose
        # handler. This will use stdout, so we can assert that no stack trace
        # is logged.
        logger = debug_logger()
        with patch("sys.stdout", fake_stdout), \
                patch('swift.common.memcached.logging', logger):
            mock.read_return_empty_str = True
            self.assertRaises(memcached.MemcacheConnectionError,
                              memcache_client.incr, 'some_key', delta=1)
        log_lines = logger.get_lines_for_level('error')
        self.assertIn('Error talking to memcached', log_lines[0])
        self.assertFalse(log_lines[1:])
        self.assertNotIn('Traceback', fake_stdout.getvalue())

 def test_reap_object(self):
     conf = {
         'mount_check': 'false',
     }
     r = reaper.AccountReaper(conf, logger=unit.debug_logger())
     mock_path = 'swift.account.reaper.direct_delete_object'
     for policy in POLICIES:
         r.reset_stats()
         with patch(mock_path) as fake_direct_delete:
             with patch('swift.account.reaper.time') as mock_time:
                 mock_time.return_value = 1429117638.86767
                 r.reap_object('a', 'c', 'partition', cont_nodes, 'o',
                               policy.idx)
                 mock_time.assert_called_once_with()
                 for i, call_args in enumerate(
                         fake_direct_delete.call_args_list):
                     cnode = cont_nodes[i % len(cont_nodes)]
                     host = '%(ip)s:%(port)s' % cnode
                     device = cnode['device']
                     headers = {
                         'X-Container-Host': host,
                         'X-Container-Partition': 'partition',
                         'X-Container-Device': device,
                         'X-Backend-Storage-Policy-Index': policy.idx,
                         'X-Timestamp': '1429117638.86767'
                     }
                     ring = r.get_object_ring(policy.idx)
                     expected = call(dict(ring.devs[i], index=i), 0,
                                     'a', 'c', 'o',
                                     headers=headers, conn_timeout=0.5,
                                     response_timeout=10)
                     self.assertEqual(call_args, expected)
                 self.assertEqual(policy.object_ring.replicas - 1, i)
         self.assertEqual(r.stats_objects_deleted,
                          policy.object_ring.replicas)

    def setUp(self):
        TestRingBase.setUp(self)
        self.logger = debug_logger()
        self.container_ring = FakeRing(max_more_nodes=9)
        self.app = proxy_server.Application(None, FakeMemcache(),
                                            logger=self.logger,
                                            account_ring=FakeRing(),
                                            container_ring=self.container_ring)

        self.account_info = {
            'status': 200,
            'container_count': '10',
            'total_object_count': '100',
            'bytes': '1000',
            'meta': {},
            'sysmeta': {},
        }

        class FakeAccountInfoContainerController(
                proxy_server.ContainerController):

            def account_info(controller, *args, **kwargs):
                patch_path = 'swift.proxy.controllers.base.get_info'
                with mock.patch(patch_path) as mock_get_info:
                    mock_get_info.return_value = dict(self.account_info)
                    return super(FakeAccountInfoContainerController,
                                 controller).account_info(
                                     *args, **kwargs)
        _orig_get_controller = self.app.get_controller

        def wrapped_get_controller(*args, **kwargs):
            with mock.patch('swift.proxy.server.ContainerController',
                            new=FakeAccountInfoContainerController):
                return _orig_get_controller(*args, **kwargs)
        self.app.get_controller = wrapped_get_controller

    def setUp(self):
        TestRingBase.setUp(self)
        self.logger = debug_logger()
        self.container_ring = FakeRing(replicas=self.CONTAINER_REPLICAS, max_more_nodes=9)
        self.app = proxy_server.Application(
            None, FakeMemcache(), logger=self.logger, account_ring=FakeRing(), container_ring=self.container_ring
        )

        self.account_info = {
            "status": 200,
            "container_count": "10",
            "total_object_count": "100",
            "bytes": "1000",
            "meta": {},
            "sysmeta": {},
        }

        class FakeAccountInfoContainerController(proxy_server.ContainerController):
            def account_info(controller, *args, **kwargs):
                patch_path = "swift.proxy.controllers.base.get_account_info"
                with mock.patch(patch_path) as mock_get_info:
                    mock_get_info.return_value = dict(self.account_info)
                    return super(FakeAccountInfoContainerController, controller).account_info(*args, **kwargs)

        _orig_get_controller = self.app.get_controller

        def wrapped_get_controller(*args, **kwargs):
            with mock.patch("swift.proxy.server.ContainerController", new=FakeAccountInfoContainerController):
                return _orig_get_controller(*args, **kwargs)

        self.app.get_controller = wrapped_get_controller

 def setUp(self):
     self.testdir = os.path.join(mkdtemp(), "tmp_test_container_auditor")
     self.logger = debug_logger()
     rmtree(self.testdir, ignore_errors=1)
     os.mkdir(self.testdir)
     fnames = ["true1.db", "true2.db", "true3.db", "fail1.db", "fail2.db"]
     for fn in fnames:
         with open(os.path.join(self.testdir, fn), "w+") as f:
             f.write(" ")

 def setUp(self):
     self.testdir = os.path.join(mkdtemp(), 'tmp_test_account_auditor')
     self.logger = debug_logger()
     rmtree(self.testdir, ignore_errors=1)
     os.mkdir(self.testdir)
     fnames = ['true1.db', 'true2.db', 'true3.db',
               'fail1.db', 'fail2.db']
     for fn in fnames:
         with open(os.path.join(self.testdir, fn), 'w+') as f:
             f.write(' ')

    def setUp(self):
        self.app = proxy.Application(None, FakeMemcache(),
                                     logger=debug_logger('proxy-ut'),
                                     account_ring=FakeRing(replicas=1),
                                     container_ring=FakeRing(replicas=1))
        monkey_patch_mimetools()
        self.testdir = \
            os.path.join(mkdtemp(), 'tmp_test_object_server_ObjectController')
        mkdirs(os.path.join(self.testdir, 'sda1', 'tmp'))
        conf = {'devices': self.testdir, 'mount_check': 'false'}
        self.obj_ctlr = object_server.ObjectController(
            conf, logger=debug_logger('obj-ut'))

        http_connect = get_http_connect(fake_http_connect(200),
                                        fake_http_connect(200),
                                        FakeServerConnection(self.obj_ctlr))

        swift.proxy.controllers.base.http_connect = http_connect
        swift.proxy.controllers.obj.http_connect = http_connect

    def init_reaper(self, conf=None, myips=None, fakelogger=False):
        if conf is None:
            conf = {}
        if myips is None:
            myips = ['10.10.10.1']

        r = reaper.AccountReaper(conf)
        r.myips = myips
        if fakelogger:
            r.logger = unit.debug_logger('test-reaper')
        return r

 def __init__(self, testdir, policy=None):
     self.logger = debug_logger("test-ssync-sender")
     self.conn_timeout = 1
     self.node_timeout = 2
     self.http_timeout = 3
     self.network_chunk_size = 65536
     self.disk_chunk_size = 4096
     conf = {"devices": testdir, "mount_check": "false"}
     policy = POLICIES.default if policy is None else policy
     self._diskfile_router = diskfile.DiskFileRouter(conf, self.logger)
     self._diskfile_mgr = self._diskfile_router[policy]

 def setUp(self):
     self.tmpdir = mkdtemp()
     self.testdir = os.path.join(self.tmpdir,
                                 'tmp_test_sof_TestObjectController')
     conf = {'devices': self.testdir, 'mount_check': 'false'}
     self.object_controller = object_server.ObjectController(
         conf, logger=debug_logger())
     self.object_controller.bytes_per_sync = 1
     self._orig_tpool_exc = tpool.execute
     tpool.execute = lambda f, *args, **kwargs: f(*args, **kwargs)
     self.df_mgr = diskfile.DiskFileManager(conf,
                                            self.object_controller.logger)

    def setUp(self):
        global not_sleep

        self.old_loadapp = internal_client.loadapp
        self.old_sleep = internal_client.sleep

        internal_client.loadapp = lambda *a, **kw: None
        internal_client.sleep = not_sleep

        self.rcache = mkdtemp()
        self.conf = {'recon_cache_path': self.rcache}
        self.logger = debug_logger('test-recon')