def test_fixture_when_explicitly_passing_down_cell_mappings(self):
     # The test setup creates two cell mappings (cell0 and cell1) by
     # default. We'll create one instance per cell and pass cell0 as
     # the down cell. We should thus get db_exc.DBError for cell0 and
     # correct InstanceList object from cell1.
     ctxt = context.get_admin_context()
     cell0 = self.cell_mappings['cell0']
     cell1 = self.cell_mappings['cell1']
     with context.target_cell(ctxt, cell0) as cctxt:
         inst1 = fake_instance.fake_instance_obj(cctxt)
         if 'id' in inst1:
             delattr(inst1, 'id')
         inst1.create()
     with context.target_cell(ctxt, cell1) as cctxt:
         inst2 = fake_instance.fake_instance_obj(cctxt)
         if 'id' in inst2:
             delattr(inst2, 'id')
         inst2.create()
     with fixtures.DownCellFixture([cell0]):
         results = context.scatter_gather_all_cells(
             ctxt, objects.InstanceList.get_all)
         self.assertEqual(2, len(results))
         for cell_uuid, result in results.items():
             if cell_uuid == cell0.uuid:
                 self.assertIsInstance(result, db_exc.DBError)
             else:
                 self.assertIsInstance(result, objects.InstanceList)
                 self.assertEqual(1, len(result))
                 self.assertEqual(inst2.uuid, result[0].uuid)

    def test_fixture_for_an_individual_down_cell_targeted_call(self):
        # We have cell0 and cell1 by default in the setup. We try targeting
        # both the cells. We should get a db error for the down cell and
        # the correct result for the up cell.
        ctxt = context.get_admin_context()
        cell0 = self.cell_mappings['cell0']
        cell1 = self.cell_mappings['cell1']
        with context.target_cell(ctxt, cell0) as cctxt:
            inst1 = fake_instance.fake_instance_obj(cctxt)
            if 'id' in inst1:
                delattr(inst1, 'id')
            inst1.create()
        with context.target_cell(ctxt, cell1) as cctxt:
            inst2 = fake_instance.fake_instance_obj(cctxt)
            if 'id' in inst2:
                delattr(inst2, 'id')
            inst2.create()

        def dummy_tester(ctxt, cell_mapping, uuid):
            with context.target_cell(ctxt, cell_mapping) as cctxt:
                return objects.Instance.get_by_uuid(cctxt, uuid)

        # Scenario A: We do not pass any down cells, fixture automatically
        # assumes the targeted cell is down whether its cell0 or cell1.
        with fixtures.DownCellFixture():
            self.assertRaises(
                db_exc.DBError, dummy_tester, ctxt, cell1, inst2.uuid)
        # Scenario B: We pass cell0 as the down cell.
        with fixtures.DownCellFixture([cell0]):
            self.assertRaises(
                db_exc.DBError, dummy_tester, ctxt, cell0, inst1.uuid)
            # Scenario C: We get the correct result from the up cell
            # when targeted.
            result = dummy_tester(ctxt, cell1, inst2.uuid)
            self.assertEqual(inst2.uuid, result.uuid)

    def _create_instances(self, pre_newton=2, deleted=0, total=5,
                          target_cell=None):
        if not target_cell:
            target_cell = self.cells[1]

        instances = []
        with context.target_cell(self.context, target_cell) as cctxt:
            flav_dict = objects.Flavor._flavor_get_from_db(cctxt, 1)
            flavor = objects.Flavor(**flav_dict)
            for i in range(0, total):
                inst = objects.Instance(
                    context=cctxt,
                    project_id=self.api.project_id,
                    user_id=FAKE_UUID,
                    vm_state='active',
                    flavor=flavor,
                    created_at=datetime.datetime(1985, 10, 25, 1, 21, 0),
                    launched_at=datetime.datetime(1985, 10, 25, 1, 22, 0),
                    host=self.computes[0].host,
                    hostname='%s-inst%i' % (target_cell.name, i))
                inst.create()

                info_cache = objects.InstanceInfoCache(context=cctxt)
                info_cache.updated_at = timeutils.utcnow()
                info_cache.network_info = network_model.NetworkInfo()
                info_cache.instance_uuid = inst.uuid
                info_cache.save()

                instances.append(inst)

                im = objects.InstanceMapping(context=cctxt,
                    project_id=inst.project_id,
                    user_id=inst.user_id,
                    instance_uuid=inst.uuid,
                    cell_mapping=target_cell)
                im.create()

        # Attach fake interfaces to instances
        network_id = list(self.neutron._networks.keys())[0]
        for i in range(0, len(instances)):
            for k in range(0, 4):
                self.api.attach_interface(instances[i].uuid,
                    {"interfaceAttachment": {"net_id": network_id}})

        with context.target_cell(self.context, target_cell) as cctxt:
            # Fake the pre-newton behaviour by removing the
            # VirtualInterfacesList objects.
            if pre_newton:
                for i in range(0, pre_newton):
                    _delete_vif_list(cctxt, instances[i].uuid)

        if deleted:
            # Delete from the end of active instances list
            for i in range(total - deleted, total):
                instances[i].destroy()

        self.instances += instances

    def delete_all(self, req, server_id):
        context = req.environ["nova.context"]
        context.can(st_policies.POLICY_ROOT % 'delete_all')
        im = _get_instance_mapping(context, server_id)

        with nova_context.target_cell(context, im.cell_mapping) as cctxt:
            self._check_instance_in_valid_state(cctxt, server_id,
                                                'delete tags')

        try:
            with nova_context.target_cell(context, im.cell_mapping) as cctxt:
                objects.TagList.destroy(cctxt, server_id)
        except exception.InstanceNotFound as e:
            raise webob.exc.HTTPNotFound(explanation=e.format_message())

    def update(self, req, server_id, id, body):
        context = req.environ["nova.context"]
        context.can(st_policies.POLICY_ROOT % 'update')
        im = _get_instance_mapping(context, server_id)

        with nova_context.target_cell(context, im.cell_mapping) as cctxt:
            instance = self._check_instance_in_valid_state(
                cctxt, server_id, 'update tag')

        try:
            jsonschema.validate(id, parameter_types.tag)
        except jsonschema.ValidationError as e:
            msg = (_("Tag '%(tag)s' is invalid. It must be a non empty string "
                     "without characters '/' and ','. Validation error "
                     "message: %(err)s") % {'tag': id, 'err': e.message})
            raise webob.exc.HTTPBadRequest(explanation=msg)

        try:
            with nova_context.target_cell(context, im.cell_mapping) as cctxt:
                tags = objects.TagList.get_by_resource_id(cctxt, server_id)
        except exception.InstanceNotFound as e:
            raise webob.exc.HTTPNotFound(explanation=e.format_message())

        if len(tags) >= objects.instance.MAX_TAG_COUNT:
            msg = (_("The number of tags exceeded the per-server limit %d")
                   % objects.instance.MAX_TAG_COUNT)
            raise webob.exc.HTTPBadRequest(explanation=msg)

        if id in _get_tags_names(tags):
            # NOTE(snikitin): server already has specified tag
            return webob.Response(status_int=204)

        try:
            with nova_context.target_cell(context, im.cell_mapping) as cctxt:
                tag = objects.Tag(context=cctxt, resource_id=server_id, tag=id)
                tag.create()
                instance.tags = objects.TagList.get_by_resource_id(cctxt,
                                                                   server_id)
        except exception.InstanceNotFound as e:
            raise webob.exc.HTTPNotFound(explanation=e.format_message())

        notifications_base.send_instance_update_notification(
            context, instance, service="nova-api")

        response = webob.Response(status_int=201)
        response.headers['Location'] = self._view_builder.get_location(
            req, server_id, id)
        return response

    def test_connection_switch(self):
        # Use a file-based sqlite database so data will persist across new
        # connections
        fake_conn = 'sqlite:///' + self.test_filename

        # The 'main' database connection will stay open, so in-memory is fine
        self.useFixture(fixtures.Database(database='main'))
        self.useFixture(fixtures.Database(connection=fake_conn))

        # Make a request context with a cell mapping
        mapping = objects.CellMapping(database_connection=fake_conn)
        # In the tests, the admin context is required in order to read
        # an Instance back after write, for some reason
        ctxt = context.get_admin_context()
        # Create an instance in the cell database
        uuid = uuidutils.generate_uuid()
        with context.target_cell(ctxt, mapping):
            instance = objects.Instance(context=ctxt, uuid=uuid)
            instance.create()

            # Verify the instance is found in the cell database
            inst = objects.Instance.get_by_uuid(ctxt, uuid)
            self.assertEqual(uuid, inst.uuid)

        # Verify the instance isn't found in the main database
        self.assertRaises(exception.InstanceNotFound,
                          objects.Instance.get_by_uuid, ctxt, uuid)

    def _create_service_in_cell(ctxt, cell, binary, is_deleted=False,
                                disabled=False, version=None,
                                create_token_auth=False):
        with context.target_cell(ctxt, cell) as cctxt:
            service = objects.Service(context=cctxt, binary=binary,
                                      disabled=disabled, host='dontcare')
            if version:
                service.version = version
            service.create()

            if is_deleted:
                service.destroy()

            if create_token_auth:
                # We have to create an instance in order to create a token
                # auth.
                inst = objects.Instance(context=cctxt,
                                        uuid=uuidutils.generate_uuid())
                inst.create()
                auth = objects.ConsoleAuthToken(context=cctxt,
                                                console_type='novnc',
                                                host='hostname', port=6080,
                                                instance_uuid=inst.uuid)
                auth.authorize(CONF.consoleauth.token_ttl)

        return service

    def update_all(self, req, server_id, body):
        context = req.environ["nova.context"]
        context.can(st_policies.POLICY_ROOT % 'update_all')
        im = _get_instance_mapping(context, server_id)

        with nova_context.target_cell(context, im.cell_mapping) as cctxt:
            self._check_instance_in_valid_state(cctxt, server_id,
                                                'update tags')

        try:
            with nova_context.target_cell(context, im.cell_mapping) as cctxt:
                tags = objects.TagList.create(cctxt, server_id, body['tags'])
        except exception.InstanceNotFound as e:
            raise webob.exc.HTTPNotFound(explanation=e.format_message())

        return {'tags': _get_tags_names(tags)}

def get_metadata_by_instance_id(instance_id, address, ctxt=None):
    ctxt = ctxt or context.get_admin_context()
    attrs = ['ec2_ids', 'flavor', 'info_cache',
             'metadata', 'system_metadata',
             'security_groups', 'keypairs',
             'device_metadata']

    if CONF.api.local_metadata_per_cell:
        instance = objects.Instance.get_by_uuid(ctxt, instance_id,
                                                expected_attrs=attrs)
        return InstanceMetadata(instance, address)

    try:
        im = objects.InstanceMapping.get_by_instance_uuid(ctxt, instance_id)
    except exception.InstanceMappingNotFound:
        LOG.warning('Instance mapping for %(uuid)s not found; '
                    'cell setup is incomplete', {'uuid': instance_id})
        instance = objects.Instance.get_by_uuid(ctxt, instance_id,
                                                expected_attrs=attrs)
        return InstanceMetadata(instance, address)

    with context.target_cell(ctxt, im.cell_mapping) as cctxt:
        instance = objects.Instance.get_by_uuid(cctxt, instance_id,
                                                expected_attrs=attrs)
        return InstanceMetadata(instance, address)

def _get_not_deleted(context, uuids):
    mappings = objects.InstanceMappingList.get_by_instance_uuids(
        context, uuids)
    inst_by_cell = collections.defaultdict(list)
    cell_mappings = {}
    found_inst_uuids = []

    # Get a master list of cell mappings, and a list of instance
    # uuids organized by cell
    for im in mappings:
        if not im.cell_mapping:
            # Not scheduled yet, so just throw it in the final list
            # and move on
            found_inst_uuids.append(im.instance_uuid)
            continue
        if im.cell_mapping.uuid not in cell_mappings:
            cell_mappings[im.cell_mapping.uuid] = im.cell_mapping
        inst_by_cell[im.cell_mapping.uuid].append(im.instance_uuid)

    # Query each cell for the instances that are inside, building
    # a list of non-deleted instance uuids.
    for cell_uuid, cell_mapping in cell_mappings.items():
        inst_uuids = inst_by_cell[cell_uuid]
        LOG.debug('Querying cell %(cell)s for %(num)i instances',
                  {'cell': cell_mapping.identity, 'num': len(uuids)})
        filters = {'uuid': inst_uuids, 'deleted': False}
        with nova_context.target_cell(context, cell_mapping) as ctx:
            found_inst_uuids.extend([
                inst.uuid for inst in objects.InstanceList.get_by_filters(
                    ctx, filters=filters)])

    return found_inst_uuids

    def _create_instance_in_cell(ctxt, cell, node, is_deleted=False,
                                 flavor_migrated=False):
        with context.target_cell(ctxt, cell) as cctxt:
            inst = objects.Instance(
                context=cctxt,
                host=node.host,
                node=node.hypervisor_hostname,
                uuid=uuidutils.generate_uuid())
            inst.create()

            if is_deleted:
                inst.destroy()
            else:
                # Create an embedded flavor for the instance. We don't create
                # this because we're in a cell context and flavors are global,
                # but we don't actually care about global flavors in this
                # check.
                extra_specs = {}
                if flavor_migrated:
                    extra_specs['resources:CUSTOM_BAREMETAL_GOLD'] = '1'
                inst.flavor = objects.Flavor(cctxt, extra_specs=extra_specs)
                inst.old_flavor = None
                inst.new_flavor = None
                inst.save()

            return inst

def handle_password(req, meta_data):
    ctxt = context.get_admin_context()
    if req.method == 'GET':
        return meta_data.password
    elif req.method == 'POST':
        # NOTE(vish): The conflict will only happen once the metadata cache
        #             updates, but it isn't a huge issue if it can be set for
        #             a short window.
        if meta_data.password:
            raise exc.HTTPConflict()
        if (req.content_length > MAX_SIZE or len(req.body) > MAX_SIZE):
            msg = _("Request is too large.")
            raise exc.HTTPBadRequest(explanation=msg)

        if CONF.api.local_metadata_per_cell:
            instance = objects.Instance.get_by_uuid(ctxt, meta_data.uuid)
        else:
            im = objects.InstanceMapping.get_by_instance_uuid(
                ctxt, meta_data.uuid)
            with context.target_cell(ctxt, im.cell_mapping) as cctxt:
                try:
                    instance = objects.Instance.get_by_uuid(
                        cctxt, meta_data.uuid)
                except exception.InstanceNotFound as e:
                    raise exc.HTTPBadRequest(explanation=e.format_message())

        instance.system_metadata.update(convert_password(ctxt, req.body))
        instance.save()
    else:
        msg = _("GET and POST only are supported.")
        raise exc.HTTPBadRequest(explanation=msg)

    def test_fixture(self):
        # The test setup creates two cell mappings (cell0 and cell1) by
        # default. Let's first list servers across all cells while they are
        # "up" to make sure that works as expected. We'll create a single
        # instance in cell1.
        ctxt = context.get_admin_context()
        cell1 = self.cell_mappings[test.CELL1_NAME]
        with context.target_cell(ctxt, cell1) as cctxt:
            inst = fake_instance.fake_instance_obj(cctxt)
            if 'id' in inst:
                delattr(inst, 'id')
            inst.create()

        # Now list all instances from all cells (should get one back).
        results = context.scatter_gather_all_cells(
            ctxt, objects.InstanceList.get_all)
        self.assertEqual(2, len(results))
        self.assertEqual(0, len(results[objects.CellMapping.CELL0_UUID]))
        self.assertEqual(1, len(results[cell1.uuid]))

        # Now do the same but with the DownCellFixture which should result
        # in exception results from both cells.
        with fixtures.DownCellFixture():
            results = context.scatter_gather_all_cells(
                ctxt, objects.InstanceList.get_all)
            self.assertEqual(2, len(results))
            for result in results.values():
                self.assertIsInstance(result, db_exc.DBError)

    def setUp(self):
        super(TestMigrationListObjects, self).setUp()

        self.context = context.RequestContext('fake', 'fake')
        self.num_migrations = 3
        self.migrations = []

        start = datetime.datetime(1985, 10, 25, 1, 21, 0)

        self.cells = objects.CellMappingList.get_all(self.context)
        # Create three migrations in each of the real cells. Leave the
        # first cell empty to make sure we don't break with an empty
        # one.
        for cell in self.cells[1:]:
            for i in range(0, self.num_migrations):
                with context.target_cell(self.context, cell) as cctx:
                    mig = objects.Migration(cctx,
                                            uuid=getattr(
                                                uuidsentinel,
                                                '%s_mig%i' % (cell.name, i)
                                            ),
                                            created_at=start,
                                            migration_type='resize',
                                            instance_uuid=getattr(
                                                uuidsentinel,
                                                'inst%i' % i)
                                            )
                    mig.create()
                self.migrations.append(mig)

    def _get_computes_for_cells(self, context, cells, compute_uuids=None):
        """Get a tuple of compute node and service information.

        Returns a tuple (compute_nodes, services) where:
         - compute_nodes is cell-uuid keyed dict of compute node lists
         - services is a dict of services indexed by hostname
        """

        compute_nodes = collections.defaultdict(list)
        services = {}
        for cell in cells:
            LOG.debug('Getting compute nodes and services for cell %(cell)s',
                      {'cell': cell.identity})
            with context_module.target_cell(context, cell) as cctxt:
                if compute_uuids is None:
                    compute_nodes[cell.uuid].extend(
                        objects.ComputeNodeList.get_all(cctxt))
                else:
                    compute_nodes[cell.uuid].extend(
                        objects.ComputeNodeList.get_all_by_uuids(
                            cctxt, compute_uuids))
                services.update(
                    {service.host: service
                     for service in objects.ServiceList.get_by_binary(
                             cctxt, 'nova-compute',
                             include_disabled=True)})
        return compute_nodes, services

    def test_populate_user_id_instance_get_fail(self, mock_inst_get):
        cells = []
        celldbs = fixtures.CellDatabases()

        # Create two cell databases and map them
        for uuid in (uuidsentinel.cell1, uuidsentinel.cell2):
            cm = cell_mapping.CellMapping(context=self.context, uuid=uuid,
                                          database_connection=uuid,
                                          transport_url='fake://')
            cm.create()
            cells.append(cm)
            celldbs.add_cell_database(uuid)
        self.useFixture(celldbs)

        # Create one instance per cell
        for cell in cells:
            with context.target_cell(self.context, cell) as cctxt:
                inst = instance.Instance(
                    cctxt,
                    project_id=self.context.project_id,
                    user_id=self.context.user_id)
                inst.create()
            create_mapping(project_id=self.context.project_id,
                           user_id=None, cell_id=cell.id,
                           instance_uuid=inst.uuid)

        # Simulate the first cell is down/has some error
        mock_inst_get.side_effect = [test.TestingException(),
                                     instance.InstanceList(objects=[inst])]

        found, done = instance_mapping.populate_user_id(self.context, 1000)
        # Verify we continue to the next cell when a down/error cell is
        # encountered.
        self.assertEqual(2, found)
        self.assertEqual(1, done)

    def test_connection_switch(self):
        ctxt = context.RequestContext('fake-user', 'fake-project')
        # Make a request context with a cell mapping
        mapping = objects.CellMapping(context=ctxt,
                                      uuid=uuidutils.generate_uuid(),
                                      database_connection=self.fake_conn,
                                      transport_url='none:///')
        mapping.create()
        # Create an instance in the cell database
        uuid = uuidutils.generate_uuid()
        with context.target_cell(ctxt, mapping):
            # Must set project_id because instance get specifies
            # project_only=True to model_query, which means non-admin
            # users can only read instances for their project
            instance = objects.Instance(context=ctxt, uuid=uuid,
                                        project_id='fake-project')
            instance.create()

            # Verify the instance is found in the cell database
            inst = objects.Instance.get_by_uuid(ctxt, uuid)
            self.assertEqual(uuid, inst.uuid)

        # Verify the instance isn't found in the main database
        self.assertRaises(exception.InstanceNotFound,
                          objects.Instance.get_by_uuid, ctxt, uuid)

    def _get_instances_all_cells(self, context, period_start, period_stop,
                                 tenant_id, limit, marker):
        all_instances = []
        cells = objects.CellMappingList.get_all(context)
        for cell in cells:
            with nova_context.target_cell(context, cell):
                try:
                    instances = (
                        objects.InstanceList.get_active_by_window_joined(
                            context, period_start, period_stop, tenant_id,
                            expected_attrs=['flavor'], limit=limit,
                            marker=marker))
                except exception.MarkerNotFound:
                    # NOTE(danms): We need to keep looking through the later
                    # cells to find the marker
                    continue
                all_instances.extend(instances)
                # NOTE(danms): We must have found a marker if we had one,
                # so make sure we don't require a marker in the next cell
                marker = None
                if limit:
                    limit -= len(instances)
                    if limit <= 0:
                        break
        if marker is not None and len(all_instances) == 0:
            # NOTE(danms): If we did not find the marker in any cell,
            # mimic the db_api behavior here
            raise exception.MarkerNotFound(marker=marker)

        return all_instances

    def test_instances_cores_ram_count(self):
        ctxt = context.RequestContext('fake-user', 'fake-project')
        mapping1 = objects.CellMapping(context=ctxt,
                                       uuid=uuidutils.generate_uuid(),
                                       database_connection='cell1',
                                       transport_url='none:///')
        mapping2 = objects.CellMapping(context=ctxt,
                                       uuid=uuidutils.generate_uuid(),
                                       database_connection='cell2',
                                       transport_url='none:///')
        mapping1.create()
        mapping2.create()

        # Create an instance in cell1
        with context.target_cell(ctxt, mapping1) as cctxt:
            instance = objects.Instance(context=cctxt,
                                        project_id='fake-project',
                                        user_id='fake-user',
                                        vcpus=2, memory_mb=512)
            instance.create()

        # Create an instance in cell2
        with context.target_cell(ctxt, mapping2) as cctxt:
            instance = objects.Instance(context=cctxt,
                                        project_id='fake-project',
                                        user_id='fake-user',
                                        vcpus=4, memory_mb=1024)
            instance.create()

        # Create an instance in cell2 for a different user
        with context.target_cell(ctxt, mapping2) as cctxt:
            instance = objects.Instance(context=cctxt,
                                        project_id='fake-project',
                                        user_id='other-fake-user',
                                        vcpus=4, memory_mb=1024)
            instance.create()

        # Count instances, cores, and ram across cells
        count = quota._instances_cores_ram_count(ctxt, 'fake-project',
                                                 user_id='fake-user')

        self.assertEqual(3, count['project']['instances'])
        self.assertEqual(10, count['project']['cores'])
        self.assertEqual(2560, count['project']['ram'])
        self.assertEqual(2, count['user']['instances'])
        self.assertEqual(6, count['user']['cores'])
        self.assertEqual(1536, count['user']['ram'])

        def _async_init_instance_info(computes_by_cell):
            context = context_module.RequestContext()
            self._load_cells(context)
            LOG.debug("START:_async_init_instance_info")
            self._instance_info = {}

            count = 0
            if not computes_by_cell:
                computes_by_cell = {}
                for cell in self.cells:
                    with context_module.target_cell(context, cell) as cctxt:
                        cell_cns = objects.ComputeNodeList.get_all(
                            cctxt).objects
                        computes_by_cell[cell] = cell_cns
                        count += len(cell_cns)

            LOG.debug("Total number of compute nodes: %s", count)

            for cell, compute_nodes in computes_by_cell.items():
                # Break the queries into batches of 10 to reduce the total
                # number of calls to the DB.
                batch_size = 10
                start_node = 0
                end_node = batch_size
                while start_node <= len(compute_nodes):
                    curr_nodes = compute_nodes[start_node:end_node]
                    start_node += batch_size
                    end_node += batch_size
                    filters = {"host": [curr_node.host
                                        for curr_node in curr_nodes],
                               "deleted": False}
                    with context_module.target_cell(context, cell) as cctxt:
                        result = objects.InstanceList.get_by_filters(
                            cctxt.elevated(), filters)
                    instances = result.objects
                    LOG.debug("Adding %s instances for hosts %s-%s",
                              len(instances), start_node, end_node)
                    for instance in instances:
                        host = instance.host
                        if host not in self._instance_info:
                            self._instance_info[host] = {"instances": {},
                                                         "updated": False}
                        inst_dict = self._instance_info[host]
                        inst_dict["instances"][instance.uuid] = instance
                    # Call sleep() to cooperatively yield
                    time.sleep(0)
                LOG.debug("END:_async_init_instance_info")