def _wait_for_nova_action(self):
        # Wait for the flavor to change.
        def update_server_info():
            self.instance._refresh_compute_server_info()
            return self.instance.server.status != "RESIZE"

        utils.poll_until(update_server_info, sleep_time=2, time_out=RESIZE_TIME_OUT)

    def _wait_for_revert_nova_action(self):
        # Wait for the server to return to ACTIVE after revert.
        def update_server_info():
            self.instance._refresh_compute_server_info()
            return self.instance.server.status == "ACTIVE"

        utils.poll_until(update_server_info, sleep_time=2, time_out=REVERT_TIME_OUT)

    def _do_resize(self, new_size):
        try:
            self.volume_client.volumes.extend(self.volume_id, new_size)
        except cinder_exceptions.ClientException:
            LOG.exception(
                "Error encountered trying to rescan or resize the "
                "attached volume filesystem for volume: "
                "%s" % self.volume_id
            )
            raise

        try:
            volume = self.volume_client.volumes.get(self.volume_id)
            utils.poll_until(
                lambda: self.volume_client.volumes.get(self.volume_id),
                lambda volume: volume.size == int(new_size),
                sleep_time=2,
                time_out=CONF.volume_time_out,
            )
            self.update_db(volume_size=new_size)
        except PollTimeOut as pto:
            LOG.error(
                "Timeout trying to rescan or resize the attached volume " "filesystem for volume: %s" % self.volume_id
            )
        except Exception as e:
            LOG.error(e)
            LOG.error(
                "Error encountered trying to rescan or resize the "
                "attached volume filesystem for volume: %s" % self.volume_id
            )
        finally:
            self.update_db(task_status=inst_models.InstanceTasks.NONE)

 def resize_volume(self, new_size):
     old_volume_size = self.volume_size
     new_size = int(new_size)
     LOG.debug("%s: Resizing volume for instance: %s from %s to %r GB"
               % (greenthread.getcurrent(), self.server.id,
                  old_volume_size, new_size))
     self.volume_client.volumes.resize(self.volume_id, new_size)
     try:
         utils.poll_until(
             lambda: self.volume_client.volumes.get(self.volume_id),
             lambda volume: volume.status == 'in-use',
             sleep_time=2,
             time_out=CONF.volume_time_out)
         volume = self.volume_client.volumes.get(self.volume_id)
         self.update_db(volume_size=volume.size)
         self.nova_client.volumes.rescan_server_volume(self.server,
                                                       self.volume_id)
         self.send_usage_event('modify_volume',
                               old_volume_size=old_volume_size,
                               launched_at=timeutils.isotime(),
                               modify_at=timeutils.isotime(),
                               volume_size=new_size)
     except PollTimeOut as pto:
         LOG.error("Timeout trying to rescan or resize the attached volume "
                   "filesystem for volume: %s" % self.volume_id)
     except Exception as e:
         LOG.error(e)
         LOG.error("Error encountered trying to rescan or resize the "
                   "attached volume filesystem for volume: %s"
                   % self.volume_id)
     finally:
         self.update_db(task_status=inst_models.InstanceTasks.NONE)

    def _create_server_volume_heat(self, flavor, image_id, security_groups, service_type, volume_size):
        client = create_heat_client(self.context)
        novaclient = create_nova_client(self.context)
        cinderclient = create_cinder_client(self.context)
        heat_template = template.HeatTemplate().template()
        parameters = {
            "KeyName": "heatkey",
            "Flavor": flavor["name"],
            "VolumeSize": volume_size,
            "ServiceType": "mysql",
            "InstanceId": self.id,
        }
        stack_name = "trove-%s" % self.id
        stack = client.stacks.create(stack_name=stack_name, template=heat_template, parameters=parameters)
        stack = client.stacks.get(stack_name)

        utils.poll_until(
            lambda: client.stacks.get(stack_name),
            lambda stack: stack.stack_status in ["CREATE_COMPLETE", "CREATE_FAILED"],
            sleep_time=2,
            time_out=HEAT_TIME_OUT,
        )

        resource = client.resources.get(stack.id, "BaseInstance")
        server = novaclient.servers.get(resource.physical_resource_id)

        resource = client.resources.get(stack.id, "DataVolume")
        volume = cinderclient.volumes.get(resource.physical_resource_id)
        volume_info = self._build_volume(volume)

        self.update_db(compute_instance_id=server.id, volume_id=volume.id)

        return server, volume_info

    def _resize_active_volume(self, new_size):
        try:
            LOG.debug(_("Instance %s calling stop_db...") % self.server.id)
            self.guest.stop_db()

            LOG.debug(_("Detach volume %(vol_id)s from instance %(id)s") %
                      {'vol_id': self.volume_id, 'id': self.server.id})
            self.volume_client.volumes.detach(self.volume_id)

            utils.poll_until(
                lambda: self.volume_client.volumes.get(self.volume_id),
                lambda volume: volume.status == 'available',
                sleep_time=2,
                time_out=CONF.volume_time_out)

            LOG.debug(_("Successfully detach volume %s") % self.volume_id)
        except Exception as e:
            LOG.debug(_("end _resize_active_volume for id: %s") %
                      self.server.id)
            LOG.exception(_("Failed to detach volume %(volume_id)s "
                            "instance %(id)s: %(e)s" %
                          {'volume_id': self.volume_id, 'id':
                            self.server.id, 'e': str(e)}))
            self.restart()
            raise

        self._do_resize(new_size)
        self.volume_client.volumes.attach(self.server.id, self.volume_id)
        LOG.debug(_("end _resize_active_volume for id: %s") % self.server.id)
        self.restart()

def poll_until_then_raise(event, exception):
    try:
        utils.poll_until(event,
                         sleep_time=RESET_ROOT_SLEEP_INTERVAL,
                         time_out=RESET_ROOT_RETRY_TIMEOUT)
    except exception.PollTimeOut:
        raise exception

    def test_bad_resize_vol_data(self):
        def _check_instance_status():
            inst = self.dbaas.instances.get(self.instance)
            if inst.status == "ACTIVE":
                return True
            else:
                return False

        poll_until(_check_instance_status)
        data = "bad data"
        try:
            self.dbaas.instances.resize_volume(self.instance.id, data)
        except Exception as e:
            resp, body = self.dbaas.client.last_response
            httpCode = resp.status
            asserts.assert_equal(httpCode, 400,
                                 "Resize instance failed with code %s, "
                                 "exception %s" % (httpCode, e))
            data = "u'bad data'"
            assert_contains(
                e.message,
                ["Validation error:",
                 "resize['volume']['size'] %s is not valid under "
                 "any of the given schemas" % data,
                 "%s is not of type 'integer'" % data,
                 "%s does not match '^[0-9]+$'" % data])

    def assert_all_instance_states(self, instance_ids, expected_states,
                                   fast_fail_status=None,
                                   require_all_states=False):
        self.report.log("Waiting for states (%s) for instances: %s" %
                        (expected_states, instance_ids))

        def _make_fn(inst_id):
            return lambda: self._assert_instance_states(
                inst_id, expected_states,
                fast_fail_status=fast_fail_status,
                require_all_states=require_all_states)

        tasks = [
            build_polling_task(
                _make_fn(instance_id),
                sleep_time=self.def_sleep_time,
                time_out=self.def_timeout) for instance_id in instance_ids]
        poll_until(lambda: all(poll_task.ready() for poll_task in tasks),
                   sleep_time=self.def_sleep_time, time_out=self.def_timeout)

        for task in tasks:
            if task.has_result():
                self.assert_true(
                    task.poll_result(),
                    "Some instances failed to acquire all expected states.")
            elif task.has_exception():
                self.fail(str(task.poll_exception()))

    def metadata(self):
        """pg_basebackup may complete, and we arrive here before the
        history file is written to the wal archive. So we need to
        handle two possibilities:
        - this is the first backup, and no history file exists yet
        - this isn't the first backup, and so the history file we retrieve
        isn't the one we just ran!
         """
        def _metadata_found():
            LOG.debug("Polling for backup metadata... ")
            self.mrb = self.most_recent_backup_file()
            if not self.mrb:
                LOG.debug("No history files found!")
                return False
            metadata = self.base_backup_metadata(
                os.path.join(WAL_ARCHIVE_DIR, self.mrb))
            LOG.debug("Label to pg_basebackup: %s label found: %s" %
                      (self.base_filename, metadata['label']))
            LOG.info(_("Metadata for backup: %s.") % str(metadata))
            return metadata['label'] == self.base_filename

        try:
            utils.poll_until(_metadata_found, sleep_time=5, time_out=60)
        except exception.PollTimeOut:
            raise RuntimeError(_("Timeout waiting for backup metadata for"
                                 " backup %s") % self.base_filename)

        return self.base_backup_metadata(
            os.path.join(WAL_ARCHIVE_DIR, self.mrb))

    def test_unassign_configuration_from_instances(self):
        # test to unassign configuration from instance
        instance_info.dbaas.instances.modify(configuration_instance.id,
                                             configuration="")
        resp, body = instance_info.dbaas.client.last_response
        assert_equal(resp.status, 202)
        instance_info.dbaas.instances.get(configuration_instance.id)
        # test that config group is not removed
        instance_info.dbaas.instances.modify(instance_info.id,
                                             configuration=None)
        resp, body = instance_info.dbaas.client.last_response
        assert_equal(resp.status, 202)
        instance_info.dbaas.instances.get(instance_info.id)

        def result_has_no_configuration():
            instance = instance_info.dbaas.instances.get(inst_info.id)
            if hasattr(instance, 'configuration'):
                return False
            else:
                return True
        inst_info = instance_info
        poll_until(result_has_no_configuration)
        inst_info = configuration_instance
        poll_until(result_has_no_configuration)
        instance = instance_info.dbaas.instances.get(instance_info.id)
        assert_equal('RESTART_REQUIRED', instance.status)

    def _do_resize(self, new_size):
        try:
            self.volume_client.volumes.extend(self.volume_id, new_size)
        except cinder_exceptions.ClientException:
            LOG.exception(_("Error encountered trying to rescan or resize the "
                            "attached volume filesystem for volume: "
                            "%s") % self.volume_id)
            raise

        try:
            volume = self.volume_client.volumes.get(self.volume_id)
            if not volume:
                raise (cinder_exceptions.
                       ClientException(_('Failed to get volume with '
                                       'id: %(id)s') %
                                       {'id': self.volume_id}))
            utils.poll_until(
                lambda: self.volume_client.volumes.get(self.volume_id),
                lambda volume: volume.size == int(new_size),
                sleep_time=2,
                time_out=CONF.volume_time_out)
            self.update_db(volume_size=new_size)
        except PollTimeOut:
            LOG.error(_("Timeout trying to rescan or resize the attached "
                      "volume filesystem for volume %(vol_id)s of "
                      "instance: %(id)s") %
                      {'vol_id': self.volume_id, 'id': self.id})
        except Exception as e:
            LOG.exception(_("Error encountered trying to rescan or resize the "
                          "attached volume filesystem of volume %(vol_id)s of "
                          "instance %(id)s: %(e)s") %
                          {'vol_id': self.volume_id, 'id': self.id, 'e': e})
        finally:
            self.update_db(task_status=inst_models.InstanceTasks.NONE)

    def test_wait_until_cluster_is_active(self):
        if not getattr(self, 'cluster', None):
            raise SkipTest(
                "Skipping this test since cluster is not available.")

        def result_is_active():
            cluster = self.rd_client.clusters.get(self.cluster.id)
            cluster_instances = [
                self.rd_client.instances.get(instance['id'])
                for instance in cluster.instances]
            self.report.log("Cluster info %s." % cluster._info)
            self.report.log("Cluster instances info %s." % cluster_instances)
            if cluster.task['name'] == "NONE":

                if ["ERROR"] * len(cluster_instances) == [
                   str(instance.status) for instance in cluster_instances]:
                    self.report.log("Cluster provisioning failed.")
                    asserts.fail("Cluster provisioning failed.")

                if ["ACTIVE"] * len(cluster_instances) == [
                   str(instance.status) for instance in cluster_instances]:
                    self.report.log("Cluster is ready.")
                    return True
            else:
                asserts.assert_not_equal(
                    ["ERROR"] * len(cluster_instances),
                    [instance.status
                     for instance in cluster_instances])
            self.report.log("Continue polling, cluster is not ready yet.")

        poll_until(result_is_active, sleep_time=SLEEP_TIME, time_out=TIMEOUT)
        self.report.log("Created cluster, ID = %s." % self.cluster.id)

    def _wait_for_slave_status(self, status, client, max_time):

        def verify_slave_status():
            actual_status = client.execute(
                "SHOW GLOBAL STATUS like 'slave_running'").first()
            if actual_status:
                return actual_status[1].upper() == status.upper()
            # The slave_running status is no longer available in MySql 5.7
            # Need to query the performance_schema instead.
            LOG.debug("slave_running global status doesn't exist, checking "
                      "service_state in performance_schema instead.")
            q = sql_query.Query()
            q.columns = ["a.service_state", "c.service_state"]
            q.tables = ["performance_schema.replication_applier_status a",
                        "performance_schema.replication_connection_status c"]
            q.where = ["a.channel_name = ''", "c.channel_name = ''"]
            t = text(str(q))
            actual_status = client.execute(t).first()
            if (actual_status and actual_status[0].upper() == 'ON' and
                    actual_status[1].upper() == 'ON'):
                actual_status_str = 'ON'
            else:
                actual_status_str = 'OFF'
            return actual_status_str == status.upper()

        LOG.debug("Waiting for SLAVE_RUNNING to change to %s.", status)
        try:
            utils.poll_until(verify_slave_status, sleep_time=3,
                             time_out=max_time)
            LOG.info(_("Replication is now %s.") % status.lower())
        except PollTimeOut:
            raise RuntimeError(
                _("Replication is not %(status)s after %(max)d seconds.") % {
                    'status': status.lower(), 'max': max_time})

    def test_bad_change_user_password(self):
        password = ""
        users = [{"name": password}]

        def _check_instance_status():
            inst = self.dbaas.instances.get(self.instance)
            if inst.status == "ACTIVE":
                return True
            else:
                return False

        poll_until(_check_instance_status)
        try:
            self.dbaas.users.change_passwords(self.instance, users)
        except Exception as e:
            resp, body = self.dbaas.client.last_response
            httpCode = resp.status
            assert_equal(httpCode, 400,
                         "Change usr/passwd failed with code %s, exception %s"
                         % (httpCode, e))
            if not isinstance(self.dbaas.client,
                              troveclient.compat.xml.TroveXmlClient):
                password = "u''"
                assert_equal(e.message,
                             "Validation error: "
                             "users[0] 'password' is a required property; "
                             "users[0]['name'] %s is too short; "
                             "users[0]['name'] %s does not match "
                             "'^.*[0-9a-zA-Z]+.*$'" %
                             (password, password))

    def test_backup_delete(self):
        """test delete"""

        # Test to make sure that user in other tenant is not able
        # to DELETE this backup
        reqs = Requirements(is_admin=False)
        other_user = CONFIG.users.find_user(
            reqs,
            black_list=[instance_info.user.auth_user])
        other_client = create_dbaas_client(other_user)
        assert_raises(exceptions.NotFound, other_client.backups.delete,
                      backup_info.id)

        instance_info.dbaas.backups.delete(backup_info.id)
        assert_equal(202, instance_info.dbaas.last_http_code)

        def backup_is_gone():
            result = instance_info.dbaas.instances.backups(instance_info.id)
            if len(result) == 0:
                return True
            else:
                return False
        poll_until(backup_is_gone)
        assert_raises(exceptions.NotFound, instance_info.dbaas.backups.get,
                      backup_info.id)

    def test_nova_resize_timeout(self):
        self._stop_db()
        self.server.resize(NEW_FLAVOR_ID)

        self.mock.StubOutWithMock(utils, 'poll_until')
        utils.poll_until(mox.IgnoreArg(), sleep_time=2, time_out=120)\
            .AndRaise(PollTimeOut)

    def reboot(self):
        try:
            LOG.debug(_("Instance %s calling stop_db...") % self.id)
            self.guest.stop_db()
            LOG.debug(_("Rebooting instance %s") % self.id)
            self.server.reboot()

            # Poll nova until instance is active
            reboot_time_out = CONF.reboot_time_out

            def update_server_info():
                self._refresh_compute_server_info()
                return self.server.status == 'ACTIVE'

            utils.poll_until(
                update_server_info,
                sleep_time=2,
                time_out=reboot_time_out)

            # Set the status to PAUSED. The guest agent will reset the status
            # when the reboot completes and MySQL is running.
            self._set_service_status_to_paused()
            LOG.debug(_("Successfully rebooted instance %s") % self.id)
        except Exception as e:
            LOG.error(_("Failed to reboot instance %(id)s: %(e)s") %
                      {'id': self.id, 'e': str(e)})
        finally:
            LOG.debug(_("Rebooting FINALLY  %s") % self.id)
            self.update_db(task_status=inst_models.InstanceTasks.NONE)

    def test_instance_resize_flavor(self):
        """Tests the resize instance/flavor API."""

        flavor_name = CONFIG.values.get('instance_bigger_flavor_name',
                                        'm1.medium')
        flavors = self.instance.dbaas.find_flavors_by_name(flavor_name)
        new_flavor = flavors[0]

        asserts.assert_true(new_flavor is not None,
                            "Flavor '%s' not found!" % flavor_name)

        if not getattr(self, 'instance', None):
            raise SkipTest(
                "Skipping this test since instance is not available.")

        self.rd_client = create_dbaas_client(self.instance.user)
        self.rd_client.instances.resize_instance(self.instance.id,
                                                 new_flavor.id)

        asserts.assert_equal(202, self.rd_client.last_http_code)
        test_instance = self.rd_client.instances.get(self.instance.id)
        asserts.assert_equal("RESIZE", test_instance.status)

        poll_until(lambda: self._find_status(self.rd_client,
                                             self.instance.id, "ACTIVE"),
                   sleep_time=SLEEP_TIME, time_out=TIMEOUT)

        test_instance = self.rd_client.instances.get(self.instance.id)
        asserts.assert_equal(int(test_instance.flavor['id']), new_flavor.id)
        self.report.log("Resized Flavor for Instance ID: %s to %s." % (
            self.instance.id, new_flavor.id))

    def _resize_active_volume(self, new_size):
        try:
            LOG.debug("Instance %s calling stop_db..." % self.server.id)
            self.guest.stop_db()

            LOG.debug("Detach volume %s from instance %s" % (self.volume_id,
                                                             self.server.id))
            self.volume_client.volumes.detach(self.volume_id)

            utils.poll_until(
                lambda: self.volume_client.volumes.get(self.volume_id),
                lambda volume: volume.status == 'available',
                sleep_time=2,
                time_out=CONF.volume_time_out)

            LOG.debug("Successfully detach volume %s" % self.volume_id)
        except Exception as e:
            LOG.error("Failed to detach volume %s instance %s: %s" % (
                self.volume_id, self.server.id, str(e)))
            self.restart()
            raise

        self._do_resize(new_size)
        self.volume_client.volumes.attach(self.server.id, self.volume_id)

        self.restart()