def _update_stack(self, stack, template, parameters=None,
                      files=None, environment=None):
        """Update an existing stack

        :param stack: stack that need to be updated
        :param template: Updated template
        :param parameters: template parameters for stack update
        :param files: additional files used in template
        :param environment: stack environment definition

        :returns: object of updated stack
        """

        kw = {
            "stack_name": stack.stack_name,
            "disable_rollback": True,
            "parameters": parameters or {},
            "template": template,
            "files": files or {},
            "environment": environment or {}
        }
        self.clients("heat").stacks.update(stack.id, **kw)

        time.sleep(CONF.benchmark.heat_stack_update_prepoll_delay)
        stack = utils.wait_for(
            stack,
            is_ready=utils.resource_is("UPDATE_COMPLETE"),
            update_resource=utils.get_from_manager(["UPDATE_FAILED"]),
            timeout=CONF.benchmark.heat_stack_update_timeout,
            check_interval=CONF.benchmark.heat_stack_update_poll_interval)
        return stack

    def _create_volume(self, size, **kwargs):
        """Create one volume.

        Returns when the volume is actually created and is in the "Available"
        state.

        :param size: int be size of volume in GB, or
                     dictionary, must contain two values:
                         min - minimum size volumes will be created as;
                         max - maximum size volumes will be created as.
        :param kwargs: Other optional parameters to initialize the volume
        :returns: Created volume object
        """
        kwargs["display_name"] = kwargs.get("display_name",
                                            self._generate_random_name())

        if isinstance(size, dict):
            size = random.randint(size["min"], size["max"])

        volume = self.clients("cinder").volumes.create(size, **kwargs)
        # NOTE(msdubov): It is reasonable to wait 5 secs before starting to
        #                check whether the volume is ready => less API calls.
        time.sleep(CONF.benchmark.cinder_volume_create_prepoll_delay)
        volume = bench_utils.wait_for(
            volume,
            is_ready=bench_utils.resource_is("available"),
            update_resource=bench_utils.get_from_manager(),
            timeout=CONF.benchmark.cinder_volume_create_timeout,
            check_interval=CONF.benchmark.cinder_volume_create_poll_interval
        )
        return volume

    def _create_share(self, share_proto, size=1, **kwargs):
        """Create a share.

        :param share_proto: share protocol for new share,
            available values are NFS, CIFS, GlusterFS and HDFS.
        :param size: size of a share in GB
        :param snapshot_id: ID of the snapshot
        :param name: name of new share
        :param description: description of a share
        :param metadata: optional metadata to set on share creation
        :param share_network: either instance of ShareNetwork or str with ID
        :param share_type: either instance of ShareType or str with ID
        :param is_public: defines whether to set share as public or not.
        :returns: instance of :class:`Share`
        """
        if not kwargs.get("name"):
            kwargs["name"] = self._generate_random_name()

        share = self.clients("manila").shares.create(
            share_proto, size, **kwargs)
        time.sleep(CONF.benchmark.manila_share_create_prepoll_delay)
        share = utils.wait_for(
            share,
            is_ready=utils.resource_is("available"),
            update_resource=utils.get_from_manager(),
            timeout=CONF.benchmark.manila_share_create_timeout,
            check_interval=CONF.benchmark.manila_share_create_poll_interval,
        )
        return share

    def _migrate(self, server, skip_host_check=False):
        """Run migration of the given server.

        :param server: Server object
        :param skip_host_check: Specifies whether to verify the targeted host
                                availability
        """
        server_admin = self.admin_clients("nova").servers.get(server.id)
        host_pre_migrate = getattr(server_admin, "OS-EXT-SRV-ATTR:host")
        server_admin.migrate()
        utils.wait_for(
            server,
            is_ready=utils.resource_is("VERIFY_RESIZE"),
            update_resource=utils.get_from_manager(),
            timeout=CONF.benchmark.nova_server_migrate_timeout,
            check_interval=(
                CONF.benchmark.nova_server_migrate_poll_interval)
        )
        if not skip_host_check:
            server_admin = self.admin_clients("nova").servers.get(server.id)
            host_after_migrate = getattr(server_admin, "OS-EXT-SRV-ATTR:host")
            if host_pre_migrate == host_after_migrate:
                raise exceptions.MigrateException(
                    "Migration complete but instance did not change host: %s" %
                    host_pre_migrate)

    def get_master_agent(self, router_id):
        net_admin = self._admin_clients.neutron()

        def get_actives(r):
            agents = net_admin.list_l3_agent_hosting_routers(r)
            active_agents = filter(
                lambda d: d.get("ha_state") == "active",
                agents.get("agents", []))
            LOG.info("Router %s is ACTIVE on: %s" % (r, [(a["id"], a["host"])
                                                         for a in
                                                         active_agents]))
            return active_agents

        utils.wait_is_ready(
            router_id,
            is_ready=utils.resource_is(str(1),
                                       lambda x: str(len(get_actives(x)))),
            timeout=vmutils.CONF.benchmark.vm_ping_timeout,
            check_interval=vmutils.CONF.benchmark.vm_ping_poll_interval

        )
        masters = get_actives(router_id)
        LOG.info("Found router %s master on agent %s" % (router_id,
                                                         (masters[0]["id"],
                                                          masters[0]["host"])))
        return masters[0]

    def _live_migrate(self, server, target_host, block_migration=False,
                      disk_over_commit=False, skip_host_check=False):
        """Run live migration of the given server.

        :param server: Server object
        :param target_host: Specifies the target compute node to migrate
        :param block_migration: Specifies the migration type
        :param disk_over_commit: Specifies whether to overcommit migrated
                                 instance or not
        :param skip_host_check: Specifies whether to verify the targeted host
                                availability
        """
        server_admin = self.admin_clients("nova").servers.get(server.id)
        host_pre_migrate = getattr(server_admin, "OS-EXT-SRV-ATTR:host")
        server_admin.live_migrate(target_host,
                                  block_migration=block_migration,
                                  disk_over_commit=disk_over_commit)
        utils.wait_for(
            server,
            is_ready=utils.resource_is("ACTIVE"),
            update_resource=utils.get_from_manager(),
            timeout=CONF.benchmark.nova_server_live_migrate_timeout,
            check_interval=(
                CONF.benchmark.nova_server_live_migrate_poll_interval)
        )
        server_admin = self.admin_clients("nova").servers.get(server.id)
        if (host_pre_migrate == getattr(server_admin, "OS-EXT-SRV-ATTR:host")
                and not skip_host_check):
            raise exceptions.LiveMigrateException(
                "Migration complete but instance did not change host: %s" %
                host_pre_migrate)

    def _boot_servers(self, image_id, flavor_name,
                      instance_num=1, **kwargs):
        """Boot multiple servers.

        Returns when all the servers are actually booted and are in the
        "Running" state.

        :param image_id: ID of the image to be used for server creation
        :param flavor_name: Name of the flavor to be used for server creation
        :param instance_num: Number of instances to boot
        :param kwargs: Other optional parameters to boot servers

        :returns: List of created server objects
        """
        reservation = self.clients("ec2").run_instances(
            image_id=image_id,
            instance_type=flavor_name,
            min_count=instance_num,
            max_count=instance_num,
            **kwargs)
        servers = [instance for instance in reservation.instances]

        time.sleep(CONF.benchmark.ec2_server_boot_prepoll_delay)
        servers = [utils.wait_for(
            server,
            is_ready=utils.resource_is("RUNNING"),
            update_resource=self._update_resource,
            timeout=CONF.benchmark.ec2_server_boot_timeout,
            check_interval=CONF.benchmark.ec2_server_boot_poll_interval
        ) for server in servers]
        return servers

    def _create_snapshot(self, volume_id, force=False, **kwargs):
        """Create one snapshot.

        Returns when the snapshot is actually created and is in the "Available"
        state.

        :param volume_id: volume uuid for creating snapshot
        :param force: flag to indicate whether to snapshot a volume even if
                      it's attached to an instance
        :param kwargs: Other optional parameters to initialize the volume
        :returns: Created snapshot object
        """
        kwargs["display_name"] = kwargs.get("display_name",
                                            self._generate_random_name())
        kwargs["force"] = force
        snapshot = self.clients("cinder").volume_snapshots.create(volume_id,
                                                                  **kwargs)
        time.sleep(CONF.benchmark.cinder_volume_create_prepoll_delay)
        snapshot = bench_utils.wait_for(
            snapshot,
            is_ready=bench_utils.resource_is("available"),
            update_resource=bench_utils.get_from_manager(),
            timeout=CONF.benchmark.cinder_volume_create_timeout,
            check_interval=CONF.benchmark.cinder_volume_create_poll_interval
        )
        return snapshot

 def _wait_for_ping(self, server_ip):
     server_ip = netaddr.IPAddress(server_ip)
     utils.wait_for(
         server_ip,
         is_ready=utils.resource_is(ICMP_UP_STATUS,
                                    self._ping_ip_address),
         timeout=120
     )

 def _wait_for_ping(self, server_ip):
     server_ip = netaddr.IPAddress(server_ip)
     utils.wait_for(
         server_ip,
         is_ready=utils.resource_is(ICMP_UP_STATUS, self._ping_ip_address),
         timeout=CONF.benchmark.vm_ping_timeout,
         check_interval=CONF.benchmark.vm_ping_poll_interval
     )

 def _wait_for_swarm_ping(self, swarm_connection):
     utils.wait_for(
         None,
         is_ready=utils.resource_is(
             "UP ALL",
             swarm_connection.status),
         timeout=CONF.benchmark.vm_swarm_ping_timeout,
         check_interval=CONF.benchmark.vm_swarm_ping_poll_interval
     )

 def _do_server_reboot(self, server, reboottype):
     server.reboot(reboot_type=reboottype)
     time.sleep(CONF.benchmark.nova_server_reboot_prepoll_delay)
     utils.wait_for(
         server, is_ready=utils.resource_is("ACTIVE"),
         update_resource=utils.get_from_manager(),
         timeout=CONF.benchmark.nova_server_reboot_timeout,
         check_interval=CONF.benchmark.nova_server_reboot_poll_interval
     )

 def _resize(self, server, flavor):
     server.resize(flavor)
     utils.wait_for(
         server,
         is_ready=utils.resource_is("VERIFY_RESIZE"),
         update_resource=utils.get_from_manager(),
         timeout=CONF.benchmark.nova_server_resize_timeout,
         check_interval=CONF.benchmark.nova_server_resize_poll_interval
     )

 def _resize_revert(self, server, status="ACTIVE"):
     server.revert_resize()
     utils.wait_for(
         server,
         is_ready=utils.resource_is(status),
         update_resource=utils.get_from_manager(),
         timeout=CONF.benchmark.nova_server_resize_revert_timeout,
         check_interval=(
             CONF.benchmark.nova_server_resize_revert_poll_interval)
     )

 def _detach_volume(self, server, volume):
     server_id = server.id
     volume_id = volume.id
     self.clients("nova").volumes.delete_server_volume(server_id,
                                                       volume_id)
     utils.wait_for(
         volume,
         is_ready=utils.resource_is("available"),
         update_resource=utils.get_from_manager(),
         timeout=CONF.benchmark.nova_detach_volume_timeout,
         check_interval=CONF.benchmark.nova_detach_volume_poll_interval
     )

def assert_server_status(server, **kwargs):

    LOG.debug("WAITING FOR SERVER TO GO ACTIVE")
    server = task_utils.wait_for(
        server,
        is_ready=task_utils.resource_is("ACTIVE"),
        update_resource=task_utils.get_from_manager(),
        timeout=kwargs["nova_server_boot_timeout"],
        check_interval=5,
    )
    LOG.debug("SERVER STATUS: %s", server.status)
    assert "ACTIVE" == server.status, "THE INSTANCE IS NOT IN ACTIVE STATE"

    def _create_image(self, container_format, image_location, disk_format,
                      name=None, prefix=None, length=None, **kwargs):
        """Create a new image.

        :param container_format: container format of image. Acceptable
                                 formats: ami, ari, aki, bare, and ovf
        :param image_location: image file location
        :param disk_format: disk format of image. Acceptable formats:
                            ami, ari, aki, vhd, vmdk, raw, qcow2, vdi, and iso
        :param name: string used to name the image
        :param prefix: prefix of generated image name if name not specified
        ignore if name specified
        :param length: length of autometic generated part in image name
        ignore if name specified
        :param kwargs: optional parameters to create image

        :returns: image object
        """
        name = name or self._generate_random_name(prefix, length)
        kw = {
            "name": name,
            "container_format": container_format,
            "disk_format": disk_format,
        }

        kw.update(kwargs)
        image_location = os.path.expanduser(image_location)

        try:
            if os.path.isfile(image_location):
                kw["data"] = open(image_location)
            else:
                kw["copy_from"] = image_location

            image = self.clients("glance").images.create(**kw)

            time.sleep(CONF.benchmark.glance_image_create_prepoll_delay)

            image = utils.wait_for(
                image,
                is_ready=utils.resource_is("active"),
                update_resource=utils.get_from_manager(),
                timeout=CONF.benchmark.glance_image_create_timeout,
                check_interval=CONF.benchmark.
                glance_image_create_poll_interval)

        finally:
            if "data" in kw:
                kw["data"].close()

        return image

    def _upload_volume_to_image(self, volume, force=False,
                                container_format="bare", disk_format="raw"):
        """Upload the given volume to image.

        Returns created image.

        :param volume: volume object
        :param force: flag to indicate whether to snapshot a volume even if
                      it's attached to an instance
        :param container_format: container format of image. Acceptable
                                 formats: ami, ari, aki, bare, and ovf
        :param: disk_format: disk format of image. Acceptable formats:
                             ami, ari, aki, vhd, vmdk, raw, qcow2, vdi
                             and iso
        :returns: Returns created image object
        """
        resp, img = volume.upload_to_image(force, self._generate_random_name(),
                                           container_format, disk_format)
        # NOTE (e0ne): upload_to_image changes volume status to uploading so
        # we need to wait until it will be available.
        volume = bench_utils.wait_for(
            volume,
            is_ready=bench_utils.resource_is("available"),
            update_resource=bench_utils.get_from_manager(),
            timeout=CONF.benchmark.cinder_volume_create_timeout,
            check_interval=CONF.benchmark.cinder_volume_create_poll_interval
        )
        image_id = img["os-volume_upload_image"]["image_id"]
        image = self.clients("glance").images.get(image_id)
        image = bench_utils.wait_for(
            image,
            is_ready=bench_utils.resource_is("active"),
            update_resource=bench_utils.get_from_manager(),
            timeout=CONF.benchmark.glance_image_create_timeout,
            check_interval=CONF.benchmark.glance_image_create_poll_interval
        )

        return image

    def _deploy_environment(self, environment, session):
        """Deploy environment.

        :param environment: Environment instance
        :param session: Session instance
        """
        self.clients("murano").sessions.deploy(environment.id,
                                               session.id)
        utils.wait_for(
            environment, is_ready=utils.resource_is("READY"),
            update_resource=utils.get_from_manager(["DEPLOY FAILURE"]),
            timeout=CONF.benchmark.deploy_environment_timeout,
            check_interval=CONF.benchmark.deploy_environment_check_interval
        )

    def _check_stack(self, stack):
        """Check given stack.

        Check the stack and stack resources.

        :param stack: stack that needs to be checked
        """
        self.clients("heat").actions.check(stack.id)
        utils.wait_for(
            stack,
            is_ready=utils.resource_is("CHECK_COMPLETE"),
            update_resource=utils.get_from_manager(["CHECK_FAILED"]),
            timeout=CONF.benchmark.heat_stack_check_timeout,
            check_interval=CONF.benchmark.heat_stack_check_poll_interval)