def _validate_config_semantic(self, config):
        LOG.info("Check health of the environment '%s'." % self.env.uuid)
        failed = []
        for p, res in self.env.check_health().items():
            LOG.info("Platform %s (available: %s): %s" %
                     (p, res["available"], res["message"]))
            if not res["available"]:
                failed.append(p)
                if logging.is_debug():
                    LOG.error(res["traceback"])
        if failed:
            raise exceptions.ValidationError(
                "One or several platforms are not available: %s. Check logs "
                "for more details." % ", ".join(failed))
        validation_ctx = self.env.get_validation_context()

        env_data = self.env.data
        env_data["platforms"] = dict(
            (p["platform_name"], p["platform_data"])
            for p in env_data["platforms"].values())

        ctx_obj = {"task": self.task,
                   "config": validation_ctx,
                   "env": env_data}

        with context.ContextManager(ctx_obj):
            for subtask in config.subtasks:
                for workload in subtask["workloads"]:
                    self._validate_workload(
                        workload, vcontext=ctx_obj, vtype="semantic")

    def create(cls, config, name):
        """Create a deployment.

        :param config: a dict with deployment configuration
        :param name: a str represents a name of the deployment
        :returns: Deployment object
        """

        try:
            deployment = objects.Deployment(name=name, config=config)
        except exceptions.DeploymentNameExists as e:
            if logging.is_debug():
                LOG.exception(e)
            raise

        deployer = deploy_engine.Engine.get_engine(
            deployment["config"]["type"], deployment)
        try:
            deployer.validate()
        except jsonschema.ValidationError:
            LOG.error(_LE("Deployment %s: Schema validation error.") %
                      deployment["uuid"])
            deployment.update_status(consts.DeployStatus.DEPLOY_FAILED)
            raise

        with deployer:
            credentials = deployer.make_deploy()
            deployment.update_credentials(credentials)
            return deployment

    def setup(self):
        """Create list of flavors."""
        from novaclient import exceptions as nova_exceptions

        self.context["flavors"] = {}

        clients = osclients.Clients(self.context["admin"]["credential"])
        for flavor_config in self.config:

            extra_specs = flavor_config.get("extra_specs")

            flavor_config = FlavorConfig(**flavor_config)
            try:
                flavor = clients.nova().flavors.create(**flavor_config)
            except nova_exceptions.Conflict:
                msg = "Using existing flavor %s" % flavor_config["name"]
                if logging.is_debug():
                    LOG.exception(msg)
                else:
                    LOG.warning(msg)
                continue

            if extra_specs:
                flavor.set_keys(extra_specs)

            self.context["flavors"][flavor_config["name"]] = flavor.to_dict()
            LOG.debug("Created flavor with id '%s'" % flavor.id)

    def _delete_single_resource(self, resource):
        """Safe resource deletion with retries and timeouts.

        Send request to delete resource, in case of failures repeat it few
        times. After that pull status of resource until it's deleted.

        Writes in LOG warning with UUID of resource that wasn't deleted

        :param resource: instance of resource manager initiated with resource
                         that should be deleted.
        """

        msg_kw = {
            "uuid": resource.id(),
            "name": resource.name() or "",
            "service": resource._service,
            "resource": resource._resource,
        }

        LOG.debug("Deleting %(service)s %(resource)s object %(name)s (%(uuid)s)" % msg_kw)

        try:
            rutils.retry(resource._max_attempts, resource.delete)
        except Exception as e:
            msg_kw["reason"] = e
            LOG.warning(
                _(
                    "Resource deletion failed, max retries exceeded for "
                    "%(service)s.%(resource)s: %(uuid)s. Reason: %(reason)s"
                )
                % msg_kw
            )
            if logging.is_debug():
                LOG.exception(e)
        else:
            started = time.time()
            failures_count = 0
            while time.time() - started < resource._timeout:
                try:
                    if resource.is_deleted():
                        return
                except Exception as e:
                    LOG.warning(
                        _("Seems like %s.%s.is_deleted(self) method is broken " "It shouldn't raise any exceptions.")
                        % (resource.__module__, type(resource).__name__)
                    )
                    LOG.exception(e)

                    # NOTE(boris-42): Avoid LOG spamming in case of bad
                    #                 is_deleted() method
                    failures_count += 1
                    if failures_count > resource._max_attempts:
                        break

                finally:
                    time.sleep(resource._interval)

            LOG.warning(
                _("Resource deletion failed, timeout occurred for " "%(service)s.%(resource)s: %(uuid)s.") % msg_kw
            )

    def setup(self):
        """Create kuryr or non-kuryr docker network, and prepare image cache"""
        try:
            docker_client = docker.APIClient(base_url="tcp://0.0.0.0:2375")

            if self.config["is_kuryr"]:
                ipam = {
                    "Driver": "kuryr",
                    "Options": {},
                    "Config": [
                        {
                            "Subnet": self.config.get("Subnet"),
                            "IPRange": self.config.get("IPRange"),
                            "Gateway": self.config.get("Gateway")
                        }
                    ]
                }
                res = docker_client.create_network(name="kuryr_network",
                                                   driver="kuryr",
                                                   ipam=ipam)
                self.context["netid"] = res.get("Id")
                self.context["netname"] = "kuryr_network"
            else:
                res = docker_client.create_network(name="docker_network")
                self.context["netid"] = res.get("Id")
                self.context["netname"] = "docker_network"
            LOG.debug("Container network id is '%s'" % self.context["netid"])
        except Exception as e:
            msg = "Can't create docker network: %s" % e.message
            if logging.is_debug():
                LOG.exception(msg)
            else:
                LOG.warning(msg)

def import_modules_by_entry_point(_packages=None):
    """Import plugins by entry-point 'rally_plugins'."""
    loaded_packages = _packages or find_packages_by_entry_point()

    for package in loaded_packages:
        if "plugins_path" in package:
            em = pkg_resources.get_entry_map(package["name"])
            ep = em["rally_plugins"]["path"]
            try:
                m = ep.load()
                if hasattr(m, "__path__"):
                    path = pkgutil.extend_path(m.__path__, m.__name__)
                else:
                    path = [m.__file__]
                prefix = m.__name__ + "."
                for loader, name, _is_pkg in pkgutil.walk_packages(
                        path, prefix=prefix):
                    sys.modules[name] = importlib.import_module(name)
            except Exception as e:
                msg = ("\t Failed to load plugins from module '%(module)s' "
                       "(package: '%(package)s')" %
                       {"module": ep.module_name,
                        "package": "%s %s" % (package["name"],
                                              package["version"])})
                if logging.is_debug():
                    LOG.exception(msg)
                else:
                    LOG.warning(msg + (": %s" % six.text_type(e)))
    return loaded_packages

def _run_scenario_once(cls, method_name, context_obj, scenario_kwargs):
    iteration = context_obj["iteration"]

    # provide arguments isolation between iterations
    scenario_kwargs = copy.deepcopy(scenario_kwargs)

    LOG.info("Task %(task)s | ITER: %(iteration)s START" %
             {"task": context_obj["task"]["uuid"], "iteration": iteration})

    scenario_inst = cls(context_obj)
    error = []
    try:
        with rutils.Timer() as timer:
            getattr(scenario_inst, method_name)(**scenario_kwargs)
    except Exception as e:
        error = utils.format_exc(e)
        if logging.is_debug():
            LOG.exception(e)
    finally:
        status = "Error %s: %s" % tuple(error[0:2]) if error else "OK"
        LOG.info("Task %(task)s | ITER: %(iteration)s END: %(status)s" %
                 {"task": context_obj["task"]["uuid"], "iteration": iteration,
                  "status": status})

        return {"duration": timer.duration() - scenario_inst.idle_duration(),
                "timestamp": timer.timestamp(),
                "idle_duration": scenario_inst.idle_duration(),
                "error": error,
                "output": scenario_inst._output,
                "atomic_actions": scenario_inst.atomic_actions()}

    def validate(self, only_syntax=False):
        """Perform full task configuration validation.

        :param only_syntax: Check only syntax of task configuration
        """
        self.task.update_status(consts.TaskStatus.VALIDATING)
        try:
            self._validate_config_syntax(self.config)
            if only_syntax:
                return
            self._validate_config_platforms(self.config)
            self._validate_config_semantic(self.config)
        except Exception as e:
            exception_info = json.dumps(traceback.format_exc(), indent=2,
                                        separators=(",", ": "))
            self.task.set_failed(type(e).__name__, str(e), exception_info)
            expected_errors = (
                # this error is a wrapper for all error messages from
                # validators.
                exceptions.InvalidTaskConfig,
                # rally.task.task_cfg raises it
                # _validate_config_semantic raises this error in case of
                # failed platform check{s}
                exceptions.ValidationError)
            if logging.is_debug() and not isinstance(e, expected_errors):
                LOG.exception("Unexpected error had happened while validating "
                              "task.")
            raise

def load_plugins(dir_or_file):
    if os.path.isdir(dir_or_file):
        directory = dir_or_file
        LOG.info(
            _("Loading plugins from directories %s/*") % directory.rstrip("/"))

        to_load = []
        for root, dirs, files in os.walk(directory, followlinks=True):
            to_load.extend((plugin[:-3], root) for plugin in files
                           if plugin.endswith(".py"))
        for plugin, directory in to_load:
            if directory not in sys.path:
                sys.path.append(directory)

            fullpath = os.path.join(directory, plugin)
            try:
                fp, pathname, descr = imp.find_module(plugin, [directory])
                imp.load_module(plugin, fp, pathname, descr)
                fp.close()
                LOG.info(_("\t Loaded module with plugins: %s.py") % fullpath)
            except Exception as e:
                LOG.warning(
                    "\t Failed to load module with plugins %(path)s.py: %(e)s"
                    % {
                        "path": fullpath,
                        "e": e
                    })
                if logging.is_debug():
                    LOG.exception(e)
    elif os.path.isfile(dir_or_file):
        plugin_file = dir_or_file
        LOG.info(_("Loading plugins from file %s") % plugin_file)
        if plugin_file not in sys.path:
            sys.path.append(plugin_file)
        try:
            plugin_name = os.path.splitext(plugin_file.split("/")[-1])[0]
            imp.load_source(plugin_name, plugin_file)
            LOG.info(_("\t Loaded module with plugins: %s.py") % plugin_name)
        except Exception as e:
            LOG.warning(
                _("\t Failed to load module with plugins %(path)s: %(e)s") % {
                    "path": plugin_file,
                    "e": e
                })
            if logging.is_debug():
                LOG.exception(e)

 def cleanup(self):
     """Clean up network"""
     try:
         self.docker_client.remove_network(self.context["netid"])
         LOG.debug("Docker network '%s' deleted" % self.context["netid"])
     except Exception as e:
         msg = "Can't delete docker network: %s" % e.message
         if logging.is_debug():
             LOG.exception(msg)
         else:
             LOG.warning(msg)

 def cleanup(self):
     """This method is called after the task finishes"""
     try:
         nova = osclients.Clients(self.config["admin"]["credential"]).nova()
         nova.flavor.delete(self.context["flavor"]["id"])
         LOG.debug("Flavor '%s' deleted" % self.context["flavor"]["id"])
     except Exception as e:
         msg = "can't delete flavor: %s" % e.message
         if logging.is_debug():
             LOG.exception(msg)
         else:
             LOG.warning(msg)

    def list_verifier_exts(self, api, verifier_id=None):
        """List all verifier extensions."""

        verifier_exts = api.verifier.list_extensions(verifier_id=verifier_id)
        if verifier_exts:
            fields = ["Name", "Entry point"]
            if logging.is_debug():
                fields.append("Location")
            cliutils.print_list(verifier_exts, fields,
                                normalize_field_names=True)
        else:
            print("There are no verifier extensions. You can add verifier "
                  "extension, using command `rally verify add-verifier-ext`.")

    def create_client(self, version=None, service_type=None):
        """Return cinder client."""
        from cinderclient import client as cinder

        client = cinder.Client(self.choose_version(version),
                               http_log_debug=logging.is_debug(),
                               timeout=CONF.openstack_client_http_timeout,
                               insecure=self.credential.insecure,
                               **self._get_auth_info(password_key="api_key"))
        kc = self.keystone()
        client.client.management_url = self._get_endpoint(service_type)
        client.client.auth_token = kc.auth_token
        return client

    def export(self, uuid, connection_string):
        """Export task results to the custom task's exporting system.

        :param uuid: UUID of the task
        :param connection_string: string used to connect to the system
        """

        parsed_obj = urlparse.urlparse(connection_string)
        try:
            client = exporter.TaskExporter.get(parsed_obj.scheme)(
                connection_string)
        except exceptions.InvalidConnectionString as e:
            if logging.is_debug():
                LOG.exception(e)
            print (e)
            return 1
        except exceptions.PluginNotFound as e:
            if logging.is_debug():
                LOG.exception(e)
            msg = ("\nPlease check your connection string. The format of "
                   "`connection` should be plugin-name://"
                   "<user>:<pwd>@<full_address>:<port>/<path>.<type>")
            print (str(e) + msg)
            return 1

        try:
            client.export(uuid)
        except (IOError, exceptions.RallyException) as e:
            if logging.is_debug():
                LOG.exception(e)
            print (e)
            return 1
        print(_("Task %(uuid)s results was successfully exported to %("
                "connection)s using %(name)s plugin.") % {
                    "uuid": uuid,
                    "connection": connection_string,
                    "name": parsed_obj.scheme
        })

    def list_plugins(self, api, platform=None):
        """List all plugins for verifiers management."""

        if platform:
            platform = platform.lower()
        verifier_plugins = api.verifier.list_plugins(platform=platform)

        fields = ["Plugin name", "Platform", "Description"]
        if logging.is_debug():
            fields.append("Location")

        cliutils.print_list(verifier_plugins, fields,
                            formatters={"Plugin name": lambda p: p["name"]},
                            normalize_field_names=True)

def _publisher(publish, queue):
    """Calls a publish method that fills queue with jobs.

    :param publish: method that fills the queue
    :param queue: deque object to be filled by the publish() method
    """
    try:
        publish(queue)
    except Exception as e:
        msg = "Failed to publish a task to the queue"
        if logging.is_debug():
            LOG.exception(msg)
        else:
            LOG.warning("%s: %s" % (msg, e))

def _run_scenario_once(args):
    iteration, cls, method_name, context_obj, kwargs = args

    LOG.info("Task %(task)s | ITER: %(iteration)s START" %
             {"task": context_obj["task"]["uuid"], "iteration": iteration})

    context_obj["iteration"] = iteration
    scenario_inst = cls(context_obj)

    error = []
    output = {"additive": [], "complete": []}
    try:
        with rutils.Timer() as timer:
            # NOTE(amaretskiy): Output as return value is deprecated
            #     but supported for backward compatibility
            deprecated_output = getattr(scenario_inst, method_name)(**kwargs)
            warning = ""
            if deprecated_output:
                warning = ("Returning output data by scenario is deprecated "
                           "in favor of calling add_output().")
            if scenario_inst._output != {"complete": [], "additive": []}:
                output = scenario_inst._output
                if deprecated_output:
                    warning += (" Output data both returned and passed to "
                                "add_output() so returned one is ignored!")
            elif deprecated_output:
                output["additive"].append({
                    "title": "Scenario output",
                    "description": "",
                    "chart_plugin": "StackedArea",
                    "data": [list(item)
                             for item in deprecated_output["data"].items()]})
            if warning:
                LOG.warning(warning)
    except Exception as e:
        error = utils.format_exc(e)
        if logging.is_debug():
            LOG.exception(e)
    finally:
        status = "Error %s: %s" % tuple(error[0:2]) if error else "OK"
        LOG.info("Task %(task)s | ITER: %(iteration)s END: %(status)s" %
                 {"task": context_obj["task"]["uuid"], "iteration": iteration,
                  "status": status})

        return {"duration": timer.duration() - scenario_inst.idle_duration(),
                "timestamp": timer.timestamp(),
                "idle_duration": scenario_inst.idle_duration(),
                "error": error,
                "output": output,
                "atomic_actions": scenario_inst.atomic_actions()}

 def create_client(self, version=None, service_type=None):
     """Return nova client."""
     from novaclient import client as nova
     kc = self.keystone()
     compute_api_url = kc.service_catalog.url_for(
         service_type=self.choose_service_type(service_type),
         endpoint_type=self.credential.endpoint_type,
         region_name=self.credential.region_name)
     client = nova.Client(self.choose_version(version),
                          auth_token=kc.auth_token,
                          http_log_debug=logging.is_debug(),
                          timeout=CONF.openstack_client_http_timeout,
                          insecure=self.credential.insecure,
                          **self._get_auth_info(password_key="api_key"))
     client.set_management_url(compute_api_url)
     return client

 def create_client(self, version=None, service_type=None):
     """Return cinder client."""
     from cinderclient import client as cinder
     client = cinder.Client(self.choose_version(version),
                            http_log_debug=logging.is_debug(),
                            timeout=CONF.openstack_client_http_timeout,
                            insecure=self.credential.insecure,
                            cacert=self.credential.cacert,
                            **self._get_auth_info(password_key="api_key"))
     kc = self.keystone()
     volume_api_url = kc.service_catalog.url_for(
         service_type=self.choose_service_type(service_type),
         endpoint_type=self.credential.endpoint_type,
         region_name=self.credential.region_name)
     client.client.management_url = volume_api_url
     client.client.auth_token = kc.auth_token
     return client

 def auth_ref(self):
     try:
         if "keystone_auth_ref" not in self.cache:
             sess, plugin = self.get_session()
             self.cache["keystone_auth_ref"] = plugin.get_access(sess)
     except Exception as e:
         if logging.is_debug():
             LOG.exception("Unable to authenticate for user"
                           " %(username)s in project"
                           " %(tenant_name)s" %
                           {"username": self.credential.username,
                            "tenant_name": self.credential.tenant_name})
         raise exceptions.AuthenticationFailed(
             username=self.credential.username,
             project=self.credential.tenant_name,
             url=self.credential.auth_url,
             etype=e.__class__.__name__,
             error=str(e))
     return self.cache["keystone_auth_ref"]