def _load_img(self):
     cirros_url = ('http://download.cirros-cloud.net/%s/%s' %
                   (CONF.image.cirros_version,
                    CONF.image.cirros_image))
     try:
         response = requests.get(cirros_url, stream=True)
     except requests.ConnectionError as err:
         msg = _('Error on downloading cirros image, possibly'
                 ' no connection to Internet with message %s') % str(err)
         raise TempestConfigCreationFailure(msg)
     if response.status_code == 200:
         with open(self.img_path + '.tmp', 'wb') as img_file:
             for chunk in response.iter_content(chunk_size=1024):
                 if chunk:   # filter out keep-alive new chunks
                     img_file.write(chunk)
                     img_file.flush()
         os.rename(self.img_path + '.tmp', self.img_path)
     else:
         if response.status_code == 404:
             msg = _('Error on downloading cirros image, possibly'
                     'invalid cirros_version or cirros_image in rally.conf')
         else:
             msg = _('Error on downloading cirros image, '
                     'HTTP error code %s') % response.getcode()
         raise TempestConfigCreationFailure(msg)

 def _initialize_testr(self):
     if not os.path.isdir(self.path(".testrepository")):
         msg = _("Test Repository initialization.")
         LOG.info(_("Starting: ") + msg)
         subprocess.check_call("%s testr init" % self.venv_wrapper,
                               shell=True, cwd=self.path())
         LOG.info(_("Completed: ") + msg)

def required_openstack(config, clients, task, admin=False, users=False):
    """Validator that requires OpenStack admin or (and) users.

    This allows us to create 4 kind of benchmarks:
    1) not OpenStack related (validator is not specified)
    2) requires OpenStack admin
    3) requires OpenStack admin + users
    4) requires OpenStack users

    :param admin: requires OpenStack admin
    :param users: requires OpenStack users
    """

    if not (admin or users):
        return ValidationResult(
            False, _("You should specify admin=True or users=True or both."))

    deployment = objects.Deployment.get(task["deployment_uuid"])

    if deployment["admin"] and deployment["users"]:
        return ValidationResult()

    if deployment["admin"]:
        if users and not config.get("context", {}).get("users"):
            return ValidationResult(False,
                                    _("You should specify 'users' context"))
        return ValidationResult()

    if deployment["users"] and admin:
        return ValidationResult(False, _("Admin credentials required"))

    return ValidationResult()

def image_valid_on_flavor(config, clients, task, flavor_name, image_name):
    """Returns validator for image could be used for current flavor

    :param flavor_name: defines which variable should be used
                       to get flavor id value.
    :param image_name: defines which variable should be used
                       to get image id value.

    """
    valid_result, flavor = _get_validated_flavor(config, clients, flavor_name)
    if not valid_result.is_valid:
        return valid_result

    valid_result, image = _get_validated_image(config, clients, image_name)
    if not valid_result.is_valid:
        return valid_result

    if flavor.ram < (image.min_ram or 0):
        message = _("The memory size for flavor '%s' is too small "
                    "for requested image '%s'") % (flavor.id, image.id)
        return ValidationResult(False, message)

    if flavor.disk:
        if (image.size or 0) > flavor.disk * (1024 ** 3):
            message = _("The disk size for flavor '%s' is too small "
                        "for requested image '%s'") % (flavor.id, image.id)
            return ValidationResult(False, message)

        if (image.min_disk or 0) > flavor.disk:
            message = _("The disk size for flavor '%s' is too small "
                        "for requested image '%s'") % (flavor.id, image.id)
            return ValidationResult(False, message)
    return ValidationResult()

 def wrapper(self, *args, **kwargs):
     params = {"msg": msg % kw, "obj_name": obj.title(),
               "uuid": getattr(self, obj)["uuid"]}
     log(_("%(obj_name)s %(uuid)s | Starting:  %(msg)s") % params)
     result = f(self, *args, **kwargs)
     log(_("%(obj_name)s %(uuid)s | Completed: %(msg)s") % params)
     return result

 def _initialize_testr(self):
     if not os.path.isdir(self.path(".testrepository")):
         print(_("Test Repository initialization."))
         try:
             check_output("%s testr init" % self.venv_wrapper,
                          shell=True, cwd=self.path())
         except subprocess.CalledProcessError:
             if os.path.exists(self.path(".testrepository")):
                 shutil.rmtree(self.path(".testrepository"))
             raise TempestSetupFailure(_("failed to initialize testr"))

    def _delete_single_resource(self, resource):
        """Safe resource deletion with retries and timeouts.

        Send request to delete resource, in case of failures repeat it few
        times. After that pull status of resource until it's deleted.

        Writes in LOG warning with UUID of resource that wasn't deleted

        :param resource: instance of resource manager initiated with resource
                         that should be deleted.
        """

        msg_kw = {
            "uuid": resource.id(),
            "service": resource._service,
            "resource": resource._resource
        }

        try:
            rutils.retry(resource._max_attempts, resource.delete)
        except Exception as e:
            msg_kw["reason"] = e
            LOG.warning(
                _("Resource deletion failed, max retries exceeded for "
                  "%(service)s.%(resource)s: %(uuid)s. Reason: %(reason)s")
                % msg_kw)
            if CONF.debug:
                LOG.exception(e)
        else:
            started = time.time()
            failures_count = 0
            while time.time() - started < resource._timeout:
                try:
                    if resource.is_deleted():
                        return
                except Exception as e:
                    LOG.warning(
                        _("Seems like %s.%s.is_deleted(self) method is broken "
                          "It shouldn't raise any exceptions.")
                        % (resource.__module__, type(resource).__name__))
                    LOG.exception(e)

                    # NOTE(boris-42): Avoid LOG spaming in case of bad
                    #                 is_deleted() method
                    failures_count += 1
                    if failures_count > resource._max_attempts:
                        break

                finally:
                    time.sleep(resource._interval)

            LOG.warning(_("Resource deletion failed, timeout occurred for "
                          "%(service)s.%(resource)s: %(uuid)s.")
                        % msg_kw)

    def generate_config_file(self):
        """Generate configuration file of tempest for current deployment."""

        LOG.debug("Tempest config file: %s " % self.config_file)
        if not self.is_configured():
            msg = _("Creation of configuration file for tempest.")
            LOG.info(_("Starting: ") + msg)

            config.TempestConf(self.deploy_id).generate(self.config_file)
            LOG.info(_("Completed: ") + msg)
        else:
            LOG.info("Tempest is already configured.")

def required_services(config, clients, task, *required_services):
    """Validator checks if specified OpenStack services are available.

    :param *required_services: list of services names
    """
    available_services = clients.services().values()
    for service in required_services:
        if service not in consts.Service:
            return ValidationResult(False, _("Unknown service: %s") % service)
        if service not in available_services:
            return ValidationResult(
                False, _("Service is not available: %s") % service)

    return ValidationResult()

def create_deploy(config, name):
    """Create a deployment.

    :param config: a dict with deployment configuration
    :param name: a str represents a name of the deployment
    """

    try:
        deployment = objects.Deployment(name=name, config=config)
    except exceptions.DeploymentNameExists as e:
        if CONF.debug:
            LOG.exception(e)
        raise

    deployer = deploy.EngineFactory.get_engine(deployment['config']['type'],
                                               deployment)
    try:
        deployer.validate()
    except jsonschema.ValidationError:
        LOG.error(_('Deployment %(uuid)s: Schema validation error.') %
                  {'uuid': deployment['uuid']})
        deployment.update_status(consts.DeployStatus.DEPLOY_FAILED)
        raise

    with deployer:
        endpoints = deployer.make_deploy()
        deployment.update_endpoints(endpoints)
        return deployment

def _consumer(consume, queue, is_published):
    """Infinity worker that consumes tasks from queue.

    This finishes it's work only in case if is_published.isSet().

    :param consume: method that consumes an object removed from the queue
    :param queue: deque object to popleft() objects from
    :param is_published: threading.Event that is used to stop the consumer
                         when the queue is empty
    """
    cache = {}
    while True:
        if queue:
            try:
                consume(cache, queue.popleft())
            except IndexError:
                # NOTE(boris-42): queue is accessed from multiple threads so
                #                 it's quite possible to have 2 queue accessing
                #                 at the same point queue with only 1 element
                pass
            except Exception as e:
                LOG.warning(_("Failed to consume a task from the queue: "
                              "%s") % e)
                if CONF.debug:
                    LOG.exception(e)
        elif is_published.isSet():
            break
        else:
            time.sleep(0.1)

    def start(self, task, deploy_id=None, tag=None, do_use=False):
        """Start benchmark task.

        :param task: a file with yaml/json configration
        :param deploy_id: a UUID of a deployment
        :param tag: optional tag for this task
        """
        task = os.path.expanduser(task)
        with open(task, 'rb') as task_file:
            config_dict = yaml.safe_load(task_file.read())
            try:
                task = api.create_task(deploy_id, tag)
                print("=" * 80)
                print(_("Task %(tag)s %(uuid)s is started")
                      % {"uuid": task["uuid"], "tag": task["tag"]})
                print("-" * 80)
                api.start_task(deploy_id, config_dict, task=task)
                self.detailed(task_id=task['uuid'])
                if do_use:
                    use.UseCommands().task(task['uuid'])
            except exceptions.InvalidConfigException:
                return(1)
            except KeyboardInterrupt:
                api.abort_task(task['uuid'])
                raise

    def flavors(self, deployment=None):
        """Display available flavors.

        :param deployment: UUID or name of a deployment
        """

        headers = ['ID', 'Name', 'vCPUs', 'RAM (MB)', 'Swap (MB)', 'Disk (GB)']
        mixed_case_fields = ['ID', 'Name', 'vCPUs']
        float_cols = ['RAM (MB)', 'Swap (MB)', 'Disk (GB)']
        formatters = dict(zip(float_cols,
                              [cliutils.pretty_float_formatter(col)
                               for col in float_cols]))
        table_rows = []
        try:
            for endpoint_dict in self._get_endpoints(deployment):
                clients = osclients.Clients(endpoint.Endpoint(**endpoint_dict))
                nova_client = clients.nova()
                for flavor in nova_client.flavors.list():
                    data = [flavor.id, flavor.name, flavor.vcpus,
                            flavor.ram, flavor.swap, flavor.disk]
                    table_rows.append(utils.Struct(**dict(zip(headers, data))))

                common_cliutils.print_list(table_rows,
                                           fields=headers,
                                           formatters=formatters,
                                           mixed_case_fields=mixed_case_fields)

        except exceptions.InvalidArgumentsException as e:
            print(_("Authentication Issues: %s") % e)
            return(1)

 def check(criterion_value, result):
     durations = [r["duration"] for r in result if not r.get("error")]
     avg = putils.mean(durations)
     success = avg < criterion_value
     msg = (_("Maximum average duration per iteration %ss, found with %ss")
            % (criterion_value, avg))
     return SLAResult(success, msg)

    def images(self, deployment=None):
        """Display available images.

        :param deployment: UUID or name of a deployment
        """

        headers = ['UUID', 'Name', 'Size (B)']
        mixed_case_fields = ['UUID', 'Name']
        float_cols = ["Size (B)"]
        table_rows = []
        formatters = dict(zip(float_cols,
                              [cliutils.pretty_float_formatter(col)
                               for col in float_cols]))

        try:
            for endpoint_dict in self._get_endpoints(deployment):
                clients = osclients.Clients(endpoint.Endpoint(**endpoint_dict))
                glance_client = clients.glance()
                for image in glance_client.images.list():
                    data = [image.id, image.name, image.size]
                    table_rows.append(utils.Struct(**dict(zip(headers, data))))

                common_cliutils.print_list(table_rows,
                                           fields=headers,
                                           formatters=formatters,
                                           mixed_case_fields=mixed_case_fields)

        except exceptions.InvalidArgumentsException as e:
            print(_("Authentication Issues: %s") % e)
            return(1)

    def _cleanup_resources(self):
        for user in self.users_endpoints:
            clients = osclients.Clients(user)
            tenant_id = clients.keystone().tenant_id
            cleanup_methods = {
                "nova": (utils.delete_nova_resources, clients.nova),
                "glance": (utils.delete_glance_resources, clients.glance,
                           tenant_id),
                "cinder": (utils.delete_cinder_resources, clients.cinder),
                "neutron": (utils.delete_neutron_resources, clients.neutron,
                            tenant_id),
                "ceilometer": (utils.delete_ceilometer_resources,
                               clients.ceilometer, tenant_id),
                "heat": (utils.delete_heat_resources, clients.heat),
                "sahara": (utils.delete_sahara_resources, clients.sahara),
                "designate": (utils.delete_designate_resources,
                              clients.designate),
                "zaqar": (utils.delete_zaqar_resources, clients.zaqar),
            }

            for service_name in self.config:
                cleanup_method = cleanup_methods[service_name]
                method = cleanup_method[0]
                client = cleanup_method[1]()
                try:
                    method(client, *cleanup_method[2:])
                except Exception as e:
                    LOG.debug("Not all user resources were cleaned.",
                              exc_info=sys.exc_info())
                    LOG.warning(_('Unable to fully cleanup the cloud: %s') %
                                (six.text_type(e)))

    def check(self, deployment=None):
        """Check keystone authentication and list all available services.

        :param deployment: a UUID or name of the deployment
        """

        headers = ['services', 'type', 'status']
        table_rows = []
        try:
            admin = db.deployment_get(deployment)['admin']
            # TODO(boris-42): make this work for users in future
            for endpoint_dict in [admin]:
                clients = osclients.Clients(endpoint.Endpoint(**endpoint_dict))
                client = clients.verified_keystone()
                print("keystone endpoints are valid and following "
                      "services are available:")
                for service in client.services.list():
                    data = [service.name, service.type, 'Available']
                    table_rows.append(utils.Struct(**dict(zip(headers, data))))
        except exceptions.InvalidArgumentsException:
            data = ['keystone', 'identity', 'Error']
            table_rows.append(utils.Struct(**dict(zip(headers, data))))
            print(_("Authentication Issues: %s.")
                  % sys.exc_info()[1])
            return(1)
        common_cliutils.print_list(table_rows, headers)

def main():
    # Initialize configuration and logging.
    CONF(sys.argv[1:], project='rally')
    log.setup('rally')
    # Prepare application and bind to the service socket.
    host = CONF.rest.host
    port = CONF.rest.port
    app = rally_app.make_app()
    server = simple_server.make_server(host, port, app)
    # Start application.
    LOG.info(_('Starting server in PID %s') % os.getpid())
    LOG.info(_("Configuration:"))
    CONF.log_opt_values(LOG, logging.INFO)
    try:
        server.serve_forever()
    except KeyboardInterrupt:
        pass

 def validate(self):
     super(LxcEngine, self).validate()
     if 'start_lxc_network' not in self.config:
         return
     lxc_net = netaddr.IPNetwork(self.config['start_lxc_network'])
     num_containers = self.config['containers_per_host']
     if lxc_net.size - 3 < num_containers:
         message = _("Network size is not enough for %d hosts.")
         raise exceptions.InvalidConfigException(message % num_containers)

 def _publish(admin, user, manager):
     try:
         for raw_resource in rutils.retry(3, manager.list):
             queue.append((admin, user, raw_resource))
     except Exception as e:
         LOG.warning(
             _("Seems like %s.%s.list(self) method is broken. "
               "It shouldn't raise any exceptions.")
             % (manager.__module__, type(manager).__name__))
         LOG.exception(e)