def check_starting_resources(self):
        """Check starting pacemaker resources"""

        logger.info(
            "Waiting {} seconds for changing pacemaker status of {}".format(
                self.pacemaker_restart_timeout,
                self.primary_controller_fqdn))
        time.sleep(self.pacemaker_restart_timeout)

        with self.fuel_web.get_ssh_for_node(
                self.primary_controller.name) as remote:

            def checking_health_disk_attribute_is_not_present():
                logger.info(
                    "Checking for '#health_disk' attribute "
                    "is not present on node {}".format(
                        self.primary_controller_fqdn))
                cibadmin_status_xml = remote.check_call(
                    'cibadmin --query --scope status').stdout_str
                pcs_attribs = get_pacemaker_nodes_attributes(
                    cibadmin_status_xml)
                return '#health_disk' not in pcs_attribs[
                    self.primary_controller_fqdn]

            wait(checking_health_disk_attribute_is_not_present,
                 timeout=self.pcs_check_timeout,
                 timeout_msg="Attribute #health_disk was appeared "
                             "in attributes on node {} in {} seconds".format(
                                 self.primary_controller_fqdn,
                                 self.pcs_check_timeout))

            self.fuel_web.assert_ha_services_ready(self.cluster_id)

    def check_vm_connect(self):
        """Ensure connectivity between VMs."""
        if self.vip_contr:
            primary_ctrl_name = self._get_controller_with_vip()
        else:
            primary_ctrl_name = self.fuel_web.get_nailgun_primary_node(
                self.env.d_env.nodes().slaves[0]).name

        private_ips = {}
        floating_ips = {}

        for srv in self.vms_to_ping:
            floating = self.os_conn.assign_floating_ip(srv)
            floating_ips[srv] = floating.ip
            logger.info("Floating address {0} was associated with instance "
                        "{1}".format(floating_ips[srv], srv.name))

            private_ips[srv] = self.os_conn.get_nova_instance_ip(
                srv, net_name=self.net_name)

        for vm in itertools.combinations(self.vms_to_ping, 2):
            logger.info('Try to ping from {src} ({src_vm}) to {dst} '
                        '({dst_vm})'.format(src=floating_ips[vm[0]],
                                            dst=private_ips[vm[1]],
                                            src_vm=vm[0].name,
                                            dst_vm=vm[1].name))

            assert_true(self.ping_from_instance(floating_ips[vm[0]],
                                                private_ips[vm[1]],
                                                primary_ctrl_name),
                        'Ping between VMs failed')

 def enable_plugins(self):
     """Enable plugins for Fuel if it is required"""
     for plugin_name in self.required_plugins:
         self.plugin_name = plugin_name
         self.plugin_path = self.plugins_paths[plugin_name]
         self.enable_plugin()
         logger.info("{} plugin has been enabled.".format(plugin_name))

 def install_plugins(self):
     """Install plugins for Fuel if it is required"""
     for plugin_name in self.required_plugins:
         self.plugin_name = plugin_name
         self.plugin_path = self.plugins_paths[plugin_name]
         self.install_plugin()
         logger.info("{} plugin has been installed.".format(plugin_name))

 def upload_plugins(self):
     """Upload plugins for Fuel if it is required"""
     for plugin_name in self.required_plugins:
         self.plugin_name = plugin_name
         self.plugin_path = self.plugins_paths[plugin_name]
         self.upload_plugin()
         logger.info("{} plugin has been uploaded.".format(plugin_name))

    def check_nova_conf(self):
        """Verify nova-compute vmware configuration."""
        nodes = self.fuel_web.client.list_cluster_nodes(self.cluster_id)
        vmware_attr = self.fuel_web.client.get_cluster_vmware_attributes(
            self.cluster_id)
        az = vmware_attr['editable']['value']['availability_zones'][0]
        nova_computes = az['nova_computes']

        data = []
        ctrl_nodes = self.fuel_web.get_nailgun_cluster_nodes_by_roles(
            self.cluster_id, ["controller"])
        for nova in nova_computes:
            target_node = nova['target_node']['current']['id']
            conf_dict = self.get_nova_conf_dict(az, nova)
            if target_node == 'controllers':
                conf_path = '/etc/nova/nova-compute.d/vmware-vcenter_{0}.' \
                            'conf'.format(nova['service_name'])
                for node in ctrl_nodes:
                    params = (node['hostname'], node['ip'], conf_path,
                              conf_dict)
                    data.append(params)
            else:
                conf_path = '/etc/nova/nova-compute.conf'
                for node in nodes:
                    if node['hostname'] == target_node:
                        params = (node['hostname'], node['ip'], conf_path,
                                  conf_dict)
                        data.append(params)

        for hostname, ip, conf_path, conf_dict in data:
            logger.info("Check nova conf of {0}".format(hostname))
            self.check_config(ip, conf_path, conf_dict)

    def check_vmware_service_actions(self):
        """Disable vmware host (cluster) and check instance creation on
        enabled cluster."""
        public_ip = self.fuel_web.get_public_vip(self.cluster_id)
        os_conn = OpenStackActions(public_ip)

        services = os_conn.get_nova_service_list()
        vmware_services = []
        for service in services:
            if service.binary == 'nova-compute' and \
               service.zone == self.vcenter_az:
                vmware_services.append(service)
                os_conn.disable_nova_service(service)

        image = os_conn.get_image(self.vmware_image)
        sg = os_conn.get_security_group(self.sg_name)
        net = os_conn.get_network(self.net_name)

        for service in vmware_services:
            logger.info("Check {}".format(service.host))
            os_conn.enable_nova_service(service)
            vm = os_conn.create_server(image=image, timeout=210,
                                       availability_zone=self.vcenter_az,
                                       net_id=net['id'], security_groups=[sg])
            vm_host = getattr(vm, 'OS-EXT-SRV-ATTR:host')
            assert_true(service.host == vm_host, 'Instance was launched on a'
                                                 ' disabled vmware cluster')
            os_conn.delete_instance(vm)
            os_conn.verify_srv_deleted(vm)
            os_conn.disable_nova_service(service)

    def check_example_plugin(self):
        """Check if service ran on controller"""

        cmd_curl = "curl localhost:8234"
        cmd = "pgrep -f fuel-simple-service"

        n_ctrls = self.fuel_web.get_nailgun_cluster_nodes_by_roles(cluster_id=self.cluster_id, roles=["controller"])
        d_ctrls = self.fuel_web.get_devops_nodes_by_nailgun_nodes(n_ctrls)

        for node in d_ctrls:
            logger.info("Check plugin service on node {0}".format(node.name))
            with self.fuel_web.get_ssh_for_node(node.name) as remote:
                res_pgrep = remote.execute(cmd)
                assert_equal(
                    0, res_pgrep["exit_code"], "Failed with error {0} " "on node {1}".format(res_pgrep["stderr"], node)
                )
                assert_equal(
                    1,
                    len(res_pgrep["stdout"]),
                    "Failed with error {0} on the " "node {1}".format(res_pgrep["stderr"], node),
                )
                # curl to service
                res_curl = remote.execute(cmd_curl)
                assert_equal(
                    0, res_pgrep["exit_code"], "Failed with error {0} " "on node {1}".format(res_curl["stderr"], node)
                )

    def check_up_vips(self):
        """Ensure that VIPs are moved to another controller."""
        vip_contr = self._get_controller_with_vip()

        assert_true(vip_contr and vip_contr != self.vip_contr,
                    'VIPs have not been moved to another controller')
        logger.info('VIPs have been moved to another controller')

    def fill_root_below_rabbit_disk_free_limit(self):
        """Fill root more to below rabbit disk free limit"""

        node = self.fuel_web.get_nailgun_node_by_name(
            self.primary_controller.name)
        pacemaker_attributes = self.ssh_manager.execute_on_remote(
            ip=node['ip'],
            cmd='cibadmin --query --scope status'
        )['stdout_str']
        controller_space_on_root = get_pacemaker_nodes_attributes(
            pacemaker_attributes)[self.primary_controller_fqdn]['root_free']

        logger.info("Free space in root on primary controller - {}".format(
                    controller_space_on_root))

        controller_space_to_filled = str(
            int(controller_space_on_root) - self.rabbit_disk_free_limit - 1
        )

        logger.info("Need to fill space on root - {}".format(
            controller_space_to_filled))

        self.ssh_manager.execute_on_remote(
            ip=node['ip'],
            cmd='fallocate -l {}M /root/bigfile2 && sync'.format(
                controller_space_to_filled)
        )
        self.ssh_manager.execute_on_remote(
            ip=node['ip'],
            cmd='ls /root/bigfile2',
            assert_ec_equal=[0])

    def check_example_plugin(self):
        """Check if service ran on controller"""

        cmd_curl = 'curl localhost:8234'
        cmd = 'pgrep -f fuel-simple-service'

        n_ctrls = self.fuel_web.get_nailgun_cluster_nodes_by_roles(
            cluster_id=self.cluster_id,
            roles=['controller'])
        d_ctrls = self.fuel_web.get_devops_nodes_by_nailgun_nodes(n_ctrls)

        for node in d_ctrls:
            logger.info("Check plugin service on node {0}".format(node.name))
            with self.fuel_web.get_ssh_for_node(node.name) as remote:
                res_pgrep = remote.execute(cmd)
                assert_equal(0, res_pgrep['exit_code'],
                             'Failed with error {0} '
                             'on node {1}'.format(res_pgrep['stderr'], node))
                assert_equal(1, len(res_pgrep['stdout']),
                             'Failed with error {0} on the '
                             'node {1}'.format(res_pgrep['stderr'], node))
                # curl to service
                res_curl = remote.execute(cmd_curl)
                assert_equal(0, res_pgrep['exit_code'],
                             'Failed with error {0} '
                             'on node {1}'.format(res_curl['stderr'], node))

    def check_batch_instance_creation(self):
        """Create several instance simultaneously."""
        count = 10
        vm_name = 'vcenter_vm'

        public_ip = self.fuel_web.get_public_vip(self.cluster_id)
        os_conn = OpenStackActions(public_ip)

        image = os_conn.get_image(self.vmware_image)
        net = os_conn.get_network(self.net_name)
        sg = os_conn.get_security_group(self.sg_name)
        os_conn.create_server(name=vm_name, image=image,
                              availability_zone=self.vcenter_az,
                              net_id=net['id'], security_groups=[sg],
                              min_count=count, timeout=210)

        for i in range(1, count + 1):
            vm = os_conn.get_server_by_name('{name}-{index}'.format(
                name=vm_name, index=i))
            logger.info("Check state for {} instance".format(vm.name))
            helpers.wait(
                lambda: os_conn.get_instance_detail(vm).status == "ACTIVE",
                timeout=180, timeout_msg="Instance state is not active"
            )

        for i in range(1, count + 1):
            vm = os_conn.get_server_by_name('{name}-{index}'.format(
                name=vm_name, index=i))
            os_conn.delete_instance(vm)
            os_conn.verify_srv_deleted(vm)

    def fill_root_below_rabbit_disk_free_limit(self):
        """Fill root more to below rabbit disk free limit"""

        with self.fuel_web.get_ssh_for_node(
                self.primary_controller.name) as remote:

            pacemaker_attributes = run_on_remote_get_results(
                remote, 'cibadmin --query --scope status')['stdout_str']

            controller_space_on_root = get_pacemaker_nodes_attributes(
                pacemaker_attributes)[self.primary_controller_fqdn][
                'root_free']

            logger.info(
                "Free space in root on primary controller - {}".format(
                    controller_space_on_root
                ))

            controller_space_to_filled = str(
                int(
                    controller_space_on_root
                ) - self.rabbit_disk_free_limit - 1)

            logger.info(
                "Need to fill space on root - {}".format(
                    controller_space_to_filled
                ))

            run_on_remote_get_results(
                remote, 'fallocate -l {}M /root/bigfile2'.format(
                    controller_space_to_filled))
            check_file_exists(remote, '/root/bigfile2')

    def create_env(self):
        """Create Fuel Environment

        For configure Environment use environment-config section in config file

        Skip action if we have snapshot with Environment name

        """
        self.check_run(self.env_config['name'])

        logger.info("Create env {}".format(
            self.env_config['name']))
        cluster_settings = {
            "murano": self.env_settings['components'].get('murano', False),
            "sahara": self.env_settings['components'].get('sahara', False),
            "ceilometer": self.env_settings['components'].get('ceilometer',
                                                              False),
            "ironic": self.env_settings['components'].get('ironic', False),
            "user": self.env_config.get("user", "admin"),
            "password": self.env_config.get("password", "admin"),
            "tenant": self.env_config.get("tenant", "admin"),
            "volumes_lvm": self.env_settings['storages'].get("volume-lvm",
                                                             False),
            "volumes_ceph": self.env_settings['storages'].get("volume-ceph",
                                                              False),
            "images_ceph": self.env_settings['storages'].get("image-ceph",
                                                             False),
            "ephemeral_ceph": self.env_settings['storages'].get(
                "ephemeral-ceph", False),
            "objects_ceph": self.env_settings['storages'].get("rados-ceph",
                                                              False),
            "osd_pool_size": str(self.env_settings['storages'].get(
                "replica-ceph", 2)),
            "net_provider": self.env_config['network'].get('provider',
                                                           'neutron'),
            "net_segment_type": self.env_config['network'].get('segment-type',
                                                               'vlan'),
            "assign_to_all_nodes": self.env_config['network'].get(
                'pubip-to-all',
                False),
            "neutron_l3_ha": self.env_config['network'].get(
                'neutron-l3-ha', False),
            "neutron_dvr": self.env_config['network'].get(
                'neutron-dvr', False),
            "neutron_l2_pop": self.env_config['network'].get(
                'neutron-l2-pop', False),
            "neutron_qos": self.env_config['network'].get(
                'neutron-qos', False),
        }

        self.cluster_id = self.fuel_web.create_cluster(
            name=self.env_config['name'],
            mode=settings.DEPLOYMENT_MODE,
            release_name=settings.OPENSTACK_RELEASE_UBUNTU
            if self.env_config['release'] == 'ubuntu'
            else settings.OPENSTACK_RELEASE,
            settings=cluster_settings)

        logger.info("Cluster created with ID:{}".format(self.cluster_id))

 def check_stopping_resources():
     logger.info(
         "Checking for 'running_resources "
         "attribute have '0' value")
     pcs_status = parse_pcs_status_xml(remote)
     pcs_attribs = get_pcs_nodes(pcs_status)
     return pcs_attribs[self.primary_controller_fqdn][
         'resources_running'] == '0'

 def checking_health_disk_attribute():
     logger.info("Checking for '#health_disk' attribute")
     cibadmin_status_xml = remote.check_call(
         'cibadmin --query --scope status').stdout_str
     pcs_attribs = get_pacemaker_nodes_attributes(
         cibadmin_status_xml)
     return '#health_disk' in pcs_attribs[
         self.primary_controller_fqdn]

 def checking_health_disk_attribute():
     logger.info("Checking for '#health_disk' attribute")
     cibadmin_status_xml = run_on_remote_get_results(
         remote, 'cibadmin --query --scope status')[
         'stdout_str']
     pcs_attribs = get_pacemaker_nodes_attributes(
         cibadmin_status_xml)
     return '#health_disk' in pcs_attribs[
         self.primary_controller_fqdn]

 def _finish_case(self):
     """Finish test case"""
     case_time = time.time() - self._start_time
     minutes = int(round(case_time)) / 60
     seconds = int(round(case_time)) % 60
     name = getattr(self, "__doc__", self.__class__.__name__).splitlines()[0]
     finish_case = "[ FINISH {} CASE TOOK {} min {} sec ]".format(name, minutes, seconds)
     footer = "<<< {:=^142} >>>".format(finish_case)
     logger.info("\n{footer}\n".format(footer=footer))

 def fail_deploy_cluster(self):
     """Deploy environment (must fail)."""
     try:
         self.fuel_web.deploy_cluster_wait(self.cluster_id)
         failed = False
     except AssertionError:
         failed = True
     assert_true(failed, 'Deploy passed with incorrect parameters')
     logger.info('Deploy failed')

 def check_started_resources():
     logger.info(
         "Checking for 'running_resources' attribute "
         "have {} value on node {}".format(
             self.slave_node_running_resources,
             self.primary_controller_fqdn))
     pcs_status = parse_pcs_status_xml(remote)
     pcs_attribs = get_pcs_nodes(pcs_status)
     return pcs_attribs[self.primary_controller_fqdn][
         'resources_running'] == self.slave_node_running_resources