def test_cpu_pinning_one_numa_cell(
            self, env, os_conn, networks, flavors, security_group,
            aggregate):
        """This test checks that cpu pinning executed successfully for
        instances created on computes with 1 NUMA
        Steps:
            1. Create net1 with subnet, net2 with subnet and router1 with
            interfaces to both nets
            2. Launch instances vm1, vm3 in net1 with m1.small.performance on
            compute-1, vm2 on compute-2.
            3. Check numa nodes for all vms
            4. Check parameter in /etc/defaults/grub
            5. Check vms connectivity
        """
        hosts = aggregate.hosts
        vms = []
        network_for_instances = [networks[0], networks[1], networks[0]]
        hosts_for_instances = [hosts[0], hosts[1], hosts[0]]
        cpus = get_cpu_distribition_per_numa_node(env)

        for i in range(2):
            vms.append(os_conn.create_server(
                name='vm{}'.format(i),
                flavor=flavors[0].id,
                nics=[{'net-id': network_for_instances[i]}],
                availability_zone='nova:{}'.format(hosts_for_instances[i]),
                security_groups=[security_group.id]))

        for vm in vms:
            host = getattr(vm, "OS-EXT-SRV-ATTR:host")
            assert host in hosts
            self.check_cpu_for_vm(os_conn, vm, 1, cpus[host])

        network_checks.check_vm_connectivity(env, os_conn)

    def test_vms_with_cpu_thread_policy_wo_hyper_threading(
            self, env, os_conn, hosts_without_hyper_threading, flavors,
            networks, keypair, security_group):
        """This test checks vms with cpu_thread_policy parameter in case of
        disabled hyper-threading

        Steps:
            1. Create net1 with subnet and router1 with interface to net1
            2. Create cpu pinning flavors with hw:numa_nodes=1 and
            cpu_thread_policy
            3. Boot vm and check that all vcpus are on the different core
            4. Redo for all flavors
            5. Check vms connectivity
        """

        host = hosts_without_hyper_threading[0]
        zone = 'nova:{}'.format(host)

        for flv in flavors:
            vm = os_conn.create_server(name='vm{}'.format(flv.name),
                                       flavor=flv.id,
                                       key_name=keypair.name,
                                       nics=[{'net-id': networks[0]}],
                                       security_groups=[security_group.id],
                                       availability_zone=zone)

            used_ts_list = self.get_vm_thread_siblings_lists(os_conn, vm)
            assert len(used_ts_list) == flv.vcpus, (
                "vcpus should be on the different cores")

        network_checks.check_vm_connectivity(env, os_conn)

    def test_l3_agent_after_drop_rabbit_port(self):
        """Drop rabbit port and check l3-agent work

        Scenario:
            1. Revert snapshot with neutron cluster
            2. Create network1, network2
            3. Create router1 and connect it with network1, network2 and
               external net
            4. Boot vm1 in network1 and associate floating ip
            5. Boot vm2 in network2
            6. Add rules for ping
            7. ping 8.8.8.8, vm1 (both ip) and vm2 (fixed ip) from each other
            8. with iptables in CLI drop rabbit's port #5673 on what router1 is
            9. Wait for route rescheduling
            10. Check that router moved to the health l3-agent
            11. Boot one more VM (VM3) in network1
            12. Boot vm3 in network1
            13. ping 8.8.8.8, vm1 (both ip), vm2 (fixed ip) and vm3 (fixed ip)
                from each other

        Duration 10m

        """
        # drop rabbit port
        self.drop_rabbit_port(router_name="router01")

        # check pings
        network_checks.check_vm_connectivity(self.env, self.os_conn)

    def test_vms_connectivity_after_ovs_restart_on_computes(
            self, env, os_conn, computes_with_dpdk_hp, flavors, networks,
            keypair, security_group):
        """This test checks connectivity between VMs with DPDK after ovs
        restart on computes. Please note we're not able to count DPDK huge
        pages only, they're added to count of 2Mb huge pages.
            Steps:
            1. Create net1 with subnet, net2 with subnet and  router1 with
            interfaces to both nets
            2. Create flavor for huge pages with 512Mb ram, 1 vcpu and 1Gb disk
            3. Launch vm1, vm2, vm3 on compute-1 and vm4 on compute-2, vm1 and
            vm2 in net1, vm3 and vm4 in net2
            4. Check that neutron port has binding:vif_type = vhostuser
            5. Check instance page size
            6. Restart ovs on computes
            7. Check vms connectivity after ovs restart
        """

        hosts = computes_with_dpdk_hp
        vms_param = [(hosts[0], networks[0], None),
                     (hosts[0], networks[0], None),
                     (hosts[0], networks[1], None),
                     (hosts[1], networks[1], None)]
        self.create_vms(os_conn, hosts, networks, flavors[0], keypair,
                        security_group, vms_param)

        network_checks.check_vm_connectivity(env, os_conn, vm_keypair=keypair)
        self.restart_ovs_on_computes(env, os_conn)
        network_checks.check_vm_connectivity(env, os_conn, vm_keypair=keypair)

    def test_shutdown_primary_controller_with_l3_agt(self):
        """[Neutron VLAN and VXLAN] Shut down primary controller
           and check l3-agent

        Steps:
            1. Check on what agents is router1:
                neutron l3-agent-list-hosting-router router1
            2. If there isn't agent on the primary controller:
                neutron l3-agent-router-remove non_on_primary_agent_id router1
                neutron l3-agent-router-add on_primary_agent_id router1
            3. Destroy primary controller
                virsh destroy <primary_controller>
            4. Wait some time until all agents are up
                neutron-agent-list
            5. Check that all routers reschedule from primary controller:
                neutron router-list-on-l3-agent <on_primary_agent_id>
            6. Boot vm3 in network1
            7. ping 8.8.8.8 from vm3
            8. ping between vm1 and vm3 by internal ip
            9. ping between vm1 and vm2 by floating ip
        """

        self._prepare_openstack()
        # Get current L3 agent on router01
        l3_agent = self.os_conn.neutron.list_l3_agent_hosting_routers(
            self.router['id'])['agents'][0]
        # Check if the agent is not on the primary controller
        # Reschedule if needed
        if l3_agent['host'] != self.primary_host:

            self.os_conn.reschedule_router_to_primary_host(self.router['id'],
                                                           self.primary_host)
            l3_agent = self.os_conn.neutron.list_l3_agent_hosting_routers(
                self.router['id'])['agents'][0]

        # virsh destroy of the primary controller
        self.env.destroy_nodes([self.primary_node])

        # Excluding the id of the l3_agent from the list
        # since it will stay on the destroyed controller
        # and remain disabled
        self.l3_agent_ids.remove(l3_agent['id'])

        # Then check that the rest l3 agents are alive
        self.os_conn.wait_agents_alive(self.l3_agent_ids)

        # Check that there are no routers on the first agent
        self.check_no_routers_on_l3_agent(l3_agent['id'])

        # Waiting for messaging layer recovery
        # https://bugs.launchpad.net/mos/+bug/1592312
        logger.debug('Waiting 5 minutes for messaging layer recovery')
        time.sleep(5 * 60)

        self.os_conn.add_server(self.networks[0],
                                self.instance_keypair.name,
                                self.hosts[0],
                                self.security_group.id)
        # Create one more server and check connectivity
        network_checks.check_vm_connectivity(self.env, self.os_conn)

    def test_cpu_pinning_resize(
            self, env, os_conn, networks, flavors, security_group,
            aggregate, aggregate_n):
        """This test checks that cpu pinning executed successfully for
        instances created on computes with 1 NUMA
        Steps:
            1. Create net1 with subnet, net2 with subnet and router1 with
            interfaces to both nets
            2. Launch vm1 using m1.small.performance-1 flavor on compute-1 and
            vm2 on compute-2 with m1.small.old flavor.
            3. Resize vm1 to m1.small.performance-2
            4. Ping vm1 from vm2
            5. Resize vm1 to m1.small.performance-3
            6. Ping vm1 from vm2
            7. Resize vm1 to m1.small.performance-1
            8. Ping vm1 from vm2
            9. Resize vm1 to m1.small.old
            10. Ping vm1 from vm2
            11. Resize vm1 to m1.small.performance-4
            12. Ping vm1 from vm2
            13. Resize vm1 to m1.small.performance-1
            14. Ping vm1 from vm2
        """
        hosts = aggregate.hosts
        vms = []
        cpus = get_cpu_distribition_per_numa_node(env)
        flavors_for_resize = ['m1.small.perfomance-2',
                              'm1.small.perfomance-3',
                              'm1.small.perfomance-1',
                              'm1.small.old', 'm1.small.perfomance-4',
                              'm1.small.perfomance-1']

        for i in range(2):
            vms.append(os_conn.create_server(
                name='vm{}'.format(i),
                flavor=flavors[i].id,
                nics=[{'net-id': networks[i]}],
                availability_zone='nova:{}'.format(hosts[i]),
                security_groups=[security_group.id]))
        vm = vms[0]

        for flavor in flavors_for_resize:
            numas = 2
            for object_flavor in flavors:
                if object_flavor.name == flavor:
                    vm = self.resize(os_conn, vm, object_flavor.id)
                    break
            if flavor is not 'm1.small.old':
                if flavor in ['m1.small.perfomance-4',
                              'm1.small.perfomance-1']:
                    numas = 1
                host = getattr(vm, "OS-EXT-SRV-ATTR:host")
                assert host in hosts
                self.check_cpu_for_vm(os_conn,
                                      os_conn.get_instance_detail(vm),
                                      numas, cpus[host])
            os_conn.wait_servers_ssh_ready(vms)
            network_checks.check_vm_connectivity(env, os_conn)

    def test_ban_l3_agents_and_clear_last(self):
        """Ban all l3-agents, clear last of them and check health of l3-agent

        Scenario:
            1. Revert snapshot with neutron cluster
            2. Create network1, network2
            3. Create router1 and connect it with network1, network2 and
               external net
            4. Boot vm1 in network1 and associate floating ip
            5. Boot vm2 in network2
            6. Add rules for ping
            7. ping 8.8.8.8, vm1 (both ip) and vm2 (fixed ip) from each other
            8. Ban l3-agent on what router1 is
            9. Wait for route rescheduling
            10. Repeat steps 7-8 twice
            11. Clear last L3 agent
            12. Check that router moved to the health l3-agent
            13. Boot one more VM (VM3) in network1
            14. Boot vm3 in network1
            15. ping 8.8.8.8, vm1 (both ip), vm2 (fixed ip) and vm3 (fixed ip)
                from each other

        Duration 10m

        """
        net_id = self.os_conn.neutron.list_networks(
            name="net01")['networks'][0]['id']
        devops_node = self.get_node_with_dhcp(net_id)
        ip = devops_node.data['ip']

        # ban l3 agents
        for _ in range(2):
            self.ban_l3_agent(router_name="router01", _ip=ip)
        last_banned_node = self.ban_l3_agent(router_name="router01",
                                             _ip=ip,
                                             wait_for_migrate=False)

        # clear last banned l3 agent
        self.clear_l3_agent(_ip=ip,
                            router_name="router01",
                            node=last_banned_node,
                            wait_for_alive=True)

        # create another server on net01
        net01 = self.os_conn.nova.networks.find(label="net01")
        self.os_conn.create_server(
            name='server03',
            availability_zone='{}:{}'.format(self.zone.zoneName,
                                             self.hosts[0]),
            key_name=self.instance_keypair.name,
            nics=[{'net-id': net01.id}],
            security_groups=[self.security_group.id])

        # check pings
        network_checks.check_vm_connectivity(self.env, self.os_conn)

    def _prepare_openstack(self):
        """Prepare OpenStack for scenarios run

        Steps:
            1. Create network1
            2. Create router1 and connect it with network1 and external net
            3. Boot vm1 in network1 and associate floating ip
            4. Boot vm2 in network2
            5. Add rules for ping
            6. ping 8.8.8.8 from vm2
            7. ping vm1 from vm2 and vm1 from vm2
        """

        # init variables
        exist_networks = self.os_conn.list_networks()['networks']
        ext_network = [x for x in exist_networks
                       if x.get('router:external')][0]
        self.zone = self.os_conn.nova.availability_zones.find(zoneName="nova")
        self.hosts = self.zone.hosts.keys()
        self.instance_keypair = self.os_conn.create_key(key_name='instancekey')
        self.setup_rules_for_default_sec_group()

        # create router
        self.router = self.os_conn.create_router(name="router01")['router']
        self.os_conn.router_gateway_add(router_id=self.router['id'],
                                        network_id=ext_network['id'])
        logger.info('router {} was created'.format(self.router['id']))

        # create one network by amount of the compute hosts
        self.net_id = self.os_conn.add_net(self.router['id'])

        # create two instances in that network
        # each instance is on the own compute
        for i, hostname in enumerate(self.hosts, 1):
            self.os_conn.create_server(
                name='server%02d' % i,
                availability_zone='{}:{}'.format(self.zone.zoneName, hostname),
                key_name=self.instance_keypair.name,
                nics=[{'net-id': self.net_id}])

        # check pings
        network_checks.check_vm_connectivity(self.env, self.os_conn)

        # make a list of all ovs agent ids
        self.ovs_agent_ids = [
            agt['id'] for agt in self.os_conn.neutron.list_agents(
                binary='neutron-openvswitch-agent')['agents']]
        # make a list of ovs agents that resides only on controllers
        controllers = [node.data['fqdn']
                       for node in self.env.get_nodes_by_role('controller')]
        ovs_agts = self.os_conn.neutron.list_agents(
            binary='neutron-openvswitch-agent')['agents']
        self.ovs_conroller_agents = [agt['id'] for agt in ovs_agts
                                     if agt['host'] in controllers]

def test_evacuate(devops_env, env, os_conn, instances, keypair):
    """Evacuate instances from failed compute node

    Scenario:
        1. Create net01, net01__subnet:
            neutron net-create net01
            neutron subnet-create net01 192.168.1.0/24 --enable-dhcp \
            --name net01__subnet
        2. Boot instances vm1 and vm2 in net01 on a single compute node:
        3. Destroy a compute node where instances are scheduled
        4. Evacuate instances vm1 and vm2:
            nova evacuate vm1 && nova evacuate vm2
        5. Check that they are rescheduled onto another compute node and
            are in ACTIVE state:
        6. Check that pings between vm1 and vm2 are successful
    """
    compute_host = getattr(instances[0], 'OS-EXT-SRV-ATTR:hypervisor_hostname')
    compute_node = env.find_node_by_fqdn(compute_host)
    devops_node = devops_env.get_node_by_mac(compute_node.data['mac'])
    devops_node.destroy()

    def is_hypervisor_down():
        hypervisor = os_conn.nova.hypervisors.find(
            hypervisor_hostname=compute_host)
        return hypervisor.state == 'down'

    common.wait(
        is_hypervisor_down,
        timeout_seconds=5 * 60,
        waiting_for='hypervisor {0} to be in down state'.format(compute_host))

    for instance in instances:
        os_conn.nova.servers.evacuate(instance)

    def is_instances_migrate():
        for instance in os_conn.nova.servers.list():
            if instance not in instances:
                continue
            if instance.status == 'ERROR':
                raise Exception('Instance {0.name} is in ERROR status\n'
                                '{0.fault[message]}\n'
                                '{0.fault[details]}'.format(instance))
            if not os_conn.server_status_is(instance, 'ACTIVE'):
                return False
            if getattr(instance,
                       'OS-EXT-SRV-ATTR:hypervisor_hostname') == compute_host:
                return False
        return True

    common.wait(is_instances_migrate,
                timeout_seconds=5 * 60,
                waiting_for='instances to migrate to another compute')

    network_checks.check_vm_connectivity(env, os_conn, vm_keypair=keypair)

    def test_ban_l3_agents_many_times(self):
        """Ban l3-agent many times and check health of l3-agent

        Scenario:
            1. Revert snapshot with neutron cluster
            2. Create network1, network2
            3. Create router1 and connect it with network1, network2 and
               external net
            4. Boot vm1 in network1 and associate floating ip
            5. Boot vm2 in network2
            6. Add rules for ping
            7. ping 8.8.8.8, vm1 (both ip) and vm2 (fixed ip) from each other
            8. Ban l3-agent on what router1 is
            9. Wait for route rescheduling
            10. Repeat steps 7-8
            11. Ban l3-agent on what router1 is
            12. Wait for L3 agent dies
            13. Clear last banned L3 agent
            14. Wait for L3 agent alive
            15. Repeat steps 11-14 40 times
            16. Boot one more VM (VM3) in network1
            17. Boot vm3 in network1
            18. ping 8.8.8.8, vm1 (both ip), vm2 (fixed ip) and vm3 (fixed ip)
                from each other vm

        Duration 30m

        """
        net_id = self.os_conn.neutron.list_networks(
            name="net01")['networks'][0]['id']
        devops_node = self.get_node_with_dhcp(net_id)
        ip = devops_node.data['ip']

        # ban 2 l3 agents
        for _ in range(2):
            self.ban_l3_agent(router_name="router01", _ip=ip)

        for i in range(40):
            # ban l3 agent
            logger.info('Ban/clear L3 agent. Iteration #{} from 40'.format(i))
            last_banned_node = self.ban_l3_agent(router_name="router01",
                                                 _ip=ip,
                                                 wait_for_migrate=False,
                                                 wait_for_die=True)
            # clear last banned l3 agent
            self.clear_l3_agent(_ip=ip,
                                router_name="router01",
                                node=last_banned_node,
                                wait_for_alive=True)

        # check pings
        network_checks.check_vm_connectivity(self.env, self.os_conn,
                                             timeout=10 * 60)

    def test_vms_connectivity_after_evacuation(self, env, os_conn, networks,
                                               flavors, aggregate,
                                               security_group, devops_env):
        """This test checks vms connectivity for vms with cpu pinning with 1
        NUMA after evacuation

        Steps:
            1. Create net1 with subnet, net2 with subnet and router1 with
            interfaces to both nets
            2. Boot vm0 with cpu flavor on host0 and net0
            3. Boot vm1 with old flavor on host1 and net1
            4. Check vms connectivity
            5. Kill compute0 and evacuate vm0 to compute1 with
            --on-shared-storage parameter
            6. Check vms connectivity
            7. Check numa nodes for vm0
            8. Make compute0 alive
            9. Check that resources for vm0 were deleted from compute0
        """
        cpus = get_cpu_distribition_per_numa_node(env)
        hosts = aggregate.hosts
        vms = []

        for i in range(2):
            vm = os_conn.create_server(
                name='vm{}'.format(i), flavor=flavors[i].id,
                nics=[{'net-id': networks[i]}],
                availability_zone='nova:{}'.format(hosts[i]),
                security_groups=[security_group.id])
            vms.append(vm)
        network_checks.check_vm_connectivity(env, os_conn)
        self.check_cpu_for_vm(os_conn, vms[0], 1, cpus[hosts[0]])

        self.compute_change_state(os_conn, devops_env, hosts[0], state='down')
        vm0_new = self.evacuate(os_conn, devops_env, vms[0])
        vm0_new.get()
        new_host = getattr(vm0_new, "OS-EXT-SRV-ATTR:host")
        assert new_host in hosts, "Unexpected host after evacuation"
        assert new_host != hosts[0], "Host didn't change after evacuation"
        os_conn.wait_servers_ssh_ready(vms)
        network_checks.check_vm_connectivity(env, os_conn)
        self.check_cpu_for_vm(os_conn, vm0_new, 1, cpus[new_host])

        self.compute_change_state(os_conn, devops_env, hosts[0], state='up')
        old_hv = os_conn.nova.hypervisors.find(hypervisor_hostname=hosts[0])
        assert old_hv.running_vms == 0, (
            "Old hypervisor {0} shouldn't have running vms").format(hosts[0])

        instance_name = getattr(vm0_new, "OS-EXT-SRV-ATTR:instance_name")
        assert instance_name in self.get_instances(os_conn, new_host), (
            "Instance should be in the list of instances on the new host")
        assert instance_name not in self.get_instances(os_conn, hosts[0]), (
            "Instance shouldn't be in the list of instances on the old host")

    def test_hp_distribution_1g_2m_for_vms(self, env, os_conn,
                                           computes_with_mixed_hp, networks,
                                           flavors, security_group):
        """This test checks huge pages 1Gb and 2Mb distribution for vms
            Steps:
            1. Create net1 with subnet, net2 with subnet and  router1 with
            interfaces to both nets
            2. Create vm1 in net1 on compute1 with 1Gb flavor
            3. Create vm2 in net2 on compute2 with 2Mb flavor
            4. Create vm3 in net2 on compute1 with 2Mb flavor
            5. Check instances configuration (about huge pages)
            6. Check quantity of HP on computes
            7. Check pings from all vms to all vms by all ips
        """
        small_nfv_flavor, medium_nfv_flavor = flavors[0], flavors[1]
        count_to_allocate_2mb = small_nfv_flavor.ram * 1024 / page_2mb
        count_to_allocate_1gb = medium_nfv_flavor.ram * 1024 / page_1gb

        initial_conf = computes_configuration(env)

        hosts = computes_with_mixed_hp
        vms_params = [
            (hosts[0], networks[0], medium_nfv_flavor, page_1gb),
            (hosts[1], networks[1], small_nfv_flavor, page_2mb),
            (hosts[0], networks[1], small_nfv_flavor, page_2mb), ]

        vms = {}

        for i, (host, network, flavor, size) in enumerate(vms_params):
            vm = os_conn.create_server(
                name='vm{}'.format(i), flavor=flavor.id,
                nics=[{'net-id': network}],
                availability_zone='nova:{}'.format(host),
                security_groups=[security_group.id])
            vms.update({vm: size})

        for vm, exp_size in vms.items():
            assert self.get_instance_page_size(os_conn, vm) == exp_size

        vms_distribution = [(hosts[0], 1, 1), (hosts[1], 0, 1), ]
        final_conf = computes_configuration(env)
        for (host, nr_1gb, nr_2mb) in vms_distribution:
            exp_free_1g = (initial_conf[host][page_1gb]['total'] -
                           nr_1gb * count_to_allocate_1gb)
            exp_free_2m = (initial_conf[host][page_2mb]['total'] -
                           nr_2mb * count_to_allocate_2mb)
            assert exp_free_1g == final_conf[host][page_1gb]['free']
            assert exp_free_2m == final_conf[host][page_2mb]['free']

        os_conn.wait_servers_ssh_ready(vms.keys())
        network_checks.check_vm_connectivity(env, os_conn)

    def test_kill_active_l3_agt(self):
        """[Neutron VLAN and VXLAN] Kill l3-agent process

            8. get node with l3 agent where is the router1:
                neutron l3-agent-hosting-router router1
            9. on this node find l3-agent process:
                ps aux | grep l3-agent
            10. Kill it:
                kill -9 <pid>
            11. Wait some time until all agents are up
                neutron-agent-list
            12. Boot vm3 in network1
            13. ping 8.8.8.8 from vm3
            14. ping between vm1 and vm3 by internal ip
            15. ping between vm1 and vm2 by floating ip
        """

        self._prepare_openstack()
        # Get current L3 agent on router01
        router_agt = self.os_conn.neutron.list_l3_agent_hosting_routers(
                self.router['id'])['agents'][0]

        # Find the current controller ip with the router01
        controller_ip = ''
        for node in self.env.get_all_nodes():
            if node.data['fqdn'] == router_agt['host']:
                controller_ip = node.data['ip']
                break

        # If ip is empty than no controller with the router was found
        assert controller_ip, "No controller with the router was found"

        with self.env.get_ssh_to_node(controller_ip) as remote:
            cmd = "ps -aux | grep [n]eutron-l3-agent | awk '{print $2}'"
            result = remote.execute(cmd)
            pid = result['stdout'][0]
            logger.info('Got l3 agent pid  {}'.format(pid))
            logger.info('Now going to kill it on the controller {}'.format(
                        controller_ip))
            result = remote.execute('kill -9 {}'.format(pid))
            assert result['exit_code'] == 0, "kill failed {}".format(result)

        self.os_conn.wait_agents_alive(self.l3_agent_ids)

        # Create one more server and check connectivity
        self.os_conn.add_server(self.networks[0],
                                self.instance_keypair.name,
                                self.hosts[0],
                                self.security_group.id)
        network_checks.check_vm_connectivity(self.env, self.os_conn)

    def test_allocation_huge_pages_2m_for_vms(self, env, os_conn, networks,
                                              nfv_flavor, security_group,
                                              aggregate):
        """This test checks allocation 2M HugePages for instances
            Steps:
            1. Create net1 with subnet, net2 with subnet and  router1 with
            interfaces to both nets
            2. Launch vm1 and vm2 using net1 on the first compute
            3. Launch vm3 using net1 on the second compute
            4. Launch vm4 using net2 on the second compute
            5. Check instances configuration (about huge pages)
            6. Check quantity of HP on computes
            7. Associate floating to vm1
            8. Check pings from all vms to all vms by all ips
        """
        free_pages = {1: 768, 3: 256}
        hosts = aggregate.hosts
        vms = []

        for i in range(2):
            vm_name = 'vm{}'.format(i)
            vm = os_conn.create_server(
                name=vm_name, flavor=nfv_flavor[0].id,
                nics=[{'net-id': networks[0]}],
                availability_zone='nova:{}'.format(hosts[0]),
                security_groups=[security_group.id])
            vms.append(vm)
        vm2 = os_conn.create_server(
            name='vm2', flavor=nfv_flavor[0].id,
            nics=[{'net-id': networks[1]}],
            availability_zone='nova:{}'.format(hosts[0]),
            security_groups=[security_group.id])
        vm3 = os_conn.create_server(
            name='vm3', flavor=nfv_flavor[0].id,
            nics=[{'net-id': networks[1]}],
            availability_zone='nova:{}'.format(hosts[1]),
            security_groups=[security_group.id])
        vms.extend([vm2, vm3])

        for vm in vms:
            self.check_instance_page_size(os_conn, vm, size=2048)

        self.check_pages(os_conn, hosts[0], total_pages=1024,
                         free_pages=free_pages[3])
        self.check_pages(os_conn, hosts[1], total_pages=1024,
                         free_pages=free_pages[1])

        os_conn.assign_floating_ip(vms[0])
        network_checks.check_vm_connectivity(env, os_conn)

    def test_allocation_huge_pages_2m_for_vms(self, env, os_conn, networks,
                                              computes_with_hp_2mb,
                                              flavors, security_group):
        """This test checks allocation 2M HugePages for instances
            Steps:
            1. Create net1 with subnet, net2 with subnet and  router1 with
            interfaces to both nets
            2. Launch vm1 and vm2 using net1 on the first compute
            3. Launch vm3 using net1 on the second compute
            4. Launch vm4 using net2 on the second compute
            5. Check instances configuration (about huge pages)
            6. Check quantity of HP on computes
            7. Associate floating to vm1
            8. Check pings from all vms to all vms by all ips
        """
        small_nfv_flavor = flavors[0]
        count_to_allocate_2mb = small_nfv_flavor.ram * 1024 / page_2mb
        initial_conf = computes_configuration(env)
        hosts = computes_with_hp_2mb

        vms = []
        vms_params = [
            (hosts[0], networks[0]),
            (hosts[0], networks[0]),
            (hosts[0], networks[1]),
            (hosts[1], networks[1]),
        ]
        for i, (host, network) in enumerate(vms_params):
            vm = os_conn.create_server(
                name='vm{}'.format(i), flavor=small_nfv_flavor.id,
                nics=[{'net-id': network}],
                availability_zone='nova:{}'.format(host),
                security_groups=[security_group.id])
            vms.append(vm)

        for vm in vms:
            assert self.get_instance_page_size(os_conn, vm) == page_2mb

        vms_distribution = [(hosts[0], 3), (hosts[1], 1), ]
        final_conf = computes_configuration(env)
        for (host, nr_2mb) in vms_distribution:
            exp_free_2m = (initial_conf[host][page_2mb]['total'] -
                           nr_2mb * count_to_allocate_2mb)
            assert exp_free_2m == final_conf[host][page_2mb]['free']

        os_conn.assign_floating_ip(vms[0])
        os_conn.wait_servers_ssh_ready(vms)
        network_checks.check_vm_connectivity(env, os_conn)

    def test_cold_migration_for_huge_pages_2m(
            self, env, os_conn, networks, nfv_flavor, security_group,
            aggregate):
        """This test checks that cold migration executed successfully for
            instances created on computes with huge pages 2M
            Steps:
            1. Create net1 with subnet, net2 with subnet and  router1 with
            interfaces to both nets
            2. Launch instance vm1 in net1 with m1.small.hpgs
            3. Check that vm1 is created on compute with huge pages
            4. Launch instance vm2 in net2 with m1.small.hpgs
            5. Check that vm2 is created on compute with huge pages
            6. Check vms connectivity
            7. Cold migrate vm1 and check that vm moved to other compute with
            huge pages
            8. Check vms connectivity
        """
        free_pages = {0: 1024, 1: 768, 2: 512}
        hosts = aggregate.hosts
        vms = []
        vm_hosts = []
        for i in range(2):
            vm = os_conn.create_server(
                name='vm{}'.format(i), flavor=nfv_flavor[0].id,
                security_groups=[security_group.id],
                nics=[{'net-id': networks[i]}])
            vms.append(vm)
        for vm in vms:
            host = getattr(vm, "OS-EXT-SRV-ATTR:host")
            assert host in hosts
            vm_hosts.append(host)
        for host in hosts:
            self.check_pages(os_conn, host, total_pages=1024,
                             free_pages=free_pages[vm_hosts.count(host)])
        for vm in vms:
            self.check_instance_page_size(os_conn, vm, size=2048)
        network_checks.check_vm_connectivity(env, os_conn)

        vm_0_new = self.migrate(os_conn, vms[0])
        vm_host_0_new = getattr(vm_0_new, "OS-EXT-SRV-ATTR:host")
        assert vm_host_0_new in hosts
        assert vm_host_0_new != vm_hosts.pop(0)
        vm_hosts.append(vm_host_0_new)
        for host in hosts:
            self.check_pages(os_conn, host, total_pages=1024,
                             free_pages=free_pages[vm_hosts.count(host)])
        self.check_instance_page_size(os_conn, vm_0_new, size=2048)
        network_checks.check_vm_connectivity(env, os_conn)

    def test_restart_primary_controller_with_l3_agt(self):
        """[Neutron VLAN and VXLAN] Reset primary controller and check l3-agent

        Steps:
            1. Check on what agents is router1:
                neutron l3-agent-list-hosting-router router1
            2. If there isn't agent on the primary controller:
                neutron l3-agent-router-remove non_on_primary_agent_id router1
                neutron l3-agent-router-add on_primary_agent_id router1
            3. Restart primary controller
            4. Wait some time until all agents are up
                neutron-agent-list
            5. Check that all routers reschedule from primary controller:
                neutron router-list-on-l3-agent <on_primary_agent_id>
            6. Boot vm3 in network1
            7. ping 8.8.8.8 from vm3
            8. ping between vm1 and vm3 by internal ip
            9. ping between vm1 and vm2 by floating ip
        """

        self._prepare_openstack()
        # Get current L3 agent on router01
        router_agt = self.os_conn.neutron.list_l3_agent_hosting_routers(
                        self.router['id'])['agents'][0]
        # Check if the agent is not on the primary controller
        # Reschedule if needed
        if router_agt['host'] != self.primary_host:
            self.os_conn.reschedule_router_to_primary_host(self.router['id'],
                                                           self.primary_host)
            router_agt = self.os_conn.neutron.list_l3_agent_hosting_routers(
                            self.router['id'])['agents'][0]

        # virsh destroy of the primary controller
        self.env.warm_restart_nodes([self.primary_node])

        # Check that the all l3 are alive
        self.os_conn.wait_agents_alive(self.l3_agent_ids)

        # Check that there are no routers on the first agent
        self.check_no_routers_on_l3_agent(router_agt['id'])

        # Create one more server and check connectivity
        self.os_conn.add_server(self.networks[0],
                                self.instance_keypair.name,
                                self.hosts[0],
                                self.security_group.id)
        network_checks.check_vm_connectivity(self.env, self.os_conn)

    def prepare_openstack(self, init):
        """Prepare OpenStack for scenarios run

        Steps:
            1. Create network1, network2
            2. Create router1 and connect it with network1, network2 and
                external net
            3. Boot vm1 in network1 and associate floating ip
            4. Boot vm2 in network2
            5. Add rules for ping
            6. Ping 8.8.8.8, vm1 (both ip) and vm2 (fixed ip) from each other
        """
        # init variables
        self.zone = self.os_conn.nova.availability_zones.find(zoneName="nova")
        self.security_group = self.os_conn.create_sec_group_for_ssh()
        self.hosts = self.zone.hosts.keys()[:2]
        self.instance_keypair = self.os_conn.create_key(key_name='instancekey')

        # create router
        router = self.os_conn.create_router(name="router01")
        self.os_conn.router_gateway_add(
            router_id=router['router']['id'],
            network_id=self.os_conn.ext_network['id'])

        # create 2 networks and 2 instances
        for i, hostname in enumerate(self.hosts, 1):
            network = self.os_conn.create_network(name='net%02d' % i)
            subnet = self.os_conn.create_subnet(
                network_id=network['network']['id'],
                name='net%02d__subnet' % i,
                cidr="192.168.%d.0/24" % i)
            self.os_conn.router_interface_add(
                router_id=router['router']['id'],
                subnet_id=subnet['subnet']['id'])
            self.os_conn.create_server(
                name='server%02d' % i,
                availability_zone='{}:{}'.format(self.zone.zoneName, hostname),
                key_name=self.instance_keypair.name,
                nics=[{'net-id': network['network']['id']}],
                s