def _install_zstack_nodes_ha(self):
     for node in self.nodes:
         cmd = "/root/scripts/network-setting -b %s %s %s %s %s" % (node.ip_, node.netmask_, node.gateway_, node.nic_, node.bridge_)
         ssh.execute(cmd, node.ip_, node.username_, node.password_)
         ssh.scp_file(self.zstack_pkg, "/root/zstack-installer.bin", node.ip_, node.username_, node.password_)
         cmd = "bash %s -o -i -I %s" % ("/root/zstack-installer.bin", node.bridge_)
         ssh.execute(cmd, node.ip_, node.username_, node.password_)

 def _install_zstack_nodes_ha(self):
     for node in self.nodes:
         cmd = "/usr/local/bin/zs-network-setting -b %s %s %s %s" % (node.ip_, node.netmask_, node.gateway_, node.nic_)
         ssh.execute(cmd, node.ip_, node.username_, node.password_)
         ssh.scp_file(self.zstack_pkg, "/root/zstack-installer.bin", node.ip_, node.username_, node.password_)
         cmd = "deactivate; which python; bash %s -o -i -I %s" % ("/root/zstack-installer.bin", node.bridge_)
         ssh.execute(cmd, node.ip_, node.username_, node.password_)

def test():

    image_name = os.environ.get('ipv6ImageName')
    ipv4_net_uuid = test_lib.lib_get_l3_by_name(os.environ.get('l3PublicNetworkName')).uuid
    ipv6_net_uuid = test_lib.lib_get_l3_by_name(os.environ.get('l3PublicNetworkName1')).uuid
    print "ipv4_net_uuid is : %s , ipv6_net_uuid is : %s" %(ipv4_net_uuid, ipv6_net_uuid)
    vm1 = test_stub.create_vm(l3_name = os.environ.get('l3PublicNetworkName'), vm_name = 'vm_1 IPv6 2 stack test', system_tags = ["dualStackNic::%s::%s" %(ipv4_net_uuid, ipv6_net_uuid)], image_name = image_name)
    vm2 = test_stub.create_vm(l3_name = os.environ.get('l3PublicNetworkName'), vm_name = 'vm_2 IPv6 2 stack test', system_tags = ["dualStackNic::%s::%s" %(ipv4_net_uuid, ipv6_net_uuid)], image_name = image_name)
    time.sleep(120) #waiting for vm bootup
    ipv4 = None
    ipv6 = None
    vms = res_ops.query_resource(res_ops.VM_INSTANCE)
    vm1_nic1 = vms[1].vmNics[0].usedIps[0].ip
    vm1_nic2 = vms[1].vmNics[0].usedIps[1].ip
    vm2_nic1 = vms[0].vmNics[0].usedIps[0].ip
    vm2_nic2 = vms[0].vmNics[0].usedIps[1].ip

    for ip in [vm1_nic1, vm1_nic2]:
        if "172.20" in ip:
            ipv4 = ip
    for ip in [vm2_nic1, vm2_nic2]:
        if "1000:2000" in ip:
            ipv6 = ip
    for ip in [vm2_nic1, vm2_nic2]:
        if "172" in ip:
            vm2_ipv4 = ip
    print "vm1_nic1 : %s, vm1_nic2: %s, vm2_nic1 :%s,vm2_nic2 :%s, ipv4 :%s, ipv6 :%s." %(vm1_nic1, vm1_nic2, vm2_nic1, vm2_nic2, ipv4, ipv6)
    cmd = "ping6 -c 4 %s" %(ipv6)
    (retcode, output, erroutput) = ssh.execute(cmd, ipv4, "root", "password", True, 22)
    cmd1 = "ping -c 4 %s" %(vm2_ipv4)
    (retcode1, output1, erroutput1) = ssh.execute(cmd1, ipv4, "root", "password", True, 22)
    print "retcode is: %s; output is : %s.; erroutput is: %s" %(retcode, output , erroutput)
    print "retcode1 is: %s; output1 is : %s.; erroutput1 is: %s" %(retcode1, output1 , erroutput1)
    if retcode != 0 and retcode1 != 0:
        test_util.test_fail('Test Create IPv6 VM Failed.')

def cleanup_sftp_backup_storage():
    backup_obj = test_lib.deploy_config.backupStorages
    sftp_backupstorages = backup_obj.get_child_node_as_list("sftpBackupStorage")
    for storage in sftp_backupstorages:
        cmd = "rm -rf %s/rootVolumeTemplates/*" % storage.url_
        ssh.execute(cmd, storage.hostname_, storage.username_, storage.password_)
        cmd = "rm -rf %s/dataVolumeTemplates/*" % storage.url_
        ssh.execute(cmd, storage.hostname_, storage.username_, storage.password_)

    def _install_zstack_ha(self):
        node1 = self.nodes[0]
	host_info = ""
        host_id = 1
        for node in self.nodes:
            host_info += "--host%s-info %s:%[email protected]%s " % (host_id, node.username_, node.password_, node.ip_)
	    host_id += 1
        cmd = "zstack-ctl install_ha %s --vip %s" % (host_info, self.zstack_ha_vip)
        ssh.execute(cmd, node1.ip_, node1.username_, node1.password_)

 def _change_node_ip(self):
     for node in self.nodes:
         cmd = 'zstack-ctl change_ip --ip=%s' % (node.ip_)
         if not linux.is_ip_existing(node.ip_):
             ssh.execute(cmd, node.ip_, node.username_, node.password_)
         else:
             thread = threading.Thread(target=shell_cmd_thread, args=(cmd,))
             thread.start()
             self._wait_for_thread_completion('change management node ip', 60)

 def _set_extra_node_config(self):
     node1 = self.nodes[0]
     for node in self.nodes:
         cmd = 'zstack-ctl configure --duplicate-to-remote=%s; zstack-ctl configure --host=%s management.server.ip=%s' % \
                 (node.ip_, node.ip_, node.ip_)
         if not linux.is_ip_existing(node.ip_):
             ssh.execute(cmd, node1.ip_, node.username_, node.password_)
         else:
             thread = threading.Thread(target=shell_cmd_thread, args=(cmd,))
             thread.start()
             self._wait_for_thread_completion('set extra management node config', 60)

def check_chrony_status(node_ip, port):
    test_util.test_dsc("Check all hosts chrony status.")
    node_ip = node_ip
    port = port
    cmd1 = "chronyc tracking"
    cmd2 = "chronyc sources"
    (retcode1, output, erroutput) = ssh.execute(cmd1, node_ip, 'root', 'password', True, port)
    (retcode2, output, erroutput) = ssh.execute(cmd2, node_ip, 'root', 'password', True, port)
    if retcode1 == 0 and retcode2 == 0:
        test_util.test_logger('@@@DEBUG-> check chrony "chronyc tracking", "chronyc sources" pass')
    else:
        test_util.test_fail('@@@DEBUG-> check chrony "chronyc tracking", "chronyc sources" failed.')

def test():

    img_option = test_util.ImageOption()
    ipv6_image_url = os.environ.get('ipv6ImageUrl')
    image_name = os.environ.get('ipv6ImageName')
    img_option.set_name(image_name)
    bs_uuid = res_ops.query_resource_fields(res_ops.BACKUP_STORAGE, [], None)[0].uuid
    img_option.set_backup_storage_uuid_list([bs_uuid])
    img_option.set_format('qcow2')
    img_option.set_url(ipv6_image_url)
    image_inv = img_ops.add_root_volume_template(img_option)
    image = test_image.ZstackTestImage()
    image.set_image(image_inv)
    image.set_creation_option(img_option)
    test_obj_dict.add_image(image)
    vm1 = test_stub.create_vm(l3_name = "%s,%s" %(os.environ.get('l3PublicNetworkName1'), os.environ.get('l3PublicNetworkName')), vm_name = 'IPv6 2 stack test ipv4 and ipv6', image_name = image_name)
    vm2 = test_stub.create_vm(l3_name = os.environ.get('l3PublicNetworkName1'), vm_name = 'IPv6 2 stack test ipv6', image_name = image_name)
    time.sleep(90) #waiting for vm bootup
    vm1_nic1 = vm1.get_vm().vmNics[0].ip
    vm1_nic2 = vm1.get_vm().vmNics[1].ip
    vm2_nic1 = vm2.get_vm().vmNics[0].ip
    for ip in (vm1_nic1, vm1_nic2):
        if "." in ip:
            ipv4 = ip
    print "vm1_nic1 : %s, vm1_nic2: %s, vm2_nic1 :%s, ipv4 :%s." %(vm1_nic1, vm1_nic2, vm2_nic1,ipv4)
    cmd = "ping6 -c 4 %s" %(vm2_nic1)
    (retcode, output, erroutput) = ssh.execute(cmd, ipv4, "root", "password", True, 22)
    print "retcode is: %s; output is : %s.; erroutput is: %s" %(retcode, output , erroutput)
    if retcode != 0:
        test_util.test_fail('Test Create IPv6 VM Failed.')

def test():

    image_name = os.environ.get('ipv6ImageName')
    vm1 = test_stub.create_vm(l3_name = "%s,%s" %(os.environ.get('l3PublicNetworkName1'), os.environ.get('l3PublicNetworkName')), vm_name = 'IPv6 2 stack test ipv4 and ipv6', image_name = image_name)
    vm2 = test_stub.create_vm(l3_name = os.environ.get('l3PrivateNetworkName'), vm_name = 'IPv6 private vm', image_name = image_name)
    time.sleep(90) #waiting for vm bootup
    vm1_nic1 = vm1.get_vm().vmNics[0].ip
    vm1_nic2 = vm1.get_vm().vmNics[1].ip
    vm2_nic1 = vm2.get_vm().vmNics[0].ip
    for ip in (vm1_nic1, vm1_nic2):
        if "." in ip:
            ipv4 = ip
    print "vm1_nic1 : %s, vm1_nic2: %s, vm2_nic1 :%s, ipv4 :%s." %(vm1_nic1, vm1_nic2, vm2_nic1,ipv4)
#######################################################################
    pub_l3_name = os.environ.get('l3PublicNetworkName1')
    pub_l3_uuid = test_lib.lib_get_l3_by_name(pub_l3_name).uuid

    vm2_nic_uuid = vm2.get_vm().vmNics[0].uuid
    vip = test_stub.create_vip('create_eip_test', pub_l3_uuid)
    test_obj_dict.add_vip(vip)
    eip = test_stub.create_eip('create eip test', vip_uuid=vip.get_vip().uuid, vnic_uuid=vm2_nic_uuid, vm_obj=vm2)

    vip.attach_eip(eip)
    eip_ip = res_ops.query_resource(res_ops.EIP)[0].vipIp

    print "eip_ip : %s, vm1_nic2: %s, vm2_nic1 :%s, ipv4 :%s." %(eip_ip, vm1_nic2, vm2_nic1,ipv4)
    cmd = "ping6 -c 4 %s" %(eip_ip)
    (retcode, output, erroutput) = ssh.execute(cmd, ipv4, "root", "password", True, 22)
    print "ping6 EIP retcode is: %s; output is : %s.; erroutput is: %s" %(retcode, output , erroutput)
    if retcode != 0:
        test_util.test_fail('Test Create IPv6 VM Failed.')

def set_quick_ha_properties():
    host_username = os.environ.get('hostUsername')
    host_password = os.environ.get('hostPassword')
    mn_ip = res_ops.query_resource(res_ops.MANAGEMENT_NODE)[0].hostName
    cmd = 'zstack-ctl status|grep properties|cut -d: -f2'
    ret, properties_file, stderr = ssh.execute(cmd, mn_ip, host_username, host_password, False, 22)

    cmd = "zstack-ctl stop"
    if test_lib.lib_execute_ssh_cmd(mn_ip, host_username, host_password, cmd,  timeout = 120) == False:
        test_util.test_fail("CMD:%s execute failed on %s" %(cmd, mn_ip))

    cmd = "sed 's/RESTFacade.connectTimeout.*$//g' -i " + properties_file
    if test_lib.lib_execute_ssh_cmd(mn_ip, host_username, host_password, cmd,  timeout = 10) == False:
        test_util.test_fail("CMD:%s execute failed on %s" %(cmd, mn_ip))

    cmd = "sed 's/RESTFacade.readTimeout.*$//g' -i " + properties_file
    if test_lib.lib_execute_ssh_cmd(mn_ip, host_username, host_password, cmd,  timeout = 10) == False:
        test_util.test_fail("CMD:%s execute failed on %s" %(cmd, mn_ip))

    cmd = "echo \"RESTFacade.readTimeout = 10000\" >>" + properties_file
    if test_lib.lib_execute_ssh_cmd(mn_ip, host_username, host_password, cmd,  timeout = 10) == False:
        test_util.test_fail("CMD:%s execute failed on %s" %(cmd, mn_ip))

    cmd = "echo \"RESTFacade.connectTimeout = 5000\" >>" + properties_file
    if test_lib.lib_execute_ssh_cmd(mn_ip, host_username, host_password, cmd,  timeout = 10) == False:
        test_util.test_fail("CMD:%s execute failed on %s" %(cmd, mn_ip))

    cmd = "nohup zstack-ctl start &"
    if test_lib.lib_execute_ssh_cmd(mn_ip, host_username, host_password, cmd,  timeout = 180) == False:
        test_util.test_fail("CMD:%s execute failed on %s" %(cmd, mn_ip))

    time.sleep(60)

def ensure_storage_online(vm):
    ret, output, stderr = ssh.execute("o2cb.init status", vm.get_vm().vmNics[0].ip, "root", "password", False, 22)
    if ret != 0:
        test_util.test_fail( cmd + " failed")

    if "online" not in output.lower():
        test_util.test_fail("not found storage online")

    def _change_node_ip(self):
        for node in self.nodes:
            cmd = 'zstack-ctl change_ip --ip=%s --mysql_root_password=%s' % (node.ip_, os.environ.get('DBAdminPassword'))
            if not linux.is_ip_existing(node.ip_):
                ssh.execute(cmd, node.ip_, node.username_, node.password_)
            else:
                thread = threading.Thread(target=shell_cmd_thread, args=(cmd,))
                thread.start()
                self._wait_for_thread_completion('change management node ip', 120)

            cmd = 'zstack-ctl configure InfluxDB.startupTimeout=1200'
            if not linux.is_ip_existing(node.ip_):
                ssh.execute(cmd, node.ip_, node.username_, node.password_)
            else:
                thread = threading.Thread(target=shell_cmd_thread, args=(cmd,))
                thread.start()
                self._wait_for_thread_completion('InfluxDB startup time setting to 10 mins', 120)

 def copy_data(self, vm=None):
     vm = vm if vm else self.vm
     vm_ip = vm.get_vm().vmNics[0].ip
     test_lib.lib_wait_target_up(vm_ip, '22', timeout=600)
     cmd = "find /home -iname 'zstack-all-in-one.*'"
     file_path = commands.getoutput(cmd).split('\n')[0]
     file_name = os.path.basename(file_path)
     dst_file = os.path.join('/mnt', file_name)
     src_file_md5 = commands.getoutput('md5sum %s' % file_path).split(' ')[0]
     ssh.scp_file(file_path, dst_file, vm_ip, 'root', 'password')
     (_, dst_md5, _)= ssh.execute('sync; sync; sleep 60; md5sum %s' % dst_file, vm_ip, 'root', 'password')
     dst_file_md5 = dst_md5.split(' ')[0]
     test_util.test_dsc('src_file_md5: [%s], dst_file_md5: [%s]' % (src_file_md5, dst_file_md5))
     assert dst_file_md5 == src_file_md5, 'dst_file_md5 [%s] and src_file_md5 [%s] is not match, stop test' % (dst_file_md5, src_file_md5)
     extract_cmd = 'tar xvf /mnt/zstack-all-in-one.* -C /mnt > /dev/null 2>&1'
     ssh.execute(extract_cmd, vm_ip, 'root', 'password')
     return self

def check_qemu_ga_is_alive(vm):
    global ga_process_not_alive_num
    vm.check()
    cmd = "ps -aux|grep ga|grep qemu"
    ret, output, stderr = ssh.execute(cmd, vm.get_vm().vmNics[0].ip, "root", "password", False, 22)
    if ret != 0:
        test_util.test_logger("qemu-ga is not alived when exception triggered: ip:%s; cmd:%s; user:%s; password:%s; stdout:%s, stderr:%s" %(vm.get_vm().vmNics[0].ip, cmd, "root", "password", output, stderr))
        ga_process_not_alive_num = ga_process_not_alive_num + 1
    return ret

def create_user_in_vm(vm, username, password):
    """
    create non-root user with password setting
    """
    global original_root_password
    test_util.test_logger("create_user_in_vm: %s:%s" %(username, password))

    vm_ip = vm.vmNics[0].ip

    cmd = "adduser %s" % (username)
    ret, output, stderr = ssh.execute(cmd, vm_ip, "root", original_root_password, False, 22)
    if ret != 0:
        test_util.test_fail("User created failure, cmd[%s], output[%s], stderr[%s]" %(cmd, output, stderr))

    cmd = "echo -e \'%s\n%s\' | passwd %s" % (password, password, username)
    ret, output, stderr = ssh.execute(cmd, vm_ip, "root", original_root_password, False, 22)
    if ret != 0:
        test_util.test_fail("set non-root password failure, cmd[%s], output[%s], stderr[%s]" %(cmd, output, stderr))

    def check_data(self, vm=None):
        vm = vm if vm else self.vm
        vm_ip = vm.get_vm().vmNics[0].ip
        test_lib.lib_wait_target_up(vm_ip, '22', timeout=600)
#         check_cmd = "if [ ! -d /mnt/zstackwoodpecker ];then tar xvf /mnt/zstack-all-in-one.tar -C /mnt > /dev/null 2>&1; fi; \
#                      grep scenario_config_path /mnt/zstackwoodpecker/zstackwoodpecker/test_lib.py > /dev/null 2>&1 && echo 0 || echo 1"
        check_cmd = "grep APIQueryCephPrimaryStorageMsg /mnt/woodpecker/apibinding/inventory.py > /dev/null 2>&1 && echo 0 || echo 1"
        (_, ret, _)= ssh.execute(check_cmd, vm_ip, 'root', 'password')
        ret = ret.split('\n')[0]
        assert ret == '0', "data check failed!, the return code is %s, 0 is expected" % ret
        return self

def test():
    test_util.test_dsc('Create test vm with EIP and check.')
    vm1 = test_stub.create_vlan_vm(os.environ.get('l3VlanNetworkName1'))
    test_obj_dict.add_vm(vm1)

    vm2 = test_stub.create_vlan_vm(os.environ.get('l3VlanNetworkName1'))
    test_obj_dict.add_vm(vm2)

    pri_l3_name = os.environ.get('l3VlanNetworkName1')
    pri_l3_uuid = test_lib.lib_get_l3_by_name(pri_l3_name).uuid

    pub_l3_name = os.environ.get('l3PublicNetworkName')
    pub_l3_uuid = test_lib.lib_get_l3_by_name(pub_l3_name).uuid

    vm1_nic = vm1.vm.vmNics[0]
    vm1_nic_uuid = vm1_nic.uuid

    vm2_nic = vm2.vm.vmNics[0]
    vm2_nic_uuid = vm2_nic.uuid

    [test_stub.run_command_in_vm(vm.get_vm(), 'iptables -F') for vm in (vm1, vm2)]

    vip = test_stub.create_vip('create_eip_test', pub_l3_uuid)
    vip_ip = vip.get_vip().ip
    test_obj_dict.add_vip(vip)
    eip = test_stub.create_eip('create eip test', vip_uuid=vip.get_vip().uuid, vnic_uuid=vm1_nic_uuid, vm_obj=vm1)

    vip.attach_eip(eip)

    vm1.check()

    pri_l3_gateway = os.environ.get('vlanIpRangeGateway1')

    cmd1 = "ping -c 5 %s" % (vip_ip)
    (retcode1, output1, erroutput1) = ssh.execute(cmd1, ext_host_ip, "root", ext_host_pwd, True, 22)
    print "retcode1 is: %s; output1 is : %s.; erroutput1 is: %s" % (retcode1, output1, erroutput1)

    cmd2 = "arp -n | grep -w %s" % (pri_l3_gateway)
    if test_lib.lib_execute_command_in_flat_vm(vm2.vm, cmd2, l3_uuid=pri_l3_uuid):
        vm1.destroy()
        vm2.destroy()
        vm1.expunge()
        vm2.expunge()
        eip.delete()
        vip.delete()
        test_util.test_fail('Gateway leak to other vm, test failed.')

    vm1.destroy()
    vm2.destroy()
    vm1.expunge()
    vm2.expunge()
    eip.delete()
    vip.delete()

def test():
    recnt_timeout = 30000
    test_util.test_dsc('Test Host Reconnect within %s ms' % recnt_timeout)
    vm = test_stub.create_vlan_vm(os.environ.get('l3VlanNetworkName1'))
    test_obj_dict.add_vm(vm)

    zone_uuid = vm.get_vm().zoneUuid
    host = test_lib.lib_get_vm_host(vm.get_vm())
    host_uuid = host.uuid
    host_management_ip = host.managementIp
    cmd = "mkdir /var/lib/ebtables/; touch /var/lib/ebtables/lock; touch /run/ebtables.lock"
    try:
        ssh.execute(cmd, host_management_ip, "root", "password", True, 22)
    except:
        ssh.execute(cmd, host_management_ip, "root", "password", True, 22)
    try:
        host_ops.reconnect_host(host_uuid, timeout=recnt_timeout)
    except:
        host_ops.reconnect_host(host_uuid, timeout=recnt_timeout)

    test_util.test_pass('Reconnect Host within %s ms' % recnt_timeout)

        def login_vm():
            try:
                ret, output, stderr = ssh.execute(cmd.command, cmd.ip, cmd.username, cmd.password, False)
                if ret != 0:
                    rsp.success = False
                    rsp.error = '%s\n%s' % (output, stderr)
                else:
                    rsp.result = output

                rsp_dict['completion'] = True
                return True

            except Exception as e:
                logger.debug('[SSH] unable to ssh in vm[ip:%s], assume its not ready. Exception: %s' % (cmd.ip, str(e)))
                rsp_dict['error'] = True
                rsp_dict['completion'] = True
                
            return False