def check_installation(ssh_cmd, tmp_file, vm_inv):
    cmd = '%s "/usr/bin/zstack-cli LogInByAccount accountName=admin password=password"' % ssh_cmd
    process_result = execute_shell_in_process(cmd, tmp_file)
    if process_result != 0:
        test_util.test_fail('zstack-cli login failed')

    vm_passwd = test_lib.lib_get_vm_password(vm_inv)
    vm_ip = vm_ip = vm_inv.vmNics[0].ip
    cmd = '%s "/usr/bin/zstack-cli AddSftpBackupStorage name=bs1 description=bs hostname=%s username=root password=%s url=/home/bs"' % (ssh_cmd, vm_ip, vm_passwd)
    process_result = execute_shell_in_process(cmd, tmp_file)
    if process_result != 0:
        test_util.test_fail('zstack-cli create Backup Storage failed')

    cmd = '%s "/usr/bin/zstack-cli QuerySftpBackupStorage name=bs1"' % ssh_cmd
    process_result = execute_shell_in_process(cmd, tmp_file)
    if process_result != 0:
        test_util.test_fail('zstack-cli Query Backup Storage failed')
    cmd = '%s "/usr/bin/zstack-cli QuerySftpBackupStorage name=bs1 fields=uuid" | grep uuid | awk \'{print $2}\'' % ssh_cmd
    (process_result, bs_uuid) = execute_shell_in_process_stdout(cmd, tmp_file)
    if process_result != 0:
        test_util.test_fail('zstack-cli Query Backup Storage failed')

    cmd = '%s "/usr/bin/zstack-cli DeleteBackupStorage uuid=%s"' % (ssh_cmd, bs_uuid.split('"')[1])
    process_result = execute_shell_in_process(cmd, tmp_file)
    if process_result != 0:
        test_util.test_fail('zstack-cli Delete Backup Storage failed')

def test():
    test_util.test_dsc('Create test vm to test zstack all installation in CentOS7.')
#    image_name = os.environ.get('imageName_i_c7')
#    image_name = os.environ.get('imageName_i_offline')
    image_name = 'ZStack-Community-Original-Image-152'
    vm = test_stub.create_vlan_vm(image_name)
    test_obj_dict.add_vm(vm)
    if os.environ.get('zstackManagementIp') == None:
        vm.check()
    else:
        time.sleep(60)

    vm_inv = vm.get_vm()
    vm_ip = vm_inv.vmNics[0].ip
    target_file = '/root/zstack-all-in-one.tgz'
#    test_stub.prepare_test_env(vm_inv, target_file)
    zstack_install_script = os.environ['zstackInstallScript']
    installer_file = '/root/zstack_installer.sh'
    vm_ip = vm_inv.vmNics[0].ip
    vm_username = test_lib.lib_get_vm_username(vm_inv)
    vm_password = test_lib.lib_get_vm_password(vm_inv)
    test_stub.scp_file_to_vm(vm_inv, zstack_install_script, installer_file)

    all_in_one_pkg = os.environ['zstackPkg_1.4']
    test_stub.scp_file_to_vm(vm_inv, all_in_one_pkg, target_file)

    ssh.make_ssh_no_password(vm_ip, vm_username, vm_password)

    ssh_cmd = 'ssh  -oStrictHostKeyChecking=no -oCheckHostIP=no -oUserKnownHostsFile=/dev/null %s' % vm_ip
    execute_all_install(ssh_cmd, target_file, tmp_file)
    test_stub.check_installation(ssh_cmd, tmp_file, vm_inv)

    os.system('rm -f %s' % tmp_file)
#    vm.destroy()
    test_util.test_pass('ZStack installation Test Success')

def prepare_mevoco_test_env(vm_inv):
    all_in_one_pkg = os.environ['zstackPkg']
    test_lib.lib_scp_file_to_vm(vm_inv, all_in_one_pkg, '/root/zizhu.bin')

    vm_ip = vm_inv.vmNics[0].ip
    ssh.make_ssh_no_password(vm_ip, test_lib.lib_get_vm_username(vm_inv), \
            test_lib.lib_get_vm_password(vm_inv))

def prepare_yum_repo(vm_inv):
    origin_file = '/etc/yum.repos.d/epel.repo'
    target_file = '/etc/yum.repos.d/epel.repo'
    vm_ip = vm_inv.vmNics[0].ip
    vm_username = test_lib.lib_get_vm_username(vm_inv)
    vm_password = test_lib.lib_get_vm_password(vm_inv)
    scp_file_to_vm(vm_inv, origin_file, target_file)

    ssh.make_ssh_no_password(vm_ip, vm_username, vm_password)

def test():
    test_util.test_dsc('Create test vm to test zstack upgrade by -u.')

    if os.path.exists('/home/installation-package/zstack'):
        image_name = os.environ.get('imageName_i_c7_z_1.7')
    elif os.path.exists('/home/installation-package/mevoco'):
        image_name = os.environ.get('imageName_i_c7_m_1.7')

    vm = test_stub.create_vlan_vm(image_name)
    test_obj_dict.add_vm(vm)
    if os.environ.get('zstackManagementIp') == None:
        vm.check()
    else:
        time.sleep(60)

    vm_inv = vm.get_vm()
    vm_ip = vm_inv.vmNics[0].ip
    vm_username = test_lib.lib_get_vm_username(vm_inv)
    vm_password = test_lib.lib_get_vm_password(vm_inv)

    ssh_cmd = "ssh -oStrictHostKeyChecking=no -oCheckHostIP=no -oUserKnownHostsFile=/dev/null %s"%vm_ip
    ssh.make_ssh_no_password(vm_ip,vm_username,vm_password)
    test_stub.copy_id_dsa(vm_inv,ssh_cmd,tmp_file)
    test_stub.copy_id_dsa_pub(vm_inv)

    test_stub.update_iso(ssh_cmd, tmp_file, vm_inv, update_file)

    test_util.test_dsc('Update MN IP')
    cmd = '%s "zstack-ctl change_ip --ip="%s' % (ssh_cmd,vm_ip)
    test_stub.execute_shell_in_process(cmd,tmp_file)
#    cmd = '%s "zstack-ctl start"' % ssh_cmd
#    test_stub.execute_shell_in_process(cmd,tmp_file)
#    test_stub.check_installation(ssh_cmd,tmp_file,vm_inv)

    pkg_num = 1.8
    curren_num = float(os.environ.get('releasePkgNum'))
    while pkg_num<=curren_num:
        test_util.test_dsc('Upgrade zstack to %s' % pkg_num)
        upgrade_target_file = '/root/zstack-upgrade-all-in-one.tgz'
        upgrade_pkg = os.environ.get('zstackPkg_%s' % pkg_num)
        test_stub.prepare_upgrade_test_env(vm_inv, upgrade_target_file, upgrade_pkg)
        test_stub.upgrade_zstack(ssh_cmd, upgrade_target_file, tmp_file)
        test_stub.check_zstack_version(ssh_cmd, tmp_file, vm_inv, str(pkg_num))
        test_stub.check_installation(ssh_cmd, tmp_file, vm_inv)
        pkg_num = pkg_num + 0.1

    test_util.test_dsc('Upgrade zstack to latest')
    upgrade_target_file = '/root/zstack-upgrade-all-in-one.tgz'
    test_stub.prepare_test_env(vm_inv, upgrade_target_file)
    test_stub.upgrade_zstack(ssh_cmd, upgrade_target_file, tmp_file)
    zstack_latest_version = os.environ.get('zstackLatestVersion')
    test_stub.check_zstack_version(ssh_cmd, tmp_file, vm_inv, zstack_latest_version)
    test_stub.check_installation(ssh_cmd, tmp_file, vm_inv)

    os.system('rm -f %s'%tmp_file)
    vm.destroy()
    test_util.test_pass('ZStack upgrade Test Success')

def test():
    test_util.test_dsc('Create test vm to test zstack upgrade by -u.')

    if os.path.exists('/home/installation-package/zstack'):
        image_name = os.environ.get('imageName_i_c7_z_1.6')
    elif os.path.exists('/home/installation-package/mevoco'):
        image_name = os.environ.get('imageName_i_c7_m_1.6')

    vm = test_stub.create_vlan_vm(image_name)
    test_obj_dict.add_vm(vm)
    if os.environ.get('zstackManagementIp') == None:
        vm.check()
    else:
        time.sleep(60)

    vm_inv = vm.get_vm()
    vm_ip = vm_inv.vmNics[0].ip

    ssh_cmd = 'ssh  -oStrictHostKeyChecking=no -oCheckHostIP=no -oUserKnownHostsFile=/dev/null %s' % vm_ip
    ssh.make_ssh_no_password(vm_ip, test_lib.lib_get_vm_username(vm_inv), \
            test_lib.lib_get_vm_password(vm_inv))
    test_stub.copy_id_dsa(vm_inv, ssh_cmd, tmp_file)
    test_stub.copy_id_dsa_pub(vm_inv)

    test_stub.update_iso(ssh_cmd, tmp_file, vm_inv, update_file)

    test_util.test_dsc('Update MN IP')
    cmd = '%s "zstack-ctl change_ip --ip="%s ' % (ssh_cmd, vm_ip)
    process_result = test_stub.execute_shell_in_process(cmd, tmp_file)

    test_util.test_dsc('Update /etc/host/, just for mevoco 1.6')
    cmd = '%s "sed -i \'3d\' /etc/hosts"' % (ssh_cmd)
    process_result = test_stub.execute_shell_in_process(cmd, tmp_file)

    test_util.test_dsc('Reboot kairosdb, just for mevoco 1.6')
    cmd = '%s "zstack-ctl kairosdb --stop"' % (ssh_cmd)
    process_result = test_stub.execute_shell_in_process(cmd, tmp_file)
    cmd = '%s "zstack-ctl kairosdb --start"' % (ssh_cmd)
    process_result = test_stub.execute_shell_in_process(cmd, tmp_file)

#    cmd = '%s "zstack-ctl start"' % ssh_cmd
#    process_result = test_stub.execute_shell_in_process(cmd, tmp_file)
#    test_stub.check_installation(ssh_cmd, tmp_file, vm_inv)

    test_util.test_dsc('Upgrade zstack to latest') 
    upgrade_target_file = '/root/zstack-upgrade-all-in-one.tgz' 
    test_stub.prepare_test_env(vm_inv, upgrade_target_file)
    test_stub.upgrade_zstack(ssh_cmd, upgrade_target_file, tmp_file) 
    zstack_latest_version = os.environ.get('zstackLatestVersion')
    test_stub.check_zstack_version(ssh_cmd, tmp_file, vm_inv, zstack_latest_version)
    test_stub.check_installation(ssh_cmd, tmp_file, vm_inv)

    os.system('rm -f %s' % tmp_file)
    vm.destroy()
    test_util.test_pass('ZStack upgrade Test Success')

def prepare_upgrade_test_env(vm_inv, aio_target, upgrade_pkg):
    zstack_install_script = os.environ['zstackInstallScript']
    target_file = '/root/zstack_installer.sh'
    vm_ip = vm_inv.vmNics[0].ip
    vm_username = test_lib.lib_get_vm_username(vm_inv)
    vm_password = test_lib.lib_get_vm_password(vm_inv)
    scp_file_to_vm(vm_inv, zstack_install_script, target_file)

    scp_file_to_vm(vm_inv, upgrade_pkg, aio_target)

    ssh.make_ssh_no_password(vm_ip, vm_username, vm_password)

def prepare_test_env(vm_inv, aio_target):
    zstack_install_script = os.environ['zstackInstallScript']
    target_file = '/root/zstack_installer.sh'
    test_lib.lib_scp_file_to_vm(vm_inv, zstack_install_script, target_file)

    all_in_one_pkg = os.environ['zstackPkg']
    test_lib.lib_scp_file_to_vm(vm_inv, all_in_one_pkg, aio_target)

    vm_ip = vm_inv.vmNics[0].ip
    ssh.make_ssh_no_password(vm_ip, test_lib.lib_get_vm_username(vm_inv), \
            test_lib.lib_get_vm_password(vm_inv))

def test():
    test_util.test_dsc('Create test vm to test zstack installation.')
    vm = create_vlan_vm()
    test_obj_dict.add_vm(vm)
    vm.check()

    vm_inv = vm.get_vm()
    vm_ip = vm_inv.vmNics[0].ip
    zstack_install_script = test_lib.test_config.zstackInstaller.text_
    target_file = '/root/zstack_installer.sh'
    test_lib.lib_scp_file_to_vm(vm_inv, zstack_install_script, target_file)

    ssh.make_ssh_no_password(vm_ip, test_lib.lib_get_vm_username(vm_inv), \
            test_lib.lib_get_vm_password(vm_inv))

    env_var = "ZSTACK_ALL_IN_ONE='%s' ZSTACK_PYPI_URL='%s' WEBSITE='%s'" % \
            (check_str(os.environ.get('ZSTACK_ALL_IN_ONE')), \
            check_str(os.environ.get('ZSTACK_PYPI_URL')), \
            check_str(os.environ.get('WEBSITE')))
    ssh_cmd = 'ssh  -oStrictHostKeyChecking=no -oCheckHostIP=no -oUserKnownHostsFile=/dev/null %s' % vm_ip
    cmd = '%s "%s bash %s -d -a"' % (ssh_cmd, env_var, target_file)
    tmp_file = '/tmp/%s' % uuid.uuid1().get_hex()
    process_result = execute_shell_in_process(cmd, tmp_file)

    if process_result != 0:
        cmd = '%s "cat /tmp/zstack_installation.log"' % ssh_cmd
        execute_shell_in_process(cmd, tmp_file)
        test_util.test_fail('zstack installation failed')

    cmd = '%s "/usr/bin/zstack-cli LogInByAccount accountName=admin password=password"' % ssh_cmd
    process_result = execute_shell_in_process(cmd, tmp_file)
    if process_result != 0:
        test_util.test_fail('zstack-cli login failed')

    cmd = '%s "/usr/bin/zstack-cli CreateZone name=zone1 description=zone1"' % ssh_cmd
    process_result = execute_shell_in_process(cmd, tmp_file)
    if process_result != 0:
        test_util.test_fail('zstack-cli create zone failed')

    cmd = '%s "/usr/bin/zstack-cli QueryZone name=zone1 description=zone1"' % ssh_cmd
    process_result = execute_shell_in_process(cmd, tmp_file)
    if process_result != 0:
        test_util.test_fail('zstack-cli Query zone failed')

    os.system('rm -f %s' % tmp_file)
    vm.destroy()
    test_util.test_pass('ZStack installation Test Success')

    def get_ssh_ip_result(self):
        vm = self.vm_list[0]
        host = test_lib.lib_get_vm_host(vm)
        vm_command = '/sbin/ip a|grep inet'
        vm_cmd_result = test_lib.lib_execute_ssh_cmd(self.vip_ip, \
                test_lib.lib_get_vm_username(vm), \
                test_lib.lib_get_vm_password(vm), \
                vm_command)
        if not vm_cmd_result:
            test_util.test_logger('Checker result: FAIL to execute test ssh command in vip: %s for lb: %s.' % (self.vip_ip, self.lbl_uuid))
            return False
        for key, values in self.vm_ip_test_dict.iteritems():
            if key in vm_cmd_result:
                self.vm_ip_test_dict[key] += 1
                break

        return True

def check_installation(ssh_cmd, tmp_file, vm_inv):
    cmd = '%s "/usr/bin/zstack-cli LogInByAccount accountName=admin password=password"' % ssh_cmd
    process_result = execute_shell_in_process(cmd, tmp_file)
    if process_result != 0:
        test_util.test_fail('zstack-cli login failed')

    vm_passwd = test_lib.lib_get_vm_password(vm_inv)
    vm_ip = vm_ip = vm_inv.vmNics[0].ip
    cmd = '%s "/usr/bin/zstack-cli AddSftpBackupStorage name=bs1 description=bs hostname=%s username=root password=%s url=/home/bs"' % (ssh_cmd, vm_ip, vm_passwd)
    process_result = execute_shell_in_process(cmd, tmp_file)
    if process_result != 0:
        test_util.test_fail('zstack-cli create Backup Storage failed')

    cmd = '%s "/usr/bin/zstack-cli QuerySftpBackupStorage name=bs1 description=bs"' % ssh_cmd
    process_result = execute_shell_in_process(cmd, tmp_file)
    if process_result != 0:
        test_util.test_fail('zstack-cli Query Backup Storage failed')

def test():
    global vm_inv
    
    iso_path = os.environ.get('iso_path')
    upgrade_script_path = os.environ.get('upgradeScript')
    test_util.test_dsc('Create test vm to test zstack installation with console proxy.')

    conditions = res_ops.gen_query_conditions('name', '=', os.environ.get('imageNameBase_21_ex'))
    image = res_ops.query_resource(res_ops.IMAGE, conditions)[0]

    vm_inv = create_vm(image) 

    time.sleep(60)

    vm_ip = vm_inv.vmNics[0].ip
    vip = '172.20.61.253'
    if vip == vm_ip:
        vip = '172.20.61.254'

    ssh_cmd = 'ssh  -oStrictHostKeyChecking=no -oCheckHostIP=no -oUserKnownHostsFile=/dev/null %s' % vm_ip
    ssh.make_ssh_no_password(vm_ip, test_lib.lib_get_vm_username(vm_inv), test_lib.lib_get_vm_password(vm_inv))
    cmd = '%s ifconfig eth0:0 %s up' % (ssh_cmd, vip)
    process_result = test_stub.execute_shell_in_process(cmd, tmp_file)

    test_stub.update_iso(vm_ip, tmp_file, iso_path, upgrade_script_path)

    target_file = '/root/zstack-all-in-one.tgz'
    test_stub.prepare_test_env(vm_inv, target_file)
    args = '-o -C %s -I %s' % (vip, vm_ip)
    test_stub.execute_install_with_args(ssh_cmd, args, target_file, tmp_file)
    test_stub.check_installation(vm_ip, tmp_file)

    cmd = '%s cat /usr/local/zstack/apache-tomcat/webapps/zstack/WEB-INF/classes/zstack.properties | grep \'consoleProxyOverriddenIp = %s\'' % (ssh_cmd, vip)
    (process_result, check_result) = test_stub.execute_shell_in_process_stdout(cmd, tmp_file)
    check_result = check_result[:-1]
    test_util.test_dsc('cat result: |%s|' % check_result)
    expect_result = "consoleProxyOverriddenIp = %s" % vip
    if check_result != expect_result:
        test_util.test_fail('Fail to install ZStack with console proxy')

    os.system('rm -f %s' % tmp_file)
    sce_ops.destroy_vm(zstack_management_ip, vm_inv.uuid)    
    test_util.test_pass('ZStack installation Test Success')

def test():
    if os.path.exists('/home/zstack-package/') != True:
        test_util.test_skip("current test suite is zstack, but this case is for mevoco. Skip test")

    test_util.test_dsc('Create test vm to test zstack upgrade by -u.')
    image_name = os.environ.get('imageName_i_c7_z_1.2')
    vm = test_stub.create_vlan_vm(image_name)
    test_obj_dict.add_vm(vm)
    if os.environ.get('zstackManagementIp') == None:
        vm.check()
    else:
        time.sleep(60)

    vm_inv = vm.get_vm()
    vm_ip = vm_inv.vmNics[0].ip

    ssh_cmd = 'ssh  -oStrictHostKeyChecking=no -oCheckHostIP=no -oUserKnownHostsFile=/dev/null %s' % vm_ip
    ssh.make_ssh_no_password(vm_ip, test_lib.lib_get_vm_username(vm_inv), \
            test_lib.lib_get_vm_password(vm_inv))
    test_stub.copy_id_dsa(vm_inv, ssh_cmd, tmp_file)
    test_stub.copy_id_dsa_pub(vm_inv)

    test_util.test_dsc('Update MN IP')
    cmd = '%s "zstack-ctl change_ip --ip="%s ' % (ssh_cmd, vm_ip)
    process_result = test_stub.execute_shell_in_process(cmd, tmp_file)
    cmd = '%s "zstack-ctl start"' % ssh_cmd
    process_result = test_stub.execute_shell_in_process(cmd, tmp_file)
    test_stub.check_installation(ssh_cmd, tmp_file, vm_inv)

    test_util.test_dsc('Upgrade zstack to latest mevoco') 
    upgrade_target_file = '/root/mevoco-upgrade-all-in-one.tgz' 
    test_stub.prepare_test_env(vm_inv, upgrade_target_file)
    test_stub.upgrade_zstack(ssh_cmd, upgrade_target_file, tmp_file) 
    zstack_latest_version = os.environ.get('zstackLatestVersion')
    test_stub.check_zstack_version(ssh_cmd, tmp_file, vm_inv, zstack_latest_version)
    test_stub.check_zstack_or_mevoco(ssh_cmd, tmp_file, vm_inv, 'mevoco')
    test_stub.check_installation(ssh_cmd, tmp_file, vm_inv)

    os.system('rm -f %s' % tmp_file)
    vm.destroy()
    test_util.test_pass('ZStack upgrade Test Success')

def test():
    test_util.test_dsc('Create test vm to test zstack installation with console proxy.')

    image_name = os.environ.get('imageName_i_offline')

    vm = test_stub.create_vlan_vm(image_name)
    test_obj_dict.add_vm(vm)
    if os.environ.get('zstackManagementIp') == None:
        vm.check()
    else:
        time.sleep(60)

    vm_inv = vm.get_vm()
    vm_ip = vm_inv.vmNics[0].ip
    vip = '172.20.198.1'
    if vip == vm_ip:
        vip = '172.20.198.2'

    ssh_cmd = 'ssh  -oStrictHostKeyChecking=no -oCheckHostIP=no -oUserKnownHostsFile=/dev/null %s' % vm_ip
    ssh.make_ssh_no_password(vm_ip, test_lib.lib_get_vm_username(vm_inv), test_lib.lib_get_vm_password(vm_inv))
    cmd = '%s ifconfig eth0:0 %s up' % (ssh_cmd, vip)
    process_result = test_stub.execute_shell_in_process(cmd, tmp_file)

    target_file = '/root/zstack-all-in-one.tgz'
    test_stub.prepare_test_env(vm_inv, target_file)
    args = '-o -C %s -I %s' % (vip, vm_ip)
    test_stub.execute_install_with_args(ssh_cmd, args, target_file, tmp_file)
    test_stub.check_installation(ssh_cmd, tmp_file, vm_inv)

    cmd = '%s cat /usr/local/zstack/apache-tomcat/webapps/zstack/WEB-INF/classes/zstack.properties | grep \'consoleProxyOverriddenIp = %s\'' % (ssh_cmd, vip)
    (process_result, check_result) = test_stub.execute_shell_in_process_stdout(cmd, tmp_file)
    check_result = check_result[:-1]
    test_util.test_dsc('cat result: |%s|' % check_result)
    expect_result = "consoleProxyOverriddenIp = %s" % vip
    if check_result != expect_result:
        test_util.test_fail('Fail to install ZStack with console proxy')

    os.system('rm -f %s' % tmp_file)
    vm.destroy()
    test_util.test_pass('ZStack installation Test Success')

def check_installation(ssh_cmd, tmp_file, vm_inv):
    cmd = '%s "/usr/bin/zstack-cli LogInByAccount accountName=admin password=password"' % ssh_cmd
    process_result = execute_shell_in_process(cmd, tmp_file)
    if process_result != 0:
        test_util.test_fail('zstack-cli login failed')

    vm_passwd = test_lib.lib_get_vm_password(vm_inv)
    vm_ip = vm_ip = vm_inv.vmNics[0].ip
    cmd = '%s "/usr/bin/zstack-cli AddSftpBackupStorage name=bs1 description=bs hostname=%s username=root password=%s url=/home/bs"' % (ssh_cmd, vm_ip, vm_passwd)
    process_result = execute_shell_in_process(cmd, tmp_file)
    if process_result != 0:
        test_util.test_fail('zstack-cli create Backup Storage failed')

    cmd = '%s "/usr/bin/zstack-cli QuerySftpBackupStorage name=bs1"' % ssh_cmd
    process_result = execute_shell_in_process(cmd, tmp_file)
    if process_result != 0:
        test_util.test_fail('zstack-cli Query Backup Storage failed')
    cmd = '%s "/usr/bin/zstack-cli QuerySftpBackupStorage name=bs1 fields=uuid" | grep uuid | awk \'{print $2}\'' % ssh_cmd
    (process_result, bs_uuid) = execute_shell_in_process_stdout(cmd, tmp_file)
    if process_result != 0:
        test_util.test_fail('zstack-cli Query Backup Storage failed')

    cmd = '%s "/usr/bin/zstack-cli DeleteBackupStorage uuid=%s"' % (ssh_cmd, bs_uuid.split('"')[1])
    process_result = execute_shell_in_process(cmd, tmp_file)
    if process_result != 0:
        test_util.test_fail('zstack-cli Delete Backup Storage failed')

# check zone
    cmd = '%s "/usr/bin/zstack-cli CreateZone name=ZONE1"' % ssh_cmd
    process_result = execute_shell_in_process(cmd, tmp_file)
    if process_result != 0:
        test_util.test_fail('zstack-cli Create Zone failed')

    cmd = '%s "/usr/bin/zstack-cli QueryZone name=ZONE1"' % ssh_cmd
    process_result = execute_shell_in_process(cmd, tmp_file)
    if process_result != 0:
        test_util.test_fail('zstack-cli Query Zone failed')
    cmd = '%s "/usr/bin/zstack-cli QueryZone name=ZONE1 fields=uuid" | grep uuid | awk \'{print $2}\'' % ssh_cmd
    (process_result, zone_uuid) = execute_shell_in_process_stdout(cmd, tmp_file)
    if process_result != 0:
        test_util.test_fail('zstack-cli Query Zone failed')

    cmd = '%s "/usr/bin/zstack-cli DeleteZone uuid=%s"' % (ssh_cmd, zone_uuid.split('"')[1])
    process_result = execute_shell_in_process(cmd, tmp_file)
    if process_result != 0:
        test_util.test_fail('zstack-cli Delete Zone failed')

# add check item
    cmd = '%s "/usr/bin/zstack-ctl status" | grep \'^version\' | awk \'{print $2}\'' % ssh_cmd
    (process_result, version_info) = execute_shell_in_process_stdout(cmd, tmp_file)
    if process_result != 0:
        test_util.test_fail('zstack-ctl get version failed')
    if '1.3' in version_info or '1.2' in version_info:
        cmd = '%s "/usr/bin/zstack-ctl status" | grep \'^status\' | awk \'{print $2}\'' % ssh_cmd
        (process_result, status_info) = execute_shell_in_process_stdout(cmd, tmp_file)
        if process_result != 0:
            test_util.test_fail('zstack-ctl get status failed')
        if not 'Running' in status_info:
            test_util.test_dsc('zstack is not running, try to start zstack')
            cmd = '%s "/usr/bin/zstack-ctl start"' % ssh_cmd
            process_result = process_result = execute_shell_in_process(cmd, tmp_file)
            if process_result != 0:
                test_util.test_fail('zstack-ctl start failed')
            time.sleep(5)
            cmd = '%s "/usr/bin/zstack-ctl status" | grep \'^status\' | awk \'{print $2}\'' % ssh_cmd
            (process_result, status_info) = execute_shell_in_process_stdout(cmd, tmp_file)
            if process_result != 0:
                test_util.test_fail('zstack-ctl get status failed')
            if not 'Running' in status_info:
                test_util.test_fail('zstack is not running, start zstack failed')
        test_util.test_dsc('check zstack status, zstack is running')
    else:
        cmd = '%s "/usr/bin/zstack-ctl status" | grep \'^MN status\' | awk \'{print $3}\'' % ssh_cmd
        (process_result, mn_status) = execute_shell_in_process_stdout(cmd, tmp_file)
        if process_result != 0:
            test_util.test_fail('zstack-ctl get MN status failed')
        if not 'Running' in mn_status:
            test_util.test_dsc('management node is not running, try to start management node')
            cmd = '%s "/usr/bin/zstack-ctl start_node"' % ssh_cmd
            process_result = process_result = execute_shell_in_process(cmd, tmp_file)
            if process_result != 0:
                test_util.test_fail('zstack-ctl start_node failed')
            time.sleep(5)
            cmd = '%s "/usr/bin/zstack-ctl status" | grep \'^MN status\' | awk \'{print $3}\'' % ssh_cmd
            (process_result, mn_status) = execute_shell_in_process_stdout(cmd, tmp_file)
            if process_result != 0:
                test_util.test_fail('zstack-ctl get MN status failed')
            if not 'Running' in mn_status:
                test_util.test_fail('management node is not running, start management node failed')
        test_util.test_dsc('check MN, MN is running')
        cmd = '%s "/usr/bin/zstack-ctl status" | grep \'^UI status\' | awk \'{print $3}\'' % ssh_cmd
        (process_result, ui_status) = execute_shell_in_process_stdout(cmd, tmp_file)
        if process_result != 0:
            test_util.test_fail('zstack-ctl get UI status failed')
        if not 'Running' in ui_status:
            test_util.test_dsc('UI is not running, try to start UI')
            cmd = '%s "/usr/bin/zstack-ctl start_ui"' % ssh_cmd
            process_result = process_result = execute_shell_in_process(cmd, tmp_file)
            if process_result != 0:
                test_util.test_fail('zstack-ctl start_ui failed')
#.........这里部分代码省略.........

def test():
    global vm
    if test_lib.lib_get_ha_enable() != 'true':
        test_util.test_skip("vm ha not enabled. Skip test")

    vm = test_stub.create_vm()
    vm.check()
    ha_ops.set_vm_instance_ha_level(vm.get_vm().uuid, "NeverStop")
    rsp = test_lib.lib_execute_ssh_cmd(vm.get_vm().vmNics[0].ip, test_lib.lib_get_vm_username(vm.get_vm()), test_lib.lib_get_vm_password(vm.get_vm()), 'init 0')
    time.sleep(60)
    vm.set_state(vm_header.RUNNING)
    vm.check()
    vm.destroy()
    vm.check()
    test_util.test_pass('VM ha never stop Test Success')

    def check(self):
        super(zstack_kvm_vm_snat_checker, self).check()
        vm = self.test_obj.vm
        test_lib.lib_install_testagent_to_vr(vm)
        host = test_lib.lib_get_vm_host(vm)

        vm_cmd_result = None
        vr_vms = test_lib.lib_find_vr_by_vm(vm)
        test_lib.lib_set_vm_host_l2_ip(vm)
        for vr_vm in vr_vms:
            test_util.test_logger("Begin to check [vm:] %s SNAT" % vm.uuid)
            nic = test_lib.lib_get_vm_nic_by_vr(vm, vr_vm)
            if not 'SNAT' in test_lib.lib_get_l3_service_type(nic.l3NetworkUuid):
                test_util.test_logger("Skip [VR:] %s, since it doesn't provide SNAT service" % vr_vm.uuid)
                continue

            ping_target = test_lib.test_config.pingTestTarget.text_
            #Check if there is a SG rule to block ICMP checking
            if test_lib.lib_is_sg_rule_exist(nic.uuid, None, None, inventory.EGRESS):
                if not test_lib.lib_is_sg_rule_exist(nic.uuid, inventory.ICMP, ping_target, inventory.EGRESS):
                    test_util.test_warn('Skip SNAT checker: because there is ICMP Egress Rule was assigned to [nic:] %s and the allowed target ip is not %s' % (nic.uuid, ping_target))
                    return self.judge(self.exp_result)

            guest_ip = nic.ip
            vm_command = 'ping -c 5 -W 5 %s >/tmp/ping_result 2>&1; ret=$?; cat /tmp/ping_result; exit $ret' % ping_target
            vm_cmd_result = test_lib.lib_ssh_vm_cmd_by_agent_with_retry(host.managementIp, nic.ip, test_lib.lib_get_vm_username(vm), test_lib.lib_get_vm_password(vm), vm_command, self.exp_result)
            if not vm_cmd_result:
                test_util.test_logger('Checker result: FAIL to ping [target:] %s from [vm:] %s .' % (ping_target, vm.uuid))
                if self.exp_result == True:
                    test_util.test_logger("network connection result is not expected pass, will print VR's network configuration:")
                    test_lib.lib_print_vr_network_conf(vr_vm)
                return self.judge(False)
            else:
                test_util.test_logger('Checker result: SUCCEED to ping [target:] %s from [vm:] %s .' % (ping_target, vm.uuid))
                return self.judge(True)

def test():
    def test_fail(msg):
        os.system('rm -f %s' % tmp_file)
        test_util.test_fail(msg)

    test_util.test_dsc('Create 3 CentOS7 vm to test multi management node installation')

#    image_name = os.environ.get('imageName_i_c7')
    image_name = os.environ.get('imageName_i_offline')

    vm1 = test_stub.create_vlan_vm(image_name)
    test_obj_dict.add_vm(vm1)
    vm2 = test_stub.create_vlan_vm(image_name)
    test_obj_dict.add_vm(vm2)
    vm3 = test_stub.create_vlan_vm(image_name)
    test_obj_dict.add_vm(vm3)
    if os.environ.get('zstackManagementIp') == None:
        vm1.check()
        vm2.check()
        vm3.check()
    else:
        time.sleep(60)

    vm1_inv = vm1.get_vm()
    vm1_ip = vm1_inv.vmNics[0].ip
    vm2_inv = vm2.get_vm()
    vm2_ip = vm2_inv.vmNics[0].ip
    vm3_inv = vm3.get_vm()
    vm3_ip = vm3_inv.vmNics[0].ip
    target_file = '/root/zstack-all-in-one.tgz'
    test_stub.prepare_test_env(vm1_inv, target_file)
    ssh_cmd1 = 'ssh  -oStrictHostKeyChecking=no -oCheckHostIP=no -oUserKnownHostsFile=/dev/null %s' % vm1_ip
    ssh_cmd2 = 'ssh  -oStrictHostKeyChecking=no -oCheckHostIP=no -oUserKnownHostsFile=/dev/null %s' % vm2_ip
    ssh_cmd3 = 'ssh  -oStrictHostKeyChecking=no -oCheckHostIP=no -oUserKnownHostsFile=/dev/null %s' % vm3_ip

    test_util.test_dsc('Install ZStack on vm1')
    test_stub.copy_id_dsa(vm1_inv, ssh_cmd1, tmp_file)
    test_stub.copy_id_dsa_pub(vm1_inv)
    test_stub.execute_all_install(ssh_cmd1, target_file, tmp_file)
    test_stub.check_installation(ssh_cmd1, tmp_file, vm1_inv)

    test_util.test_dsc('Install multi management node on vm2 and vm3')
    host_list = 'root:passw[email protected]%s root:[email protected]%s' % (vm2_ip, vm3_ip)
    cmd = '%s "zstack-ctl add_multi_management --host-list %s"' % (ssh_cmd1, host_list)
    process_result = test_stub.execute_shell_in_process(cmd, tmp_file)

    test_util.test_dsc('Check installation on vm1')
    test_stub.check_installation(ssh_cmd1, tmp_file, vm1_inv)

    test_util.test_dsc('Check installation on vm2')
    ssh.make_ssh_no_password(vm2_ip, test_lib.lib_get_vm_username(vm2_inv), \
            test_lib.lib_get_vm_password(vm2_inv))
    cmd = '%s "zstack-ctl start"' % ssh_cmd2
    process_result = test_stub.execute_shell_in_process(cmd, tmp_file)
    test_stub.check_installation(ssh_cmd2, tmp_file, vm2_inv)

    test_util.test_dsc('Check installation on vm3')
    ssh.make_ssh_no_password(vm3_ip, test_lib.lib_get_vm_username(vm3_inv), \
            test_lib.lib_get_vm_password(vm3_inv))
    cmd = '%s "zstack-ctl start"' % ssh_cmd3
    process_result = test_stub.execute_shell_in_process(cmd, tmp_file)
    test_stub.check_installation(ssh_cmd3, tmp_file, vm3_inv)

    test_util.test_dsc('Upgrade multi management node on vm2 and vm3')
    cmd = '%s "zstack-ctl upgrade_multi_management_node --installer-bin %s"' % (ssh_cmd1, target_file) 
    process_result = test_stub.execute_shell_in_process(cmd, tmp_file)

    test_util.test_dsc('After upgrade, check installation on vm1')
    test_stub.check_installation(ssh_cmd1, tmp_file, vm1_inv)

    test_util.test_dsc('After upgrade, check installation on vm2')
    ssh.make_ssh_no_password(vm2_ip, test_lib.lib_get_vm_username(vm2_inv), \
            test_lib.lib_get_vm_password(vm2_inv))
    cmd = '%s "zstack-ctl start"' % ssh_cmd2
    process_result = test_stub.execute_shell_in_process(cmd, tmp_file)
    test_stub.check_installation(ssh_cmd2, tmp_file, vm2_inv)

    test_util.test_dsc('After upgrade, check installation on vm3')
    ssh.make_ssh_no_password(vm3_ip, test_lib.lib_get_vm_username(vm3_inv), \
            test_lib.lib_get_vm_password(vm3_inv))
    cmd = '%s "zstack-ctl start"' % ssh_cmd3
    process_result = test_stub.execute_shell_in_process(cmd, tmp_file)
    test_stub.check_installation(ssh_cmd3, tmp_file, vm3_inv)

    os.system('rm -f %s' % tmp_file)
    vm1.destroy()
    test_obj_dict.rm_vm(vm1)
    vm2.destroy()
    test_obj_dict.rm_vm(vm2)
    vm3.destroy()
    test_obj_dict.rm_vm(vm3)
    test_util.test_pass('ZStack multi management nodes installation Test Success')

def make_ssh_no_password(vm_inv):
    vm_ip = vm_inv.vmNics[0].ip
    ssh.make_ssh_no_password(vm_ip, test_lib.lib_get_vm_username(vm_inv), \
            test_lib.lib_get_vm_password(vm_inv))

    def check(self):
        super(zstack_kvm_vm_data_integrity_checker, self).check()

        if isinstance(self.test_obj, volume_header.TestVolume):
            volume = self.test_obj.volume
            vm = self.test_obj.target_vm.vm
        if isinstance(self.test_obj, vm_header.TestVm):
            vm = self.test_obj.vm
        if isinstance(self.test_obj, zstack_sp_header.ZstackVolumeSnapshot):
            volume_obj = self.test_obj.get_target_volume()
            vm = volume_obj.get_target_vm()

        if vm.state != "Running":
            test_util.test_logger('Check result: Skip attach_volume_checker since VM is not in Running state')
            return self.judge(True)
     
        time.sleep(30)

        #test_lib.lib_install_testagent_to_vr(vm)
        host = test_lib.lib_get_vm_host(vm)
        test_lib.lib_install_testagent_to_host(host)
        test_lib.lib_set_vm_host_l2_ip(vm)
        default_l3_uuid = vm.defaultL3NetworkUuid
        #vr = test_lib.lib_find_vr_by_pri_l3(default_l3_uuid)
        #nic = test_lib.lib_get_vm_nic_by_vr(vm, vr)
        nic = vm.vmNics[0]
        
        #print partition information
        cmd = 'ls -l /dev/disk/by-id/'
        cmd_res = test_lib.lib_ssh_vm_cmd_by_agent_with_retry(host.managementIp, nic.ip, test_lib.lib_get_vm_username(vm), test_lib.lib_get_vm_password(vm), cmd, self.exp_result)
        test_util.test_logger("partition information: %s" % cmd_res)
        #exec vdbench
        command = 'python /root/vdbench_test.py | tee result'
        cmd_result = test_lib.lib_ssh_vm_cmd_by_agent_with_retry(host.managementIp, nic.ip, test_lib.lib_get_vm_username(vm), test_lib.lib_get_vm_password(vm), command, self.exp_result)
        test_util.test_logger("czhou: %s" % cmd_result)

        if isinstance(cmd_result, str) and "generate successfully" in cmd_result:
            test_util.test_logger("Checker result: Success to validate data integrity, output: %s" % cmd_result)
            return self.judge(True)
      
        if isinstance(cmd_result, str) and "no disk attached, skip generating" in cmd_result:
            test_util.test_logger("Checker result: No validationg and no generating, output: %s" % cmd_result)
            return self.judge(True)

        #if isinstance(cmd_result, str) and "All old disks have been removed,skip validation" in cmd_result:
        #    if "generate successfully" in cmd_result or "skip generating" in cmd_result:
        #        test_util.test_logger("Checker result: Skip validation checker since all disks have been removed")
        #        return self.judge(True)
        #    else:
        #        test_util.test_logger("Checker result: Skip validation checker since all disks have been removed, but generating data failed on volume output: %s" % cmd_result)
        #        return self.judge(False)

        #if isinstance(cmd_result, str) and "validate successfully" in cmd_result:
        #    if "generate successfully" in cmd_result or "skip generating" in cmd_result:
        #        test_util.test_logger("Checker result: Success to validate data integrity, output: %s" % cmd_result)
        #        return self.judge(True)
        #    else:
        #        test_util.test_logger("Checker result: Success to validate data integrity, but generating data failed on volume output: %s" % cmd_result)
        #        return self.judge(False)
 
        #if isinstance(cmd_result, str) and "validate failed on" in cmd_result:
        #    test_util.test_logger("Checker result: Fail to validate data integrity, output: %s" % cmd_result)
        #    return self.judge(False)


        return self.judge(False)