def ensure_bss_host_connected_from_sep_net_down(scenarioFile, scenarioConfig, downMagt=True):
    '''
        This function is only support for separated network case
    '''
    bss_host_ip = []
    bs_list = res_ops.query_resource(res_ops.BACKUP_STORAGE)
    if bs_list[0].type == "SftpBackupStorage":
        for bs in bs_list:
            bss_host_ip.append(bs.hostname)

        for bs_host_ip in bss_host_ip:
            if test_lib.lib_wait_target_up(bs_host_ip, '22', 300):
                bss_host_ip.remove(bs_host_ip)

        if downMagt:
            l2network_nic = test_lib.lib_get_l2_magt_nic_by_vr_offering()
        else:
            l2network_nic = test_lib.lib_get_l2_pub_nic_by_vr_offering()

        mha_s_vm_list = get_mha_s_vm_list_from_scenario_file(scenarioConfig, scenarioFile)
        for bs_host_ip in bss_host_ip:
            for mn_host in mha_s_vm_list:
                if mn_host.managementIp_ == bs_host_ip or mn_host.ip_ == bs_host_ip:
                    reopen_host_network(mn_host, scenarioConfig, param_l2_nic=l2network_nic)

        for bs_host_ip in bss_host_ip:
            if test_lib.lib_wait_target_up(bs_host_ip, '22', 300):
                bss_host_ip.remove(bs_host_ip)

        if bss_host_ip:
            test_util.test_fail("still have bs host not started.")
    else:
        test_util.test_logger("the current bs is %s type, is not expected sftp, therefore, skip ensure bss host connected" %(bs_list[0].type))

def test():
    global vm_inv
    test_util.test_dsc('Create test vm to test zstack upgrade by -u.')

    image_name = os.environ.get('imageName_i_c7_z_1.9')
    #iso_path = os.environ.get('iso_path')
    zstack_latest_version = os.environ.get('zstackLatestVersion')
    zstack_latest_path = os.environ.get('zstackLatestInstaller')
    vm_name = os.environ.get('vmName')
    #upgrade_script_path = os.environ.get('upgradeScript')

    vm_inv = test_stub.create_vm_scenario(image_name, vm_name)
    vm_ip = vm_inv.vmNics[0].ip
    test_lib.lib_wait_target_up(vm_ip, 22)

    test_stub.make_ssh_no_password(vm_ip, tmp_file)

    test_util.test_logger('Update MN IP')
    test_stub.update_mn_hostname(vm_ip, tmp_file)
    test_stub.update_mn_ip(vm_ip, tmp_file)
    test_stub.start_mn(vm_ip, tmp_file)
    test_stub.check_installation(vm_ip, tmp_file)

    test_util.test_logger('Upgrade zstack to latest with repo') 
    #test_stub.update_iso(vm_ip, tmp_file, iso_path, upgrade_script_path)
    test_stub.update_repo(vm_ip, tmp_file)
    test_stub.upgrade_zstack(vm_ip, zstack_latest_path, tmp_file) 
    test_stub.check_zstack_version(vm_ip, tmp_file, zstack_latest_version)
    test_stub.start_mn(vm_ip, tmp_file)
    test_stub.check_mn_running(vm_ip, tmp_file)
    test_stub.check_installation(vm_ip, tmp_file)

    os.system('rm -f %s' % tmp_file)
    test_stub.destroy_vm_scenario(vm_inv.uuid)
    test_util.test_pass('ZStack upgrade Test Success')

def test():
    instance_offering_name = os.environ.get('instanceOfferingName_m')
    instance_offering_uuid = test_lib.lib_get_instance_offering_by_name(instance_offering_name).uuid

    vm = test_stub.create_vm(image_name = os.environ.get('sshkeyImageName'), instance_offering_uuid = instance_offering_uuid)
    test_obj_dict.add_vm(vm)
    vm_ip = vm.get_vm().vmNics[0].ip
    time.sleep(10)
    test_lib.lib_add_vm_sshkey(vm.get_vm().uuid, os.environ.get('sshkeyPubKey'))
    host = test_lib.lib_get_vm_host(vm.get_vm())
    host_uuid = host.uuid
    host_ops.reconnect_host(host_uuid)
    vm.reboot()
    test_lib.lib_wait_target_up(vm_ip, '22', 240)
    time.sleep(10)
    for i in range(5):
        ssh_cmd = 'timeout 5 ssh -i %s -oPasswordAuthentication=no -oStrictHostKeyChecking=no -oCheckHostIP=no -oUserKnownHostsFile=/dev/null %s echo pass' % (os.environ.get('sshkeyPriKey_file'), vm_ip)
        process_result = test_stub.execute_shell_in_process(ssh_cmd, tmp_file)
        if process_result == 0:
            break
        else:
            time.sleep(10)
    else:
        test_util.test_fail("fail to use ssh key connect to VM")

    vm.destroy()
    test_util.test_pass('Create VM Test Success')

def test():
    global vm_inv
    test_util.test_dsc('Create test vm to test zstack upgrade by -u.')
    image_name = os.environ.get('imageName_i_c7_z_1.10')
    iso_path = os.environ.get('iso_path')
    iso_20_path = os.environ.get('iso_20_path')
    iso_21_path = os.environ.get('iso_21_path')
    iso_230_path = os.environ.get('iso_230_path')
    zstack_latest_version = os.environ.get('zstackLatestVersion')
    zstack_latest_path = os.environ.get('zstackLatestInstaller')
    vm_name = os.environ.get('vmName')
    upgrade_script_path = os.environ.get('upgradeScript')

    vm_inv = test_stub.create_vm_scenario(image_name, vm_name)
    vm_ip = vm_inv.vmNics[0].ip
    test_lib.lib_wait_target_up(vm_ip, 22)

    test_stub.make_ssh_no_password(vm_ip, tmp_file)


    test_util.test_dsc('Update MN IP')
    test_stub.update_mn_hostname(vm_ip, tmp_file)
    test_stub.update_mn_ip(vm_ip, tmp_file)
    #test_stub.reset_rabbitmq(vm_ip, tmp_file)
    test_stub.start_mn(vm_ip, tmp_file)
    test_stub.check_installation(vm_ip, tmp_file)

    #test_stub.update_iso(vm_ip, tmp_file, iso_path, upgrade_script_path)
    test_stub.update_20_iso(vm_ip, tmp_file, iso_20_path, upgrade_script_path)
    #pkg_num = 1.9

    release_ver=['2.0.0','2.1.0','2.2.0','2.3.0','2.3.1']
    curren_num = float(os.environ.get('releasePkgNum'))
    for pkg_num in release_ver:
    #while pkg_num <= curren_num:
	#if str(pkg_num) == '1.10':
	#	test_stub.update_iso(vm_ip, tmp_file, iso_path, upgrade_script_path)
        if str(pkg_num) == '2.1.0':
		test_stub.update_21_iso(vm_ip, tmp_file, iso_21_path, upgrade_script_path)
        if str(pkg_num) == '2.3.0':
		test_stub.update_230_iso(vm_ip, tmp_file, iso_230_path, upgrade_script_path)
        test_util.test_logger('Upgrade zstack to %s' % pkg_num)
        upgrade_pkg = os.environ.get('zstackPkg_%s' % pkg_num)
        test_stub.upgrade_zstack(vm_ip, upgrade_pkg, tmp_file) 
        test_stub.start_mn(vm_ip, tmp_file)
        test_stub.check_zstack_version(vm_ip, tmp_file, str(pkg_num))
        #test_stub.check_installation(vm_ip, tmp_file)
        #pkg_num = pkg_num + 0.1

    test_util.test_dsc('Upgrade zstack to latest') 
    test_stub.update_iso(vm_ip, tmp_file, iso_path, upgrade_script_path)
    test_stub.upgrade_zstack(vm_ip, zstack_latest_path, tmp_file) 
    test_stub.start_mn(vm_ip, tmp_file)
    test_stub.check_mn_running(vm_ip, tmp_file)
    test_stub.check_zstack_version(vm_ip, tmp_file, zstack_latest_version)
    test_stub.check_installation(vm_ip, tmp_file)

    os.system('rm -f %s' % tmp_file)
    test_stub.destroy_vm_scenario(vm_inv.uuid)
    test_util.test_pass('ZStack upgrade Test Success')

def ensure_bss_host_connected_from_stop(scenarioFile, scenarioConfig, deploy_config):
    '''
        This function is only support for not separated network case
    '''
    bss_host_ip = []
    bs_list = res_ops.query_resource(res_ops.BACKUP_STORAGE)
    if bs_list[0].type == "SftpBackupStorage":
        for bs in bs_list:
            bss_host_ip.append(bs.hostname)

        for bs_host_ip in bss_host_ip:
            if test_lib.lib_wait_target_up(bs_host_ip, '22', 300):
                bss_host_ip.remove(bs_host_ip)

        mha_s_vm_list = get_mha_s_vm_list_from_scenario_file(scenarioConfig, scenarioFile)
        for bs_host_ip in bss_host_ip:
            for mn_host in mha_s_vm_list:
                if mn_host.managementIp_ == bs_host_ip or mn_host.ip_ == bs_host_ip:
                    recover_host(mn_host, scenarioConfig, deploy_config)

        for bs_host_ip in bss_host_ip:
            if test_lib.lib_wait_target_up(bs_host_ip, '22', 300):
                bss_host_ip.remove(bs_host_ip)

        if bss_host_ip:
            test_util.test_fail("still have bs host not started.")
    else:
        test_util.test_logger("the current bs is %s type, is not expected sftp, therefore, skip ensure bss host connected" %(bs_list[0].type))

def test():
    #skip ceph in c74
    cmd = "cat /etc/redhat-release | grep '7.4'"
    mn_ip = res_ops.query_resource(res_ops.MANAGEMENT_NODE)[0].hostName
    rsp = test_lib.lib_execute_ssh_cmd(mn_ip, 'root', 'password', cmd, 180)
    if rsp != False:
        ps = res_ops.query_resource(res_ops.PRIMARY_STORAGE)
        for i in ps:
            if i.type == 'Ceph':
                test_util.test_skip('cannot hotplug iso to the vm in ceph,it is a libvirt bug:https://bugzilla.redhat.com/show_bug.cgi?id=1541702.')    

    global iso
    global test_obj_dict

    # run condition
    allow_bs_list = [inventory.IMAGE_STORE_BACKUP_STORAGE_TYPE, inventory.CEPH_BACKUP_STORAGE_TYPE]
    test_lib.skip_test_when_bs_type_not_in_list(allow_bs_list)

    hosts = res_ops.query_resource(res_ops.HOST)
    if len(hosts) <= 1:
        test_util.test_skip("skip for host_num is not satisfy condition host_num>1")

    # add iso and create vm from iso
    iso = test_stub.add_test_minimal_iso('minimal_iso')
    test_obj_dict.add_image(iso)
    root_volume_offering = test_stub.add_test_root_volume_offering('root-disk-iso', 10737418240)
    test_obj_dict.add_disk_offering(root_volume_offering)
    vm_offering = test_stub.add_test_vm_offering(2, 1024*1024*1024, 'iso-vm-offering')
    test_obj_dict.add_instance_offering(vm_offering)
    vm = test_stub.create_vm_with_iso_for_test(vm_offering.uuid, iso.image.uuid, root_volume_offering.uuid, 'iso-vm')
    test_obj_dict.add_vm(vm)

    # check vm
    vm_inv = vm.get_vm()
    test_lib.lib_set_vm_host_l2_ip(vm_inv)
    test_lib.lib_wait_target_up(vm.get_vm().vmNics[0].ip, 22, 1800)

    # clone vm
    cloned_vm_name = ['cloned_vm']
    cloned_vm_obj = vm.clone(cloned_vm_name)[0]
    test_obj_dict.add_vm(cloned_vm_obj)

    # delete iso
    iso.delete()

    # vm ops test
    test_stub.vm_ops_test(cloned_vm_obj, "VM_TEST_ATTACH")

    # expunge iso
    iso.expunge()

    #detach iso
    img_ops.detach_iso(vm.vm.uuid)

    # vm ops test
    test_stub.vm_ops_test(cloned_vm_obj, "VM_TEST_MIGRATE")

    test_lib.lib_robot_cleanup(test_obj_dict)
    test_util.test_pass('Cloned VM ops for BS Success')

def test():
    global vm_inv,host_name, host_uuid, host_management_ip, vm_ip, bs_name, bs_uuid
    test_util.test_dsc('Create test vm to test zstack upgrade by -u.')

    image_name = os.environ.get('imageTestAlarm_230_mn')
    iso_path = os.environ.get('iso_path')
    zstack_latest_version = os.environ.get('zstackLatestVersion')
    zstack_latest_path = os.environ.get('zstackLatestInstaller')
    vm_name = os.environ.get('vmName') + image_name
    upgrade_script_path = os.environ.get('upgradeScript')

    vm_inv = test_stub.create_vm_scenario(image_name, vm_name)
    vm_ip = vm_inv.vmNics[0].ip
    test_lib.lib_wait_target_up(vm_ip, 22)
    #vm_ip = '172.20.197.159'
    os.environ['ZSTACK_BUILT_IN_HTTP_SERVER_IP'] = vm_ip
    test_stub.make_ssh_no_password(vm_ip, tmp_file)

    test_util.test_logger('Update MN IP')
    test_stub.update_mn_hostname(vm_ip, tmp_file)
    test_stub.update_mn_ip(vm_ip, tmp_file)
    test_stub.start_mn(vm_ip, tmp_file)


    test_util.test_logger('Update host management IP and reconnect host')
    host_name = 'Host-1'
    cond1 = res_ops.gen_query_conditions('name', '=', host_name)
    test2 = res_ops.query_resource(res_ops.HOST, cond1)
    test_util.test_logger('aaaaa %s' % test2)
    #host_uuid = res_ops.query_resource(res_ops.HOST, conditions)[0].inventories[0]
    management_ip = vm_ip
    host_uuid = scen_ops.query_resource(management_ip, res_ops.HOST, cond1).inventories[0].uuid
    print 'test = ' 
    print  host_uuid
    host_management_ip = vm_ip
    host_ops.update_host(host_uuid, 'managementIp', host_management_ip)
    host_ops.reconnect_host(host_uuid)

    test_util.test_logger('Update bs IP and reconnect bs')
    bs_name = 'BS-1'
    cond2 = res_ops.gen_query_conditions('name', '=', bs_name)
    #bs_uuid = res_ops.query_resource(res_ops.BACKUP_STORAGE, conditions)[0].inventories[0].uuid    
    bs_uuid = scen_ops.query_resource(management_ip, res_ops.BACKUP_STORAGE, cond2).inventories[0].uuid
    bs_ip = vm_ip
    bs_ops.update_image_store_backup_storage_info(bs_uuid, 'hostname', bs_ip)
    bs_ops.reconnect_backup_storage(bs_uuid)

    #test_stub.check_installation(vm_ip, tmp_file)

    test_util.test_logger('Upgrade zstack to latest') 
    test_stub.update_iso(vm_ip, tmp_file, iso_path, upgrade_script_path)
    test_stub.upgrade_zstack(vm_ip, zstack_latest_path, tmp_file) 
    test_stub.check_zstack_version(vm_ip, tmp_file, zstack_latest_version)
    test_stub.start_mn(vm_ip, tmp_file)
    test_stub.check_installation(vm_ip, tmp_file)

    os.system('rm -f %s' % tmp_file)
    test_stub.destroy_vm_scenario(vm_inv.uuid)
    test_util.test_pass('ZStack upgrade Test Success')

def resize_rvol(vm_obj, dvol):
    vol_size = dvol.volume.size
    volume_uuid = dvol.volume.uuid
    set_size = 1024 * 1024 * 1024 + int(vol_size)
    vol_ops.resize_data_volume(volume_uuid, set_size)
    vm_obj.update()
    # if set_size/vol_size_after > 0.9:
    #     test_util.test_fail('Resize Root Volume failed, size = %s' % vol_size_after)
    vm_obj.check()
    test_lib.lib_wait_target_up(vm_obj.get_vm().vmNics[0].ip, 22, 300)

def resize_rvol(vm_obj):
    vol_size = test_lib.lib_get_root_volume(vm_obj.get_vm()).size
    volume_uuid = test_lib.lib_get_root_volume(vm_obj.get_vm()).uuid
    set_size = 1024 * 1024 * 1024 + int(vol_size)
    vol_ops.resize_volume(volume_uuid, set_size)
    vm_obj.update()
    vol_size_after = test_lib.lib_get_root_volume(vm_obj.get_vm()).size
    # if set_size != vol_size_after:
    #     test_util.test_fail('Resize Root Volume failed, size = %s' % vol_size_after)
    # vm_obj.check()
    test_lib.lib_wait_target_up(vm_obj.get_vm().vmNics[0].ip, 22, 300)

    def check_data(self, vm=None):
        vm = vm if vm else self.vm
        vm_ip = vm.get_vm().vmNics[0].ip
        test_lib.lib_wait_target_up(vm_ip, '22', timeout=600)
#         check_cmd = "if [ ! -d /mnt/zstackwoodpecker ];then tar xvf /mnt/zstack-all-in-one.tar -C /mnt > /dev/null 2>&1; fi; \
#                      grep scenario_config_path /mnt/zstackwoodpecker/zstackwoodpecker/test_lib.py > /dev/null 2>&1 && echo 0 || echo 1"
        check_cmd = "grep APIQueryCephPrimaryStorageMsg /mnt/woodpecker/apibinding/inventory.py > /dev/null 2>&1 && echo 0 || echo 1"
        (_, ret, _)= ssh.execute(check_cmd, vm_ip, 'root', 'password')
        ret = ret.split('\n')[0]
        assert ret == '0', "data check failed!, the return code is %s, 0 is expected" % ret
        return self

def test():
    from vncdotool import api
    global vm1
    global vm2

    vm1 = test_stub.create_vm()
    vm1.check()
    console = test_lib.lib_get_vm_console_address(vm1.get_vm().uuid)
    test_util.test_logger('[vm:] %s console is on %s:%s' % (vm1.get_vm().uuid, console.hostIp, console.port))
    display = str(int(console.port)-5900)

    test_lib.lib_set_vm_console_password(vm1.get_vm().uuid, password1)
    test_util.test_logger('set [vm:] %s console with password %s' % (vm1.get_vm().uuid, password1))
    vm1.reboot()

    test_lib.lib_delete_vm_console_password(vm1.get_vm().uuid)
    test_util.test_logger('delete [vm:] %s console password after reboot' % (vm1.get_vm().uuid))
    vm1.reboot()
    if not test_lib.lib_wait_target_up(console.hostIp, console.port, timeout=60):
        test_util.test_fail('[vm:] %s console on %s:%s is not connectable' % (vm1.get_vm().uuid, console.hostIp, console.port))

    try:
        client = api.connect(console.hostIp+":"+display)
        client.keyPress('k')
        test_util.test_logger('[vm:] %s console on %s:%s is connectable without password' % (vm1.get_vm().uuid, console.hostIp, console.port))
    except:
        test_util.test_fail('[vm:] %s console on %s:%s is not connectable without password' % (vm1.get_vm().uuid, console.hostIp, console.port))

    vm2 = test_stub.create_vm()
    vm2.check()
    console = test_lib.lib_get_vm_console_address(vm2.get_vm().uuid)
    test_util.test_logger('[vm:] %s console is on %s:%s' % (vm2.get_vm().uuid, console.hostIp, console.port))
    display = str(int(console.port)-5900)

    test_lib.lib_set_vm_console_password(vm2.get_vm().uuid, password1)
    test_util.test_logger('set [vm:] %s console with password %s' % (vm2.get_vm().uuid, password1))
    test_lib.lib_delete_vm_console_password(vm2.get_vm().uuid)
    test_util.test_logger('delete [vm:] %s console password without reboot' % (vm2.get_vm().uuid))
    vm1.reboot()
    if not test_lib.lib_wait_target_up(console.hostIp, console.port, timeout=60):
        test_util.test_fail('[vm:] %s console on %s:%s is not connectable' % (vm2.get_vm().uuid, console.hostIp, console.port))

    try:
        client = api.connect(console.hostIp+":"+display)
        client.keyPress('k')
        test_util.test_logger('[vm:] %s console on %s:%s is connectable without password' % (vm2.get_vm().uuid, console.hostIp, console.port))
    except:
        test_util.test_fail('[vm:] %s console on %s:%s is not connectable without password' % (vm2.get_vm().uuid, console.hostIp, console.port))

    vm1.destroy()
    vm2.destroy()

    test_util.test_pass('Delete VM Console Password Test Success')

def migrate(vm_obj):
    ps = test_lib.lib_get_primary_storage_by_vm(vm_obj.get_vm())
    if vm_obj.vm.state == "Running" and ps.type in [inventory.CEPH_PRIMARY_STORAGE_TYPE, 'SharedMountPoint', inventory.NFS_PRIMARY_STORAGE_TYPE,
                   'SharedBlock', inventory.LOCAL_STORAGE_TYPE]:
        target_host = test_lib.lib_find_random_host(vm_obj.vm)
        vm_obj.migrate(target_host.uuid)
    elif ps.type in [inventory.LOCAL_STORAGE_TYPE]:
        vm_obj.check()
        target_host = test_lib.lib_find_random_host(vm_obj.vm)
        vol_ops.migrate_volume(vm_obj.get_vm().allVolumes[0].uuid, target_host.uuid)
        vm_obj.start()
        test_lib.lib_wait_target_up(vm_obj.get_vm().vmNics[0].ip, 22, 300)
    else:
        test_util.test_fail("FOUND NEW STORAGTE TYPE. FAILED")

def recover_host_vlan(host_vm, scenarioConfig, deploy_config):
    host_ip = host_vm.ip_
    test_lib.lib_wait_target_up(host_ip, '22', 120)
    host_config = sce_ops.get_scenario_config_vm(host_vm.name_,scenarioConfig)
    for l3network in xmlobject.safe_list(host_config.l3Networks.l3Network):
        if hasattr(l3network, 'l2NetworkRef'):
            for l2networkref in xmlobject.safe_list(l3network.l2NetworkRef):
                nic_name = sce_ops.get_ref_l2_nic_name(l2networkref.text_, deploy_config)
                if nic_name.find('.') >= 0 :
                    vlan = nic_name.split('.')[1]
                    test_util.test_logger('[vm:] %s %s is created.' % (host_ip, nic_name.replace("eth", "zsn")))
                    cmd = 'vconfig add %s %s' % (nic_name.split('.')[0].replace("eth", "zsn"), vlan)
                    test_lib.lib_execute_ssh_cmd(host_ip, host_config.imageUsername_, host_config.imagePassword_, cmd)
    return True

def test():
    global iso
    global test_obj_dict

    # run condition
    allow_bs_list = [inventory.IMAGE_STORE_BACKUP_STORAGE_TYPE, inventory.CEPH_BACKUP_STORAGE_TYPE]
    test_lib.skip_test_when_bs_type_not_in_list(allow_bs_list)

    hosts = res_ops.query_resource(res_ops.HOST)
    if len(hosts) <= 1:
        test_util.test_skip("skip for host_num is not satisfy condition host_num>1")

    # add iso and create vm from iso
    iso = test_stub.add_test_minimal_iso('minimal_iso')
    test_obj_dict.add_image(iso)
    root_volume_offering = test_stub.add_test_root_volume_offering('root-disk-iso', 10737418240)
    test_obj_dict.add_disk_offering(root_volume_offering)
    vm_offering = test_stub.add_test_vm_offering(2, 1024*1024*1024, 'iso-vm-offering')
    test_obj_dict.add_instance_offering(vm_offering)
    vm = test_stub.create_vm_with_iso_for_test(vm_offering.uuid, iso.image.uuid, root_volume_offering.uuid, 'iso-vm')
    test_obj_dict.add_vm(vm)

    # check vm
    vm_inv = vm.get_vm()
    test_lib.lib_set_vm_host_l2_ip(vm_inv)
    test_lib.lib_wait_target_up(vm.get_vm().vmNics[0].ip, 22, 1800)

    # clone vm
    cloned_vm_name = ['cloned_vm']
    cloned_vm_obj = vm.clone(cloned_vm_name)[0]
    test_obj_dict.add_vm(cloned_vm_obj)

    # delete iso
    iso.delete()

    # vm ops test
    test_stub.vm_ops_test(cloned_vm_obj, "VM_TEST_RESIZE_RVOL")

    # expunge iso
    iso.expunge()

    #detach iso
    img_ops.detach_iso(vm.vm.uuid)

    # vm ops test
    test_stub.vm_ops_test(cloned_vm_obj, "VM_TEST_STATE")

    test_lib.lib_robot_cleanup(test_obj_dict)
    test_util.test_pass('Cloned VM ops for BS Success')

def test():

    test_util.test_dsc("create vpc vrouter")

    vr = test_stub.create_vpc_vrouter()

    test_util.test_dsc("attach vpc l3 to vpc vrouter")
    test_stub.attach_l3_to_vpc_vr(vr, test_stub.L3_SYSTEM_NAME_LIST)

    test_util.test_dsc("Create one neverstop vm in random L3")
    vm = test_stub.create_vm_with_random_offering(vm_name='vpc_vm1', l3_name=random.choice(test_stub.L3_SYSTEM_NAME_LIST))
    test_obj_dict.add_vm(vm)
    vm.check()

    host_list = res_ops.query_resource(res_ops.HOST, [], None)
    for host in host_list:
        host_ops.change_host_state(host.uuid, "maintain")
    
    time.sleep(30)
    assert vr.state == 'Stopped'
    assert vm.state == 'Stopped'
 
    for host in host_list:
        host_ops.change_host_state(host.uuid, "enable")

    test_stub.ensure_hosts_connected()
    test_stub.ensure_pss_connected()

    vm.start()
    assert vr.state == 'Running'

    ha_ops.set_vm_instance_ha_level(vm.get_vm().uuid, "NeverStop")
    for host in host_list:
        host_ops.change_host_state(host.uuid, "maintain")
    assert vr.state == 'Stopped'
    assert vm.state == 'Stopped'

    for host in host_list:
        host_ops.change_host_state(host.uuid, "enable")

    test_stub.ensure_hosts_connected()
    test_stub.ensure_pss_connected()
    
    test_lib.lib_wait_target_up(vm.vmNics[0].ip, '22', 120)
    assert vr.state == 'Running'
    
    
    test_lib.lib_error_cleanup(test_obj_dict)
    test_stub.remove_all_vpc_vrouter()

def test():
    global test_obj_dict
    global ps_uuid
    global host_uuid
    global vr_uuid
    test_util.test_dsc('Create test vm and check')
    l3_1_name = os.environ.get('l3VlanNetworkName1')
    vm = test_stub.create_vlan_vm(l3_name=l3_1_name)
    #l3_1 = test_lib.lib_get_l3_by_name(l3_1_name)
    #vr = test_lib.lib_find_vr_by_l3_uuid(l3_1.uuid)[0]
    #vr_uuid = vr.uuid
    
    host = test_lib.lib_get_vm_host(vm.get_vm())
    host_uuid = host.uuid
    test_obj_dict.add_vm(vm)
    vm.check()
    #ps = test_lib.lib_get_primary_storage_by_vm(vm.get_vm())
    #ps_uuid = ps.uuid
    #ps_ops.change_primary_storage_state(ps_uuid, 'maintain')
    test_stub.maintain_all_pss()
    if not test_lib.lib_wait_target_down(vm.get_vm().vmNics[0].ip, '22', 90):
        test_util.test_fail('VM is expected to stop when PS change to maintain state')

    vm.set_state(vm_header.STOPPED)
    vm.check()
    #ps_ops.change_primary_storage_state(ps_uuid, 'enable')
    test_stub.enable_all_pss()
    host_ops.reconnect_host(host_uuid)
    #vm_ops.reconnect_vr(vr_uuid)
    test_stub.ensure_pss_connected()
    vrs = test_lib.lib_get_all_vrs()
    for vr in vrs:
        vr_cond = res_ops.gen_query_conditions('uuid', '=', vr.uuid)
        vr_inv = res_ops.query_resource(res_ops.VM_INSTANCE, vr_cond)[0]
        if vr_inv.state == 'Stopped':
            vm_ops.start_vm(vr.uuid)
        else:
            test_lib.lib_wait_target_up(vr_inv.vmNics[0].ip, '22', 360)
            for _ in xrange(100):
                if res_ops.query_resource(res_ops.VM_INSTANCE, vr_cond)[0].state != 'Running':
                    time.sleep(3)
                else:
                    break

    vm.start()
    vm.check()
    vm.destroy()
    test_util.test_pass('PS maintain mode Test Success')

 def _copy_sshkey_from_node(self):
     node = self.nodes[0]
     if not node.dockerImage__:
         print 'Copy sshkey from mn node'
         #consider some zstack-server is running in vm, the server 
         # startup speed is slow. Increase timeout to 180s.
         private_key_path = '%s/webapps/zstack/WEB-INF/classes/ansible/rsaKeys/id_rsa' % self.catalina_home
         public_key_path = '%s/webapps/zstack/WEB-INF/classes/ansible/rsaKeys/id_rsa.pub' % self.catalina_home
         if linux.is_ip_existing(node.ip_):
             cmd = 'scp %s /root/.ssh/id_rsa' % (private_key_path)
             shell.call(cmd)
             cmd = 'scp %s /root/.ssh/id_rsa.pub' % (public_key_path)
             shell.call(cmd)
         elif not linux.is_ip_existing(node.ip_):
             import zstackwoodpecker.test_lib as test_lib
             if test_lib.lib_wait_target_up(node.ip_, '22', 120):
                 node_ip = node.ip_
             else:
                 node_ip = os.environ['zstackHaVip']
             ssh.scp_file(private_key_path, "/root/.ssh/id_rsa", node_ip, node.username_, node.password_)
             ssh.scp_file(public_key_path, "/root/.ssh/id_rsa.pub", node_ip, node.username_, node.password_)
             cmd = 'scp %s /root/.ssh/id_rsa' % (private_key_path)
             shell.call(cmd)
             cmd = 'scp %s /root/.ssh/id_rsa.pub' % (public_key_path)
             shell.call(cmd)
         else:
             cmd = 'scp %s /root/.ssh/id_rsa' % (private_key_path)
             shell.call(cmd)
             cmd = 'scp %s /root/.ssh/id_rsa.pub' % (public_key_path)
             shell.call(cmd)

def test():
    global test_obj_dict
    global ps_uuid
    global host_uuid
    global vr_uuid
    allow_ps_list = [inventory.CEPH_PRIMARY_STORAGE_TYPE, "SharedBlock"]
    test_lib.skip_test_when_ps_type_not_in_list(allow_ps_list)

    test_util.test_dsc('Create test vm and check')

    bs_cond = res_ops.gen_query_conditions("status", '=', "Connected")
    bss = res_ops.query_resource_fields(res_ops.BACKUP_STORAGE, bs_cond, \
            None)
    if not bss:
        test_util.test_skip("not find available backup storage. Skip test")

    test_lib.lib_set_delete_policy('vm', 'Delay')
    test_lib.lib_set_delete_policy('volume', 'Delay')
    l3_1_name = os.environ.get('l3VlanNetworkName1')
    vm = test_stub.create_vlan_vm(l3_name=l3_1_name)
    l3_1 = test_lib.lib_get_l3_by_name(l3_1_name)
    vr = test_lib.lib_find_vr_by_l3_uuid(l3_1.uuid)[0]
    vr_uuid = vr.uuid
    
    host = test_lib.lib_get_vm_host(vm.get_vm())
    host_uuid = host.uuid
    test_obj_dict.add_vm(vm)
    vm.check()

    disk_offering = test_lib.lib_get_disk_offering_by_name(os.environ.get('rootDiskOfferingName'))
    volume_creation_option = test_util.VolumeOption()
    volume_creation_option.set_disk_offering_uuid(disk_offering.uuid)
    volume_creation_option.set_system_tags(['ephemeral::shareable', 'capability::virtio-scsi'])
    volume = test_stub.create_volume(volume_creation_option)
    test_obj_dict.add_volume(volume)
    volume.check()
    volume.attach(vm)

    #ps = test_lib.lib_get_primary_storage_by_vm(vm.get_vm())
    #ps_uuid = ps.uuid
    #ps_ops.change_primary_storage_state(ps_uuid, 'disable')
    test_stub.disable_all_pss()
    if not test_lib.lib_wait_target_up(vm.get_vm().vmNics[0].ip, '22', 90):
        test_util.test_fail('VM is expected to running when PS change to disable state')

    vm.set_state(vm_header.RUNNING)
    vm.check()
    volume.delete()
    volume.check()
    volume.expunge()
    volume.check()

    #ps_ops.change_primary_storage_state(ps_uuid, 'enable')
    test_stub.enable_all_pss()
    host_ops.reconnect_host(host_uuid)
    vm_ops.reconnect_vr(vr_uuid)
    vm.destroy()
    test_lib.lib_set_delete_policy('vm', 'Direct')
    test_lib.lib_set_delete_policy('volume', 'Direct')
    test_util.test_pass('Delete volume under PS disable mode Test Success')

def test():
    global session_uuid
    session_uuid = acc_ops.login_as_admin()
    l3_1_name = os.environ.get('l3VlanNetworkName1')
    l3_1 = test_lib.lib_get_l3_by_name(l3_1_name)

    #create VRs.
    vrs = test_lib.lib_find_vr_by_l3_uuid(l3_1.uuid)
    if not vrs:
        vm = test_stub.create_vlan_vm(l3_name=l3_1_name)
        vm.destroy()
        vr1 = test_lib.lib_find_vr_by_l3_uuid(l3_1.uuid)[0]
    else:
        vr1 = vrs[0]

    vm_ops.stop_vm(vr1.uuid)
    if test_lib.lib_get_ha_enable() != 'true' or vr1.applianceVmType != "vrouter":
        vm_ops.start_vm(vr1.uuid)

    vr1_mgmt_ip = test_lib.lib_find_vr_mgmt_ip(vr1)
    if not test_lib.lib_wait_target_up(vr1_mgmt_ip, '7272', 120):
        test_util.test_fail('vr: %s is not startup in 120 seconds. Fail to reboot it. ' % vr1.uuid)

    timeout = time.time() + 60
    while time.time() < timeout:
        cond = res_ops.gen_query_conditions('uuid', '=', vr1.uuid) 
        vms = res_ops.query_resource(res_ops.VM_INSTANCE, cond)
        if vms[0].status == 'Connected':
            test_util.test_pass('Test Reconnect VR VM start VRs success')
        time.sleep(1)

    test_util.test_fail('VR VM: %s is not connected within 60 seconds' % vr1.uuid)

def test():
    global test_obj_dict
    global ps_uuid
    global host_uuid
    global vr_uuid
    test_util.test_dsc('Create test vm and check')
    l3_1_name = os.environ.get('l3VlanNetworkName1')
    vm = test_stub.create_vlan_vm(l3_name=l3_1_name)
    l3_1 = test_lib.lib_get_l3_by_name(l3_1_name)
    vr = test_lib.lib_find_vr_by_l3_uuid(l3_1.uuid)[0]
    vr_uuid = vr.uuid
    
    l3_1 = test_lib.lib_get_l3_by_name(l3_1_name)
    host = test_lib.lib_get_vm_host(vm.get_vm())
    host_uuid = host.uuid
    test_obj_dict.add_vm(vm)
    vm.check()
    #ps = test_lib.lib_get_primary_storage_by_vm(vm.get_vm())
    #ps_uuid = ps.uuid
    #ps_ops.change_primary_storage_state(ps_uuid, 'disable')
    test_stub.disable_all_pss()
    if not test_lib.lib_wait_target_up(vm.get_vm().vmNics[0].ip, '22', 90):
        test_util.test_fail('VM is expected to runnning when PS change to disable state')

    vm.set_state(vm_header.RUNNING)
    vm.check()
    vm.suspend()
    vm.check()

    #ps_ops.change_primary_storage_state(ps_uuid, 'enable')
    test_stub.enable_all_pss()
    host_ops.reconnect_host(host_uuid)
    vm_ops.reconnect_vr(vr_uuid)
    test_util.test_pass('PS disable mode Test Success')