def add_checking_point(self):
        volume_obj = self.get_target_volume()
        volume = volume_obj.get_volume()
        if volume.type == 'Root':
            test_util.test_logger('Can not add checking point file for Root Volume: %s, since it can not be detached and reattached to utility vm for checking.' % volume.uuid)
            return

        volume_vm = volume_obj.get_target_vm()
        #check if volume has been attached to the living VM.
        if volume_obj.get_state() == volume_header.ATTACHED:
            if volume_vm.get_state() == vm_header.STOPPED or \
                    volume_vm.get_state() == vm_header.RUNNING:
                test_util.test_logger('volume has been attached to living VM.')

                volume_obj.detach()
                volume_obj.attach(self.utility_vm)
                #add checking point
                self._create_checking_file()
                volume_obj.detach()
                volume_obj.attach(volume_vm)
                return 
        volume_obj.attach(self.utility_vm)
        #add_checking_point
        self._create_checking_file()
        volume_obj.detach()

def vm_op_test(vm, op):
    test_util.test_logger(vm.vm.name + "-------" + op)
    ops = {
        "VM_TEST_STOP": stop,
        "VM_TEST_REBOOT": reboot,
        "VM_TEST_NONE": do_nothing,
        "VM_TEST_MIGRATE": migrate,
        "VM_TEST_SNAPSHOT": create_snapshot,
        "VM_TEST_CREATE_IMG": create_image,
        "VM_TEST_RESIZE_RVOL": resize_rvol,
        "RVOL_DEL_SNAPSHOT": delete_snapshot,
        "VM_TEST_CHANGE_OS": change_os,
        "VM_TEST_RESET": reset,
        "VM_TEST_BACKUP": back_up,
        "VM_TEST_REVERT_BACKUP": revert_backup,
        "VM_TEST_REVERT_VM_BACKUP": revert_vm_backup,
        "VM_TEST_BACKUP_IMAGE": backup_image, 
        "DVOL_TEST_SNAPSHOT": create_dvol_snapshot,
        "DVOL_DEL_SNAPSHOT": delete_dvol_snapshot,
        "DVOL_TEST_CREATE_IMG": create_dvol_image,
        "DVOL_TEST_RESIZE": resize_dvol,
        "DVOL_BACKUP": dvol_back_up,
        "DVOL_TEST_BACKUP_IMAGE": dvol_backup_image,
        "CREATE_ATTACH_VOLUME": create_attach_volume

    }
    ops[op](vm)

def test():

    global bs_username, bs_hostname, bs_password, bs_name, bs_username, bs_url, bs_sshport
    global new_image    

    file_path = test_stub.gen_license('woodpecker', '[email protected]', '1', 'Prepaid', '1', '')
    test_stub.load_license(file_path)
    issued_date = test_stub.get_license_info().issuedDate
    expired_date = test_stub.license_date_cal(issued_date, 86400 * 1)
    test_stub.check_license("[email protected]", 1, None, False, 'Paid', issued_date=issued_date, expired_date=expired_date)

    test_util.test_logger('create zone and add the bs of the imagestore')
    node_uuid = res_ops.query_resource(res_ops.MANAGEMENT_NODE)[0].uuid
    test_stub.create_zone()
    zone_uuid = res_ops.query_resource(res_ops.ZONE)[0].uuid

    bs_name = 'BS1'
    bs_hostname = os.environ.get('node1Ip')
    bs_username = os.environ.get('nodeUserName')
    bs_password = os.environ.get('nodePassword')
    bs_url = '/zstack_bs'
    bs_sshport = '22'
    test_stub.create_image_store_backup_storage(bs_name, bs_hostname, bs_username, bs_password, bs_url, bs_sshport)
    bs_uuid = res_ops.query_resource(res_ops.BACKUP_STORAGE)[0].uuid

    test_stub.reload_default_license()
    test_util.test_logger('Check default community license')
    #test_stub.check_license(None, None, 2147483647, False, 'Community')

    try:
        bs_ops.reconnect_backup_storage(bs_uuid)
    except Exception, e:
        if "commercial" in str(e):
            test_util.test_pass('test reconnect bs failed, An operation failed, details: commercial license is required to use ImageStore.')

def test():
    global vm
    global vip_s_vm_cfg_lst

    vip_s_vm_cfg_lst = test_stub.get_s_vm_cfg_lst_vip_bind(test_lib.all_scenario_config, test_lib.scenario_file)
    if len(vip_s_vm_cfg_lst) != 1:
        test_util.test_fail('vip has been running on %d host(s)' % len(vip_s_vm_cfg_lst))

    test_util.test_logger("disconnect host [%s]" % (vip_s_vm_cfg_lst[0].ip_))
    #test_stub.down_host_network(vip_s_vm_cfg_lst[0].ip_, test_lib.all_scenario_config)  
    test_stub.exec_zsha2_demote(vip_s_vm_cfg_lst[0].ip_, "root", "password")

    time.sleep(5)

    expected_vip_s_vm_cfg_lst_ip = test_stub.get_expected_vip_s_vm_cfg_lst_after_switch(test_lib.all_scenario_config, test_lib.scenario_file, vip_s_vm_cfg_lst[0].ip_)
    if not test_stub.check_if_vip_is_on_host(test_lib.all_scenario_config, test_lib.scenario_file, expected_vip_s_vm_cfg_lst_ip):
        test_util.test_fail("find vip should drift on ip %s, but is not on it." %(expected_vip_s_vm_cfg_lst_ip))

    vip_s_vm_cfg_lst_new = test_stub.get_s_vm_cfg_lst_vip_bind(test_lib.all_scenario_config, test_lib.scenario_file)
    if len(vip_s_vm_cfg_lst_new) != 1:
        test_util.test_fail('vip has been running on %d host(s)' % len(vip_s_vm_cfg_lst_new))

    test_stub.wrapper_of_wait_for_management_server_start(600)

    test_stub.ensure_hosts_connected(exclude_host=[vip_s_vm_cfg_lst[0]])
    test_stub.ensure_bss_connected()
    test_stub.ensure_pss_connected()

    vm = test_stub.create_basic_vm()
    vm.check()
    vm.destroy()

    test_util.test_pass('Create VM Test Success')

def deploy_initial_database(deploy_config):
    operations = [
            add_backup_storage,
            add_zone,
            add_l2_network,
            add_primary_storage,
            add_cluster,
            add_host,
            add_l3_network,
            add_image,
            add_disk_offering,
            add_instance_offering,
            add_virtual_router
            ]
    for operation in operations:
        session_uuid = account_operations.login_as_admin()
        try:
            operation(deploy_config, session_uuid)
        except Exception as e:
            test_util.test_logger('[Error] zstack deployment meets exception when doing: %s . The real exception are:.' % operation.__name__)
            print('----------------------Exception Reason------------------------')
            traceback.print_exc(file=sys.stdout)
            print('-------------------------Reason End---------------------------\n')
            raise e
        finally:
            account_operations.logout(session_uuid)

    test_util.test_logger('[Done] zstack initial database was created successfully.')

def test():
    h1_name = os.environ.get("hostName")
    cond = res_ops.gen_query_conditions('name', '=', h1_name)
    h1 = res_ops.query_resource(res_ops.HOST, cond)
    ag1 = ag_ops.create_affinity_group(name="ag1", policy="antiHard")
    vm1 = test_stub.create_ag_vm(host_uuid=h1[0].uuid)
    assert vm1.get_vm().hostUuid == h1[0].uuid
    test_obj_dict.add_vm(vm1)

    new_vm = vm1.clone(names=["clone-vm1", "clone-vm2", "clone-vm3"], systemtag=["affinityGroupUuid::%s" % ag1.uuid])
    test_obj_dict.add_vm(new_vm[0])
    test_obj_dict.add_vm(new_vm[1])
    test_obj_dict.add_vm(new_vm[2])
    vmuuids = []
    ag = test_lib.lib_get_affinity_group_by_name(name="ag1")
    for usage in ag.usages:
        vmuuids.append(usage.resourceUuid)
    assert new_vm[0].get_vm().uuid in vmuuids
    assert new_vm[1].get_vm().uuid in vmuuids
    assert new_vm[2].get_vm().uuid in vmuuids
    assert len(vmuuids) == 3
    
    try:
        ag_ops.add_vm_to_affinity_group(ag1.uuid, vm1.get_vm().uuid) 
    except:
        test_util.test_logger("vm1 is not expected to add into affinity group [uuid: %s]" % ag1.uuid)
    vmuuids = []
    ag = test_lib.lib_get_affinity_group_by_name(name="ag1")
    for usage in ag.usages:
        vmuuids.append(usage.resourceUuid)
    assert vm1.get_vm().uuid not in vmuuids
 
    test_lib.lib_error_cleanup(test_obj_dict)
    ag_ops.delete_affinity_group(ag1.uuid)
    test_util.test_pass("Affinity Group antiHard policy pass")

def test():
    test_util.test_dsc('''
    Test Description:
        Will create 1 VM with 3 l3 networks. 1 l3_network is not using VR; 1 l3_network is using novlan VR; 1 l3_network is using vlan VR. 
    Resource required:
        Need support 3 VMs (1 test VM + 2 VR VMs) existing at the same time. 
        This test required a special image, which was configed with at least 3 enabled NICs (e.g. eth0, eth1, eth2).
    ''')
    image_name = os.environ.get('imageName_net')
    image_uuid = test_lib.lib_get_image_by_name(image_name).uuid
    l3_name = os.environ.get('l3VlanNetworkName1')
    l3_net_uuid = test_lib.lib_get_l3_by_name(l3_name).uuid
    l3_net_list = [l3_net_uuid]
    l3_name = os.environ.get('l3VlanNetworkName3')
    l3_net_uuid = test_lib.lib_get_l3_by_name(l3_name).uuid
    l3_net_list.append(l3_net_uuid)
    l3_name = os.environ.get('l3VlanNetworkName4')
    l3_net_uuid = test_lib.lib_get_l3_by_name(l3_name).uuid
    l3_net_list.append(l3_net_uuid)

    vm = test_stub.create_vm(l3_net_list, image_uuid, '3_l3_vm')
    test_obj_dict.add_vm(vm)
    vm.check()

    if len(vm.vm.vmNics) == 3:
        test_util.test_logger("Find 3 expected Nics in new created VM.")
    else:
        test_util.test_fail("New create VM doesn't not have 3 Nics. It only have %s" % len(vm.get_vm().vmNics))

    vm.destroy()
    test_util.test_pass('Create 1 VM with 3 l3_network (1 vlan VR, 1 novlan VR and 1 no VR L3network) successfully.')

    def create_checker(self, test_obj): 
        kvm_volume_checker_chain = checker_header.CheckerChain()
        checker_dict = {}
        if test_obj.state == volume_header.CREATED:
            checker_dict[db_checker.zstack_volume_db_checker] = True
            checker_dict[volume_checker.zstack_kvm_volume_file_checker] = False

        elif test_obj.state == volume_header.ATTACHED:
            checker_dict[db_checker.zstack_volume_db_checker] = True
            checker_dict[volume_checker.zstack_kvm_volume_file_checker] = True
            if not test_obj.target_vm.state == vm_header.DESTROYED:
                checker_dict[db_checker.zstack_volume_attach_db_checker] = True
                if test_obj.target_vm.state == vm_header.RUNNING:
                    checker_dict[volume_checker.zstack_kvm_volume_attach_checker] = True
            else:
                checker_dict[db_checker.zstack_volume_attach_db_checker] = False

        elif test_obj.state == volume_header.DETACHED:
            checker_dict[db_checker.zstack_volume_db_checker] = True
            checker_dict[db_checker.zstack_volume_attach_db_checker] = False
            checker_dict[volume_checker.zstack_kvm_volume_attach_checker] = False
            checker_dict[volume_checker.zstack_kvm_volume_file_checker] = True

        elif test_obj.state == volume_header.DELETED:
            test_util.test_logger('volume has been deleted: %s' % test_obj.volume.uuid)
            checker_dict[db_checker.zstack_volume_db_checker] = False
            checker_dict[volume_checker.zstack_kvm_volume_file_checker] = False

        kvm_volume_checker_chain.add_checker_dict(checker_dict, test_obj)
        return kvm_volume_checker_chain

def revert_vm_from_backup(group_uuid, session_uuid=None):
    action = api_actions.RevertVmFromVmBackupAction()
    action.groupUuid = group_uuid
    action.timeout = 1800000
    evt = account_operations.execute_action_with_session(action, session_uuid) 
    test_util.test_logger('Revert [volume_uuid:] %s ' %  group_uuid)
    return evt.inventory

    def check_nfs(self, volume, volume_installPath):
        host = test_lib.lib_get_volume_object_host(self.test_obj)
        if not host:
            test_util.test_logger('Check result: can not find Host, who is belonged to same Zone Uuid of [volume uuid: ] %s. Can not check volume file existence' % volume.uuid)
            return self.judge(False)

        self.check_file_exist(volume, volume_installPath, host)

def dump_zstack_deployment_config(deployConfig = None):
    '''
    deployConfig is the original zstack config. We need this conifg to set 
    username/password, as they are not get from ZStack API

    will return an xmlobject
    '''
    if not deployConfig:
        deployConfig = xmlobject.XmlObject('fake')

    root_xml = etree.Element("deployerConfig")
    session_uuid = account_operations.login_as_admin()
    try:
        add_nodes_config(root_xml, deployConfig.nodes__, session_uuid)
        add_sftp_backup_stroage_config(root_xml, \
                deployConfig.backupStorages__, session_uuid)
        add_instance_offering_config(root_xml, session_uuid)
        add_disk_offering_config(root_xml, session_uuid)
        add_image_config(root_xml, deployConfig.images__, session_uuid)
        add_zone_config(root_xml, deployConfig.zones, session_uuid)
    except Exception as e:
        test_util.test_logger('[Error] export zstack deployment configuration meets exception.')
        traceback.print_exc(file=sys.stdout)
        raise e
    finally:
        account_operations.logout(session_uuid)

    return root_xml

def check_detach_l2(pre_cluster_uuid, l2_uuid, vm, is_other_cluster):
    l2 = res_ops.get_resource(res_ops.L2_NETWORK, uuid = l2_uuid)[0]

    attached_clusters = l2.attachedClusterUuids

    if pre_cluster_uuid in attached_clusters:
        test_util.test_fail('[cluster:] %s is still in [l2:] %s attached list.'\
                % (pre_cluster_uuid, l2_uuid))

    test_util.test_dsc('start vm again. vm should be started in different cluster, if there has.')
    if attached_clusters :
        if not is_other_cluster:
            test_util.test_fail('There should not be available cluster for [l2:] %s. But catch some.' % l2_uuid)

        vm.start()
        new_cluster_uuid = vm.get_vm().clusterUuid
        if new_cluster_uuid == pre_cluster_uuid : 
            test_util.test_fail('\
            VM start on old [cluster]: %s, which is detached by [l2:] %s ' \
                    % (vm.get_vm().uuid, new_cluster_uuid, l2_uuid))
        vm.check()
    else:
        if is_other_cluster:
            test_util.test_fail('There should be available cluster for [l2:] %s. But did not catch.' % l2_uuid)
        #no cluster is attached with l2. vm will start failure.
        try:
            vm.start()
        except:
            test_util.test_logger('\
Expected: VM start failed, since there is not cluster is attached to [l2]: %s, \
after [cluster:] %s is detached' % (l2_uuid, pre_cluster_uuid))
        else:
            test_util.test_fail('[vm]: %s is Wrongly started up, since there is\
not cluster is attached with [l2]: %s, after previous detaching ops' % \
                (vm.get_vm().uuid, l2_uuid))

    def check(self):
        super(zstack_kvm_vm_snat_checker, self).check()
        vm = self.test_obj.vm
        test_lib.lib_install_testagent_to_vr(vm)
        host = test_lib.lib_get_vm_host(vm)

        vm_cmd_result = None
        vr_vms = test_lib.lib_find_vr_by_vm(vm)
        test_lib.lib_set_vm_host_l2_ip(vm)
        for vr_vm in vr_vms:
            test_util.test_logger("Begin to check [vm:] %s SNAT" % vm.uuid)
            nic = test_lib.lib_get_vm_nic_by_vr(vm, vr_vm)
            if not 'SNAT' in test_lib.lib_get_l3_service_type(nic.l3NetworkUuid):
                test_util.test_logger("Skip [VR:] %s, since it doesn't provide SNAT service" % vr_vm.uuid)
                continue

            ping_target = test_lib.test_config.pingTestTarget.text_
            #Check if there is a SG rule to block ICMP checking
            if test_lib.lib_is_sg_rule_exist(nic.uuid, None, None, inventory.EGRESS):
                if not test_lib.lib_is_sg_rule_exist(nic.uuid, inventory.ICMP, ping_target, inventory.EGRESS):
                    test_util.test_warn('Skip SNAT checker: because there is ICMP Egress Rule was assigned to [nic:] %s and the allowed target ip is not %s' % (nic.uuid, ping_target))
                    return self.judge(self.exp_result)

            guest_ip = nic.ip
            vm_command = 'ping -c 5 -W 5 %s >/tmp/ping_result 2>&1; ret=$?; cat /tmp/ping_result; exit $ret' % ping_target
            vm_cmd_result = test_lib.lib_ssh_vm_cmd_by_agent_with_retry(host.managementIp, nic.ip, test_lib.lib_get_vm_username(vm), test_lib.lib_get_vm_password(vm), vm_command, self.exp_result)
            if not vm_cmd_result:
                test_util.test_logger('Checker result: FAIL to ping [target:] %s from [vm:] %s .' % (ping_target, vm.uuid))
                if self.exp_result == True:
                    test_util.test_logger("network connection result is not expected pass, will print VR's network configuration:")
                    test_lib.lib_print_vr_network_conf(vr_vm)
                return self.judge(False)
            else:
                test_util.test_logger('Checker result: SUCCEED to ping [target:] %s from [vm:] %s .' % (ping_target, vm.uuid))
                return self.judge(True)

    def _create_checking_file(self):
        #make fs for volume, if it doesn't exist
        if not self.parent and not self.child_list:
            test_lib.lib_mkfs_for_volume(self.target_volume.get_volume().uuid, \
                    self.utility_vm.get_vm())

        import tempfile
        with tempfile.NamedTemporaryFile() as script:
            script.write('''
device=/dev/`ls -ltr --file-type /dev | grep disk | awk '{print $NF}' | grep -v '[[:digit:]]' | tail -1`1
mkdir -p %s
mount $device %s
mkdir -p %s
touch %s/%s
umount %s
            ''' % (test_lib.WOODPECKER_MOUNT_POINT, \
                    test_lib.WOODPECKER_MOUNT_POINT, \
                    checking_point_folder, checking_point_folder, \
                    self.checking_point, test_lib.WOODPECKER_MOUNT_POINT))
            script.flush()
            test_lib.lib_execute_shell_script_in_vm(self.utility_vm.get_vm(),
                    script.name)

        if self.parent:
            test_util.test_logger('[snapshot:] %s checking file: %s is created.\
Its [parent:] %s' % \
                    (self.snapshot_option.get_name(), \
                        self.checking_point, self.parent.get_snapshot().uuid))
        else:
            test_util.test_logger('[snapshot:] %s checking file: %s is created.'% (self.snapshot_option.get_name(), self.checking_point))

def test_scp_vm_inbound_speed(vm_inv, bandwidth):
    '''
    bandwidth unit is KB
    '''
    timeout = TEST_TIME + 30
    vm_ip = vm_inv.vmNics[0].ip
    file_size = bandwidth * TEST_TIME
    seek_size = file_size / 1024 - 1
    cmd = 'dd if=/dev/zero of=%s bs=1M count=1 seek=%d' \
            % (test_file, seek_size)
    os.system(cmd)
    cmd = 'scp -oStrictHostKeyChecking=no -oCheckHostIP=no -oUserKnownHostsFile=/dev/null %s %s:/dev/null' \
            % (test_file, vm_ip)
    start_time = time.time()
    if execute_shell_in_process(cmd, timeout) != 0:
        test_util.test_fail('scp test file failed')

    end_time = time.time()
    os.system('rm -f %s' % test_file)

    scp_time = end_time - start_time
    if scp_time < TEST_TIME:
        test_util.test_fail('network inbound QOS test file failed, since the scp time: %d is smaller than the expected test time: %d. It means the bandwidth limitation: %d KB/s is not effect. ' % (scp_time, TEST_TIME, bandwidth))
    else:
        test_util.test_logger('network inbound QOS test file pass, since the scp time: %d is bigger than the expected test time: %d. It means the bandwidth limitation: %d KB/s is effect. ' % (scp_time, TEST_TIME, bandwidth))

    return True

def get_datacenter(content, name=None):
    from pyVmomi import vim
    dc = get_obj(content, [vim.Datacenter], name=name)
    if isinstance(dc, list):
        test_util.test_logger("do not find datacenter named %s, now return all datacenter" % name)
        return dc
    return [dc]

def test():
    global vm_inv
    test_util.test_dsc('Create test vm to test zstack upgrade by -u.')

    image_name = os.environ.get('imageName_i_c7_z_1.9')
    #iso_path = os.environ.get('iso_path')
    zstack_latest_version = os.environ.get('zstackLatestVersion')
    zstack_latest_path = os.environ.get('zstackLatestInstaller')
    vm_name = os.environ.get('vmName')
    #upgrade_script_path = os.environ.get('upgradeScript')

    vm_inv = test_stub.create_vm_scenario(image_name, vm_name)
    vm_ip = vm_inv.vmNics[0].ip
    test_lib.lib_wait_target_up(vm_ip, 22)

    test_stub.make_ssh_no_password(vm_ip, tmp_file)

    test_util.test_logger('Update MN IP')
    test_stub.update_mn_hostname(vm_ip, tmp_file)
    test_stub.update_mn_ip(vm_ip, tmp_file)
    test_stub.start_mn(vm_ip, tmp_file)
    test_stub.check_installation(vm_ip, tmp_file)

    test_util.test_logger('Upgrade zstack to latest with repo') 
    #test_stub.update_iso(vm_ip, tmp_file, iso_path, upgrade_script_path)
    test_stub.update_repo(vm_ip, tmp_file)
    test_stub.upgrade_zstack(vm_ip, zstack_latest_path, tmp_file) 
    test_stub.check_zstack_version(vm_ip, tmp_file, zstack_latest_version)
    test_stub.start_mn(vm_ip, tmp_file)
    test_stub.check_mn_running(vm_ip, tmp_file)
    test_stub.check_installation(vm_ip, tmp_file)

    os.system('rm -f %s' % tmp_file)
    test_stub.destroy_vm_scenario(vm_inv.uuid)
    test_util.test_pass('ZStack upgrade Test Success')

def get_cluster(content, name=None):
    from pyVmomi import vim
    cluster = get_obj(content, [vim.ClusterComputeResource], name=name)
    if isinstance(cluster, list):
        test_util.test_logger("do not find cluster named %s, now return all cluster" % name)
        return cluster
    return [cluster]

def detach_l3(nic_uuid, session_uuid = None):
    action = api_actions.DetachL3NetworkFromVmAction()
    action.vmNicUuid = nic_uuid
    test_util.action_logger('[Detach L3 Network Nic]: %s' % nic_uuid)
    evt = acc_ops.execute_action_with_session(action, session_uuid)
    test_util.test_logger('[L3 Network Nic]: %s has been detached'% nic_uuid)
    return evt.inventory

def get_host(content, name=None):
    from pyVmomi import vim
    host = get_obj(content, [vim.HostSystem], name=name)
    if isinstance(host, list):
        test_util.test_logger("do not find host named %s, now return all host" % name)
        return host
    return [host]