def test():
    '''
    Test Description:
        Will check if test VM's mac/ip assignment will be removed in VR's DNS config, after VM is shutdown. And if VM's mac/ip will be added back after it restart. 
    Resource required:
        Need support 2 VMs. 
    '''
    global test_obj_dict
    test_util.test_dsc('Create test vm and check. VR only has DNS and DHCP services')
    vm = test_stub.create_vlan_vm()
    test_obj_dict.add_vm(vm)

    vm.check()
    vm.stop()
    if linux.wait_callback_success(check_dhcp_remove, vm.vm, 5, 0.2):
        test_util.test_logger('[vm:] %s mac/ip was removed in VR /etc/hosts.dhcp' % vm.vm.uuid)
    else:
        test_util.test_logger('check dhcp remove failure, will try to print VR DHCP tables')
        test_lib.lib_print_vr_dhcp_tables(test_lib.lib_find_vr_by_vm(vm.vm)[0])
        test_util.test_fail('[vm:] %s mac was not removed in VR /etc/hosts.dhcp, after vm was stopped.' % vm.vm.uuid)
    if linux.wait_callback_success(check_dhcp_release, vm.vm, 5, 0.2):
        test_util.test_logger('[vm:] %s mac/ip was removed in VR /etc/hosts.leases' % vm.vm.uuid)
    else:
        test_util.test_logger('check dhcp lease failure, will try to print VR DHCP tables')
        test_lib.lib_print_vr_dhcp_tables(test_lib.lib_find_vr_by_vm(vm.vm)[0])
        test_util.test_fail('[vm:] %s mac was not removed in VR /etc/hosts.leases, after vm was stopped.' % vm.vm.uuid)

    vm.start()
    vm.check()
    vm.destroy()
    vm.check()
    test_util.test_pass('Successfully check VM DHCP release when VM stopped')

 def save_pid():
     linux.wait_callback_success(os.path.exists, v2v_pid_path)
     with open(v2v_pid_path, 'r') as fd:
         new_task.current_pid = fd.read().strip()
     new_task.current_process_cmd = echo_pid_cmd
     new_task.current_process_name = "virt_v2v_cmd"
     logger.debug("longjob[uuid:%s] saved process[pid:%s, name:%s]" %
                  (cmd.longJobUuid, new_task.current_pid, new_task.current_process_name))

    def stop(self, graceful=True, timeout=5, undefine=True):
        def cleanup_addons():
            for chan in self.domain_xmlobject.devices.get_child_node_as_list('channel'):
                if chan.type_ == 'unix':
                    path = chan.source.path_
                    shell.call('rm -f %s' % path)

        def loop_shutdown(_):
            try:
                self.domain.shutdown()
            except:
                #domain has been shutdown
                pass

            return self.wait_for_state_change(self.VM_STATE_SHUTDOWN)
        
        def loop_undefine(_):
            if not undefine:
                return True

            if not self.is_alive():
                return True

            try:
                self.domain.undefineFlags(libvirt.VIR_DOMAIN_UNDEFINE_MANAGED_SAVE|libvirt.VIR_DOMAIN_UNDEFINE_SNAPSHOTS_METADATA)
            except libvirt.libvirtError:
                self.domain.undefine()

            return self.wait_for_state_change(None)

        def loop_destroy(_):
            try:
                self.domain.destroy()
            except:
                #domain has been destroyed
                pass

            return self.wait_for_state_change(self.VM_STATE_SHUTDOWN)

        do_destroy = True
        if graceful:
            if linux.wait_callback_success(loop_shutdown, None, timeout=60):
                do_destroy = False

        if do_destroy:
            if not linux.wait_callback_success(loop_destroy, None, timeout=60):
                raise kvmagent.KvmError('failed to destroy vm, timeout after 60 secs')

        cleanup_addons()
        if not linux.wait_callback_success(loop_undefine, None, timeout=60):
            raise kvmagent.KvmError('failed to undefine vm, timeout after 60 secs')

def test():
    global host
    if test_lib.lib_get_active_host_number() < 2:
        test_util.test_fail('Not available host to do maintenance, since there are not 2 hosts')

    vm_cpu = 1
    vm_memory = 1073741824 #1G
    cond = res_ops.gen_query_conditions('name', '=', 'ttylinux')
    image_uuid = res_ops.query_resource(res_ops.IMAGE, cond)[0].uuid
    l3_network_uuid = res_ops.query_resource(res_ops.L3_NETWORK)[0].uuid
    vm = test_stub.create_mini_vm([l3_network_uuid], image_uuid, cpu_num = vm_cpu, memory_size = vm_memory)
    test_obj_dict.add_vm(vm)

    host_uuid = test_lib.lib_get_vm_host(vm.vm).uuid
    host_ops.change_host_state(host_uuid, 'maintain')

    #need to update vm's inventory, since they will be changed by maintenace mode
    vm.update()
    vm.set_state(vm_header.STOPPED)

    vm.check()

    host_ops.change_host_state(host_uuid, 'enable')
    if not linux.wait_callback_success(is_host_connected, host_uuid, 120):
        test_util.test_fail('host status is not changed to connected, after changing its state to Enable')

    vm.start()
    vm.check()
    vm.destroy()
    test_obj_dict.rm_vm(vm)
    test_util.test_pass('Maintain Host Test Success')

    def _restart_dnsmasq(self):
        self._do_dnsmasq_restart()
        
        def check_start(_):
            dnsmasq_pid = linux.get_pid_by_process_name('dnsmasq')
            return dnsmasq_pid is not None 

        if not linux.wait_callback_success(check_start, None, 5, 0.5):
            logger.debug('dnsmasq is not running, former start failed, try to start it again ...')
            cmd = self._do_dnsmasq_start()
            
            if cmd.return_code != 0:
                raise virtualrouter.VirtualRouterError('dnsmasq in virtual router is not running, we try to start it but fail, error is %s' % cmd.stdout)

            if not linux.wait_callback_success(check_start, None, 5, 0.5):
                raise virtualrouter.VirtualRouterError('dnsmasq in virtual router is not running, "/etc/init.d/dnsmasq start" returns success, but the process is not running after 5 seconds')

        def rebase_all_to_active_file():
            self.domain.blockRebase(disk_name, None, 0, 0)

            def wait_job(_):
                return not self._wait_for_block_job(disk_name, abort_on_error=True)

            if not linux.wait_callback_success(wait_job, timeout=300):
                raise kvmagent.KvmError('live full snapshot merge failed')

    def _start_multi_nodes(self, restart = False):
        nodes = []
        threads = []
        for node in self.nodes:
            #The reserved node is used by test cases. 
            if not restart and node.reserve__:
                continue

            if not node.dockerImage__:
                print 'Deploy node in hosts'
                #consider some zstack-server is running in vm, the server 
                # startup speed is slow. Increase timeout to 180s.
                cmd = 'zstack-ctl stop_node --host=%s ; zstack-ctl start_node --host=%s --timeout=180' % (node.ip_, node.ip_)
                thread = threading.Thread(target=shell_cmd_thread, args=(cmd, True, ))
                threads.append(thread)
            else:
                print 'Deploy node in docker'
                docker_node = DockerNode(self)
                docker_node.set_docker_image(node.dockerImage__)
                docker_node.set_node_ip(node.ip__)
                docker_node.prepare_node()
                nodes.append(docker_node)
                thread = threading.Thread(target=docker_node.start_node)
                threads.append(thread)

        for thread in threads:
            thread.start()

        self._wait_for_thread_completion('start management node', 200)

        if node_exception:
            print 'node start meets exception:'
            info1 = node_exception[0][1]
            info2 = node_exception[0][2]
            raise info1, None, info2
                
        current_time = time.time()
        #largest timeout time for multi nodes startup is 300s
        timeout_time = current_time + 300
        for node in self.nodes:
            #The reserved node is used by test cases. 
            if node.reserve__:
                continue
            new_time = time.time() 
            if new_time >= timeout_time:
                new_timeout = 1
            else:
                new_timeout = timeout_time - new_time

            if not linux.wait_callback_success(\
                    node_ops.is_management_node_start, \
                    node.ip_, timeout=new_timeout, interval=0.5):
                raise ActionError('multi node does not startup on host: %s' \
                        % node.ip_)

        zstack_home = '%s/apache-tomcat/webapps/zstack/' % self.install_path
        cmd = 'zstack-ctl setenv ZSTACK_HOME=%s' % zstack_home
        shell.call(cmd)

 def start(self, timeout=60):
     #TODO: 1. enbale hair_pin mode
     logger.debug('creating vm:\n%s' % self.domain_xml)
     conn = kvmagent.get_libvirt_connection()
     domain = conn.defineXML(self.domain_xml)
     self.domain = domain
     self.domain.createWithFlags(0)    
     if not linux.wait_callback_success(self.wait_for_state_change, self.VM_STATE_RUNNING, timeout=timeout):
         raise kvmagent.KvmError('unable to start vm[uuid:%s, name:%s], vm state is not changing to running after %s seconds' % (self.uuid, self.get_name(), timeout))

def wait_for_management_server_start(wait_start_timeout=120, \
        node_uuid=None):
    if not linux.wait_callback_success(is_management_node_ready, \
            node_uuid, \
            wait_start_timeout, 1, True):
        raise Exception('waiting for management server start up time out\
                after %s seconds' % wait_start_timeout)
    else:
        print('management starts successfully, is running now ...')

 def wait_for_node_start(self, timeout=120):
     if not linux.wait_callback_success(node_ops.is_management_node_start, \
             self.node_option.get_management_ip(), timeout=timeout, \
             interval=0.5):
         test_util.test_logger('multi node does not startup on host: %s' \
                 % self.node_option.get_management_ip())
         return False
     test_util.test_logger('multi node startup on host: %s' \
             % self.node_option.get_management_ip())
     return True

    def delete(self, req):
        cmd = jsonobject.loads(req[http.REQUEST_BODY])
        pool, image_name = self._parse_install_path(cmd.installPath)

        def delete_image(_):
            # in case image is deleted, we don't have to wait for timeout
            img = "%s/%s" % (pool, image_name)
            shell.check_run('rbd info %s && rbd rm %s' % (img, img))
            return True

        # 'rbd rm' might fail due to client crash. We wait for 30 seconds as suggested by 'rbd'.
        #
        # rbd: error: image still has watchers
        # This means the image is still open or the client using it crashed. Try again after
        # closing/unmapping it or waiting 30s for the crashed client to timeout.
        linux.wait_callback_success(delete_image, interval=5, timeout=30, ignore_exception_in_callback=True)

        rsp = AgentResponse()
        self._set_capacity_to_response(rsp)
        return jsonobject.dumps(rsp)

        def do_commit(base, top, flags=0):
            self.domain.blockCommit(disk_name, base, top, 0, flags)

            logger.debug('start block commit %s --> %s' % (top, base))
            def wait_job(_):
                logger.debug('merging snapshot chain is waiting for blockCommit job completion')
                return not self._wait_for_block_job(disk_name, abort_on_error=True)

            if not linux.wait_callback_success(wait_job, timeout=300):
                raise kvmagent.KvmError('live merging snapshot chain failed, timeout after 300s')

            logger.debug('end block commit %s --> %s' % (top, base))

def check_sanlock_renewal_failure(lockspace):
    last_record = linux.wait_callback_success(get_sanlock_renewal, lockspace, 10, 1, True)
    if last_record is False:
        logger.warn("unable find correct sanlock renewal record, may be rotated")
        return True, ""
    if "next_errors=" not in last_record:
        return True, ""
    errors = int(last_record.split("next_errors=")[-1])
    if errors > 2:
        return False, "sanlock renew lease of lockspace %s failed for %s times, storage may failed" % (
        lockspace, errors)
    return True, ""

    def _start_multi_nodes(self, restart = False):
        nodes = []
        threads = []
        for node in self.nodes:
            #The reserved node is used by test cases. 
            if not restart and node.reserve__:
                continue

            if not node.dockerImage__:
                print 'Deploy node in hosts'
                cmd = 'zstack-ctl stop_node --remote=%s ; zstack-ctl start_node --remote=%s' % (node.ip_, node.ip_)
                thread = threading.Thread(target=shell_cmd_thread, args=(cmd,))
                threads.append(thread)
            else:
                print 'Deploy node in docker'
                docker_node = DockerNode(self)
                docker_node.set_docker_image(node.dockerImage__)
                docker_node.set_node_ip(node.ip__)
                docker_node.prepare_node()
                nodes.append(docker_node)
                thread = threading.Thread(target=docker_node.start_node)
                threads.append(thread)

        for thread in threads:
            thread.start()

        self._wait_for_thread_completion('start management node', 200)

        if node_exception:
            print 'node start meets exception:'
            info1 = node_exception[0][1]
            info2 = node_exception[0][2]
            raise info1, None, info2
                
        current_time = time.time()
        #largest timeout time for multi nodes startup is 300s
        timeout_time = current_time + 300
        for node in self.nodes:
            #The reserved node is used by test cases. 
            if node.reserve__:
                continue
            new_time = time.time() 
            if new_time >= timeout_time:
                new_timeout = 1
            else:
                new_timeout = timeout_time - new_time

            if not linux.wait_callback_success(\
                    node_ops.is_management_node_start, \
                    node.ip_, timeout=new_timeout, interval=0.5):
                raise ActionError('multi node does not startup on host: %s' \
                        % node.ip_)

def test():
    test_util.test_dsc('Test Host Start/Stop function')
    vm_cpu = 1
    vm_memory = 1073741824 #1G
    cond = res_ops.gen_query_conditions('name', '=', 'ttylinux')
    image_uuid = res_ops.query_resource(res_ops.IMAGE, cond)[0].uuid
    l3_network_uuid = res_ops.query_resource(res_ops.L3_NETWORK)[0].uuid
    vm = test_stub.create_mini_vm([l3_network_uuid], image_uuid, cpu_num = vm_cpu, memory_size = vm_memory)
    test_obj_dict.add_vm(vm)

    host_uuid = test_lib.lib_get_vm_host(vm.vm).uuid
    host_ops.change_host_state(host_uuid, 'disable')
    if not linux.wait_callback_success(is_host_disabled, host_uuid, 120):
        test_util.test_fail('host state is not changed to disabled')

    host_ops.change_host_state(host_uuid, 'enable')
    if not linux.wait_callback_success(is_host_enabled, host_uuid, 120):
        test_util.test_fail('host state is not changed to enabled')

    vm.destroy()
    test_obj_dict.rm_vm(vm)
    test_util.test_pass('Stop/Start Host Test Pass')

        def take_full_snapshot():
            logger.debug('start rebasing to make a full snapshot')
            self.domain.blockRebase(disk_name, None, 0, 0)

            logger.debug('rebasing full snapshot is in processing')
            def wait_job(_):
                logger.debug('full snapshot is waiting for blockRebase job completion')
                return not self._wait_for_block_job(disk_name, abort_on_error=True)

            if not linux.wait_callback_success(wait_job, timeout=300):
                raise kvmagent.KvmError('live full snapshot failed')

            return take_delta_snapshot()

        def run_convert_if_need():
            def do_run():
                save_pid()
                ret = shell.run(echo_pid_cmd)
                new_task.current_process_return_code = ret
                return ret

            pid = linux.read_file(v2v_pid_path)
            if not pid:
                return do_run()

            pid = int(pid.strip())
            process_completed = os.path.exists(v2v_cmd_ret_path)
            process_has_been_killed = not os.path.exists(v2v_cmd_ret_path) and not os.path.exists('/proc/%d' % pid)
            process_still_running = not os.path.exists(v2v_cmd_ret_path) and os.path.exists('/proc/%d' % pid)
            if process_has_been_killed:
                return do_run()

            if process_still_running:
                linux.wait_callback_success(os.path.exists, v2v_cmd_ret_path, timeout=259200, interval=60)

            ret = linux.read_file(v2v_cmd_ret_path)
            return int(ret.strip() if ret else 126)

    def cancel_sftp_download(self, req):
        cmd = jsonobject.loads(req[http.REQUEST_BODY])
        rsp = AgentResponse()

        def check():
            return shell.run("rbd ls %s | grep -q %s" % (pool, image_name)) != 0

        def remove(target_name):
            return shell.run("rbd info {0}/{1} || rbd rm {0}/{1}".format(pool, target_name)) == 0

        pool, image_name = self._parse_install_path(cmd.primaryStorageInstallPath)
        tmp_image_name = 'tmp-%s' % image_name

        if check():
            return jsonobject.dumps(rsp)

        for image in (tmp_image_name, image_name):
            shell.run("pkill -9 -f '%s'" % image)
            linux.wait_callback_success(remove, image, timeout=30)

        if not check():
            rsp.set_err("remove image %s/%s fail" % (pool, image_name))

        return jsonobject.dumps(rsp)

def check_sg_rule_exist(test_vm, protocol, direction, extra_str, exp_result):
    if linux.wait_callback_success(do_check_sg_rule_exist, (test_vm, direction, extra_str, exp_result), 5, 0.2):
        test_result = exp_result
        if exp_result:
            test_util.test_logger('Expected result: [Security Group] find %s %s rule for [vm:] %s. ' % (protocol, direction, test_vm.uuid))
        else:
            test_util.test_logger('Expected result: [Security Group] FAIL to find %s %s rule for [vm:] %s in 5 seconds. ' % (protocol, direction, test_vm.uuid))
    else:
        test_result = not exp_result
        if exp_result:
            test_util.test_logger('Unexpected result: [Security Group] Not find %s %s rule for [vm:] %s. ' % (protocol, direction, test_vm.uuid))
        else:
            test_util.test_logger('Unexpected result: [Security Group]  find %s %s rule for [vm:] %s in 5 seconds. ' % (protocol, direction, test_vm.uuid))

    return test_result

    def _restart_dnsmasq(self, ns_name, conf_file_path):
        pid = linux.find_process_by_cmdline([conf_file_path])
        if pid:
            linux.kill_process(pid)

        NS_NAME = ns_name
        CONF_FILE = conf_file_path
        DNSMASQ = bash_errorout('which dnsmasq').strip(' \t\r\n')
        bash_errorout('ip netns exec {{NS_NAME}} {{DNSMASQ}} --conf-file={{CONF_FILE}} ')

        def check(_):
            pid = linux.find_process_by_cmdline([conf_file_path])
            return pid is not None

        if not linux.wait_callback_success(check, None, 5):
            raise Exception('dnsmasq[conf-file:%s] is not running after being started %s seconds' % (conf_file_path, 5))