def _deploy_db(self, keep_db = False):
        if not keep_db:
            extra_opts = '--drop'
        else:
            extra_opts = '--keep-db'

        if not self.need_deploy_db:
            return
        ssh.make_ssh_no_password(self.db_server, 'root', \
                self.db_server_root_password)

        if not self.db_admin_password:
            cmd = 'zstack-ctl install_db --debug --host=%s --login-password=zstack.mysql.password' % self.db_server
        else:
            cmd = 'zstack-ctl install_db --debug --host=%s \
                    --login-password=%s' \
                    % (self.db_server, \
                    self.db_admin_password)

        print('installing db ...')
        shell.call(cmd)

        cmd = 'zstack-ctl deploydb %s --host=%s' % (extra_opts, self.db_server)
        if self.db_admin_password:
            cmd = '%s --root-password=%s' % (cmd, self.db_admin_password )
        else:
            cmd = '%s --root-password=zstack.mysql.password' % cmd

        if self.db_password:
            cmd = '%s --zstack-password=%s' % (cmd, self.db_password)

        print('deploying db ...')
        shell.call(cmd)

def prepare_mevoco_test_env(vm_inv):
    all_in_one_pkg = os.environ['zstackPkg']
    test_lib.lib_scp_file_to_vm(vm_inv, all_in_one_pkg, '/root/zizhu.bin')

    vm_ip = vm_inv.vmNics[0].ip
    ssh.make_ssh_no_password(vm_ip, test_lib.lib_get_vm_username(vm_inv), \
            test_lib.lib_get_vm_password(vm_inv))

def test():
    test_util.test_dsc('Create test vm to test zstack all installation in CentOS7.')
#    image_name = os.environ.get('imageName_i_c7')
#    image_name = os.environ.get('imageName_i_offline')
    image_name = 'ZStack-Community-Original-Image-152'
    vm = test_stub.create_vlan_vm(image_name)
    test_obj_dict.add_vm(vm)
    if os.environ.get('zstackManagementIp') == None:
        vm.check()
    else:
        time.sleep(60)

    vm_inv = vm.get_vm()
    vm_ip = vm_inv.vmNics[0].ip
    target_file = '/root/zstack-all-in-one.tgz'
#    test_stub.prepare_test_env(vm_inv, target_file)
    zstack_install_script = os.environ['zstackInstallScript']
    installer_file = '/root/zstack_installer.sh'
    vm_ip = vm_inv.vmNics[0].ip
    vm_username = test_lib.lib_get_vm_username(vm_inv)
    vm_password = test_lib.lib_get_vm_password(vm_inv)
    test_stub.scp_file_to_vm(vm_inv, zstack_install_script, installer_file)

    all_in_one_pkg = os.environ['zstackPkg_1.4']
    test_stub.scp_file_to_vm(vm_inv, all_in_one_pkg, target_file)

    ssh.make_ssh_no_password(vm_ip, vm_username, vm_password)

    ssh_cmd = 'ssh  -oStrictHostKeyChecking=no -oCheckHostIP=no -oUserKnownHostsFile=/dev/null %s' % vm_ip
    execute_all_install(ssh_cmd, target_file, tmp_file)
    test_stub.check_installation(ssh_cmd, tmp_file, vm_inv)

    os.system('rm -f %s' % tmp_file)
#    vm.destroy()
    test_util.test_pass('ZStack installation Test Success')

    def _deploy_rabbitmq(self):
        ssh.make_ssh_no_password(self.rabbitmq_server, 'root', \
                self.rabbitmq_server_root_passwd)

        cmd = "zstack-ctl install_rabbitmq --host=%s" % self.rabbitmq_server

        print('deploying rabbitmq ...')
        shell.call(cmd)

def test():
    test_util.test_dsc('Create test vm to test zstack upgrade by -u.')

    if os.path.exists('/home/installation-package/zstack'):
        image_name = os.environ.get('imageName_i_c7_z_1.7')
    elif os.path.exists('/home/installation-package/mevoco'):
        image_name = os.environ.get('imageName_i_c7_m_1.7')

    vm = test_stub.create_vlan_vm(image_name)
    test_obj_dict.add_vm(vm)
    if os.environ.get('zstackManagementIp') == None:
        vm.check()
    else:
        time.sleep(60)

    vm_inv = vm.get_vm()
    vm_ip = vm_inv.vmNics[0].ip
    vm_username = test_lib.lib_get_vm_username(vm_inv)
    vm_password = test_lib.lib_get_vm_password(vm_inv)

    ssh_cmd = "ssh -oStrictHostKeyChecking=no -oCheckHostIP=no -oUserKnownHostsFile=/dev/null %s"%vm_ip
    ssh.make_ssh_no_password(vm_ip,vm_username,vm_password)
    test_stub.copy_id_dsa(vm_inv,ssh_cmd,tmp_file)
    test_stub.copy_id_dsa_pub(vm_inv)

    test_stub.update_iso(ssh_cmd, tmp_file, vm_inv, update_file)

    test_util.test_dsc('Update MN IP')
    cmd = '%s "zstack-ctl change_ip --ip="%s' % (ssh_cmd,vm_ip)
    test_stub.execute_shell_in_process(cmd,tmp_file)
#    cmd = '%s "zstack-ctl start"' % ssh_cmd
#    test_stub.execute_shell_in_process(cmd,tmp_file)
#    test_stub.check_installation(ssh_cmd,tmp_file,vm_inv)

    pkg_num = 1.8
    curren_num = float(os.environ.get('releasePkgNum'))
    while pkg_num<=curren_num:
        test_util.test_dsc('Upgrade zstack to %s' % pkg_num)
        upgrade_target_file = '/root/zstack-upgrade-all-in-one.tgz'
        upgrade_pkg = os.environ.get('zstackPkg_%s' % pkg_num)
        test_stub.prepare_upgrade_test_env(vm_inv, upgrade_target_file, upgrade_pkg)
        test_stub.upgrade_zstack(ssh_cmd, upgrade_target_file, tmp_file)
        test_stub.check_zstack_version(ssh_cmd, tmp_file, vm_inv, str(pkg_num))
        test_stub.check_installation(ssh_cmd, tmp_file, vm_inv)
        pkg_num = pkg_num + 0.1

    test_util.test_dsc('Upgrade zstack to latest')
    upgrade_target_file = '/root/zstack-upgrade-all-in-one.tgz'
    test_stub.prepare_test_env(vm_inv, upgrade_target_file)
    test_stub.upgrade_zstack(ssh_cmd, upgrade_target_file, tmp_file)
    zstack_latest_version = os.environ.get('zstackLatestVersion')
    test_stub.check_zstack_version(ssh_cmd, tmp_file, vm_inv, zstack_latest_version)
    test_stub.check_installation(ssh_cmd, tmp_file, vm_inv)

    os.system('rm -f %s'%tmp_file)
    vm.destroy()
    test_util.test_pass('ZStack upgrade Test Success')

def prepare_yum_repo(vm_inv):
    origin_file = '/etc/yum.repos.d/epel.repo'
    target_file = '/etc/yum.repos.d/epel.repo'
    vm_ip = vm_inv.vmNics[0].ip
    vm_username = test_lib.lib_get_vm_username(vm_inv)
    vm_password = test_lib.lib_get_vm_password(vm_inv)
    scp_file_to_vm(vm_inv, origin_file, target_file)

    ssh.make_ssh_no_password(vm_ip, vm_username, vm_password)

def test():
    test_util.test_dsc('Create test vm to test zstack upgrade by -u.')

    if os.path.exists('/home/installation-package/zstack'):
        image_name = os.environ.get('imageName_i_c7_z_1.6')
    elif os.path.exists('/home/installation-package/mevoco'):
        image_name = os.environ.get('imageName_i_c7_m_1.6')

    vm = test_stub.create_vlan_vm(image_name)
    test_obj_dict.add_vm(vm)
    if os.environ.get('zstackManagementIp') == None:
        vm.check()
    else:
        time.sleep(60)

    vm_inv = vm.get_vm()
    vm_ip = vm_inv.vmNics[0].ip

    ssh_cmd = 'ssh  -oStrictHostKeyChecking=no -oCheckHostIP=no -oUserKnownHostsFile=/dev/null %s' % vm_ip
    ssh.make_ssh_no_password(vm_ip, test_lib.lib_get_vm_username(vm_inv), \
            test_lib.lib_get_vm_password(vm_inv))
    test_stub.copy_id_dsa(vm_inv, ssh_cmd, tmp_file)
    test_stub.copy_id_dsa_pub(vm_inv)

    test_stub.update_iso(ssh_cmd, tmp_file, vm_inv, update_file)

    test_util.test_dsc('Update MN IP')
    cmd = '%s "zstack-ctl change_ip --ip="%s ' % (ssh_cmd, vm_ip)
    process_result = test_stub.execute_shell_in_process(cmd, tmp_file)

    test_util.test_dsc('Update /etc/host/, just for mevoco 1.6')
    cmd = '%s "sed -i \'3d\' /etc/hosts"' % (ssh_cmd)
    process_result = test_stub.execute_shell_in_process(cmd, tmp_file)

    test_util.test_dsc('Reboot kairosdb, just for mevoco 1.6')
    cmd = '%s "zstack-ctl kairosdb --stop"' % (ssh_cmd)
    process_result = test_stub.execute_shell_in_process(cmd, tmp_file)
    cmd = '%s "zstack-ctl kairosdb --start"' % (ssh_cmd)
    process_result = test_stub.execute_shell_in_process(cmd, tmp_file)

#    cmd = '%s "zstack-ctl start"' % ssh_cmd
#    process_result = test_stub.execute_shell_in_process(cmd, tmp_file)
#    test_stub.check_installation(ssh_cmd, tmp_file, vm_inv)

    test_util.test_dsc('Upgrade zstack to latest') 
    upgrade_target_file = '/root/zstack-upgrade-all-in-one.tgz' 
    test_stub.prepare_test_env(vm_inv, upgrade_target_file)
    test_stub.upgrade_zstack(ssh_cmd, upgrade_target_file, tmp_file) 
    zstack_latest_version = os.environ.get('zstackLatestVersion')
    test_stub.check_zstack_version(ssh_cmd, tmp_file, vm_inv, zstack_latest_version)
    test_stub.check_installation(ssh_cmd, tmp_file, vm_inv)

    os.system('rm -f %s' % tmp_file)
    vm.destroy()
    test_util.test_pass('ZStack upgrade Test Success')

def prepare_upgrade_test_env(vm_inv, aio_target, upgrade_pkg):
    zstack_install_script = os.environ['zstackInstallScript']
    target_file = '/root/zstack_installer.sh'
    vm_ip = vm_inv.vmNics[0].ip
    vm_username = test_lib.lib_get_vm_username(vm_inv)
    vm_password = test_lib.lib_get_vm_password(vm_inv)
    scp_file_to_vm(vm_inv, zstack_install_script, target_file)

    scp_file_to_vm(vm_inv, upgrade_pkg, aio_target)

    ssh.make_ssh_no_password(vm_ip, vm_username, vm_password)

def prepare_test_env(vm_inv, aio_target):
    zstack_install_script = os.environ['zstackInstallScript']
    target_file = '/root/zstack_installer.sh'
    test_lib.lib_scp_file_to_vm(vm_inv, zstack_install_script, target_file)

    all_in_one_pkg = os.environ['zstackPkg']
    test_lib.lib_scp_file_to_vm(vm_inv, all_in_one_pkg, aio_target)

    vm_ip = vm_inv.vmNics[0].ip
    ssh.make_ssh_no_password(vm_ip, test_lib.lib_get_vm_username(vm_inv), \
            test_lib.lib_get_vm_password(vm_inv))

def test():
    test_util.test_dsc('Create test vm to test zstack installation.')
    vm = create_vlan_vm()
    test_obj_dict.add_vm(vm)
    vm.check()

    vm_inv = vm.get_vm()
    vm_ip = vm_inv.vmNics[0].ip
    zstack_install_script = test_lib.test_config.zstackInstaller.text_
    target_file = '/root/zstack_installer.sh'
    test_lib.lib_scp_file_to_vm(vm_inv, zstack_install_script, target_file)

    ssh.make_ssh_no_password(vm_ip, test_lib.lib_get_vm_username(vm_inv), \
            test_lib.lib_get_vm_password(vm_inv))

    env_var = "ZSTACK_ALL_IN_ONE='%s' ZSTACK_PYPI_URL='%s' WEBSITE='%s'" % \
            (check_str(os.environ.get('ZSTACK_ALL_IN_ONE')), \
            check_str(os.environ.get('ZSTACK_PYPI_URL')), \
            check_str(os.environ.get('WEBSITE')))
    ssh_cmd = 'ssh  -oStrictHostKeyChecking=no -oCheckHostIP=no -oUserKnownHostsFile=/dev/null %s' % vm_ip
    cmd = '%s "%s bash %s -d -a"' % (ssh_cmd, env_var, target_file)
    tmp_file = '/tmp/%s' % uuid.uuid1().get_hex()
    process_result = execute_shell_in_process(cmd, tmp_file)

    if process_result != 0:
        cmd = '%s "cat /tmp/zstack_installation.log"' % ssh_cmd
        execute_shell_in_process(cmd, tmp_file)
        test_util.test_fail('zstack installation failed')

    cmd = '%s "/usr/bin/zstack-cli LogInByAccount accountName=admin password=password"' % ssh_cmd
    process_result = execute_shell_in_process(cmd, tmp_file)
    if process_result != 0:
        test_util.test_fail('zstack-cli login failed')

    cmd = '%s "/usr/bin/zstack-cli CreateZone name=zone1 description=zone1"' % ssh_cmd
    process_result = execute_shell_in_process(cmd, tmp_file)
    if process_result != 0:
        test_util.test_fail('zstack-cli create zone failed')

    cmd = '%s "/usr/bin/zstack-cli QueryZone name=zone1 description=zone1"' % ssh_cmd
    process_result = execute_shell_in_process(cmd, tmp_file)
    if process_result != 0:
        test_util.test_fail('zstack-cli Query zone failed')

    os.system('rm -f %s' % tmp_file)
    vm.destroy()
    test_util.test_pass('ZStack installation Test Success')

def test():
    global vm_inv
    
    iso_path = os.environ.get('iso_path')
    upgrade_script_path = os.environ.get('upgradeScript')
    test_util.test_dsc('Create test vm to test zstack installation with console proxy.')

    conditions = res_ops.gen_query_conditions('name', '=', os.environ.get('imageNameBase_21_ex'))
    image = res_ops.query_resource(res_ops.IMAGE, conditions)[0]

    vm_inv = create_vm(image) 

    time.sleep(60)

    vm_ip = vm_inv.vmNics[0].ip
    vip = '172.20.61.253'
    if vip == vm_ip:
        vip = '172.20.61.254'

    ssh_cmd = 'ssh  -oStrictHostKeyChecking=no -oCheckHostIP=no -oUserKnownHostsFile=/dev/null %s' % vm_ip
    ssh.make_ssh_no_password(vm_ip, test_lib.lib_get_vm_username(vm_inv), test_lib.lib_get_vm_password(vm_inv))
    cmd = '%s ifconfig eth0:0 %s up' % (ssh_cmd, vip)
    process_result = test_stub.execute_shell_in_process(cmd, tmp_file)

    test_stub.update_iso(vm_ip, tmp_file, iso_path, upgrade_script_path)

    target_file = '/root/zstack-all-in-one.tgz'
    test_stub.prepare_test_env(vm_inv, target_file)
    args = '-o -C %s -I %s' % (vip, vm_ip)
    test_stub.execute_install_with_args(ssh_cmd, args, target_file, tmp_file)
    test_stub.check_installation(vm_ip, tmp_file)

    cmd = '%s cat /usr/local/zstack/apache-tomcat/webapps/zstack/WEB-INF/classes/zstack.properties | grep \'consoleProxyOverriddenIp = %s\'' % (ssh_cmd, vip)
    (process_result, check_result) = test_stub.execute_shell_in_process_stdout(cmd, tmp_file)
    check_result = check_result[:-1]
    test_util.test_dsc('cat result: |%s|' % check_result)
    expect_result = "consoleProxyOverriddenIp = %s" % vip
    if check_result != expect_result:
        test_util.test_fail('Fail to install ZStack with console proxy')

    os.system('rm -f %s' % tmp_file)
    sce_ops.destroy_vm(zstack_management_ip, vm_inv.uuid)    
    test_util.test_pass('ZStack installation Test Success')

def test():
    if os.path.exists('/home/zstack-package/') != True:
        test_util.test_skip("current test suite is zstack, but this case is for mevoco. Skip test")

    test_util.test_dsc('Create test vm to test zstack upgrade by -u.')
    image_name = os.environ.get('imageName_i_c7_z_1.2')
    vm = test_stub.create_vlan_vm(image_name)
    test_obj_dict.add_vm(vm)
    if os.environ.get('zstackManagementIp') == None:
        vm.check()
    else:
        time.sleep(60)

    vm_inv = vm.get_vm()
    vm_ip = vm_inv.vmNics[0].ip

    ssh_cmd = 'ssh  -oStrictHostKeyChecking=no -oCheckHostIP=no -oUserKnownHostsFile=/dev/null %s' % vm_ip
    ssh.make_ssh_no_password(vm_ip, test_lib.lib_get_vm_username(vm_inv), \
            test_lib.lib_get_vm_password(vm_inv))
    test_stub.copy_id_dsa(vm_inv, ssh_cmd, tmp_file)
    test_stub.copy_id_dsa_pub(vm_inv)

    test_util.test_dsc('Update MN IP')
    cmd = '%s "zstack-ctl change_ip --ip="%s ' % (ssh_cmd, vm_ip)
    process_result = test_stub.execute_shell_in_process(cmd, tmp_file)
    cmd = '%s "zstack-ctl start"' % ssh_cmd
    process_result = test_stub.execute_shell_in_process(cmd, tmp_file)
    test_stub.check_installation(ssh_cmd, tmp_file, vm_inv)

    test_util.test_dsc('Upgrade zstack to latest mevoco') 
    upgrade_target_file = '/root/mevoco-upgrade-all-in-one.tgz' 
    test_stub.prepare_test_env(vm_inv, upgrade_target_file)
    test_stub.upgrade_zstack(ssh_cmd, upgrade_target_file, tmp_file) 
    zstack_latest_version = os.environ.get('zstackLatestVersion')
    test_stub.check_zstack_version(ssh_cmd, tmp_file, vm_inv, zstack_latest_version)
    test_stub.check_zstack_or_mevoco(ssh_cmd, tmp_file, vm_inv, 'mevoco')
    test_stub.check_installation(ssh_cmd, tmp_file, vm_inv)

    os.system('rm -f %s' % tmp_file)
    vm.destroy()
    test_util.test_pass('ZStack upgrade Test Success')

def test():
    test_util.test_dsc('Create test vm to test zstack installation with console proxy.')

    image_name = os.environ.get('imageName_i_offline')

    vm = test_stub.create_vlan_vm(image_name)
    test_obj_dict.add_vm(vm)
    if os.environ.get('zstackManagementIp') == None:
        vm.check()
    else:
        time.sleep(60)

    vm_inv = vm.get_vm()
    vm_ip = vm_inv.vmNics[0].ip
    vip = '172.20.198.1'
    if vip == vm_ip:
        vip = '172.20.198.2'

    ssh_cmd = 'ssh  -oStrictHostKeyChecking=no -oCheckHostIP=no -oUserKnownHostsFile=/dev/null %s' % vm_ip
    ssh.make_ssh_no_password(vm_ip, test_lib.lib_get_vm_username(vm_inv), test_lib.lib_get_vm_password(vm_inv))
    cmd = '%s ifconfig eth0:0 %s up' % (ssh_cmd, vip)
    process_result = test_stub.execute_shell_in_process(cmd, tmp_file)

    target_file = '/root/zstack-all-in-one.tgz'
    test_stub.prepare_test_env(vm_inv, target_file)
    args = '-o -C %s -I %s' % (vip, vm_ip)
    test_stub.execute_install_with_args(ssh_cmd, args, target_file, tmp_file)
    test_stub.check_installation(ssh_cmd, tmp_file, vm_inv)

    cmd = '%s cat /usr/local/zstack/apache-tomcat/webapps/zstack/WEB-INF/classes/zstack.properties | grep \'consoleProxyOverriddenIp = %s\'' % (ssh_cmd, vip)
    (process_result, check_result) = test_stub.execute_shell_in_process_stdout(cmd, tmp_file)
    check_result = check_result[:-1]
    test_util.test_dsc('cat result: |%s|' % check_result)
    expect_result = "consoleProxyOverriddenIp = %s" % vip
    if check_result != expect_result:
        test_util.test_fail('Fail to install ZStack with console proxy')

    os.system('rm -f %s' % tmp_file)
    vm.destroy()
    test_util.test_pass('ZStack installation Test Success')

def enable_ansible_connection(target, username, password, exc_info): 
    ansible_config = '/etc/ansible/ansible.cfg'
    host_config = '/etc/ansible/hosts'
    ansible_config_content='''
[defaults]
forks = 100
host_key_checking = False
pipelining = True

'''
    add_host_cmd = "grep '^%s$' %s; if [ $? -ne 0 ]; then echo -e '\n%s\n' >> %s; sed -i '/^$/d' %s; fi; " % (target, host_config, target, host_config, host_config)
    if not os.path.exists(os.path.dirname(ansible_config)):
        os.system('mkdir -p %s' % os.path.dirname(ansible_config))
    if not os.path.exists(ansible_config):
        open(ansible_config).write(ansible_config_content)

    shell.call(add_host_cmd)
    print('Create no ssh password for: %s ' % target)
    try:
        ssh.make_ssh_no_password(target, username, password)
    except Exception as e:
        exc_info.append(sys.exc_info())
        raise e

def make_ssh_no_password(vm_inv):
    vm_ip = vm_inv.vmNics[0].ip
    ssh.make_ssh_no_password(vm_ip, test_lib.lib_get_vm_username(vm_inv), \
            test_lib.lib_get_vm_password(vm_inv))

def test():
    def test_fail(msg):
        os.system('rm -f %s' % tmp_file)
        test_util.test_fail(msg)

    test_util.test_dsc('Create 3 CentOS7 vm to test multi management node installation')

#    image_name = os.environ.get('imageName_i_c7')
    image_name = os.environ.get('imageName_i_offline')

    vm1 = test_stub.create_vlan_vm(image_name)
    test_obj_dict.add_vm(vm1)
    vm2 = test_stub.create_vlan_vm(image_name)
    test_obj_dict.add_vm(vm2)
    vm3 = test_stub.create_vlan_vm(image_name)
    test_obj_dict.add_vm(vm3)
    if os.environ.get('zstackManagementIp') == None:
        vm1.check()
        vm2.check()
        vm3.check()
    else:
        time.sleep(60)

    vm1_inv = vm1.get_vm()
    vm1_ip = vm1_inv.vmNics[0].ip
    vm2_inv = vm2.get_vm()
    vm2_ip = vm2_inv.vmNics[0].ip
    vm3_inv = vm3.get_vm()
    vm3_ip = vm3_inv.vmNics[0].ip
    target_file = '/root/zstack-all-in-one.tgz'
    test_stub.prepare_test_env(vm1_inv, target_file)
    ssh_cmd1 = 'ssh  -oStrictHostKeyChecking=no -oCheckHostIP=no -oUserKnownHostsFile=/dev/null %s' % vm1_ip
    ssh_cmd2 = 'ssh  -oStrictHostKeyChecking=no -oCheckHostIP=no -oUserKnownHostsFile=/dev/null %s' % vm2_ip
    ssh_cmd3 = 'ssh  -oStrictHostKeyChecking=no -oCheckHostIP=no -oUserKnownHostsFile=/dev/null %s' % vm3_ip

    test_util.test_dsc('Install ZStack on vm1')
    test_stub.copy_id_dsa(vm1_inv, ssh_cmd1, tmp_file)
    test_stub.copy_id_dsa_pub(vm1_inv)
    test_stub.execute_all_install(ssh_cmd1, target_file, tmp_file)
    test_stub.check_installation(ssh_cmd1, tmp_file, vm1_inv)

    test_util.test_dsc('Install multi management node on vm2 and vm3')
    host_list = 'root:[email protected]%s root:[email protected]%s' % (vm2_ip, vm3_ip)
    cmd = '%s "zstack-ctl add_multi_management --host-list %s"' % (ssh_cmd1, host_list)
    process_result = test_stub.execute_shell_in_process(cmd, tmp_file)

    test_util.test_dsc('Check installation on vm1')
    test_stub.check_installation(ssh_cmd1, tmp_file, vm1_inv)

    test_util.test_dsc('Check installation on vm2')
    ssh.make_ssh_no_password(vm2_ip, test_lib.lib_get_vm_username(vm2_inv), \
            test_lib.lib_get_vm_password(vm2_inv))
    cmd = '%s "zstack-ctl start"' % ssh_cmd2
    process_result = test_stub.execute_shell_in_process(cmd, tmp_file)
    test_stub.check_installation(ssh_cmd2, tmp_file, vm2_inv)

    test_util.test_dsc('Check installation on vm3')
    ssh.make_ssh_no_password(vm3_ip, test_lib.lib_get_vm_username(vm3_inv), \
            test_lib.lib_get_vm_password(vm3_inv))
    cmd = '%s "zstack-ctl start"' % ssh_cmd3
    process_result = test_stub.execute_shell_in_process(cmd, tmp_file)
    test_stub.check_installation(ssh_cmd3, tmp_file, vm3_inv)

    test_util.test_dsc('Upgrade multi management node on vm2 and vm3')
    cmd = '%s "zstack-ctl upgrade_multi_management_node --installer-bin %s"' % (ssh_cmd1, target_file) 
    process_result = test_stub.execute_shell_in_process(cmd, tmp_file)

    test_util.test_dsc('After upgrade, check installation on vm1')
    test_stub.check_installation(ssh_cmd1, tmp_file, vm1_inv)

    test_util.test_dsc('After upgrade, check installation on vm2')
    ssh.make_ssh_no_password(vm2_ip, test_lib.lib_get_vm_username(vm2_inv), \
            test_lib.lib_get_vm_password(vm2_inv))
    cmd = '%s "zstack-ctl start"' % ssh_cmd2
    process_result = test_stub.execute_shell_in_process(cmd, tmp_file)
    test_stub.check_installation(ssh_cmd2, tmp_file, vm2_inv)

    test_util.test_dsc('After upgrade, check installation on vm3')
    ssh.make_ssh_no_password(vm3_ip, test_lib.lib_get_vm_username(vm3_inv), \
            test_lib.lib_get_vm_password(vm3_inv))
    cmd = '%s "zstack-ctl start"' % ssh_cmd3
    process_result = test_stub.execute_shell_in_process(cmd, tmp_file)
    test_stub.check_installation(ssh_cmd3, tmp_file, vm3_inv)

    os.system('rm -f %s' % tmp_file)
    vm1.destroy()
    test_obj_dict.rm_vm(vm1)
    vm2.destroy()
    test_obj_dict.rm_vm(vm2)
    vm3.destroy()
    test_obj_dict.rm_vm(vm3)
    test_util.test_pass('ZStack multi management nodes installation Test Success')

def test():

    test_util.test_dsc('Create test vm and check')
    mn_ip = os.environ['ZSTACK_BUILT_IN_HTTP_SERVER_IP']
    test_util.test_dsc("wait 10s for start")
    nodeUserName = os.environ.get('nodeUserName')
    nodePassword = os.environ.get('nodePassword')
    ssh.make_ssh_no_password(mn_ip, nodeUserName, nodePassword)

    #Query image
    image_name = os.environ.get('imageName_s')
    for i in range(0, 4):
        time.sleep(10)
        try:
            cmd = "zstack-cli LogInByAccount accountName=admin password=password;"
            fi, fo, fe = os.popen3('ssh ' + mn_ip + ' ' + cmd )
            cmd = 'zstack-cli QueryImage fields=uuid name="' + image_name  + '"'
            fi, fo, fe = os.popen3('ssh ' + mn_ip + ' ' + cmd )
            test_util.test_logger('ssh ' + mn_ip + ' ' + cmd)
            test_out = fo.read()
            query_result = json.loads(test_out)
            break
        except Exception as e:
            test_util.test_logger(str(e))
            test_util.test_logger('fail to query image retry: ' + str(i))
    test_util.test_logger(query_result)
    image_uuid = str(query_result['inventories'][0]['uuid'])

    #Query l3
    l3_1_name = os.environ.get('l3PublicNetworkName')
    test_util.test_logger(l3_1_name)
    for i in range(0, 4):
        try:
            cmd = "zstack-cli LogInByAccount accountName=admin password=password;"
            fi, fo, fe = os.popen3('ssh ' + mn_ip + ' ' + cmd )
            cmd = '"zstack-cli QueryL3Network fields=uuid name=\'' + l3_1_name + '\'"'
            test_util.test_logger(cmd)
            fi, fo, fe = os.popen3('ssh ' + mn_ip + ' ' + cmd )
            test_err = fe.read()
            test_out = fo.read()
            query_result = json.loads(test_out)
            break
        except Exception as e:
            test_util.test_logger(str(e))
            test_util.test_logger('fail to query l3network retry: ' + str(i))
        time.sleep(2)
    test_util.test_logger(query_result)
    #l3_1_uuid = str(query_result['inventories'][0]['uuid'])
    l3_1_uuid = test_lib.lib_get_l3_by_name(l3_1_name).uuid

    #Query instanceOffering
    for i in range(0, 4):
        try:
            cmd = "zstack-cli QueryInstanceOffering fields=uuid"
            fi, fo, fe = os.popen3('ssh ' + mn_ip + ' ' + cmd )
            test_out = fo.read()

            query_result = json.loads(test_out)
            test_util.test_logger(query_result)
            break
        except Exception as e:
            test_util.test_logger(str(e))
            test_util.test_logger('fail to query l3network retry: ' + str(i))
        time.sleep(2)

    instance_offering_uuid = str(query_result['inventories'][0]['uuid'])

    for i in range(0, 4):
        try:
            test_util.test_logger('CLI query')
            cmd = "zstack-cli LogInByAccount accountName=admin password=password;\
                   zstack-cli QueryHost"
            fi, fo, fe = os.popen3('ssh ' + mn_ip + ' ' + cmd )
            testout = fo.read()
            query_result = json.loads(test_out)
            test_util.test_logger(query_result)
            break
        except Exception as e:
            test_util.test_logger(str(e))
            test_util.test_logger('fail to query l3network retry: ' + str(i))
        time.sleep(2)
    cmd = "zstack-cli LogInByAccount accountName=admin password=password;\
           zstack-cli CreateVmInstance instanceOfferingUuid=" + instance_offering_uuid \
           + " l3NetworkUuids=" + l3_1_uuid \
           + " imageUuid=" + image_uuid + " name=VM1" \
           + " description='Thisisatestvm.\\\nShouldsuccessfullycreateaVM'"

    test_util.test_logger(cmd)
    fi, fo, fe = os.popen3('ssh ' + mn_ip + ' ' + cmd )
    textOut = fo.read()
    test_util.test_logger(textOut)
    fo.close
    fi.close
    fe.close

def test():
    global vm1_inv
    global vm2_inv
    global vm3_inv

    iso_path = os.environ.get('iso_path')
    upgrade_script_path = os.environ.get('upgradeScript')

    test_util.test_dsc('Create 3 CentOS7 vm to test multi management node installation')

    conditions = res_ops.gen_query_conditions('name', '=', os.environ.get('imageNameBase_21_ex'))
    image = res_ops.query_resource(res_ops.IMAGE, conditions)[0]
    
    vm1_inv = create_vm(image) 
    vm2_inv = create_vm(image)
    vm3_inv = create_vm(image)

    vm1_ip = vm1_inv.vmNics[0].ip
    vm2_ip = vm2_inv.vmNics[0].ip
    vm3_ip = vm3_inv.vmNics[0].ip

    time.sleep(60)

    target_file = '/root/zstack-all-in-one.tgz'
    test_stub.prepare_test_env(vm1_inv, target_file)
    ssh_cmd1 = 'ssh  -oStrictHostKeyChecking=no -oCheckHostIP=no -oUserKnownHostsFile=/dev/null %s' % vm1_ip
    ssh_cmd2 = 'ssh  -oStrictHostKeyChecking=no -oCheckHostIP=no -oUserKnownHostsFile=/dev/null %s' % vm2_ip
    ssh_cmd3 = 'ssh  -oStrictHostKeyChecking=no -oCheckHostIP=no -oUserKnownHostsFile=/dev/null %s' % vm3_ip

    #test_util.test_dsc('Update MN IP and check installation on vm1')
    #cmd='%s "zstack-ctl change_ip --ip="%s'%(ssh_cmd1,vm1_ip)
    #test_stub.execute_shell_in_process(cmd, tmp_file)
    #cmd='%s "zstack-ctl start"'%ssh_cmd1
    #test_stub.execute_shell_in_process(cmd, tmp_file)
    #test_stub.check_installation(vm1_ip, tmp_file)

    test_util.test_dsc('Update master iso')
    test_stub.update_iso(vm1_ip, tmp_file, iso_path, upgrade_script_path)

    test_util.test_dsc('Install zstack mangement node on vm1')
    cmd= '%s "[ -e /usr/local/zstack ] && echo yes || echo no"' % ssh_cmd1
    (process_result, cmd_stdout) = test_stub.execute_shell_in_process_stdout(cmd, tmp_file)
    if process_result != 0:
        test_util.test_fail('check /usr/local/zstack fail, cmd_stdout:%s' % cmd_stdout)
    cmd_stdout = cmd_stdout[:-1]
    if cmd_stdout == "yes":
        cmd = '%s "rm -rf /usr/local/zstack"' % ssh_cmd
        (process_result, cmd_stdout) = test_stub.execute_shell_in_process_stdout(cmd, tmp_file)
        if process_result != 0:
            test_util.test_fail('delete /usr/local/zstack fail')
    test_stub.execute_all_install(ssh_cmd1, target_file, tmp_file)

    test_util.test_dsc('Install multi management node on vm2 and vm3')
    host_list = 'root:[email protected]%s root:[email protected]%s' % (vm2_ip, vm3_ip)
    cmd = '%s "zstack-ctl add_multi_management --host-list %s"' % (ssh_cmd1, host_list)
    process_result = test_stub.execute_shell_in_process(cmd, tmp_file)

    test_util.test_dsc('Check installation on vm1')
    test_stub.check_installation(vm1_ip, tmp_file)

    test_util.test_dsc('Check installation on vm2')
    ssh.make_ssh_no_password(vm2_ip, test_lib.lib_get_vm_username(vm2_inv), \
            test_lib.lib_get_vm_password(vm2_inv))
    cmd = '%s "zstack-ctl start"' % ssh_cmd2
    process_result = test_stub.execute_shell_in_process(cmd, tmp_file)
    test_stub.check_installation(vm2_ip, tmp_file)

    test_util.test_dsc('Check installation on vm3')
    ssh.make_ssh_no_password(vm3_ip, test_lib.lib_get_vm_username(vm3_inv), \
            test_lib.lib_get_vm_password(vm3_inv))
    cmd = '%s "zstack-ctl start"' % ssh_cmd3
    process_result = test_stub.execute_shell_in_process(cmd, tmp_file)
    test_stub.check_installation(vm3_ip, tmp_file)

    #test_util.test_dsc('Upgrade multi management node on vm2 and vm3')
    #cmd = '%s "zstack-ctl upgrade_multi_management_node --installer-bin %s"' % (ssh_cmd1, target_file) 
    #process_result = test_stub.execute_shell_in_process(cmd, tmp_file)

    #test_util.test_dsc('After upgrade, check installation on vm1')
    #test_stub.check_installation(vm1_ip, tmp_file)

    #test_util.test_dsc('After upgrade, check installation on vm2')
    #ssh.make_ssh_no_password(vm2_ip, test_lib.lib_get_vm_username(vm2_inv), \
    #        test_lib.lib_get_vm_password(vm2_inv))
    #cmd = '%s "zstack-ctl start"' % ssh_cmd2
    #process_result = test_stub.execute_shell_in_process(cmd, tmp_file)
    #test_stub.check_installation(vm2_ip, tmp_file)

    #test_util.test_dsc('After upgrade, check installation on vm3')
    #ssh.make_ssh_no_password(vm3_ip, test_lib.lib_get_vm_username(vm3_inv), \
    #        test_lib.lib_get_vm_password(vm3_inv))
    #cmd = '%s "zstack-ctl start"' % ssh_cmd3
    #process_result = test_stub.execute_shell_in_process(cmd, tmp_file)
    #test_stub.check_installation(vm3_ip, tmp_file)

    os.system('rm -f %s' % tmp_file)
    sce_ops.destroy_vm(zstack_management_ip, vm1_inv.uuid)
    sce_ops.destroy_vm(zstack_management_ip, vm2_inv.uuid)
    sce_ops.destroy_vm(zstack_management_ip, vm3_inv.uuid)

#.........这里部分代码省略.........