def migrate_host(self, src_host, new_host_list, ulockmgr):
        [status, vcluster_list] = self.get_all_clusterinfo()
        if not status:
            return [False, vcluster_list]
        auth_key = env.getenv('AUTH_KEY')
        res = post_to_user("/master/user/groupinfo/", {'auth_key':auth_key})
        groups = json.loads(res['groups'])
        quotas = {}
        for group in groups:
            quotas[group['name']] = group['quotas']

        for vcluster in vcluster_list:
            if 'ownername' not in vcluster.keys():
                return [Flase, 'Ownername not in vcluster(%s).keys' % str(vcluster) ]
            try:
                username = vcluster['ownername']
                ulockmgr.acquire(username)
                clustername = vcluster['clustername']
                rc_info = post_to_user("/master/user/recoverinfo/", {'username':username,'auth_key':auth_key})
                groupname = rc_info['groupname']
                user_info = {"data":{"id":rc_info['uid'],"groupinfo":quotas[groupname]}}
                self.migrate_cluster(clustername, username, src_host, new_host_list, user_info)
            except Exception as ex:
                ulockmgr.release(username)
                logger.error(traceback.format_exc())
                return [False, str(ex)]
            ulockmgr.release(username)
        return [True, ""]

def _setup_provider(provider_key, request=None):
    def skip(provider_key, previous_fail=False):
        if request:
            node = request.node
            name, location = get_test_idents(node)
            skip_data = {'type': 'provider', 'reason': provider_key}
            art_client.fire_hook('skip_test', test_location=location, test_name=name,
                skip_data=skip_data)
        if previous_fail:
            raise pytest.skip('Provider {} failed to set up previously in another test, '
                              'skipping test'.format(provider_key))
        else:
            raise pytest.skip('Provider {} failed to set up this time, '
                              'skipping test'.format(provider_key))
    # This function is dynamically "fixturized" to setup up a specific provider,
    # optionally skipping the provider setup if that provider has previously failed.
    if provider_key in _failed_providers:
        skip(provider_key, previous_fail=True)

    try:
        providers.setup_provider(provider_key)
    except Exception as ex:
        logger.error('Error setting up provider %s', provider_key)
        logger.exception(ex)
        _failed_providers.add(provider_key)
        skip(provider_key)

def test_tables_fields(provider, test_item, soft_assert):

    navigate_to(test_item.obj, 'All')
    tb.select('List View')
    # NOTE: We must re-instantiate here table
    # in order to prevent StaleElementException or UsingSharedTables
    # TODO: Switch to widgetastic
    paged_tbl = PagedTable(table_locator="//div[@id='list_grid']//table")
    for row in paged_tbl.rows():
        cell = row[2]  # We're using indexing since it could be either 'name' or 'host'
        if cell:
            name = cell.text
        else:
            logger.error('Could not find NAME header on {}s list...'
                         .format(test_item.obj.__name__))
            continue
        for field in test_item.fields_to_verify:

            try:
                value = getattr(row, field)
            except AttributeError:
                soft_assert(False, '{}\'s list table: field  not exist: {}'
                            .format(test_item.obj.__name__, field))
                continue

            soft_assert(value, '{}\'s list table: {} row - has empty field: {}'
                        .format(test_item.obj.__name__, name, field))

    def stop_vnode(self, request, context):
        logger.info('stop vnode with config: ' + str(request))
        taskid = request.taskid
        username = request.username
        vnodeid = request.vnodeid
        brname = request.vnode.network.brname
        mount_list = request.vnode.mount
        lxcname = '%s-batch-%s-%s' % (username,taskid,str(vnodeid))

        logger.info("Stop the task with lxc:"+lxcname)
        container = lxc.Container(lxcname)
        if container.stop():
            logger.info("stop container %s success" % lxcname)
        else:
            logger.error("stop container %s failed" % lxcname)

        #umount oss
        self.umount_oss("/var/lib/lxc/%s/oss" % (lxcname), mount_list)

        logger.info("deleting container:%s" % lxcname)
        if self.imgmgr.deleteFS(lxcname):
            logger.info("delete container %s success" % lxcname)
        else:
            logger.error("delete container %s failed" % lxcname)

        #del ovs bridge
        if brname is not None:
            netcontrol.del_bridge(brname)

        #release gpu
        self.release_gpu_device(lxcname)

        return rpc_pb2.Reply(status=rpc_pb2.Reply.ACCEPTED,message="")

def pytest_runtest_teardown(item, nextitem):
    name, location = get_test_idents(item)
    app = get_or_create_current_appliance()
    ip = app.address
    fire_art_test_hook(
        item, 'finish_test',
        slaveid=store.slaveid, ip=ip, wait_for_task=True)
    fire_art_test_hook(item, 'sanitize', words=words)
    jenkins_data = {
        'build_url': os.environ.get('BUILD_URL'),
        'build_number': os.environ.get('BUILD_NUMBER'),
        'git_commit': os.environ.get('GIT_COMMIT'),
        'job_name': os.environ.get('JOB_NAME')
    }
    try:
        caps = app.browser.widgetastic.selenium.capabilities
        param_dict = {
            'browserName': caps['browserName'],
            'browserPlatform': caps['platform'],
            'browserVersion': caps['version']
        }
    except Exception as e:
        logger.error(e)
        param_dict = None

    fire_art_test_hook(
        item, 'ostriz_send', env_params=param_dict,
        slaveid=store.slaveid, polarion_ids=extract_polarion_ids(item), jenkins=jenkins_data)

 def _custom_click_handler(self, wait_ajax):
     """Handler called from pytest_selenium"""
     if self.is_dimmed and not self._force:
         logger.error("Could not click %s because it was dimmed", repr(self))
         return
     sel.wait_for_element(self, timeout=5)
     return sel.click(self, no_custom_handler=True, wait_ajax=wait_ajax)

 def migrate_cluster(self, clustername, username, src_host, new_host_list, user_info):
     [status, info] = self.get_clusterinfo(clustername, username)
     if not status:
         return [False, "cluster not found"]
     prestatus = info['status']
     self.stop_cluster(clustername, username)
     for container in info['containers']:
         if not container['host'] == src_host:
             continue
         random.shuffle(new_host_list)
         for new_host in new_host_list:
             status,msg = self.migrate_container(clustername,username,container['containername'],new_host,user_info)
             if status:
                 break
             else:
                 logger.error(msg)
         else:
             if prestatus == 'running':
                 self.start_cluster(clustername, username, user_info)
             return [False, msg]
     logger.info("[Migrate] prestatus:%s for cluster(%s) user(%s)"%(prestatus, clustername, username))
     if prestatus == 'running':
         status, msg = self.start_cluster(clustername, username, user_info)
         if not status:
             return [False, msg]
     return [True, ""]

    def createImage(self,user,image,lxc,description="Not thing", imagenum=10):
        fspath = self.NFS_PREFIX + "/local/volume/" + lxc
        imgpath = self.imgpath + "private/" + user + "/"
        #tmppath = self.NFS_PREFIX + "/local/tmpimg/"
        #tmpimage = str(random.randint(0,10000000)) + ".tz"

        if not os.path.exists(imgpath+image) and os.path.exists(imgpath):
            cur_imagenum = 0
            for filename in os.listdir(imgpath):
                if os.path.isdir(imgpath+filename):
                    cur_imagenum += 1
            if cur_imagenum >= int(imagenum):
                return [False,"image number limit exceeded"]
        #sys_run("mkdir -p %s" % tmppath, True)
        sys_run("mkdir -p %s" % imgpath,True)
        try:
            sys_run("tar -cvf %s -C %s ." % (imgpath+image+".tz",self.dealpath(fspath)), True)
        except Exception as e:
            logger.error(e)
        #try:
            #sys_run("cp %s %s" % (tmppath+tmpimage, imgpath+image+".tz"), True)
            #sys_run("rsync -a --delete --exclude=lost+found/ --exclude=root/nfs/ --exclude=dev/ --exclude=mnt/ --exclude=tmp/ --exclude=media/ --exclude=proc/ --exclude=sys/ %s/ %s/" % (self.dealpath(fspath),imgpath+image),True)
        #except Exception as e:
        #    logger.error(e)
        #sys_run("rm -f %s" % tmppath+tmpimage, True)
        #sys_run("rm -f %s" % (imgpath+"."+image+"_docklet_share"),True)
        self.updateinfo(user,image,description)
        logger.info("image:%s from LXC:%s create success" % (image,lxc))
        return [True, "create image success"]

def db_commit():
    try:
        db.session.commit()
    except Exception as err:
        db.session.rollback()
        logger.error(traceback.format_exc())
        raise

 def copyImage(self,user,image,token,target):
     path = "/opt/docklet/global/images/private/"+user+"/"
     '''image_info_file = open(path+"."+image+".info", 'r')
     [createtime, isshare] = image_info_file.readlines()
     recordshare = isshare
     isshare = "unshared"
     image_info_file.close()
     image_info_file = open(path+"."+image+".info", 'w')
     image_info_file.writelines([createtime, isshare])
     image_info_file.close()'''
     try:
         sys_run('ssh [email protected]%s "mkdir -p %s"' % (target,path))
         sys_run('scp %s%s.tz [email protected]%s:%s' % (path,image,target,path))
         #sys_run('scp %s.%s.description [email protected]%s:%s' % (path,image,target,path))
         #sys_run('scp %s.%s.info [email protected]%s:%s' % (path,image,target,path))
         resimage = Image.query.filter_by(ownername=user,imagename=image).first()
         auth_key = env.getenv('AUTH_KEY')
         url = "http://" + target + ":" + master_port + "/image/copytarget/"
         data = {"token":token,"auth_key":auth_key,"user":user,"imagename":image,"description":resimage.description}
         result = requests.post(url, data=data).json()
         logger.info("Response from target master: " + str(result))
     except Exception as e:
         logger.error(e)
         '''image_info_file = open(path+"."+image+".info", 'w')
         image_info_file.writelines([createtime, recordshare])
         image_info_file.close()'''
         return {'success':'false', 'message':str(e)}
     '''image_info_file = open(path+"."+image+".info", 'w')
     image_info_file.writelines([createtime, recordshare])
     image_info_file.close()'''
     logger.info("copy image %s of %s to %s success" % (image,user,target))
     return {'success':'true', 'action':'copy image'}

 def detachFS(self, lxc, vgname="docklet-group"):
     rootfs = "/var/lib/lxc/%s/rootfs" % lxc
     Ret = sys_run("umount %s" % rootfs)
     if Ret.returncode != 0:
         logger.error("cannot umount rootfs:%s" % rootfs)
         return False
     return True

    def prepareImage(self,user,image,fspath):
        imagename = image['name']
        imagetype = image['type']
        imageowner = image['owner']
        #tmppath = self.NFS_PREFIX + "/local/tmpimg/"
        #tmpimage = str(random.randint(0,10000000)) + ".tz"
        if imagename == "base" and imagetype == "base":
            return
        if imagetype == "private":
            imgpath = self.imgpath + "private/" + user + "/"
        else:
            imgpath = self.imgpath + "public/" + imageowner + "/"
        #try:
        #    sys_run("cp %s %s" % (imgpath+imagename+".tz", tmppath+tmpimage))
        #except Exception as e:
        #    logger.error(e)
        try:
            sys_run("tar -C %s -xvf %s" % (self.dealpath(fspath),imgpath+imagename+".tz"), True)
            #sys_run("rsync -a --delete --exclude=lost+found/ --exclude=root/nfs/ --exclude=dev/ --exclude=mnt/ --exclude=tmp/ --exclude=media/ --exclude=proc/ --exclude=sys/ %s/ %s/" % (imgpath+imagename,self.dealpath(fspath)),True)
        except Exception as e:
            logger.error(e)
        #sys_run("rm -f %s" % tmppath+tmpimage)

        #self.sys_call("rsync -a --delete --exclude=nfs/ %s/ %s/" % (imgpath+image,self.dealpath(fspath)))
        #self.updatetime(imgpath,image)
        return

def set_rails_loglevel(level, validate_against_worker='MiqUiWorker'):
    """Sets the logging level for level_rails and detects when change occured."""
    ui_worker_pid = '#{}'.format(get_worker_pid(validate_against_worker))

    logger.info('Setting log level_rails on appliance to {}'.format(level))
    yaml = store.current_appliance.get_yaml_config()
    if not str(yaml['log']['level_rails']).lower() == level.lower():
        logger.info('Opening /var/www/miq/vmdb/log/evm.log for tail')
        evm_tail = SSHTail('/var/www/miq/vmdb/log/evm.log')
        evm_tail.set_initial_file_end()

        yaml['log']['level_rails'] = level
        store.current_appliance.set_yaml_config(yaml)

        attempts = 0
        detected = False
        while (not detected and attempts < 60):
            logger.debug('Attempting to detect log level_rails change: {}'.format(attempts))
            for line in evm_tail:
                if ui_worker_pid in line:
                    if 'Log level for production.log has been changed to' in line:
                        # Detects a log level change but does not validate the log level
                        logger.info('Detected change to log level for production.log')
                        detected = True
                        break
            time.sleep(1)  # Allow more log lines to accumulate
            attempts += 1
        if not (attempts < 60):
            # Note the error in the logger but continue as the appliance could be slow at logging
            # that the log level changed
            logger.error('Could not detect log level_rails change.')
        evm_tail.close()
    else:
        logger.info('Log level_rails already set to {}'.format(level))

def main(args):
    try:
        readconfig(args['--config-file'])
    except Exception, e:
        logger.error("Error reading config file from location: {0}".format(args['--config-file']))
        logger.error(str(e))
        sys.exit(1)

def map_vms_to_ids(provider_names_to_vm_names):
    """Takes a dictionary of providers with a list of vms and generates a list of vm_ids for each
    vm in the data structure.  We need this because more than one provider can lead to a """
    starttime = time.time()
    expected_num_ids = sum(len(x) for x in provider_names_to_vm_names.itervalues())
    expected_num_providers = len(provider_names_to_vm_names.keys())
    # Intended ouput here (List of vm ids):
    vm_ids = []
    # Intermediate data structure holding provider_id to list of vm names
    provider_ids_to_vm_names = {}

    # First get all providers details
    all_providers_details = []
    for pro_id in get_all_provider_ids():
        details = get_provider_details(pro_id)
        all_providers_details.append(details)

    providers_to_vms_copy = dict(provider_names_to_vm_names)
    # Next map provider_name to the provider_id
    for provider_name in provider_names_to_vm_names:
        for provider_detail in all_providers_details:
            if provider_name == provider_detail['name']:
                # Copy VMs from that provider to the Intermediate data structure
                provider_ids_to_vm_names[provider_detail['id']] = list(
                    provider_names_to_vm_names[provider_name])
                del providers_to_vms_copy[provider_name]
                break

    if len(providers_to_vms_copy) > 0:
        # Error, we did not find all providers, likely there is an issue with the scenario data
        # inside of cfme_performance.yml or cfme_performance.local.yml
        logger.error('Provider(s) + vm(s) not found in CFME Inventory: {}'.format(
            providers_to_vms_copy))

    provider_ids_to_vm_names_copy = copy.deepcopy(provider_ids_to_vm_names)
    # Now map each vm_name+ems_id to the actual vm_id and append to our list
    for vm_id in get_all_vm_ids():
        vm_details = get_vm_details(vm_id)
        for provider_id in provider_ids_to_vm_names:
            if ('ems_id' in vm_details and provider_id == vm_details['ems_id']):
                # Match provider_id, now check vm_name
                for vm_name in provider_ids_to_vm_names[provider_id]:
                    if vm_name == vm_details['name']:
                        logger.debug('Matching {} to vm id: {}'.format(vm_name, vm_id))
                        vm_ids.append(vm_id)
                        del (provider_ids_to_vm_names_copy[provider_id]
                            [provider_ids_to_vm_names_copy[provider_id].index(vm_name)])
                        break
        if (sum(len(x) for x in provider_ids_to_vm_names_copy.itervalues()) == 0):
            break

    # Now check for left over vms that we did not match:
    leftover_num_ids = sum(len(x) for x in provider_ids_to_vm_names_copy.itervalues())
    if leftover_num_ids > 0:
        logger.error('(Provider_id(s)) + VM(s) not found in CFME inventory: {}'.format(
            provider_ids_to_vm_names_copy))
    logger.debug('Mapped {}/{} vm ids/names over {}/{} provider ids/names in {}s'.format(
        len(vm_ids), expected_num_ids, len(provider_ids_to_vm_names.keys()), expected_num_providers,
        round(time.time() - starttime, 2)))
    return vm_ids

def process_provider_vms(provider_key, matchers, delta, vms_to_delete):
    with lock:
        print '{} processing'.format(provider_key)
    try:
        now = datetime.datetime.now()
        with lock:
            # Known conf issue :)
            provider = get_mgmt(provider_key)
        for vm_name in provider.list_vm():
            if not match(matchers, vm_name):
                continue

            try:
                vm_creation_time = provider.vm_creation_time(vm_name)
            except:
                logger.error('Failed to get creation/boot time for %s on %s' % (
                    vm_name, provider_key))
                continue

            if vm_creation_time + delta < now:
                vm_delta = now - vm_creation_time
                with lock:
                    vms_to_delete[provider_key].add((vm_name, vm_delta))
        with lock:
            print '{} finished'.format(provider_key)
    except Exception as ex:
        with lock:
            # Print out the error message too because logs in the job get deleted
            print '{} failed ({}: {})'.format(provider_key, type(ex).__name__, str(ex))
        logger.error('failed to process vms from provider %s', provider_key)
        logger.exception(ex)

def get_datastores_per_host(provider_key):
    print('{} processing to get datastores per host'.format(provider_key))
    try:
        provider = get_mgmt(provider_key)

        vm_registered_files = get_registered_vm_files(provider_key)
        hosts = provider.list_host()
        host_datastore_url = {host: provider.list_host_datastore_url(host) for host in hosts}
        unregistered_files = []

        print("\n*********************UNREGISTERED FILES ON: {}**********************\n".format(
            provider_key))
        print('HOST_NAME\t\tFILE_PATH\t\tTEMPLATE_VM_ISO\t\tNUMBER_OF_FILES\n')
        for host in host_datastore_url:
            try:
                list_orphaned_files_per_host(host, host_datastore_url[host],
                                             provider_key, vm_registered_files,
                                             unregistered_files)
            except Exception as e:
                logger.error(e)
                continue

    except Exception as ex:
            # Print out the error message too because logs in the job get deleted
        print('{} failed ({}: {})'.format(provider_key, type(ex).__name__, str(ex)))
        logger.error('failed to process vms from provider {}'.format(provider_key))
        logger.exception(ex)

def process_provider_vms(provider_key, matchers, delta, vms_to_delete):
    with lock:
        print '%s processing' % provider_key
    try:
        now = datetime.datetime.now()
        provider = provider_factory(provider_key)
        for vm_name in provider.list_vm():
            if not match(matchers, vm_name):
                continue

            try:
                vm_creation_time = provider.vm_creation_time(vm_name)
            except:
                logger.error('Failed to get creation/boot time for %s on %s' % (
                    vm_name, provider_key))
                continue

            if vm_creation_time + delta < now:
                vm_delta = now - vm_creation_time
                with lock:
                    vms_to_delete[provider_key].add((vm_name, vm_delta))
        with lock:
            print '%s finished' % provider_key
    except Exception as ex:
        with lock:
            print '%s failed' % provider_key
        logger.error('failed to process vms from provider %s', provider_key)
        logger.exception(ex)

def save_cluster(user, beans, form):
    global G_vclustermgr
    clustername = form.get('clustername', None)
    if (clustername == None):
        return json.dumps({'success':'false', 'message':'clustername is null'})

    imagename = form.get("image", None)
    description = form.get("description", None)
    containername = form.get("containername", None)
    isforce = form.get("isforce", None)
    G_ulockmgr.acquire(user)
    try:
        if not isforce == "true":
            [status,message] = G_vclustermgr.image_check(user,imagename)
            if not status:
                return json.dumps({'success':'false','reason':'exists', 'message':message})

        user_info = post_to_user("/user/selfQuery/", {'token':form.get("token")})
        [status,message] = G_vclustermgr.create_image(user,clustername,containername,imagename,description,user_info["data"]["groupinfo"]["image"])
        if status:
            logger.info("image has been saved")
            return json.dumps({'success':'true', 'action':'save'})
        else:
            logger.debug(message)
            return json.dumps({'success':'false', 'reason':'exceed', 'message':message})
    except Exception as ex:
        logger.error(str(ex))
        return json.dumps({'success':'false', 'message': str(ex)})
    finally:
        G_ulockmgr.release(user)

 def add_msg(self,taskid,username,instanceid,status,token,errmsg):
     self.msgslock.acquire()
     try:
         self.taskmsgs.append(rpc_pb2.TaskMsg(taskid=str(taskid),username=username,instanceid=int(instanceid),instanceStatus=status,token=token,errmsg=errmsg))
     except Exception as err:
         logger.error(traceback.format_exc())
     self.msgslock.release()