def run_command(self, command, timeout=RUNCMD_TIMEOUT):
        logger.info("Running command `{}`".format(command))
        template = '%s\n'
        command = template % command

        try:
            session = self.get_transport().open_session()
            if timeout:
                session.settimeout(float(timeout))
            session.exec_command(command)
            stdout = session.makefile()
            stderr = session.makefile_stderr()
            output = ''
            while True:
                if session.recv_ready:
                    for line in stdout:
                        output += line
                        if self._streaming:
                            sys.stdout.write(line)

                if session.recv_stderr_ready:
                    for line in stderr:
                        output += line
                        if self._streaming:
                            sys.stderr.write(line)

                if session.exit_status_ready():
                    break
            exit_status = session.recv_exit_status()
            return SSHResult(exit_status, output)
        except paramiko.SSHException as exc:
            logger.exception(exc)

        # Returning two things so tuple unpacking the return works even if the ssh client fails
        return SSHResult(1, None)

def _setup_provider(provider_key, request=None):
    def skip(provider_key, previous_fail=False):
        if request:
            node = request.node
            name, location = get_test_idents(node)
            skip_data = {'type': 'provider', 'reason': provider_key}
            art_client.fire_hook('skip_test', test_location=location, test_name=name,
                skip_data=skip_data)
        if previous_fail:
            raise pytest.skip('Provider {} failed to set up previously in another test, '
                              'skipping test'.format(provider_key))
        else:
            raise pytest.skip('Provider {} failed to set up this time, '
                              'skipping test'.format(provider_key))
    # This function is dynamically "fixturized" to setup up a specific provider,
    # optionally skipping the provider setup if that provider has previously failed.
    if provider_key in _failed_providers:
        skip(provider_key, previous_fail=True)

    try:
        providers.setup_provider(provider_key)
    except Exception as ex:
        logger.error('Error setting up provider %s', provider_key)
        logger.exception(ex)
        _failed_providers.add(provider_key)
        skip(provider_key)

    def pytest_sessionfinish(self, exitstatus):
        # Now master/standalone needs to move all the reports to an appliance for the source report
        if store.parallelizer_role != 'master':
            manager().collect()

        # for slaves, everything is done at this point
        if store.parallelizer_role == 'slave':
            return

        # on master/standalone, merge all the collected reports and bring them back
        manager().merge()

        try:
            global ui_coverage_percent
            last_run = json.load(log_path.join('coverage', 'merged', '.last_run.json').open())
            ui_coverage_percent = last_run['result']['covered_percent']
            style = {'bold': True}
            if ui_coverage_percent > 40:
                style['green'] = True
            else:
                style['red'] = True
            store.write_line('UI Coverage Result: {}%'.format(ui_coverage_percent),
                **style)
        except Exception as ex:
            logger.error('Error printing coverage report to terminal')
            logger.exception(ex)

def appliance_preupdate(old_version, appliance):

    series = appliance.version.series()
    update_url = "update_url_{}".format(series.replace('.', ''))

    """Requests appliance from sprout based on old_versions, edits partitions and adds
    repo file for update"""

    usable = []
    sp = SproutClient.from_config()
    available_versions = set(sp.call_method('available_cfme_versions'))
    for a in available_versions:
        if a.startswith(old_version):
            usable.append(Version(a))
    usable.sort(reverse=True)
    try:
        apps, pool_id = sp.provision_appliances(count=1, preconfigured=True,
            lease_time=180, version=str(usable[0]))
    except Exception as e:
        logger.exception("Couldn't provision appliance with following error:{}".format(e))
        raise SproutException('No provision available')

    apps[0].db.extend_partition()
    urls = process_url(cfme_data['basic_info'][update_url])
    output = build_file(urls)
    with tempfile.NamedTemporaryFile('w') as f:
        f.write(output)
        f.flush()
        os.fsync(f.fileno())
        apps[0].ssh_client.put_file(
            f.name, '/etc/yum.repos.d/update.repo')
    yield apps[0]
    apps[0].ssh_client.close()
    sp.destroy_pool(pool_id)

def process_provider_vms(provider_key, matchers, delta, vms_to_delete):
    with lock:
        print '{} processing'.format(provider_key)
    try:
        now = datetime.datetime.now()
        with lock:
            # Known conf issue :)
            provider = get_mgmt(provider_key)
        for vm_name in provider.list_vm():
            if not match(matchers, vm_name):
                continue

            try:
                vm_creation_time = provider.vm_creation_time(vm_name)
            except:
                logger.error('Failed to get creation/boot time for %s on %s' % (
                    vm_name, provider_key))
                continue

            if vm_creation_time + delta < now:
                vm_delta = now - vm_creation_time
                with lock:
                    vms_to_delete[provider_key].add((vm_name, vm_delta))
        with lock:
            print '{} finished'.format(provider_key)
    except Exception as ex:
        with lock:
            # Print out the error message too because logs in the job get deleted
            print '{} failed ({}: {})'.format(provider_key, type(ex).__name__, str(ex))
        logger.error('failed to process vms from provider %s', provider_key)
        logger.exception(ex)

def cleanup_templates(api, edomain, days, max_templates):
    try:
        templates = api.storagedomains.get(edomain).templates.list()
        thread_queue = []
        delete_templates = []
        for template in templates:
            delta = datetime.timedelta(days=days)
            now = datetime.datetime.now(pytz.utc)
            template_creation_time = template.get_creation_time().astimezone(pytz.utc)

            if template.get_name().startswith('auto-tmp'):
                if now > (template_creation_time + delta):
                    delete_templates.append(template)

        if not delete_templates:
            print("RHEVM: No old templates to delete in {}".format(edomain))

        for delete_template in delete_templates[:max_templates]:
            thread = Thread(target=delete_edomain_templates,
                            args=(api, delete_template, edomain))
            thread.daemon = True
            thread_queue.append(thread)
            thread.start()

        for thread in thread_queue:
            thread.join()
    except Exception as e:
        logger.exception(e)
        return False

def get_registered_vm_files(provider_key):
    try:
        print("{} processing all the registered files..".format(provider_key))
        vm_registered_files = defaultdict(set)
        provider = get_mgmt(provider_key)
        for vm_name in provider.list_vm():
            try:
                vm_file_path = provider.get_vm_config_files_path(vm_name)
                vm_directory_name = re.findall(r'\s(.*)/\w*', vm_file_path)
                vm_registered_files[vm_directory_name[0]] = vm_name
            except Exception as e:
                logger.error(e)
                logger.error('Failed to get creation/boot time for {} on {}'.format(
                    vm_name, provider_key))
                continue
        print("\n**************************REGISTERED FILES ON {}***********************\n".format(
            provider_key))
        for k, v in vm_registered_files.items():
            print('FILE_NAME: {}\nVM_NAME: {}\n'.format(k, v))
        return vm_registered_files
    except Exception as ex:
            # Print out the error message too because logs in the job get deleted
        print('{} failed ({}: {})'.format(provider_key, type(ex).__name__, str(ex)))
        logger.error('failed to process vms from provider {}'.format(provider_key))
        logger.exception(ex)

def process_provider_vms(provider_key, matchers, delta, vms_to_delete):
    with lock:
        print '%s processing' % provider_key
    try:
        now = datetime.datetime.now()
        provider = provider_factory(provider_key)
        for vm_name in provider.list_vm():
            if not match(matchers, vm_name):
                continue

            try:
                vm_creation_time = provider.vm_creation_time(vm_name)
            except:
                logger.error('Failed to get creation/boot time for %s on %s' % (
                    vm_name, provider_key))
                continue

            if vm_creation_time + delta < now:
                vm_delta = now - vm_creation_time
                with lock:
                    vms_to_delete[provider_key].add((vm_name, vm_delta))
        with lock:
            print '%s finished' % provider_key
    except Exception as ex:
        with lock:
            print '%s failed' % provider_key
        logger.error('failed to process vms from provider %s', provider_key)
        logger.exception(ex)

def testing_instance(setup_provider, provider):
    """ Fixture to provision instance on the provider
    """
    instance = Instance.factory(random_vm_name('pwr-c'), provider)
    if not provider.mgmt.does_vm_exist(instance.name):
        instance.create_on_provider(allow_skip="default")
    elif instance.provider.type == "ec2" and \
            provider.mgmt.is_vm_state(instance.name, provider.mgmt.states['deleted']):
        provider.mgmt.set_name(
            instance.name, 'test_terminated_{}'.format(fauxfactory.gen_alphanumeric(8)))
        instance.create_on_provider(allow_skip="default", find_in_cfme=True)
    provider.refresh_provider_relationships()

    # Make sure the instance shows up
    try:
        wait_for(lambda: instance.exists,
                 fail_condition=False,
                 num_sec=600,
                 delay=15,
                 fail_func=provider.refresh_provider_relationships)
    except TimedOutError:
        pytest.fail('Failed to find instance in CFME after creating on provider: {}'
                    .format(instance.name))

    yield instance

    logger.info('Fixture cleanup, deleting test instance: %s', instance.name)
    try:
        provider.mgmt.delete_vm(instance.name)
    except Exception:
        logger.exception('Exception when deleting testing_instance: %s', instance.name)

 def _cleanup_vm():
     try:
         if initialize_provider.mgmt.does_vm_exist(name):
             initialize_provider.mgmt.delete_vm(name)
         initialize_provider.refresh_provider_relationships()
     except Exception as e:
         logger.exception(e)

def scan_provider(provider_key, matchers, match_queue, scan_failure_queue):
    """
    Process the VMs on a given provider, comparing name and creation time.
    Append vms meeting criteria to vms_to_delete

    Args:
        provider_key (string): the provider key from yaml
        matchers (list): A list of regex objects with match() method
        match_queue (Queue.Queue): MP queue to hold VMs matching age requirement
        scan_failure_queue (Queue.Queue): MP queue to hold vms that we could not compare age
    Returns:
        None: Uses the Queues to 'return' data
    """
    logger.info('%s: Start scan for vm text matches', provider_key)
    try:
        vm_list = get_mgmt(provider_key).list_vm()
    except Exception:  # noqa
        scan_failure_queue.put(VmReport(provider_key, FAIL, NULL, NULL, NULL))
        logger.exception('%s: Exception listing vms', provider_key)
        return

    text_matched_vms = [name for name in vm_list if match(matchers, name)]
    for name in text_matched_vms:
        match_queue.put(VmProvider(provider_key, name))

    non_text_matching = set(vm_list) - set(text_matched_vms)
    logger.info('%s: NOT matching text filters: %s', provider_key, non_text_matching)
    logger.info('%s: MATCHED text filters: %s', provider_key, text_matched_vms)

def get_datastores_per_host(provider_key):
    print('{} processing to get datastores per host'.format(provider_key))
    try:
        provider = get_mgmt(provider_key)

        vm_registered_files = get_registered_vm_files(provider_key)
        hosts = provider.list_host()
        host_datastore_url = {host: provider.list_host_datastore_url(host) for host in hosts}
        unregistered_files = []

        print("\n*********************UNREGISTERED FILES ON: {}**********************\n".format(
            provider_key))
        print('HOST_NAME\t\tFILE_PATH\t\tTEMPLATE_VM_ISO\t\tNUMBER_OF_FILES\n')
        for host in host_datastore_url:
            try:
                list_orphaned_files_per_host(host, host_datastore_url[host],
                                             provider_key, vm_registered_files,
                                             unregistered_files)
            except Exception as e:
                logger.error(e)
                continue

    except Exception as ex:
            # Print out the error message too because logs in the job get deleted
        print('{} failed ({}: {})'.format(provider_key, type(ex).__name__, str(ex)))
        logger.error('failed to process vms from provider {}'.format(provider_key))
        logger.exception(ex)

        def _get_canvas_element(provider):
            try:
                canvas = provider.get_remote_console_canvas()
            except ItemNotFound:
                logger.exception('Could not find canvas element.')
                return False

            return canvas

def is_ovirt_engine_running(provider_mgmt):
    try:
        stdout = make_ssh_client(provider_mgmt).run_command('service ovirt-engine status')[1]
        if 'running' not in stdout:
            return False
        return True
    except Exception as e:
        logger.exception(e)
        return False

 def stop(self):
     if self.pid is not None:
         if process_running(self.pid):
             os.kill(self.pid, SIGINT)
             os.waitpid(self.pid, 0)
             logger.info("Recording finished")
             self.pid = None
         else:
             logger.exception("Could not find recordmydesktop process #%d" % self.pid)

 def rename_vm(self, instance_name, new_name):
     instance = self._find_instance_by_name(instance_name)
     try:
         instance.update(new_name)
     except Exception as e:
         logger.exception(e)
         return instance_name
     else:
         return new_name

 def rename_vm(self, vm_name, new_vm_name):
     vm = self._get_vm(vm_name)
     try:
         vm.set_name(new_vm_name)
         vm.update()
     except Exception as e:
         logger.exception(e)
         return vm_name
     else:
         return new_vm_name

def is_ovirt_engine_running(rhevm_ip, sshname, sshpass):
    try:
        ssh_client = make_ssh_client(rhevm_ip, sshname, sshpass)
        stdout = ssh_client.run_command('service ovirt-engine status')[1]
        if 'running' not in stdout:
            return False
        return True
    except Exception as e:
        logger.exception(e)
        return False

 def connect(self):
     """连接指定IP、端口"""
     if not self.connected:
         try:
             self._sock.connect((self.domain, self.port))
         except socket.error as e:
             logger.exception(e)
         else:
             self.connected = 1
             logger.debug('TCPClient connect to {0}:{1} success.'.format(self.domain, self.port))

def deploy_template(provider_key, vm_name, template_name=None, timeout=900,
        **deploy_args):
    """
    Args:
        provider_key: Provider key on which the VM is to be created
        vm_name: Name of the VM to be deployed
        template_name: Name of the template that the VM is deployed from
    """
    allow_skip = deploy_args.pop("allow_skip", ())
    if isinstance(allow_skip, dict):
        skip_exceptions = allow_skip.keys()
        callable_mapping = allow_skip
    elif isinstance(allow_skip, basestring) and allow_skip.lower() == "default":
        skip_exceptions = (OSOverLimit, RHEVRequestError, exceptions.VMInstanceNotCloned, SSLError)
        callable_mapping = {}
    else:
        skip_exceptions = allow_skip
        callable_mapping = {}
    provider_crud = get_crud(provider_key)

    deploy_args.update(vm_name=vm_name)

    if template_name is None:
        try:
            deploy_args.update(template=provider_crud.data['small_template'])
        except KeyError:
            raise ValueError('small_template not defined for Provider {} in cfme_data.yaml'.format(
                provider_key))
    else:
        deploy_args.update(template=template_name)

    deploy_args.update(provider_crud.deployment_helper(deploy_args))

    logger.info("Getting ready to deploy VM/instance %s from template %s on provider %s",
        vm_name, deploy_args['template'], provider_crud.data['name'])
    try:
        try:
            logger.debug("Deploy args: %s", deploy_args)
            vm_name = provider_crud.mgmt.deploy_template(timeout=timeout, **deploy_args)
            logger.info("Provisioned VM/instance %s", vm_name)  # instance ID in case of EC2
        except Exception as e:
            logger.error('Could not provisioning VM/instance %s (%s: %s)',
                vm_name, type(e).__name__, str(e))
            _vm_cleanup(provider_crud.mgmt, vm_name)
            raise
    except skip_exceptions as e:
        e_c = type(e)
        if e_c in callable_mapping and not callable_mapping[e_c](e):
            raise
        # Make it visible also in the log.
        store.write_line(
            "Skipping due to a provider error: {}: {}\n".format(e_c.__name__, str(e)), purple=True)
        logger.exception(e)
        pytest.skip("{}: {}".format(e_c.__name__, str(e)))
    return vm_name