def setup(sphinx):
    """Main sphinx entry point, calls sphinx-apidoc"""
    for module in modules_to_document:
        module_path = project_path.join(module).strpath
        tests_exclude_path = project_path.join(module, "tests").strpath
        output_module_path = _doc_modules_path.join(module).strpath

        # Shove stdout into a pipe to supress the output, but still let stderr out
        args = ["sphinx-apidoc", "-T", "-e", "-o", output_module_path, module_path, tests_exclude_path]
        proc = subprocess.Popen(args, stdout=subprocess.PIPE)
        proc.wait()
    sphinx.add_config_value("clean_autogenerated_docs", False, rebuild="")
    sphinx.connect("build-finished", purge_module_apidoc)

def compute_coverage(branch):
    coverage_data = CoverageData()
    try:
        coverage_data.read_file(project_path.join('.coverage').strpath)
    except misc.CoverageException:
        sys.stderr.write("No coverage data found")

    git_proc = subprocess.Popen(['git', 'diff', '-U0', branch],
        stdout=subprocess.PIPE, stderr=subprocess.PIPE)
    git_output = git_proc.stdout.read()
    files = git_output.split("diff --git")

    from collections import defaultdict
    file_data = defaultdict(list)

    for the_file in files:
        filenames = re.findall('a/(.*?) b/(.*)', the_file)
        if not filenames:
            continue
        filename = project_path.join(filenames[0][1])
        if '.py' not in filename.strpath:
            continue
        the_file += "git_output_checker"
        the_diffs = re.findall('(@@.*[email protected]@.*?([email protected]@|git_output_checker))', the_file, re.M | re.S, )
        for diff in the_diffs:
            diff_args = re.match('@@ -(\d+)(,(\d+))*\s+\+(\d+)(,(\d+))*', diff).groups()
            if diff_args[5]:
                for extra_line in range(int(diff_args[5])):
                    file_data[filename].append(extra_line + int(diff_args[3]))
            else:
                file_data[filename].append(int(diff_args[3]))

    line_count = 0
    completed_lines = 0
    for file_changed, lines in file_data.iteritems():
        for line in lines:
            line_count += 1
            used_lines = coverage_data.lines(file_changed)
            if not used_lines:
                continue
            if isinstance(used_lines, int):
                used_lines = set([used_lines])
            else:
                used_lines = set(used_lines)
            if line in used_lines:
                completed_lines += 1

    return float(completed_lines) / line_count * 100

def pytest_configure(config):
    if config.getoption("appliances"):
        return
    if not config.getoption('--use-sprout'):
        return

    provision_request = SproutProvisioningRequest.from_config(config)

    mgr = config._sprout_mgr = SproutManager()
    requested_appliances = mgr.request_appliances(provision_request)
    config.option.appliances[:] = []
    appliances = config.option.appliances
    # Push an appliance to the stack to have proper reference for test collection
    # FIXME: this is a bad hack based on the need for controll of collection partitioning
    appliance_stack.push(
        IPAppliance(address=requested_appliances[0]["ip_address"]))
    log.info("Appliances were provided:")
    for appliance in requested_appliances:
        url = "https://{}/".format(appliance["ip_address"])
        appliances.append(url)
        log.info("- %s is %s", url, appliance['name'])

    mgr.reset_timer()
    # Set the base_url for collection purposes on the first appliance
    conf.runtime["env"]["base_url"] = appliances[0]
    # Retrieve and print the template_name for Jenkins to pick up
    template_name = requested_appliances[0]["template_name"]
    conf.runtime["cfme_data"]["basic_info"]["appliance_template"] = template_name
    log.info("appliance_template: %s", template_name)
    with project_path.join('.appliance_template').open('w') as template_file:
        template_file.write('export appliance_template="{}"'.format(template_name))
    log.info("Sprout setup finished.")

def pytest_configure(config):
    global appliance
    global pool_id
    global sprout
    yield
    if not config.option.appliances and (config.option.use_sprout
            and config.option.sprout_appliances == 1):
        terminal = reporter()
        sprout = SproutClient.from_config()
        terminal.write("Requesting single appliance from sprout...\n")
        pool_id = sprout.request_appliances(
            config.option.sprout_group,
            count=config.option.sprout_appliances,
            version=config.option.sprout_version,
            date=config.option.sprout_date,
            lease_time=config.option.sprout_timeout
        )
        terminal.write("Appliance pool {}. Waiting for fulfillment ...\n".format(pool_id))
        at_exit(destroy_the_pool)
        if config.option.sprout_desc is not None:
            sprout.set_pool_description(pool_id, str(config.option.sprout_desc))
        try:
            result = wait_for(
                lambda: sprout.request_check(pool_id)["fulfilled"],
                num_sec=config.option.sprout_provision_timeout * 60,
                delay=5,
                message="requesting appliance was fulfilled"
            )
        except:
            pool = sprout.request_check(pool_id)
            dump_pool_info(lambda x: terminal.write("{}\n".format(x)), pool)
            terminal.write("Destroying the pool on error.\n")
            sprout.destroy_pool(pool_id)
            raise
        terminal.write("Provisioning took {0:.1f} seconds\n".format(result.duration))
        request = sprout.request_check(pool_id)
        ip_address = request["appliances"][0]["ip_address"]
        terminal.write("Appliance requested at address {} ...\n".format(ip_address))
        reset_timer(sprout, pool_id, config.option.sprout_timeout)
        terminal.write("Appliance lease timer is running ...\n")
        appliance = IPAppliance(address=ip_address)
        # Retrieve and print the template_name for Jenkins to pick up
        template_name = request["appliances"][0]["template_name"]
        conf.runtime["cfme_data"]["basic_info"]["appliance_template"] = template_name
        terminal.write("appliance_template=\"{}\";\n".format(template_name))
        with project_path.join('.appliance_template').open('w') as template_file:
            template_file.write('export appliance_template="{}"'.format(template_name))
        terminal.write("Single appliance Sprout setup finished.\n")
        # And set also the appliances_provider
        provider = request["appliances"][0]["provider"]
        terminal.write("appliance_provider=\"{}\";\n".format(provider))
        conf.runtime["cfme_data"]["basic_info"]["appliances_provider"] = provider

def pytest_configure(config):
    path = cfme_data.get("cfme_annotations_path")
    if path:
        to_parse = project_path.join(path)
        parsed = parse(to_parse)
        if not parsed:
            store.terminalreporter.line("no test annotation found in {}".format(to_parse), yellow=True)
    else:
        store.terminalreporter.line("no test annotation found in {}".format(path), yellow=True)
        parsed = []
    config.pluginmanager.register(MarkFromMap.from_parsed_list(parsed, "tier", pytest.mark.tier))
    config.pluginmanager.register(MarkFromMap.from_parsed_list(parsed, "requirement", pytest.mark.requirement))
    config.pluginmanager.register(MarkFromMap.from_parsed_list(parsed, "type", pytest.mark.__getattr__))

def pytest_configure(config):
    path = cfme_data.get('cfme_annotations_path')
    if path:
        to_parse = project_path.join(path)
        parsed = parse(to_parse)
        if not parsed:
            store.terminalreporter.line('no test annotation found in %s' % to_parse, yellow=True)
    else:
        store.terminalreporter.line('no test annotation found in %s' % path, yellow=True)
        parsed = []
    config.pluginmanager.register(MarkFromMap.from_parsed_list(
        parsed, 'tier', pytest.mark.tier))
    config.pluginmanager.register(MarkFromMap.from_parsed_list(
        parsed, 'requirement', pytest.mark.requirement))
    config.pluginmanager.register(MarkFromMap.from_parsed_list(parsed, 'type',
                                                               pytest.mark.__getattr__))

def pytest_configure(config):
    global proc
    if not SLAVEID and not proc and isinstance(art_client, ArtifactorClient):
        import subprocess
        path = project_path.join('utils', 'artifactor_start.py')
        cmd = [path.strpath]
        cmd.append('--port')
        cmd.append(str(art_client.port))
        if config.getvalue('run_id'):
            cmd.append('--run-id')
            cmd.append(str(config.getvalue('run_id')))
        proc = subprocess.Popen(cmd)
        wait_for(net_check, func_args=[art_client.port, '127.0.0.1'], func_kwargs={'force': True},
                 num_sec=10, message="wait for artifactor to start")
        config.option.artifactor_port = art_client.port
    elif isinstance(art_client, ArtifactorClient):
        art_client.port = config.option.artifactor_port

def get_template_from_config(template_config_name):
    """
    Convenience function to grab the details for a template from the yamls.
    """

    template_config = conf.cfme_data.get('customization_templates', {})[template_config_name]

    script_data = load_data_file(str(project_path.join(template_config['script_file'])),
                                 replacements=template_config['replacements'])

    script_data = script_data.read()

    return CustomizationTemplate(name=template_config['name'],
                                 description=template_config['description'],
                                 image_type=template_config['image_type'],
                                 script_type=template_config['script_type'],
                                 script_data=script_data)

import json
import os
import os.path
from datetime import datetime

from artifactor.plugins.post_result import test_report
from utils import read_env
from utils.path import project_path
from utils.trackerbot import post_jenkins_result

job_name = os.environ["JOB_NAME"]
number = int(os.environ["BUILD_NUMBER"])
date = str(datetime.now())

# reduce returns to bools for easy logic
runner_src = read_env(project_path.join(".jenkins_runner_result"))
runner_return = runner_src.get("RUNNER_RETURN", "1") == "0"
test_return = runner_src.get("TEST_RETURN", "1") == "0"


# 'stream' environ is set by jenkins for all stream test jobs
# but not in the template tester
if job_name not in (
    "template-tester",
    "template-tester-openstack",
    "template-tester-rhevm",
    "template-tester-virtualcenter",
):
    # try to pull out the appliance template name
    template_src = read_env(project_path.join(".appliance_template"))
    template = template_src.get("appliance_template", "Unknown")

        file_part = caseid[: -needle - 1].replace(".", os.sep)
    else:
        file_part = caseid

    return "{}.py::{}{}".format(file_part, attribute_part, parameter_part)


def _clean(mapping):
    mapping.pop("", "")
    try:
        return {
            "requirement": int(mapping["Requirement"]),
            "tier": int(mapping["TestTier"]),
            "id": generate_nodeid(mapping),
            "type": mapping["TestType"].lower(),
        }
    except (TypeError, ValueError):
        return None


def parse(path):
    if not path.check():
        return []
    with path.open() as fp:
        return filter(None, map(_clean, csv.DictReader(fp)))


if __name__ == "__main__":
    mapping_file = project_path.join(py.std.sys.argv[1])
    print(yaml.dump(parse(mapping_file), default_flow_style=False))

def import_all_modules_of(loc):
    path = project_path.join('{}'.format(loc.replace('.', '/'))).strpath
    for _, name, _ in pkgutil.iter_modules([path]):
        importlib.import_module('{}.{}'.format(loc, name))

    def __init__(self, config):
        self.config = config
        self.session = None
        self.session_finished = False
        self.countfailures = 0
        self.collection = OrderedDict()
        self.sent_tests = 0
        self.log = create_sublogger('master')
        self.maxfail = config.getvalue("maxfail")
        self._failed_collection_errors = {}
        self.terminal = store.terminalreporter
        self.trdist = None
        self.slaves = SlaveDict()
        self.slave_urls = SlaveDict()
        self.slave_tests = defaultdict(set)
        self.test_groups = self._test_item_generator()
        self.failed_slave_test_groups = deque()
        self.slave_spawn_count = 0
        self.sprout_client = None
        self.sprout_timer = None
        self.sprout_pool = None
        if not self.config.option.use_sprout:
            # Without Sprout
            self.appliances = self.config.option.appliances
        else:
            # Using sprout
            self.sprout_client = SproutClient.from_config()
            self.terminal.write(
                "Requesting {} appliances from Sprout at {}\n".format(
                    self.config.option.sprout_appliances, self.sprout_client.api_entry))
            pool_id = self.sprout_client.request_appliances(
                self.config.option.sprout_group,
                count=self.config.option.sprout_appliances,
                version=self.config.option.sprout_version,
                date=self.config.option.sprout_date,
                lease_time=self.config.option.sprout_timeout
            )
            self.terminal.write("Pool {}. Waiting for fulfillment ...\n".format(pool_id))
            self.sprout_pool = pool_id
            at_exit(self.sprout_client.destroy_pool, self.sprout_pool)
            if self.config.option.sprout_desc is not None:
                self.sprout_client.set_pool_description(
                    pool_id, str(self.config.option.sprout_desc))
            try:
                result = wait_for(
                    lambda: self.sprout_client.request_check(self.sprout_pool)["fulfilled"],
                    num_sec=self.config.option.sprout_provision_timeout * 60,
                    delay=5,
                    message="requesting appliances was fulfilled"
                )
            except:
                pool = self.sprout_client.request_check(self.sprout_pool)
                dump_pool_info(lambda x: self.terminal.write("{}\n".format(x)), pool)
                self.terminal.write("Destroying the pool on error.\n")
                self.sprout_client.destroy_pool(pool_id)
                raise
            else:
                pool = self.sprout_client.request_check(self.sprout_pool)
                dump_pool_info(lambda x: self.terminal.write("{}\n".format(x)), pool)
            self.terminal.write("Provisioning took {0:.1f} seconds\n".format(result.duration))
            request = self.sprout_client.request_check(self.sprout_pool)
            self.appliances = []
            # Push an appliance to the stack to have proper reference for test collection
            IPAppliance(address=request["appliances"][0]["ip_address"]).push()
            self.terminal.write("Appliances were provided:\n")
            for appliance in request["appliances"]:
                url = "https://{}/".format(appliance["ip_address"])
                self.appliances.append(url)
                self.terminal.write("- {} is {}\n".format(url, appliance['name']))
            map(lambda a: "https://{}/".format(a["ip_address"]), request["appliances"])
            self._reset_timer()
            # Set the base_url for collection purposes on the first appliance
            conf.runtime["env"]["base_url"] = self.appliances[0]
            # Retrieve and print the template_name for Jenkins to pick up
            template_name = request["appliances"][0]["template_name"]
            conf.runtime["cfme_data"]["basic_info"]["appliance_template"] = template_name
            self.terminal.write("appliance_template=\"{}\";\n".format(template_name))
            with project_path.join('.appliance_template').open('w') as template_file:
                template_file.write('export appliance_template="{}"'.format(template_name))
            self.terminal.write("Parallelized Sprout setup finished.\n")
            self.slave_appliances_data = {}
            for appliance in request["appliances"]:
                self.slave_appliances_data[appliance["ip_address"]] = (
                    appliance["template_name"], appliance["provider"]
                )

        # set up the ipc socket
        zmq_endpoint = 'tcp://127.0.0.1:{}'.format(random_port())
        ctx = zmq.Context.instance()
        self.sock = ctx.socket(zmq.ROUTER)
        self.sock.bind('%s' % zmq_endpoint)

        # clean out old slave config if it exists
        slave_config = conf_path.join('slave_config.yaml')
        slave_config.check() and slave_config.remove()

        # write out the slave config
        conf.runtime['slave_config'] = {
            'args': self.config.args,
            'options': self.config.option.__dict__,
#.........这里部分代码省略.........

    def __int__(self):
        # handling int(x)
        return self.rc

    def __cmp__(self, other):
        # Handling comparison to strings or numbers
        if isinstance(other, int):
            return cmp(self.rc, other)
        elif isinstance(other, basestring):
            return cmp(self.output, other)
        else:
            raise ValueError('You can only compare SSHResult with str or int')


_ssh_key_file = project_path.join('.generated_ssh_key')
_ssh_pubkey_file = project_path.join('.generated_ssh_key.pub')

_client_session = []


class SSHClient(paramiko.SSHClient):
    """paramiko.SSHClient wrapper

    Allows copying/overriding and use as a context manager
    Constructor kwargs are handed directly to paramiko.SSHClient.connect()

    If ``container`` param is specified, then it is assumed that the VM hosts a container of CFME.
    The ``container`` param then contains the name of the container.
    """
    def __init__(self, stream_output=False, **connect_kwargs):

#.........这里部分代码省略.........
                self.sprout_client.set_pool_description(
                    pool_id, str(self.config.option.sprout_desc))

            def detailed_check():
                try:
                    result = self.sprout_client.request_check(self.sprout_pool)
                except SproutException as e:
                    # TODO: ensure we only exit this way on sprout usage
                    try:
                        self.sprout_client.destroy_pool(pool_id)
                    except Exception:
                        pass
                    self.println(
                        "sprout pool could not be fulfilled\n{}".format(e))
                    pytest.exit(1)

                self.println("[{now:%H:%M}] fulfilled at {progress:2}%".format(
                    now=datetime.now(),
                    progress=result['progress']
                ))
                return result["fulfilled"]
            try:
                result = wait_for(
                    detailed_check,
                    num_sec=self.config.option.sprout_provision_timeout * 60,
                    delay=5,
                    message="requesting appliances was fulfilled"
                )
            except Exception:
                pool = self.sprout_client.request_check(self.sprout_pool)
                dump_pool_info(self.println, pool)
                self.println("Destroying the pool on error.")
                self.sprout_client.destroy_pool(pool_id)
                raise
            else:
                pool = self.sprout_client.request_check(self.sprout_pool)
                dump_pool_info(self.println, pool)
            self.println("Provisioning took {0:.1f} seconds".format(result.duration))
            request = self.sprout_client.request_check(self.sprout_pool)
            self.appliances = []
            # Push an appliance to the stack to have proper reference for test collection
            # FIXME: this is a bad hack based on the need for controll of collection partitioning
            appliance_stack.push(
                IPAppliance(address=request["appliances"][0]["ip_address"]))
            self.println("Appliances were provided:")
            for appliance in request["appliances"]:
                url = "https://{}/".format(appliance["ip_address"])
                self.appliances.append(url)
                self.println("- {} is {}".format(url, appliance['name']))
            map(lambda a: "https://{}/".format(a["ip_address"]), request["appliances"])
            self._reset_timer()
            # Set the base_url for collection purposes on the first appliance
            conf.runtime["env"]["base_url"] = self.appliances[0]
            # Retrieve and print the template_name for Jenkins to pick up
            template_name = request["appliances"][0]["template_name"]
            conf.runtime["cfme_data"]["basic_info"]["appliance_template"] = template_name
            self.terminal.write("appliance_template=\"{}\";\n".format(template_name))
            with project_path.join('.appliance_template').open('w') as template_file:
                template_file.write('export appliance_template="{}"'.format(template_name))
            self.println("Parallelized Sprout setup finished.")
            self.slave_appliances_data = {}
            for appliance in request["appliances"]:
                self.slave_appliances_data[appliance["ip_address"]] = (
                    appliance["template_name"], appliance["provider"]
                )

        # set up the ipc socket
        zmq_endpoint = 'tcp://127.0.0.1:{}'.format(random_port())
        ctx = zmq.Context.instance()
        self.sock = ctx.socket(zmq.ROUTER)
        self.sock.bind('{}'.format(zmq_endpoint))

        # clean out old slave config if it exists
        slave_config = conf_path.join('slave_config.yaml')
        slave_config.check() and slave_config.remove()

        # write out the slave config
        conf.runtime['slave_config'] = {
            'args': self.config.args,
            'options': self.config.option.__dict__,
            'zmq_endpoint': zmq_endpoint,
            'sprout': self.sprout_client is not None and self.sprout_pool is not None,
        }
        if hasattr(self, "slave_appliances_data"):
            conf.runtime['slave_config']["appliance_data"] = self.slave_appliances_data
        conf.runtime['slave_config']['options']['use_sprout'] = False  # Slaves don't use sprout
        conf.save('slave_config')

        for i, base_url in enumerate(self.appliances):
            self.slave_urls.add(base_url)

        for slave in sorted(self.slave_urls):
            self.print_message("using appliance {}".format(self.slave_urls[slave]),
                slave, green=True)

        # Start the recv queue
        self._recv_queue = deque()
        recv_queuer = Thread(target=_recv_queue, args=(self,))
        recv_queuer.daemon = True
        recv_queuer.start()

import json
import os
import os.path
from datetime import datetime

from artifactor.plugins.post_result import test_report
from utils import read_env
from utils.path import project_path
from utils.trackerbot import post_jenkins_result

job_name = os.environ['JOB_NAME']
number = int(os.environ['BUILD_NUMBER'])
date = str(datetime.now())

# reduce returns to bools for easy logic
runner_src = read_env(project_path.join('.jenkins_runner_result'))
runner_return = runner_src.get('RUNNER_RETURN', '1') == '0'
test_return = runner_src.get('TEST_RETURN', '1') == '0'


# 'stream' environ is set by jenkins for all stream test jobs
# but not in the template tester
if job_name not in ('template-tester', 'template-tester-openstack',
                    'template-tester-rhevm', 'template-tester-virtualcenter'):
    # try to pull out the appliance template name
    template_src = read_env(project_path.join('.appliance_template'))
    template = template_src.get('appliance_template', 'Unknown')
    stream = os.environ['stream']
else:
    tester_src = read_env(project_path.join('.template_tester'))
    stream = tester_src['stream']

    @contextmanager
    def appliances_ignored_when_renaming(self, *appliances):
        with self.atomic() as client:
            ignored_appliances = client._get("renaming_appliances")
            if ignored_appliances is None:
                ignored_appliances = set([])
            for appliance in appliances:
                ignored_appliances.add(appliance)
            client._set("renaming_appliances", ignored_appliances)
        yield
        with self.atomic() as client:
            ignored_appliances = client._get("renaming_appliances")
            if ignored_appliances is None:
                ignored_appliances = set([])
            for appliance in appliances:
                try:
                    ignored_appliances.remove(appliance)
                except KeyError:
                    # Something worng happened, ignore
                    pass
            client._set("renaming_appliances", ignored_appliances)

    @property
    def renaming_appliances(self):
        return self.get("renaming_appliances") or set([])


redis = RedisWrapper(redis_client)
sprout_path = project_path.join("sprout")

        the_diffs = re.findall('(@@.*[email protected]@.*?([email protected]@|git_output_checker))', the_file, re.M | re.S, )
        for diff in the_diffs:
            diff_args = re.match('@@ -(\d+)(,(\d+))*\s+\+(\d+)(,(\d+))*', diff).groups()
            if diff_args[5]:
                for extra_line in range(int(diff_args[5])):
                    file_data[filename].append(extra_line + int(diff_args[3]))
            else:
                file_data[filename].append(int(diff_args[3]))

    line_count = 0
    completed_lines = 0
    for file_changed, lines in file_data.iteritems():
        for line in lines:
            line_count += 1
            used_lines = coverage_data.lines(file_changed)
            if not used_lines:
                continue
            if isinstance(used_lines, int):
                used_lines = set([used_lines])
            else:
                used_lines = set(used_lines)
            if line in used_lines:
                completed_lines += 1

    return float(completed_lines) / line_count * 100

if __name__ == "__main__":
    result = compute_coverage(sys.argv[1])
    with open(project_path.join('coverage_result.txt').strpath, "w") as f:
        f.write("{}".format(result))