def pull(dry_run, flavor):
    """ Pull down tasks from forges and add them to your taskwarrior tasks.

    Relies on configuration in bugwarriorrc
    """
    twiggy.quickSetup()
    try:
        main_section = _get_section_name(flavor)

        # Load our config file
        config = load_config(main_section)

        tw_config = TaskWarriorBase.load_config(get_taskrc_path(config, main_section))
        lockfile_path = os.path.join(os.path.expanduser(tw_config["data"]["location"]), "bugwarrior.lockfile")

        lockfile = PIDLockFile(lockfile_path)
        lockfile.acquire(timeout=10)
        try:
            # Get all the issues.  This can take a while.
            issue_generator = aggregate_issues(config, main_section)

            # Stuff them in the taskwarrior db as necessary
            synchronize(issue_generator, config, main_section, dry_run)
        finally:
            lockfile.release()
    except LockTimeout:
        log.name("command").critical(
            "Your taskrc repository is currently locked. "
            "Remove the file at %s if you are sure no other "
            "bugwarrior processes are currently running." % (lockfile_path)
        )
    except:
        log.name("command").trace("error").critical("oh noes")

    def issues(self):
        user = self.config.get(self.target, 'bitbucket.username')
        response = self.get_data('/users/' + user + '/')
        repos = [
            repo.get('slug') for repo in response.get('repositories')
            if repo.get('has_issues')
        ]

        issues = sum([self.pull(user + "/" + repo) for repo in repos], [])
        log.name(self.target).debug(" Found {0} total.", len(issues))

        closed = ['resolved', 'duplicate', 'wontfix', 'invalid']
        not_resolved = lambda tup: tup[1]['status'] not in closed
        issues = filter(not_resolved, issues)
        issues = filter(self.include, issues)
        log.name(self.target).debug(" Pruned down to {0}", len(issues))

        for tag, issue in issues:
            issue_obj = self.get_issue_for_record(issue)
            extras = {
                'project': tag.split('/')[1],
                'url': self.BASE_URL + '/'.join(
                    issue['resource_uri'].split('/')[3:]
                ).replace('issues', 'issue'),
                'annotations': self.get_annotations(tag, issue, issue_obj)
            }
            issue_obj.update_extra(extras)
            yield issue_obj

def aggregate_issues(conf):
    """ Return all issues from every target.

    Takes a config object and a callable which returns a shortened url.
    """
    log.name('bugwarrior').info("Starting to aggregate remote issues.")

    # Create and call service objects for every target in the config
    targets = [t.strip() for t in conf.get('general', 'targets').split(',')]

    # This multiprocessing stuff is kind of experimental.
    use_multiprocessing = conf.has_option('general', 'multiprocessing') and \
        asbool(conf.get('general', 'multiprocessing'))

    if use_multiprocessing:
        log.name('bugwarrior').info("Spawning %i workers." % len(targets))
        pool = multiprocessing.Pool(processes=len(targets))
        map_function = pool.map
    else:
        log.name('bugwarrior').info("Processing targets in serial.")
        map_function = map

    issues_by_target = map_function(
        _aggregate_issues,
        zip([conf] * len(targets), targets)
    )
    log.name('bugwarrior').info("Done aggregating remote issues.")

    if WORKER_FAILURE in issues_by_target:
        log.name('bugwarrior').critical("A worker failed.  Aborting.")
        raise RuntimeError('Worker failure')

    return sum(issues_by_target, [])

def validate_config(config):
    if not config.has_section("general"):
        die("No [general] section found.")

    twiggy.quickSetup(name2level(config.get("general", "log.level")), config.get("general", "log.file"))

    if not config.has_option("general", "targets"):
        die("No targets= item in [general] found.")

    targets = config.get("general", "targets")
    targets = [t.strip() for t in targets.split(",")]

    for target in targets:
        if target not in config.sections():
            die("No [%s] section found." % target)

    for option in ["bitly.api_user", "bitly.api_key"]:
        if not config.has_option("general", option):
            log.name("config").warning("URLs will not be shortened with bit.ly")

    # Validate each target one by one.
    for target in targets:
        service = config.get(target, "service")
        if not service:
            die("No 'service' in [%s]" % target)

        if service not in SERVICES:
            die("'%s' in [%s] is not a valid service." % (service, target))

        # Call the service-specific validator
        SERVICES[service].validate_config(config, target)

    def issues(self):
        if self.tag:
            url = self.base_url + "/api/0/projects?tags=" + self.tag
            response = requests.get(url)
            if not bool(response):
                raise IOError('Failed to talk to %r %r' % (url, response))

            all_repos = [r['name'] for r in response.json()['projects']]
        else:
            all_repos = [self.repo]

        repos = filter(self.filter_repos, all_repos)

        issues = []
        for repo in repos:
            issues.extend(self.get_issues(repo, ('issues', 'issues')))
            issues.extend(self.get_issues(repo, ('pull-requests', 'requests')))

        log.name(self.target).debug(" Found {0} issues.", len(issues))
        issues = filter(self.include, issues)
        log.name(self.target).debug(" Pruned down to {0} issues.", len(issues))

        for repo, issue in issues:
            # Stuff this value into the upstream dict for:
            # https://pagure.com/ralphbean/bugwarrior/issues/159
            issue['repo'] = repo

            issue_obj = self.get_issue_for_record(issue)
            extra = {
                'project': repo,
                'type': 'pull_request' if 'branch' in issue else 'issue',
                'annotations': self.annotations(issue, issue_obj)
            }
            issue_obj.update_extra(extra)
            yield issue_obj

    def __init__(self, *args, **kw):
        super(BugzillaService, self).__init__(*args, **kw)
        self.base_uri = self.config_get('base_uri')
        self.username = self.config_get('username')
        self.password = self.config_get('password')
        self.ignore_cc = self.config_get_default('ignore_cc', default=False,
                                                 to_type=lambda x: x == "True")
        self.query_url = self.config_get_default('query_url', default=None)
        self.include_needinfos = self.config_get_default(
            'include_needinfos', False, to_type=lambda x: x == "True")
        self.open_statuses = self.config_get_default(
            'open_statuses', _open_statuses, to_type=lambda x: x.split(','))
        log.name(self.target).debug(" filtering on statuses: {0}", self.open_statuses)

        # So more modern bugzilla's require that we specify
        # query_format=advanced along with the xmlrpc request.
        # https://bugzilla.redhat.com/show_bug.cgi?id=825370
        # ...but older bugzilla's don't know anything about that argument.
        # Here we make it possible for the user to specify whether they want
        # to pass that argument or not.
        self.advanced = asbool(self.config_get_default('advanced', 'no'))

        if not self.password or self.password.startswith("@oracle:"):
            self.password = get_service_password(
                self.get_keyring_service(self.config, self.target),
                self.username, oracle=self.password,
                interactive=self.config.interactive
            )

        url = 'https://%s/xmlrpc.cgi' % self.base_uri
        self.bz = bugzilla.Bugzilla(url=url)
        self.bz.login(self.username, self.password)

    def issues(self):
        base_url = "https://" + self.config.get(self.target, 'trac.base_uri')
        tickets = self.trac.query_tickets('status!=closed&max=0')
        tickets = map(self.trac.get_ticket, tickets)
        issues = [(self.target, ticket[3]) for ticket in tickets]
        log.name(self.target).debug(" Found {0} total.", len(issues))

        # Build a url for each issue
        for i in range(len(issues)):
            issues[i][1]['url'] = "%s/ticket/%i" % (base_url, tickets[i][0])
            issues[i][1]['number'] = tickets[i][0]

        issues = filter(self.include, issues)
        log.name(self.target).debug(" Pruned down to {0}", len(issues))

        return [dict(
            description=self.description(
                issue['summary'], issue['url'],
                issue['number'], cls="issue"),
            project=tag,
            priority=self.priorities.get(
                issue['priority'],
                self.default_priority,
            ),
            **self.annotations(tag, issue)
        ) for tag, issue in issues]

def _aggregate_issues(conf, main_section, target, queue, service_name):
    """ This worker function is separated out from the main
    :func:`aggregate_issues` func only so that we can use multiprocessing
    on it for speed reasons.
    """

    start = time.time()

    try:
        service = SERVICES[service_name](conf, main_section, target)
        issue_count = 0
        for issue in service.issues():
            queue.put(issue)
            issue_count += 1
    except Exception as e:
        log.name(target).trace('error').critical(
            "Worker for [%s] failed: %s" % (target, e)
        )
        queue.put(
            (SERVICE_FINISHED_ERROR, (target, e))
        )
    else:
        queue.put(
            (SERVICE_FINISHED_OK, (target, issue_count, ))
        )
    finally:
        duration = time.time() - start
        log.name(target).info("Done with [%s] in %fs" % (target, duration))

    def issues(self):
        user = self.config.get(self.target, 'github.username')

        all_repos = githubutils.get_repos(username=user, auth=self.auth)
        assert(type(all_repos) == list)
        repos = filter(self.filter_repos, all_repos)

        issues = {}
        if self.involved_issues:
            issues.update(
                self.get_involved_issues(user)
            )
        else:
            for repo in repos:
                issues.update(
                    self.get_owned_repo_issues(user + "/" + repo['name'])
                )
        issues.update(self.get_directly_assigned_issues())
        log.name(self.target).debug(" Found {0} issues.", len(issues))
        issues = filter(self.include, issues.values())
        log.name(self.target).debug(" Pruned down to {0} issues.", len(issues))

        for tag, issue in issues:
            # Stuff this value into the upstream dict for:
            # https://github.com/ralphbean/bugwarrior/issues/159
            issue['repo'] = tag

            issue_obj = self.get_issue_for_record(issue)
            extra = {
                'project': tag.split('/')[1],
                'type': 'pull_request' if 'pull_request' in issue else 'issue',
                'annotations': self.annotations(tag, issue, issue_obj)
            }
            issue_obj.update_extra(extra)
            yield issue_obj

def merge_left(field, local_task, remote_issue, hamming=False):
    """ Merge array field from the remote_issue into local_task

    * Local 'left' entries are preserved without modification
    * Remote 'left' are appended to task if not present in local.

    :param `field`: Task field to merge.
    :param `local_task`: `taskw.task.Task` object into which to merge
        remote changes.
    :param `remote_issue`: `dict` instance from which to merge into
        local task.
    :param `hamming`: (default `False`) If `True`, compare entries by
        truncating to maximum length, and comparing hamming distances.
        Useful generally only for annotations.

    """

    # Ensure that empty defaults are present
    local_field = local_task.get(field, [])
    remote_field = remote_issue.get(field, [])

    # We need to make sure an array exists for this field because
    # we will be appending to it in a moment.
    if field not in local_task:
        local_task[field] = []

    # If a remote does not appear in local, add it to the local task
    new_count = 0
    for remote in remote_field:
        found = False
        for local in local_field:
            if (
                # For annotations, they don't have to match *exactly*.
                (
                    hamming
                    and get_annotation_hamming_distance(remote, local) == 0
                )
                # But for everything else, they should.
                or (
                    remote == local
                )
            ):
                found = True
                break
        if not found:
            log.name('db').debug(
                "%s not found in %r" % (remote, local_field)
            )
            local_task[field].append(remote)
            new_count += 1
    if new_count > 0:
        log.name('db').debug(
            'Added %s new values to %s (total: %s)' % (
                new_count,
                field,
                len(local_task[field]),
            )
        )

    def issues(self):
        email = self.config.get(self.target, 'bugzilla.username')
        # TODO -- doing something with blockedby would be nice.

        query = dict(
            column_list=self.column_list,
            bug_status=self.not_closed_statuses,
            email1=email,
            emailreporter1=1,
            emailcc1=1,
            emailassigned_to1=1,
            emailqa_contact1=1,
            emailtype1="substring",
        )

        if self.advanced:
            # Required for new bugzilla
            # https://bugzilla.redhat.com/show_bug.cgi?id=825370
            query['query_format'] = 'advanced'

        bugs = self.bz.query(query)

        # Convert to dicts
        bugs = [
            dict(
                ((col, getattr(bug, col)) for col in self.column_list)
            ) for bug in bugs
        ]

        issues = [(self.target, bug) for bug in bugs]
        log.name(self.target).debug(" Found {0} total.", len(issues))

        # Build a url for each issue
        base_url = "https://%s/show_bug.cgi?id=" % \
            self.config.get(self.target, 'bugzilla.base_uri')
        for i in range(len(issues)):
            issues[i][1]['url'] = base_url + str(issues[i][1]['id'])
            issues[i][1]['component'] = \
                issues[i][1]['component'].lower().replace(' ', '-')

        # XXX - Note that we don't use the .include() method like all the other
        # IssueService child classes.  That's because the bugzilla xmlrpc API
        # can already do a lot of the filtering we want for us.

        #issues = filter(self.include, issues)
        #log.name(self.target).debug(" Pruned down to {0}", len(issues))

        return [dict(
            description=self.description(
                issue['summary'], issue['url'],
                issue['id'], cls="issue"),
            project=issue['component'],
            priority=self.priorities.get(
                issue['priority'],
                self.default_priority,
            ),
            **self.annotations(tag, issue)
        ) for tag, issue in issues]

 def get_involved_issues(self, user):
     """ Grab all 'interesting' issues """
     issues = {}
     for issue in githubutils.get_involved_issues(user, auth=self.auth):
         url = issue['html_url']
         tag = re.match('.*github\\.com/(.*)/(issues|pull)/[^/]*$', url)
         if tag is None:
             log.name(self.target).critical(" Unrecognized issue URL: {0}.", url)
             continue
         issues[url] = (tag.group(1), issue)
     return issues

    def issues(self):
        issues = self.client.get_task_list()
        log.name(self.target).debug(
            " Remote has {0} total issues.", len(issues))

        # Filter out closed tasks.
        issues = filter(lambda i: i["status"] == 1, issues)
        log.name(self.target).debug(
            " Remote has {0} active issues.", len(issues))

        for issue in issues:
            yield self.get_issue_for_record(issue)

def pull():
    try:
        # Load our config file
        config = load_config()

        # Get all the issues.  This can take a while.
        issues = aggregate_issues(config)

        # Stuff them in the taskwarrior db as necessary
        synchronize(issues, config)
    except:
        log.name('command').trace('error').critical('oh noes')

    def issues(self):
        issues = self.client.find_issues(self.user_id)
        log.name(self.target).debug(" Found {0} total.", len(issues))

        return [dict(
            description=self.description(
                issue["subject"],
                self.get_issue_url(issue),
                issue["id"], cls="issue"),
            project=self.get_project_name(issue),
            priority=self.default_priority,
        ) for issue in issues]

    def issues(self):
        cases = self.jira.search_issues(self.query, maxResults=-1)

        jira_version = 5  # Default version number
        if self.config.has_option(self.target, 'jira.version'):
            jira_version = self.config.getint(self.target, 'jira.version')
        if jira_version == 4:
            # Convert for older jira versions that don't support the new API
            cases = [self.__convert_for_jira4(case) for case in cases]

        log.name(self.target).debug(" Found {0} total.", len(cases))
        return [self.__issue(case, jira_version) for case in cases]

    def issues(self):
        issues = self.client.get_actual_tasks()
        log.name(self.target).debug(" Found {0} total.", len(issues))

        return [dict(
            description=self.description(
                self.get_issue_title(issue),
                self.get_issue_url(issue),
                self.get_issue_id(issue),
                cls="issue"),
            project=self.project_name,
            priority=self.default_priority,
        ) for issue in issues]

 def __init__(self, config, target, shorten):
     self.config = config
     self.target = target
     self.shorten = shorten
     if config.has_option('general', 'description_length'):
         self.desc_len = self.config.getint('general', 'description_length')
     else:
         self.desc_len = 35
     if config.has_option('general', 'annotation_length'):
         self.anno_len = self.config.getint('general', 'annotation_length')
     else:
         self.anno_len = 45
     log.name(target).info("Working on [{0}]", self.target)

def run_hooks(conf, name):
    if conf.has_option('hooks', name):
        pre_import = [
            t.strip() for t in conf.get('hooks', name).split(',')
        ]
        if pre_import is not None:
            for hook in pre_import:
                exit_code = subprocess.call(hook, shell=True)
                if exit_code is not 0:
                    msg = 'Non-zero exit code %d on hook %s' % (
                        exit_code, hook
                    )
                    log.name('hooks:%s' % name).error(msg)
                    raise RuntimeError(msg)

    def __init__(self, root_src, root_dst, debug=False):
        '''
            Setup the base options of the copy/convert setup
        '''
        self.src_root_dir = root_src
        self.dst_root_dir = root_dst
        self.log_root_dir = os.path.join(self.dst_root_dir, 'logs')
        self._safe_rmtree(self.log_root_dir)
        self._safe_mkdir(self.log_root_dir)
        self.log_processing = os.path.join(self.log_root_dir, 'processing.log')

        quick_setup(file=self.log_processing)
        self.log_name = log.name('files')
        self.ressources_path = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'data')
        os.environ["PATH"] += os.pathsep + self.ressources_path

        self.cur_file = None

        self.debug = debug
        if self.debug:
            self.log_debug_err = os.path.join(self.log_root_dir, 'debug_stderr.log')
            self.log_debug_out = os.path.join(self.log_root_dir, 'debug_stdout.log')
        else:
            self.log_debug_err = os.devnull
            self.log_debug_out = os.devnull