def show_single_coverage_mask(qreq_, cm, weight_mask_m, weight_mask, daids, fnum=None):
    import plottool as pt
    from ibeis import viz
    fnum = pt.ensure_fnum(fnum)
    idx_list = ut.dict_take(cm.daid2_idx, daids)
    nPlots = len(idx_list) + 1
    nRows, nCols = pt.get_square_row_cols(nPlots)
    pnum_ = pt.make_pnum_nextgen(nRows, nCols)
    pt.figure(fnum=fnum, pnum=(1, 2, 1))
    # Draw coverage masks with bbox
    # <FlipHack>
    #weight_mask_m = np.fliplr(np.flipud(weight_mask_m))
    #weight_mask = np.fliplr(np.flipud(weight_mask))
    # </FlipHack>
    stacked_weights, offset_tup, sf_tup = vt.stack_images(weight_mask_m, weight_mask, return_sf=True)
    (woff, hoff) = offset_tup[1]
    wh1 = weight_mask_m.shape[0:2][::-1]
    wh2 = weight_mask.shape[0:2][::-1]
    pt.imshow(255 * (stacked_weights), fnum=fnum, pnum=pnum_(0), title='(query image) What did match vs what should match')
    pt.draw_bbox((   0,    0) + wh1, bbox_color=(0, 0, 1))
    pt.draw_bbox((woff, hoff) + wh2, bbox_color=(0, 0, 1))
    # Get contributing matches
    qaid = cm.qaid
    daid_list = daids
    fm_list = ut.take(cm.fm_list, idx_list)
    fs_list = ut.take(cm.fs_list, idx_list)
    # Draw matches
    for px, (daid, fm, fs) in enumerate(zip(daid_list, fm_list, fs_list), start=1):
        viz.viz_matches.show_matches2(qreq_.ibs, qaid, daid, fm, fs,
                                      draw_pts=False, draw_lines=True,
                                      draw_ell=False, fnum=fnum, pnum=pnum_(px),
                                      darken=.5)
    coverage_score = score_matching_mask(weight_mask_m, weight_mask)
    pt.set_figtitle('score=%.4f' % (coverage_score,))

def get_summary(profile_block_list, maxlines=20):
    """
    References:
        https://github.com/rkern/line_profiler
    """
    time_list = [get_block_totaltime(block) for block in profile_block_list]
    time_list = [time if time is not None else -1 for time in time_list]
    blockid_list = [get_block_id(block) for block in profile_block_list]
    sortx = ut.list_argsort(time_list)
    sorted_time_list = ut.take(time_list, sortx)
    sorted_blockid_list = ut.take(blockid_list, sortx)

    aligned_blockid_list = ut.util_str.align_lines(sorted_blockid_list, ':')
    summary_lines = [('%6.2f seconds - ' % time) + line
                     for time, line in
                     zip(sorted_time_list, aligned_blockid_list)]
    #summary_header = ut.codeblock(
    #    '''
    #    CLEANED PROFILE OUPUT

    #    The Pystone timings are not from kernprof, so they may include kernprof
    #    overhead, whereas kernprof timings do not (unless the line being
    #    profiled is also decorated with kernrof)

    #    The kernprof times are reported in Timer Units

    #    ''')
    summary_lines_ = ut.listclip(summary_lines, maxlines, fromback=True)
    summary_text = '\n'.join(summary_lines_)
    return summary_text

def make_factor_text(factor, name):
    collapse_uniform = True
    if collapse_uniform and ut.almost_allsame(factor.values):
        # Reduce uniform text
        ftext = name + ':\nuniform(%.3f)' % (factor.values[0],)
    else:
        values = factor.values
        try:
            rowstrs = ['p(%s)=%.3f' % (','.join(n), v,)
                       for n, v in zip(zip(*factor.statenames), values)]
        except Exception:
            rowstrs = ['p(%s)=%.3f' % (','.join(n), v,)
                       for n, v in zip(factor._row_labels(False), values)]
        idxs = ut.list_argmaxima(values)
        for idx in idxs:
            rowstrs[idx] += '*'
        thresh = 4
        always_sort = True
        if len(rowstrs) > thresh:
            sortx = factor.values.argsort()[::-1]
            rowstrs = ut.take(rowstrs, sortx[0:(thresh - 1)])
            rowstrs += ['... %d more' % ((len(values) - len(rowstrs)),)]
        elif always_sort:
            sortx = factor.values.argsort()[::-1]
            rowstrs = ut.take(rowstrs, sortx)
        ftext = name + ': \n' + '\n'.join(rowstrs)
    return ftext

 def unnest_data(data):
     unnested_cols = list(zip(ut.take(data, idxs2)))
     nested_cols = ut.take(data, idxs1)
     grouped_items = [nested_cols, unnested_cols]
     groupxs = [idxs1, idxs2]
     unflat = ut.ungroup(grouped_items, groupxs, nested_nCols - 1)
     return tuple(ut.flatten(unflat))

def get_topname_training_idxs(cm, num=5):
    """
    gets the index of the annots in the top groundtrue name and the top
    groundfalse names.

    Args:
        cm (ibeis.ChipMatch):  object of feature correspondences and scores
        num(int): number of false names (default = 5)

    Returns:
        tuple: (tp_idxs, tn_idxs)
            cm.daid_list[tp_idxs] are all of the
               annotations in the correct name.
            cm.daid_list[tn_idxs] are all of the
               annotations in the top `num_false` incorrect names.

    CommandLine:
        python -m ibeis --tf get_topname_training_idxs --show

    Example:
        >>> # ENABLE_DOCTEST
        >>> from ibeis.algo.hots.scorenorm import *  # NOQA
        >>> import ibeis
        >>> cm, qreq_ = ibeis.testdata_cm('PZ_MTEST', a='default:dindex=0:10,qindex=0:1', t='best')
        >>> num = 1
        >>> (tp_idxs, tn_idxs) = get_topname_training_idxs(cm, num)
        >>> result = ('(tp_idxs, tn_idxs) = %s' % (ut.repr2((tp_idxs, tn_idxs), nl=1),))
        >>> print(result)
        (tp_idxs, tn_idxs) = (
            np.array([0, 1, 2], dtype=np.int64),
            [3, 4, 5, 6],
        )
    """
    if num is None:
        num = 5
    sortx = cm.name_argsort()
    sorted_nids = vt.take2(cm.unique_nids, sortx)
    sorted_groupxs = ut.take(cm.name_groupxs, sortx)
    # name ranks of the groundtrue name
    tp_ranks = np.where(sorted_nids == cm.qnid)[0]
    if len(tp_ranks) == 0:
        #if ut.STRICT:
        #    raise Exception('tp_ranks=0')
        #else:
            raise UnbalancedExampleException('tp_ranks=0')

    # name ranks of the top groundfalse names
    tp_rank = tp_ranks[0]
    tn_ranks = [rank for rank in range(num + 1)
                if rank != tp_rank and rank < len(sorted_groupxs)]
    if len(tn_ranks) == 0:
        #if ut.STRICT:
        #    raise Exception('tn_ranks=0')
        #else:
            raise UnbalancedExampleException('tn_ranks=0')
    # annot idxs of the examples
    tp_idxs = sorted_groupxs[tp_rank]
    tn_idxs = ut.flatten(ut.take(sorted_groupxs, tn_ranks))
    return tp_idxs, tn_idxs

def find_requested_hrefs(all_href_list, py_version, pkg_list):
    """
    Filters out everything but the requested urls
    Returns the urls to download the requested installers
    """
    print('Filtering to only requested HREFS')
    href_list1, missing1  = filter_href_list(all_href_list, pkg_list, OS_VERSION, py_version)
    #print('missing1 = %r' % (missing1,))
    href_list2, missing2 = filter_href_list(all_href_list, missing1, OS_VERSION, py_version)
    #print('missing2 = %r' % (missing2,))
    #print(href_list2)
    href_list3, missing3 = filter_href_list(all_href_list, missing2, 'x64', py_version.replace('p', 'P'))
    #print('missing3 = %r' % (missing3,))
    href_list4, missing4 = filter_href_list(all_href_list, missing3, 'any', py_version.replace('cp', 'py')[0:3])

    if len(missing4) > 0:
        print('Could not find a match for missing4=%r' % (missing4,))
        #import Levenshtein
        for pkg in missing4:
            #dist_list = [Levenshtein.distance(href, pkg) for href in all_href_list]
            dist_list = [0 if (href.find(pkg) > -1) else 100 for href in all_href_list]
            closest_matche_xs = ut.list_argsort(dist_list)[::1]
            print('Perhaps pkg=%r could match one of these?' % (pkg,))
            closest_hrefs = ut.take(all_href_list, closest_matche_xs[0:3])
            print(ut.indentjoin(closest_hrefs, '\n   '))

    href_list = href_list1 + href_list2 + href_list3 + href_list4
    return href_list

    def get_patches(invassign, wx):
        ax_list = invassign.wx2_axs[wx]
        fx_list = invassign.wx2_fxs[wx]
        config = invassign.fstack.config
        ibs = invassign.fstack.ibs

        unique_axs, groupxs = vt.group_indices(ax_list)
        fxs_groups = vt.apply_grouping(fx_list, groupxs)

        unique_aids = ut.take(invassign.fstack.ax2_aid, unique_axs)

        all_kpts_list = ibs.depc.d.get_feat_kpts(unique_aids, config=config)
        sub_kpts_list = vt.ziptake(all_kpts_list, fxs_groups, axis=0)

        chip_list = ibs.depc_annot.d.get_chips_img(unique_aids)
        # convert to approprate colorspace
        #if colorspace is not None:
        #    chip_list = vt.convert_image_list_colorspace(chip_list, colorspace)
        # ut.print_object_size(chip_list, 'chip_list')
        patch_size = 64
        grouped_patches_list = [
            vt.get_warped_patches(chip, kpts, patch_size=patch_size)[0]
            for chip, kpts in ut.ProgIter(zip(chip_list, sub_kpts_list),
                                          nTotal=len(unique_aids),
                                          lbl='warping patches')
        ]
        # Make it correspond with original fx_list and ax_list
        word_patches = vt.invert_apply_grouping(grouped_patches_list, groupxs)
        return word_patches

def report_partitioning_statistics(new_reduced_joint):
    # compute partitioning statistics
    import vtool as vt
    vals, idxs = vt.group_indices(new_reduced_joint.values.ravel())
    #groupsize = list(map(len, idxs))
    #groupassigns = ut.unflat_vecmap(new_reduced_joint.assignment, idxs)
    all_states = new_reduced_joint._row_labels(asindex=True)
    clusterstats = [tuple(sorted(list(ut.dict_hist(a).values())))
                    for a in all_states]
    grouped_vals = ut.group_items(new_reduced_joint.values.ravel(),
                                  clusterstats)

    #probs_assigned_to_clustertype = [(
    #    sorted(np.unique(np.array(b).round(decimals=5)).tolist())[::-1], a)
    #    for a, b in grouped_vals.items()]
    probs_assigned_to_clustertype = [(
        ut.dict_hist(np.array(b).round(decimals=5)), a)
        for a, b in grouped_vals.items()]
    sortx = ut.argsort([max(c[0].keys())
                        for c in probs_assigned_to_clustertype])
    probs_assigned_to_clustertype = ut.take(probs_assigned_to_clustertype, sortx)

    # This list of 2-tuples with the first item being the unique
    # probabilies that are assigned to a cluster type along with the number
    # of times they were assigned. A cluster type is the second item. Every
    # number represents how many annotations were assigned to a specific
    # label. The length of that list is the number of total labels.  For
    # all low scores you will see [[{somenum: 1}, {0: 800}], [1, 1, 1, ... 1]]
    # indicating that that the assignment of everyone to a different label happend once
    # where the probability was somenum and a 800 times where the probability was 0.

    #print(sorted([(b, a) for a, b in ut.map_dict_vals(sum, x)]).items())
    #z = sorted([(b, a) for a, b in ut.map_dict_vals(sum, grouped_vals).items()])
    print(ut.repr2(probs_assigned_to_clustertype, nl=2, precision=2, sorted_=True))

 def compute_annot_occurrence_ids(ibs, aid_list):
     from ibeis.algo.preproc import preproc_occurrence
     gid_list = ibs.get_annot_gids(aid_list)
     gid2_aids = ut.group_items(aid_list, gid_list)
     flat_imgsetids, flat_gids = preproc_occurrence.ibeis_compute_occurrences(ibs, gid_list, seconds_thresh=4 * 60 * 60, verbose=False)
     occurid2_gids = ut.group_items(flat_gids, flat_imgsetids)
     occurid2_aids = {oid: ut.flatten(ut.take(gid2_aids, gids)) for oid, gids in occurid2_gids.items()}
     return occurid2_aids

 def _update_state(model):
     import networkx as nx
     nodes = sorted(list(model.graph.nodes()))
     edges = list(model.graph.edges())
     edge2_weights = nx.get_edge_attributes(model.graph, 'weight')
     node2_labeling = nx.get_node_attributes(model.graph, 'name_label')
     edge_weights = ut.take(edge2_weights, edges)
     labeling = ut.take(node2_labeling, nodes)
     n_nodes = len(nodes)
     # Model state
     model.n_nodes = n_nodes
     model.edges = edges
     model.edge_weights = edge_weights
     # Model parameters
     model.labeling = np.zeros(model.n_nodes, dtype=np.int32)
     model._update_labels(labeling=labeling)
     model._update_weights()

def chain_crf():
    from pystruct.datasets import load_letters
    letters = load_letters()
    X, y, folds = ut.take(letters, ['data', 'labels', 'folds'])
    X, y = np.array(X), np.array(y)
    X_train, X_test = X[folds == 1], X[folds != 1]
    y_train, y_test = y[folds == 1], y[folds != 1]
    len(X_train)

def get_name_shortlist_aids(daid_list, dnid_list, annot_score_list,
                            name_score_list, nid2_nidx,
                            nNameShortList, nAnnotPerName):
    r"""
    CommandLine:
        python -m ibeis.algo.hots.scoring --test-get_name_shortlist_aids

    Example:
        >>> # ENABLE_DOCTEST
        >>> from ibeis.algo.hots.scoring import *  # NOQA
        >>> # build test data
        >>> daid_list        = np.array([11, 12, 13, 14, 15, 16, 17])
        >>> dnid_list        = np.array([21, 21, 21, 22, 22, 23, 24])
        >>> annot_score_list = np.array([ 6,  2,  3,  5,  6,  3,  2])
        >>> name_score_list  = np.array([ 8,  9,  5,  4])
        >>> nid2_nidx        = {21:0, 22:1, 23:2, 24:3}
        >>> nNameShortList, nAnnotPerName = 3, 2
        >>> # execute function
        >>> args = (daid_list, dnid_list, annot_score_list, name_score_list,
        ...         nid2_nidx, nNameShortList, nAnnotPerName)
        >>> top_daids = get_name_shortlist_aids(*args)
        >>> # verify results
        >>> result = str(top_daids)
        >>> print(result)
        [15, 14, 11, 13, 16]
    """
    unique_nids, groupxs    = vt.group_indices(np.array(dnid_list))
    grouped_annot_scores    = vt.apply_grouping(annot_score_list, groupxs)
    grouped_daids           = vt.apply_grouping(np.array(daid_list), groupxs)
    # Ensure name score list is aligned with the unique_nids
    aligned_name_score_list = name_score_list.take(ut.dict_take(nid2_nidx, unique_nids))
    # Sort each group by the name score
    group_sortx             = aligned_name_score_list.argsort()[::-1]
    _top_daid_groups        = ut.take(grouped_daids, group_sortx)
    _top_annot_score_groups = ut.take(grouped_annot_scores, group_sortx)
    top_daid_groups         = ut.listclip(_top_daid_groups, nNameShortList)
    top_annot_score_groups  = ut.listclip(_top_annot_score_groups, nNameShortList)
    # Sort within each group by the annotation score
    top_daid_sortx_groups   = [annot_score_group.argsort()[::-1]
                               for annot_score_group in top_annot_score_groups]
    top_sorted_daid_groups  = vt.ziptake(top_daid_groups, top_daid_sortx_groups)
    top_clipped_daids = [ut.listclip(sorted_daid_group, nAnnotPerName)
                         for sorted_daid_group in top_sorted_daid_groups]
    top_daids = ut.flatten(top_clipped_daids)
    return top_daids

def show_name(ibs, nid, in_image=True, fnum=0, sel_aids=[], subtitle='',
              annote=False, aid_list=None, index_list=None,  **kwargs):
    r"""
    Args:
        ibs (IBEISController):  ibeis controller object
        nid (?):
        in_image (bool):
        fnum (int):  figure number
        sel_aids (list):
        subtitle (str):
        annote (bool):

    CommandLine:

        python -m ibeis.viz.viz_name --test-show_name --dpath ~/latex/crall-candidacy-2015 --save 'figures/{name}.jpg' --no-figtitle --notitle --db NNP_Master3 --figsize=9,4 --clipwhite --dpi=180 --adjust=.05 --index_list=[0,1,2,3] --rc=2,4 --append temp_out_figure.tex --name=IBEIS_PZ_0739 --no-draw_lbls --doboth --no-inimage  --diskshow

        python -m ibeis.viz.viz_name --test-show_name --no-figtitle --notitle --db NNP_Master3 --figsize=9,4 --clipwhite --dpi=180 --adjust=.05 --index_list=[0,1,2,3] --rc=2,4 --append temp_out_figure.tex --name=IBEIS_PZ_0739 --no-draw_lbls --doboth --no-inimage  --show

        python -m ibeis.viz.viz_name --test-show_name --show

    Example:
        >>> # DISABLE_DOCTEST
        >>> from ibeis.viz.viz_name import *  # NOQA
        >>> ibs, nid, in_image, index_list = testdata_showname()
        >>> fnum = 0
        >>> sel_aids = []
        >>> subtitle = ''
        >>> annote = False
        >>> # execute function
        >>> show_name(ibs, nid, in_image, fnum, sel_aids, subtitle, annote, index_list=index_list)
        >>> ut.show_if_requested()
    """
    print('[viz_name] show_name nid=%r, index_list=%r, aid_list=%r' % (nid, index_list, aid_list))
    if aid_list is None:
        aid_list = ibs.get_name_aids(nid)
    else:
        assert ut.list_all_eq_to(ibs.get_annot_nids(aid_list), nid)

    if index_list is not None:
        aid_list = ut.take(aid_list, index_list)

    name = ibs.get_name_texts((nid,))
    print('[viz_name] * name=%r aid_list=%r' % (name, aid_list))

    show_multiple_chips(ibs, aid_list, in_image=in_image, fnum=fnum,
                        sel_aids=sel_aids, annote=annote, **kwargs)

    if isinstance(nid, np.ndarray):
        nid = nid[0]
    if isinstance(name, np.ndarray):
        name = name[0]

    use_figtitle = not ut.get_argflag('--no-figtitle')

    if use_figtitle:
        figtitle = 'Name View nid=%r name=%r' % (nid, name)
        df2.set_figtitle(figtitle)

    def issue(repo, command, sudo=False, dry=False, error='raise', return_out=False):
        """
        issues a command on a repo

        CommandLine:
            python -m utool.util_git --exec-repocmd

        Example:
            >>> # DISABLE_DOCTEST
            >>> from utool.util_git import *  # NOQA
            >>> import utool as ut
            >>> repo = dirname(ut.get_modpath(ut, prefer_pkg=True))
            >>> command = 'git status'
            >>> sudo = False
            >>> result = repocmd(repo, command, sudo)
            >>> print(result)
        """
        import utool as ut
        if ut.WIN32:
            assert not sudo, 'cant sudo on windows'
        if command == 'short_status':
            return repo.short_status()
        command_list = ut.ensure_iterable(command)
        cmdstr = '\n        '.join([cmd_ for cmd_ in command_list])
        if not dry:
            print('+--- *** repocmd(%s) *** ' % (cmdstr,))
            print('repo=%s' % ut.color_text(repo.dpath, 'yellow'))
        verbose = True
        with repo.chdir_context():
            ret = None
            for count, cmd in enumerate(command_list):
                if dry:
                    print(cmd)
                    continue
                if not sudo or ut.WIN32:
                    # ret = os.system(cmd)
                    cmdinfo = ut.cmd2(cmd, verbout=True)
                    out, err, ret = ut.take(cmdinfo, ['out', 'err', 'ret'])
                else:
                    # cmdinfo = ut.cmd2('sudo ' + cmd, verbose=1)
                    out, err, ret = ut.cmd(cmd, sudo=True)
                if verbose > 1:
                    print('ret(%d) = %r' % (count, ret,))
                if ret != 0:
                    if error == 'raise':
                        raise Exception('Failed command %r' % (cmd,))
                    elif error == 'return':
                        return out
                    else:
                        raise ValueError('unknown flag error=%r' % (error,))
                if return_out:
                    return out
        if not dry:
            print('L____')

 def jagged_stats_info(arr_, lbl, col_lbls):
     arr = ut.recursive_replace(arr_, np.inf, np.nan)
     # Treat infinite as nan
     stat_dict = ut.get_jagged_stats(arr, use_nan=True, use_sum=True)
     sel_stat_dict, sel_indices = ut.find_interesting_stats(stat_dict, col_lbls)
     sel_col_lbls = ut.take(col_lbls, sel_indices)
     statstr_kw   = dict(precision=3, newlines=True, lbl=lbl, align=True)
     stat_str     = ut.get_stats_str(stat_dict=stat_dict, **statstr_kw)
     sel_stat_str = ut.get_stats_str(stat_dict=sel_stat_dict, **statstr_kw)
     sel_stat_str = 'sel_col_lbls = %s' % (ut.list_str(sel_col_lbls),) + '\n' + sel_stat_str
     return stat_str, sel_stat_str

 def __init__(invassign, fstack, vocab, wx2_idxs, wx2_maws, wx2_fxs, wx2_axs):
     invassign.fstack = fstack
     invassign.vocab = vocab
     invassign.wx2_idxs = wx2_idxs
     invassign.wx2_maws = wx2_maws
     invassign.wx2_fxs = wx2_fxs
     invassign.wx2_axs = wx2_axs
     invassign.wx2_num = ut.map_dict_vals(len, invassign.wx2_axs)
     invassign.wx_list = sorted(invassign.wx2_num.keys())
     invassign.num_list = ut.take(invassign.wx2_num, invassign.wx_list)
     invassign.perword_stats = ut.get_stats(invassign.num_list)

    def make_prob_names(infr):
        cm_list = infr.cm_list
        # Consolodate information from a series of chip matches
        unique_nids = sorted(ut.list_union(*[cm.unique_nids for cm in cm_list]))
        #nid2_nidx = ut.make_index_lookup(unique_nids)
        # Populate matrix of raw name scores
        prob_names = np.zeros((len(cm_list), len(unique_nids)))
        for count, cm in enumerate(cm_list):
            try:
                name_scores = ut.dict_take(cm.nid2_name_score, unique_nids, 0)
            except AttributeError:
                unique_nidxs = ut.take(cm.nid2_nidx, unique_nids)
                name_scores = ut.take(cm.name_score_list, unique_nidxs)
            prob_names[count][:] = name_scores

        # Normalize to row stochastic matrix
        prob_names /= prob_names.sum(axis=1)[:, None]
        #print(ut.hz_str('prob_names = ', ut.array2string2(prob_names,
        #precision=2, max_line_width=140, suppress_small=True)))
        return unique_nids, prob_names

    def intersect_hack():
        failed = testres.rank_mat > 0
        colx2_failed = [np.nonzero(failed_col)[0] for failed_col in failed.T]
        #failed_col2_only = np.setdiff1d(colx2_failed[1], colx2_failed[0])
        #failed_col2_only_aids = ut.take(testres.qaids, failed_col2_only)
        failed_col1_only = np.setdiff1d(colx2_failed[0], colx2_failed[1])
        failed_col1_only_aids = ut.take(testres.qaids, failed_col1_only)
        gt_aids1 = ibs.get_annot_groundtruth(failed_col1_only_aids, daid_list=testres.cfgx2_qreq_[0].daids)
        gt_aids2 = ibs.get_annot_groundtruth(failed_col1_only_aids, daid_list=testres.cfgx2_qreq_[1].daids)

        qaids_expt = failed_col1_only_aids
        gt_avl_aids1 = ut.flatten(gt_aids1)
        gt_avl_aids2 = list(set(ut.flatten(gt_aids2)).difference(gt_avl_aids1))

        ibs.print_annotconfig_stats(qaids_expt, gt_avl_aids1)
        ibs.print_annotconfig_stats(qaids_expt, gt_avl_aids2)
        #jsontext = ut.to_json({
        #    'qaids': list(qaids_expt),
        #    'dinclude_aids1': list(gt_aids_expt1),
        #    'dinclude_aids2': list(gt_aids_expt2),
        #})
        #annotation_configs.varysize_pzm
        #from ibeis.expt import annotation_configs

        acfg = testres.acfg_list[0]
        import copy
        acfg1 = copy.deepcopy(acfg)
        acfg2 = copy.deepcopy(acfg)
        acfg1['qcfg']['min_pername'] = None
        acfg2['qcfg']['min_pername'] = None
        acfg1['dcfg']['min_pername'] = None
        acfg2['dcfg']['min_gt_per_name'] = None

        acfg1['qcfg']['default_aids'] = qaids_expt
        acfg1['dcfg']['gt_avl_aids'] = gt_avl_aids1
        acfg2['qcfg']['default_aids'] = qaids_expt
        acfg2['dcfg']['gt_avl_aids'] = gt_avl_aids2

        from ibeis.init import filter_annots
        from ibeis.expt import experiment_helpers

        annots1 = filter_annots.expand_acfgs(ibs, acfg1, verbose=True)
        annots2 = filter_annots.expand_acfgs(ibs, acfg2, verbose=True)

        acfg_name_list = dict(  # NOQA
            acfg_list=[acfg1, acfg2],
            expanded_aids_list=[annots1, annots2],
        )
        test_cfg_name_list = ['candidacy_k']
        cfgdict_list, pipecfg_list = experiment_helpers.get_pipecfg_list(test_cfg_name_list, ibs=ibs)

        t1, t2 = testres_list  # NOQA

def get_training_fsv(cm, namemode=True, num=None, top_percent=None):
    """
    CommandLine:
        python -m ibeis.algo.hots.scorenorm --exec-get_training_fsv --show

    Example:
        >>> # ENABLE_DOCTEST
        >>> from ibeis.algo.hots.scorenorm import *  # NOQA
        >>> import ibeis
        >>> num = None
        >>> cm, qreq_ = ibeis.testdata_cm('PZ_MTEST', a='default:dindex=0:10,qindex=0:1', t='best')
        >>> (tp_fsv, tn_fsv) = get_training_fsv(cm, namemode=False)
        >>> result = ('(tp_fsv, tn_fsv) = %s' % (ut.repr2((tp_fsv, tn_fsv), nl=1),))
        >>> print(result)
    """
    if namemode:
        tp_idxs, tn_idxs = get_topname_training_idxs(cm, num=num)
    else:
        tp_idxs, tn_idxs = get_topannot_training_idxs(cm, num=num)

    # Keep only the top scoring half of the feature matches
    # top_percent = None
    if top_percent is not None:
        cm_orig = cm
        #cm_orig.assert_self(qreq_)

        tophalf_indicies = [
            ut.take_percentile(fs.argsort()[::-1], top_percent)
            for fs in cm.get_fsv_prod_list()
        ]
        cm = cm_orig.take_feature_matches(tophalf_indicies, keepscores=True)

        assert np.all(cm_orig.daid_list.take(tp_idxs) == cm.daid_list.take(tp_idxs))
        assert np.all(cm_orig.daid_list.take(tn_idxs) == cm.daid_list.take(tn_idxs))
        #cm.assert_self(qreq_)

    tp_fsv = np.vstack(ut.take(cm.fsv_list, tp_idxs))
    tn_fsv = np.vstack(ut.take(cm.fsv_list, tn_idxs))
    return tp_fsv, tn_fsv

def group_images_by_label(label_arr, gid_arr):
    """
    Input: Length N list of labels and ids
    Output: Length M list of unique labels, and lenth M list of lists of ids
    """
    # Reverse the image to cluster index mapping
    import vtool as vt
    labels_, groupxs_ = vt.group_indices(label_arr)
    sortx = np.array(list(map(len, groupxs_))).argsort()[::-1]
    labels  = labels_.take(sortx, axis=0)
    groupxs = ut.take(groupxs_, sortx)
    label_gids = vt.apply_grouping(gid_arr, groupxs)
    return labels, label_gids