def preload_commands(dbdir, **kwargs):
    """ Preload commands work with command line arguments and global caches """
    #print('[main_cmd] preload_commands')
    if params.args.dump_argv:
        print(ut.dict_str(vars(params.args), sorted_=False))
    if params.args.dump_global_cache:
        ut.global_cache_dump()  # debug command, dumps to stdout
    if params.args.set_workdir is not None:
        sysres.set_workdir(params.args.set_workdir)
    if params.args.get_workdir:
        print(' Current work dir = %s' % sysres.get_workdir())
    if params.args.logdir is not None:
        sysres.set_logdir(params.args.logdir)
    if params.args.get_logdir:
        print(' Current log dir = %s' % (sysres.get_logdir(),))
    if params.args.view_logdir:
        ut.view_directory(sysres.get_logdir())
    if ut.get_argflag('--vwd'):
        vwd()
    if ut.get_argflag('--vdq'):
        print('got arg --vdq')
        vdq(dbdir)
    if kwargs.get('delete_ibsdir', False):
        ibsfuncs.delete_ibeis_database(dbdir)
    if params.args.convert:
        preload_convert_hsdb(dbdir)
    if params.args.preload_exit:
        print('[main_cmd] preload exit')
        sys.exit(1)

def main():
    r"""
    python win32bootstrap.py --dl numpy --nocache
    python win32bootstrap.py --dl numpy-1.9.2rc1 --force
    python win32bootstrap.py --dl numpy-1.9.2rc1 --run
    python win32bootstrap.py --force
    python win32bootstrap.py --dryrun
    python win32bootstrap.py --dryrun --dl numpy scipy
    python win32bootstrap.py --dl numpy

    C:\Users\jon.crall\AppData\Roaming\utool\numpy-1.9.2rc1+mkl-cp27-none-win32.whl
    pip install C:/Users/jon.crall/AppData/Roaming/utool/numpy-1.9.2rc1+mkl-cp27-none-win32.whl

    """
    # Packages that you are requesting
    pkg_list = []
    if ut.get_argflag('--all'):
        pkg_list = KNOWN_PKG_LIST
    else:
        print('specify --all to download all packages')
        print('or specify --dl pkgname to download that package')
    pkg_list.extend(ut.get_argval('--dl', list, []))
    dryrun = ut.get_argflag('--dryrun')
    pkg_exe_list = bootstrap_sysreq(pkg_list, dryrun=dryrun)
    if ut.get_argflag('--run'):
        for pkg_exe in pkg_exe_list:
            if pkg_exe.endswith('.whl'):
                ut.cmd('pip install ' + pkg_exe)

def test_zmq_task():
    """
    CommandLine:
        python -m ibeis.web.zmq_task_queue --exec-test_zmq_task
        python -b -m ibeis.web.zmq_task_queue --exec-test_zmq_task

        python -m ibeis.web.zmq_task_queue --main
        python -m ibeis.web.zmq_task_queue --main --bg
        python -m ibeis.web.zmq_task_queue --main --fg

    Example:
        >>> # SCRIPT
        >>> from ibeis.web.zmq_task_queue import *  # NOQA
        >>> test_zmq_task()
    """
    _init_signals()
    # now start a few clients, and fire off some requests
    client_id = np.random.randint(1000)
    jobiface = JobInterface(client_id)
    reciever = JobBackend()
    if ut.get_argflag('--bg'):
        from ibeis.init import sysres
        dbdir = sysres.get_args_dbdir('cache', False, None, None,
                                      cache_priority=False)
        reciever.initialize_background_processes(dbdir)
        print('[testzmq] parent process is looping forever')
        while True:
            time.sleep(1)
    elif ut.get_argflag('--fg'):
        jobiface.initialize_client_thread()
    else:
        dbdir = sysres.get_args_dbdir('cache', False, None, None,
                                      cache_priority=False)
        reciever.initialize_background_processes(dbdir)
        jobiface.initialize_client_thread()

    # Foreground test script
    print('... waiting for jobs')
    if ut.get_argflag('--cmd'):
        ut.embed()
        jobiface.queue_job()
    else:
        print('[test] ... emit test1')
        jobid1 = jobiface.queue_job('helloworld', 1)
        jobiface.wait_for_job_result(jobid1)
        #jobiface.get_job_status(jobid1)
        #jobid_list = [jobiface.queue_job('helloworld', 5) for _ in range(NUM_JOBS)]
        #jobid_list += [jobiface.queue_job('get_valid_aids')]
        jobid_list = []

        #identify_jobid = jobiface.queue_job('query_chips', [1], [3, 4, 5], cfgdict={'K': 1})
        identify_jobid = jobiface.queue_job('query_chips_simple_dict', [1], [3, 4, 5], cfgdict={'K': 1})

        for jobid in jobid_list:
            jobiface.wait_for_job_result(jobid)

        jobiface.wait_for_job_result(identify_jobid)
    print('FINISHED TEST SCRIPT')

def autogen_ipynb(ibs, launch=None, run=None):
    r"""
    Autogenerates standard IBEIS Image Analysis IPython notebooks.

    CommandLine:
        python -m ibeis --tf autogen_ipynb --run --db lynx

        python -m ibeis --tf autogen_ipynb --ipynb --db PZ_MTEST --asreport
        python -m ibeis --tf autogen_ipynb --ipynb --db PZ_MTEST --noexample --withtags

        python -m ibeis --tf autogen_ipynb --db PZ_MTEST
        # TODO: Add support for dbdir to be specified
        python -m ibeis --tf autogen_ipynb --db ~/work/PZ_MTEST

        python -m ibeis --tf autogen_ipynb --ipynb --db Oxford -a default:qhas_any=\(query,\),dpername=1,exclude_reference=True,dminqual=good
        python -m ibeis --tf autogen_ipynb --ipynb --db PZ_MTEST -a default -t best:lnbnn_normalizer=[None,normlnbnn-test]

        python -m ibeis.templates.generate_notebook --exec-autogen_ipynb --db wd_peter_blinston --ipynb

        python -m ibeis --tf autogen_ipynb --db PZ_Master1 --ipynb
        python -m ibeis --tf autogen_ipynb --db PZ_Master1 -a timectrl:qindex=0:100 -t best best:normsum=True --ipynb --noexample
        python -m ibeis --tf autogen_ipynb --db PZ_Master1 -a timectrl --run
        jupyter-notebook Experiments-lynx.ipynb
        killall python

        python -m ibeis --tf autogen_ipynb --db humpbacks --ipynb -t default:proot=BC_DTW -a default:has_any=hasnotch
        python -m ibeis --tf autogen_ipynb --db humpbacks --ipynb -t default:proot=BC_DTW default:proot=vsmany -a default:has_any=hasnotch,mingt=2,qindex=0:50 --noexample

    Example:
        >>> # SCRIPT
        >>> from ibeis.templates.generate_notebook import *  # NOQA
        >>> import ibeis
        >>> ibs = ibeis.opendb(defaultdb='testdb1')
        >>> result = autogen_ipynb(ibs)
        >>> print(result)
    """
    dbname = ibs.get_dbname()
    fname = 'Experiments-' + dbname
    nb_fpath = fname + '.ipynb'
    if ut.get_argflag('--cells'):
        notebook_cells = make_ibeis_cell_list(ibs)
        print('\n# ---- \n'.join(notebook_cells))
        return
    # TODO: Add support for dbdir to be specified
    notebook_str = make_ibeis_notebook(ibs)
    ut.writeto(nb_fpath, notebook_str)
    run = ut.get_argflag('--run') if run is None else run
    launch = launch if launch is not None else ut.get_argflag('--ipynb')
    if run:
        run_nb = ut.run_ipython_notebook(notebook_str)
        output_fpath = ut.export_notebook(run_nb, fname)
        ut.startfile(output_fpath)
    elif launch:
        ut.cmd('jupyter-notebook', nb_fpath, detatch=True)
        #ut.cmd('ipython-notebook', nb_fpath)
        #ut.startfile(nb_fpath)
    else:
        print('notebook_str =\n%s' % (notebook_str,))

def testdata_show_qres():
    import ibeis
    cm, qreq_ = ibeis.testdata_cm()
    kwargs = dict(
        top_aids=ut.get_argval('--top-aids', type_=int, default=3),
        sidebyside=not ut.get_argflag('--no-sidebyside'),
        annot_mode=ut.get_argval('--annot_mode', type_=int, default=1),
        viz_name_score=not ut.get_argflag('--no-viz_name_score'),
        max_nCols=ut.get_argval('--max_nCols', type_=int, default=None)
    )
    return qreq_.ibs, cm, qreq_, kwargs

def are_you_sure(parent=None, msg=None, title='Confirmation', default=None):
    """ Prompt user for conformation before changing something """
    msg = 'Are you sure?' if msg is None else msg
    print('[gt] Asking User if sure')
    print('[gt] title = %s' % (title,))
    print('[gt] msg =\n%s' % (msg,))
    if ut.get_argflag('-y') or ut.get_argflag('--yes'):
        # DONT ASK WHEN SPECIFIED
        return True
    ans = user_option(parent=parent, msg=msg, title=title, options=['Yes', 'No'],
                           use_cache=False, default=default)
    return ans == 'Yes'

 def __init__(self):
     #self.num_engines = 3
     self.num_engines = NUM_ENGINES
     self.engine_queue_proc = None
     self.collect_queue_proc = None
     self.engine_procs = None
     self.collect_proc = None
     # --
     only_engine = ut.get_argflag('--only-engine')
     self.spawn_collector = not only_engine
     self.spawn_engine = not ut.get_argflag('--no-engine')
     self.spawn_queue = not only_engine

def std_build_command(repo="."):
    """
    DEPRICATE
    My standard build script names.

    Calls mingw_build.bat on windows and unix_build.sh  on unix
    """
    import utool as ut

    print("+**** stdbuild *******")
    print("repo = %r" % (repo,))
    if sys.platform.startswith("win32"):
        # vtool --rebuild-sver didnt work with this line
        # scriptname = './mingw_build.bat'
        scriptname = "mingw_build.bat"
    else:
        scriptname = "./unix_build.sh"
    if repo == "":
        # default to cwd
        repo = "."
    else:
        os.chdir(repo)
    ut.assert_exists(scriptname)
    normbuild_flag = "--no-rmbuild"
    if ut.get_argflag(normbuild_flag):
        scriptname += " " + normbuild_flag
    # Execute build
    ut.cmd(scriptname)
    # os.system(scriptname)
    print("L**** stdbuild *******")

            def exec_(script):
                import utool as ut

                print("+**** exec %s script *******" % (script.type_))
                print("repo = %r" % (repo,))
                with ut.ChdirContext(repo.dpath):
                    if script.is_fpath_valid():
                        normbuild_flag = "--no-rmbuild"
                        if ut.get_argflag(normbuild_flag):
                            ut.cmd(script.fpath + " " + normbuild_flag)
                        else:
                            ut.cmd(script.fpath)
                    else:
                        if script.text is not None:
                            print("ABOUT TO EXECUTE")
                            ut.print_code(script.text, "bash")
                            if ut.are_you_sure("execute above script?"):
                                from os.path import join

                                scriptdir = ut.ensure_app_resource_dir("utool", "build_scripts")
                                script_path = join(
                                    scriptdir, "script_" + script.type_ + "_" + ut.hashstr27(script.text) + ".sh"
                                )
                                ut.writeto(script_path, script.text)
                                _ = ut.cmd("bash ", script_path)  # NOQA
                        else:
                            print("CANT QUITE EXECUTE THIS YET")
                            ut.print_code(script.text, "bash")
                # os.system(scriptname)
                print("L**** exec %s script *******" % (script.type_))

def std_build_command(repo='.'):
    """
    My standard build script names.

    Calls mingw_build.bat on windows and unix_build.sh  on unix
    """
    import utool as ut
    print("+**** stdbuild *******")
    print('repo = %r' % (repo,))
    if sys.platform.startswith('win32'):
        #scriptname = './mingw_build.bat'  # vtool --rebuild-sver didnt work with this line
        scriptname = 'mingw_build.bat'
    else:
        scriptname = './unix_build.sh'
    if repo == '':
        # default to cwd
        repo = '.'
    else:
        os.chdir(repo)
    ut.assert_exists(scriptname)
    normbuild_flag = '--no-rmbuild'
    if ut.get_argflag(normbuild_flag):
        scriptname += ' ' + normbuild_flag
    # Execute build
    ut.cmd(scriptname)
    #os.system(scriptname)
    print("L**** stdbuild *******")

def ensure_text(fname, text, repo_dpath='.', force=None, locals_={}, chmod=None):
    """
    Args:
        fname (str):  file name
        text (str):
        repo_dpath (str):  directory path string(default = '.')
        force (bool): (default = False)
        locals_ (dict): (default = {})

    Example:
        >>> # DISABLE_DOCTEST
        >>> from utool.util_project import *  # NOQA
        >>> import utool as ut
        >>> result = setup_repo()
        >>> print(result)
    """
    import utool as ut
    ut.colorprint('Ensuring fname=%r' % (fname), 'yellow')

    # if not fname.endswith('__init__.py'):
    #     # HACK
    #     return

    if force is None and ut.get_argflag('--force-%s' % (fname,)):
        force = True
    text_ = ut.remove_codeblock_syntax_sentinals(text)
    fmtkw = locals_.copy()
    fmtkw['fname'] = fname
    text_ = text_.format(**fmtkw) + '\n'

    fpath = join(repo_dpath, fname)
    ut.dump_autogen_code(fpath, text_)

def initialize_job_manager(ibs):
    """
    Run from the webserver

    Example:
        >>> # DISABLE_DOCTEST
        >>> from ibeis.web.zmq_task_queue import *  # NOQA
        >>> import ibeis
        >>> ibs = ibeis.opendb('testdb1')

    Example:
        >>> # WEB_DOCTEST
        >>> from ibeis.web.zmq_task_queue import *  # NOQA
        >>> import ibeis
        >>> web_instance = ibeis.opendb_bg_web(db='testdb1', wait=10)
        >>> baseurl = 'http://127.0.1.1:5000'
        >>> _payload = {'image_attrs_list': [], 'annot_attrs_list': []}
        >>> payload = ut.map_dict_vals(ut.to_json, _payload)
        >>> #resp = requests.post(baseurl + '/api/core/helloworld/?f=b', data=payload)
        >>> resp = requests.post(baseurl + '/api/core/add_images_json/', data=payload)
        >>> print(resp)
        >>> web_instance.terminate()
        >>> json_dict = resp.json()
        >>> text = json_dict['response']
        >>> print(text)
    """
    ibs.job_manager = ut.DynStruct()
    ibs.job_manager.jobiface = JobInterface(0)

    if not ut.get_argflag('--fg'):
        ibs.job_manager.reciever = JobBackend()
        ibs.job_manager.reciever.initialize_background_processes(dbdir=ibs.get_dbdir())

    ibs.job_manager.jobiface.initialize_client_thread()

def vizualize_vocabulary(ibs, invindex):
    """
    cleaned up version of dump_word_patches. Makes idf scatter plots and dumps
    the patches that contributed to each word.

    CommandLine:
        python -m ibeis.algo.hots.smk.smk_plots --test-vizualize_vocabulary
        python -m ibeis.algo.hots.smk.smk_plots --test-vizualize_vocabulary --vf

    Example:
        >>> from ibeis.algo.hots.smk.smk_plots import *  # NOQA
        >>> from ibeis.algo.hots.smk import smk_debug
        >>> from ibeis.algo.hots.smk import smk_repr
        >>> #tup = smk_debug.testdata_raw_internals0(db='GZ_ALL', nWords=64000)
        >>> #tup = smk_debug.testdata_raw_internals0(db='GZ_ALL', nWords=8000)
        >>> tup = smk_debug.testdata_raw_internals0(db='PZ_Master0', nWords=64000)
        >>> #tup = smk_debug.testdata_raw_internals0(db='PZ_Mothers', nWords=8000)
        >>> ibs, annots_df, daids, qaids, invindex, qreq_ = tup
        >>> smk_repr.compute_data_internals_(invindex, qreq_.qparams, delete_rawvecs=False)
        >>> vizualize_vocabulary(ibs, invindex)
    """
    invindex.idx2_wxs = np.array(invindex.idx2_wxs)

    print('[smk_plots] Vizualizing vocabulary')

    # DUMPING PART --- dumps patches to disk
    figdir = ibs.get_fig_dir()
    ut.ensuredir(figdir)
    if ut.get_argflag('--vf'):
        ut.view_directory(figdir)

    # Compute Word Statistics
    metrics = compute_word_metrics(invindex)
    wx2_nMembers, wx2_pdist_stats, wx2_wdist_stats = metrics
    #(wx2_pdist, wx2_wdist, wx2_nMembers, wx2_pdist_stats, wx2_wdist_stats) = metrics

    #wx2_prad = {wx: pdist_stats['max'] for wx, pdist_stats in six.iteritems(wx2_pdist_stats) if 'max' in pdist_stats}
    #wx2_wrad = {wx: wdist_stats['max'] for wx, wdist_stats in six.iteritems(wx2_wdist_stats) if 'max' in wdist_stats}

    wx2_prad = {wx: stats['max'] for wx, stats in wx2_pdist_stats.items() if 'max' in stats}
    wx2_wrad = {wx: stats['max'] for wx, stats in wx2_wdist_stats.items() if 'max' in stats}
    #wx2_prad = get_metric(metrics, 'wx2_pdist_stats', 'max')
    #wx2_wrad = get_metric(metrics, 'wx2_wdist_stats', 'max')

    wx_sample1 = select_by_metric(wx2_nMembers)
    wx_sample2 = select_by_metric(wx2_prad)
    wx_sample3 = select_by_metric(wx2_wrad)

    wx_sample = wx_sample1 + wx_sample2 + wx_sample3
    overlap123 = len(wx_sample) - len(set(wx_sample))
    print('overlap123 = %r' % overlap123)
    wx_sample  = set(wx_sample)
    print('len(wx_sample) = %r' % len(wx_sample))

    #make_scatterplots(ibs, figdir, invindex, metrics)

    vocabdir = join(figdir, 'vocab_patches2')
    wx2_dpath = get_word_dpaths(vocabdir, wx_sample, metrics)

    make_wordfigures(ibs, metrics, invindex, figdir, wx_sample, wx2_dpath)

def show_chip_distinctiveness_plot(chip, kpts, dstncvs, fnum=1, pnum=None):
    import plottool as pt
    pt.figure(fnum, pnum=pnum)
    ax = pt.gca()
    divider = pt.ensure_divider(ax)
    #ax1 = divider.append_axes("left", size="50%", pad=0)
    ax1 = ax
    ax2 = divider.append_axes("bottom", size="100%", pad=0.05)
    #f, (ax1, ax2) = pt.plt.subplots(1, 2, sharex=True)
    cmapstr = 'rainbow'  # 'hot'
    color_list = pt.df2.plt.get_cmap(cmapstr)(ut.norm_zero_one(dstncvs))
    sortx = dstncvs.argsort()
    #pt.df2.plt.plot(qfx2_dstncvs[sortx], c=color_list[sortx])
    pt.plt.sca(ax1)
    pt.colorline(np.arange(len(sortx)), dstncvs[sortx],
                 cmap=pt.plt.get_cmap(cmapstr))
    pt.gca().set_xlim(0, len(sortx))
    pt.dark_background()
    pt.plt.sca(ax2)
    pt.imshow(chip, darken=.2)
    # MATPLOTLIB BUG CANNOT SHOW DIFFERENT ALPHA FOR POINTS AND KEYPOINTS AT ONCE
    #pt.draw_kpts2(kpts, pts_color=color_list, ell_color=color_list, ell_alpha=.1, ell=True, pts=True)
    #pt.draw_kpts2(kpts, color_list=color_list, pts_alpha=1.0, pts_size=1.5,
    #              ell=True, ell_alpha=.1, pts=False)
    ell = ut.get_argflag('--ell')
    pt.draw_kpts2(kpts, color_list=color_list, pts_alpha=1.0, pts_size=1.5,
                  ell=ell, ell_alpha=.3, pts=not ell)
    pt.plt.sca(ax)

def testdata_ibeis(**kwargs):
    """
    DEPRICATE
    Step 1

    builds ibs for testing

    Example:
        >>> from ibeis.algo.hots.smk.smk_debug import *  # NOQA
        >>> kwargs = {}
    """
    print(' === Test Data IBEIS ===')
    print('kwargs = ' + ut.dict_str(kwargs))
    print('[smk_debug] testdata_ibeis')
    db = kwargs.get('db', ut.get_argval('--db', str, 'PZ_MTEST'))
    #with ut.Indenter('ENSURE'):
    if db == 'PZ_MTEST':
        ibeis.ensure_pz_mtest()
    ibs = ibeis.opendb(db=db)
    ibs._default_config()
    aggregate = kwargs.get('aggregate', ut.get_argflag(('--agg', '--aggregate')))
    nWords    = kwargs.get(   'nWords',  ut.get_argval(('--nWords', '--nCentroids'), int, default=8E3))
    nAssign   = kwargs.get(  'nAssign',  ut.get_argval(('--nAssign', '--K'), int, default=10))
    # Configs
    ibs.cfg.query_cfg.pipeline_root = 'smk'
    ibs.cfg.query_cfg.smk_cfg.aggregate = aggregate
    ibs.cfg.query_cfg.smk_cfg.smk_alpha = 3
    ibs.cfg.query_cfg.smk_cfg.smk_thresh = 0
    ibs.cfg.query_cfg.smk_cfg.vocabtrain_cfg.nWords = nWords
    ibs.cfg.query_cfg.smk_cfg.vocabassign_cfg.nAssign = nAssign
    if ut.VERYVERBOSE:
        ibs.cfg.query_cfg.smk_cfg.printme3()
    return ibs

def _init_ibeis(dbdir=None, verbose=None, use_cache=True, web=None, **kwargs):
    """
    Private function that calls code to create an ibeis controller
    """
    import utool as ut
    from ibeis import params
    from ibeis.control import IBEISControl
    if verbose is None:
        verbose = ut.VERBOSE
    if verbose and NOT_QUIET:
        print('[main] _init_ibeis()')
    # Use command line dbdir unless user specifies it
    if dbdir is None:
        ibs = None
        ut.printWARN('[main!] WARNING args.dbdir is None')
    else:
        kwargs = kwargs.copy()
        request_dbversion = kwargs.pop('request_dbversion', None)
        asproxy = kwargs.pop('asproxy', None)
        ibs = IBEISControl.request_IBEISController(
            dbdir=dbdir, use_cache=use_cache,
            request_dbversion=request_dbversion,
            asproxy=asproxy)
        if web is None:
            web = ut.get_argflag(('--webapp', '--webapi', '--web', '--browser'),
                                 help_='automatically launch the web app / web api')
            #web = params.args.webapp
        if web:
            from ibeis.web import app
            port = params.args.webport
            app.start_from_ibeis(ibs, port=port, **kwargs)
    return ibs

def _sed(r, regexpr, repl, force=False, recursive=False, dpath_list=None):
    if True:
        import utool as ut

        force = ut.smart_cast2(force)
        ut.sed(regexpr, repl, force=force, recursive=recursive, dpath_list=dpath_list, verbose=True)
        return
    else:
        # _grep(r, [repl], dpath_list=dpath_list, recursive=recursive)
        force = rutil.cast(force, bool)
        recursive = rutil.cast(recursive, bool)
        import utool as ut

        pyext = ut.get_argflag("--pyext")
        if pyext:
            include_patterns = ["*.py"]
        else:
            include_patterns = ["*.py", "*.cxx", "*.cpp", "*.hxx", "*.hpp", "*.c", "*.h", "*.pyx", "*.pxi"]
        if ut.get_argflag("--all"):
            include_patterns = ["*"]
        # if ut.get_argflag('--tex'):
        include_patterns = ["*.tex"]
        if dpath_list is None:
            dpath_list = [os.getcwd()]
        regexpr = extend_regex(regexpr)
        # import re
        print("sed-ing %r" % (dpath_list,))
        print(" * regular include_patterns : %r" % (include_patterns,))
        print(" * (orig) regular expression : %r" % (regexpr,))
        print(" * (origstr) regular expression : %s" % (regexpr,))
        # regexpr = re.escape(regexpr)
        print(" * regular expression : %r" % (regexpr,))
        print(" * (str)regular expression : %s" % (regexpr,))
        print(" * replacement        : %r" % (repl,))
        print(" * recursive: %r" % (recursive,))
        print(" * force: %r" % (force,))
        if "\x08" in regexpr:
            print("Remember \\x08 != \\b")
            print("subsituting for you for you")
            regexpr = regexpr.replace("\x08", "\\b")
            print(" * regular expression : %r" % (regexpr,))

        # Walk through each directory recursively
        num_changed = 0
        for fpath in _matching_fnames(dpath_list, include_patterns, recursive=recursive):
            num_changed += len(__regex_sedfile(fpath, regexpr, repl, force))
        print("total lines changed = %r" % (num_changed,))

 def get_extract_tuple(aid, fx, k=-1):
     rchip = ibs.get_annot_chips(aid)
     kp    = ibs.get_annot_kpts(aid)[fx]
     sift  = ibs.get_annot_vecs(aid)[fx]
     if not ut.get_argflag('--texknormplot'):
         aidstr = vh.get_aidstrs(aid)
         nidstr = vh.get_nidstrs(ibs.get_annot_nids(aid))
         id_str = ' ' + aidstr + ' ' + nidstr + ' fx=%r' % (fx,)
     else:
         id_str = nidstr = aidstr = ''
     info = ''
     if k == -1:
         if pt.is_texmode():
             info = '\\vspace{1cm}'
             info += 'Query $\\mathbf{d}_i$'
             info += '\n\\_'
             info += '\n\\_'
         else:
             if len(id_str) > '':
                 info = 'Query: %s' % (id_str,)
             else:
                 info = 'Query'
         type_ = 'Query'
     elif k < K:
         type_ = 'Match'
         if ut.get_argflag('--texknormplot') and  pt.is_texmode():
             #info = 'Match:\n$k=%r$, $\\frac{||\\mathbf{d}_i - \\mathbf{d}_j||}{Z}=%.3f$' % (k, qfx2_dist[0, k])
             info = '\\vspace{1cm}'
             info += 'Match: $\\mathbf{d}_{j_%r}$\n$\\textrm{dist}=%.3f$' % (k, qfx2_dist[0, k])
             info += '\n$s_{\\tt{LNBNN}}=%.3f$' % (qfx2_dist[0, K + Knorm - 1] - qfx2_dist[0, k])
         else:
             info = 'Match:%s\nk=%r, dist=%.3f' % (id_str, k, qfx2_dist[0, k])
             info += '\nLNBNN=%.3f' % (qfx2_dist[0, K + Knorm - 1] - qfx2_dist[0, k])
     elif k < Knorm + K:
         type_ = 'Norm'
         if ut.get_argflag('--texknormplot') and  pt.is_texmode():
             #info = 'Norm: $j_%r$\ndist=%.3f' % (id_str, k, qfx2_dist[0, k])
             info = '\\vspace{1cm}'
             info += 'Norm: $j_%r$\n$\\textrm{dist}=%.3f$' % (k, qfx2_dist[0, k])
             info += '\n\\_'
         else:
             info = 'Norm: %s\n$k=%r$, dist=$%.3f$' % (id_str, k, qfx2_dist[0, k])
     else:
         raise Exception('[viz] problem k=%r')
     return (rchip, kp, sift, fx, aid, info, type_)

def testdata_expts(defaultdb='testdb1',
                   default_acfgstr_name_list=['default:qindex=0:10:4,dindex=0:20'],
                   default_test_cfg_name_list=['default'],
                   a=None,
                   t=None,
                   qaid_override=None,
                   daid_override=None,
                   initial_aids=None,
                   ):
    """
    Use this if you want data from an experiment.
    Command line interface to quickly get testdata for test_results.

    Command line flags can be used to specify db, aidcfg, pipecfg, qaid
    override, daid override (and maybe initial aids).

    """
    print('[main_helpers] testdata_expts')
    import ibeis
    from ibeis.expt import experiment_harness
    from ibeis.expt import test_result
    if a is not None:
        default_acfgstr_name_list = a
    if t is not None:
        default_test_cfg_name_list = t

    if isinstance(default_acfgstr_name_list, six.string_types):
        default_acfgstr_name_list = [default_acfgstr_name_list]
    if isinstance(default_test_cfg_name_list, six.string_types):
        default_test_cfg_name_list = [default_test_cfg_name_list]

    #from ibeis.expt import experiment_helpers
    ibs = ibeis.opendb(defaultdb=defaultdb)
    acfg_name_list = ut.get_argval(('--aidcfg', '--acfg', '-a'), type_=list,
                                   default=default_acfgstr_name_list)
    test_cfg_name_list = ut.get_argval(('-t', '-p'), type_=list, default=default_test_cfg_name_list)
    daid_override = ut.get_argval(('--daid-override', '--daids-override'), type_=list, default=daid_override)
    qaid_override = ut.get_argval(('--qaid', '--qaids-override', '--qaid-override'), type_=list, default=qaid_override)

    # Hack a cache here
    use_bigtest_cache3 = not ut.get_argflag(('--nocache', '--nocache-hs'))
    use_bigtest_cache3 &= ut.is_developer()
    use_bigtest_cache3 &= False
    if use_bigtest_cache3:
        from os.path import dirname, join
        cache_dir = ut.ensuredir(join(dirname(ut.get_module_dir(ibeis)), 'BIG_TESTLIST_CACHE3'))
        load_testres = ut.cached_func('testreslist', cache_dir=cache_dir)(experiment_harness.run_test_configurations2)
    else:
        load_testres = experiment_harness.run_test_configurations2
    testres_list = load_testres(
        ibs, acfg_name_list, test_cfg_name_list, qaid_override=qaid_override,
        daid_override=daid_override, initial_aids=initial_aids)
    testres = test_result.combine_testres_list(ibs, testres_list)

    print(testres)

    return ibs, testres

def _load_named_config(ibs, cfgname=None):
    r"""
    """
    # TODO: update cfgs between versions
    # Try to load previous config otherwise default
    #use_config_cache = not (ut.is_developer() and not ut.get_argflag(('--nocache-pref',)))
    use_config_cache = not ut.get_argflag(('--nocache-pref',))
    ibs.cfg = Config.load_named_config(cfgname, ibs.get_dbdir(), use_config_cache)
    ibs.reset_table_cache()