def test_vectorize_multipath():
    N = as_apply(15)

    p0 = hp_uniform('p0', 0, 1)
    loss = hp_choice('p1', [1, p0, -p0]) ** 2
    expr_idxs = scope.range(N)
    vh = VectorizeHelper(loss, expr_idxs, build=True)

    vloss = vh.v_expr
    print vloss

    full_output = as_apply([vloss,
        vh.idxs_by_label(),
        vh.vals_by_label()])

    new_vc = recursive_set_rng_kwarg(
            full_output,
            as_apply(np.random.RandomState(1)),
            )

    losses, idxs, vals = rec_eval(new_vc)
    print 'losses', losses
    print 'idxs p0', idxs['p0']
    print 'vals p0', vals['p0']
    print 'idxs p1', idxs['p1']
    print 'vals p1', vals['p1']
    p0dct = dict(zip(idxs['p0'], vals['p0']))
    p1dct = dict(zip(idxs['p1'], vals['p1']))
    for ii, li in enumerate(losses):
        print ii, li
        if p1dct[ii] != 0:
            assert li == p0dct[ii] ** 2
        else:
            assert li == 1

def test_vectorize_simple():
    N = as_apply(15)

    p0 = hp_uniform('p0', 0, 1)
    loss = p0 ** 2
    print loss
    expr_idxs = scope.range(N)
    vh = VectorizeHelper(loss, expr_idxs, build=True)
    vloss = vh.v_expr

    full_output = as_apply([vloss,
        vh.idxs_by_label(),
        vh.vals_by_label()])
    fo2 = replace_repeat_stochastic(full_output)

    new_vc = recursive_set_rng_kwarg(
            fo2,
            as_apply(np.random.RandomState(1)),
            )

    #print new_vc
    losses, idxs, vals = rec_eval(new_vc)
    print 'losses', losses
    print 'idxs p0', idxs['p0']
    print 'vals p0', vals['p0']
    p0dct = dict(zip(idxs['p0'], vals['p0']))
    for ii, li in enumerate(losses):
        assert p0dct[ii] ** 2 == li

def get_performance(slm, decisions, preproc, comparison,
                    namebase=None, progkey='result_w_cleanup',
                    return_multi=False, ctrl=None):
    if decisions is None:
        decisions = np.zeros((1, 3200))
    else:
        decisions = np.asarray(decisions)
    assert decisions.shape == (1, 3200)
    if namebase is None:
        namebase = 'memmap_' + str(np.random.randint(1e8))
    image_features = scope.slm_memmap(
            desc=slm,
            X=scope.get_images('float32', preproc=preproc),
            name=namebase + '_img_feat')
    if return_multi:
        comps = ['mult', 'sqrtabsdiff']
    else:
        comps = [comparison]
    cmp_progs = []
    for comp in comps:
        sresult = screening_program(
                    slm_desc=slm,
                    preproc=preproc,
                    comparison=comp,
                    namebase=namebase,
                    decisions=decisions,
                    image_features=image_features,
                    ctrl=ctrl)[1][progkey]
        cmp_progs.append([comp, sresult])
    cmp_results = pyll.rec_eval(cmp_progs)
    if return_multi:
        return cmp_results
    else:
        return cmp_results[0][1]

    def suggest(self, new_ids, trials):
        """
        new_ids - a list of unique identifiers (not necessarily ints!)
                  for the suggestions that this function should return.

        All lists have the same length.
        """
        # XXX: this used to be the implementation for the Random class and the
        # base class.  But then I was doing an experiment with Random() a
        # different seed every time and I was surprised to see it generating
        # the same thing all the time!  In response, I gave the Random
        # subclass its own simpler and more random implementation of suggest
        # that does not re-seed self.rng based on the new_ids. That leaves
        # this strange implementation here in the base class, and I'm not sure
        # whether to delete it. -JB June 19 2012
        #
        # -- install new_ids as program arguments
        rval = []
        for new_id in new_ids:
            # the results are not computed all at once so that we can
            # seed the generator based on each new_id
            sh1 = hashlib.sha1()
            sh1.update(str(new_id))
            self.rng.seed(int(int(sh1.hexdigest(), base=16) % (2 ** 31)))

            # -- sample new specs, idxs, vals
            idxs, vals = pyll.rec_eval(self.s_idxs_vals, memo={self.s_new_ids: [new_id]})
            # print 'BandigAlgo.suggest IDXS', idxs
            # print 'BandigAlgo.suggest VALS', vals
            new_result = self.bandit.new_result()
            new_misc = dict(tid=new_id, cmd=self.cmd, workdir=self.workdir)
            miscs_update_idxs_vals([new_misc], idxs, vals)
            rval.extend(trials.new_trial_docs([new_id], [None], [new_result], [new_misc]))
        return rval

def test_qlognormal_never_0():
    rng = np.random.RandomState(234)
    s = scope.qlognormal(-5, 3, 0.1)
    recursive_set_rng_kwarg(s, rng)
    results = [rec_eval(s) for i in range(100)]
    assert min(results) == 0.1
    assert max(results) != 0.1

        def work(self):
            bandit = self.bandit
            random_algo = Random(bandit)
            # build an experiment of 10 trials
            trials = Trials()
            exp = Experiment(trials, random_algo)
            #print random_algo.s_specs_idxs_vals
            exp.run(10)
            ids = trials.tids
            assert len(ids) == 10
            tpe_algo = TreeParzenEstimator(bandit)
            #print pyll.as_apply(tpe_algo.post_idxs)
            #print pyll.as_apply(tpe_algo.post_vals)
            argmemo = {}

            print trials.miscs
            idxs, vals = miscs_to_idxs_vals(trials.miscs)
            argmemo[tpe_algo.observed['idxs']] = idxs
            argmemo[tpe_algo.observed['vals']] = vals
            argmemo[tpe_algo.observed_loss['idxs']] = trials.tids
            argmemo[tpe_algo.observed_loss['vals']] = trials.losses()
            stuff = pyll.rec_eval([tpe_algo.post_below['idxs'],
                        tpe_algo.post_below['vals']],
                        memo=argmemo)
            print stuff

 def evaluate(self, config, ctrl):
     memo = self.memo_from_config(config)
     memo[self.pyll_ctrl] = ctrl
     if self.init_pyll_memo:
         memo = self.init_pyll_memo(memo, config=config, ctrl=ctrl)
     if self.rng is not None and not self.installed_rng:
         # -- N.B. this modifies the expr graph in-place
         #    XXX this feels wrong
         self.expr = recursive_set_rng_kwarg(self.expr,
             pyll.as_apply(self.rng))
         self.installed_rng = True
     try:
         # -- the "work" of evaluating `config` can be written
         #    either into the pyll part (self.expr)
         #    or the normal Python part (self.fn)
         pyll_rval = pyll.rec_eval(self.expr, memo=memo)
         rval = self.fn(pyll_rval)
     except Exception, e:
         n_match = 0
         for match, match_pair in self.exceptions:
             if match(e):
                 rval = match_pair(e)
                 logger.info('Caught fn exception %s' % str(rval))
                 n_match += 1
                 break
         if n_match == 0:
             raise

def test_recursive_set_rng_kwarg():
    uniform = scope.uniform
    a = as_apply([uniform(0, 1), uniform(2, 3)])
    rng = np.random.RandomState(234)
    recursive_set_rng_kwarg(a, rng)
    print a
    val_a = rec_eval(a)
    assert 0 < val_a[0] < 1
    assert 2 < val_a[1] < 3

def test_clone():
    config = config0()
    config2 = clone(config)

    nodeset = set(dfs(config))
    assert not any(n in nodeset for n in dfs(config2))

    foo = recursive_set_rng_kwarg(
                config,
                scope.rng_from_seed(5))
    r = rec_eval(foo)
    print r
    r2 = rec_eval(
            recursive_set_rng_kwarg(
                config2,
                scope.rng_from_seed(5)))

    print r2
    assert r == r2

def suggest_batch(new_ids, domain, trials, seed):

    rng = np.random.RandomState(seed)
    # -- sample new specs, idxs, vals
    idxs, vals = pyll.rec_eval(
        domain.s_idxs_vals,
        memo={
            domain.s_new_ids: new_ids,
            domain.s_rng: rng,
        })
    return idxs, vals

def test_screening_prog_for_smoke():
    # smoke test
    prog = toyproblem.screening_prog(ctrl=Ctrl(None), **config_tiny)
    sprog = str(prog)
    #print sprog
    rval = pyll.rec_eval(prog)
    #print rval
    assert 'loss' in rval
    assert 'decisions' in rval
    assert len(rval['splits']) == 2
    assert rval['splits'][0] != rval['splits'][1]

def suggest(new_ids, domain, trials, seed=123):

    rval = []
    for new_id in new_ids:
        # -- sample new specs, idxs, vals
        idxs, vals = pyll.rec_eval(domain.s_idxs_vals,
                memo={domain.s_new_ids: [new_id]})
        new_result = domain.new_result()
        new_misc = dict(tid=new_id, cmd=domain.cmd, workdir=domain.workdir)
        miscs_update_idxs_vals([new_misc], idxs, vals)
        rval.extend(trials.new_trial_docs([new_id],
                [None], [new_result], [new_misc]))
    return rval

    def evaluate(self, config, ctrl, attach_attachments=True):
        memo = self.memo_from_config(config)
        self.use_obj_for_literal_in_memo(ctrl, base.Ctrl, memo)
        if self.rng is not None and not self.installed_rng:
            # -- N.B. this modifies the expr graph in-place
            #    XXX this feels wrong
            self.expr = recursive_set_rng_kwarg(self.expr,
                pyll.as_apply(self.rng))
            self.installed_rng = True
        if self.pass_expr_memo_ctrl:
            rval = self.fn(
                    expr=self.expr,
                    memo=memo,
                    ctrl=ctrl,
                    *self.args)
        else:
            # -- the "work" of evaluating `config` can be written
            #    either into the pyll part (self.expr)
            #    or the normal Python part (self.fn)
            pyll_rval = pyll.rec_eval(self.expr, memo=memo,
                    print_node_on_error=self.rec_eval_print_node_on_error)
            rval = self.fn(pyll_rval, *self.args)

        if isinstance(rval, (float, int, np.number)):
            dict_rval = {'loss': rval}
        elif isinstance(rval, (dict,)):
            dict_rval = rval
            if 'loss' not in dict_rval:
                raise ValueError('dictionary must have "loss" key',
                        dict_rval.keys())
        else:
            raise TypeError('invalid return type (neither number nor dict)', rval)

        if dict_rval['loss'] is not None:
            # -- fail if cannot be cast to float
            dict_rval['loss'] = float(dict_rval['loss'])

        dict_rval.setdefault('status', base.STATUS_OK)
        if dict_rval['status'] not in base.STATUS_STRINGS:
            raise ValueError('invalid status string', dict_rval['status'])

        if attach_attachments:
            attachments = dict_rval.pop('attachments', {})
            for key, val in attachments.items():
                ctrl.attachments[key] = val

        # -- don't do this here because SON-compatibility is only a requirement
        #    for trials destined for a mongodb. In-memory rvals can contain
        #    anything.
        #return base.SONify(dict_rval)
        return dict_rval

 def logp(apply_node):
     val = memo_cpy[apply_node]
     if val is pyll.base.GarbageCollected:
         # -- XXX: confirm this happens because the hyperparam is unused.
         return 0
     if 'uniform' in apply_node.name:
         low = apply_node.arg['low'].obj
         high = apply_node.arg['high'].obj
         if 'q' in apply_node.name:
             q = apply_node.arg['q'].obj
         if apply_node.name == 'uniform':
             return rdists.uniform_gen(a=low, b=high).logpdf(
                 val, loc=low, scale=(high - low))
         elif apply_node.name == 'quniform':
             return rdists.quniform_gen(
                 low=low, high=high, q=q).logpmf(val)
         elif apply_node.name == 'loguniform':
             return rdists.loguniform_gen(
                 low=low, high=high).logpdf(val)
         elif apply_node.name == 'qloguniform':
             return rdists.qloguniform_gen(
                 low=low, high=high, q=q).logpmf(val)
         else:
             raise NotImplementedError(apply_node.name) 
     elif 'normal' in apply_node.name:
         mu = apply_node.arg['mu'].obj
         sigma = apply_node.arg['sigma'].obj
         if 'q' in apply_node.name:
             q = apply_node.arg['q'].obj
         if apply_node.name == 'normal':
             return scipy.stats.norm(
                 loc=mu, scale=sigma).logpdf(val)
         elif apply_node.name == 'qnormal':
             return rdists.qnormal_gen(
                 mu=mu, sigma=sigma, q=q).logpmf(val)
         elif apply_node.name == 'lognormal':
             return rdists.lognorm_gen(
                 mu=mu, sigma=sigma).logpdf(val)
         elif apply_node.name == 'qlognormal':
             return rdists.qlognormal_gen(
                 mu=mu, sigma=sigma, q=q).logpmf(val)
         else:
             raise NotImplementedError(apply_node.name) 
     elif apply_node.name == 'randint':
         return -math.log(apply_node.arg['upper'].obj)
     elif apply_node.name == 'categorical':
         assert val == int(val), val
         p = pyll.rec_eval(apply_node.arg['p'])
         return math.log(p[int(val)])
     else:
         raise NotImplementedError(apply_node.name)

def suggest(new_ids, domain, trials, seed=123):
    logger.info('generating trials for new_ids: %s' % str(new_ids))

    rval = []
    for new_id in new_ids:
        # -- hack - domain should be read-only here :/
        #    in fact domain should not have its own seed or rng
        domain.rng.seed(seed + new_id)
        # -- sample new specs, idxs, vals
        idxs, vals = pyll.rec_eval(domain.s_idxs_vals,
                memo={domain.s_new_ids: [new_id]})
        new_result = domain.new_result()
        new_misc = dict(tid=new_id, cmd=domain.cmd, workdir=domain.workdir)
        miscs_update_idxs_vals([new_misc], idxs, vals)
        rval.extend(trials.new_trial_docs([new_id],
                [None], [new_result], [new_misc]))
    return rval

def space_eval(space, hp_assignment):
    """Compute a point in a search space from a hyperparameter assignment.

    Parameters:
    -----------
    space - a pyll graph involving hp nodes (see `pyll_utils`).

    hp_assignment - a dictionary mapping hp node labels to values.
    """
    nodes = pyll.toposort(space)
    memo = {}
    for node in nodes:
        if node.name == 'hyperopt_param':
            label = node.arg['label'].eval()
            if label in hp_assignment:
                memo[node] = hp_assignment[label]
    rval = pyll.rec_eval(space, memo=memo)
    return rval

    def evaluate_async(self, config, ctrl, attach_attachments=True,):
        '''
        this is the first part of async evaluation for ipython parallel engines (see ipy.py)

        This breaks evaluate into two parts to allow for the apply_async call
        to only pass the objective function and arguments.
        '''
        memo = self.memo_from_config(config)
        use_obj_for_literal_in_memo(self.expr, ctrl, Ctrl, memo)
        if self.pass_expr_memo_ctrl:
            rval = self.fn(expr=self.expr, memo=memo, ctrl=ctrl)
        else:
            # -- the "work" of evaluating `config` can be written
            #    either into the pyll part (self.expr)
            #    or the normal Python part (self.fn)
            pyll_rval = pyll.rec_eval(
                self.expr,
                memo=memo,
                print_node_on_error=self.rec_eval_print_node_on_error)
            return (self.fn,pyll_rval)

def suggest(new_ids, domain, trials, seed):
    #logger.debug("in suggest with seed: %s" % (str(seed)))
    #logger.debug('generating trials for new_ids: %s' % str(new_ids))

    rng = np.random.RandomState(seed)
    rval = []
    for ii, new_id in enumerate(new_ids):
        # -- sample new specs, idxs, vals
        idxs, vals = pyll.rec_eval(
            domain.s_idxs_vals,
            memo={
                domain.s_new_ids: [new_id],
                domain.s_rng: rng,
            })
        new_result = domain.new_result()
        new_misc = dict(tid=new_id, cmd=domain.cmd, workdir=domain.workdir)
        miscs_update_idxs_vals([new_misc], idxs, vals)
        rval.extend(trials.new_trial_docs([new_id],
                    [None], [new_result], [new_misc]))
    return rval

 def evaluate(self, config, ctrl):
     """Return a result document
     """
     memo = self.memo_from_config(config)
     self.use_obj_for_literal_in_memo(ctrl, Ctrl, memo)
     if self.rng is not None and not self.installed_rng:
         # -- N.B. this modifies the expr graph in-place
         #    XXX this feels wrong
         self.expr = recursive_set_rng_kwarg(self.expr, pyll.as_apply(self.rng))
         self.installed_rng = True
     try:
         r_dct = pyll.rec_eval(self.expr, memo=memo)
     except Exception, e:
         n_match = 0
         for match, match_pair in self.exceptions:
             if match(e):
                 r_dct = match_pair(e)
                 n_match += 1
                 break
         if n_match == 0:
             raise

    def evaluate(self, config, ctrl, attach_attachments=True):
        memo = self.memo_from_config(config)
        use_obj_for_literal_in_memo(self.expr, ctrl, Ctrl, memo)
        if self.pass_expr_memo_ctrl:
            rval = self.fn(expr=self.expr, memo=memo, ctrl=ctrl)
        else:
            # -- the "work" of evaluating `config` can be written
            #    either into the pyll part (self.expr)
            #    or the normal Python part (self.fn)
            pyll_rval = pyll.rec_eval(
                self.expr,
                memo=memo,
                print_node_on_error=self.rec_eval_print_node_on_error)
            rval = self.fn(pyll_rval)

        if isinstance(rval, (float, int, np.number)):
            dict_rval = {'loss': float(rval), 'status': STATUS_OK}
        else:
            dict_rval = dict(rval)
            status = dict_rval['status']
            if status not in STATUS_STRINGS:
                raise InvalidResultStatus(dict_rval)

            if status == STATUS_OK:
                # -- make sure that the loss is present and valid
                try:
                    dict_rval['loss'] = float(dict_rval['loss'])
                except (TypeError, KeyError):
                    raise InvalidLoss(dict_rval)

        if attach_attachments:
            attachments = dict_rval.pop('attachments', {})
            for key, val in attachments.items():
                ctrl.attachments[key] = val

        # -- don't do this here because SON-compatibility is only a requirement
        #    for trials destined for a mongodb. In-memory rvals can contain
        #    anything.
        #return base.SONify(dict_rval)
        return dict_rval