def most_popular_cat_from_ing(
    df, df_i, num_ingredients, category, freq=True, top_n=3, fname=None):
    from foodessentials import get_perc
    counts = df_i['ingredient'].value_counts()
    ing_names = counts.index.values[:num_ingredients]
    if fname:
        with open(fname, 'wb') as f_out:
            for ing in ing_names:
                x = get_perc(ing, category, df, df_i)
                if freq:
                    # Sort by freq rather than percentage.
                    x = sorted(x, key=lambda x : x[2])
                cats = np.array([str(i[1]) for i in x[-top_n:][::-1]])
                f_out.write('{} --> {}\n'.format(ing, 
                    np.array_str(cats, 
                        max_line_width=10000).replace('\n', '')
                    ))
    else:
        for ing in ing_names:
            x = get_perc(ing, category, df, df_i)
            if freq:
                # Sort by freq rather than percentage.
                x = sorted(x, key=lambda x : x[2])
            cats = np.array([str(i[1]) for i in x[-top_n:][::-1]])
            print '{} --> {}'.format(ing, 
                np.array_str(cats, 
                    max_line_width=10000).replace('\n', '')
                )

def main():
    if not os.path.exists("./logfiles"):
        os.makedirs("logfiles")
    logging.basicConfig(filename="./logfiles/test_ensemble.log",
                        level=logging.INFO)

    print("\nNumber of threads: 4")
    print("Maximum number of evaluations: 50")
    print("Search strategy: CandidateSRBF")
    print("Experimental design: Latin Hypercube + point [0.1, 0.5, 0.8]")
    print("Surrogate: Cubic RBF, Linear RBF, Thin-plate RBF, MARS")

    nthreads = 4
    maxeval = 50
    nsamples = nthreads

    data = Hartman3()
    print(data.info)

    # Use 3 differents RBF's and MARS as an ensemble surrogate
    models = [
        RBFInterpolant(surftype=CubicRBFSurface, maxp=maxeval),
        RBFInterpolant(surftype=LinearRBFSurface, maxp=maxeval),
        RBFInterpolant(surftype=TPSSurface, maxp=maxeval)
    ]
    response_surface = EnsembleSurrogate(models, maxeval)

    # Add an additional point to the experimental design. If a good
    # solution is already known you can add this point to the
    # experimental design
    extra = np.atleast_2d([0.1, 0.5, 0.8])

    # Create a strategy and a controller
    controller = ThreadController()
    controller.strategy = \
        SyncStrategyNoConstraints(
            worker_id=0, data=data,
            response_surface=response_surface,
            maxeval=maxeval, nsamples=nsamples,
            exp_design=LatinHypercube(dim=data.dim, npts=2*(data.dim+1)),
            search_procedure=CandidateSRBF(data=data, numcand=100*data.dim),
            extra=extra)

    # Launch the threads and give them access to the objective function
    for _ in range(nthreads):
        worker = BasicWorkerThread(controller, data.objfunction)
        controller.launch_worker(worker)

    # Run the optimization strategy
    result = controller.run()

    response_surface.compute_weights()
    print('Final weights: {0}'.format(
        np.array_str(response_surface.weights, max_line_width=np.inf,
                     precision=5, suppress_small=True)))

    print('Best value found: {0}'.format(result.value))
    print('Best solution found: {0}\n'.format(
        np.array_str(result.params[0], max_line_width=np.inf,
                     precision=5, suppress_small=True)))

def shrink_data(inX, iny, n):
    outX = []
    outy = []
    last_ip = 0
    nrounds = 0
    duplicates = set()
    for i in range(inX.shape[0]):
        cur_ip = inX[i][1]
        if cur_ip == last_ip:
            # Haven't yet filled up all of cur_y yet
            if nrounds < n:
                if np.array_str(inX[i]) in duplicates:
                    print "found duplicate at ip", cur_ip, "prognum", inX[i][0]
                    continue
                else:
                    outX.append(inX[i])
                    outy.append(iny[i])
                    duplicates.add(np.array_str(inX[i]))
                    nrounds += 1
        # First round of new IP value
        else:
            last_ip = cur_ip
            nrounds = 1
            if np.array_str(inX[i]) in duplicates:
                print "found duplicate at ip", cur_ip, "prognum", inX[i][0]
                continue
            else:
                outX.append(inX[i])
                outy.append(iny[i])
                duplicates.add(np.array_str(inX[i]))
    X_shrunk = np.array(outX)
    y_shrunk = np.array(outy)
    np.save("X-shrunk.npy",X_shrunk)
    np.save("y-shrunk.npy",y_shrunk)
    return X_shrunk, y_shrunk

def run():
    """Run the evolution."""
    if args.verbose and __name__ == '__main__':
        print "objective: minimise", eval_func.__doc__

    if args.seed is not None:
        np.random.seed(args.seed)
    hof = tools.HallOfFame(1)
    stats = tools.Statistics(lambda ind: ind.fitness.values)
    stats.register("min", np.min)

    try:
        algorithms.eaGenerateUpdate(toolbox, ngen=args.generations,
                                    stats=stats, halloffame=hof, verbose=True)
    except KeyboardInterrupt:
        print 'user terminated early'

    (score,) = hof[0].fitness.values
    print 'Score: %.2f $/MWh' % score
    print 'List:', [max(0, param) for param in hof[0]]

    set_generators(hof[0])
    nem.run(context)
    context.verbose = True
    print context
    if args.transmission:
        x = context.exchanges.max(axis=0)
        print np.array_str(x, precision=1, suppress_small=True)
        f = open('results.json', 'w')
        obj = {'exchanges': x.tolist(), 'generators': context}
        json.dump(obj, f, cls=nem.Context.JSONEncoder)
        f.close()

def topic_model_on_zlda(docs, vocab, num_topics=5, zlabels=None, eta=0.95, file_out=None):
    """
    See http://pages.cs.wisc.edu/~andrzeje/research/zl_lda.html
    :param docs:
    :param vocab:
    :param num_topics:
    :param zlabels: each entry is ignored unless it is a List.
    :param eta: confidence in the our labels. If eta = 0 --> don't use z-labels, if eta = 1 --> "hard" z-labels.
    :param file_out:
    :return: Phi - P(w|z), Theta - P(z|d)
    """
    alpha = .1 * np.ones((1, num_topics))
    beta = .1 * np.ones((num_topics, len(vocab)))
    numsamp = 100
    randseed = 194582

    if not zlabels:
        zlabels = [[0]*len(text) for text in docs]

    phi, theta, sample = zlabelLDA(docs, zlabels, eta, alpha, beta, numsamp, randseed)
    if file_out:
        print('\nTheta - P(z|d)\n', np.array_str(theta, precision=2), file=file_out)
        print('\n\nPhi - P(w|z)\n', np.array_str(phi,precision=2), file=file_out)
        print('\n\nsample', file=file_out)
        for doc in range(len(docs)):
            print(sample[doc], file=file_out)

    return phi, theta

    def pretty_string_samples(self, idx_start=0, idx_end=20, precision=4, header=False):
        s = ''
        if header:
            t = '  '
            u = 'ch'
            for i in range(self.ch):
                t += '-------:'
                u += '  %2i   :' %(i+1)
            t += '\n'
            u += '\n'
            
            s += t  #   -------:-------:-------:
            s += u  # ch   1   :   2   :   3   :
            s += t  #   -------:-------:-------:

        s += np.array_str(self.samples[idx_start:idx_end,:],
                          max_line_width=260,   # we can print 32 channels before linewrap
                          precision=precision,
                          suppress_small=True)
        if (idx_end-idx_start) < self.nofsamples:
            s  = s[:-1] # strip the right ']' character
            s += '\n ...,\n'
            lastlines = np.array_str(self.samples[-3:,:],
                                     max_line_width=260,
                                     precision=precision,
                                     suppress_small=True)
            s += ' %s\n' %lastlines[1:] # strip first '['
        return s

	def __str__(self):
		s = str("\n\tObservationType:" + self.observeType + "\tPARAM Arr: " + np.array_str(self.ParamArr) + "\tTARGET Arr: " + np.array_str(self.TargetArr))
		if(self.PredictionErrArr != None):
			s = s + "\tPREDICTED Arr: " + np.array_str(self.PredictedArr)
			s = s + "\tPREDICTION ERROR: " + str(self.PredictionErrArr)
			s = s + "\tDISTANCE: " + str(self.DistanceToTargetArr)
		return s

def do_everything(input_file = 'experiments.txt', output_file = 'results.txt', mp=True, oci=False):
    '''Automate clustering process
       input: input_file:  a 5-column text file with 1 line per clustering run
                           each line lists the 4 filters to be used to construct colours, plus number of clusters
              mp: make output plots?
              oci: output cluster IDs for each object?
       output: output_file: a text file listing input+results from each clustering run'''
    
    run = np.genfromtxt(input_file, dtype='str')

    # TODO: check whether results file already exists; if not, open it and print a header line
    # if it does already exist, just open it
    results = open(output_file, 'a') 
    
    for i in range(0, len(run)):
        
        input_str =  '{} {}'.format(np.array_str(run[i][:-1])[1:-1],int(run[i,4])) # list of input parameters: bands and num of clusters

        score, num_obj =  do_cluster(run[i,0], run[i,1], run[i,2], run[i,3], int(run[i,4]), make_plots=True, output_cluster_id=oci)
        total_obj = num_obj.sum()
        output_str = ' {:.4f} {:5d} {}'.format(score, total_obj, np.array_str(num_obj)[1:-1])
        
        results.write(input_str + ' ' + output_str + '\n')
        
       

    results.close()

    return

def train_policy(policy, optimizer, estimator, continuous, n_iters, t_len):
    trials = []
    grads = []

    for i in range(n_iters):
        print 'Trial %d...' % i
        print 'A:\n%s' % np.array_str(policy.A)
        print 'B:\n%s' % np.array_str(policy.B)
        states, actions, rewards, logprobs = run_trial(policy,
                                                       preprocess,
                                                       max_len=t_len,
                                                       continuous=continuous)
        estimator.report_episode(states, actions, rewards, logprobs)

        trials.append(len(rewards))

        start_theta = policy.get_theta()
        theta, _ = optimizer.optimize(x_init=start_theta,
                                      func=estimator.estimate_reward_and_gradient)
        if np.any(theta != start_theta):
            policy.set_theta(theta)
            estimator.update_buffer()
            estimator.remove_unlikely_trajectories(-3)
            print '%d trajectories remaining' % estimator.num_samples

        if len(trials) > 3 and np.mean(trials[-3:]) >= t_len:
            print 'Convergence achieved'
            break

    return trials, grads

def plotVec( antList=phasedAnts ) :
  color = ['r','g','b','m','c','r','g','b','m','c','c','m','c','m','c']
  pbCorrectedVis = numpy.load("pbCorrectedVis.npy")
  #print pbCorrectedVis
  scalarSum = 0.
  vecList = [0.+0.j]	
  for n in antList :
    for m in range (0,16) :
      ilast = len(vecList) - 1
      if ( numpy.isnan( numpy.real( pbCorrectedVis[m][n-1] ) ) or \
           numpy.isnan( numpy.imag( pbCorrectedVis[m][n-1] ) ) ) :
        vecList = numpy.append( vecList, vecList[ilast] )
      else : 
        vecList = numpy.append( vecList, vecList[ilast] + pbCorrectedVis[m][n] )
        scalarSum += numpy.abs( pbCorrectedVis[m][n] )
  print "\nvecList:"
  print numpy.array_str( vecList, precision=2, max_line_width=200 )
  istart = 0
  i = 0
  while (istart < len(vecList) ) :
    x = numpy.real( vecList[istart:(istart+16)] )
    y = numpy.imag( vecList[istart:(istart+16)] )
    pylab.plot( x, y, color=color[i] )
    i = i+1
    istart = istart+16
  pylab.axis( [-scalarSum,scalarSum,-scalarSum,scalarSum] )
  pylab.grid(True)
  pylab.axes().set_aspect('equal')
  pylab.draw()

def regenerate(request, dbn_id):
    dbn = get_object_or_404(DBNModel , pk=dbn_id)
    data = [map(int, request.POST['data'].split(','))]
    (visible_state, hidden_state) = dbn.regenerate(data)
    visible_state = np.array_str(visible_state)
    hidden_state = np.array_str(hidden_state)
    return render(request, 'rbm/regenerate.html', {'old_data': request.POST['data'], 'dbn': dbn, 'visible_state': visible_state, 'hidden_state': hidden_state})

def evaluate_input( proxy, inval, num_retries=1 ):
    """Query the optimization function.

    Parameters
    ----------
    proxy : rospy.ServiceProxy
        Service proxy to call the GetCritique service for evaluation.

    inval : numeric array
        Input values to evaluate.

    Return
    ------
    reward : numeric
        The reward of the input values
    feedback : list
        List of feedback
    """
    req = GetCritiqueRequest()
    req.input = inval

    for i in range(num_retries+1):
        try:
            res = proxy.call( req )
            break
        except rospy.ServiceException:
            rospy.logerr( 'Could not evaluate item: ' + np.array_str( inval ) )
    
    reward = res.critique
    rospy.loginfo( 'Evaluated input: %s\noutput: %f\nfeedback: %s', 
                   np.array_str( inval, max_line_width=sys.maxint ),
                   reward,
                   str( res.feedback ) )
    return (reward, res.feedback)

def main():
  np.set_printoptions(precision=3)
  Xtrain, ytrain, Xval, yval, Xtest, ytest = data_processing()
  # =========================Q3.1 linear_regression=================================
  w = linear_regression_noreg(Xtrain, ytrain)
  print("======== Question 3.1 Linear Regression ========")
  print("dimensionality of the model parameter is ", len(w), ".", sep="")
  print("model parameter is ", np.array_str(w))
  
  # =========================Q3.2 regularized linear_regression=====================
  lambd = 5.0
  wl = regularized_linear_regression(Xtrain, ytrain, lambd)
  print("\n")
  print("======== Question 3.2 Regularized Linear Regression ========")
  print("dimensionality of the model parameter is ", len(wl), sep="")
  print("lambda = ", lambd, ", model parameter is ", np.array_str(wl), sep="")

  # =========================Q3.3 tuning lambda======================
  lambds = [0, 10 ** -4, 10 ** -3, 10 ** -2, 10 ** -1, 1, 10, 10 ** 2]
  bestlambd = tune_lambda(Xtrain, ytrain, Xval, yval, lambds)
  print("\n")
  print("======== Question 3.3 tuning lambdas ========")
  print("tuning lambda, the best lambda =  ", bestlambd, sep="")

  # =========================Q3.4 report mse on test ======================
  wbest = regularized_linear_regression(Xtrain, ytrain, bestlambd)
  mse = test_error(wbest, Xtest, ytest)
  print("\n")
  print("======== Question 3.4 report MSE ========")
  print("MSE on test is %.3f" % mse)

 def pprint(self, file=sys.stdout):
     """
     Parameters
     ----------
     file : file-like, optional
         An object with `write()` method
     """
     p = partial(print, file=file)
     ## Print summary of the fitting
     p("degree of freedom: {0}".format(self.ndf))
     p("iterations: {0}".format(self.info[2]))
     p("reason for stop: {0}".format(self.info[3]))
     p("")
     p(":parameters:")
     for i, (q, dq) in enumerate(zip(self.p, self.p_stdv)):
         rel = 100 * abs(dq/q)
         p("  p[{0}]: {1:+12.5g} +/- {2:>12.5g}  ({3:4.1f}%)".format(i, q, dq, rel))
     p("")
     p(":covariance:")
     p(np.array_str(self.covr, precision=2, max_line_width=200))
     p("")
     p(":correlation:")
     p(np.array_str(self.corr, precision=2, max_line_width=200))
     p("")
     p(":r2:")
     p("  {0:6g}".format(self.r2))

 def __str__(self, z=False, precision=3):
     s = ''
     if z:
         s += '# q(z):\n{:s}\n'.format(np.array_str(self.z, precision=precision))
         s += '# q(pi):\n{:s}'.format(np.array_str(self.pi, precision=precision))
         for i, nw in enumerate(self.nw):
             s += '\n\n# q(nw[{:d}]):\n{:s}'.format(i, nw.__str__(precision=precision))
     return s

def main():
    """
    DO NOT TOUCH THIS FUNCTION. IT IS USED FOR COMPUTER EVALUATION OF YOUR CODE
    """
    results = my_info() + '\t\t'
    results += np.array_str(np.diagonal(one_vs_all())) + '\t\t'
    results += np.array_str(np.diagonal(all_vs_all()))
    print results + '\t\t'

def main():
    """
    DO NOT TOUCH THIS FUNCTION. IT IS USED FOR COMPUTER EVALUATION OF YOUR CODE
    """
    results = my_info() + "\t\t"
    results += np.array_str(np.diagonal(simple_EC_classifier())) + "\t\t"
    results += np.array_str(np.diagonal(KNN()))
    print results + "\n"

def export_off(mesh):
    faces_stacked = np.column_stack((np.ones(len(mesh.faces))*3, mesh.faces)).astype(np.int64)
    export = 'OFF\n'
    export += str(len(mesh.vertices)) + ' ' + str(len(mesh.faces)) + ' 0\n'
    export += np.array_str(mesh.vertices, precision=9).replace('[', '').replace(']', '').strip()
    export += np.array_str(faces_stacked).replace('[', '').replace(']', '').strip()

    return export

 def writeLogNP(self, beliefs, cards, reward): #[[cards],[beliefs]]
     array = [np.array_str(cards),np.array_str(beliefs), reward]
     logname = "log-" + str(self.logNr) +".csv"
     ##print(logname)
     with open(self.logPath + logname, "a", newline='') as csv_file:
         writer = csv.writer(csv_file, delimiter=',')
         writer.writerow(array)
     return logname

  def ecologyMetric(self):
    '''Measuring the ecology potential in the population.'''

    unique_gen = set()
    for org in self.orgs:
      if numpy.array_str(org.genome) not in unique_gen:
        unique_gen.add(numpy.array_str(org.genome))

    return len(unique_gen)