def evaluation(y_test=None, y_predict=None, n_classes=None):
    """
    Input the predicted results, targets results and
    the number of class, return the confusion matrix, F1-score of each class,
    accuracy and macro F1-score.

    Parameters
    ----------
    y_test : list
        The target results
    y_predict : list
        The predicted results
    n_classes : int
        The number of classes

    Examples
    --------
    >>> c_mat, f1, acc, f1_macro = tl.utils.evaluation(y_test, y_predict, n_classes)

    """
    c_mat = confusion_matrix(y_test, y_predict, labels=[x for x in range(n_classes)])
    f1 = f1_score(y_test, y_predict, average=None, labels=[x for x in range(n_classes)])
    f1_macro = f1_score(y_test, y_predict, average='macro')
    acc = accuracy_score(y_test, y_predict)
    tl.logging.info('confusion matrix: \n%s' % c_mat)
    tl.logging.info('f1-score        : %s' % f1)
    tl.logging.info('f1-score(macro) : %f' % f1_macro)  # same output with > f1_score(y_true, y_pred, average='macro')
    tl.logging.info('accuracy-score  : %f' % acc)
    return c_mat, f1, acc, f1_macro

def compute_ref(true_tags, out_file, data_type='svm_light'):
    tag_map = {'OK': 1, 'BAD': 0, u'OK': 1, u'BAD': 0}
    predicted = []
    if data_type == 'svm_light':
        tag_map_pred = {'+1': 1, '-1': 0}
        for line in open(out_file):
            label = line[line.find(':')+1:line.find(' ')]
            predicted.append(tag_map_pred[label])
    elif data_type == 'crfpp' or data_type == 'crf_suite':
        for line in open(out_file):
            line = line.strip('\n')
            if line == '':
                continue
            tag = line.split('\t')[-1]
            if tag == 'OK' or tag == 'BAD':
                predicted.append(tag)
        predicted = [tag_map[t] for t in predicted]
#    if (type(true_tags[0]) is str or type(true_tags[0]) is unicode) and not true_tags[0].isdigit():
    true_tags = [tag_map[t] for t in true_tags]
#    if type(predicted[0]) is str and not predicted[0].isdigit():
    print(true_tags[:10])
    print(predicted[:10])

    print(f1_score(predicted, true_tags, average=None))
    print(f1_score(predicted, true_tags, average='weighted', pos_label=None))

    def compare_2_models(model1, model2, X, y, h):
        h = min(X.shape[0], h)
        hidden_layer = features[np.random.choice(X.shape[0],
                                                 h,
                                                 replace=False)]
        print('training 1st model')
        pr = cProfile.Profile()
        pr.enable()
        model1.fit(X, y, hidden_layer=hidden_layer)
        y1 = model1.predict(X)
        pr.disable()
        ps = pstats.Stats(pr).sort_stats('cumulative')
        ps.print_stats()

        print('training 2nd model')
        pr = cProfile.Profile()
        pr.enable()
        model2.fit(X, y, hidden_layer=hidden_layer)
        y2 = model2.predict(X)
        pr.disable()
        ps = pstats.Stats(pr).sort_stats('cumulative')
        ps.print_stats()

        print(f1_score(y, y2))
        print(f1_score(y, y1))

        return np.allclose(y1, y2)

    def on_epoch_end(self, epoch, logs={}):
        print logs

        corr=0
        tot=0
        preds = self.model.predict(self.dev_data, verbose=1)
        preds_text=[]
        for l in preds:
            preds_text.append(self.index2label[np.argmax(l)])

        print "Micro f-score:", f1_score(self.dev_labels_text,preds_text,average=u"micro")
        print "Macro f-score:", f1_score(self.dev_labels_text,preds_text,average=u"macro")
        print "Macro recall:", recall_score(self.dev_labels_text,preds_text,average=u"macro")

        if self.best_mr < recall_score(self.dev_labels_text,preds_text,average=u"macro"):
            self.best_mr = recall_score(self.dev_labels_text,preds_text,average=u"macro")
            model.save_weights(self.model_name + '_full_' + str(epoch) + '_MR_' + str(self.best_mr) + '.hdf5')
            print 'Saved Weights!'


        print classification_report(self.dev_labels_text, preds_text)
        for i in xrange(len(self.dev_labels)):

        #    next_index = sample(preds[i])
            next_index = np.argmax(preds[i])
            # print preds[i],next_index,index2label[next_index]

            l = self.index2label[next_index]

            # print "correct:", index2label[np.argmax(dev_labels[i])], "predicted:",l
            if self.index2label[np.argmax(self.dev_labels[i])]==l:
                corr+=1
            tot+=1
        print corr,"/",tot

def kernel_graph_experiment(model, data_fn, data_name, model_name):
    print "Running graph experiment (%s)..." % (data_name,)

    A, X, Y = data_fn()

    n_nodes = len(A)

    indices = np.arange(n_nodes)
    np.random.shuffle(indices)

    print indices

    train_indices = indices[: n_nodes // 3]
    valid_indices = indices[n_nodes // 3 : (2 * n_nodes) // 3]
    test_indices = indices[(2 * n_nodes) // 3 :]
    # train_indices = indices[:int(n_nodes*0.8)]
    # valid_indices = indices[int(n_nodes*0.8):int(n_nodes*0.9)]
    # test_indices = indices[int(n_nodes*0.9):]

    model.fit_with_validation(Y, train_indices, valid_indices, test_indices)

    preds = model.predict(Y, np.asarray([]), test_indices)
    actuals = Y[test_indices, :]

    accuracy = accuracy_score(actuals, preds)
    f1_micro = f1_score(actuals, preds, average="micro")
    f1_macro = f1_score(actuals, preds, average="macro")

    print "form: name,micro_f,macro_f,accuracy"
    print "###RESULTS###: %s,%s,%.8f,%.8f,%.8f" % (data_name, model_name, f1_micro, f1_macro, accuracy)

def getScores(y, yPredTrain, yTest, yPredTest):

    scores = dict()

    scores['f1Train'] = f1_score(y, yPredTrain)
    scores['f1Test'] = f1_score(yTest, yPredTest)


    scores['accTrain'] = accuracy_score(y, yPredTrain)
    scores['accTest'] = accuracy_score(yTest, yPredTest)
    

    scores['rocTrain'] = roc_auc_score(y, yPredTrain)
    scores['rocTest'] = roc_auc_score(yTest, yPredTest)
    

    scores['cMatrixTrain'] = confusion_matrix(y, yPredTrain)
    scores['cMatrixTest'] = confusion_matrix(yTest, yPredTest)

    proba = float(len(np.where(y==1)[0]))/len(y)
    if proba < 0.50:
        proba = 1 - proba
    scores['random'] = proba
    
    return scores

 def on_epoch_end(self, batch, logs={}):
     # losses
     self.losses_train.append(self.model.evaluate(X_train, Y_train, batch_size=128,verbose =0))
     self.losses_val.append(self.model.evaluate(X_val, Y_val, batch_size=128,verbose = 0))
     
     # Roc train
     train_preds = self.model.predict_proba(X_train, verbose=0)
     train_preds = train_preds[:, 1]
     roc_train = metrics.roc_auc_score(y_train, train_preds)
     self.roc_train.append(roc_train)
     
     # Roc val
     val_preds = self.model.predict_proba(X_val, verbose=0)
     val_preds = val_preds[:, 1]
     roc_val = metrics.roc_auc_score(y_val, val_preds)
     self.roc_val.append(roc_val)
     
     # Metrics train
     y_preds = self.model.predict_classes(X_train,verbose = 0)
     self.f1_train.append(metrics.f1_score(y_train,y_preds))
     self.recal_train.append(metrics.recall_score(y_train,y_preds))
     self.preci_train.append(metrics.precision_score(y_train,y_preds))
     
     # Metrics val
     y_preds = self.model.predict_classes(X_val,verbose =0)
     self.f1_val.append(metrics.f1_score(y_val,y_preds))
     self.recal_val.append(metrics.recall_score(y_val,y_preds))
     self.preci_val.append(metrics.precision_score(y_val,y_preds))

    def on_epoch_end(self, epoch, logs={}):
        print logs

        corr=0
        tot=0
        preds = self.model.predict(self.dev_data, verbose=1)

        preds_text=[]
        for l in preds:
            preds_text.append(self.index2label[np.argmax(l)])

        print "Micro f-score:", f1_score(self.dev_labels_text,preds_text,average=u"micro")
        print "Macro f-score:", f1_score(self.dev_labels_text,preds_text,average=u"macro")
        print classification_report(self.dev_labels_text, preds_text)

        for i in xrange(len(self.dev_labels)):

        #    next_index = sample(preds[i])
            next_index = np.argmax(preds[i])
            # print preds[i],next_index,index2label[next_index]

            l = self.index2label[next_index]

            # print "correct:", index2label[np.argmax(dev_labels[i])], "predicted:",l
            if self.index2label[np.argmax(self.dev_labels[i])]==l:
                corr+=1
            tot+=1
        print corr,"/",tot

def cv_model():
    DATA_FILE  = './data/train-set-ru-b64-utf-8.txt'
    all_data = []
    target = []
    with open(DATA_FILE) as df:
        for i, line in enumerate(df):
            print i
            line = line.strip()
            parts = line.split()
            stats_collector = StatsCollector()
            #print parts[2]
            #print base64.b64decode(parts[3])#.decode('utf-8')
            #print parts[2].decode('utf-8'), parts[3].decode('utf-8'), "\n"
            stats_collector.collect(int(parts[1]), parts[3], parts[2])
            # mark page url
            all_data.append(stats_collector.get_features())
            target.append(stats_collector.get_target())
            #print all_data[-1]

    data = np.asarray(all_data, dtype = np.float)
    target = np.asarray(target, dtype = np.float)

    clf = GradientBoostingClassifier(loss='deviance', learning_rate=0.05, n_estimators=400,\
     min_samples_split=30, min_samples_leaf=15, max_depth=5)

    kf = KFold(data.shape[0], n_folds = 3, shuffle = True)

    for train_index, test_index in kf:
        X_train, X_test = data[train_index], data[test_index]
        y_train, y_test = target[train_index], target[test_index]
        clf.fit(X_train, y_train)
        y_pred = clf.predict(X_test)
        print f1_score(y_test, y_pred)

def cross_val(data_x, data_y, classifier, kFold, b_cost=1, h_cost=1, w=0.5):
    e_h, e_b = 0, 0
    y_tests, pred_probas = [], []
    
    for train_index, test_index in kFold:
        data_x_, data_y_ = np.array(data_x), np.array(data_y)
        X_train, X_test = list(data_x_[train_index]), list(data_x_[test_index])
        y_train, y_test = list(data_y_[train_index]), list(data_y_[test_index])
        classifier.fit(X_train, y_train)
        pred_proba = [r[0] for r in classifier.predict_proba(X_test)]
        y_tests += y_test
        pred_probas += pred_proba
    
    predictions = [0 if p*b_cost > (1-p)*h_cost else 1 for p in pred_probas]
    roc_auc = roc_auc_score(y_tests, pred_probas)
    total_acc = accuracy_score(y_tests, predictions)
    precision, recall, thresholds = precision_recall_curve(y_tests, pred_probas, pos_label=0)
    fpr, tpr, thresholds = roc_curve(y_tests, pred_probas, pos_label=0)
    precision_bots = precision_score(y_tests, predictions, pos_label = 0)
    precision_humans = precision_score(y_tests, predictions, pos_label = 1)
    recall_bots = recall_score(y_tests, predictions, pos_label = 0)
    recall_humans = recall_score(y_tests, predictions, pos_label = 1)
    f1_bots = f1_score(y_tests, predictions, pos_label = 0)
    f1_humans = f1_score(y_tests, predictions, pos_label = 1)
    conf_matrix = np.matrix(list(confusion_matrix(y_tests, predictions)))
    
    #plot_curve(fpr, tpr, 'ROC', w)
    #plot_curve(recall, precision, 'PR', w)
    
    return [total_acc, precision_bots, precision_humans, recall_bots, recall_humans, f1_bots, f1_humans, roc_auc, conf_matrix]

def fit_model():
    DATA_FILE  = './data/train-set-ru-b64-utf-8.txt'
    stats_collector = StatsCollector()
    i=0
    data = []
    target = []

    with open (DATA_FILE) as df:
         for i, line in enumerate(df):
            print i
            line = line.strip()
            parts = line.split()
            stats_collector = StatsCollector()
            stats_collector.collect(int(parts[1]), parts[3], parts[2])
            data.append(stats_collector.get_features())
            target.append(stats_collector.get_target())
            #print len(data[-1])


    data = np.asarray(data, dtype = np.float)
    target = np.asarray(target, dtype = np.float)
    print data.shape, target.shape
    df.close()
    clf = GradientBoostingClassifier(loss='deviance', learning_rate=0.07, n_estimators=300, min_samples_split=30,\
         min_samples_leaf=15, max_depth=4)

    clf.fit(data, target)
    y_pred = clf.predict(data)
    print f1_score(target, y_pred)

    joblib.dump(clf, 'model/model.pkl') 

    def logistic_regression_sklearn(self, features, labels):
        """Run a logistic regression, evaluate it, return the LR object
        """

        print '\n**** Running logistic regression...'

        # Split into train / test segments
        features_train, features_test, target_train, target_test = cross_validation.train_test_split(features, labels, test_size=0.20, random_state=0)

        lr = LogisticRegression()
        lr.fit(features_train, target_train)

        # Evaluate the regression
        target_predicted = lr.predict(features_test)
        accuracy = accuracy_score(target_test, target_predicted)

        print 'Logistic regression accuracy score: {0:.0f}%'.format(100 * accuracy)

        # coefs = pd.DataFrame(zip(feature_cols, np.transpose(lr.coef_[0])), columns=['Feature', 'Coefficient'])

        print 'F1: ',
        print f1_score(target_test, target_predicted)

        # preds = lr.predict_proba(features_test)[:,1]
        # fpr, tpr, _ = roc_curve(target_test, preds)

        # print 'AOC: ',
        # print '{:.2f}'.format(auc(fpr,tpr))

        return lr

def evaluate_fold(clf, X_train, y_train, X_test, y_test):
    """
    This is the business section
    """
    tmp = dict()
    tmp['X_train.shape'] = X_train.shape
    tmp['X_test.shape'] = X_test.shape
    try:
        pred_test = clf.predict_proba(X_test)
        pred_train = clf.predict_proba(X_train)
        tmp['roc'] = roc_info(y_test, pred_test[:,1])   
        tmp['roc_area'] = roc_auc_score(y_test, pred_test[:,1])
        pred_test = clf.predict(X_test)
        pred_train = clf.predict(X_train)
        tmp['f1_test'] = f1_score(y_test, pred_test, pos_label=1)        
        tmp['f1_train'] = f1_score(y_train, pred_train, pos_label=1) 

    except (AttributeError, NotImplementedError):
        pred_test = clf.predict(X_test)
        pred_train = clf.predict(X_train)
        tmp['roc'] = roc_info(y_test, pred_test)
        tmp['roc_area'] = roc_auc_score(y_test, pred_test)
        tmp['f1_test'] = f1_score(y_test, pred_test, pos_label=1)        
        tmp['f1_train'] = f1_score(y_train, pred_train, pos_label=1) 

    return tmp

def benchmark(clf_current):
    print('_' * 80)
    print("Test performance for: ")
    clf_descr = str(clf_current).split('(')[0]
    print(clf_descr)
    t0 = time()
    classif = OneVsRestClassifier(clf_current)
    classif.fit(X_train, Y_train.toarray())
    train_time = time() - t0
    print("train time: %0.3fs" % train_time)
    t0 = time()
    if hasattr(clf_current,"decision_function"):
        dfmatrix = classif.decision_function(X_test)
        score = metrics.f1_score(Y_test.toarray(), df_to_preds(dfmatrix, k = 5))
    else:
        probsmatrix = classif.predict_proba(X_test)
        score = metrics.f1_score(Y_test.toarray(), probs_to_preds(probsmatrix, k = 5))
        
    test_time = time() - t0

    
    print("f1-score:   %0.7f" % score)
    print("test time:  %0.3fs" % test_time)

    print('_' * 80)
    return clf_descr, score, train_time, test_time

 def cutoff_f1(clf, X, y):
     y_pred = (clf.predict_proba(X)[:,1] > cutoff_value).astype(int)
     y_pred2 = clf.predict(X)
     s1 = f1_score(y, y_pred)
     s2 = f1_score(y, y_pred2)
     # print 'f1 = %.4f, %.4f' % (s1, s2)
     return s1

def main():
	f = open("me.stdout", "r").read()

	print f
	
	(confusionMatrix, labels, ytrue, ypred, trueCount) = readConfusionMatrix.readText(f)
	for row in confusionMatrix:
		print row

	precisionMicro = np.float(metrics.precision_score(ytrue, ypred, average="micro"))
	recallMicro = np.float(metrics.recall_score(ytrue, ypred, average="micro"))
	f1Micro = np.float(metrics.f1_score(ytrue, ypred, average="micro"))
	f1Macro = np.float(metrics.f1_score(ytrue, ypred, pos_label=1, average="macro"))
	precisionMacro = np.float(metrics.precision_score(ytrue, ypred, average="macro"))
	recallMacro = np.float(metrics.recall_score(ytrue, ypred, average="macro"))

	mConf = metrics.confusion_matrix(ytrue, ypred)
	print mConf

	print labels
	print len(ytrue)
	print len(ypred)
	print trueCount

	print metrics.accuracy_score(ytrue, ypred)

	print precisionMicro
	print recallMicro
	print f1Micro
	print f1Macro
	print precisionMacro
	print recallMacro

def scnn_proportion_experiment(data_fn, name, n_hops, prop_valid, prop_test, transform_fn=util.rw_laplacian, transform_name='rwl'):
    print 'Running node experiment (%s)...' % (name,)

    A, X, Y = data_fn()

    n_nodes = A.shape[0]

    indices = np.arange(n_nodes)

    valid_start = int(n_nodes * (1 - (prop_valid + prop_test)))
    test_start = int(n_nodes * (1 - prop_test))

    valid_indices = indices[valid_start:test_start]
    test_indices  = indices[test_start:]

    for train_prop in [x / 10.0 for x in range(1, 11)]:
        train_end = int(valid_start * train_prop)
        train_indices = indices[:train_end]

        scnn = SCNN(n_hops=n_hops, transform_fn=transform_fn)
        scnn.fit(A, X, Y, train_indices=train_indices, valid_indices=valid_indices)

        probs = scnn.predict_proba(X, test_indices)
        print probs

        preds = scnn.predict(X, test_indices)
        actuals = np.argmax(Y[test_indices,:], axis=1)

        f1_micro = f1_score(actuals, preds, average='micro')
        f1_macro = f1_score(actuals, preds, average='macro')
        accuracy = accuracy_score(actuals, preds)

        print 'form: name,n_hops,transform_name,micro_f,macro_f,accuracy'
        print '###RESULTS###: %s,%d,%.2f,%s,%.8f,%.8f,%.8f' % (name, n_hops, train_prop, transform_name, f1_micro, f1_macro, accuracy)

def baseline_graph_experiment(model, data_fn, data_name, model_name):
    print "Running graph experiment (%s)..." % (data_name,)

    A, X, Y = data_fn()

    A = np.asarray(A)
    X = np.asarray(X)
    Y = np.asarray(Y)

    n_nodes = A.shape[0]

    indices = np.arange(n_nodes)
    np.random.shuffle(indices)

    train_indices = indices[: n_nodes // 3]
    valid_indices = indices[n_nodes // 3 : (2 * n_nodes) // 3]
    test_indices = indices[(2 * n_nodes) // 3 :]

    model.fit_with_validation(A, X, Y, train_indices, valid_indices)

    preds = model.predict(A, X, test_indices)
    actuals = Y[test_indices, :]

    accuracy = accuracy_score(actuals, preds)
    f1_micro = f1_score(actuals, preds, average="micro")
    f1_macro = f1_score(actuals, preds, average="macro")

    print "form: name,micro_f,macro_f,accuracy"
    print "###RESULTS###: %s,%s,%.8f,%.8f,%.8f" % (data_name, model_name, f1_micro, f1_macro, accuracy)

def compareModels(model):
    """
    This evaluates the pre-trained model agaisnt
    metamind's API on sentences in `data/validation`

    Parameters
    ----------
    model: test.MODEL
        Namedtuple containing model parameters (dictionary, tfidf
        learner and labels)

    """
    
    set_api_key("MohJ53r6kUvoPjHS8tStX1vnfssvN5EDetVcp2uCNISwXus2BS")

    with open('data/validation', 'r') as fin:
        validations = fin.read()
        truth = [model.labels.label2class[i] for i in
                 ['positive']*9 + ['negative']*8]          

    scores_mm = []
    scores_joe = []
    for validation in validations.split('\n'):
        mmLabel = testMetaMind(validation)[0]['label']
        scores_mm.append(model.labels.label2class[mmLabel])
        joeLabel = testDeepModel(validation, model)
        scores_joe.append(model.labels.label2class[joeLabel])
        
    print 'MetaMind F1 score is %s' % f1_score(truth, scores_mm)
    print 'My F1 score is %s' % f1_score(truth, scores_joe)

def predict_evaluate_models(fn ,ax=None, sel=["Penalties_Conceeded","Tries_Scored"], goal="Referee", verbosity=0):
    class_weight = 'auto'
    X, y, names = data_prepare(fn, sel=sel, goal=goal, verbosity=verbosity-1)
    if verbosity > 2:
        y_shuffled = y.copy()
        np.random.shuffle(y_shuffled)
        print ("All zeros accuracy:",1.0-np.sum(y)/len(y)) 
        print ("y_shuffled f1_csore:",metrics.f1_score(y, y_shuffled))

    n_folds = 10
    cv = cross_validation.StratifiedKFold(y, n_folds=n_folds)
    #cv = cross_validation.LeaveOneOut(n=len(y))
    results = []
    for sclf in ('svm','svmp','svmr','lgCV','gnb','rf','knc'):
        clf = get_clf(sclf,class_weight=class_weight)
        y_pred = cross_validation.cross_val_predict(clf, X, y, cv=cv)
        #print "pred:",y_pred
        res = [
            metrics.accuracy_score(y, y_pred),
            metrics.precision_score(y, y_pred),
            metrics.recall_score(y, y_pred),
            metrics.f1_score(y, y_pred),
            ]
        if verbosity > 0:
            print (sclf,res) 
        results.append( (sclf,res) )

    return results