def gnb_estimators_growing(clf, x_test, y_test, x_train, y_train):
    penalty = [2**i for i in range(-5, 15, 3)]
    gamma = [2**i for i in range(-15, 3, 2)]
    
    err_train = []
    err_test = []
    
   
    clf.C = 1.0
    for n_gamma in gamma:
        print('For n_gamma:', n_gamma)

        clf.gamma = n_gamma
        for n_penalty in penalty:
            print('For n_penalty:', n_penalty)

            clf.C = n_penalty
            clf.fit(x_train, y_train)
            err_train.append(hamming_loss(y_train, clf.predict(x_train)))
            err_test.append(hamming_loss(y_test, clf.predict(x_test)))
        
        plt.plot(penalty, err_train, color="blue",label="train")
        plt.plot(penalty, err_test, color="red",label="test")
        plt.xlabel("Penalty")
        plt.ylabel("Error")
        plt.legend(loc="upper right",fancybox=True);      
        plt.show()
    
        err_train = []
        err_test = []
    
    print('Growing algorithm ended')

def calculate_result(actual,pred):  
    m_precision = metrics.precision_score(actual,pred)  
    m_recall = metrics.recall_score(actual,pred)
    print 'Hamming_loss:{0:.3f}'.format(hamming_loss(actual, pred, classes=None))
    print 'Precision:{0:.3f}'.format(m_precision)
    print 'Recall:{0:0.3f}'.format(m_recall)  
    print 'F1-score:{0:.3f}'.format(metrics.f1_score(actual,pred,average='micro'))

def compare_manual_vs_model():

    with open(DATA_FOLDER + "labels_int.p", "r") as f:
        y_dict = pickle.load(f)

    print "Loading test data"
    X_test, y_test, filenames_test = dataset.load_test()
    y_pred = joblib.load("../models/pred_ml_improved.pkl")

    relevant = []
    for pred, correct, filename in zip(y_pred, y_test, filenames_test):
        if filename in FILES:
            relevant.append((pred, correct, filename, CLASSIFICATIONS[filename]))

    model_predictions, correct, filename, manual_predictions = zip(*relevant)
    manual_predictions = learn.multilabel_binary_y(manual_predictions)
    model_predictions = np.array(model_predictions)
    correct = learn.multilabel_binary_y(correct)

    rules = infer_topology.infer_topology_rules()
    improved_manual = infer_topology.apply_topology_rules(rules, manual_predictions)

    prediction_names = ["MODEL", "MANUAL", "IMPROVED_MANUAL"]
    predictions = [model_predictions, manual_predictions, improved_manual]

    for name, pred in zip(prediction_names, predictions):

        print "\n{}\n--".format(name)
        print "Zero-one classification loss", zero_one_loss(correct, pred)
        print "Hamming loss", hamming_loss(correct, pred)
        print "Precision:", precision_score(correct, pred, average="weighted", labels=label_list)
        print "Recall   :", recall_score(correct, pred, average="weighted", labels=label_list)
        print "F1 score :", f1_score(correct, pred, average="weighted", labels=label_list)

    def train_and_eval(x_train, y_train, x_test, y_test, model, param_result):
        print("\nTraining and evaluating...")

        for result_list in param_result:
            print("Fitting: " + str(result_list[2]))

            opt_model = result_list[2]
            opt_model.fit(x_train, y_train)
            y_pred = opt_model.predict(x_test)

            print("\nClassification Report:")
            print(metrics.classification_report(y_test, y_pred))
            print("\nAccuracy Score:")
            print(metrics.accuracy_score(y_test, y_pred))
            print("\nConfusion Matrix:")
            print(metrics.confusion_matrix(y_test, y_pred))
            print("\nF1-Score:")
            print(metrics.f1_score(y_test, y_pred))
            print("\nHamming Loss:")
            print(metrics.hamming_loss(y_test, y_pred))
            print("\nJaccard Similarity:")
            print(metrics.jaccard_similarity_score(y_test, y_pred))
            # vvv Not supported due to ValueError: y_true and y_pred have different number of classes 3, 2
            # print('\nLog Loss:')
            # print(metrics.log_loss(y_test, y_pred))
            # vvv multiclass not supported
            # print('\nMatthews Correlation Coefficient:')
            # print(metrics.matthews_corrcoef(y_test, y_pred))
            print("\nPrecision:")
            print(metrics.precision_score(y_test, y_pred))
            # vvv Not supported due to ValueError: y_true and y_pred have different number of classes 3, 2
            # print('\nRecall:')
            # print(metrics.recall(y_test, y_pred))
            print()

 def report_dataset(X, y_true, title):
     y_proba = model.predict_proba(X, batch_size=batch_size)
     # multi-label classes with default threshold
     y_pred = y_proba >= 0.5
     print(title + ' accuracy (exatch match):', accuracy_score(y_true, y_pred))
     print(title + ' hamming score (non-exatch match):', 1 - hamming_loss(y_true, y_pred))
     print(title + 'AUC:', roc_auc_score(y_true.flatten(), y_proba.flatten()))

 def svmDesc(lab_pred,lab_test, title='Confusion matrix', cmap=plot.cm.Blues,taskLabels=taskLabels,normal=True):
     #build confussion matrix itself
     conM = confusion_matrix(lab_test, lab_pred)
     if normal== True:
         conM = conM.astype('float') / conM.sum(axis=1)[:, np.newaxis]
     #build heatmap graph of matrix
     plot.imshow(conM, interpolation='nearest', cmap=cmap)
     plot.title(title)
     plot.colorbar()
     tick_marks = np.arange(len(taskLabels))
     plot.xticks(tick_marks, taskLabels, rotation=45)
     plot.yticks(tick_marks, taskLabels)
     plot.tight_layout()
     plot.ylabel('True label')
     plot.xlabel('Predicted label')
     
     #classification report
     creport = classification_report(lab_test,lab_pred)
     print "CLASSIFICATION REPORT: "  
     print creport
     
     #hamming distance
     hamming = hamming_loss(lab_test,lab_pred)
     print "HAMMING DISTANCE:              %s" % str(hamming)
     
     #jaccard similarity score
     jaccard = jaccard_similarity_score(lab_test,lab_pred)
     print "JACCARD SIMILARITY SCORE:      %s" % str(jaccard)
     
     #precision score    
     pscore = precision_score(lab_test,lab_pred)
     print "PRECISION SCORE:               %s" % str(pscore)

def err(k):
    #tmp=Ysub[k,:].dot(proj1000T)
    tmp=Ysub[k,:].dot(proj)
    pred=(tmp>0.5).astype(int)
    #return absolute num incorrect labels per sample (hamming loss is normalized by #cols)
    #return metrics.hamming_loss(Ytrunc[k,:].todense(),pred)*42048
    return metrics.hamming_loss(y_testBin[k,:],pred)*42048

    def hamming_loss(self, classifier_name, context, information, pattern_kind):
        from sklearn.metrics import hamming_loss

        self.measures[classifier_name]["hamming_loss"] = \
            hamming_loss(
                context["patterns"].patterns[classifier_name][pattern_kind],
                information.info[classifier_name]["discretized_outputs"][pattern_kind])

    def test_calc_hamming_loss(self):
        labels_true = [s['label'] for s in self.samples]
        data = [s['info'] for s in self.samples]

        labels_pred = self.model.predict(data)
        loss = hamming_loss(labels_true, labels_pred)

        self.assertLess(loss, self.BASELINE_LOSS)

    def randomClassifyClasses(self, ticketsToClasses):
        #conditionally classify something correctly as a class.
        #we need labeled data with the classes changed in that commit
        tickets = [(ticket, classes) for ticket, classes in ticketsToClasses.items()]
        random.shuffle(tickets)
        trainIndex = int(len(tickets) * .8)
        trainTickets = tickets[:trainIndex] 
        testTickets = tickets[trainIndex:]
        print(len(tickets))
        print(trainIndex)
        trainText = np.array([ticket[0].summary for ticket in trainTickets])
        trainLabels = np.array([ticket[1] for ticket in trainTickets])
        testText =  np.array([ticket[0].summary for ticket in testTickets])
        testLabels = np.array([ticket[1] for ticket in testTickets])
        ticketLabels = [ticket[1] for ticket in tickets]

        target_names = list(set([label for labelList in ticketLabels for label in labelList]))

        print ("Total of %d labels, so %.5f *x accuracy is baseline" % (len(target_names), (1.0 / (len(target_names) * 1.0))))
        lb = preprocessing.LabelBinarizer()
        Y = lb.fit_transform(trainLabels)
        #dv = DictVectorizer()
        classifier = Pipeline([
        ('hash', HashingVectorizer()),
        ('tfidf', TfidfTransformer()),
        ('clf', OneVsRestClassifier(LinearSVC()))])

        classifier.fit(trainText, Y)
        #predicted = classifier.predict(testText)
        predictedLabels = []

        numLabels = len(lb.classes_)
        for i in range(0, len(testTickets)):
            labelList = [lb.classes_[random.randrange(0, numLabels - 1)] for j in range(0, random.randrange(0, numLabels))]
            predictedLabels.append(labelList)
        predictedLabels = np.array(predictedLabels)

        fpredictedLabels = [pred for pred in predictedLabels if len(pred) != 0]
        ftestLabels = [testLabels[i] for i in range(0, len(testLabels)) if len(predictedLabels[i]) != 0]
        ftestText = [testText[i] for i in range(0, len(testLabels)) if len(predictedLabels[i]) != 0]
        
        print("original: %d filtered %d" % (len(predictedLabels), len(fpredictedLabels)))
        for i in range(0, len(predictedLabels)):
                if len(predictedLabels[i]) == 0:
                    print(i)
        for item, plabels, alabels in zip(ftestText, fpredictedLabels, ftestLabels):
            print ('TICKET: \n%s PREDICTED => \n\t\t%s' % (item, ', '.join(plabels)))
            print ('\n\t\ttACTUAL => \n\t\t%s' % ', '.join(alabels))
        #classification_report(testLabels, predictedLabels)
        f1Score = f1_score(ftestLabels, fpredictedLabels)
        precision = precision_score(ftestLabels, fpredictedLabels)
        accuracy = accuracy_score(ftestLabels, fpredictedLabels)
        recall = recall_score(ftestLabels, fpredictedLabels)
        hamming = hamming_loss(ftestLabels, fpredictedLabels)
        self.classifier = classifier
        
        return (precision, recall, accuracy, f1Score, hamming)

def label_based_measures(y_true, y_pred):
    """
    Evaluation measures used to assess the predictive performance in multi-label
    label-based learning: hamming_loss, precision, recall and f1
    """
    m = {}
    m['hamming_accuracy'] = 1 - hamming_loss(y_true, y_pred)
    m['precision'], m['recall'], m['f1'], _ = precision_recall_fscore_support(y_true, y_pred)
    return m

def rf_estimators_growing(clf, x_test, y_test, x_train, y_train):
    estimators = [i for i in range(145, 150, 1)]#[1, 2, 3, 4, 5, 10, 20, 30, 40, 50]# + \
                 #[i for i in range(100, 1000, 100)]# + \
                # [i for i in range(1000, 6000, 1000)]

    err_train = []
    err_test = []
    clf.max_features = 5
    for n_estimators in estimators:
        print('For n_estimators:', n_estimators)

        clf.n_estimators = n_estimators
        clf.fit(x_train, y_train)
        err_train.append(hamming_loss(y_train, clf.predict(x_train)))
        err_test.append(hamming_loss(y_test, clf.predict(x_test)))

    plt.plot(estimators, err_test, 'r-')
    plt.show()
    print('Growing algorithm ended')

def test_losses():
    """Test loss functions"""
    y_true, y_pred, _ = make_prediction(binary=True)
    n_samples = y_true.shape[0]
    n_classes = np.size(unique_labels(y_true))

    # Classification
    # --------------
    with warnings.catch_warnings(True):
    # Throw deprecated warning
        assert_equal(zero_one(y_true, y_pred), 13)
        assert_almost_equal(zero_one(y_true, y_pred, normalize=True),
                            13 / float(n_samples), 2)

    assert_almost_equal(zero_one_loss(y_true, y_pred),
                        13 / float(n_samples), 2)
    assert_equal(zero_one_loss(y_true, y_pred, normalize=False), 13)
    assert_almost_equal(zero_one_loss(y_true, y_true), 0.0, 2)
    assert_almost_equal(zero_one_loss(y_true, y_true, normalize=False), 0, 2)

    assert_almost_equal(hamming_loss(y_true, y_pred),
                        2 * 13. / (n_samples * n_classes), 2)

    assert_equal(accuracy_score(y_true, y_pred),
                 1 - zero_one_loss(y_true, y_pred))

    assert_equal(accuracy_score(y_true, y_pred, normalize=False),
                 n_samples - zero_one_loss(y_true, y_pred, normalize=False))

    with warnings.catch_warnings(True):
    # Throw deprecated warning
        assert_equal(zero_one_score(y_true, y_pred),
                     1 - zero_one_loss(y_true, y_pred))

    # Regression
    # ----------
    assert_almost_equal(mean_squared_error(y_true, y_pred),
                        12.999 / n_samples, 2)
    assert_almost_equal(mean_squared_error(y_true, y_true),
                        0.00, 2)

    # mean_absolute_error and mean_squared_error are equal because
    # it is a binary problem.
    assert_almost_equal(mean_absolute_error(y_true, y_pred),
                        12.999 / n_samples, 2)
    assert_almost_equal(mean_absolute_error(y_true, y_true), 0.00, 2)

    assert_almost_equal(explained_variance_score(y_true, y_pred), -0.04, 2)
    assert_almost_equal(explained_variance_score(y_true, y_true), 1.00, 2)
    assert_equal(explained_variance_score([0, 0, 0], [0, 1, 1]), 0.0)

    assert_almost_equal(r2_score(y_true, y_pred), -0.04, 2)
    assert_almost_equal(r2_score(y_true, y_true), 1.00, 2)
    assert_equal(r2_score([0, 0, 0], [0, 0, 0]), 1.0)
    assert_equal(r2_score([0, 0, 0], [0, 1, 1]), 0.0)

def print_results(Y_test, Y_pred, classes_, f):
    print("Hamming score (acc)\t", 1 - hamming_loss(Y_test, Y_pred), file=f)
    print("F1 (micro-averaged)\t", f1_score(Y_test, Y_pred, average='micro'), file=f)
    print("F1 (macro-averaged)\t", f1_score(Y_test, Y_pred, average='macro'), file=f)
    print("\nLabel\tAccuracy\tPrecision\tRecall\tF1", file=f)
    for i, label in enumerate(classes_):
        print(label + "\t" +
            "%.4f" % accuracy_score(Y_test[:, i], Y_pred[:, i]) + "\t" +
            "%.4f" % precision_score(Y_test[:, i], Y_pred[:, i]) + "\t" +
            "%.4f" % recall_score(Y_test[:, i], Y_pred[:, i]) + "\t" +
            "%.4f" % f1_score(Y_test[:, i], Y_pred[:, i]), file=f)

def run(y_true, y_pred):
    perf = {}
    
    perf['accuracy'] = accuracy_score(y_true, y_pred)
    perf['precision'] = precision_score(y_true, y_pred, average='micro')
    perf['recall'] = recall_score(y_true, y_pred, average='micro')
    perf['fbeta_score'] = fbeta_score(y_true, y_pred, average='macro', beta=1.0)
    perf['hamming_loss'] = hamming_loss(y_true, y_pred)
    perf['cm'] = confusion_matrix(y_true, y_pred)
    
    return perf

def evalClassifier(vScore_test, thePredictedScores):  
  target_names = ['Low_Risk', 'High_Risk']
  '''
    the way skelarn treats is the following: first index -> lower index -> 0 -> 'Low'
    the way skelarn treats is the following: next index after first  -> next lower index -> 1 -> 'high'    
  '''
  print "precison, recall, F-stat"
  print(classification_report(vScore_test, thePredictedScores, target_names=target_names))
  print"*********************"
  # preserve the order first test(real values from dataset), then predcited (from the classifier )
  '''
    are under the curve values .... reff: http://gim.unmc.edu/dxtests/roc3.htm 
    0.80~0.90 -> good, any thing less than 0.70 bad , 0.90~1.00 -> excellent 
  '''
  area_roc_output = roc_auc_score(vScore_test, thePredictedScores)
  # preserve the order first test(real values from dataset), then predcited (from the classifier )  
  print "Area under the ROC curve is ", area_roc_output
  print"*********************"  
  '''
    mean absolute error (mae) values .... reff: http://scikit-learn.org/stable/modules/generated/sklearn.metrics.mean_absolute_error.html
    the smaller the better , ideally expect 0.0 
  '''
  mae_output = mean_absolute_error(vScore_test, thePredictedScores)
  # preserve the order first test(real values from dataset), then predcited (from the classifier )  
  print "Mean absolute errro output  is ", mae_output  
  print"*********************"  
  '''
  accuracy_score ... reff: http://scikit-learn.org/stable/modules/model_evaluation.html#scoring-parameter .... percentage of correct predictions 
  ideally 1.0, higher the better 
  '''
  accuracy_score_output = accuracy_score(vScore_test, thePredictedScores)
  # preserve the order first test(real values from dataset), then predcited (from the classifier )  
  print "Accuracy output  is ", accuracy_score_output   
  print"*********************"  
  
  '''
  hamming_loss ... reff: http://scikit-learn.org/stable/modules/model_evaluation.html#scoring-parameter .... percentage of correct predictions 
  ideally 0.0, lower the better 
  '''
  hamming_loss_output = hamming_loss(vScore_test, thePredictedScores)
  # preserve the order first test(real values from dataset), then predcited (from the classifier )  
  print "Hamming loss output  is ", hamming_loss_output    
  print"*********************"  
  
  
  '''
  jaccardian score ... reff: http://scikit-learn.org/stable/modules/model_evaluation.html#scoring-parameter .... percentage of correct predictions 
  ideally 1.0, higher the better 
  '''
  jaccardian_output = jaccard_similarity_score(vScore_test, thePredictedScores)
  # preserve the order first test(real values from dataset), then predcited (from the classifier )  
  print "Jaccardian output  is ", jaccardian_output     
  print"*********************"  

def printscore(true_matrix,max_matrix,voting_matrix,mean_matrix):

    print "\t\tvoting classification report"
    print classification_report(true_matrix,voting_matrix)
    print "\t\tmax classification report"
    print classification_report(true_matrix,max_matrix)
    print "\t\tmean classification report"
    print classification_report(true_matrix,mean_matrix)

    print "------------------------------------------"

    print "\tvoting accuracy :{}\n".format(accuracy_score(true_matrix,voting_matrix))
    print "max accuracy :{}\n".format(accuracy_score(true_matrix,max_matrix))
    print "mean accuracy :{}\n".format(accuracy_score(true_matrix,mean_matrix))

    print "------------------------------------------"

    print "\tvoting Hamming loss:{}\n".format(hamming_loss(true_matrix,voting_matrix))
    print "max Hamming loss:{}\n".format(hamming_loss(true_matrix,max_matrix))
    print "mean Hamming loss:{}\n".format(hamming_loss(true_matrix,mean_matrix))

    print "------------------------------------------"

    print "\tvoting f1 score:{}\n".format(f1_score(true_matrix,voting_matrix,average='macro'))
    print "max f1 score:{}\n".format(f1_score(true_matrix,max_matrix,average='macro'))
    print "mean f1 score:{}\n".format(f1_score(true_matrix,mean_matrix,average='macro'))

    print "------------------------------------------"

    fpr, tpr, thresholds = roc_curve(true_matrix, voting_matrix, pos_label=2)
    print "\tvoting auc:{}\n".format(metrics.auc(fpr, tpr))
    fpr, tpr, thresholds = roc_curve(true_matrix, max_matrix, pos_label=2)
    print "max auc:{}\n".format(metrics.auc(fpr, tpr))
    fpr, tpr, thresholds = roc_curve(true_matrix, mean_matrix, pos_label=2)
    print "mean auc:{}\n".format(metrics.auc(fpr, tpr))


    return

def classify(XTrain,XTest,YTrain,YTest,c,cv=False):
	# print XTrain.shape,XTest.shape,YTrain.shape,YTest.shape;
	

	# print classifier
	# YTrain = YTrain.todense()
	# XTrain = XTrain.todense()

	# print type(XTrain)
	# print type(YTrain)
	start_time = time.time()
	# print XTrain.shape

	classifier = OneVsRestClassifier(LinearSVC(penalty='L1',loss='L2',C=c,dual=False,multi_class='ovr'))#,verbose=1));

	classifier.fit(XTrain, YTrain)

	predicted = classifier.predict(XTest)

	
	# if not a cross-validation instance, need to print results
	if not cv:
		qbGbl.weights = classifier.coef_

		# print report
		print metrics.classification_report(YTest, predicted,target_names=yTransformer.classes_)

		# ##  Collective Statistics
		print 'accuracy score of the classifier: {0}'.format(1.0-metrics.hamming_loss(YTest,predicted))
		print 'value of the C\t\t\t: {0}'.format(c)

		print 'Time taken to classify\t\t: {0} seconds'.format(time.time()-start_time);

		# print 'precision score of the classifier: {0}'.format(metrics.precision_score(YTest,predicted))
		# print 'recall score the classifier: {0}'.format(metrics.recall_score(YTest,predicted))
		# print 'F1 score of the classifier: {0}'.format(metrics.f1_score(YTest,predicted))

	return float(1.0-metrics.hamming_loss(YTest,predicted))

def test_multilabel_hamming_loss():
    # Dense label indicator matrix format
    y1 = np.array([[0, 1, 1], [1, 0, 1]])
    y2 = np.array([[0, 0, 1], [1, 0, 1]])

    assert_equal(hamming_loss(y1, y2), 1 / 6)
    assert_equal(hamming_loss(y1, y1), 0)
    assert_equal(hamming_loss(y2, y2), 0)
    assert_equal(hamming_loss(y2, 1 - y2), 1)
    assert_equal(hamming_loss(y1, 1 - y1), 1)
    assert_equal(hamming_loss(y1, np.zeros(y1.shape)), 4 / 6)
    assert_equal(hamming_loss(y2, np.zeros(y1.shape)), 0.5)

    def performance(self, preds):
        accuracy = accuracy_score(self.y_test, preds)
        precision = precision_score(self.y_test, preds)
        recall = recall_score(self.y_test, preds)
        f1 = f1_score(self.y_test, preds)
        jss = jaccard_similarity_score(self.y_test, preds)
        hl = hamming_loss(self.y_test, preds)
        zol = zero_one_loss(self.y_test, preds)

        return {'accuracy_score': accuracy,
                'precision_score': precision,
                'recall_score': recall,
                'f1_score': f1,
                'jaccard_similarity_score': jss,
                'hamming_loss': hl,
                'zero_one_loss': zol}