def test_classifier_chain_vs_independent_models():
    # Verify that an ensemble of classifier chains (each of length
    # N) can achieve a higher Jaccard similarity score than N independent
    # models
    yeast = fetch_mldata('yeast')
    X = yeast['data']
    Y = yeast['target'].transpose().toarray()
    X_train = X[:2000, :]
    X_test = X[2000:, :]
    Y_train = Y[:2000, :]
    Y_test = Y[2000:, :]

    ovr = OneVsRestClassifier(LogisticRegression())
    ovr.fit(X_train, Y_train)
    Y_pred_ovr = ovr.predict(X_test)

    chain = ClassifierChain(LogisticRegression(),
                            order=np.array([0, 2, 4, 6, 8, 10,
                                            12, 1, 3, 5, 7, 9,
                                            11, 13]))
    chain.fit(X_train, Y_train)
    Y_pred_chain = chain.predict(X_test)

    assert_greater(jaccard_similarity_score(Y_test, Y_pred_chain),
                   jaccard_similarity_score(Y_test, Y_pred_ovr))

    def test_jaccard_similarity_score(self):
        result = self.df.metrics.jaccard_similarity_score()
        expected = metrics.jaccard_similarity_score(self.target, self.pred)
        self.assertEqual(result, expected)

        result = self.df.metrics.jaccard_similarity_score(normalize=False)
        expected = metrics.jaccard_similarity_score(self.target, self.pred, normalize=False)
        self.assertEqual(result, expected)

 def tribunalTrain(data,predict,tribunal,split=.2,stat=False,statLis=None):
     #data for testing the tribunal performance, not in actual judge training
     dat_train, dat_test, lab_train, lab_test = train_test_split(data,predict, test_size=split)
     verdict = []
      
     print 'Tribunal in session'
     
     for judge in tribunal:
         jdat_train, jdat_test, jlab_train, jlab_test = train_test_split(dat_train,lab_train, test_size=split)
         judge.fit(jdat_train, jlab_train)
         print 'judge trained'
 
     for d in dat_test:
         votes = []
         for judge in tribunal:
             v = judge.predict(d)
             votes.append(v)
         decision = stats.mode(votes,axis=None)
         verdict.append(decision[0])
     npVerdict = np.array(verdict)
     
     if stat == False:        
         svmDesc(npVerdict,lab_test,title='Tribunal Confusion Matrix')
     else:
         jac = jaccard_similarity_score(npVerdict,lab_test)
         statLis.append(jac)

def calc_jacc(model):
    img = np.load(xtmp_file)
    msk = np.load(ytmp_file)

    prd = model.predict(img, batch_size=4)
    print prd.shape, msk.shape
    avg, trs = [], []

    for i in range(num_classes):
        t_msk = msk[:, i, :, :]
        t_prd = prd[:, i, :, :]
        t_msk = t_msk.reshape(msk.shape[0] * msk.shape[2], msk.shape[3])
        t_prd = t_prd.reshape(msk.shape[0] * msk.shape[2], msk.shape[3])

        m, b_tr = 0, 0
        for j in range(10):
            tr = j / 10.0
            pred_binary_mask = t_prd > tr

            jk = jaccard_similarity_score(t_msk, pred_binary_mask)
            if jk > m:
                m = jk
                b_tr = tr
        print i, m, b_tr
        avg.append(m)
        trs.append(b_tr)

    score = sum(avg) / 10.0
    return score, trs

 def svmDesc(lab_pred,lab_test, title='Confusion matrix', cmap=plot.cm.Blues,taskLabels=taskLabels,normal=True):
     #build confussion matrix itself
     conM = confusion_matrix(lab_test, lab_pred)
     if normal== True:
         conM = conM.astype('float') / conM.sum(axis=1)[:, np.newaxis]
     #build heatmap graph of matrix
     plot.imshow(conM, interpolation='nearest', cmap=cmap)
     plot.title(title)
     plot.colorbar()
     tick_marks = np.arange(len(taskLabels))
     plot.xticks(tick_marks, taskLabels, rotation=45)
     plot.yticks(tick_marks, taskLabels)
     plot.tight_layout()
     plot.ylabel('True label')
     plot.xlabel('Predicted label')
     
     #classification report
     creport = classification_report(lab_test,lab_pred)
     print "CLASSIFICATION REPORT: "  
     print creport
     
     #hamming distance
     hamming = hamming_loss(lab_test,lab_pred)
     print "HAMMING DISTANCE:              %s" % str(hamming)
     
     #jaccard similarity score
     jaccard = jaccard_similarity_score(lab_test,lab_pred)
     print "JACCARD SIMILARITY SCORE:      %s" % str(jaccard)
     
     #precision score    
     pscore = precision_score(lab_test,lab_pred)
     print "PRECISION SCORE:               %s" % str(pscore)

    def train_and_eval(x_train, y_train, x_test, y_test, model, param_result):
        print("\nTraining and evaluating...")

        for result_list in param_result:
            print("Fitting: " + str(result_list[2]))

            opt_model = result_list[2]
            opt_model.fit(x_train, y_train)
            y_pred = opt_model.predict(x_test)

            print("\nClassification Report:")
            print(metrics.classification_report(y_test, y_pred))
            print("\nAccuracy Score:")
            print(metrics.accuracy_score(y_test, y_pred))
            print("\nConfusion Matrix:")
            print(metrics.confusion_matrix(y_test, y_pred))
            print("\nF1-Score:")
            print(metrics.f1_score(y_test, y_pred))
            print("\nHamming Loss:")
            print(metrics.hamming_loss(y_test, y_pred))
            print("\nJaccard Similarity:")
            print(metrics.jaccard_similarity_score(y_test, y_pred))
            # vvv Not supported due to ValueError: y_true and y_pred have different number of classes 3, 2
            # print('\nLog Loss:')
            # print(metrics.log_loss(y_test, y_pred))
            # vvv multiclass not supported
            # print('\nMatthews Correlation Coefficient:')
            # print(metrics.matthews_corrcoef(y_test, y_pred))
            print("\nPrecision:")
            print(metrics.precision_score(y_test, y_pred))
            # vvv Not supported due to ValueError: y_true and y_pred have different number of classes 3, 2
            # print('\nRecall:')
            # print(metrics.recall(y_test, y_pred))
            print()

    def calc_thresholds(self, patches_in, patches_out):
        prediction = self.model.predict(patches_in, batch_size=4)
        avg, trs = [], []

        for i in range(self.out_chan):
            t_prd = prediction [:, :, :, i]
            t_msk = patches_out[:, :, :, i]

            t_prd = t_prd.reshape(t_msk.shape[0] * t_msk.shape[1], t_msk.shape[2])
            t_msk = t_msk.reshape(t_msk.shape[0] * t_msk.shape[1], t_msk.shape[2])

            t_msk = t_msk > 0.5 
            # threshold finder
            best_score = 0
            best_threashold = 0
            for j in range(10):
                threashold = (j+1) / 10.0
                threshold_mask = (t_prd > threashold) 

                jk = jaccard_similarity_score(t_msk, threshold_mask)
                if jk > best_score:
                    best_score = jk
                    best_threashold = threashold
            print " -- output:", i, "best:", best_score, "threashold:", best_threashold
            avg.append(best_score)
            trs.append(best_threashold)

        score = sum(avg) / 10.0
        return score, trs

def analise():
    datasets = load_data_from_pickle()
    classifier = get_conv_classifier()
    given_answers = list(classifier.predict(datasets.test.data)['classes'])

    wrong_answer_buckets = np.zeros(5)
    for i, test_data in enumerate(datasets.test.data):
        right_answer = datasets.test.target[i]
        given_answer = given_answers[i]
        if right_answer != given_answer:
            wrong_answer_buckets[right_answer] += 1
    print(wrong_answer_buckets / sum(wrong_answer_buckets))

    confusion_matrix = metrics.confusion_matrix(datasets.test.target, given_answers, range(5))
    print(confusion_matrix)

    cohen_kappa_score = metrics.cohen_kappa_score(datasets.test.target, given_answers, range(5))
    print(cohen_kappa_score)

    jaccard_similarity_score = metrics.jaccard_similarity_score(datasets.test.target, given_answers)
    print(jaccard_similarity_score)

    report = metrics.classification_report(datasets.test.target, given_answers, labels=range(5),
                                           target_names=['NORTH', 'EAST', 'SOUTH', 'WEST', 'STILL'])
    print(report)

def calculateSimilarityItems(item1, item2):
  try:
    result = jaccard_similarity_score(Utility[item1], Utility[item2])
  except Warning:
    #print "Exception at %d : %d" % (item1, item2)
    result = 0.5
  return result

 def neighbor_rating(self, neighbor, itemID, sigma, threshold):
     self.ratings_sum=0
     self.similarity_sum=0
     #print(type(user))
     #print(user)
     #print(neighbor[2:10])
     #print(df.itemID)
     ratings = df[(df.userID == neighbor.userID) & (df.itemID == itemID)]
     #print(ratings.shape)
     for index, user_rating in ratings.iterrows():
         similarity = jaccard_similarity_score(neighbor[2:-1],user_rating[2:-1])
         #print(similarity)
         if similarity > threshold:
             self.ratings_sum += user_rating.rating * similarity
             self.similarity_sum += similarity
             #print(self.similarity_sum)
         #print(self.ratings_sum)
         #print(neighbor[-10:],user_rating[-10:])
     #print("Rating sum")
     #print(self.ratings_sum)
     #print("Similarity Sum")
     #print(self.similarity_sum)
     try:
         rating = self.ratings_sum / self.similarity_sum
         #rating = ratings.rating.mean()
     except:
         rating = 0
     #print(rating)
     return rating

def ComputeMetrics(prob, batch_labels, p1, p2, rgb=None, save_path=None, ind=0):
    GT = label(batch_labels.copy())
    PRED = PostProcess(prob, p1, p2)
    lbl = GT.copy()
    pred = PRED.copy()
    aji = AJI_fast(lbl, pred)
    lbl[lbl > 0] = 1
    pred[pred > 0] = 1 
    l, p = lbl.flatten(), pred.flatten()
    acc = accuracy_score(l, p)
    roc = roc_auc_score(l, p)
    jac = jaccard_similarity_score(l, p)
    f1 = f1_score(l, p)
    recall = recall_score(l, p)
    precision = precision_score(l, p)
    if rgb is not None:
        xval_n = join(save_path, "xval_{}.png").format(ind)
        yval_n = join(save_path, "yval_{}.png").format(ind)
        prob_n = join(save_path, "prob_{}.png").format(ind)
        pred_n = join(save_path, "pred_{}.png").format(ind)
        c_gt_n = join(save_path, "C_gt_{}.png").format(ind)
        c_pr_n = join(save_path, "C_pr_{}.png").format(ind)
        ## CHECK PLOT FOR PROB AS IT MIGHT BE ILL ADAPTED

        imsave(xval_n, rgb)
        imsave(yval_n, color_bin(GT))
        imsave(prob_n, prob)
        imsave(pred_n, color_bin(PRED))
        imsave(c_gt_n, add_contours(rgb, GT))
        imsave(c_pr_n, add_contours(rgb, PRED))

    return acc, roc, jac, recall, precision, f1, aji

def test_jaccard_binary_index():
    y_test = np.array([0, 0, 1, 0, 0, 1, 1, 0, 0, 0, 0])
    y_pred = np.array([0, 1, 1, 0, 0, 0, 1, 0, 0, 0, 0])
    sk_jaccard_score = metrics.jaccard_similarity_score(y_test, y_pred)
    print(sk_jaccard_score)
    jaccard_index = jaccard_binary_index(y_test, y_pred)
    print(jaccard_index)
    assert jaccard_index == 0.5

 def jaccard_score(self, row):
     query = row['search_term']
     title = row['product_title']
     
     corpus = np.array([query, title])
     tfidf_matrix = self.tfidf_vectorizer.fit_transform(corpus)
     
     return jaccard_similarity_score(tfidf_matrix[0], tfidf_matrix[1])

def test_classifier_chain_crossval_fit_and_predict():
    # Fit classifier chain with cross_val_predict and verify predict
    # performance
    X, Y = generate_multilabel_dataset_with_correlations()
    classifier_chain_cv = ClassifierChain(LogisticRegression(), cv=3)
    classifier_chain_cv.fit(X, Y)

    classifier_chain = ClassifierChain(LogisticRegression())
    classifier_chain.fit(X, Y)

    Y_pred_cv = classifier_chain_cv.predict(X)
    Y_pred = classifier_chain.predict(X)

    assert_equal(Y_pred_cv.shape, Y.shape)
    assert_greater(jaccard_similarity_score(Y, Y_pred_cv), 0.4)

    assert_not_equal(jaccard_similarity_score(Y, Y_pred_cv),
                     jaccard_similarity_score(Y, Y_pred))

def getJaccardSimilarity(user1=None, user2=None):
    if user1.ndim != 1 or user2.ndim != 1:
        print 'Input arrays must be 1-dimensional'
        return
    elif user1.shape != user2.shape:
        print 'Input arrays must have the same length'
        return
    else:
        return jaccard_similarity_score(user1, user2)

def jaccard_driver(a_driver):

    a_driver["DStats"] = (a_driver["DStats"] * 100).round()
    a_driver["Baseline"] = (a_driver["Baseline"] * 100).round()
    a_driver["Predicts"] = []

    for i in range(0, len(a_driver["DStats"])):
        a_driver["Predicts"].append(metrics.jaccard_similarity_score(a_driver["DStats"][i], a_driver["Baseline"]))

    return a_driver["Predicts"]

def jaccard_index(y, y_pred):
  """Computes Jaccard Index which is the Intersection Over Union metric
       which is commonly used in image segmentation tasks

      Parameters
      ----------
      y: ground truth array
      y_pred: predicted array
    """
  return jaccard_similarity_score(y, y_pred)

def jaccard_driver(a_driver):
    
    a_driver['DStats'] = (a_driver['DStats']*100).round()
    a_driver['Baseline'] = (a_driver['Baseline']*100).round()
    a_driver['Predicts'] = []
    
    for i in range (0,len(a_driver['DStats'])):
        a_driver['Predicts'].append(metrics.jaccard_similarity_score(a_driver['DStats'][i],a_driver['Baseline']))
                
    
    return a_driver['Predicts']

def eval_mclf(y, y_hat):
    results = {
        "jaccard": jaccard_similarity_score(numpy.array(y),
                                            numpy.array(y_hat)),
        "f1-macro": f1_score(numpy.array(y), numpy.array(y_hat),
                             average='macro'),
        "f1-micro": f1_score(numpy.array(y), numpy.array(y_hat),
                             average='micro')
    }

    return results

def _get_max_similarity(list1,list2, coassoc_vec):
    n = len(coassoc_vec.keys())
    max = 0
    for i in range(len(list1)):
        checkee = coassoc_vec[coassoc_vec.keys()[list1[i]]]
        for j in range(len(list2)):
            neu = coassoc_vec[coassoc_vec.keys()[list2[j]]]
            jaccard = jaccard_similarity_score(checkee.binarycoassoc_vs,neu.binarycoassoc_vs)
            if jaccard> max:
                max = jaccard
    return max