def test_classification():
    from sklearn.datasets import load_digits
    from sklearn.cross_validation import KFold
    from sklearn.metrics import normalized_mutual_info_score
    digits = load_digits()
    X, y = digits.data, digits.target
    folds = 3
    cv = KFold(y.shape[0], folds)
    total = 0.0
    oo_score_bag = []
    for tr, te in cv:
        mlp = MLPClassifier(use_dropout=True, n_hidden=200, lr=1.)
        print(mlp)
        mlp.fit(X[tr], y[tr], max_epochs=100, staged_sample=X[te])
        t = normalized_mutual_info_score(mlp.predict(X[te]), y[te])
        print("Fold training accuracy: %f" % t)
        total += t
        this_score = []
        for i in mlp.oo_score:
            this_score.append(normalized_mutual_info_score(i, y[te]))
        oo_score_bag.append(this_score)
    from matplotlib import pyplot as plt
    plt.plot(oo_score_bag[0])
    plt.show()

    print("training accuracy: %f" % (total / float(folds)))

  def loss_augmented_fit(self, feat, y, loss_mult):
    """Fit K-Medoids to the provided data."""
    self._check_init_args()
    # Check that the array is good and attempt to convert it to
    # Numpy array if possible.
    feat = self._check_array(feat)
    # Apply distance metric to get the distance matrix.
    pdists = pairwise_distance_np(feat)

    num_data = feat.shape[0]
    candidate_ids = list(range(num_data))
    candidate_scores = np.zeros(num_data,)
    subset = []

    k = 0
    while k < self.n_clusters:
      candidate_scores = []
      for i in candidate_ids:
        # push i to subset.
        subset.append(i)
        marginal_cost = -1.0 * np.sum(np.min(pdists[:, subset], axis=1))
        loss = 1.0 - metrics.normalized_mutual_info_score(
            y, self._get_cluster_ics(pdists, subset))
        candidate_scores.append(marginal_cost + loss_mult * loss)
        # remove i from subset.
        subset.pop()

      # push i_star to subset.
      i_star = candidate_ids[np.argmax(candidate_scores)]
      subset.append(i_star)
      # remove i_star from candidate indices.
      candidate_ids.remove(i_star)
      k += 1

    # Expose labels_ which are the assignments of
    # the training data to clusters.
    self.labels_ = self._get_cluster_ics(pdists, subset)
    # Expose cluster centers, i.e. medoids.
    self.cluster_centers_ = feat.take(subset, axis=0)
    # Expose indices of chosen cluster centers.
    self.center_ics_ = subset
    # Expose the score = -\sum_{i \in V} min_{j \in S} || x_i - x_j ||
    self.score_ = np.float32(-1.0) * self._get_facility_distance(pdists, subset)
    self.score_aug_ = self.score_ + loss_mult * (
        1.0 - metrics.normalized_mutual_info_score(
            y, self._get_cluster_ics(pdists, subset)))
    self.score_aug_ = self.score_aug_.astype(np.float32)
    # Expose the chosen cluster indices.
    self.subset_ = subset
    return self

def evaluate_label(A, H, W, corr, K):
    label = H.argmax(axis=1)
    km = KMeans(K)
    label2 = km.fit_predict(H)
    nmi = normalized_mutual_info_score(label, corr)
    nmi2 = normalized_mutual_info_score(label2, corr)
    print("NMI by argmax: " + str(nmi))
    print("NMI by kmeans: " + str(nmi2))
    A = np.matrix(A)
    W = np.matrix(W)
    H = np.matrix(H)
    loss = np.power(A - W * H.T, 2).sum()
    print(loss)
    return nmi, nmi2, loss

def test_diffusion_embedding_two_components_no_diffusion_time(seed=36):
    """Test spectral embedding with two components"""
    random_state = np.random.RandomState(seed)
    n_sample = 100
    affinity = np.zeros(shape=[n_sample * 2,
                               n_sample * 2])
    # first component
    affinity[0:n_sample,
             0:n_sample] = np.abs(random_state.randn(n_sample, n_sample)) + 2
    # second component
    affinity[n_sample::,
             n_sample::] = np.abs(random_state.randn(n_sample, n_sample)) + 2
    # connection
    affinity[0, n_sample + 1] = 1
    affinity[n_sample + 1, 0] = 1
    affinity.flat[::2 * n_sample + 1] = 0
    affinity = 0.5 * (affinity + affinity.T)

    true_label = np.zeros(shape=2 * n_sample)
    true_label[0:n_sample] = 1
    geom_params = {'laplacian_method':'geometric'}
    se_precomp = SpectralEmbedding(n_components=1,
                                   random_state=np.random.RandomState(seed),
                                   eigen_solver = 'arpack',
                                   diffusion_maps = True,
                                   geom = geom_params)
    embedded_coordinate = se_precomp.fit_transform(affinity,
                                                   input_type='affinity')

    # thresholding on the first components using 0.
    label_ = np.array(embedded_coordinate.ravel() < 0, dtype="float")
    assert_equal(normalized_mutual_info_score(true_label, label_), 1.0)

def compare_direct_undir():
    from sklearn import metrics
    g = gt.Graph.Read_GraphML('ed_tag.graphml')
    gt.net_stat(g)
    gu = gt.Graph.Read_GraphML('ed_tag_undir.graphml')
    gt.net_stat(gu)
    com = g.community_infomap(edge_weights='weight', vertex_weights='weight')
    comu1 = gu.community_infomap(edge_weights='weight', vertex_weights='weight')
    comu2 = gu.community_infomap(edge_weights='weight', vertex_weights='weight')
    mem = com.membership
    memu1 = comu1.membership
    memu2 = comu2.membership
    print metrics.adjusted_rand_score(mem, memu1)
    print metrics.normalized_mutual_info_score(mem, memu1)
    print metrics.adjusted_rand_score(memu2, memu1)
    print metrics.normalized_mutual_info_score(memu2, memu1)

def plotMI(dat, lab, width = 0.35, signed = 0):
	'''
	Draw a bar chart of the normalized MI between each X and Y
	'''
	X = dat.drop(lab, 1)
	Y = dat[[lab]].values
	cols = X.columns.values
	mis = []

	#Start by getting MI
	for c in cols:
		mis.append(skm.normalized_mutual_info_score(Y.ravel(), X[[c]].values.ravel()))

	#Get signs by correlation
	corrs = dat.corr()[lab]
	corrs[corrs.index != lab]
	df = pd.DataFrame(list(zip(mis, cols)), columns = ['MI', 'Lab'])
	df = pd.concat([df, pd.DataFrame(list(corrs), columns = ['corr'])], axis=1, join_axes=[df.index])


	
	if signed == 0:
		makeBar(df, 'MI', 'Lab', width)

	else:
		makeBarSigned(df, 'MI', 'Lab', width)

def compare(method1, method2, fig=False):
    X1 = np.load('{0}_{1}_X_2d.npy'.format(species, method1))
    X2 = np.load('{0}_{1}_X_2d.npy'.format(species, method2))
    
    print 'n_cluster\tHomo\tCompl\tNMI\tARI'
    for i in range(2, 6):
        clust1 = Clustering(species, method1, X1, None, n_clusters=i)
        clust2 = Clustering(species, method2, X2, None, n_clusters=i)
        
        clust1.agglomerative(linkage='ward')
        clust2.agglomerative(linkage='ward')
        
        label1 = clust1.pred_labels('ward')
        label2 = clust2.pred_labels('ward')
        
        
        if i == 3 and fig:
            names = np.unique(label1)
            figName = '{0}_{1}_on_{2}'.format(species, method1, method2)
            plot2d(X2, label1, names, figName, figName)

            names = np.unique(label2)
            figName = '{0}_{1}_on_{2}'.format(species, method2, method1)
            plot2d(X1, label2, names, figName, figName)
    
        print '{0}\t{1}\t{2}\t{3}\t{4}\n'.format(i, metrics.homogeneity_score(label1, label2),
                                                metrics.completeness_score(label1, label2),
                                                metrics.normalized_mutual_info_score(label1, label2),
                                                metrics.adjusted_rand_score(label1, label2))

    def get_normalized_mutual_info(standard_file, prediction_file, isjson=False, isint=False):
        """Get normalized mutual information (NMI) [Strehl2002]_.

        Parameters
        ----------
        standard_file   : str
            The ground truth or standard filename.
        prediction_file : str
            The analyzed or predicted filename.
        isjson          : bool
            The flag for standard_file.
        isint           : bool
            The flag for value in prediction_file.

        Returns
        -------
        normalized_mutual_info  : float
            Normalized mutual information score.

        References
        ----------
        .. [Strehl2002] Alexander Strehl and Joydeep Ghosh. Cluster ensembles A knowledge reuse framework
                        for combining multiple partitions. Journal of Machine Learning Research,
                        3(Dec):583-617, 2002.
        """
        if isjson:
            standard_data = AbstractionUtility.read_json(standard_file)
            standard_labels = standard_data.values()
        else:
            standard_labels = ExternalEvaluation.get_evaluated(standard_file)

        prediction_labels = ExternalEvaluation.get_evaluated(prediction_file, isint=isint)
        normalized_mutual_info = metrics.normalized_mutual_info_score(standard_labels, prediction_labels)

        return normalized_mutual_info

 def evaluate(self):
     ARI = round(metrics.adjusted_rand_score(self.labels, self.pred), 4)
     AMI = round(metrics.adjusted_mutual_info_score(self.labels, self.pred), 4)
     NMI = round(metrics.normalized_mutual_info_score(self.labels, self.pred), 4)
     print("Adjusted Rand index:", "%.4f" % ARI)
     print("Adjusted Mutual Information:", "%.4f" % AMI)
     print("Normalized Mutual Information:", "%.4f" % NMI)

  def pam_augmented_fit(self, feat, y, loss_mult):
    pam_max_iter = 5
    self._check_init_args()
    feat = self._check_array(feat)
    pdists = pairwise_distance_np(feat)
    self.loss_augmented_fit(feat, y, loss_mult)
    print('PAM -1 (before PAM): score: %f, score_aug: %f' % (
        self.score_, self.score_aug_))
    # Initialize from loss augmented facility location
    subset = self.center_ics_
    for iter_ in range(pam_max_iter):
      # update the cluster assignment
      cluster_ics = self._get_cluster_ics(pdists, subset)
      # update the medoid for each clusters
      self._augmented_update_medoid_ics_in_place(pdists, y, cluster_ics, subset,
                                                 loss_mult)
      self.score_ = np.float32(-1.0) * self._get_facility_distance(
          pdists, subset)
      self.score_aug_ = self.score_ + loss_mult * (
          1.0 - metrics.normalized_mutual_info_score(
              y, self._get_cluster_ics(pdists, subset)))
      self.score_aug_ = self.score_aug_.astype(np.float32)
      print('PAM iter: %d, score: %f, score_aug: %f' % (iter_, self.score_,
                                                        self.score_aug_))

    self.center_ics_ = subset
    self.labels_ = cluster_ics
    return self

def test_spectral_embedding_two_components(seed=36):
    """Test spectral embedding with two components"""
    random_state = np.random.RandomState(seed)
    n_sample = 100
    affinity = np.zeros(shape=[n_sample * 2,
                               n_sample * 2])
    # first component
    affinity[0:n_sample,
             0:n_sample] = np.abs(random_state.randn(n_sample, n_sample)) + 2
    # second component
    affinity[n_sample::,
             n_sample::] = np.abs(random_state.randn(n_sample, n_sample)) + 2
    # connection
    affinity[0, n_sample + 1] = 1
    affinity[n_sample + 1, 0] = 1
    affinity.flat[::2 * n_sample + 1] = 0
    affinity = 0.5 * (affinity + affinity.T)

    true_label = np.zeros(shape=2 * n_sample)
    true_label[0:n_sample] = 1

    se_precomp = SpectralEmbedding(n_components=1, affinity="precomputed",
                                   random_state=np.random.RandomState(seed))
    embedded_coordinate = se_precomp.fit_transform(affinity)
    # Some numpy versions are touchy with types
    embedded_coordinate = \
        se_precomp.fit_transform(affinity.astype(np.float32))
    # thresholding on the first components using 0.
    label_ = np.array(embedded_coordinate.ravel() < 0, dtype="float")
    assert_equal(normalized_mutual_info_score(true_label, label_), 1.0)

def calculate_nmi_kmeans(H, correct_label):
    km = KMeans(H.shape[1])
    try:
        com = km.fit_predict(H)
    except:
        com = [0] * H.shape[0]
    nmi = normalized_mutual_info_score(com, correct_label)
    return nmi

def cluster_metrics(labels_1, labels_2):
    print("\n".join(
        [
            "Normalized Mutual Information: %f" % (normalized_mutual_info_score(labels_1, labels_2)),
            "Adjusted Rand Score: %f" % (adjusted_rand_score(labels_1, labels_2)),
            "Homogeneity: %f" % (homogeneity_score(labels_1, labels_2)),
            "Completeness: %f" % (completeness_score(labels_1, labels_2))
        ]
    ))

def normalized_mutual_info_score_scorefunc(X, y):

    scores = []
    pvals = []
    for col in range(X.shape[1]):
        scores.append(normalized_mutual_info_score(X[:, col], y))
        pvals.append(1)

    return np.array(scores), np.array(pvals)

def model_metrics(model, X, y, batch_size):
    loss_and_metrics = model.evaluate(X, y, batch_size=batch_size)
    predicted_classes = model.predict_classes(X, batch_size=batch_size)
    predicted_probas = model.predict_proba(X, batch_size=batch_size)

    accuracy = loss_and_metrics[1]
    roc_auc = roc_auc_score(y, predicted_probas)
    nmi = normalized_mutual_info_score(y, predicted_classes.flatten())
    return accuracy, roc_auc, nmi

def calc(gr_truth, predicted):
    # precision, recall, fscore, _ = score(gr_truth, predicted, average='micro')
    # print('precision: {}'.format(precision))
    # print('recall: {}'.format(recall))
    # print('fscore: {}'.format(fscore))
    # print('jaccard: {}'.format(jaccard_similarity_score(gr_truth, predicted, normalize=True)))
    # print('mutual: {}'.format(mutual_info_score(gr_truth, predicted)))
    # print('mutual adj: {}'.format(adjusted_mutual_info_score(gr_truth, predicted)))
    # print('mutual norm: {}'.format(normalized_mutual_info_score(gr_truth, predicted)))
    return normalized_mutual_info_score(gr_truth, predicted)

def observe_correlations(nrows=1000000):
    training_file = "data/train.csv"
    dataframe = pd.read_csv(training_file,nrows=nrows,header=0)
    info_variable=pd.DataFrame(index=dataframe.columns[2:])
    info_variable["levels"]=dataframe.iloc[:,2:].apply(lambda t: np.unique(t).shape[0])
    info_variable["Entropy"]=dataframe.iloc[:,2:].apply(lambda t: entropy(t))
    info_variable["MutualInformation"]=dataframe.iloc[:,2:].apply(lambda t: metrics.normalized_mutual_info_score(dataframe["click"],t))
    info_variable["clustering"]=cluster_var(dataframe)
    info_variable=info_variable.sort("clustering")
    return info_variable

def mRMR(dfin, target, adjusted = False, n_features_to_select = 10):
    '''
    :param dfin: dataframe of features
    :param target: target (0,1)
    :return:
    '''

    df = dfin.copy()

    final_features = []
    importance = []

    if n_features_to_select > df.shape[1]:
        n_features_to_select = df.shape[1]-1

    #Iteratively create a subset of final features the feature that maximizes
    # MI(C, X_j) - (1/(m-1) * SUM_i (MI(X_i, X_j))
    #where C = the ultimate target, X_i = the feature to test, and X_j = features already selected
    while len(final_features) < n_features_to_select:
        features = np.array([c for c in df.columns.tolist() if c not in final_features])

        if adjusted:
            mi = np.array([adjusted_mutual_info_score(df[f], target) for f in features])
            if len(final_features) > 0:
                mr = np.array([normalized_mutual_info_score(df[x_j], df[x_i]) for x_j in features for x_i in features])
            else:
                mr = np.zeros(len(features))
        else:
            mi = [normalized_mutual_info_score(df[f], target) for f in features]
            if len(final_features) > 0:
                mr = np.array([normalized_mutual_info_score(df[x_j], df[x_i]) for x_j in features for x_i in features])
            else:
                mr = np.zeros(len(features))

        if len(final_features) > 0:
            mr = (1./(len(features) - 1)) * mr.reshape(len(features), len(features)).sum(axis = 1)

        mrmr = mi - mr
        final_features.append(features[mrmr == max(mrmr)][0])
        importance.append(max(mrmr))

    return final_features, importance

def test_pipeline_spectral_clustering(seed=36):
    # Test using pipeline to do spectral clustering
    random_state = np.random.RandomState(seed)
    se_rbf = SpectralEmbedding(n_components=n_clusters, affinity="rbf", random_state=random_state)
    se_knn = SpectralEmbedding(
        n_components=n_clusters, affinity="nearest_neighbors", n_neighbors=5, random_state=random_state
    )
    for se in [se_rbf, se_knn]:
        km = KMeans(n_clusters=n_clusters, random_state=random_state)
        km.fit(se.fit_transform(S))
        assert_array_almost_equal(normalized_mutual_info_score(km.labels_, true_labels), 1.0, 2)

def compare_clusters(nn_clusters, tf_clusters):
    """prints some comparisons"""
    print('~~Adjusted mutual information: {}'.format(
        metrics.adjusted_mutual_info_score(nn_clusters.labels_,
                                           tf_clusters.labels_)))
    print('~~Normalized mutual information: {}'.format(
        metrics.normalized_mutual_info_score(nn_clusters.labels_,
                                             tf_clusters.labels_)))
    print('~~Adjusted Rand Index: {}'.format(
        metrics.adjusted_rand_score(nn_clusters.labels_,
                                    tf_clusters.labels_)))