def test_permutation_test_score_allow_nans():
    # Check that permutation_test_score allows input data with NaNs
    X = np.arange(200, dtype=np.float64).reshape(10, -1)
    X[2, :] = np.nan
    y = np.repeat([0, 1], X.shape[0] / 2)
    p = Pipeline([("imputer", Imputer(strategy="mean", missing_values="NaN")), ("classifier", MockClassifier())])
    cval.permutation_test_score(p, X, y, cv=5)

def test_permutation_score():
    iris = load_iris()
    X = iris.data
    X_sparse = coo_matrix(X)
    y = iris.target
    svm = SVC(kernel="linear")
    cv = cval.StratifiedKFold(y, 2)

    score, scores, pvalue = cval.permutation_test_score(svm, X, y, n_permutations=30, cv=cv, scoring="accuracy")
    assert_greater(score, 0.9)
    assert_almost_equal(pvalue, 0.0, 1)

    score_label, _, pvalue_label = cval.permutation_test_score(
        svm, X, y, n_permutations=30, cv=cv, scoring="accuracy", labels=np.ones(y.size), random_state=0
    )
    assert_true(score_label == score)
    assert_true(pvalue_label == pvalue)

    # check that we obtain the same results with a sparse representation
    svm_sparse = SVC(kernel="linear")
    cv_sparse = cval.StratifiedKFold(y, 2)
    score_label, _, pvalue_label = cval.permutation_test_score(
        svm_sparse,
        X_sparse,
        y,
        n_permutations=30,
        cv=cv_sparse,
        scoring="accuracy",
        labels=np.ones(y.size),
        random_state=0,
    )

    assert_true(score_label == score)
    assert_true(pvalue_label == pvalue)

    # test with custom scoring object
    def custom_score(y_true, y_pred):
        return ((y_true == y_pred).sum() - (y_true != y_pred).sum()) / y_true.shape[0]

    scorer = make_scorer(custom_score)
    score, _, pvalue = cval.permutation_test_score(svm, X, y, n_permutations=100, scoring=scorer, cv=cv, random_state=0)
    assert_almost_equal(score, 0.93, 2)
    assert_almost_equal(pvalue, 0.01, 3)

    # set random y
    y = np.mod(np.arange(len(y)), 3)

    score, scores, pvalue = cval.permutation_test_score(svm, X, y, n_permutations=30, cv=cv, scoring="accuracy")

    assert_less(score, 0.5)
    assert_greater(pvalue, 0.2)

def permutation_cross_validation(estimator, X, y, n_fold=3, isshuffle=True, cvmeth='shufflesplit', score_type='r2', n_perm=1000):
    """
    An easy way to evaluate the significance of a cross-validated score by permutations
    -------------------------------------------------
    Parameters:
        estimator: linear model estimator
        X: IV
        y: DV
        n_fold: fold number cross validation
        cvmeth: kfold or shufflesplit. 
                shufflesplit is the random permutation cross-validation iterator
        score_type: scoring type, 'r2' as default
        n_perm: permutation numbers
    Return:
        score: model scores
        permutation_scores: model scores when permutation labels
        pvalues: p value of permutation scores
    """
    try:
        from sklearn import cross_validation, preprocessing
    except ImportError:
        raise Exception('To call this function, please install sklearn')
    if X.ndim == 1:
        X = np.expand_dims(X, axis = 1)
    if y.ndim == 1:
        y = np.expand_dims(y, axis = 1)
    X = preprocessing.scale(X)
    y = preprocessing.scale(y)
    if cvmeth == 'kfold':
        cvmethod = cross_validation.KFold(y.shape[0], n_fold, shuffle = isshuffle)
    elif cvmeth == 'shufflesplit':
        testsize = 1.0/n_fold
        cvmethod = cross_validation.ShuffleSplit(y.shape[0], n_iter = 100, test_size = testsize, random_state = 0)
    score, permutation_scores, pvalues = cross_validation.permutation_test_score(estimator, X, y, scoring = score_type, cv = cvmethod, n_permutations = n_perm)
    return score, permutation_scores, pvalues

def automatic_bernulli():
    data = pd.read_csv('/home/vasiliy/Study/StadiumProject/Classifier/signs.csv', sep=';')
    Y = np.array(data['fight'].get_values())
    np.random.shuffle(Y)
    data.drop(['match', 'city', 'date', 'fight'], 1, inplace=True)
    # data = data[['anger_over_value_relation', 'avg_likes', 'sc_max_surprise', 'sc_median_fear',
    #              'fear_over_value_relation']]

    X = data.as_matrix()

    features_number = 0
    result = {}
    for features_number in range(3, 16):
        X_new = SelectKBest(f_classif, k=features_number).fit_transform(X, Y)
        # X_new = X
        classifier = ExtraTreesClassifier()
        super_means = []
        for i in range(1000):
            kf = KFold(len(X_new), n_folds=6, shuffle=True)
            means = []
            for training, testing in kf:
                classifier.fit(X_new[training], Y[training])
                prediction = classifier.predict(X_new[testing])
                curmean = np.mean(prediction == Y[testing])
                means.append(curmean)
            super_means.append(np.mean(means))
        print 'features_number=', features_number, 'Mean accuracy: {:.1%} '.format(
                np.mean(super_means))
            # result['fn'+str(features_number)+'n_n'+str(n_neib)] = np.mean(super_means)
        score, permutation_scores, pvalue = permutation_test_score(classifier, X_new, Y, scoring="accuracy", cv=kf,
                                                                n_permutations=len(Y), n_jobs=1)
        print ("Classification score %s (pvalue : %s)" % (score, pvalue))

def classify(x, y, classifier='lda', kern='rbf', n_folds=10, rep=10, kind='sf', n_jobs=1, n_knn=3, n_perm=0, n_tree=100,
             cvkind='skfold'):
    "da, all_scores, permutation_scores, pvalue"
    # Check format :
    x = checkfeat(x,y)
    n_epoch, n_feat = x.shape
    priors = n.array([1/len(n.unique(y))]*len(n.unique(y)))

    # - Classifier's choice :
    if (type(classifier) is int) | (type(classifier) is str):
        clf = classifier_choice(classifier, kern=kern, n_knn=n_knn, n_tree=n_tree, priors=priors)
    else : clf = classifier

    # - Cross validation definition :
    if kind == 'mf' and n_perm == 0:  # Multi feature classification
        da, all_scores, cv_model = classify_fcn(x, y, clf, n_folds=n_folds, rep=rep, n_jobs=n_jobs, cvkind=cvkind)
    elif kind == 'sf' and n_perm == 0:  # Single features classification
        da = n.zeros((1, n_feat))
        all_scores = n.zeros((rep, n_folds, n_feat))
        for k in range(0, n_feat):
            da[:, k], all_scores[:, :, k], cv_model = classify_fcn(x[:, k], y, clf, n_folds=n_folds, rep=rep,
                                                                   n_jobs=n_jobs, cvkind=cvkind)

    # Statistical evaluation :
    if n_perm == 0:
        permutation_scores, pvalue = 0, [[0]]
    else:
        all_scores = 0
        cv_model = crossval_choice(y, cvkind=cvkind, n_folds=n_folds, rndstate=0)
        if kind == 'mf':  # Multi feature classification
            da, permutation_scores, pvalue = cross_validation.permutation_test_score(clf, x, y, scoring="accuracy",
                                                                                     cv=cv_model, n_permutations=n_perm,
                                                                                     n_jobs=n_jobs)
        elif kind == 'sf':  # Single features classification
            permutation_scores = n.zeros((n_perm, n_feat))
            da = n.zeros((1, n_feat))
            pvalue = n.zeros((1, n_feat))
            for k in range(0, n_feat):
                da[0, k], permutation_scores[:, k], pvalue[0, k] = cross_validation.permutation_test_score(clf, x[:, k], y,
                                                                                                           scoring="accuracy",
                                                                                                           cv=cv_model,
                                                                                                           n_permutations=n_perm,
                                                                                                           n_jobs=n_jobs)

    return 100*da, 100*all_scores, permutation_scores, list(pvalue[0])

def computeScore(svm, X, y, cv):
    score, permutation_scores, pvalue = permutation_test_score(svm, \
                                                               X, y, \
                                                               scoring='accuracy', \
                                                               cv=cv, \
                                                               n_permutations=100, \
                                                               n_jobs=1)
    print("Classification score %s (pvalue: %s)" % (score, pvalue))
    return score, permutation_scores, pvalue

def check_trop_score(X_data, trop_clusters):
    
    cv = Bootstrap(X_data.shape[0], n_iter=3, train_size=0.7)
    pred = KMeans(n_clusters=len(set(trop_clusters)))
    t_score, scores, pval = permutation_test_score(pred, X_data, 
                                                   n_permutations=100,
                                                   y = trop_clusters,
                                                   n_jobs=20,
                                                   scoring=rand_linker,
                                                   cv=cv)
    return t_score, scores, pval

    def test_permutation_test_score(self):
        import sklearn.svm as svm
        iris = datasets.load_iris()

        df = pdml.ModelFrame(iris)
        clf = svm.SVC(kernel=str('linear'), C=1)
        result = df.cross_validation.permutation_test_score(clf, cv=5)
        expected = cv.permutation_test_score(clf, iris.data, y=iris.target, cv=5)

        self.assertEqual(len(result), 3)
        self.assertEqual(result[0], expected[0])
        self.assert_numpy_array_almost_equal(result[1], expected[1])
        self.assertEqual(result[2], expected[2])

def test_permutation_score():
    iris = load_iris()
    X = iris.data
    X_sparse = coo_matrix(X)
    y = iris.target
    svm = SVC(kernel='linear')
    cv = cval.StratifiedKFold(y, 2)

    score, scores, pvalue = cval.permutation_test_score(
        svm, X, y, zero_one_score, cv)

    assert_greater(score, 0.9)
    np.testing.assert_almost_equal(pvalue, 0.0, 1)

    score_label, _, pvalue_label = cval.permutation_test_score(
        svm, X, y, zero_one_score, cv, labels=np.ones(y.size), random_state=0)

    assert_true(score_label == score)
    assert_true(pvalue_label == pvalue)

    # check that we obtain the same results with a sparse representation
    svm_sparse = SVC(kernel='linear')
    cv_sparse = cval.StratifiedKFold(y, 2, indices=True)
    score_label, _, pvalue_label = cval.permutation_test_score(
        svm_sparse, X_sparse, y, zero_one_score, cv_sparse,
        labels=np.ones(y.size), random_state=0)

    assert_true(score_label == score)
    assert_true(pvalue_label == pvalue)

    # set random y
    y = np.mod(np.arange(len(y)), 3)

    score, scores, pvalue = cval.permutation_test_score(svm, X, y,
            zero_one_score, cv)

    assert_less(score, 0.5)
    assert_greater(pvalue, 0.4)

def permutation():
    file = 'data/n228_bcdefgh.mat'
    dat = data.load(file)
    X, y = data.build(dat, range(0, 96), 'fr1', 17)

    # Univariate Feature Selection
    select = SelectKBest(f_classif,k=27).fit(X,y)
    Xa = select.transform(X)

    # Select good cell with heuristic
    channel = data.goodCell(dat)
    Xb, y = data.build(dat, channel, 'fr1', 17)

    # PCA Dimentionnality Reduction
    pca = PCA(n_components=38)
    Xc = pca.fit_transform(X)


    dat = [X, Xa, Xb, X, Xc,Xa]
    pNB = PoissonNB()
    gNB = GaussianNB()
    classifiers = [pNB,pNB,pNB,gNB,gNB,gNB]
    label = ['Poisson Unreduced', 'Poisson Univariate Reduction', 'Poisson Heuristic Reduction', 'Gaussion No reduction', 'Gaussian PCA reduction', 'Gaussian Univariate Reduction']
    scores = []
    perm_scores = []
    p_value = []

    for i in range(0,len(dat)):
        score, permutation_score, pvalue = permutation_test_score(classifiers[i], dat[i], y, cv=StratifiedKFold(y, n_folds=3, shuffle=True, random_state=42),n_permutations=100, n_jobs=-1, random_state=42, scoring=make_scorer(error_distance, greater_is_better=False))
        scores.append(score)
        perm_scores.append(np.mean(permutation_score))
        p_value.append(pvalue)

    ind = np.arange(len(scores))
    plt.bar(ind, scores)
#    ax.set_xticks(ind)
#    ax.set_xticklabels(label)
    plt.plot(ind, perm_scores)


    plt.show()


    print "Average Distance between real location and predicted location"
    print score
    print "Chance Performance, from permutation"
    print np.mean(permutation_score)
    print "p-value"
    print pvalue

def handle_bayes():
    input_data = pd.read_csv('/home/vasiliy/Study/StadiumProject/Classifier/signs.csv', sep=';')
    signs = ['ms_avg_sadness', 'ms_avg_sadness', 'ms_disgust', 'ms_contempt', 'ms_max_sadness', 'ms_median_surprise',
             'ms_avg_happiness']
    signs = ['ms_avg_sadness', 'ms_avg_sadness', 'ms_disgust', 'ms_contempt', 'ms_max_sadness', 'ms_median_surprise']
    signs = ['ms_median_sadness', 'likes', 'ms_min_anger', 'ms_min_disgust', 'ms_min_fear', 'ms_avg_anger' ]
    X = input_data[signs]
    X = X.as_matrix()
    Y = np.array(input_data['fight'].get_values())
    classifier = GaussianNB()
    kf = KFold(len(signs), n_folds=6, shuffle=True)
    for training, testing in kf:
        classifier.fit(X[training], Y[training])
    score, permutation_scores, pvalue = permutation_test_score(
    classifier, X, Y, scoring="accuracy", cv=kf, n_permutations=len(Y), n_jobs=1)
    print ("Classification score %s (pvalue : %s)" % (score, pvalue))

def test_permutation_score():
    iris = load_iris()
    X = iris.data
    X_sparse = coo_matrix(X)
    y = iris.target
    svm = SVC(kernel='linear')
    cv = cval.StratifiedKFold(y, 2)

    score, scores, pvalue = cval.permutation_test_score(
        svm, X, y, cv=cv, scoring="accuracy")
    assert_greater(score, 0.9)
    assert_almost_equal(pvalue, 0.0, 1)

    score_label, _, pvalue_label = cval.permutation_test_score(
        svm, X, y, cv=cv, scoring="accuracy", labels=np.ones(y.size),
        random_state=0)
    assert_true(score_label == score)
    assert_true(pvalue_label == pvalue)

    # test with custom scoring object
    scorer = make_scorer(fbeta_score, beta=2)
    score_label, _, pvalue_label = cval.permutation_test_score(
        svm, X, y, scoring=scorer, cv=cv, labels=np.ones(y.size),
        random_state=0)
    assert_almost_equal(score_label, .97, 2)
    assert_almost_equal(pvalue_label, 0.01, 3)

    # check that we obtain the same results with a sparse representation
    svm_sparse = SVC(kernel='linear')
    cv_sparse = cval.StratifiedKFold(y, 2)
    score_label, _, pvalue_label = cval.permutation_test_score(
        svm_sparse, X_sparse, y, cv=cv_sparse,
        scoring="accuracy", labels=np.ones(y.size), random_state=0)

    assert_true(score_label == score)
    assert_true(pvalue_label == pvalue)

    # set random y
    y = np.mod(np.arange(len(y)), 3)

    score, scores, pvalue = cval.permutation_test_score(svm, X, y, cv=cv,
                                                        scoring="accuracy")

    assert_less(score, 0.5)
    assert_greater(pvalue, 0.2)

    # test with deprecated interface
    with warnings.catch_warnings(record=True):
        score, scores, pvalue = cval.permutation_test_score(
            svm, X, y, score_func=accuracy_score, cv=cv)
    assert_less(score, 0.5)
    assert_greater(pvalue, 0.2)

def bayes_classification(permutation, test):
    input_data = pd.read_csv('/home/vasiliy/Study/StadiumProject/Classifier/signs.csv', sep=';')
    output_data = []
    Y = np.array(input_data['fight'].get_values())
    if permutation == True:
        np.random.shuffle(Y)
    input_data = input_data.drop(['match', 'city', 'date', 'fight'], 1)
    data_array = input_data.as_matrix()
    for features_number in range(3,30,1):
        X_new = SelectKBest(f_classif, k=features_number).fit_transform(data_array, Y)
        classifier = GaussianNB()
        kf = KFold(len(X_new), n_folds=6, shuffle=True)
        means = []
        for training, testing in kf:
            classifier.fit(X_new[training], Y[training])
            prediction = classifier.predict(X_new[testing])
            curmean = np.mean(classifier.score(X_new[testing], Y[testing]))
            means.append(curmean)
        output_data.append(np.mean(means))
        score, permutation_scores, pvalue = permutation_test_score(
    classifier, X_new, Y, scoring="accuracy", cv=kf, n_permutations=len(Y), n_jobs=1)
        if test:
            print ("Classification score %s (pvalue : %s)" % (score, pvalue))
    return output_data

#print "The different cross_scores: ", cross_score_LDA
   

#### Naive bayes ####

from sklearn.naive_bayes import GaussianNB
ngb = GaussianNB()

cross_score_NB = cross_val_score(ngb, X_scl, y, scoring="accuracy", cv = loo, 
                    n_jobs = 8, verbose = True)
                    
print "Cross val score: ", cross_score_NB.mean() 
print "The different cross_scores: ", cross_score_NB

score_NB, permutation_score_NB, pvalue_NB = permutation_test_score(ngb, X_scl, y,
        scoring="accuracy", cv = cv, n_permutations = 2000, 
        n_jobs = n_jobs, verbose = True)
print 'Classification score:', score_NB, 'p-value:', pvalue_NB

#### SVM ####
from sklearn.svm import LinearSVC
svc = LinearSVC()

cross_score_SVM = cross_val_score(svc, X_scl, y, scoring="accuracy", cv = loo, 
                    n_jobs = 8, verbose = True)
                    
print "Cross val score: ", cross_score_SVM.mean() 
print "The different cross_scores: ", cross_score_SVM


score_SVM, permutation_score_SVM, pvalue_SVM = permutation_test_score(svc, X, y,

#.........这里部分代码省略.........
                (n, subjid, thresh_dens)
            el = nx.read_edgelist(os.path.join(directory, g_name),
                                  nodetype=int)
            g = nx.Graph()
            # there are 148 regions, or nodes
            g.add_nodes_from(range(148))
            g.add_edges_from(el.edges())
            graphs.append(g)
            subjects.append(subj_name)
            classes.append(subjid)
            vects.append(complex_network_mapping(graphs[-1]))
            print "Graph built for subject %s and class %s." % \
                (subj_name, subjid)

    # Reordering data for the leave-one-subject-out cross-validation
    nm_graphs = [None] * len(graphs)
    nm_classes = [None] * len(classes)
    nm_subjects = [None] * len(subjects)
    nm_vects = [None] * len(vects)

    for i in range(len(graphs) / 2):
        nm_graphs[i*2] = graphs[i]
        nm_graphs[i*2 + 1] = graphs[(len(graphs) / 2) + i]
        nm_classes[i*2] = classes[i]
        nm_classes[i*2 + 1] = classes[(len(classes) / 2) + i]
        nm_subjects[i*2] = subjects[i]
        nm_subjects[i*2 + 1] = subjects[(len(subjects) / 2) + i]
        nm_vects[i*2] = vects[i]
        nm_vects[i*2 + 1] = vects[(len(vects) / 2) + i]

    print nm_subjects
    print nm_classes

    nm_vects = np.array(nm_vects)
#    nm_vects = np.where(nm_vects == inf, 10, nm_vects)
#    nm_vects = np.where(nm_vects == nan, 10, nm_vects)

    ss = StandardScaler()
    X = ss.fit_transform(nm_vects)
    print X
    print np.mean(X)
    print np.max(X)

    tuned_parameters = [{'kernel': ['rbf'], 'gamma': [1e-3, 1e-4],
                     'C': [1, 10, 100, 1000]},
                    {'kernel': ['linear'], 'C': [1, 10, 100, 1000]}]

    tuned_parameters2 = [{'kernel': ['rbf'], 'gamma': [1e-3, 1e-4],
                     'C': [1, 10, 100, 1000]},
                     {'kernel': ['sigmoid'], 'gamma': [1e-3, 1e-4],
                     'C': [1, 10, 100, 1000]},
                     {'kernel': ['linear'], 'C': [1, 10, 100, 1000]}]


    from sklearn.grid_search import GridSearchCV
    clf = GridSearchCV(SVC(C=1), tuned_parameters, cv=KFold(len(nm_classes), niter, shuffle=False))
    clf2 = GridSearchCV(SVC(C=1), tuned_parameters2, cv=KFold(len(nm_classes), niter, shuffle=False))
    clf.fit(X, np.array(nm_classes))
    clf.best_params_
    clf = SVC(C=100, kernel='linear')
    print "Now getting cross validation "
    cvr = SVC(C=1000, gamma=.001, kernel='rbf')
    cv_scores = cross_val_score(cvr, X, np.array(nm_classes),
                                cv=KFold(len(nm_classes),
                                         niter, shuffle=False))

    cv_scores = cross_val_score(cvr, X, np.array(nm_classes),
                                cv=KFold(len(nm_classes),
                                         niter, shuffle=False))


    cv_scores = cross_val_score(clf, X, np.array(nm_classes),
                                cv=KFold(len(nm_classes),
                                         niter, shuffle=False))


    from sklearn.linear_model import SGDClassifier
    clfGD = SGDClassifier(loss='log')
    clfGD.fit(X, np.array(nm_classes))
    cv_scores = cross_val_score(clfGD, X, np.array(nm_classes),
                                cv=KFold(len(nm_classes),
                                         niter, shuffle=False))
    print cv_scores
    print np.mean(cv_scores)
    print("Accuracy: %0.2f (+/- %0.2f)" %
        (cv_scores.mean(), cv_scores.std() * 2))

     from sklearn.dummy import DummyClassifier
     null_scores = cross_val_score(DummyClassifier(), X, np.array(nm_classes),
                                   cv=KFold(len(nm_classes),
                                            niter, shuffle=False))

     print null_scores.mean()

     from sklearn.cross_validation import permutation_test_score
     null_scores_perm = permutation_test_score(cvr, X, np.array(nm_classes),
                                               cv=KFold(len(nm_classes),
                                                        niter, shuffle=False))

     print null_scores_perm.mean()

    clf = RandomForestClassifier(n_estimators=500, max_features=None)
elif args.clf == "GradientBoostingClassifier":   
    clf = GradientBoostingClassifier(
            n_estimators=100, learning_rate=1.0, 
            max_depth=1, random_state=prng
            )
else:
    raise ValueError("--clf not understood")

# Go
acc, perm, p = permutation_test_score(
        clf, X, y, 
        score_func=None, 
        cv=cv, 
        n_permutations=args.null, 
        n_jobs=5, 
        labels=None, 
        random_state=prng, 
        verbose=0, 
        scoring="accuracy"
        )

# Save
f = open(args.o[0], "a")
f.write("{0},{1},{2},{3}\n".format(    
        np.round(acc, decimals=3), 
        np.round(np.mean(perm), decimals=3),
        np.round(p, decimals=4),
        args.name)
        )
f.close()

for k, band in enumerate(bands.keys()):
    data_cls = []
    for j in range(len(cls_all)):
        tmp = cls_all[j][band]
        data_cls.append(
            np.asarray([bct.strengths_und(g) for g in tmp]).mean(axis=0))
    data_pln = []
    for j in range(len(pln_all)):
        tmp = pln_all[j][band]
        data_pln.append(
            np.asarray([bct.strengths_und(g) for g in tmp]).mean(axis=0))
    data_cls = np.asarray(data_cls)
    data_pln = np.asarray(data_pln)

    X = np.vstack([data_cls, data_pln])
    y = np.concatenate([np.zeros(len(data_cls)), np.ones(len(data_pln))])

    cv = StratifiedKFold(y, n_folds=6, shuffle=True)

    model = joblib.load(source_folder +
                        "graph_data/sk_models/eigen_ada_pln_%s.plk" % band)

    score, perm_scores, pval = permutation_test_score(
        model, X, y, cv=cv, n_permutations=5000, n_jobs=1)

    result = {"score": score, "perm_scores": perm_scores, "pval": pval}
    results_all[band] = result

np.save(source_folder + "graph_data/perm_test_eigen_pln.npy", results_all)

X = iris.data
y = iris.target
n_classes = np.unique(y).size

# Some noisy data not correlated
random = np.random.RandomState(seed=0)
E = random.normal(size=(len(X), 2200))

# Add noisy data to the informative features for make the task harder
X = np.c_[X, E]

svm = SVC(kernel="linear")
cv = StratifiedKFold(y, 2)

score, permutation_scores, pvalue = permutation_test_score(
    svm, X, y, scoring="accuracy", cv=cv, n_permutations=100, n_jobs=1
)

print("Classification score %s (pvalue : %s)" % (score, pvalue))

###############################################################################
# View histogram of permutation scores
pl.hist(permutation_scores, 20, label="Permutation scores")
ylim = pl.ylim()
# BUG: vlines(..., linestyle='--') fails on older versions of matplotlib
# pl.vlines(score, ylim[0], ylim[1], linestyle='--',
#          color='g', linewidth=3, label='Classification Score'
#          ' (pvalue %s)' % pvalue)
# pl.vlines(1.0 / n_classes, ylim[0], ylim[1], linestyle='--',
#          color='k', linewidth=3, label='Luck')
pl.plot(2 * [score], ylim, "--g", linewidth=3, label="Classification Score" " (pvalue %s)" % pvalue)

def do_session(
    ds,
    clf=SVC(kernel="linear", probability=True),
    scoring=score,
    targets="quantized_distance",
    n_jobs=1,
    learning_curve=False,
    permutation_test=False,
):

    ds.sa["chunks"] = ["{}:{}".format(sid, scan) for sid, scan in zip(ds.sa["session_id"], ds.sa["run"])]

    ds.sa["targets"] = ds.sa[targets]

    # fixme: do wiener filter here

    from mvpa2.mappers.detrend import PolyDetrendMapper

    detrender = PolyDetrendMapper(polyord=1, chunks_attr="chunks")

    ds = ds.get_mapped(detrender)

    ds = ds[numpy.logical_not(numpy.logical_or(ds.sa.move, ds.sa.cue)), :]

    if ds.nfeatures > 3000:
        fs = SelectKBest(k=3000)
        fs.fit(ds.samples, ds.sa.search > 0)

    ds = ds[ds.sa.search > 0, :]

    if ds.nfeatures > 3000:
        ds = ds[:, fs.get_support()]

    logger.info("Configuring cross validation")
    cv = StratifiedKFold(ds.sa.quantized_distance, n_folds=6)  # FIXME: make this a function parameter

    logger.info("Beginning cross validation")
    scores = cross_val(clf, ds.samples, ds.targets, cv, scoring)

    if learning_curve:
        from sklearn.learning_curve import learning_curve

        logger.info("Beginning learning curve analysis")

        train_sizes_abs, train_scores, test_scores = learning_curve(
            clf, ds.samples, ds.targets, n_jobs=n_jobs, verbose=50, scoring="accuracy"
        )

    if permutation_test:
        logger.info("Beginning permutation test")
        score,
        permutation_scores,
        pvalue = permutation_test_score(
            clf, ds.samples, ds.targets, cv=cv, n_jobs=n_jobs, verbose=50, scoring="accuracy"
        )

    result = {}
    result["datetime"] = datetime.datetime.now()
    if ds.nfeatures > 3000:
        result["fs"] = fs
    result["mapper"] = ds.mapper
    # result['clf'] = clf
    # result['cv'] = cv
    # result['scoring'] = scoring
    result["scores"] = scores
    if learning_curve:
        result["learning_curve"] = (train_sizes_abs, train_scores, test_scores)
    else:
        result["learning_curve"] = None

    if permutation_test:
        result["pvalue"] = pvalue
    else:
        result["pvalue"] = None

    return result

    data_cls = []
    for j in range(len(cls_all)):
        tmp = cls_all[j][band]
        data_cls.append(
            np.asarray([bct.strengths_und(g) for g in tmp]).mean(axis=0))
    data_pln = []
    for j in range(len(pln_all)):
        tmp = pln_all[j][band]
        data_pln.append(
            np.asarray([bct.strengths_und(g) for g in tmp]).mean(axis=0))
    data_cls = np.asarray(data_cls)
    data_pln = np.asarray(data_pln)

    X = np.vstack([data_cls, data_pln])
    y = np.concatenate([np.zeros(len(data_cls)), np.ones(len(data_pln))])

    cv = StratifiedKFold(y, n_folds=6, shuffle=True)

    model = joblib.load(source_folder +
                        "graph_data/sk_models/path-strength_ada_%s_pln.plk" %
                        band)

    score, perm_scores, pval = permutation_test_score(
        model, X, y, cv=cv, n_permutations=10000, n_jobs=1, verbose=2)

    result = {"score": score, "perm_scores": perm_scores, "pval": pval}
    results_all[band] = result

np.save(source_folder + "graph_data/perm_test_path-strength_pln.npy",
        results_all)