def getClfScore(classifier, features, labels, cv):
    '''Evaluating performance of estimator

    param:
        classifier : classifiers list
        features   : data to fit
        labels     : samples data
        cv         : cross validation iterator
    return:
        test_score : dict of classification score
    '''
    
    test_score = {}

    for idx, clfname in enumerate(sorted(classifier.keys())):
        clf_score = {}
        clf = classifier[clfname]
        precision = cross_val_score(clf, features, labels, 'precision', cv)
        recall    = cross_val_score(clf, features, labels, 'recall', cv)
        
        clf_score['precision'] = np.mean(precision)
        clf_score['recall']    = np.mean(recall)
        
        test_score[clfname] = clf_score
    return test_score

def test_cross_val_score_fit_params():
    clf = MockClassifier()
    n_samples = X.shape[0]
    n_classes = len(np.unique(y))
    fit_params = {'sample_weight': np.ones(n_samples),
                  'class_prior': np.ones(n_classes) / n_classes}
    cval.cross_val_score(clf, X, y, fit_params=fit_params)

    def cv(self, parameters, scoring="roc_auc"):
        """ Evaluate score by cross validation. """

        X = self.data.values.astype(np.float)
        y = self.label.values

        print cross_val_score(self.estimator, X, y, scoring=scoring, cv=3)

def importData(datadirectory):
	#categories = ['n','u', 'y']
	categories = ['n', 'y']

	data = load_files(datadirectory,categories=categories, shuffle=True, random_state=42, encoding='latin-1') 
	X_train, X_test, y_train, y_test = cross_validation.train_test_split(data.data, data.target, test_size = 0.4, random_state=0)
	print X_train 
	# count_vect = CountVectorizer()
	# X_train_vec = count_vect.fit_transform(X_train)
	# X_test_vec = count_vect.fit_transform(X_test)
	# clf = svm.SVC(kernel='linear', C=1).fit(X_train_vec, y_train)
	# clf.score(X_test_vec, y_test) 

	text_clf = Pipeline([('vect', TfidfVectorizer()), ('clf', MultinomialNB())])
	#print text_clf.named_steps['clf']
	print str(sum(cross_val_score(text_clf, data.data,data.target ))/3.0) + ' Tfidf NB'
	#array([ 0.62376238,  0.57      ,  0.6122449 ])
	text_clf = Pipeline([('vect', CountVectorizer()),('clf', MultinomialNB()),]) 
	print str(sum(cross_val_score(text_clf, data.data,data.target ))/3.0) + ' CountVec NB'                                         #array([ 0.56435644,  0.5       ,  0.57142857])
	clf = Pipeline([('vect', CountVectorizer()), ('svm', LinearSVC())])                        
	print str(sum(cross_val_score(clf, data.data,data.target ))/3.0) + ' CountVec SVM'
	#array([ 0.55445545,  0.48      ,  0.54081633])
	clf = Pipeline([('vect', TfidfVectorizer()), ('svm', LinearSVC())])                    
	print str(sum(cross_val_score(clf, data.data,data.target ))/3.0) + ' Tfidf SVM'
	#array([ 0.62376238,  0.57      ,  0.6122449 ])
	clf_sgdc = Pipeline([('vect', CountVectorizer()),('clf', linear_model.SGDClassifier()),])
	print str(sum(cross_val_score(clf_sgdc, data.data,data.target ))/3.0) + ' SGDC' 

def crossValidation():
    data2010, labels2010 = read_tac('2010')

    #classifiers
    gnb = naive_bayes.GaussianNB()
    Svm = svm.SVC(kernel = "linear")
    logReg = linear_model.LogisticRegression()

    GNBscores = cross_validation.cross_val_score(gnb, data2010, labels2010, cv=2)
    SVMscores = cross_validation.cross_val_score(Svm, data2010, labels2010, cv=2)
    logRegscores = cross_validation.cross_val_score(logReg, data2010, labels2010, cv=2)

    print "Results:"
    print "Gaussian Naive Bayes: " 
    print str(GNBscores.mean())
    print "Support Vector Machine: " 
    print str(SVMscores.mean())
    print "Logistic Regression: " 
    print str(logRegscores.mean())

    fh.write("Results:" + "\n")
    fh.write("Gaussian Naive Bayes: "  + "\n")
    fh.write(str(GNBscores.mean()) + "\n")
    fh.write("Support Vector Machine: "  + "\n")
    fh.write(str(SVMscores.mean()) + "\n")
    fh.write("Logistic Regression: "  + "\n")
    fh.write(str(logRegscores.mean()) + "\n")
    fh.write("-------------------------------------------------\n")
    fh.write("\n\n")

def dofitSVMstd(X_train, Y_train, X_test):
    shape = X_train.shape
    b = []
    for j in range(shape[0]):
        a1 = [np.std(X_train[j, :, i]) for i in range(shape[2])]
        a2 = [getEntropy(list(X_train[j, :, i].astype(int))) for i in range(shape[2])]
        a1.sort(reverse=True)
        a2.sort()
        b.append(a1[0:16] + a2[0:16])

    x1 = np.array(b)
    clf = RandomForestClassifier()
    dummy = clf.fit(x1, Y_train)
    scores = cross_validation.cross_val_score(clf, x1, Y_train)
    p1 = clf.predict(x1)
    shape = X_test.shape
    b = []
    for j in range(shape[0]):
        a1 = [np.std(X_test[j, :, i]) for i in range(shape[2])]
        a2 = [getEntropy(list(X_test[j, :, i].astype(int))) for i in range(shape[2])]
        a1.sort(reverse=True)
        a2.sort()
        b.append(a1[0:16] + a2[0:16])

    x2 = np.array(b)
    y2 = clf.predict(x2)
    xx = np.concatenate((x1, x2))
    yy = np.concatenate((Y_train, y2))
    dummy = clf.fit(xx, yy)
    p2 = clf.predict(x2)
    scores = cross_validation.cross_val_score(clf, x1, Y_train)
    # sum(clf.predict(x2))
    return [scores, np.concatenate((p1, p2))]

def test_cross_val_score_filter_feature_selection_threshold():

    threshold = 1.0
    scikit_data,scikit_target = dfm.get_expression_scikit_data_target(expression_file, ic50_file,normalized=True,trimmed=True,threshold=None)
    model = classify.construct_svc_model(kernel='linear')
    non_thresholded_test_1 = cv.cross_val_score_filter_feature_selection(model,cv.trim_X_threshold,threshold,scikit_data,scikit_target,cv=5)

    m = classify.construct_svc_model(kernel='linear')
    s_data,s_target = dfm.get_expression_scikit_data_target(expression_file, ic50_file,normalized=True,trimmed=True,threshold=threshold)
    non_thresholded_test_2 = cross_val_score(m,s_data,s_target,cv=5)

    threshold = .05
    scikit_data,scikit_target = dfm.get_expression_scikit_data_target(expression_file, ic50_file,normalized=True,trimmed=True,threshold=None)
    model = classify.construct_svc_model(kernel='linear')
    thresholded_test_1 = cv.cross_val_score_filter_feature_selection(model,cv.trim_X_threshold,threshold,scikit_data,scikit_target,cv=5)

    m = classify.construct_svc_model(kernel='linear')
    s_data,s_target = dfm.get_expression_scikit_data_target(expression_file, ic50_file,normalized=True,trimmed=True,threshold=threshold)
    thresholded_test_2 = cross_val_score(m,s_data,s_target,cv=5)


    #The non-thresholded tests should be the same because if we are not thresholding, it doesn't matter where we perform thresholding
    assert(math.fabs(non_thresholded_test_1.mean() - non_thresholded_test_2.mean()) < .001)

    #The first non_thresholded test should have lower accuracy because we are doing thresholding within the cross-validation,
    #which will reduce cross-validation overfitting and as a consequence reported cross-validation accuracy.
    assert(thresholded_test_1.mean() - thresholded_test_2.mean() < 0)

def validate_model(model, features, labels):
    accuracy = cross_val_score(model, features, labels, scoring='accuracy', cv=4).mean()
    precision = cross_val_score(model, features, labels, scoring='precision', cv=4).mean()
    recall = cross_val_score(model, features, labels, scoring='recall', cv=4).mean()
    f1 = cross_val_score(model, features, labels, scoring='f1', cv=4).mean()
    print "\n(METRICS) Accuracy: {:.3f}   Precision: {:.3f}   Recall: {:.3f}   F1-Score: {:.3f}".\
        format(accuracy,precision, recall, f1)

def experiment_zero(data,company):
	print '___Experiment One___'
	# Experiment Parameters
	finance_datatype = 0    # finance_datatype: Integer 2 = Stock price change, 1 = Percentage stock price change, 0 = Only direction
	finance_n = 2           # finance_n: Integer >=0 Number of days of finance data to include
	sentiment_datatype = 1	# sentiment_datatype: Boolean 1 = all sentiment featues, 0 = Total
	sentiment_n = 1 		# sentiment_n: Integer >=0 Number of days of sentiment data to include
	day = 0                 # day: Boolean 1 = Include day of the week, 0 = do not
	target = 0				# target: Boolean 1 = Amount, 0 = Direction
	volume = 0 				# volume: boolean 1 = Yes, 0 = No
	if (finance_n + sentiment_n + day + volume) == 0:
		print 'Insufficient parameters set'
		return 

	# Data Processing
	feature_vector_meaning(company, finance_datatype, finance_n, sentiment_datatype, sentiment_n, day, target, volume)
	matrix = create_feature_matrix(company, data, finance_datatype, finance_n, sentiment_datatype, sentiment_n, day, target, volume)
	end = len(matrix[0])
	train_x = matrix[:,0:end-1]
	train_y = matrix[:,end-1]

	# Classifier training
	scaler = preprocessing.StandardScaler().fit(train_x)
	train_x = scaler.transform(train_x)

	clf = direction_classifier(train_x,train_y)
	cv = cross_validation.ShuffleSplit(len(train_x), n_iter=5, test_size=0.2, random_state=0)
	print ' _ _ _Evaluation_ _ _'
	if target == 0:
		scores = cross_validation.cross_val_score(clf, train_x, train_y, cv=cv, scoring='accuracy')
		print("  Accuracy: %0.2f (+/- %0.2f)" % (scores.mean(), scores.std() * 2))
	elif target == 1:
		scores = cross_validation.cross_val_score(clf, train_x, train_y, cv=cv, scoring='mean_squared_error')
		print("  MSE: %0.2f (+/- %0.2f)" % (scores.mean(), scores.std() * 2))		
	print '====================='

def run_model(data):
    """Do some label bucketing, print model output."""
    features = data.ix[:, :-1]

    # more categories <--> less accuracy
    # labels = data.ix[:, -1].map(lambda k: 1 if k > 10 else 0)
    labels = data.ix[:, -1].map(lambda k: int(k / 5))     # bucketing trick
    print 'num classes = {}\n'.format(len(set(labels)))

    # weak (base) classifier
    print 'fitting weak classifier...'
    weak_clf = DecisionTreeClassifier(max_depth=MAX_DEPTH)

    weak_cv_results = cross_val_score(weak_clf, features, labels,
        cv=N_FOLDS)
    print 'weak_cv_results = {}'.format(weak_cv_results)
    print 'avg accuracy = {}\n'.format(weak_cv_results.mean())
    
    # strong (ensemble) classifier
    print 'fitting strong classifier...'
    strong_clf = RandomForestClassifier(
        max_depth=MAX_DEPTH,
        n_estimators=N_TREES,
        n_jobs=N_JOBS)

    strong_cv_results = cross_val_score(strong_clf, features, labels,
        cv=N_FOLDS)
    print 'strong_cv_results = {}'.format(strong_cv_results)
    print 'avg accuracy = {}'.format(strong_cv_results.mean())

def coeff_of_deterimination(classifier, X, y, K=10):
    # Perform a cross-validation estimate of the coefficient of determination using
    # the cross_validation module using all CPUs available on the machine
    R21 = cross_val_score(classifier, X, y=y, n_jobs=1).mean()
    R2 = cross_val_score(classifier, X, y=y, cv=KFold(y.size, K), n_jobs=1).mean()
    print "The %d-Folds est coeff. of determ. R2 = %s" % (K, R2)
    print "basic cross val ", R21

def run_conventional_linkage(x, y, n_comps, linker_model, verbose=0, k_folds=3):
  print "---->Cross validating"
  cvs = cross_val_score(linker_model, x, y, cv=k_folds, scoring='r2', verbose=verbose)
  mse = cross_val_score(linker_model, x, y, cv=k_folds, scoring='mean_squared_error', verbose=verbose)
  print '---->R2: ', np.mean(cvs)
  print '---->MSE: ', np.mean(mse)
  return np.mean(cvs), np.std(cvs), np.mean(mse), np.std(mse)

def _run_classifier(X, Y, parent, child, max_depth):
    X_train, X_test, y_train, y_test = train_test_split(X, Y, test_size=0.333, random_state=0)
    clf = tree.DecisionTreeClassifier(min_samples_split=parent, min_samples_leaf=child, max_depth=max_depth)
    clf = clf.fit(X_train, y_train)

    print 'model score on train data data:'
    print clf.score(X_train, y_train)
    print 'ten fold cross-validation results on train data:'
    scores = cross_val_score(clf, X_train, y_train, cv=10)
    print("Accuracy: %0.2f (+/- %0.2f)" % (scores.mean(), scores.std() * 2))

    print 'model score on test data'
    print clf.score(X_test, y_test)
    print 'ten fold cross-validation results on test data:'
    scores = cross_val_score(clf, X_test, y_test, cv=10)
    print("Accuracy: %0.2f (+/- %0.2f)" % (scores.mean(), scores.std() * 2))

    print 'Gini Importance'
    print clf.feature_importances_

    'Classification Report'
    y_true, y_pred = y_test, clf.predict(X_test)
    print(classification_report(y_true, y_pred))

    'Confusion Matrix'
    print(confusion_matrix(y_true, y_pred))
    cm = confusion_matrix(y_true, y_pred)
    print _calc_error_rate_conf_int(cm)
    return _calc_error_rate_conf_int(cm) + [len(y_test)]

 def fit_from_prep(self, infile):
     H, y, w = self._da.load_from_file(infile)
     self._vq = VQ(w, hist=w.shape[0])
     self._cl.fit(H, y)
     if self._verbose:
         print cross_validation.cross_val_score(
             self._cl, H, y, cv=3).mean()

def analytics():
    trainer_data = get_thing_from_file("training_dataset.txt")
    tester_data = get_thing_from_file("test_dataset.txt")

    bayes_clf = get_thing_from_file("bayes_model.txt")
    svm_clf = get_thing_from_file("svm_model.txt")

    # we load the fitted models from file so we don't need these lines
    # bayes_clf.fit(trainer_data.data, trainer_data.target)
    # svm_clf.fit(trainer_data.data, trainer_data.target)

    test = tester_data.data

    predicted_bayes = bayes_clf.predict(test)
    predicted_svm = svm_clf.predict(test)

    print "** ACCURACIES **"
    print numpy.mean(predicted_bayes == tester_data.target)
    print numpy.mean(predicted_svm == tester_data.target)

    print "** K-FOLD VALIDATION ACCURACY"

    bayes_scores = cross_validation.cross_val_score(bayes_clf,
                                    tester_data.data, tester_data.target, cv=10)

    svm_scores = cross_validation.cross_val_score(svm_clf, tester_data.data,
                                              tester_data.target, cv=10)

    print max(bayes_scores)
    print max(svm_scores)
    print "**"

    def lda_run(self, k_folds = 5):
        self.r_forest_lda = RandomForestClassifier(n_estimators=2000,n_jobs=5, max_depth=None, min_samples_split=1, random_state =0)
        self.lda_scores = cross_validation.cross_val_score(self.r_forest_lda, self.lda_iss_features, self.labels, cv=k_folds,n_jobs=5)
        print("Cross validation Random Forest performance LDA: Accuracy: %0.2f (std %0.2f)" % (self.lda_scores.mean()*100, self.lda_scores.std()*100))
        self.r_forest_lda.fit(self.lda_iss_features,self.labels)
        print self.r_forest_lda.score(self.lda_iss_validation_features, self.validation_labels)*100, 'LDA test-set performance \n'

        '''
        C_range = np.logspace(-2, 10, 13)
        gamma_range = np.logspace(-9, 3, 13)
        param_grid = dict(gamma=gamma_range, C=C_range)
        cv = StratifiedShuffleSplit(self.labels, n_iter=5, test_size=0.2, random_state=42)
        grid = GridSearchCV(SVC(), param_grid=param_grid, cv=cv)
        grid.fit(self.lda_iss_features, self.labels)
        print("The best parameters are %s with a score of %0.2f"% (grid.best_params_, grid.best_score_))
        '''

        self.svc_lda = SVC(kernel='rbf',C = 1,gamma = 'auto')
        self.svc_lda_scores = cross_validation.cross_val_score(self.svc_lda, self.lda_iss_features, self.labels, cv=k_folds,n_jobs=5)
        print("Cross validation SVM performance LDA: Accuracy: %0.2f (std %0.2f)" % (self.svc_lda_scores.mean()*100, self.svc_lda_scores.std()*100))
        self.svc_lda.fit(self.lda_iss_features,self.labels)
        print self.svc_lda.score(self.lda_iss_validation_features, self.validation_labels)*100, 'LDA test-set performance \n'

        y_true = self.validation_labels
        y_pred = self.svc_lda.predict(self.lda_iss_validation_features)
        target_names = ['S1','S2','S3','S4']
        t = classification_report(y_true, y_pred, target_names=target_names)
        print 'Support vector report lda'
        print t

def training(matrix, Y, SVM):
    """ def  training(matrix , Y , svm ):
			matrix: is the train data
			Y: is the labels in array
			svm: is a boolean. If svm == True we perform svm otherwise we perform AdaBoostClassifier

			return: cross_validation scores
	"""

    if SVM:
        classifier = svm.SVC()
    else:
        classifier = AdaBoostClassifier(n_estimators=300)

    precision_micro_scorer = metrics.make_scorer(custom_precision_micro_score)
    precision_macro_scorer = metrics.make_scorer(custom_precision_macro_score)
    recall_micro_scorer = metrics.make_scorer(custom_recall_micro_score)
    recall_macro_scorer = metrics.make_scorer(custom_recall_macro_score)

    precision_micro = cross_val_score(classifier, matrix, Y, cv=10, scoring=precision_micro_scorer)
    precision_macro = cross_val_score(classifier, matrix, Y, cv=10, scoring=precision_macro_scorer)
    recall_micro = cross_val_score(classifier, matrix, Y, cv=10, scoring=recall_micro_scorer)
    recall_macro = cross_val_score(classifier, matrix, Y, cv=10, scoring=recall_macro_scorer)

    return {"micro": (precision_micro, recall_micro), "macro": (precision_macro, recall_macro)}

    def randomforest_info(self, max_trees = 1000, step = 40, k_folds = 5):
        print 'Characterising R_forest. Looping through trees: ',
        self.treedata = np.zeros((max_trees/step, 10))
        for i,n_trees in enumerate(np.arange(0, max_trees,step)):
            if n_trees == 0:
                n_trees = 1
            print n_trees,
            r_forest = RandomForestClassifier(n_estimators=n_trees, n_jobs=5, max_depth=None, min_samples_split=1, random_state=0)
            scores = cross_validation.cross_val_score(r_forest, self.iss_features, self.labels, cv=k_folds,n_jobs=5)
            r_forest_full = RandomForestClassifier(n_estimators=n_trees, n_jobs=5, max_depth=None, min_samples_split=1, random_state=0)
            r_forest_full.fit(self.iss_features,self.labels)
            self.treedata[i,0] = n_trees
            self.treedata[i,1] = scores.mean()
            self.treedata[i,2] = scores.std()
            # now add the test dataset - score
            self.treedata[i,3] = r_forest_full.score(self.iss_validation_features, self.validation_labels)

            r_forest_lda = RandomForestClassifier(n_estimators=n_trees, n_jobs=5, max_depth=None, min_samples_split=1, random_state=0)
            r_forest_lda_full = RandomForestClassifier(n_estimators=n_trees, n_jobs=5, max_depth=None, min_samples_split=1, random_state=0)
            r_forest_lda_full.fit(self.lda_iss_features,self.labels)
            lda_scores = cross_validation.cross_val_score(r_forest_lda, self.lda_iss_features, self.labels, cv=k_folds,n_jobs=5)
            self.treedata[i,4] = lda_scores.mean()
            self.treedata[i,5] = lda_scores.std()
            self.treedata[i,6] = r_forest_lda_full.score(self.lda_iss_validation_features, self.validation_labels)
            print self.treedata[i,6]

            r_forest_pca = RandomForestClassifier(n_estimators=n_trees, n_jobs=5, max_depth=None, min_samples_split=1, random_state=0)
            r_forest_pca_full = RandomForestClassifier(n_estimators=n_trees, n_jobs=5, max_depth=None, min_samples_split=1, random_state=0)
            r_forest_pca_full.fit(self.pca_iss_features,self.labels)
            pca_scores = cross_validation.cross_val_score(r_forest_pca, self.pca_iss_features, self.labels, cv=k_folds,n_jobs=5)
            self.treedata[i,7] = pca_scores.mean()
            self.treedata[i,8] = pca_scores.std()
            self.treedata[i,9] = r_forest_pca_full.score(self.pca_iss_validation_features, self.validation_labels)

def svmByPackageMachineLearning(xList, yList):
	
	'''
	Example:
	
	iris = datasets.load_iris()
	clf = svm.SVC(kernel='linear', C=1)
	score = cross_validation.cross_val_score(clf, iris.data, iris.target, cv=10)
	print(score)
	print("Accuracy: %0.2f (+/- %0.2f)" % (score.mean(), score.std() * 2))
	'''
	
	#SVM with kernel
	clf_rbf = svm.SVC(decision_function_shape='ovo', C=10.0, kernel='rbf',  		gamma=5)
	clf_sig = svm.SVC(decision_function_shape='ovo', C=10.0, kernel='sigmoid',  	gamma=5, 		coef0=100.0)
	#clf_pol = svm.SVC(decision_function_shape='ovo', C=10.0, kernel='polynomial', 	gamma=5,		coef0=100.0, degree=4)
	clf_lin = svm.SVC(decision_function_shape='ovo', C=10.0, kernel='linear')

	
	#cross validation 
	score_rbf = cross_validation.cross_val_score(clf_rbf, xList, yList, cv=10)
	score_sig = cross_validation.cross_val_score(clf_sig, xList, yList, cv=10)
	#score_pol = cross_validation.cross_val_score(clf_pol, xList, yList, cv=10)
	score_lin = cross_validation.cross_val_score(clf_lin, xList, yList, cv=10)
	
	
	print("rbf: %0.2f (+/- %0.2f)" % (score_rbf.mean(), score_rbf.std() * 2))
	print("sig: %0.2f (+/- %0.2f)" % (score_sig.mean(), score_sig.std() * 2))
	#print("pol: %0.2f (+/- %0.2f)" % (score_pol.mean(), score_pol.std() * 2))
	print("lin: %0.2f (+/- %0.2f)" % (score_lin.mean(), score_lin.std() * 2))

def rf_cross_val(x,y):
    X_train, X_test, y_train, y_test = train_test_split(x,y,test_size = 0.33, random_state = 42)

    random_forest_grid = {'n_estimators': [100],
                                    'n_jobs': [-1]}

    rf_gridsearch = GridSearchCV(RandomForestRegressor(),
                                 random_forest_grid,
                                 n_jobs=-1,
                                 verbose=True,
                                 cv=3)

    rf_gridsearch.fit(X_train, y_train)

    print "best parameters:", rf_gridsearch.best_params_

    best_rf_model = rf_gridsearch.best_estimator_

    y_pred = best_rf_model.predict(X_test)

    print "Accuracy with best rf:", cross_val_score(best_rf_model, X_test, y_test).mean()

    rf = RandomForestRegressor(n_estimators=10, n_jobs = -1)

    print "Accuracy with default param rf:", cross_val_score(rf, X_test, y_test).mean()
    return best_rf_model