def compute_mse(model,x_train_current_tmp,YTrain,x_test_current_tmp,YTest, score ,values_TM = []):
    model.fit(x_train_current_tmp, YTrain)
    y_pred_train = model.predict(x_train_current_tmp)
    y_pred_test = model.predict(x_test_current_tmp)

    if len(values_TM)!=0:
        abs_error_train = 100.*mean_absolute_error(YTrain,y_pred_train)*len(YTrain)/(89.7* values_TM[0, 0] * values_TM[0,1])
        print("abs train", abs_error_train)

        abs_error_test = 100.*mean_absolute_error(YTest,y_pred_test)*len(YTest)/(89.7* values_TM[1, 0] * values_TM[1,1])
        print("abs test", abs_error_test)

        mse_error_train = 100.*np.sqrt(mean_squared_error(YTrain,y_pred_train)*len(YTrain)/(values_TM[0, 0] * values_TM[0, 1]))/(89.7)
        print("mean squared error train", mse_error_train )

        mse_error_test = 100.*np.sqrt(mean_squared_error(YTest,y_pred_test)*len(YTest)/(values_TM[1, 0] * values_TM[1, 1]))/(89.7)
        print("mean squared error test", mse_error_test )

    if score=="mean_squared_error":
        new_loss = mean_squared_error(YTest,y_pred_test)
    elif score== "mean_absolute_error":
        new_loss = mean_absolute_error(YTest,y_pred_test)
    else:
        new_loss = r2_score(YTest,y_pred_test)
    beta = model.coef_

    if x_train_current_tmp.shape[1]==1:
        beta = np.array([beta])
    beta = beta.reshape([len(beta),1])

    return new_loss, beta

def make_model(data,tc):

    train_data = data.sample(frac=.8)
    test_data = data.drop(train_data.index)
    train_y = train_data['T/Tc']
    train_X = train_data.drop(['T/Tc','temperature'], axis=1)
    test_y = test_data['T/Tc']
    test_X = test_data.drop(['T/Tc','temperature'], axis=1)

#    model = XGBClassifier(n_estimators = 1000,max_depth=8, learning_rate=0.05)
#    model.fit(train_X, train_y, early_stopping_rounds=10,
#                 eval_set=[(test_X, test_y)], verbose=True)
#    xgb.plot_tree(model)

    model = svm.SVC(kernel='rbf', gamma=1, C=1, verbose = True)
    model.fit(train_X, train_y)
    predictions = model.predict(test_X)
    print("Mean Absolute Error : " + str(mean_absolute_error(np.array(predictions), test_y)))

    train_y = train_data['temperature']/tc
    test_y = test_data['temperature']/tc

#    model2 = XGBRegressor(n_estimators = 1000,max_depth=8, learning_rate=0.05)
#    model2.fit(train_X, train_y, early_stopping_rounds=10,eval_metric='mae',
#                 eval_set=[(test_X, test_y)], verbose=True)

    model2 = svm.SVR(kernel='rbf', gamma=.5, C=1, verbose = True)
    model2.fit(train_X, train_y)

    predictions = model2.predict(test_X)
    print("Mean Absolute Error : " + str(mean_absolute_error(np.array(predictions), test_y)))

    return [model,model2]

def prediction_performance(model, Xtest, Ytest, numberCategories):
    # Calculate metric for logistic regression performance.
    if(numberCategories == 1):
        # Get metrics for binary classification.
        YDistribution = model.predict_proba(Xtest)[:,1]
        YClassification = model.predict(Xtest)
        auc = roc_auc_score(Ytest, YDistribution)
        print("AUC", auc)
        MAE = mean_absolute_error(Ytest, YDistribution)
        print("MAE", MAE)
        accuracy = 1 - mean_absolute_error(YClassification, Ytest)
        print("Accuracy", accuracy)
        metrics = [accuracy, auc, MAE]
    else:
        # Get metric for multiple class classification.
        YPredictions = model.predict(Xtest)
        YDistribution = model.predict_proba(Xtest)
        YTestLabels = label_data(Ytest)
        accuracy = model.score(Xtest, YTestLabels)
        print("Accuracy", accuracy)
        avAUC = evaluate_auc_score(model, Xtest, Ytest)
        print("Av AUC", avAUC)
        #auc = roc_auc_score(Ytest, YPredictions)
        MAE = mean_absolute_error(Ytest, YDistribution)
        print("MAE", MAE)
        metrics = [accuracy, avAUC, MAE]
    return metrics

def test_regressor(train, test, feature_extractor, target_transformer, regressor):
	(train_raw_X, train_raw_y) = (train, train['SalaryNormalized'])
	(test_raw_X, test_raw_y) = (test, test['SalaryNormalized'])


	print 'feature extraction ...'
	train_y = target_transformer.transform(train_raw_y)
	test_y = target_transformer.transform(test_raw_y)
	train_X = feature_extractor.fit_transform(train_raw_X, train_y)
	test_X = feature_extractor.transform(test_raw_X)

	print 'fit regression model ...'
	try:
		regressor.fit(train_X, train_y)
		train_raw_yhat = target_transformer.r_transform(regressor.predict(train_X))
		test_raw_yhat = target_transformer.r_transform(regressor.predict(test_X))
	except TypeError:
		regressor.fit(train_X.toarray(), train_y)
		train_raw_yhat = target_transformer.r_transform(regressor.predict(train_X.toarray()))
		test_raw_yhat = target_transformer.r_transform(regressor.predict(test_X.toarray()))

	print 'evaluate error metrics ...'
	train_error = metrics.mean_absolute_error(train_raw_y, train_raw_yhat)
	test_error = metrics.mean_absolute_error(test_raw_y, test_raw_yhat)
	print 'Train error: ', train_error
	print 'Test error:', test_error

def predict_variance_inf_phase1(budget, hum_train_means, temp_train_means, hum_train_vars, temp_train_vars):
    """Method to make predictions based on max-variance active inference."""         
    start_hum = 0
    window_hum = None
    window_temp = None
    i = 0

    hum_preds = np.ones((50, 96))
    temp_preds = np.ones((50, 96))

    for t in global_times:
        if budget > 0:
            window_hum = np.argpartition(hum_train_vars[t], -budget)[-budget:]
            window_temp = np.argpartition(temp_train_vars[t], -budget)[-budget:]
        else:
            window_hum = np.array([])
            window_temp = np.array([])

        hum_pred, temp_pred = makePreds_phase1(window_hum, window_temp, hum_train_means, temp_train_means, i, t)

        hum_preds[:, i] = copy.deepcopy(hum_pred)
        temp_preds[:, i] = copy.deepcopy(temp_pred)
        
        i += 1

    hum_mean_err = mean_absolute_error(hum_test, hum_preds)
    temp_mean_err = mean_absolute_error(temp_test, temp_preds)

    return hum_preds, temp_preds, hum_mean_err, temp_mean_err

def normalEquation(features, features_validation, values, values_validation):

    M = numpy.dot(features.T, features)
    print "Transposta de f por f"
    print M.shape
    M = numpy.array(M)
    print "Transformou em array"
    print M.shape
    M = numpy.linalg.pinv(M)
    print "Inversa"
    print M.shape
    M = numpy.dot(M, features.T)
    print "Multiplicou por transposta de f"
    print M.shape
    theta = numpy.dot(M, values)
    #M = numpy.linalg.pinv(M)


    print theta.shape
    print features.shape
    print theta

    predictions = numpy.dot(theta, features.T)
    pred_validation = numpy.dot(theta, features_validation.T)

    print predictions

    print "MEAN ABSOLUTE ERROR "
    print mean_absolute_error(values, predictions)

    print "MEAN ABSOLUTE ERROR (validation) "
    print mean_absolute_error(values_validation, pred_validation)

 def test_continue_train(self):
     X, y = load_boston(True)
     X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.1, random_state=42)
     params = {
         'objective': 'regression',
         'metric': 'l1',
         'verbose': -1
     }
     lgb_train = lgb.Dataset(X_train, y_train, free_raw_data=False)
     lgb_eval = lgb.Dataset(X_test, y_test, reference=lgb_train, free_raw_data=False)
     init_gbm = lgb.train(params, lgb_train, num_boost_round=20)
     model_name = 'model.txt'
     init_gbm.save_model(model_name)
     evals_result = {}
     gbm = lgb.train(params, lgb_train,
                     num_boost_round=30,
                     valid_sets=lgb_eval,
                     verbose_eval=False,
                     # test custom eval metrics
                     feval=(lambda p, d: ('mae', mean_absolute_error(p, d.get_label()), False)),
                     evals_result=evals_result,
                     init_model='model.txt')
     ret = mean_absolute_error(y_test, gbm.predict(X_test))
     self.assertLess(ret, 3.5)
     self.assertAlmostEqual(evals_result['valid_0']['l1'][-1], ret, places=5)
     for l1, mae in zip(evals_result['valid_0']['l1'], evals_result['valid_0']['mae']):
         self.assertAlmostEqual(l1, mae, places=5)
     os.remove(model_name)

def evaluate(ytest, ypred, filename='metrics.txt'):
    true_result = [1 if item > 0.5 else 0 for item in ytest]
    pred_result = [1 if item > 0.5 else 0 for item in ypred]
    
    cm = confusion_matrix(true_result, pred_result)
    print('\nConfusion matrix:')
    print(cm)
    print("\nLoss classified as loss", cm[0][0])
    print("Wins classified as wins", cm[1][1])
    print("Wins classified as loss", cm[1][0])
    print("Loss classified as wins", cm[0][1])
    print('\nAccuracy:\t', accuracy_score(true_result, pred_result))
    print('Precision:\t', precision_score(true_result, pred_result))
    print('Recall: \t', recall_score(true_result, pred_result))
    print('F1 score:\t', f1_score(true_result, pred_result))
    print('Mean absolute error:\t', mean_absolute_error(ytest, ypred))
    
    # print to file
    print("Loss classified as loss", cm[0][0], file=open(filename, "a"))
    print("Wins classified as wins", cm[1][1], file=open(filename, "a"))
    print("Wins classified as loss", cm[1][0], file=open(filename, "a"))
    print("Loss classified as wins", cm[0][1], file=open(filename, "a"))
    print('\nAccuracy:\t', accuracy_score(true_result, pred_result), file=open(filename, "a"))
    print('Precision:\t', precision_score(true_result, pred_result), file=open(filename, "a"))
    print('Recall: \t', recall_score(true_result, pred_result), file=open(filename, "a"))
    print('F1 score:\t', f1_score(true_result, pred_result), file=open(filename, "a"))
    print('Mean absolute error:\t', mean_absolute_error(ytest, ypred), file=open(filename, "a"))

def cross_val(regressor_high,regressor_low,classifier,train):
	rows=random.sample(train.index, int(train.shape[0]*0.75))
	sample = train.ix[rows]

	crime=pd.DataFrame(sample.Total_Crime_Risk,dtype=int)
	crime['highcrime']=0
	crime.highcrime[crime.Total_Crime_Risk>crime.Total_Crime_Risk.median()]=1
	crime['GEOGRAPHY_ID']=sample.GEOGRAPHY_ID
	sample=sample.drop(train.columns[[0,-2,-1]], axis=1)
		
	model=classifier.fit(sample, crime.highcrime)
	Highcrime=model.predict(sample)
	Highcrime=np.array(Highcrime)
	sample['predicted_highcrime']=Highcrime
	
	high_areas=sample.ix[sample.predicted_highcrime==1]
	high_areas=pd.merge(high_areas, crime, on='GEOGRAPHY_ID', how= 'inner')
	high_areas_crime=high_areas.Total_Crime_Risk
	high_areas=high_areas.drop(high_areas.columns[[-1,-2,-3]],axis=1)

	low_areas=sample.ix[sample.predicted_highcrime==0]
	low_areas=pd.merge(low_areas, crime, on='GEOGRAPHY_ID', how= 'inner')
	low_areas_crime=low_areas.Total_Crime_Risk
	low_areas=low_areas.drop(low_areas.columns[[-1,-2,-3]],axis=1)

	model_high=regressor_high.fit(high_areas, high_areas_crime)
	high_crime=model_high.predict(high_areas)
	model_low=regressor_low.fit(low_areas, low_areas_crime)
	low_crime=model_low.predict(low_areas)

	high_error=mean_absolute_error(high_areas_crime,high_crime)
	low_error=mean_absolute_error(low_areas_crime,low_crime)
	print high_error,low_error, ((high_error+low_error)/2)

def tst(X, Y, k=3, rad=4, mode='k'):
    trX = X[:-1200]
    trY = Y[:-1200]
    tstX = X[-400:]
    tstY = Y[-400:]

    nnlr = NNLR(k, rad, mode)

    nnlr.fit(trX, trY)

    pred = nnlr.predict(trX)
    print 'Training Set'
    print 'Root Mean Squared Error'
    print mean_squared_error(trY, pred)**.5
    print 'Root Mean Error'
    print mean_absolute_error(trY, pred)
    # print zip(pred, trX)[:5]
    print nnlr.active

    pred = nnlr.predict(tstX)
    print 'Test Set'
    print 'Root Mean Squared Error'
    print mean_squared_error(tstY, pred)**.5
    print 'Root Mean Error'
    print mean_absolute_error(tstY, pred)
    # print zip(pred, tstY)[:5]
    print nnlr.active

def main():
    DOC = """
================================================================================
    Compare the prediction accuracy of different models on the boston dataset
================================================================================
    """
    print(DOC)
    from sklearn import cross_validation, datasets
    boston = datasets.load_boston()
    X, y = boston.data, np.round(boston.target)
    #X -= X.mean()
    y -= y.min()

    idx = np.argsort(y)
    X = X[idx]
    y = y[idx]
    cv = cross_validation.ShuffleSplit(y.size, n_iter=50, test_size=.1, random_state=0)
    score_logistic = []
    score_ordinal_logistic = []
    score_ridge = []
    for i, (train, test) in enumerate(cv):
        #test = train
        if not np.all(np.unique(y[train]) == np.unique(y)):
            # we need the train set to have all different classes
            continue
        assert np.all(np.unique(y[train]) == np.unique(y))
        train = np.sort(train)
        test = np.sort(test)
        w, theta = ordinal_logistic_fit(X[train], y[train], verbose=True,
                                        solver='TNC')
        pred = ordinal_logistic_predict(w, theta, X[test])
        s = metrics.mean_absolute_error(y[test], pred)
        print('ERROR (ORDINAL)  fold %s: %s' % (i+1, s))
        score_ordinal_logistic.append(s)

        from sklearn import linear_model
        clf = linear_model.LogisticRegression(C=1.)
        clf.fit(X[train], y[train])
        pred = clf.predict(X[test])
        s = metrics.mean_absolute_error(y[test], pred)
        print('ERROR (LOGISTIC) fold %s: %s' % (i+1, s))
        score_logistic.append(s)

        from sklearn import linear_model
        clf = linear_model.Ridge(alpha=1.)
        clf.fit(X[train], y[train])
        pred = np.round(clf.predict(X[test]))
        s = metrics.mean_absolute_error(y[test], pred)
        print('ERROR (RIDGE)    fold %s: %s' % (i+1, s))
        score_ridge.append(s)


    print()
    print('MEAN ABSOLUTE ERROR (ORDINAL LOGISTIC):    %s' % np.mean(score_ordinal_logistic))
    print('MEAN ABSOLUTE ERROR (LOGISTIC REGRESSION): %s' % np.mean(score_logistic))
    print('MEAN ABSOLUTE ERROR (RIDGE REGRESSION):    %s' % np.mean(score_ridge))
    # print('Chance level is at %s' % (1. / np.unique(y).size))
    
    return np.mean(score_ridge)

def testModel(
    model, layerSizes, Xtrain, Ytrain, Xtest, Ytest, learningRate, epochs,
    batchSize, optimizer, resultsFile = "lossOptLog.txt", printResults = False,
    elapsedTime = False):
    lossCategories = Ytrain.shape[1]
    numberHiddenLayers = len(layerSizes) - 2
    inputLayerSize = layerSizes[0]
    units1 = layerSizes[1]
    dropout1 = dropouts[0]
    dropout2 = dropouts[1]
    dropout3 = dropouts[2]
    # Test MAE of model on training data (to check for overfitting).
    trainingPredY = model.predict_proba(Xtrain, verbose = 0)
    MAETrain = mean_absolute_error(Ytrain, trainingPredY)

    # Test MAE on test data.
    testPredY = model.predict(Xtest, verbose = 0)
    MAE = mean_absolute_error(Ytest, testPredY)

    # Calculate AUC for each category.
    auc = [0] * lossCategories
    """
    for i in range(0, lossCategories):
        categoryValues = Ytest[:][i:(i+1)]
        categoryPredictions = testPredY[:][i:(i+1)]
        auc[i] = roc_auc_score(categoryPredictions, categoryValues)
    aucAverage = (sum(auc) / len(auc))
    """
    aucAverage = 0
    # Evaluate the model and write results to a file.
    scores = model.evaluate(Xtest, Ytest, verbose = 0)
    testAccuracy = scores[1]
    scores = model.evaluate(Xtrain, Ytrain, verbose = 0)
    trainAccuracy = scores[1]
    if(printResults):
        print("Training MAE: %.2f%%" % (MAETrain * 100))
        print("acc: %.2f%%" % (testAccuracy*100))
        print("auc: %.2f%%" % (aucAverage*100))
        print("MAE: %.2f%%" % (MAE*100))
        print("%s , %s , %s, %s, %s , %s , %s , %s , %s , %s, %s \n"
            % (units1, units2, units3, learningRate, epochs, batchSize,
            patience, optimizer, dropout1, dropout2, dropout3))
        print("\n")        
    # Write model results to a file.
    if(elapsedTime is not False):
        with open(resultsFile, "a") as text_file:
            text_file.write(
                "%s , %s , %s, %s , %s , %s , %s , %s , %s , %s, %s , %s , %s , %s , %s , %s, %s \n"
                % (elapsedTime, MAETrain, trainAccuracy, testAccuracy, aucAverage, MAE, units1,
                units2, units3, learningRate, epochs, batchSize, patience,
                optimizer, dropout1, dropout2, dropout3))
    else:
        with open(resultsFile, "a") as text_file:
            text_file.write(
                "%s , %s , %s , %s , %s , %s ,%s , %s, %s, %s , %s , %s , %s , %s , %s, %s \n"
                % (MAETrain, trainAccuracy, testAccuracy, aucAverage, MAE, units1, units2, units3,
                learningRate, epochs, batchSize, optimizer, dropout1, dropout2,
                dropout3))

def blended_scorer(estimator, X, y):
    ols_preds = ols_preds_for_Xs(X)
    pred_y = estimator.predict(X)
    msg("BLENDED SCORES FOR a CV GROUP:")
    for blend in np.arange(0, 1.01, 0.1):
        blended_prediction = (blend * ols_preds) + ((1.0 - blend) * pred_y)
        blended_score = mean_absolute_error(blended_prediction, y)
        msg("%f * OLS yields score of %f" % (blend, blended_score))
    return mean_absolute_error(y, pred_y)

def main():
    #load da serie
    dtst = Datasets()
    serie = dtst.Leitura_dados(dtst.bases_linear_graduais(3, 35))
    serie = np.asarray(serie)
    particao = Particionar_series(serie, [0.0, 0.0, 0.0], 0)
    serie = particao.Normalizar(serie)
    
    '''
    ELM = ELMRegressor()
    ELM.Tratamento_dados(serie, [0.8, 0.2, 0.2], 4)
    
    #criando uma lista para os dados
    lista_dados = []
    lista_dados.append(ELM.train_entradas)
    lista_dados.append(ELM.train_saidas)
    lista_dados.append(ELM.val_entradas)
    lista_dados.append(ELM.val_saidas)
    lista_dados.append(ELM.teste_entradas)
    lista_dados.append(ELM.teste_saidas)
    
    #Otimizando a arquitetura de uma ELM
    ELM.Otimizar_rede(10, lista_dados)
    '''
        
    #ELM treinando com a entrada e a saida
    #ELM = ELMRegressor(ELM.neuronios_escondidos)
    ELM = ELMRegressor(5)
    ELM.Tratamento_dados(serie, [0.8, 0.2, 0.2], 4)
    ELM.Treinar(ELM.train_entradas, ELM.train_saidas)
    
    #previsao do ELM para o conjunto de treinamento
    prediction_train = ELM.Predizer(ELM.train_entradas)
    MAE_train = mean_absolute_error(ELM.train_saidas, prediction_train)
    print('MAE Treinamento: ', MAE_train)
    
    #previsao do ELM para o conjunto de teste
    prediction_test = ELM.Predizer(ELM.teste_entradas)
    MAE_test = mean_absolute_error(ELM.teste_saidas, prediction_test)
    print('MAE Teste: ', MAE_test)
    
    
    #grafico de previsao para treinamento
    plt.plot(ELM.train_saidas, label = 'Real Treinamento', color = 'Blue')
    plt.plot(prediction_train, label = 'Real Previsão', color = 'Red')
    plt.title('Gráfico Treinamento, MAE: %s' %MAE_train)
    plt.legend()
    plt.tight_layout()
    plt.show()
    
    #grafico de previsao para teste
    plt.plot(ELM.teste_saidas, label = 'Real Teste', color = 'Blue')
    plt.plot(prediction_test, label = 'Previsao Teste', color = 'Red')
    plt.title('Gráfico Teste, MAE: %s' %MAE_test)
    plt.legend()
    plt.tight_layout()
    plt.show()

def build_SGDRegressor(train_X, train_y, test_X, test_y):
    ##########
    log_train_y = np.log(train_y)
    ##########
    sgd_regressor = linear_model.SGDRegressor(loss='huber', penalty='l1', alpha=0.001, l1_ratio=0.15, verbose=True, n_iter = 50)
    sgd_regressor.fit(train_X, log_train_y)
    train_yhat = np.exp(sgd_regressor.predict(train_X))
    test_yhat = np.exp(sgd_regressor.predict(test_X))
    print metrics.mean_absolute_error(train_y, train_yhat)
    print metrics.mean_absolute_error(test_y, test_yhat)

def test_clf(X, y, clf, test_size=0.2, num=20):
    ylist = y.T.tolist()[0]
    train = numpy.zeros(num)
    cross = numpy.zeros(num)
    for i in xrange(num):
        X_train, X_test, y_train, y_test = cross_validation.train_test_split(X, ylist, test_size=test_size)
        clf.fit(X_train, y_train)
        train[i] = mean_absolute_error(clf.predict(X_train), y_train)
        cross[i] = mean_absolute_error(clf.predict(X_test), y_test)
    return (train.mean(), train.std()), (cross.mean(), cross.std())

def test_losses():
    """Test loss functions"""
    y_true, y_pred, _ = make_prediction(binary=True)
    n_samples = y_true.shape[0]
    n_classes = np.size(unique_labels(y_true))

    # Classification
    # --------------
    with warnings.catch_warnings(True):
    # Throw deprecated warning
        assert_equal(zero_one(y_true, y_pred), 13)
        assert_almost_equal(zero_one(y_true, y_pred, normalize=True),
                            13 / float(n_samples), 2)

    assert_almost_equal(zero_one_loss(y_true, y_pred),
                        13 / float(n_samples), 2)
    assert_equal(zero_one_loss(y_true, y_pred, normalize=False), 13)
    assert_almost_equal(zero_one_loss(y_true, y_true), 0.0, 2)
    assert_almost_equal(zero_one_loss(y_true, y_true, normalize=False), 0, 2)

    assert_almost_equal(hamming_loss(y_true, y_pred),
                        2 * 13. / (n_samples * n_classes), 2)

    assert_equal(accuracy_score(y_true, y_pred),
                 1 - zero_one_loss(y_true, y_pred))

    assert_equal(accuracy_score(y_true, y_pred, normalize=False),
                 n_samples - zero_one_loss(y_true, y_pred, normalize=False))

    with warnings.catch_warnings(True):
    # Throw deprecated warning
        assert_equal(zero_one_score(y_true, y_pred),
                     1 - zero_one_loss(y_true, y_pred))

    # Regression
    # ----------
    assert_almost_equal(mean_squared_error(y_true, y_pred),
                        12.999 / n_samples, 2)
    assert_almost_equal(mean_squared_error(y_true, y_true),
                        0.00, 2)

    # mean_absolute_error and mean_squared_error are equal because
    # it is a binary problem.
    assert_almost_equal(mean_absolute_error(y_true, y_pred),
                        12.999 / n_samples, 2)
    assert_almost_equal(mean_absolute_error(y_true, y_true), 0.00, 2)

    assert_almost_equal(explained_variance_score(y_true, y_pred), -0.04, 2)
    assert_almost_equal(explained_variance_score(y_true, y_true), 1.00, 2)
    assert_equal(explained_variance_score([0, 0, 0], [0, 1, 1]), 0.0)

    assert_almost_equal(r2_score(y_true, y_pred), -0.04, 2)
    assert_almost_equal(r2_score(y_true, y_true), 1.00, 2)
    assert_equal(r2_score([0, 0, 0], [0, 0, 0]), 1.0)
    assert_equal(r2_score([0, 0, 0], [0, 1, 1]), 0.0)

def build_boostedTree(train_X, train_y, test_X, test_y):
    bt = ensemble.GradientBoostingRegressor(loss = 'lad', 
        learning_rate= 0.1, n_estimators=100, subsample=0.3, max_depth=3, max_features=50, 
        verbose = 1)
    bt_train_X = train_X
    bt_test_X = test_X
    bt.fit(bt_train_X.toarray(), train_y)
    train_yhat = sgd_regressor.predict(bt_train_X)
    test_yhat = sgd_regressor.predict(bt_test_X)
    print metrics.mean_absolute_error(train_y, train_yhat)
    print metrics.mean_absolute_error(test_y, test_yhat)

def zillow_keras(parameters, X_train, X_dev, Y_train, Y_dev, random_seed=None):
    #random seed
    if random_seed != None:
        np.random.seed(random_seed)
    
    hidden_1, hidden_2, activation, epochs, learning_rate, batch_size, method, momentum = parameters

    if activation == 'tanh':
        initializer = keras.initializers.lecun_normal()
    elif activation == 'relu':
        initializer = keras.initializers.he_normal()
    else:
        raise ValueError('activation')

    regularizer = None
    #regularizer = regularizers.l2(0.01)

    model = Sequential()
    model.add(Dense(hidden_1, input_dim=X_train.shape[1], kernel_initializer=initializer, activation=activation, kernel_regularizer=regularizer))
    model.add(Dense(hidden_2, kernel_initializer=initializer, activation=activation, kernel_regularizer=regularizer))
    model.add(Dense(1, kernel_initializer=initializer, kernel_regularizer=regularizer))

    if method == 'GD':
        optimizer = optimizers.SGD(lr=learning_rate, momentum=momentum)
    elif method == 'RMSProp':
        optimizer = optimizers.RMSprop(lr=learning_rate)
    elif method == 'Adam':
        optimizer = optimizers.Adam(lr=learning_rate)
    else:
        raise ValueError('method')

    model.compile(loss='mean_squared_error', optimizer=optimizer, metrics=['mae'])

    initial_epoch = 0
    #verbose = 1
    verbose = 0
    verbose2 = 100
    #verbose2 = None
    results = model.fit(X_train, Y_train, batch_size=batch_size, epochs=epochs, callbacks=[KerasCallback(verbose2)], validation_data = (X_dev, Y_dev), verbose=verbose, initial_epoch=initial_epoch)

    Y_predict = model.predict(X_train)
    try:
        accuracy_train = mean_absolute_error(Y_train, Y_predict)
    except ValueError:
        accuracy_train = 0
    Y_predict = model.predict(X_dev)
    try:
        accuracy_dev = mean_absolute_error(Y_dev, Y_predict)
    except ValueError:
        accuracy_dev = 0

    print(accuracy_train, accuracy_dev, hidden_1, hidden_2, activation, epochs, learning_rate, batch_size, method, momentum)
    
    '''

def test_clf_kfold(X, y, clf, folds=10):
    train = numpy.zeros(folds)
    cross = numpy.zeros(folds)
    for i, (train_idx, test_idx) in enumerate(cross_validation.KFold(y.shape[0], n_folds=folds)):
        X_train = X[train_idx]
        X_test = X[test_idx]
        y_train = y[train_idx].T.tolist()[0]
        y_test = y[test_idx].T.tolist()[0]
        clf.fit(X_train, y_train)
        train[i] = mean_absolute_error(clf.predict(X_train), y_train)
        cross[i] = mean_absolute_error(clf.predict(X_test), y_test)
    return (train.mean(), train.std()), (cross.mean(), cross.std())