def test_load_diabetes():
    res = load_diabetes()
    assert_equal(res.data.shape, (442, 10))
    assert_true(res.target.size, 442)

    # test return_X_y option
    X_y_tuple = load_diabetes(return_X_y=True)
    bunch = load_diabetes()
    assert_true(isinstance(X_y_tuple, tuple))
    assert_array_equal(X_y_tuple[0], bunch.data)
    assert_array_equal(X_y_tuple[1], bunch.target)

    def test_Lasso_Path(self):
        diabetes = datasets.load_diabetes()
        X = diabetes.data
        y = diabetes.target
        X /= X.std(axis=0)

        df = pdml.ModelFrame(diabetes)
        df.data /= df.data.std(axis=0, ddof=False)

        self.assert_numpy_array_almost_equal(df.data.values, X)

        eps = 5e-3
        expected = lm.lasso_path(X, y, eps, fit_intercept=False)
        result = df.lm.lasso_path(eps=eps, fit_intercept=False)
        self.assert_numpy_array_almost_equal(expected[0], result[0])
        self.assert_numpy_array_almost_equal(expected[1], result[1])
        self.assert_numpy_array_almost_equal(expected[2], result[2])

        expected = lm.enet_path(X, y, eps=eps, l1_ratio=0.8, fit_intercept=False)
        result = df.lm.enet_path(eps=eps, l1_ratio=0.8, fit_intercept=False)
        self.assert_numpy_array_almost_equal(expected[0], result[0])
        self.assert_numpy_array_almost_equal(expected[1], result[1])
        self.assert_numpy_array_almost_equal(expected[2], result[2])

        expected = lm.enet_path(X, y, eps=eps, l1_ratio=0.8, positive=True, fit_intercept=False)
        result = df.lm.enet_path(eps=eps, l1_ratio=0.8, positive=True, fit_intercept=False)
        self.assert_numpy_array_almost_equal(expected[0], result[0])
        self.assert_numpy_array_almost_equal(expected[1], result[1])
        self.assert_numpy_array_almost_equal(expected[2], result[2])

        expected = lm.lars_path(X, y, method='lasso', verbose=True)
        result = df.lm.lars_path(method='lasso', verbose=True)
        self.assert_numpy_array_almost_equal(expected[0], result[0])
        self.assert_numpy_array_almost_equal(expected[1], result[1])
        self.assert_numpy_array_almost_equal(expected[2], result[2])

def test_ElasticnetWeights():
    """Test elastic net with different weight for each predictor
    alpha: a vector of weight, small # means prior knowledge
            1 : means no prior knowledge
    """

    # Has 10 features
    diabetes = datasets.load_diabetes()
    # pprint(diabetes)
    print("Size of data:{}".format(diabetes.data.shape))
    X = diabetes.data
    y = diabetes.target

    X /= X.std(axis=0)  # Standardize data (easier to set the l1_ratio parameter)

    eps = 5e-3   # the smaller it is the longer is the path
    alphas = np.arange(2, 4, 0.2)
    alphas = np.append(alphas, 2.27889) # best aplpha from cv

    # Computing regularization path using the lasso
    alphas_lasso, coefs_lasso, _ = lasso_path(X, y, eps, fit_intercept=False,
                                              alphas=alphas)

    # Computing regularization path using the elastic net
    alphas_enet, coefs_enet, _ = enet_path(
        X, y, eps=eps, l1_ratio=0.8, fit_intercept=False, alphas=alphas)


    # ElasticnetCV
    num_predict = X.shape[1]
    alphas = np.zeros(num_predict)
    alphas.fill(1)
    val = 0.1
    alphas[2] = val
    alphas[3] = val
    alphas[6] = val
    enetCV_alpha, enetCV_coef = runPrintResults(X,y, None, "EnetCV")
    runPrintResults(X,y, alphas, "EnetCVWeight 1")

    # print("coefs_enet: {}".format(coefs_enet[:, -1]))
    # print("coefs_lasso: {}".format(coefs_lasso[:, -1]))

    # Display results
    plt.figure(1)
    ax = plt.gca()
    ax.set_color_cycle(2 * ['b', 'r', 'g', 'c', 'k'])
    l1 = plt.plot(alphas_lasso, coefs_lasso.T)
    l2 = plt.plot(alphas_enet, coefs_enet.T, linestyle='--')

    # repeat alpha for x-axis values for plotting
    enetCV_alphaVect = [enetCV_alpha] * num_predict
    l3 = plt.scatter(enetCV_alphaVect, enetCV_coef, marker='x')

    plt.xlabel('alpha')
    plt.ylabel('coefficients')
    plt.title('Lasso and Elastic-Net Paths')
    plt.legend((l1[-1], l2[-1]), ('Lasso', 'Elastic-Net'),
                loc='upper right')
    plt.axis('tight')
    plt.savefig("fig/lassoEnet")

    def test_simple_grnn(self):
        dataset = datasets.load_diabetes()
        x_train, x_test, y_train, y_test = train_test_split(
            dataset.data, dataset.target, train_size=0.7
        )

        x_train_before = x_train.copy()
        x_test_before = x_test.copy()
        y_train_before = y_train.copy()

        grnnet = algorithms.GRNN(std=0.1, verbose=False)
        grnnet.train(x_train, y_train)
        result = grnnet.predict(x_test)
        error = rmsle(result, y_test)

        old_result = result.copy()
        self.assertAlmostEqual(error, 0.4245, places=4)

        # Test problem with variable links
        np.testing.assert_array_equal(x_train, x_train_before)
        np.testing.assert_array_equal(x_test, x_test_before)
        np.testing.assert_array_equal(y_train, y_train_before)

        x_train[:, :] = 0
        result = grnnet.predict(x_test)
        total_classes_prob = np.round(result.sum(axis=1), 10)
        np.testing.assert_array_almost_equal(result, old_result)

    def test_simple_grnn(self):
        dataset = datasets.load_diabetes()
        x_train, x_test, y_train, y_test = train_test_split(
            dataset.data, dataset.target, train_size=0.7
        )

        x_train_before = x_train.copy()
        x_test_before = x_test.copy()
        y_train_before = y_train.copy()

        grnnet = algorithms.GRNN(std=0.1, verbose=False)
        grnnet.train(x_train, y_train)
        result = grnnet.predict(x_test)
        error = metrics.mean_absolute_error(result, y_test)

        old_result = result.copy()
        self.assertAlmostEqual(error, 46.3358, places=4)

        # Test problem with variable links
        np.testing.assert_array_equal(x_train, x_train_before)
        np.testing.assert_array_equal(x_test, x_test_before)
        np.testing.assert_array_equal(y_train, y_train_before)

        x_train[:, :] = 0
        result = grnnet.predict(x_test)

        np.testing.assert_array_almost_equal(result, old_result)

    def test_grid_search(self):
        def scorer(network, X, y):
            result = network.predict(X)
            return rmsle(result, y)

        dataset = datasets.load_diabetes()
        x_train, x_test, y_train, y_test = train_test_split(
            dataset.data, dataset.target, train_size=0.7
        )

        grnnet = algorithms.GRNN(std=0.5, verbose=False)
        grnnet.train(x_train, y_train)
        error = scorer(grnnet, x_test, y_test)

        self.assertAlmostEqual(0.513, error, places=3)

        random_search = grid_search.RandomizedSearchCV(
            grnnet,
            param_distributions={'std': np.arange(1e-2, 1, 1e-4)},
            n_iter=10,
            scoring=scorer
        )
        random_search.fit(dataset.data, dataset.target)
        scores = random_search.grid_scores_

        best_score = min(scores, key=itemgetter(1))
        self.assertAlmostEqual(0.452, best_score[1], places=3)

def main():
    diabetes = datasets.load_diabetes()
    # Use only one feature
    diabetes_X = diabetes.data[:, np.newaxis, 2]
    diabetes_X = scale(diabetes_X)
    diabetes_y = scale(diabetes.target)

    diabetes_X_train = diabetes_X[:-20]
    diabetes_X_test = diabetes_X[-20:]
    # diabetes_y_train = diabetes.target[:-20]
    # diabetes_y_test = diabetes.target[-20:]
    diabetes_y_train = diabetes_y[:-20]
    diabetes_y_test = diabetes_y[-20:]

    # regr = linear_model.LinearRegression()
    regr = LinearRegression(n_iter=50, fit_alg="batch")
    # regr = LinearRegressionNormal()
    regr.fit(diabetes_X_train, diabetes_y_train)

    # regr.fit(np.array([[0, 0], [1, 1], [2, 2]]), np.array([0, 1, 2]))
    # print(regr.predict(np.array([[3, 3]])))

    # print('Coefficients: \n', regr.coef_)
    # print("Residual sum of squares: %.2f"
    #       % np.mean((regr.predict(diabetes_X_test) - diabetes_y_test) ** 2))
    print("Variance score: %.2f" % regr.score(diabetes_X_test, diabetes_y_test))

    def test_pipeline(self):
        dataset = datasets.load_diabetes()
        target_scaler = preprocessing.MinMaxScaler()
        target = dataset.target.reshape(-1, 1)

        x_train, x_test, y_train, y_test = train_test_split(
            dataset.data,
            target_scaler.fit_transform(target),
            train_size=0.85
        )

        network = algorithms.Backpropagation(
            connection=[
                layers.SigmoidLayer(10),
                layers.SigmoidLayer(40),
                layers.OutputLayer(1),
            ],
            use_bias=True,
            show_epoch=100,
            verbose=False,
        )
        pipeline = Pipeline([
            ('min_max_scaler', preprocessing.MinMaxScaler()),
            ('backpropagation', network),
        ])
        pipeline.fit(x_train, y_train, backpropagation__epochs=1000)
        y_predict = pipeline.predict(x_test)

        error = rmsle(target_scaler.inverse_transform(y_test),
                      target_scaler.inverse_transform(y_predict).round())
        self.assertAlmostEqual(0.4481, error, places=4)

    def test_pipeline(self):
        dataset = datasets.load_diabetes()
        target_scaler = preprocessing.MinMaxScaler()
        target = dataset.target.reshape(-1, 1)

        x_train, x_test, y_train, y_test = train_test_split(
            dataset.data,
            target_scaler.fit_transform(target),
            train_size=0.85
        )

        network = algorithms.GradientDescent(
            connection=[
                layers.Input(10),
                layers.Sigmoid(25),
                layers.Sigmoid(1),
            ],
            show_epoch=100,
            verbose=False,
        )
        pipeline = Pipeline([
            ('min_max_scaler', preprocessing.MinMaxScaler()),
            ('gd', network),
        ])
        pipeline.fit(x_train, y_train, gd__epochs=50)
        y_predict = pipeline.predict(x_test)

        error = rmsle(target_scaler.inverse_transform(y_test),
                      target_scaler.inverse_transform(y_predict).round())
        self.assertAlmostEqual(0.48, error, places=2)

    def test_grid_search(self):
        def scorer(network, X, y):
            result = network.predict(X)
            return rmsle(result[:, 0], y)

        dataset = datasets.load_diabetes()
        x_train, x_test, y_train, y_test = train_test_split(
            dataset.data, dataset.target, train_size=0.7
        )

        grnnet = algorithms.GRNN(std=0.5, verbose=False)
        grnnet.train(x_train, y_train)
        error = scorer(grnnet, x_test, y_test)

        self.assertAlmostEqual(0.513, error, places=3)

        random_search = model_selection.RandomizedSearchCV(
            grnnet,
            param_distributions={'std': np.arange(1e-2, 0.1, 1e-4)},
            n_iter=10,
            scoring=scorer,
            random_state=self.random_seed
        )
        random_search.fit(dataset.data, dataset.target)
        scores = random_search.cv_results_

        best_score = min(scores['mean_test_score'])
        self.assertAlmostEqual(0.4266, best_score, places=3)

def gmm_clustering():
    conversion = {
        0: 2,
        1: 0,
        2: 1,
    }

    g = mixture.GMM(n_components=3)

    iris_data = datasets.load_iris()
    diabetes_data = datasets.load_diabetes()
    data = iris_data

    # Generate random observations with two modes centered on 0
    # and 10 to use for training.
    np.random.seed(0)
    obs = np.concatenate((np.random.randn(100, 1), 10 + np.random.randn(300, 1)))
    g.fit(data.data)

    print("Target classification")
    print(data.target)
    results = g.predict(data.data)
    results = [conversion[item] for item in results]

    print("\nResults")
    print(np.array(results))
    compare = [results[i] == data.target[i] for i in range(len(results))]

    accuracy_count = [item for item in compare if item == True]

    print("\nAccuracy: {:.0%}".format(float(len(accuracy_count)) / len(compare)))
    print(max(data.target))

    def test_hessian_diagonal(self):
        dataset = datasets.load_diabetes()
        data, target = dataset.data, dataset.target

        input_scaler = preprocessing.StandardScaler()
        target_scaler = preprocessing.StandardScaler()

        x_train, x_test, y_train, y_test = cross_validation.train_test_split(
            input_scaler.fit_transform(data),
            target_scaler.fit_transform(target.reshape(-1, 1)),
            train_size=0.8
        )

        nw = algorithms.HessianDiagonal(
            connection=[
                layers.SigmoidLayer(10),
                layers.SigmoidLayer(20),
                layers.OutputLayer(1)
            ],
            step=1.5,
            shuffle_data=False,
            verbose=False,
            min_eigenvalue=1e-10
        )
        nw.train(x_train, y_train, epochs=10)
        y_predict = nw.predict(x_test)

        error = rmsle(target_scaler.inverse_transform(y_test),
                      target_scaler.inverse_transform(y_predict).round())

        self.assertAlmostEqual(0.5032, error, places=4)

def bagging_regression():
	digits = load_diabetes()
	x = digits.data
	y = digits.target

	sample_parameter = {
		'n_jobs': -1,
		'min_samples_leaf': 2.0,
		'n_estimators': 500,
		'max_features': 0.55,
		'criterion': 'mse',
		'min_samples_split': 4.0,
		'model': 'RFREG',
		'max_depth': 4.0
	}

	x_train,x_test,y_train,y_test = train_test_split(x,y,test_size=0.2,random_state=42)

	clf_layer = mlc.layer.layer.RegressionLayer()
	print "single prediction"
	#y_train_predict,y_test_predict = clf_layer.predict(x_train,y_train,x_test,sample_parameter)
	#print y_test_predict
	y_train_predict_proba,y_test_predict_proba = clf_layer.predict(x_train,y_train,x_test,sample_parameter)
	#print y_test_predict_proba
	print evaluate_function(y_test,y_test_predict_proba,'mean_squared_error')

	print "multi ensamble prediction"

	multi_bagging_clf = mlc.layer.layer.RegressionBaggingLayer()
	y_train_predict_proba,y_test_predict_proba = multi_bagging_clf.predict(x_train,y_train,x_test,sample_parameter,times=5)

	print evaluate_function(y_test,y_test_predict_proba,'mean_squared_error')

def test_linearsvr_fit_sampleweight():
    # check correct result when sample_weight is 1
    # check that SVR(kernel='linear') and LinearSVC() give
    # comparable results
    diabetes = datasets.load_diabetes()
    n_samples = len(diabetes.target)
    unit_weight = np.ones(n_samples)
    lsvr = svm.LinearSVR(C=1e3).fit(diabetes.data, diabetes.target,
                                    sample_weight=unit_weight)
    score1 = lsvr.score(diabetes.data, diabetes.target)

    lsvr_no_weight = svm.LinearSVR(C=1e3).fit(diabetes.data, diabetes.target)
    score2 = lsvr_no_weight.score(diabetes.data, diabetes.target)

    assert_allclose(np.linalg.norm(lsvr.coef_),
                    np.linalg.norm(lsvr_no_weight.coef_), 1, 0.0001)
    assert_almost_equal(score1, score2, 2)

    # check that fit(X)  = fit([X1, X2, X3],sample_weight = [n1, n2, n3]) where
    # X = X1 repeated n1 times, X2 repeated n2 times and so forth
    random_state = check_random_state(0)
    random_weight = random_state.randint(0, 10, n_samples)
    lsvr_unflat = svm.LinearSVR(C=1e3).fit(diabetes.data, diabetes.target,
                                           sample_weight=random_weight)
    score3 = lsvr_unflat.score(diabetes.data, diabetes.target,
                               sample_weight=random_weight)

    X_flat = np.repeat(diabetes.data, random_weight, axis=0)
    y_flat = np.repeat(diabetes.target, random_weight, axis=0)
    lsvr_flat = svm.LinearSVR(C=1e3).fit(X_flat, y_flat)
    score4 = lsvr_flat.score(X_flat, y_flat)

    assert_almost_equal(score3, score4, 2)

def get_data(n_clients):
    """
    Import the dataset via sklearn, shuffle and split train/test.
    Return training, target lists for `n_clients` and a holdout test set
    """
    print("Loading data")
    diabetes = load_diabetes()
    y = diabetes.target
    X = diabetes.data
    # Add constant to emulate intercept
    X = np.c_[X, np.ones(X.shape[0])]

    # The features are already preprocessed
    # Shuffle
    perm = np.random.permutation(X.shape[0])
    X, y = X[perm, :], y[perm]

    # Select test at random
    test_size = 50
    test_idx = np.random.choice(X.shape[0], size=test_size, replace=False)
    train_idx = np.ones(X.shape[0], dtype=bool)
    train_idx[test_idx] = False
    X_test, y_test = X[test_idx, :], y[test_idx]
    X_train, y_train = X[train_idx, :], y[train_idx]

    # Split train among multiple clients.
    # The selection is not at random. We simulate the fact that each client
    # sees a potentially very different sample of patients.
    X, y = [], []
    step = int(X_train.shape[0] / n_clients)
    for c in range(n_clients):
        X.append(X_train[step * c: step * (c + 1), :])
        y.append(y_train[step * c: step * (c + 1)])

    return X, y, X_test, y_test

    def load_diabetes_data():
        """
        Load the diabetes data set from scikit learn

        Args:
            None

        Returns:
            diabetes_X_train: Training features for diabetes data set
            diabetes_X_test: Test set features for diabetes data set
            diabetes_y_train: Target variables of the training set
            diabetes_y_test: Target variables of the test set
        """
        diabetes = datasets.load_diabetes()
        diabetes_X, diabetes_y = diabetes.data, diabetes.target

        # Split the data set as
        # 70 % -> Training set
        # 30 % -> Test set

        limit = 0.7 * len(diabetes_y)
        diabetes_X_train = diabetes_X[:limit]
        diabetes_X_test = diabetes_X[limit:]
        diabetes_y_train = diabetes_y[:limit]
        diabetes_y_test = diabetes_y[limit:]
        return diabetes_X_train, diabetes_X_test, diabetes_y_train, diabetes_y_test

    def test_levenberg_marquardt(self):
        dataset = datasets.load_diabetes()
        data, target = dataset.data, dataset.target

        data_scaler = preprocessing.MinMaxScaler()
        target_scaler = preprocessing.MinMaxScaler()

        x_train, x_test, y_train, y_test = train_test_split(
            data_scaler.fit_transform(data),
            target_scaler.fit_transform(target.reshape(-1, 1)),
            train_size=0.85
        )

        # Network
        lmnet = algorithms.LevenbergMarquardt(
            connection=[
                layers.SigmoidLayer(10),
                layers.SigmoidLayer(40),
                layers.OutputLayer(1),
            ],
            mu_increase_factor=2,
            mu=0.1,
            show_epoch=10,
            use_bias=False,
            verbose=False,
        )
        lmnet.train(x_train, y_train, epochs=100)
        y_predict = lmnet.predict(x_test)

        error = rmsle(target_scaler.inverse_transform(y_test),
                      target_scaler.inverse_transform(y_predict).round())
        error

        self.assertAlmostEqual(0.4372, error, places=4)

    def test_mixture_of_experts(self):
        dataset = datasets.load_diabetes()
        data, target = asfloat(dataset.data), asfloat(dataset.target)
        insize, outsize = data.shape[1], 1

        input_scaler = preprocessing.MinMaxScaler((-1 ,1))
        output_scaler = preprocessing.MinMaxScaler()
        x_train, x_test, y_train, y_test = cross_validation.train_test_split(
            input_scaler.fit_transform(data),
            output_scaler.fit_transform(target.reshape(-1, 1)),
            train_size=0.8
        )

        n_epochs = 10
        scaled_y_test = output_scaler.inverse_transform(y_test)
        scaled_y_test = scaled_y_test.reshape((y_test.size, 1))

        # -------------- Train single GradientDescent -------------- #

        bpnet = algorithms.GradientDescent(
            (insize, 20, outsize),
            step=0.1,
            verbose=False
        )
        bpnet.train(x_train, y_train, epochs=n_epochs)
        network_output = bpnet.predict(x_test)
        network_error = rmsle(output_scaler.inverse_transform(network_output),
                              scaled_y_test)

        # -------------- Train ensemlbe -------------- #

        moe = algorithms.MixtureOfExperts(
            networks=[
                algorithms.Momentum(
                    (insize, 20, outsize),
                    step=0.1,
                    batch_size=1,
                    verbose=False
                ),
                algorithms.Momentum(
                    (insize, 20, outsize),
                    step=0.1,
                    batch_size=1,
                    verbose=False
                ),
            ],
            gating_network=algorithms.Momentum(
                layers.Softmax(insize) > layers.Output(2),
                step=0.1,
                verbose=False
            )
        )
        moe.train(x_train, y_train, epochs=n_epochs)
        ensemble_output = moe.predict(x_test)
        ensemlbe_error = rmsle(
            output_scaler.inverse_transform(ensemble_output),
            scaled_y_test
        )

        self.assertGreater(network_error, ensemlbe_error)

def supervisedTest02():
	import numpy as np
	from sklearn import datasets

	diabetes = datasets.load_diabetes()
	diabetes_X_train = diabetes.data[:-20]
	diabetes_X_test  = diabetes.data[-20:]
	diabetes_Y_train = diabetes.target[:-20]
	diabetes_Y_test  = diabetes.target[-20:]

	from sklearn import linear_model
	regr = linear_model.LinearRegression(copy_X=True, fit_intercept=True, normalize=False)
	regr.fit(diabetes_X_train, diabetes_Y_train)

	#print regr.coef_  #注意因为diabetes_X_train的特征是4维，所以coef_的个数是4+1 = 5

	mean_err = np.mean((regr.predict(diabetes_X_test) - diabetes_Y_test) ** 2)
	score = regr.score(diabetes_X_test, diabetes_Y_test) #这是判断test数据预测程度

	print mean_err
	print score


	print len(diabetes.data)    #样本数目
	print len(diabetes.data[0]) #特征维数

def ModelSelectionTest01():
	from sklearn import datasets, svm
	import numpy as np
	digits = datasets.load_digits()
	X_digits = digits.data
	Y_digits = digits.target
	svc = svm.SVC(C = 1, kernel = 'linear')
	score = svc.fit(X_digits[:-100], Y_digits[:-100]).score(X_digits[-100:], Y_digits[-100:])

	#print score

	X_folds = np.array_split(X_digits, 3)
	Y_folds = np.array_split(Y_digits, 3)

	#print len(X_folds[0])

	scores = list()

	for k in range(3):
		X_train = list(X_folds) #这里的X_folds是一个具有3个元素的list
		X_test = X_train.pop(k) #test是train的第K个元素
		X_train = np.concatenate(X_train) #这里是把X_train减去X_test
		#print len(X_train)
		Y_train = list(Y_folds)
		Y_test = Y_train.pop(k)
		Y_train = np.concatenate(Y_train)

		scores.append(svc.fit(X_train, Y_train).score(X_test, Y_test))

	#print scores


	from sklearn import cross_validation
	k_fold = cross_validation.KFold(n = 6, n_folds = 3)
	for train_indices, test_indices in k_fold:
		print train_indices, test_indices

	k_fold = cross_validation.KFold(len(X_digits), n_folds = 3)
	scores = [svc.fit(X_digits[train], Y_digits[train]).score(X_digits[test], Y_digits[test]) for train , test in k_fold]

	#print scores

	scores = cross_validation.cross_val_score(svc, X_digits, Y_digits, cv = k_fold, n_jobs = 1)
	#print scores

	from sklearn.grid_search import GridSearchCV
	gammas = np.logspace(-6, -1, 10)
	clf = GridSearchCV(estimator = svc, param_grid = dict(gamma = gammas), n_jobs = 1)
	clf.fit(X_digits[:1000], Y_digits[:1000])
	print clf.best_score_
	print clf.best_estimator_.gamma

	from sklearn import linear_model, datasets
	lasso = linear_model.LassoCV()    #这里的lassoCV和lasso有什么区别？
	diabetes = datasets.load_diabetes()
	X_diabetes = diabetes.data
	Y_diabetes = diabetes.target
	lasso.fit(X_diabetes, Y_diabetes)

	print lasso.alpha_