def train_data(self, num_data=2000, stddev=0.10):
        """
        generate the moon/linear data
        """
        if self.dtype == "moon":
            feat_vec, labels = datasets.make_moons(num_data, noise=stddev)
        elif self.dtype == "linear":
            feat_vec, labels = make_blobs(n_samples=num_data, n_features=2, 
                                          centers=2, cluster_std=1.7)
        else:
            feat_vec, labels = datasets.make_moons(num_data, noise=stddev)

        ##
        ## we need to have these in numpy matrix format
        ##
        feats_vecs = np.matrix(feat_vec).astype(np.float32)
        labels = np.array(labels).astype(dtype=np.uint8)

        # Convert the int numpy array into a one-hot matrix.
        labels_onehot = (np.arange(self.num_classes) == labels[:, None]).astype(np.float32)

        ##
        ## create train and test set
        ##
        train_set_size = int(self.dsplit * num_data)

        self.feats_vecs = feats_vecs[:train_set_size,:]
        self.tfeats_vecs = feats_vecs[train_set_size:,:] 
        self.labels_onehot = labels_onehot[:train_set_size]
        self.tlabels_onehot = labels_onehot[train_set_size:]

        # Return a pair of the feature matrix and the one-hot label matrix.
        return self.feats_vecs, self.labels_onehot

def plot_tree_progressive():
    fig, axes = plt.subplots(4, 2, figsize=(15, 25), subplot_kw={'xticks': (), 'yticks': ()})
    X, y = make_moons(n_samples=100, noise=0.25, random_state=3)

    for i, max_depth in enumerate([1, 2, 9]):
        tree = plot_tree(X, y, max_depth=max_depth, ax=axes[i + 1, 0])
        axes[i + 1, 1].imshow(tree_image(tree))
        axes[i + 1, 1].set_axis_off()
    axes[0, 1].set_visible(False)
    for ax in axes[:, 0]:
        ax.scatter(X[:, 0], X[:, 1], c=np.array(['r', 'b'])[y], s=60)
    X, y = make_moons(noise=0.3, random_state=0)

def _download():
    train_x, train_t = make_moons(n_samples=10000, shuffle=True, noise=0.2, random_state=1234)
    test_x, test_t = make_moons(n_samples=10000, shuffle=True, noise=0.2, random_state=1234)
    valid_x, valid_t = make_moons(n_samples=10000, shuffle=True, noise=0.2, random_state=1234)

    train_x += np.abs(train_x.min())
    test_x += np.abs(test_x.min())
    valid_x += np.abs(valid_x.min())

    train_set = (train_x, train_t)
    test_set = (test_x, test_t)
    valid_set = (valid_x, valid_t)

    return train_set, test_set, valid_set

def make_trans_moons(theta=40, nb=100, noise=.05):
    from math import cos, sin, pi
    
    X, y = make_moons(nb, noise=noise, random_state=1) 
    Xt, yt = make_moons(nb, noise=noise, random_state=2)
    
    trans = -np.mean(X, axis=0) 
    X  = 2*(X+trans)
    Xt = 2*(Xt+trans)
    
    theta = -theta*pi/180
    rotation = np.array( [  [cos(theta), sin(theta)], [-sin(theta), cos(theta)] ] )
    Xt = np.dot(Xt, rotation.T)
    
    return X, y, Xt, yt

def test_make_moons():
    X, y = make_moons(3, shuffle=False)
    for x, label in zip(X, y):
        center = [0.0, 0.0] if label == 0 else [1.0, 0.5]
        dist_sqr = ((x - center) ** 2).sum()
        assert_almost_equal(dist_sqr, 1.0,
                            err_msg="Point is not on expected unit circle")

def test_run():
    data, label = make_moons(n_samples=NSAMPLES, noise=0.4)
    scores, confusions, predictions, test_proba = \
        poly(data, label, n_folds=2, verbose=1, feature_selection=False,
             save=False, project_name='test1')
    data, label = make_classification(n_samples=NSAMPLES, n_features=20,
                                      n_informative=5, n_redundant=2,
                                      n_repeated=0, n_classes=3,
                                      n_clusters_per_class=2, weights=None,
                                      flip_y=0.01, class_sep=1.0,
                                      hypercube=True, shift=0.0,
                                      scale=1.0, shuffle=True,
                                      random_state=None)
    scores, confusions, predictions, test_proba = \
        poly(data, label, n_folds=3, verbose=1, feature_selection=False,
             save=False, project_name='test2')

    scores, confusions, predictions, test_proba = \
        poly(data, label, n_folds=3, verbose=1,
             exclude=['Multilayer Perceptron'], feature_selection=True,
             project_name='test3')
    scores, confusions, predictions, test_proba = \
        poly(data, label, n_folds=3, verbose=1,
             exclude=['Multilayer Perceptron',
                      'Voting'],
             feature_selection=True,
             project_name='test3')
    plot(scores)

def plot_adaboost():
    X, y = make_moons(noise=0.3, random_state=0)

    # Create and fit an AdaBoosted decision tree
    est = AdaBoostClassifier(DecisionTreeClassifier(max_depth=1),
                             algorithm="SAMME.R",
                             n_estimators=200)

    sample_weight = np.empty(X.shape[0], dtype=np.float)
    sample_weight[:] = 1. / X.shape[0]

    est._validate_estimator()
    est.estimators_ = []
    est.estimator_weights_ = np.zeros(4, dtype=np.float)
    est.estimator_errors_ = np.ones(4, dtype=np.float)

    plot_step = 0.02

    # Plot the decision boundaries
    x_min, x_max = X[:, 0].min() - 1, X[:, 0].max() + 1
    y_min, y_max = X[:, 1].min() - 1, X[:, 1].max() + 1
    xx, yy = np.meshgrid(np.arange(x_min, x_max, plot_step),
                         np.arange(y_min, y_max, plot_step))

    fig, axes = plt.subplots(1, 4, figsize=(14, 4), sharey=True)
    colors = ['#d7191c', '#fdae61', '#ffffbf', '#abd9e9', '#2c7bb6']
    c = lambda a, b, c: map(lambda x: x / 254.0, [a, b, c])
    colors = [c(215, 25, 28),
              c(253, 174, 97),
              c(255, 255, 191),
              c(171, 217, 233),
              c(44, 123, 182),
              ]

    for i, ax in enumerate(axes):
        sample_weight, estimator_weight, estimator_error = est._boost(i, X, y, sample_weight)
        est.estimator_weights_[i] = estimator_weight
        est.estimator_errors_[i] = estimator_error
        sample_weight /= np.sum(sample_weight)

        Z = est.predict(np.c_[xx.ravel(), yy.ravel()])
        Z = Z.reshape(xx.shape)
        ax.contourf(xx, yy, Z,
                    cmap=matplotlib.colors.ListedColormap([colors[1], colors[-2]]),
                    alpha=1.0)
        ax.axis("tight")

        # Plot the training points
        ax.scatter(X[:, 0], X[:, 1],
                   c=np.array([colors[0], colors[-1]])[y],
                   s=20 + (200 * sample_weight) ** 2, cmap=plt.cm.Paired)
        ax.set_xlim(x_min, x_max)
        ax.set_ylim(y_min, y_max)
        ax.set_xlabel('$x_0$')

        if i == 0:
            ax.set_ylabel('$x_1$')

    plt.tight_layout()
    plt.show()

def main():
    X, y = datasets.make_moons(n_samples=200, shuffle=True, noise=0.1, random_state=None)
    plt.scatter(X[:, 0], X[:, 1], c=y)
    for i in range(8):
        clf = RandomForestClassifier(n_estimators = 2**i)   
        clf.fit(X,y)
        plot_surface(clf, X, y)

def generate_noisy_data():
    blobs, _ = datasets.make_blobs(n_samples=200,
                                    centers=[(-0.75,2.25), (1.0, 2.0)],
                                    cluster_std=0.25)
    moons, _ = datasets.make_moons(n_samples=200, noise=0.05)
    noise = np.random.uniform(-1.0, 3.0, (50, 2))
    return np.vstack([blobs, moons, noise])

def single_run(te):
    print te
    data, label = make_moons(n_samples=1000, noise=0.05, shuffle=True, random_state = int(time.time()))
        
    data,validation_data,label,validation_label = train_test_split(data,label,train_size = .30)
        #separate the data set into buckets
    
    total_data = list(group_list(data,1))
    total_label = list(group_list(label,1))
    
    #The two separate site sets

    for s in range(10,150,10):
	print s
	nets = []
	nn_groups_data = []
	nn_groups_label = []
	number_of_nets = s
	for x in range(number_of_nets):
            nets.append(nnDif.nn_build(1,[2,6,6,1],eta=eta,nonlin=nonlin))
        iters = 20000
        for j in range(number_of_nets):
            x = (total_data[int(float(j)/number_of_nets*(len(total_data))):int(float((j+1))/number_of_nets*(len(total_data)))])
            nn_groups_data.append(x)
    
            nn_groups_label.append(total_label[int(float(j)/number_of_nets*(len(total_label)/number_of_nets)):int(float((j+1))/number_of_nets*(len(total_label)))])

	start = time.time()
	visitbatches(nets,nn_groups_data,nn_groups_label,[],it=iters)
	print time.time() - start
	one = accuracy(nets[0], validation_data, validation_label, thr=0.5)

	nn1Acc[te][s/10] += one
        '''

def main():
	no_of_samples = 400
	
	data = []
	data.append( datasets.make_moons(n_samples=no_of_samples, noise=0.05)[0] )
	data.append( datasets.make_circles(n_samples=no_of_samples, factor=0.5, noise=0.05)[0] )
	
	# number of clusters we expect
	K = 2

	for X in data:	
		# from dataset, create adjacency, degree, and laplacian matrix
		adjacency 	= gaussianDistance( X, sigma=0.1 )
		degree 		= degreeMatrix( adjacency )
		L 			= diag(degree) - adjacency

		# perform whitening on the Laplacian matrix
		deg_05 	= diag( degree  ** -0.5 )
		L 		= deg_05.dot( L ).dot( deg_05 )

		# use eig to obtain eigenvalues and eigenvectors
		eigenvalues, eigenvectors = linalg.eig( L )

		# Sort the eigenvalues ascending, the first K zero eigenvalues represent the connected components
		idx = eigenvalues.argsort()
		eigenvalues.sort()
		evecs = eigenvectors[:, idx]
		eigenvectors = evecs[:, 0:K]
		print eigenvalues[0:K]

		color_array = ['b', 'r', 'g', 'y']

		fig = pyplot.figure( figsize=(15, 5) )
		fig.canvas.set_window_title( 'Difference between K-means and Spectral Clusterings' )

		# First perform the normal K-means on the original dataset and plot it out
		centroids, labels = scipy.cluster.vq.kmeans2( X, K )
		data = c_[X, labels]	
		ax = fig.add_subplot( 131 )
		ax.set_title('K means clustering')
		for k in range( 0, K ):
			ax.scatter( data[data[:, 2]==k, 0], data[data[:, 2]==k, 1], c=color_array[k], marker='o')

		# Then we perform spectral clustering, i.e. K-means on eigenvectors
		centroids, labels = scipy.cluster.vq.kmeans2( eigenvectors, K )
		data = c_[X, labels]	
		ax = fig.add_subplot( 132 )
		ax.set_title('Spectral clustering')
		for k in range( 0, K ):
			ax.scatter( data[data[:, 2]==k, 0], data[data[:, 2]==k, 1], c=color_array[k], marker='o')

		# Plot out the eigenvectors too
		data = c_[eigenvectors, labels]
		ax = fig.add_subplot(133)
		ax.set_title('K-eigenvectors')
		for k in range( 0, K ):
			ax.scatter( data[data[:, 2]==k, 0], data[data[:, 2]==k, 1], c=color_array[k], marker='o')

		pyplot.show()

def generate_biclass_data(data_type, random_state):
    """ Generate biclass data to classify

    arg : data_type (str) possible type of data
            choose any in ["lin_sep", "non_lin_sep", "overlap"]
            'lin_sep' : Bi-class, linearly separable data
            'non_lin_sep' : Bi-class, non linearly separable data
            'overlap' : Bi-class, non linearly separable data with class overlap

        random_state (int) seed for numpy.random
    """

    # Set seed for reproducible results
    np.random.seed(random_state)

    # Case 1 : linearly separable data
    if data_type == "lin_sep":
        mean1 = np.array([0, 2])
        mean2 = np.array([2, 0])
        cov = np.array([[0.8, 0.6], [0.6, 0.8]])
        X1 = np.random.multivariate_normal(mean1, cov, 100)
        y1 = np.ones(len(X1))
        X2 = np.random.multivariate_normal(mean2, cov, 100)
        y2 = np.ones(len(X2)) * -1
        X = np.vstack((X1, X2))
        y = np.hstack((y1, y2))

    # Case 2 : non -linearly separable data
    elif data_type == "moons":
        X, y = make_moons(n_samples=200, noise=0.2)

    elif data_type == "circles":
        X, y = make_circles(n_samples=200, noise=0.2, factor=0.5)

    # Case 3 : data with overlap between classes
    elif data_type == "overlap":
        mean1 = np.array([0, 2])
        mean2 = np.array([2, 0])
        cov = np.array([[1.5, 1.0], [1.0, 1.5]])
        X1 = np.random.multivariate_normal(mean1, cov, 100)
        y1 = np.ones(len(X1))
        X2 = np.random.multivariate_normal(mean2, cov, 100)
        y2 = np.ones(len(X2)) * -1
        X = np.vstack((X1, X2))
        y = np.hstack((y1, y2))

    assert(X.shape[0] == y.shape[0])

    # Format target to: -1 / +1
    targets = set(y.tolist())
    t1 = min(targets)
    t2 = max(targets)
    l1 = np.where(y < t2)
    l2 = np.where(y > t1)
    y[l1] = -1
    y[l2] = 1

    return X, y

def make_datasets():
    """

    :return:
    """

    return [make_moons(n_samples=200, noise=0.3, random_state=0),
            make_circles(n_samples=200, noise=0.2, factor=0.5, random_state=1),
            make_linearly_separable()]

def loadDatasets(linearly_separable):

    datasets = [\
                make_moons(noise=0.3, random_state=0), \
                make_circles(noise=0.2, factor=0.5, random_state=1), \
                linearly_separable \
               ]

    return datasets

def test_1():
    # 读取sk里面的数据, 并且绘图
    np.random.seed(0)
    X, y = datasets.make_moons(200, noise=0.20)
    print(X)
    mpp.scatter(X[:,0], X[:,1], s=40, c=y)
    #mpp.plot(X[:,0], X[:,1])
    mpp.show()
    return X, y

    def test():
        np.random.seed(0)

        train_x, train_y = datasets.make_moons(5000, noise=.20)
        train_y = np.eye(2)[train_y]

        example_count = len(train_x)

        nn = TheanoNN(train_x.shape[1],1000,train_y.shape[1],np.float32(0.01),np.float32(0.01),train_x,train_y)
        nn.train()

def make_noisy_problem(n_samples_train=30, label_noise_rate=0.1, input_noise=0.15,
                       n_samples_test=3000, seed=0):
    rng = np.random.RandomState(seed)
    rng = np.random.RandomState(1)
    scaler = StandardScaler()

    X_train, y_train = make_moons(n_samples=n_samples_train, shuffle=True,
                                  noise=input_noise, random_state=rng)
    X_test, y_test = make_moons(n_samples=n_samples_test, shuffle=True,
                                noise=input_noise, random_state=rng)
    
    if label_noise_rate > 0:
        rnd_levels = rng.uniform(low=0., high=1., size=n_samples_train)
        noise_mask = rnd_levels <= label_noise_rate
        y_train[noise_mask] = rng.randint(low=0, high=2, size=noise_mask.sum())
    
    X_train = scaler.fit_transform(X_train)
    X_test = scaler.transform(X_test)
    return (X_train, y_train), (X_test, y_test)

    def setup_method(self, method):
        base = GradientBoostingClassifier(n_estimators=2)
        self.clf = CascadedBooster(base_clf=base)

        n_samples = 500
        np.random.seed(42)
        X, Y = make_moons(n_samples=n_samples, noise=.05)

        self.X = X
        self.Y = Y
        self.clf.fit(X, Y)

def build_datasets(n_samples=100):
    X, y = make_classification(n_samples=n_samples, n_features=2, n_redundant=0, n_informative=2, n_clusters_per_class=1)
    X += 2 * np.random.uniform(size=X.shape)
    linearly_separable = (X, y)

    names = ['moons', 'circles', 'linear', 'xor']
    datasets = [make_moons(n_samples=n_samples, noise=0.3),
                make_circles(n_samples=n_samples, noise=0.2, factor=0.5),
                linearly_separable,
                xor_scale_invariant(n_samples=n_samples)]
    return (names, datasets)

def get_dataset(dataset, n_samples):
    # Generate the new data:
    if dataset=='Noisy Circles':
        X, y = datasets.make_circles(n_samples=n_samples, factor=.5, noise=.05)
    elif dataset=='Noisy Moons':
        X, y = datasets.make_moons(n_samples=n_samples, noise=.05)
    elif dataset=='Blobs':
        X, y = datasets.make_blobs(n_samples=n_samples, random_state=8)
    else:
        X, y = np.random.rand(n_samples, 2), None

    return X, y