def main():
    # Load the data
    full_td, _, _ = mnist_loader.load_data_wrapper()
    td = full_td[:1000]  # Just use the first 1000 items of training data
    epochs = 500  # Number of epochs to train for

    print "\nTwo hidden layers:"
    net = network2.Network([784, 30, 30, 10])
    initial_norms(td, net)
    abbreviated_gradient = [ag[:6] for ag in get_average_gradient(net, td)[:-1]]
    print "Saving the averaged gradient for the top six neurons in each " + "layer.\nWARNING: This will affect the look of the book, so be " + "sure to check the\nrelevant material (early chapter 5)."
    f = open("initial_gradient.json", "w")
    json.dump(abbreviated_gradient, f)
    f.close()
    shutil.copy("initial_gradient.json", "../../js/initial_gradient.json")
    training(td, net, epochs, "norms_during_training_2_layers.json")
    plot_training(epochs, "norms_during_training_2_layers.json", 2)

    print "\nThree hidden layers:"
    net = network2.Network([784, 30, 30, 30, 10])
    initial_norms(td, net)
    training(td, net, epochs, "norms_during_training_3_layers.json")
    plot_training(epochs, "norms_during_training_3_layers.json", 3)

    print "\nFour hidden layers:"
    net = network2.Network([784, 30, 30, 30, 30, 10])
    initial_norms(td, net)
    training(td, net, epochs, "norms_during_training_4_layers.json")
    plot_training(epochs, "norms_during_training_4_layers.json", 4)

def visualize():
  training_data, validation_data, test_data = mnist_loader.load_data_wrapper()
  # Unzipping gives tuples, but we want arrays of values.
  training_input = [x.transpose()[0] for x in zip(*training_data)[0]]
  test_input = [x.transpose()[0] for x in zip(*test_data)[0]]
  # Get the y values.
  test_target = [y for y in zip(*test_data)[1]]

  # Apply SVD to the training input.
  u, s, v = np.linalg.svd(training_input, full_matrices=False)
  print u.shape
  print s.shape
  print v.shape
  
  print "Generating embeddings..."
  #print v[0]
  print v[0].shape
  embeddings = [np.dot(test_inp, np.transpose(v[:10][:])) for test_inp in test_input]
  print embeddings[0].shape
  
  # Do dimensionality reduction into 2 dimensions.
  print "Performing dimensionality reduction using t-sne..."
  tsne = TSNE()
  reduced_vecs = tsne.fit_transform(embeddings)
  print reduced_vecs[0]

  # Graph all of the points, where points corresponding to the same digit will have the same color.
  colors = ['r', 'b', 'g', 'c', 'm', 'k', 'y', (.2, .2, .2), (.4, 0, .5), (.8, .2, 0)]
  red_patch = mpatches.Patch(color='red', label='1')
  patches = [mpatches.Patch(color=colors[i], label='%i'% i) for i in range(len(colors))]
  plt.legend(handles=patches)
  for i in range(len(reduced_vecs)):
    plt.plot([reduced_vecs[i][0]], [reduced_vecs[i][1]], 'o', color=colors[test_target[i]])
  plt.show()

def run_network(filename, n, eta):
    """Train the network using both the default and the large starting
    weights.  Store the results in the file with name ``filename``,
    where they can later be used by ``make_plots``.

    """
    # Make results more easily reproducible
    random.seed(12345678)
    np.random.seed(12345678)
    training_data, validation_data, test_data = mnist_loader.load_data_wrapper()
    net = network2.Network([784, n, 10], cost=network2.CrossEntropyCost)
    print "Train the network using the default starting weights."
    default_vc, default_va, default_tc, default_ta = net.SGD(
        training_data, 30, 10, eta, lmbda=5.0, evaluation_data=validation_data, monitor_evaluation_accuracy=True
    )
    print "Train the network using the large starting weights."
    net.large_weight_initializer()
    large_vc, large_va, large_tc, large_ta = net.SGD(
        training_data, 30, 10, eta, lmbda=5.0, evaluation_data=validation_data, monitor_evaluation_accuracy=True
    )
    f = open(filename, "w")
    json.dump(
        {
            "default_weight_initialization": [default_vc, default_va, default_tc, default_ta],
            "large_weight_initialization": [large_vc, large_va, large_tc, large_ta],
        },
        f,
    )
    f.close()

def run_network(filename, num_epochs, training_set_size=1000, lmbda=0.0):
    """Train the network for ``num_epochs`` on ``training_set_size``
    images, and store the results in ``filename``.  Those results can
    later be used by ``make_plots``.  Note that the results are stored
    to disk in large part because it's convenient not to have to
    ``run_network`` each time we want to make a plot (it's slow).

    """
    # Make results more easily reproducible
    random.seed(12345678)
    np.random.seed(12345678)
    training_data, validation_data, test_data = mnist_loader.load_data_wrapper()
    #net = network2.Network([784, 30, 10], cost=network2.CrossEntropyCost)
    net = network2.Network([784, 30, 10], cost=network2.LogLikelihoodCost)
    net.large_weight_initializer()
    test_cost, test_accuracy, training_cost, training_accuracy, output_activations \
        = net.SGD(training_data[:training_set_size], num_epochs, 10, 0.5,
                  evaluation_data=test_data, lmbda = lmbda,
                  monitor_evaluation_cost=True, 
                  monitor_evaluation_accuracy=True, 
                  monitor_training_cost=True, 
                  monitor_training_accuracy=True)
    f = open(filename, "w")
    '''
    print "test_cost",type(test_cost),test_cost
    print
    print "test_accuracy",type(test_accuracy), test_accuracy
    print
    print "training_cost",type(training_cost), training_cost
    print 
    print "training_accuracy",type(training_accuracy), training_cost
    print
    '''
    json.dump([test_cost, test_accuracy, training_cost, training_accuracy, output_activations], f)
    f.close()

def run():
    np.random.seed(0)
    data = load_data_wrapper()
    initial_learning_rate = 0.2
    network = Network(
        [784, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 10],
        activations_function=ReLU(),
        cost=CrossEntropy(),
        stopping_criteria=NEpochs(50),  # LearningRateDecreaseLimit(
        #     initial_learning_rate=initial_learning_rate,
        #     limit=1/2
        # ),
        learning_rate=FixedLearningRate(0.01),  # HalfLRIfNoDecreaseInNEpochs(
        #     monitor_parameter='validation_cost',
        #     max_epochs=1,
        #     initial_learning_rate=initial_learning_rate
        # ),
        update_algorithm=Momentum(momentum=0.5, base_algorithm=L2UpdateAlgorithm(lmbda=5)),
        weight_initializer=initialize_input_dim_normalized_weights,
    )
    t0 = datetime.utcnow()
    network.sgd(training_data=data[0], validation_data=data[1], test_data=data[2], mini_batch_size=12)
    print("Total time fast {}".format(datetime.utcnow() - t0))
    plot_stats(network)
    return network.log

def main():
    f = open('myNeural.txt', 'r')
    net = pickle.load(f)
    training_data, validation_data, test_data = mnist_loader.load_data_wrapper()
    # 处理为 scikit_learn所需的数据结构
    X_train = [np.reshape(x, (1, -1))[0] for (x, y) in training_data]
    y_train = [np.argmax(np.reshape(y, (1, -1))[0]) for (x, y) in training_data]

    # Fit estimators
    ESTIMATORS = {

        # KNN
        "K-nn": neighbors.KNeighborsClassifier().fit(X_train, y_train),
        # 朴素贝叶斯
        "native-bayes": BernoulliNB().fit(X_train, y_train)
        # 决策树
        # 聚类
    }

    for i in test_data:
        print '=================================='
        testdata = i[0]
        print '正确的结果为%d' % i[1]
        # 使用神经元网络验证
        result = np.argmax(net.feedforward(testdata))
        print '使用神经元网络分类的结果为', result
        for name, estimator in ESTIMATORS.items():
            print '使用%s进行分类，分类结果为%s' % (name, estimator.predict(np.reshape(testdata, (1, -1))))

def run_network(filename):
    """Train the network using both the default and the large starting
    weights.  Store the results in the file with name ``filename``,
    where they can later be used by ``make_plots``.

    """
    training_data, validation_data, test_data = mnist_loader.load_data_wrapper()
    net = network2.Network([784, 30, 10], cost=network2.CrossEntropyCost())
    print "Train the network using the default starting weights."
    default_vc, default_va, default_tc, default_ta \
        = net.SGD(training_data, 30, 10, 0.01,
                  evaluation_data=validation_data, lmbda = 0.001,
                  monitor_evaluation_accuracy=True)
    print "Train the network using the large starting weights."
    net.large_weight_initializer()
    large_vc, large_va, large_tc, large_ta \
        = net.SGD(training_data, 30, 10, 0.01,
                  evaluation_data=validation_data, lmbda = 0.001,
                  monitor_evaluation_accuracy=True)
    f = open(filename, "w")
    json.dump({"default_weight_initialization":
               [default_vc, default_va, default_tc, default_ta],
               "large_weight_initialization":
               [large_vc, large_va, large_tc, large_ta]}, 
              f)
    f.close()

    def __init__(self, fileNames=[]):
        self.fileNames = fileNames
        self.trainingData,self.valData,self.testData = \
            mnist_loader.load_data_wrapper()
        self.data = [self.trainingData, self.valData, self.testData]

        self.fileNameDataPairs = zip(self.fileNames, self.data)

def trainnetwork():
     
     training_data, validation_data, test_data = mnist_loader.load_data_wrapper()
     
     network= Network([1200,30,10])
     network.SGD(training_data,30,10,.5,test_data=test_data)
     network.save("network.txt")

def run_networks():
    """Train networks using three different values for the learning rate,
    and store the cost curves in the file ``multiple_eta.json``, where
    they can later be used by ``make_plot``.

    """
    # Make results more easily reproducible
    random.seed(12345678)
    np.random.seed(12345678)
    training_data, validation_data, test_data = mnist_loader.load_data_wrapper()
    results = []
    for eta in LEARNING_RATES:
        print "\nTrain a network using eta = " + str(eta)
        net = network2.Network([784, 30, 10])
        results.append(
            net.SGD(
                training_data,
                NUM_EPOCHS,
                10,
                eta,
                lmbda=5.0,
                evaluation_data=validation_data,
                monitor_training_cost=True,
            )
        )
    f = open("multiple_eta.json", "w")
    json.dump(results, f)
    f.close()

def load_data():
    train, val, test = mnist_loader.load_data_wrapper()
    train = list(train)
    val = list(val)
    test = list(test)

    return train, val, test

def train_mnist_worker(params):
    net_id = params.get('net-id', 'nn')
    layers = [784]
    layers.extend([int(i) for i in params.get('layers', [15])])
    layers.append(10)
    net_params                    = {}
    net_params['epochs']          = int(params.get('epochs', 1))
    net_params['mini_batch_size'] = int(params.get('mini-batch-size', 4))
    net_params['eta']             = float(params.get('eta', 0.1))
    net_params['lmbda']           = float(params.get('lmbda', 0.0001))
    net_params['layers']          = layers

    redis.set(redis_key('params', net_id), json.dumps(net_params))
    redis.set(redis_key('status', net_id), 'train_mnist: started')

    net = Network(layers)
    training_data, validation_data, test_data = load_data_wrapper()
    redis.set(redis_key('status', net_id), 'train_mnist: training with mnist data')
    net.SGD(training_data, net_params['epochs'],
                           net_params['mini_batch_size'],
                           net_params['eta'],
                           net_params['lmbda'])

    redis.set(redis_key('data', net_id), net.tostring())
    redis.set(redis_key('status', net_id), 'train_mnist: trained')

def convert_test():  
    fulltrain = pd.read_csv('dat/train.csv')
    trainx = fulltrain.drop(['Id','Cover_Type'], axis=1) # Features
    trainy = fulltrain['Cover_Type'] # Target

    newtrain = train_pd_to_nielsen(trainx, trainy, trainy.max())

    print np.asarray(newtrain).shape
    print np.asarray(newtrain)[0][0].shape
    print np.asarray(newtrain)[0][1].shape

    nisttrain, nistvalid, nisttest = mnist_loader.load_data_wrapper(nielsen_path + 'data/mnist.pkl.gz')
    print np.asarray(nisttrain).shape
    print np.asarray(nisttrain)[0][0].shape
    print np.asarray(nisttrain)[0][1].shape

    print '-'*50
    newtest = test_pd_to_nielsen(trainx, trainy)

    print np.asarray(newtest).shape
    print np.asarray(newtest)[0][0].shape
    print np.asarray(newtest)[0][1].shape

    print np.asarray(nistvalid).shape
    print np.asarray(nistvalid)[0][0].shape
    print np.asarray(nistvalid)[0][1].shape

    def __init__(self, sizes):
        print("Starting...");
        start = time.time()
        training_data, validation_data, test_data = \
	       mnist_loader.load_data_wrapper()
        self.net = network.Network(sizes)
        (self.net).SGD(training_data, 30, 10, 3.0, test_data=test_data)
        end = time.time()
        print("Done!\nTime Taken: " + str(end - start));

def learn(request):
    tr_d, v_d, t_d = mnist_loader.load_data_wrapper()
    net.SGD(tr_d,30,10,3.0)
    print net.evaluate(t_d)
    return HttpResponseRedirect('/')  



            

def recognize(img_name):
	training_data, validation_data, test_data = mnist_loader.load_data_wrapper()
	img_list= loadImage(img_name)
	net = network.Network([784, 100, 10])  #input,hidden,output
	#net.SGD(training_data, 5, 10, 2.0, test_data = test_data)
	f=open('100hl.bin','rb')
	net.biases = np.load(f)	
	net.weights = np.load(f)	 # load trained weights  and biases
	f.close() #close the file after reading weights and biases
	#check for image
	return net.feedforward(img_list)

def main():
    # Time to processs dataset
    start_process = timeit.default_timer()

    training_data, validation_data, test_data = mnist_loader.load_data_wrapper()

    net = NeuralNetwork([784, 30, 10])
    net.SGD(training_data, 30, 10, 3.0, test_data=test_data)

    time_neural = timeit.default_timer() - start_process

    print "The total time to run the neural network was: %d seconds" %(int(time_neural))

def main():
    filename = 'test'
    # load data
    training_data, validation_data, test_data = mnist_loader.load_data_wrapper()
    # example
    epochs = 5
    net = network2.Network([784, 10], cost=network2.CrossEntropyCost)
    vc, va, tc, ta = net.SGD(training_data=training_data, epochs=epochs, mini_batch_size=100, eta=0.1, lmbda = 0.1, reg = 2,
        evaluation_data=validation_data, 
        monitor_evaluation_cost=True,
        monitor_evaluation_accuracy=True,
        monitor_training_cost=True,
        monitor_training_accuracy=True)

def run_networks():
    # Make results more easily reproducible    
    random.seed(12345678)
    np.random.seed(12345678)
    training_data, validation_data, test_data = mnist_loader.load_data_wrapper()
    # instantiate network
    net = nnetwork.Network([INPUT_NEURONS,HIDDEN_NEURONS,OUTPUT_NEURONS])
    # run SGD
    results = net.gradientDescent(training_data, BATCH_SIZE, eta, NUM_EPOCHS,
                    test_data=test_data)
    f = open("learning.json", "w")
    json.dump(results, f)
    f.close()

def test_cross_entropy_cost():
	training_data, validation_data, test_data = mnist_loader.load_data_wrapper()
	net2 = network2.Network([784, 30, 10], cost=network2.CrossEntropyCost)
	net2.large_weight_initializer()
	net2.SGD(training_data, 30, 10, 0.5, evaluation_data=test_data, monitor_evaluation_accuracy=True)


# import network
# from mnist_load import MnistLoad
# loader = MnistLoad('../data-new/')
# train_data, test_data = loader.loadAsNumpyData()
# net = network.Network([784, 30, 10])
# net.SGD(train_data, 30, 10, 3.0, test_data = test_data)