def __init__(
        self,
        input,
        n_in=784,
        n_hidden=500,
        W=None,
        hbias=None,
        vbias=None,
        numpy_rng=None,
        transpose=False,
        activation=T.nnet.sigmoid,
        theano_rng=None,
        name="grbm",
        W_r=None,
        dropout=0,
        dropconnect=0,
    ):

        # initialize parent class (RBM)
        RBM.__init__(
            self,
            input=input,
            n_visible=n_in,
            n_hidden=n_hidden,
            W=W,
            hbias=hbias,
            vbias=vbias,
            numpy_rng=numpy_rng,
            theano_rng=theano_rng,
        )

    def __init__(self, input=None, n_visible=784, n_hidden=500,
                 W=None, h_bias=None, v_bias=None, numpy_rng=None, theano_rng=None):
        """
        GBRBM constructor. Defines the parameters of the model along with
        basic operations for inferring hidden from visible (and vice-versa).
        It initialize parent class (RBM).

        :param input: None for standalone RBMs or symbolic variable if RBM is part of a larger graph.

        :param n_visible: number of visible units

        :param n_hidden: number of hidden units

        :param W: None for standalone RBMs or symbolic variable pointing to a
        shared weight matrix in case RBM is part of a DBN network; in a DBN,
        the weights are shared between RBMs and layers of a MLP

        :param h_bias: None for standalone RBMs or symbolic variable pointing
        to a shared hidden units bias vector in case RBM is part of a
        different network

        :param v_bias: None for standalone RBMs or a symbolic variable
        pointing to a shared visible units bias
        """
        RBM.__init__(
            self,
            input=input,
            n_visible=n_visible,
            n_hidden=n_hidden,
            W=W, h_bias=h_bias,
            v_bias=v_bias,
            numpy_rng=numpy_rng,
            theano_rng=theano_rng)

 def __init__(
     self,
     input,
     n_visible=784,
     n_hidden=500,
     W=None,
     hbias=None,
     vbias=None,
     numpy_rng=None,
     transpose=False,
     theano_rng=None,
     weight_decay=0.0002,
 ):
     RBM.__init__(
         self,
         input=input,
         n_visible=n_visible,
         n_hidden=n_hidden,
         W=W,
         hbias=hbias,
         vbias=vbias,
         numpy_rng=numpy_rng,
         theano_rng=theano_rng,
         weight_decay=weight_decay,
     )

def pretrain_rbm_layers(v, validation_v=None, n_hidden=[], gibbs_steps=[], batch_size=[], num_epochs=[], learning_rate=[], probe_epochs=[]):
    rbm_layers = []
    n_rbm = len(n_hidden)
    # create rbm layers
    for i in range(n_rbm):
        rbm = RBM(n_hidden=n_hidden[i],
                    gibbs_steps=gibbs_steps[i],
                    batch_size=batch_size[i],
                    num_epochs=num_epochs[i],
                    learning_rate=learning_rate[i],
                    probe_epochs=probe_epochs[i])
        rbm_layers.append(rbm)
    # pretrain rbm layers
    input = v
    validation_input = validation_v
    for rbm, i in zip(rbm_layers, range(len(rbm_layers))):
        print '### pretraining RBM Layer {i}'.format(i=i)
        rbm.fit(input, validation_input)
        output = rbm.sample_h_given_v(input, rbm.params['W'], rbm.params['c'])
        if validation_input is not None:
            validation_output = rbm.sample_h_given_v(validation_input, rbm.params['W'], rbm.params['c'])
        else:
            validation_output = None
        input = output
        validation_input = validation_output
    return rbm_layers

class SFG:
  
  def __init__(self):
    self.image_width = self.image_height = 28
    self.visible_units = self.image_width * self.image_height
    self.hidden_units = self.visible_units / 10
    self.rbm = RBM(self.visible_units, self.hidden_units)

  #assumes there are only training images in the training_folder
  def train(self, training_folder, epochs = 500):
    data = []
    for training_image in os.listdir(training_folder):
      image = pil.open(training_folder + '/' + training_image)
      image = self.array_for_image(image)
      data.append(image)

    self.rbm.train(data, epochs)
  
  #takes a pil Image and returns an arary of 1s and 0s
  def array_for_image(self, image):
    return np.array(image.convert("L")).flatten() / 255

  def regen_image(self, image, samples):
    data = self.array_for_image(image)
    (v, _) = self.rbm.regenerate([data],samples)
    return self.image_for_array(v[0])

  def image_for_array(self, array):
    img_array = []
    for row in range(0, self.image_height):
      img_array.append(array[row * self.image_width : (row+1) * self.image_width])

    img_array = np.asarray(img_array, np.uint8) * 255
    return pil.fromarray(img_array)

    def pretrainRBM(self,trainset):
        trainv = np.mat(trainset[1])   # 1xn
        vlen = trainv.shape[1]
        trainnum = len(trainset)
        hlen = 500
        weights = []
        print "vlen = %d" %(vlen)
        print "Trainnum = %d" %(trainnum)
        for i in range(self.nlayers):
            rbm = RBM(vlen,hlen)
            T,e = 3,0.05
            if i == 0:
                traindata = trainset
            else:
                traindata = outdata
            outdata = np.zeros((trainnum,hlen))
            for j in range(trainnum):
                print "layer:%d CD sample %d..." %(i,j)
                trainv = np.mat(traindata[j])
                rbm.train_CD(trainv,T,e)
                outdata[j] = np.mat(rbm.sample(rbm.calc_forward(trainv)))   # 1xhlen
            self.rbm_layers.append(rbm)
            weights.append(rbm.W)
            vlen = hlen
#            hlen -= 100
        dump_data("data/dbn.pkl",weights)
        print "========= pretrainRBM complete ==========="

    def __init__(self,
                 input,
                 n_visible=16,
                 n_hidden=20,                 
                 W=None, hbias=None, vbias=None,
                 numpy_rng=None, theano_rng=None):

            # initialize parent class (RBM)
            RBM.__init__(self,
                         input=input,
                         n_visible=n_visible,
                         n_hidden=n_hidden,
                         W=W, hbias=hbias, vbias=vbias,
                         numpy_rng=numpy_rng, theano_rng=theano_rng)

 def load_dbn_param(self,dbnpath,softmaxpath):
     weights = cPickle.load(open(dbnpath,'rb'))
     vlen,hlen = 0,0
     self.nlayers = len(weights)
     for i in range(self.nlayers):
         weight = weights[i]
         vlen,hlen = weight.shape[0],weight.shape[1]
         rbm = RBM(vlen,hlen)
         rbm.W = weight
         self.rbm_layers.append(rbm)
         print "RBM layer%d shape:%s" %(i,str(rbm.W.shape))
     self.softmax = SoftMax()
     self.softmax.load_theta(softmaxpath)
     print "softmax parameter: "+str(self.softmax.theta.shape)

 def load_from_matfile(cls, matfilename):
     data = loadmat(matfilename)
     stack_data = data.get('stack_data')
     numrbms = data.get('numrbms')
     rbms = []
     for mac_i in range(numrbms):
         vbias = data.get(str(mac_i)+"_visbias")
         hbias = data.get(str(mac_i)+"_hidbias")
         vishid = data.get(str(mac_i)+"_vishid")
         rbm = RBM(vbias.size, hbias.size)
         rbm.get_vislayer().bias = vbias
         rbm.get_hidlayer().bias = hbias
         rbm.weights[0] = vishid
         rbms.append(rbm)
     return cls(stack_data, rbms)

class RBMTest(unittest.TestCase):
    def setUp(self):
        self.rbm = RBM(10,10)

    def can_make_rbm_test(self):
        rbm = RBM(10, 10)

    def logistic_function_test(self):
        self.assertEquals(self.rbm.logistic(0), 1)

    def train_throws_error_with_inconsistent_matrix_sizes_test(self):
        with self.assertRaises(TypeError):
            self.rbm.train([[1,0,1,1,1,1,0,1], [1,1,1,1,0], [1,1,1,1,1,1]])

    def regenerate_throws_error_with_inconsistent_matrix_sizes_test(self):
        with self.assertRaises(TypeError):
            self.rbm.regenerate([[1,0,1,1,1,1,0,1], [1,1,1,1,0], [1,1,1,1,1,1]])

    def fit_network(self, X, labels=None):
        if labels is None:
            labels = numpy.zeros((X.shape[0], 2))
        self.layers = []
        temp_X = X
        for j in range(self.num_layers):

            print "\nTraining Layer %i" % (j + 1)
            print "components: %i" % self.components[j]
            print "batch_size: %i" % self.batch_size[j]
            print "learning_rate: %0.3f" % self.learning_rate[j]
            print "bias_learning_rate: %0.3f" % self.bias_learning_rate[j]
            print "epochs: %i" % self.epochs[j]
            print "Sparsity: %s" % str(self.sparsity_rate[j])
            print "Sparsity Phi: %s" % str(self.phi)
            if j != 0:
                self.plot_weights = False

            model = RBM(n_components=self.components[j], batch_size=self.batch_size[j],
                        learning_rate=self.learning_rate[j], regularization_mu=self.sparsity_rate[j],
                        n_iter=self.epochs[j], verbose=True, learning_rate_bias=self.bias_learning_rate[j],
                        plot_weights=self.plot_weights, plot_histograms=self.plot_histograms, phi=self.phi)

            if j + 1 == self.num_layers and labels is not None:
                model.fit(numpy.asarray(temp_X), numpy.asarray(labels))
            else:
                model.fit(numpy.asarray(temp_X))

            temp_X = model._mean_hiddens(temp_X)  # hidden layer given visable units
            print "Trained Layer %i\n" % (j + 1)

            self.layers.append(model)

	def _ulogprob_hid(self, Y, num_is_samples=100):
		"""
		Estimates the unnormalized marginal log-probabilities of hidden states.
		
		Use this method only if you know what you are doing.
		"""

		# approximate this SRBM with an RBM
		rbm = RBM(self.X.shape[0], self.Y.shape[0])
		rbm.W = self.W
		rbm.b = self.b
		rbm.c = self.c

		# allocate memory
		Q = np.asmatrix(np.zeros([num_is_samples, Y.shape[1]]))

		for k in range(num_is_samples):
			# draw importance samples
			X = rbm.backward(Y)

			# store importance weights
			Q[k, :] = self._ulogprob(X, Y) - rbm._clogprob_vis_hid(X, Y)

		# average importance weights to get estimates
		return utils.logmeanexp(Q, 0)

def pretrain_rbm_layers(v, validation_v=None, n_hidden=[], gibbs_steps=[], batch_size=[], num_epochs=[], learning_rate=[], probe_epochs=[]):
    """
    Fake pre-training, just randomly initialising the weights of RBM layers
    :param v:
    :param validation_v:
    :param n_hidden:
    :param gibbs_steps:
    :param batch_size:
    :param num_epochs:
    :param learning_rate:
    :param probe_epochs:
    :return:
    """
    rbm_layers = []
    n_rbm = len(n_hidden)
    # create rbm layers
    for i in range(n_rbm):
        rbm = RBM(n_hidden=n_hidden[i],
                    gibbs_steps=gibbs_steps[i],
                    batch_size=batch_size[i],
                    num_epochs=num_epochs[i],
                    learning_rate=learning_rate[i],
                    probe_epochs=probe_epochs[i])
        rbm_layers.append(rbm)
    # pretrain rbm layers
    n_v = v.shape[1]
    for rbm, i in zip(rbm_layers, range(len(rbm_layers))):
        print '### pretraining RBM Layer {i}'.format(i=i)
        n_h = n_hidden[i]
        initial_W = np.float32(np.random.uniform(
            low=-4 * np.sqrt(6.0 / (n_h + n_v)),
            high=4 * np.sqrt(6.0 / (n_h + n_v)),
            size=(n_v, n_h)
        ))
        rbm.params['W'] = initial_W
        rbm.params['c'] = np.zeros((n_h, ), np.float32)
        n_v = n_h
    return rbm_layers

def test(learning_rate=0.1, k=1, training_epochs=15):
  print '... loading data'

  datasets = load_data('mnist.pkl.gz')
  train_set_x, train_set_y = datasets[0]
  test_set_x, test_set_y = datasets[2]

  print '... modeling'

  rbm = RBM(input=train_set_x, n_visible=28 * 28, n_hidden=500)

  print '... training'

  start_time = time.clock()

  for epoch in xrange(training_epochs):
    cost = rbm.get_cost_updates(lr=learning_rate, k=k)
    print 'Training epoch %d, cost is ' % epoch, cost

  end_time = time.clock()
  pretraining_time = (end_time - start_time)

  print ('Training took %f minutes' % (pretraining_time / 60.))

    def __init__(self,knapsack_file="weing1.pkl"):
        super(ES, self).__init__()
        # GA stuff
        self.generations = 100
        self.knapsack = pickle.load(open(knapsack_file))
        print "k:",self.knapsack
        self.N = int(self.knapsack.items)
        # RMB stuff
        self.RBM = RBM(n_visible=self.N,n_hidden=50) 
        self.sample_RBM()

        # Stats stuff
        self.population_snapshots = []
        self.genotypes_history = Genotypes(min=False)

def get_representation():
	# Load the dictionary and corresponding args.
	(W, b, hidden_size) = pickle.load(open("Models/RBM/model%d.pkl"%experiment_number,'rb'))

	# Set the constructor
	myObject = RBM(hidden_size=hidden_size)

	print "Loading dataset..."
	trainset,validset,testset = dataset_store.get_classification_problem('ocr_letters')

	encoded_trainset = []
	encoded_validset = []
	encoded_testset = []

	print "Initializing..."
	myObject.initialize(W,b)

	print "Encoding the trainset..."
	counter = 0 #Inelegant, I know! I use this to only use the first 1000 values.
	for input,target in trainset:    
		#Encode the sample.
		h = myObject.encode(input)
		encoded_trainset.append(h)

		# counter +=1
		# if counter == 1000:
		#     break

	# Save the datasets to files. 
	filename = "Models/RBM/trainset%d.pkl"%(experiment_number)
	pickle.dump( np.asarray(encoded_trainset) , open(filename, 'wb'))

	counter = 0
	print "Encoding the validset..."
	for input,target in validset:
		#Encode the sample.
		h = myObject.encode(input)
		encoded_validset.append(h)

		# counter +=1
		# if counter == 1000:
		#     break

	filename = "Models/RBM/validset%d.pkl"%(experiment_number)
	pickle.dump( np.asarray(encoded_validset) , open(filename, 'wb'))

	#Note: only need to do it for the best hyper-params at the end.	
	print "Encoding the testset..."
	for input,target in testset:
		#Encode the sample.
		h = myObject.encode(input)
		encoded_testset.append(h)	    

	filename = "Models/RBM/testset%d.pkl"%(experiment_number)
	pickle.dump( np.asarray(encoded_testset), open(filename, 'wb'))

def train(dim_h, lrate, max_epoches, batch_size, train_ratio):
    """
    traing a model and save to disk
    """
    result_file = "mnist_%s.pkl" % strftime("%b_%d_%H_%M_%S", gmtime())

    if os.access(result_file, os.R_OK):
        raise RuntimeError("%s already exists" % result_file)

    data = load_digits()
    learner = RBM.train(
        data, dim_h=dim_h, lrate=lrate, max_epoches=max_epoches, batch_size=batch_size, train_ratio=train_ratio
    )

    with open(result_file, "wb") as handle:
        pickle.dump(learner, handle)
    print('learner sleeping at "%s"' % result_file)

def test_rbm_mnist(learning_rate=0.01, training_epochs=10, batch_size=20,
                   n_chains=30, n_samples=5, output_folder=None, isPCD=0,
                   n_hidden=500):
    """
    Demonstrate how to train and afterwards sample from it using Theano.

    This is demonstrated on MNIST.

    :param learning_rate: learning rate used for training the RBM

    :param training_epochs: number of epochs used for training

    :param dataset: path the the pickled dataset

    :param batch_size: size of a batch used to train the RBM

    :param n_chains: number of parallel Gibbs chains to be used for sampling

    :param n_samples: number of samples to plot for each chain

    e.g.
        test_rbm_mnist(output_folder='/home/eric/Desktop/rbm_plots')

    """

    assert output_folder is not None

    from rbm_variants import RBM_Orthogonal as RBM
    # from rbm import RBM

    #################################
    #     Data Constructing         #
    #################################

    from sklearn.datasets import fetch_mldata

    mnist = fetch_mldata('MNIST original')

    from xylearn.utils.data_util import get_train_test
    from xylearn.utils.data_normalization import rescale

    data = get_train_test(rescale(mnist.data), mnist.target, useGPU=1, shuffle=True)

    train_x, train_y = data['train']
    n_vis = train_x.get_value(borrow=True).shape[1]

    print numpy.linalg.matrix_rank(train_x.get_value(borrow=True))

    n_train_batches = train_x.get_value(borrow=True).shape[0] / batch_size


    # construct the RBM class
    rbm = RBM(n_visible=n_vis, n_hidden=n_hidden, isPCD=isPCD)
    train_fn = rbm.get_train_fn(train_x, batch_size)


    #################################
    #     Training the RBM          #
    #################################
    if not os.path.isdir(output_folder):
        os.makedirs(output_folder)
    os.chdir(output_folder)

    plotting_time = 0.
    start_time = time.clock()
    import PIL.Image
    from visualizer import tile_raster_images

    # go through training epochs
    for epoch in xrange(training_epochs):

        # go through the training set
        mean_cost = []
        for batch_index in xrange(n_train_batches):
            # for each batch, we extract the gibbs chain
            new_cost = train_fn(index=batch_index, lr=learning_rate)
            mean_cost += [new_cost]

        print 'Training epoch %d, cost is ' % epoch, numpy.mean(mean_cost)
        # monitor projected rank
        projection = rbm.project(train_x)
        print 'rank: ' + str(numpy.linalg.matrix_rank(projection))

        # W shape is [784 500]
        # Plot filters after each training epoch
        plotting_start = time.clock()
        # Construct image from the weight matrix
        image = PIL.Image.fromarray(tile_raster_images(
            X=rbm.W.get_value(borrow=True).T,
            img_shape=(28, 28), tile_shape=(20, 20),
            tile_spacing=(1, 1)))
        image.save('filters_at_epoch_%i.png' % epoch)
        plotting_stop = time.clock()
        plotting_time += (plotting_stop - plotting_start)

    end_time = time.clock()
    pretraining_time = (end_time - start_time) - plotting_time
    print ('Training took %f minutes' % (pretraining_time / 60.))


#.........这里部分代码省略.........

def toy_test(learning_rate=0.01, training_epochs=100, batch_size=50,
             output_folder=None, isPCD=0,
             n_hidden=3):
    assert output_folder is not None
    # toy_data, word count vector, [num_terms, num_doc].
    # each cell represents the number of times a term occurs
    #                          d1 d2 d3 d4 d5
    toy_data = numpy.asarray([[0, 2, 0, 1, 0],
                              [9, 0, 3, 1, 1],
                              [4, 1, 1, 2, 1],
                              [10, 10, 1, 1, 0],
                              [1, 0, 8, 0, 10],
                              [0, 1, 10, 1, 0],
                              [1, 0, 2, 6, 1],
                              [0, 0, 1, 0, 0],
                              [1, 0, 0, 0, 0],
                              [1, 0, 1, 0, 0],
                              [1, 1, 0, 0, 1],
                              [10, 2, 0, 1, 0],
                              [0, 0, 1, 0, 10],
                              [1, 0, 0, 3, 0],
                              [0, 0, 2, 0, 1],
                              [10, 0, 1, 0, 0],
                              [0, 1, 0, 0, 0],
                              [0, 1, 0, 1, 0],
                              [1, 0, 1, 0, 0],
                              [1, 0, 0, 0, 1],
                              [1, 0, 1, 0, 0],
                              [0, 0, 1, 0, 0]])

    # from rbm import RBM
    from rbm_variants import RBM_Orthogonal as RBM
    # from rbm_variants import PoissonRBM as RBM


    train_x = toSharedX(toy_data, name="toy_data")

    n_vis = train_x.get_value(borrow=True).shape[1]

    n_samples = train_x.get_value(borrow=True).shape[0]

    if batch_size >= n_samples:
        batch_size = n_samples

    n_train_batches = n_samples / batch_size


    # construct the RBM class
    rbm = RBM(n_visible=n_vis, n_hidden=n_hidden, isPCD=isPCD)
    train_fn = rbm.get_train_fn(train_x, batch_size)

    print "... projecting"
    print rbm.project(train_x, hidSample=1)

    #################################
    #     Training the RBM          #
    #################################
    if not os.path.isdir(output_folder):
        os.makedirs(output_folder)
    os.chdir(output_folder)

    plotting_time = 0.
    start_time = time.clock()
    import PIL.Image
    from visualizer import tile_raster_images

    # go through training epochs
    for epoch in xrange(training_epochs):

        # go through the training set
        mean_cost = []
        for batch_index in xrange(n_train_batches):
            # for each batch, we extract the gibbs chain
            new_cost = train_fn(index=batch_index, lr=learning_rate)
            mean_cost += [new_cost]

        print 'Training epoch %d, cost is ' % epoch, numpy.mean(mean_cost)

        if numpy.mean(mean_cost) >= 0:
            break

        # W shape is [784 500]
        # Plot filters after each training epoch
        plotting_start = time.clock()
        # Construct image from the weight matrix
        image = PIL.Image.fromarray(tile_raster_images(
            X=rbm.W.get_value(borrow=True).T,
            # weight is [n_vis, n_hidden]
            # so, among 'n_hidden' rows,
            # each row corresponds to propdown one hidden unit
            img_shape=(1, n_vis), tile_shape=(n_hidden, 1),
            tile_spacing=(1, 1)))
        image.save('filters_at_epoch_%i.png' % epoch)
        plotting_stop = time.clock()
        plotting_time += (plotting_stop - plotting_start)

    end_time = time.clock()
    pretraining_time = (end_time - start_time) - plotting_time
    print ('Training took %f minutes' % (pretraining_time / 60.))

#.........这里部分代码省略.........

 def __init__(self):
   self.image_width = self.image_height = 25
   self.visible_units = self.image_width * self.image_height
   self.hidden_units = 400
   self.rbm = RBM(self.visible_units, self.hidden_units, 0.05)