def __init__(self,
                gen_params, # dictionary of generative model parameters
                GEN_MODEL,  # class that inherits from GenerativeModel
                rec_params, # dictionary of approximate posterior ("recognition model") parameters
                REC_MODEL, # class that inherits from RecognitionModel
                xDim=2, # dimensionality of latent state
                yDim=2 # dimensionality of observations
                ):

        # instantiate rng's
        self.srng = RandomStreams(seed=234)
        self.nrng = np.random.RandomState(124)

        #---------------------------------------------------------
        ## actual model parameters
        self.X, self.Y = T.matrices('X','Y')   # symbolic variables for the data

        self.xDim   = xDim
        self.yDim   = yDim

        # instantiate our prior & recognition models
        self.mrec   = REC_MODEL(rec_params, self.Y, self.xDim, self.yDim, self.srng, self.nrng)
        self.mprior = GEN_MODEL(gen_params, self.xDim, self.yDim, srng=self.srng, nrng = self.nrng)

        self.isTrainingRecognitionModel = True;
        self.isTrainingGenerativeModel = True;

    def _gpu_matrix_dot(matrix_a, matrix_b, matrix_c=None):
        """
        Performs matrix multiplication.
        Attempts to use the GPU if it's available. If the matrix multiplication
        is too big to fit on the GPU, this falls back to the CPU after throwing
        a warning.
        Parameters
        ----------
        matrix_a : WRITEME
        matrix_b : WRITEME
        matrix_c : WRITEME
        """
        if not hasattr(ZCA._gpu_matrix_dot, 'theano_func'):
            ma, mb = T.matrices('A', 'B')
            mc = T.dot(ma, mb)
            ZCA._gpu_matrix_dot.theano_func = \
                theano.function([ma, mb], mc, allow_input_downcast=True)

        theano_func = ZCA._gpu_matrix_dot.theano_func

        try:
            if matrix_c is None:
                return theano_func(matrix_a, matrix_b)
            else:
                matrix_c[...] = theano_func(matrix_a, matrix_b)
                return matrix_c
        except MemoryError:
            warnings.warn('Matrix multiplication too big to fit on GPU. '
                          'Re-doing with CPU. Consider using '
                          'THEANO_FLAGS="device=cpu" for your next '
                          'preprocessor run')
            return np.dot(matrix_a, matrix_b, matrix_c)

def sample_parallel():
    print "並列"
    x, y = T.matrices("a", "b")
    diff = x - y
    abs_diff = abs(diff)
    diff_sq = diff**2
    # 2つの行列を入力, 3つの行列のベクトルを出力
    f = theano.function([x, y], [diff, abs_diff, diff_sq])
    print f([[0,1],[2,3]], [[10,11],[12,13]])
    print

 def test_grad(self, cls_ofg):
     x, y, z = T.matrices('xyz')
     e = x + y * z
     op = cls_ofg([x, y, z], [e])
     f = op(x, y, z)
     f = f - T.grad(T.sum(f), y)
     fn = function([x, y, z], f)
     xv = np.ones((2, 2), dtype=config.floatX)
     yv = np.ones((2, 2), dtype=config.floatX) * 3
     zv = np.ones((2, 2), dtype=config.floatX) * 5
     assert np.all(11.0 == fn(xv, yv, zv))

 def test_grad(self):
     x, y, z = T.matrices('xyz')
     e = x + y * z
     op = OpFromGraph([x, y, z], [e], mode='FAST_RUN', grad_depth=2)
     f = op(x, y, z)
     f = f - T.grad(T.sum(f), y)
     fn = function([x, y, z], f)
     xv = numpy.ones((2, 2), dtype=config.floatX)
     yv = numpy.ones((2, 2), dtype=config.floatX)*3
     zv = numpy.ones((2, 2), dtype=config.floatX)*5
     assert numpy.all(11.0 == fn(xv, yv, zv))

    def test_connection_pattern(self, cls_ofg):
        # Basic case
        x, y, z = T.matrices('xyz')
        out1 = x * y
        out2 = y * z

        op1 = cls_ofg([x, y, z], [out1, out2])
        results = op1.connection_pattern(None)
        expect_result = [[True, False],
                         [True, True],
                         [False, True]]
        assert results == expect_result

        # Graph with ops that don't have a 'full' connection pattern
        # and with ops that have multiple outputs
        m, n, p, q = T.matrices('mnpq')
        o1, o2 = op1(m, n, p)
        out1, out2 = op1(o1, q, o2)
        op2 = cls_ofg([m, n, p, q], [out1, out2])

        results = op2.connection_pattern(None)
        expect_result = [[True, False],
                         [True, True],
                         [False, True],
                         [True, True]]
        assert results == expect_result

        # Inner graph where some computation doesn't rely on explicit inputs
        srng = RandomStreams(seed=234)
        rv_u = srng.uniform((2, 2))
        x, y = T.matrices('xy')
        out1 = x + rv_u
        out2 = y + 3
        out3 = 3 + rv_u
        op3 = cls_ofg([x, y], [out1, out2, out3])

        results = op3.connection_pattern(None)
        expect_result = [[True, False, False],
                         [False, True, False],
                         [True, False, True]]
        assert results == expect_result

 def test_grad_grad(self):
     x, y, z = T.matrices('xyz')
     e = x + y * z
     op = OpFromGraph([x, y, z], [e])
     f = op(x, y, z)
     f = f - T.grad(T.sum(f), y)
     f = f - T.grad(T.sum(f), y)
     fn = function([x, y, z], f)
     xv = numpy.ones((2, 2), dtype=config.floatX)
     yv = numpy.ones((2, 2), dtype=config.floatX) * 3
     zv = numpy.ones((2, 2), dtype=config.floatX) * 5
     assert numpy.allclose(6.0, fn(xv, yv, zv))

 def _build(self):
     if self._debug:
         theano.config.compute_test_value = 'warn'
     X,W = T.matrices('X','W')
     if self._debug:
         X.tag.test_value = np.random.rand(3,1)
         W.tag.test_value = np.random.rand(5,3)
     Z = T.dot(W,X)
     A = self._activation(Z)
     self._fpropagate = function([X, W],A)
     self._layers = []
     self._generate_initial_weights()

    def __init__(self, shape):
        self.in_size, self.out_size = shape

        self.W = init_weights(shape)
        self.b = init_bias(self.out_size)

        self.gW = init_gradws(shape)
        self.gb = init_bias(self.out_size)

        D, X = T.matrices("D", "X")
        def _active(X):
            return T.nnet.sigmoid(T.dot(X, self.W) + self.b)
        self.active = theano.function(inputs = [X], outputs = _active(X))
        
        def _derive(D, X):
            return D * ((1 - X) * X)
        self.derive = theano.function(
            inputs = [D, X],
            outputs = _derive(D, X)
        )

        def _propagate(D):
            return T.dot(D, self.W.T)
        self.propagate = theano.function(inputs = [D], outputs = _propagate(D))

        x, dy = T.rows("x","dy")
        updates_grad = [(self.gW, self.gW + T.dot(x.T, dy)),
               (self.gb, self.gb + dy)]
        self.grad = theano.function(
            inputs = [x, dy],
            updates = updates_grad
        )

        updates_clear = [
               (self.gW, self.gW * 0),
               (self.gb, self.gb * 0)]
        self.clear_grad = theano.function(
            inputs = [],
            updates = updates_clear
        )

        lr = T.scalar()
        t = T.scalar()
        updates_w = [
               (self.W, self.W - self.gW * lr / t),
               (self.b, self.b - self.gb * lr / t)]
        self.update = theano.function(
            inputs = [lr, t],
            updates = updates_w
        )

 def test_straightforward(self):
     x, y, z = T.matrices('xyz')
     e = x + y * z
     op = OpFromGraph([x, y, z], [e], mode='FAST_RUN')
     f = op(x, y, z) - op(y, z, x)  # (1+3*5=array of 16) - (3+1*5=array of 8)
     fn = function([x, y, z], f)
     xv = numpy.ones((2, 2), dtype=config.floatX)
     yv = numpy.ones((2, 2), dtype=config.floatX)*3
     zv = numpy.ones((2, 2), dtype=config.floatX)*5
     #print function, function.__module__
     #print fn.maker.fgraph.toposort()
     fn(xv, yv, zv)
     assert numpy.all(8.0 == fn(xv, yv, zv))
     assert numpy.all(8.0 == fn(xv, yv, zv))

 def test_size_changes(self):
     x, y, z = T.matrices('xyz')
     e = T.dot(x, y)
     op = OpFromGraph([x, y], [e], mode='FAST_RUN')
     f = op(x, op(y, z))
     fn = function([x, y, z], f)
     xv = numpy.ones((2, 3), dtype=config.floatX)
     yv = numpy.ones((3, 4), dtype=config.floatX)*3
     zv = numpy.ones((4, 5), dtype=config.floatX)*5
     res = fn(xv, yv, zv)
     assert res.shape == (2, 5)
     assert numpy.all(180.0 == res)
     res = fn(xv, yv, zv)
     assert res.shape == (2, 5)
     assert numpy.all(180.0 == res)

 def test_size_changes(self, cls_ofg):
     x, y, z = T.matrices('xyz')
     e = T.dot(x, y)
     op = cls_ofg([x, y], [e])
     f = op(x, op(y, z))
     fn = function([x, y, z], f)
     xv = np.ones((2, 3), dtype=config.floatX)
     yv = np.ones((3, 4), dtype=config.floatX) * 3
     zv = np.ones((4, 5), dtype=config.floatX) * 5
     res = fn(xv, yv, zv)
     assert res.shape == (2, 5)
     assert np.all(180.0 == res)
     res = fn(xv, yv, zv)
     assert res.shape == (2, 5)
     assert np.all(180.0 == res)

    def test_straightforward(self, cls_ofg):
        x, y, z = T.matrices('xyz')
        e = x + y * z
        op = cls_ofg([x, y, z], [e])
        # (1+3*5=array of 16) - (3+1*5=array of 8)
        f = op(x, y, z) - op(y, z, x)

        fn = function([x, y, z], f)
        xv = np.ones((2, 2), dtype=config.floatX)
        yv = np.ones((2, 2), dtype=config.floatX) * 3
        zv = np.ones((2, 2), dtype=config.floatX) * 5
        # print function, function.__module__
        # print fn.maker.fgraph.toposort()
        fn(xv, yv, zv)
        assert np.all(8.0 == fn(xv, yv, zv))
        assert np.all(8.0 == fn(xv, yv, zv))

    def test_shared(self, cls_ofg):
        x, y, z = T.matrices('xyz')
        s = shared(np.random.rand(2, 2).astype(config.floatX))
        e = x + y * z + s
        op = cls_ofg([x, y, z], [e])
        # (1+3*5=array of 16) - (3+1*5=array of 8)
        f = op(x, y, z) - op(y, z, x)

        fn = function([x, y, z], f)
        xv = np.ones((2, 2), dtype=config.floatX)
        yv = np.ones((2, 2), dtype=config.floatX) * 3
        zv = np.ones((2, 2), dtype=config.floatX) * 5
        # print function, function.__module__
        # print fn.maker.fgraph.toposort()
        assert np.allclose(8.0, fn(xv, yv, zv))
        assert np.allclose(8.0, fn(xv, yv, zv))

    def __init__(self, shape, X):
        prefix = "Softmax_"
        self.in_size, self.out_size = shape
        self.W = init_weights(shape, prefix + "W")
        self.b = init_bias(self.out_size, prefix + "b")

        self.gW = init_gradws(shape, prefix + "gW")
        self.gb = init_bias(self.out_size, prefix + "gb")

        D = T.matrices("D")
        self.X = X
        def _active(X):
            return T.nnet.softmax(T.dot(X, self.W) + self.b)
        self.active = theano.function(inputs = [self.X], outputs = _active(self.X))

        def _propagate(D):
            return T.dot(D, self.W.T)
        self.propagate = theano.function(inputs = [D], outputs = _propagate(D))

        x, dy = T.rows("x","dy")
        updates_grad = [(self.gW, self.gW + T.dot(x.T, dy)),
               (self.gb, self.gb + dy)]
        self.grad = theano.function(
            inputs = [x, dy],
            updates = updates_grad
        )

        updates_clear = [
               (self.gW, self.gW * 0),
               (self.gb, self.gb * 0)]
        self.clear_grad = theano.function(
            inputs = [],
            updates = updates_clear
        )

        lr = T.scalar()
        t = T.scalar()
        updates_w = [
               (self.W, self.W - self.gW * lr / t),
               (self.b, self.b - self.gb * lr / t)]
        self.update = theano.function(
            inputs = [lr, t],
            updates = updates_w
        )

        self.params = [self.W, self.b]

    def test_shared_grad(self, cls_ofg):
        x, y, z = T.matrices('xyz')
        s = shared(np.random.rand(2, 2).astype(config.floatX))
        e = x + y * z + s
        op = cls_ofg([x, y, z], [e])
        f = op(x, y, z)
        f = f - T.grad(T.sum(f), y)
        fn = function([x, y, z], f)
        xv = np.ones((2, 2), dtype=config.floatX)
        yv = np.ones((2, 2), dtype=config.floatX) * 3
        zv = np.ones((2, 2), dtype=config.floatX) * 5
        assert np.allclose(11.0 + s.get_value(), fn(xv, yv, zv))

        # grad again the shared variable
        f = op(x, y, z)
        f = f - T.grad(T.sum(f), s)
        fn = function([x, y, z], f)
        assert np.allclose(15.0 + s.get_value(),
                           fn(xv, yv, zv))

    def test_shared_grad(self):
        x, y, z = T.matrices('xyz')
        s = shared(numpy.random.rand(2, 2).astype(config.floatX))
        e = x + y * z + s
        op = OpFromGraph([x, y, z], [e], mode='FAST_RUN')
        f = op(x, y, z)
        f = f - T.grad(T.sum(f), y)
        fn = function([x, y, z], f)
        xv = numpy.ones((2, 2), dtype=config.floatX)
        yv = numpy.ones((2, 2), dtype=config.floatX) * 3
        zv = numpy.ones((2, 2), dtype=config.floatX) * 5
        assert numpy.allclose(11.0 + s.get_value(), fn(xv, yv, zv))

        # grad again the shared variable
        f = op(x, y, z)
        f = f - T.grad(T.sum(f), s)
        fn = function([x, y, z], f)
        assert numpy.allclose(15.0 + s.get_value(),
                              fn(xv, yv, zv))

def test_4_conditionals():
    a, b = T.scalars('a', 'b')
    x, y = T.matrices('x', 'y')

    f_switch = theano.function([a, b, x, y], T.switch(T.lt(a, b), T.mean(x), T.mean(y)))
    f_lazy_ifelse = theano.function([a, b, x, y], ifelse(T.lt(a, b), T.mean(x), T.mean(y)))

    x_val = np.ones((100, 100), dtype=theano.config.floatX)*1
    y_val = np.ones((100, 100), dtype=theano.config.floatX)*2

    # vectorized switch is going to evaluate both options
    np.testing.assert_almost_equal(
        f_switch(1, 2, x_val, y_val),
        1
    )

    # lazy evaluation is going to evaluate only single option
    np.testing.assert_almost_equal(
        f_lazy_ifelse(2, 1, x_val, y_val),
        2
    )

    def compile(self,X,n_negative_samples=None):
        if n_negative_samples is None:
            n_negative_samples = 1000
        
        pos_samples = X.loc[:, self.column_ranges.keys()].values.astype(floatX)

        pos_data, neg_data = T.matrices('SigData', 'BckData')
        pos_w, neg_w, parameters = T.vectors('SigW', 'BckW', 'parameters')

        neg_samples, neg_weight = self.generate_negative_samples(n_negative_samples=n_negative_samples,
                                                                 strategy=self.sampling_strategy)

        givens = {pos_data: pos_samples, neg_data: neg_samples,  neg_w: neg_weight}

        pdf = self.prepare_pdf()
        pdfs, summands = pdf(pos_data, neg_data, neg_weights=neg_w, weights=parameters)
        result = - T.mean(pos_w * T.log(pdfs))

        self.Tfunction = theano.function([parameters,pos_w], result, givens=givens)
        self.Tderivative = theano.function([parameters,pos_w], T.grad(result, parameters), givens=givens)
        self.X=X

def SGD(eta, minibatch_size, n_minibatch, n_epochs):
    # Testing and Validation data are the outputs of the last inputs.
    print 'Calling SGD() ..'
    index = T.iscalar('index')
    x, y = T.matrices('x', 'y')

    tree = TreeLSTM(x, n_in)
    updates = [(param, param - eta * gparam) for param, gparam in zip(tree.params, T.grad(tree.loss(y), tree.params))]
    train_fn = theano.function([index], tree.loss(y), updates=updates,
                               givens={x: train_x[:, minibatch_size * n_in * index: (minibatch_size + 1) * n_in * index],
                                       y: train_y[:, minibatch_size * index: (minibatch_size + 1) * index]}
                               )

    # Compilation over
    #################
    ## TRAIN MODEL ##
    #################

    for epoch in n_epochs:
        for idx in range(n_minibatch):
            train_fn(idx)