def main():

    """
    a = tensor.tensor3('a')
    b = tensor.tensor3('b')
    c = tensor.tensor3('c')
    d = tensor.concatenate([a,b,c], axis=0)
    f = theano.function([a,b,c],d)
    aval = np.array([[[2,2,2,2]],[[2,2,2,2]], [[2,2,2,2]]])
    bval = 2*aval
    cval = 3*aval
    ans = f(a=aval, b=bval, c = None)
    print(ans)
    print(ans.shape)
    """

    a = tensor.tensor3("a")
    b = tensor.tensor3("b")
    c = tensor.tensor3("c")
    d = con(rep0=a, rep1=b, rep2=c)
    f = theano.function([a, b, c], d)
    aval = np.array([[[2, 2, 2, 2]], [[2, 2, 2, 2]], [[2, 2, 2, 2]]])
    tval = np.zeros(aval.shape)
    print tval
    print tval.shape
    bval = 2 * aval
    cval = 3 * aval
    ans = f(a=aval, b=bval, c=cval)
    print (ans)
    print (ans.shape)
    print (tensor.dim(ans))

    """

    def _get_net(self):
        net = OrderedDict()

        net['l_in_x'] = InputLayer(shape=(None, None, TOKEN_REPRESENTATION_SIZE),
                                   input_var=T.tensor3(name="enc_ix"),
                                   name="encoder_seq_ix")

        net['l_in_y'] = InputLayer(shape=(None, None, TOKEN_REPRESENTATION_SIZE),
                                   input_var=T.tensor3(name="dec_ix"),
                                   name="decoder_seq_ix")

        # encoder ###############################################
        net['l_enc'] = LSTMLayer(
            incoming=net['l_in_x'],
            num_units=HIDDEN_LAYER_DIMENSION,
            grad_clipping=GRAD_CLIP,
            only_return_final=True,
            name='lstm_encoder'
        )

        # decoder ###############################################

        net['l_dec'] = LSTMLayer(
            incoming=net['l_in_y'],
            num_units=HIDDEN_LAYER_DIMENSION,
            hid_init=net['l_enc'],
            grad_clipping=GRAD_CLIP,
            name='lstm_decoder'
        )

        # decoder returns the batch of sequences of though vectors, each corresponds to a decoded token
        # reshape this 3d tensor to 2d matrix so that the next Dense layer can convert each though vector to
        # probability distribution vector

        # output ###############################################
        # cut off the last prob vectors for every prob sequence:
        # they correspond to the tokens that go after EOS_TOKEN and we are not interested in it
        net['l_slice'] = SliceLayer(
            incoming=net['l_dec'],
            indices=slice(0, -1),  # keep all but the last token
            axis=1,  # sequneces axis
            name='slice_layer'
        )

        net['l_dec_long'] = ReshapeLayer(
            incoming=net['l_slice'],
            shape=(-1, HIDDEN_LAYER_DIMENSION),
            name='reshape_layer'
        )

        net['l_dist'] = DenseLayer(
            incoming=net['l_dec_long'],
            num_units=self.vocab_size,
            nonlinearity=lasagne.nonlinearities.softmax,
            name="dense_output_probas"
        )

        # don't need to reshape back, can compare this "long" output with true one-hot vectors

        return net

def set_tf_update_function(input_emb_param,
                           generator_rnn_model,
                           generator_output_model,
                           generator_optimizer,
                           generator_grad_clipping):

    # input sequence data (time_length * num_samples * input_dims)
    input_sequence  = tensor.tensor3(name='input_sequence',
                                     dtype=floatX)
    target_sequence  = tensor.tensor3(name='target_sequence',
                                     dtype=floatX)

    # embedding sequence
    input_emb_sequence  = tensor.dot(input_sequence, input_emb_param)
    target_emb_sequence = tensor.dot(target_sequence, input_emb_param)

    # set generator input data list
    generator_input_data_list = [input_emb_sequence,]

    # get generator output data
    generator_output = generator_rnn_model[0].forward(generator_input_data_list, is_training=True)
    generator_hidden = generator_output[0]
    generator_cell   = generator_output[1]

    generator_emb_sequence = get_tensor_output(generator_hidden, generator_output_model, is_training=True)
    generator_sequence     = tensor.dot(generator_emb_sequence, tensor.transpose(input_emb_param))

    # get square error
    square_error = tensor.sqr(target_sequence-generator_sequence).sum(axis=2)

    # set generator update
    tf_updates_cost = square_error.mean()
    tf_updates_dict = get_model_and_params_updates(layers=generator_rnn_model+generator_output_model,
                                                   params=[input_emb_param,],
                                                   cost=tf_updates_cost,
                                                   optimizer=generator_optimizer)

    generator_gradient_dict  = get_model_and_params_gradients(layers=generator_rnn_model+generator_output_model,
                                                              params=[input_emb_param,],
                                                              cost=tf_updates_cost)
    generator_gradient_norm  = 0.
    for grad in generator_gradient_dict:
        generator_gradient_norm += tensor.sum(grad**2)
    generator_gradient_norm  = tensor.sqrt(generator_gradient_norm)

    # set tf update inputs
    tf_updates_inputs  = [input_sequence,
                          target_sequence]

    # set tf update outputs
    tf_updates_outputs = [square_error,
                          generator_gradient_norm,]

    # set tf update function
    tf_updates_function = theano.function(inputs=tf_updates_inputs,
                                          outputs=tf_updates_outputs,
                                          updates=tf_updates_dict,
                                          on_unused_input='ignore')

    return tf_updates_function

def build_model(args):
    x = tensor.tensor3('features', dtype=floatX)
    y = tensor.tensor3('targets', dtype=floatX)

    linear = Linear(input_dim=1, output_dim=4 * args.units)
    rnn = LSTM(dim=args.units, activation=Tanh())
    linear2 = Linear(input_dim=args.units, output_dim=1)

    prediction = Tanh().apply(linear2.apply(rnn.apply(linear.apply(x))))

    prediction = prediction[:-1, :, :]

    # SquaredError does not work on 3D tensor
    y = y.reshape((y.shape[0] * y.shape[1], y.shape[2]))
    prediction = prediction.reshape((prediction.shape[0] * prediction.shape[1],
                                     prediction.shape[2]))

    cost = SquaredError().apply(y, prediction)

    # Initialization
    linear.weights_init = IsotropicGaussian(0.1)
    linear2.weights_init = IsotropicGaussian(0.1)
    linear.biases_init = Constant(0)
    linear2.biases_init = Constant(0)
    rnn.weights_init = Orthogonal()

    return cost

    def init_exprs(self):
        inpt_mean = T.tensor3('inpt_mean')
        inpt_var = T.tensor3('inpt_var')
        target = T.tensor3('target')
        pars = self.parameters

        hidden_to_hiddens = [getattr(pars, 'hidden_to_hidden_%i' % i)
                             for i in range(len(self.n_hiddens) - 1)]
        hidden_biases = [getattr(pars, 'hidden_bias_%i' % i)
                         for i in range(len(self.n_hiddens))]
        hidden_var_biases_sqrt = [1 if i else 0 for i in self.use_varprop_at]
        recurrents = [getattr(pars, 'recurrent_%i' % i)
                      for i in range(len(self.n_hiddens))]
        initial_hiddens = [getattr(pars, 'initial_hidden_%i' % i)
                           for i in range(len(self.n_hiddens))]

        self.exprs = self.make_exprs(
            inpt_mean, inpt_var, target,
            pars.in_to_hidden, hidden_to_hiddens, pars.hidden_to_out,
            hidden_biases, hidden_var_biases_sqrt,
            initial_hiddens, recurrents, pars.out_bias,
            self.hidden_transfers, self.out_transfer, self.loss,
            self.pooling, self.leaky_coeffs,
            [self.p_dropout_inpt] + [self.p_dropout_hidden] * len(recurrents),
            self.hotk_inpt)

def generate_subpop_input(r_E, r_I, n_pairs):
    
    c = T.scalar("c", dtype='float32')
    h = T.matrix("h", dtype='float32')
    W_EE = T.tensor3("W_EE", dtype='float32')
    W_EI = T.tensor3("W_EI", dtype='float32')
    W_IE = T.tensor3("W_IE", dtype='float32')
    W_II = T.tensor3("W_II", dtype='float32')

    r_e = T.matrix("r_e", dtype='float32')
    r_i = T.matrix("r_i", dtype='float32')

    I_E = T.matrix('I_E', dtype='float32')
    I_I = T.matrix('I_I', dtype='float32')

    I_thresh_E = T.matrix('I_thresh_E', dtype='float32')
    I_thresh_I = T.matrix('I_thresh_I', dtype='float32')

    # Compile functions:
    I_E = c*h + T.sum(T.sum(W_EE*r_e,1),1).reshape((n_pairs, n_pairs)).T - T.sum(T.sum(W_EI*r_i,1),1).reshape((n_pairs, n_pairs)).T
    I_I = c*h + T.sum(T.sum(W_IE*r_e,1),1).reshape((n_pairs, n_pairs)).T - T.sum(T.sum(W_II*r_i,1),1).reshape((n_pairs, n_pairs)).T

    I_thresh_E = T.switch(T.lt(I_E,0), 0, I_E)
    I_thresh_I = T.switch(T.lt(I_I,0), 0, I_I)

    inputs = theano.function(inputs=[c,h,W_EE,W_EI,W_IE,W_II],
                                outputs=[I_thresh_E, I_thresh_I],
                                givens={r_e:r_E, r_i:r_I},
                                allow_input_downcast=True)
    return inputs

    def __init__(self, input_size, hidden_size, output_size):
        self.input_size = input_size
        self.hidden_size = hidden_size
        self.output_size = output_size

        x = tensor.tensor3('x', dtype=floatX)
        y = tensor.tensor3('y', dtype=floatX)

        x_to_lstm = Linear(name="x_to_lstm", input_dim=input_size, output_dim=4 * hidden_size,
                           weights_init=IsotropicGaussian(), biases_init=Constant(0))
        lstm = LSTM(dim=hidden_size, name="lstm", weights_init=IsotropicGaussian(), biases_init=Constant(0))
        lstm_to_output = Linear(name="lstm_to_output", input_dim=hidden_size, output_dim=output_size,
                                weights_init=IsotropicGaussian(), biases_init=Constant(0))

        x_transform = x_to_lstm.apply(x)
        h, c = lstm.apply(x_transform)

        y_hat = lstm_to_output.apply(h)
        y_hat = Logistic(name="y_hat").apply(y_hat)

        self.cost = BinaryCrossEntropy(name="cost").apply(y, y_hat)

        x_to_lstm.initialize()
        lstm.initialize()
        lstm_to_output.initialize()

        self.computation_graph = ComputationGraph(self.cost)

def build_model_EvoMN(options, tparams):
    trng = RandomStreams(SEED)
    use_noise = theano.shared(numpy_floatX(0.))
    use_linear = theano.shared(numpy_floatX(0.))
    
    x = tensor.tensor3('x', dtype='int64') # x is n_sent * n_word * n_samples
    xmask = tensor.tensor3('xmask', dtype=config.floatX) # same as x
    q = tensor.matrix('q', dtype='int64') # q is nword * n_samples
    qmask = tensor.matrix('qmask', dtype=config.floatX)
    y = tensor.vector('y',dtype='int64') # nsamples * 1
    nhops = tensor.scalar('nhops',dtype='int64') # nhops, used to loop.
    wmat = tensor.matrix('wmat',dtype=config.floatX) # dim_word * (maxSentLen+1)
    
    aEmbSeq, bEmbSeq, qSeq = memLayers(tparams, options, x, xmask, q, qmask, nhops, wmat, use_linear)
    proj = qSeq[-1] # nsamples * dim_hidden
    if options['use_dropout']:
        proj = dropout_layer(proj, use_noise, trng)
    
    pred = tensor.nnet.softmax(tensor.dot(proj, tparams['Wemb_B_' + str(options['nhops'])].T)) # nsamples * vocab_size
    pred_ans = pred.argmax(axis=1) # nsamples vector
    off = 1e-7
    cost = -tensor.log(pred[tensor.arange(y.shape[0]), y] + off).sum()
    
    f_debug = theano.function([x,xmask,q,qmask,y,nhops,wmat], [x,xmask,q,qmask,y,nhops,wmat,proj,pred,pred_ans,aEmbSeq,bEmbSeq,qSeq],name='f_debug')
    print 'f_debug complete~'
    f_pred = theano.function([x,xmask,q,qmask,nhops,wmat], pred, name='f_pred')
    print 'f_pred complete~'
    f_ans = theano.function([x,xmask,q,qmask,nhops,wmat], pred_ans, name='f_ans')
    print 'f_ans complete~'
    
    return use_noise, use_linear, x, xmask, q, qmask, y, nhops, wmat, proj, pred, pred_ans, cost, f_debug, f_pred, f_ans

def main():
    pars = "model/4GRAM_BI/76.69"
    print("Loading data...")
    (feats_in,feats_out) = iodata.iodata_forPre()
    feats_in = np.array(feats_in).astype(theano.config.floatX)
    print("{}".format((QUESTION_SIZE*(NGRAMS+1)*NUM_CHOICES,1,WORD_2_VEC_FEATURES)))
    feats_out = np.array(feats_out).astype(theano.config.floatX).reshape((QUESTION_SIZE*(NGRAMS+1)*NUM_CHOICES,1,WORD_2_VEC_FEATURES))
    #print(feats_out.shape)
    #print(feats_in)
    #print(lenfeats_out)
    output_layer = build_model(bi_directional = True)
    network.layers.set_all_param_values(output_layer, pickle.load(open(pars, "r")))
    x = T.tensor3('x', dtype=theano.config.floatX)
    y  =T.tensor3('y',dtype = theano.config.floatX)  
    cos_distance_ls = np.zeros((QUESTION_SIZE,NUM_CHOICES))
    predict = theano.function([x,y],calculate_cos_dis(output_layer.get_output(x,deterministic=True),y),on_unused_input='ignore')

    for index in range(QUESTION_SIZE):
        try:
            print(feats_in[(index)*NUM_CHOICES:(index+1)*NUM_CHOICES].shape)
            print(feats_out[(index)*NUM_CHOICES:(index+1)*NUM_CHOICES].shape)
            pred  = predict(feats_in[(index)*NUM_CHOICES:(index+1)*NUM_CHOICES],feats_out[(index)*NUM_CHOICES:(index+1)*NUM_CHOICES])
            print("OHOHOH")
        except RuntimeError:
            pass
        cos_distance_ls[index,:] = cos_distance_ls[index,:] + pred

    def __init__(self):
        print("Initialising network...")
        import theano
        import theano.tensor as T
        import lasagne
        from lasagne.layers import (InputLayer, LSTMLayer, ReshapeLayer, 
                                    ConcatLayer, DenseLayer)
        theano.config.compute_test_value = 'raise'

        # Construct LSTM RNN: One LSTM layer and one dense output layer
        l_in = InputLayer(shape=input_shape)

        # setup fwd and bck LSTM layer.
        l_fwd = LSTMLayer(
            l_in, N_HIDDEN, backwards=False, learn_init=True, peepholes=True)
        l_bck = LSTMLayer(
            l_in, N_HIDDEN, backwards=True, learn_init=True, peepholes=True)

        # concatenate forward and backward LSTM layers
        concat_shape = (N_SEQ_PER_BATCH * SEQ_LENGTH, N_HIDDEN)
        l_fwd_reshape = ReshapeLayer(l_fwd, concat_shape)
        l_bck_reshape = ReshapeLayer(l_bck, concat_shape)
        l_concat = ConcatLayer([l_fwd_reshape, l_bck_reshape], axis=1)

        l_recurrent_out = DenseLayer(l_concat, num_units=N_OUTPUTS, 
                                     nonlinearity=None)
        l_out = ReshapeLayer(l_recurrent_out, output_shape)

        input = T.tensor3('input')
        target_output = T.tensor3('target_output')

        # add test values
        input.tag.test_value = rand(
            *input_shape).astype(theano.config.floatX)
        target_output.tag.test_value = rand(
            *output_shape).astype(theano.config.floatX)

        print("Compiling Theano functions...")
        # Cost = mean squared error
        cost = T.mean((l_out.get_output(input) - target_output)**2)

        # Use NAG for training
        all_params = lasagne.layers.get_all_params(l_out)
        updates = lasagne.updates.nesterov_momentum(cost, all_params, LEARNING_RATE)

        # Theano functions for training, getting output, and computing cost
        self.train = theano.function(
            [input, target_output],
            cost, updates=updates, on_unused_input='warn',
            allow_input_downcast=True)

        self.y_pred = theano.function(
            [input], l_out.get_output(input), on_unused_input='warn',
            allow_input_downcast=True)

        self.compute_cost = theano.function(
            [input, target_output], cost, on_unused_input='warn',
            allow_input_downcast=True)

        print("Done initialising network.")

    def init_model(self):
        print('Initializing model...')
        ra_input_var = T.tensor3('raw_audio_input')
        mc_input_var = T.tensor3('melody_contour_input')
        target_var = T.imatrix('targets')
        network = self.build_network(ra_input_var, mc_input_var)
        prediction = layers.get_output(network)
        prediction = T.clip(prediction, 1e-7, 1.0 - 1e-7)
        loss = lasagne.objectives.categorical_crossentropy(prediction, target_var)
        loss = loss.mean()
        params = layers.get_all_params(network, trainable=True)
        updates = lasagne.updates.sgd(loss, params, learning_rate=0.02)

        test_prediction = layers.get_output(network, deterministic=True)
        test_loss = lasagne.objectives.categorical_crossentropy(test_prediction,
                                                                target_var)
        test_loss = test_loss.mean()
        test_acc = T.mean(T.eq(T.argmax(test_prediction, axis=1), T.argmax(target_var, axis=1)),
                          dtype=theano.config.floatX)

        print('Building functions...')
        self.train_fn = theano.function([ra_input_var, mc_input_var, target_var], 
                                        [loss, prediction], 
                                        updates=updates, 
                                        on_unused_input='ignore')
        self.val_fn = theano.function([ra_input_var, mc_input_var, target_var], 
                                        [test_loss, test_acc, test_prediction], 
                                        on_unused_input='ignore')
        self.run_fn = theano.function([ra_input_var, mc_input_var],
                                        [prediction],
                                        on_unused_input='ignore')

 def step_fun(self):
     if self._step_fun is None:
         inputs = T.matrix('inputs')
         states_tm1 = [T.matrix('state_%d_%d_tm1' % (layer, state))
                       for layer in range(self.n_layers)
                       for state in range(self.gate0.n_states)]
         if self.gates[-1].use_attention:
             raise NotImplementedError('Stacked RNN with attention')
             attended=T.tensor3('attended')
             attended_dot_u=T.tensor3('attended_dot_u')
             attention_mask=T.matrix('attention_mask')
             self._step_fun = function(
                     [inputs] + states_tm1 + [
                         attended, attended_dot_u, attention_mask],
                     self.step(*([inputs, T.ones(inputs.shape[:-1])] +
                                 states_tm1 + [T.ones_like(states_tm1[0]),
                                 attended, attended_dot_u,
                                 attention_mask])),
                     name='%s_step_fun'%self.name)
         else:
             self._step_fun = function(
                     [inputs] + states_tm1,
                     self.step(*([inputs, T.ones(inputs.shape[:-1])] +
                               states_tm1 + [T.ones_like(states_tm1[0])])),
                     name='%s_step_fun'%self.name)
     return self._step_fun

  def GetClassifier(self):

    def PairwiseLoss(x, mutation):
      """
      This function takes two matrix and return a vector by first
      calculating the loss for each row and then take element-wise
      maximum for each row.
      """
      return T.maximum(0., 1. - self.F(x) + self.F(mutation)).sum()

    inputs = T.tensor3(name='input', dtype='int32')
    mutations = T.tensor3(name='mutations', dtype='int32')

    components, updates = theano.scan(fn=PairwiseLoss,
                                      outputs_info=None,
                                      sequences=[inputs, mutations])
    loss = components.sum()

    gparams = [T.grad(loss, param) for param in self.params]

    updates = [(param, param - self.learning_rate * gparam)
               for param, gparam in zip(self.params, gparams)]

    return theano.function(inputs=[inputs, mutations],
                           outputs=loss,
                           updates=updates)

    def _setup_vars(self, sparse_input):
        '''Setup Theano variables for our network.

        Parameters
        ----------
        sparse_input : bool
            Not used -- sparse inputs are not supported for recurrent networks.

        Returns
        -------
        vars : list of theano variables
            A list of the variables that this network requires as inputs.
        '''
        _warn_dimshuffle()

        assert not sparse_input, 'Theanets does not support sparse recurrent models!'

        # the first dimension indexes time, the second indexes the elements of
        # each minibatch, and the third indexes the variables in a given frame.
        self.x = TT.tensor3('x')

        # for a regressor, this specifies the correct outputs for a given input.
        self.targets = TT.tensor3('targets')

        # the weights are the same shape as the output and specify the strength
        # of each entries in the error computation.
        self.weights = TT.tensor3('weights')

        if self.weighted:
            return [self.x, self.targets, self.weights]
        return [self.x, self.targets]

def set_evaluation_function(generator_model):
    # input sequence data (time_length * num_samples * input_dims)
    input_sequence  = tensor.tensor3(name='input_sequence',
                                     dtype=floatX)
    target_sequence  = tensor.tensor3(name='target_sequence',
                                    dtype=floatX)
    # set generator input data list
    generator_input_data_list = [input_sequence,]

    # get generator output data
    generator_output = generator_model[0].forward(generator_input_data_list,
                                                  is_training=True)
    output_sequence  = generator_output[0]
    generator_random = generator_output[-1]

    # get square error
    sample_cost = tensor.sqr(target_sequence-output_sequence).sum(axis=2)

    # set evaluation inputs
    evaluation_inputs  = [input_sequence,
                          target_sequence]

    # set evaluation outputs
    evaluation_outputs = [sample_cost,
                          output_sequence]

    # set evaluation function
    evaluation_function = theano.function(inputs=evaluation_inputs,
                                          outputs=evaluation_outputs,
                                          updates=generator_random,
                                          on_unused_input='ignore')

    return evaluation_function

def test7():
    morph = T.tensor3("morph")
    morph_mask = T.tensor3("mask")
    rel = T.matrix("rel")
    morphStruct = MorphStruct()
    morph_out = morphStruct.apply(morph , morph_mask , rel)
    fn = theano.function(inputs = [morph , morph_mask , rel] ,outputs = [morph_out] , on_unused_input='ignore')
    #rel : batch * sentence
    #state_below_morph : batch * sentence * n_emb_morph
    i = [
            [
                [1,1,1,1,1] , [2,2,2,2,2] , [3,3,3,3,3] , [4,4,4,4,4] ,
                [1,1,1,1,1] , [2,2,2,2,2] , [3,3,3,3,3] , [4,4,4,4,4] ,
                [1,1,1,1,1] , [2,2,2,2,2] , [3,3,3,3,3] , [4,4,4,4,4]
            ],
            [
                [1,1,1,1,1] , [2,2,2,2,2] , [3,3,3,3,3] , [4,4,4,4,4] ,
                [1,1,1,1,1] , [2,2,2,2,2] , [3,3,3,3,3] , [4,4,4,4,4] ,
                [1,1,1,1,1] , [2,2,2,2,2] , [3,3,3,3,3] , [4,4,4,4,4]
            ],
            [
                [1,1,1,1,1] , [2,2,2,2,2] , [3,3,3,3,3] , [4,4,4,4,4] ,
                [1,1,1,1,1] , [2,2,2,2,2] , [3,3,3,3,3] , [4,4,4,4,4] ,
                [1,1,1,1,1] , [2,2,2,2,2] , [3,3,3,3,3] , [4,4,4,4,4]
            ]
        ]
    m = i
    r = [[1,1,2,1,3,1,3,0,0,0] , [1,2,1,2,1,1,1,1,1,1],[3,1,1,5,1,1,0,0,0,0]]
    #res = fn(i , m , r)
    mat = np.array(i)
    print mat.shape
    print mat.sum(2)
    print mat.sum(1).shape

def build_decoder(tparams, options):
    """
    build an encoder, given pre-computed word embeddings
    """

    # description string: #words x #samples
    # text: text sentence
    # hypothesis: hypothesis sentence
    text_embedding = tensor.tensor3('text_embedding', dtype='float32')
    # text = tensor.matrix('text', dtype='int64')
    text_mask = tensor.matrix('text_mask', dtype='float32')
    hypothesis_embedding = tensor.tensor3('hypothesis_embedding', dtype='float32')
    # hypothesis = tensor.matrix('hypothesis', dtype='int64')
    hypothesis_mask = tensor.matrix('hypothesis_mask', dtype='float32')

    # encoder
    proj = get_layer(options['encoder'])[1](tparams, text_embedding, None, options,
                                            prefix='encoder',
                                            mask=text_mask)
    ctx = proj[0][-1]
    dec_ctx = ctx

    # decoder (hypothesis)
    proj_hypo = get_layer(options['decoder'])[1](tparams, hypothesis_embedding, dec_ctx, options,
                                             prefix='decoder_f',
                                             mask=hypothesis_mask)

    hypo_ctx = proj_hypo[0][-1]

    return text_embedding, text_mask, hypothesis_embedding, hypothesis_mask, hypo_ctx

def set_evaluation_function(generator_rnn_model,
                            generator_output_model):

    # input sequence data (time_length * num_samples * input_dims)
    input_sequence  = tensor.tensor3(name='input_sequence',
                                     dtype=floatX)
    target_sequence  = tensor.tensor3(name='target_sequence',
                                    dtype=floatX)
    # set generator input data list
    generator_input_data_list = [input_sequence,]

    # get generator output data
    generator_output = generator_rnn_model[0].forward(generator_input_data_list, is_training=True)
    generator_hidden = generator_output[0]
    generator_cell   = generator_output[1]

    generator_sample = get_tensor_output(generator_hidden, generator_output_model, is_training=True)

    # get square error
    square_error = tensor.sqr(target_sequence-generator_sample).sum(axis=2)

    # set evaluation inputs
    evaluation_inputs  = [input_sequence,
                          target_sequence]

    # set evaluation outputs
    evaluation_outputs = [square_error,]

    # set evaluation function
    evaluation_function = theano.function(inputs=evaluation_inputs,
                                          outputs=evaluation_outputs,
                                          on_unused_input='ignore')

    return evaluation_function

def initialize_data_nodes(loss_function, input_type, out_every_t):
    x = T.tensor3() if input_type == 'real' else T.matrix(dtype=INT_STR)
    if loss_function == 'CE':
        y = T.matrix(dtype=INT_STR) if out_every_t else T.vector(dtype=INT_STR)
    else:
        y = T.tensor3() if out_every_t else T.matrix()
    return x, y

    def __init__(self,n_in,n_hidden,n_out):
        self.n_in=int(n_in)
        self.n_hidden=int(n_hidden)
        self.n_out=int(n_out)
        self.input= T.tensor3()
        self.output= T.tensor3()
        self.x_mask=T.matrix()
        #self.y_mask=T.matrix()

        
        self.W_z = glorot_normal((n_out,n_hidden))
        self.U_z = glorot_normal((n_hidden,n_hidden))
        self.b_z = zero((n_hidden,))

        self.W_r = glorot_normal((n_out,n_hidden))
        self.U_r = glorot_normal((n_hidden,n_hidden))
        self.b_r = zero((n_hidden,))

        self.W_h = glorot_normal((n_out,n_hidden)) 
        self.U_h = glorot_normal((n_hidden,n_hidden))
        self.b_h = zero((n_hidden,))
        
        self.U_att= glorot_normal((self.n_in,1)) 
        self.b_att= zero((1,))

        self.W_yc=glorot_normal((self.n_out,))
        

        self.W_cy = glorot_normal((self.n_in,self.n_hidden))
        self.W_cs= glorot_normal((self.n_in,self.n_hidden))

        
        self.W_ha = glorot_normal((self.n_in,self.n_in))
        self.W_sa= glorot_normal((self.n_hidden,self.n_in))
        

        
        self.W_cl= glorot_normal((self.n_in,self.n_out))
        self.W_yl= glorot_normal((self.n_out,self.n_out))
        self.W_hl= glorot_normal((self.n_hidden,self.n_out))
        
        self.params=[self.W_z,self.U_z,self.b_z,self.W_r,self.U_r,self.b_r,
                   self.W_h,self.U_h,self.b_h,self.W_cy,self.W_cs,self.W_ha,self.W_sa
                     ,self.W_cl,self.W_yl,self.W_hl,self.U_att,self.b_att]
        

        self.L1 = T.sum(abs(self.W_z))+T.sum(abs(self.U_z))+\
                  T.sum(abs(self.W_r))+T.sum(abs(self.U_r))+\
                  T.sum(abs(self.W_h))+T.sum(abs(self.U_h))+\
                  T.sum(abs(self.W_cy))+T.sum(abs(self.W_cs))+\
                  T.sum(abs(self.W_ha))+T.sum(abs(self.W_sa))+\
                  T.sum(abs(self.W_cl))+T.sum(abs(self.W_yl))+\
                  T.sum(abs(self.W_hl))+T.sum(abs(self.U_att))
        self.L2_sqr = T.sum(self.W_z**2) + T.sum(self.U_z**2)+\
                      T.sum(self.W_r**2) + T.sum(self.U_r**2)+\
                      T.sum(self.W_h**2) + T.sum(self.U_h**2)+\
                      T.sum(self.W_cy**2) + T.sum(self.W_cs**2)+\
                      T.sum(self.W_ha**2) + T.sum(self.W_sa**2)+\
                      T.sum(self.W_cl**2) + T.sum(self.W_yl**2)+\
                      T.sum(self.W_hl**2) + T.sum(self.U_att**2)