def _plain_ctc(self, ):
        labels2 = tt.concatenate((self.labels, [self.blank, self.blank]))
        sec_diag = tt.neq(labels2[:-2], labels2[2:]) * tt.eq(labels2[1:-1], self.blank)
        # Last two entries of sec_diag do not matter as they multiply zero rows below.

        recurrence_relation = \
            tt.eye(self.n) + \
            tt.eye(self.n, k=1) + \
            tt.eye(self.n, k=2) * sec_diag.dimshuffle((0, 'x'))

        pred_y = self.inpt[:, self.labels]

        fwd_pbblts, _ = th.scan(
            lambda curr, accum: curr * tt.dot(accum, recurrence_relation),
            sequences=[pred_y],
            outputs_info=[tt.eye(self.n)[0]]
        )

        # TODO: Add probabilites[-1, -2] only if last label is blank.
        # liklihood = ifelse(tt.eq(self.n, 1), fwd_pbblts[-1, -1],
        #                        ifelse(tt.neq(self.labels[-1], self.blank), fwd_pbblts[-1, -1],
        #                               fwd_pbblts[-1, -1] + fwd_pbblts[-1, -2]))
        liklihood = fwd_pbblts[-1, -1]
        self.cost = -tt.log(liklihood)
        self.debug = fwd_pbblts.T

    def __init__(self, inpt, labels):
        '''
        Recurrent Relation:
        A matrix that specifies allowed transistions in paths.
        At any time, one could
        0) Stay at the same label (diagonal is identity)
        1) Move to the next label (first upper diagonal is identity)
        2) Skip to the next to next label if
            a) next label is blank and
            b) the next to next label is different from the current
            (Second upper diagonal is product of conditons a & b)
        '''
        n_labels = labels.shape[0]

        big_I = T.cast(T.eye(n_labels+2), 'float64')
        recurrence_relation1 = T.cast(T.eye(n_labels), 'float64') + big_I[2:,1:-1] + big_I[2:,:-2] * T.cast((T.arange(n_labels) % 2), 'float64')
        recurrence_relation = T.cast(recurrence_relation1, 'float64')

        '''
        Forward path probabilities
        '''
        pred_y = inpt[:, labels]

        probabilities, _ = theano.scan(
            lambda curr, prev: curr * T.dot(prev, recurrence_relation),
            sequences=[pred_y],
            outputs_info=[T.cast(T.eye(n_labels)[0], 'float64')]
        )

        # Final Costs
        labels_probab = T.sum(probabilities[-1, -2:])
        self.cost = -T.log(labels_probab)
        self.params = []

def likelihood(f, l, R, mu, eps, sigma2, lambda_1=1e-4):
    # The similarity matrix W is a linear combination of the slices in R
    W = T.tensordot(R, mu, axes=1)

    # The following indices correspond to labeled and unlabeled examples
    labeled = T.eq(l, 1).nonzero()

    # Calculating the graph Laplacian of W
    D = T.diag(W.sum(axis=0))
    L = D - W

    # The Covariance (or Kernel) matrix is the inverse of the (regularized) Laplacian
    epsI = eps * T.eye(L.shape[0])
    rL = L + epsI
    Sigma = nlinalg.matrix_inverse(rL)

    # The marginal density of labeled examples uses Sigma_LL as covariance (sub-)matrix
    Sigma_LL = Sigma[labeled][:, labeled][:, 0, :]

    # We also consider additive Gaussian noise with variance sigma2
    K_L = Sigma_LL + (sigma2 * T.eye(Sigma_LL.shape[0]))

    # Calculating the inverse and the determinant of K_L
    iK_L = nlinalg.matrix_inverse(K_L)
    dK_L = nlinalg.det(K_L)

    f_L = f[labeled]

    # The (L1-regularized) log-likelihood is given by the summation of the following four terms
    term_A = - (1 / 2) * f_L.dot(iK_L.dot(f_L))
    term_B = - (1 / 2) * T.log(dK_L)
    term_C = - (1 / 2) * T.log(2 * np.pi)
    term_D = - lambda_1 * T.sum(abs(mu))

    return term_A + term_B + term_C + term_D

	def rbm_K(self, X, effective_batch_size):
		D, DH = self.n_visible, self.n_hidden
		W, bh, bv = self.W, self.bh, self.bv

		#one bit flipped connected states
		Y = X.reshape((effective_batch_size, 1, D), 3) * T.ones((1, D, 1)) #tile out data vectors (repeat each one D times)
		Y1 = (Y + T.eye(D).reshape((1, D, D), 3))%2 # flip each bit once 

		# minimal activation connected states
		onehots = T.eye(D)
		blanks = T.zeros(D)
	
		eX = energy(X, W, bh, bv)	
		eY = energy(Y1, W, bh, bv)
		eO = energy(onehots, W, bh, bv)
		eB = energy(blanks, W, bh, bv)
		
		edif = eX.dimshuffle(0, 'x') - eY #- eB #- eO.dimshuffle('x', 0) - eB

		Z = T.exp(0.5*edif)
		K = T.sum(Z) / effective_batch_size
		K.name = 'K'

		K = T.cast(K, 'float32')
		return K 

def recurrence_relation(size):
    """
    Based on code from Shawn Tan
    """

    eye2 = T.eye(size + 2)
    return T.eye(size) + eye2[2:, 1:-1] + eye2[2:, :-2] * (T.arange(size) % 2)

def recurrence(blanked_label,blank_symbol):
    '''
    A(s) = alpha(t,s) + alpha(t,s-1)  if l_s = blank or l_s = l_{s-2}
           = alpha(t,s) + alpha(t,s-1) + alpha(t,s-2) if l_s != l_{s-2}
    we can define a L' x L' matrix R to help do this(L' = 2L+1)
    A = alpha(t,:) * R
    '''
    length = blanked_label.shape[1]
    blanks = T.zeros((1,2)) + blank_symbol
    ybb = T.concatenate((blanked_label, blanks), axis=1)
    '''
    ybb: 1 x L'+2    L' = 2*L+1
    ybb[0,:-2] == blanked_label  ybb[0,2:] = blanked_label move along 2 label
    T.neq(ybb[:, :-2], ybb[:, 2:]) -> l'=b or l'_u == l'_(u-2) set 0
    T.eq(ybb[:, 1:-1], blank_symbol) -> [0,?,0,?,...0,1,0] ? depends on 
    whether l_s = l_{s-2}  result[0,i]==1 means ybb[0,i] != ybb[0,i+2]
    '''
    setDiagMatrix = T.neq(ybb[:,:-2],ybb[:,2:])*T.eq(ybb[:,1:-1],blank_symbol)
    '''
    r2: L' x L'
    r3: L' x L'
    '''
    r2 = T.eye(length,k=1)
    r3 = T.eye(length,k=2)*(setDiagMatrix.T)
    return r2,r3

    def vanilla_ctc(self, ):
        my_labels = TT.concatenate((self.labels, [self.blank, self.blank]))
        pre_V = TT.neq(my_labels[:-2], my_labels[2:]) * \
                   TT.eq(my_labels[1:-1], self.blank)

        capLambda = \
            TT.eye(self.n) + \
            TT.eye(self.n, k=1) + \
            TT.eye(self.n, k=2) * pre_V.dimshuffle((0, 'x'))

        softmax_outputs = self.inpt[:, self.labels]

        alphas, _ = theano.scan(
            lambda outPuts, old_alpha: outPuts * TT.dot(old_alpha, capLambda),
            sequences=[softmax_outputs],
            outputs_info=[TT.eye(self.n)[0]]
        )

        # TODO: This is what we really should use for the initialization.
        # Need to debug and make sure there are no errors.
        # initial_alphas = TT.zeros(n)
        # initial_alphas[0]=inpt[0][-1]
        # initial_alphas[1]=inpt[0][labels[1]]
        # alphas, _ = theano.scan(
        #     lambda outPuts, old_alpha: outPuts * TT.dot(old_alpha, capLambda),
        #     sequences=[softmax_outputs],
        #     outputs_info=[initial_alphas]
        # )

        transcript_prob = TT.sum(alphas[-1, -2:])
        self.cost = -TT.log(transcript_prob)
        self.debug = alphas.T

        def each_loss(outpt, inpt):
            # y 是填充了blank之后的ans
            blank = 26
            y_nblank = T.neq(inpt, blank)
            n = T.dot(y_nblank, y_nblank)  # 真实的字符长度
            N = 2 * n + 1  # 填充后的字符长度，去除尾部多余的填充
            labels = inpt[:N]
            labels2 = T.concatenate((labels, [blank, blank]))
            sec_diag = T.neq(labels2[:-2], labels2[2:]) * T.eq(labels2[1:-1], blank)
            recurrence_relation = \
                T.eye(N) + \
                T.eye(N, k=1) + \
                T.eye(N, k=2) * sec_diag.dimshuffle((0, 'x'))

            pred_y = outpt[:, labels]

            fwd_pbblts, _ = theano.scan(
                lambda curr, accum: T.switch(T.eq(curr*T.dot(accum, recurrence_relation), 0.0),
                                             T.dot(accum, recurrence_relation)
                                             , curr*T.dot(accum, recurrence_relation)),
                sequences=[pred_y],
                outputs_info=[T.eye(N)[0]]
            )
            #return fwd_pbblts
            #liklihood = fwd_pbblts[0, 0]
            liklihood = fwd_pbblts[-1, -1] + fwd_pbblts[-1, -2]
            #liklihood = T.switch(T.lt(liklihood, 1e-35), 1e-35, liklihood)
            #loss = -T.log(T.cast(liklihood, "float32"))
            #loss = 10 * (liklihood - 1) * (liklihood - 100)
            loss = (T.le(liklihood, 1.0)*(10*(liklihood-1)*(liklihood-100)))+(T.gt(liklihood, 1.0)*(-T.log(T.cast(liklihood, "float32"))))
            return loss

def recurrence_relation(y, y_mask):
    # with blank symbol of -1 this falls back to the recurrence that fails
    # with repeating symbols!
    blank_symbol = 2
    n_y = y.shape[0]
    blanks = tensor.zeros((2, y.shape[1])) + blank_symbol
    ybb = tensor.concatenate((y, blanks), axis=0).T
    # ybb = B x (L'+2)   L'=2*label_noblank_length+1
    # ybb[:,:-2] == y.T   ybb[:,2:] = y.T move along 2 label
    # see Alex's paper:
    # tensor.neq(ybb[:, :-2], ybb[:, 2:]) -> l'=b or l'_u == l'_(u-2) set 0
    # tensor.eq(ybb[:, 1:-1], blank_symbol) -> [0,?,0,?,...0,1,0] ? depends on whether l'_u == l'_(u-2)  result[0,i]==1 means ybb[0,i] != ybb[0,i+2]
    # sec_diag = B x L'
    sec_diag = (tensor.neq(ybb[:, :-2], ybb[:, 2:]) *
                tensor.eq(ybb[:, 1:-1], blank_symbol) *
                y_mask.T)

    # r2: L'xL'
    # r3: L'xL'xB
    r2 = tensor.eye(n_y, k=1)
    # tensor.eye(n_y, k=2).dimshuffle(0, 1, 'x')   L' x L' x 1 
    # sec_diag.dimshuffle(1, 'x', 0)  L' x 1 x B
    r3 = (tensor.eye(n_y, k=2).dimshuffle(0, 1, 'x') *
          sec_diag.dimshuffle(1, 'x', 0))
    return r2, r3

	def recurrence_relation(self, y):
		def sec_diag_i(yt, ytp1, ytp2):
			return T.neq(yt, ytp2) * T.eq(ytp1, self.n_out)

		y_extend = T.concatenate((y, [self.n_out, self.n_out]))
		sec_diag, _ = theano.scan(sec_diag_i,
				sequences={'input':y_extend, 'taps':[0, 1, 2]})

		y_sz = y.shape[0]
		return T.eye(y_sz) + \
			T .eye(y_sz, k=1) + \
			T.eye(y_sz, k=2) * sec_diag.dimshuffle((0, 'x'))

 def __init__(self, v=None, **kwargs):
     super(HouseholderFlow, self).__init__(**kwargs)
     v = self.add_param(v, 'v')
     self.shared_params = dict(v=v)
     if self.batched:
         vv = v.dimshuffle(0, 1, 'x') * v.dimshuffle(0, 'x', 1)
         I = tt.eye(self.dim).dimshuffle('x', 0, 1)
         vvn = (1e-10+(v**2).sum(-1)).dimshuffle(0, 'x', 'x')
     else:
         vv = tt.outer(v, v)
         I = tt.eye(self.dim)
         vvn = ((v**2).sum(-1)+1e-10)
     self.H = I - 2. * vv / vvn

    def _mb_normal_ctc(self, network_output,   labels, mask):


        n_y = labels.shape[1] / 2
        y = labels[:,:n_y]
        y = y.dimshuffle(1,0)
        y_mask = labels[:,n_y:].astype(theano.config.floatX)

        # y_row = labels.dimshuffle(1,0)
        # n_y = y_row.shape[0] / 2
        # y = y_row[:n_y,:]
        # y_mask = y_row[n_y:,:].astype(theano.config.floatX)

        y_hat = network_output.dimshuffle(0, 2, 1)

        pred_y = y_hat[:, y.astype('int32'), T.arange(self.tpo["batch_size"])]


        ybb = T.concatenate((y, self.blanks), axis=0).T
        sec_diag = (T.neq(ybb[:, :-2], ybb[:, 2:]) *
                    T.eq(ybb[:, 1:-1], self.tpo["CTC_blank"]) *
                    y_mask)



        # r1: LxL
        # r2: LxL
        # r3: LxLxB
        r2 = T.eye(n_y, k=1)
        r3 = (T.eye(n_y, k=2).dimshuffle(0, 1, 'x') *
              sec_diag.dimshuffle(1, 'x', 0))



        def step(p_curr, p_prev):
            # instead of dot product, we * first
            # and then sum oven one dimension.
            # objective: T.dot((p_prev)BxL, LxLxB)
            # solusion: Lx1xB * LxLxB --> LxLxB --> (sumover)xLxB
            dotproduct = (p_prev + T.dot(p_prev, r2) +
                          (p_prev.dimshuffle(1, 'x', 0) * r3).sum(axis=0).T)
            return p_curr.T * dotproduct * y_mask  # B x L

        probabilities, _ = theano.scan(
            step,
            sequences=[pred_y],
            outputs_info=[T.eye(n_y)[0] * T.ones([self.tpo["batch_size"], n_y])])


        labels_probab = T.sum(probabilities[-1,:, -2:])
        return T.mean(-T.log(labels_probab))

    def grad(self, inputs, cost_grad):
        """
        Notes:
        1. The gradient is computed under the assumption that perturbations
        of the input array respect triangularity, i.e. partial derivatives wrt
        triangular region are zero.
        2. In contrast with the usual mathematical presentation, in order to
        apply theano's 'reshape' function wich implements row-order (i.e. C
        order), the differential expressions below have been derived based on
        the row-vectorizations of inputs 'a' and 'b'.

        See The Matrix Reference Manual,
        Copyright 1998-2011 Mike Brookes, Imperial College, London, UK
        """

        a, b = inputs
        ingrad = cost_grad
        ingrad = tensor.as_tensor_variable(ingrad)
        shp_a = (tensor.shape(inputs[0])[1],
                               tensor.shape(inputs[0])[1])
        I_M = tensor.eye(*shp_a)
        if self.lower:
            inv_a = solve_triangular(a, I_M, lower=True)
            tri_M = tril(tensor.ones(shp_a))
        else:
            inv_a = solve_triangular(a, I_M, lower=False)
            tri_M = triu(tensor.ones(shp_a))
        if b.ndim == 1:
            prod_a_b = tensor.tensordot(-b.T, inv_a.T, axes=1)
            prod_a_b = tensor.shape_padleft(prod_a_b)
            jac_veca = kron(inv_a, prod_a_b)
            jac_b = inv_a
            outgrad_veca = tensor.tensordot(ingrad, jac_veca, axes=1)
            outgrad_a = tensor.reshape(outgrad_veca,
                        (inputs[0].shape[0], inputs[0].shape[0])) * tri_M
            outgrad_b = tensor.tensordot(ingrad, jac_b, axes=1).flatten(ndim=1)
        else:
            ingrad_vec = ingrad.flatten(ndim=1)
            prod_a_b = tensor.tensordot(-b.T, inv_a.T, axes=1)
            jac_veca = kron(inv_a, prod_a_b)
            I_N = tensor.eye(tensor.shape(inputs[1])[1],
                               tensor.shape(inputs[1])[1])
            jac_vecb = kron(inv_a, I_N)
            outgrad_veca = tensor.tensordot(ingrad_vec, jac_veca, axes=1)
            outgrad_a = tensor.reshape(outgrad_veca,
                        (inputs[0].shape[0], inputs[0].shape[0])) * tri_M
            outgrad_vecb = tensor.tensordot(ingrad_vec, jac_vecb, axes=1)
            outgrad_b = tensor.reshape(outgrad_vecb,
                        (inputs[1].shape[0], inputs[1].shape[1]))
        return [outgrad_a, outgrad_b]

    def compute_log_averaged_ei(self, x, X, randomness, incumbent):

        # We compute the old predictive mean at x
        
        Kzz = compute_kernel(self.lls, self.lsf, self.z, self.z) + T.eye(self.z.shape[ 0 ]) * self.jitter * T.exp(self.lsf)
        KzzInv = T.nlinalg.MatrixInversePSD()(Kzz)
        LLt = T.dot(self.LParamPost, T.transpose(self.LParamPost))
        covCavityInv = KzzInv + LLt * casting(self.n_points - self.set_for_training) / casting(self.n_points)
        covCavity = T.nlinalg.MatrixInversePSD()(covCavityInv)
        meanCavity = T.dot(covCavity, casting(self.n_points - self.set_for_training) / casting(self.n_points) * self.mParamPost)
        KzzInvmeanCavity = T.dot(KzzInv, meanCavity)
        Kxz = compute_kernel(self.lls, self.lsf, x, self.z)
        m_old_x = T.dot(Kxz, KzzInvmeanCavity)

        # We compute the old predictive mean at X

        KXz = compute_kernel(self.lls, self.lsf, X, self.z)
        m_old_X = T.dot(KXz, KzzInvmeanCavity)

        # We compute the required cross covariance matrices

        KXX = compute_kernel(self.lls, self.lsf, X, X) - T.dot(T.dot(KXz, KzzInv), KXz.T) + T.eye(X.shape[ 0 ]) * self.jitter * T.exp(self.lsf)
        KXXInv = T.nlinalg.MatrixInversePSD()(KXX)

        KxX = compute_kernel(self.lls, self.lsf, x, X)
        xX = T.concatenate([ x, X ], 0)
        KxXz = compute_kernel(self.lls, self.lsf, xX, self.z)
        KxX = KxX - T.dot(T.dot(KxXz[ 0 : x.shape[ 0], : ], KzzInv), KxXz[ x.shape[ 0 ] : xX.shape[ 0 ], : ].T)

        # We compute the new posterior mean

        samples_internal = T.dot(MatrixChol()(KXX), randomness)

        new_predictive_mean = T.tile(m_old_x, [ 1, randomness.shape[ 1 ] ]) + T.dot(KxX, T.dot(KXXInv, samples_internal))

        # We compute the new posterior variance

        z_expanded = T.concatenate([ self.z, X ], 0)
        Kxz_expanded = compute_kernel(self.lls, self.lsf, x, z_expanded)
        Kzz_expanded = compute_kernel(self.lls, self.lsf, z_expanded, z_expanded) + T.eye(z_expanded.shape[ 0 ]) * self.jitter * T.exp(self.lsf)
        Kzz_expandedInv = T.nlinalg.MatrixInversePSD()(Kzz_expanded)
        v_out = T.exp(self.lsf) - T.dot(Kxz_expanded * T.dot(Kxz_expanded, Kzz_expandedInv), T.ones_like(z_expanded[ : , 0 : 1 ]))
        new_predictive_var = T.tile(v_out, [ 1, randomness.shape[ 1 ] ])

        s = (incumbent - new_predictive_mean) / T.sqrt(new_predictive_var)

        log_ei = T.log((incumbent - new_predictive_mean) * ratio(s) + T.sqrt(new_predictive_var)) + log_n_pdf(s)

        return T.mean(LogSumExp(log_ei, 1), 1)

    def recurrence_relation_(y_, blank_symbol):
        y = y_.dimshuffle(0,'x')
        n_y = y.shape[0]
        blanks = T.zeros((2, y.shape[1])) + blank_symbol
        ybb = T.concatenate((y, blanks), axis=0).T
        sec_diag = (T.neq(ybb[:, :-2], ybb[:, 2:]) *
                    T.eq(ybb[:, 1:-1], blank_symbol))

        # r1: LxL
        # r2: LxL
        # r3: LxL
        r2 = T.eye(n_y, k=1)
        r3 = (T.eye(n_y, k=2) * sec_diag)

        return r2, r3

def recurrence_relation(y, y_mask):
    # with blank symbol of -1 this falls back to the recurrence that fails
    # with repeating symbols!
    blank_symbol = -1
    n_y = y.shape[0]
    blanks = tensor.zeros((2, y.shape[1])) + blank_symbol
    ybb = tensor.concatenate((y, blanks), axis=0).T
    sec_diag = tensor.neq(ybb[:, :-2], ybb[:, 2:]) * tensor.eq(ybb[:, 1:-1], blank_symbol) * y_mask.T

    # r1: LxL
    # r2: LxL
    # r3: LxLxB
    r2 = tensor.eye(n_y, k=1)
    r3 = tensor.eye(n_y, k=2).dimshuffle(0, 1, "x") * sec_diag.dimshuffle(1, "x", 0)
    return r2, r3

 def __init__(self, z0=None, dim=None, v=None, jitter=.1):
     super(HouseholderFlow, self).__init__(dim=dim, z0=z0, jitter=jitter)
     if v is None:
         v = self.add_param(dim, 'v')
     self.shared_params = dict(v=v)
     v = v.dimshuffle(0, 'x')
     self.H = tt.eye(dim) - 2. * v.dot(v.T) / ((v**2).sum()+1e-10)

def _path_probabs(y, y_hat, y_mask, y_hat_mask, blank_symbol):
    """Compute the probabilities of the paths that are compatible with the
    sequence y.

    This function uses scan to get the forward probabilities (often denoted
    with the symbol alpha in the literature).

    See _log_path_probabs for a version that works in log domain.
    """

    pred_y = _class_batch_to_labeling_batch(y, y_hat, y_hat_mask)
    pred_y = pred_y.dimshuffle(0, 2, 1)
    n_labels = y.shape[0]

    r2, r3 = _recurrence_relation(y, y_mask, blank_symbol)

    def step(p_curr, p_prev):
        # instead of dot product, we * first
        # and then sum oven one dimension.
        # objective: T.dot((p_prev)BxL, LxLxB)
        # solusion: Lx1xB * LxLxB --> LxLxB --> (sumover)xLxB
        dotproduct = p_prev + tensor.dot(p_prev, r2) + (p_prev.dimshuffle(1, "x", 0) * r3).sum(axis=0).T
        return p_curr.T * dotproduct * y_mask.T  # B x L

    probabilities, _ = theano.scan(
        step, sequences=[pred_y], outputs_info=[tensor.eye(n_labels)[0] * tensor.ones(y.T.shape)]
    )
    return probabilities

    def get_output_for(self, input, init=False, **kwargs):
        if input.ndim > 2:
            # if the input has more than two dimensions, flatten it into a
            # batch of feature vectors.
            input = input.flatten(2)
        
        activation = T.tensordot(input, self.W, [[1], [0]])
        abs_dif = (T.sum(abs(activation.dimshuffle(0,1,2,'x') - activation.dimshuffle('x',1,2,0)),axis=2)
                    + 1e6 * T.eye(input.shape[0]).dimshuffle(0,'x',1))

        if init:
            mean_min_abs_dif = 0.5 * T.mean(T.min(abs_dif, axis=2),axis=0)
            abs_dif /= mean_min_abs_dif.dimshuffle('x',0,'x')
            self.init_updates = [(self.log_weight_scale, self.log_weight_scale-T.log(mean_min_abs_dif).dimshuffle(0,'x'))]
        
        f = T.sum(T.exp(-abs_dif),axis=2)

        if init:
            mf = T.mean(f,axis=0)
            f -= mf.dimshuffle('x',0)
            self.init_updates.append((self.b, -mf))
        else:
            f += self.b.dimshuffle('x',0)

        return T.concatenate([input, f], axis=1)

 def __init__(self, incoming, dimension, params_init=(GlorotUniform(),
                                                      GlorotUniform(),
                                                      Uniform([0, 0.1])),
              addition_parameters=[False], **kwargs):
     '''
     init parameters
     :param incoming: input to the LISTA layer
     :param dimension: 2 numbers list.
      dimension[0] is dict_size, length of dictionary vector in LISTA. dimension[1] is T a.k.a depth
     :param params_init: init value or init method for LISTA
     :transposed: = True if the input dictionary D is the transpose matrix of a theano.compile.SharedVariable V.
      In that case self.W = D^T = V^T^T = V
     :param kwargs: parameters of super class
     :return:
     '''
     super(LISTA, self).__init__(incoming, **kwargs)
     self.transposed = addition_parameters[0]
     num_inputs = incoming.output_shape[-1]
     self.dict_size = dimension[0]
     self.T = dimension[1]
     self.W = self.add_param(params_init[0], [num_inputs, self.dict_size], name='W',
                             lista=True, lista_weight_W=True, sparse_dictionary=True, regularizable=True)
     # self.S = self.add_param(params_init[1], [self.dict_size, self.dict_size], name='S',
     #                         lista=True, lista_weight_W=True, regularizable=True)
     if T > 0:
         self.S = T.eye(self.dict_size) - T.dot(self.get_dictionary(), self.get_dictionary().T)
         self.S = self.add_param(theano.shared(floatX(self.S.eval())), [self.dict_size, self.dict_size], name='S',
                                 lista=True, lista_weight_S=True, regularizable=True)
     self.theta = self.add_param(theano.shared(floatX(0.01 * np.ones([self.dict_size, ]))), [self.dict_size, ],
                                 name='theta',
                                 lista=True, lista_fun_param=True, regularizable=False)
     self.eps = 1e-6
     self.clipped_theta = T.clip(self.theta, self.eps, 10)