def ternary_operation(x):
    """Ternary operation use threshold computed with weights."""
    g = tf.compat.v1.get_default_graph()
    with g.gradient_override_map({"Sign": "Identity"}):
        threshold = _compute_threshold(x)
        x = tf.sign(tf.add(tf.sign(tf.add(x, threshold)), tf.sign(tf.add(x, -threshold))))
        return x

def noisy_activation(x, generic, linearized, training, alpha=1.1, c=0.5):
    """
    Implements the noisy activation with Half-Normal Noise for Hard-Saturation
    functions. See http://arxiv.org/abs/1603.00391, Algorithm 1.

    Args:

        x: Tensor which is an input to the activation function

        generic: The generic formulation of the activation function. (denoted
            as h in the paper)

        linearized: Linearization of the activation based on the first-order
            Tailor expansion around zero. (denoted as u in the paper)

        training: A boolean tensor telling whether we are in the training stage
            (and the noise is sampled) or in runtime when the expactation is
            used instead.

        alpha: Mixing hyper-parameter. The leakage rate from the linearized
            function to the nonlinear one.

        c: Standard deviation of the sampled noise.

    """

    delta = generic(x) - linearized(x)
    d = -tf.sign(x) * tf.sign(1 - alpha)
    p = tf.Variable(1.0)
    scale = c * (tf.sigmoid(p * delta) - 0.5)  ** 2
    noise = tf.select(training, tf.abs(tf.random_normal([])), math.sqrt(2 / math.pi))
    activation = alpha * generic(x) + (1 - alpha) * linearized(x) + d * scale * noise
    return activation

def get_accuracy_loss(arg,x,y,y_):
    '''
    Note: when the task is regression accuracy = loss but for classification
    loss = cross_entropy,svm_loss, surrogate_loss, etc and accuracy = 1 - {0-1 loss}.
    '''
    with tf.name_scope("loss_and_acc") as scope:
        # loss
        if arg.softmax:
            #cross_entropy = tf.reduce_mean(-tf.rduce_sum(y_ * tf.log(y), reduction_indices=[1]))
            diff = tf.nn.softmax_cross_entropy_with_logits(labels=y_, logits=y)
            cross_entropy = tf.reduce_mean(diff)
            loss = cross_entropy
            correct_prediction = tf.equal(tf.argmax(y,1), tf.argmax(y_,1)) # list of booleans indicating correct predictions
            accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))
        else:
            l2_loss = tf.reduce_sum( tf.reduce_mean(tf.square(y_-y), 0))
            loss = l2_loss
            y = tf.cast(tf.sign(y),tf.float32)
            y_ = tf.cast(tf.sign(y_),tf.float32)
            correct_prediction = tf.equal(y, y_) # list of booleans indicating correct predictions
            accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))
        # accuracy
        # if arg.classification:
        #     correct_prediction = tf.equal(tf.argmax(y,1), tf.argmax(y_,1)) # list of booleans indicating correct predictions
        #     accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))
        # else:
        #     accuracy = tf.equal(tf.argmax(y,1), tf.argmax(y_,1))
    ##
    tf.summary.scalar('loss', loss)
    tf.summary.scalar('accuracy', accuracy)
    return loss, accuracy

def loss_func(score_op):
    final_scores = tf.placeholder(tf.float32, shape=[None])

    squared_errors = tf.square(tf.reshape(score_op, [-1]) - final_scores)
    #mean_sq_err = tf.reduce_mean(squared_errors, name='mean_sq_err')
    cross_entropy_ish_loss = tf.reduce_mean(-tf.log(tf.constant(1.0) - tf.constant(0.5) * tf.abs(tf.reshape(score_op, [-1]) - final_scores), name='cross-entropy-ish-loss'))

    correct_prediction = tf.equal(tf.sign(tf.reshape(score_op, [-1])), tf.sign(final_scores))
    accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32), name='accuracy')
    #return final_scores, mean_sq_err, accuracy, squared_errors
    return final_scores, cross_entropy_ish_loss, accuracy

def loss_func(logits):
    final_maps = tf.placeholder(tf.float32, shape=[None, 361])

    # final maps are originally -1 to 1. rescale them to 0 to 1 probabilities:
    final_prob_maps = final_maps * tf.constant(0.5) + tf.constant(0.5)
    cross_entropy = tf.nn.sigmoid_cross_entropy_with_logits(logits=logits, targets=final_prob_maps)
    cross_entropy_mean = tf.reduce_mean(cross_entropy, name='cross_entropy_mean')

    correct_prediction = tf.equal(tf.sign(logits), tf.sign(final_maps))
    accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))
    return final_maps, cross_entropy_mean, accuracy

  def angular_symmetry(self, d_cutoff, d, atom_numbers, coordinates):
    """ Angular Symmetry Function """

    max_atoms = self.max_atoms
    embedding = tf.eye(np.max(self.atom_cases) + 1)
    atom_numbers_embedded = tf.nn.embedding_lookup(embedding, atom_numbers)

    Rs = np.linspace(0., self.angular_cutoff, self.angular_length)
    ita = 3 / (Rs[1] - Rs[0])**2
    thetas = np.linspace(0., np.pi, self.angular_length)
    zeta = float(self.angular_length**2)

    ita, zeta, Rs, thetas = np.meshgrid(ita, zeta, Rs, thetas)
    zeta = tf.cast(np.reshape(zeta, (1, 1, 1, 1, -1)), tf.float32)
    ita = tf.cast(np.reshape(ita, (1, 1, 1, 1, -1)), tf.float32)
    Rs = tf.cast(np.reshape(Rs, (1, 1, 1, 1, -1)), tf.float32)
    thetas = tf.cast(np.reshape(thetas, (1, 1, 1, 1, -1)), tf.float32)
    length = zeta.get_shape().as_list()[-1]

    vector_distances = tf.stack([coordinates] * max_atoms, 1) - tf.stack(
        [coordinates] * max_atoms, 2)
    R_ij = tf.stack([d] * max_atoms, axis=3)
    R_ik = tf.stack([d] * max_atoms, axis=2)
    f_R_ij = tf.stack([d_cutoff] * max_atoms, axis=3)
    f_R_ik = tf.stack([d_cutoff] * max_atoms, axis=2)

    # Define angle theta = arccos(R_ij(Vector) dot R_ik(Vector)/R_ij(distance)/R_ik(distance))
    vector_mul = tf.reduce_sum(tf.stack([vector_distances] * max_atoms, axis=3) * \
                               tf.stack([vector_distances] * max_atoms, axis=2), axis=4)
    vector_mul = vector_mul * tf.sign(f_R_ij) * tf.sign(f_R_ik)
    theta = tf.acos(tf.math.divide(vector_mul, R_ij * R_ik + 1e-5))

    R_ij = tf.stack([R_ij] * length, axis=4)
    R_ik = tf.stack([R_ik] * length, axis=4)
    f_R_ij = tf.stack([f_R_ij] * length, axis=4)
    f_R_ik = tf.stack([f_R_ik] * length, axis=4)
    theta = tf.stack([theta] * length, axis=4)

    out_tensor = tf.pow((1. + tf.cos(theta - thetas)) / 2., zeta) * \
                 tf.exp(-ita * tf.square((R_ij + R_ik) / 2. - Rs)) * f_R_ij * f_R_ik * 2

    if self.atomic_number_differentiated:
      out_tensors = []
      for id_j, atom_type_j in enumerate(self.atom_cases):
        for atom_type_k in self.atom_cases[id_j:]:
          selected_atoms = tf.stack([atom_numbers_embedded[:, :, atom_type_j]] * max_atoms, axis=2) * \
                           tf.stack([atom_numbers_embedded[:, :, atom_type_k]] * max_atoms, axis=1)
          selected_atoms = tf.expand_dims(
              tf.expand_dims(selected_atoms, axis=1), axis=4)
          out_tensors.append(
              tf.reduce_sum(out_tensor * selected_atoms, axis=(2, 3)))
      return tf.concat(out_tensors, axis=2)
    else:
      return tf.reduce_sum(out_tensor, axis=(2, 3))

def main():
    data, labels = input()
    #     logits=inference()
    #     loss_step=loss(logits)
    #     train_step = train(loss_step,0.001)
    #     sess=tf.Session()
    #     sess.run(tf.global_variables_initializer())
    #     print(sess.run([w,b]))
    #     labels.shape=(6000,1)
    #     for i in range(1000):
    #         sess.run(train_step,feed_dict={x_placehold:data,y_placehold:labels})
    #     wc,bc=sess.run([w,b],feed_dict={x_placehold:data,y_placehold:labels})
    #     print(wc,bc)

    labels.shape = (6000, 1)
    print(data)
    print(labels)
    print(data.shape)
    print(labels.shape)
    print(feed_dict('D'))
    sess = tf.Session()
    sess.run(tf.global_variables_initializer())
    # sess.run(tf.sign(tf.matmul(x_placehold, w) + b),
    #          feed_dict=feed_dict('D'))
    sess.run(tf.sign(tf.matmul(x_placehold, w) + b) - y_placehold,
             feed_dict=feed_dict())
    sess.run(tf.square(tf.sign(tf.matmul(x_placehold, w) + b) -
                       y_placehold), feed_dict=feed_dict())
    sess.run(tf.reduce_sum(tf.square(tf.sign(tf.matmul(x_placehold, w) + b) -
                                     y_placehold)), feed_dict=feed_dict())

    logits = tf.matmul(x_placehold, w) + b
    loss_op = tf.reduce_sum(tf.square(logits - y_placehold))

    with tf.name_scope('loss'):
        tf.summary.scalar('error', loss_op)

    with tf.name_scope('w'):
        tf.summary.scalar('x', w[0, 0])
        tf.summary.scalar('y', w[1, 0])

    merged = tf.summary.merge_all()
    train_writer = tf.summary.FileWriter('./train', sess.graph)

    optimizer = tf.train.GradientDescentOptimizer(0.1)

    for i in range(1000):
        sess.run(optimizer.minimize(loss_op), feed_dict=feed_dict())
        summary = sess.run(merged, feed_dict=feed_dict())
        train_writer.add_summary(summary, i)

    wc, bc = sess.run([w, b], feed_dict=feed_dict())
    print(wc, bc)

def neural_attention(embedding_dim=384, encoding_dim=128):
    embeddings = tf.Variable(tf.random_normal([vocab_size, embedding_dim], stddev=0.22), dtype=tf.float32)
    tf.contrib.layers.apply_regularization(tf.contrib.layers.l2_regularizer(1e-4), [embeddings])

    with tf.variable_scope('encode'):
        with tf.variable_scope('X'):
            X_lens = tf.reduce_sum(tf.sign(tf.abs(X)), 1)
            embedded_X = tf.nn.embedding_lookup(embeddings, X)
            encoded_X = tf.nn.dropout(embedded_X, keep_prob)
            gru_cell = tf.contrib.rnn.core_rnn_cell.GRUCell(encoding_dim)
            outputs, output_states = tf.nn.bidirectional_dynamic_rnn(gru_cell, gru_cell, encoded_X,
                                                                     sequence_length=X_lens, dtype=tf.float32,
                                                                     swap_memory=True)
            encoded_X = tf.concat(outputs, 2)
        with tf.variable_scope('Q'):
            Q_lens = tf.reduce_sum(tf.sign(tf.abs(Q)), 1)
            embedded_Q = tf.nn.embedding_lookup(embeddings, Q)
            encoded_Q = tf.nn.dropout(embedded_Q, keep_prob)
            gru_cell = tf.contrib.rnn.core_rnn_cell.GRUCell(encoding_dim)
            outputs, output_states = tf.nn.bidirectional_dynamic_rnn(gru_cell, gru_cell, encoded_Q,
                                                                     sequence_length=Q_lens, dtype=tf.float32,
                                                                     swap_memory=True)
            encoded_Q = tf.concat(outputs, 2)

    W_q = tf.Variable(tf.random_normal([2 * encoding_dim, 4 * encoding_dim], stddev=0.22), dtype=tf.float32)
    b_q = tf.Variable(tf.random_normal([2 * encoding_dim, 1], stddev=0.22), dtype=tf.float32)
    W_d = tf.Variable(tf.random_normal([2 * encoding_dim, 6 * encoding_dim], stddev=0.22), dtype=tf.float32)
    b_d = tf.Variable(tf.random_normal([2 * encoding_dim, 1], stddev=0.22), dtype=tf.float32)
    g_q = tf.Variable(tf.random_normal([10 * encoding_dim, 2 * encoding_dim], stddev=0.22), dtype=tf.float32)
    g_d = tf.Variable(tf.random_normal([10 * encoding_dim, 2 * encoding_dim], stddev=0.22), dtype=tf.float32)

    with tf.variable_scope('attend') as scope:
        infer_gru = tf.contrib.rnn.core_rnn_cell.GRUCell(4 * encoding_dim)
        infer_state = infer_gru.zero_state(batch_size, tf.float32)
        for iter_step in range(8):
            if iter_step > 0:
                scope.reuse_variables()

            _, q_glimpse = glimpse(W_q, b_q, encoded_Q, infer_state)
            d_attention, d_glimpse = glimpse(W_d, b_d, encoded_X, tf.concat([infer_state, q_glimpse], 1 ))

            gate_concat = tf.concat([infer_state, q_glimpse, d_glimpse, q_glimpse * d_glimpse], 1)

            r_d = tf.sigmoid(tf.matmul(gate_concat, g_d))
            r_d = tf.nn.dropout(r_d, keep_prob)
            r_q = tf.sigmoid(tf.matmul(gate_concat, g_q))
            r_q = tf.nn.dropout(r_q, keep_prob)

            combined_gated_glimpse = tf.concat([r_q * q_glimpse, r_d * d_glimpse], 1)
            _, infer_state = infer_gru(combined_gated_glimpse, infer_state)

    return tf.to_float(tf.sign(tf.abs(X))) * d_attention

 def one_bp_iteration(self, xe_v2c_pre_iter, H_sumC_to_V, H_sumV_to_C, xe_0):
     xe_tanh = tf.tanh(tf.to_double(tf.truediv(xe_v2c_pre_iter, [2.0])))
     xe_tanh = tf.to_float(xe_tanh)
     xe_tanh_temp = tf.sign(xe_tanh)
     xe_sum_log_img = tf.matmul(H_sumC_to_V, tf.multiply(tf.truediv((1 - xe_tanh_temp), [2.0]), [3.1415926]))
     xe_sum_log_real = tf.matmul(H_sumC_to_V, tf.log(1e-8 + tf.abs(xe_tanh)))
     xe_sum_log_complex = tf.complex(xe_sum_log_real, xe_sum_log_img)
     xe_product = tf.real(tf.exp(xe_sum_log_complex))
     xe_product_temp = tf.multiply(tf.sign(xe_product), -2e-7)
     xe_pd_modified = tf.add(xe_product, xe_product_temp)
     xe_v_sumc = tf.multiply(self.atanh(xe_pd_modified), [2.0])
     xe_c_sumv = tf.add(xe_0, tf.matmul(H_sumV_to_C, xe_v_sumc))
     return xe_v_sumc, xe_c_sumv

 def _apply(self, grad, var, indices=None):
   lr = tf.cast(self._learning_rate_tensor, var.dtype.base_dtype)
   m = self.get_slot(var, "m")
   # m_t = beta1 * m + (1 - beta1) * g_t
   beta1_t = tf.cast(self._beta1_t, var.dtype.base_dtype)
   m_scaled_g_values = grad * (1 - beta1_t)
   m_t = tf.assign(m, m * beta1_t, use_locking=self._use_locking)
   with tf.control_dependencies([m_t]):
     m_t = self._assign_add(m, updates=m_scaled_g_values, indices=indices)
   # update = lr * grad * where(...)
   m_gathered = self._gather(m_t, indices=indices)
   ones = tf.ones_like(grad)
   update = lr * grad * tf.where(tf.equal(tf.sign(m_gathered), tf.sign(grad)), ones, ones * self._decrease_factor)
   var_update = self._assign_sub(ref=var, updates=update, indices=indices)
   return tf.group(*[var_update, m_t])

    def _apply_dense(self, grad, var):
        lr_t = math_ops.cast(self._lr_t, var.dtype.base_dtype)
        alpha_t = math_ops.cast(self._alpha_t, var.dtype.base_dtype)
        beta_t = math_ops.cast(self._beta_t, var.dtype.base_dtype)

        eps = 1e-7  # cap for moving average

        m = self.get_slot(var, "m")
        m_t = m.assign(tf.maximum(beta_t * m + eps, tf.abs(grad)))

        var_update = state_ops.assign_sub(var, lr_t * grad * tf.exp(
            tf.log(alpha_t) * tf.sign(grad) * tf.sign(m_t)))  # Update 'ref' by subtracting 'value
        # Create an op that groups multiple operations.
        # When this op finishes, all ops in input have finished
        return control_flow_ops.group(*[var_update, m_t])

 def build(self):
   """ tensorflow computation graph for transform """
   graph = tf.Graph()
   with graph.as_default():
     self.inputs = tf.placeholder(tf.float32, shape=(None, self.max_atoms, 4))
     atom_numbers = tf.cast(self.inputs[:, :, 0], tf.int32)
     flags = tf.sign(atom_numbers)
     flags = tf.cast(
         tf.expand_dims(flags, 1) * tf.expand_dims(flags, 2), tf.float32)
     coordinates = self.inputs[:, :, 1:]
     if self.coordinates_in_bohr:
       coordinates = coordinates * 0.52917721092
     d = self.distance_matrix(coordinates, flags)
     d_radial_cutoff = self.distance_cutoff(d, self.radial_cutoff, flags)
     d_angular_cutoff = self.distance_cutoff(d, self.angular_cutoff, flags)
     radial_sym = self.radial_symmetry(d_radial_cutoff, d, atom_numbers)
     angular_sym = self.angular_symmetry(d_angular_cutoff, d, atom_numbers,
                                         coordinates)
     self.outputs = tf.concat(
         [
             tf.cast(tf.expand_dims(atom_numbers, 2), tf.float32), radial_sym,
             angular_sym
         ],
         axis=2)
   return graph

        def __graph__():
            """Building the inference graph"""

            with tf.name_scope('input'):
                # [BATCH_SIZE, NUM_FEATURES]
                x_input = tf.placeholder(dtype=tf.float32, shape=[None, self.num_features], name='x_input')

                # [BATCH_SIZE]
                y_input = tf.placeholder(dtype=tf.uint8, shape=[None], name='y_input')

                # [BATCH_SIZE, NUM_CLASSES]
                y_onehot = tf.one_hot(indices=y_input, depth=self.num_classes, on_value=1, off_value=-1,
                                      name='y_onehot')

            learning_rate = tf.placeholder(dtype=tf.float32, name='learning_rate')

            with tf.name_scope('training_ops'):
                with tf.name_scope('weights'):
                    weight = tf.get_variable(name='weights',
                                             initializer=tf.random_normal([self.num_features, self.num_classes],
                                                                          stddev=0.01))
                    self.variable_summaries(weight)
                with tf.name_scope('biases'):
                    bias = tf.get_variable(name='biases', initializer=tf.constant([0.1], shape=[self.num_classes]))
                    self.variable_summaries(bias)
                with tf.name_scope('Wx_plus_b'):
                    output = tf.matmul(x_input, weight) + bias
                    tf.summary.histogram('pre-activations', output)

            with tf.name_scope('svm'):
                regularization = tf.reduce_mean(tf.square(weight))
                hinge_loss = tf.reduce_mean(tf.square(tf.maximum(tf.zeros([self.batch_size, self.num_classes]),
                                                                 1 - tf.cast(y_onehot, tf.float32) * output)))
                with tf.name_scope('loss'):
                    loss = regularization + self.svm_c * hinge_loss
            tf.summary.scalar('loss', loss)

            optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate).minimize(loss)

            with tf.name_scope('accuracy'):
                predicted_class = tf.sign(output)
                predicted_class = tf.identity(predicted_class, name='prediction')
                with tf.name_scope('correct_prediction'):
                    correct = tf.equal(tf.argmax(predicted_class, 1), tf.argmax(y_onehot, 1))
                with tf.name_scope('accuracy'):
                    accuracy = tf.reduce_mean(tf.cast(correct, 'float'))
            tf.summary.scalar('accuracy', accuracy)

            merged = tf.summary.merge_all()

            self.x_input = x_input
            self.y_input = y_input
            self.y_onehot = y_onehot
            self.learning_rate = learning_rate
            self.loss = loss
            self.optimizer = optimizer
            self.output = output
            self.predicted_class = predicted_class
            self.accuracy = accuracy
            self.merged = merged

def triangle_wave(frequency):
  """Emit a triangle wave at the given frequency."""
  xs = tf.reshape(tf.range(_samples(), dtype=tf.float32), [1, _samples(), 1])
  ts = xs / FLAGS.sample_rate
  #
  # A triangle wave looks like this:
  #
  #      /\      /\
  #     /  \    /  \
  #         \  /    \  /
  #          \/      \/
  #
  # If we look at just half a period (the first four slashes in the
  # diagram above), we can see that it looks like a transformed absolute
  # value function.
  #
  # Let's start by computing the times relative to the start of each
  # half-wave pulse (each individual "mountain" or "valley", of which
  # there are four in the above diagram).
  half_pulse_index = ts * (frequency * 2)
  half_pulse_angle = half_pulse_index % 1.0  # in [0, 1]
  #
  # Now, we can see that each positive half-pulse ("mountain") has
  # amplitude given by A(z) = 0.5 - abs(z - 0.5), and then normalized:
  absolute_amplitude = (0.5 - tf.abs(half_pulse_angle - 0.5)) / 0.5
  #
  # But every other half-pulse is negative, so we should invert these.
  half_pulse_parity = tf.sign(1 - (half_pulse_index % 2.0))
  amplitude = half_pulse_parity * absolute_amplitude
  #
  # This is precisely the desired result, so we're done!
  return amplitude

    def binomial_sampling(self, pr):

        """
        Binomial sampling of hidden units activations using a rejection method.

        Basic mechanics:
            1) Extract a random number from a uniform distribution (g) and compare it with
                the unit's probability (pr)

            2) Choose 0 if pr<g, 1 otherwise. It is convenient to implement this condtion using
               the relu function.

        Args:
            pr (tensor, float32): input conditional probability
            g  (np.array, float32):  uniform probability used for comparison

        Returns:
            h_sampled (tensor, float32): sampled units. The value is 1 if pr>g and 0 otherwise.
        """

        np.random.seed(self.seed)

        # sample from a Bernoulli distribution with same dimensions as input distribution
        g = tf.convert_to_tensor(np.random.uniform(size=pr.shape[1]), dtype=tf.float32)

        # sample the value of the hidden units
        h_sampled = tf.nn.relu(tf.sign(pr - g))

        return h_sampled

def retrieve_seq_length_op(data):
    """An op to compute the length of a sequence. 0 are masked. """
    with tf.name_scope('GetLength'):
        used = tf.sign(x=tf.reduce_max(tf.abs(data), axis=2))
        length = tf.reduce_sum(input_tensor=used, axis=1)
        length = tf.cast(x=length, dtype=tf.int32)
    return length

def loss(logits, labels):
    """Calculates Mean Pixel Error.
    
    Args:
      logits: Logits from inference().
      labels: Labels from distorted_inputs or inputs(). 1-D tensor
              of shape [batch_size]
    
    Returns:
      Loss tensor of type float.
    """
    
    labelValidity = tf.sign(labels, name='label_validity')
    
    minop = tf.sub(logits, labels, name='Diff_Op')
    
    absop = tf.abs(minop, name='Abs_Op')
    
    lossValues = tf.mul(labelValidity, absop, name='lossValues')
    
    loss_mean = tf.reduce_mean(lossValues, name='MeanPixelError')
    
    tf.add_to_collection('losses', loss_mean)
    
    return tf.add_n(tf.get_collection('losses'), name='total_loss'), loss_mean

 def encode(self, x, noise):
   x = tf.to_float(x)
   # we can't use tf.pow(..., 8.0) because of a high-error approximation
   # on TPU.  Instead we square three times.
   x = tf.sign(x) * tf.square(tf.square(tf.square(tf.abs(x) * 128.0)))
   x = _to_bfloat16_unbiased(x, noise)
   return x

 def _non_linear_grad(cls, op, grad):
     LRP.logger.debug("Computing non-linear gradient with activation type {}".format(op.type))
     op_out = op.outputs[0]
     op_in = op.inputs[0]
     stabilizer_epsilon = cls._eps * tf.sign(op_in)
     op_in += stabilizer_epsilon
     return grad * op_out / op_in

 def __call__(self,x,keep_prob=1.0,seq_length=None):  #__call__ is very efficient when the state of instance changes frequently 
   with tf.variable_scope(self.name,reuse = self.reuse) as vs:
     self.fw_cell =tf.contrib.rnn.LSTMCell(self.cell_size,state_is_tuple=True,reuse=tf.get_variable_scope().reuse)
     self.fw_cell1 =tf.contrib.rnn.LSTMCell(self.cell_size,state_is_tuple=True,reuse=tf.get_variable_scope().reuse)
     
     self.bw_cell =tf.contrib.rnn.LSTMCell(self.cell_size,state_is_tuple=True,reuse=tf.get_variable_scope().reuse)
     self.bw_cell1 =tf.contrib.rnn.LSTMCell(self.cell_size,state_is_tuple=True,reuse=tf.get_variable_scope().reuse)
 
 
     self.fw_cells = tf.contrib.rnn.MultiRNNCell([self.fw_cell,self.fw_cell1],state_is_tuple=True)
     self.bw_cells = tf.contrib.rnn.MultiRNNCell([self.bw_cell,self.bw_cell1],state_is_tuple=True)
     
     if seq_length ==None:  #get the real sequence length (suppose that the padding are zeros)
       used = tf.sign(tf.reduce_max(tf.abs(x),reduction_indices=2))
       seq_length = tf.cast(tf.reduce_sum(used,reduction_indices=1),tf.int32)
     
     lstm_out,_,_ =  tf.contrib.rnn.static_bidirectional_rnn(self.fw_cells,self.bw_cells,tf.unstack(tf.transpose(x,[1,0,2])),dtype=tf.float32,sequence_length=seq_length)
     
     lstm_out = tf.transpose(tf.stack(lstm_out),[1,0,2])
     print 'lstm_out: ',lstm_out
     
     #shape(lstm_out) = (self.batch_size,sequence_length,2*cell_size)
     
     #if keep_prob < 1.:
     #  lstm_out = tf.nn.dropout(lstm_out,keep_prob)
       
     if self.reuse is None:
       self.trainable_weights = vs.global_variables()
       
   self.reuse =True
   return lstm_out,seq_length