def __call__(self, inputs, state, scope=None):
     """Run the cell with the declared dropouts."""
     if not isinstance(self._input_keep_prob, float) or self._input_keep_prob < 1:
         inputs = nn_ops.dropout(inputs, self._input_keep_prob, seed=self._seed)
     output, new_state = self._cell(inputs, state)
     if not isinstance(self._output_keep_prob, float) or self._output_keep_prob < 1:
         output = nn_ops.dropout(output, self._output_keep_prob, seed=self._seed)
     return output, new_state

 def __call__(self, inputs, state, scope=None):
     """Run the cell with the declared dropouts."""
     if (not isinstance(self._input_keep_prob, float) or
                 self._input_keep_prob < 1):
         do_inputs = dropout(inputs, self._input_keep_prob, seed=self._seed)
         inputs = tf.cond(self._is_train, lambda: do_inputs, lambda: inputs)
     output, new_state = self._cell(inputs, state)
     if (not isinstance(self._output_keep_prob, float) or
                 self._output_keep_prob < 1):
         do_output = dropout(output, self._output_keep_prob, seed=self._seed)
         output = tf.cond(self._is_train, lambda: do_output, lambda: output)
     return output, new_state

 def testInvalidKeepProb(self):
   x_dim = 40
   y_dim = 30
   t = constant_op.constant(1.0, shape=[x_dim, y_dim], dtype=dtypes.float32)
   with self.assertRaises(ValueError):
     nn_ops.dropout(t, -1.0)
   with self.assertRaises(ValueError):
     nn_ops.dropout(t, 1.1)
   with self.assertRaises(ValueError):
     nn_ops.dropout(t, [0.0, 1.0])
   with self.assertRaises(ValueError):
     nn_ops.dropout(t, array_ops.placeholder(dtypes.float64))
   with self.assertRaises(ValueError):
     nn_ops.dropout(t, array_ops.placeholder(dtypes.float32, shape=[2]))

 def testDropoutPlaceholderKeepProb(self):
   # Runs dropout with 0-1 tensor 10 times, sum the number of ones and validate
   # that it is producing approximately the right number of ones over a large
   # number of samples, based on the keep probability.
   x_dim = 40
   y_dim = 30
   num_iter = 10
   for keep_prob in [0.1, 0.5, 0.8]:
     with self.test_session():
       t = constant_op.constant(
           1.0, shape=[x_dim, y_dim], dtype=dtypes.float32)
       keep_prob_placeholder = array_ops.placeholder(dtypes.float32)
       dropout = nn_ops.dropout(t, keep_prob_placeholder)
       final_count = 0
       self.assertEqual([x_dim, y_dim], dropout.get_shape())
       for _ in xrange(0, num_iter):
         value = dropout.eval(feed_dict={keep_prob_placeholder: keep_prob})
         final_count += np.count_nonzero(value)
         # Verifies that there are only two values: 0 and 1/keep_prob.
         sorted_value = np.unique(np.sort(value))
         self.assertEqual(0, sorted_value[0])
         self.assertAllClose(1 / keep_prob, sorted_value[1])
     # Check that we are in the 15% error range
     expected_count = x_dim * y_dim * keep_prob * num_iter
     rel_error = math.fabs(final_count - expected_count) / expected_count
     print(rel_error)
     self.assertTrue(rel_error < 0.15)

  def testNoDropoutFast(self):
    x = array_ops.zeros((5,))
    y = nn_ops.dropout(x, keep_prob=1)
    self.assertTrue(x is y)

    y = nn_ops.dropout_v2(x, rate=0)
    self.assertTrue(x is y)

  def testShapedDropout(self):
    # Runs dropout with 0-1 tensor 10 times, sum the number of ones and validate
    # that it is producing approximately the right number of ones over a large
    # number of samples, based on the keep probability. This time with shaped
    # noise.
    x_dim = 40 * 30
    y_dim = 3
    num_iter = 10
    for keep_prob in [0.1, 0.5, 0.8]:
      t = constant_op.constant(1.0, shape=[x_dim, y_dim], dtype=dtypes.float32)
      dropout = nn_ops.dropout(t, keep_prob, noise_shape=[x_dim, 1])
      self.assertEqual([x_dim, y_dim], dropout.get_shape())
      final_count = 0
      for _ in xrange(0, num_iter):
        value = self.evaluate(dropout)
        final_count += np.count_nonzero(value)
        # Verifies that there are only two values: 0 and 1/keep_prob.
        sorted_value = np.unique(np.sort(value))
        self.assertEqual(0, sorted_value[0])
        self.assertAllClose(1 / keep_prob, sorted_value[1])

      # Check that we are in the 15% error range
      expected_count = x_dim * y_dim * keep_prob * num_iter
      rel_error = math.fabs(final_count - expected_count) / expected_count
      print(rel_error)
      self.assertTrue(rel_error < 0.15)

  def __call__(self, inputs, state, scope=None):
    """LSTM cell with layer normalization and recurrent dropout."""

    with vs.variable_scope(scope or type(self).__name__) as scope:  # LayerNormBasicLSTMCell  # pylint: disable=unused-variables
      c, h = state
      args = array_ops.concat(1, [inputs, h])
      concat = self._linear(args)

      i, j, f, o = array_ops.split(1, 4, concat)
      if self._layer_norm:
        i = self._norm(i, "input")
        j = self._norm(j, "transform")
        f = self._norm(f, "forget")
        o = self._norm(o, "output")

      g = self._activation(j)
      if (not isinstance(self._keep_prob, float)) or self._keep_prob < 1:
        g = nn_ops.dropout(g, self._keep_prob, seed=self._seed)

      new_c = (c * math_ops.sigmoid(f + self._forget_bias)
               + math_ops.sigmoid(i) * g)
      if self._layer_norm:
        new_c = self._norm(new_c, "state")
      new_h = self._activation(new_c) * math_ops.sigmoid(o)

      new_state = rnn_cell.LSTMStateTuple(new_c, new_h)
      return new_h, new_state

 def testShapedDropoutUnknownShape(self):
   x_dim = 40
   y_dim = 30
   keep_prob = 0.5
   x = constant_op.constant(1.0, shape=[x_dim, y_dim], dtype=dtypes.float32)
   dropout_x = nn_ops.dropout(
       x, keep_prob, noise_shape=array_ops.placeholder(dtypes.int32))
   self.assertEqual(x.get_shape(), dropout_x.get_shape())

    def body(i, prev_c, prev_h, actions, log_probs):
      # pylint: disable=g-long-lambda
      signal = control_flow_ops.cond(
          math_ops.equal(i, 0),
          lambda: array_ops.tile(device_go_embedding,
                                 [self.hparams.num_children, 1]),
          lambda: embedding_ops.embedding_lookup(device_embeddings,
                                                 actions.read(i - 1))
      )
      if self.hparams.keep_prob is not None:
        signal = nn_ops.dropout(signal, self.hparams.keep_prob)
      next_c, next_h = lstm(signal, prev_c, prev_h, w_lstm, forget_bias)
      query = math_ops.matmul(next_h, attn_w_2)
      query = array_ops.reshape(
          query, [self.hparams.num_children, 1, self.hparams.hidden_size])
      query = math_ops.tanh(query + attn_mem)
      query = array_ops.reshape(query, [
          self.hparams.num_children * self.num_groups, self.hparams.hidden_size
      ])
      query = math_ops.matmul(query, attn_v)
      query = array_ops.reshape(query,
                                [self.hparams.num_children, self.num_groups])
      query = nn_ops.softmax(query)
      query = array_ops.reshape(query,
                                [self.hparams.num_children, self.num_groups, 1])
      query = math_ops.reduce_sum(attn_mem * query, axis=1)
      query = array_ops.concat([next_h, query], axis=1)
      logits = math_ops.matmul(query, device_softmax)
      logits /= self.hparams.temperature
      if self.hparams.tanh_constant > 0:
        logits = math_ops.tanh(logits) * self.hparams.tanh_constant
      if self.hparams.logits_std_noise > 0:
        num_in_logits = math_ops.cast(
            array_ops.size(logits), dtype=dtypes.float32)
        avg_norm = math_ops.divide(
            linalg_ops.norm(logits), math_ops.sqrt(num_in_logits))
        logits_noise = random_ops.random_normal(
            array_ops.shape(logits),
            stddev=self.hparams.logits_std_noise * avg_norm)
        logits = control_flow_ops.cond(
            self.global_step > self.hparams.stop_noise_step, lambda: logits,
            lambda: logits + logits_noise)

      if mode == "sample":
        next_y = random_ops.multinomial(logits, 1, seed=self.hparams.seed)
      elif mode == "greedy":
        next_y = math_ops.argmax(logits, 1)
      elif mode == "target":
        next_y = array_ops.slice(y, [0, i], [-1, 1])
      else:
        raise NotImplementedError
      next_y = math_ops.to_int32(next_y)
      next_y = array_ops.reshape(next_y, [self.hparams.num_children])
      actions = actions.write(i, next_y)
      log_probs += nn_ops.sparse_softmax_cross_entropy_with_logits(
          logits=logits, labels=next_y)
      return i + 1, next_c, next_h, actions, log_probs

 def testShapedDropoutCorrelation(self):
   # Runs a shaped dropout and tests that the correlations are correct.
   x_dim = 40
   y_dim = 30
   num_iter = 10
   for keep_prob in [0.1, 0.5, 0.8]:
     t = constant_op.constant(1.0, shape=[x_dim, y_dim], dtype=dtypes.float32)
     dropout = nn_ops.dropout(t, keep_prob, noise_shape=[x_dim, 1])
     self.assertEqual([x_dim, y_dim], dropout.get_shape())
     for _ in xrange(0, num_iter):
       value = self.evaluate(dropout)
       # Verifies that each y column as only one type of activation.
       for i in xrange(x_dim):
         sorted_value = np.unique(np.sort(value[i, :]))
         self.assertEqual(sorted_value.size, 1)

 def testShapedDropoutShapeError(self):
   # Runs shaped dropout and verifies an error is thrown on misshapen noise.
   x_dim = 40
   y_dim = 30
   keep_prob = 0.5
   t = constant_op.constant(1.0, shape=[x_dim, y_dim], dtype=dtypes.float32)
   with self.assertRaises(ValueError):
     _ = nn_ops.dropout(t, keep_prob, noise_shape=[x_dim, y_dim + 10])
   with self.assertRaises(ValueError):
     _ = nn_ops.dropout(t, keep_prob, noise_shape=[x_dim, y_dim, 5])
   with self.assertRaises(ValueError):
     _ = nn_ops.dropout(t, keep_prob, noise_shape=[x_dim + 3])
   with self.assertRaises(ValueError):
     _ = nn_ops.dropout(t, keep_prob, noise_shape=[x_dim])
   # test that broadcasting proceeds
   _ = nn_ops.dropout(t, keep_prob, noise_shape=[y_dim])
   _ = nn_ops.dropout(t, keep_prob, noise_shape=[1, y_dim])
   _ = nn_ops.dropout(t, keep_prob, noise_shape=[x_dim, 1])
   _ = nn_ops.dropout(t, keep_prob, noise_shape=[1, 1])

  def testPartialShapedDropout(self):
    x_dim = 40 * 30
    y_dim = 3
    num_iter = 10
    for keep_prob in [0.1, 0.5, 0.8]:
      t = constant_op.constant(1.0, shape=[x_dim, y_dim], dtype=dtypes.float32)
      # Set noise_shape=[None, 1] which means [x_dim, 1].
      dropout = nn_ops.dropout(t, keep_prob, noise_shape=[None, 1])
      self.assertEqual([x_dim, y_dim], dropout.get_shape())
      final_count = 0
      for _ in xrange(0, num_iter):
        value = self.evaluate(dropout)
        final_count += np.count_nonzero(value)
        # Verifies that there are only two values: 0 and 1/keep_prob.
        sorted_value = np.unique(np.sort(value))
        self.assertEqual(0, sorted_value[0])
        self.assertAllClose(1 / keep_prob, sorted_value[1])

      # Check that we are in the 15% error range
      expected_count = x_dim * y_dim * keep_prob * num_iter
      rel_error = math.fabs(final_count - expected_count) / expected_count
      print(rel_error)
      self.assertTrue(rel_error < 0.15)

 def dropout(i, do_dropout, v):
   if not isinstance(do_dropout, bool) or do_dropout:
     return nn_ops.dropout(
         v, keep_prob=keep_prob, seed=self._gen_seed(salt_prefix, i))
   else:
     return v

 def dropout(i, v):
   return nn_ops.dropout(
       v, keep_prob=keep_prob, seed=self._gen_seed(salt_prefix, i))

 def testNoDropoutFast(self):
   x = array_ops.zeros((5,))
   for p in 1, constant_op.constant(1.0):
     y = nn_ops.dropout(x, keep_prob=p)
     self.assertTrue(x is y)

 def testDropoutWithIntegerInputs(self):
   x = constant_op.constant([1, 1, 1, 1, 1])
   with self.assertRaises(ValueError):
     _ = nn_ops.dropout(x, 0.5)