def testFunctionalDenseTwiceReuse(self):
   inputs = random_ops.random_uniform((5, 3), seed=1)
   core_layers.dense(inputs, 2, name='my_dense')
   vars1 = variables.trainable_variables()
   core_layers.dense(inputs, 2, name='my_dense', reuse=True)
   vars2 = variables.trainable_variables()
   self.assertEqual(vars1, vars2)

 def testFunctionalDenseTwice(self):
   inputs = random_ops.random_uniform((5, 3), seed=1)
   core_layers.dense(inputs, 2)
   vars1 = variables.trainable_variables()
   core_layers.dense(inputs, 2)
   vars2 = variables.trainable_variables()
   self.assertEqual(len(vars1), 2)
   self.assertEqual(len(vars2), 4)

 def testFunctionalDenseTwice(self):
   inputs = random_ops.random_uniform((5, 3), seed=1)
   core_layers.dense(inputs, 2)
   vars1 = _get_variable_dict_from_varstore().values()
   core_layers.dense(inputs, 2)
   vars2 = _get_variable_dict_from_varstore().values()
   self.assertEqual(len(vars1), 2)
   self.assertEqual(len(vars2), 4)

  def dnn_logit_fn(features, mode):
    """Deep Neural Network logit_fn.

    Args:
      features: This is the first item returned from the `input_fn`
                passed to `train`, `evaluate`, and `predict`. This should be a
                single `Tensor` or `dict` of same.
      mode: Optional. Specifies if this training, evaluation or prediction. See
            `ModeKeys`.

    Returns:
      A `Tensor` representing the logits, or a list of `Tensor`'s representing
      multiple logits in the MultiHead case.
    """
    with variable_scope.variable_scope(
        'input_from_feature_columns',
        values=tuple(six.itervalues(features)),
        partitioner=input_layer_partitioner):
      net = feature_column_lib.input_layer(
          features=features, feature_columns=feature_columns)

    for layer_id, num_hidden_units in enumerate(hidden_units):
      with variable_scope.variable_scope(
          'hiddenlayer_%d' % layer_id, values=(net,)) as hidden_layer_scope:
        net = core_layers.dense(
            net,
            units=num_hidden_units,
            activation=activation_fn,
            kernel_initializer=init_ops.glorot_uniform_initializer(),
            name=hidden_layer_scope)
        if dropout is not None and mode == model_fn.ModeKeys.TRAIN:
          net = core_layers.dropout(net, rate=dropout, training=True)
      _add_hidden_layer_summary(net, hidden_layer_scope.name)

    if isinstance(units, int):
      with variable_scope.variable_scope(
          'logits', values=(net,)) as logits_scope:
        logits = core_layers.dense(
            net,
            units=units,
            activation=None,
            kernel_initializer=init_ops.glorot_uniform_initializer(),
            name=logits_scope)
      _add_hidden_layer_summary(logits, logits_scope.name)
    else:
      logits = []
      for head_index, logits_dimension in enumerate(units):
        with variable_scope.variable_scope(
            'logits_head_{}'.format(head_index), values=(net,)) as logits_scope:
          these_logits = core_layers.dense(
              net,
              units=logits_dimension,
              activation=None,
              kernel_initializer=init_ops.glorot_uniform_initializer(),
              name=logits_scope)
        _add_hidden_layer_summary(these_logits, logits_scope.name)
        logits.append(these_logits)
    return logits

 def testFunctionalDenseWithCustomGetter(self):
   called = [0]
   def custom_getter(getter, *args, **kwargs):
     called[0] += 1
     return getter(*args, **kwargs)
   with tf.variable_scope('test', custom_getter=custom_getter):
     inputs = tf.random_uniform((5, 3), seed=1)
     core_layers.dense(inputs, 2)
   self.assertEqual(called[0], 2)

  def dnn_logit_fn(features, mode):
    """Deep Neural Network logit_fn.

    Args:
      features: This is the first item returned from the `input_fn`
                passed to `train`, `evaluate`, and `predict`. This should be a
                single `Tensor` or `dict` of same.
      mode: Optional. Specifies if this training, evaluation or prediction. See
            `ModeKeys`.

    Returns:
      A `Tensor` representing the logits, or a list of `Tensor`'s representing
      multiple logits in the MultiHead case.
    """
    is_training = mode == model_fn.ModeKeys.TRAIN
    with variable_scope.variable_scope(
        'input_from_feature_columns',
        values=tuple(six.itervalues(features)),
        partitioner=input_layer_partitioner):
      net = feature_column_lib.input_layer(
          features=features, feature_columns=feature_columns)
    for layer_id, num_hidden_units in enumerate(hidden_units):
      with variable_scope.variable_scope(
          'hiddenlayer_%d' % layer_id, values=(net,)) as hidden_layer_scope:
        net = core_layers.dense(
            net,
            units=num_hidden_units,
            activation=activation_fn,
            kernel_initializer=init_ops.glorot_uniform_initializer(),
            name=hidden_layer_scope)
        if dropout is not None and is_training:
          net = core_layers.dropout(net, rate=dropout, training=True)
        if batch_norm:
          # TODO(hjm): In future, if this becomes popular, we can enable
          # customization of the batch normalization params by accepting a
          # list of `BatchNormalization` instances as `batch_norm`.
          net = normalization.batch_normalization(
              net,
              # The default momentum 0.99 actually crashes on certain
              # problem, so here we use 0.999, which is the default of
              # tf.contrib.layers.batch_norm.
              momentum=0.999,
              training=is_training,
              name='batchnorm_%d' % layer_id)
      _add_hidden_layer_summary(net, hidden_layer_scope.name)

    with variable_scope.variable_scope('logits', values=(net,)) as logits_scope:
      logits = core_layers.dense(
          net,
          units=units,
          activation=None,
          kernel_initializer=init_ops.glorot_uniform_initializer(),
          name=logits_scope)
    _add_hidden_layer_summary(logits, logits_scope.name)

    return logits

 def testFunctionalDenseTwiceReuseFromScope(self):
   with self.test_session():
     with variable_scope.variable_scope('scope'):
       inputs = random_ops.random_uniform((5, 3), seed=1)
       core_layers.dense(inputs, 2, name='my_dense')
       vars1 = variables.trainable_variables()
     with variable_scope.variable_scope('scope', reuse=True):
       core_layers.dense(inputs, 2, name='my_dense')
       vars2 = variables.trainable_variables()
     self.assertEqual(vars1, vars2)

 def testKernelRegularizerWithReuse(self):
   regularizer = lambda x: math_ops.reduce_sum(x) * 1e-3
   inputs = random_ops.random_uniform((5, 3), seed=1)
   _ = core_layers.dense(
       inputs, 2, name='my_dense', kernel_regularizer=regularizer)
   self.assertEqual(
       len(ops.get_collection(ops.GraphKeys.REGULARIZATION_LOSSES)), 1)
   _ = core_layers.dense(
       inputs, 2, name='my_dense', kernel_regularizer=regularizer, reuse=True)
   self.assertEqual(
       len(ops.get_collection(ops.GraphKeys.REGULARIZATION_LOSSES)), 1)

 def testFunctionalDenseInitializerFromScope(self):
   with self.test_session() as sess:
     with variable_scope.variable_scope(
         'scope', initializer=init_ops.ones_initializer()):
       inputs = random_ops.random_uniform((5, 3), seed=1)
       core_layers.dense(inputs, 2)
       sess.run(variables.global_variables_initializer())
       weights = sess.run(variables.trainable_variables())
       self.assertEqual(len(weights), 2)
       # Check that the matrix weights got initialized to ones (from scope).
       self.assertAllClose(weights[0], np.ones((3, 2)))
       # Check that the bias still got initialized to zeros.
       self.assertAllClose(weights[1], np.zeros((2)))

 def testFunctionalDenseInitializerFromScope(self):
   with variable_scope.variable_scope(
       'scope', initializer=init_ops.ones_initializer()), self.test_session():
     inputs = random_ops.random_uniform((5, 3), seed=1)
     core_layers.dense(inputs, 2)
     variables.global_variables_initializer().run()
     weights = _get_variable_dict_from_varstore()
     self.assertEqual(len(weights), 2)
     # Check that the matrix weights got initialized to ones (from scope).
     self.assertAllClose(weights['scope/dense/kernel'].read_value().eval(),
                         np.ones((3, 2)))
     # Check that the bias still got initialized to zeros.
     self.assertAllClose(weights['scope/dense/bias'].read_value().eval(),
                         np.zeros((2)))

 def testFunctionalDenseInitializerFromScope(self):
   with variable_scope.variable_scope(
       'scope', initializer=init_ops.ones_initializer()):
     inputs = random_ops.random_uniform((5, 3), seed=1)
     core_layers.dense(inputs, 2)
     if context.in_graph_mode():
       self.evaluate(variables.global_variables_initializer())
     weights = variables.trainable_variables()
     self.assertEqual(len(weights), 2)
     # Check that the matrix weights got initialized to ones (from scope).
     self.assertAllClose(
         self.evaluate(weights[0].read_value()), np.ones((3, 2)))
     # Check that the bias still got initialized to zeros.
     self.assertAllClose(self.evaluate(weights[1].read_value()), np.zeros((2)))

 def _fn(x, output_units):
   """Fully connected MLP parameterized via `real_nvp_template`."""
   for units in hidden_layers:
     x = layers.dense(
         inputs=x, units=units, activation=activation, *args, **kwargs)
   x = layers.dense(
       inputs=x,
       units=(1 if shift_only else 2) * output_units,
       activation=None,
       *args,
       **kwargs)
   if shift_only:
     return x, None
   shift, log_scale = array_ops.split(x, 2, axis=-1)
   return shift, log_scale

 def fn(a, b, c):
   return core_layers.dense(
       a,
       10,
       use_bias=False,
       kernel_initializer=lambda shape, dtype, partition_info: w
   ) + math_ops.matmul(b, c)

 def testEagerExecution(self):
   with context.eager_mode():
     container = variable_scope.EagerVariableStore()
     x = constant_op.constant([[2.0]])
     with container.as_default():
       y = core_layers.dense(
           x, 1, name='my_dense',
           kernel_initializer=init_ops.ones_initializer())
     self.assertAllEqual(y, [[2.0]])
     self.assertEqual(len(container.variables()), 2)
     # Recreate the layer to test reuse.
     with container.as_default():
       core_layers.dense(
           x, 1, name='my_dense',
           kernel_initializer=init_ops.ones_initializer())
     self.assertEqual(len(container.variables()), 2)

 def testFunctionalDense(self):
   with self.test_session():
     inputs = random_ops.random_uniform((5, 3), seed=1)
     outputs = core_layers.dense(
         inputs, 2, activation=nn_ops.relu, name='my_dense')
     self.assertEqual(
         len(ops.get_collection(ops.GraphKeys.TRAINABLE_VARIABLES)), 2)
     self.assertEqual(outputs.op.name, 'my_dense/Relu')

 def testFunctionalDense(self):
   inputs = tf.random_uniform((5, 3), seed=1)
   outputs = core_layers.dense(
       inputs, 2, activation=tf.nn.relu, name='my_dense')
   self.assertEqual(
       len(tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES)), 2)
   self.assertEqual(outputs.op.name, 'my_dense/Relu')
   self.assertEqual(outputs.get_shape().as_list(), [5, 2])

    def F(x):
      out = core_layers.dense(x, 3, use_bias=False)

      def Grad(out_grad, variables=None):  # pylint: disable=redefined-outer-name
        self.assertEqual(1, len(variables))
        grads = gradients.gradients(out, [x, variables[0]], grad_ys=out_grad)
        return grads[0], [array_ops.ones((4, 3))]

      return out, Grad

 def layer_with_recompute(inputs, is_recomputing=False):
   kwarg_values.append(is_recomputing)
   out = core_layers.dense(inputs, 2)
   out = normalization_layers.batch_normalization(out, training=True)
   if is_recomputing:
     # Ensure that the updates are not duplicated by popping off the latest
     # 2 additions.
     update_ops = ops.get_collection_ref(ops.GraphKeys.UPDATE_OPS)
     update_ops.pop()
     update_ops.pop()
   return out

  def rnn_logit_fn(features, mode):
    """Recurrent Neural Network logit_fn.

    Args:
      features: This is the first item returned from the `input_fn`
                passed to `train`, `evaluate`, and `predict`. This should be a
                single `Tensor` or `dict` of same.
      mode: Optional. Specifies if this training, evaluation or prediction. See
            `ModeKeys`.

    Returns:
      A `Tensor` representing the logits.
    """
    with variable_scope.variable_scope(
        'sequence_input_layer',
        values=tuple(six.itervalues(features)),
        partitioner=input_layer_partitioner):
      sequence_input, sequence_length = seq_fc.sequence_input_layer(
          features=features, feature_columns=sequence_feature_columns)
      summary.histogram('sequence_length', sequence_length)

      if context_feature_columns:
        context_input = feature_column_lib.input_layer(
            features=features,
            feature_columns=context_feature_columns)
        sequence_input = seq_fc.concatenate_context_input(
            context_input, sequence_input)

    cell = rnn_cell_fn(mode)
    # Ignore output state.
    rnn_outputs, _ = rnn.dynamic_rnn(
        cell=cell,
        inputs=sequence_input,
        sequence_length=sequence_length,
        dtype=dtypes.float32,
        time_major=False)
    last_activations = _select_last_activations(rnn_outputs, sequence_length)

    with variable_scope.variable_scope('logits', values=(rnn_outputs,)):
      logits = core_layers.dense(
          last_activations,
          units=output_units,
          activation=None,
          kernel_initializer=init_ops.glorot_uniform_initializer())
    return logits

 def testFunctionalDenseInScope(self):
   with variable_scope.variable_scope('test'):
     inputs = random_ops.random_uniform((5, 3), seed=1)
     core_layers.dense(inputs, 2, name='my_dense')
     var = variables.trainable_variables()[0]
     self.assertEqual(var.name, 'test/my_dense/weights:0')
   with variable_scope.variable_scope('test1') as scope:
     inputs = random_ops.random_uniform((5, 3), seed=1)
     core_layers.dense(inputs, 2, name=scope)
     var = variables.trainable_variables()[2]
     self.assertEqual(var.name, 'test1/weights:0')
   with variable_scope.variable_scope('test2'):
     inputs = random_ops.random_uniform((5, 3), seed=1)
     core_layers.dense(inputs, 2)
     var = variables.trainable_variables()[4]
     self.assertEqual(var.name, 'test2/dense/weights:0')