def testGradient(self):
    if not test.is_gpu_available(cuda_only=True):
      self.skipTest('GPU required')

    random_seed.set_random_seed(0)
    x = random_ops.truncated_normal([1, 200, 200, 3], seed=0)
    y = conv_layers.conv2d(x, 32, [3, 3])
    z = conv_layers.conv2d(y, 32, [3, 3])
    optimizer = gradient_descent.GradientDescentOptimizer(1e-4)
    loss = math_ops.reduce_mean(z)
    train_op = optimizer.minimize(loss)
    graph = ops.get_default_graph()
    graph.add_to_collection('train_op', train_op)
    meta_graph = saver_lib.export_meta_graph(graph_def=graph.as_graph_def())

    rewrite_options = rewriter_config_pb2.RewriterConfig(
        optimize_tensor_layout=True)
    optimized_graph = tf_optimizer.OptimizeGraph(rewrite_options, meta_graph)

    found = 0
    for node in optimized_graph.node:
      if node.op in ['Conv2D', 'Conv2DBackpropFilter', 'Conv2DBackpropInput']:
        found += 1
        self.assertEqual(node.attr['data_format'].s, 'NCHW')
    self.assertEqual(found, 5)

 def testFunctionalConv2DNoReuse(self):
   height, width = 7, 9
   images = random_ops.random_uniform((5, height, width, 3), seed=1)
   conv_layers.conv2d(images, 32, [3, 3])
   self.assertEqual(len(variables.trainable_variables()), 2)
   conv_layers.conv2d(images, 32, [3, 3])
   self.assertEqual(len(variables.trainable_variables()), 4)

  def _train(self, checkpoint_path, layout_optimizer=False, restore=False):
    ops.reset_default_graph()
    graph = ops.get_default_graph()
    with session.Session(
        config=get_config(layout_optimizer), graph=graph) as sess:
      batch = 2
      height = 6
      width = 7
      input_channels = 3
      shape = [batch, height, width, input_channels]
      image = array_ops.placeholder(dtype='float32', shape=shape)
      conv1 = conv_layers.conv2d(image, 32, [3, 3])
      conv2 = conv_layers.conv2d(conv1, 32, [3, 3])
      optimizer = gradient_descent.GradientDescentOptimizer(0.01)
      loss = math_ops.reduce_mean(conv2)
      train_op = optimizer.minimize(loss)
      saver = saver_lib.Saver(write_version=saver_pb2.SaverDef.V2)

      if restore:
        saver.restore(sess, checkpoint_path)
      else:
        sess.run(variables.global_variables_initializer())

      np.random.seed(0)
      for _ in range(2):
        image_val = np.random.rand(*shape).astype(np.float32)
        sess.run([loss, train_op], feed_dict={image: image_val})

      if restore:
        all_vars = ops.get_collection(ops.GraphKeys.GLOBAL_VARIABLES)
        all_vars_values = [var.eval(session=sess) for var in all_vars]
        return all_vars_values
      else:
        saver.save(sess, checkpoint_path)

  def testInvalidKernelSize(self):
    height, width = 7, 9
    images = random_ops.random_uniform((5, height, width, 3), seed=1)
    with self.assertRaisesRegexp(ValueError, 'kernel_size'):
      conv_layers.conv2d(images, 32, (1, 2, 3))

    with self.assertRaisesRegexp(ValueError, 'kernel_size'):
      conv_layers.conv2d(images, 32, None)

  def testInvalidStrides(self):
    height, width = 7, 9
    images = random_ops.random_uniform((5, height, width, 3), seed=1)
    with self.assertRaisesRegexp(ValueError, 'strides'):
      conv_layers.conv2d(images, 32, 3, strides=(1, 2, 3))

    with self.assertRaisesRegexp(ValueError, 'strides'):
      conv_layers.conv2d(images, 32, 3, strides=None)

 def testFunctionalConv2DReuseFromScope(self):
   with variable_scope.variable_scope('scope'):
     height, width = 7, 9
     images = random_ops.random_uniform((5, height, width, 3), seed=1)
     conv_layers.conv2d(images, 32, [3, 3], name='conv1')
     self.assertEqual(len(variables.trainable_variables()), 2)
   with variable_scope.variable_scope('scope', reuse=True):
     conv_layers.conv2d(images, 32, [3, 3], name='conv1')
     self.assertEqual(len(variables.trainable_variables()), 2)

 def testFunctionalConv2DNoReuse(self):
   height, width = 7, 9
   images = random_ops.random_uniform((5, height, width, 3), seed=1)
   conv_layers.conv2d(images, 32, [3, 3])
   self.assertEqual(
       len(ops.get_collection(ops.GraphKeys.TRAINABLE_VARIABLES)), 2)
   conv_layers.conv2d(images, 32, [3, 3])
   self.assertEqual(
       len(ops.get_collection(ops.GraphKeys.TRAINABLE_VARIABLES)), 4)

 def _conv2d_impl(self, input_layer, num_channels_in, filters, kernel_size,
                  strides, padding, kernel_initializer):
     if self.use_tf_layers:
         return conv_layers.conv2d(
             input_layer,
             filters,
             kernel_size,
             strides,
             padding,
             self.channel_pos,
             kernel_initializer=kernel_initializer,
             use_bias=False)
     else:
         weights_shape = [
             kernel_size[0], kernel_size[1], num_channels_in, filters
         ]
         weights = self.get_variable(
             'conv2d/kernel',
             weights_shape,
             self.variable_dtype,
             self.dtype,
             initializer=kernel_initializer)
         if self.data_format == 'NHWC':
             strides = [1] + strides + [1]
         else:
             strides = [1, 1] + strides
         return tf.nn.conv2d(
             input_layer,
             weights,
             strides,
             padding,
             data_format=self.data_format)

 def testFunctionalConv2DInitializerFromScope(self):
   with self.test_session() as sess:
     with variable_scope.variable_scope(
         'scope', initializer=init_ops.ones_initializer()):
       height, width = 7, 9
       images = random_ops.random_uniform((5, height, width, 3), seed=1)
       conv_layers.conv2d(images, 32, [3, 3], name='conv1')
       weights = variables.trainable_variables()
       # Check the names of weights in order.
       self.assertTrue('kernel' in weights[0].name)
       self.assertTrue('bias' in weights[1].name)
       sess.run(variables.global_variables_initializer())
       weights = sess.run(weights)
       # Check that the kernel weights got initialized to ones (from scope)
       self.assertAllClose(weights[0], np.ones((3, 3, 3, 32)))
       # Check that the bias still got initialized to zeros.
       self.assertAllClose(weights[1], np.zeros((32)))

 def training_step(inputs, scale):
   outputs = convolutional.conv2d(
       inputs,
       filters=16,
       kernel_size=(3, 3),
       data_format="channels_first",
       kernel_regularizer=make_regularizer(scale))
   loss = math_ops.reduce_mean(math_ops.square(outputs))
   return loss.op

 def _conv2d_impl(self, input_layer, num_channels_in, filters, kernel_size,
                  strides, padding, kernel_initializer):
   if self.use_tf_layers:
     return conv_layers.conv2d(input_layer, filters, kernel_size, strides,
                               padding, self.channel_pos,
                               kernel_initializer=kernel_initializer,
                               use_bias=False)
   else:
     weights_shape = [kernel_size[0], kernel_size[1], num_channels_in, filters]
     # We use the name 'conv2d/kernel' so the variable has the same name as its
     # tf.layers equivalent. This way, if a checkpoint is written when
     # self.use_tf_layers == True, it can be loaded when
     # self.use_tf_layers == False, and vice versa.
     weights = self.get_variable('conv2d/kernel', weights_shape,
                                 self.variable_dtype, self.dtype,
                                 initializer=kernel_initializer)
     if self.data_format == 'NHWC':
       strides = [1] + strides + [1]
     else:
       strides = [1, 1] + strides
     return tf.nn.conv2d(input_layer, weights, strides, padding,
                         data_format=self.data_format)

 def _simple_model(self, image, fused, freeze_mode):
   output_channels, kernel_size = 2, 3
   conv = conv_layers.conv2d(
       image,
       output_channels,
       kernel_size,
       use_bias=False,
       kernel_initializer=init_ops.ones_initializer())
   bn_layer = normalization_layers.BatchNormalization(fused=fused)
   bn_layer._bessels_correction_test_only = False
   training = not freeze_mode
   bn = bn_layer.apply(conv, training=training)
   loss = math_ops.reduce_sum(math_ops.abs(bn))
   optimizer = gradient_descent.GradientDescentOptimizer(0.01)
   if not freeze_mode:
     update_ops = ops.get_collection(ops.GraphKeys.UPDATE_OPS)
     with ops.control_dependencies(update_ops):
       train_op = optimizer.minimize(loss)
   else:
     train_op = optimizer.minimize(loss)
   saver = saver_lib.Saver(write_version=saver_pb2.SaverDef.V2)
   return loss, train_op, saver

 def testConv2DFloat16(self):
   height, width = 7, 9
   images = random_ops.random_uniform((5, height, width, 4), dtype='float16')
   output = conv_layers.conv2d(images, 32, [3, 3], activation=nn_ops.relu)
   self.assertListEqual(output.get_shape().as_list(),
                        [5, height - 2, width - 2, 32])

 def testInvalidDataFormat(self):
   height, width = 7, 9
   images = random_ops.random_uniform((5, height, width, 3), seed=1)
   with self.assertRaisesRegexp(ValueError, 'data_format'):
     conv_layers.conv2d(images, 32, 3, data_format='invalid')

 def test_invalid_shape(self):
   inputs = random_ops.random_uniform((10, 100, 100, 3), seed=1)
   graph = conv_layers.conv2d(inputs, 3, 10, strides=(1, 1))
   with self.assertRaisesRegexp(ValueError, 'number of features'):
     graph = maxout.maxout(graph, num_units=2)