def testNCELoss(self):
        # A simple test to verify the numerics.

        def _SigmoidCrossEntropyWithLogits(logits, targets):
            # logits, targets: float arrays of the same shape.
            assert logits.shape == targets.shape
            pred = 1.0 / (1.0 + np.exp(-logits))
            eps = 0.0001
            pred = np.minimum(np.maximum(pred, eps), 1 - eps)
            return -targets * np.log(pred) - (1.0 - targets) * np.log(1.0 - pred)

        weights, biases, hidden_acts, sharded_weights = self._GenerateTestInputs()
        labels = [0, 1, 2]
        true_w, true_b = weights[labels], biases[labels]
        sampled = [1, 0, 2, 3]
        num_sampled = len(sampled)
        true_exp = np.empty([self._batch_size, 1], dtype=np.float32)
        true_exp.fill(0.5)
        sampled_exp = np.empty([num_sampled], dtype=np.float32)
        sampled_exp.fill(0.5)
        sampled_w, sampled_b = weights[sampled], biases[sampled]
        test_sampled_vals = (sampled, true_exp, sampled_exp)

        with self.test_session():
            logits_np, labels_np = self._ComputeSampledLogitsNP(
                true_w, true_b, sampled_w, sampled_b, hidden_acts, true_expected=true_exp, sampled_expected=sampled_exp
            )
            nce_loss_np = np.sum(_SigmoidCrossEntropyWithLogits(logits_np, labels_np), 1)

            labels_tf = constant_op.constant(labels, shape=(self._batch_size, 1))
            weights_tf = constant_op.constant(weights)
            biases_tf = constant_op.constant(biases)
            inputs_tf = constant_op.constant(hidden_acts)

            nce_loss_tf = nn.nce_loss(
                weights_tf,
                biases_tf,
                inputs_tf,
                labels_tf,
                num_sampled=1,
                num_classes=self._num_classes,
                num_true=1,
                sampled_values=test_sampled_vals,
            )

            self.assertAllClose(nce_loss_np, nce_loss_tf.eval(), 1e-4)

            # Test with sharded weights
            nce_loss_tf = nn.nce_loss(
                [constant_op.constant(shard) for shard in sharded_weights],
                biases_tf,
                inputs_tf,
                labels_tf,
                num_sampled=1,
                num_classes=self._num_classes,
                num_true=1,
                sampled_values=test_sampled_vals,
            )

            self.assertAllClose(nce_loss_np, nce_loss_tf.eval(), 1e-4)

  def testRandomization(self):
    # Run 1x1 crop num_samples times in an image and ensure that one finds each
    # pixel 1/num_pixels of the time.
    num_samples = 1000
    height = 5
    width = 4

    num_pixels = height * width
    data = np.arange(num_pixels).reshape([height, width, 1])
    x_np = np.array(data).astype(np.int32)

    target_shape_np = np.array([1, 1], dtype=np.int64)

    y = []
    with self.test_session():
      x = constant_op.constant(x_np, shape=x_np.shape)
      target_shape = constant_op.constant(target_shape_np, shape=[2])
      y_tf = image_ops.random_crop(x, target_shape)
      for _ in xrange(num_samples):
        y_np = y_tf.eval()
        self.assertAllEqual(y_np.shape, [1, 1, 1])
        y.extend(y_np.flatten())

    # Calculate the mean and 4 * standard deviation.
    mean = [num_samples / num_pixels] * num_pixels
    four_stddev = 4.0 * np.sqrt(mean)

    # Ensure that each entry is observed in 1/num_pixels of the samples
    # within 4 standard deviations.
    counts = np.bincount(y)
    self.assertAllClose(counts, mean, atol=four_stddev)

def ones(shape, dtype=dtypes.float32, name=None):
  """Creates a tensor with all elements set to 1.

  This operation returns a tensor of type `dtype` with shape `shape` and all
  elements set to 1.

  For example:

  ```python
  tf.ones([2, 3], int32) ==> [[1, 1, 1], [1, 1, 1]]
  ```

  Args:
    shape: Either a list of integers, or a 1-D `Tensor` of type `int32`.
    dtype: The type of an element in the resulting `Tensor`.
    name: A name for the operation (optional).

  Returns:
    A `Tensor` with all elements set to 1.
  """
  with ops.op_scope([shape], name, "ones") as name:
    if isinstance(shape, list):
      output = constant(1, shape=shape, dtype=dtype, name=name)
    else:
      shape = ops.convert_to_tensor(shape, name="shape")
      output = fill(shape, constant(1, dtype=dtype), name=name)
  assert output.dtype.base_dtype == dtypes.as_dtype(dtype).base_dtype
  return output

 def testFetchIndexedSlicesWithoutDenseShape(self):
   with session.Session() as s:
     indices = np.array([[3, 2, 0], [4, 5, 1]]).astype(np.int64)
     values = np.array([1.0, 2.0]).astype(np.float32)
     dense_shape = None
     ind = ops.IndexedSlices(
         constant_op.constant(values), constant_op.constant(indices), None)
     # Single fetch, use as tuple
     ind_out = s.run(ind)
     values_out, indices_out, dense_shape_out = ind_out
     self.assertAllEqual(values_out, values)
     self.assertAllEqual(indices_out, indices)
     self.assertAllEqual(dense_shape_out, dense_shape)
     # Single fetch, use as IndexedSlicesValue
     ind_out = s.run(ind)
     self.assertAllEqual(ind_out.values, values)
     self.assertAllEqual(ind_out.indices, indices)
     self.assertAllEqual(ind_out.dense_shape, dense_shape)
     # Tuple fetch, use as tuple
     values_out, indices_out, dense_shape_out = s.run(ind)
     self.assertAllEqual(values_out, values)
     self.assertAllEqual(indices_out, indices)
     self.assertAllEqual(dense_shape_out, dense_shape)
     # List fetch, use as tuple
     (values_out, indices_out, dense_shape_out), = s.run([ind])
     self.assertAllEqual(values_out, values)
     self.assertAllEqual(indices_out, indices)
     self.assertAllEqual(dense_shape_out, dense_shape)
     # List fetch, use as IndexedSlicesValue
     ind_out, = s.run([ind])
     self.assertAllEqual(ind_out.values, values)
     self.assertAllEqual(ind_out.indices, indices)
     self.assertAllEqual(ind_out.dense_shape, dense_shape)

  def testDeConv2DSame(self):
    with self.test_session():
      strides = [1, 2, 2, 1]

      # Input, output: [batch, height, width, depth]
      x_shape = [2, 6, 4, 3]
      y_shape = [2, 12, 8, 2]

      # Filter: [kernel_height, kernel_width, output_depth, input_depth]
      f_shape = [3, 3, 2, 3]

      x = constant_op.constant(1.0, shape=x_shape, name="x",
                               dtype=types.float32)
      f = constant_op.constant(1.0, shape=f_shape, name="filter",
                               dtype=types.float32)
      output = nn.deconv2d(x, f, y_shape, strides=strides, padding="SAME")
      value = output.eval()

      for n in xrange(x_shape[0]):
        for k in xrange(f_shape[2]):
          for w in xrange(y_shape[2]):
            for h in xrange(y_shape[1]):
              target = 3.0
              # We add a case for locations divisible by the stride.
              h_in = h % strides[1] == 0 and h > 0 and h < y_shape[1] - 1
              w_in = w % strides[2] == 0 and w > 0 and w < y_shape[2] - 1
              if h_in and w_in:
                target += 9.0
              elif h_in or w_in:
                target += 3.0
              self.assertAllClose(target, value[n, h, w, k])

 def testFetchSparseTensor(self):
   with session.Session() as s:
     indices = np.array([[3, 2, 0], [4, 5, 1]]).astype(np.int64)
     values = np.array([1.0, 2.0]).astype(np.float32)
     shape = np.array([7, 9, 2]).astype(np.int64)
     sp = ops.SparseTensor(
         constant_op.constant(indices),
         constant_op.constant(values),
         constant_op.constant(shape))
     # Single fetch, use as tuple
     sp_out = s.run(sp)
     indices_out, values_out, shape_out = sp_out
     self.assertAllEqual(indices_out, indices)
     self.assertAllEqual(values_out, values)
     self.assertAllEqual(shape_out, shape)
     # Single fetch, use as SparseTensorValue
     sp_out = s.run(sp)
     self.assertAllEqual(sp_out.indices, indices)
     self.assertAllEqual(sp_out.values, values)
     self.assertAllEqual(sp_out.shape, shape)
     # Tuple fetch, use as tuple
     indices_out, values_out, shape_out = s.run(sp)
     self.assertAllEqual(indices_out, indices)
     self.assertAllEqual(values_out, values)
     self.assertAllEqual(shape_out, shape)
     # List fetch, use as tuple
     (indices_out, values_out, shape_out), = s.run([sp])
     self.assertAllEqual(indices_out, indices)
     self.assertAllEqual(values_out, values)
     self.assertAllEqual(shape_out, shape)
     # List fetch, use as SparseTensorValue
     sp_out, = s.run([sp])
     self.assertAllEqual(sp_out.indices, indices)
     self.assertAllEqual(sp_out.values, values)
     self.assertAllEqual(sp_out.shape, shape)

  def testNoOp(self):
    img_shape = [1, 6, 4, 1]
    single_shape = [6, 4, 1]
    # This test is also conducted with int8, so 127 is the maximum
    # value that can be used.
    data = [127, 127, 64, 64,
            127, 127, 64, 64,
            64, 64, 127, 127,
            64, 64, 127, 127,
            50, 50, 100, 100,
            50, 50, 100, 100]
    target_height = 6
    target_width = 4

    for nptype in self.TYPES:
      img_np = np.array(data, dtype=nptype).reshape(img_shape)

      for opt in self.OPTIONS:
        with self.test_session() as sess:
          image = constant_op.constant(img_np, shape=img_shape)
          y = image_ops.resize_images(image, target_height, target_width, opt)
          yshape = array_ops.shape(y)
          resized, newshape = sess.run([y, yshape])
          self.assertAllEqual(img_shape, newshape)
          self.assertAllClose(resized, img_np, atol=1e-5)

      # Resizing with a single image must leave the shape unchanged also.
      with self.test_session():
        img_single = img_np.reshape(single_shape)
        image = constant_op.constant(img_single, shape=single_shape)
        y = image_ops.resize_images(image, target_height, target_width,
                                    self.OPTIONS[0])
        yshape = array_ops.shape(y)
        newshape = yshape.eval()
        self.assertAllEqual(single_shape, newshape)

    def testDeConv2DSingleStride(self):
        with self.test_session():
            strides = [1, 1, 1, 1]

            # Input, output: [batch, height, width, depth]
            x_shape = [2, 6, 4, 3]
            y_shape = [2, 6, 4, 2]

            # Filter: [kernel_height, kernel_width, output_depth, input_depth]
            f_shape = [3, 3, 2, 3]

            x = constant_op.constant(1.0, shape=x_shape, name="x", dtype=dtypes.float32)
            f = constant_op.constant(1.0, shape=f_shape, name="filter", dtype=dtypes.float32)
            output = nn.deconv2d(x, f, y_shape, strides=strides, padding="SAME")
            value = output.eval()

            # We count the number of cells being added at the locations in the output.
            # At the center, #cells=kernel_height * kernel_width
            # At the corners, #cells=ceil(kernel_height/2) * ceil(kernel_width/2)
            # At the borders, #cells=ceil(kernel_height/2)*kernel_width or
            #                        kernel_height * ceil(kernel_width/2)

            for n in xrange(x_shape[0]):
                for k in xrange(f_shape[2]):
                    for w in xrange(y_shape[2]):
                        for h in xrange(y_shape[1]):
                            target = 4 * 3.0
                            h_in = h > 0 and h < y_shape[1] - 1
                            w_in = w > 0 and w < y_shape[2] - 1
                            if h_in and w_in:
                                target += 5 * 3.0
                            elif h_in or w_in:
                                target += 2 * 3.0
                            self.assertAllClose(target, value[n, h, w, k])

  def testFetchByNameDifferentStringTypes(self):
    with session.Session() as sess:
      c = constant_op.constant(42.0, name='c')
      d = constant_op.constant(43.0, name=u'd')
      e = constant_op.constant(44.0, name=b'e')
      f = constant_op.constant(45.0, name=r'f')

      self.assertTrue(isinstance(c.name, six.text_type))
      self.assertTrue(isinstance(d.name, six.text_type))
      self.assertTrue(isinstance(e.name, six.text_type))
      self.assertTrue(isinstance(f.name, six.text_type))

      self.assertEqual(42.0, sess.run('c:0'))
      self.assertEqual(42.0, sess.run(u'c:0'))
      self.assertEqual(42.0, sess.run(b'c:0'))
      self.assertEqual(42.0, sess.run(r'c:0'))

      self.assertEqual(43.0, sess.run('d:0'))
      self.assertEqual(43.0, sess.run(u'd:0'))
      self.assertEqual(43.0, sess.run(b'd:0'))
      self.assertEqual(43.0, sess.run(r'd:0'))

      self.assertEqual(44.0, sess.run('e:0'))
      self.assertEqual(44.0, sess.run(u'e:0'))
      self.assertEqual(44.0, sess.run(b'e:0'))
      self.assertEqual(44.0, sess.run(r'e:0'))

      self.assertEqual(45.0, sess.run('f:0'))
      self.assertEqual(45.0, sess.run(u'f:0'))
      self.assertEqual(45.0, sess.run(b'f:0'))
      self.assertEqual(45.0, sess.run(r'f:0'))

 def testColocationIgnoreStack(self):
   a = constant_op.constant([2.0], name="a")
   b = constant_op.constant(3.0, name="b")
   with ops.colocate_with(a.op):
     with ops.colocate_with(b.op, ignore_existing=True):
       c = constant_op.constant(4.0)
   self.assertEqual(set([b"loc:@b"]), set(c.op.colocation_groups()))

 def testCircularConvolution(self):
     v = constant_op.constant([1,2,3,4,5,6,7], dtype=tf.float32)
     k = constant_op.constant([0,0,1], dtype=tf.float32)
     for use_gpu in [True, False]:
         with self.test_session(use_gpu=use_gpu):
             cir_conv = circular_convolution(v, k).eval()
             self.assertAllEqual(cir_conv, [7,1,2,3,4,5,6])

 def testNegation(self):
   with self.test_session():
     values = constant_op.constant([2, 3, 5, 7], shape=[2, 2])
     indices = constant_op.constant([0, 2])
     x = -ops.IndexedSlices(values, indices)
     self.assertAllEqual(x.values.eval(), [[-2, -3], [-5, -7]])
     self.assertAllEqual(x.indices.eval(), [0, 2])

    def testFetchByNameDifferentStringTypes(self):
        with session.Session() as sess:
            c = constant_op.constant(42.0, name="c")
            d = constant_op.constant(43.0, name=u"d")
            e = constant_op.constant(44.0, name=b"e")
            f = constant_op.constant(45.0, name=r"f")

            self.assertTrue(isinstance(c.name, six.text_type))
            self.assertTrue(isinstance(d.name, six.text_type))
            self.assertTrue(isinstance(e.name, six.text_type))
            self.assertTrue(isinstance(f.name, six.text_type))

            self.assertEqual(42.0, sess.run("c:0"))
            self.assertEqual(42.0, sess.run(u"c:0"))
            self.assertEqual(42.0, sess.run(b"c:0"))
            self.assertEqual(42.0, sess.run(r"c:0"))

            self.assertEqual(43.0, sess.run("d:0"))
            self.assertEqual(43.0, sess.run(u"d:0"))
            self.assertEqual(43.0, sess.run(b"d:0"))
            self.assertEqual(43.0, sess.run(r"d:0"))

            self.assertEqual(44.0, sess.run("e:0"))
            self.assertEqual(44.0, sess.run(u"e:0"))
            self.assertEqual(44.0, sess.run(b"e:0"))
            self.assertEqual(44.0, sess.run(r"e:0"))

            self.assertEqual(45.0, sess.run("f:0"))
            self.assertEqual(45.0, sess.run(u"f:0"))
            self.assertEqual(45.0, sess.run(b"f:0"))
            self.assertEqual(45.0, sess.run(r"f:0"))

  def _testTypesForAdam(self, var, m, v, grad, use_gpu):
    self.setUp()
    with self.test_session(use_gpu=use_gpu):
      var_t = variables.Variable(var)
      m_t = variables.Variable(m)
      v_t = variables.Variable(v)

      t = 1
      beta1 = np.array(0.9, dtype=var.dtype)
      beta2 = np.array(0.999, dtype=var.dtype)
      beta1_power = beta1**t
      beta2_power = beta2**t
      lr = np.array(0.001, dtype=var.dtype)
      epsilon = np.array(1e-8, dtype=var.dtype)
      beta1_t = constant_op.constant(beta1, self._toType(var.dtype), [])
      beta2_t = constant_op.constant(beta2, self._toType(var.dtype), [])
      beta1_power_t = variables.Variable(beta1_power)
      beta2_power_t = variables.Variable(beta2_power)
      lr_t = constant_op.constant(lr, self._toType(var.dtype), [])
      epsilon_t = constant_op.constant(epsilon, self._toType(var.dtype), [])
      variables.initialize_all_variables().run()

      self.assertAllCloseAccordingToType(var, var_t.eval())
      new_var, _, _ = self._adamUpdateNumpy(var, grad, t, m, v,
                                            lr, beta1, beta2, epsilon)
      apply_adam = training_ops.apply_adam(var_t, m_t, v_t, beta1_power_t,
                                           beta2_power_t, lr_t,
                                           beta1_t, beta2_t, epsilon_t, grad)
      out = apply_adam.eval()
      self.assertShapeEqual(out, apply_adam)
      self.assertAllCloseAccordingToType(new_var, out)

 def testScalarMul(self):
   with self.test_session():
     values = constant_op.constant([2, 3, 5, 7], shape=[2, 2])
     indices = constant_op.constant([0, 2])
     x = math_ops.scalar_mul(-2, ops.IndexedSlices(values, indices))
     self.assertAllEqual(x.values.eval(), [[-4, -6], [-10, -14]])
     self.assertAllEqual(x.indices.eval(), [0, 2])

 def _testBatchNormGradient(self, param_index, tag, scale_after_normalization,
                            err_tolerance=1e-11):
   x_shape = [3, 5, 4, 5]
   param_shape = [5]
   np.random.seed(1)  # Make it reproducible.
   x_val = np.random.random_sample(x_shape).astype(np.float64)
   m_val = np.random.random_sample(param_shape).astype(np.float64)
   v_val = np.random.random_sample(param_shape).astype(np.float64)
   beta_val = np.random.random_sample(param_shape).astype(np.float64)
   gamma_val = np.random.random_sample(param_shape).astype(np.float64)
   with self.test_session():
     x = constant_op.constant(x_val, name="x")
     m = constant_op.constant(m_val, name="m")
     v = constant_op.constant(v_val, name="v")
     beta = constant_op.constant(beta_val, name="beta")
     gamma = constant_op.constant(gamma_val, name="gamma")
     epsilon = 0.001
     # If scale_after_normalization is False, backprop for gamma
     # will be 0. gamma is unchanged.
     output = nn.batch_norm_with_global_normalization(
         x, m, v, beta, gamma, epsilon, scale_after_normalization)
     all_params = [x, m, v, beta, gamma]
     all_shapes = [x_shape, param_shape, param_shape, param_shape, param_shape]
     err = gc.ComputeGradientError(all_params[param_index],
                                   all_shapes[param_index], output, x_shape)
   print "Batch normalization %s gradient %s scale err = " % (
       tag, "with" if scale_after_normalization else "without"
   ), err
   self.assertLess(err, err_tolerance)

 def testBatchNorm(self):
   x_shape = [3, 5, 4, 2]
   param_shape = [2]
   x_val = np.random.random_sample(x_shape).astype(np.float32)
   m_val = np.random.random_sample(param_shape).astype(np.float32)
   v_val = np.random.random_sample(param_shape).astype(np.float32)
   beta_val = np.random.random_sample(param_shape).astype(np.float32)
   gamma_val = np.random.random_sample(param_shape).astype(np.float32)
   for use_gpu in [True, False]:
     with self.test_session(use_gpu=use_gpu) as sess:
       x = constant_op.constant(x_val, name="x")
       m = constant_op.constant(m_val, name="m")
       v = constant_op.constant(v_val, name="v")
       beta = constant_op.constant(beta_val, name="beta")
       gamma = constant_op.constant(gamma_val, name="gamma")
       epsilon = 0.001
       for scale_after_normalization in [True, False]:
         bn = nn.batch_norm_with_global_normalization(
             x, m, v, beta, gamma, epsilon, scale_after_normalization)
         on = self._opsBatchNorm(
             x, m, v, beta, gamma, epsilon, scale_after_normalization)
         np_batch_norm = self._npBatchNorm(
             x_val, m_val, v_val, beta_val, gamma_val, epsilon,
             scale_after_normalization)
         tf_batch_norm, ops_batch_norm = sess.run([bn, on])
         self.assertAllClose(np_batch_norm, tf_batch_norm, atol=0.000001)
         self.assertAllClose(np_batch_norm, ops_batch_norm, atol=0.000001)
         self.assertAllClose(tf_batch_norm, ops_batch_norm, atol=0.000001)

  def _testDefaultGraphInThread(self, constructed_event, continue_event, i):
    with session.Session() as s:
      self.assertEqual(ops.get_default_graph(), s.graph)
      a = constant_op.constant(1.0, shape=[1, 2])
      b = constant_op.constant(2.0, shape=[2, 3])
      c = math_ops.matmul(a, b)
      v = variables.Variable(c, name='var_%d' % i)

      # Block here until all threads have constructed their graph.
      constructed_event.set()
      continue_event.wait()

      assign_c_to_v = state_ops.assign(v, c)
      v.initializer.run()
      assign_c_to_v.eval()
      v_val = v.eval()
      self.assertAllEqual([[4.0, 4.0, 4.0]], v_val)
      d = constant_op.constant(3.0, shape=[2, 3])
      e = math_ops.matmul(a, d)
      assign_e_to_v = state_ops.assign(v, e)
      e_val = e.eval()
      self.assertAllEqual([[6.0, 6.0, 6.0]], e_val)
      v_val = v.eval()
      self.assertAllEqual([[4.0, 4.0, 4.0]], v_val)
      s.run(assign_e_to_v)
      v_val = v.eval()
      self.assertAllEqual([[6.0, 6.0, 6.0]], v_val)
      self.assertEqual(ops.get_default_graph(), s.graph)

  def testWarnings(self):
    # Smaller than the threshold: no warning.
    c_sparse = ops.IndexedSlices(array_ops.placeholder(dtypes.float32),
                                 array_ops.placeholder(dtypes.int32),
                                 constant([4, 4, 4, 4]))
    with warnings.catch_warnings(record=True) as w:
      math_ops.mul(c_sparse, 1.0)
    self.assertEqual(0, len(w))

    # Greater than or equal to the threshold: warning.
    c_sparse = ops.IndexedSlices(array_ops.placeholder(dtypes.float32),
                                 array_ops.placeholder(dtypes.int32),
                                 constant([100, 100, 100, 100]))
    with warnings.catch_warnings(record=True) as w:
      math_ops.mul(c_sparse, 1.0)
    self.assertEqual(1, len(w))
    self.assertTrue(
        "with 100000000 elements. This may consume a large amount of memory."
        in str(w[0].message))

    # Unknown dense shape: warning.
    c_sparse = ops.IndexedSlices(array_ops.placeholder(dtypes.float32),
                                 array_ops.placeholder(dtypes.int32),
                                 array_ops.placeholder(dtypes.int32))
    with warnings.catch_warnings(record=True) as w:
      math_ops.mul(c_sparse, 1.0)
    self.assertEqual(1, len(w))
    self.assertTrue(
        "of unknown shape. This may consume a large amount of memory."
        in str(w[0].message))

 def testMultiColocationGroups(self):
   a = constant_op.constant([2.0], name="a")
   b = constant_op.constant(3.0, name="b")
   with ops.colocate_with(a.op):
     with ops.colocate_with(b.op):
       c = constant_op.constant(4.0)
   self.assertEqual(set([b"loc:@a", b"loc:@b"]), set(c.op.colocation_groups()))