def testContinueTraining(self):
    boston = base.load_boston()
    output_dir = tempfile.mkdtemp()
    est = estimator.SKCompat(
        estimator.Estimator(
            model_fn=linear_model_fn, model_dir=output_dir))
    float64_labels = boston.target.astype(np.float64)
    est.fit(x=boston.data, y=float64_labels, steps=50)
    scores = est.score(
        x=boston.data,
        y=float64_labels,
        metrics={'MSE': metric_ops.streaming_mean_squared_error})
    del est
    # Create another estimator object with the same output dir.
    est2 = estimator.SKCompat(
        estimator.Estimator(
            model_fn=linear_model_fn, model_dir=output_dir))

    # Check we can evaluate and predict.
    scores2 = est2.score(
        x=boston.data,
        y=float64_labels,
        metrics={'MSE': metric_ops.streaming_mean_squared_error})
    self.assertAllClose(scores['MSE'], scores2['MSE'])
    predictions = np.array(list(est2.predict(x=boston.data)))
    other_score = _sklearn.mean_squared_error(predictions, float64_labels)
    self.assertAllClose(scores['MSE'], other_score)

    # Check we can keep training.
    est2.fit(x=boston.data, y=float64_labels, steps=100)
    scores3 = est2.score(
        x=boston.data,
        y=float64_labels,
        metrics={'MSE': metric_ops.streaming_mean_squared_error})
    self.assertLess(scores3['MSE'], scores['MSE'])

  def testContinueTrainingDictionaryInput(self):
    boston = tf.contrib.learn.datasets.load_boston()
    output_dir = tempfile.mkdtemp()
    est = tf.contrib.learn.Estimator(model_fn=linear_model_fn,
                                     model_dir=output_dir)
    boston_input = {'input': boston.data}
    float64_target = {'labels': boston.target.astype(np.float64)}
    est.fit(x=boston_input, y=float64_target, steps=50)
    scores = est.evaluate(
      x=boston_input,
      y=float64_target,
      metrics={'MSE': tf.contrib.metrics.streaming_mean_squared_error})
    del est
    # Create another estimator object with the same output dir.
    est2 = tf.contrib.learn.Estimator(model_fn=linear_model_fn,
                                      model_dir=output_dir)

    # Check we can evaluate and predict.
    scores2 = est2.evaluate(
      x=boston_input,
      y=float64_target,
      metrics={'MSE': tf.contrib.metrics.streaming_mean_squared_error})
    self.assertAllClose(scores2['MSE'],
                        scores['MSE'])
    predictions = np.array(list(est2.predict(x=boston_input)))
    other_score = _sklearn.mean_squared_error(predictions, float64_target['labels'])
    self.assertAllClose(other_score, scores['MSE'])

  def testContinueTraining(self):
    boston = tf.contrib.learn.datasets.load_boston()
    output_dir = tempfile.mkdtemp()
    est = tf.contrib.learn.Estimator(model_fn=linear_model_fn,
                                     model_dir=output_dir)
    float64_target = boston.target.astype(np.float64)
    est.fit(x=boston.data, y=float64_target, steps=50)
    scores = est.evaluate(
        x=boston.data,
        y=float64_target,
        metrics={'MSE': tf.contrib.metrics.streaming_mean_squared_error})
    del est
    # Create another estimator object with the same output dir.
    est2 = tf.contrib.learn.Estimator(model_fn=linear_model_fn,
                                      model_dir=output_dir)

    # Check we can evaluate and predict.
    scores2 = est2.evaluate(
        x=boston.data,
        y=float64_target,
        metrics={'MSE': tf.contrib.metrics.streaming_mean_squared_error})
    self.assertAllClose(scores2['MSE'],
                        scores['MSE'])
    predictions = est2.predict(x=boston.data)
    other_score = _sklearn.mean_squared_error(predictions, float64_target)
    self.assertAllClose(other_score, scores['MSE'])

    # Check we can keep training.
    est2.fit(x=boston.data, y=float64_target, steps=100)
    scores3 = est2.evaluate(
        x=boston.data,
        y=float64_target,
        metrics={'MSE': tf.contrib.metrics.streaming_mean_squared_error})
    self.assertLess(scores3['MSE'], scores['MSE'])

  def testCustomMetrics(self):
    """Tests custom evaluation metrics."""

    def _input_fn(num_epochs=None):
      # Create 4 rows, one of them (y = x), three of them (y=Not(x))
      labels = constant_op.constant([[1.], [0.], [0.], [0.]])
      features = {
          'x':
              input_lib.limit_epochs(
                  array_ops.ones(
                      shape=[4, 1], dtype=dtypes.float32),
                  num_epochs=num_epochs),
      }
      return features, labels

    def _my_metric_op(predictions, labels):
      return math_ops.reduce_sum(math_ops.multiply(predictions, labels))

    regressor = dnn.DNNRegressor(
        feature_columns=[feature_column.real_valued_column('x')],
        hidden_units=[3, 3],
        config=run_config.RunConfig(tf_random_seed=1))

    regressor.fit(input_fn=_input_fn, steps=5)
    scores = regressor.evaluate(
        input_fn=_input_fn,
        steps=1,
        metrics={
            'my_error': metric_ops.streaming_mean_squared_error,
            ('my_metric', 'scores'): _my_metric_op
        })
    self.assertIn('loss', set(scores.keys()))
    self.assertIn('my_error', set(scores.keys()))
    self.assertIn('my_metric', set(scores.keys()))
    predict_input_fn = functools.partial(_input_fn, num_epochs=1)
    predictions = np.array(list(regressor.predict(input_fn=predict_input_fn)))
    self.assertAlmostEqual(
        _sklearn.mean_squared_error(np.array([1, 0, 0, 0]), predictions),
        scores['my_error'])

    # Tests the case that the 2nd element of the key is not "scores".
    with self.assertRaises(KeyError):
      regressor.evaluate(
          input_fn=_input_fn,
          steps=1,
          metrics={
              ('my_error', 'predictions'):
                  metric_ops.streaming_mean_squared_error
          })

    # Tests the case where the tuple of the key doesn't have 2 elements.
    with self.assertRaises(ValueError):
      regressor.evaluate(
          input_fn=_input_fn,
          steps=1,
          metrics={
              ('bad_length_name', 'scores', 'bad_length'):
                  metric_ops.streaming_mean_squared_error
          })

 def testOneDim(self):
   random.seed(42)
   x = np.random.rand(1000)
   y = 2 * x + 3
   regressor = learn.TensorFlowLinearRegressor()
   regressor.fit(x, y)
   score = mean_squared_error(y, regressor.predict(x))
   self.assertLess(score, 1.0, "Failed with score = {0}".format(score))

 def testBoston(self):
   random.seed(42)
   boston = datasets.load_boston()
   regressor = learn.LinearRegressor(
       feature_columns=learn.infer_real_valued_columns_from_input(boston.data))
   regressor.fit(boston.data, boston.target, max_steps=500)
   score = mean_squared_error(boston.target, regressor.predict(boston.data))
   self.assertLess(score, 150, "Failed with score = {0}".format(score))

 def testBoston(self):
   random.seed(42)
   boston = datasets.load_boston()
   regressor = learn.TensorFlowLinearRegressor(batch_size=boston.data.shape[0],
                                               steps=500,
                                               learning_rate=0.001)
   regressor.fit(boston.data, boston.target)
   score = mean_squared_error(boston.target, regressor.predict(boston.data))
   self.assertLess(score, 150, "Failed with score = {0}".format(score))

 def testOneDim(self):
   random.seed(42)
   x = np.random.rand(1000)
   y = 2 * x + 3
   feature_columns = learn.infer_real_valued_columns_from_input(x)
   regressor = learn.TensorFlowLinearRegressor(feature_columns=feature_columns)
   regressor.fit(x, y)
   score = mean_squared_error(y, regressor.predict(x))
   self.assertLess(score, 1.0, "Failed with score = {0}".format(score))

 def testMultiRegression(self):
   random.seed(42)
   rng = np.random.RandomState(1)
   x = np.sort(200 * rng.rand(100, 1) - 100, axis=0)
   y = np.array([np.pi * np.sin(x).ravel(), np.pi * np.cos(x).ravel()]).T
   regressor = learn.TensorFlowLinearRegressor(learning_rate=0.01)
   regressor.fit(x, y)
   score = mean_squared_error(regressor.predict(x), y)
   self.assertLess(score, 10, "Failed with score = {0}".format(score))

 def testBostonAll(self):
   boston = tf.contrib.learn.datasets.load_boston()
   est = tf.contrib.learn.Estimator(model_fn=linear_model_fn,
                                    classification=False)
   est.fit(x=boston.data, y=boston.target.astype(np.float32), steps=100)
   scores = est.evaluate(
       x=boston.data,
       y=boston.target.astype(np.float32))
   predictions = est.predict(x=boston.data)
   other_score = mean_squared_error(predictions, boston.target)
   self.assertAllClose(other_score, scores['mean_squared_error'])

 def testBostonAll(self):
   boston = tf.contrib.learn.datasets.load_boston()
   est = tf.contrib.learn.Estimator(model_fn=linear_model_fn)
   est.fit(x=boston.data, y=boston.target.astype(np.float32), steps=100)
   scores = est.evaluate(
       x=boston.data,
       y=boston.target.astype(np.float32),
       metrics={'MSE': tf.contrib.metrics.streaming_mean_squared_error})
   predictions = est.predict(x=boston.data)
   other_score = _sklearn.mean_squared_error(predictions, boston.target)
   self.assertAllClose(other_score, scores['MSE'])

 def testMultiRegression(self):
   random.seed(42)
   rng = np.random.RandomState(1)
   x = np.sort(200 * rng.rand(100, 1) - 100, axis=0)
   y = np.array([np.pi * np.sin(x).ravel(), np.pi * np.cos(x).ravel()]).T
   regressor = learn.LinearRegressor(
       feature_columns=learn.infer_real_valued_columns_from_input(x),
       target_dimension=2)
   regressor.fit(x, y, steps=100)
   score = mean_squared_error(regressor.predict(x), y)
   self.assertLess(score, 10, "Failed with score = {0}".format(score))

  def testCustomMetrics(self):
    """Tests custom evaluation metrics."""

    def _input_fn(num_epochs=None):
      # Create 4 rows, one of them (y = x), three of them (y=Not(x))
      labels = constant_op.constant([[1.], [0.], [0.], [0.]])
      features = {
          'x':
              input_lib.limit_epochs(
                  array_ops.ones(shape=[4, 1], dtype=dtypes.float32),
                  num_epochs=num_epochs),
      }
      return features, labels

    def _my_metric_op(predictions, labels):
      return math_ops.reduce_sum(math_ops.multiply(predictions, labels))

    regressor = debug.DebugRegressor(
        config=run_config.RunConfig(tf_random_seed=1))

    regressor.fit(input_fn=_input_fn, steps=5)
    scores = regressor.evaluate(
        input_fn=_input_fn,
        steps=1,
        metrics={
            'my_error':
                MetricSpec(
                    metric_fn=metric_ops.streaming_mean_squared_error,
                    prediction_key='scores'),
            'my_metric':
                MetricSpec(metric_fn=_my_metric_op, prediction_key='scores')
        })
    self.assertIn('loss', set(scores.keys()))
    self.assertIn('my_error', set(scores.keys()))
    self.assertIn('my_metric', set(scores.keys()))
    predict_input_fn = functools.partial(_input_fn, num_epochs=1)
    predictions = np.array(
        list(regressor.predict_scores(input_fn=predict_input_fn)))
    self.assertAlmostEqual(
        _sklearn.mean_squared_error(np.array([1, 0, 0, 0]), predictions),
        scores['my_error'])

    # Tests the case where the prediction_key is not "scores".
    with self.assertRaisesRegexp(KeyError, 'bad_type'):
      regressor.evaluate(
          input_fn=_input_fn,
          steps=1,
          metrics={
              'bad_name':
                  MetricSpec(
                      metric_fn=metric_ops.streaming_auc,
                      prediction_key='bad_type')
          })

 def testBostonAll(self):
     boston = tf.contrib.learn.datasets.load_boston()
     est = tf.contrib.learn.Estimator(model_fn=linear_model_fn)
     float64_target = boston.target.astype(np.float64)
     est.fit(x=boston.data, y=float64_target, steps=100)
     scores = est.evaluate(
         x=boston.data, y=float64_target, metrics={"MSE": tf.contrib.metrics.streaming_mean_squared_error}
     )
     predictions = est.predict(x=boston.data)
     other_score = _sklearn.mean_squared_error(predictions, boston.target)
     self.assertAllClose(other_score, scores["MSE"])
     self.assertTrue("global_step" in scores)
     self.assertEqual(scores["global_step"], 100)

 def testBostonAll(self):
     boston = tf.contrib.learn.datasets.load_boston()
     est = tf.contrib.learn.SKCompat(tf.contrib.learn.Estimator(model_fn=linear_model_fn))
     float64_labels = boston.target.astype(np.float64)
     est.fit(x=boston.data, y=float64_labels, steps=100)
     scores = est.score(
         x=boston.data, y=float64_labels, metrics={"MSE": tf.contrib.metrics.streaming_mean_squared_error}
     )
     predictions = np.array(list(est.predict(x=boston.data)))
     other_score = _sklearn.mean_squared_error(predictions, boston.target)
     self.assertAllClose(scores["MSE"], other_score)
     self.assertTrue("global_step" in scores)
     self.assertEqual(100, scores["global_step"])

 def testBostonAll(self):
   boston = base.load_boston()
   est = estimator.SKCompat(estimator.Estimator(model_fn=linear_model_fn))
   float64_labels = boston.target.astype(np.float64)
   est.fit(x=boston.data, y=float64_labels, steps=100)
   scores = est.score(
       x=boston.data,
       y=float64_labels,
       metrics={'MSE': metric_ops.streaming_mean_squared_error})
   predictions = np.array(list(est.predict(x=boston.data)))
   other_score = _sklearn.mean_squared_error(predictions, boston.target)
   self.assertAllClose(scores['MSE'], other_score)
   self.assertTrue('global_step' in scores)
   self.assertEqual(100, scores['global_step'])

    def testCustomMetrics(self):
        """Tests custom evaluation metrics."""

        def _input_fn(num_epochs=None):
            # Create 4 rows, one of them (y = x), three of them (y=Not(x))
            labels = tf.constant([[1.], [0.], [0.], [0.]])
            features = {
                'x':
                tf.train.limit_epochs(
                    tf.ones(shape=[4, 1], dtype=tf.float32),
                    num_epochs=num_epochs),
            }
            return features, labels

        def _my_metric_op(predictions, labels):
            return tf.reduce_sum(tf.mul(predictions, labels))

        regressor = tf.contrib.learn.DNNRegressor(
            feature_columns=[tf.contrib.layers.real_valued_column('x')],
            hidden_units=[3, 3],
            config=tf.contrib.learn.RunConfig(tf_random_seed=1))

        regressor.fit(input_fn=_input_fn, steps=5)
        scores = regressor.evaluate(
            input_fn=_input_fn,
            steps=1,
            metrics={
                'my_error': tf.contrib.metrics.streaming_mean_squared_error,
                'my_metric': _my_metric_op
            })
        self.assertIn('loss', set(scores.keys()))
        self.assertIn('my_error', set(scores.keys()))
        self.assertIn('my_metric', set(scores.keys()))
        predict_input_fn = functools.partial(_input_fn, num_epochs=1)
        predictions = np.array(
            list(regressor.predict(input_fn=predict_input_fn)))
        self.assertAlmostEqual(
            _sklearn.mean_squared_error(np.array([1, 0, 0, 0]), predictions),
            scores['my_error'])

        # Tests that when the key is a tuple, an error is raised.
        with self.assertRaises(KeyError):
            regressor.evaluate(
                input_fn=_input_fn,
                steps=1,
                metrics={
                    ('my_error', 'predictions'):
                    tf.contrib.metrics.streaming_mean_squared_error
                })

 def testBostonAllDictionaryInput(self):
   boston = tf.contrib.learn.datasets.load_boston()
   est = tf.contrib.learn.Estimator(model_fn=linear_model_fn)
   boston_input = {'input': boston.data}
   float64_target = {'labels': boston.target.astype(np.float64)}
   est.fit(x=boston_input, y=float64_target, steps=100)
   scores = est.evaluate(
     x=boston_input,
     y=float64_target,
     metrics={'MSE': tf.contrib.metrics.streaming_mean_squared_error})
   predictions = np.array(list(est.predict(x=boston_input)))
   other_score = _sklearn.mean_squared_error(predictions, boston.target)
   self.assertAllClose(other_score, scores['MSE'])
   self.assertTrue('global_step' in scores)
   self.assertEqual(scores['global_step'], 100)

 def testBostonDNN(self):
   boston = tf.contrib.learn.datasets.load_boston()
   regressor = tf.contrib.learn.TensorFlowDNNRegressor(
       hidden_units=[10, 20, 10], n_classes=0,
       batch_size=boston.data.shape[0], steps=300, learning_rate=0.01)
   regressor.fit(boston.data, boston.target)
   score = mean_squared_error(boston.target, regressor.predict(boston.data))
   self.assertLess(score, 110, "Failed with score = {0}".format(score))
   weights = regressor.weights_
   self.assertEqual(weights[0].shape, (13, 10))
   self.assertEqual(weights[1].shape, (10, 20))
   self.assertEqual(weights[2].shape, (20, 10))
   self.assertEqual(weights[3].shape, (10, 1))
   biases = regressor.bias_
   self.assertEqual(len(biases), 5)

 def testBostonDNN(self):
   boston = tf.contrib.learn.datasets.load_boston()
   feature_columns = [tf.contrib.layers.real_valued_column("", dimension=13)]
   regressor = tf.contrib.learn.DNNRegressor(
       feature_columns=feature_columns, hidden_units=[10, 20, 10],
       config=tf.contrib.learn.RunConfig(tf_random_seed=1))
   regressor.fit(
       boston.data, boston.target, steps=300, batch_size=boston.data.shape[0])
   score = mean_squared_error(boston.target, regressor.predict(boston.data))
   self.assertLess(score, 110, "Failed with score = {0}".format(score))
   weights = regressor.weights_
   self.assertEqual(weights[0].shape, (13, 10))
   self.assertEqual(weights[1].shape, (10, 20))
   self.assertEqual(weights[2].shape, (20, 10))
   self.assertEqual(weights[3].shape, (10, 1))
   biases = regressor.bias_
   self.assertEqual(len(biases), 5)