def testIrisStreaming(self):
    iris = datasets.load_iris()

    def iris_data():
      while True:
        for x in iris.data:
          yield x

    def iris_predict_data():
      for x in iris.data:
        yield x

    def iris_target():
      while True:
        for y in iris.target:
          yield y

    classifier = learn.TensorFlowLinearClassifier(n_classes=3, steps=100)
    classifier.fit(iris_data(), iris_target())
    score1 = accuracy_score(iris.target, classifier.predict(iris.data))
    score2 = accuracy_score(iris.target,
                            classifier.predict(iris_predict_data()))
    self.assertGreater(score1, 0.5, "Failed with score = {0}".format(score1))
    self.assertEqual(score2, score1, "Scores from {0} iterator doesn't "
                     "match score {1} from full "
                     "data.".format(score2, score1))

  def testIrisStreaming(self):
    iris = datasets.load_iris()

    def iris_data():
      while True:
        for x in iris.data:
          yield x

    def iris_predict_data():
      for x in iris.data:
        yield x

    def iris_target():
      while True:
        for y in iris.target:
          yield y

    classifier = learn.LinearClassifier(
        feature_columns=learn.infer_real_valued_columns_from_input(iris.data),
        n_classes=3)
    classifier.fit(iris_data(), iris_target(), max_steps=500)
    score1 = accuracy_score(iris.target, classifier.predict(iris.data))
    score2 = accuracy_score(iris.target,
                            classifier.predict(iris_predict_data()))
    self.assertGreater(score1, 0.5, "Failed with score = {0}".format(score1))
    self.assertEqual(score2, score1, "Scores from {0} iterator doesn't "
                     "match score {1} from full "
                     "data.".format(score2, score1))

  def testIrisES(self):
    random.seed(42)

    iris = datasets.load_iris()
    x_train, x_test, y_train, y_test = train_test_split(iris.data,
                                                        iris.target,
                                                        test_size=0.2,
                                                        random_state=42)

    x_train, x_val, y_train, y_val = train_test_split(
        x_train, y_train, test_size=0.2)
    val_monitor = learn.monitors.ValidationMonitor(x_val, y_val,
                                                   early_stopping_rounds=100)

    # classifier without early stopping - overfitting
    classifier1 = learn.TensorFlowDNNClassifier(hidden_units=[10, 20, 10],
                                                n_classes=3,
                                                steps=1000)
    classifier1.fit(x_train, y_train)
    accuracy_score(y_test, classifier1.predict(x_test))

    # classifier with early stopping - improved accuracy on testing set
    classifier2 = learn.TensorFlowDNNClassifier(hidden_units=[10, 20, 10],
                                                n_classes=3,
                                                steps=1000)

    classifier2.fit(x_train, y_train, monitors=[val_monitor])
    accuracy_score(y_test, classifier2.predict(x_test))

 def testIrisContinueTraining(self):
     iris = datasets.load_iris()
     classifier = learn.TensorFlowLinearClassifier(n_classes=3,
         learning_rate=0.01, continue_training=True, steps=250)
     classifier.fit(iris.data, iris.target)
     score1 = accuracy_score(iris.target, classifier.predict(iris.data))
     classifier.fit(iris.data, iris.target)
     score2 = accuracy_score(iris.target, classifier.predict(iris.data))
     self.assertGreater(score2, score1,
                        "Failed with score = {0}".format(score2))

 def testIrisContinueTraining(self):
   iris = datasets.load_iris()
   classifier = learn.LinearClassifier(
       feature_columns=learn.infer_real_valued_columns_from_input(iris.data),
       n_classes=3)
   classifier.fit(iris.data, iris.target, steps=100)
   score1 = accuracy_score(iris.target, classifier.predict(iris.data))
   classifier.fit(iris.data, iris.target, steps=500)
   score2 = accuracy_score(iris.target, classifier.predict(iris.data))
   self.assertGreater(
       score2, score1,
       "Failed with score2 {0} <= score1 {1}".format(score2, score1))

    def testIrisES(self):
        random.seed(42)

        iris = datasets.load_iris()
        x_train, x_test, y_train, y_test = train_test_split(iris.data, iris.target, test_size=0.2, random_state=42)

        x_train, x_val, y_train, y_val = train_test_split(x_train, y_train, test_size=0.2, random_state=42)
        val_monitor = learn.monitors.ValidationMonitor(
            x_val,
            y_val,
            every_n_steps=50,
            early_stopping_rounds=100,
            early_stopping_metric="accuracy",
            early_stopping_metric_minimize=False,
        )

        # classifier without early stopping - overfitting
        classifier1 = learn.TensorFlowDNNClassifier(hidden_units=[10, 20, 10], n_classes=3, steps=1000)
        classifier1.fit(x_train, y_train)
        _ = accuracy_score(y_test, classifier1.predict(x_test))

        # Full 1000 steps, 12 summaries and no evaluation summary.
        # 12 summaries = global_step + first + every 100 out of 1000 steps.
        self.assertEqual(12, len(_get_summary_events(classifier1.model_dir)))
        with self.assertRaises(ValueError):
            _get_summary_events(classifier1.model_dir + "/eval")

        # classifier with early stopping - improved accuracy on testing set
        classifier2 = learn.TensorFlowDNNClassifier(
            hidden_units=[10, 20, 10],
            n_classes=3,
            steps=2000,
            config=tf.contrib.learn.RunConfig(save_checkpoints_secs=1),
        )

        classifier2.fit(x_train, y_train, monitors=[val_monitor])
        _ = accuracy_score(y_val, classifier2.predict(x_val))
        _ = accuracy_score(y_test, classifier2.predict(x_test))

        # Note, this test is unstable, so not checking for equality.
        # See stability_test for examples of stability issues.
        if val_monitor.early_stopped:
            self.assertLess(val_monitor.best_step, 2000)
            # Note, due to validation monitor stopping after the best score occur,
            # the accuracy at current checkpoint is less.
            # TODO(ipolosukhin): Time machine for restoring old checkpoints?
            # flaky, still not always best_value better then score2 value.
            # self.assertGreater(val_monitor.best_value, score2_val)

            # Early stopped, unstable so checking only < then max.
            self.assertLess(len(_get_summary_events(classifier2.model_dir)), 21)
            # Eval typically has ~6 events, but it varies based on the run.
            self.assertLess(len(_get_summary_events(classifier2.model_dir + "/eval")), 8)

 def testDNNDropout0_1(self):
     # Dropping only a little.
     iris = datasets.load_iris()
     classifier = learn.TensorFlowDNNClassifier(hidden_units=[10, 20, 10], n_classes=3, dropout=0.1)
     classifier.fit(iris.data, iris.target)
     score = accuracy_score(iris.target, classifier.predict(iris.data))
     self.assertGreater(score, 0.9, "Failed with score = {0}".format(score))

  def testCustomMetrics(self):
    """Tests weight column in evaluation."""

    def _input_fn_train():
      # Create 4 rows, one of them (y = x), three of them (y=Not(x))
      target = tf.constant([[1], [0], [0], [0]])
      features = {'x': tf.ones(shape=[4, 1], dtype=tf.float32),}
      return features, target

    classifier = tf.contrib.learn.DNNLinearCombinedClassifier(
        linear_feature_columns=[tf.contrib.layers.real_valued_column('x')],
        dnn_feature_columns=[tf.contrib.layers.real_valued_column('x')],
        dnn_hidden_units=[3, 3])

    classifier.train(input_fn=_input_fn_train, steps=100)
    scores = classifier.evaluate(
        input_fn=_input_fn_train,
        steps=100,
        metrics={
            'my_accuracy': tf.contrib.metrics.streaming_accuracy,
            'my_precision': tf.contrib.metrics.streaming_precision
        })
    self.assertTrue(set(['loss', 'my_accuracy', 'my_precision']).issubset(set(
        scores.keys())))
    predictions = classifier.predict(input_fn=_input_fn_train)
    self.assertEqual(_sklearn.accuracy_score([1, 0, 0, 0], predictions),
                     scores['my_accuracy'])

 def testIrisAllDictionaryInput(self):
   iris = base.load_iris()
   est = estimator.Estimator(model_fn=logistic_model_no_mode_fn)
   iris_data = {'input': iris.data}
   iris_target = {'labels': iris.target}
   est.fit(iris_data, iris_target, steps=100)
   scores = est.evaluate(
       x=iris_data,
       y=iris_target,
       metrics={
           ('accuracy', 'class'): metric_ops.streaming_accuracy
       })
   predictions = list(est.predict(x=iris_data))
   predictions_class = list(est.predict(x=iris_data, outputs=['class']))
   self.assertEqual(len(predictions), iris.target.shape[0])
   classes_batch = np.array([p['class'] for p in predictions])
   self.assertAllClose(classes_batch,
                       np.array([p['class'] for p in predictions_class]))
   self.assertAllClose(classes_batch,
                       np.argmax(
                           np.array([p['prob'] for p in predictions]), axis=1))
   other_score = _sklearn.accuracy_score(iris.target, classes_batch)
   self.assertAllClose(other_score, scores['accuracy'])
   self.assertTrue('global_step' in scores)
   self.assertEqual(scores['global_step'], 100)

  def testIrisMomentum(self):
    random.seed(42)

    iris = datasets.load_iris()
    x_train, x_test, y_train, y_test = train_test_split(iris.data,
                                                        iris.target,
                                                        test_size=0.2,
                                                        random_state=42)
    # setup exponential decay function
    def exp_decay(global_step):
      return tf.train.exponential_decay(learning_rate=0.1,
                                        global_step=global_step,
                                        decay_steps=100,
                                        decay_rate=0.001)

    def custom_optimizer(learning_rate):
      return tf.train.MomentumOptimizer(learning_rate, 0.9)

    classifier = learn.TensorFlowDNNClassifier(hidden_units=[10, 20, 10],
                                               n_classes=3,
                                               steps=400,
                                               learning_rate=exp_decay,
                                               optimizer=custom_optimizer)
    classifier.fit(x_train, y_train)
    score = accuracy_score(y_test, classifier.predict(x_test))

    self.assertGreater(score, 0.65, "Failed with score = {0}".format(score))

 def testIrisClassWeight(self):
     iris = datasets.load_iris()
     classifier = learn.TensorFlowLinearClassifier(
         n_classes=3, class_weight=[0.1, 0.8, 0.1])
     classifier.fit(iris.data, iris.target)
     score = accuracy_score(iris.target, classifier.predict(iris.data))
     self.assertLess(score, 0.7, "Failed with score = {0}".format(score))

  def testIrisMomentum(self):
    random.seed(42)

    iris = datasets.load_iris()
    x_train, x_test, y_train, y_test = train_test_split(iris.data,
                                                        iris.target,
                                                        test_size=0.2,
                                                        random_state=42)

    def custom_optimizer(learning_rate):
      return tf.train.MomentumOptimizer(learning_rate, 0.9)

    cont_features = [
        tf.contrib.layers.real_valued_column("", dimension=4)]
    classifier = learn.TensorFlowDNNClassifier(
        feature_columns=cont_features,
        hidden_units=[10, 20, 10],
        n_classes=3,
        steps=400,
        learning_rate=0.01,
        optimizer=custom_optimizer)
    classifier.fit(x_train, y_train)
    score = accuracy_score(y_test, classifier.predict(x_test))

    self.assertGreater(score, 0.65, "Failed with score = {0}".format(score))

 def testDNNDropout0(self):
   # Dropout prob == 0.
   iris = tf.contrib.learn.datasets.load_iris()
   classifier = tf.contrib.learn.TensorFlowDNNClassifier(
       hidden_units=[10, 20, 10], n_classes=3, dropout=0.0)
   classifier.fit(iris.data, iris.target)
   score = accuracy_score(iris.target, classifier.predict(iris.data))
   self.assertGreater(score, 0.9, "Failed with score = {0}".format(score))

 def testIris(self):
   iris = datasets.load_iris()
   classifier = learn.TensorFlowLinearClassifier(
       feature_columns=learn.infer_real_valued_columns_from_input(iris.data),
       n_classes=3)
   classifier.fit(iris.data, [x for x in iris.target])
   score = accuracy_score(iris.target, classifier.predict(iris.data))
   self.assertGreater(score, 0.7, "Failed with score = {0}".format(score))

 def testIrisSummaries(self):
   iris = datasets.load_iris()
   output_dir = tempfile.mkdtemp() + "learn_tests/"
   classifier = learn.TensorFlowLinearClassifier(n_classes=3,
                                                 model_dir=output_dir)
   classifier.fit(iris.data, iris.target)
   score = accuracy_score(iris.target, classifier.predict(iris.data))
   self.assertGreater(score, 0.5, "Failed with score = {0}".format(score))

  def testCustomMetrics(self):
    """Tests custom evaluation metrics."""

    def _input_fn(num_epochs=None):
      # Create 4 rows, one of them (y = x), three of them (y=Not(x))
      target = tf.constant([[1], [0], [0], [0]])
      features = {
          'x': tf.train.limit_epochs(
              tf.ones(shape=[4, 1], dtype=tf.float32), num_epochs=num_epochs)}
      return features, target

    def _my_metric_op(predictions, targets):
      # For the case of binary classification, the 2nd column of "predictions"
      # denotes the model predictions.
      targets = tf.to_float(targets)
      predictions = tf.slice(predictions, [0, 1], [-1, 1])
      return tf.reduce_sum(tf.mul(predictions, targets))

    classifier = tf.contrib.learn.DNNLinearCombinedClassifier(
        linear_feature_columns=[tf.contrib.layers.real_valued_column('x')],
        dnn_feature_columns=[tf.contrib.layers.real_valued_column('x')],
        dnn_hidden_units=[3, 3])

    classifier.fit(input_fn=_input_fn, steps=100)
    scores = classifier.evaluate(
        input_fn=_input_fn,
        steps=100,
        metrics={
            'my_accuracy': tf.contrib.metrics.streaming_accuracy,
            ('my_precision', 'classes'): tf.contrib.metrics.streaming_precision,
            ('my_metric', 'probabilities'): _my_metric_op
        })
    self.assertTrue(
        set(['loss', 'my_accuracy', 'my_precision', 'my_metric'
            ]).issubset(set(scores.keys())))
    predict_input_fn = functools.partial(_input_fn, num_epochs=1)
    predictions = np.array(
        list(classifier.predict(input_fn=predict_input_fn)))
    self.assertEqual(_sklearn.accuracy_score([1, 0, 0, 0], predictions),
                     scores['my_accuracy'])

    # Test the case where the 2nd element of the key is neither "classes" nor
    # "probabilities".
    with self.assertRaises(KeyError):
      classifier.evaluate(
          input_fn=_input_fn,
          steps=100,
          metrics={('bad_name', 'bad_type'): tf.contrib.metrics.streaming_auc})

    # Test the case where the tuple of the key doesn't have 2 elements.
    with self.assertRaises(ValueError):
      classifier.evaluate(
          input_fn=_input_fn,
          steps=100,
          metrics={
              ('bad_length_name', 'classes', 'bad_length'):
                  tf.contrib.metrics.streaming_accuracy
          })

  def testCustomMetrics(self):
    """Tests custom evaluation metrics."""
    def _input_fn(num_epochs=None):
      # Create 4 rows, one of them (y = x), three of them (y=Not(x))
      labels = tf.constant([[1], [0], [0], [0]])
      features = {
          'x': tf.train.limit_epochs(
              tf.ones(shape=[4, 1], dtype=tf.float32), num_epochs=num_epochs),
      }
      return features, labels

    def _my_metric_op(predictions, labels):
      # For the case of binary classification, the 2nd column of "predictions"
      # denotes the model predictions.
      labels = tf.to_float(labels)
      predictions = tf.strided_slice(
          predictions, [0, 1], [-1, 2], end_mask=1)
      labels = math_ops.cast(labels, predictions.dtype)
      return tf.reduce_sum(tf.multiply(predictions, labels))

    classifier = tf.contrib.learn.DNNClassifier(
        feature_columns=[tf.contrib.layers.real_valued_column('x')],
        hidden_units=[3, 3],
        config=tf.contrib.learn.RunConfig(tf_random_seed=1))

    classifier.fit(input_fn=_input_fn, steps=5)
    scores = classifier.evaluate(
        input_fn=_input_fn,
        steps=5,
        metrics={
            'my_accuracy': MetricSpec(
                metric_fn=tf.contrib.metrics.streaming_accuracy,
                prediction_key='classes'),
            'my_precision': MetricSpec(
                metric_fn=tf.contrib.metrics.streaming_precision,
                prediction_key='classes'),
            'my_metric': MetricSpec(
                metric_fn=_my_metric_op,
                prediction_key='probabilities')
        })
    self.assertTrue(
        set(['loss', 'my_accuracy', 'my_precision', 'my_metric'
            ]).issubset(set(scores.keys())))
    predict_input_fn = functools.partial(_input_fn, num_epochs=1)
    predictions = np.array(list(classifier.predict(input_fn=predict_input_fn)))
    self.assertEqual(_sklearn.accuracy_score([1, 0, 0, 0], predictions),
                     scores['my_accuracy'])

    # Test the case where the 2nd element of the key is neither "classes" nor
    # "probabilities".
    with self.assertRaisesRegexp(KeyError, 'bad_type'):
      classifier.evaluate(
          input_fn=_input_fn,
          steps=5,
          metrics={
              'bad_name': MetricSpec(
                  metric_fn=tf.contrib.metrics.streaming_auc,
                  prediction_key='bad_type')})

 def testIrisClassWeight(self):
   iris = datasets.load_iris()
   # Note, class_weight are not supported anymore :( Use weight_column.
   with self.assertRaises(ValueError):
     classifier = learn.TensorFlowLinearClassifier(
         n_classes=3, class_weight=[0.1, 0.8, 0.1])
     classifier.fit(iris.data, iris.target)
     score = accuracy_score(iris.target, classifier.predict(iris.data))
     self.assertLess(score, 0.7, "Failed with score = {0}".format(score))

 def testIrisSummaries(self):
   iris = datasets.load_iris()
   output_dir = tempfile.mkdtemp() + "learn_tests/"
   classifier = learn.LinearClassifier(
       feature_columns=learn.infer_real_valued_columns_from_input(iris.data),
       n_classes=3, model_dir=output_dir)
   classifier.fit(iris.data, iris.target, max_steps=100)
   score = accuracy_score(iris.target, classifier.predict(iris.data))
   self.assertGreater(score, 0.5, "Failed with score = {0}".format(score))

  def testCustomMetrics(self):
    """Tests custom evaluation metrics."""

    def _input_fn_train():
      # Create 4 rows, one of them (y = x), three of them (y=Not(x))
      target = tf.constant([[1], [0], [0], [0]], dtype=tf.float32)
      features = {'x': tf.ones(shape=[4, 1], dtype=tf.float32)}
      return features, target

    def _my_metric_op(predictions, targets):
      # For the case of binary classification, the 2nd column of "predictions"
      # denotes the model predictions.
      predictions = tf.slice(predictions, [0, 1], [-1, 1])
      return tf.reduce_sum(tf.mul(predictions, targets))

    classifier = tf.contrib.learn.LinearClassifier(
        feature_columns=[tf.contrib.layers.real_valued_column('x')])

    classifier.fit(input_fn=_input_fn_train, steps=100)
    scores = classifier.evaluate(
        input_fn=_input_fn_train,
        steps=100,
        metrics={
            'my_accuracy': MetricSpec(
                metric_fn=tf.contrib.metrics.streaming_accuracy,
                prediction_key='classes'),
            'my_precision': MetricSpec(
                metric_fn=tf.contrib.metrics.streaming_precision,
                prediction_key='classes'),
            'my_metric': MetricSpec(metric_fn=_my_metric_op,
                                    prediction_key='probabilities')
        })
    self.assertTrue(
        set(['loss', 'my_accuracy', 'my_precision', 'my_metric'
            ]).issubset(set(scores.keys())))
    predictions = classifier.predict(input_fn=_input_fn_train)
    self.assertEqual(_sklearn.accuracy_score([1, 0, 0, 0], predictions),
                     scores['my_accuracy'])

    # Test the case where the 2nd element of the key is neither "classes" nor
    # "probabilities".
    with self.assertRaises(ValueError):
      classifier.evaluate(
          input_fn=_input_fn_train,
          steps=100,
          metrics={('bad_name', 'bad_type'): tf.contrib.metrics.streaming_auc})

    # Test the case where the tuple of the key doesn't have 2 elements.
    with self.assertRaises(ValueError):
      classifier.evaluate(
          input_fn=_input_fn_train,
          steps=100,
          metrics={
              ('bad_length_name', 'classes', 'bad_length'):
                  tf.contrib.metrics.streaming_accuracy
          })