def testScoringAllMetrics(self):
   """
   This tests an example set of detections, where all metrics have counts > 0.
   """
   start = datetime.datetime.now()
   increment = datetime.timedelta(minutes=5)
   length = 100
   numWindows = 2
   windowSize = 5
   
   timestamps = generateTimestamps(start, increment, length)
   windows = generateWindows(timestamps, numWindows, windowSize)
   labels = generateLabels(timestamps, windows)
   predictions = pandas.Series([0]*length)
   
   index = timestamps[timestamps == windows[0][0]].index[0]
   # TP, add'l TP, and FP
   predictions[index] = 1
   predictions[index+1] = 1
   predictions[index+7] = 1
   
   scorer = Scorer(timestamps, predictions, labels, windows, self.costMatrix,
     probationaryPeriod=0)
   (_, score) = scorer.getScore()
   
   self.assertAlmostEquals(score, -0.9540, 4)
   self._checkCounts(scorer.counts, length-windowSize*numWindows-1, 2, 1, 8)

  def testRowsLabeledAnomalousWithinAWindow(self):
    """
    All timestamps labeled as anomalous should be within a label window.
    """
    data = pandas.DataFrame({"timestamp" :
      generateTimestamps(strp("2014-01-01"),
      datetime.timedelta(minutes=5), 10)})

    windows = [["2014-01-01 00:15", "2014-01-01 00:30"]]

    writeCorpus(self.tempCorpusPath, {"test_data_file.csv": data})
    writeCorpusLabel(self.tempCorpusLabelPath, {"test_data_file.csv": windows})

    corpus = nab.corpus.Corpus(self.tempCorpusPath)

    corpusLabel = nab.labeler.CorpusLabel(self.tempCorpusLabelPath, corpus)

    for relativePath, lab in corpusLabel.labels.iteritems():
      windows = corpusLabel.windows[relativePath]

      for row in lab[lab["label"] == 1].iterrows():
        self.assertTrue(
          all([w[0] <= row[1]["timestamp"] <= w[1] for w in windows]),
            "The label at %s of file %s is not within a label window"
            % (row[1]["timestamp"], relativePath))

  def testFalsePositiveMeansNegativeScore(self):
    """
    A false positive should make the score negative.
    """
    start = datetime.datetime.now()
    increment = datetime.timedelta(minutes=5)
    length = 1000
    numWindows = 1
    windowSize = 10
    threshold = 0.5

    timestamps = generateTimestamps(start, increment, length)
    windows = generateWindows(timestamps, numWindows, windowSize)
    anomalyScores = pandas.Series([0]*length)

    anomalyScores[0] = 1
    sweeper = Sweeper(probationPercent=0, costMatrix=self.costMatrix)
    (scores, matchingRow) = sweeper.scoreDataSet(
      timestamps,
      anomalyScores,
      windows,
      "testData",
      threshold
    )
    self.assertTrue(matchingRow.score < 0)
    self._checkCounts(matchingRow, length-windowSize*numWindows-1, 0, 1,
      windowSize*numWindows)

  def test_oneFalsePositiveNoWindow(self):
    """
    When there is no window (meaning no anomaly), a false positive should still
    result in a negative score.
    """
    start = datetime.datetime.now()
    increment = datetime.timedelta(minutes=5)
    length = 1000
    numWindows = 0
    windowSize = 10

    timestamps = generateTimestamps(start, increment, length)

    predictions = pandas.Series([0]*length)
    windows = generateWindows(timestamps, numWindows, windowSize)
    labels = generateLabels(timestamps, windows)

    costMatrix = {"tpWeight": 1.0,
    "fnWeight": 1.0,
    "fpWeight": 1.0,
    "tnWeight": 1.0}

    predictions[0] = 1

    scorer = Scorer(timestamps, predictions, labels, windows, costMatrix,
      probationaryPeriod=0)

    self.assertTrue(scorer.getScore() == -costMatrix["fpWeight"])

    # Ensure counts are correct.
    self.assertEqual(scorer.counts['tn'], length-windowSize*numWindows-1)
    self.assertEqual(scorer.counts['tp'], 0)
    self.assertEqual(scorer.counts['fp'], 1)
    self.assertEqual(scorer.counts['fn'], windowSize*numWindows)

  def testGetLabels(self):
    """
    Labels dictionary generated by CorpusLabel.getLabels() should match the
    label windows.
    """
    data = pandas.DataFrame({"timestamp" :
      generateTimestamps(strp("2014-01-01"),
      datetime.timedelta(minutes=5), 10)})

    windows = [["2014-01-01 00:00", "2014-01-01 00:10"],
               ["2014-01-01 00:10", "2014-01-01 00:15"]]
    
    writeCorpus(self.tempCorpusPath, {"test_data_file.csv" : data})
    writeCorpusLabel(self.tempCorpusLabelPath, {"test_data_file.csv": windows})

    corpus = nab.corpus.Corpus(self.tempCorpusPath)

    corpusLabel = nab.labeler.CorpusLabel(self.tempCorpusLabelPath, corpus)

    for relativePath, l in corpusLabel.labels.iteritems():
      windows = corpusLabel.windows[relativePath]

      for t, lab in corpusLabel.labels["test_data_file.csv"].values:
        for w in windows:
          if (w[0] <= t and t <= w[1]):
            self.assertEqual(lab, 1,
              "Incorrect label value for timestamp %r" % t)

  def testNonexistentDatafileOrLabelsThrowsError(self):
    """
    A KeyError should be thrown when there are not corresponding windows labels
    for a data file (or vice-versa) in the corpus.
    """
    data = pandas.DataFrame({"timestamp" :
      generateTimestamps(strp("2014-01-01"),
      datetime.timedelta(minutes=5), 10)})

    windows = [["2014-01-01 00:15", "2014-01-01 00:30"]]

    # Case 1: nonexistent datafile for window labels
    writeCorpus(self.tempCorpusPath, {"test_data_file.csv": data})
    writeCorpusLabel(self.tempCorpusLabelPath,
      {"test_data_file.csv": windows, "non_existent_data_file.csv": windows})
    
    corpus = nab.corpus.Corpus(self.tempCorpusPath)

    self.assertRaises(
      KeyError, nab.labeler.CorpusLabel, self.tempCorpusLabelPath, corpus)
  
    # Case 2: nonexistent window labels for datafile
    writeCorpus(self.tempCorpusPath,
      {"test_data_file.csv": data, "non_existent_data_file.csv": data})
    writeCorpusLabel(self.tempCorpusLabelPath, {"test_data_file.csv": windows})
    
    corpus = nab.corpus.Corpus(self.tempCorpusPath)

    self.assertRaises(
      KeyError, nab.labeler.CorpusLabel, self.tempCorpusLabelPath, corpus)

  def testTwoFalsePositivesIsWorseThanOne(self):
    """
    For two false positives A and B in a file, the score given A and B should be
    more negative than the score given just A.
    """
    start = datetime.datetime.now()
    increment = datetime.timedelta(minutes=5)
    length = 1000
    numWindows = 1
    windowSize = 10

    timestamps = generateTimestamps(start, increment, length)
    windows = generateWindows(timestamps, numWindows, windowSize)
    labels = generateLabels(timestamps, windows)
    predictions = pandas.Series([0]*length)

    predictions[0] = 1
    scorer1 = Scorer(timestamps, predictions, labels, windows, self.costMatrix,
      probationaryPeriod=0)
    (_, score1) = scorer1.getScore()

    predictions[1] = 1
    scorer2 = Scorer(timestamps, predictions, labels, windows, self.costMatrix,
      probationaryPeriod=0)
    (_, score2) = scorer2.getScore()

    self.assertTrue(score2 < score1)
    self._checkCounts(scorer1.counts, length-windowSize*numWindows-1, 0, 1,
      windowSize*numWindows)
    self._checkCounts(scorer2.counts, length-windowSize*numWindows-2, 0, 2,
      windowSize*numWindows)

  def testEarlierFalsePositiveAfterWindowIsBetter(self):
    """For two false positives A and B, where A occurs earlier than B, the
    score change due to A will be less than the score change due to B.
    """
    start = datetime.datetime.now()
    increment = datetime.timedelta(minutes=5)
    length = 10
    numWindows = 1
    windowSize = 2

    timestamps = generateTimestamps(start, increment, length)
    windows = generateWindows(timestamps, numWindows, windowSize)
    labels = generateLabels(timestamps, windows)
    predictions1 = pandas.Series([0]*length)
    predictions2 = pandas.Series([0]*length)
    t1, t2 = windows[0]

    index1 = timestamps[timestamps == t2].index[0] + 1
    predictions1[index1] = 1
    scorer1 = Scorer(timestamps, predictions1, labels, windows, self.costMatrix,
      probationaryPeriod=0)
    (_, score1) = scorer1.getScore()

    predictions2[index1+1] = 1
    scorer2 = Scorer(timestamps, predictions2, labels, windows, self.costMatrix,
      probationaryPeriod=0)
    (_, score2) = scorer2.getScore()

    self.assertTrue(score1 > score2)
    self._checkCounts(scorer1.counts, length-windowSize*numWindows-1, 0, 1,
      windowSize*numWindows)
    self._checkCounts(scorer2.counts, length-windowSize*numWindows-1, 0, 1,
      windowSize*numWindows)

  def test_firstTruePositiveWithinWindow(self):
    """
    First record within window has a score close to costMatrix["tpWeight"].
    Since we use Sigmoids, it will never be exactly 1.
    """

    start = datetime.datetime.now()
    increment = datetime.timedelta(minutes=5)
    length = 10
    numWindows = 1
    windowSize = 2

    timestamps = generateTimestamps(start, increment, length)
    predictions = pandas.Series([0]*length)
    windows = generateWindows(timestamps, numWindows, windowSize)
    labels = generateLabels(timestamps, windows)

    costMatrix = {"tpWeight": 1.0,
                  "fnWeight": 2.0,
                  "fpWeight": 3.0,
                  "tnWeight": 4.0}

    index = timestamps[timestamps == windows[0][0]].index[0]
    predictions[index] = 1

    scorer = Scorer(timestamps, predictions, labels, windows, costMatrix,
      probationaryPeriod=0)

    self.assertTrue(costMatrix["tpWeight"] - scorer.getScore() <= 1)

  def testOnlyScoreFirstTruePositiveWithinWindow(self):
    """
    An algorithm making multiple detections within a window (i.e. true positive)
    should only be scored for the earliest true positive.
    """
    start = datetime.datetime.now()
    increment = datetime.timedelta(minutes=5)
    length = 10
    numWindows = 1
    windowSize = 2

    timestamps = generateTimestamps(start, increment, length)
    windows = generateWindows(timestamps, numWindows, windowSize)
    labels = generateLabels(timestamps, windows)
    predictions = pandas.Series([0]*length)
    window = windows[0]
    t1, t2 = window

    index1 = timestamps[timestamps == t1].index[0]
    predictions[index1] = 1
    scorer1 = Scorer(timestamps, predictions, labels, windows, self.costMatrix,
      probationaryPeriod=0)
    (_, score1) = scorer1.getScore()

    index2 = timestamps[timestamps == t2].index[0]
    predictions[index2] = 1
    scorer2 = Scorer(timestamps, predictions, labels, windows, self.costMatrix,
      probationaryPeriod=0)
    (_, score2) = scorer2.getScore()

    self.assertEqual(score1, score2)
    self._checkCounts(scorer1.counts, length-windowSize*numWindows, 1, 0,
      windowSize*numWindows-1)
    self._checkCounts(scorer2.counts, length-windowSize*numWindows, 2, 0,
      windowSize*numWindows-2)

  def testOneFalsePositiveNoWindow(self):
    """
    When there is no window (i.e. no anomaly), a false positive should still
    result in a negative score, specifically negative the FP weight.
    """
    start = datetime.datetime.now()
    increment = datetime.timedelta(minutes=5)
    length = 1000
    numWindows = 0
    windowSize = 10
    threshold = 0.5

    timestamps = generateTimestamps(start, increment, length)
    windows = generateWindows(timestamps, numWindows, windowSize)
    anomalyScores = pandas.Series([0]*length)

    anomalyScores[0] = 1
    sweeper = Sweeper(probationPercent=0, costMatrix=self.costMatrix)
    (scores, matchingRow) = sweeper.scoreDataSet(
      timestamps,
      anomalyScores,
      windows,
      "testData",
      threshold
    )
    
    self.assertEqual(matchingRow.score, -self.costMatrix["fpWeight"])
    self._checkCounts(matchingRow, length-windowSize*numWindows-1, 0, 1,
      windowSize*numWindows)

  def testFirstTruePositiveWithinWindow(self):
    """
    First record within window has a score approximately equal to 
    self.costMatrix["tpWeight"]; within 4 decimal places is more than enough
    precision.
    """
    start = datetime.datetime.now()
    increment = datetime.timedelta(minutes=5)
    length = 10
    numWindows = 1
    windowSize = 2

    timestamps = generateTimestamps(start, increment, length)
    windows = generateWindows(timestamps, numWindows, windowSize)
    labels = generateLabels(timestamps, windows)
    predictions = pandas.Series([0]*length)

    index = timestamps[timestamps == windows[0][0]].index[0]
    predictions[index] = 1
    scorer = Scorer(timestamps, predictions, labels, windows, self.costMatrix,
      probationaryPeriod=0)
    (_, score) = scorer.getScore()

    self.assertAlmostEquals(score, self.costMatrix["tpWeight"], 4)
    self._checkCounts(scorer.counts, length-windowSize*numWindows, 1, 0,
      windowSize*numWindows-1)

  def testRewardLowFalsePositives(self):
    """
    Given false positives in the set of detections, the score output with the
    Reward Low False Positives application profile will be greater than with
    the Standard application profile.
    """
    start = datetime.datetime.now()
    increment = datetime.timedelta(minutes=5)
    length = 100
    numWindows = 0
    windowSize = 10
    
    timestamps = generateTimestamps(start, increment, length)
    windows = []
    labels = generateLabels(timestamps, windows)
    predictions = pandas.Series([0]*length)
    
    costMatrixFP = copy.deepcopy(self.costMatrix)
    costMatrixFP["fpWeight"] = 2.0
    costMatrixFP["fnWeight"] = 0.5
    # FP
    predictions[0] = 1

    scorer1 = Scorer(timestamps, predictions, labels, windows, self.costMatrix,
      probationaryPeriod=0)
    (_, score1) = scorer1.getScore()
    scorer2 = Scorer(timestamps, predictions, labels, windows, costMatrixFP,
      probationaryPeriod=0)
    (_, score2) = scorer2.getScore()
    
    self.assertEqual(score1, 0.5*score2)
    self._checkCounts(scorer1.counts, length-windowSize*numWindows-1, 0, 1, 0)
    self._checkCounts(scorer2.counts, length-windowSize*numWindows-1, 0, 1, 0)

  def testScoringAllMetrics(self):
    """
    This tests an example set of detections, where all metrics have counts > 0.
    """
    start = datetime.datetime.now()
    increment = datetime.timedelta(minutes=5)
    length = 100
    numWindows = 2
    windowSize = 5
    threshold = 0.5
    
    timestamps = generateTimestamps(start, increment, length)
    windows = generateWindows(timestamps, numWindows, windowSize)
    anomalyScores = pandas.Series([0]*length)
    
    index = timestamps[timestamps == windows[0][0]].index[0]
    # TP, add'l TP, and FP
    anomalyScores[index] = 1
    anomalyScores[index+1] = 1
    anomalyScores[index+7] = 1

    sweeper = Sweeper(probationPercent=0, costMatrix=self.costMatrix)

    (scores, matchingRow) = sweeper.scoreDataSet(
      timestamps,
      anomalyScores,
      windows,
      "testData",
      threshold
    )
    
    self.assertAlmostEquals(matchingRow.score, -0.9540, 4)
    self._checkCounts(matchingRow, length-windowSize*numWindows-1, 2, 1, 8)

  def test_FourFalseNegatives(self):
    """
    A false negative with four windows should have exactly four times
    the negative of the false negative score.
    """
    start = datetime.datetime.now()
    increment = datetime.timedelta(minutes=5)
    length = 2000
    numWindows = 4
    windowSize = 10

    timestamps = generateTimestamps(start, increment, length)
    predictions = pandas.Series([0]*length)
    windows = generateWindows(timestamps, numWindows, windowSize)
    labels = generateLabels(timestamps, windows)

    costMatrix = {"tpWeight": 1.0,
                  "fnWeight": 2.0,
                  "fpWeight": 3.0,
                  "tnWeight": 4.0}

    scorer = Scorer(timestamps, predictions, labels, windows, costMatrix,
      probationaryPeriod=0)

    self.assertTrue(abs(scorer.getScore() + 4*costMatrix['fnWeight']) < 0.01)

    # Ensure counts are correct.
    self.assertEqual(scorer.counts['tn'], length-windowSize*numWindows)
    self.assertEqual(scorer.counts['tp'], 0)
    self.assertEqual(scorer.counts['fp'], 0)
    self.assertEqual(scorer.counts['fn'], windowSize*numWindows)

  def testBucketMerge(self):
    data = pandas.DataFrame({"timestamp" :
      generateTimestamps(strp("2015-12-01"),
      datetime.timedelta(days=1), 31)})
    dataFileName = "test_data_file.csv"
    writeCorpus(self.tempCorpusPath, {dataFileName : data})

    rawLabels = (["2015-12-24 00:00:00",
                  "2015-12-31 00:00:00"],
                 ["2015-12-01 00:00:00",
                  "2015-12-25 00:00:00",
                  "2015-12-31 00:00:00"],
                 ["2015-12-25 00:00:00"])

    for i, labels in enumerate(rawLabels):
      labelsPath = self.tempCorpusLabelPath.replace(
        os.path.sep+"label.json", os.path.sep+"raw"+os.path.sep+"label{}.json".format(i))
      writeCorpusLabel(labelsPath, {"test_data_file.csv": labels})
    labelsDir = labelsPath.replace(os.path.sep+"label{}.json".format(i), "")

    corpus = nab.corpus.Corpus(self.tempCorpusPath)
    labelCombiner = nab.labeler.LabelCombiner(
      labelsDir, corpus, 0.5, 0.10, 0.15, 0)
    labelCombiner.getRawLabels()
    labelTimestamps, _ = labelCombiner.combineLabels()

    expectedLabels = ['2015-12-25 00:00:00', '2015-12-31 00:00:00']
    self.assertEqual(expectedLabels, labelTimestamps[dataFileName],
      "The combined labels did not bucket and merge as expected.")

  def testFourFalseNegatives(self):
    """
    A false negative with four windows should have exactly four times
    the negative of the false negative score.
    """
    start = datetime.datetime.now()
    increment = datetime.timedelta(minutes=5)
    length = 2000
    numWindows = 4
    windowSize = 10

    timestamps = generateTimestamps(start, increment, length)
    windows = generateWindows(timestamps, numWindows, windowSize)
    anomalyScores = pandas.Series([0] * length)
    threshold = 1

    sweeper = Sweeper(probationPercent=0, costMatrix=self.costMatrix)
    (scores, matchingRow) = sweeper.scoreDataSet(
      timestamps,
      anomalyScores,
      windows,
      "testData",
      threshold
    )

    self.assertEqual(matchingRow.score, 4 * -self.costMatrix["fnWeight"])
    self._checkCounts(matchingRow, length - windowSize * numWindows, 0, 0,
                      windowSize * numWindows)

  def test_falsePositiveMeansNegativeScore(self):
    """
    A false positive should make the score negative.
    """
    start = datetime.datetime.now()
    increment = datetime.timedelta(minutes=5)
    length = 1000
    numWindows = 1
    windowSize = 10

    timestamps = generateTimestamps(start, increment, length)
    predictions = pandas.Series([0]*length)
    windows = generateWindows(timestamps, numWindows, windowSize)
    labels = generateLabels(timestamps, windows)

    costMatrix = {"tpWeight": 1.0,
    "fnWeight": 1.0,
    "fpWeight": 1.0,
    "tnWeight": 1.0}

    predictions[0] = 1

    scorer = Scorer(timestamps, predictions, labels, windows, costMatrix,
      probationaryPeriod=0)
    score = scorer.getScore()

    self.assertTrue(score < 0)

    # Ensure counts are correct.
    self.assertEqual(scorer.counts['tn'], length-windowSize*numWindows-1)
    self.assertEqual(scorer.counts['tp'], 0)
    self.assertEqual(scorer.counts['fp'], 1)
    self.assertEqual(scorer.counts['fn'], windowSize*numWindows)

  def test_earlierFalsePositiveAfterWindowIsBetter(self):
    """Imagine there are two false positives A and B that both occur right after
    a window. If A occurs earlier than B, then the score change due to A will be
    less than the score change due to B.
    """
    start = datetime.datetime.now()
    increment = datetime.timedelta(minutes=5)
    length = 10
    numWindows = 1
    windowSize = 2

    timestamps = generateTimestamps(start, increment, length)

    predictions1 = pandas.Series([0]*length)
    predictions2 = pandas.Series([0]*length)

    windows = generateWindows(timestamps, numWindows, windowSize)
    labels = generateLabels(timestamps, windows)

    window = windows[0]
    t1, t2 = window

    costMatrix = {"tpWeight": 1.0,
                  "fnWeight": 1.0,
                  "fpWeight": 1.0,
                  "tnWeight": 1.0}

    index1 = timestamps[timestamps == t2].index[0] + 1
    predictions1[index1] = 1

    scorer1 = Scorer(timestamps, predictions1, labels, windows, costMatrix,
      probationaryPeriod=0)
    score1 = scorer1.getScore()

    predictions2[index1+1] = 1

    scorer2 = Scorer(timestamps, predictions2, labels, windows, costMatrix,
      probationaryPeriod=0)
    score2 = scorer2.getScore()

    self.assertTrue(score1 > score2)

    # Ensure counts are correct.
    self.assertEqual(scorer1.counts['tn'], length-windowSize*numWindows-1)
    self.assertEqual(scorer1.counts['tp'], 0)
    self.assertEqual(scorer1.counts['fp'], 1)
    self.assertEqual(scorer1.counts['fn'], windowSize*numWindows)

    self.assertEqual(scorer2.counts['tn'], length-windowSize*numWindows-1)
    self.assertEqual(scorer2.counts['tp'], 0)
    self.assertEqual(scorer2.counts['fp'], 1)
    self.assertEqual(scorer2.counts['fn'], windowSize*numWindows)

  def test_twoFalsePositivesIsWorseThanOne(self):
    """False positives have an additive effect on the score. If there are two
    false positives, A and B, in a file, then the score given A and B should be
    larger than the score given just A.
    """
    start = datetime.datetime.now()
    increment = datetime.timedelta(minutes=5)
    length = 1000
    numWindows = 1
    windowSize = 10

    timestamps = generateTimestamps(start, increment, length)
    predictions = pandas.Series([0]*length)
    windows = generateWindows(timestamps, numWindows, windowSize)
    labels = generateLabels(timestamps, windows)


    costMatrix = {"tpWeight": 1.0,
    "fnWeight": 1.0,
    "fpWeight": 1.0,
    "tnWeight": 1.0}

    predictions[0] = 1

    scorer1 = Scorer(timestamps, predictions, labels, windows, costMatrix,
      probationaryPeriod=0)

    score1 = scorer1.getScore()


    predictions[1] = 1

    scorer2 = Scorer(timestamps, predictions, labels, windows, costMatrix,
      probationaryPeriod=0)

    score2 = scorer2.getScore()

    self.assertTrue(score1 > score2)

    # Ensure counts are correct.
    self.assertEqual(scorer1.counts['tn'], length-windowSize*numWindows-1)
    self.assertEqual(scorer1.counts['tp'], 0)
    self.assertEqual(scorer1.counts['fp'], 1)
    self.assertEqual(scorer1.counts['fn'], windowSize*numWindows)

    self.assertEqual(scorer2.counts['tn'], length-windowSize*numWindows-2)
    self.assertEqual(scorer2.counts['tp'], 0)
    self.assertEqual(scorer2.counts['fp'], 2)
    self.assertEqual(scorer2.counts['fn'], windowSize*numWindows)