def test_fill_should_not_override_destination(self):
        testdb = "test-%s" % self.db
        self._removedb()

        try:
            os.unlink(testdb)
        except (IOError, OSError):
            pass

        schema = [(1, 20)]
        data = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10,
                11, 12, 13, 14, 15, 16, 17, 18, 19, 20]

        end = int(time.time()) + schema[0][0]
        start = end - (schema[0][1] * schema[0][0])
        times = range(start, end, schema[0][0])

        override_data = zip(times, data)

        emptyData = [data]
        startTime = time.time()
        self._createdb(self.db, schema)
        self._createdb(testdb, schema, override_data)

        fill_archives(self.db, testdb, startTime)

        original_data = whisper.fetch(self.db, 0)
        filled_data = whisper.fetch(testdb, 0)
        self.assertEqual(data, filled_data[1])

def benchmark_create_update_fetch():
	path, archive_list, tear_down = set_up_create()
	# start timer
	start_time = time.clock()
	for i in range(100):
		whisper.create(path, archive_list)

		seconds_ago = 3500
		current_value = 0.5
		increment = 0.2
		now = time.time()
		# file_update closes the file so we have to reopen every time
		for i in range(seconds_ago):
			whisper.update(path, current_value, now - seconds_ago + i)
			current_value += increment

		from_time = now - seconds_ago
		until_time = from_time + 1000

		whisper.fetch(path, from_time, until_time)
		tear_down()

	# end timer
	end_time = time.clock()
	elapsed_time = end_time - start_time

	print "Executed 100 iterations in %ss (%i ns/op)" % (elapsed_time, (elapsed_time * 1000 * 1000 * 1000) / 100)

    def test_fill_should_handle_gaps(self):
        testdb = "test-%s" % self.db
        self._removedb()

        try:
            os.unlink(testdb)
        except (IOError, OSError):
            pass

        schema = [(1, 20)]
        complete = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10,
                    11, 12, 13, 14, 15, 16, 17, 18, 19, 20]
        holes = [1, 2, 3, 4, 5, 6, None, None, None, None,
                 11, 12, 13, 14, 15, 16, 17, 18, 19, 20]

        end = int(time.time()) + schema[0][0]
        start = end - (schema[0][1] * schema[0][0])
        times = range(start, end, schema[0][0])

        complete_data = zip(times, complete)

        holes_data = [t for t in zip(times, holes) if t[1] is not None]
        self._createdb(self.db, schema, complete_data)
        self._createdb(testdb, schema, holes_data)

        fill_archives(self.db, testdb, time.time())

        original_data = whisper.fetch(self.db, 0)
        filled_data = whisper.fetch(testdb, 0)
        self.assertEqual(original_data, filled_data)

    def test_fetch(self):
        """fetch info from database """

        # check a db that doesnt exist
        with self.assertRaises(Exception):
            whisper.fetch("this_db_does_not_exist", 0)

        # SECOND MINUTE HOUR DAY
        retention = [(1, 60), (60, 60), (3600, 24), (86400, 365)]
        whisper.create(self.db, retention)

        # check a db with an invalid time range
        with self.assertRaises(whisper.InvalidTimeInterval):
            whisper.fetch(self.db, time.time(), time.time()-6000)

        fetch = whisper.fetch(self.db, 0)

        # check time range
        self.assertEqual(fetch[0][1] - fetch[0][0],
                         retention[-1][0] * retention[-1][1])

        # check number of points
        self.assertEqual(len(fetch[1]), retention[-1][1])

        # check step size
        self.assertEqual(fetch[0][2], retention[-1][0])

        self._removedb()

    def test_fetch(self):
        """
        fetch info from database
        """
        # Don't use AssertRaisesException due to a super obscure bug in
        # python2.6 which returns an IOError in the 2nd argument of __exit__
        # in a context manager as a tuple. See this for a minimal reproducer:
        #    http://git.io/cKz30g
        with self.assertRaises(IOError):
            # check a db that doesnt exist
            whisper.fetch("this_db_does_not_exist", 0)

        # SECOND MINUTE HOUR DAY
        retention = [(1, 60), (60, 60), (3600, 24), (86400, 365)]
        whisper.create(self.filename, retention)

        # check a db with an invalid time range
        now = int(time.time())
        past = now - 6000

        msg = "Invalid time interval: from time '{0}' is after until time '{1}'"
        with AssertRaisesException(whisper.InvalidTimeInterval(msg.format(now, past))):
            whisper.fetch(self.filename, now, past)

        fetch = whisper.fetch(self.filename, 0)

        # check time range
        self.assertEqual(fetch[0][1] - fetch[0][0],
                         retention[-1][0] * retention[-1][1])

        # check number of points
        self.assertEqual(len(fetch[1]), retention[-1][1])

        # check step size
        self.assertEqual(fetch[0][2], retention[-1][0])

    def test_update_single_archive(self):
        """
        Update with a single leveled archive
        """
        retention_schema = [(1, 20)]
        data = self._update(schema=retention_schema)
        # fetch the data
        fetch = whisper.fetch(self.filename, 0)   # all data
        fetch_data = fetch[1]

        for i, (timestamp, value) in enumerate(data):
            # is value in the fetched data?
            self.assertEqual(value, fetch_data[i])

        # check TimestampNotCovered
        with AssertRaisesException(
                whisper.TimestampNotCovered(
                    'Timestamp not covered by any archives in this database.')):
            # in the futur
            whisper.update(self.filename, 1.337, time.time() + 1)

        with AssertRaisesException(
                whisper.TimestampNotCovered(
                    'Timestamp not covered by any archives in this database.')):
            # before the past
            whisper.update(self.filename, 1.337, time.time() - retention_schema[0][1] - 1)

        # When no timestamp is passed in, it should use the current time
        original_lock = whisper.LOCK
        whisper.LOCK = True
        whisper.update(self.filename, 3.7337, None)
        fetched = whisper.fetch(self.filename, 0)[1]
        self.assertEqual(fetched[-1], 3.7337)

        whisper.LOCK = original_lock

    def test_fill_endat(self, unused_mock_time):
        dst_db = "dst-%s" % self.db
        self._removedb()
        try:
            os.unlink(dst_db)
        except (IOError, OSError):
            pass

        complete = range(1, 21)
        seconds_per_point = 1
        seconds_per_point_l2 = seconds_per_point * 4
        points_number = len(complete)
        schema = [(seconds_per_point, points_number),
                  (seconds_per_point_l2, points_number),
        ]
        empty_data = []

        end = int(time.time()) + seconds_per_point
        start = end - (points_number * seconds_per_point)
        times = range(start, end, seconds_per_point)

        complete_data = zip(times, complete)
        self._createdb(self.db, schema, complete_data)
        self._createdb(dst_db, schema, empty_data)

        quarter = points_number // 4
        half = points_number // 2
        three_quarter =  points_number * 3 // 4

        # fills a fourth of data, from 2/4th to 3/4th
        fill_archives(self.db, dst_db, time.time()-quarter, time.time()-half)
        quarter_filled_data = whisper.fetch(dst_db, start-seconds_per_point)[1]
        expected = [None]*half + complete[half:three_quarter] + [None]*quarter
        self.assertEqual(expected, quarter_filled_data)
        # Fetching data older than start forces the use of the second level of aggregation
        # We get a first empty cell and then
        quarter_filled_data_l2 = whisper.fetch(dst_db, 0)[1]
        average_l1 = _average(quarter_filled_data)
        average_l2 = _average(quarter_filled_data_l2)
        self.assertEqual(average_l1, average_l2)

        # fills a half of data, from 2/4th to 4/4th
        fill_archives(self.db, dst_db, time.time(), time.time()-half)
        half_filled_data = whisper.fetch(dst_db, start-seconds_per_point)[1]
        expected = [None]*half + complete[half:]
        self.assertEqual(expected, half_filled_data)

        # Explicitly passes the default value of endAt=now (excluded)
        fill_archives(self.db, dst_db, time.time(), endAt=0)
        filled_data = whisper.fetch(dst_db, start-seconds_per_point)[1]
        self.assertEqual(complete[:-1], filled_data[:-1])
        self.assertEqual(filled_data[-1], None)

    def test_heal_target_corrupt(self):
        testdb = "/dev/null"
        self._removedb()

        schema = [(1, 20)]
        self._createdb(self.db, schema)
        original_data = whisper.fetch(self.db, 0)

        # This should log complaints but exit successfully as it cannot
        # heal its target /dev/null
        heal_metric(self.db, testdb)
        data = whisper.fetch(self.db, 0)
        self.assertEqual(original_data, data)

def fill_archives(src, dst, startFrom):
    header = whisper.info(dst)
    archives = header['archives']
    archives = sorted(archives, key=lambda t: t['retention'])

    for archive in archives:
        fromTime = time.time() - archive['retention']
        if fromTime >= startFrom:
            continue

        (timeInfo, values) = whisper.fetch(dst, fromTime, startFrom)
        (start, end, step) = timeInfo
        gapstart = None
        for v in values:
            if not v and not gapstart:
                gapstart = start
            elif v and gapstart:
                # ignore single units lost
                if (start - gapstart) > archive['secondsPerPoint']:
                    fill(src, dst, gapstart - step, start)
                gapstart = None
            elif gapstart and start == end - step:
                fill(src, dst, gapstart - step, start)

            start += step

        startFrom = fromTime

def getReadings(uuid, start, end):
    assert time()-int(start) >= 60
    dataFile = os.path.join(WHISPER_DATA,str(uuid) + ".wsp")
    try:
        (timeInfo, values) = whisper.fetch(dataFile, start, end)
    except whisper.WhisperException, exc:
      raise SystemExit('[ERROR] %s' % str(exc))

    def fetch(self, from_time, until_time=None):
        """
        This method fetch data from the database according to the period
        given

        fetch(path, fromTime, untilTime=None)

        fromTime is an datetime
        untilTime is also an datetime, but defaults to now.

        Returns a tuple of (timeInfo, valueList)
        where timeInfo is itself a tuple of (fromTime, untilTime, step)

        Returns None if no data can be returned
        """
        until_time = until_time or datetime.now()
        time_info, values = whisper.fetch(self.path,
                                          from_time.strftime('%s'),
                                          until_time.strftime('%s'))
        # build up a list of (timestamp, value)
        start_time, end_time, step = time_info
        current = start_time
        times = []
        while current <= end_time:
            times.append(current)
            current += step
        return zip(times, values)

def fetch(args):
    now = int(time.time())
    yesterday = now - (24 * 60 * 60)

    if args.from_time:
        from_time = time.mktime(
            dt.datetime.strptime(args.from_time, FORMAT).timetuple())
    else:
        from_time = now

    if args.until_time:
        until_time = time.mktime(
            dt.datetime.strptime(args.until_time, FORMAT).timetuple())
    else:
        until_time = yesterday

    times, values = whisper.fetch(args.file, from_time, until_time)

    if args.drop:
        fn = _DROP_FUNCTIONS.get(args.drop)
        values = [x for x in values if fn(x)]

    start, end, step = times

    t = start
    for value in values:

        if args.time_format:
            s = time.strftime(args.time_format, time.localtime(t))
        else:
            s = time.ctime(t)

        print("%s\t%f" % (s, value))

        t += step

    def extract_data(self, metric, start, end=None):
        f = self.map_metric_to_files(metric)
        if not f:
            return None

        fetched = whisper.fetch(f[0], start, end)
        return fetched

  def fetch(self, startTime, endTime):
    data = whisper.fetch(self.fs_path, startTime, endTime)
    if not data:
      return None

    time_info, values = data
    (start,end,step) = time_info

    # Merge in data from carbon's cache
    try:
      cached_datapoints = CarbonLink.query(self.real_metric_path)
    except:
      log.exception("Failed CarbonLink query '%s'" % self.real_metric_path)
      cached_datapoints = []

    for (timestamp, value) in cached_datapoints:
      interval = timestamp - (timestamp % step)

      try:
        i = int(interval - start) / step
        values[i] = value
      except:
        pass

    return (time_info, values)

    def test_heal_mixed_data(self):
        testdb = "test-%s" % self.db
        self._removedb()

        try:
            os.unlink(testdb)
        except (IOError, OSError):
            pass

        schema = [(1, 20)]
        have = [1, 2, 3, None, 5, 6, 7, 8, 9, 10,
                    11, 12, 13, 14, 15, None, 17, 18, 19, None]
        remote = [1, 2, 3, 4, 5, 6, None, None, None, None,
                 11, 12, 13, 14, 15, 16, 17, 18, None, 20]

        end = int(time.time()) + schema[0][0]
        start = end - (schema[0][1] * schema[0][0])
        times = range(start, end, schema[0][0])

        have_data = [t for t in zip(times, have) if t[1] is not None]
        remote_data = [t for t in zip(times, remote) if t[1] is not None]

        self._createdb(self.db, schema, remote_data)
        self._createdb(testdb, schema, have_data)

        heal_metric(self.db, testdb, overwrite=True)

        final_data = whisper.fetch(testdb, 0)
        self.assertEqual(final_data[1], list(range(1,21)))

  def fetch(self, startTime, endTime):
    try:
      data = whisper.fetch(self.fs_path, startTime, endTime)
    except IOError:
      log.exception("Failed fetch of whisper file '%s'" % self.fs_path)
      return None
    if not data:
      return None

    time_info, values = data
    (start,end,step) = time_info

    meta_info = whisper.info(self.fs_path)
    aggregation_method = meta_info['aggregationMethod']
    lowest_step = min([i['secondsPerPoint'] for i in meta_info['archives']])
    # Merge in data from carbon's cache
    cached_datapoints = []
    try:
      cached_datapoints = CarbonLink().query(self.real_metric_path)
    except:
      log.exception("Failed CarbonLink query '%s'" % self.real_metric_path)
      cached_datapoints = []

    if isinstance(cached_datapoints, dict):
      cached_datapoints = cached_datapoints.items()

    values = merge_with_cache(cached_datapoints,
                              start,
                              step,
                              values,
                              aggregation_method)

    return time_info, values

  def fetch(self, startTime, endTime):
    data = whisper.fetch(self.fs_path, startTime, endTime)
    if not data:
      return None
    consolidationFunc = ""
    whisper_info = whisper.info(self.fs_path)
    if "aggregationMethod" in whisper_info:
      aggregationMethod = whisper_info["aggregationMethod"]
      if aggregationMethod == 'min' or aggregationMethod == 'max':
        consolidationFunc = aggregationMethod
    time_info, values = data
    (start,end,step) = time_info

    # Merge in data from carbon's cache
    try:
      cached_datapoints = CarbonLink.query(self.real_metric_path)
    except:
      log.exception("Failed CarbonLink query '%s'" % self.real_metric_path)
      cached_datapoints = []

    for (timestamp, value) in cached_datapoints:
      interval = timestamp - (timestamp % step)

      try:
        i = int(interval - start) / step
        values[i] = value
      except:
        pass

    return (time_info, values, consolidationFunc)

    def fetch(self, startTime, endTime):
        data = whisper.fetch(self.fs_path, startTime, endTime)
        if not data:
            return None

        time_info, values = data
        (start, end, step) = time_info

        meta_info = whisper.info(self.fs_path)
        lowest_step = min([i['secondsPerPoint'] for i in meta_info['archives']])
        # Merge in data from carbon's cache
        cached_datapoints = []
        try:
            if step == lowest_step:
                cached_datapoints = CarbonLink.query(self.real_metric_path)
        except:
            log.exception("Failed CarbonLink query '%s'" % self.real_metric_path)
            cached_datapoints = []

        if isinstance(cached_datapoints, dict):
            cached_datapoints = cached_datapoints.items()

        for (timestamp, value) in cached_datapoints:
            interval = timestamp - (timestamp % step)

            try:
                i = int(interval - start) / step
                values[i] = value
            except:
                pass

        return (time_info, values)

    def test_update_many_excess(self):
        # given an empty db
        wsp = "test_update_many_excess.wsp"
        self.addCleanup(self._remove, wsp)
        archive_len = 3
        archive_step = 1
        whisper.create(wsp, [(archive_step, archive_len)])

        # given too many points than the db can hold
        excess_len = 1
        num_input_points = archive_len + excess_len
        test_now = int(time.time())
        input_start = test_now - num_input_points + archive_step
        input_points = [(input_start + i, random.random() * 10)
                        for i in range(num_input_points)]

        # when the db is updated with too many points
        whisper.update_many(wsp, input_points, now=test_now)

        # then only the most recent input points (those at the end) were written
        actual_time_info = whisper.fetch(wsp, 0, now=test_now)[0]
        self.assertEqual(actual_time_info,
                         (input_points[-archive_len][0],
                          input_points[-1][0] + archive_step,  # untilInterval = newest + step
                          archive_step))

    def fetch(self, startTime, endTime):
        data = whisper.fetch(self.fs_path, startTime, endTime)
        if not data:
            return None

        time_info, values = data
        start, end, step = time_info
        return (time_info, values)