def _populate_data(self):
     self.db = os.path.join(settings.WHISPER_DIR, 'test.wsp')
     whisper.create(self.db, [(1, 60)])
     ts = int(time.time())
     for i, value in enumerate(reversed(self._test_data)):
         whisper.update(self.db, value, ts - i)
     self.ts = ts

def benchmark_create_update_fetch():
	path, archive_list, tear_down = set_up_create()
	# start timer
	start_time = time.clock()
	for i in range(100):
		whisper.create(path, archive_list)

		seconds_ago = 3500
		current_value = 0.5
		increment = 0.2
		now = time.time()
		# file_update closes the file so we have to reopen every time
		for i in range(seconds_ago):
			whisper.update(path, current_value, now - seconds_ago + i)
			current_value += increment

		from_time = now - seconds_ago
		until_time = from_time + 1000

		whisper.fetch(path, from_time, until_time)
		tear_down()

	# end timer
	end_time = time.clock()
	elapsed_time = end_time - start_time

	print "Executed 100 iterations in %ss (%i ns/op)" % (elapsed_time, (elapsed_time * 1000 * 1000 * 1000) / 100)

 def save(self, value, timestamp, lenient=False):
     logger.debug("Saving %s: %f" % (self.name, value))
     try:
         whisper.update(self.path, value, timestamp)
     except whisper.TimestampNotCovered as exc:
         # The timestamp we were given is either "in the future" (perhaps
         # because our own clock is delayed) or "before the time the
         # database remembers". If we're willing to fudge the timestamp,
         # check whether the difference is less than the configured
         # epsilon for clock drift. If it is, then try saving the value
         # again using a timestamp from one second earlier than reported.
         # If that's still not accepted, a new (unhandled) TimestampNot-
         # Covered exception will be raised for the caller to handle.
         statsd.incr('error.timestampnotcovered')
         if lenient:
             delta = timestamp - time.time() # in seconds
             statsd.timing('timestamp_drift', delta * 1000) # report in ms
             if abs(delta) < settings.drift_epsilon:
                 # Ensure lenient is set to False for the next attempt so
                 # that we don't end up in a loop
                 self.save(value, timestamp-1, lenient=False)
                 # Report only successful lenient saves
                 statsd.incr('lenient_save')
         else:
             raise

    def record_metering_data(self, data):

        record = copy.deepcopy(data)

        timestamp = record["timestamp"].replace(second=0, microsecond=0)
        timestamp = int((timestamp - datetime.datetime(1970, 1, 1)).total_seconds())
        value = float(record["counter_volume"])

        record_path = (
            env_variables["whisper_path"] + data["resource_id"] + "_" + data["counter_name"].replace(".", "_") + ".wsp"
        )
        # if not os.path.isdir(os.path.dirname(record_path)):
        #     os.makedirs(os.path.dirname(record_path))

        if not os.path.isfile(record_path):
            whisper.create(record_path, archieve_list)

        whisper.update(record_path, value, timestamp)

        # add resource & meter to sqlite db
        conn = sqlite3.connect(env_variables["sql_db_path"])
        c = conn.cursor()
        c.execute("select count(*) from resources where resource_id=?", (data["resource_id"],))
        r = c.fetchone()[0]
        if r == 0:
            c.execute(
                "insert into resources (resource_id, user_id, project_id, source_id, resource_metadata)"
                + "values (?,?,?,?,?)",
                (
                    data["resource_id"],
                    data["user_id"],
                    data["project_id"],
                    data["source"],
                    json.dumps(data["resource_metadata"]),
                ),
            )

        c.execute(
            "select count(*) from meters where name=? and resource_id=?", (data["counter_name"], data["resource_id"])
        )
        r = c.fetchone()[0]
        if r == 0:
            c.execute(
                "insert into meters (name, type, unit, resource_id, project_id, user_id, source)"
                + "values (?,?,?,?,?,?,?)",
                (
                    data["counter_name"],
                    data["counter_type"],
                    data["counter_unit"],
                    data["resource_id"],
                    data["project_id"],
                    data["user_id"],
                    data["source"],
                ),
            )

        conn.commit()
        conn.close()

def store(file, data):
	print "store"
	import whisper
	try:
		whisper.create(file, [(1,60*60*24), (10, 60*60*24), (60, 60*60*24*30) ])
	except:
		pass
	print "update"
	whisper.update(file, data)

    def _write_data(self, hostID, plugin, data_json):

        # For each plugin
        if plugin == None:
            return False

        data =  self.jsonToPython(data_json)
        data_timestamp = data.get('TimeStamp')

        # TODO dont write data if no infos

        # Get data_path
        data_path = self._redis_connexion.redis_hget("WSP_PATH", hostID)
        if data_path is None:
            return False

        wspPath = '%s/%s' % (data_path, plugin)
        for ds_name, value in data.get("Values", {}).iteritems():

            ds_path = '%s/%s.wsp' % (wspPath, ds_name)
            # Create wsp file - config wsp here
            if not os.path.isfile(ds_path):
                self._logger.warning("writewsp host : %s Create wsp file : %s"
                                        % (hostID, ds_path))
                # Create directory
                if not os.path.exists(wspPath):
                    try:
                        os.makedirs(wspPath)
                        self._logger.info("writewsp host : %s make directory : %s"
                                            % (hostID, wspPath))
                    except OSError:
                        self._logger.error("writewsp host : %s can't make directory : %s"
                                            % (hostID, wspPath))
                        continue
                try:
                    whisper.create(ds_path,
                                   [(60, 1440),   # --- Daily (1 Minute Average)
                                   (300, 2016),   # --- Weekly (5 Minute Average)
                                   (600, 4608),   # --- Monthly (10 Min Average)
                                   (3600, 8784)]) # --- Yearly (1 Hour Average)
                except Exception as e:
                    self._logger.error("writewsp host : %s Create wsp Error %s"
                        % (hostID, str(e)))
                    continue
            # Update whisper
            try:
                self._logger.debug("writewsp host : %s Update wsp "
                                   "Timestamp %s For value %s in file %s"
                                    % (hostID, data_timestamp, value, ds_path))
                whisper.update(ds_path, str(value), str(data_timestamp) )
            except Exception as e:
                self._logger.error("writewsp host : %s Update Error %s - %s"
                                    % (hostID, ds_path, e))
                continue
        self._dataNumber += 1
        return True

 def test_01_add_point(self):
     """
     Add a point and check the created time range.
     """
     now = int(time.time())
     whisper.update(FILENAME, 1234, timestamp=now)
     (fromInterval, toInterval, step), points = whisper.fetch(FILENAME, 0, None)
     now = now - (now % 60)
     # The upper bound is (roughly) 'now'.
     self.assertEqual(toInterval, now + SECONDS_PER_POINT)
     # The lower bound is (roughly) now minus the covered time.
     self.assertEqual(fromInterval, now - (NUMBER_OF_POINTS - 1) * SECONDS_PER_POINT)

    def _update(self, datapoints):
        """
        This method store in the datapoints in the current database.

            :datapoints: is a list of tupple with the epoch timestamp and value
                 [(1368977629,10)]
        """
        if len(datapoints) == 1:
            timestamp, value = datapoints[0]
            whisper.update(self.path, value, timestamp)
        else:
            whisper.update_many(self.path, datapoints)

def update(path, datapoints):
    nrOfPoints = len(datapoints),
    if nrOfPoints == 1:
        (timestamp, value) = datapoints[0]
        timestamp = timegm(timestamp.timetuple())
        whisper.update(path, value, timestamp)
    elif nrOfPoints > 1:
        whisper.update_many(path + '.wsp', [
            (timegm(t.timetuple()), v) for (t,v) in datapoints])
    else:
        raise Exception("No Datapoint given")

    return True

    def update(self, metric, dataline):
        data = dataline.split()
        metric = os.path.join(self.root, metric)
        metric += '.wsp'

        if not os.path.exists(metric):
            # create this database
            mkdirs(os.path.dirname(metric))
            whisper.create(metric, [(METRIC_SEC_PER_POINT, METRIC_POINTS_TO_STORE)])

        value = float(data[0])
        timestamp = float(data[1])

        whisper.update(metric, value, timestamp)

    def create_whisper_hosts(self, ts=None):
        worker1 = self.hostcpu.replace('hostname', 'worker1')
        worker2 = self.hostcpu.replace('hostname', 'worker2')
        try:
            os.makedirs(worker1.replace('cpu.wsp', ''))
            os.makedirs(worker2.replace('cpu.wsp', ''))
        except OSError:
            pass

        whisper.create(worker1, [(1, 60)])
        whisper.create(worker2, [(1, 60)])

        ts = ts or int(time.time())
        whisper.update(worker1, 1, ts)
        whisper.update(worker2, 2, ts)

    def test_diff(self):
        testdb = "test-%s" % self.filename

        now = time.time()

        whisper.create(testdb, self.retention)
        whisper.create(self.filename, self.retention)
        whisper.update(testdb, 1.0, now)
        whisper.update(self.filename, 2.0, now)

        results = whisper.diff(testdb, self.filename)
        self._remove(testdb)

        expected = [(0, [(int(now), 1.0, 2.0)], 1), (1, [], 0)]

        self.assertEqual(results, expected)

    def _update(self, wsp=None, schema=None, sparse=False, useFallocate=False):
        wsp = wsp or self.filename
        schema = schema or [(1, 20)]

        num_data_points = 20

        # create sample data
        whisper.create(wsp, schema, sparse=sparse, useFallocate=useFallocate)
        tn = time.time() - num_data_points
        data = []
        for i in range(num_data_points):
            data.append((tn + 1 + i, random.random() * 10))

        # test single update
        whisper.update(wsp, data[0][1], data[0][0])

        # test multi update
        whisper.update_many(wsp, data[1:])
        return data

    def _update(self, wsp=None, schema=None):
        wsp = wsp or self.db
        schema = schema or [(1, 20)]
        num_data_points = 20

        whisper.create(wsp, schema)

        # create sample data
        tn = time.time() - num_data_points
        data = []
        for i in range(num_data_points):
            data.append((tn + 1 + i, random.random() * 10))

        # test single update
        whisper.update(wsp, data[0][1], data[0][0])

        # test multi update
        whisper.update_many(wsp, data[1:])
        return data

    def test_file_diff(self):
        testdb = "test-%s" % self.filename

        now = time.time()

        whisper.create(testdb, self.retention)
        whisper.create(self.filename, self.retention)
        whisper.update(testdb, 1.0, now)
        whisper.update(self.filename, 2.0, now)

        # Merging 2 archives with different retentions should fail
        with open(testdb, 'rb') as fh_1:
            with open(self.filename, 'rb+') as fh_2:
                results = whisper.file_diff(fh_1, fh_2)
        self._remove(testdb)

        expected = [(0, [(int(now), 1.0, 2.0)], 1), (1, [], 0)]

        self.assertEqual(results, expected)

    def create_whisper_hosts(self):
        self.start_ts = int(time.time())
        try:
            os.makedirs(self.worker1.replace('cpu.wsp', ''))
            os.makedirs(self.worker2.replace('cpu.wsp', ''))
            os.makedirs(self.worker3.replace('cpu.wsp', ''))
            os.makedirs(self.worker4.replace('cpu.wsp.gz', ''))
        except OSError:
            pass

        whisper.create(self.worker1, [(1, 60)])
        whisper.create(self.worker2, [(1, 60)])
        open(self.worker3, 'a').close()

        whisper.update(self.worker1, 1, self.start_ts)
        whisper.update(self.worker2, 2, self.start_ts)

        with open(self.worker1, 'rb') as f_in, gzip.open(self.worker4, 'wb') as f_out:
            shutil.copyfileobj(f_in, f_out)

  def create_whisper_hosts(self):
    worker1 = self.hostcpu.replace('hostname', 'worker1')
    worker2 = self.hostcpu.replace('hostname', 'worker2')
    bogus_file = os.path.join(settings.WHISPER_DIR, 'a/b/c/bogus_file.txt')

    try:
      os.makedirs(worker1.replace('cpu.wsp', ''))
      os.makedirs(worker2.replace('cpu.wsp', ''))
      os.makedirs(bogus_file.replace('bogus_file.txt', ''))
    except OSError:
      pass

    open(bogus_file, 'a').close()
    whisper.create(worker1, [(1, 60)])
    whisper.create(worker2, [(1, 60)])

    ts = int(time.time())
    whisper.update(worker1, 1, ts)
    whisper.update(worker2, 2, ts)

    def test_update_single_archive(self):
        """
        Update with a single leveled archive
        """
        retention_schema = [(1, 20)]
        data = self._update(schema=retention_schema)
        # fetch the data
        fetch = whisper.fetch(self.filename, 0)   # all data
        fetch_data = fetch[1]

        for i, (timestamp, value) in enumerate(data):
            # is value in the fetched data?
            self.assertEqual(value, fetch_data[i])

        # check TimestampNotCovered
        with AssertRaisesException(
                whisper.TimestampNotCovered(
                    'Timestamp not covered by any archives in this database.')):
            # in the futur
            whisper.update(self.filename, 1.337, time.time() + 1)

        with AssertRaisesException(
                whisper.TimestampNotCovered(
                    'Timestamp not covered by any archives in this database.')):
            # before the past
            whisper.update(self.filename, 1.337, time.time() - retention_schema[0][1] - 1)

        # When no timestamp is passed in, it should use the current time
        original_lock = whisper.LOCK
        whisper.LOCK = True
        whisper.update(self.filename, 3.7337, None)
        fetched = whisper.fetch(self.filename, 0)[1]
        self.assertEqual(fetched[-1], 3.7337)

        whisper.LOCK = original_lock

    def test_render_view(self):
        url = reverse('graphite.render.views.renderView')

        response = self.client.get(url, {'target': 'test', 'format': 'json'})
        self.assertEqual(json.loads(response.content), [])
        self.assertTrue(response.has_header('Expires'))
        self.assertTrue(response.has_header('Last-Modified'))
        self.assertTrue(response.has_header('Cache-Control'))

        response = self.client.get(url, {'target': 'test'})
        self.assertEqual(response['Content-Type'], 'image/png')
        self.assertTrue(response.has_header('Expires'))
        self.assertTrue(response.has_header('Last-Modified'))
        self.assertTrue(response.has_header('Cache-Control'))

        self.addCleanup(self.wipe_whisper)
        whisper.create(self.db, [(1, 60)])

        ts = int(time.time())
        whisper.update(self.db, 0.5, ts - 2)
        whisper.update(self.db, 0.4, ts - 1)
        whisper.update(self.db, 0.6, ts)

        response = self.client.get(url, {'target': 'test', 'format': 'json'})
        data = json.loads(response.content)
        end = data[0]['datapoints'][-4:]
        self.assertEqual(
            end, [[None, ts - 3], [0.5, ts - 2], [0.4, ts - 1], [0.6, ts]])

    def test_update_single_archive(self):
        """Update with a single leveled archive"""
        retention_schema = [(1, 20)]
        data = self._update(schema=retention_schema)
        # fetch the data
        fetch = whisper.fetch(self.db, 0)   # all data
        fetch_data = fetch[1]

        for i, (timestamp, value) in enumerate(data):
            # is value in the fetched data?
            self.assertEqual(value, fetch_data[i])

        # check TimestampNotCovered
        with self.assertRaises(whisper.TimestampNotCovered):
            # in the future
            whisper.update(self.db, 1.337, time.time() + 1)
        with self.assertRaises(whisper.TimestampNotCovered):
            # before the past
            whisper.update(self.db, 1.337,
                           time.time() - retention_schema[0][1] - 1)

        self._removedb()