def generate_sample_datasets (host_ips, metric_ids, year, month, day, hour):
    avro_schema = ''
    #load data from hdfs
    cat = subprocess.Popen(['sudo', '-u', 'hdfs', 'hadoop', 'fs', '-cat', '/user/pnda/PNDA_datasets/datasets/.metadata/schema.avsc'], stdout=subprocess.PIPE)
    for line in cat.stdout:
        avro_schema = avro_schema + line
    schema = avro.schema.parse(avro_schema)
    bytes_writer = io.BytesIO()
    encoder = avro.io.BinaryEncoder(bytes_writer)
    #create hdfs folder structure
    dir = create_hdfs_dirs (year, month, day, hour)
    filename = str(uuid.uuid4()) + '.avro'
    filepath = dir + filename
    tmp_file = '/tmp/' + filename
    
    writer = DataFileWriter(open(tmp_file, "w"), DatumWriter(), schema)
    
    start_dt = datetime.datetime(year, month, day, hour, 0, 0) 
    start_ts = int(time.mktime(start_dt.timetuple()))
    end_dt = start_dt.replace(hour=hour+1)
    end_ts = int(time.mktime(end_dt.timetuple()))

    for ts in xrange(start_ts, end_ts, 1):
        #generate random pnda record on per host ip basis
        for host_ip in host_ips:
           record = {}
           record['timestamp'] = (ts * 1000)
           record['src'] = 'test'
           record['host_ip'] = host_ip
           record['rawdata'] = generate_random_metrics(metric_ids)
           #encode avro
           writer.append(record)
    writer.close()
    subprocess.Popen(['sudo', '-u', 'hdfs', 'hadoop', 'fs', '-copyFromLocal', tmp_file, dir])
    return filepath

    def loadAccountInfo(self, user, req):
        validuntil = None
        trafficleft = None
        premium = None

        html = req.load("http://uploading.com/")

        premium = re.search(self.PREMIUM_PATTERN, html) is None

        m = re.search(self.VALID_UNTIL_PATTERN, html)
        if m:
            expiredate = m.group(1).strip()
            self.logDebug("Expire date: " + expiredate)

            try:
                validuntil = time.mktime(time.strptime(expiredate, "%b %d, %Y"))

            except Exception, e:
                self.logError(e)

            else:
                if validuntil > time.mktime(time.gmtime()):
                    premium = True
                else:
                    premium = False
                    validuntil = None

def validate_data(info):

    hack_license,pick_datetime,drop_datetime,n_passengers,trip_dist,pick_long,\
    pick_lat,drop_long,drop_lat,payment_type,fare_amount,\
    surcharge,tip_amount,mta_tax,tolls_amount,total_amount=info

    time_in_seconds = time.mktime(time.strptime(drop_datetime,'%Y-%m-%d %H:%M:%S'))-\
                      time.mktime(time.strptime(pick_datetime,'%Y-%m-%d %H:%M:%S'))
    try:
        pick_long = float(pick_long.strip())
        pick_lat = float(pick_lat.strip())
        drop_long = float(drop_long.strip())
        drop_lat = float(drop_lat.strip())
        trip_dist = float(trip_dist.strip())
        total_amount = float(total_amount.strip())
        n_passengers = int(n_passengers.strip())
    except ValueError:
        sys.stderr.write('CASTING TO FLOATS FAILED')
        return False
    # Is the straight distance shorter than the reported distance?
    euclidean = validate_euclidean(trip_dist,pick_long,pick_lat,drop_long,drop_lat)
    gps_pickup = validate_gps(pick_long,pick_lat) # Are the GPS coordinates present in Manhattan
    gps_dropoff = validate_gps(drop_long,drop_lat)
    distance = validate_distance(trip_dist,pick_long,pick_lat,drop_long,drop_lat) # Are distances too big
    val_time = validate_time(time_in_seconds) # Are times too long or 0? Are they positive?
    velocity = validate_velocity(time_in_seconds,trip_dist) # Is velocity too out of reach
    amount = validate_amount(total_amount)
    pass_validate = validate_passengers(n_passengers)

    return(euclidean and gps_pickup and gps_dropoff and distance and val_time and velocity and amount and pass_validate)

    def add_separator(self, timestamp):
        '''Add whitespace and timestamp between chat sessions.'''
        time_with_current_year = \
            (time.localtime(time.time())[0], ) + \
            time.strptime(timestamp, '%b %d %H:%M:%S')[1:]

        timestamp_seconds = time.mktime(time_with_current_year)
        if timestamp_seconds > time.time():
            time_with_previous_year = \
                (time.localtime(time.time())[0] - 1, ) + \
                time.strptime(timestamp, '%b %d %H:%M:%S')[1:]
            timestamp_seconds = time.mktime(time_with_previous_year)

        message = TextBox(self,
                          style.COLOR_BUTTON_GREY, style.COLOR_BUTTON_GREY,
                          style.COLOR_WHITE, style.COLOR_BUTTON_GREY, False,
                          None, timestamp_to_elapsed_string(timestamp_seconds))
        self._message_list.append(message)
        box = Gtk.HBox()
        align = Gtk.Alignment.new(
            xalign=0.5, yalign=0.0, xscale=0.0, yscale=0.0)
        box.pack_start(align, True, True, 0)
        align.show()
        align.add(message)
        message.show()
        self._conversation.attach(box, 0, self._row_counter, 1, 1)
        box.show()
        self._row_counter += 1
        self.add_log_timestamp(timestamp)
        self._last_msg_sender = None

def log_query(request, requestOptions, requestContext, renderingTime):
  timeRange = requestOptions['endTime'] - requestOptions['startTime']
  logdata = {
    'graphType': requestOptions['graphType'],
    'graphClass': requestOptions.get('graphClass'),
    'format': requestOptions.get('format'),
    'start': int(mktime(requestOptions['startTime'].utctimetuple())),
    'end': int(mktime(requestOptions['endTime'].utctimetuple())),
    'range': timeRange.days * 24 * 3600 + int(round(timeRange.seconds/60.)),
    'localOnly': requestOptions['localOnly'],
    'useCache': 'noCache' not in requestOptions,
    'cachedResponse': requestContext.get('cachedResponse', False),
    'cachedData': requestContext.get('cachedData', False),
    'maxDataPoints': requestOptions.get('maxDataPoints', 0),
    'renderingTime': renderingTime,
  }

  if 'HTTP_X_REAL_IP' in request.META:
    logdata['source'] = request.META['HTTP_X_REAL_IP']
  else:
    logdata['source'] = request.get_host()

  for target,retrievalTime in requestContext['targets']:
    if isinstance(target, list):
      for t in target:
        logdata['target'] = t
        logdata['retrievalTime'] = retrievalTime
        query_log.info(logdata)
    else:
      logdata['target'] = target
      logdata['retrievalTime'] = retrievalTime
      query_log.info(logdata)

def get_feed_entries(feeds):
    entries = []

    for feed in feeds:
        d = feedparser.parse(feed.get('feed_url'))

        for entry in d.entries:
            entry.publisher = feed['title']
            # entry.publisher_icon = feed['icon']

            if 'media_content' in entry:
                if entry.media_content[0]['medium'] == 'image':
                    entry.image = entry.media_content[0]['url']
            elif 'content' in entry:
                soup = BeautifulSoup(entry.content[0]['value'], 'html.parser')
                image = soup.find_all('img')[0]
                entry.image = image.get('src')

            published = datetime.fromtimestamp(mktime(entry.published_parsed))
            updated = datetime.fromtimestamp(mktime(entry.updated_parsed))

            entry.published = published
            entry.updated = updated

            entries.append(entry)

    return sorted(entries, key=attrgetter('published'), reverse=True)

    def _createSearchRequest(self, search=None, tags=None,
                             notebooks=None, date=None,
                             exact_entry=None, content_search=None):

        request = ""
        if notebooks:
            for notebook in tools.strip(notebooks.split(',')):
                if notebook.startswith('-'):
                    request += '-notebook:"%s" ' % tools.strip(notebook[1:])
                else:
                    request += 'notebook:"%s" ' % tools.strip(notebook)

        if tags:
            for tag in tools.strip(tags.split(',')):

                if tag.startswith('-'):
                    request += '-tag:"%s" ' % tag[1:]
                else:
                    request += 'tag:"%s" ' % tag

        if date:
            date = tools.strip(date.split('-'))
            try:
                dateStruct = time.strptime(date[0] + " 00:00:00", "%d.%m.%Y %H:%M:%S")
                request += 'created:%s ' % time.strftime("%Y%m%d", time.localtime(time.mktime(dateStruct)))
                if len(date) == 2:
                    dateStruct = time.strptime(date[1] + " 00:00:00", "%d.%m.%Y %H:%M:%S")
                request += '-created:%s ' % time.strftime("%Y%m%d", time.localtime(time.mktime(dateStruct) + 60 * 60 * 24))
            except ValueError, e:
                out.failureMessage('Incorrect date format in --date attribute. '
                                   'Format: %s' % time.strftime("%d.%m.%Y", time.strptime('19991231', "%Y%m%d")))
                return tools.exitErr()

def __validateArgs():
#===============================================================================
  if len(sys.argv) < 5:
    print "python",sys.argv[0], "CIK ALIAS SINCE UNTIL"
    print "where CIK: one platform client key"
    print "    ALIAS: dataport alias"
    print "    SINCE: MM/DD/YYYY"
    print "    UNTIL: MM/DD/YYYY"
    sys.exit(1)
  cik, alias, since, until = sys.argv[1], sys.argv[2], sys.argv[3], sys.argv[4]
  if len(cik) != 40:
    print "Invalid cik"
    sys.exit(1)
  since = since + " 00:00:00"
  until = until + " 23:59:59"
  try:
    start = datetime.strptime(since, "%m/%d/%Y %H:%M:%S")
    end   = datetime.strptime(until, "%m/%d/%Y %H:%M:%S")
  except ValueError as err:
    print "Invalid time format."
    sys.exit(1)
  start_timestamp = int(time.mktime(start.timetuple()))
  end_timestamp = int(time.mktime(end.timetuple()))
  if start_timestamp > end_timestamp:
    print "SINCE must not be greater than UNTIL"
    sys.exit(1)
  return cik, alias, start_timestamp, end_timestamp

    def __init__(
            self, timestr=None, timezone=LOCALTZ,
            allowpast=True, allowfuture=True):
        """ Converts input to UTC timestamp. """

        if timestr is None:
            timestr = time.time()
        self.timezone = timezone

        if type(timestr) == str:
            self.__timestamp__ = self.__fromstring__(timestr)

        elif type(timestr) in [int, float]:
            self.__timestamp__ = timestr + self.timezone

        elif type(timestr) in [
                datetime.datetime,
                datetime.date,
                datetime.time
            ]:
            self.__timestamp__ = time.mktime(timestr.timetuple())

        elif type(timestr) == time.struct_time:
            self.__timestamp__ = time.mktime(timestr) + self.timezone

        else:
            raise TypeError("Invalid type specified.")

        if not allowpast and self.__timestamp__ < currentutc():
            raise DateRangeError("Values from the past are not allowed.")
        if not allowfuture and self.__timestamp__ > currentutc():
            raise DateRangeError("Values from the future are not allowed.")

 def ger(self):
     query = self.get_argument('q')
     client = tornado.httpclient.AsyncHTTPClient()
     # yield关键字以及tornado.gen.Task对象的一个实例实现函数的调用和参数的传入
     # yield的使用返回程序对Tornado的控制，允许在HTTP请求进行中执行其他任务。
     # HTTP请求完成时，RequestHandler方法在其停止的地方恢复。
     response = yield tornado.gen.Task(client.fetch,
             "http://search.twitter.com/search.json?" + \
                     urllib.urlencode({"q": query, "result_type": "recent", "rpp": 100}))
     body = json.loads(response.body)
     result_count = len(body['results'])
     now = datetime.datetime.utcnow()
     raw_oldest_tweet_at = body['results'][-1]['created_at']
     oldest_tweet_at = datetime.datetime.strptime(raw_oldest_tweet_at,
             "%a, %d %b %Y %H:%M:%S +0000")
     seconds_diff = time.mktime(now.timetuple()) - \
             time.mktime(oldest_tweet_at.timetuple())
     tweets_per_second = float(result_count) / seconds_diff
     self.write("""
     <div style="text-align: center">
         <div style="font-size: 72px">%s</div>
         <div style="font-size: 144px">%.02f</div>
         <div style="font-size: 24px">tweets per second</div>
     </div>""" % (query, tweets_per_second))
     self.finish()

 def findSchedule(self):
     for event in self.new_events:
        is_scheduled = False
        curr_time = self.end_date - event.duration
        while not is_scheduled and not curr_time < self.start_date:
            event.start = curr_time
            event.end = curr_time + event.duration
            is_valid = True
            # check conflicts with current schedule
            for component in self.ical.walk():
                if component.name == 'VEVENT':
                    #try:
                    dc = component.decoded
                    dtstart = time.mktime(time.strptime(str(dc('dtstart')), '%Y-%m-%d %H:%M:%S+00:00'))/60
                    dtend = time.mktime(time.strptime(str(dc('dtend')), '%Y-%m-%d %H:%M:%S+00:00'))/60
                    if curr_time > dtstart and curr_time < dtend or curr_time + event.duration > dtstart and curr_time + event.duration < dtend or curr_time < dtstart and curr_time + event.duration > dtend or curr_time > dtstart and curr_time + event.duration < dtend or curr_time == dtstart or curr_time + event.duration == dtend:
                        is_valid = False
                        break
            if is_valid:
                for constraint in event.constraints:
                    if not constraint.isValid(event, self.ical):
                        is_valid = False
                        break
            if is_valid:
                self.addToCalendar(event)
                is_scheduled = True
            else:
                curr_time -= 30

  def runUntil(self, stopDate=None, **kw):
     """Runs the EventLoop until the given time plus interval have been
     reached or it runs out of things to monitor. This method
     should not be called when the EventLoop is already running.

     The current time is assumed, if no date time is passed in.

     Examples:(note, these aren't real doctests yet)

     Run until a given date, say St. Patty's day
     >> date=datetime.datetime(2007, 03,17, 17,00)
     >> EventLoop.currentEventLoop().runUntil(dateAndTime)

     Additionally you can pass in any keyword argument normally
     taken by daetutilse.relativedelta to derive the date. These
     include:

     years, months, weeks, days, hours, minutes, seconds, microseconds

     These are moste useful when you want to compute the relative
     offset from now. For example to run the EventLoop for 5 seconds
     you could do this.

     >> EventLoop.currentEventLoop().runUntil(seconds=5)

     Or, probably not as practical but still possible, wait one
     year and 3 days

     >> EventLoop.currentEventLoop().runUntil(years=1, days=3)
     

     """

     if self.running:
        raise RuntimeError("EventLoop is already running.")
     else:
        self.running = True

     delta = relativedelta(**kw)
     now = datetime.datetime.now()
     
     if stopDate is None:
        stopDate = now

     stopDate = now + delta

     # convert the time back into seconds since the epoch,
     # subtract now from it, and this will then be the delay we
     # can use

     seconds2Run = time.mktime(stopDate.timetuple()) - time.mktime(now.timetuple())
     self.waitBeforeCalling(seconds2Run, self.stop)
     
     while self._shouldRun(1):
        try:
           self.runOnce()
        except:
           self.log.exception("Caught unexpected error in RunOnce.")
           
     self.reset()

def cont_position(posdata):
	#returns a list with a continious position of the user for every minute, None if unkown position
	cont_pos = []
	for t in range(24*60):
		prev = prevpos(posdata, time.mktime(config.SAMPLE_DAY.timetuple())+60*t)
		next = nextpos(posdata, time.mktime(config.SAMPLE_DAY.timetuple())+60*t)

		closest = None
		if prev != None and next != None:
			if abs(prev[0]-60*t) <= abs(next[0]-60*t): #select the closest position
				closest = prev
			else:
				closest = next
		elif prev != None:
			closest = prev
		elif next != None:
			closest = next
		else:
			closest = None

		if closest == None: #no position found
			cont_pos.append((None, None, 0.0)) #lat, lon, confidence
		elif abs(closest[0]-(time.mktime(config.SAMPLE_DAY.timetuple())+60*t)) < 10*60: #known position
			cont_pos.append((closest[1], closest[2], 1.0)) #lat, lon, confidence
		elif prev != None and next != None and (prev[1:2] == next[1:2]) and abs(prev[0]-next[0]) < 3*60*60: #probable position, if previous and next cell are the same
			cont_pos.append((closest[1], closest[2], 0.2)) #lat, lon, confidence
		else: #position too old
			cont_pos.append((None, None, 0.0)) #lat, lon, confidence
	
	assert(len(cont_pos) == 24*60)
	return cont_pos

    def get_actions(self,coordinator):
        accumulator=dict()
        accumulator['total']=0
        try:
            url = "http://" + self.host + ":" + str(self.port) + self.api_url['actions_from_coordinator'] % (coordinator,0,0)
            response = requests.get(url, auth=self.html_auth)
            if not response.ok:
                return {}
            total_actions=json.loads(response.content)['total']

            url = "http://" + self.host + ":" + str(self.port) + self.api_url['actions_from_coordinator'] % (coordinator,total_actions-self.query_size,self.query_size)
            response = requests.get(url, auth=self.html_auth)
            if not response.ok:
                return {}

            actions = json.loads(response.content)['actions']

            for action in actions:
                created=time.mktime(self.time_conversion(action['createdTime']))
                modified=time.mktime(self.time_conversion(action['lastModifiedTime']))
                runtime=modified-created
                if accumulator.get(action['status']) is None:
                    accumulator[action['status']]=defaultdict(int)
                accumulator[action['status']]['count']+=1
                accumulator[action['status']]['runtime']+=runtime
                accumulator['total']+=1
        except:
            logging.error('http request error: "%s"' % url)
            return {} 
        return accumulator

    def _add_event(self, title, date, start, end, all_day, 
                   url=None, description=None):
        if isinstance(title, unicode):
            title = title.encode('utf8')
        values = dict(title=title,
                      all_day=all_day and 'true' or 'false',
                      guid=self.guid,
                      )
        if date:
            values['date'] = mktime(date.timetuple())
        else:
            values['start'] = mktime(start.timetuple())
            values['end'] = mktime(end.timetuple())

        if url is not None:
            values['url'] = url
        if description is not None:
            values['description'] = description
        data = urlencode(values)
        url = self.base_url + '/api/events'
        req = urllib2.Request(url, data)
        response = urllib2.urlopen(req)
        content = response.read()
        event = anyjson.deserialize(content)['event']
        self._massage_event(event)
        return event, response.code == 201

    def pop_second(self):
        parsed_sec = AbstractReader.pop_second(self)
        if parsed_sec:
            self.pending_second_data_queue.append(parsed_sec)
        else:
            self.log.debug("No new seconds present")   
            
        if not self.pending_second_data_queue:
            self.log.debug("pending_second_data_queue empty")
            return None
        else:
            self.log.debug("pending_second_data_queue: %s", self.pending_second_data_queue)


        next_time = int(time.mktime(self.pending_second_data_queue[0].time.timetuple()))
            
        if self.last_sample_time and (next_time - self.last_sample_time) > 1:
            self.last_sample_time += 1
            self.log.debug("Adding phantom zero sample: %s", self.last_sample_time)
            res = self.get_zero_sample(datetime.datetime.fromtimestamp(self.last_sample_time))
        else:
            res = self.pending_second_data_queue.pop(0)
        
        self.last_sample_time = int(time.mktime(res.time.timetuple()))
        res.overall.planned_requests = self.__get_expected_rps()
        self.log.debug("Pop result: %s", res)
        return res

def _calculate_offset(date, local_tz):
    """
    input :
    date : date type
    local_tz : if true, use system timezone, otherwise return 0

    return the date of UTC offset.
    If date does not have any timezone info, we use local timezone,
    otherwise return 0
    """
    if local_tz:
        # handle year before 1970 most sytem there is no timezone information before 1970.
        if date.year < 1970:
            # Use 1972 because 1970 doesn't have a leap day
            t = time.mktime(date.replace(year=1972).timetuple)
        else:
            t = time.mktime(date.timetuple())

        # handle daylightsaving, if daylightsaving use altzone, otherwise use timezone
        if time.localtime(t).tm_isdst:
            return -time.altzone
        else:
            return -time.timezone
    else:
        return 0

def create_id_token(user, aud, nonce):
    """
    Receives a user object and aud (audience).
    Then creates the id_token dictionary.
    See: http://openid.net/specs/openid-connect-core-1_0.html#IDToken

    Return a dic.
    """
    sub = settings.get('OIDC_IDTOKEN_SUB_GENERATOR')(user=user)

    expires_in = settings.get('OIDC_IDTOKEN_EXPIRE')

    # Convert datetimes into timestamps.
    now = timezone.now()
    iat_time = int(time.mktime(now.timetuple()))
    exp_time = int(time.mktime((now + timedelta(seconds=expires_in)).timetuple()))
    user_auth_time = user.last_login or user.date_joined
    auth_time = int(time.mktime(user_auth_time.timetuple()))

    dic = {
        'iss': get_issuer(),
        'sub': sub,
        'aud': str(aud),
        'exp': exp_time,
        'iat': iat_time,
        'auth_time': auth_time,
    }

    if nonce:
        dic['nonce'] = str(nonce)

    return dic

def do_offset(tuples_list, filename, format ='%b %d, %Y %H:%M:%S', offset_val=0):
    new_tuples_list = []
    firstval = time.strptime(tuples_list[0][0], format)
    if filename != "slide_timestamps.txt":
        def_time = 'Apr 01, 2000 00:00:00'
    else :
        def_time = 'Apr 01 2000 00:00:00'
    conversion_timer = time.mktime(time.strptime(def_time, format))

    for item in tuples_list:
        t= item[0]
        timer = time.strptime(t, format)  ##3,4,5
        timer = time.mktime(timer) - time.mktime(firstval) + conversion_timer + offset_val
        timer = time.strftime("%H:%M:%S",time.localtime(timer))
        if filename == "spectrum.txt":
            line_list = [timer]
            for i in json.loads(item[1]):
                line_list.append(i)
            #print line_list
            new_tuples_list.append(tuple(line_list))
            
        else:
            line_list = [timer]
            for i in item[1:]:
                line_list.append(i)
            #print line_list
            new_tuples_list.append(tuple(line_list))       
    return new_tuples_list

    def get_all_bw_usage(self, instances, start_time, stop_time=None):
        """Return bandwidth usage info for each interface on each
           running VM"""

        # we only care about VMs that correspond to a nova-managed
        # instance:
        imap = dict([(inst.name, inst.uuid) for inst in instances])

        bwusage = []
        start_time = time.mktime(start_time.timetuple())
        if stop_time:
            stop_time = time.mktime(stop_time.timetuple())

        # get a dictionary of instance names.  values are dictionaries
        # of mac addresses with values that are the bw stats:
        # e.g. {'instance-001' : { 12:34:56:78:90:12 : {'bw_in': 0, ....}}
        iusages = self._vmops.get_all_bw_usage(start_time, stop_time)
        for instance_name in iusages:
            if instance_name in imap:
                # yes these are stats for a nova-managed vm
                # correlate the stats with the nova instance uuid:
                iusage = iusages[instance_name]

                for macaddr, usage in iusage.iteritems():
                    bwusage.append(dict(mac_address=macaddr,
                                        uuid=imap[instance_name],
                                        bw_in=usage['bw_in'],
                                        bw_out=usage['bw_out']))
        return bwusage