def userLogin(self, userId, userCookie,
         lastSeen = utils.timestamp(False),
         email = None, openid = None,
         firstName = None, lastName = None, cellPhone = None,
         memberSince = utils.timestamp(False), uType = 'manual'):
     cursor = self.__conn.cursor()
     try:
         uid = self.getUser(uid = userId, ck = userCookie)['_id']
         cursor.execute("""UPDATE """
                 + DbWorker.dbCfg['table.user'] +
                 """ 
                 SET _firstName = %s
                 , _lastName = %s
                 , _cellPhone = %s
                 , _lastSeen = %s
                 , _type = %s
                 , _cookie = %s
                  WHERE _id = %s
                 """ , (firstName, lastSeen,
                 cellPhone, lastSeen, uType, userCookie, uid))
     except RecordNotFound as e:
         cursor.execute("""
                 INSERT INTO """ + DbWorker.dbCfg['table.user'] + 
                 """ VALUES(%s, %s, %s, %s, %s, %s, %s, %s, %s, %s)
                 """,
                 (userId,
                 email, openid, firstName, lastName, cellPhone,
                 memberSince, lastSeen, uType, userCookie))
     finally:
         self.__conn.commit()
         cursor.close()

 def save_ignored_relationship(self, uid, tid):
     if not self.__ignored_relationship.get(uid):
         self.__ignored_relationship[uid] = [utils.timestamp(), [tid]]
     else:
         self.__ignored_relationship[uid][0] = utils.timestamp()
         self.__ignored_relationship[uid][1].append(tid)
         self.__ignored_relationship[uid][1] = self.__ignored_relationship[uid][1][-2:]

 def test_event_comment_notification(self):
     
     to_notify = self.make_user(username='noticeuser')
     
     self.register_for_notifications(user=to_notify)
     self.register_for_notifications()
     
     event = Event(**{
         u'where': 'test',
         u'when': timestamp(),
         u'what': 'test',
         u'broadcast': False,
         u'posted_from': [37.422834216666665, -122.08536667833332],
         u'creator': self.get_user()[u'username'],
     })
     event.save()
     
     Attendant(**{
         u'status': status.ATTENDING,
         u'timestamp': timestamp(),
         u'event': event[u'id'],
         u'user': to_notify[u'id'],
     }).save()
     
     comment = 'the test comment'
     
     response = self.post('/events/%s/comments/' % event[u'id'], 
                          {'comment': comment})
     self.assertEqual(response.status, 200, 'comments POST 200')
     
     nots = self.get_new_notifications(user=to_notify)
     
     self.assertEqual(len(nots), 1, 'one new notification')
     
     notification = nots[0]
     
     self.assertTrue(u'type' in notification, 
                     'poll response has type')
     self.assertEqual(notification[u'type'], 'comment', 
                      'event has the correct type')
     
     self.assertTrue(u'event_revision' in notification, 
                     'poll response has event rev')
     self.assertEqual(notification[u'event_revision'], event[u'revision'], 
                      'event has the correct revision')
     
     self.assertTrue(u'comment' in notification, 
                     'poll response has comment')
     self.assertEqual(notification[u'comment'], comment, 
                      'event has the correct comment')
     
     self.assertTrue(u'commenter' in notification, 
                     'poll response has commenter')
     self.assertEqual(notification[u'commenter'], self.get_user()[u'username'], 
                      'event has the correct commenter')
     
     nots = self.get_new_notifications()
     self.assertEqual(len(nots), 0, 'no notification for the poster')

def get(gkey,itm=[],start_time=0,stop_time=0,sort='clock asc',groupby=0,page=None):
	"""
	获取统计数据
	参数说明
	   itm itemid列表 为空时提取整个group的记录
	   start_time 开始时间戮
	   stop_time  结构时间戮
	   sort       排序方式 
	   groupby    分组方式 
	   page       分页参数集 {'site':每页数据量,'num':页码} 默认返回所有记录
	"""
	stat_db = mysqldb.get_db()
	rdb = rediswrap.get_redis()
	sql_item = {'fields':'*'}
	r_itmkey = RD_ITM_KEY_PRFX+gkey
	if itm:
		itm.append('mrk')
		itmids = rdb.hmget(r_itmkey,itm)
		mrk = itmids.pop().split(',')
		
	else:
		mrk = rdb.hget(r_itmkey,'mrk')
		itmids = rdb.hvals(r_itmkey)
		itmids.remove(mrk)
		mrk = mrk.split(',')	

	ids = [k for k in itmids if k ]	
	sql_item['table'] = get_hst_name(mrk[1]) 
	sql_item['where'] = " itemid in (%s) " % ",".join(ids)
	start_time = utils.timestamp(start_time) if start_time else utils.timestamp(0,'d')
	stop_time = utils.timestamp(stop_time) if stop_time else int(time.time())
	sql_item['where'] += " and clock>=%s and clock <%s" % (start_time,stop_time)
	sql_item['order'] = sort

	if groupby:
		if groupby ==1:
			sql_item['group'] = 'itemid'
		elif groupby == 2:
			sql_item['group'] = 'clock'
		else:
			sql_item['group'] = 'itemid,clock' 
		sql_item['fields'] = "itemid,sum(val) as val,clock"
	#分页这个mark一下。待定
	if page:
		s = page['num']*page['site']
		sql_item['limit'] = "%s,%s" %(s,page['site'])
	
	res,desc = stat_db.query(sql_item)
	#取得items的名称
	item_lab = {}
	if res == 0 and desc:
		itm_tb = "stat_item_" + mrk[0] if mrk[0] else "stat_item"
		rs, ds = stat_db.query("select name,id from %s where id in(%s)" %(itm_tb,",".join(ids) ))
		if rs==0 and ds:
			for row in ds :
				item_lab[row['id']]=row['name']
		return 0,[item_lab,desc]
	return 0,[{},[]]

    def test_event_list_sort(self):
        t1 = timestamp()
        t2 = timestamp()
        t3 = timestamp()
        
        e1 = Event(**{
            u'when': t1,
            u'what': 'user2 created, broadcast',
            u'broadcast': True,
            u'creator': self.get_user()[u'username'],
            u'created': t1,
        })
        e1.save()
        
        e2 = Event(**{
            u'when': t2,
            u'what': 'user2 created, broadcast',
            u'broadcast': True,
            u'creator': self.get_user()[u'username'],
            u'created': t3,
        })
        e2.save()
        
        e3 = Event(**{
            u'when': t2,
            u'what': 'user2 created, broadcast',
            u'broadcast': True,
            u'creator': self.get_user()[u'username'],
            u'created': t2,
        })
        e3.save()
        
        e4 = Event(**{
            u'when': t3,
            u'what': 'user2 created, broadcast',
            u'broadcast': True,
            u'creator': self.get_user()[u'username'],
            u'created': t1,
        })
        e4.save()
        
        response = self.get('/events/?sort=soon&filter=creator&'
                            'username=%s' % self.get_user()[u'username'])
        self.assertEqual(response.status, 200, 'response OK')
        
        events = json.loads(response.read())[u'events']
        self.assertEqual(len(events), 4, 'correct number of events returned')
        
        self.assertEqual(events[0], e1[u'revision'], 
                         'event 1 when correct, primary sort')
        #unconnect to test secondary sort
#        self.assertEqual(events[1], e3[u'revision'], 
#                         'event 3 when correct, secondary sort')
#        self.assertEqual(events[2], e2[u'revision'], 
#                         'event 2 when correct, secondary sort')
        self.assertEqual(events[3], e4[u'revision'], 
                         'event 4 when correct, primary sort')

 def test_event_list_invited(self):
     user = self.get_user()
     user2 = self.make_user(username='user2')
     user3 = self.make_user(username='user3')
     
     self.follow(user, user2, reciprocal=True)
     self.follow(user3, user, reciprocal=True)
     
     event1 = Event(**{
         u'where': 'test',
         u'when': timestamp(),
         u'what': 'user2 created',
         u'broadcast': False,
         u'posted_from': [37.422834216666665, -122.08536667833332],
         u'creator': user2[u'username'],
     })
     event1.save()
     Attendant(user=user[u'id'], event=event1[u'id']).save()
     
     event2 = Event(**{
         u'where': 'test',
         u'what': 'user3 created',
         u'broadcast': False,
         u'creator': user3[u'username'],
     })
     event2.save()
     Attendant(user=user[u'id'], event=event2[u'id'], 
               status=status.ATTENDING).save()
     
     event3 = Event(**{
         u'where': 'test',
         u'what': 'user2 created, broadcast',
         u'broadcast': True,
         u'posted_from': [37.422834216666665, -122.08536667833332],
         u'creator': user2[u'username'],
     })
     event3.save()
     
     event4 = Event(**{
         u'where': 'test',
         u'when': timestamp(),
         u'what': 'user3 created',
         u'broadcast': False,
         u'creator': user3[u'username'],
     })
     event4.save()
     
     response = self.get('/events/?filter=invited')
     self.assertEqual(response.status, 200, 'response OK')
     
     events = json.loads(response.read())[u'events']
     
     self.assertEqual(len(events), 3, 'correct number of events returned')
     self.assertTrue(event1[u'revision'] in events, 'event 1 returned')
     self.assertTrue(event2[u'revision'] in events, 'event 2 returned')
     self.assertTrue(event3[u'revision'] in events, 'event 3 returned')

 def check_login(self, stage=0):
     # self.curl.request("http:/pan.baidu.com/")
     
     ret = self.api_request("https://pan.baidu.com/api/account/thirdinfo")
     if ret["errno"] == 0:
         logger.debug("Login check success!")
         return True
     
     # More than twice landing check
     if stage >= 2:
         logger.debug("Login check failed!")
         return False
     
     # Get token
     token = self.get_token()
     
     # Check require verifycode
     params = dict(token=token,
                   tpl="netdisk",
                   apiver="v3",
                   tt=utils.timestamp(),
                   username=self.username,
                   isphone="false")
     check_login_url = "https://passport.baidu.com/v2/api/?logincheck&" + urllib.urlencode(params)
     ret = self.api_request(check_login_url)
     code_string =  ret["data"]["codeString"]
     
     if code_string:
         logger.debug("Login check require verifycode")
         verifycode = self.get_verifycode(code_string)
     else:    
         verifycode = ""
         
     # try to login    
     login_params = dict(staticpage="http://pan.baidu.com/res/static/thirdparty/pass_v3_jump.html",
                         charset="utf-8",
                         token=token,
                         tpl="netdisk",
                         tt=utils.timestamp(),
                         codestring=code_string,
                         isPhone="false",
                         safeflg=0,
                         u="http://pan.baidu.com/",
                         username=self.username,
                         password=self.password,
                         verifycode=verifycode,
                         mem_pass="on",
                         )    
     login_url = "https://passport.baidu.com/v2/api/?login"
     html = self.curl.request(login_url, data=login_params, method="POST")
     url = re.findall(r"encodeURI\('(.*?)'\)", html)[0]
     self.curl.request(url)
     return self.check_login(stage + 1)

def getLibrarySymbols():
    global libSyms
    global wsSyms
    global wsLibs
    global scannerProcess
    global scanq
    if not libSyms:
        if not scanq:
            libSyms = {}
            wsSyms = {}
            wsLibs = {}
        else:
            utils.timestamp("Getting scan results from queue")
            (libSyms, wsSyms, wsLibs) = scanq.get()
            utils.timestamp("Done queue get")
        if scannerProcess:
            utils.timestamp("Joining scan process")
            scannerProcess.join()
            utils.timestamp("Done join")
        scannerProcess = None
        if scanq:
            scanq.close()
        scanq = None
        import symbolscanner

        symbolscanner.setInitialResults(workspacePath, libSyms, wsSyms, wsLibs)
    return libSyms

 def list(self, dir="/", page=1, num='15'):
     # None for error
     params = dict(channel='chunlei',
                   clienttype=0,
                   web=1,
                   num=num,
                   t=utils.timestamp(),
                   page=page,
                   dir=dir,
                   _=utils.timestamp())
     ret = self.api_request("http://pan.baidu.com/api/list", 
                            extra_data=params)
     files = ret['list']
     return files

 def test_event_notification_attending_creator(self):
     
     to_attend = self.make_user(username='attending_user')
     
     self.register_for_notifications(user=self.get_user())
     
     event = Event(**{
         u'where': 'test',
         u'when': timestamp(),
         u'what': 'test',
         u'broadcast': False,
         u'posted_from': [37.422834216666665, -122.08536667833332],
         u'creator': self.get_user()[u'username'],
     })
     event.save()
     
     # this attendant will trigger the notification
     Attendant(**{
         u'status': status.INVITED,
         u'timestamp': timestamp(),
         u'event': event[u'id'],
         u'user': to_attend[u'id'],
     }).save()
     
     response = self.post('/events/%s/attendants/' % event[u'id'], 
                          {u'status': status.ATTENDING}, 
                          auth_user=to_attend)
     self.assertEqual(response.status, 200, 'attendants POST 200')
     
     nots = self.get_new_notifications(user=self.get_user())
     
     self.assertEqual(len(nots), 1, 'one new notification')
     
     notification = nots[0]
     
     self.assertTrue(u'type' in notification, 
                     'poll response has type')
     self.assertEqual(notification[u'type'], 'attendant', 
                      'event has the correct type')
     
     self.assertTrue(u'event_revision' in notification, 
                     'poll response has event rev')
     self.assertEqual(notification[u'event_revision'], event[u'revision'], 
                      'event has the corrrect revision')
     
     self.assertTrue(u'attendant' in notification, 
                     'poll response has attendant')
     self.assertEqual(notification[u'attendant'], to_attend[u'username'], 
                      'event has the corrrect attendant')

    def read_request_proxy(self, client_conn):
        line = self.get_line(self.rfile)
        if line == "":
            return None

        if not self.proxy_connect_state:
            connparts = http.parse_init_connect(line)
            if connparts:
                host, port, httpversion = connparts
                headers = self.read_headers(authenticate=True)
                self.wfile.write(
                            'HTTP/1.1 200 Connection established\r\n' +
                            ('Proxy-agent: %s\r\n'%self.server_version) +
                            '\r\n'
                            )
                self.wfile.flush()
                self.establish_ssl(client_conn, host, port)
                self.proxy_connect_state = (host, port, httpversion)
                line = self.rfile.readline(line)

        if self.proxy_connect_state:
            r = http.parse_init_http(line)
            if not r:
                raise ProxyError(400, "Bad HTTP request line: %s"%repr(line))
            method, path, httpversion = r
            headers = self.read_headers(authenticate=False)

            host, port, _ = self.proxy_connect_state
            content = http.read_http_body_request(
                self.rfile, self.wfile, headers, httpversion, self.config.body_size_limit
            )
            return flow.Request(
                        client_conn, httpversion, host, port, "https", method, path, headers, content,
                        self.rfile.first_byte_timestamp, utils.timestamp()
                   )
        else:
            r = http.parse_init_proxy(line)
            if not r:
                raise ProxyError(400, "Bad HTTP request line: %s"%repr(line))
            method, scheme, host, port, path, httpversion = r
            headers = self.read_headers(authenticate=True)
            content = http.read_http_body_request(
                self.rfile, self.wfile, headers, httpversion, self.config.body_size_limit
            )
            return flow.Request(
                        client_conn, httpversion, host, port, scheme, method, path, headers, content,
                        self.rfile.first_byte_timestamp, utils.timestamp()
                    )

def make_logfile_name(params=None, net_save_filename=None, timestamp = None, train=True):
    '''
    Returns the name of the logfile for the current training/forward pass run
    '''

    #Need to specify either a params object, or the net save prefix
    utils.assert_arglist(params,
        [net_save_filename])

    if params is not None:
        if train:
            _net_save_filename = params['train_save_net']
        else:
            _net_save_filename = params['output_prefix']

    if net_save_filename is not None:
        _net_save_filename = net_save_filename

    save_prefix = os.path.splitext( _net_save_filename )[0]

    save_prefix_valid = len(save_prefix) > 0
    assert(save_prefix_valid)

    if timestamp is None:
        timestamp = utils.timestamp()
    mode = "train" if train else "forward"

    directory_name = os.path.dirname( save_prefix )
    if not os.path.exists(directory_name):
        os.mkdir(directory_name)

    save_filename = "{}_{}_{}.log".format(save_prefix, mode, timestamp)

    return save_filename

    def read_request_transparent(self, client_conn):
        orig = self.config.transparent_proxy["resolver"].original_addr(self.connection)
        if not orig:
            raise ProxyError(502, "Transparent mode failure: could not resolve original destination.")
        self.log(client_conn, "transparent to %s:%s"%orig)

        host, port = orig
        if port in self.config.transparent_proxy["sslports"]:
            scheme = "https"
            if not self.ssl_established:
                self.establish_ssl(client_conn, host, port)
        else:
            scheme = "http"
        line = self.get_line(self.rfile)
        if line == "":
            return None
        r = http.parse_init_http(line)
        if not r:
            raise ProxyError(400, "Bad HTTP request line: %s"%repr(line))
        method, path, httpversion = r
        headers = self.read_headers(authenticate=False)
        content = http.read_http_body_request(
                    self.rfile, self.wfile, headers, httpversion, self.config.body_size_limit
                )
        return flow.Request(
                    client_conn,httpversion, host, port, scheme, method, path, headers, content,
                    self.rfile.first_byte_timestamp, utils.timestamp()
               )

 def _read_request_transparent(self, client_conn, scheme, host, port):
     """
     Read a transparent HTTP request. Transparent means that the client isn't aware of proxying.
     In other words, the client request starts with
     "GET /foo.html HTTP/1.1"
     rather than
     "CONNECT example.com:80 HTTP/1.1"
     """
     if scheme.lower() == "https" and not self.ssl_established:
         self.establish_ssl(client_conn, host, port)
     line = self.get_line(self.rfile)
     if line == "":
         return None
     r = http.parse_init_http(line)
     if not r:
         raise ProxyError(400, "Bad HTTP request line: %s"%repr(line))
     method, path, httpversion = r
     headers = self.read_headers(authenticate=False)
     content = http.read_http_body_request(
                 self.rfile, self.wfile, headers, httpversion, self.config.body_size_limit
             )
     return flow.Request(
                 client_conn,httpversion, host, port, scheme, method, path, headers, content,
                 self.rfile.first_byte_timestamp, utils.timestamp()
            )

    def __init__(self, username, password):
        self.username = username
        self.password = password
        self.session = requests.session()

        # https://ss0.bdstatic.com/5LMZfyabBhJ3otebn9fN2DJv/passApi/js/login_tangram_f2e986d5.js


        # self.index_url = 'https://www.baidu.com/'

        # href="https://passport.baidu.com/v2/?login&tpl=mn&u=http%3A%2F%2Fwww.baidu.com%2F"
        # https://www.baidu.com/   主页
        # #https://passport.baidu.com/v2/?login　　　登录界面
        # get token
        # https://passport.baidu.com/v2/api/?getapi&tpl=pp&apiver=v3&tt=1431305718340&class=login&logintype=basicLogin&callback=bd__cbs__k7txq4

        ##
        ##要先访问主页，自动保存cookies ，再访问sign_url　才能显示页面内容
        self.sign_url = 'https://passport.baidu.com/v2/api/?login'
        # self.mission_url = 'http://v2ex.com/mission/daily'

        self.config_section = 'baidu'
        '''配置文件字段'''
        self.config_cookies = 'cookies'
        '''配置文件字段'''

        self.tt = utils.timestamp();

    def POST(self):
        x = web.input()
        # check csrf token
        if (not x.has_key('csrf_token') or x["csrf_token"]!=session["csrf_token"]):
            print "tokens aren't matching"
            session["csrf_token"] = uuid4().hex        
        else :
            # protect using csrf_token
                if "data" in x :
                    fname = os.path.join("logs","wavs", \
                                         session.dialog_state.id + "_" + utils.timestamp(include_date=False) + ".wav")
                    savefile = open(fname,"wb")
                    savefile.write(x["data"])
                    savefile.close()
                    # Warn if this is not for the most recent utterance
                    if not x.has_key("utterance_key") or int(session["utterance_key"]) != int(x["utterance_key"]) :
                        print "got recording with wrong utterance_key"
                        print "comparing", x["utterance_key"], session["utterance_key"]                    
                    uttIndex = int(x["utterance_key"])
                    for i in range(len(session.dialog_state.recordings), uttIndex+1):
                        session.dialog_state.recordings.append("")
                    session.dialog_state.recordings[uttIndex] = fname
                    session["utterance_key"] = max(uttIndex, session["utterance_key"])

        web.header('Content-Type', 'application/json')
        return json.dumps({
            "csrf_token":session["csrf_token"]
            })

 def get_verify_code(self):
     url = 'https://passport.baidu.com/v2/api/?logincheck&'         
     data = {
         "callback" : "bdPass.api.login._needCodestringCheckCallback",
         "tpl" : "mn",
         "charset" : "UTF-8",
         "index" : 0,
         "username" : self.username,
         "time" : utils.timestamp()
         }
     data_query = urllib.urlencode(data)
     url = "%s&%s" % (url, data_query)
     req = urllib2.Request(url)
     data = self.opener.open(req).read().strip()
     data = data[data.find("(") + 1: data.rfind(")")]
     data = eval(data, type('Dummy', (dict,), dict(__getitem__=lambda s,n:n))())
     codestring = data["codestring"]
     if codestring != "":
         url = "https://passport.baidu.com/cgi-bin/genimage?%s" % codestring
         req = urllib2.Request(url)
         ret_data = self.opener.open(req).read()
         pic_image = utils.get_cache_file("pic")
         with open(pic_image, "wb") as fp:
             fp.write(ret_data)
         self.loginfo("Verify code pic download ok!")
         return raw_input("piz input code > ").strip()    

 def reportRequest(self, userCookie, startDate,
         endDate = utils.timestamp(False),
         category = None):
     try:
         uid = self.getUser(ck = userCookie)['_id']
         if uid is None:
             return
         cursor = self.__conn.cursor(MySQLdb.cursors.DictCursor)
         if category is None:
             cursor.execute("""
             SELECT _billId, _amount, _date, _category 
             FROM """ + DbWorker.dbCfg['table.expense'] + """ 
             WHERE _userId = %s AND _date >= %s 
             AND _date <= %s
             """ , (uid, startDate, endDate))
         else:
             cursor.execute("""
             SELECT _billId, _amount, _date, _category 
             FROM """ + DbWorker.dbCfg['table.expense'] + """ 
             WHERE _userId = %s AND _date >= %s 
             AND _date <= %s AND _category = %s
             """ , (uid, startDate, endDate, category))
         result = cursor.fetchall()
         expenses = [self.__transformDateInRec(row) for row in result]
         cursor.close()
         return json.dumps(expenses)
     except:
         raise

 def prep_for_response_sms(self):
     user = User(username='smstestuser',
                 password='password',
                 number='+16666666666')
     user.save()
     
     event = Event(**{
         u'where': 'test',
         u'when': timestamp(),
         u'what': 'test',
         u'broadcast': False,
         u'posted_from': [37.422834216666665, -122.08536667833332],
         u'creator': user[u'username'],
     })
     event.save()
     
     smsuser = User(number='+16475555555', display_name='Testy Smoth')
     smsuser.save()
     
     Attendant(user=user[u'id'], event=event[u'id']).save()
     
     SMSRegister(contact_number=smsuser[u'number'], 
                 twilio_number=settings.TWILIO_NUMBERS[0], 
                 event=event[u'id'], 
                 expires=event[u'when'],
                 user=user[u'id']).save()
     
     return user, event, smsuser

def create_pdf(folder, images):
    filename = folder + '/' + 'pdf_' + utils.timestamp() + '.pdf'
    pdf_creator = PdfBuilder(filename)
    for image in images:
        pdf_creator.add_image(image, 687, 809)
    pdf_creator.images_on_pdf()
    pdf_creator.save()