def monitor_exception_count(self, e, isTimeout_retry, url):
     if str(e).find('urlopen error timed out') > -1:
         if not isTimeout_retry:
             self.monitor.urls_serial_timeout.append(url)
         else:
             self.monitor.urls_retry_failure.append(url)
     Util.printf('urlopen-->%s | exception--> %s' % (url, e))

    def select_map_store(self):
        map_store={}
        table = "video_report_%d" % int(self.monitorReport.mydate.get_now(format='ym'))
        fields = []
        fiedls_position={}
        re=ReportEntity.ReportEntity()
        count = 0
        for f in re.__dict__:
            fields.append(f)
            fiedls_position.setdefault(f,count)
            count+=1

        sql = ' select '
        sql += ','.join(fields)
        sql += ' from '+ table
        sql += ' where dtime='+ self.monitorReport.mydate.get_now() +' and play_plat=2 '
        Util.printf(sql)
        rows = self.mysql_select(sql)

        for row in rows:
            video_id= row[fiedls_position.get('video_id')]
            report = {}
            for k , v  in fiedls_position.iteritems():
                report.setdefault(k,row[v])
            map_store.setdefault(('%d' % video_id),report)

        return map_store

    def proxy(self, path, func):
        if not self.file_exist(path,True):
            return

        if func.has_key('preprocess'):
            self.preprocess = func.get('preprocess')
        if func.has_key('process'):
            self.process = func.get('process')
        if func.has_key('postprocess'):
            self.postprocess = func.get('postprocess')

        context = self.preprocess()
        file_object = open(path)
        try:
            for line in file_object:
                try:
                    exitcode = self.process(line, context)
                    if exitcode == 0:
                        continue
                    elif exitcode == -1:
                        return
                except Exception, e:
                    Util.printf(e)
                    Util.printf("first item:"+line)
            return self.postprocess(context)

 def get_conn_qs(self):
     try:
         conn = MySQLdb.connect(host=self.host, user=self.user, passwd=self.passwd,
                                db=self.db, charset=self.charset, port=self.port)
         return conn
     except Exception, e:
         Util.printf(e)

 def monitor_exception_count(self, e, isTimeout_retry, url):
     if str(e).find("timed out") > -1 or str(e).find("Time-out") > -1:
         if not isTimeout_retry:
             self.monitor.urls_parallel_timeout.append(url)
         else:
             self.monitor.urls_retry_failure.append(url)
     Util.printf("urlopen-->%s | exception--> %s" % (url, e))

    def run(self):
        Util.printf("thread running".ljust(50,"."))

        while self.isRun:
            if not self.q.isEmpty():
                Util.printf("proccess running".ljust(50,"."))
                singal =  str(self.q.get_singal())
                p = MyProcess(singal,self.path,self.mysqlWrap,self.constants,self.monitor,self.lock)
                self.jobs.setdefault(singal,p)
                p.start()
                Util.printf("queque pop %s" % singal)
            else:
                jbs = [job.is_alive() for job in self.jobs.values() ]
                if jbs.count(False) == self.jobs_size :
                    self.isRun = False

                if jbs.__len__() == self.jobs_size:
                    self.isRun = False

                if jbs.__len__() > 0 and jbs.__len__() == self.jobs_size - 1:
                    self.q.put_singal("URLRequest")

                jps_show = [(name,job.is_alive()) for name,job in self.jobs.items() ]
                Util.printf("current process is_alive:%s" % jps_show)

                time.sleep(2)

        Util.printf("thread stop".ljust(50,"."))

    def insert_update_report(self, **args):
        map_store=args.get("map_store")
        update_fields=args.get("update_fields")
        table_name = "video_report_%d" % int(self.monitorReport.mydate.get_now(format='ym'))
        table_field = []
        update_field=[]
        datalist = []
        table_field_len = 0

        for report in map_store.values():
            if table_field_len == 0:
                for k in report.keys():
                    table_field.append(k)
                table_field_len = len(table_field)

            table_value = []
            for v in table_field:
                table_value.append(report[v])
            datalist.append(tuple(table_value))

        for field in update_fields:
            update_field.append( "%s=values(%s)" % (field,field))

        sql_head = ' insert into ' + table_name
        sql_body = ' ( ' + ','.join(table_field) + ' ) '
        sql_blank = ' , %s ' * (table_field_len - 1)
        sql_tail = ' values ( %s ' + sql_blank + ' ) '
        sql_duplicate=' ON DUPLICATE KEY UPDATE '+','.join(update_field)
        sql = sql_head + sql_body + sql_tail + sql_duplicate
        Util.printf(sql)
        self.mysql_executemany(sql, datalist)

 def mysql_select(self,sql):
     conn = self.get_conn_qs()
     cursor = conn.cursor()
     try:
         n = cursor.execute(sql)
         return cursor.fetchall()
     except Exception, e:
         Util.printf(e)

 def oracle_fetchall(self, sql):
     conn = self.get_conn()
     cursor = conn.cursor()
     try:
         cursor.execute(sql)
         rows = cursor.fetchall()
         return rows
     except Exception, e:
         Util.printf(e)

    def hive_shell_exe(self, sql, log_path):
        Util.file_mkdirs(os.path.dirname(log_path))
        exec_sql = 'touch %s' % log_path
        os.popen(exec_sql)

        exec_sql = str('$HIVE_HOME/bin/hive -S -e "%s" > %s' % (sql, log_path))
        os.popen(exec_sql)

        self.q.put_singal(log_path[os.path.dirname(log_path).__len__()+1:])

    def write_hive_shell(self, sql_path, log_path):
        FileBase.file_exist(sql_path)
        Util.file_remove(log_path)
        exec_sql = 'touch %s' % log_path
        os.popen(exec_sql)

        sql = Template(FileRead.file_readlines(sql_path)[0])
        temp_sql = sql.substitute(dtime=Util.get_yesterday())
        exec_sql = "$HIVE_HOME/bin/hive -S -e '%s' > %s" % (temp_sql, log_path)
        os.popen(exec_sql)

 def sub_job_join(self):
     while True:
         if self.jobs.__len__() == self.jobs_size:
             for k,job in self.jobs.items() :
                 if job.is_alive():
                     Util.printf(("%s proccess join".ljust(50,".")) % k)
                     job.join()
             break
         else:
             Util.printf("main proccess sleeping".ljust(50,"."))
             time.sleep(10)

 def mysql_execute_new(self, sql, datalist):
     conn = self.get_conn_qs()
     cursor = conn.cursor()
     try:
         for  data in datalist:
             tmpsql = sql % tuple(data)
             n = cursor.execute(sql,tuple(data))
             conn.commit()
             Util.printf('n=%s | sql=%s' % (n, tmpsql))
     except Exception, e:
         Util.printf(e)

 def update_mysql_field(self, *args):
     update_map = args[0]
     update_field = args[1]
     uids = args[2]
     table = "video_report_%d" % int(self.monitorReport.mydate.get_now(format='ym'))
     base_sql = "update %s set %s" % (table, update_field)
     sql = base_sql + "=%s where dtime=%s and play_plat=2 and video_id=%s and uid=%s"
     Util.printf(sql)
     datalist = []
     for k, v in update_map.items():
         datalist.append((v, self.monitorReport.mydate.get_now('-'), k, uids.get(k, 0)))
     self.mysql_executemany(sql, datalist)

 def read_keyvalue(self, path):
     update_map = {}
     self.file_exist(path)
     file_object = open(path)
     try:
         for line in file_object:
             kvs = line.split()
             if len(kvs) < 2: continue
             update_map.setdefault(kvs[0], kvs[1])
         return update_map
     except Exception, e:
         Util.printf(e)

def create_tree(offline):
    global model
    module_path = dirname(__file__)
    if offline:
       digits = load_data("video_report_201507_v5.txt")
       X = digits.data[:,1:]
       y = digits.target.astype(np.int64)
       model = DecisionTreeRegressor(max_depth=80)
       model.fit(X, y)
       Util.store_object(model,join(module_path, 'data', 'tree_model'))
    else:
        model = Util.grab_object(join(module_path, 'data', 'tree_model'))
    return model

 def mysql_executemany(self, sql, datalist):
     conn = self.get_conn_qs()
     cursor = conn.cursor()
     try:
         datalen = len(datalist) + 1
         for i in range(datalen):
             if i % 500 == 0:
                 n = cursor.executemany(sql, datalist[i:500 + i])
                 Util.printf(n)
                 conn.commit()
     except Exception, e:
         Util.printf(e)
         import traceback
         traceback.print_exc()

 def mysql_executemany(self, sql, datalist):
     conn = self.get_conn_qs()
     cursor = conn.cursor()
     try:
         datalen = len(datalist) + 1
         for i in range(datalen):
             if i % 500 == 0:
                 n = cursor.executemany(sql, datalist[i:500 + i])
                 data_leave = datalen - i - 500
                 prefix = str("data_leave:%s" % data_leave).ljust(20)
                 postfix = str("| commit_already:%s" % n).ljust(20)
                 Util.printf(prefix + postfix)
                 conn.commit()
     except Exception, e:
         Util.printf(e)
         import traceback
         traceback.print_exc()

 def __init__(self):
     ReportCDNEntity.__init__(self)
     ReportDMEntity.__init__(self)
     ReportADEntity.__init__(self)
     ReportURLEntity.__init__(self)
     self.video_id = 0
     self.play_plat = 2
     self.is_user_share = 1
     self.create_time = time.strftime('%Y-%m-%d %H:%M:%S')
     self.dtime = Util.get_yesterday()

    def insert_report(self, map_store):
        table_name = "video_report_%d" % int(self.monitorReport.mydate.get_now(format='ym'))
        table_field = []
        datalist = []
        table_field_len = 0

        for report in map_store.values():
            if table_field_len == 0:
                for k in report.keys():
                    table_field.append(k)
                table_field_len = len(table_field)

            table_value = []
            for v in table_field:
                table_value.append(report[v])
            datalist.append(tuple(table_value))

        sql_head = ' insert into ' + table_name
        sql_body = ' ( ' + ','.join(table_field) + ' ) '
        sql_blank = ' , %s ' * (table_field_len - 1)
        sql_tail = ' values ( %s ' + sql_blank + ' ) '
        sql = sql_head + sql_body + sql_tail
        Util.printf(sql)
        self.mysql_executemany(sql, datalist)