def getHistKData(self, code, start, end, ktype):
        '''
        历史K线
        '''
        print 'getHistKData work'

        if len(start) == 0 or len(end) == 0 :   
            df = ts.get_hist_data(code)
            ktype = 'D'

        elif len(start) == 0 :
            df = ts.get_hist_data(code, start, end)
            
        elif cmp(ktype, 'D') == 0 or \
             cmp(ktype, 'W') == 0 or \
             cmp(ktype, 'M') == 0 or \
             cmp(ktype, '5') == 0 or \
             cmp(ktype, '15') == 0 or \
             cmp(ktype, '30') == 0 or \
             cmp(ktype, '60') == 0 :

            df = ts.get_hist_data(code, start, end, ktype)
            
        else:
            print 'input error'
            return

        df.to_csv(self.__filename)

        
        self.__write2DB(ktype, code)

def get_ts_start_date(stock, datelen, type=None):
    '''
    :param stock: code
    :param datelen: date lenth
    :param type: day or week
    :return:start_date
    '''

    datelen = int(datelen)
    # start_date=get_day_of_day(datelen)
    # print start_date,stock
    if type != None:
        df = ts.get_hist_data(stock,start=get_day_of_day(datelen*3),ktype=type)
    else:
        df = ts.get_hist_data(stock,start=get_day_of_day(datelen*3))
    # day_t=get_today()
    # if day_t in df.index:
    #     # print "daynow"
    #     df=df.drop(day_t)
    if df.empty:
        print ("data err,pls check dataframe")
    else:
        # print df['open'].count()
        if df['open'].count() < datelen:
            start_date = (df.index)[-1]
            # print start_date
        else:
            start_date = (df.index)[datelen - 1]
    return start_date

def init_day_data(start=None):
    SjLshqD.objects.all().delete()
    stocks = SjGpJb.objects.order_by('dm').all()
    t1 = datetime.now()
    for basic in stocks:
        print('fetch stock: %s\'s data ' % basic.dm)
        if start is None:
            df = ts.get_hist_data(basic.dm, ktype='D')
        else:
            df = ts.get_hist_data(basic.dm, start=start, ktype='D')

        if df is not None and not df.empty:
            records = df.to_records()
            pattern = '%Y-%m-%d'
            for record in records:
                sj = str_utils.to_timezone(record['date'], pattern)
                data = SjLshqD.objects.get_or_create(gpjb=basic, sj=sj)[0]
                data.kpj = record['open']
                data.spj = record['close']
                data.zgj = record['high']
                data.zdj = record['low']
                data.cjl = record['volume']
                data.zde = record['price_change']
                data.zdf = record['p_change']
                data.jj5 = record['ma5']
                data.jj10 = record['ma10']
                data.jj20 = record['ma20']
                data.jl5 = record['v_ma5']
                data.jl10 = record['v_ma10']
                data.jl20 = record['v_ma20']
                data.hsl = record['turnover']
                data.save()

    t2 = datetime.now()
    print('init_day_data use totaol\' time: %s' % (t2 - t1).total_seconds())

    def get_hist_data(self,code = '600848', start = 0, end = 0, ktype = 'D', option = 'append'):
    #option = 'replace','fail','append'
    #code：股票代码，即6位数字代码，或者指数代码（sh=上证指数 sz=深圳成指 hs300=沪深300指数 sz50=上证50 zxb=中小板 cyb=创业板）
    #start：开始日期，格式YYYY-MM-DD
    #end：结束日期，格式YYYY-MM-DD
    #ktype：数据类型，D=日k线 W=周 M=月 5=5分钟 15=15分钟 30=30分钟 60=60分钟，默认为D
    #retry_count：当网络异常后重试次数，默认为3
    #pause:重试时停顿秒数，默认为0
        if(start != 0 and end != 0):
            df = ts.get_hist_data(code,start=start,end=end,ktype=ktype)
        else:
            df = ts.get_hist_data(code,ktype=ktype)
        #cnx = self.conn.getCNX()
        #df.to_sql("hist_data_"+ktype,cnx,flavor='mysql',if_exists=option,index=False)
        #print df.values
        #for tt in df.values:
        #    print tt
        #    for i in range(len(tt)):
        #        print tt[i]
        #date = datetime.datetime.strptime(start, "%Y-%m-%d")
        #curdate = date + datetime.timedelta(days=dd)
        #print d
        #print date
        #return

        for j in range(len(df.values)):
            date = df.index[j]
            tt = df.values[j]
            sqlPre = "insert into "+"hist_data_"+ktype+" (`code`, `date`, `open`, `high`, `close`, `low`, `volume`, `price_change`, `p_change`, `ma5`, `ma10`, `ma20`, `v_ma5`, `v_ma10`, `v_ma20`, `turnover`) VALUES ('"+code+"','"+date+"',"
            for i in range(len(tt)):
                sqlPre += "'"+tt[i].astype("str")+"',"
            sqlPre = sqlPre[:(len(sqlPre)-1)] + ")"
            self.conn.execute(sqlPre)
            #break
        print "get_hist_data executed"

def _DP(datapath, sid, ktype):
    '''
        获取不同种类的大盘指数函数
        20160720    改一下，保存临时文件
    :param datapath:
    :param zs:
    :param ktypes:
    :return:
    '''

    # for fn in os.listdir(datapath):
    # if fn.find()
    # for id in zs:
    timestemp = datetime.strftime(datetime.now(), '%Y%m%d%H%M%S')
    ffn = os.path.join(datapath, sid + '_' + ktype + '.csv')
    if os.path.isfile(ffn):
        oldps = pandas.read_csv(ffn, index_col='date')
        # start = datetime.strftime(datetime.strptime(oldps.index.max(), '%Y-%m-%d') + timedelta(days=1), '%Y-%m-%d')
        start = datetime.strptime(oldps.index.max().split(' ')[0], '%Y-%m-%d') + timedelta(days=1)
        # 判断现有数据的时间
        if (start - datetime.now()).days >= 0:
            pass
        else:
            tmpffn = ffn + '.bak.csv'
            toSaveData = tushare.get_hist_data(code=sid, ktype=ktype, start=datetime.strftime(start, '%Y-%m-%d'))
            # toSaveData['p_change_level'] = numpy.nan
            # toSaveData['volume_level'] = numpy.nan
            toSaveData.to_csv(tmpffn.lower())
    else:
        tmpffn = ffn
        toSaveData = tushare.get_hist_data(code=sid, ktype=ktype)
        # toSaveData['p_change_level'] = numpy.nan
        # toSaveData['volume_level'] = numpy.nan
        toSaveData.to_csv(ffn.lower())

 def get_data_by_count(self, stock_code, trade_date, count, kline_type, dataPath=None, dataType=None):
     """
     获取到指定日期的count根k线数据，通达信目前只支持日线数据
     :param stock_code:
     :param trade_date: 指定日期的数据
     :param count:
     :param kline_type: 数据类型，D=日k线 W=周 M=月 5=5分钟 15=15分钟 30=30分钟 60=60分钟，默认为D
     :param dataPath: 数据路径
     :param dataType: 数据类型， 0 通达信数据
     :return: dataframe 从小到大日期排序
     """
     # 获取count日内的k线数据
     holidays = (count / 5) * 3
     startDate = trade_date + datetime.timedelta(days=-(count + holidays))
     try:
         spy = ts.get_hist_data(stock_code, start=startDate.strftime("%Y-%m-%d"),
                                end=trade_date.strftime("%Y-%m-%d"),
                                ktype=kline_type)
         for i in range(4):
             if len(spy) < count:
                 holidays *= 2
                 startDate = trade_date + datetime.timedelta(days=-(count + holidays))
                 spy = ts.get_hist_data(stock_code, start=startDate.strftime("%Y-%m-%d"),
                                        end=trade_date.strftime("%Y-%m-%d"),
                                        ktype=kline_type)
             else:
                 break
     except (RuntimeError, TypeError, NameError, IOError, ValueError):
         return []
     return spy[:count].sort_index()

def dataframe2csv_test():

    '''
    问题描述:
    dataframe存csv后, 再从csv里面读到dataframe里面, 多一列序号...

    试验发现:
    对于通过调用pd.read_csv('xxx.csv')返回dataframe, 会自动增加一列序号

    解决办法:
    调用read_csv时使用index_col参数设置当前第一列为index, 就不会再增加索引列了.
    pd.read_csv('xxx.csv', index_col=0)
    '''
    df1 = tushare.get_hist_data('600004', start='2016-07-18', end='2016-07-19')
    print('----------------- df1 -----------------')
    print(df1)
    df1.to_csv('result.csv')

    df2 = tushare.get_hist_data('600004', start='2016-07-20', end='2016-07-21')
    print('----------------- df2 -----------------')
    print(df2)

    old_data = pd.read_csv('result.csv', index_col=0)
    print(old_data)
    old_data.to_csv('result.csv')

    df3 = df2.append(old_data)
    print('----------------- df3 -----------------')
    print(df3)

    df3.to_csv('result.csv')

def getIndexChangeRate(startDate,endDate):    
    df_result = DataFrame()
    df = ts.get_hist_data('sh',start =startDate,end = endDate ).reset_index()
    df['gap'] = df['high'] - df['low']
    df['gap_rate'] = df['gap']/df['close']*100
    df['mkt'] = 'sh'
    df_result = df_result.append(df)
    
    df = ts.get_hist_data('sz',start =startDate,end = endDate ).reset_index()
    df['gap'] = df['high'] - df['low']
    df['gap_rate'] = df['gap']/df['close']*100
    df['mkt'] = 'sz'
    df_result = df_result.append(df)
    
    df = ts.get_hist_data('zxb',start =startDate,end = endDate ).reset_index()
    df['gap'] = df['high'] - df['low']
    df['gap_rate'] = df['gap']/df['close']*100
    df['mkt'] = 'zxb'
    df_result = df_result.append(df)
    
    df = ts.get_hist_data('cyb',start =startDate,end = endDate ).reset_index()
    df['gap'] = df['high'] - df['low']
    df['gap_rate'] = df['gap']/df['close']*100
    df['mkt'] = 'cyb'
    df_result = df_result.append(df)
    
    fileName = r'D:\stock\index_changeRate_' +startDate+'_' + endDate + '.csv'
    df_result = df_result.loc[:,['date','mkt','close','volume','price_change','p_change','gap','gap_rate']]
    df_result = df_result.sort_index(by='date',ascending=False)
    df_result.to_csv(fileName,index = False)

def download_dailydata():
    start = clock()
    today = strftime("%Y-%m-%d")
    with open(PATH_ALL_CODE, 'r') as csv_file:
        spam_reader = reader(csv_file, delimiter=' ', quotechar='|')
        for row in spam_reader:
            #df = ts.get_hist_data(row[0][2:8], start='2016-03-31', end='2016-03-31')
            df = ts.get_hist_data(row[0][2:8], today)
            stockcode = pd.DataFrame(row[0], index=df.index, columns= (['stockcode']))
            df.insert(0, 'stockcode', stockcode)
            del df['price_change']
            del df['ma20']
            del df['v_ma20']
            df.to_sql('dailydata', ENGINE, if_exists='append')
    csv_file.close()

    index_list = ['sz399001','sz399006']
    for i in range(0, len(index_list)):
        df = ts.get_hist_data(index_list[i][2:8],today)
        stockcode = pd.DataFrame(index_list[i], index=df.index, columns= (['stockcode']))
        df.insert(0, 'stockcode', stockcode)
        del df['price_change']
        del df['ma20']
        del df['v_ma20']
        df.to_sql('index', ENGINE, if_exists='append')
    end = clock()
    print 'download data costs %d seconds' % (end - start)

def getdata_ema_trend_silent(stock, datelenth, type=None):
    # if datalen == '10':
    #     print ("datalen not 10")
    # else:
    #     print ("len:",datalen)
    # df = ts.get_tick_data(stock)
    start_date = get_ts_start_date(stock, datelenth, type)
    # print ("\nstart: %s  stock: %s" %(start_date[-5:],stock))
    if type != None:
        df = ts.get_hist_data(stock, start=start_date, ktype=type)
    else:
        df = ts.get_hist_data(stock, start=start_date)
    # countnum = df['open'].count()
    # print ('countnum:',countnum)
    # open_p=df['open'].mean()
    # low_p=df['open'].mean()
    # close_p=df['close'].mean()
    # print ("open_p:",open_p)
    # print  ("close_p:",close_p)
    # real_df=ts.get_realtime_quotes(stock)
    # real_o=(real_df['open'])[0]
    # real_c=(real_df['price'])[0]
    # real_perc=(real_df['pre_close'])[0]
    # print ("open:%s  lastc:%s nowp:%s"%(real_o,real_perc,real_c))
    return df

def getLast3YearsData(datapath, sid):
    '''

    :param datapath:
    :param sid:
    :return:
    '''
    tushare.get_hist_data(sid).to_csv(os.path.join(datapath, sid + '.csv').lower())

 def run(self):
     print "Starting " + self.name
     while True:
         code = get_code()
         if not code:
             return
         try:
             end = datetime.date.today()
             start = datetime.date.today() - datetime.timedelta(days=200)
             ts.get_hist_data(code, start='%s' % (start), end='%s' % (end)).to_csv('pd_5days/%s.csv' % (code))
         except:
             print 'something wrong with code: %s' % (code)

def crawl_data_00():
	today = getToday()

	for f_code in xrange(1,100):
		num_str = "%06d" % f_code
		code_str = str(num_str)

		data_t = ts.get_hist_data(code_str,start=today,end=today)
		
		if( data_t is not None):
			yield code_str
			ts.get_hist_data(code_str)

def do_it(stock_code,num4days,day_list):
	now = datetime.date.today()
	yestoday = now - datetime.timedelta(days=1)
	end_day = now - datetime.timedelta(days=num4days+day_list[-1]+10)
	workday = pd.bdate_range(start=str(end_day),end=str(yestoday))

	try:
		df = ts.get_hist_data(stock_code,start=str(end_day),end=str(yestoday))
		df_sh = ts.get_hist_data('sh',start=str(end_day),end=str(yestoday))
	except:
		print('timeout!')
		sys.exit(1)

	days = len(workday.date)
	p_change_sum = 0
	sh_p_change_sum = 0
	count = 0

	for i in range(days-1,day_list[-1]+10,-1):
		my_str = ''
		date_today = str(workday.date[i])
		date_yestoday = str(workday.date[i-1])
		#print(date_today,date_yestoday,date_today)

		try:
			price_open = df[df.index == date_today].open[0]
		except:
			continue

		if price_open != price_open:
			continue

		info_para_list = [df,date_today,workday,day_list,i]
		price_info_list = get_info(info_para_list) #获取股票信息
		color,persent = color4rules(date_today,price_info_list)

		info_para_list = [df_sh,date_today,workday,day_list,i]
		sh_info_list = get_info(info_para_list)
		color_sh,persent_sh = color4rules(date_today,sh_info_list)

		color4output(date_today,price_info_list,sh_info_list,color,persent,persent_sh)

		price_open,price_min,price_max,p_change,p_change_list,day_data = price_info_list
		sh_open,sh_min,sh_max,sh_p_change,sh_p_change_list,day_data_sh = sh_info_list
		if p_change >=0:
			p_change_sum +=1
		else:
			p_change_sum -=1
		if sh_p_change>=0:
			sh_p_change_sum +=1
		else:
			sh_p_change_sum -=1
	print('code:\t'+get_color(str(p_change_sum))+'\t'+'sh:\t'+get_color(str(sh_p_change_sum)))

def do_it(code, basics):

    stock_name = str(basics[basics.index == code][["name"]].values[0][0])
    stock_code = code

    num4days = 200
    now = datetime.date.today()
    yestoday = now - datetime.timedelta(days=1)
    end_day = now - datetime.timedelta(days=num4days)
    workday = pd.bdate_range(start=str(end_day), end=str(yestoday))

    try:
        df = ts.get_hist_data(stock_code, start=str(end_day), end=str(yestoday))
        df_sh = ts.get_hist_data("sh", start=str(end_day), end=str(yestoday))
    except:
        print("timeout!")
        sys.exit(1)

    days = len(workday.date)

    for i in range(days - 1, days - 2, -1):
        my_str = ""
        date_today = str(workday.date[i])
        date_yestoday = str(workday.date[i - 1])

        try:
            price_open = df[df.index == date_today].open[0]
            # yestoday_price_open = df[df.index == date_yestoday].open[0]
        except:
            continue

        if price_open != price_open:
            continue

        info_para_list = [df, date_today, workday, day_list, i]
        price_info_list = get_info(info_para_list)  # 获取股票信息
        color, persent = color4rules(date_today, price_info_list)

        info_para_list = [df_sh, date_today, workday, day_list, i]
        sh_info_list = get_info(info_para_list)
        color_sh, persent_sh = color4rules(date_today, sh_info_list)

        if color != "no":
            stock_industry = str(basics[basics.index == code][["industry"]].values[0][0])  # 行业
            stock_area = str(basics[basics.index == code][["area"]].values[0][0])  # 区域
            stock_pe = str(basics[basics.index == code][["pe"]].values[0][0])  # 市盈率
            stock_pb = str(basics[basics.index == code][["pb"]].values[0][0])  # 市净率

            stock_basics_list = [stock_code, stock_name, stock_industry, stock_area, stock_pe, stock_pb]
            color4output(date_today, stock_basics_list, price_info_list, sh_info_list, color, persent, persent_sh)

def getStockClose(stockCode):
    client = MongoClient()
    db = client[stockCode+'eastmoney']

    # Todo:fix the time
    dates = ts.get_hist_data(stockCode,start = '2014-03-13',end  = '2016-03-10').index.tolist()
    prices = ts.get_hist_data(stockCode,start = '2014-03-13',end  = '2016-03-10')['close'].tolist()

    for i in range(len(dates)):
        result =  db.price.insert_one({
        "close" : prices[i],
        "create_date" : dates[i]
     })
        print result.inserted_id

def sync_stock_minutes_history():
	'''
	stock minutes history
	'''
	today_str = datetime.datetime.now().strftime(u'%Y-%m-%d')
	yesterday_str = datetime.date.fromordinal(datetime.date.today().toordinal()-1).strftime(u'%Y-%m-%d')
	for k in stocklist:
		df = ts.get_hist_data(code=k, start=yesterday_str, end=today_str, ktype='5')
		DataFrameToMongo(df, MongoClient(mongourl)['stoinfo']['history_5min'], ['__index__'], k)
		df = ts.get_hist_data(code=k, start=yesterday_str, end=today_str, ktype='15')
		DataFrameToMongo(df, MongoClient(mongourl)['stoinfo']['history_15min'], ['__index__'], k)
		df = ts.get_hist_data(code=k, start=yesterday_str, end=today_str, ktype='30')
		DataFrameToMongo(df, MongoClient(mongourl)['stoinfo']['history_30min'], ['__index__'], k)
		df = ts.get_hist_data(code=k, start=yesterday_str, end=today_str, ktype='60')
		DataFrameToMongo(df, MongoClient(mongourl)['stoinfo']['history_60min'], ['__index__'], k)

def fetch_day_data2file(start='2016-01-01'):
    stocks = SjGpJb.objects.order_by('dm').all()
    t1 = datetime.now()
    for basic in stocks:
        print('fetch stock: %s\'s data to file' % basic.dm)
        if start is None:
            df = ts.get_hist_data(basic.dm, ktype='D')
        else:
            df = ts.get_hist_data(basic.dm, start=start, ktype='D')

        if df is not None and not df.empty:
            df.to_csv('/Users/lmclinux/code/IWANTYOURMONEY/data-files/%s.csv' %basic.ggid)

    t2 = datetime.now()
    print('init_day_data2file use totaol\' time: %s' % (t2 - t1).total_seconds())

def detect_saling_stocks():
	global saling_stocks, request_date, last_request_date
	# get request date
	tm_wday = time.localtime(time.time()).tm_wday
	tm_hour = time.localtime(time.time()).tm_hour
	if tm_hour < 15:
		today = datetime.date.today()
		oneday = datetime.timedelta(days=1)
		now = today - oneday
	else:
		now = datetime.date.today()
	if tm_wday == 5:
		delta = datetime.timedelta(days=-1)
		last_date = now + delta
		request_date = last_date.strftime("%Y-%m-%d")
	elif tm_wday == 6:
		delta = datetime.timedelta(days=-2)
		last_date = now + delta
		request_date = last_date.strftime("%Y-%m-%d")		
	else:
		request_date = now.strftime("%Y-%m-%d")

	#request_date = "2017-01-04"	
	yesterday = datetime.timedelta(days=2)
	yesterday_date = now - yesterday
	weekday = yesterday_date.weekday()
	if weekday == 5:
		delta = datetime.timedelta(days=-1)
		last_date = yesterday_date + delta
		last_request_date = last_date.strftime("%Y-%m-%d")
	elif weekday == 6:
		delta = datetime.timedelta(days=-2)
		last_date = yesterday_date + delta
		last_request_date = last_date.strftime("%Y-%m-%d")		
	else:
		last_request_date = yesterday_date.strftime("%Y-%m-%d")

	#last_request_date = "2016-12-30"
	for stock_code in stocks:
		curr_stock = ts.get_hist_data(stock_code, start=request_date,end=request_date)
		last_stock = ts.get_hist_data(stock_code, start=last_request_date,end=last_request_date)
		# if stock ma5 < ma10, sale it
		if curr_stock["ma5"][0] < curr_stock["ma10"][0]:
			saling_stocks.append(stock_code)
			continue
		if (curr_stock["ma5"][0] - last_stock["ma5"][0]) / last_stock["ma5"][0] * 100 < fixed_down_rate:
			saling_stocks.append(stock_code)
			continue

def main():
    now = time.strftime("%Y-%m-%d")
    # print(now)
    token = '60517739976b768e07823056c6f9cb0fee33ed55a1709b3eaa14a76c6a1b7a56'
    sb = StockBox()
    # sb.looper(id)
    id = '300333'
    # sb.realtime(id)
    sb.base_function("300333")
    # pandas_test=Pandas_test()
    # pandas_test.test_function()
    # sb.longhuban('2016-04-05')
    # sb.getNews()
    # sb.fund()
    # sb.get_stock_chengfeng()
    # sb.date_store()
    # sb.profit_test()
    # sb.daily_longhu()

    # 获取历史数据 近3年的数据
    history = ts.get_hist_data(id)

    print(u"历史3年的数据")

    print(history.head(10))

    history_all = ts.get_h_data(id, '20015101', '20160101')

    print(u'所有的历史数据')
    print(history_all)