def collect_basic_data():
    basic_data_path = os.path.join(BASE_FOLDER, 'basics')
    if not os.path.exists(basic_data_path):
        os.makedirs(basic_data_path)
    path = os.path.join(basic_data_path, 'basic.csv')
    if not os.path.exists(path):
        ts.get_stock_basics().to_csv(path)
    return path

def load_file():
    db_full_path = get_db_file_name()
    if os.path.exists(db_full_path):
        if os.path.getsize(db_full_path) == 0:
            stock_list = ts.get_stock_basics()
            save_file(stock_list)
    else:
        stock_list = ts.get_stock_basics()
        save_file(stock_list)
    f = open(db_full_path, 'rb')
    objects = pickle.load(f)
    f.close
    return objects

def get_stock_basics():
    try:
        df = ts.get_stock_basics()
        df.reset_index(level=0, inplace=True)
        return json.loads(df.to_json(orient='records'))
    except IOError:
        pass

def get_stock_his_day_Data(code, startDay, endDay):###generator for the stock data share by year
	df = ts.get_stock_basics()
	tmDate = df.ix[code]['timeToMarket']

	if '-' in startDay:
		_d = startDay.split('-')
		startDay = _d[0]+_d[1]+_d[2]

	if '-' in endDay:
		_d = endDay.split('-')
		endDay = _d[0]+_d[1]+_d[2]

	if not isinstance(startDay, np.int64):
		startDay = np.int64(startDay)
	if not isinstance(endDay, np.int64):
		endDay = np.int64(endDay)

	if startDay < tmDate:
		startDay = tmDate

	today = np.int64( str(datetime.date.today()).replace('-','') )

	if endDay > today:
		endDay = today
 
 	#search by year, for the reliability
 	nyears = endDay/10000 - startDay/10000 + 1
 	sstartDay, sendDay = str(startDay), str(endDay)
	for nyear in xrange(startDay/10000,endDay/10000+1):
		tmpStart = sstartDay[0:4]+'-'+sstartDay[4:6]+'-'+sstartDay[6:8] if nyear==startDay/10000 else str(nyear)+'-01-01'
		tmpEnd = sendDay[0:4]+'-'+sendDay[4:6]+'-'+sendDay[6:8] if nyear==(endDay/10000) else str(nyear)+'-12-31'
		logging.debug("get code:%s history data from %s to %s" %(code, tmpStart, tmpEnd))
		tmpdata = ts.get_h_data(code, start=tmpStart, end=tmpEnd)
		yield(tmpdata)

def download_stock_basic_info():
    """
    获取股票基本信息
    :return:
    """
    
    try:
        df = ts.get_stock_basics()

        print(df.columns)
        df['code'] = df.index

        print(df.head())
        if len(df):
            engine = db.get_w_engine()
            to_sql(STOCK_BASIC_TABLE, engine, df, type='replace')
            # df.to_sql(STOCK_BASIC_TABLE, engine, if_exists='append', index=False)

        # 添加指数
        indexs = [('sh', '上证指数', '指数','全国','19910715'),
                  ('sz', '深圳成指', '指数','全国','19940720'),
                  ('hs300', '沪深300指数', '指数','全国','20050408'),
                  ('sz50', '上证50指数', '指数','全国','20040102'),
                  ('zxb', '中小板指数', '指数','全国','20050607'),
                  ('cyb', '创业板指数', '指数','全国','20100531'),]
        df = pd.DataFrame(indexs, columns=[KEY_CODE,KEY_NAME, KEY_INDUSTRY, KEY_AREA, KEY_TimeToMarket])
        print(df)
        to_sql(STOCK_BASIC_TABLE, engine, df, type='replace')

           
    except Exception as e:
        print(str(e))

 def process(self):
     fo = open('shrink_code', 'w')
     oversold = open('oversold', 'w')
     week_shrink = open('week_shrink','w')
     #raw_data = TS.memchaced_data(ts.get_stock_basics,'get_stock_basics')
     raw_data = ts.get_stock_basics()
     raw_data['earn_ratio'] = raw_data['esp'] / raw_data['bvps']
     for code in raw_data.index:
         if code in bad_container:
             continue
         try:
             daydata = ts.get_k_data(code, ktype='D')
             ratio = judge_oversold(daydata)
             if ratio < -0.45:
                 oversold.write("{0}\t{1}\n".format(code, ratio))
                 weekdata = ts.get_k_data(code, ktype='W')
             if judge_week_shrinkage(code, 0.02):
                 week_shrink.write(code + '\n')
             if ratio > -0.3:
                 continue
             #if raw_data.ix[code]['earn_ratio'] < 0.05:
             #    continue
             totals = raw_data.ix[code]['totals']
             close = np.array(daydata['close'])[-1]
             if close < 6.5:
                 continue
             if close * totals > 360:
                 continue
             flag,mean = judge_shrinkage(daydata,0.02)
             if ratio > -0.3:
                 continue
             if flag != -1:
                 fo.write("{0}\t{1}\t{2:.1%}\n".format(flag,code, mean))
         except Exception, e:
                 print e

def collect_hist_data(start=None, end=None, type='D', exclude_cyb=True, sample=0, persist=False):
    folder_name = 'hist-{}-{:%y%m%d}-{:%y%m%d}'.format(
        type,
        datetime.datetime.strptime(start, '%Y-%m-%d'),
        datetime.datetime.today())
    storage_path = os.path.join(BASE_FOLDER, folder_name)

    # if target storage path already exists, consider the data has been collected already
    if not os.path.isdir(storage_path):
        os.makedirs(storage_path)
    if not os.listdir(storage_path):
        basics = ts.get_stock_basics()
        codes = basics.index if not exclude_cyb else [x for x in basics.index if not x.startswith('300')]
        hist_data = {}
        codes_selected = codes if sample == 0 else random.sample(codes, sample)
        
        for code in codes_selected:
            try:
                df = ts.get_hist_data(code, start, end, ktype=type)
                hist_data[code] = df
                if persist:
                    df.to_csv(os.path.join(storage_path, '%s.csv' % code))
                print('retrieved hist data for %s' % code)
            except Exception as ex:
                try:
                    print('error occurred in retrieving {}: {}'.format(code, ex))
                except Exception as innerex:
                    print('exception: {}'.format(innerex))
    # return pd.Panel(hist_data)
    return storage_path

 def process(self):
     week_shrink = open('week_shrink','w')
     raw_data = ts.get_stock_basics()
     raw_data['earn_ratio'] = raw_data['esp'] / raw_data['bvps']
     container = defaultdict(list)
     for code in raw_data.index:
         if code in bad_container:
             continue
         weekdata = ts.get_k_data(code, ktype='W')
         try:
             info = compute_foundation_info(code, weekdata, 36)
             if info['ratio'] > 0.2:
                 continue
             if not info['cross_flag'] and not info['nearcross_flag']:
                 continue
             if info['macdmean'] > 0.3:
                 continue
             assment = info['close'] * raw_data.ix[code]['totals']
             if info['week_shrink'] < 2 or info['close'] < 6 or assment > 1800:
                 continue
             for feature in features:
                 feature_value = info[feature]
                 container[feature].append(feature_value)
         except Exception,e:
             print "ERROR:{0}".format(code)

def ggdr(year1,month1,day1,year2,month2,day2):
    gongsilist = ts.get_stock_basics().index
    a = len(gongsilist)
    for gongsi in gongsilist:
        basic_info_insert(gonggao_dict(gongsi,year1,month1,day1,year2,month2,day2))
        a -= 1
        print '还剩',a,'家'

def save_stock_basics_to_sql():
    back = ts.get_stock_basics()
    back['code'] = back.index
    back.index = range(back.shape[0])
    engine = create_engine('mysql://root:[email protected]/stock?charset=utf8')
    filename = "stock_basics"
    back.to_sql(filename, engine, if_exists='append',index_label=['index'])

def get_all_index_K_yangbaoyin(data):
    good = {}
    stock_list = ts.get_stock_basics()
    for code in stock_list.index:
        hist = ts.get_hist_data(code)
        good[code] = get_index_K_yangbaoyin(hist)
    return good

    def basic_info(self,retry=5):
        engine = get_engine('db_stock')

        # 需要添加异常处理 重试次数
        count = 0

        while count < retry:
            try:
                df = ts.get_stock_basics()

            except Exception as e:
                logger.info(e)
                time.sleep(10)
                count+=1
                continue
            else:
                if df is not None:
                    df=df.reset_index()
                    df['更新日期']=datetime.datetime.now()

                    df.to_sql('tb_basic_info',engine,if_exists='replace')
                    logger.info('入库成功')
                    break
                else:
                    count+=1
                    time.sleep(10)
                    continue

def today_df_filter0(today_df):
    #"""
    today_df = ts.get_today_all()
    today_df = today_df[today_df.amount>0]
    today_df_high_open = today_df[today_df.open>today_df.settlement*1.005]
    all_trade_code = today_df['code'].values.tolist()
    all_a_code = ps.get_all_code(hist_dir="C:/中国银河证券海王星/T0002/export/")
    all_stop_code = list(set(all_a_code).difference(set(all_trade_code)))
    print('\n')
    print('all_stop_code=%s' % all_stop_code)
    print(len(all_stop_code))
    high_open_code_str = today_df_high_open['code'].values.tolist()
    print('all_trade_code = %s'%all_trade_code)
    print(len(all_trade_code))
    print('today_df_high_open = %s'%high_open_code_str)
    today_df['star'] = ((today_df['trade']-today_df['open'])/(today_df['high']-today_df['low'])).round(3)
    today_df['star_h'] = np.where(today_df['star']>=0, ((today_df['high']-today_df['trade'])/(today_df['high']-today_df['low'])).round(3),
                                  ((today_df['high']-today_df['open'])/(today_df['high']-today_df['low'])).round(3))
    today_df['atr'] = np.where((today_df['high']-today_df['low'])<(today_df['high']-today_df['settlement']),
                                today_df['high']-today_df['settlement'],today_df['high']-today_df['low']) #temp_df['close'].shift(1)-temp_df['low'])
    today_df['atr'] = np.where(today_df['atr']<(today_df['settlement']-today_df['low']),
                             (today_df['settlement']-today_df['low']),today_df['atr'])
    today_df['atr_r'] = ((today_df['atr']/today_df['settlement']).round(3))*100.0
    today_df['star_chg'] = today_df['star'] * today_df['changepercent']
    #del today_df['atr']
    describe_df = today_df.describe().round(3)
    #print(type(describe_df))
    lt_describe = describe_df.loc['25%']#.iloc[3].values
    mean_chg = describe_df.loc['mean','changepercent']
    most_chg = describe_df.loc['75%','changepercent']
    least_chg = describe_df.loc['25%','changepercent']
    most_atr_r = describe_df.loc['75%','atr_r']
    least_atr_r = describe_df.loc['25%','atr_r']
    print(mean_chg,most_chg,least_chg,most_atr_r,least_atr_r)
    print(describe_df)
    great_rate = 2.0
    gt_today_df = today_df[today_df.changepercent> great_rate]
    great_atd_df = today_df[today_df['atr_r']>11]
    stock_basic_df=ts.get_stock_basics()
    #stock_basic_df['outstanding'] = stock_basic_df['outstanding'] * 0.0001
    #stock_basic_df['totals'] = stock_basic_df['totals'] * 0.0001
    lt_outstanding_df = stock_basic_df[stock_basic_df.outstanding<100000] #流通股本小于10亿
    print(lt_outstanding_df)
    
    today_df['real'] = 0.00000001 *today_df['amount']/today_df['turnoverratio']
    min_atr_df = today_df[today_df.real<10.0]
    min_atr_df = min_atr_df[min_atr_df.atr_r<least_atr_r]
    lt_outstanding_df_list = lt_outstanding_df.index.values.tolist()
    print(lt_outstanding_df_list)
    min_atr_df = min_atr_df.set_index('code')
    min_atr_df_list = min_atr_df.index.values.tolist()
    print(min_atr_df_list)
    inter_list = list(set(min_atr_df_list).intersection(set(lt_outstanding_df_list)))
    print(len(inter_list))
    filter_df = min_atr_df[min_atr_df.index.isin(inter_list)]
    #print(filter_df)
    #print(type(filter_df))
    min_atr_df = filter_df.sort_values(axis=0, by='atr_r', ascending=True)
    min_atr_df = filter_df.sort_values(axis=0, by='star', ascending=False)
    print(min_atr_df)

	def get_code_list():
		stock_list_csv_path = STOCK_BASICS_DATA_PATH
		if os.path.isfile(stock_list_csv_path):
			stock_list_df = pd.read_csv(stock_list_csv_path, dtype={"code":"object"})
		else:
			stock_list_df = ts.get_stock_basics()
		return stock_list_df["code"]

def QA_fetch_get_stock_info(name):
    data = QATs.get_stock_basics()
    data_json = QA_util_to_json_from_pandas(data)

    for i in range(0, len(data_json) - 1, 1):
        data_json[i]['code'] = data.index[i]
    return data_json

def load_comp_basic(session):
    log_file = (os.getcwd() + os.sep + "logs" + os.sep + "load_comp_basic_%s.log") % (
    dt.datetime.now().strftime('%Y-%m-%d'))
    logger = get_logger(log_file,'basic_load')

    logger.info('Daily company basic information load begin')
    logger.info("##########################################")

    logger.info("Begin load data from Tushare")
    # 获得上市公司的基本数据
    code_list = ts.get_stock_basics()
    code_list = code_list.sort_index()
    logger.info(("Get total %d stocks") % (code_list.index.__len__()))
    try:
        logger.info("Truncate table stock.comp_basic")
        session.execute(str("truncate table stock.comp_basic"))
        session.commit()

        logger.info("Insert data into table stock.comp_basic")
        code_list.insert(0,'code',code_list.index)
        code_list.timeToMarket  =code_list.timeToMarket.apply(lambda x:dt.datetime.strptime(x.__str__(),"%Y%m%d") if x !=0 else None)

        engine = session.get_bind()
        code_list.to_sql('comp_basic', engine, schema='stock', index=False, if_exists='append')

        logger.info("Load successfully")

        return code_list
    except Exception as e:
        logger.error("Error when get basic data")
        return pd.DataFrame()
    finally:
        session.close()

def stock_list():
    df = ts.get_stock_basics()
    data = df.to_dict('index')
    for code, value in sorted(data.items()):
        # print(code)
        # print(value['name'])
        Stock.objects(code=code).update_one(code=code, name=value['name'], upsert=True)

    def save_data_excel(self):
        df = ts.get_stock_basics()

        df.to_csv(self.today + '.csv', encoding='gbk')
        df_x = pd.read_csv(self.today + '.csv', encoding='gbk')
        df_x.to_excel(self.today + '.xls', encoding='gbk')
        os.remove(self.today + '.csv')

def QA_save_stock_day_with_fqfactor(client=DATABASE):
    df = ts.get_stock_basics()

    __coll = client.stock_day
    __coll.ensure_index('code')

    def saving_work(i):
        QA_util_log_info('