def get_grouped_data(self, forecast=False):
        cdf = self.cumulative_sum()
        gdf = self.group_by('M')

        if cdf.shape[0] > gdf.shape[0]:
            df = cdf.to_frame()
            df.columns = ['cumulative sum']
            df['total added'] = gdf.to_frame()['event']
        else:
            df = gdf.to_frame()
            df.columns = ['total added']
            df['cumulative sum'] = cdf.to_frame()['event']

        if forecast:
            mtotals = pd.to_numeric(df['cumulative sum'], downcast='float')
            model = ARIMA(mtotals, order=(10,1,0))
            model_fit = model.fit(disp=0)
            forecast = model_fit.forecast(steps=12)
            dates = pd.date_range('2017-04-30', '2018-06-01', freq='M')
            records = zip([x.to_datetime() for x in dates], forecast[0])
            ndf = pd.DataFrame.from_records(records)
            ndf.columns = ['date', 'forecast']
            ndf.set_index(['date'], inplace=True)
            df = pd.concat([df, ndf], axis=1)

        return df

def arima_predict(train_dat, n_predictions, p=2, d=0, q=0):
    arima = ARIMA(np.array(train_dat).astype(np.float), [p, d, q])
    diffed_logged_results = arima.fit(trend='c', disp=False)
    preds = diffed_logged_results.predict(len(train_dat),
                                          len(train_dat) + n_predictions - 1,
                                          exog=None, dynamic=False)
    return preds

def arimamodel(ts):
	ts_log, ts_log_diff = trend(ts)
	model = ARIMA(ts_log, order = (2,1,2))
	result_ARIMA = model.fit(disp = -1)

	m = ARIMA(ts, order = (2,1,2)).fit()

	arimares = ARMAResults(m, params = '')

	pre = arimares.forcast(steps = 60)


	# pre = m.predict('20150901', '20151230', dynamic = True)
	print pre

	# prediction back to the original scale
	predictions_ARIMA = backorg(result_ARIMA, ts_log)
	plt.plot(predictions_ARIMA)
	# print (predictions_ARIMA - ts)[40:80]

	plt.plot(ts, color = 'red')

	# plt.plot(ts_log_diff)
	# plt.plot(result_ARIMA.fittedvalues, color = 'red')
	plt.title('RSS: %.4F' % np.sum((result_ARIMA.fittedvalues - ts_log_diff)**2))
	plt.show()

 def ARIMA_forcast2(self):
     # this approach forecast 1 data pt at a time, then add the new forecast datapoint to the training data
     # then repeat
     import warnings
     warnings.filterwarnings('ignore')
     
     # test without taking log of data
     # using rolling avg 
     y = vr_df2_ts.values
     train = vr_df2_ts.values[286:574]
     prediction = list()
     for t in range(288):
         modelY = ARIMA(y, order=(1,1,1))
         results = modelY.fit(disp=-1)
         out = results.forecast()
         yhat = out[0]
         prediction.append(yhat)
         y = np.append(y,train[t])
         
     forecast = pd.Series(prediction,index=pd.date_range(start='2017-02-09 00:00:00', periods=288,freq='5min'))
     exog = vr_df2_ts.iloc[286:574]
     exog.set_index(pd.date_range(start='2017-02-09 00:00:00', periods=288,freq='5min'),inplace=True)
     
     plt.plot(vr_df2_ts)
     plt.plot(exog,'g')
     plt.plot(forecast,'r')

    def forecast_by_cluster(self, hold_out_n, n_ahead, order, exog):
        dfit = self.ds_agg_by_c
        
        efit = efor = None
        if hold_out_n > 0:
            # hold out validation required
            dfit = dfit[:-hold_out_n]
            if (exog is not None):
                efit = exog[:-hold_out_n]
                efor = exog[-hold_out_n:]
        else:
            if (exog is not None):
                efit = exog[:-n_ahead]
                efor = exog[-n_ahead:]
        ds_c_for = np.zeros((n_ahead, self.n_clusters))

        for c in tqdm(range(self.n_clusters)):
            cdfit = dfit[:,c]
            if sum(cdfit) == 0:
                ds_c_for[:,c] = 0
                continue
            m = ARIMA(cdfit, exog = efit, order = order)
            mf = m.fit()
            f = mf.forecast(n_ahead, exog = efor, alpha = .95)[0]
            ds_c_for[:,c] = f
        
        self.ds_c_for = ds_c_for

def mamodel(ts):
	ts_log, ts_log_diff = trend(ts)
	model = ARIMA(ts_log, order = (0,1,1))
	result_MA = model.fit(disp = -1)
	plt.plot(ts_log_diff)
	plt.plot(result_MA.fittedvalues, color = 'red')
	plt.title('RSS: %.4F' % np.sum((result_MA.fittedvalues - ts_log_diff)**2))
	plt.show(block = False)

def armodel(ts):
	ts_log, ts_log_diff = trend(ts)
	model = ARIMA(ts_log, order = (1,1,0))
	result_AR = model.fit(disp = -1)
	plt.plot(ts_log_diff)
	plt.plot(result_AR.fittedvalues, color = 'red')
	# pdb.set_trace()
	plt.title('RSS: %.4F' % np.sum((result_AR.fittedvalues - ts_log_diff)**2))
	plt.show(block = False)

 def ARIMA_fit(self):
     # order=(p,d,q) AR and MA can also be modeled separately by enter 0 for either p or q
     model = ARIMA(ts_log, order=(5,1,5))
     self.results_ARIMA = model.fit(disp=-1)
     
     print(results_ARIMA.summary())
     
     plt.plot(ts_log_diff)
     plt.plot(results_ARIMA.fittedvalues, color='r')
     plt.title('RSS: %.4f'% sum((results_ARIMA.fittedvalues-ts_log_diff['in_tpkts'])**2))

def ARIMA_fun( data ):
    lag_pacf = pacf( data, nlags=20, method='ols' )
    lag_acf, ci2, Q  = acf( data, nlags=20 , qstat=True, unbiased=True)

    model = ARIMA(orig_data, order=(1, 1, int(ci2[0]) ) )  
    results_ARIMA = model.fit(disp=-1)
    plt.subplot(121)
    plt.plot( data )
    plt.plot(results_ARIMA.fittedvalues)
    #plt.show()
    return results_ARIMA.fittedvalues

 def fit(self):
   if len(self.df) < self.t_window: return None
   model = ARIMA(self.df, order=(2, 1, 1))
   results_ARIMA = model.fit(disp=-1)
   forecast = results_ARIMA.predict(start = self.t_window, end= self.t_window+2, dynamic= True)
   forecast = forecast.cumsum()
   predictions_ARIMA_log = pd.Series(self.df.ix[self.t_window-1], index=forecast.index)
   predictions_ARIMA_log = predictions_ARIMA_log.add(forecast,fill_value=0)
   predictions_ARIMA = np.exp(predictions_ARIMA_log)
   #print self.df
   return predictions_ARIMA

    def objfunc(order, *params):
        series = params

        try:
            mod = ARIMA(series, order, exog=None)
            with warnings.catch_warnings():
                warnings.simplefilter("ignore")
                res = mod.fit(disp=0, solver='bfgs', maxiter=5000)
        except:
            return float('inf')
        if math.isnan(res.aic):
            return float('inf')
        return res.aic

    def pridictNextNdays(self,train):
        timeSerize = train[self.selected]
        timeSerize = timeSerize[self.start_train:self.end_train]
        model = ARIMA(timeSerize, order=(self.p,self.d,self.q), freq='D') # build a model
        fitting = model.fit(disp=False)
        forecast, fcasterr, conf_int = fitting.forecast(steps=self.next_ndays, alpha=.05)

        # params = fitting.params
        # residuals = fitting.resid
        # p = fitting.k_ar
        # q = fitting.k_ma
        # k_exog = fitting.k_exog
        # k_trend = fitting.k_trend
        # forecast = _arma_predict_out_of_sample(params,self.next_ndays,residuals, p, q, k_trend, k_exog, endog=timeSerize, exog=None, start=len(timeSerize))
        return  forecast

 def testArima(self,train):
     realSerize = train[self.selected]
     timeSerize = realSerize[self.start_train:self.end_train]
     realData = train[self.selected][self.end_train:self.next_ndays]
     model = ARIMA(timeSerize, order=(self.p,self.d, self.q)) # build a model
     fitting = model.fit(disp=False)
     forecast, fcasterr, conf_int = fitting.forecast(steps=self.next_ndays, alpha=.05)
     # params = fitting.params
     # residuals = fitting.resid
     # p = fitting.k_ar
     # q = fitting.k_ma
     # k_exog = fitting.k_exog
     # k_trend = fitting.k_trend
     # forecast = _arma_predict_out_of_sample(params,self.next_ndays,residuals, p, q, k_trend, k_exog, endog=timeSerize, exog=None, start=len(timeSerize))
     return  {'real':list(realSerize)[self.end_train:self.end_train+self.next_ndays],'pridiction':forecast}

    def predict_arima_next_days(self, item):
        ts = df_train[item]
        ts = ts.sort_index() # sorting index Date
        ts_last_day = ts[self.fc] # real last data
        ts = ts[0:self.fc] # index 0 until last data - 1

        model = ARIMA(ts, order=(self.p, self.d, self.q)) # build a model
        fitting = model.fit(disp=False)

        # n_days forecasting
        forecast, fcasterr, conf_int = fitting.forecast(steps=self.n_days, alpha=.05)
        # ts:          history until 1 day before self.fc
        # ts[self.fc]: last day
        # forecast:    1 day forecast (time equalto ts[self.fc])
        return ts, ts_last_day, forecast

class ARIMAModelResult:
    def __init__(self, autoregressive_periods, integrated_order, moving_average_model_periods, training_data, test):
        self.autoregressive_periods = autoregressive_periods
        self.integrated_order = integrated_order
        self.moving_average_model_periods = moving_average_model_periods
        self.model = ARIMA(training_data, order=(
            self.autoregressive_periods,
            self.integrated_order,
            self.moving_average_model_periods
        )
                           )
        self.fit = self.model.fit()
        self.aic = self.fit.aic
        self.predictions = self.fit.forecast(steps=len(test))[0]
        self.model_fitness = mean_squared_error(test, self.predictions)

    def __eq__(self, other):
        return self.model_fitness == other.model_fitness

    def __lt__(self, other):
        return self.model_fitness < other.model_fitness

    def __gt__(self, other):
        return self.model_fitness > other.model_fitness

    def __str__(self):
        return "Autoregressive periods: {}\nIntegraded Order: {}\nMoving Average Model Periods: {}\n Predictions: {}\nMSE: {}".format(
            self.autoregressive_periods,
            self.integrated_order,
            self.moving_average_model_periods,
            self.predictions,
            self.model_fitness
        )

def arima(ts, forecast_window):
    logger.info(ts)
    start = int(ts.count() - 1)
    end = int(start + forecast_window)

    ts_log = np.log(ts)
    model = ARIMA(ts_log, order=(0, 1, 2))
    results = model.fit(disp=-1)
    prediction = results.predict(start=start, end=end, dynamic=True)
    future = pd.Series(prediction, copy=True)
    cumsum = future.cumsum()
    prediction_future = future.add(ts_log.ix[-1])
    prediction_future = prediction_future.add(cumsum)
    ts_future = np.exp(prediction_future)

    return ts_future

def predictFutureProfit(df, forward):
	results = {}

	for asset in get_assets(df):
		ts = df[asset]
		ts_log = np.log(ts)

		model = ARIMA(ts_log, order=(1, 1, 0))  
		results_ARIMA = model.fit(disp=-1)  
		predictions_diff = results_ARIMA.predict(2, len(ts.index)-1, dynamic=True)
		predictions_diff_cumsum = predictions_diff.cumsum()
		predictions_log = pd.Series(ts_log.ix[0], index=ts_log.index)
		predictions_log = predictions_log.add(predictions_diff_cumsum,fill_value=0)
		predictions = np.exp(predictions_log)
		results[asset] = predictions[-1]

	return results

def fitArima(ts):
    import statsmodels.api as sm
    logged_ts = np.log(ts)
    diffed_logged_ts = (logged_ts - logged_ts.shift(7))[7:]
    p = 0
    d = 1
    q = 1
    arima = ARIMA(diffed_logged_ts, [p, d, q], exog=None, freq='D', missing='none')
    diffed_logged_results = arima.fit(trend='c', disp=False)
    predicted_diffed_logged = diffed_logged_results.predict(exog=None, dynamic=False)
    #a=pd.date_range(diffed_logged_ts.index[1], periods=90, freq='D')
    predicted_diffed_logged_ts = pd.Series(predicted_diffed_logged, index=diffed_logged_ts.index[d:])
    predicted_diffed_logged_ts = np.exp(logged_ts.shift(7) + diffed_logged_ts.shift(d) + predicted_diffed_logged_ts)
    
    concatenated = pd.concat([ts, predicted_diffed_logged_ts], axis=1, keys=['original', 'predicted'])
    #a= concatenated
    #a.plot()
    #plt.show()
    return concatenated

    def arima(self):
        kl = self.get_kline()
        cp = self.get_close_price(kl)
        date = self.get_date(kl)
        #t = datetime.fromtimestamp(date[-1].timestamp()+24*60*60)
        t = date[-1] + timedelta(days=int(self.day_history/5)) #days seconds ...
        print("predict date:", date[-1],"--->", t)

        dta = pd.Series(cp, index=date)
        print(dta)
        model=ARIMA(dta,order=(4,1,3)) #P D Q
        result=model.fit()
        pred=result.predict( date[-10], t,dynamic=True,typ='levels')
        plt.figure(figsize=(12,8))
        plt.plot(dta, 'ro-')
        plt.xticks(rotation=45)
        plt.plot(pred, 'go-')

        plt.show()

def programmer_5():
    discfile = "data/discdata_processed.xls"
    # 残差延迟个数
    lagnum = 12

    data = pd.read_excel(discfile, index_col="COLLECTTIME")
    data = data.iloc[:len(data) - 5]
    xdata = data["CWXT_DB:184:D:\\"]

    # 训练模型并预测，计算残差
    arima = ARIMA(xdata, (0, 1, 1)).fit()
    xdata_pred = arima.predict(typ="levels")
    pred_error = (xdata_pred - xdata).dropna()

    lb, p = acorr_ljungbox(pred_error, lags=lagnum)
    h = (p < 0.05).sum()
    if h > 0:
        print(u"模型ARIMA（0,1,1)不符合白噪声检验")
    else:
        print(u"模型ARIMA（0,1,1)符合白噪声检验")
    print(lb)