def cluster_vs_meta_granger_TM(c,X,M,Ml,lags=7,thresh=0.05):
	# use the Toda Yamamoto method (environmental data is stationary, but clusters are not)
	x1 = X[c].sum(0)
	adf = stattools.adfuller(x1,maxlag=lags)
	if (adf[0] > adf[4]['5%']):
		m1 = adf[2]
	else:
		m1 = 0
	R = []
	for j,x2 in enumerate(M):
		have_values = np.isfinite(x2)
		xi = x1[have_values]
		x2i = x2[have_values]
		adf = stattools.adfuller(x2i,maxlag=lags)
		if (adf[0] > adf[4]['5%']):
			m2 = adf[2]
		else:
			m2 = 0
		m = max(m1,m2)
		y = [xi[i+max(0,m2-m1):len(xi)+i-(m1+lags)] for i in range(m1+lags)] + [x2i[i+max(0,m1-m2):len(xi)+i-(m2+lags)] for i in range(m2+lags)]
		y = np.array(y).T
		lm = linear_model.OLS(xi[max(m1,m2)+lags:],y)
		result = lm.fit()
		Restr = np.eye(y.shape[1])[m+lags:]
		wald = result.wald_test(Restr)
		if wald.pvalue < thresh:
			R.append((wald.pvalue,Ml[j]))
	return m,sorted(R)

 def __init__(self):
     self.res1 = adfuller(self.y, regression="nc", autolag=None, maxlag=1)
     self.teststat = -2.4511596
     self.pvalue = 0.013747  # Stata does not return a p-value for noconstant
     # this value is just taken from our results
     self.critvalues = [-2.587, -1.950, -1.617]
     _, _1, _2, self.store = adfuller(self.y, regression="nc", autolag=None, maxlag=1, store=True)

def diff_nonstationary(x, alpha):
    
    """Returns number of differentiations required to transform
    a non-stationary time series into a stationary one. If 0 (zero) is
    returned, there's no need to differentiate."""
    """
    PARAMETERS:
    1) x - input time series
    2) alpha - significance level
    """
    
    i = 0 # no need to differentiate
    pvalue = adfuller(x, regression =
            ('ct' if
            stats.linregress( pd.Series(range(1, len(x)+1)), x ).pvalue<alpha
            else 'c')
            )[1]
    while pvalue>alpha:
        x = x.diff()
        pvalue = adfuller(x.dropna(),
            regression = 'c')[1]
        i += 1
        if pvalue<=alpha:
            break
    return(int(i))
    
### End of code

    def ADF(self, v, crit='5%', max_d=6, reg='nc', autolag='AIC'):
        """ Augmented Dickey Fuller test

        Parameters
        ----------
        v: ndarray matrix
            residuals matrix

        Returns
        -------
        bool: boolean
            true if v pass the test
        """

        boolean = True
        try:
            l = v.shape[1]
            for j in range(l):
                adf = adfuller(v[:, j], max_d, reg, autolag)

                if(adf[0] < adf[4][crit]):
                    pass
                else:
                    boolean = False
                    break
        except:
            adf = adfuller(v, max_d, reg, autolag)
            if(adf[0] > adf[4][crit]):
                boolean = False

        return boolean

 def setup_class(cls):
     cls.res1 = adfuller(cls.y, regression="nc", autolag=None,
             maxlag=1)
     cls.teststat = -2.4511596
     cls.pvalue = 0.013747 # Stata does not return a p-value for noconstant
                            # this value is just taken from our results
     cls.critvalues = [-2.587,-1.950,-1.617]
     _, _1, _2, cls.store = adfuller(cls.y, regression="nc", autolag=None,
                                      maxlag=1, store=True)

def test_adfuller_short_series(reset_randomstate):
    y = np.random.standard_normal(7)
    res = adfuller(y, store=True)
    assert res[-1].maxlag == 1
    y = np.random.standard_normal(2)
    with pytest.raises(ValueError, match='sample size is too short'):
        adfuller(y)
    y = np.random.standard_normal(3)
    with pytest.raises(ValueError, match='sample size is too short'):
        adfuller(y, regression='ct')

def testADFTest():
    import statsmodels.tsa.stattools as sts
    import statsmodels.stats.stattools as sss
    import numpy as np
    data =np.random.randn(100)
    #http://statsmodels.sourceforge.net/stable/generated/statsmodels.tsa.stattools.adfuller.html
    print sts.adfuller(data)
    
    #http://statsmodels.sourceforge.net/stable/generated/statsmodels.stats.stattools.jarque_bera.html
    print sss.jarque_bera(data)

def adftest(y, short_flag):
	'''Augmented Dicky-Fuller test for given timeseries.
	When test-statistics (first returned value) is absolutely less than critical values,
	process could be considered as stationary one.'''
	sep = 32 * '--'
	print "\n\t\tAugmented Dicky-Fuller test\n"
	if short_flag:
		stationarity = ["stationary", "nonstationary"]

		test_c = adfuller(y, regression='c')
		stat_c = 1 if test_c[0] > test_c[4]['5%'] else 0

		test_ct = adfuller(y, regression='ct')
		stat_ct = 1 if test_ct[0] > test_ct[4]['5%'] else 0

		test_ctt = adfuller(y, regression='ctt')
		stat_ctt = 1 if test_ctt[0] > test_ctt[4]['5%'] else 0
		
		test_nc = adfuller(y, regression='nc')
		stat_nc = 1 if test_nc[0] > test_nc[4]['5%'] else 0

		print sep
		print "- constant only:\t\t\t\t{}".format(stationarity[stat_c])
		print "- constant and trend:\t\t\t\t{}".format(stationarity[stat_ct])
		print "- constant, and linear and quadratic trend:\t{}".format(stationarity[stat_ctt])
		print "\n- no constant, no trend:\t\t\t{}".format(stationarity[stat_nc])
		print sep	
	else:
		print "- constant only\n{}".format(adfuller(y,regression='c'))
		print "- constant and trend\n{}".format(adfuller(y,regression='ct'))
		print "- constant, and linear and quadratic trend\n{}".format(adfuller(y,regression='ctt'))
		print "\n- no constant, no trend\n{}".format(adfuller(y,regression='nc'))
		print sep

 def summarize_all(self):
     if len(self.independent) == 1:
         dependent = self.dependent
         independent = self.independent[0]
         params = self.result.params
         result = self.result
         k = params[1]
         b = params[0]
         conf = result.conf_int()
         cadf = adfuller(result.resid)
         if cadf[0] <= cadf[4]['5%']:
             boolean = 'likely'
         else:
             boolean = 'unlikely'
         print
         print ("{:^40}".format("{} vs {}".format(dependent.upper(), independent.upper())))
         print ("%20s %s = %.4f * %s + %.4f" % ("Model:", dependent, k, independent, b))
         print ("%20s %.4f" % ("R square:", result.rsquared))
         print ("%20s [%.4f, %.4f]" % ("Confidence interval:", conf.iloc[1, 0], conf.iloc[1, 1]))
         print ("%20s %.4f" % ("Model error:", result.resid.std()))
         print ("%20s %s" % ("Mean reverting:", boolean))
         print ("%20s %d" % ("Half life:", half_life(result.resid)))
     else:
         dependent = self.dependent
         independent = self.independent  # list
         params = self.result.params
         result = self.result
         b = params[0]
         conf = result.conf_int()  # pandas
         cadf = adfuller(result.resid)
         if cadf[0] <= cadf[4]['5%']:
             boolean = 'likely'
         else:
             boolean = 'unlikely'
         print
         print ("{:^40}".format("{} vs {}".format(dependent.upper(), (', '.join(independent)).upper())))
         string = []
         for i in range(len(independent)):
             string.append("%.4f * %s" % (params[independent[i]], independent[i]))
         print ("%20s %s = %s + %.4f" % ("Model:", dependent, ' + '.join(string), b))
         print ("%20s %.4f" % ("R square:", result.rsquared))
         string = []
         for i in range(len(independent)):
             string.append("[%.4f, %.4f]" % (conf.loc[independent[i], 0], conf.loc[independent[i], 1]))
         print ("%20s %s" % ("Confidence interval:", ' , '.join(string)))
         print ("%20s %.4f" % ("Model error:", result.resid.std()))
         print ("%20s %s" % ("Mean reverting:", boolean))
         print ("%20s %d" % ("Half life:", half_life(result.resid)))

def ADF(ticker,start,end):
    print('ADF')
    
    stock = DataReader(ticker, "yahoo", start, end)
    
    result = ts.adfuller(stock['Adj Close'], 1)
    print(result)
    print('')
    
    test = result[0]
    crit = result[4]
    one = crit['1%']
    five = crit['5%']
    ten = crit['10%']
    
    if test<one:
        print('Lesser than 1%')
        print('-----------------------------------------')
        return stock
        
    if test<five:
        print('Lesser than 5%')
        print('-----------------------------------------')
        return stock
        
    if test<ten:
        print('Lesser than 10%')
        print('-----------------------------------------')
        return stock
        
    print('Cannot reject Null Hypothesis')
    print('-----------------------------------------')
    return stock

def is_stationary(x, p = 10):

    x = np.array(x)
    result = ts.adfuller(x, regression='ctt')
    #1% level
    if p == 1:
        #if DFStat <= critical value
        if result[0] >= result[4]['1%']:        #DFstat is less negative
            #is stationary
            return True
        else:
            #is nonstationary
            return False
    #5% level
    if p == 5:
        #if DFStat <= critical value
        if result[0] >= result[4]['5%']:        #DFstat is less negative
            #is stationary
            return True
        else:
            #is nonstationary
            return False
    #10% level
    if p == 10:
        #if DFStat <= critical value
        if result[0] >= result[4]['10%']:        #DFstat is less negative
            #is stationary
            return True
        else:
            #is nonstationary
            return False

def testStationarity(ts):
    dftest = adfuller(ts)
    # 对上述函数求得的值进行语义描述
    dfoutput = pd.Series(dftest[0:4], index=['Test Statistic','p-value','#Lags Used','Number of Observations Used'])
    for key,value in dftest[4].items():
        dfoutput['Critical Value (%s)'%key] = value
    return dfoutput

def dickeyfuller_fcn(data,maxlag):
    #@FORMAT: data = np(values)
    try:
        df_fcn = adfuller(data,maxlag)
        return df_fcn[1]
    except:
        return np.nan

    def test_stationarity(self, timeseries, window, return_plot=False):
        
        dftest = adfuller(timeseries, autolag='AIC')
        dfoutput = pd.Series(dftest[0:4], index=['Test Statistic', 'p-value', '#Lags Used', 'Number of Observations Used'])
        for key, value in dftest[4].items():
            dfoutput['Critical Value (%s)' % key] = value       
        
        if return_plot:
            # Determing rolling statistics
            rolmean = timeseries.rolling(window = window, center = False).mean()

            rolstd = timeseries.rolling(window = window, center = False).std()
        
            # Plot rolling statistics:
            orig = plt.plot(timeseries, color='blue', label='Original')
            mean = plt.plot(rolmean, color='red', label='Rolling Mean')
            std = plt.plot(rolstd, color='black', label='Rolling Std')
            plt.legend(loc='best')
            plt.title('Rolling Mean & Standard Deviation')
            plt.show(block=False)
        
            # Perform Dickey-Fuller test:
            print('Results of Dickey-Fuller Test:')
                
            print(dfoutput)
            
            
        return dfoutput

    def test_frame_timeseries_dickey_fuller_constant_trend_squared(self):
        """Test Augmented Dickey Fuller with constant, trend, and trend squared regression"""
        result = self.frame.timeseries_augmented_dickey_fuller_test("logM", max_lag=1, regression="ctt")
        df_ctt_result = smtsa.adfuller(self.pandaframe["logM"], maxlag=1, regression="ctt")

        self.assertAlmostEqual(result.p_value, df_ctt_result[1], delta=0.0001)
        self.assertAlmostEqual(result.test_stat, df_ctt_result[0], delta=0.01)

    def test_frame_timeseries_dickey_fuller_no_constant(self):
        """Test Augmented Dickey Fuller with no constant regression"""
        result = self.frame.timeseries_augmented_dickey_fuller_test("logM", max_lag=1, regression="nc")
        df_nc_result = smtsa.adfuller(self.pandaframe["logM"], maxlag=1, regression="nc")

        self.assertAlmostEqual(result.p_value, df_nc_result[1], delta=0.0001)
        self.assertAlmostEqual(result.test_stat, df_nc_result[0], delta=0.01)

def passes_dftest(data):
	if statmodel.adfuller(data[0][1][1], 250, 'ctt', 't-stat', False, False)[0] < 1:
		data[0][1][0] = True
		return data
	else:
		data[0][1][0] = False
		return data

def runSingleMRTest(ticker):
	dr = DataReader(ticker, "yahoo", datetime(datetime.now().year-1,datetime.now().month,datetime.now().day), datetime.now())
	ts = dr['Adj Close']

	# ADF test with a lag order value of 2
	adf = ts_tool.adfuller(ts, 2)
	print adf[0] # adf test-statistic
	print adf[1] # p-value
	print adf[4] # 1/5/10 test statistic
	print hurst(ts)

	# calculate half-life  
	md = sm.OLS(ts.diff(), sm.add_constant(ts.shift()), missing='drop')  
	mdf = md.fit()  
	half_life = -np.log(2)/mdf.params[1]  
	lookback = np.round(half_life) 

	print lookback

	# calculate VaR using Variance-covariance
	c = 0.95
	rets = ts.pct_change()
	mu = np.mean(rets)
	sigma = np.std(rets)

	# The VaR returned is daily possible loss
	# to convert it to monthly, mu = mu * sqrt(20)
	# or annually, mu = mu * sqrt(250)
	alpha = norm.ppf(1-c, mu, sigma)

	print -alpha

def stationarity_test(ts):
    """@ brief Helper function to determine if a series is stationary."""
    # Determining rolling mean and variance.
    rol_mean = ts.rolling(window=12, center=False).mean()
    rol_std = ts.rolling(window=12, center=False).std()

    # Plot rolling statistics.
    plt.plot(ts, color='#da4264', label='Original')
    plt.plot(rol_mean, color='#391a5a', label='Rolling Mean')
    plt.plot(rol_std, color='#369acd', label='Rolling Std')
    plt.legend(loc='best')
    plt.title('Rolling Mean and Standard Deviation')
    plt.show()

    # Perform Dickey-Fuller test:
    print 'Results of Dickey-Fuller Test:'
    dftest = adfuller(ts, autolag='AIC')
    dfoutput = pd.Series(dftest[0:4], index=['Test Statistic',
                                             'p-value',
                                             '#Lags Used',
                                             'Number of Observations Used'])
    for key, value in dftest[4].items():
        dfoutput['Critical Value (%s)' % key] = value
    print dfoutput
    print "\nConclusion:"
    for key, value in dftest[4].items():
        if dfoutput['Test Statistic'] < dfoutput['Critical Value (%s)' % key]:
            print "Non-stationary series: Reject",
        else:
            print "Non-stationary series: Accept",
        print "at %s level" % key

 def teste_estacionariedade(self, timeseries):
     
     '''
     Este metodo tem por testar a estacionariedade de uma serie com o teste adfuller
     :param: timeseries: serie temporal, array
     :return: print com as estatisticas do teste
     '''
     
     #Determing rolling statistics
     timeseries = pd.DataFrame(timeseries)
     rolmean = timeseries.rolling(window=12, center=False).mean()
     rolstd = timeseries.rolling(window=12, center=False).std()
         
     #Perform Dickey-Fuller test:
     print('Results of Dickey-Fuller Test:')
     timeseries = timeseries[1:].values
     dftest = adfuller(timeseries, autolag='AIC')
     dfoutput = pd.Series(dftest[0:4], index=['Test Statistic','p-value','#Lags Used','Number of Observations Used'])
     for key,value in dftest[4].items():
         dfoutput['Critical Value (%s)'%key] = value
     print(dfoutput)
     
 
     #Plot rolling statistics:
     orig = plt.plot(timeseries, color='blue',label='Original')
     mean = plt.plot(rolmean, color='red', label='Rolling Mean')
     std = plt.plot(rolstd, color='black', label = 'Rolling Std')
     plt.legend(loc='best')
     plt.title('Rolling Mean & Standard Deviation')
     plt.show()