def test_convert_to_annual(self):
     "Test convert_to_annual"
     base = dict(D=1, H=24, T=24 * 60, S=24 * 3600)
     #for fq in ('D', 'H', 'T', 'S'):
     # Don't test for minuTe and Second frequency, too time consuming.
     for fq in ('D', 'H'):
         dates = date_array(start_date=Date(fq, '2001-01-01 00:00:00'),
                            end_date=Date(fq, '2004-12-31 23:59:59'))
         bq = base[fq]
         series = time_series(range(365 * bq) * 3 + range(366 * bq),
                              dates=dates)
         control = ma.masked_all((4, 366 * bq), dtype=series.dtype)
         control[0, :58 * bq] = range(58 * bq)
         control[0, 59 * bq:] = range(58 * bq, 365 * bq)
         control[[1, 2]] = control[0]
         control[3] = range(366 * bq)
         test = convert_to_annual(series)
         assert_equal(test, control)
     #
     series = time_series(range(59, 365) + range(366) + range(365),
                          start_date=Date('D', '2003-03-01'))
     test = convert_to_annual(series)
     assert_equal(test[:, 59:62],
                  ma.masked_values([[-1, 59, 60], [59, 60, 61], [-1, 59, 60]],
                                   - 1))

def add_diurnal(tseries, sine_period, peak_mag):
    """
    Scales a time series to a sine wave of peak_mag with sine_period.
    Input: tseries, sine_period (float, hrs), peak_mag (float)
    Output: scaled_data (array-like)
    """
    # Convert sine_period to same frequency as tseries
    # Create a time delta of magnitude sine_period
    # Convert that time delta into frequency units same as tseries
    zero_date = ts.now('H')
    second_date = zero_date + sine_period
    time_delta = ts.date_array([zero_date, second_date])
    time_delta = time_delta.asfreq(tseries.freq)
    sine_period = float(time_delta[1] - time_delta[0])
    
    angular_freq = (2. * np.pi) / sine_period
    
    for i in range(len(tseries)-1):
        passed_time = float(tseries.dates[i]- tseries.start_date)
        sine_factor = peak_mag * np.sin(angular_freq * passed_time)
        tseries[i] = tseries[i] + tseries[i] * sine_factor
    
    return tseries

# Generate power density function (pdf) to create synthetic TPM from
# mean, stdev, autocorr, npointsx

# def gen_pdf(desired_mean, desired_stdev, bin_width):
    ## TODO
    return 0

 def test_sorted(self):
     dates = [ts.Date('D', string='2007-01-%02i' % i) for i in (3, 2, 1)]
     (a, b) = zip(*[(3., 30), (2., 20), (1., 10), ])
     ndtype = [('a', np.float), ('b', np.int)]
     controldates = date_array(dates, freq='D')
     controldates.sort_chronologically()
     series = time_series(zip(*(a, b)), dates, freq='D', dtype=ndtype)
     assert_equal(series._data.tolist(), [(1., 10), (2., 20), (3., 30)])
     assert_equal(series._dates, controldates)
     #
     trec = time_records(zip(*(a, b)), dates, freq='D', dtype=ndtype)
     assert_equal(trec._data.tolist(), [(1., 10), (2., 20), (3., 30)])
     assert_equal(trec._dates, controldates)
     assert_equal(trec['a'], [1., 2., 3.])
     assert_equal(trec.a, [1., 2., 3.])
     #
     trec = fromrecords(zip(a, b), dates, names=('a', 'b'))
     assert_equal(trec._data.tolist(), [(1., 10), (2., 20), (3., 30)])
     assert_equal(trec._dates, controldates)
     assert_equal(trec['a'], [1., 2., 3.])
     assert_equal(trec.a, [1., 2., 3.])
     #
     trec = fromarrays([a, b], dates, names=('a', 'b'))
     assert_equal(trec._data.tolist(), [(1., 10), (2., 20), (3., 30)])
     assert_equal(trec._dates, controldates)
     assert_equal(trec['a'], [1., 2., 3.])
     assert_equal(trec.a, [1., 2., 3.])

def _collect_and_plot(files):
    TS = []
    location = []
    for f in files:
        temperatures = [ section[1] for section in parse.parse_file(f)[1:-1] if section[1]['Plant'] == ['tmp'] ]
        for t in temperatures:
            if t['Step'][0] != '0000-00-00.01:00:00':
                print 'Not hourly readings of temperature. Abort.'
                break
            dates = ts.date_array(start_date=ts.Date('H', t['Start'][0]), length=len(t['Value']))
            data = [ float(value.rsplit('/')[0]) for value in t['Value'] ]
            TS.append(ts.TimeSeries(data=data, dates=dates))
            if location and t['Installation'][0] != location:
                print 'Location changed during reading of gs2 files. Probably some bad grouping of gs2 files.'
            location = t['Installation'][0]
    if TS:
        path = '/Users/tidemann/Documents/NTNU/devel/data/eklima/Telemark/'
        for file in os.listdir(path):
            try:
                series = xml.parse(path + file)
                sg.utils.plot_time_series([ts.concatenate((TS)), series], ['b-','r-'], [location, file])
            except:
                print file, 'had no data.'
    else:
        print 'No temperature data.'

    def test_tsfromtxt(self):
        "Tests reading from a text file."
        fcontent = """#
'Dates', 'One (S)','Two (I)','Three (F)','Four (M)','Five (-)','Six (C)'
'2007-01', 'strings',1,1.0,'mixed column',,1
'2007-02', 'with embedded "double quotes"',2,2.0,1.0,,1
'2007-03', 'strings',3,3.0E5,3,,1
'2007-05','strings',4,-1e-10,,,1
"""
        import os
        from datetime import datetime
        import tempfile
        (tmp_fd, tmp_fl) = tempfile.mkstemp()
        os.write(tmp_fd, fcontent)
        os.close(tmp_fd)

        mrectxt = tsfromtxt(tmp_fl, delimiter=',', names=tuple('ABCDEFG'),
                               datecols=0, skip_header=2, asrecarray=True)
        os.remove(tmp_fl)
        #
        dlist = ['2007-%02i' % i for i in (1, 2, 3, 5)]
        self.failUnless(isinstance(mrectxt, TimeSeriesRecords))
        assert_equal(mrectxt._dates, date_array(dlist, 'M'))
        assert_equal(mrectxt.dtype.names, ['A', 'B', 'C', 'D', 'E', 'F'])
        assert_equal(mrectxt.F, [1, 1, 1, 1])
        assert_equal(mrectxt.E._mask, [1, 1, 1, 1])
        assert_equal(mrectxt.C, [1, 2, 300000, -1e-10])

 def _get_tseriesD(freq,date_values,kw):
     v = [ xlrd.xldate_as_tuple(int(d),0) 
           for i,d in enumerate(date_values.flatten()) 
           if not np.isnan(d)  ]
     D = [ ts.Date(freq=str(freq),year=_v[0],month=_v[1],day=_v[2]) for _v in v]
     date_array = ts.date_array(D)
     return date_array

def _get_year(year,d,n=1):
    """
    Return a time-series with the same frequency of the input time-series
    with n complete years from input year and values taken from input series

    :param year: base year
    :type year: integer
    :param d: time-series object
    :type d: time-series
    :param n: number of periods to take
    :type n: integer
    :return: output time-series
    :rtype: time-series
    
    """

    f    = d.freqstr      # frequenza d'ingresso
    nels = _ts_nels(f)    # numero di elementi da considerare in un anno (M=12, Q=4, A=1)

    N=n*nels              # Numero di elementi totali da considerare
    
    startd = d.start_date
    endd   = d.end_date

    if f[0]=='M':
        starty = ts.Date(f,year=year,month=1)
        endy   = ts.Date(f,year=year,month=N)
    elif f[0]=='Q':
        starty = ts.Date(f,year=year,quarter=1)
        endy   = ts.Date(f,year=year,quarter=N)
    elif f[0]=='A':
        starty = ts.Date(f,year=year)
        endy   = ts.Date(f,year=year+N-1)
    else:
        raise UnknownFrequencyError, f

    # Create a timeseries with N elements np.nan
    # from starty with frequency f
    s = ts.time_series([ np.nan for i in range(0,N)],
                       start_date=starty,
                       freq=f)

    # create date range
    da = ts.date_array(start_date=starty,
                       end_date=endy,
                       freq=f)

    d.fill_missing_dates()
    d.adjust_endpoints()

    # copy values from d to s
    d.mask=False
    for _d in da:
        s[_d]=np.nan
        if _d <= d.end_date:
            s[_d]=d[_d]
        else:
            s[_d]=np.nan

    return s

    def request(self,reqs,**kw):
        """Random Request

        >>> ds = Random()
        >>> xs = ds.request([{'Instrument':'NAME=PIPPO~:2012-12-31~2007-01-01~M', 'Source': 'FRED'},])
        >>> ts = xs['PIPPO']._data
        >>> print ts.freqstr
        M
        """
        logger.debug('request')
        rx={}
        kw=udict(kw)
        if 'SEED' in kw:
            seed = int(kw['SEED'])
            rand.seed(seed)
        for rq in reqs:
            logger.debug("Request: %s",rq)
            s = parse_instrument(rq['Instrument'])
            logger.debug(s)
            params = ldict(mk_params(s['TICKER']))
            if 'name' in params:
                name = params['name'].upper()
                del params['name']
            else:
                name = rq['Source'].upper()
            if 'NAME' in kw and kw['NAME']:
                name = kw['NAME'].upper()
            kw['NAME']=name
            if s['FREQ']!='0':
                dr = ts.date_array(freq=s['FREQ'],start_date=s['START'],end_date=s['END'])
                L = len(dr)
            else:
                L = s['END']-s['START']
            params['size']=L
            dd = np.zeros(L)
            logger.debug(params)
            try:
                if re.match('^constant$',rq['Source'],re.I):
                    name = 'CONSTANT'
                    v = float(name)
                    params['LOW']=v
                    params['HIGH']=v
                    dd = rand.uniform(**params)
                elif re.match('^walk$',rq['Source'],re.I):
                    dd = RandomWalk(**params)
                elif re.match('^uniform|beta|binomial|chisquare|exponential|gamma|geometric|gumbel|hypergeometric|laplace|logistic|lognormal|logseries|multinomial|multivariate_normal|negative_binomial|noncentral_chisquare|noncentral_f|normal|pareto|poisson|power|rayleigh|standard_cauchy|standard_exponential|standard_gamma|standard_normal|standard_t|triangular|uniform|vonmises|wald|weibull|zipf$',rq['Source'],re.I):
                    generator = rq['Source'].lower()
                    if hasattr(rand,generator):
                        f = getattr(rand,generator)
                        dd = f(**params)
                ## Add other distributions here
                else:
                    raise TypeError, 'Unknown generator'
            except TypeError, exc:
                logging.error('%s generator - %s',rq['Source'],exc.args[0])
            except:

 def _make_predict_dates(self):
     data = self._data
     dtstart = data.predict_start
     dtend = data.predict_end
     freq = data.freq
     #pandas_freq = _freq_to_pandas[freq]
     dates = date_array(start_date=dtstart, end_date=dtend,
                             freq=freq).toordinal().astype(int)
     self._data.predict_dates = asarray(
             [datetime.datetime.fromordinal(i) for i in dates])

 def _get_tseriesQ(freq,date_values,kw):
     # print date_values
     by=0
     if kw.has_key('YEAR'):
         by = eval(kw['YEAR'])-1
     v = [ (int(d)-1)%4+1+(int(_i/4)*4)+by*4 
           for _i,d in enumerate(date_values.flatten()) 
           if not np.isnan(d) ]
     D = [ ts.Date(freq=str(freq),value=_v) for _v in v]
     date_array = ts.date_array(D)
     return date_array

 def setup(self):
     "Generic setup"
     d = np.arange(5)
     m = ma.make_mask([1, 0, 0, 1, 1])
     base_d = np.r_[d, d[::-1]].reshape(2, -1).T
     base_m = np.r_[[m, m[::-1]]].T
     base = ma.array(base_d, mask=base_m)
     mrec = mr.fromarrays(base.T,)
     dlist = ['2007-%02i' % (i + 1) for i in d]
     dates = date_array(dlist)
     mts = time_series(mrec, dates)
     rts = time_records(mrec, dates)
     self.data = [d, m, mrec, dlist, dates, mts, rts]

 def test_dates_on_several_columns(self):
     "Test tsfromtxt when the date spans several columns."
     datatxt = """
     2001, 01, 0.0, 10.
     2001, 02, 1.1, 11.
     2001, 02, 2.2, 12.
     """
     data = StringIO.StringIO(datatxt)
     dateconverter = lambda y, m: Date('M', year=int(y), month=int(m))
     test = tsfromtxt(data, delimiter=',', dtype=float, datecols=(0, 1),
                      dateconverter=dateconverter)
     assert_equal(test, [[0., 10.], [1.1, 11.], [2.2, 12.]])
     assert_equal(test.dates,
                  date_array(['2001-01', '2001-02', '2001-02'], freq='M'))

 def _make_predict_dates(self):
     try:
         from scikits.timeseries import date_array
     except ImportError:
         self._data.predict_dates = None
     data = self._data
     dtstart = data.predict_start
     dtend = data.predict_end
     freq = data.freq
     #pandas_freq = _freq_to_pandas[freq]
     dates = date_array(start_date=dtstart, end_date=dtend,
                             freq=freq).toordinal().astype(int)
     self._data.predict_dates = asarray(
             [datetime.datetime.fromordinal(i) for i in dates])

    def test_with_datecols(self):
        "Test two datecols"
        fcontent = StringIO.StringIO("""
year, month, A, B
2009, 01, 1, 1.
2009, 03, 3, 3.
""")
        dateconv = lambda y, m: Date("M", year=int(y), month=int(m))
        test = tsfromtxt(fcontent, delimiter=",", skip_header=1, names=True,
                         converters={'dates': dateconv}, datecols=(0, 1))
        dates = date_array(['2009-01', '2009-03'], freq='M')
        assert_equal(test.dates.tovalue(), dates)
        assert_equal(test['A'], [1, 3])
        assert_equal(test['B'], [1., 3.])
        assert_equal(test.dtype, np.dtype([('A', int), ('B', float)]))

 def _get_tseriesM(freq,date_values,kw):
     dformat='INT'
     if kw.has_key('DFORMAT'):
         dformat = kw['DFORMAT']
     if dformat=='INT':            
         v = [ (int(d)-1)*12+i%12+1 for i,d in enumerate(date_values.flatten()) if not np.isnan(d)  ]
         D = [ ts.Date(freq=str(freq),value=_v) for _v in v]
     elif dformat=='XL_DATE':
         v = [ xlrd.xldate_as_tuple(d,0) for i,d in enumerate(date_values.flatten()) if not np.isnan(d)  ]
         D = [ ts.Date(freq=str(freq),year=_v[0],month=_v[1]) for _v in v]
     else:
         logger.error('DATE FORMAT NOT SUPPORTED ON EXCEL READING')
         raise ValueError, dformat
                     #            print "FREQ=|%s|"%freq,D
     date_array = ts.date_array(D)
     return date_array

 def setUp(self):
     "Initializes"
     ndtype = [('lin',float),('rand',float)]
     dlin = np.linspace(0,10,120)
     drnd = np.random.rand(120)
     data = np.array(zip(dlin, drnd), dtype=ndtype)
     dates = ts.date_array(start_date=ts.now('M')-120, length=120, freq='M')
     enso = ENSOIndicator(np.random.rand(120) + np.linspace(-1,1,120), 
                          dates=dates,
                          thresholds=(-0.5,0.5),
                          full_year='False', refseason='NDH', minsize=5)
     cdat = data.view(ClimateRecords)
     cdat._dates = dates
     cdat.ensoindicator = enso
     self.dlin = dlin
     self.cdat=cdat
     self.enso=enso

    def test_with_names(self):
        "Tests w/ names"
        fcontent = StringIO.StringIO("""#
'Dates', 'One (S)','Two (I)','Three (F)','Four (M)','Five (-)','Six (C)'
'2007-01', 'strings',1,1.0,'mixed column',,1
'2007-02', 'with embedded "double quotes"',2,2.0,1.0,,1
'2007-03', 'strings',3,3.0E5,3,,1
'2007-05','strings',4,-1e-10,,,1
""")
        test = tsfromtxt(fcontent, delimiter=",", datecols=0, skip_header=2,
                         names="A,B,C,D,E,F", freq='M')
        assert(isinstance(test, TimeSeries))
        dlist = ['2007-%02i' % i for i in (1, 2, 3, 5)]
        assert_equal(test.dates.tovalue(),
                     date_array(dlist, freq='M').tovalue())
        assert_equal(test.dtype.names, ['A', 'B', 'C', 'D', 'E', 'F'])
        assert_equal(test['F'], [1, 1, 1, 1])
        assert_equal(test['E'].mask, [1, 1, 1, 1])
        assert_equal(test['C'], [1, 2, 300000, -1e-10])

#    t = timer()
#    mod_tb = tb.lpc(y, 2)
#    t_end = timer()
#    print str(t_end - t) + " seconds for talkbox.lpc"
#    print """For higher lag lengths ours quickly fills up memory and starts
#thrashing the swap.  Should we include talkbox C code or Cythonize the
#Levinson recursion algorithm?"""

    ## Try with a pandas series
    import pandas
    import scikits.timeseries as ts
    d1 = ts.Date(year=1700, freq='A')
    #NOTE: have to have yearBegin offset for annual data until parser rewrite
    #should this be up to the user, or should it be done in TSM init?
    #NOTE: not anymore, it's end of year now
    ts_dr = ts.date_array(start_date=d1, length=len(sunspots.endog))
    pandas_dr = pandas.DateRange(start=d1.datetime,
                                 periods=len(sunspots.endog), timeRule='[email protected]')
    #pandas_dr = pandas_dr.shift(-1, pandas.datetools.yearBegin)

    dates = np.arange(1700, 1700 + len(sunspots.endog))
    dates = ts.date_array(dates, freq='A')
    #sunspots = pandas.Series(sunspots.endog, index=dates)

    #NOTE: pandas only does business days for dates it looks like
    import datetime
    dt_dates = np.asarray(lmap(datetime.datetime.fromordinal,
                              ts_dr.toordinal().astype(int)))
    sunspots = pandas.Series(sunspots.endog, index=dt_dates)

    #NOTE: pandas can't handle pre-1900 dates

rearranged
1.00796791   0.24449867(-0.00521004)   0.50554663
garch11:
[ 1.01258264  0.24149155  0.50479994]
-2056.3877404
R include_constant=False
Final Estimate:
 LLH:  2056.397    norm LLH:  2.056397
    omega    alpha1     beta1
1.0123560 0.2409589 0.5049154
'''


erro,ho, etaxo = generate_gjrgarch(20, ar, ma, mu=0.04, scale=0.01,
                  varinnovation = np.ones(20))

if 'sp500' in examples:
    import tabular as tb
    import scikits.timeseries as ts

    a = tb.loadSV(r'C:\Josef\work-oth\gspc_table.csv')

    s = ts.time_series(a[0]['Close'][::-1],
                dates=ts.date_array(a[0]['Date'][::-1],freq="D"))

    sp500 = a[0]['Close'][::-1]
    sp500r = np.diff(np.log(sp500))


plt.show()

                        dd = f(**params)
                ## Add other distributions here
                else:
                    raise TypeError, 'Unknown generator'
            except TypeError, exc:
                logging.error('%s generator - %s',rq['Source'],exc.args[0])
            except:
                raise
            if s['FREQ']!='0':
                data = ts.time_series(dd,
                                      freq=s['FREQ'],
                                      dates=dr)            
                if s['NAN']:
                    t=ts.now(data.freqstr)
                    if t<data.end_date:
                        da = ts.date_array(start_date=t,end_date=data.end_date)
                        data[da]=np.nan
                rx[kw['NAME']]=Timeseries(data=data,name=name)
            else: # for a vector
                print type(dd),dd.shape
                xx = dd.reshape((1,-1)) 
                print type(xx),xx.shape
                rx[kw['NAME']]=np.array(dd)
        return rx

# random://normal/loc=10,scale=.2~:2012-12-31~2007-01-01~M?name=UNI

if __name__=="__main__":
    parse_instrument('NAME=PIPPO')
    parse_instrument('NAME=PIPPO~M')
    parse_instrument('NAME=PIPPO~Q')