def test_dot_real(data_dict):
    """Dot operator testing with real datasets"""
    data_dir = os.path.join(os.getcwd(), 'data')

    path = os.path.join(data_dir, data_dict['data_name'])
    if not os.path.exists(path):
        get_data(
            data_dir,
            data_dict['data_name'],
            data_dict['url'],
            data_dict['data_origin_name']
        )
        assert os.path.exists(path)

    k = data_dict['feature_dim']
    m = data_dict['m']
    batch_size_list = data_dict['batch_size']

    default_output_index = data_dict['default_index']['output_dim']
    default_batch_size_index = data_dict['default_index']['batch_size']
    density = estimate_density(path, data_dict['feature_dim'])
    num_batches = data_dict['num_batches']

    assert default_batch_size_index < len(batch_size_list)
    assert default_output_index < len(m)
    if ARGS.verbose:
        print("Running Benchmarking on %r data") % data_dict['data_mini']
    print('{:>15} {:>10} {:>10} {:>10} {:>20} {:>15} {:>15} {:>10} {:>10}'.format('density(%)',
                                                                                 'n',
                                                                                 'm',
                                                                                 'k',
                                                                                 't_dense/t_sparse',
                                                                                 't_dense(ms)',
                                                                                 't_sparse(ms)',
                                                                                 'is_transpose',
                                                                                 'rhs_rsp'))


    for output_dim in m:
        _compare_sparse_dense(data_dir, data_dict['data_name'], data_dict['data_mini'],
                              k, output_dim, density,
                              batch_size_list[default_batch_size_index], num_batches)
        _compare_sparse_dense(data_dir, data_dict['data_name'], data_dict['data_mini'],
                              k, output_dim, density,
                              batch_size_list[default_batch_size_index], num_batches,
                              transpose=True)
        _compare_sparse_dense(data_dir, data_dict['data_name'], data_dict['data_mini'],
                              k, output_dim, density,
                              batch_size_list[default_batch_size_index], num_batches, rsp=True)

    for batch_size in batch_size_list:
        _compare_sparse_dense(data_dir, data_dict['data_name'], data_dict['data_mini'],
                              k, m[default_output_index], density, batch_size, num_batches)
        _compare_sparse_dense(data_dir, data_dict['data_name'], data_dict['data_mini'],
                              k, m[default_output_index], density, batch_size, num_batches,
                              transpose=True)
        _compare_sparse_dense(data_dir, data_dict['data_name'], data_dict['data_mini'],
                              k, output_dim, density,
                              batch_size_list[default_batch_size_index], num_batches, rsp=True)

def compute_portvals(start_date, end_date, orders, startval):
    # get the trading days using SPY as reference
    dates = pd.date_range(start_date, end_date)
    df = get_data(['SPY'], dates)
    # Make the sell orders a negative value
    orders['Shares'][orders['Order'].str.upper()=='SELL'] = -orders['Shares'][orders['Order'].str.upper()=='SELL']

    # Create a data frame to hold a matrix of all the stocks
    symbols = np.unique(orders['Symbol'].values.ravel())  
    for stock in symbols: 
        df[stock]=0   
    
    # Get the prices for each day in the index
    # Front fill the prices where we have an NA, then backfill
    prices = get_data(symbols, df.index, False)
    prices = prices.fillna(method='ffill', axis=0)
    prices = prices.fillna(method='bfill', axis=0)

    # Add the starting value and a cash value
    df['Cash'] = startval + 0.0
    prices['Cash'] = 1
    orders['Prices'] = 0
    for ind, row in orders.iterrows():
        # calculate leverage        
        # leverage = (sum(longs) + sum(abs(shorts)) / ((sum(longs) - sum(abs(shorts)) + cash)
        # get temporary table after the transaction is made, and before the transaction is made
        df_chk, df_chk_b4 = df.ix[ind,1:], df.ix[ind,1:]
        df_chk [row['Symbol']] = df[row['Symbol']][ind] + row['Shares']
        df_chk ['Cash'] = df['Cash'][ind] - prices[row['Symbol']][ind] * row['Shares']
        df_chk        = prices.ix[ind] * df_chk
        df_chk_b4  = prices.ix[ind] * df_chk_b4
        # calculate the leverage after and before 
        lev_after = sum(abs(df_chk[:-1])) / sum(df_chk )
        lev_before = sum(abs(df_chk_b4[:-1])) / sum(df_chk_b4 )
        # print lev_after, lev_before, ind
        #if lev_after < 1000.0 or lev_after < lev_before :      
        df[row['Symbol']][ind:end_date] = df[row['Symbol']][ind:end_date] + row['Shares']
        df['Cash'][ind:end_date] = df['Cash'][ind:end_date] - prices[row['Symbol']][ind] * row['Shares']
        #else:
        #    print "Cancel the order", ind, row['Symbol'], row['Shares'], "Lev before", lev_before , "Lev after",  lev_after 
    
    df = df.iloc[:,1:] * prices
    portvals = df.sum(axis=1)
    # print portvals
    return portvals   


#def test_run():
    """Driver function."""
    # Define input parameters
    start_date = '2011-01-05'
    end_date = '2011-01-20'
    orders_file = os.path.join("orders", "orders-short.csv")
    start_val = 1000000

def test_run():
	symbols = ['IBM']
	train_dates = pd.date_range('2008-1-1', '2010-12-31')
	test_dates = pd.date_range('2011-1-1', '2011-12-31')
	training = get_data(symbols, train_dates)
	testing = get_data(symbols, test_dates)
	trainingIBM = training[symbols]
	testingIBM = testing[symbols]

	testing_result = train_model(trainingIBM, testingIBM)
	ax = testing_result[['IBM', 'pred']].plot(title='Predicted market')
	ax.set_xlabel('Date')
	ax.set_ylabel('Price')
	fig = ax.get_figure()
	fig.savefig("output/predicted_market.png")

	generate_orders(testing_result)

	orders_file = os.path.join("orders", "orders.csv")
	start_val = 10000
	# Process orders
	portvals = compute_portvals('2011-1-1', '2011-12-31', orders_file, start_val)
	if isinstance(portvals, pd.DataFrame):
	    portvals = portvals[portvals.columns[0]]  # if a DataFrame is returned select the first column to get a Series
	# print portvals
	# Get portfolio stats
	cum_ret, avg_daily_ret, std_daily_ret, sharpe_ratio = get_portfolio_stats(portvals)
	# Simulate a $SPX-only reference portfolio to get stats
	prices_SPX = get_data(['$SPX'], test_dates)
	prices_SPX = prices_SPX[['$SPX']]  # remove SPY
	portvals_SPX = get_portfolio_value(prices_SPX, [1.0])
	cum_ret_SPX, avg_daily_ret_SPX, std_daily_ret_SPX, sharpe_ratio_SPX = get_portfolio_stats(portvals_SPX)

	print "Data Range: {} to {}".format('2011-1-1', '2011-12-31')
	print
	print "Sharpe Ratio of Fund: {}".format(sharpe_ratio)
	print "Sharpe Ratio of $SPX: {}".format(sharpe_ratio_SPX)
	print
	print "Cumulative Return of Fund: {}".format(cum_ret)
	print "Cumulative Return of $SPX: {}".format(cum_ret_SPX)
	print
	print "Standard Deviation of Fund: {}".format(std_daily_ret)
	print "Standard Deviation of $SPX: {}".format(std_daily_ret_SPX)
	print
	print "Average Daily Return of Fund: {}".format(avg_daily_ret)
	print "Average Daily Return of $SPX: {}".format(avg_daily_ret_SPX)
	print
	print "Final Portfolio Value: {}".format(portvals[-1])

	# Plot computed daily portfolio value
	df_temp = pd.concat([portvals, prices_SPX['$SPX']], keys=['Portfolio', '$SPX'], axis=1)
	plot_normalized_data(df_temp, title="Daily portfolio value and $SPX")

def assess_portfolio(sd = dt.datetime(2008,1,1), ed = dt.datetime(2009,1,1), \
    syms = ['GOOG','AAPL','GLD','XOM'], \
    allocs=[0.1,0.2,0.3,0.4], \
    sv=1000000, rfr=0.0, sf=252.0, \
    gen_plot=False):

    # Read in adjusted closing prices for given symbols, date range
    dates = pd.date_range(sd, ed)
    prices_all = get_data(syms, dates)  # automatically adds SPY
    prices = prices_all[syms]  # only portfolio symbols
    prices_SPY = prices_all['SPY']  # only SPY, for comparison later
    prices_SPY = (prices_SPY/prices_SPY.iloc[0])*sv

    # Get daily portfolio value
    port_val = get_portfolio_value(prices, allocs, sv)
    
    # Get portfolio statistics (note: std_daily_ret = volatility)
    cr, adr, sddr, sr = get_portfolio_stats(port_val, rfr, sf)
    
    # Compare daily portfolio value with SPY using a normalized plot
    if gen_plot:
        # Plot normalized portfolio value.
        df_temp = pd.concat([port_val/sv, prices_SPY/sv], keys=['Portfolio', 'SPY'], axis=1)
        plot_data(df_temp,
                  title="Daily portfolio value and SPY",
                  ylabel="Normalized price")
        
    # Compute end value
    ev = port_val[-1]

    return cr, adr, sddr, sr, ev

def run_strategy():
    symbol = 'IBM'
    start_date = '2007-12-31'
    end_date = '2009-12-31'

    prices_IBM = get_data([symbol], pd.date_range(start_date, end_date))

    bollinger_df = calc_bollinger_bands(prices_IBM[symbol], 20)
    order_df = build_orders(bollinger_df)

    order_df.index.name = 'Date'
    order_df.to_csv("bollinger_order.csv")

    #Build Plot
    plt.style.use('ggplot')
    ax = bollinger_df.plot()
    #Add lines showing buy/sells
    for index, row in order_df.iterrows():
        if row['Order'] == 'BUY':
            ax.axvline(x=index, color='g')
        elif row['Order'] == 'SELL':
            ax.axvline(x=index, color='r')


    plt.show()

def compute_insample_data_ML4T(start_date,end_date):
    dates=pd.date_range(start_date,end_date)
    prices=get_data(['ML4T-399'],dates,True)
    prices=prices.drop('SPY',axis=1)
    #calculating volatility
    daily_returns = prices.copy()
    daily_returns[1:] = (prices[1:]/prices[:-1].values)-1.0
    daily_returns.ix[0,:]=0
    vol=pd.rolling_std(daily_returns,window=10)
    vol1=vol[9:-5]
    
    #calculating momentum
    momentum=prices.copy()
    momentum[9:] = (prices[9:]/prices[:-9].values)-1.0
    momentum1=momentum[9:-5]
    
    #calculating bollinger values
    sma = pd.rolling_mean(prices,window=10)
    sma1 = sma.dropna()
    std=pd.rolling_std(prices,window=10)
    std1=std[9:]
    bb_prices=prices[9:]
    bb_value=(bb_prices - sma1)/(2*std1)
    bb_value1=bb_value[0:-5]
    
    #shifting prices
    shifted_prices=prices.shift(-5)
    future_prices = prices.copy()
    prices1=prices[9:-5]
    future_return=(shifted_prices/prices) - 1.0
    future_return1=future_return[9:-5]
    
    vol_array=vol1.values
    momentum_array=momentum1.values
    bb_value_array=bb_value1.values

    data=np.concatenate((vol_array,momentum_array,bb_value_array,future_return1), axis=1)
    predY = knn_learner(data)
    predY_df_return=pd.DataFrame(predY,index=future_return1.index,columns=['predY'])
    predY_df_price=prices1*(predY_df_return.values+1)
    future_prices=prices1*(future_return1.values+1)

    predY_df_price=predY_df_price.rename(columns={'ML4T-399':'Predicted Y'})
    ax=predY_df_price.plot(title="Sine Data Training Y/Price/Predicted Y[2008-2009]")
    
    future_prices = future_prices.rename(columns={'ML4T-399':'Training Y'})
    prices1 = prices1.rename(columns={'ML4T-399':'Price'})
    future_prices.plot(ax=ax)
    prices1.plot(ax=ax)
    plt.ylim((0,100))
    start_date='2008-01-15'
    end_date='2009-12-23'
    print
    print "Sine Data In Sample Statistics"
    print
    my_strategy_ML4T(prices1,predY_df_price,start_date,end_date,"ML4T_insample_orders","Sine Data In Sample Entries/Exits","Sine Data In Sample Backtest")

    start_date='2010-01-01'
    end_date='2010-12-31'
    compute_outsample_data_ML4T(data,start_date,end_date)

def get_sequence_list_and_phyche_value_pseknc(input_data, extra_phyche_index=None):
    """For PseDNC, PseKNC, make sequence_list and phyche_value.

    :param input_data: file type or handle.
    :param extra_phyche_index: dict, the key is the dinucleotide (string),
                                     the value is its physicochemical property value (list).
                               It means the user-defined physicochemical indices.
    """
    if extra_phyche_index is None:
        extra_phyche_index = {}

    original_phyche_value = {
        'AA': [0.06, 0.5, 0.09, 1.59, 0.11, -0.11],
        'AC': [1.5, 0.5, 1.19, 0.13, 1.29, 1.04],
        'GT': [1.5, 0.5, 1.19, 0.13, 1.29, 1.04],
        'AG': [0.78, 0.36, -0.28, 0.68, -0.24, -0.62],
        'CC': [0.06, 1.08, -0.28, 0.56, -0.82, 0.24],
        'CA': [-1.38, -1.36, -1.01, -0.86, -0.62, -1.25],
        'CG': [-1.66, -1.22, -1.38, -0.82, -0.29, -1.39],
        'TT': [0.06, 0.5, 0.09, 1.59, 0.11, -0.11],
        'GG': [0.06, 1.08, -0.28, 0.56, -0.82, 0.24],
        'GC': [-0.08, 0.22, 2.3, -0.35, 0.65, 1.59],
        'AT': [1.07, 0.22, 0.83, -1.02, 2.51, 1.17],
        'GA': [-0.08, 0.5, 0.09, 0.13, -0.39, 0.71],
        'TG': [-1.38, -1.36, -1.01, -0.86, -0.62, -1.25],
        'TA': [-1.23, -2.37, -1.38, -2.24, -1.51, -1.39],
        'TC': [-0.08, 0.5, 0.09, 0.13, -0.39, 0.71],
        'CT': [0.78, 0.36, -0.28, 0.68, -0.24, -0.62]}

    sequence_list = get_data(input_data)
    phyche_value = extend_phyche_index(original_phyche_value, extra_phyche_index)

    return sequence_list, phyche_value

def optimize_portfolio(sd=dt.datetime(2008,1,1), ed=dt.datetime(2009,1,1), \
    syms=['GOOG','AAPL','GLD','XOM'], gen_plot=False):

    # Read in adjusted closing prices for given symbols, date range
    dates = pd.date_range(sd, ed)
    prices_all = get_data(syms, dates)  # automatically adds SPY
    prices = prices_all[syms]  # only portfolio symbols
    prices_SPY = prices_all['SPY']  # only SPY, for comparison later

    # find the allocations for the optimal portfolio
    # note that the values here ARE NOT meant to be correct for a test case
    allocs = optimize_allocs(prices, min_sharpe_fun)
    cr, adr, sddr, sr = compute_portfolio_stats(get_port_val(prices, allocs), allocs)

    # Get daily portfolio value
    port_val = get_port_val(prices, allocs)

    # Compare daily portfolio value with SPY using a normalized plot
    if gen_plot:
        # add code to plot here
        df_temp = pd.concat([port_val, prices_SPY], keys=['Portfolio', 'SPY'], axis=1)
        df_temp = df_temp / df_temp.iloc[0]
        plot_stock_data(df_temp.ix[sd : ed, ['Portfolio', 'SPY']])
        pass

    return allocs, cr, adr, sddr, sr

def test_run():
    # Read data
    dates = pd.date_range('2012-01-01', '2012-12-31')
    symbols = ['SPY']
    df = get_data(symbols, dates)

    # Compute Bollinger Bands
    # 1. Compute rolling mean
    rm_SPY = get_rolling_mean(df['SPY'], window=20)

    # 2. Compute rolling standard deviation
    rstd_SPY = get_rolling_std(df['SPY'], window=20)

    # 3. Compute upper and lower bands
    upper_band, lower_band = get_bollinger_bands(rm_SPY, rstd_SPY)

    # Plot raw SPY values, rolling mean and Bollinger Bands
    ax = df['SPY'].plot(title="Bollinger Bands", label='SPY')
    rm_SPY.plot(label='Rolling mean', ax=ax)
    upper_band.plot(label='upper band', ax=ax)
    lower_band.plot(label='lower band', ax=ax)

    # Add axis labels and legend
    ax.set_xlabel("Date")
    ax.set_ylabel("Price")
    ax.legend(loc='upper left')
    plt.show()

def ipseknc(input_data, k, w, lamada, phyche_list, alphabet, extra_index_file=None, all_prop=False):
    """This is a complete process in iPseKNC, k is kmer, but the index is just for dinucleotide.

    :param k: int, the value of k-tuple.
    :param phyche_list: list, the input physicochemical properties list.
    :param extra_index_file: a file path includes the user-defined phyche_index.
    :param all_prop: bool, choose all physicochemical properties or not.
    """
    phyche_list = get_phyche_list(k=2, phyche_list=phyche_list,
                                  extra_index_file=extra_index_file, alphabet=alphabet, all_prop=all_prop)

    # Get phyche_vals.
    if extra_index_file is not None:
        extra_phyche_index = get_extra_index(extra_index_file)
        from util import normalize_index

        phyche_vals = get_phyche_value(k=2, phyche_list=phyche_list, alphabet=alphabet,
                                       extra_phyche_index=normalize_index(extra_phyche_index, alphabet,
                                                                          is_convert_dict=True))
    else:
        phyche_vals = get_phyche_value(k=2, phyche_list=phyche_list, alphabet=alphabet)

    seq_list = get_data(input_data, alphabet)

    return make_pseknc_vector(seq_list, phyche_vals, k, w, lamada, alphabet, theta_type=3)

def test_run():
    # Read data
    dates = pd.date_range('2009-01-01', '2012-12-31')  # one month only
    symbols = ['SPY','XOM','GLD']
    df = get_data(symbols, dates)
    plot_data(df)

    # Compute daily returns
    daily_returns = compute_daily_returns(df)
    #plot_data(daily_returns, title="Daily returns", ylabel="Daily returns")

    # Scatterplot SPY vs XOM
    daily_returns.plot(kind='scatter',x='SPY',y='XOM')
    beta_XOM,alpha_XOM=np.polyfit(daily_returns['SPY'],daily_returns['XOM'],1)
    print "beta_XOM= ",beta_XOM
    print "alpha_XOM= ",alpha_XOM
    plt.plot(daily_returns['SPY'],beta_XOM*daily_returns['SPY']+alpha_XOM,'-',color='r')
    plt.grid()
    plt.show()

    # Scatterplot SPY vs GLD
    daily_returns.plot(kind='scatter',x='SPY',y='GLD')
    beta_GLD,alpha_GLD=np.polyfit(daily_returns['SPY'],daily_returns['GLD'],1)
    print "beta_GLD= ",beta_GLD
    print "alpha_GLD= ",alpha_GLD
    plt.plot(daily_returns['SPY'],beta_GLD*daily_returns['SPY']+alpha_GLD,'-',color='r')
    plt.grid()
    plt.show()

    # Calculate correlation coefficient
    print daily_returns.corr(method='pearson')

def get_indicators(start_date, end_date, symbols):
    """Simulate and assess the performance of a stock portfolio."""
    # Read in adjusted closing prices for given symbols, date range
    dates = pd.date_range(start_date, end_date)
    prices_all = get_data(symbols, dates)  # automatically adds SPY
    prices = prices_all[symbols]  # only portfolio symbols
    # prices_SPY = prices_all['SPY']  # only SPY, for comparison later

    sym = symbols[1]

    x1 = (prices[sym] - pd.rolling_mean(prices[sym], 20)) / (2 * pd.rolling_std(prices[sym], 20))
    x1_dis = pd.cut(x1, 10, labels=False)

    x2 = prices[sym].pct_change(20)
    x2_dis = pd.cut(x2, 10, labels=False)
    x3 = pd.rolling_std(prices[sym].pct_change(1), 20)
    x3_dis = pd.cut(x3, 10, labels=False)

    # return pd.concat([x1_,x2_0,x3_0], axis=1).dropna(), prices
    tempdf = pd.concat([x1_dis, x2_dis, x3_dis], axis=1).dropna()
    tempdf.columns = ["x1", "x2", "x3"]

    print tempdf.dtypes

    tempdf["holding"] = np.random.randint(0, 3, size=len(tempdf))
    # 0 = no position , 1 = negative positin 2 =holding long
    tempdf["s"] = 1000 * tempdf["holding"] + 100 * tempdf["x3"] + 10 * tempdf["x2"] + 1 * tempdf["x1"]
    print tempdf.head(50)
    return tempdf, prices

def optimize_portfolio(start_date, end_date, symbols):
    """Simulate and optimize portfolio allocations."""
    # Read in adjusted closing prices for given symbols, date range
    dates = pd.date_range(start_date, end_date)
    prices_all = get_data(symbols, dates)  # automatically adds SPY
    prices = prices_all[symbols]  # only portfolio symbols
    prices_SPY = prices_all['SPY']  # only SPY, for comparison later

    # Get optimal allocations


    allocs = find_optimal_allocations(prices)

    allocs = allocs / np.sum(allocs)  # normalize allocations, if they don't sum to 1.0

    # Get daily portfolio value (already normalized since we use default start_val=1.0)
    port_val = get_portfolio_value(prices, allocs)

    # Get portfolio statistics (note: std_daily_ret = volatility)
    cum_ret, avg_daily_ret, std_daily_ret, sharpe_ratio = get_portfolio_stats(port_val)

    # Print statistics
    print "Start Date:", start_date
    print "End Date:", end_date
    print "Symbols:", symbols
    print "Optimal allocations:", allocs
    print "Sharpe Ratio:", sharpe_ratio
    print "Volatility (stdev of daily returns):", std_daily_ret
    print "Average Daily Return:", avg_daily_ret
    print "Cumulative Return:", cum_ret

    # Compare daily portfolio value with normalized SPY
    normed_SPY = prices_SPY / prices_SPY.ix[0, :]
    df_temp = pd.concat([port_val, normed_SPY], keys=['Portfolio', 'SPY'], axis=1)
    plot_data(df_temp, title="Daily Portfolio Value and SPY")

def get_vals(start_date, end_date, symbol):
    # Read in adjusted closing prices for given symbols, date range
    # to allow getting SMA from day 1 on start_date we initially read in an earlier date to calc SMA
    dates = pd.date_range(start_date, end_date)
    prices_all = util.get_data(symbol, dates)  # automatically adds SPY
    prices = prices_all[symbol]  # only portfolio symbols       
    return prices

def compute_portvals(start_date, end_date, orders_file, start_val):
	"""Compute daily portfolio value given a sequence of orders in a CSV file.

	Parameters
	----------
	    start_date: first date to track
	    end_date: last date to track
	    orders_file: CSV file to read orders from
	    start_val: total starting cash available

	Returns
	-------
	    portvals: portfolio value for each trading day from start_date to end_date (inclusive)
	"""
	dates = pd.date_range(start_date, end_date)
	orders = construct_orders(orders_file, dates)
	# print orders
	symbols = list(set(orders.Symbol))

	prices_all = get_data(symbols, dates)
	prices = prices_all[symbols]
	trades = calculate_trades(prices.index, orders, symbols)
	# print trades

	holdings = pd.DataFrame(index=prices.index)
	holdings['cash'] = start_val + (-1.0 * (prices * trades).sum(axis=1)).cumsum()
	holdings['stock'] = (prices * trades.cumsum()).sum(axis=1)
	portvals = holdings.cash + holdings.stock
	return portvals

def compute_portvals(start_date, end_date, orders_file, start_val):
    """Compute daily portfolio value given a sequence of orders in a CSV file.

    Parameters
    ----------
        start_date: first date to track
        end_date: last date to track
        orders_file: CSV file to read orders from
        start_val: total starting cash available

    Returns
    -------
        portvals: portfolio value for each trading day from start_date to end_date (inclusive)
    """
    # TODO: Your code here
    #create df_prices
    df_temp = pd.read_csv(orders_file, index_col='Date', parse_dates=True)
    symbols = []
    for index,row in df_temp.iterrows():		
    	symbols.append(row['Symbol'])
    symbols = list(set(symbols))
    dates = pd.date_range(start_date, end_date)
    df_prices = get_data(symbols, dates)
    df_prices = df_prices.drop('SPY',1)
    df_prices['CASH'] = 1.0
    #print df_prices
    
    #Create df_trade.
    #Check for leverage by create a curr_list that save the cumulative holding.
    #When a new order comes, create a temp_list with update holding and multiply it with current prices, 
    #then check to see if leverage exceeds 2 or not. If it's not, then process the order,
    #change curr_list to temp_list and update df_trade
    #If it exceeds 2, then don't process the order and do nothing 
    df_trade = df_prices.copy()
    df_trade[df_trade != 0] = 0
    df_trade.ix[start_date,'CASH'] = start_val
    curr_list = df_trade.ix[start_date].copy()
    for index, row in df_temp.iterrows():
    	temp_list = curr_list.copy()
    	temp_list.ix[row['Symbol']] += (1 if row['Order'] == 'BUY' else -1)*float(row['Shares'])
    	temp_list.ix['CASH'] += (-1 if row['Order'] == 'BUY' else 1)*float(row['Shares'])*df_prices.ix[index,row['Symbol']]
    	sum_abs_all  = abs(temp_list).dot(df_prices.ix[index])
    	sum_cash = abs(temp_list['CASH'])
    	sum_all = temp_list.dot(df_prices.ix[index])
    	leverage = (sum_abs_all-sum_cash)/sum_all
    	#print df_prices.ix[index,row['Symbol']], sum_abs_all , sum_cash, sum_all, leverage
    	if (leverage <= 2.0):
    		curr_list = temp_list.copy()
    		df_trade.ix[index,row['Symbol']] += (1 if row['Order'] == 'BUY' else -1)*float(row['Shares'])
    		df_trade.ix[index,'CASH'] += (-1 if row['Order'] == 'BUY' else 1)*float(row['Shares'])*df_prices.ix[index,row['Symbol']]
    #print df_trade
    
    #calculate holding from df_trade and portvals
    portvals = pd.Series(index = df_prices.index)
    portvals.ix[0] = df_prices.ix[0].dot(df_trade.ix[0])
    for i in range(1,df_trade.shape[0]):
    	df_trade.ix[i] += df_trade.ix[i-1]
    	portvals.ix[i] = df_prices.ix[i].dot(df_trade.ix[i])
    print portvals
    return portvals

def define_y(symbol, startdate_string ='12/31/07', enddate_string ='12/31/09', window=5):
    """

    :param symbol: STRING
    :param startdate_string: STRING 'MM/DD/YY'
    :param enddate_string: STRING 'MM/DD/YY'
    :param window: size of rolling averages for 5 day forecast
    :return: data, data_np.  Features in both Pandas and Numpy formats.  4 columns each of ['bb_value', 'momentum', 'daily_returns', 'volatility']
    """
    # Import Orders into DataFrame (CURRENTLY HAS ALL DATES including non-trading)
    start_date = pd.to_datetime(startdate_string) #StartDate per Instructions
    end_date = pd.to_datetime(enddate_string) #EndDate per Instructions
    dates = pd.date_range(start_date, end_date)

    symbols = [symbol, '$SPX']

    # Read in adjusted closing prices for given symbols, date range
    prices_all = get_data(symbols, dates)  # automatically adds SPY
    prices = prices_all[[symbol]]  # only portfolio symbols
    #prices_np = prices.as_matrix()
    #index_df = prices.index

    # Compute SMA
    sma = pd.rolling_mean(prices, window)
    sma.columns = prices.columns
    #sma_np = sma.as_matrix()

    y = (prices.shift(-5)/prices)-1
    y_np = y.as_matrix().transpose() #need to transpose y. As a 1d Output variable, need to have it be a series.  Not sure why, but whatever

    return y, y_np, prices

def assess_portfolio(start_date, end_date, symbols, allocs, start_val=1):
    """Simulate and assess the performance of a stock portfolio."""
    # Read in adjusted closing prices for given symbols, date range
    dates = pd.date_range(start_date, end_date)
    prices_all = get_data(symbols, dates)  # automatically adds SPY
    prices = prices_all[symbols]  # only portfolio symbols
    prices_SPY = prices_all['SPY']  # only SPY, for comparison later

    # Get daily portfolio value
    port_val = get_portfolio_value(prices, allocs, start_val)
    plot_data(port_val, title="Daily Portfolio Value")

    # Get portfolio statistics (note: std_daily_ret = volatility)
    cum_ret, avg_daily_ret, std_daily_ret, sharpe_ratio = get_portfolio_stats(port_val)

    # Print statistics
    print "Start Date:", start_date
    print "End Date:", end_date
    print "Symbols:", symbols
    print "Allocations:", allocs
    print "Sharpe Ratio:", sharpe_ratio
    print "Volatility (stdev of daily returns):", std_daily_ret
    print "Average Daily Return:", avg_daily_ret
    print "Cumulative Return:", cum_ret

    # Compare daily portfolio value with SPY using a normalized plot
    df_temp = pd.concat([port_val, prices_SPY], keys=['Portfolio', 'SPY'], axis=1)
    plot_normalized_data(df_temp, title="Daily portfolio value and SPY")

def optimize_portfolio(sd=dt.datetime(2008,1,1), ed=dt.datetime(2009,1,1), \
    syms=['GOOG','AAPL','GLD','XOM'], gen_plot=False):

    # Read in adjusted closing prices for given symbols, date range
    dates = pd.date_range(sd, ed)
    prices_all = get_data(syms, dates, sd, ed)  # automatically adds SPY
    prices = prices_all[syms]  # only portfolio symbols
    prices_SPY = prices_all['SPY']  # only SPY, for comparison later

    # find the allocations for the optimal portfolio
    sv = 1000000
    normalized_prices = prices / prices.ix[0, :]
    x0 = np.array([0.2, 0.2, 0.3, 0.3, 0.0])
    optimal_allocs = spo.minimize(f, x0, args=(normalized_prices, sv),
                        method='SLSQP', options={'disp': True},
                        bounds=tuple((0, 1) for i in range(0, x0.size)),
                        constraints = ({'type': 'eq', 'fun': \
                        lambda inputs: 1.0 - np.sum(inputs)})
                        )
    allocs = optimal_allocs.x
    port_val, cr, adr, sddr, sr = calc_portfolio_stats(allocs, normalized_prices, sv)

    # Compare daily portfolio value with SPY using a normalized plot
    if gen_plot:
        df_temp = pd.concat([port_val, prices_SPY], keys=['Portfolio', 'SPY'], axis=1)
        df_temp = df_temp / df_temp.ix[0, :]
        df_temp.plot()
        plt.show()
        pass

    return allocs, cr, adr, sddr, sr

def main():
	dates = pd.date_range('2009-01-01', '2012-12-31')
	symbols = ['SPY']
	df = get_data(symbols, dates)
	plot_data(df)

	daily_returns = compute_daily_returns(df)

	# histogram
	daily_returns.hist(bins=20)
	
	'''	
	call this twice if wanting to plot 2+ charts on same chart:
	daily_returns['SPY'].hist(bins=20, label="SPY")
	daily_returns['XOM'].hist(bins=20, label="XOM")
	'''

	mean = daily_returns['SPY'].mean()
	std = daily_returns['SPY'].std()

	plt.axvline(mean, color='w', linestyle='dashed', linewidth=2)
	plt.axvline(std, color='r', linestyle='dashed', linewidth=2)
	plt.axvline(-std, color='r', linestyle='dashed', linewidth=2)
	plt.show()

	print daily_returns.kurtosis()