def display(self):
        c = self.db.cursor()

        now = datetime.datetime.now()
        if now.weekday() == 5:
            now = now - timedelta(days=1)
        elif now.weekday() == 6:
            now = now - timedelta(days=2)
        now = now.strftime("%Y%m%d")

        spynow = ystockquote.get_historical_prices('SPY', now, now)
        spynow = float(spynow[1][4])

        for row in c.execute("SELECT * FROM stocks ORDER BY tradeDate"):
            cur = ystockquote.get_historical_prices(row[1], now, now)
            cur = float(cur[1][4])
            orig = row[5] * row[6]
            spy_orig = row[7] * row[8]
            if row[5] < 0:
                continue
            print "Ticker:", row[1]
            print "  Date:", row[3]
            print "  Unit Price:", row[6]
            print "  SPY Price:", row[8]
            print "  Current Price:", cur
            print "  Current SPY:", spynow
            print "  Return:", 100 * ((cur * row[5]) - orig) / orig
            print "  SPY Return:", 100 * ((spynow * row[7]) - spy_orig) / spy_orig

def getsavedata(stk):
    firstday = '2013-01-01'
    today    = datetime.date.today().strftime('%Y-%m-%d')

    savename=getsavename(stk,'.p')

    # TODO: make path for savename
    # mkdir(dirname(savename))

    # load (and update if needed)
    if os.path.exists(savename):
      quotes=pickle.load( open(savename, "rb") )
      lastquote = sorted(quotes.keys())[-1]

      # update with new values
      prevdate = datetime.datetime.strptime(today,'%Y-%m-%d') - datetime.timedelta(days=1)
      prevdate=prevdate.strftime('%Y-%m-%d')
      if lastquote != prevdate:
         nextdate = datetime.datetime.strptime(lastquote,'%Y-%m-%d') + datetime.timedelta(days=1)
         nextdate=nextdate.strftime('%Y-%m-%d')
         pprint([prevdate, lastquote,nextdate,today])
         quotes.update( ystockquote.get_historical_prices(stk,nextdate,today) )
         savestock(stk,quotes)

    # get all new
    else:
      quotes  = ystockquote.get_historical_prices(stk,firstday,today)
      savestock(stk,quotes)

    return quotes

def getsavedata(stk,firstday='2013-01-01',lastday='today',forceupdate=False):
    if lastday == 'today':
      lastday    = datetime.date.today().strftime('%Y-%m-%d')

    savename=getsavename(stk,'.p')

    #did we dl anything? do we need to wait
    dl=0

    # TODO: make path for savename
    # mkdir(dirname(savename))

    # load (and update if needed)
    if os.path.exists(savename):
      quotes=pickle.load( open(savename, "rb") )
      lastquote = sorted(quotes.keys())[-1]

      # what is the last possible day we could have values for 
      # this is only meaningful of lastday is "today"
      prevdate = datetime.datetime.strptime(lastday,'%Y-%m-%d') - datetime.timedelta(days=1)
      prevdate=prevdate.strftime('%Y-%m-%d')

      # if we dont have yestrdays quotes (and we arn't forcing a different date range)
      if lastquote != prevdate and not forceupdate:
         nextdate = datetime.datetime.strptime(lastquote,'%Y-%m-%d') + datetime.timedelta(days=1)
         nextdate=nextdate.strftime('%Y-%m-%d')
         # set the first day of data to retrieve to the 
         # next day (first missing day) in the data we have
         firstday = nextdate
         forceupdate=True

      if forceupdate:
         pprint([prevdate, lastquote,firstday,lastday])
         quotes.update( ystockquote.get_historical_prices(stk,firstday,lastday) )
         savestock(stk,quotes)
         dl=1

    # get all new
    else:
      quotes  = ystockquote.get_historical_prices(stk,firstday,lastday)
      savestock(stk,quotes)
      dl=1

    if dl: time.sleep(10)

    # did we miss anything?
    populateMissing(stk,quotes)
    return quotes

    def import_historic_quotes(self, years=15):
        result = []
        today = datetime.date.today()
        first_day = datetime.date.today() - datetime.timedelta(days=int(years * 365))
        today_str = today.strftime('%Y-%m-%d')
        first_day_str = first_day.strftime('%Y-%m-%d')
        logger.info('Start import of historic quotes')
        for sec in Security.objects.all():
            logger.debug('Security ' + str(sec))
            no_quote = False
            no_yahoo_id = False
            if sec.yahoo_id != '' and not sec.yahoo_id.startswith('unknown'):

                try:
                    quote = ystockquote.get_historical_prices(sec.yahoo_id, first_day_str, today_str)
                except urllib.error.HTTPError:
                    no_quote = True
                else:
                    logger.debug('Found quotes')
                    for key in quote:
                        logger.debug('Security ' + str(sec))
                        self.add(sec, key, quote[key]['Close'])
            else:
                no_yahoo_id = True
            result.append({'stock_id': sec.id,
                           'yahoo_id': sec.yahoo_id,
                           'name': sec.name,
                           'no_quote': no_quote,
                           'no_yahoo_id': no_yahoo_id})
        return result

def getBench(f_tdelta,bench):

	sDate = datetime.today().date() + timedelta(days=-f_tdelta)
	sDate = sDate.strftime("%Y-%m-%d")
	eDate = datetime.today().date()
	eDate = eDate.strftime("%Y-%m-%d")

	tdate = []
	temp = []

	print "Getting prices for " + str(bench)
	temp.append(ystockquote.get_historical_prices(bench, sDate, eDate))

	for y in temp[0]:
		tdate.append(y)

	tdate = sorted(tdate, reverse=True)
	ptemp = [[0 for x in xrange(1)] for x in xrange(len(tdate))]

	for datee in range(len(tdate)):
		for assetnum in range(1):
			try:
				q = temp[assetnum][tdate[datee]]['Close']
				ptemp[datee][assetnum] = q
			except KeyError:
				print "wiww"
				ptemp[datee][assetnum] = ptemp[datee-1][assetnum]

	return ptemp

def main():
    error_file = "no_quote.txt"
    ticker_file = "tickers.txt"
    quotes_file = "quotes.txt"

    error_file = open(error_file, 'w')
    ticker_file = open(ticker_file, 'r')
    quotes_file = open(quotes_file, 'w')

    for ticker in ticker_file:
        try:
            ticker_arr = ticker.split('\t')
            curr_ticker = ticker_arr[0]
            curr_cik = ticker_arr[1]
            curr_date = ticker_arr[2]
            if not '-' in curr_date:
                curr_date = curr_date[0:4] + '-' + curr_date[4:6] + '-' + curr_date[6:8]
            # crteDateObj(curr_date)
            price_dict = ystockquote.get_historical_prices(curr_ticker, curr_date, curr_date)
            if curr_date in price_dict and 'Close' in price_dict[curr_date] and 'Open' in price_dict[curr_date]:
                curr_close = price_dict[curr_date]['Close']
                curr_open = price_dict[curr_date]['Open']
                quotes_file.write(curr_ticker + '\t' + curr_cik + '\t' + curr_date + '\t' + curr_open + '\t' + curr_close)

        except:
            error_file.write(curr_ticker + ', ' + curr_cik + ', ' + curr_date) 

    error_file.close()
    ticker_file.close()
    quotes_file.close()

def stockmine(ticker, dayweek="d", splitsize=60):  # splitsize <= 0 will not split, "d" for day, "w" for weeks
    count = 0  # Keeps count of the number of days currently stored before writing
    writeCount = 0  # how many file writes have been made
    dayStat = []
    monthStat = []
    if ticker != None:
        header = True
        today = date.today()
        stock = ystockquote.get_historical_prices(
            str(ticker), "20000101", today.strftime("%Y%m%d"), dayweek
        )  # retrieve financial data from Yahoo
        for data in stock:  # go through stock data individually
            if header == False:
                dayStat.append([data[0], data[5]])  # appends date and volume
                count += 1
                if count >= splitsize and splitsize > 0:  # when splitsize days have been reached
                    monthStat.append(
                        dayStat
                    )  # list containing sublists, where each sublist contains the day, which contains sublist of date/vol
                    dayStat = None
                    dayStat = []
                    count = 0
                    writeCount += 1
            else:
                header = False  # skips first line
        if splitsize <= 0:
            monthStat = dayStat
    return monthStat

    def getQuotes(self, companyLabel):
        
        # TODO: if NO CHANGE:
        # return nothing

        print 'Fetching Quotes', companyLabel
        results =  ystockquote.get_historical_prices(companyLabel, '20100101', '20130301')
        div =  ystockquote.get_dividend_yield(companyLabel)
        print 'Fetched Quotes'
        
          
        if str(div) == 'N/A':
            div = 0
        else:
            div = float(div) * .01
            
        del results[0]  # column names
        
        quotes = []
        for i,entry in enumerate(results):
            # second column is opening price
            if i+1 >= len(results) or self.isFirstWorkDay(entry[0], results[i+1][0]):
                gain = float(entry[1+2]) * (1+div)
                quotes.append(gain)

        return quotes

    def get_percent_change(self, start_date, end_date):
        """Pulls data from Yahoo's API and calculates the percent change from the start data to the end date."""
        # q = 'select * from yahoo.finance.symbol where symbol in ("'
        # q += self.symbol + '")'

        # Format Query for YQL
        q = 'select * from yahoo.finance.historicaldata where symbol = "%s" and startDate = "%s" and endDate = "%s"' % (self.symbol, start_date, end_date)
        query = urllib.quote_plus(q)

        # Format URL for YQL
        url = "http://query.yahooapis.com/v1/public/yql?q="
        url += query + "&env=http%3A%2F%2Fdatatables.org%2Falltables.env"

        # Launch Yahoo Request
        r = BeautifulSoup(requests.get(url).text)
        symbols = r.find_all("symbol")
        # print r.prettify()

        # If YQL Api is not down, simply calculate percent change
        if(len(symbols) > 0):
            p2 = float(symbols[0].close.string)
            p1 = float(symbols[1].close.string)
            self.percent_change = (p2 - p1) / (.5 * (p1 + p2)) * 100
        # Otherwise call the ystocksymbol gem
        else:
            self.data = ystockquote.get_historical_prices(self.symbol, convert_date(start_date), convert_date(end_date))
            days = len(self.data) - 1
            # print self.data
            p2 = float(self.data[1][4])
            p1 = float(self.data[days][4])
            self.percent_change = (p2 - p1) / (.5 * (p1 + p2)) * 100

def fetchQuotes(sym, start=FROM_DATE, end=CURRENT_DATE):
    his = None
    data = None
    try:
        # print start, end
        data = ystockquote.get_historical_prices(sym, start, end)
    except Exception:
        print "Please check the dates. Data might not be available. 404 returned"

        # 404 due to data yet not available
    if data:
        his = DataFrame(collections.OrderedDict(sorted(data.items()))).T
        his = his.convert_objects(convert_numeric=True)
        his.index = pd.to_datetime(his.index)
        his.insert(0, 'symbol', sym, allow_duplicates=True)
        # insert the date as dataframe too
        his.insert(1, 'date', his.index)
        # his.columns = getColumns('stock_quote_historical')   # Removing as db dependency is removed
        his.columns = getColumnsNoSql('stock_quote_historical')

    daily = ystockquote.get_all(sym)
    # print daily
    # persist(his, daily, sym, end)

    return his, daily

def DownLoadStocks():
    #StockNameList = 'C:\Users\Makaye\Desktop\Investment\Code\yahoo_stock_symbols_4.csv'
    StockNameWithExchange = 'C:\Users\Makaye\Desktop\Investment\Code\TickersNamesExhanges_4.csv'
    fid = open(StockNameWithExchange,'r')
    Data = fid.readlines()
    StockTicker = [day.split(',')[0] for day in Data]
    #StockName = [day.split(',')[1].split('\n')[0] for day in Data]
    StockExchange = [day.split(',')[2].split('\n')[0] for day in Data]
    fid.close()
    
    # Create the directories if needed
    BaseDir =  'C:\Users\Makaye\Desktop\Investment\Stocks'   
    start_date = '20100101'
    end_date = '20111111'
    for i in range(0,len(StockTicker)):
        CurExh = StockExchange[i]   
        CurTick = StockTicker[i]
        CurDir= os.path.join(BaseDir,CurExh)
        if not os.path.exists(CurDir):
            os.makedirs(CurDir)
        # download the data for each exchange
        OutDir = os.path.join(BaseDir,CurExh,CurTick+".csv")
        if not os.path.exists(OutDir):
            try:
                print "DownLoading: "+CurExh+": "+CurTick+", "+str(i)+"/"+str(len(StockTicker)) 
                fid = open(OutDir,'w')
                Y=ys.get_historical_prices(StockTicker[i],start_date,end_date)
                
                for j in Y:
                    
                    temp=",".join(["%s"% el for el in j])+'\n'  
                    fid.write(temp)
                fid.close()
            except:
                print "Problem with: "+CurExh+": "+CurTick

    def import_ofx(self, ofx_file):
        ofx = OfxParser.parse(file(ofx_file))
        idx = {}
        for s in ofx.security_list:
            idx[s.uniqueid] = s.ticker

        c = self.db.cursor()

        for t in ofx.account.statement.transactions:
            c.execute("SELECT id FROM stocks WHERE id = ?", [t.id])
            row = c.fetchone()
            if row:
                print "Skipping duplicate transaction:", t.id
                continue

            spydate = t.tradeDate
            # Fidelity transactions can "close" on a weekend?!?
            if spydate.weekday() == 5:
                spydate = spydate - timedelta(days=1)
            elif spydate.weekday() == 6:
                spydate = spydate - timedelta(days=2)
            spy = ystockquote.get_historical_prices('SPY',
                    spydate.strftime("%Y%m%d"), spydate.strftime("%Y%m%d"))
            spy_price = float(spy[1][4])
            spy_units = (float(t.units) * float(t.unit_price)) / spy_price

            c.execute("INSERT INTO stocks VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)",
                (t.id, idx[t.security], t.security, t.tradeDate, t.settleDate,
                float(t.units), float(t.unit_price), spy_units, spy_price))

        self.db.commit()

    def get_historical_prices(self):
	data = ystockquote.get_historical_prices(self.symbol, self.start_date, self.end_date)
	data_headers = data[0]
	data.pop(0)
	
	self.data = data[::-1]
	self.data_headers = data_headers

  def yahoo(symbol, start_date=None, stop_date=None):
    """
    Loads the prices from the start date for the given symbol
    Only new quotes are downloaded.
    """
    if not stop_date:
      stop_date = date.today().strftime("%Y%m%d")
    if not start_date:
      query = db.Query(Quote)
      query.order('-date')
      query.filter('symbol = ', symbol)
      latest_quote = query.get()
      if latest_quote:
        start_date = latest_quote.date
      else:
        start_date = date.today() - timedelta(days=120)
      if start_date == date.today():
        return

    start_date = start_date.strftime("%Y%m%d")
    prices = ystockquote.get_historical_prices(symbol, start_date, stop_date)
    headers = prices[0]
    try:
      close = Quote.get_idx(headers, 'Close')
      date_ = Quote.get_idx(headers, 'Date')
      open = Quote.get_idx(headers, 'Open')
      high = Quote.get_idx(headers, 'High')
      low = Quote.get_idx(headers, 'Low')
    except Exception, e:
      logging.warning('Could not download %s:%s', symbol, e)
      return None

    def __init__(self, name, start_date):
        self.name = name.upper()
        try:
            self.stockDate = dt.date(int(start_date[0:4]), int(start_date[5:7]), int(start_date[8:10]))
        except ValueError:
            print "Error: bad date"
        except IndexError:
            print "Error: dates must be iso format"

        got_price = 0
        attempts = 0
        while not got_price and attempts < 5:
            try:
                self.price = float(
                    ystock.get_historical_prices(self.name, self.stockDate.isoformat(), self.stockDate.isoformat())[1][
                        4
                    ]
                )
            except:
                self.stockDate = self.stockDate + self.one_day
                attempts += 1
            else:
                got_price = 1

        if self.price == 0:
            print "Unable to find stock name"

def prices(symbol):
  """
  Loads the prices from the start date for the given symbol
  Only new quotes are downloaded.
  """
  to = date.today().strftime("%Y%m%d")
  c = db.cursor()
  c.execute("SELECT DATE_ADD(max(date), INTERVAL 1 DAY) FROM quote where symbol = %s",
               (symbol))
  (_from, ) = c.fetchone()
  if _from == date.today():
    print "Skipping %s" % symbol
    return
  print "Downloading %s" % symbol
  if _from is None: 
    _from = start_date
  else:
    _from = _from.strftime("%Y%m%d")
  prices = ystockquote.get_historical_prices(symbol, _from, to)
  headers = prices[0]
  try:
    close = get_idx(headers, 'Close')
    date_ = get_idx(headers, 'Date')
    open = get_idx(headers, 'Open')
    high = get_idx(headers, 'High')
    low = get_idx(headers, 'Low')
    quotes = prices[1:]
    for l in quotes:
      #print "%s %s" % (l[date_], l[close])
      insert(symbol, l[date_], l[close], l[high], l[low], l[open])
    print "Inserted %s new quotes for %s" % (len(quotes), symbol)
  except:
    print "Could not download %s" % symbol

	def get_high_prices(self, date):
		date = datetime.date(2015, 10, 21)
		day_before = date - timedelta(1)
		day_after = date + timedelta(1)
		price = ystockquote.get_historical_prices(self.symbol, str(day_before), str(day_after))
		self.day_before_price = price[str(day_before)]['High']
		self.day_after_price = price[str(day_after)]['High']

def yahooExtract(ticker, country):

# This function uses the built-in python library 'ystockquote', which inquires
# historical prices and return them as dictionaries. We then determine the
# average prices within each year, storing these averages into a csv file in a
# separate folder.

	rawdata = ystockquote.get_historical_prices(ticker, '1995-01-01', '2011-12-31')

	prices = {}
	for date in rawdata.keys():
		if date[0:4] in prices.keys():
			prices[date[0:4]].append(float(rawdata[date]['Close']))
		else:
			prices[date[0:4]] = []
			prices[date[0:4]].append(float(rawdata[date]['Close']))

	data = []
	for year in sorted(prices.keys()):
		data.append([year, float(sum(prices[year]))/len(prices[year])])

	with open('done/' + country + '.csv', 'w') as file1:
		file1write = csv.writer(file1, delimiter = ',')
		file1write.writerows(data)

	file1.close()

def main():
    # ticker_file = "input/tickers.txt"
    # bucket_name = "edgarsentimentbucket"

    # conn = S3Connection(argv[1], argv[2])
    # path_bucket = conn.get_bucket(bucket_name)
    # k = Key(path_bucket)
    # k.key = ticker_file
    # pathfile = k.get_contents_as_string()
    # try:
    #     lines = pathfile.split('\n')
    # except AttributeError:
    #     lines = pathfile

    
    try:
        print "started"
        # for linie in open(ticker_file, "r"):
        for linie in sys.stdin:
            try:
                print linie
                ticker_arr = linie.split('\t')
                curr_ticker = ticker_arr[1]
                curr_cik = ticker_arr[2]
                curr_date = ticker_arr[3]
                if not '-' in curr_date:
                    curr_date = curr_date[0:4] + '-' + curr_date[4:6] + '-' + curr_date[6:8]
                curr_date = curr_date.strip()
                curr_date_obj = crteDateObj(curr_date)
                yest_date_obj = curr_date_obj - timedelta(days=1)
                yest_date = crteDateStr(yest_date_obj)
                try:
                    price_dict = ystockquote.get_historical_prices(curr_ticker, yest_date, curr_date)
                    curr_close = price_dict[curr_date]['Close']
                    curr_adj_close = price_dict[curr_date]['Adj Close']
                    curr_open = price_dict[curr_date]['Open']
                    yest_close = price_dict[yest_date]['Close']
                    yest_adj_close = price_dict[yest_date]['Adj Close']
                    yest_open = price_dict[yest_date]['Close']
                except:
                    curr_close = "NA"
                    curr_adj_close = "NA"
                    curr_open = "NA"
                    yest_close = "NA"
                    yest_adj_close = "NA"
                    yest_open = "NA"

                try:
                    all_dict = ystockquote.get_all(curr_ticker)
                    curr_mkt_cap = all_dict['market_cap']
                except:
                    curr_mkt_cap = "NA"

                print curr_ticker + '\t' + curr_cik + '\t' + curr_date + '\t' + curr_open + '\t' + \
                 curr_close + '\t' + curr_adj_close + '\t' + yest_open + '\t' + yest_close + '\t' + \
                 yest_adj_close + '\t' + curr_mkt_cap
            except:
                print "bad"
    except:
        print "didn't start"

 def calcOutcome(self, call):
     #print 'getting outcome'
     try:
         nextWeekRes = ystockquote.get_historical_prices(call['ticker'],call['callDate'].strftime("%Y%m%d"),call['weekACall'].strftime("%Y%m%d"))
     except Exception,e:
         nextWeekRes = None
         print 'Exception in next week on ', call['ticker'], 'call date: ', call['callDate']