def __threadMain(self):
     try:
         logger.info("Initializing client.")
         self.__stream.filter(track=self.__track, follow=self.__follow, languages=self.__languages)
     finally:
         logger.info("Client finished.")
         self.__running = False

    def dispatchImpl(self, eventFilter):
        try:
            eventType, eventData = self.__queue.get(True, Client.QUEUE_TIMEOUT)
            if eventFilter is not None and eventType not in eventFilter:
                return

            if eventType == WSClient.ON_TICKER:
                self.__tickerEvent.emit(eventData)
            elif eventType == WSClient.ON_TRADE:
                self.__tradeEvent.emit(eventData)
            elif eventType == WSClient.ON_USER_ORDER:
                self.__userOrderEvent.emit(eventData)
            elif eventType == WSClient.ON_RESULT:
                requestId, result = eventData
                logger.info("Result: %s - %s" % (requestId, result))
            elif eventType == WSClient.ON_REMARK:
                requestId, data = eventData
                logger.info("Remark: %s - %s" % (requestId, data))
            elif eventType == WSClient.ON_CONNECTED:
                self.__onConnected()
            elif eventType == WSClient.ON_DISCONNECTED:
                self.__onDisconnected()
            else:
                logger.error("Invalid event received to dispatch: %s - %s" % (eventType, eventData))
        except Queue.Empty:
            pass

def update_bars(dbFilePath, symbolsFile, timezone, fromYear, toYear):
	db = sqlitefeed.Database(dbFilePath)
	for symbol in open(symbolsFile, "r"):
		symbol = symbol.strip()
		logger.info("Downloading %s bars" % (symbol))
		for year in range(fromYear, toYear+1):
			download_bars(db, symbol, year, timezone)

 def stop(self):
     try:
         if self.__thread is not None and self.__thread.is_alive():
             logger.info("Shutting down client.")
             self.__stream.disconnect()
     except Exception, e:
         logger.error("Error disconnecting stream: %s." % (str(e)))

def build_feed(instruments, fromYear, toYear, storage, frequency=bar.Frequency.DAY, timezone=None, skipErrors=False):
    logger = pyalgotrade.logger.getLogger("yahoofinance")
    ret = yahoofeed.Feed(frequency, timezone)

    if not os.path.exists(storage):
        logger.info("Creating %s directory" % (storage))
        os.mkdir(storage)

    for year in range(fromYear, toYear+1):
        for instrument in instruments:
            fileName = os.path.join(storage, "%s-%d-yahoofinance.csv" % (instrument, year))
            if not os.path.exists(fileName):
                logger.info("Downloading %s %d to %s" % (instrument, year, fileName))
                try:
                    if frequency == bar.Frequency.DAY:
                        download_daily_bars(instrument, year, fileName)
                    elif frequency == bar.Frequency.WEEK:
                        download_weekly_bars(instrument, year, fileName)
                    else:
                        raise Exception("Invalid frequency")
                except Exception, e:
                    if skipErrors:
                        logger.error(str(e))
                        continue
                    else:
                        raise e
            ret.addBarsFromCSV(instrument, fileName)

 def OnRtnDepthMarketData(self, *args):
     
     logger.info("OnRtnDepthMarketData")
     logger.info("id: "+args[0].InstrumentID)
     logger.info("TradingDay: " + args[0].TradingDay + " " + args[0].UpdateTime)
     logger.info("LastPrice: " + str(args[0].LastPrice))
     if self.__insertIntoMysql:
         if self.__mysqlCon == None:
             self.__mysqlCon = client.mysqlConnection(CONSTANTS.HOST,
                                                      CONSTANTS.USERNAME,
                                                      CONSTANTS.PASSWORD,
                                                      CONSTANTS.DATABASE)
         date = datetime.strptime(args[0].TradingDay, "%Y%m%d")
         dateStr = date.strftime("%Y-%m-%d")
         dateStr = dateStr + " " + args[0].UpdateTime
         self.__mysqlCon.addBar(args[0].InstrumentID,
                         RealTimeBar(dateStr,
                                     args[0].LastPrice,
                                     args[0].Volume))
     if self.__dumpToFile:
         try:
             self.dumpToFile(args[0])
         except Exception as e:
             print "except", e
     logger.info("OnRtnDepthMarketData End")

def download_symbols(market):
    market = market.upper()
    pages = ["A", "B", "C", "D", "E", "F", "G", "H", "I", "J", "K", "L", "M", "N", "O", "P", "Q", "R", "S", "T", "U", "V", "W", "X", "Y", "Z"]
    for x in pages:
        url = "http://www.eoddata.com/stocklist/%s/%s.htm" % (market, x)
        logger.info("Processing %s" % url)
        for symbol in get_symbols_from_page(url):
            yield symbol

 def stop(self):
     try:
         self.__stopped = True
         if self.__thread is not None and self.__thread.is_alive():
             logger.info("Shutting down MtGox client.")
             self.__wsClient.stopClient()
     except Exception, e:
         logger.error("Error shutting down MtGox client: %s" % (str(e)))

 def deleteDataFromDate(self, date):
     cursor = self.__con.cursor()
     query = "delete from data where date(date) >= '{0:s}' and date(date) <= '{0:s}'"
     query = query.format(date)
     logger.info(query)
     cursor.execute(query)
     self.__con.commit()
     cursor.close()

 def OnRspSubMarketData(self, *args):
     logger.info("OnRspSubMarketData")
     if args:
         rspInfoField = args[1]
         logger.info("Subscribe Instrument " + args[0].InstrumentID)
         logger.info("errorID " + str(rspInfoField.ErrorID))
         logger.info("errorMsg " + str(rspInfoField.ErrorMsg))
     logger.info("OnRspSubMarketData End")

 def run(self):
     logger.info("Thread started")
     while not self.__stopped:
         self.__wait()
         if not self.__stopped:
             try:
                 self.doCall()
             except Exception, e:
                 logger.critical("Unhandled exception", exc_info=e)

def build_feed(sourceCode, tableCodes, fromYear, toYear, storage, frequency=bar.Frequency.DAY, timezone=None, skipErrors=False, noAdjClose=False, authToken=None):
    """Build and load a :class:`pyalgotrade.barfeed.quandlfeed.Feed` using CSV files downloaded from Quandl.
    CSV files are downloaded if they haven't been downloaded before.

    :param sourceCode: The dataset source code.
    :type sourceCode: string.
    :param tableCodes: The dataset table codes.
    :type tableCodes: list.
    :param fromYear: The first year.
    :type fromYear: int.
    :param toYear: The last year.
    :type toYear: int.
    :param storage: The path were the files will be loaded from, or downloaded to.
    :type storage: string.
    :param frequency: The frequency of the bars. Only **pyalgotrade.bar.Frequency.DAY** or **pyalgotrade.bar.Frequency.WEEK**
        are supported.
    :param timezone: The default timezone to use to localize bars. Check :mod:`pyalgotrade.marketsession`.
    :type timezone: A pytz timezone.
    :param skipErrors: True to keep on loading/downloading files in case of errors.
    :type skipErrors: boolean.
    :param noAdjClose: True if the instruments don't have adjusted close values.
    :type noAdjClose: boolean.
    :param authToken: Optional. An authentication token needed if you're doing more than 50 calls per day.
    :type authToken: string.
    :rtype: :class:`pyalgotrade.barfeed.quandlfeed.Feed`.
    """

    logger = pyalgotrade.logger.getLogger("quandl")
    ret = quandlfeed.Feed(frequency, timezone)
    if noAdjClose:
        ret.setNoAdjClose()

    if not os.path.exists(storage):
        logger.info("Creating %s directory" % (storage))
        os.mkdir(storage)

    for year in range(fromYear, toYear+1):
        for tableCode in tableCodes:
            fileName = os.path.join(storage, "%s-%s-%d-quandl.csv" % (sourceCode, tableCode, year))
            if not os.path.exists(fileName):
                logger.info("Downloading %s %d to %s" % (tableCode, year, fileName))
                try:
                    if frequency == bar.Frequency.DAY:
                        download_daily_bars(sourceCode, tableCode, year, fileName, authToken)
                    elif frequency == bar.Frequency.WEEK:
                        download_weekly_bars(sourceCode, tableCode, year, fileName, authToken)
                    else:
                        raise Exception("Invalid frequency")
                except Exception as e:
                    if skipErrors:
                        logger.error(str(e))
                        continue
                    else:
                        raise e
            ret.addBarsFromCSV(tableCode, fileName)
    return ret

 def __onDisconnected(self):
     if self.__enableReconnection:
         initialized = False
         while not self.__stopped and not initialized:
             logger.info("Reconnecting")
             initialized = self.__initializeClient()
             if not initialized:
                 time.sleep(5)
     else:
         self.__stopped = True

 def OnFrontConnected(self):
     logger.info("OnFrontConnected")
     f = CThostFtdcReqUserLoginField()
     f.BrokerID = self.__broker_id
     f.UserUD = self.__user_id
     f.Password = self.__password
     self.__md.ReqUserLogin(f, self.__reqNum)
     self.__reqNum = self.__reqNum + 1
     if (datetime.now() - self.__starttime) > timedelta(seconds=60*60*10):
         self.stopClient()

def main():
    try:
        writer = symbolsxml.Writer()
        for symbol in open("merval-symbols.txt", "r"):
            symbol = symbol.strip()
            process_symbol(writer, symbol)
        logger.info("Writing merval.xml")
        writer.write("merval.xml")
    except Exception, e:
        logger.error(str(e))

def main():
    try:
        htmlTree = get_html()
        table = find_table(htmlTree)
        if table is None:
            raise Exception("S&P 500 Component Stocks table not found")
        symbolsXML = parse_results(table)

        logger.info("Writing sp500.xml")
        symbolsXML.write("sp500.xml")
    except Exception, e:
        logger.error(str(e))

    def backFill(self, freq,filename, startDate, endDate):
        if freq == 86400:
            final_table = "1day_data"
        elif freq == 1800:
            final_table = "30mins_data"
        elif freq == 300:
            final_table = "5mins_data"
        cursor = self.__con.cursor() 
        cleanup_qry_text = ("""DELETE FROM %s WHERE date(date) >= '%s' and date(date) <= '%s'""" % (final_table, startDate, endDate))
        logger.info(cleanup_qry_text)
        cursor.execute(cleanup_qry_text)

        insert_qry_text = """INSERT INTO {0:s} (symbol, date, open, close, high, low, volume) 
                                select t0.symbol, t0.new_date date, t2.open, t1.close, t0.high,
                                t0.low, t0.volume
                                from (select data.symbol symbol,from_unixtime((floor((unix_timestamp(data.date) / {1:d})) * {1:d}){2:s}) new_date, 
                                        max(data.high) high, min(data.low) low, max(data.date) max_ts, min(data.date) min_ts, 
                                        avg(data.volume) volume 
                                        from data where date(date) >= '{3:s}' and date(date) <= '{4:s}' group by 1,2) t0 
                                join 
                                    (select tbl2.symbol, tbl2.date,  tbl2.close close from data tbl2
                                    join
                                        (select symbol, date, max(milliseconds) max_milli_secs from data
                                         where date(date) >= '{3:s}' and date(date) <= '{4:s}'
                                        group by 1,2) tbl1
                                    on tbl2.symbol = tbl1.symbol and tbl2.date = tbl1.date and tbl2.milliseconds = tbl1.max_milli_secs
                                    where date(tbl2.date) >= '{3:s}' and date(tbl2.date) <= '{4:s}'
                                    group by 1,2) t1 
                                on t0.symbol = t1.symbol and t0.max_ts = t1.date 
                                join 
                                    (select tbl2.symbol, tbl2.date,  tbl2.open open from data tbl2
                                    join
                                        (select symbol, date, min(milliseconds) min_milli_secs from data
                                        where date(date) >= '{3:s}' and date(date) <= '{4:s}' 
                                        group by 1,2) tbl1
                                    on tbl2.symbol = tbl1.symbol and tbl2.date = tbl1.date and tbl2.milliseconds = tbl1.min_milli_secs
                                    where date(tbl2.date) >= '{3:s}' and date(tbl2.date) <= '{4:s}' 
                                    group by 1,2) t2 
                                on t0.symbol = t2.symbol and t0.min_ts = t2.date""" 
        if freq == CONSTANTS.ONE_DAY:
            insert_qry_text = insert_qry_text.format(final_table, freq, '+8*3600', startDate, endDate)
        else:
            insert_qry_text = insert_qry_text.format(final_table, freq, '', startDate, endDate)
        logger.info(insert_qry_text)
        cursor.execute(insert_qry_text)
        self.__con.commit()
        cursor.close()
        text_file = open(filename, "a")
        text_file.writelines("Back fill job %s started at: %s, complete at: %s , end date %s\n" % (final_table, startDate, datetime.now(), endDate))
        text_file.close()

def build_feed(instruments, fromYear, toYear, storage, frequency=bar.Frequency.DAY, timezone=None, skipErrors=False):
    """Build and load a :class:`pyalgotrade.barfeed.googlefeed.Feed` using CSV files downloaded from Google Finance.
    CSV files are downloaded if they haven't been downloaded before.

    :param instruments: Instrument identifiers.
    :type instruments: list.
    :param fromYear: The first year.
    :type fromYear: int.
    :param toYear: The last year.
    :type toYear: int.
    :param storage: The path were the files will be loaded from, or downloaded to.
    :type storage: string.
    :param frequency: The frequency of the bars. Only **pyalgotrade.bar.Frequency.DAY** is currently supported.
    :param timezone: The default timezone to use to localize bars. Check :mod:`pyalgotrade.marketsession`.
    :type timezone: A pytz timezone.
    :param skipErrors: True to keep on loading/downloading files in case of errors.
    :type skipErrors: boolean.
    :rtype: :class:`pyalgotrade.barfeed.googlefeed.Feed`.
    """

    logger = pyalgotrade.logger.getLogger("googlefinance")
    ret = googlefeed.Feed(frequency, timezone)

    if not os.path.exists(storage):
        logger.info("Creating {dirname} directory".format(dirname=storage))
        os.mkdir(storage)

    for year in range(fromYear, toYear+1):
        for instrument in instruments:
            fileName = os.path.join(
                storage,
                "{instrument}-{year}-googlefinance.csv".format(
                    instrument=instrument, year=year))
            if not os.path.exists(fileName):
                logger.info(
                    "Downloading {instrument} {year} to {filename}".format(
                        instrument=instrument, year=year, filename=fileName))
                try:
                    if frequency == bar.Frequency.DAY:
                        download_daily_bars(instrument, year, fileName)
                    else:
                        raise Exception("Invalid frequency")
                except Exception as e:
                    if skipErrors:
                        logger.error(str(e))
                        continue
                    else:
                        raise e
            ret.addBarsFromCSV(instrument, fileName)
    return ret

    def push(self, result, parameters):
        """
        Push strategy results obtained by running the strategy with the given parameters.

        :param result: The result obtained by running the strategy with the given parameters.
        :type result: float
        :param parameters: The parameters that yield the given result.
        :type parameters: Parameters
        """
        with self.__lock:
            if result is not None and (self.__bestResult is None or result > self.__bestResult):
                self.__bestResult = result
                self.__bestParameters = parameters
                logger.info("Best result so far %s with parameters %s" % (result, parameters.args))

def build_feed(instruments, fromYear, toYear, storage, frequency=bar.Frequency.DAY, timezone=None, skipErrors=False):
    """Build and load a :class:`pyalgotrade.barfeed.yahoofeed.Feed` using CSV files downloaded from Yahoo! Finance.
    CSV files are downloaded if they haven't been downloaded before.

    :param instruments: Instrument identifiers.
    :type instruments: list.
    :param fromYear: The first year.
    :type fromYear: int.
    :param toYear: The last year.
    :type toYear: int.
    :param storage: The path were the files will be loaded from, or downloaded to.
    :type storage: string.
    :param frequency: The frequency of the bars. Only **pyalgotrade.bar.Frequency.DAY** or **pyalgotrade.bar.Frequency.WEEK**
        are supported.
    :param timezone: The default timezone to use to localize bars. Check :mod:`pyalgotrade.marketsession`.
    :type timezone: A pytz timezone.
    :param skipErrors: True to keep on loading/downloading files in case of errors.
    :type skipErrors: boolean.
    :rtype: :class:`pyalgotrade.barfeed.tusharefeed.Feed`.
    """

    logger = pyalgotrade.logger.getLogger("tushare")
    ret = tusharefeed.Feed(frequency, timezone)
    
    
    datapath = os.path.join(storage, "data")
    if not os.path.exists(datapath):
        logger.info("Creating %s directory" % (datapath))
        os.mkdir(storage)

    for year in range(fromYear, toYear+1):
        for instrument in instruments:
            fileName = os.path.join(datapath, "%s-%d-tushare.csv" % (instrument, year))
            if not os.path.exists(fileName):
                logger.info("Downloading %s %d to %s" % (instrument, year, fileName))
                try:
                    if frequency == bar.Frequency.DAY:
                        download_daily_bars(instrument, year, fileName)
                    elif frequency == bar.Frequency.WEEK:
                        download_weekly_bars(instrument, year, fileName)
                    else:
                        raise Exception("Invalid frequency")
                except Exception, e:
                    if skipErrors:
                        logger.error(str(e))
                        continue
                    else:
                        raise e
            ret.addBarsFromCSV(instrument, fileName)