def test_fetch_csv_genmix(self):
     c = client_factory("NYISO")
     now = pytz.utc.localize(datetime.utcnow())
     today = now.astimezone(pytz.timezone(c.TZ_NAME)).date()
     content_list = c.fetch_csvs(today, "rtfuelmix")
     self.assertEqual(len(content_list), 1)
     self.assertEqual(content_list[0].split("\r\n")[0], "Time Stamp,Time Zone,Fuel Category,Gen MWh")

 def test_fetch_csv_load(self):
     c = client_factory("NYISO")
     now = pytz.utc.localize(datetime.utcnow())
     today = now.astimezone(pytz.timezone(c.TZ_NAME)).date()
     content_list = c.fetch_csvs(today, "pal")
     self.assertEqual(len(content_list), 1)
     self.assertEqual(content_list[0].split("\r\n")[0], '"Time Stamp","Time Zone","Name","PTID","Load"')

 def create_client(self, ba_name):
     # set up client with logging
     c = client_factory(ba_name)
     handler = logging.StreamHandler()
     c.logger.addHandler(handler)
     c.logger.setLevel(logging.INFO)
     return c

    def setUp(self):
        self.c = client_factory('AZPS')
        handler = logging.StreamHandler()
        self.c.logger.addHandler(handler)
        self.c.logger.setLevel(logging.INFO)
        self.TZ_NAME = 'America/Phoenix'

        self.now = datetime.now(pytz.timezone(self.TZ_NAME))
        self.today = datetime.combine(self.now.date(), time())
        self.yesterday = self.today - timedelta(days=1)
        self.tomorrow = self.today + timedelta(days=1)
        self.last_month = self.today - timedelta(days=32)
        self.next_month = self.today + timedelta(days=32)

        self.sample = '"Time (MST)","Solar Aggregate (MW)","Wind Aggregate (MW)","Other Renewables Aggregate (MW)","Hydro Aggregate (MW)"\n\
"2015-07-18 00:00:05",0.028038,134.236,116.294,835.628\n\
"2015-07-18 00:00:15",0.079792,132.875,115.742,817.136\n\
"2015-07-18 00:00:25",0.101218,131.663,115.727,802.076\n\
"2015-07-18 00:00:35",0.119465,130.456,115.712,798.891\n\
"2015-07-18 00:00:45",0.180171,129.403,115.697,820.483\n\
"2015-07-18 00:00:55",0.268247,129.221,115.681,849.028\n\
"2015-07-18 00:01:05",0.29724,128.836,115.666,865.843\n\
"2015-07-18 00:01:15",0.281987,127.656,115.354,862.289\n\
'
        self.sample_start = pytz.timezone(self.TZ_NAME).localize(datetime(2015, 7, 18, 0, 0))
        self.sample_end = pytz.timezone(self.TZ_NAME).localize(datetime(2015, 7, 19, 0, 0))

    def _run_test(self, ba_name, **kwargs):
        # set up
        c = client_factory(ba_name)

        # get data
        data = c.get_generation(**kwargs)

        # test number
        self.assertGreater(len(data), 1)

        # test contents
        for dp in data:
            # test key names
            self.assertEqual(set(['gen_MW', 'ba_name', 'fuel_name',
                                  'timestamp', 'freq', 'market']),
                             set(dp.keys()))

            # test values
            self.assertEqual(dp['timestamp'].tzinfo, pytz.utc)
            self.assertIn(dp['fuel_name'], self.FUEL_CHOICES)
            self.assertIn(dp['ba_name'], self.BA_CHOICES)

            # test for numeric gen
            self.assertGreaterEqual(dp['gen_MW']+1, dp['gen_MW'])

            # test earlier than now
            if c.options.get('forecast', False):
                self.assertGreater(dp['timestamp'], pytz.utc.localize(datetime.utcnow()))
            else:
                self.assertLess(dp['timestamp'], pytz.utc.localize(datetime.utcnow()))

        # return
        return data

    def test_parse_ren_report_bot(self):
        c = client_factory("CAISO")

        # bottom half
        bot_df = c.parse_to_df(self.ren_report_tsv, skiprows=29, nrows=24, header=0, delimiter="\t+", engine="python")
        self.assertEqual(list(bot_df.columns), ["Hour", "RENEWABLES", "NUCLEAR", "THERMAL", "IMPORTS", "HYDRO"])
        self.assertEqual(len(bot_df), 24)

 def test_pjm_latest(self):
     expected = client_factory('PJM').get_generation(**self.latest_kwargs)
     received = tasks.get_generation('PJM', **self.latest_kwargs)
     for i in range(len(expected)):
         if expected[i]['timestamp'] == received[i]['timestamp']:
             self.assertEqual(expected[i]['gen_MW'], received[i]['gen_MW'])
             self.assertEqual(expected[i]['fuel_name'], received[i]['fuel_name'])

 def test_ercot_latest(self):
     # before min 32, will not have wind data
     if datetime.now().minute >= 32:
         # basic test
         data = self._run_test('ERCOT', latest=True)
         
         # test all timestamps are equal
         timestamps = [d['timestamp'] for d in data]
         self.assertEqual(len(set(timestamps)), 1)
         
         # test flags
         for dp in data:
             self.assertEqual(dp['market'], self.MARKET_CHOICES.hourly)
             self.assertEqual(dp['freq'], self.FREQUENCY_CHOICES.hourly)                
 
         # test fuel names
         fuels = set([d['fuel_name'] for d in data])
         expected_fuels = ['wind', 'nonwind']
         for expfuel in expected_fuels:
             self.assertIn(expfuel, fuels)
             
     else:
         c = client_factory('ERCOT')
         data = c.get_generation(latest=True)
         self.assertEqual(len(data), 0)

    def test_fetch_oasis_ren_dam(self):
        c = client_factory("CAISO")
        ts = c.utcify("2014-05-08 12:00")
        payload = {
            "queryname": "SLD_REN_FCST",
            "market_run_id": "DAM",
            "startdatetime": (ts - timedelta(minutes=20)).strftime(c.oasis_request_time_format),
            "enddatetime": (ts + timedelta(minutes=40)).strftime(c.oasis_request_time_format),
        }
        payload.update(c.base_payload)
        data = c.fetch_oasis(payload=payload)
        self.assertEqual(len(data), 4)
        self.assertEqual(
            str(data[0]),
            "<report_data>\n\
<data_item>RENEW_FCST_DA_MW</data_item>\n\
<opr_date>2014-05-08</opr_date>\n\
<interval_num>13</interval_num>\n\
<interval_start_gmt>2014-05-08T19:00:00-00:00</interval_start_gmt>\n\
<interval_end_gmt>2014-05-08T20:00:00-00:00</interval_end_gmt>\n\
<value>813.7</value>\n\
<trading_hub>NP15</trading_hub>\n\
<renewable_type>Solar</renewable_type>\n\
</report_data>",
        )

    def setUp(self):
        environ['EIA_KEY'] = 'test'

        self.c = client_factory("EIA")
        self.longMessage = True
        self.BA_CHOICES = EIAClient.EIA_BAs
        self.BALists = BALists()

    def test_get_ancillary_services(self):
        c = client_factory('CAISO')
        ts = datetime(2015, 3, 1, 11, 0, 0, tzinfo=pytz.utc)
        start = ts - timedelta(days=2)

        as_prc = c.get_ancillary_services('AS_CAISO_EXP', start_at=start, end_at=ts,
                                          market_run_id='DAM')

        self.assertEqual(len(as_prc), 48)
        self.assertGreaterEqual(min([i['timestamp'] for i in as_prc]),
                                start - timedelta(minutes=5))
        self.assertLessEqual(max([i['timestamp'] for i in as_prc]),
                             ts + timedelta(minutes=5))

        means = {
            'SR': 1.685417,
            'RU': 3.074583,
            'RMU': 5.729167e-02,
            'RMD': 3.620833e-01,
            'RD': 3.901667,
            'NR': 9.000000e-02,
        }

        for anc_type in means:
            dp = [i[anc_type] for i in as_prc]
            self.assertAlmostEqual(numpy.mean(dp), means[anc_type], places=6)

 def test_utcify_index(self):
     c = client_factory('BPA')
     df = c.parse_to_df(self.wind_tsv, skiprows=6, header=0, delimiter='\t',
                        index_col=0, parse_dates=True)
     utc_index = c.utcify_index(df.index)
     self.assertEqual(utc_index[0].to_pydatetime(), datetime(2014, 4, 15, 10+7, 10, tzinfo=pytz.utc))
     self.assertEqual(len(df), len(utc_index))

 def test_get_AS_dataframe_empty(self):
     c = client_factory('CAISO')
     st = pytz.utc.localize(datetime.now() + timedelta(days=2))
     et = st + timedelta(days=1)
     as_prc = c.get_AS_dataframe('AS_CAISO_EXP', start_at=st, end_at=et,
                                 market_run_id='DAM', anc_type='RU')
     self.assertTrue(as_prc.empty)

    def test_get_AS_dataframe_latest(self):
        c = client_factory('CAISO')
        as_prc = c.get_AS_dataframe('AS_CAISO_EXP')

        # Could be 1 or 2 prices in last 61 minutes
        self.assertLessEqual(len(as_prc), 12)
        self.assertGreaterEqual(len(as_prc), 6)

 def _run_test(self, ba, node_list, kwargs):
     expected = client_factory(ba).get_lmp(node_list, **kwargs)
     received = tasks.get_lmp(ba, node_list, **kwargs)
     for i in range(len(expected)):
         if expected[i]['timestamp'] == received[i]['timestamp']:
             self.assertEqual(expected[i]['lmp'], received[i]['lmp'])
             self.assertEqual(expected[i]['lmp_type'], received[i]['lmp_type'])

    def test_get_AS_dataframe(self):
        c = client_factory('CAISO')
        ts = datetime(2015, 3, 1, 11, 0, 0, tzinfo=pytz.utc)
        start = ts - timedelta(days=2)

        as_prc = c.get_AS_dataframe(node_id='AS_CAISO_EXP', start_at=start, end_at=ts,
                                    market_run_id='DAM')

        self.assertEqual(len(as_prc), 288)
        self.assertAlmostEqual(as_prc['MW'].mean(), 1.528506944444443)

        grouped = as_prc.groupby('XML_DATA_ITEM')
        self.assertEqual(len(grouped), 6)
        means = {
            'SP_CLR_PRC': 1.685417,
            'RU_CLR_PRC': 3.074583,
            'RMU_CLR_PRC': 5.729167e-02,
            'RMD_CLR_PRC': 3.620833e-01,
            'RD_CLR_PRC': 3.901667,
            'NS_CLR_PRC': 9.000000e-02,
        }

        for group in means:
            self.assertAlmostEqual(grouped.get_group(group)['MW'].mean(), means[group], places=6)
            self.assertEqual(len(grouped.get_group(group)), 48)

 def _run_test(self, ba, kwargs):
     expected = client_factory(ba).get_generation(**kwargs)
     received = tasks.get_generation(ba, **kwargs)
     for i in range(len(expected)):
         if expected[i]['timestamp'] == received[i]['timestamp']:
             self.assertEqual(expected[i]['gen_MW'], received[i]['gen_MW'])
             self.assertEqual(expected[i]['fuel_name'], received[i]['fuel_name'])

    def _run_pairwise_test(self, ba_name, **kwargs):
        # set up
        c = client_factory(ba_name)

        # get data
        data = c.get_trade(**kwargs)

        # test number
        self.assertGreaterEqual(len(data), 1)

        # test contents
        for dp in data:
            # test key names
            for key in ['source_ba_name', 'dest_ba_name', 'timestamp', 'freq', 'market']:
                self.assertIn(key, dp.keys())
            self.assertEqual(len(dp.keys()), 6)

            # test values
            self.assertEqual(dp['timestamp'].tzinfo, pytz.utc)
            self.assertIn(dp['source_ba_name'], self.BA_CHOICES)

            # test for numeric value
            self.assertGreaterEqual(dp['export_MW']+1, dp['export_MW'])

            # test correct temporal relationship to now
            if c.options['forecast']:
                self.assertGreaterEqual(dp['timestamp'], datetime.now(pytz.utc))
            else:
                self.assertLess(dp['timestamp'], datetime.now(pytz.utc))

        # return
        return data

 def test_names(self):
     for name in self.expected_names:
         if 'env_vars' in name:
           for var in name['env_vars']:
             environ[var] = name['env_vars'][var];
         c = client_factory(name['name'])
         self.assertIsNotNone(c)

 def test_fetch_csv_load_forecast(self):
     c = client_factory('NYISO')
     now = pytz.utc.localize(datetime.utcnow())
     today = now.astimezone(pytz.timezone(c.TZ_NAME)).date()
     content = c.fetch_csv(today, 'isolf')
     self.assertEqual(content.split('\n')[0],
                      '"Time Stamp","Capitl","Centrl","Dunwod","Genese","Hud Vl","Longil","Mhk Vl","Millwd","N.Y.C.","North","West","NYISO"')