class MSGWeatherDataRetriever(object):
    """
    Retrieve national NOAA weather data relevant to the MSG project and save it
    to local storage in the path given in the configuration file for [Weather
     Data], weather_data_path.

    Unzip the retrieved data and recompress the hourly data using gzip.
    """

    def __init__(self):
        self.configer = MSGConfiger()
        self.pool = None
        self.fileList = []
        self.dateList = []
        self.weatherUtil = MSGWeatherDataUtil()

        global weatherDataPath
        weatherDataPath = self.configer.configOptionValue(WEATHER_DATA, "weather_data_path")
        global weatherDataURL
        weatherDataURL = self.configer.configOptionValue(WEATHER_DATA, "weather_data_url")

        global weatherDataPattern
        weatherDataPattern = self.configer.configOptionValue(WEATHER_DATA, "weather_data_pattern")

    def fileExists(self, filename):
        # @todo Move to external module.
        try:
            with open(filename):
                return True

        except IOError, e:
            # self.logger.log("IO Error: %s" % e, 'error')
            return False

        return False

class MECODataAutoloader(object):
    """
    Provide automated loading of MECO energy data from exports in gzip-compressed XML source data.
    """

    def __init__(self):
        """
        Constructor.
        """

        self.logger = MSGLogger(__name__)
        self.configer = MSGConfiger()
        self.fileUtil = MSGFileUtil()


    def newDataExists(self):
        """
        Check the data autoload folder for the presence of new data.

        :returns: True if new data exists.
        """

        autoloadPath = self.configer.configOptionValue('MECO Autoload', 'meco_new_data_path')
        if not self.fileUtil.validDirectory(autoloadPath):
            raise Exception('InvalidDirectory', '%s' % autoloadPath)

        patterns = ['*.gz']
        matchCnt = 0
        for root, dirs, filenames in os.walk(autoloadPath):
            for pat in patterns:
                for filename in fnmatch.filter(filenames, pat):
                    print filename
                    matchCnt += 1
        if matchCnt > 0:
            return True
        else:
            return False


    def loadNewData(self):
        """
        Load new data contained in the new data path.
        """

        autoloadPath = self.configer.configOptionValue('MECO Autoload', 'meco_new_data_path')
        command = self.configer.configOptionValue('MECO Autoload', 'data_load_command')
        os.chdir(autoloadPath)

        try:
            subprocess.check_call(command, shell = True)
        except subprocess.CalledProcessError, e:
            self.logger.log("An exception occurred: %s" % e, 'error')

class WeatherDataLoadingTester(unittest.TestCase):
    def setUp(self):
        self.weatherUtil = MSGWeatherDataUtil()
        self.logger = SEKLogger(__name__, 'DEBUG')
        self.dbConnector = MSGDBConnector()
        self.cursor = self.dbConnector.conn.cursor()
        self.configer = MSGConfiger()


    def testLoadDataSinceLastLoaded(self):
        """
        Data should be loaded since the last data present in the database.
        """
        pass


    def testRetrieveDataSinceLastLoaded(self):
        """
        Data since the last loaded date is retrieved.
        """
        pass


    def testGetLastLoadedDate(self):
        myDate = self.weatherUtil.getLastDateLoaded(self.cursor).strftime(
            "%Y-%m-%d %H:%M:%S")
        pattern = '^(\d+-\d+-\d+\s\d+:\d+:\d+)$'
        match = re.match(pattern, myDate)
        assert match and (match.group(1) == myDate), "Date format is valid."


    def testWeatherDataPattern(self):
        myPattern = self.configer.configOptionValue('Weather Data',
                                                    'weather_data_pattern')
        testString = """<A HREF="someURL">QCLCD201208.zip</A>"""

        match = re.match(myPattern, testString)
        self.logger.log("pattern = %s" % myPattern, 'info')
        if match:
            self.logger.log("match = %s" % match)
            self.logger.log("match group = %s" % match.group(1))
        else:
            self.logger.log("match not found")
        assert match and match.group(
            1) == 'QCLCD201208.zip', "Download filename was matched."


    def testWeatherDataURL(self):
        myURL = self.configer.configOptionValue('Weather Data',
                                                'weather_data_url')
        pass

    def __init__(self):
        """
        Constructor.
        """

        warnings.simplefilter('default')
        warnings.warn("This module is deprecated in favor of SEKNotifier.",
                      DeprecationWarning)

        self.config = MSGConfiger()
        self.logger = SEKLogger(__name__, 'info')
        self.connector = MSGDBConnector()
        self.conn = self.connector.connectDB()
        self.cursor = self.conn.cursor()
        self.dbUtil = MSGDBUtil()
        self.noticeTable = 'NotificationHistory'
        self.notificationHeader = "This is a message from the Hawaii Smart " \
                                  "Energy Project MSG Project notification " \
                                  "system.\n\n"

        self.noReplyNotice = '\n\nThis email account is not monitored. No ' \
                             'replies will originate from this ' \
                             'account.\n\nYou are receiving this message ' \
                             'because you are on the recipient list for ' \
                             'notifications for the Hawaii Smart Energy ' \
                             'Project.'

    def setUp(self):
        self.logger = SEKLogger(__name__, 'DEBUG')
        self.configer = MSGConfiger()
        self.exporter = MSGDBExporter()
        self.testDir = 'db_exporter_test'
        self.uncompressedTestFilename = 'meco_v3_test_data.sql'
        self.compressedTestFilename = 'meco_v3_test_data.sql.gz'
        self.exportTestDataPath = self.configer.configOptionValue('Testing',
                                                                  'export_test_data_path')
        self.fileUtil = MSGFileUtil()
        self.fileChunks = []
        self.testDataFileID = ''
        self.pyUtil = MSGPythonUtil()
        self.timeUtil = MSGTimeUtil()

        conn = None
        try:
            conn = MSGDBConnector().connectDB()
        except Exception as detail:
            self.logger.log("Exception occurred: {}".format(detail), 'error')
            exit(-1)

        self.logger.log("conn = {}".format(conn), 'debug')
        self.assertIsNotNone(conn)

        # Create a temporary working directory.
        try:
            os.mkdir(self.testDir)
        except OSError as detail:
            self.logger.log(
                'Exception during creation of temp directory: %s' % detail,
                'ERROR')

class TestMSGDBConnect(unittest.TestCase):
    """
    These tests require a database connection to be available.
    """

    def setUp(self):
        self.connector = MSGDBConnector(True)
        self.conn = self.connector.connectDB()
        self.configer = MSGConfiger()

    def test_init(self):
        self.assertTrue(
            isinstance(self.connector, msg_db_connector.MSGDBConnector),
            "self.connection is an instance of MECODBConnector.")

    def test_db_connection(self):
        """
        DB can be connected to.
        """
        self.assertIsNotNone(self.conn, 'DB connection not available.')

        # Get the name of the database.
        self.assertEqual(
            self.configer.configOptionValue('Database', 'testing_db_name'),
            self.connector.dbName, 'Testing DB name is not correct.')

    def tearDown(self):
        self.connector.closeDB(self.conn)

    def __init__(self):
        """
        Constructor.
        """

        self.logger = MSGLogger(__name__)
        self.configer = MSGConfiger()
        self.fileUtil = MSGFileUtil()

    def __init__(self, testing = False):
        """
        Constructor.

        :param testing: Use testing mode if True.
        """

        self.reader = MECODBReader(testing)
        self.configer = MSGConfiger()
        self.dpi = 300

    def __init__(self):
        """
        Constructor.
        """

        self.logger = SEKLogger(__name__, 'DEBUG', useColor = False)
        self.timeUtil = MSGTimeUtil()
        self.configer = MSGConfiger()
        self.fileUtil = MSGFileUtil()
        self.pythonUtil = MSGPythonUtil()  # for debugging
        self.connector = MSGDBConnector()
        self.conn = self.connector.connectDB()
        self.cursor = self.conn.cursor()
        self.dbUtil = MSGDBUtil()
        self.notifier = SEKNotifier(connector = self.connector,
                                    dbUtil = self.dbUtil,
                                    user = self.configer.configOptionValue(
                                        'Notifications', 'email_username'),
                                    password = self.configer.configOptionValue(
                                        'Notifications', 'email_password'),
                                    fromaddr = self.configer.configOptionValue(
                                        'Notifications', 'email_from_address'),
                                    toaddr = self.configer.configOptionValue(
                                        'Notifications', 'email_recipients'),
                                    testing_toaddr =
                                    self.configer.configOptionValue(
                                        'Notifications',
                                        'testing_email_recipients'),
                                    smtp_server_and_port =
                                    self.configer.configOptionValue(
                                        'Notifications',
                                        'smtp_server_and_port'))

        # Google Drive parameters.
        self.clientID = self.configer.configOptionValue('Export',
                                                        'google_api_client_id')
        self.clientSecret = self.configer.configOptionValue('Export',
                                                            'google_api_client_secret')
        self.oauthScope = 'https://www.googleapis.com/auth/drive'
        self.oauthConsent = 'urn:ietf:wg:oauth:2.0:oob'
        self.googleAPICredentials = ''
        self.exportTempWorkPath = self.configer.configOptionValue('Export',
                                                                  'db_export_work_path')

        self.credentialPath = self.configer.configOptionValue('Export',
                                                              'google_api_credentials_path')
        self.credentialStorage = Storage(
            '{}/google_api_credentials'.format(self.credentialPath))

        self._driveService = None
        self._cloudFiles = None
        self.postAgent = 'Maui Smart Grid 1.0.0 DB Exporter'
        self.retryDelay = 10
        self.availableFilesURL = ''

    def __init__(self):
        """
        Constructor.
        """

        self.config = MSGConfiger()
        self.logger = MSGLogger(__name__, 'info')
        self.notificationHeader = "This is a message from the Hawaii Smart " \
                                  "Energy Project MSG Project notification " \
                                  "system.\n\n"

        self.noReplyNotice = """\n\nThis email account is not monitored. No

class TestMECONotifier(unittest.TestCase):
    """
    Unit tests for the MECO Notifier.
    """

    def setUp(self):
        self.logger = MSGLogger(__name__)
        self.notifier = MSGNotifier()
        self.configer = MSGConfiger()

    def tearDown(self):
        pass

    def testInit(self):
        self.assertIsNotNone(self.notifier, "Notifier has been initialized.")


    def testEmailServer(self):
        """
        Test connecting to the email server.
        """

        errorOccurred = False
        user = self.configer.configOptionValue('Notifications', 'email_username')
        password = self.configer.configOptionValue('Notifications',
                                                 'email_password')

        server = smtplib.SMTP(
            self.configer.configOptionValue('Notifications', 'email_smtp_server'))

        try:
            server.starttls()
        except smtplib.SMTPException, e:
            print "Exception = %s" % e

        try:
            server.login(user, password)
        except smtplib.SMTPException, e:
            print "Exception = %s" % e

    def __init__(self):
        self.configer = MSGConfiger()
        self.pool = None
        self.fileList = []
        self.dateList = []
        self.weatherUtil = MSGWeatherDataUtil()

        global weatherDataPath
        weatherDataPath = self.configer.configOptionValue(WEATHER_DATA, "weather_data_path")
        global weatherDataURL
        weatherDataURL = self.configer.configOptionValue(WEATHER_DATA, "weather_data_url")

        global weatherDataPattern
        weatherDataPattern = self.configer.configOptionValue(WEATHER_DATA, "weather_data_pattern")

    def __init__(self, exitOnError=True, commitOnEveryInsert=False, testing=False):
        """
        Constructor.

        :param testing: if True, the testing DB will be connected instead of
        the production DB.
        """

        self.logger = SEKLogger(__name__, "info")
        self.configer = MSGConfiger()
        self.conn = MSGDBConnector().connectDB()
        self.cursor = self.conn.cursor()
        self.dbUtil = MSGDBUtil()
        self.notifier = MSGNotifier()
        self.mathUtil = MSGMathUtil()
        self.timeUtil = MSGTimeUtil()
        self.nextMinuteCrossing = {}
        self.nextMinuteCrossingWithoutSubkeys = None
        self.exitOnError = exitOnError
        self.commitOnEveryInsert = commitOnEveryInsert
        section = "Aggregation"
        tableList = [
            "irradiance",
            "agg_irradiance",
            "weather",
            "agg_weather",
            "circuit",
            "agg_circuit",
            "egauge",
            "agg_egauge",
        ]
        self.dataParams = {
            "weather": ("agg_weather", "timestamp", ""),
            "egauge": ("agg_egauge", "datetime", "egauge_id"),
            "circuit": ("agg_circuit", "timestamp", "circuit"),
            "irradiance": ("agg_irradiance", "timestamp", "sensor_id"),
        }
        self.columns = {}

        # tables[datatype] gives the table name for datatype.
        self.tables = {t: self.configer.configOptionValue(section, "{}_table".format(t)) for t in tableList}

        for t in self.tables.keys():
            self.logger.log("t:{}".format(t), "DEBUG")
            try:
                self.columns[t] = self.dbUtil.columnsString(self.cursor, self.tables[t])
            except TypeError as error:
                self.logger.log("Ignoring missing table: Error is {}.".format(error), "error")

    def __init__(self):
        self.configer = MSGConfiger()
        self.pool = None
        self.fileList = []
        self.dateList = []
        self.weatherUtil = MSGWeatherDataUtil()

        global weatherDataPath
        weatherDataPath = self.configer.configOptionValue('Weather Data',
                                                          'weather_data_path')
        global weatherDataURL
        weatherDataURL = self.configer.configOptionValue('Weather Data',
                                                         'weather_data_url')

        global weatherDataPattern
        weatherDataPattern = self.configer.configOptionValue('Weather Data',
                                                             'weather_data_pattern')

    def __init__(self):
        """
        Constructor.

        A database connection is not maintained here to keep this class
        lightweight.
        """

        self.logger = SEKLogger(__name__, DEBUG)
        self.configer = MSGConfiger()
        self.url = self.configer.configOptionValue('Weather Data',
                                                   'weather_data_url')
        self.pattern = self.configer.configOptionValue('Weather Data',
                                                       'weather_data_pattern')
        self.fileList = []
        self.dateList = [] # List of dates corresponding weather data files.
        self.fillFileListAndDateList()
        self.dbUtil = MSGDBUtil()

    def __init__(self, exitOnError = True, commitOnEveryInsert = False,
                 testing = False):
        """
        Constructor.

        :param testing: if True, the testing DB will be connected instead of
        the production DB.
        """

        self.logger = MSGLogger(__name__, 'info')
        self.configer = MSGConfiger()
        self.conn = MSGDBConnector().connectDB()
        self.cursor = self.conn.cursor()
        self.dbUtil = MSGDBUtil()
        self.notifier = MSGNotifier()
        self.mathUtil = MSGMathUtil()
        self.timeUtil = MSGTimeUtil()
        self.nextMinuteCrossing = {}
        self.nextMinuteCrossingWithoutSubkeys = None
        self.exitOnError = exitOnError
        self.commitOnEveryInsert = commitOnEveryInsert
        section = 'Aggregation'
        tableList = ['irradiance', 'agg_irradiance', 'weather', 'agg_weather',
                     'circuit', 'agg_circuit', 'egauge', 'agg_egauge']
        self.dataParams = {'weather': ('agg_weather', 'timestamp', ''),
                           'egauge': ('agg_egauge', 'datetime', 'egauge_id'),
                           'circuit': ('agg_circuit', 'timestamp', 'circuit'),
                           'irradiance': (
                               'agg_irradiance', 'timestamp', 'sensor_id')}
        self.columns = {}

        # tables[datatype] gives the table name for datatype.
        self.tables = {
            t: self.configer.configOptionValue(section, '%s_table' % t) for t in
            tableList}

        for t in self.tables.keys():
            self.logger.log('t:%s' % t, 'DEBUG')
            try:
                self.columns[t] = self.dbUtil.columnsString(self.cursor,
                                                            self.tables[t])
            except TypeError as error:
                self.logger.log('Ignoring missing table: Error is %s.' % error,
                                'error')

    def __init__(self):
        """
        Constructor.

        A database connection is not maintained here to keep this class
        lightweight. This results in the class not having a parameter for
        TESTING MODE.
        """

        self.logger = MSGLogger(__name__, 'info')
        self.configer = MSGConfiger()
        self.url = self.configer.configOptionValue('Weather Data',
                                                   'weather_data_url')
        self.pattern = self.configer.configOptionValue('Weather Data',
                                                       'weather_data_pattern')
        self.fileList = []
        self.dateList = [] # List of dates corresponding weather data files.
        self.fillFileListAndDateList()
        self.dbUtil = MSGDBUtil()

    def __init__(self, testing = False, logLevel = 'silent'):
        """
        Constructor.

        :param testing: Boolean indicating if Testing Mode is on. When
        testing mode is on, a connection to the testing database will be made
        instead of the production database. This is useful for unit testing.
        :param logLevel
        """

        self.logger = SEKLogger(__name__, logLevel)

        if testing:
            self.logger.log("Testing Mode is ON.")

        self.configer = MSGConfiger()
        self.dbPassword = self.configer.configOptionValue("Database",
                                                          'db_password')
        self.dbHost = self.configer.configOptionValue("Database", 'db_host')
        self.dbPort = self.configer.configOptionValue("Database", 'db_port')

        if testing:
            self.dbName = self.configer.configOptionValue("Database",
                                                          'testing_db_name')
        else:
            self.dbName = self.configer.configOptionValue("Database", 'db_name')

        self.logger.log(
            "Instantiating DB connector with database {}.".format(self.dbName))

        self.dbUsername = self.configer.configOptionValue("Database",
                                                          'db_username')

        self.conn = self.connectDB()
        if not self.conn:
            raise Exception('DB connection not available.')

        try:
            self.dictCur = self.conn.cursor(
                cursor_factory = psycopg2.extras.DictCursor)
        except AttributeError as error:
            self.logger.log('Error while getting DictCursor: {}'.format(error))

class TestMECOConfig(unittest.TestCase):
    def setUp(self):
        self.configer = MSGConfiger()

    def test_init(self):
        # @todo Improve this test by using direct introspection instead of
        # this roundabout method.
        localConfiger = MSGConfiger()
        self.assertIsInstance(self.configer, type(localConfiger))

    def test_get_debugging(self):
        """
        Verify the debugging option in the configuration file.
        """

        debugging = self.configer.configOptionValue("Debugging", "debug")

        if debugging is False:
            self.assertFalse(debugging, "Debugging/debug is not set to False.")
        elif debugging is True:
            self.assertTrue(debugging, "Debugging/debug is not set to True.")
        else:
            self.assertTrue(False,
                            "Debugging/debug does not have a valid value.")

 def setUp(self):
     self.weatherUtil = MSGWeatherDataUtil()
     self.logger = SEKLogger(__name__, 'DEBUG')
     self.dbConnector = MSGDBConnector()
     self.cursor = self.dbConnector.conn.cursor()
     self.configer = MSGConfiger()