def test_copy_to_multiple_directories(self):
        """Copy files into multiple directories.
        """
        dir1 = tempfile.mkdtemp()
        dir2 = tempfile.mkdtemp()
        dirs = [dir1, dir2]
        file_1_obj = tempfile.NamedTemporaryFile(suffix='.txt')
        file_2_obj = tempfile.NamedTemporaryFile(suffix='.txt')
        file_3_obj = tempfile.NamedTemporaryFile(suffix='.txt')
        files = [file_1_obj.name, file_2_obj.name, file_3_obj.name]

        self._ftp.copy_to_multiple_directories(dirs, files)

        # Check target dirs.
        for dir in dirs:
            tmp_files = get_directory_files_list(dir)
            received = [os.path.basename(x) for x in tmp_files]
            expected = [os.path.basename(x) for x in files]
            msg = 'Mutliple dir copy - %s listing error' % dir
            self.assertListEqual(sorted(received), sorted(expected), msg)

        # Clean up.
        file_1_obj.close()
        file_2_obj.close()
        file_3_obj.close()
        for dir in dirs:
            remove_files(get_directory_files_list(dir))
            os.removedirs(dir)

    def test_signature_archive_selective(self):
        """Archive signature file -- only *.ps files.
        """
        # Define the archive directory.
        archive_dir = tempfile.mkdtemp()
        self._e.set_signature_dir(self._dir)

        # Create a dummy signature files.
        dummy_files = ['22222.ps', '22222.png']
        for i in dummy_files:
            sig_file = os.path.join(self._e.signature_dir, i)
            fh = open(sig_file, 'w')
            fh.close()

        archive_control = {'ps': True,
                           'png': False}
        self._e.archive_signature_file(22222,
                                       archive_control,
                                       self._e.signature_dir,
                                       archive_dir)

        # Check that the file now exists in archive.
        digest_path = gen_digest_path('22222')
        archive_digest_path = os.path.join(archive_dir, *digest_path)
        received = get_directory_files_list(archive_digest_path)
        expected = [os.path.join(archive_digest_path, '22222.ps')]
        msg = 'Signature file archive directory should succeed -- ps only'
        self.assertListEqual(received, expected, msg)

        # Cleanup
        self._e.reset()
        remove_files(get_directory_files_list(self._e.signature_dir))
        remove_files(received)
        self._e.set_signature_dir(None)
        os.removedirs(archive_digest_path)

    def get_files(self):
        """Searches the :attr:`adp_in_dirs` directories for TCD report
        files.

        **Returns:**
            list of ADP bulk load files

        """
        files = []

        for dir in self.adp_in_dirs:
            log.debug('Searching "%s" for ADP bulk insert files' % dir)
            if len(self.adp_file_formats):
                for format in self.adp_file_formats:
                    files.extend(get_directory_files_list(dir, format))
            else:
                files.extend(get_directory_files_list(dir))

        # Convert xlsx file to csv.
        converted_files = []
        for f in files:
            tmp_file = xlsx_to_csv_converter(f)
            if tmp_file is not None:
                converted_files.append(tmp_file)

        log.debug('All ADP bulk load files: "%s"' % converted_files)

        return converted_files

    def test_inbound(self):
        """Inbound file transfer.
        """
        dir = self._ftp_dir
        t_files = get_directory_files_list(os.path.join(self._test_dir,
                                                        'returns'))
        for f in t_files:
            copy_file(f, os.path.join(dir, os.path.basename(f)))

        # Prepare the config.
        self._ftp.config.add_section('ftp_in')
        self._ftp.config.set('ftp_in', 'host', '127.0.0.1')
        self._ftp.config.set('ftp_in', 'port', '2121')
        self._ftp.config.set('ftp_in', 'user', 'tester')
        self._ftp.config.set('ftp_in', 'password', 'tester')
        self._ftp.config.set('ftp_in', 'filter', '.*_VANA_RE[PFI]_\d{14}\.txt')
        self._ftp.config.set('ftp_in', 'target', '')
        self._ftp.config.set('ftp_in', 'pod', 'yes')
        self._ftp._parse_config(file_based=False)

        self._ftp.inbound(self._ftp.xfers[0], dry=True)

        # Clean up.
        remove_files(get_directory_files_list(dir))
        self._ftp.reset_config()

    def test_move_signature_files_dry_pass(self):
        """Move signature files -- dry pass.
        """
        dry = True

        source_dir = tempfile.mkdtemp()
        target_dir = tempfile.mkdtemp()
        sig_files = [os.path.join(source_dir, '03970006761.ps'),
                     os.path.join(source_dir, '03970006762.png')]
        for f in sig_files:
            fh = open(f, 'w')
            fh.close

        received = self._ptd.move_signature_files('03970006762',
                                                  source_dir,
                                                  target_dir,
                                                  dry=dry)
        msg = 'Signature file move error -- dry run'
        self.assertTrue(received, msg)

        # Check the source file exists.
        received = os.path.exists(os.path.join(source_dir,
                                               '03970006762.png'))
        self.assertTrue(received, msg)

        # Clean up.
        remove_files(get_directory_files_list(source_dir))
        remove_files(get_directory_files_list(target_dir))
        os.removedirs(source_dir)
        os.removedirs(target_dir)

    def test_get_files(self):
        """Get files from remote resource.
        """
        dir = self._ftp_dir
        t_files = get_directory_files_list(os.path.join(self._test_dir,
                                                        'returns'))
        remote_files = []
        for f in t_files:
            remote_files.append(os.path.basename(f))
            copy_file(f, os.path.join(dir, os.path.basename(f)))

        # Prepare the config.
        self._ftp.config.add_section('ftp_in')
        self._ftp.config.set('ftp_in', 'host', '127.0.0.1')
        self._ftp.config.set('ftp_in', 'port', '2121')
        self._ftp.config.set('ftp_in', 'user', 'tester')
        self._ftp.config.set('ftp_in', 'password', 'tester')
        self._ftp._parse_config(file_based=False)

        self._ftp.connect_resource(self._ftp.xfers[0])

        # No files to transfer.
        files_to_transfer = []
        received = self._ftp.get_files(files_to_transfer,
                                       target_dir=self._dir,
                                       dry=False)
        expected = []
        msg = 'No files to retrieve error'
        self.assertListEqual(received, expected, msg)

        # Single file transfer.
        files_to_transfer = ['VIC_VANA_REF_20131114073201.txt']
        received = self._ftp.get_files(files_to_transfer,
                                       target_dir=self._dir,
                                       dry=False)
        expected = get_directory_files_list(self._dir)
        ret = expected
        msg = 'Single file retrieved error'
        self.assertListEqual(received, expected, msg)

        # Report files to transfer.
        del expected[:]
        received = self._ftp.get_files(remote_files,
                                       target_dir=self._dir,
                                       dry=False)
        expected = get_directory_files_list(self._dir)
        msg = 'Retrieved multiple file list error'
        self.assertListEqual(sorted(received), sorted(expected), msg)

        # Clean up.
        self._ftp.disconnect_resource()
        remove_files(get_directory_files_list(self._dir))
        remove_files(get_directory_files_list(dir))
        self._ftp.reset_config()

    def test_process(self):
        """Test the process cycle.
        """
        t_files = ['VIC_VANA_REP_20131108145146.txt',
                   '142828.ps',
                   '145563.ps',
                   '145601.ps',
                   '145661.ps']
        dir = tempfile.mkdtemp()
        for f in t_files:
            copy_file(os.path.join(self._test_dir, f), os.path.join(dir, f))

        # Prepare the config.
        self._ftp.config.add_section('ftp_t')
        self._ftp.config.set('ftp_t', 'host', '127.0.0.1')
        self._ftp.config.set('ftp_t', 'port', '2121')
        self._ftp.config.set('ftp_t', 'user', 'tester')
        self._ftp.config.set('ftp_t', 'password', 'tester')
        self._ftp.config.set('ftp_t', 'source', dir)
        self._ftp.config.set('ftp_t', 'filter', 'VIC_VANA_REP_\d{14}\.txt')
        self._ftp.config.set('ftp_t', 'target', '')
        self._ftp.config.set('ftp_t', 'pod', 'yes')
        self._ftp._parse_config(file_based=False)

        self._ftp.process(dry=False)

        # Check archive directory.
        ymd = datetime.datetime.now().strftime('%Y%m%d')
        archive_dir = os.path.join(self._ftp.archive_dir, 'tester', ymd)

        received = get_directory_files_list(archive_dir)
        expected = [os.path.join(archive_dir, x) for x in t_files]
        msg = 'Archive directory list not as expected'
        self.assertListEqual(sorted(received), sorted(expected), msg)

        # Check FTP inbound directory.
        received = get_directory_files_list(self._ftpserver.dir)
        expected = [os.path.join(self._ftpserver.dir, x) for x in t_files]
        msg = 'FTP inbound directory list not as expected'
        self.assertListEqual(sorted(received), sorted(expected), msg)

        # Clean up.
        for f in t_files:
            remove_files(os.path.join(self._ftp_dir, f))
            remove_files(os.path.join(archive_dir, f))
        os.removedirs(dir)
        os.rmdir(os.path.join(archive_dir))
        os.rmdir(os.path.join(self._ftp.archive_dir, 'tester'))
        self._ftp.reset_config()

    def test_process_pe_sms_error_comms(self):
        """Test processing -- primary elect SMS error comms.
        """
        dry = True

        files = ['email.3.pe', 'sms.3.pe']

        dir = tempfile.mkdtemp()
        comms_files = []
        for f in files:
            fh = open(os.path.join(dir, f), 'w')
            comms_files.append(fh.name)
            fh.close()

        # Provide a dodgy mobile.
        sql = """UPDATE job_item
SET phone_nbr = '0531602145'
WHERE id = 3"""
        self._c.db(sql)

        for file in comms_files:
            received = self._c.process(file, dry=dry)
            msg = 'Primary Elect comms files processed incorrect'
            if os.path.basename(file) == 'email.3.pe':
                self.assertTrue(received, msg)
            else:
                self.assertFalse(received, msg)

        # Cleanup.
        remove_files(get_directory_files_list(dir))
        os.removedirs(dir)
        self._c.db.rollback()

    def test_process_returns_err_sms(self):
        """Test processing -- returns with an invalid SMS.
        """
        dry = True

        # Provide a dodgy mobile.
        sql = """UPDATE returns
SET phone_nbr = '0531602145'
WHERE id = 2"""
        self._c.db(sql)

        files = ['email.2.ret', 'sms.2.ret']

        dir = tempfile.mkdtemp()
        comms_files = []
        for f in files:
            fh = open(os.path.join(dir, f), 'w')
            comms_files.append(fh.name)
            fh.close()

        for file in comms_files:
            received = self._c.process(file, dry=dry)
            msg = 'Returns comms files with invalid SMS error'
            if os.path.basename(file) == 'email.2.ret':
                self.assertTrue(received, msg)
            else:
                self.assertFalse(received, msg)

        # Cleanup.
        remove_files(get_directory_files_list(dir))
        os.removedirs(dir)
        self._c.db.rollback()

    def test_process_reminder_email_error_comms(self):
        """Test processing -- reminder email error comms.
        """
        dry = True

        files = ['email.6.rem', 'sms.6.rem']

        dir = tempfile.mkdtemp()
        comms_files = []
        for f in files:
            fh = open(os.path.join(dir, f), 'w')
            comms_files.append(fh.name)
            fh.close()

        # Provide a dodgy email.
        sql = """UPDATE job_item
SET email_addr = '@@@tollgroup.com'
WHERE id = 6"""
        self._c.db(sql)

        for file in comms_files:
            received = self._c.process(file, dry=dry)
            msg = 'Email error reminder comms files processed incorrect'
            if os.path.basename(file) == 'sms.6.rem':
                self.assertTrue(received, msg)
            else:
                self.assertFalse(received, msg)

        # Cleanup.
        remove_files(get_directory_files_list(dir))
        os.removedirs(dir)
        self._c.db.rollback()

    def test_processor_valid_record_with_matched_dp_trigger(self):
        """Process valid raw T1250 line -- matched delivery partner trigger.
        """
        # Seed the Agent Id.
        agent_fields = {'code': 'N031',
                        'dp_id': 1}
        self._ldr.db(self._ldr.db._agent.insert_sql(agent_fields))
        dps = ['Nparcel']

        msg = 'Valid T1250 record should process OK'
        line = self._c.get('test_lines', 'VALID_LINE_WITH_RECIPIENTS')
        self.assertTrue(self._ldr.process(self._job_ts,
                                          line,
                                          FILE_BU.get('tolp'),
                                          COND_MAP_COMMS,
                                          dps), msg)

        # With comms enabled, we should have comms flag files.
        received = get_directory_files_list(self._comms_dir)
        sql = """SELECT id
FROM job_item"""
        self._ldr.db(sql)
        expected = []
        for row in self._ldr.db.rows():
            expected.append(os.path.join(self._comms_dir, '%s.%d.%s') %
                            ('email', row[0], 'body'))
            expected.append(os.path.join(self._comms_dir, '%s.%d.%s') %
                            ('sms', row[0], 'body'))
        msg = 'Comms directory file list error'
        self.assertListEqual(sorted(received), sorted(expected), msg)

        # Restore DB state and clean.
        remove_files(received)
        self._ldr.db.rollback()

    def test_distribute_file_archive(self):
        """Distribute the T1250 file (archive).
        """
        base_dir = tempfile.mkdtemp()
        source_dir = os.path.join(base_dir, 'ipec', 'in')
        create_dir(source_dir)
        archive_dir = tempfile.mkdtemp()
        old_archive_dir = self._d.config.archive_dir
        self._d.config.set_archive_dir(archive_dir)

        # Fudge a T1250.
        test_file = 'T1250_TOLI_20130828202901.txt'
        fh = open(os.path.join(source_dir, test_file), 'w')
        fh.close()

        expected_archive_dir = os.path.join(archive_dir,
                                           'ipec',
                                           '20130828')

        # Test the archive.
        self._d.distribute_file(os.path.join(source_dir, test_file))
        received = get_directory_files_list(expected_archive_dir)
        expected = [os.path.join(expected_archive_dir, test_file)]
        msg = 'Archived file error'
        self.assertListEqual(received, expected, msg)

        # Cleanup.
        os.remove(os.path.join(expected_archive_dir, test_file))
        os.removedirs(source_dir)
        os.removedirs(expected_archive_dir)
        self._d.config.set_archive_dir(old_archive_dir)

    def test_start_non_dry(self):
        """Start non-dry loop.
        """
        comms_files = ['email.8.rem', 'sms.8.rem']
        dry = False

        old_dry = self._rd.dry
        old_batch = self._rd.batch

        # Start processing.
        self._rd.set_dry(dry)
        self._rd.set_batch()
        self._rd._start(self._rd.exit_event)

        # Check that the reminder comms event files were created.
        comms_dir = self._rd._config.comms_dir
        received = get_directory_files_list(comms_dir)
        expected = [os.path.join(comms_dir, x) for x in comms_files]
        msg = 'List of reminder comms event files not as expected'
        self.assertListEqual(sorted(received), sorted(expected), msg)

        # Clean up.
        remove_files(expected)
        self._rd.set_dry(old_dry)
        self._rd.set_batch(old_batch)
        self._rd.exit_event.clear()

    def test_filter_file_list(self):
        """Filter file list.
        """
        t_dir = os.path.join(self._test_dir, 'returns')
        t_files = get_directory_files_list(t_dir)

        format = 'banana'
        received = self._ftp.filter_file_list(t_files, format)
        expected = []
        msg = '"%s" filter list error' % format
        self.assertListEqual(received, expected, msg)

        format = 'VIC_VANA_REP_\d{14}\.txt'
        received = self._ftp.filter_file_list(t_files, format)
        priority_rep_file = ['VIC_VANA_REP_20131114044105.txt',
                             'VIC_VANA_REP_20131114050106.txt']
        expected = [os.path.join(t_dir, x) for x in priority_rep_file]
        msg = '"%s" filter list error' % format
        self.assertListEqual(sorted(received), sorted(expected), msg)

        format = 'VIC_VANA_REI_\d{14}\.txt'
        received = self._ftp.filter_file_list(t_files, format)
        priority_rep_file = ['VIC_VANA_REI_20131114044602.txt',
                             'VIC_VANA_REI_20131114045103.txt',
                             'VIC_VANA_REI_20131114045604.txt']
        expected = [os.path.join(t_dir, x) for x in priority_rep_file]
        msg = '"%s" filter list error' % format
        self.assertListEqual(sorted(received), sorted(expected), msg)

    def test_process_loader(self):
        """Test processing -- loader.
        """
        dry = True

        files = ['email.1.body', 'sms.1.body',
                 'email.2.body', 'sms.2.body',
                 'email.6.body', 'sms.6.body']
        dodgy = ['banana', 'email.rem.3']

        dir = tempfile.mkdtemp()
        comms_files = []
        for f in files + dodgy:
            fh = open(os.path.join(dir, f), 'w')
            comms_files.append(fh.name)
            fh.close()

        for file in comms_files:
            received = self._c.process(file, dry=dry)
            msg = 'Loader comms files processing error'
            filename = os.path.basename(file)
            if (filename == 'email.2.body' or
                filename == 'sms.2.body' or
                filename == 'email.6.body' or
                filename == 'sms.6.body'):
                self.assertTrue(received, msg)
            else:
                self.assertFalse(received, msg)

        # Cleanup.
        remove_files(get_directory_files_list(dir))
        os.removedirs(dir)
        self._c.db.rollback()

    def test_xlsx_to_csv_converter(self):
        """Convert a xlsx file to csv.
        """
        test_dir = os.path.join('top', 'tests', 'files')
        xlsx_file = os.path.join(test_dir, 'ADP-Bulk-Load.xlsx')
        dir = tempfile.mkdtemp()
        file_to_convert = os.path.join(dir, os.path.basename(xlsx_file))
        copy_file(xlsx_file, file_to_convert)

        csv_file = xlsx_to_csv_converter(file_to_convert)

        # Check that the csv file exists.
        msg = 'CSV file does not exist'
        received = os.path.exists(csv_file)
        self.assertTrue(received, msg)

        # Check contents.
        fh = open(csv_file)
        received = fh.readline().strip()
        fh.close()

        fh = open(os.path.join(test_dir, 'ADP-Bulk-Load.csv'))
        expected = fh.readline().strip()
        fh.close()

        msg = 'CSV file contents error'
        self.assertEqual(received, expected, msg)

        # Clean up.
        remove_files(get_directory_files_list(dir))
        os.removedirs(dir)

    def test_process(self):
        """Verify a processing loop.
        """
        # Place a copy of the test files in temp directory.
        dir = tempfile.mkdtemp()
        test_file = os.path.join('top',
                                 'tests',
                                 'files',
                                 'NSW_VANA_REF_20131121065550.txt')
        proc_file = os.path.join(dir, os.path.basename(test_file))
        copy_file(test_file, proc_file)
        sig_files = ['P3014R0-00007342.ps', 'P3014R0-00007343.ps']
        for f in sig_files:
            fh = open(os.path.join(dir, f), 'w')
            fh.close()

        # Original signature files should not exist.
        received = self._pt.process(file=proc_file, column='JOB_KEY')
        msg = 'POD translation processing loop error'
        for f in sig_files:
            self.assertFalse(os.path.exists(os.path.join(dir, f)))

        # Check that the translated file was produced.
        received = os.path.exists('%s.xlated' % proc_file)
        msg = 'Translated file error'
        self.assertTrue(received, msg)

        remove_files(get_directory_files_list(dir))
        os.removedirs(dir)

    def rename_signature_files(self, dir, old_token, new_token, dry=False):
        """Search *dir* for files which match the filter *old_token*
        and rename with *new_token*.  The original file name extension
        will be retained.

        **Args:**
            *dir*: directory to search

            *old_token*: the token to use as a filter for file matches that
            will be renamed

            *new_token*: the new filename

        **Returns:**
            for the files that were renamed, a list of the new filenames

        """
        signature_files = []

        files = get_directory_files_list(dir, filter='%s\..*' % old_token)
        log.info('Found old signature files: "%s"' % files)

        for f in files:
            (fn, ext) = os.path.splitext(f)
            target = os.path.join(os.path.dirname(f),
                                  '%s%s' % (new_token, ext))
            move_file(f, target, dry=dry)
            signature_files.append(target)

        return signature_files

    def test_get_files(self):
        """Get inbound report files list.
        """
        test_file_dir = os.path.join('top', 'tests', 'files')
        dir_1 = tempfile.mkdtemp()
        dir_2 = tempfile.mkdtemp()
        files = ['VIC_VANA_REI_20131108145146.txt',
                 'VIC_VANA_REP_20131108145146.txt']
        copy_file(os.path.join(test_file_dir, files[0]),
                  os.path.join(dir_1, files[0]))
        copy_file(os.path.join(test_file_dir, files[1]),
                  os.path.join(dir_2, files[1]))
        dirs = [dir_1, dir_2]
        filters = ['.*_REP_\d{14}\.txt$', '.*_REI_\d{14}\.txt$']

        received = self._e.get_files(dirs, filters)
        expected = [os.path.join(dir_1, files[0]),
                    os.path.join(dir_2, files[1])]
        msg = 'Exporter report files list error'
        self.assertListEqual(sorted(received), sorted(expected), msg)

        # Clean up.
        for d in dirs:
            remove_files(get_directory_files_list(d))
            os.removedirs(d)

    def test_process_inbound_pod_without_delete(self):
        """Test the process cycle for inbound transfer without delete.
        """
        t_files = get_directory_files_list(os.path.join(self._test_dir,
                                                        'returns'))
        dir = self._ftp_dir
        for f in t_files:
            copy_file(f, os.path.join(dir, os.path.basename(f)))

        # Prepare the config.
        filter = '.*_VANA_RE[PFI]_\d{14}\.txt'
        self._ftp.config.add_section('ftp_t')
        self._ftp.config.set('ftp_t', 'host', '127.0.0.1')
        self._ftp.config.set('ftp_t', 'port', '2121')
        self._ftp.config.set('ftp_t', 'user', 'tester')
        self._ftp.config.set('ftp_t', 'password', 'tester')
        self._ftp.config.set('ftp_t', 'direction', 'inbound')
        self._ftp.config.set('ftp_t', 'source', '')
        self._ftp.config.set('ftp_t', 'filter', filter)
        self._ftp.config.set('ftp_t', 'target', self._dir)
        self._ftp.config.set('ftp_t', 'pod', 'Yes')
        self._ftp.config.set('ftp_t', 'partial', 'YES')
        self._ftp.config.set('ftp_t', 'delete', 'NO')
        self._ftp._parse_config(file_based=False)

        self._ftp.process(dry=False)

        # Check FTP inbound directory (all files should be transfered).
        received = get_directory_files_list(self._dir)
        expected = [os.path.join(self._dir,
                                 os.path.basename(x)) for x in t_files]
        msg = 'FTP inbound directory list not as expected'
        self.assertListEqual(sorted(received), sorted(expected), msg)

        # Check FTP source directory (all files should be intact).
        received = get_directory_files_list(self._ftp_dir)
        expected = [os.path.join(self._ftp_dir,
                                 os.path.basename(x)) for x in t_files]
        msg = 'FTP inbound directory list not as expected'
        self.assertListEqual(sorted(received), sorted(expected), msg)

        # Clean up.
        remove_files(get_directory_files_list(self._ftp_dir))
        remove_files(get_directory_files_list(self._dir))
        self._ftp.reset_config()