def test_real_file(self):
        """
        The file used in this test, is a real file from the IDD. It contains 14 records:
        7 power records, 6 CO2 records (normal) and 1 CO2 record (blank).
        (No control files are in the real file.)
        Verify that the correct number of particles are generated
        from a real file.
        """
        log.debug('===== START TEST REAL FILE =====')

        with open(os.path.join(RESOURCE_PATH, '20140507.pco2w1.log'), 'r') as file_handle:

            num_particles_to_request = 2500
            num_expected_particles = 14

            parser = Pco2wAbcDclParser(self._recovered_parser_config,
                                       file_handle,
                                       self.exception_callback,
                                       None,
                                       None)

            particles = parser.get_records(num_particles_to_request)

            log.info(len(particles))

            self.assertEquals(len(particles), num_expected_particles)

            self.assertEquals(self.exception_callback_value, [])

        log.debug('===== END TEST REAL FILE =====')

    def launch(self):
        """
        Launches the simulator process as indicated by _COMMAND.

        @return (rsn_oms, uri) A pair with the CIOMSSimulator instance and the
                associated URI to establish connection with it.
        """
        log.debug("[OMSim] Launching: %s", _COMMAND)

        self._process = self._spawn(_COMMAND)

        if not self._process or not self.poll():
            msg = "[OMSim] Failed to launch simulator: %s" % _COMMAND
            log.error(msg)
            raise Exception(msg)

        log.debug("[OMSim] process started, pid: %s", self.getpid())

        # give it some time to start up
        sleep(5)

        # get URI:
        uri = None
        with open("logs/rsn_oms_simulator.yml", buffering=1) as f:
            # we expect one of the first few lines to be of the form:
            # rsn_oms_simulator_uri=xxxx
            # where xxxx is the uri -- see oms_simulator_server.
            while uri is None:
                line = f.readline()
                if line.index("rsn_oms_simulator_uri=") == 0:
                    uri = line[len("rsn_oms_simulator_uri="):].strip()

        self._rsn_oms = CIOMSClientFactory.create_instance(uri)
        return self._rsn_oms, uri

    def test_no_particles(self):
        """
        The file used in this test only contains DCL Logging records.
        Verify that no particles are produced if the input file
        has no instrument data records, i.e., they just contain DCL Logging records.
        """
        log.debug('===== START TEST NO PARTICLES =====')

        with open(os.path.join(RESOURCE_PATH, 'no_particles.log'), 'r') as file_handle:

            num_particles_to_request = 2
            num_expected_particles = 0

            parser = Pco2wAbcDclParser(self._recovered_parser_config,
                                       file_handle,
                                       self.exception_callback,
                                       None,
                                       None)

            particles = parser.get_records(num_particles_to_request)

            self.assertEquals(len(particles), num_expected_particles)

            self.assertEquals(self.exception_callback_value, [])

        log.debug('===== END TEST NO PARTICLES =====')

    def test_invalid_checksum(self):
        """
        The last five records in the file used in this test have a length that does not match the Length
        field in the record. This tests for this requirement:
        If the beginning of another instrument data record (* character), is encountered before "Length"
        bytes have been found, where "Length" is the record length specified in a record, then we can not
        reliably parse the record.
        This results in 5 particles being retrieved instead of 6, and also result in the exception
        callback being called.
        """
        log.debug('===== START TEST INVALID CHECKSUM =====')

        with open(os.path.join(RESOURCE_PATH, 'invalid_checksum.log'), 'r') as file_handle:

            num_particles_to_request = 1
            num_expected_particles = 1

            parser = Pco2wAbcDclParser(self._recovered_parser_config,
                                       file_handle,
                                       self.exception_callback,
                                       None,
                                       None)

            particles = parser.get_records(num_particles_to_request)

            self.assertEquals(len(particles), num_expected_particles)

            self.assert_particles(particles, "invalid_checksum.yml", RESOURCE_PATH)

            # No exception should be thrown
            self.assertEquals(self.exception_callback_value, [])

        log.debug('===== END TEST INVALID CHECKSUM =====')

 def test_schema_generation(self):
     self.maxDiff = None
     result = self.param_dict.generate_dict()
     json_result = json.dumps(result, indent=4, sort_keys=True)
     log.debug("Expected: %s", self.target_schema)
     log.debug("Result: %s", json_result)
     self.assertEqual(result, self.target_schema)

    def test_missing_file_time_recov(self):
        """
        Read a file that is missing the file time metadata
        A RecoverableException should be reported.
        """
        log.debug('===== START TEST MISSING FILE TIME RECOV =====')

        num_particles_to_request = 10
        num_expected_particles = 6

        with open(os.path.join(RESOURCE_PATH, 'pco2wXYZ_11212014_1628.DAT'), 'r') as file_handle:

            parser = Pco2wAbcImodemParser(self._recov_parser_config,
                                          file_handle,
                                          self.exception_callback)

            particles = parser.get_records(num_particles_to_request)

            self.assertEquals(len(particles), num_expected_particles)

            self.assert_particles(particles, "pco2wXYZ_11212014_1628.recov.yml", RESOURCE_PATH)

            self.assertEquals(len(self.exception_callback_value), 1)

            for exception in self.exception_callback_value:
                self.assertIsInstance(exception, RecoverableSampleException)

        log.debug('===== END TEST MISSING FILE TIME RECOV =====')

    def test_invalid_data_recov(self):
        """
        Read files and verify that all expected particles can be read.
        Verify that invalid data is handled appropriately with the
        correct exceptions being reported.
        """
        log.debug('===== START TEST INVALID DATA RECOVERED =====')

        num_particles_to_request = 10
        num_expected_particles = 7

        with open(os.path.join(RESOURCE_PATH, 'pco2wXYZ_11212014_1625.DAT'), 'r') as file_handle:

            parser = Pco2wAbcImodemParser(self._recov_parser_config,
                                          file_handle,
                                          self.exception_callback)

            particles = parser.get_records(num_particles_to_request)

            self.assertEquals(len(particles), num_expected_particles)

            self.assert_particles(particles, "pco2wXYZ_11212014_1625.recov.yml", RESOURCE_PATH)

            self.assertEquals(len(self.exception_callback_value), 2)

            for exception in self.exception_callback_value:
                self.assertIsInstance(exception, RecoverableSampleException)

        log.debug('===== END TEST INVALID DATA RECOVERED =====')

    def _build_parsed_values(self):
        """
        Take the velocity header data sample format and parse it into
        values with appropriate tags.
        @throws SampleException If there is a problem with sample creation
        """
        log.debug('VectorVelocityHeaderDataParticle: raw data =%r', self.raw_data)

        try:
            unpack_string = '<4s6sH8B20sH'
            sync, timestamp, number_of_records, noise1, noise2, noise3, _, correlation1, correlation2, correlation3, _,\
                _, cksum = struct.unpack(unpack_string, self.raw_data)

            if not validate_checksum('<20H', self.raw_data):
                log.warn("Failed checksum in %s from instrument (%r)", self._data_particle_type, self.raw_data)
                self.contents[DataParticleKey.QUALITY_FLAG] = DataParticleValue.CHECKSUM_FAILED

            timestamp = common.convert_time(timestamp)
            self.set_internal_timestamp((timestamp-datetime(1900, 1, 1)).total_seconds())

        except Exception as e:
            log.error('Error creating particle vel3d_cd_data_header, raw data: %r', self.raw_data)
            raise SampleException(e)

        result = [{VID: VectorVelocityHeaderDataParticleKey.TIMESTAMP, VAL: str(timestamp)},
                  {VID: VectorVelocityHeaderDataParticleKey.NUMBER_OF_RECORDS, VAL: number_of_records},
                  {VID: VectorVelocityHeaderDataParticleKey.NOISE1, VAL: noise1},
                  {VID: VectorVelocityHeaderDataParticleKey.NOISE2, VAL: noise2},
                  {VID: VectorVelocityHeaderDataParticleKey.NOISE3, VAL: noise3},
                  {VID: VectorVelocityHeaderDataParticleKey.CORRELATION1, VAL: correlation1},
                  {VID: VectorVelocityHeaderDataParticleKey.CORRELATION2, VAL: correlation2},
                  {VID: VectorVelocityHeaderDataParticleKey.CORRELATION3, VAL: correlation3}]

        log.debug('VectorVelocityHeaderDataParticle: particle=%s', result)
        return result

    def test_no_particles(self):
        """
        Verify that no particles are produced if the input file
        has no instrument records.
        """
        log.debug('===== START TEST NO PARTICLES =====')

        with open(os.path.join(RESOURCE_PATH, 'no_particles.txt'), 'r') as file_handle:

            NUM_PARTICLES_TO_REQUEST = 10
            NUM_EXPECTED_PARTICLES = 0

            parser = Pco2wAbcParser(self._parser_config,
                                    file_handle,
                                    self.exception_callback,
                                    None,
                                    None)

            particles = parser.get_records(NUM_PARTICLES_TO_REQUEST)

            self.assertEquals(len(particles), NUM_EXPECTED_PARTICLES)

            self.assertEqual(self._exception_occurred, False)

        log.debug('===== END TEST NO PARTICLES =====')

    def test_power_record_missing_timestamp(self):
        """
        The file used in this test has a power record with a missing timestamp.
        This results in 9 particles being retrieved instead of 10, and also result in the exception
        callback being called.
        """
        log.debug('===== START TEST POWER RECORD MISSING TIMESTAMP =====')

        with open(os.path.join(RESOURCE_PATH, 'power_record_missing_timestamp.txt'), 'r') as file_handle:

            NUM_PARTICLES_TO_REQUEST = 10
            NUM_EXPECTED_PARTICLES = 9

            parser = Pco2wAbcParser(self._parser_config,
                                    file_handle,
                                    self.exception_callback,
                                    None,
                                    None)

            particles = parser.get_records(NUM_PARTICLES_TO_REQUEST)

            self.assertEquals(len(particles), NUM_EXPECTED_PARTICLES)

            self.assert_particles(particles, "power_record_missing_timestamp.yml", RESOURCE_PATH)

            self.assertEqual(self._exception_occurred, True)

        log.debug('===== END TEST POWER RECORD MISSING TIMESTAMP =====')

    def test_invalid_checksum(self):
        """
        The first record in the file used in this test has in invalid checksum. An instrument particle will
        still get created, but the passed_checksum parameter will be 0 (no warning or error msg generated).
        """
        log.debug('===== START TEST INVALID CHECKSUM =====')

        with open(os.path.join(RESOURCE_PATH, 'invalid_checksum.DAT'), O_MODE) as file_handle:

            num_particles_to_request = 5
            num_expected_particles = 5

            parser = PhsenAbcdefImodemParser(self._recovered_parser_config,
                                             file_handle,
                                             self.exception_callback)

            particles = parser.get_records(num_particles_to_request)

            self.assertEquals(len(particles), num_expected_particles)

            self.assert_particles(particles, "invalid_checksum_rec.yml", RESOURCE_PATH)

            # No exception should be thrown
            self.assertEquals(self.exception_callback_value, [])

        log.debug('===== END TEST INVALID CHECKSUM =====')

    def test_jumbled(self):
        """
        Read files and verify that all expected particles can be read.
        This particular data file has the velocity data records
        preceded by the diagnostics records, a situation not likely
        to occur on a deployed instrument but anything is possible!
        The logic in the parser will not produce an instrument metadata
        particle (configuration data) until it encounters a velocity or
        a diagnostics record. Assumes that all the configuration records are
        at the beginning of the file. This is reasonable as the instrument is
        configured before being deployed. So the config records would be stored
        first. Verify that the contents of the particles are correct.
        There should be no exceptions generated.
        """
        log.debug('===== START TEST SIMPLE NOT IN ORDER =====')

        # Test the telemetered version
        with open(os.path.join(RESOURCE_PATH, 'jumbled_VELPT_SN_11402_2014-07-02.aqd'), 'rb') as file_handle:

            num_particles_to_request = num_expected_particles = 72

            parser = VelptAbParser(self._parser_config,
                                   file_handle,
                                   self.exception_callback)

            particles = parser.get_records(num_particles_to_request)

            self.assertEquals(len(particles), num_expected_particles)

            self.assert_particles(particles, 'jumbled_VELPT_SN_11402_2014-07-02.yml', RESOURCE_PATH)

        log.debug('===== END TEST SIMPLE NOT IN ORDER =====')

    def test_invalid_record_type(self):
        """
        The file used in this test has a record type that does not match any of the expected record types.
        This results in 9 particles being retrieved instead of 10, and also result in the exception
        callback being called.
        """
        log.debug('===== START TEST INVALID RECORD TYPE =====')

        with open(os.path.join(RESOURCE_PATH, 'invalid_record_type.txt'), 'r') as file_handle:

            NUM_PARTICLES_TO_REQUEST = 10
            NUM_EXPECTED_PARTICLES = 9

            parser = Pco2wAbcParser(self._parser_config,
                                    file_handle,
                                    self.exception_callback,
                                    None,
                                    None)

            particles = parser.get_records(NUM_PARTICLES_TO_REQUEST)

            self.assertEquals(len(particles), NUM_EXPECTED_PARTICLES)

            self.assert_particles(particles, "invalid_record_type.yml", RESOURCE_PATH)

            self.assertEqual(self._exception_occurred, True)

        log.debug('===== END TEST INVALID RECORD TYPE =====')

    def test_happy_path(self):
        """
        Read files and verify that all expected particles can be read.
        Verify that the contents of the particles are correct.
        There should be no exceptions generated.
        """
        log.debug('===== START TEST HAPPY PATH =====')

        with open(os.path.join(RESOURCE_PATH, 'happy_path.txt'), 'r') as file_handle:

            NUM_PARTICLES_TO_REQUEST = NUM_EXPECTED_PARTICLES = 10

            parser = Pco2wAbcParser(self._parser_config,
                                    file_handle,
                                    self.exception_callback,
                                    None,
                                    None)

            particles = parser.get_records(NUM_PARTICLES_TO_REQUEST)

            self.assertEquals(len(particles), NUM_EXPECTED_PARTICLES)

            self.assert_particles(particles, "happy_path.yml", RESOURCE_PATH)

            self.assertEqual(self._exception_occurred, False)

        log.debug('===== END TEST HAPPY PATH =====')

    def test_bad_diag_checksum_21_recs(self):
        """
        The file used in this test has a power record with a missing timestamp.
        This results in 9 particles being retrieved instead of 10, and also result in the exception
        callback being called.
        """
        log.debug('===== START TEST FOUND BAD DIAG HDR CHECKSUM AND TOO MANY RECS =====')

        with open(os.path.join(RESOURCE_PATH, 'bad_diag_hdr_checksum_21_diag_VELPT_SN_11402_2014-07-02.aqd'), 'rb')\
                as file_handle:

            num_particles_to_request = num_expected_particles = 118

            parser = VelptAbParser(self._parser_config,
                                   file_handle,
                                   self.exception_callback)

            particles = parser.get_records(num_particles_to_request)

            self.assertEquals(len(particles), num_expected_particles)

            self.assert_particles(particles, 'bad_diag_hdr_checksum_21_diag_VELPT_SN_11402_2014-07-02.yml',
                                  RESOURCE_PATH)

        log.debug('===== END TEST FOUND BAD DIAG HDR CHECKSUM AND TOO MANY RECS =====')

    def test_blank_line(self):
        """
        Read file and verify that all expected particles can be read.
        Verify that the contents of the particles are correct. There are
        blank lines interspersed in the file. This test verifies that
        these blank lines do not adversely affect the parser. Only one
        test is run as the content of the input files is the same for
        recovered or telemetered.
        """
        log.debug('===== START TEST BLANK LINE =====')

        num_particles_to_request = 25
        num_expected_particles = 20

        with open(os.path.join(RESOURCE_PATH, '20141207sbl.pwrsys.log'), 'rU') as file_handle:

            parser = FuelCellEngDclParser(self._recovered_parser_config,
                                          file_handle,
                                          self.exception_callback)

            particles = parser.get_records(num_particles_to_request)

            self.assertEquals(len(particles), num_expected_particles)

        log.debug('===== END TEST BLANK LINE =====')

    def test_invalid_header_fields(self):
        """
        The header in the file used in this test has in invalid Voltage and Number of Samples Written.
        A metadata particle will still get created, but there will be None in some of the parameters
        (an exception will be generated).
        """
        log.debug('===== START TEST INVALID HEADER FIELDS =====')

        with open(os.path.join(RESOURCE_PATH, 'invalid_header_fields.DAT'), O_MODE) as file_handle:

            num_particles_to_request = 5
            num_expected_particles = 5

            parser = PhsenAbcdefImodemParser(self._recovered_parser_config,
                                             file_handle,
                                             self.exception_callback)

            particles = parser.get_records(num_particles_to_request)

            self.assertEquals(len(particles), num_expected_particles)

            self.assert_particles(particles, "invalid_header_fields_rec.yml", RESOURCE_PATH)

            self.assert_(isinstance(self.exception_callback_value[0], UnexpectedDataException))

        log.debug('===== END TEST INVALID HEADER FIELDS =====')

    def test_no_science_particles(self):
        """
        The file used in this test only contains header and footer records.
        Verify that no science (pH or Control) particles are produced if the input file
        has no pH data records or control data, i.e., they just contain header and footer records.
        In this case only the metadata particle will get created.
        """
        log.debug('===== START TEST NO SCIENCE PARTICLES =====')

        with open(os.path.join(RESOURCE_PATH, 'header_and_footer_only.DAT'), O_MODE) as file_handle:

            num_particles_to_request = 2
            num_expected_particles = 1

            parser = PhsenAbcdefImodemParser(self._recovered_parser_config,
                                             file_handle,
                                             self.exception_callback)

            particles = parser.get_records(num_particles_to_request)

            self.assertEquals(len(particles), num_expected_particles)

            self.assert_particles(particles, "header_and_footer_only_rec.yml", RESOURCE_PATH)

            self.assertEquals(self.exception_callback_value, [])

        log.debug('===== END TEST NO SCIENCE PARTICLES =====')

    def test_invalid_metadata_timestamp(self):
        """
        The file used in this test has error in the timestamp for the first metadata record.
        This results in 9 particles being retrieved instead of 10, and also result in the exception
        callback being called.
        """
        log.debug('===== START TEST INVALID METADATA TIMESTAMP =====')

        with open(os.path.join(RESOURCE_PATH, 'invalid_metadata_timestamp.txt'), 'r') as file_handle:

            NUM_PARTICLES_TO_REQUEST = 10
            NUM_EXPECTED_PARTICLES = 9

            parser = Pco2wAbcParser(self._parser_config,
                                    file_handle,
                                    self.exception_callback,
                                    None,
                                    None)

            particles = parser.get_records(NUM_PARTICLES_TO_REQUEST)

            self.assertEquals(len(particles), NUM_EXPECTED_PARTICLES)

            self.assert_particles(particles, "invalid_metadata_timestamp.yml", RESOURCE_PATH)

            self.assertEqual(self._exception_occurred, True)

        log.debug('===== END TEST INVALID METADATA TIMESTAMP =====')

    def stop_launched_simulator(cls):
        """
        Utility to stop the process launched with launch_simulator.
        The stop is attempted a couple of times in case of errors (with a few
        seconds of sleep in between).

        @return None if process seems to have been stopped properly.
                Otherwise the exception of the last attempt to stop it.
        """
        if cls._sim_process:
            sim_proc, cls._sim_process = cls._sim_process, None
            attempts = 3
            attempt = 0
            while attempt <= attempts:
                attempt += 1
                log.debug("[OMSim] stopping launched simulator (attempt=%d) ...", attempt)
                try:
                    sim_proc.stop()
                    log.debug("[OMSim] simulator process seems to have stopped properly")
                    return None

                except Exception as ex:
                    if attempt < attempts:
                        sleep(10)
                    else:
                        log.warn("[OMSim] error while stopping simulator process: %s", ex)
                        return ex