def test_get_index_for_crash_dynamic_name(self):
        """Test a dynamic (date-based) index name.
        """

        # The crashstorage class looks for '%' in the index name; if that
        # symbol is present, it will attempt to generate a new date-based
        # index name. Since the test base config doesn't use this pattern,
        # we need to specify it now.
        modified_config = self.get_tuned_config(
            ESCrashStorage,
            {'resource.elasticsearch.elasticsearch_index':
                'socorro_integration_test_reports%Y%m%d'}
        )
        es_storage = ESCrashStorage(config=modified_config)

        # The date is used to generate the name of the index; it must be a
        # datetime object.
        date = string_to_datetime(
            a_processed_crash['client_crash_date']
        )
        index = es_storage.get_index_for_crash(date)

        # The base index name is obtained from the test base class and the
        # date is appended to it according to pattern specified above.
        ok_(type(index) is str)
        eq_(index, 'socorro_integration_test_reports20120408')

    def test_index_data_capture(self):
        """Verify we capture index data in ES crashstorage"""
        with MetricsMock() as mm:
            es_storage = ESCrashStorage(config=self.config, namespace='processor.es')

            mock_connection = mock.Mock()
            # Do a successful indexing
            es_storage._index_crash(
                connection=mock_connection,
                es_index=None,
                es_doctype=None,
                crash_document=None,
                crash_id=None
            )
            # Do a failed indexing
            mock_connection.index.side_effect = Exception
            with pytest.raises(Exception):
                es_storage._index_crash(
                    connection=mock_connection,
                    es_index=None,
                    es_doctype=None,
                    crash_document=None,
                    crash_id=None
                )

            assert (
                len(mm.filter_records(stat='processor.es.index', tags=['outcome:successful'])) == 1
            )
            assert (
                len(mm.filter_records(stat='processor.es.index', tags=['outcome:failed'])) == 1
            )

    def test_index_crash_with_bad_keys(self):
        a_raw_crash_with_bad_keys = {
            'foo': 'alpha',
            '': 'bad key 1',
            '.': 'bad key 2',
            u'na\xefve': 'bad key 3',
        }

        es_storage = ESCrashStorage(config=self.config)

        es_storage.save_raw_and_processed(
            raw_crash=deepcopy(a_raw_crash_with_bad_keys),
            dumps=None,
            processed_crash=deepcopy(a_processed_crash),
            crash_id=a_processed_crash['uuid']
        )

        # Ensure that the document was indexed by attempting to retreive it.
        doc = self.es_client.get(
            index=self.config.elasticsearch.elasticsearch_index,
            id=a_processed_crash['uuid']
        )
        # Make sure the invalid keys aren't in the crash.
        raw_crash = doc['_source']['raw_crash']
        assert raw_crash == {'foo': 'alpha'}
        es_storage.close()

    def test_indexing_bogus_number_field(self, es_class_mock, es_client_mock):
        """Test an index attempt that fails because of a bogus number field.

        Expected behavior is to remove that field and retry indexing.

        """
        es_storage = ESCrashStorage(config=self.config)

        crash_id = a_processed_crash['uuid']
        raw_crash = {}
        processed_crash = {
            'date_processed': '2012-04-08 10:56:41.558922',
            'bogus-field': 1234567890,
            'foo': 'bar',
        }

        def mock_index(*args, **kwargs):
            if 'bogus-field' in kwargs['body']['processed_crash']:
                raise elasticsearch.exceptions.TransportError(
                    400,
                    'RemoteTransportException[[i-f94dae31][inet[/172.31.1.54:'
                    '9300]][indices:data/write/index]]; nested: '
                    'MapperParsingException[failed to parse '
                    '[processed_crash.bogus-field]]; nested: '
                    'NumberFormatException[For input string: '
                    '"18446744073709480735"]; '
                )

            return True

        es_class_mock().index.side_effect = mock_index

        # Submit a crash and ensure that it succeeds.
        es_storage.save_raw_and_processed(
            raw_crash=deepcopy(raw_crash),
            dumps=None,
            processed_crash=deepcopy(processed_crash),
            crash_id=crash_id
        )

        expected_doc = {
            'crash_id': crash_id,
            'removed_fields': 'processed_crash.bogus-field',
            'processed_crash': {
                'date_processed': string_to_datetime(
                    '2012-04-08 10:56:41.558922'
                ),
                'foo': 'bar',
            },
            'raw_crash': {},
        }
        es_class_mock().index.assert_called_with(
            index=self.config.elasticsearch.elasticsearch_index,
            doc_type=self.config.elasticsearch.elasticsearch_doctype,
            body=expected_doc,
            id=crash_id
        )

    def test_get_index_for_crash_static_name(self):
        """Test a static index name """
        es_storage = ESCrashStorage(config=self.config)

        # The actual date isn't important since the index name won't use it.
        index = es_storage.get_index_for_crash('some_date')

        # The index name is obtained from the test base class.
        assert type(index) is str
        assert index == 'socorro_integration_test_reports'

    def test_indexing_unhandled_errors(self, es_class_mock, es_client_mock):
        """Test an index attempt that fails because of unhandled errors.

        Expected behavior is to fail indexing and raise the error.

        """
        es_storage = ESCrashStorage(config=self.config)

        crash_id = a_processed_crash['uuid']
        raw_crash = {}
        processed_crash = {
            'date_processed': '2012-04-08 10:56:41.558922',
        }

        # Test with an error from which a field name cannot be extracted.
        def mock_index_unparsable_error(*args, **kwargs):
            raise elasticsearch.exceptions.TransportError(
                400,
                'RemoteTransportException[[i-f94dae31][inet[/172.31.1.54:'
                '9300]][indices:data/write/index]]; nested: '
                'MapperParsingException[BROKEN PART]; NumberFormatException'
            )

            return True

        es_class_mock().index.side_effect = mock_index_unparsable_error

        with pytest.raises(elasticsearch.exceptions.TransportError):
            es_storage.save_raw_and_processed(
                raw_crash=deepcopy(raw_crash),
                dumps=None,
                processed_crash=deepcopy(processed_crash),
                crash_id=crash_id
            )

        # Test with an error that we do not handle.
        def mock_index_unhandled_error(*args, **kwargs):
            raise elasticsearch.exceptions.TransportError(
                400,
                'Something went wrong'
            )

            return True

        es_class_mock().index.side_effect = mock_index_unhandled_error

        with pytest.raises(elasticsearch.exceptions.TransportError):
            es_storage.save_raw_and_processed(
                raw_crash=deepcopy(raw_crash),
                dumps=None,
                processed_crash=deepcopy(processed_crash),
                crash_id=crash_id
            )

    def test_index_crash(self):
        """Mock test the entire crash submission mechanism"""
        es_storage = ESCrashStorage(config=self.config)

        # This is the function that would actually connect to ES; by mocking
        # it entirely we are ensuring that ES doesn't actually get touched.
        es_storage._submit_crash_to_elasticsearch = mock.Mock()

        es_storage.save_raw_and_processed(
            raw_crash=deepcopy(a_raw_crash),
            dumps=None,
            processed_crash=deepcopy(a_processed_crash),
            crash_id=a_processed_crash['uuid']
        )

        # Ensure that the indexing function is only called once.
        assert es_storage._submit_crash_to_elasticsearch.call_count == 1

    def test_success(self, espy_mock):
        """Test a successful index of a crash report.
        """

        # It's mocks all the way down.
        sub_mock = mock.MagicMock()
        espy_mock.Elasticsearch.return_value = sub_mock

        es_storage = ESCrashStorage(config=self.config)

        crash_id = a_processed_crash['uuid']

        # Submit a crash like normal, except that the back-end ES object is
        # mocked (see the decorator above).
        es_storage.save_raw_and_processed(
            raw_crash=a_raw_crash,
            dumps=None,
            processed_crash=a_processed_crash,
            crash_id=crash_id,
        )

        # Ensure that the ES objects were instantiated by ConnectionContext.
        ok_(espy_mock.Elasticsearch.called)

        # Ensure that the IndicesClient was also instantiated (this happens in
        # IndexCreator but is part of the crashstorage workflow).
        ok_(espy_mock.client.IndicesClient.called)

        # The actual call to index the document (crash).
        document = {
            'crash_id': crash_id,
            'processed_crash': a_processed_crash,
            'raw_crash': a_raw_crash
        }

        additional = {
            'doc_type': 'crash_reports',
            'id': crash_id,
            'index': 'socorro_integration_test_reports'
        }

        sub_mock.index.assert_called_with(
            body=document,
            **additional
        )

    def test_crash_size_capture(self):
        """Verify we capture raw/processed crash sizes in ES crashstorage"""
        es_storage = ESCrashStorage(config=self.config)

        es_storage._submit_crash_to_elasticsearch = mock.Mock()

        es_storage.save_raw_and_processed(
            raw_crash=a_raw_crash,
            dumps=None,
            processed_crash=a_processed_crash,
            crash_id=a_processed_crash['uuid']
        )

        mock_calls = [str(call) for call in self.config.metrics.mock_calls]
        # NOTE(willkg): The sizes of these json documents depend on what's in them. If we changed
        # a_processed_crash and a_raw_crash, then these numbers will change.
        assert 'call.histogram(\'processor.es.raw_crash_size\', 27)' in mock_calls
        assert 'call.histogram(\'processor.es.processed_crash_size\', 1785)' in mock_calls

    def test_crash_size_capture(self):
        """Verify we capture raw/processed crash sizes in ES crashstorage"""
        with MetricsMock() as mm:
            es_storage = ESCrashStorage(config=self.config, namespace='processor.es')

            es_storage._submit_crash_to_elasticsearch = mock.Mock()

            es_storage.save_raw_and_processed(
                raw_crash=deepcopy(a_raw_crash),
                dumps=None,
                processed_crash=deepcopy(a_processed_crash),
                crash_id=a_processed_crash['uuid']
            )

            # NOTE(willkg): The sizes of these json documents depend on what's
            # in them. If we changed a_processed_crash and a_raw_crash, then
            # these numbers will change.
            assert mm.has_record('histogram', stat='processor.es.raw_crash_size', value=27)
            assert mm.has_record('histogram', stat='processor.es.processed_crash_size', value=1738)

    def test_index_crash(self):
        """Test indexing a crash document.
        """

        es_storage = ESCrashStorage(config=self.config)

        es_storage.save_raw_and_processed(
            raw_crash=a_raw_crash,
            dumps=None,
            processed_crash=a_processed_crash,
            crash_id=a_processed_crash['uuid']
        )

        # Ensure that the document was indexed by attempting to retreive it.
        ok_(
            self.es_client.get(
                index=self.config.elasticsearch.elasticsearch_index,
                id=a_processed_crash['uuid']
            )
        )

    def test_fatal_failure(self, espy_mock):
        """Test an index attempt that fails catastrophically"""
        sub_mock = mock.MagicMock()
        espy_mock.Elasticsearch.return_value = sub_mock

        es_storage = ESCrashStorage(config=self.config)

        crash_id = a_processed_crash['uuid']

        # Oh the humanity!
        failure_exception = Exception('horrors')
        sub_mock.index.side_effect = failure_exception

        # Submit a crash and ensure that it failed.
        with pytest.raises(Exception):
            es_storage.save_raw_and_processed(
                raw_crash=deepcopy(a_raw_crash),
                dumps=None,
                processed_crsah=deepcopy(a_processed_crash),
                crash_id=crash_id
            )

    def test_success_operational_exception(self, espy_mock):
        """Test an index attempt that experiences a operational exception that
        it managed to recover from.
        """

        # It's mocks all the way down.
        sub_mock = mock.MagicMock()
        espy_mock.Elasticsearch.return_value = sub_mock

        # ESCrashStorage uses the "limited backoff" transaction executor.
        # In real life this will retry operational exceptions over time, but
        # in unit tests, we just want it to hurry up and fail.
        backoff_config = self.config
        backoff_config['backoff_delays'] = [0, 0, 0]
        backoff_config['wait_log_interval'] = 0

        es_storage = ESCrashStorage(config=self.config)

        crash_id = a_processed_crash['uuid']

        # The transaction executor will try three times, so we will fail
        # twice for the purposes of this test.
        bad_results = [
            elasticsearch.exceptions.ConnectionError,
            elasticsearch.exceptions.ConnectionError
        ]

        # Replace the underlying index method with this function.
        def esindex_fn(*args, **kwargs):
            try:
                result = bad_results.pop(0)
                raise result
            except IndexError:
                return sub_mock.index

        sub_mock.index.side_effect = esindex_fn

        # Submit a crash like normal, except that the index method will
        # raise twice, then pass as normal.
        es_storage.save_raw_and_processed(
            raw_crash=a_raw_crash,
            dumps=None,
            processed_crash=a_processed_crash,
            crash_id=crash_id,
        )

        # The actual call to index the document (crash).
        document = {
            'crash_id': crash_id,
            'processed_crash': a_processed_crash,
            'raw_crash': a_raw_crash
        }

        additional = {
            'doc_type': 'crash_reports',
            'id': crash_id,
            'index': 'socorro_integration_test_reports'
        }

        sub_mock.index.assert_called_with(
            body=document,
            **additional
        )

    def test_indexing_bogus_string_field(self, es_class_mock, es_client_mock):
        """Test an index attempt that fails because of a bogus string field.

        Expected behavior is to remove that field and retry indexing.

        """

        es_storage = ESCrashStorage(config=self.config)

        crash_id = a_processed_crash['uuid']
        raw_crash = {}
        processed_crash = {
            'date_processed': '2012-04-08 10:56:41.558922',
            'bogus-field': 'some bogus value',
            'foo': 'bar',
        }

        def mock_index(*args, **kwargs):
            if 'bogus-field' in kwargs['body']['processed_crash']:
                raise elasticsearch.exceptions.TransportError(
                    400,
                    'RemoteTransportException[[i-5exxx97][inet[/172.3.9.12:'
                    '9300]][indices:data/write/index]]; nested: '
                    'IllegalArgumentException[Document contains at least one '
                    'immense term in field="processed_crash.bogus-field.full" '
                    '(whose UTF8 encoding is longer than the max length 32766)'
                    ', all of which were skipped.  Please correct the analyzer'
                    ' to not produce such terms.  The prefix of the first '
                    'immense term is: \'[124, 91, 48, 93, 91, 71, 70, 88, 49, '
                    '45, 93, 58, 32, 65, 116, 116, 101, 109, 112, 116, 32, '
                    '116, 111, 32, 99, 114, 101, 97, 116, 101]...\', original '
                    'message: bytes can be at most 32766 in length; got 98489]'
                    '; nested: MaxBytesLengthExceededException'
                    '[bytes can be at most 32766 in length; got 98489]; '
                )

            return True

        es_class_mock().index.side_effect = mock_index

        # Submit a crash and ensure that it succeeds.
        es_storage.save_raw_and_processed(
            raw_crash=deepcopy(raw_crash),
            dumps=None,
            processed_crash=deepcopy(processed_crash),
            crash_id=crash_id
        )

        expected_doc = {
            'crash_id': crash_id,
            'removed_fields': 'processed_crash.bogus-field',
            'processed_crash': {
                'date_processed': string_to_datetime(
                    '2012-04-08 10:56:41.558922'
                ),
                'foo': 'bar',
            },
            'raw_crash': {},
        }
        es_class_mock().index.assert_called_with(
            index=self.config.elasticsearch.elasticsearch_index,
            doc_type=self.config.elasticsearch.elasticsearch_doctype,
            body=expected_doc,
            id=crash_id
        )

    def test_indexing_bogus_number_field(self, es_class_mock, es_client_mock):
        """Test an index attempt that fails because of a bogus number field.
        Expected behavior is to remove that field and retry indexing.
        """
        # ESCrashStorage uses the "limited backoff" transaction executor.
        # In real life this will retry operational exceptions over time, but
        # in unit tests, we just want it to hurry up and fail.
        backoff_config = self.config
        backoff_config['backoff_delays'] = [0, 0, 0]
        backoff_config['wait_log_interval'] = 0

        es_storage = ESCrashStorage(config=self.config)

        crash_id = a_processed_crash['uuid']
        raw_crash = {}
        processed_crash = {
            'date_processed': '2012-04-08 10:56:41.558922',
            'bogus-field': 1234567890,
            'foo': 'bar',
        }

        def mock_index(*args, **kwargs):
            if 'bogus-field' in kwargs['body']['processed_crash']:
                raise elasticsearch.exceptions.TransportError(
                    400,
                    'RemoteTransportException[[i-f94dae31][inet[/172.31.1.54:'
                    '9300]][indices:data/write/index]]; nested: '
                    'MapperParsingException[failed to parse '
                    '[processed_crash.bogus-field]]; nested: '
                    'NumberFormatException[For input string: '
                    '"18446744073709480735"]; '
                )

            return True

        es_class_mock().index.side_effect = mock_index

        # Submit a crash and ensure that it succeeds.
        es_storage.save_raw_and_processed(
            raw_crash,
            None,
            processed_crash,
            crash_id
        )

        expected_doc = {
            'crash_id': crash_id,
            'removed_fields': 'processed_crash.bogus-field',
            'processed_crash': {
                'date_processed': string_to_datetime(
                    '2012-04-08 10:56:41.558922'
                ),
                'foo': 'bar',
            },
            'raw_crash': {},
        }
        es_class_mock().index.assert_called_with(
            index=self.config.elasticsearch.elasticsearch_index,
            doc_type=self.config.elasticsearch.elasticsearch_doctype,
            body=expected_doc,
            id=crash_id
        )