def test_doing_work_with_one_worker(self):
        config = DotDict()
        config.logger = self.logger
        config.number_of_threads = 1
        config.maximum_queue_size = 1
        my_list = []

        def insert_into_list(anItem):
            my_list.append(anItem)

        ttm = ThreadedTaskManager(config,
                                  task_func=insert_into_list
                                 )
        try:
            ttm.start()
            time.sleep(0.2)
            ok_(len(my_list) == 10,
                            'expected to do 10 inserts, '
                               'but %d were done instead' % len(my_list))
            ok_(my_list == range(10),
                            'expected %s, but got %s' % (range(10), my_list))
            ttm.stop()
        except Exception:
            # we got threads to join
            ttm.wait_for_completion()
            raise

    def test_blocking_start(self):
        config = DotDict()
        config.logger = self.logger
        config.idle_delay = 1
        config.quit_on_empty_queue =  False

        class MyTaskManager(TaskManager):
            def _responsive_sleep(
                self,
                seconds,
                wait_log_interval=0,
                wait_reason=''
            ):
                try:
                    if self.count >= 2:
                        self.quit = True
                    self.count += 1
                except AttributeError:
                    self.count = 0

        tm = MyTaskManager(
            config,
            task_func=Mock()
        )

        waiting_func = Mock()

        tm.blocking_start(waiting_func=waiting_func)

        eq_(
            tm.task_func.call_count,
            10
        )
        eq_(waiting_func.call_count, 0)

    def test_action_case_1(self):
        """sentinel exsits in stack, but no secondaries"""
        pc = DotDict()
        pc.process_type = 'plugin'
        pijd = copy.deepcopy(cannonical_json_dump)
        pc.json_dump = pijd
        pc.json_dump['crashing_thread']['frames'][2]['function'] = \
            'NtUserSetWindowPos'
        f2jd = copy.deepcopy(cannonical_json_dump)
        pc.upload_file_minidump_flash2 = DotDict()
        pc.upload_file_minidump_flash2.json_dump = f2jd

        fake_processor = create_basic_fake_processor()

        rc = DotDict()
        rd = {}
        rule = SetWindowPos()
        action_result = rule.action(rc, rd, pc, fake_processor)

        ok_(action_result)
        ok_('classifications' in pc)
        ok_('skunk_works' in pc.classifications)
        eq_(
            pc.classifications.skunk_works.classification,
            'NtUserSetWindowPos | other'
        )

    def test_doing_work_with_two_workers_and_generator(self):
        config = DotDict()
        config.logger = self.logger
        config.number_of_threads = 2
        config.maximum_queue_size = 2
        my_list = []

        def insert_into_list(anItem):
            my_list.append(anItem)

        ttm = ThreadedTaskManager(config,
                                  task_func=insert_into_list,
                                  job_source_iterator=(((x,), {}) for x in
                                                       xrange(10))
                                 )
        try:
            ttm.start()
            time.sleep(0.2)
            ok_(len(ttm.thread_list) == 2,
                            "expected 2 threads, but found %d"
                              % len(ttm.thread_list))
            ok_(len(my_list) == 10,
                            'expected to do 10 inserts, '
                              'but %d were done instead' % len(my_list))
            ok_(sorted(my_list) == range(10),
                            'expected %s, but got %s' % (range(10),
                                                         sorted(my_list)))
        except Exception:
            # we got threads to join
            ttm.wait_for_completion()
            raise

    def test_action_case_4(self):
        """nothing in 1st dump, sentinel but no secondary in
        upload_file_minidump_flash2 dump"""
        pc = DotDict()
        pc.dump = DotDict()
        pijd = copy.deepcopy(cannonical_json_dump)
        pc.dump.json_dump = pijd
        f2jd = copy.deepcopy(cannonical_json_dump)
        pc.upload_file_minidump_flash2 = DotDict()
        pc.upload_file_minidump_flash2.json_dump = f2jd
        pc.upload_file_minidump_flash2.json_dump['crashing_thread']['frames'][2] \
            ['function'] = 'NtUserSetWindowPos'

        fake_processor = create_basic_fake_processor()

        rc = DotDict()
        rd = {}
        rule = SetWindowPos()
        action_result = rule.action(rc, rd, pc, fake_processor)

        ok_(action_result)
        ok_('classifications' in pc)
        ok_('skunk_works' in pc.classifications)
        eq_(
            pc.classifications.skunk_works.classification,
            'NtUserSetWindowPos | other'
        )

    def setup_mocked_s3_storage(
        self,
        executor=TransactionExecutor,
        executor_for_gets=TransactionExecutor,
        storage_class='BotoS3CrashStorage',
        host='',
        port=0,
        resource_class=S3ConnectionContext,
        **extra
    ):
        config = DotDict({
            'resource_class': resource_class,
            'logger': mock.Mock(),
            'host': host,
            'port': port,
            'access_key': 'this is the access key',
            'secret_access_key': 'secrets',
            'bucket_name': 'silliness',
            'keybuilder_class': KeyBuilderBase,
            'prefix': 'dev',
            'calling_format': mock.Mock()
        })
        config.update(extra)
        s3_conn = resource_class(config)
        s3_conn._connect_to_endpoint = mock.Mock()
        s3_conn._mocked_connection = s3_conn._connect_to_endpoint.return_value
        s3_conn._calling_format.return_value = mock.Mock()
        s3_conn._CreateError = mock.Mock()
        s3_conn.ResponseError = mock.Mock()
        s3_conn._open = mock.MagicMock()

        return s3_conn

    def test_wrong_signature(self, mocked_subprocess_module):
        config = self.get_basic_config()
        raw_crash = copy.copy(canonical_standard_raw_crash)
        raw_dumps = {config.dump_field: 'a_fake_dump.dump'}
        processed_crash = DotDict()
        processed_crash.product = 'Firefox'
        processed_crash.os_name = 'Windows NT'
        processed_crash.cpu_name = 'x86'
        processed_crash.signature = 'this-is-not-a-JIT-signature'
        processed_crash['json_dump.crashing_thread.frames'] = [
            DotDict({'not_module': 'not-a-module',}),
            DotDict({'module': 'a-module',})
        ]
        processor_meta = self.get_basic_processor_meta()

        mocked_subprocess_handle = (
            mocked_subprocess_module.Popen.return_value
        )
        mocked_subprocess_handle.stdout.read.return_value = (
            'EXTRA-SPECIAL'
        )
        mocked_subprocess_handle.wait.return_value = 0

        rule = JitCrashCategorizeRule(config)

        # the call to be tested
        rule.act(raw_crash, raw_dumps, processed_crash, processor_meta)

        ok_('classifications.jit.category' not in processed_crash)
        ok_('classifications.jit.category_return_code' not in processed_crash)

    def test_stuff_missing(self):
        config = self.get_basic_config()

        raw_crash = copy.copy(canonical_standard_raw_crash)

        raw_dumps = {}
        system_info = copy.copy(
            canonical_processed_crash['json_dump']['system_info']
        )
        del system_info['cpu_count']
        processed_crash = DotDict()
        processed_crash.json_dump = {
            'system_info': system_info
        }

        processor_meta = self.get_basic_processor_meta()

        rule = CPUInfoRule(config)

        # the call to be tested
        rule.act(raw_crash, raw_dumps, processed_crash, processor_meta)

        eq_(
            processed_crash.cpu_info,
            "GenuineIntel family 6 model 42 stepping 7"
        )
        eq_(processed_crash.cpu_name, 'x86')

        # raw crash should be unchanged
        eq_(raw_crash, canonical_standard_raw_crash)

def create_basic_fake_processor():
    fake_processor = DotDict()
    fake_processor.c_signature_tool = c_signature_tool
    fake_processor.config = DotDict()
    # need help figuring out failures? switch to FakeLogger and read stdout
    fake_processor.config.logger = SilentFakeLogger()
    #fake_processor.config.logger = FakeLogger()
    return fake_processor

    def test_constuctor1(self):
        config = DotDict()
        config.logger = self.logger
        config.quit_on_empty_queue =  False

        tm = TaskManager(config)
        ok_(tm.config == config)
        ok_(tm.logger == self.logger)
        ok_(tm.task_func == default_task_func)
        ok_(tm.quit == False)

    def test_executor_identity(self):
        config = DotDict()
        config.logger = self.logger
        tm = TaskManager(
            config,
            job_source_iterator=range(1),

        )
        tm._pid = 666
        eq_(tm.executor_identity(), '666-MainThread')

    def test_transaction_ack_crash(self):
        config = self._setup_config()
        connection = Mock()
        ack_token = DotDict()
        ack_token.delivery_tag = 1
        crash_id = 'some-crash-id'

        crash_store = RabbitMQCrashStorage(config)
        crash_store._transaction_ack_crash(connection, crash_id, ack_token)

        connection.channel.basic_ack.assert_called_once_with(delivery_tag=1)

    def _fake_processed_crash(self):
        d = DotDict()
        # these keys survive redaction
        d.a = DotDict()
        d.a.b = DotDict()
        d.a.b.c = 11
        d.sensitive = DotDict()
        d.sensitive.x = 2
        d.not_url = 'not a url'

        return d

    def test_predicate(self):
        rc = DotDict()
        rd = {}
        pc = DotDict()
        pc.classifications = DotDict()
        processor = None

        support_rule = SupportClassificationRule()
        ok_(support_rule.predicate(rc, rd, pc, processor))

        pc.classifications.support = DotDict()
        ok_(support_rule.predicate(rc, rd, pc, processor))

 def _setup_config(self):
     config = DotDict()
     config.transaction_executor_class = Mock()
     config.backoff_delays = (0, 0, 0)
     config.logger = Mock()
     config.rabbitmq_class = MagicMock()
     config.routing_key = 'socorro.normal'
     config.filter_on_legacy_processing = True
     config.redactor_class = Redactor
     config.forbidden_keys = Redactor.required_config.forbidden_keys.default
     config.throttle = 100
     return config

    def setup_mocked_s3_storage(
        self,
        executor=TransactionExecutor,
        executor_for_gets=TransactionExecutor,
        keybuilder_class=KeyBuilderBase,
        storage_class='BotoS3CrashStorage',
        bucket_name='mozilla-support-reason',
        host='',
        port=0,
    ):
        config = DotDict({
            'source': {
                'dump_field': 'dump'
            },
            'transaction_executor_class': executor,
            'transaction_executor_class_for_get': executor_for_gets,
            'resource_class': S3ConnectionContext,
            'keybuilder_class': keybuilder_class,
            'backoff_delays': [0, 0, 0],
            'redactor_class': Redactor,
            'forbidden_keys': Redactor.required_config.forbidden_keys.default,
            'logger': mock.Mock(),
            'host': host,
            'port': port,
            'access_key': 'this is the access key',
            'secret_access_key': 'secrets',
            'temporary_file_system_storage_path': self.TEMPDIR,
            'dump_file_suffix': '.dump',
            'bucket_name': bucket_name,
            'prefix': 'dev',
            'calling_format': mock.Mock(),
            'json_object_hook': DotDict,
        })

        if isinstance(storage_class, basestring):
            if storage_class == 'BotoS3CrashStorage':
                config.bucket_name = 'crash_storage'
                s3 = BotoS3CrashStorage(config)
            elif storage_class == 'SupportReasonAPIStorage':
                s3 = SupportReasonAPIStorage(config)
        else:
            s3 = storage_class(config)
        s3_conn = s3.connection_source
        s3_conn._connect_to_endpoint = mock.Mock()
        s3_conn._mocked_connection = s3_conn._connect_to_endpoint.return_value
        s3_conn._calling_format.return_value = mock.Mock()
        s3_conn._CreateError = mock.Mock()
        s3_conn._open = mock.MagicMock()

        return s3

    def test_statistics_all_missing_prefix(self):
        d = DotDict()
        d.statsd_host = 'localhost'
        d.statsd_port = 666
        d.prefix = None
        d.active_counters_list = ['x', 'y', 'z']

        with patch('socorrolib.lib.statistics.StatsClient') as StatsClientMocked:
            s = StatisticsForStatsd(d, 'processor')
            StatsClientMocked.assert_called_with(
                'localhost', 666, 'processor')

            s.incr('x')
            StatsClientMocked.assert_has_calls(
                StatsClientMocked.mock_calls,
                [call.incr('processor.x')]
            )

            s.incr('y')
            StatsClientMocked.assert_has_calls(
                StatsClientMocked.mock_calls,
                [call.incr('processor.y')]
            )

            s.incr('z')
            StatsClientMocked.assert_has_calls(
                StatsClientMocked.mock_calls,
                [call.incr('processor.z')]
            )

            s.incr('w')
            StatsClientMocked.assert_has_calls(
                StatsClientMocked.mock_calls,
                [
                    call.incr('processor.y'),
                    call.incr('processor.x'),
                    call.incr('processor.y')
                ]
            )

            s.incr(None)
            StatsClientMocked.assert_has_calls(
                StatsClientMocked.mock_calls,
                [
                    call.incr('processor.y'),
                    call.incr('processor.x'),
                    call.incr('processor.y'),
                    call.incr('processor.unknown')
                ]
            )

    def test_action_fail(self):
        jd = copy.deepcopy(cannonical_json_dump)
        pc = DotDict()
        pc.json_dump = jd

        fake_processor = create_basic_fake_processor()

        rc = DotDict()
        rd = {}
        rule = BitguardClassifier()
        action_result = rule.action(rc, rd, pc, fake_processor)

        ok_(not action_result)
        ok_('classifications' not in pc)

 def test_constuctor1(self):
     config = DotDict()
     config.logger = self.logger
     config.number_of_threads = 1
     config.maximum_queue_size = 1
     ttm = ThreadedTaskManager(config)
     try:
         ok_(ttm.config == config)
         ok_(ttm.logger == self.logger)
         ok_(ttm.task_func == default_task_func)
         ok_(ttm.quit == False)
     finally:
         # we got threads to join
         ttm._kill_worker_threads()

    def test_predicate(self):
        rc = DotDict()
        rd = {}
        pc = DotDict()
        pc.classifications = DotDict()
        processor = None

        skunk_rule = SkunkClassificationRule()
        ok_(skunk_rule.predicate(rc, rd, pc, processor))

        pc.classifications.skunk_works = DotDict()
        ok_(skunk_rule.predicate(rc, rd, pc, processor))

        pc.classifications.skunk_works.classification = 'stupid'
        ok_(not skunk_rule.predicate(rc, rd, pc, processor))