def test_interrupt_if_at_failure_limits(self):
        port = Mock()  # FIXME: This should be a tighter mock.
        port.TEST_PATH_SEPARATOR = '/'
        port._filesystem = MockFileSystem()
        manager = Manager(port=port, options=MockOptions(), printer=Mock())

        manager._options = MockOptions(exit_after_n_failures=None, exit_after_n_crashes_or_timeouts=None)
        result_summary = ResultSummary(expectations=Mock(), test_files=[])
        result_summary.unexpected_failures = 100
        result_summary.unexpected_crashes = 50
        result_summary.unexpected_timeouts = 50
        # No exception when the exit_after* options are None.
        manager._interrupt_if_at_failure_limits(result_summary)

        # No exception when we haven't hit the limit yet.
        manager._options.exit_after_n_failures = 101
        manager._options.exit_after_n_crashes_or_timeouts = 101
        manager._interrupt_if_at_failure_limits(result_summary)

        # Interrupt if we've exceeded either limit:
        manager._options.exit_after_n_crashes_or_timeouts = 10
        self.assertRaises(TestRunInterruptedException, manager._interrupt_if_at_failure_limits, result_summary)

        manager._options.exit_after_n_crashes_or_timeouts = None
        manager._options.exit_after_n_failures = 10
        exception = self.assertRaises(TestRunInterruptedException, manager._interrupt_if_at_failure_limits, result_summary)

 def get_manager_with_tests(test_names):
     port = Mock()  # FIXME: Use a tighter mock.
     port.TEST_PATH_SEPARATOR = '/'
     manager = Manager(port, options=MockOptions(http=True), printer=Mock())
     manager._test_files = set(test_names)
     manager._test_files_list = test_names
     return manager

def run(port, options, args, regular_output=sys.stderr, buildbot_output=sys.stdout):
    try:
        warnings = _set_up_derived_options(port, options)

        printer = printing.Printer(port, options, regular_output, buildbot_output, logger=logging.getLogger())

        for warning in warnings:
            _log.warning(warning)

        if options.lint_test_files:
            return lint(port, options)

        # We wrap any parts of the run that are slow or likely to raise exceptions
        # in a try/finally to ensure that we clean up the logging configuration.
        unexpected_result_count = -1

        manager = Manager(port, options, printer)
        printer.print_config()

        unexpected_result_count = manager.run(args)
        _log.debug("Testing completed, Exit status: %d" % unexpected_result_count)
    except Exception:
        exception_type, exception_value, exception_traceback = sys.exc_info()
        if exception_type not in (KeyboardInterrupt, TestRunInterruptedException, WorkerException):
            print >> sys.stderr, '\n%s raised: %s' % (exception_type.__name__, exception_value)
            stack_utils.log_traceback(_log.error, exception_traceback)
        raise
    finally:
        printer.cleanup()

    return unexpected_result_count

 def test_fallback_path_in_config(self):
     options = self.get_options()
     host = MockHost()
     port = host.port_factory.get('test-mac-leopard', options=options)
     printer = self.get_printer()
     manager = Manager(port, port.options, printer)
     manager.print_config()
     self.assertTrue('Baseline search path: test-mac-leopard -> test-mac-snowleopard -> generic' in printer.output)

def run(port, options, args, logging_stream):
    try:
        printer = printing.Printer(port, options, logging_stream, logger=logging.getLogger())

        _set_up_derived_options(port, options)
        manager = Manager(port, options, printer)
        printer.print_config(port.results_directory())

        run_details = manager.run(args)
        _log.debug("Testing completed, Exit status: %d" % run_details.exit_code)
        return run_details
    finally:
        printer.cleanup()

 def test_update_summary_with_result(self):
     host = MockHost()
     port = host.port_factory.get('test-win-xp')
     test = 'failures/expected/reftest.html'
     port.test_expectations = lambda: 'WONTFIX : failures/expected/reftest.html = IMAGE'
     expectations = TestExpectations(port, tests=[test])
     # Reftests expected to be image mismatch should be respected when pixel_tests=False.
     manager = Manager(port=port, options=MockOptions(pixel_tests=False, exit_after_n_failures=None, exit_after_n_crashes_or_timeouts=None), printer=Mock())
     manager._expectations = expectations
     result_summary = ResultSummary(expectations=expectations, test_files=[test])
     result = TestResult(test_name=test, failures=[test_failures.FailureReftestMismatchDidNotOccur()])
     manager._update_summary_with_result(result_summary, result)
     self.assertEquals(1, result_summary.expected)
     self.assertEquals(0, result_summary.unexpected)

def _print_expectations(port, options, args, logging_stream):
    logger = logging.getLogger()
    logger.setLevel(logging.DEBUG if options.debug_rwt_logging else logging.INFO)
    try:
        printer = printing.Printer(port, options, logging_stream, logger=logger)

        _set_up_derived_options(port, options)
        manager = Manager(port, options, printer)

        exit_code = manager.print_expectations(args)
        _log.debug("Printing expectations completed, Exit status: %d", exit_code)
        return exit_code
    except Exception as error:
        _log.error('Error printing expectations: {}'.format(error))
    finally:
        printer.cleanup()
        return -1

def run(port, options, args, regular_output=sys.stderr, buildbot_output=sys.stdout):
    warnings = _set_up_derived_options(port, options)

    printer = printing.Printer(port, options, regular_output, buildbot_output, logger=logging.getLogger())

    for warning in warnings:
        _log.warning(warning)

    if options.help_printing:
        printer.help_printing()
        printer.cleanup()
        return 0

    if options.lint_test_files:
        return lint(port, options)

    # We wrap any parts of the run that are slow or likely to raise exceptions
    # in a try/finally to ensure that we clean up the logging configuration.
    unexpected_result_count = -1
    try:
        manager = Manager(port, options, printer)
        manager.print_config()

        printer.print_update("Collecting tests ...")
        try:
            manager.collect_tests(args)
        except IOError, e:
            if e.errno == errno.ENOENT:
                return -1
            raise

        printer.print_update("Checking build ...")
        if not port.check_build(manager.needs_servers()):
            _log.error("Build check failed")
            return -1

        printer.print_update("Parsing expectations ...")
        manager.parse_expectations()

        unexpected_result_count = manager.run()
        _log.debug("Testing completed, Exit status: %d" % unexpected_result_count)

    def test_servers_started(self):

        def start_http_server(number_of_servers=None):
            self.http_started = True

        def start_websocket_server():
            self.websocket_started = True

        def stop_http_server():
            self.http_stopped = True

        def stop_websocket_server():
            self.websocket_stopped = True

        host = MockHost()
        port = host.port_factory.get('test-mac-leopard')
        port.start_http_server = start_http_server
        port.start_websocket_server = start_websocket_server
        port.stop_http_server = stop_http_server
        port.stop_websocket_server = stop_websocket_server

        self.http_started = self.http_stopped = self.websocket_started = self.websocket_stopped = False
        manager = Manager(port=port, options=MockOptions(http=True), printer=Mock())
        manager._test_files = ['http/tests/pass.txt']
        manager.start_servers_with_lock(number_of_servers=4)
        self.assertEquals(self.http_started, True)
        self.assertEquals(self.websocket_started, False)
        manager.stop_servers_with_lock()
        self.assertEquals(self.http_stopped, True)
        self.assertEquals(self.websocket_stopped, False)

        self.http_started = self.http_stopped = self.websocket_started = self.websocket_stopped = False
        manager = Manager(port=port, options=MockOptions(http=True), printer=Mock())
        manager._test_files = ['websocket/pass.txt']
        manager.start_servers_with_lock(number_of_servers=4)
        self.assertEquals(self.http_started, True)
        self.assertEquals(self.websocket_started, True)
        manager.stop_servers_with_lock()
        self.assertEquals(self.http_stopped, True)
        self.assertEquals(self.websocket_stopped, True)

        self.http_started = self.http_stopped = self.websocket_started = self.websocket_stopped = False
        manager = Manager(port=port, options=MockOptions(http=True), printer=Mock())
        manager._test_files = ['perf/foo/test.html']
        manager.start_servers_with_lock(number_of_servers=4)
        self.assertEquals(self.http_started, False)
        self.assertEquals(self.websocket_started, False)
        manager.stop_servers_with_lock()
        self.assertEquals(self.http_stopped, False)
        self.assertEquals(self.websocket_stopped, False)

def _run_tests(port, options, args, printer):
    _set_up_derived_options(port, options, args)
    manager = Manager(port, options, printer)
    printer.print_config(port.results_directory())
    return manager.run(args)

 def get_manager_with_tests(test_names):
     host = MockHost()
     port = host.port_factory.get()
     manager = Manager(port, options=MockOptions(test_list=None, http=True), printer=Mock())
     manager.collect_tests(test_names)
     return manager

def run(port, options, args, regular_output=sys.stderr,
        buildbot_output=sys.stdout):
    """Run the tests.

    Args:
      port: Port object for port-specific behavior
      options: a dictionary of command line options
      args: a list of sub directories or files to test
      regular_output: a stream-like object that we can send logging/debug
          output to
      buildbot_output: a stream-like object that we can write all output that
          is intended to be parsed by the buildbot to
    Returns:
      the number of unexpected results that occurred, or -1 if there is an
          error.

    """
    warnings = _set_up_derived_options(port, options)

    printer = printing.Printer(port, options, regular_output, buildbot_output,
                               configure_logging=True)
    for w in warnings:
        _log.warning(w)

    if options.help_printing:
        printer.help_printing()
        printer.cleanup()
        return 0

    # We wrap any parts of the run that are slow or likely to raise exceptions
    # in a try/finally to ensure that we clean up the logging configuration.
    num_unexpected_results = -1
    try:
        manager = Manager(port, options, printer)
        manager.print_config()

        printer.print_update("Collecting tests ...")
        try:
            manager.collect_tests(args)
        except IOError, e:
            if e.errno == errno.ENOENT:
                return -1
            raise

        if options.lint_test_files:
            return manager.lint()

        printer.print_update("Checking build ...")
        if not port.check_build(manager.needs_servers()):
            _log.error("Build check failed")
            return -1

        printer.print_update("Parsing expectations ...")
        manager.parse_expectations()

        result_summary = manager.set_up_run()
        if result_summary:
            num_unexpected_results = manager.run(result_summary)
            manager.clean_up_run()
            _log.debug("Testing completed, Exit status: %d" %
                       num_unexpected_results)

 def get_manager_with_tests(test_names):
     port = layout_tests.port.get()
     manager = Manager(port, options=MockOptions(test_list=None, http=True), printer=Mock())
     manager.collect_tests(test_names)
     return manager