def test_timeout_many(self):
        upper_daemon = UpperDaemon(TimeoutTCPHandler)
        upper_daemon.start()
        upper_daemon.wait_for_start()

        port = upper_daemon.get_port()

        self.uri_opener.settings.set_timeout(1)

        url = URL('http://127.0.0.1:%s/' % port)
        
        for _ in xrange(MAX_ERROR_COUNT):
            try:
                self.uri_opener.GET(url)
            except ScanMustStopByUnknownReasonExc:
                self.assertTrue(False, 'Not expecting this exception type.')
            except ScanMustStopOnUrlError:
                self.assertTrue(True)
            except ScanMustStopException:
                self.assertTrue(True)
                break
        else:
            self.assertTrue(False)
        
        self.uri_opener.settings.set_default_values()

    def test_exception_is_raised_always_after_stop(self):
        return_empty_daemon = UpperDaemon(EmptyTCPHandler)
        return_empty_daemon.start()
        return_empty_daemon.wait_for_start()

        port = return_empty_daemon.get_port()

        # No retries means that the test is easier to read/understand
        self.uri_opener.settings.set_max_http_retries(0)

        # Don't rate limit
        self.uri_opener._rate_limit = lambda: True

        url = URL('http://127.0.0.1:%s/' % port)
        http_exception_count = 0
        loops = 100

        # Loop until we reach a must stop exception
        for i in xrange(loops):
            try:
                self.uri_opener.GET(url, cache=False)
            except HTTPRequestException:
                http_exception_count += 1
            except ScanMustStopByKnownReasonExc, smse:
                break
            except Exception, e:
                msg = 'Not expecting: "%s"'
                self.assertTrue(False, msg % e.__class__.__name__)

    def test_http_timeout_with_plugin(self):
        """
        This is very related with the tests at:
            w3af/core/data/url/tests/test_xurllib.py

        Very similar test is TestXUrllib.test_timeout

        :see: https://github.com/andresriancho/w3af/issues/7112
        """
        upper_daemon = UpperDaemon(TimeoutTCPHandler)
        upper_daemon.start()
        upper_daemon.wait_for_start()

        port = upper_daemon.get_port()

        url = URL('http://127.0.0.1:%s/' % port)
        freq = FuzzableRequest(url)

        plugin_inst = self.w3af.plugins.get_plugin_inst('audit', 'sqli')
        plugin_inst._uri_opener.settings.set_configured_timeout(1)
        plugin_inst._uri_opener.clear_timeout()

        # We expect the server to timeout and the response to be a 204
        resp = plugin_inst.get_original_response(freq)
        self.assertEqual(resp.get_url(), url)
        self.assertEqual(resp.get_code(), 204)

        plugin_inst._uri_opener.settings.set_default_values()

    def test_timeout_many(self):
        upper_daemon = UpperDaemon(TimeoutTCPHandler)
        upper_daemon.start()
        upper_daemon.wait_for_start()

        port = upper_daemon.get_port()

        self.uri_opener.settings.set_configured_timeout(0.5)
        self.uri_opener.clear_timeout()
        # We can mock this because it's being tested at TestXUrllibDelayOnError
        self.uri_opener._pause_on_http_error = Mock()

        url = URL('http://127.0.0.1:%s/' % port)
        http_request_e = 0
        scan_stop_e = 0

        for _ in xrange(MAX_ERROR_COUNT):
            try:
                self.uri_opener.GET(url)
            except HTTPRequestException, hre:
                http_request_e += 1
                self.assertEqual(hre.message, 'HTTP timeout error')
            except ScanMustStopException:
                scan_stop_e += 1
                self.assertTrue(True)
                break

    def test_timeout_many(self):
        upper_daemon = UpperDaemon(TimeoutTCPHandler)
        upper_daemon.start()
        upper_daemon.wait_for_start()

        port = upper_daemon.get_port()

        self.uri_opener.settings.set_timeout(1)

        url = URL('http://127.0.0.1:%s/' % port)
        http_request_e = 0
        scan_stop_e = 0

        for _ in xrange(MAX_ERROR_COUNT):
            try:
                self.uri_opener.GET(url)
            except HTTPRequestException:
                http_request_e += 1
                self.assertTrue(True)
            except ScanMustStopException:
                scan_stop_e += 1
                self.assertTrue(True)
                break
            except Exception, e:
                msg = 'Not expecting: "%s"'
                self.assertTrue(False, msg % e.__class__.__name__)

class TestUpperDaemon(unittest.TestCase):
    """
    This is a unittest for the UpperDaemon which lives in upper_daemon.py
    
    @author: Andres Riancho <andres . riancho | gmail . com>
    """
    def setUp(self):
        self.upper_daemon = UpperDaemon()
        self.upper_daemon.start()
        self.upper_daemon.wait_for_start()
    
    def test_basic(self):
        sent = 'abc'
        
        sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
        
        sock.connect(('127.0.0.1', self.upper_daemon.get_port()))
        sock.sendall(sent)
        
        received = sock.recv(3)
        
        self.assertEqual(received, sent.upper())

    def test_url_port_not_http(self):
        upper_daemon = UpperDaemon(EmptyTCPHandler)
        upper_daemon.start()
        upper_daemon.wait_for_start()

        port = upper_daemon.get_port()

        url = URL('http://127.0.0.1:%s/' % port)
        self.assertRaises(ScanMustStopOnUrlError, self.uri_opener.GET, url)

    def test_ssl_fail_when_requesting_http(self):
        http_daemon = UpperDaemon(Ok200Handler)
        http_daemon.start()
        http_daemon.wait_for_start()

        port = http_daemon.get_port()

        # Note that here I'm using httpS <<---- "S" and that I've started an
        # HTTP server. We should get an exception
        url = URL('https://127.0.0.1:%s/' % port)

        self.assertRaises(HTTPRequestException, self.uri_opener.GET, url)

    def test_url_port_not_http(self):
        upper_daemon = UpperDaemon(EmptyTCPHandler)
        upper_daemon.start()
        upper_daemon.wait_for_start()

        port = upper_daemon.get_port()

        url = URL('http://127.0.0.1:%s/' % port)

        try:
            self.uri_opener.GET(url)
        except HTTPRequestException, hre:
            self.assertEqual(hre.value, "Bad HTTP response status line: ''")

    def test_timeout(self):
        upper_daemon = UpperDaemon(TimeoutTCPHandler)
        upper_daemon.start()
        upper_daemon.wait_for_start()

        port = upper_daemon.get_port()
        
        url = URL('http://127.0.0.1:%s/' % port)
        
        self.uri_opener.settings.set_timeout(1)
        
        self.assertRaises(ScanMustStopOnUrlError, self.uri_opener.GET, url)
        
        self.uri_opener.settings.set_default_values()

    def test_timeout_auto_adjust(self):
        upper_daemon = UpperDaemon(Ok200SmallDelayHandler)
        upper_daemon.start()
        upper_daemon.wait_for_start()

        port = upper_daemon.get_port()

        # Enable timeout auto-adjust
        self.uri_opener.settings.set_configured_timeout(0)
        self.uri_opener.clear_timeout()

        # We can mock this because it's being tested at TestXUrllibDelayOnError
        self.uri_opener._pause_on_http_error = Mock()

        # Mock to verify the calls
        self.uri_opener.set_timeout = Mock()

        # Make sure we start from the desired timeout value
        self.assertEqual(self.uri_opener.get_timeout('127.0.0.1'),
                         DEFAULT_TIMEOUT)

        url = URL('http://127.0.0.1:%s/' % port)
        sent_requests = 0

        for _ in xrange(TIMEOUT_ADJUST_LIMIT * 2):
            try:
                self.uri_opener.GET(url)
            except Exception:
                raise
            else:
                sent_requests += 1
                if self.uri_opener.set_timeout.call_count:
                    break

        self.assertEqual(self.uri_opener.set_timeout.call_count, 1)

        rtt = self.uri_opener.get_average_rtt()[0]
        adjusted_tout = self.uri_opener.set_timeout.call_args[0][0]
        expected_tout = TIMEOUT_MULT_CONST * rtt
        delta = rtt * 0.2

        self.assertGreaterEqual(adjusted_tout, expected_tout - delta)
        self.assertLessEqual(adjusted_tout, expected_tout + delta)
        self.assertLess(adjusted_tout, DEFAULT_TIMEOUT)
        self.assertEqual(sent_requests, TIMEOUT_ADJUST_LIMIT + 1)

    def test_timeout(self):
        upper_daemon = UpperDaemon(TimeoutTCPHandler)
        upper_daemon.start()
        upper_daemon.wait_for_start()

        port = upper_daemon.get_port()

        url = URL('http://127.0.0.1:%s/' % port)

        self.uri_opener.settings.set_configured_timeout(0.5)
        self.uri_opener.clear_timeout()
        # We can mock this because it's being tested at TestXUrllibDelayOnError
        self.uri_opener._pause_on_http_error = Mock()
        start = time.time()

        try:
            self.uri_opener.GET(url)
        except HTTPRequestException, hre:
            self.assertEqual(hre.message, 'HTTP timeout error')

    def test_ignore_errors(self):
        upper_daemon = UpperDaemon(TimeoutTCPHandler)
        upper_daemon.start()
        upper_daemon.wait_for_start()

        port = upper_daemon.get_port()

        self.uri_opener.settings.set_timeout(1)
        self.uri_opener._retry = Mock()

        url = URL('http://127.0.0.1:%s/' % port)

        try:
            self.uri_opener.GET(url, ignore_errors=True)
        except ScanMustStopOnUrlError:
            self.assertEqual(self.uri_opener._retry.call_count, 0)
        else:
            self.assertTrue(False, 'Exception not raised')

        self.uri_opener.settings.set_default_values()

    def test_error_handling_disable_per_request(self):
        upper_daemon = UpperDaemon(TimeoutTCPHandler)
        upper_daemon.start()
        upper_daemon.wait_for_start()

        port = upper_daemon.get_port()

        self.uri_opener.settings.set_configured_timeout(1)
        self.uri_opener.clear_timeout()
        self.uri_opener._retry = Mock()

        url = URL('http://127.0.0.1:%s/' % port)

        try:
            self.uri_opener.GET(url, error_handling=False)
        except HTTPRequestException:
            self.assertEqual(self.uri_opener._retry.call_count, 0)
        else:
            self.assertTrue(False, 'Exception not raised')

        self.uri_opener.settings.set_default_values()

    def test_url_port_not_http_many(self):
        upper_daemon = UpperDaemon(EmptyTCPHandler)
        upper_daemon.start()
        upper_daemon.wait_for_start()

        port = upper_daemon.get_port()

        self.uri_opener.settings.set_max_http_retries(0)

        url = URL('http://127.0.0.1:%s/' % port)
        http_request_e = 0
        scan_must_stop_e = 0

        for _ in xrange(MAX_ERROR_COUNT):
            try:
                self.uri_opener.GET(url)
            except HTTPRequestException:
                http_request_e += 1
            except ScanMustStopException, smse:
                scan_must_stop_e += 1
                break
            except Exception, e:
                msg = 'Not expecting "%s".'
                self.assertTrue(False, msg % e.__class__.__name__)

 def setUp(self):
     self.upper_daemon = UpperDaemon()
     self.upper_daemon.start()
     self.upper_daemon.wait_for_start()

    def test_increasing_delay_on_errors(self):
        expected_log = {0: False, 70: False, 40: False, 10: False, 80: False,
                        50: False, 20: False, 90: False, 60: False, 30: False,
                        100: False}
        self.assertEqual(self.uri_opener._sleep_log, expected_log)

        return_empty_daemon = UpperDaemon(EmptyTCPHandler)
        return_empty_daemon.start()
        return_empty_daemon.wait_for_start()

        port = return_empty_daemon.get_port()

        # No retries means that the test is easier to read/understand
        self.uri_opener.settings.set_max_http_retries(0)

        # We want to keep going, don't test the _should_stop_scan here.
        self.uri_opener._should_stop_scan = lambda x: False

        url = URL('http://127.0.0.1:%s/' % port)
        http_exception_count = 0
        loops = 100

        # Not check the delays
        with patch('w3af.core.data.url.extended_urllib.time.sleep') as sleepm:
            for i in xrange(loops):
                try:
                    self.uri_opener.GET(url, cache=False)
                except HTTPRequestException:
                    http_exception_count += 1
                except Exception, e:
                    msg = 'Not expecting: "%s"'
                    self.assertTrue(False, msg % e.__class__.__name__)
                else:
                    self.assertTrue(False, 'Expecting HTTPRequestException')

            self.assertEqual(loops - 1, i)

            # Note that the timeouts are increasing based on the error rate and
            # SOCKET_ERROR_DELAY
            expected_calls = [call(1.5),
                              call(3.0),
                              call(4.5),
                              call(6.0),
                              call(7.5),
                              call(9.0),
                              call(10.5),
                              call(12.0),
                              call(13.5)]

            expected_log = {0: False, 70: True, 40: True, 10: True, 80: True,
                            50: True, 20: True, 90: True, 60: True, 30: True,
                            100: False}
            self.assertEqual(expected_calls, sleepm.call_args_list)
            self.assertEqual(http_exception_count, 100)
            self.assertEqual(self.uri_opener._sleep_log, expected_log)

            # This one should also clear the log
            try:
                self.uri_opener.GET(url, cache=False)
            except HTTPRequestException:
                pass
            else:
                self.assertTrue(False, 'Expected HTTPRequestException')

            # The log was cleared, all values should be False
            self.assertTrue(all([not v for v in self.uri_opener._sleep_log.values()]))