def test_if_broken_mesos_does_not_break_marathon_cache(
            self, nginx_class, mocker, superuser_user_header):
        filter_regexp = {
            'Mesos state request failed: invalid response status: 500':
                SearchCriteria(1, True),
            'Marathon apps cache has been successfully updated': SearchCriteria(1, True),
        }

        # Break marathon
        mocker.send_command(endpoint_id='http://127.0.0.2:5050',
                            func_name='always_bork',
                            aux_data=True)

        mocker.send_command(endpoint_id='http://127.0.0.1:8080',
                            func_name='enable_nginx_app')

        ar = nginx_class()
        url = ar.make_url_from_path('/service/nginx-enabled/bar/baz')

        with GuardedSubprocess(ar):
            lbf = LineBufferFilter(filter_regexp,
                                   timeout=(CACHE_FIRST_POLL_DELAY + 1),
                                   line_buffer=ar.stderr_line_buffer)

            resp = requests.get(url,
                                allow_redirects=False,
                                headers=superuser_user_header)
            lbf.scan_log_buffer()

        assert resp.status_code == 200
        req_data = resp.json()
        assert req_data['endpoint_id'] == 'http://127.0.0.1:16001'

        assert lbf.extra_matches == {}

    def test_if_broken_marathon_does_not_break_mesos_cache(
            self, nginx_class, mocker, superuser_user_header):
        filter_regexp = {
            'Marathon app request failed: invalid response status: 500':
                SearchCriteria(1, True),
            'Mesos state cache has been successfully updated':
                SearchCriteria(1, True),
        }

        # Break marathon
        mocker.send_command(endpoint_id='http://127.0.0.1:8080',
                            func_name='always_bork',
                            aux_data=True)

        ar = nginx_class()

        with GuardedSubprocess(ar):
            lbf = LineBufferFilter(filter_regexp,
                                   timeout=(CACHE_FIRST_POLL_DELAY + 1),
                                   line_buffer=ar.stderr_line_buffer)

            ping_mesos_agent(ar, superuser_user_header)
            lbf.scan_log_buffer()

        assert lbf.extra_matches == {}

    def test_if_default_scheme_is_honoured_by_agent_endpoint(
            self, nginx_class, mocker, valid_user_header):
        filter_regexp = {'Default scheme: https://': SearchCriteria(1, False)}

        ar = nginx_class(default_scheme="https://")
        agent_id = AGENT3_ID
        url_good = ar.make_url_from_path('/agent/{}/blah/blah'.format(agent_id))
        agent_id = AGENT1_ID
        url_bad = ar.make_url_from_path('/agent/{}/blah/blah'.format(agent_id))

        with GuardedSubprocess(ar):
            lbf = LineBufferFilter(filter_regexp,
                                   line_buffer=ar.stderr_line_buffer)

            resp = requests.get(url_bad,
                                allow_redirects=False,
                                headers=valid_user_header)

            assert resp.status_code == 502

            resp = requests.get(url_good,
                                allow_redirects=False,
                                headers=valid_user_header)

            assert resp.status_code == 200
            req_data = resp.json()
            assert req_data['endpoint_id'] == 'https://127.0.0.1:15401'

            lbf.scan_log_buffer()

        assert lbf.extra_matches == {}

    def test_if_missing_mesos_leader_entry_is_handled(
            self, nginx_class, valid_user_header, dns_server_mock):
        filter_regexp = {
            'Failed to instantiate the resolver': SearchCriteria(0, True),
            'DNS server returned error code': SearchCriteria(1, True),
            '`Mesos Leader` state cache has been successfully updated':
                SearchCriteria(0, True),
        }

        ar = nginx_class()

        with GuardedSubprocess(ar):
            lbf = LineBufferFilter(filter_regexp,
                                   line_buffer=ar.stderr_line_buffer)
            # Unfortunatelly there are upstreams that use `leader.mesos` and
            # removing this entry too early will result in Nginx failing to start.
            # So we need to do it right after nginx starts, but before first
            # cache update.
            time.sleep(1)
            dns_server_mock.remove_dns_entry('leader.mesos.')

            # Now let's trigger the cache update:
            ping_mesos_agent(ar, valid_user_header)

            lbf.scan_log_buffer()

            assert lbf.extra_matches == {}

    def test_upstream_wrong_json(
            self, nginx_class, mocker, superuser_user_header):
        filter_regexp = {
            "Cannot decode Marathon apps JSON: ": SearchCriteria(1, True),
        }

        ar = nginx_class()

        # Set wrong non-json response content
        mocker.send_command(endpoint_id='http://127.0.0.1:8080',
                            func_name='set_encoded_response',
                            aux_data=b"wrong response")

        url = ar.make_url_from_path('/service/nginx-alwaysthere/foo/bar/')
        with GuardedSubprocess(ar):
            # Register Line buffer filter:
            lbf = LineBufferFilter(filter_regexp,
                                   timeout=5,  # Just to give LBF enough time
                                   line_buffer=ar.stderr_line_buffer)

            # Trigger cache update by issuing request:
            resp = requests.get(url,
                                allow_redirects=False,
                                headers=superuser_user_header)
            assert "cache state is invalid" in resp.content.decode('utf-8')
            assert resp.status_code == 503

            lbf.scan_log_buffer()

        assert lbf.extra_matches == {}

    def test_if_var_pointing_to_empty_file_is_handled(self,
                                                      ar_process_without_secret_key):
        # Scanning for the exact log entry is bad, but in this case - can't be
        # avoided.
        filter_regexp = 'Secret key not set or empty string.'

        lbf = LineBufferFilter(filter_regexp,
                               line_buffer=ar_process_without_secret_key.stderr_line_buffer)

        lbf.scan_log_buffer()

        assert lbf.all_found is True

    def test_if_not_defining_the_var_is_handled(self,
                                                ar_process_without_secret_key):
        # Scanning for the exact log entry is bad, but in this case - can't be
        # avoided.
        filter_regexp = 'SECRET_KEY_FILE_PATH not set.'

        lbf = LineBufferFilter(filter_regexp,
                               line_buffer=ar_process_without_secret_key.stderr_line_buffer)

        lbf.scan_log_buffer()

        assert lbf.all_found is True

    def test_if_broken_marathon_causes_marathon_cache_to_expire_and_requests_to_fail(
            self, nginx_class, mocker, superuser_user_header):
        filter_regexp = {
            'Marathon app request failed: invalid response status: 500':
                SearchCriteria(1, False),
            'Mesos state cache has been successfully updated':
                SearchCriteria(2, False),
            'Cache entry `svcapps` is too old, aborting request':
                SearchCriteria(1, True),
        }

        ar = nginx_class(cache_max_age_soft_limit=3,
                         cache_max_age_hard_limit=4,
                         cache_expiration=2,
                         cache_poll_period=3,
                         )

        mocker.send_command(endpoint_id='http://127.0.0.1:8080',
                            func_name='enable_nginx_app')
        url = ar.make_url_from_path('/service/nginx-enabled/foo/bar/')

        with GuardedSubprocess(ar):
            # Register Line buffer filter:
            lbf = LineBufferFilter(filter_regexp,
                                   timeout=5,  # Just to give LBF enough time
                                   line_buffer=ar.stderr_line_buffer)

            # Trigger cache update by issuing request:
            resp = requests.get(url,
                                allow_redirects=False,
                                headers=superuser_user_header)
            assert resp.status_code == 200

            # Break marathon
            mocker.send_command(endpoint_id='http://127.0.0.1:8080',
                                func_name='always_bork',
                                aux_data=True)

            # Wait for the cache to be old enough to be discarded by AR:
            # cache_max_age_hard_limit + 1s for good measure
            # must be more than cache_poll_period
            time.sleep(4 + 1)

            # Perform the main/test request:
            resp = requests.get(url,
                                allow_redirects=False,
                                headers=superuser_user_header)
            assert resp.status_code == 503

            lbf.scan_log_buffer()

        assert lbf.extra_matches == {}

    def test_if_absent_var_is_handled(self, nginx_class, mocker):
        filter_regexp = {
            'Local Mesos Master IP: unknown': SearchCriteria(1, True),
        }
        ar = nginx_class(host_ip=None)

        with GuardedSubprocess(ar):
            lbf = LineBufferFilter(filter_regexp,
                                   line_buffer=ar.stderr_line_buffer)

            lbf.scan_log_buffer()

        assert lbf.extra_matches == {}

    def test_if_var_is_honoured(self, valid_ip, nginx_class, mocker):
        filter_regexp = {
            'Local Mesos Master IP: {}'.format(valid_ip): SearchCriteria(1, True),
        }
        ar = nginx_class(host_ip=valid_ip)

        with GuardedSubprocess(ar):
            lbf = LineBufferFilter(filter_regexp,
                                   line_buffer=ar.stderr_line_buffer)

            lbf.scan_log_buffer()

        assert lbf.extra_matches == {}

    def _assert_filter_regexp_for_invalid_app(
            self,
            filter_regexp,
            app,
            nginx_class,
            mocker,
            auth_headers,
            ):
        """Helper method that will assert if provided regexp filter is found
        in nginx logs for given apps response from Marathon upstream endpoint.

        Arguments:
            filter_regexp (dict): Filter definition where key is the message
                looked up in logs and value is SearchCriteria definition
            app (dict): App that upstream endpoint should respond with
            nginx_class (Nginx): Nginx process fixture
            mocker (Mocker): Mocker fixture
            auth_header (dict): Headers that should be passed to Nginx in the
                request
        """
        ar = nginx_class()

        mocker.send_command(endpoint_id='http://127.0.0.1:8080',
                            func_name='set_apps_response',
                            aux_data={"apps": [app]})

        # Remove all entries for mesos frameworks and mesos_dns so that
        # we test only the information in Marathon
        mocker.send_command(endpoint_id='http://127.0.0.2:5050',
                            func_name='set_frameworks_response',
                            aux_data=[])
        mocker.send_command(endpoint_id='http://127.0.0.1:8123',
                            func_name='set_srv_response',
                            aux_data=[])

        url = ar.make_url_from_path('/service/scheduler-alwaysthere/foo/bar/')
        with GuardedSubprocess(ar):
            # Register Line buffer filter:
            lbf = LineBufferFilter(filter_regexp,
                                   timeout=5,  # Just to give LBF enough time
                                   line_buffer=ar.stderr_line_buffer)

            # Trigger cache update by issuing request:
            resp = requests.get(url,
                                allow_redirects=False,
                                headers=auth_headers)
            assert resp.status_code == 404

            lbf.scan_log_buffer()

        assert lbf.extra_matches == {}

    def test_if_var_pointing_to_empty_file_is_handled(
            self, nginx_class, role, empty_file):
        # Scanning for the exact log entry is bad, but in this case - can't be
        # avoided.
        filter_regexp = {'Auth token verification key not set': SearchCriteria(1, False)}
        ar = nginx_class(role=role, auth_token_verification_key_file_path=empty_file)

        with GuardedSubprocess(ar):
            lbf = LineBufferFilter(filter_regexp,
                                   line_buffer=ar.stderr_line_buffer)

            lbf.scan_log_buffer()

        assert lbf.extra_matches == {}

    def test_if_temp_marathon_borkage_does_not_disrupt_caching(
            self, nginx_class, mocker, valid_user_header):
        filter_regexp = {
            'Marathon app request failed: invalid response status: 500':
                SearchCriteria(1, False),
            'Mesos state cache has been successfully updated':
                SearchCriteria(2, False),
            'Cache entry `svcapps` is stale':
                SearchCriteria(1, True),
        }

        ar = nginx_class(cache_max_age_soft_limit=3,
                         cache_max_age_hard_limit=1200,
                         cache_expiration=2,
                         cache_poll_period=3,
                         )

        url = ar.make_url_from_path('/service/scheduler-alwaysthere/foo/bar/')

        with GuardedSubprocess(ar):
            # Register Line buffer filter:
            lbf = LineBufferFilter(filter_regexp,
                                   timeout=5,  # Just to give LBF enough time
                                   line_buffer=ar.stderr_line_buffer)

            # Trigger cache update by issuing request:
            resp = requests.get(url,
                                allow_redirects=False,
                                headers=valid_user_header)
            assert resp.status_code == 200

            # Break marathon
            mocker.send_command(endpoint_id='http://127.0.0.1:8080',
                                func_name='always_bork',
                                aux_data=True)

            # Wait for the cache to be old enough to be considered stale by AR:
            # cache_max_age_soft_limit + 1s for a good measure
            time.sleep(3 + 1)

            # Perform the main/test request:
            resp = requests.get(url,
                                allow_redirects=False,
                                headers=valid_user_header)
            assert resp.status_code == 200

            lbf.scan_log_buffer()

        assert lbf.extra_matches == {}

    def test_if_var_is_verified(self, invalid_ip, nginx_class, mocker):
        filter_regexp = {
            'Local Mesos Master IP: unknown': SearchCriteria(1, True),
            'HOST_IP var is not a valid ipv4: {}'.format(invalid_ip):
                SearchCriteria(1, True),
        }
        ar = nginx_class(host_ip=invalid_ip)

        with GuardedSubprocess(ar):
            lbf = LineBufferFilter(filter_regexp,
                                   line_buffer=ar.stderr_line_buffer)

            lbf.scan_log_buffer()

        assert lbf.extra_matches == {}

    def test_if_not_defining_the_var_is_handled(self, nginx_class, role):
        # Scanning for the exact log entry is bad, but in this case - can't be
        # avoided.
        filter_regexp = {
            'AUTH_TOKEN_VERIFICATION_KEY_FILE_PATH not set.':
                SearchCriteria(1, False)
        }
        ar = nginx_class(role=role, auth_token_verification_key_file_path=None)

        with GuardedSubprocess(ar):
            lbf = LineBufferFilter(filter_regexp,
                                   line_buffer=ar.stderr_line_buffer)
            lbf.scan_log_buffer()

        assert lbf.extra_matches == {}

    def test_if_unset_hostip_var_is_handled(self, nginx_class, valid_user_header):
        filter_regexp = {
            'Local Mesos Master IP address is unknown, cache entry is unusable':
                SearchCriteria(1, True),
            '`Mesos Leader` state cache has been successfully updated':
                SearchCriteria(1, True),
        }
        ar = nginx_class(host_ip=None)

        with GuardedSubprocess(ar):
            lbf = LineBufferFilter(filter_regexp,
                                   line_buffer=ar.stderr_line_buffer)
            # Just trigger the cache update:
            ping_mesos_agent(ar, valid_user_header)

            lbf.scan_log_buffer()

        assert lbf.extra_matches == {}

    def test_if_cache_refresh_occurs_regularly(
            self, nginx_class, mocker, superuser_user_header):
        filter_regexp = {
            'Executing cache refresh triggered by timer': SearchCriteria(3, False),
            'Cache `[\s\w]+` expired. Refresh.': SearchCriteria(6, True),
            'Mesos state cache has been successfully updated': SearchCriteria(3, True),
            'Marathon apps cache has been successfully updated': SearchCriteria(3, True),
            'Marathon leader cache has been successfully updated': SearchCriteria(3, True),
            }
        cache_poll_period = 4

        # Enable recording for marathon
        mocker.send_command(endpoint_id='http://127.0.0.1:8080',
                            func_name='record_requests')
        # Enable recording for mesos
        mocker.send_command(endpoint_id='http://127.0.0.2:5050',
                            func_name='record_requests')

        # Make regular polling occur faster than usual to speed up the tests.
        ar = nginx_class(cache_poll_period=cache_poll_period, cache_expiration=3)

        # In total, we should get three cache updates in given time frame:
        timeout = CACHE_FIRST_POLL_DELAY + cache_poll_period * 2 + 1

        with GuardedSubprocess(ar):
            lbf = LineBufferFilter(filter_regexp,
                                   timeout=timeout,
                                   line_buffer=ar.stderr_line_buffer)

            lbf.scan_log_buffer()

            # Do a request that uses cache so that we can verify that data was
            # in fact cached and no more than one req to mesos/marathon
            # backends were made
            ping_mesos_agent(ar, superuser_user_header)

        mesos_requests = mocker.send_command(endpoint_id='http://127.0.0.2:5050',
                                             func_name='get_recorded_requests')
        marathon_requests = mocker.send_command(endpoint_id='http://127.0.0.1:8080',
                                                func_name='get_recorded_requests')

        assert lbf.extra_matches == {}
        assert len(mesos_requests) == 3
        assert len(marathon_requests) == 6

    def test_if_auth_module_can_be_disabled(self, nginx_class, mocker):
        filter_regexp = {
            ("ADMINROUTER_ACTIVATE_AUTH_MODULE set to `false`. "
             "Deactivate authentication module."): SearchCriteria(1, True),
        }
        ar = nginx_class(auth_enabled='false')
        url = ar.make_url_from_path('/exhibitor/foo/bar')

        with GuardedSubprocess(ar):
            lbf = LineBufferFilter(filter_regexp,
                                   line_buffer=ar.stderr_line_buffer)

            resp = requests.get(url,
                                allow_redirects=False)

            assert resp.status_code == 200
            lbf.scan_log_buffer()

        assert lbf.extra_matches == {}

    def test_if_cache_refresh_is_triggered_by_request(
            self, nginx_class, mocker, superuser_user_header):
        """...right after Nginx has started."""
        filter_regexp = {
            'Executing cache refresh triggered by request': SearchCriteria(1, True),
            'Cache `[\s\w]+` empty. Fetching.': SearchCriteria(3, True),
            'Mesos state cache has been successfully updated': SearchCriteria(1, True),
            'Marathon apps cache has been successfully updated': SearchCriteria(1, True),
            'Marathon leader cache has been successfully updated': SearchCriteria(1, True),
            }
        # Enable recording for marathon
        mocker.send_command(endpoint_id='http://127.0.0.1:8080',
                            func_name='record_requests')
        # Enable recording for mesos
        mocker.send_command(endpoint_id='http://127.0.0.2:5050',
                            func_name='record_requests')

        # Make sure that timers will not interfere:
        ar = nginx_class(cache_first_poll_delay=120,
                         cache_poll_period=120,
                         cache_expiration=115)

        with GuardedSubprocess(ar):
            lbf = LineBufferFilter(filter_regexp,
                                   timeout=5,
                                   line_buffer=ar.stderr_line_buffer)

            ping_mesos_agent(ar, superuser_user_header)
            lbf.scan_log_buffer()

            # Do an extra request so that we can verify that data was in fact
            # cached and no more than one req to mesos/marathon backends were
            # made
            ping_mesos_agent(ar, superuser_user_header)

        mesos_requests = mocker.send_command(endpoint_id='http://127.0.0.2:5050',
                                             func_name='get_recorded_requests')
        marathon_requests = mocker.send_command(endpoint_id='http://127.0.0.1:8080',
                                                func_name='get_recorded_requests')

        assert lbf.extra_matches == {}
        assert len(mesos_requests) == 1
        assert len(marathon_requests) == 2

    def test_if_unset_hostip_var_is_handled(self, nginx_class, valid_user_header):
        filter_regexp = {
            'Private IP address of the host is unknown, ' +
            'aborting cache-entry creation for mesos leader':
                SearchCriteria(1, True),
            'mesos leader cache has been successfully updated':
                SearchCriteria(1, True),
        }
        ar = nginx_class(host_ip=None)

        with GuardedSubprocess(ar):
            lbf = LineBufferFilter(filter_regexp,
                                   line_buffer=ar.stderr_line_buffer)
            # Just trigger the cache update:
            ping_mesos_agent(ar, valid_user_header)

            lbf.scan_log_buffer()

        assert lbf.extra_matches == {}