def test_writing_event_defaults(self):
        query = SearchQuery()
        default_event = {}
        self.populate_test_event(default_event)
        self.refresh(self.event_index_name)

        query.add_must(ExistsMatch('summary'))
        results = query.execute(self.es_client)
        assert len(results['hits']) == 1
        assert sorted(results['hits'][0].keys()) == ['_id', '_index', '_score', '_source', '_type']
        saved_event = results['hits'][0]['_source']
        assert 'category' in saved_event
        assert 'details' in saved_event
        assert 'hostname' in saved_event
        assert 'mozdefhostname' in saved_event
        assert 'processid' in saved_event
        assert 'processname' in saved_event
        assert 'receivedtimestamp' in saved_event
        assert 'severity' in saved_event
        assert 'source' in saved_event
        assert 'summary' in saved_event
        assert 'tags' in saved_event
        assert 'timestamp' in saved_event
        assert 'utctimestamp' in saved_event
        assert 'category' in saved_event

def kibanaDashboards():
    resultsList = []
    try:
        es_client = ElasticsearchClient((list('{0}'.format(s) for s in options.esservers)))
        search_query = SearchQuery()
        search_query.add_must(TermMatch('_type', 'dashboard'))
        results = search_query.execute(es_client, indices=['.kibana'])

        for dashboard in results['hits']:
            resultsList.append({
                'name': dashboard['_source']['title'],
                'url': "%s#/%s/%s" % (
                    options.kibanaurl,
                    "dashboard",
                    dashboard['_id']
                )
            })

    except ElasticsearchInvalidIndex as e:
        sys.stderr.write('Kibana dashboard index not found: {0}\n'.format(e))

    except Exception as e:
        sys.stderr.write('Kibana dashboard received error: {0}\n'.format(e))

    return json.dumps(resultsList)

    def test_beginning_time_seconds_received_timestamp(self):
        query = SearchQuery(seconds=10)
        query.add_must(ExistsMatch('summary'))
        assert query.date_timedelta == {'seconds': 10}

        default_event = {
            "receivedtimestamp": UnitTestSuite.current_timestamp(),
            "summary": "Test summary",
            "details": {
                "note": "Example note",
            }
        }
        self.populate_test_event(default_event)

        too_old_event = default_event
        too_old_event['receivedtimestamp'] = UnitTestSuite.subtract_from_timestamp({'seconds': 11})
        too_old_event['utctimestamp'] = UnitTestSuite.subtract_from_timestamp({'seconds': 11})
        self.populate_test_event(too_old_event)

        not_old_event = default_event
        not_old_event['receivedtimestamp'] = UnitTestSuite.subtract_from_timestamp({'seconds': 9})
        not_old_event['utctimestamp'] = UnitTestSuite.subtract_from_timestamp({'seconds': 9})
        self.populate_test_event(not_old_event)

        self.refresh(self.event_index_name)

        results = query.execute(self.es_client)
        assert len(results['hits']) == 2

 def process_alert(self):
     search_query = SearchQuery(minutes=20)
     content = QueryStringMatch(str(self.watchterm))
     search_query.add_must(content)
     self.filtersManual(search_query)
     self.searchEventsSimple()
     self.walkEvents()

def getESAlerts(es):
    search_query = SearchQuery(minutes=50)
    # We use an ExistsMatch here just to satisfy the
    # requirements of a search query must have some "Matchers"
    search_query.add_must(ExistsMatch('summary'))
    results = search_query.execute(es, indices=['alerts'], size=10000)
    return results

    def main(self):
        search_query = SearchQuery(minutes=15)

        self.config = self.parse_json_alert_config('ssh_access_signreleng.json')

        if self.config['ircchannel'] == '':
            self.config['ircchannel'] = None

        search_query.add_must([
            TermMatch('tags', 'releng'),
            TermMatch('details.program', 'sshd'),
            QueryStringMatch('hostname: /{}/'.format(self.config['hostfilter'])),
            PhraseMatch('summary', 'Accepted publickey for ')
        ])

        for exclusion in self.config['exclusions']:
            exclusion_query = None
            for key, value in exclusion.iteritems():
                phrase_exclusion = PhraseMatch(key, value)
                if exclusion_query is None:
                    exclusion_query = phrase_exclusion
                else:
                    exclusion_query = exclusion_query + phrase_exclusion

            search_query.add_must_not(exclusion_query)

        self.filtersManual(search_query)
        self.searchEventsSimple()
        self.walkEvents()

    def main(self):
        self.parse_config("proxy_drop_executable.conf", ["extensions"])

        search_query = SearchQuery(minutes=20)

        search_query.add_must(
            [
                TermMatch("category", "proxy"),
                TermMatch("details.proxyaction", "TCP_DENIED"),
            ]
        )

        # Only notify on certain file extensions from config
        filename_regex = "/.*\.({0})/".format(self.config.extensions.replace(",", "|"))
        search_query.add_must(
            [QueryStringMatch("details.destination: {}".format(filename_regex))]
        )

        self.filtersManual(search_query)

        # Search aggregations on field 'hostname', keep X samples of
        # events at most
        self.searchEventsAggregated("details.sourceipaddress", samplesLimit=10)
        # alert when >= X matching events in an aggregation
        # I think it makes sense to alert every time here
        self.walkAggregations(threshold=1)

    def test_simple_query_execute(self):
        query = SearchQuery()
        query.add_must(ExistsMatch('note'))
        assert query.date_timedelta == {}

        self.populate_example_event()
        self.refresh(self.event_index_name)

        results = query.execute(self.es_client)

        assert results.keys() == ['hits', 'meta']
        assert results['meta'].keys() == ['timed_out']
        assert results['meta']['timed_out'] is False
        assert len(results['hits']) == 1

        assert results['hits'][0].keys() == ['_score', '_type', '_id', '_source', '_index']
        assert type(results['hits'][0]['_id']) == unicode
        assert results['hits'][0]['_type'] == 'event'

        assert results['hits'][0]['_index'] == datetime.now().strftime("events-%Y%m%d")

        assert results['hits'][0]['_source']['note'] == 'Example note'
        assert results['hits'][0]['_source']['summary'] == 'Test Summary'

        assert results['hits'][0]['_source']['details'].keys() == ['information']
        assert results['hits'][0]['_source']['details']['information'] == 'Example information'

        with pytest.raises(KeyError):
            results['abcdefg']

        with pytest.raises(KeyError):
            results['abcdefg']['test']

    def test_without_time_defined(self):
        query = SearchQuery()
        query.add_must(ExistsMatch('summary'))
        assert query.date_timedelta == {}

        default_event = {
            "utctimestamp": UnitTestSuite.current_timestamp(),
            "summary": "Test summary",
            "details": {
                "note": "Example note",
            }
        }

        self.populate_test_event(default_event)
        default_event['utctimestamp'] = UnitTestSuite.subtract_from_timestamp({'days': 11})
        default_event['receivedtimestamp'] = UnitTestSuite.subtract_from_timestamp({'days': 11})
        self.populate_test_event(default_event)

        not_old_event = default_event
        not_old_event['utctimestamp'] = UnitTestSuite.subtract_from_timestamp({'days': 9})
        not_old_event['receivedtimestamp'] = UnitTestSuite.subtract_from_timestamp({'days': 9})
        self.populate_test_event(not_old_event)

        self.refresh(self.event_index_name)

        results = query.execute(self.es_client)
        assert len(results['hits']) == 3

 def test_execute_without_size(self):
     for num in range(0, 1200):
         self.populate_example_event()
     self.refresh(self.event_index_name)
     query = SearchQuery()
     query.add_must(ExistsMatch('summary'))
     results = query.execute(self.es_client)
     assert len(results['hits']) == 1000

 def search_and_verify_event(self, expected_event):
     self.refresh('events')
     search_query = SearchQuery(minutes=5)
     search_query.add_must(ExistsMatch('tags'))
     results = search_query.execute(self.es_client)
     assert len(results['hits']) == 1
     saved_event = results['hits'][0]['_source']
     self.verify_event(saved_event, expected_event)

    def main(self):
        search_query = SearchQuery(hours=6)

        day_old_date = toUTC(datetime.now() - timedelta(days=1)).isoformat()
        search_query.add_must(LessThanMatch('utctimestamp', day_old_date))
        self.filtersManual(search_query)

        self.searchEventsAggregated('mozdefhostname', samplesLimit=1000)
        self.walkAggregations(threshold=1)

 def process_alert(self, alert_config):
     self.current_alert_time_window = int(alert_config['time_window'])
     self.current_alert_time_type = alert_config['time_window_type']
     search_query_time_window = {self.current_alert_time_type: self.current_alert_time_window}
     search_query = SearchQuery(**search_query_time_window)
     search_query.add_must(QueryStringMatch(str(alert_config['search_query'])))
     self.filtersManual(search_query)
     self.searchEventsSimple()
     self.walkEvents(description=alert_config['description'])

def getSqsStats(es):
    search_query = SearchQuery(minutes=15)
    search_query.add_must([
        TermMatch('_type', 'mozdefhealth'),
        TermMatch('category', 'mozdef'),
        TermMatch('tags', 'sqs-latest'),
    ])
    results = search_query.execute(es, indices=['mozdefstate'])

    return results['hits']

    def main(self):
        search_query = SearchQuery(minutes=30)

        search_query.add_must([
            TermMatch('tags', 'mig-runner-sshioc'),
        ])

        self.filtersManual(search_query)

        self.searchEventsSimple()
        self.walkEvents()

    def main(self):
        search_query = SearchQuery(minutes=15)
        search_query.add_must([
            TermMatch('category', 'syslog'),
            TermMatch('details.program', 'sshd'),
            PhraseMatch('summary', 'Accepted publickey')
        ])

        self.filtersManual(search_query)
        self.searchEventsAggregated('hostname', samplesLimit=10)
        self.walkAggregations(threshold=1)

    def test_aggregation_without_must_fields(self):
        event = self.generate_default_event()
        event['_source']['utctimestamp'] = event['_source']['utctimestamp']()
        event['_source']['receivedtimestamp'] = event['_source']['receivedtimestamp']()
        self.populate_test_event(event)
        self.refresh(self.event_index_name)

        search_query = SearchQuery(minutes=10)

        search_query.add_aggregation(Aggregation('source'))
        results = search_query.execute(self.es_client)
        assert results['aggregations']['source']['terms'][0]['count'] == 1

    def main(self):
        search_query = SearchQuery(minutes=20)

        search_query.add_must([
            TermMatch('source', 'cloudtrail'),
            TermMatch('details.eventname', 'PutBucketPolicy'),
            ExistsMatch('details.requestparameters.bucketpolicy.statement.principal')
        ])

        self.filtersManual(search_query)
        self.searchEventsSimple()
        self.walkEvents()

    def main(self):
        self._config = self.parse_json_alert_config('feedback_events.json')
        search_query = SearchQuery(minutes=30)

        search_query.add_must([
            TermMatch('category', 'user_feedback'),
            TermMatch('details.action', 'escalate')
        ])

        self.filtersManual(search_query)
        self.searchEventsSimple()
        self.walkEvents()

    def main(self):
        search_query = SearchQuery(minutes=5)

        search_query.add_must([
            TermMatch('category', 'execve'),
            TermMatch('processname', 'audisp-json'),
            TermMatch('details.processname', 'ssh'),
            PhraseMatch('details.parentprocess', 'sftp')
        ])

        self.filtersManual(search_query)
        self.searchEventsSimple()
        self.walkEvents()