def points_during_day(data, noon):
    """Returns all the points in the dataset that occur in the 12 hours around
    the datetime object that is passed in. data must be sorted."""
    before = datetime_to_timestamp(noon - timedelta(hours=12))
    after = datetime_to_timestamp(noon + timedelta(hours=12))

    between = [pt for pt in data if pt[1] > before and pt[1] < after]
    return between

    def test_datetime_and_timestamp_conversions(self) -> None:
        timestamp = 1483228800
        for dt in [
                parser.parse('2017-01-01 00:00:00.123 UTC'),
                parser.parse('2017-01-01 00:00:00.123').replace(tzinfo=timezone_utc),
                parser.parse('2017-01-01 00:00:00.123').replace(tzinfo=pytz.utc)]:
            self.assertEqual(timestamp_to_datetime(timestamp), dt-timedelta(microseconds=123000))
            self.assertEqual(datetime_to_timestamp(dt), timestamp)

        for dt in [
                parser.parse('2017-01-01 00:00:00.123+01:00'),
                parser.parse('2017-01-01 00:00:00.123')]:
            with self.assertRaises(TimezoneNotUTCException):
                datetime_to_timestamp(dt)

def get_gcm_payload(user_profile, message):
    # type: (UserProfile, Message) -> Dict[str, Any]
    content = message.content
    content_truncated = (len(content) > 200)
    if content_truncated:
        content = content[:200] + "..."

    android_data = {
        'user': user_profile.email,
        'event': 'message',
        'alert': get_alert_from_message(message),
        'zulip_message_id': message.id,  # message_id is reserved for CCS
        'time': datetime_to_timestamp(message.pub_date),
        'content': content,
        'content_truncated': content_truncated,
        'sender_email': message.sender.email,
        'sender_full_name': message.sender.full_name,
        'sender_avatar_url': absolute_avatar_url(message.sender),
    }

    if message.recipient.type == Recipient.STREAM:
        android_data['recipient_type'] = "stream"
        android_data['stream'] = get_display_recipient(message.recipient)
        android_data['topic'] = message.subject
    elif message.recipient.type in (Recipient.HUDDLE, Recipient.PERSONAL):
        android_data['recipient_type'] = "private"

    return android_data

    def test_start_and_end(self) -> None:
        stat = COUNT_STATS['realm_active_humans::day']
        self.insert_data(stat, [None], [])
        stat = COUNT_STATS['1day_actives::day']
        self.insert_data(stat, [None], [])
        stat = COUNT_STATS['active_users_audit:is_bot:day']
        self.insert_data(stat, ['false'], [])
        end_time_timestamps = [datetime_to_timestamp(dt) for dt in self.end_times_day]

        # valid start and end
        result = self.client_get('/json/analytics/chart_data',
                                 {'chart_name': 'number_of_humans',
                                  'start': end_time_timestamps[1],
                                  'end': end_time_timestamps[2]})
        self.assert_json_success(result)
        data = result.json()
        self.assertEqual(data['end_times'], end_time_timestamps[1:3])
        self.assertEqual(data['everyone'], {'_1day': [0, 100], '_15day': [0, 100], 'all_time': [0, 100]})

        # start later then end
        result = self.client_get('/json/analytics/chart_data',
                                 {'chart_name': 'number_of_humans',
                                  'start': end_time_timestamps[2],
                                  'end': end_time_timestamps[1]})
        self.assert_json_error_contains(result, 'Start time is later than')

 def test_messages_sent_by_client(self):
     # type: () -> None
     stat = COUNT_STATS['messages_sent:client:day']
     client1 = Client.objects.create(name='client 1')
     _client1 = Client.objects.create(name='_client 1')
     client2 = Client.objects.create(name='client 2')
     client3 = Client.objects.create(name='client 3')
     _client3 = Client.objects.create(name='_client 3')
     client4 = Client.objects.create(name='client 4')
     self.insert_data(stat, [client4.id, client3.id, client2.id],
                      [client1.id, _client1.id, client4.id, _client3.id])
     result = self.client_get('/json/analytics/chart_data',
                              {'chart_name': 'messages_sent_by_client'})
     self.assert_json_success(result)
     data = ujson.loads(result.content)
     self.assertEqual(data, {
         'msg': '',
         'end_times': [datetime_to_timestamp(dt) for dt in self.end_times_day],
         'frequency': CountStat.DAY,
         'interval': CountStat.DAY,
         'realm': {'client 4': self.data(100), 'client 3': self.data(101),
                   'client 2': self.data(102)},
         'user': {'client 1': self.data(401), 'client 4': self.data(202),
                  'client 3': self.data(203)},
         'display_order': ['client 1', 'client 2', 'client 3', 'client 4'],
         'result': 'success',
     })

def get_presence_backend(request: HttpRequest, user_profile: UserProfile,
                         email: Text) -> HttpResponse:
    try:
        target = get_user(email, user_profile.realm)
    except UserProfile.DoesNotExist:
        return json_error(_('No such user'))
    if not target.is_active:
        return json_error(_('No such user'))
    if target.is_bot:
        return json_error(_('Presence is not supported for bot users.'))

    presence_dict = UserPresence.get_status_dict_by_user(target)
    if len(presence_dict) == 0:
        return json_error(_('No presence data for %s' % (target.email,)))

    # For initial version, we just include the status and timestamp keys
    result = dict(presence=presence_dict[target.email])
    aggregated_info = result['presence']['aggregated']
    aggr_status_duration = datetime_to_timestamp(timezone_now()) - aggregated_info['timestamp']
    if aggr_status_duration > settings.OFFLINE_THRESHOLD_SECS:
        aggregated_info['status'] = 'offline'
    for val in result['presence'].values():
        val.pop('client', None)
        val.pop('pushable', None)
    return json_success(result)

 def _send_presence_for_aggregated_tests(self, email, status, validate_time):
     # type: (str, str, datetime.datetime) -> Dict[str, Dict[str, Any]]
     self.login(email)
     timezone_util = 'zerver.views.presence.timezone_now'
     with mock.patch(timezone_util, return_value=validate_time - datetime.timedelta(seconds=5)):
         self.client_post("/json/users/me/presence", {'status': status})
     with mock.patch(timezone_util, return_value=validate_time - datetime.timedelta(seconds=2)):
         self.client_post("/api/v1/users/me/presence", {'status': status},
                          HTTP_USER_AGENT="ZulipAndroid/1.0",
                          **self.api_auth(email))
     with mock.patch(timezone_util, return_value=validate_time - datetime.timedelta(seconds=7)):
         latest_result = self.client_post("/api/v1/users/me/presence", {'status': status},
                                          HTTP_USER_AGENT="ZulipIOS/1.0",
                                          **self.api_auth(email))
     latest_result_dict = latest_result.json()
     self.assertDictEqual(
         latest_result_dict['presences'][email]['aggregated'],
         {
             'status': status,
             'timestamp': datetime_to_timestamp(validate_time - datetime.timedelta(seconds=2)),
             'client': 'ZulipAndroid'
         }
     )
     result = self.client_get("/json/users/%s/presence" % (email,))
     return result.json()

def archive(request: HttpRequest,
            stream_id: int,
            topic_name: str) -> HttpResponse:

    def get_response(rendered_message_list: List[str],
                     is_web_public: bool,
                     stream_name: str) -> HttpResponse:
        return render(
            request,
            'zerver/archive/index.html',
            context={
                'is_web_public': is_web_public,
                'message_list': rendered_message_list,
                'stream': stream_name,
                'topic': topic_name,
            }
        )

    try:
        stream = get_stream_by_id(stream_id)
    except JsonableError:
        return get_response([], False, '')

    if not stream.is_web_public:
        return get_response([], False, '')

    all_messages = list(Message.objects.select_related(
        'sender').filter(recipient__type_id=stream_id, subject=topic_name).order_by('pub_date'))
    if not all_messages:
        return get_response([], True, stream.name)

    rendered_message_list = []
    prev_sender = None
    for msg in all_messages:
        include_sender = False
        status_message = Message.is_status_message(msg.content, msg.rendered_content)
        if not prev_sender or prev_sender != msg.sender or status_message:
            if status_message:
                prev_sender = None
            else:
                prev_sender = msg.sender
            include_sender = True
        if status_message:
            status_message = msg.rendered_content[4+3: -4]
        context = {
            'sender_full_name': msg.sender.full_name,
            'timestampstr': datetime_to_timestamp(msg.last_edit_time
                                                  if msg.last_edit_time
                                                  else msg.pub_date),
            'message_content': msg.rendered_content,
            'avatar_url': get_gravatar_url(msg.sender.email, 1),
            'include_sender': include_sender,
            'status_message': status_message,
        }
        rendered_msg = loader.render_to_string('zerver/archive/single_message.html', context)
        rendered_message_list.append(rendered_msg)
    return get_response(rendered_message_list, True, stream.name)

 def to_presence_dict(client_name=None, status=None, timestamp=None, push_enabled=None,
         has_push_devices=None, is_mirror_dummy=None):
     presence_val = UserPresence.status_to_string(status)
     timestamp = datetime_to_timestamp(timestamp)
     return dict(
             client=client_name,
             status=presence_val,
             timestamp=timestamp,
             pushable=(push_enabled and has_push_devices),
     )

 def test_min_length(self) -> None:
     stat = COUNT_STATS['realm_active_humans::day']
     self.insert_data(stat, [None], [])
     # test min_length is too short to change anything
     result = self.client_get('/json/analytics/chart_data',
                              {'chart_name': 'number_of_humans',
                               'min_length': 2})
     self.assert_json_success(result)
     data = result.json()
     self.assertEqual(data['end_times'], [datetime_to_timestamp(dt) for dt in self.end_times_day])
     self.assertEqual(data['realm'], {'human': self.data(100)})
     # test min_length larger than filled data
     result = self.client_get('/json/analytics/chart_data',
                              {'chart_name': 'number_of_humans',
                               'min_length': 5})
     self.assert_json_success(result)
     data = result.json()
     end_times = [ceiling_to_day(self.realm.date_created) + timedelta(days=i) for i in range(-1, 4)]
     self.assertEqual(data['end_times'], [datetime_to_timestamp(dt) for dt in end_times])
     self.assertEqual(data['realm'], {'human': [0]+self.data(100)})

 def test_aggregated_presense_idle(self) -> None:
     validate_time = timezone_now()
     result_dict = self._send_presence_for_aggregated_tests(str(self.example_email("othello")), 'idle',
                                                            validate_time)
     self.assertDictEqual(
         result_dict['presence']['aggregated'],
         {
             "status": "idle",
             "timestamp": datetime_to_timestamp(validate_time - datetime.timedelta(seconds=2))
         }
     )

 def test_min_length(self):
     # type: () -> None
     stat = COUNT_STATS['active_users:is_bot:day']
     self.insert_data(stat, ['true', 'false'], [])
     # test min_length is too short to change anything
     result = self.client_get('/json/analytics/chart_data',
                              {'chart_name': 'number_of_humans',
                               'min_length': 2})
     self.assert_json_success(result)
     data = ujson.loads(result.content)
     self.assertEqual(data['end_times'], [datetime_to_timestamp(dt) for dt in self.end_times_day])
     self.assertEqual(data['realm'], {'bot': self.data(100), 'human': self.data(101)})
     # test min_length larger than filled data
     result = self.client_get('/json/analytics/chart_data',
                              {'chart_name': 'number_of_humans',
                               'min_length': 5})
     self.assert_json_success(result)
     data = ujson.loads(result.content)
     end_times = [ceiling_to_day(self.realm.date_created) + timedelta(days=i) for i in range(-1, 4)]
     self.assertEqual(data['end_times'], [datetime_to_timestamp(dt) for dt in end_times])
     self.assertEqual(data['realm'], {'bot': [0]+self.data(100), 'human': [0]+self.data(101)})

def fill_edit_history_entries(message_history, message):
    # type: (List[Dict[str, Any]], Message) -> None
    """This fills out the message edit history entries from the database,
    which are designed to have the minimum data possible, to instead
    have the current topic + content as of that time, plus data on
    whatever changed.  This makes it much simpler to do future
    processing.

    Note that this mutates what is passed to it, which is sorta a bad pattern.
    """
    prev_content = message.content
    prev_rendered_content = message.rendered_content
    prev_topic = message.subject
    assert(datetime_to_timestamp(message.last_edit_time) == message_history[0]['timestamp'])

    for entry in message_history:
        entry['topic'] = prev_topic
        if 'prev_subject' in entry:
            # We replace use of 'subject' with 'topic' for downstream simplicity
            prev_topic = entry['prev_subject']
            entry['prev_topic'] = prev_topic
            del entry['prev_subject']

        entry['content'] = prev_content
        entry['rendered_content'] = prev_rendered_content
        if 'prev_content' in entry:
            del entry['prev_rendered_content_version']
            prev_content = entry['prev_content']
            prev_rendered_content = entry['prev_rendered_content']
            entry['content_html_diff'] = highlight_html_differences(
                prev_rendered_content,
                entry['rendered_content'])

    message_history.append(dict(
        topic = prev_topic,
        content = prev_content,
        rendered_content = prev_rendered_content,
        timestamp = datetime_to_timestamp(message.pub_date),
        user_id = message.sender_id,
    ))

 def test_aggregated_presense_offline(self) -> None:
     email = self.example_email("othello")
     self.login(email)
     validate_time = timezone_now()
     with self.settings(OFFLINE_THRESHOLD_SECS=1):
         result_dict = self._send_presence_for_aggregated_tests(str(email), 'idle', validate_time)
     self.assertDictEqual(
         result_dict['presence']['aggregated'],
         {
             "status": "offline",
             "timestamp": datetime_to_timestamp(validate_time - datetime.timedelta(seconds=2))
         }
     )

 def to_log_dict(self):
     return dict(
         id                = self.id,
         sender_email      = self.sender.email,
         sender_domain     = self.sender.realm.domain,
         sender_full_name  = self.sender.full_name,
         sender_short_name = self.sender.short_name,
         sending_client    = self.sending_client.name,
         type              = self.recipient.type_name(),
         recipient         = get_display_recipient(self.recipient),
         subject           = self.subject,
         content           = self.content,
         timestamp         = datetime_to_timestamp(self.pub_date))

def get_gcm_payload(user_profile: UserProfile, message: Message) -> Dict[str, Any]:
    data = get_common_payload(message)
    content, truncated = truncate_content(get_mobile_push_content(message.rendered_content))
    data.update({
        'user': user_profile.email,
        'event': 'message',
        'alert': get_gcm_alert(message),
        'zulip_message_id': message.id,  # message_id is reserved for CCS
        'time': datetime_to_timestamp(message.pub_date),
        'content': content,
        'content_truncated': truncated,
        'sender_full_name': message.sender.full_name,
        'sender_avatar_url': absolute_avatar_url(message.sender),
    })
    return data

 def test_number_of_humans(self) -> None:
     stat = COUNT_STATS['realm_active_humans::day']
     self.insert_data(stat, [None], [])
     result = self.client_get('/json/analytics/chart_data',
                              {'chart_name': 'number_of_humans'})
     self.assert_json_success(result)
     data = result.json()
     self.assertEqual(data, {
         'msg': '',
         'end_times': [datetime_to_timestamp(dt) for dt in self.end_times_day],
         'frequency': CountStat.DAY,
         'realm': {'human': self.data(100)},
         'display_order': None,
         'result': 'success',
     })

def update_user_activity(request, user_profile):
    # update_active_status also pushes to rabbitmq, and it seems
    # redundant to log that here as well.
    if request.META["PATH_INFO"] == '/json/users/me/presence':
        return

    if hasattr(request, '_query'):
        query = request._query
    else:
        query = request.META['PATH_INFO']

    event={'query': query,
           'user_profile_id': user_profile.id,
           'time': datetime_to_timestamp(now()),
           'client': request.client.name}
    queue_json_publish("user_activity", event, lambda event: None)

 def test_aggregated_presense_mixed(self) -> None:
     email = self.example_email("othello")
     self.login(email)
     validate_time = timezone_now()
     with mock.patch('zerver.views.presence.timezone_now',
                     return_value=validate_time - datetime.timedelta(seconds=3)):
         self.api_post(email, "/api/v1/users/me/presence", {'status': 'active'},
                       HTTP_USER_AGENT="ZulipTestDev/1.0")
     result_dict = self._send_presence_for_aggregated_tests(str(email), 'idle', validate_time)
     self.assertDictEqual(
         result_dict['presence']['aggregated'],
         {
             "status": "idle",
             "timestamp": datetime_to_timestamp(validate_time - datetime.timedelta(seconds=2))
         }
     )

 def test_messages_sent_over_time(self) -> None:
     stat = COUNT_STATS['messages_sent:is_bot:hour']
     self.insert_data(stat, ['true', 'false'], ['false'])
     result = self.client_get('/json/analytics/chart_data',
                              {'chart_name': 'messages_sent_over_time'})
     self.assert_json_success(result)
     data = result.json()
     self.assertEqual(data, {
         'msg': '',
         'end_times': [datetime_to_timestamp(dt) for dt in self.end_times_hour],
         'frequency': CountStat.HOUR,
         'everyone': {'bot': self.data(100), 'human': self.data(101)},
         'user': {'bot': self.data(0), 'human': self.data(200)},
         'display_order': None,
         'result': 'success',
     })