def incr(self, model, columns, filters, extra=None):
        """
        Increment the key by doing the following:

        - Insert/update a hashmap based on (model, columns)
            - Perform an incrby on counters
            - Perform a set (last write wins) on extra
        - Add hashmap key to pending flushes
        """
        # TODO(dcramer): longer term we'd rather not have to serialize values
        # here (unless it's to JSON)
        key = self._make_key(model, filters)
        # We can't use conn.map() due to wanting to support multiple pending
        # keys (one per Redis shard)
        conn = self.cluster.get_local_client_for_key(key)

        pipe = conn.pipeline()
        pipe.hsetnx(key, 'm', '%s.%s' % (model.__module__, model.__name__))
        pipe.hsetnx(key, 'f', pickle.dumps(filters))
        for column, amount in columns.iteritems():
            pipe.hincrby(key, 'i+' + column, amount)

        if extra:
            for column, value in extra.iteritems():
                pipe.hset(key, 'e+' + column, pickle.dumps(value))
        pipe.expire(key, self.key_expire)
        pipe.zadd(self.pending_key, time(), key)
        pipe.execute()

 def get_prep_value(self, value):
     if not value and self.null:
         # save ourselves some storage
         return None
     # enforce unicode strings to guarantee consistency
     if isinstance(value, str):
         value = unicode(value)
     return compress(pickle.dumps(value))

 def get_prep_value(self, value):
     if not value and self.null:
         # save ourselves some storage
         return None
     # enforce six.text_type strings to guarantee consistency
     if isinstance(value, six.binary_type):
         value = six.text_type(value)
     # db values need to be in unicode
     return compress(pickle.dumps(value))

 def test_process_saves_extra(self):
     group = Group.objects.create(project=Project(id=1))
     columns = {'times_seen': 1}
     filters = {'pk': group.pk}
     the_date = datetime.now() + timedelta(days=5)
     self.buf.conn.set('foo', 1)
     self.buf.conn.hset('extra', 'last_seen', pickle.dumps(the_date))
     self.buf.process(Group, columns, filters)
     group_ = Group.objects.get(pk=group.pk)
     self.assertEquals(group_.last_seen, the_date)

    def test_handles_gettext_lazy(self):
        def fake_gettext(to_translate):
            return u'Igpay Atinlay'

        fake_gettext_lazy = lazy(fake_gettext, str)

        self.assertEquals(
            pickle.loads(pickle.dumps(
                    transform(fake_gettext_lazy("something")))),
            u'Igpay Atinlay')

    def incr(self, model, columns, filters, extra=None):
        """
        Increment the key by doing the following:

        - Insert/update a hashmap based on (model, columns)
            - Perform an incrby on counters
            - Perform a set (last write wins) on extra
        - Add hashmap key to pending flushes
        """
        # TODO(dcramer): longer term we'd rather not have to serialize values
        # here (unless it's to JSON)
        key = self._make_key(model, filters)
        pending_key = self._make_pending_key_from_key(key)
        # We can't use conn.map() due to wanting to support multiple pending
        # keys (one per Redis partition)
        conn = self.cluster.get_local_client_for_key(key)

        pipe = conn.pipeline()
        pipe.hsetnx(key, 'm', '%s.%s' % (model.__module__, model.__name__))
        # TODO(dcramer): once this goes live in production, we can kill the pickle path
        # (this is to ensure a zero downtime deploy where we can transition event processing)
        pipe.hsetnx(key, 'f', pickle.dumps(filters))
        # pipe.hsetnx(key, 'f', json.dumps(self._dump_values(filters)))
        for column, amount in six.iteritems(columns):
            pipe.hincrby(key, 'i+' + column, amount)

        if extra:
            # Group tries to serialize 'score', so we'd need some kind of processing
            # hook here
            # e.g. "update score if last_seen or times_seen is changed"
            for column, value in six.iteritems(extra):
                # TODO(dcramer): once this goes live in production, we can kill the pickle path
                # (this is to ensure a zero downtime deploy where we can transition event processing)
                pipe.hset(key, 'e+' + column, pickle.dumps(value))
                # pipe.hset(key, 'e+' + column, json.dumps(self._dump_value(value)))
        pipe.expire(key, self.key_expire)
        pipe.zadd(pending_key, time(), key)
        pipe.execute()

        metrics.incr('buffer.incr', skip_internal=True, tags={
            'module': model.__module__,
            'model': model.__name__,
        })

    def test_event_node_id(self):
        # Create an event without specifying node_id. A node_id should be generated
        e1 = Event(project_id=1, event_id='abc', data={'foo': 'bar'})
        e1.save()
        e1_node_id = e1.data.id
        assert e1.data.id is not None, "We should have generated a node_id for this event"
        e1_body = nodestore.get(e1_node_id)
        assert e1_body == {'foo': 'bar'}, "The event body should be in nodestore"

        e1 = Event.objects.get(project_id=1, event_id='abc')
        assert e1.data.data == {'foo': 'bar'}, "The event body should be loaded from nodestore"
        assert e1.data.id == e1_node_id, "The event's node_id should be the same after load"

        # Create another event that references the same nodestore object as the first event.
        e2 = Event(project_id=1, event_id='def', data={'node_id': e1_node_id})
        assert e2.data.id == e1_node_id, "The event should use the provided node_id"
        e2_body = nodestore.get(e1_node_id)
        assert e2_body == {'foo': 'bar'}, "The event body should be in nodestore already"
        e2.save()
        e2_body = nodestore.get(e1_node_id)
        assert e2_body == {'foo': 'bar'}, "The event body should not be overwritten by save"

        e2 = Event.objects.get(project_id=1, event_id='def')
        assert e2.data.data == {'foo': 'bar'}, "The event body should be loaded from nodestore"
        assert e2.data.id == e1_node_id, "The event's node_id should be the same after load"

        # Create an event with a new event body that specifies the node_id to use.
        e3 = Event(project_id=1, event_id='ghi', data={'baz': 'quux', 'node_id': '1:ghi'})
        assert e3.data.id == '1:ghi', "Event should have the specified node_id"
        assert e3.data.data == {'baz': 'quux'}, "Event body should be the one provided (sans node_id)"
        e3.save()
        e3_body = nodestore.get('1:ghi')
        assert e3_body == {'baz': 'quux'}, "Event body should be saved to nodestore"

        e3 = Event.objects.get(project_id=1, event_id='ghi')
        assert e3.data.data == {'baz': 'quux'}, "Event body should be loaded from nodestore"
        assert e3.data.id == '1:ghi', "Loaded event should have the correct node_id"

        # Try load it again, but using the pickled/compressed string we would expect to find
        # in the column
        e3_pickled_id = compress(pickle.dumps({'node_id': '1:ghi'}))
        e3 = Event(project_id=1, event_id='jkl', data=e3_pickled_id)
        assert e3.data.data == {'baz': 'quux'}, "Event body should be loaded from nodestore"

        # Event with no data should not be saved (or loaded) from nodestore
        e4 = Event(project_id=1, event_id='mno', data=None)
        e4.save()
        assert nodestore.get('1:mno') is None, "We should not have saved anything to nodestore"
        e4 = Event.objects.get(project_id=1, event_id='mno')
        assert e4.data.id is None
        assert e4.data.data == {}  # NodeData returns {} by default
        Event.objects.bind_nodes([e4], 'data')
        assert e4.data.id is None
        assert e4.data.data == {}

    def incr(self, model, columns, filters, extra=None):
        for column, amount in columns.iteritems():
            key = self._make_key(model, filters, column)
            call_args = (key, str(amount), str(self.key_expire))
            self._tnt.call('box.sentry_buffer.incr', call_args)

        if extra:
            key = self._make_extra_key(model, filters)
            for column, value in extra.iteritems():
                call_args = (key, column, pickle.dumps(value),
                             str(self.key_expire))
                self._tnt.call('box.sentry_buffer.hset', call_args)
        super(TarantoolBuffer, self).incr(model, columns, filters, extra)

    def get_prep_value(self, value):
        if not value and self.null:
            # save ourselves some storage
            return None

        # TODO(dcramer): we should probably do this more intelligently
        # and manually
        if not value.id:
            value.id = nodestore.create(value.data)
        else:
            nodestore.set(value.id, value.data)

        return compress(pickle.dumps({'node_id': value.id}))

    def incr(self, model, columns, filters, extra=None):
        with self.conn.map() as conn:
            for column, amount in columns.iteritems():
                key = self._make_key(model, filters, column)
                conn.incr(key, amount)
                conn.expire(key, self.key_expire)

            # Store extra in a hashmap so it can easily be removed
            if extra:
                key = self._make_extra_key(model, filters)
                for column, value in extra.iteritems():
                    conn.hset(key, column, pickle.dumps(value))
                    conn.expire(key, self.key_expire)
        super(RedisBuffer, self).incr(model, columns, filters, extra)

    def get_prep_value(self, value):
        """
            Prepares the NodeData to be written in a Model.save() call.

            Makes sure the event body is written to nodestore and
            returns the node_id reference to be written to rowstore.
        """
        if not value and self.null:
            # save ourselves some storage
            return None

        if value.id is None:
            value.id = self.id_func()

        value.save()
        return compress(pickle.dumps({'node_id': value.id}))

    def test_does_transition_data_to_node(self):
        group = self.group
        data = {'key': 'value'}

        query_bits = [
            "INSERT INTO sentry_message (group_id, project_id, data, message, datetime)",
            "VALUES(%s, %s, %s, %s, %s)",
        ]
        params = [group.id, group.project_id, compress(pickle.dumps(data)), 'test', timezone.now()]

        # This is pulled from SQLInsertCompiler
        if connection.features.can_return_id_from_insert:
            r_fmt, r_params = connection.ops.return_insert_id()
            if r_fmt:
                query_bits.append(r_fmt % Event._meta.pk.column)
                params += r_params

        cursor = connection.cursor()
        cursor.execute(' '.join(query_bits), params)

        if connection.features.can_return_id_from_insert:
            event_id = connection.ops.fetch_returned_insert_id(cursor)
        else:
            event_id = connection.ops.last_insert_id(
                cursor, Event._meta.db_table, Event._meta.pk.column
            )

        event = Event.objects.get(id=event_id)
        assert type(event.data) == NodeData
        assert event.data == data
        assert event.data.id is None

        event.save()

        assert event.data == data
        assert event.data.id is not None

        node_id = event.data.id
        event = Event.objects.get(id=event_id)

        Event.objects.bind_nodes([event], 'data')

        assert event.data == data
        assert event.data.id == node_id

    def get_prep_value(self, value):
        if not value and self.null:
            # save ourselves some storage
            return None

        # We can't put our wrappers into the nodestore, so we need to
        # ensure that the data is converted into a plain old dict
        data = value.data
        if isinstance(data, CANONICAL_TYPES):
            data = dict(data.items())

        # TODO(dcramer): we should probably do this more intelligently
        # and manually
        if not value.id:
            value.id = nodestore.create(data)
        else:
            nodestore.set(value.id, data)

        return compress(pickle.dumps({'node_id': value.id}))

 def get_prep_value(self, value):
     if value is None:
         return
     return base64.b64encode(pickle.dumps(value).encode('zlib'))

 def get_prep_value(self, value):
     if value is None:
         return
     return base64.b64encode(pickle.dumps(transform(value)).encode("zlib"))

 def test_incr_does_buffer_extra_to_conn(self):
     model = mock.Mock()
     columns = {'times_seen': 1}
     filters = {'pk': 1}
     self.buf.incr(model, columns, filters, extra={'foo': 'bar'})
     self.assertEquals(self.buf.conn.hget('extra', 'foo'), pickle.dumps('bar'))

 def get_prep_value(self, value):
     if not value and self.null:
         # save ourselves some storage
         return None
     return base64.b64encode(pickle.dumps(value).encode('zlib'))

 def get_prep_value(self, value):
     if not value and self.null:
         # save ourselves some storage
         return None
     return compress(pickle.dumps(value))

 def _postWithKey(self, data, key=None):
     resp = self.client.post(reverse('sentry-api-store'), {
         'data': base64.b64encode(pickle.dumps(data)),
         'key': settings.KEY,
     })
     return resp

 def encode(self, value):
     return zlib.compress(pickle.dumps(value))