def _get_newest_link(sr):
    for fullname in sr.get_links('new', 'all'):
        link = Thing._by_fullname(fullname, data=True)
        if not link._spam and not link._deleted:
            return link

    return None

    def process_message(msg):
        timer = g.stats.get_timer("new_voting.%s" % queue)
        timer.start()

        vote_data = json.loads(msg.body)

        user = Account._byID(vote_data.pop("user_id"), data=True)
        thing = Thing._by_fullname(vote_data.pop("thing_fullname"), data=True)

        timer.intermediate("preamble")

        lock_key = "vote-%s-%s" % (user._id36, thing._fullname)
        with g.make_lock("voting", lock_key, timeout=5):
            print "Processing vote by %s on %s %s" % (user, thing, vote_data)

            try:
                vote = Vote(
                    user,
                    thing,
                    direction=vote_data["direction"],
                    date=datetime.utcfromtimestamp(vote_data["date"]),
                    data=vote_data["data"],
                    event_data=vote_data.get("event_data"),
                )
            except TypeError as e:
                # a vote on an invalid type got in the queue, just skip it
                g.log.error(e.message)
                return

            timer.intermediate("create_vote_obj")

            vote.commit()

            timer.flush()

def _run_changed(msgs, chan):
    '''Consume the cloudsearch_changes queue, and print reporting information
    on how long it took and how many remain
    
    '''
    start = datetime.now(g.tz)
    
    changed = [pickle.loads(msg.body) for msg in msgs]
    
    fullnames = set()
    fullnames.update(LinkUploader.desired_fullnames(changed))
    fullnames.update(SubredditUploader.desired_fullnames(changed))
    things = Thing._by_fullname(fullnames, data=True, return_dict=False)
    
    link_uploader = LinkUploader(g.CLOUDSEARCH_DOC_API, things=things)
    subreddit_uploader = SubredditUploader(g.CLOUDSEARCH_SUBREDDIT_DOC_API,
                                           things=things)
    
    link_time = link_uploader.inject()
    subreddit_time = subreddit_uploader.inject()
    cloudsearch_time = link_time + subreddit_time
    
    totaltime = (datetime.now(g.tz) - start).total_seconds()
    
    print ("%s: %d messages in %.2fs seconds (%.2fs secs waiting on "
           "cloudsearch); %d duplicates, %s remaining)" %
           (start, len(changed), totaltime, cloudsearch_time,
            len(changed) - len(things),
            msgs[-1].delivery_info.get('message_count', 'unknown')))

 def batch_lookups(self):
     try:
         self.things = Thing._by_fullname(self.fullnames, data=True, return_dict=False)
     except NotFound:
         if self.use_safe_get:
             self.things = safe_get(Thing._by_fullname, self.fullnames, data=True, return_dict=False)
         else:
             raise

def send_gift(buyer, recipient, months, days, signed, giftmessage,
              thing_fullname):
    admintools.engolden(recipient, days)

    if thing_fullname:
        thing = Thing._by_fullname(thing_fullname, data=True)
        thing._gild(buyer)
    else:
        thing = None

    if signed:
        sender = buyer.name
        md_sender = "[%s](/user/%s)" % (sender, sender)
    else:
        sender = _("An anonymous redditor")
        md_sender = _("An anonymous redditor")

    create_gift_gold(buyer._id, recipient._id, days, c.start_time, signed)

    if months == 1:
        amount = "a month"
    else:
        amount = "%d months" % months

    if not thing:
        subject = _(
            'Let there be gold! %s just sent you reddit gold!') % sender
        message = strings.youve_got_gold % dict(
            sender=md_sender, amount=amount)

        if giftmessage and giftmessage.strip():
            message += "\n\n" + strings.giftgold_note + giftmessage + '\n\n----'
    else:
        url = thing.make_permalink_slow()
        if isinstance(thing, Comment):
            subject = _('Your comment has been gilded!')
            message = strings.youve_been_gilded_comment % {'url': url}
        else:
            subject = _('Your submission has been gilded!')
            message = strings.youve_been_gilded_link % {'url': url}

    message += '\n\n' + strings.gold_benefits_msg
    if g.lounge_reddit:
        message += '\n* ' + strings.lounge_msg
    message = append_random_bottlecap_phrase(message)

    try:
        send_system_message(
            recipient, subject, message, distinguished='gold-auto')
    except MessageError:
        g.log.error('send_gift: could not send system message')

    g.log.info("%s gifted %s to %s" % (buyer.name, amount, recipient.name))
    return thing

    def process_message(msg):
		# msg is *PROBABLY* json
        timer = g.stats.get_timer("new_voting.%s" % queue)
        timer.start()

		# json being loaded into a python object
		# it has the fields "user_id", "thing_fullname"
		# a thing is a database object
		# it's a link, comment, post, whatever, everything can be upvoted/downvoted
        vote_data = json.loads(msg.body)
        hook = hooks.get_hook('vote.validate_vote_data')
        if hook.call_until_return(msg=msg, vote_data=vote_data) is False:
            # Corrupt records in the queue. Ignore them.
            print "Ignoring invalid vote by %s on %s %s" % (
                    vote_data.get('user_id', '<unknown>'),
                    vote_data.get('thing_fullname', '<unknown>'),
                    vote_data)
            return

		# this gets the user from database/cache (either memcached or postgres, whatever)
        user = Account._byID(vote_data.pop("user_id"), data=True)
        thing = Thing._by_fullname(vote_data.pop("thing_fullname"), data=True)

        timer.intermediate("preamble")

		# this gets a servers-wide lock
		# I mean, a bunch of consumers might be consuming items that use the same "thing" (same database object)
		# so, you want a global lock to avoid them from fucking eachother up 
		# memcachd stores the lock, atomically
        lock_key = "vote-%s-%s" % (user._id36, thing._fullname)
        with g.make_lock("voting", lock_key, timeout=5):
            print "Processing vote by %s on %s %s" % (user, thing, vote_data)

            try:
                vote = Vote(
                    user,
                    thing,
                    direction=vote_data["direction"],
                    date=datetime.utcfromtimestamp(vote_data["date"]),
                    data=vote_data["data"],
                    event_data=vote_data.get("event_data"),
                )
            except TypeError as e:
                # a vote on an invalid type got in the queue, just skip it
                g.log.exception("Invalid type: %r", e.message)
                return

            timer.intermediate("create_vote_obj")

            vote.commit()

            timer.flush()

def validate_blob(custom):
    """Validate payment_blob and return a dict with everything looked up."""
    ret = {}

    if not custom:
        raise GoldException('no custom')

    payment_blob = g.hardcache.get('payment_blob-%s' % str(custom))
    if not payment_blob:
        raise GoldException('no payment_blob')

    if 'account_id' in payment_blob and 'account_name' in payment_blob:
        try:
            buyer = Account._byID(payment_blob['account_id'], data=True)
            ret['buyer'] = buyer
        except NotFound:
            raise GoldException('bad account_id')

        if not buyer.name.lower() == payment_blob['account_name'].lower():
            raise GoldException('buyer mismatch')
    elif 'email' in payment_blob:
        ret['email'] = payment_blob['email']
    else:
        raise GoldException('no account_id or email')

    goldtype = payment_blob['goldtype']
    ret['goldtype'] = goldtype

    if goldtype == 'gift':
        recipient_name = payment_blob.get('recipient', None)
        if not recipient_name:
            raise GoldException('gift missing recpient')
        try:
            recipient = Account._by_name(recipient_name)
            ret['recipient'] = recipient
        except NotFound:
            raise GoldException('bad recipient')
        thing_fullname = payment_blob.get('thing', None)
        if thing_fullname:
            try:
                ret['thing'] = Thing._by_fullname(thing_fullname)
            except NotFound:
                raise GoldException('bad thing')
        ret['signed'] = payment_blob.get('signed', False)
        giftmessage = payment_blob.get('giftmessage')
        giftmessage = _force_unicode(giftmessage) if giftmessage else None
        ret['giftmessage'] = giftmessage
    elif goldtype not in ('onetime', 'autorenew', 'creddits', 'code'):
        raise GoldException('bad goldtype')

    return ret

def validate_blob(custom):
    """Validate payment_blob and return a dict with everything looked up."""
    ret = {}

    if not custom:
        raise GoldException("no custom")

    payment_blob = g.hardcache.get("payment_blob-%s" % str(custom))
    if not payment_blob:
        raise GoldException("no payment_blob")

    if not ("account_id" in payment_blob and "account_name" in payment_blob):
        raise GoldException("no account_id")

    try:
        buyer = Account._byID(payment_blob["account_id"], data=True)
        ret["buyer"] = buyer
    except NotFound:
        raise GoldException("bad account_id")

    if not buyer.name.lower() == payment_blob["account_name"].lower():
        raise GoldException("buyer mismatch")

    goldtype = payment_blob["goldtype"]
    ret["goldtype"] = goldtype

    if goldtype == "gift":
        recipient_name = payment_blob.get("recipient", None)
        if not recipient_name:
            raise GoldException("gift missing recpient")
        try:
            recipient = Account._by_name(recipient_name)
            ret["recipient"] = recipient
        except NotFound:
            raise GoldException("bad recipient")
        thing_fullname = payment_blob.get("thing", None)
        if thing_fullname:
            try:
                ret["thing"] = Thing._by_fullname(thing_fullname)
            except NotFound:
                raise GoldException("bad thing")
        ret["signed"] = payment_blob.get("signed", False)
        giftmessage = payment_blob.get("giftmessage")
        giftmessage = _force_unicode(giftmessage) if giftmessage else None
        ret["giftmessage"] = giftmessage
    elif goldtype not in ("onetime", "autorenew", "creddits", "code"):
        raise GoldException("bad goldtype")

    return ret

    def all_defendants(cls, quench=False, _update=False):
        all = cls.all_defendants_cache(_update=_update)

        defs = Thing._by_fullname(all, data=True).values()

        if quench:
            # Used for the spotlight, to filter out trials with over 20 votes;
            # otherwise, hung juries would hog the spotlight for an hour as
            # their vote counts continued to skyrocket

            return filter (lambda d:
                           not g.cache.get("quench_jurors-" + d._fullname),
                           defs)
        else:
            return defs

def get_recent_name_submissions():
    link_fullnames = list(queries.get_links(SERVERNAME_SR, "new", "all"))
    links = chain.from_iterable(Thing._by_fullname(chunk, return_dict=False)
                                for chunk in in_chunks(link_fullnames))

    for link in links:
        if link._deleted or link._spam:
            continue

        # OH GOD WHAT HAVE YOU POSTED IN MY LOVELY AUTOMATED SUBREDDIT!?
        if (not hasattr(link, "revenue_date") or
            not hasattr(link, "revenue_bucket") or
            not hasattr(link, "server_names")):
            continue

        yield link

    def from_queue(self, max_date, batch_limit = 50, kind = None):
        from r2.models import is_banned_IP, Account, Thing
        keep_trying = True
        min_id = None
        s = self.queue_table
        while keep_trying:
            where = [s.c.date < max_date]
            if min_id:
                where.append(s.c.uid > min_id)
            if kind:
                where.append(s.c.kind == kind)

            res = sa.select([s.c.to_addr, s.c.account_id,
                             s.c.from_name, s.c.fullname, s.c.body,
                             s.c.kind, s.c.ip, s.c.date, s.c.uid,
                             s.c.msg_hash, s.c.fr_addr, s.c.reply_to],
                            sa.and_(*where),
                            order_by = s.c.uid, limit = batch_limit).execute()
            res = res.fetchall()

            if not res: break

            # batch load user accounts
            aids = [x[1] for x in res if x[1] > 0]
            accts = Account._byID(aids, data = True,
                                  return_dict = True) if aids else {}

            # batch load things
            tids = [x[3] for x in res if x[3]]
            things = Thing._by_fullname(tids, data = True,
                                        return_dict = True) if tids else {}

            # make sure no IPs have been banned in the mean time
            ips = set(x[6] for x in res)
            ips = dict((ip, is_banned_IP(ip)) for ip in ips)

            # get the lower bound date for next iteration
            min_id = max(x[8] for x in res)

            # did we not fetch them all?
            keep_trying = (len(res) == batch_limit)

            for (addr, acct, fname, fulln, body, kind, ip, date, uid,
                 msg_hash, fr_addr, reply_to) in res:
                yield (accts.get(acct), things.get(fulln), addr,
                       fname, date, ip, ips[ip], kind, msg_hash, body,
                       fr_addr, reply_to)

    def add_props(cls, user, wrapped):
        user_fullnames = {w.user_fullname for w in wrapped}
        target_fullnames = {w.target_fullname for w in wrapped}

        users = Account._by_fullname(user_fullnames, data=True,
                                     return_dict=True)
        targets = Thing._by_fullname(target_fullnames, data=True,
                                     return_dict=True)

        author_ids = {t.author_id for t in targets.itervalues()
                      if hasattr(t, 'author_id')}
        link_ids = {t.link_id for t in targets.itervalues()
                    if hasattr(t, 'link_id')}
        sr_ids = {t.sr_id for t in targets.itervalues() if hasattr(t, 'sr_id')}

        authors = Account._byID(author_ids, data=True, return_dict=True)
        links = Link._byID(link_ids, data=True, return_dict=True)
        subreddits = Subreddit._byID(sr_ids, data=True, return_dict=True)

        target_things = {}
        for fullname, target in targets.iteritems():
            if isinstance(target, (Comment, Link)):
                author = authors[target.author_id]
                if isinstance(target, Link):
                    subreddit = subreddits[target.sr_id]
                    path = target.make_permalink(subreddit)
                else:
                    link = links[target.link_id]
                    subreddit = subreddits[link.sr_id]
                    path = target.make_permalink(link, subreddit)
                target_things[fullname] = GameLogTarget(target, path, author,
                                                        subreddit)
            elif isinstance(target, Account):
                target_things[fullname] = WrappedUser(target)

        for w in wrapped:
            w.is_self = (c.user_is_loggedin and
                         w.user_fullname == c.user._fullname)
            w.user = WrappedUser(users[w.user_fullname])
            w.target = target_things[w.target_fullname]
            w.item = g.f2pitems[w.item]
            w.user_team = scores.get_user_team(users[w.user_fullname])
            if isinstance(w.target, WrappedUser):
                target_user = targets[w.target.fullname]
            else:
                target_user = authors[targets[w.target_fullname].author_id]
            w.target_team = scores.get_user_team(target_user)

    def process_message(msg):
        timer = g.stats.get_timer("new_voting.%s" % queue)
        timer.start()

        vote_data = json.loads(msg.body)
        hook = hooks.get_hook('vote.validate_vote_data')
        if hook.call_until_return(msg=msg, vote_data=vote_data) is False:
            # Corrupt records in the queue. Ignore them.
            print "Ignoring invalid vote by %s on %s %s" % (
                    vote_data.get('user_id', '<unknown>'),
                    vote_data.get('thing_fullname', '<unknown>'),
                    vote_data)
            return

        # if it's an old-style vote, convert to the new format
        if "uid" in vote_data:
            vote_data = convert_old_vote_data(vote_data, msg.timestamp)

        user = Account._byID(vote_data.pop("user_id"), data=True)
        thing = Thing._by_fullname(vote_data.pop("thing_fullname"), data=True)

        timer.intermediate("preamble")

        lock_key = "vote-%s-%s" % (user._id36, thing._fullname)
        with g.make_lock("voting", lock_key, timeout=5):
            print "Processing vote by %s on %s %s" % (user, thing, vote_data)

            try:
                vote = Vote(
                    user,
                    thing,
                    direction=vote_data["direction"],
                    date=datetime.utcfromtimestamp(vote_data["date"]),
                    data=vote_data["data"],
                    event_data=vote_data.get("event_data"),
                )
            except TypeError as e:
                # a vote on an invalid type got in the queue, just skip it
                g.log.exception("Invalid type: %r", e.message)
                return

            timer.intermediate("create_vote_obj")

            vote.commit()

            timer.flush()

def send_gift(buyer, recipient, months, days, signed, giftmessage, comment_id):
    admintools.engolden(recipient, days)

    if comment_id:
        comment = Thing._by_fullname(comment_id, data=True)
        comment._gild(buyer)
    else:
        comment = None

    if signed:
        sender = buyer.name
        md_sender = "[%s](/user/%s)" % (sender, sender)
    else:
        sender = _("An anonymous redditor")
        md_sender = _("An anonymous redditor")

    create_gift_gold (buyer._id, recipient._id, days, c.start_time, signed)

    if months == 1:
        amount = "a month"
    else:
        amount = "%d months" % months

    if not comment:
        subject = _('Let there be gold! %s just sent you reddit gold!') % sender
        message = strings.youve_got_gold % dict(sender=md_sender, amount=amount)

        if giftmessage and giftmessage.strip():
            message += "\n\n" + strings.giftgold_note + giftmessage + '\n\n----'
    else:
        subject = _('Your comment has been gilded.')
        message = strings.youve_got_comment_gold % dict(
            url=comment.make_permalink_slow(),
        )

    message += '\n\n' + strings.gold_benefits_msg
    message += '\n\n' + strings.lounge_msg

    try:
        send_system_message(recipient, subject, message)
    except MessageError:
        g.log.error('send_gift: could not send system message')

    g.log.info("%s gifted %s to %s" % (buyer.name, amount, recipient.name))
    return comment

    def process_message(msg):
        if not ACCOUNT:
            return

        fullname = msg.body
        item = Thing._by_fullname(fullname, data=True)
        if not isinstance(item, (Link, Comment)):
            return

        subreddit = item.subreddit_slow

        wiki_page_id = wiki_id(subreddit._id36, "config/automoderator")
        wiki_page_fullname = "WikiPage_%s" % wiki_page_id
        last_edited = LastModified.get(wiki_page_fullname, "Edit")
        if not last_edited:
            return

        # initialize rules for the subreddit if we haven't already
        # or if the page has been edited since we last initialized
        need_to_init = False
        if subreddit._id not in rules_by_subreddit:
            need_to_init = True
        else:
            rules = rules_by_subreddit[subreddit._id]
            if last_edited > rules.init_time:
                need_to_init = True

        if need_to_init:
            wp = WikiPage.get(subreddit, "config/automoderator")
            rules = Ruleset(wp.content)
            rules_by_subreddit[subreddit._id] = rules

        if not rules:
            return

        try:
            TimeoutFunction(rules.apply_to_item, 2)(item)
            print "Checked %s from /r/%s" % (item, subreddit.name)
        except TimeoutFunctionException:
            print "Timed out on %s from /r/%s" % (item, subreddit.name)
        except KeyboardInterrupt:
            raise
        except:
            print "Error on %s from /r/%s" % (item, subreddit.name)
            print traceback.format_exc()

def update_trending_subreddits():
    try:
        trending_sr = Subreddit._by_name(g.config['trending_sr'])
    except NotFound:
        g.log.info("Unknown trending subreddit %r or trending_sr config "
                   "not set. Not updating.", g.config['trending_sr'])
        return

    iq = iter(trending_sr.get_links('new', 'all'))
    link = Thing._by_fullname(next(iq), data=True)
    subreddit_names = _SUBREDDIT_RE.findall(link.title)
    trending_data = {
        'subreddit_names': subreddit_names,
        'permalink': link.make_permalink(trending_sr),
        'link_id': link._id,
    }
    NamedGlobals.set(TRENDING_SUBREDDITS_KEY, trending_data)
    g.log.debug("Trending subreddit data set to %r", trending_data)

def send_gift(buyer, recipient, months, days, signed, giftmessage, comment_id):
    admintools.engolden(recipient, days)

    if comment_id:
        comment = Thing._by_fullname(comment_id, data=True)
        comment._gild(buyer)
    else:
        comment = None

    if signed:
        sender = buyer.name
        md_sender = "[%s](/user/%s)" % (sender, sender)
    else:
        sender = "someone"
        md_sender = "An anonymous lightnet user"

    create_gift_gold(buyer._id, recipient._id, days, c.start_time, signed)

    if months == 1:
        amount = "a month"
    else:
        amount = "%d months" % months

    if not comment:
        message = strings.youve_got_gold % dict(sender=md_sender, amount=amount)

        if giftmessage and giftmessage.strip():
            message += "\n\n" + strings.giftgold_note + giftmessage + "\n\n----"
    else:
        message = strings.youve_got_comment_gold % dict(url=comment.make_permalink_slow())

    message += "\n\n" + strings.gold_benefits_msg
    message += "\n\n" + strings.lounge_msg % {"link": "/space/" + g.lounge_reddit}

    subject = sender + " just sent you reddit gold!"

    try:
        send_system_message(recipient, subject, message)
    except MessageError:
        g.log.error("send_gift: could not send system message")

    g.log.info("%s gifted %s to %s" % (buyer.name, amount, recipient.name))
    return comment

def send_gift(buyer, recipient, months, days, signed, giftmessage, comment_id):
    admintools.engolden(recipient, days)

    if comment_id:
        comment = Thing._by_fullname(comment_id, data=True)
        comment._gild(buyer)
    else:
        comment = None

    if signed:
        sender = buyer.name
        md_sender = "[%s](/user/%s)" % (sender, sender)
    else:
        sender = "someone"
        md_sender = "An anonymous redditor"

    create_gift_gold (buyer._id, recipient._id, days, c.start_time, signed)

    if months == 1:
        amount = "a month"
    else:
        amount = "%d months" % months

    if not comment:
        message = strings.youve_got_gold % dict(sender=md_sender, amount=amount)

        if giftmessage and giftmessage.strip():
            message += "\n\n" + strings.giftgold_note + giftmessage
    else:
        message = strings.youve_got_comment_gold % dict(
            url=comment.make_permalink_slow(),
        )

    subject = sender + " just sent you reddit gold!"
    send_system_message(recipient, subject, message)

    g.log.info("%s gifted %s to %s" % (buyer.name, amount, recipient.name))
    return comment

def _run_changed(msgs, chan):
    '''Consume the cloudsearch_changes queue, and print reporting information
    on how long it took and how many remain
    
    '''
    start = datetime.now(g.tz)
    
    changed = [pickle.loads(msg.body) for msg in msgs]
    
    # Only handle links to start with
    
    fullnames = _desired_things(changed, (Link,))
    things = Thing._by_fullname(fullnames, data=True, return_dict=False)
    
    cloudsearch_time = inject(things)
    
    totaltime = (datetime.now(g.tz) - start).total_seconds()
    
    print ("%s: %d messages in %.2fs seconds (%.2fs secs waiting on "
           "cloudsearch); %d duplicates, %s remaining)" %
           (start, len(changed), totaltime, cloudsearch_time,
            len(changed) - len(things),
            msgs[-1].delivery_info.get('message_count', 'unknown')))

 def get_things(codes):
     """Fetch relevant things for a list of ad codenames in batch."""
     fullnames = [AdvertTrafficSummary.split_codename(code)[0]
                  for code in codes
                  if code.startswith(Thing._type_prefix)]
     return Thing._by_fullname(fullnames, data=True, return_dict=True)