def recompute_unread(min_date = None):
    from r2.models import Inbox, Account, Comment, Message
    from r2.lib.db import queries

    def load_accounts(inbox_rel):
        accounts = set()
        q = inbox_rel._query(eager_load = False, data = False,
                             sort = desc("_date"))
        if min_date:
            q._filter(inbox_rel.c._date > min_date)

        for i in fetch_things2(q):
            accounts.add(i._thing1_id)

        return accounts

    accounts_m = load_accounts(Inbox.rel(Account, Message))
    for i, a in enumerate(accounts_m):
        a = Account._byID(a)
        print "%s / %s : %s" % (i, len(accounts_m), a)
        queries.get_unread_messages(a).update()
        queries.get_unread_comments(a).update()
        queries.get_unread_selfreply(a).update()

    accounts = load_accounts(Inbox.rel(Account, Comment)) - accounts_m
    for i, a in enumerate(accounts):
        a = Account._byID(a)
        print "%s / %s : %s" % (i, len(accounts), a)
        queries.get_unread_comments(a).update()
        queries.get_unread_selfreply(a).update()

    def process_response(self):
        data = request.POST

        transaction_id = 'RG%s' % data['transaction_id']
        pennies = int(data['pennies'])
        months = int(data['months'])
        status = 'succeeded'

        goldtype = data['goldtype']
        buyer = Account._by_name(data['buyer'])

        if goldtype == 'gift':
            gift_kw = {
                'recipient': Account._by_name(data['recipient']),
                'giftmessage': _force_utf8(data.get('giftmessage', None)),
                'signed': data.get('signed') == 'True',
            }
        else:
            gift_kw = {}

        webhook = Webhook(
            transaction_id=transaction_id,
            pennies=pennies,
            months=months,
            goldtype=goldtype,
            buyer=buyer,
            **gift_kw)
        return status, webhook

    def get_reports(cls, wrapped, max_user_reasons=20):
        """Get two lists of mod and user reports on the item."""
        if (wrapped.reported > 0 and
                (wrapped.can_ban or
                 getattr(wrapped, "promoted", None) and c.user_is_sponsor)):
            from r2.models import SRMember

            reports = cls.for_thing(wrapped.lookups[0])

            q = SRMember._simple_query(
                ["_thing2_id", "_date"],
                SRMember.c._thing1_id == wrapped.sr_id,
                SRMember.c._name == "moderator",
            )
            mod_dates = {rel._thing2_id: rel._date for rel in q}

            if g.automoderator_account:
                automoderator = Account._by_name(g.automoderator_account)
            else:
                automoderator = None

            mod_reports = []
            user_reports = []

            for report in reports:
                # always include AutoModerator reports
                if automoderator and report._thing1_id == automoderator._id:
                    mod_reports.append(report)
                # include in mod reports if made after the user became a mod
                elif (report._thing1_id in mod_dates and
                        report._date >= mod_dates[report._thing1_id]):
                    mod_reports.append(report)
                else:
                    user_reports.append(report)

            # mod reports return as tuples with (reason, name)
            mods = Account._byID([report._thing1_id
                                  for report in mod_reports],
                                 data=True, return_dict=True)
            mod_reports = [(getattr(report, "reason", None),
                            mods[report._thing1_id].name)
                            for report in mod_reports]

            # user reports return as tuples with (reason, count)
            user_reports = Counter([getattr(report, "reason", None)
                                    for report in user_reports])
            user_reports = user_reports.most_common(max_user_reasons)

            return mod_reports, user_reports
        else:
            return [], []

def validate_blob(custom):
    """Validate payment_blob and return a dict with everything looked up."""
    ret = {}

    if not custom:
        raise GoldException('no custom')

    payment_blob = g.hardcache.get('payment_blob-%s' % str(custom))
    if not payment_blob:
        raise GoldException('no payment_blob')

    if 'account_id' in payment_blob and 'account_name' in payment_blob:
        try:
            buyer = Account._byID(payment_blob['account_id'], data=True)
            ret['buyer'] = buyer
        except NotFound:
            raise GoldException('bad account_id')

        if not buyer.name.lower() == payment_blob['account_name'].lower():
            raise GoldException('buyer mismatch')
    elif 'email' in payment_blob:
        ret['email'] = payment_blob['email']
    else:
        raise GoldException('no account_id or email')

    goldtype = payment_blob['goldtype']
    ret['goldtype'] = goldtype

    if goldtype == 'gift':
        recipient_name = payment_blob.get('recipient', None)
        if not recipient_name:
            raise GoldException('gift missing recpient')
        try:
            recipient = Account._by_name(recipient_name)
            ret['recipient'] = recipient
        except NotFound:
            raise GoldException('bad recipient')
        thing_fullname = payment_blob.get('thing', None)
        if thing_fullname:
            try:
                ret['thing'] = Thing._by_fullname(thing_fullname)
            except NotFound:
                raise GoldException('bad thing')
        ret['signed'] = payment_blob.get('signed', False)
        giftmessage = payment_blob.get('giftmessage')
        giftmessage = _force_unicode(giftmessage) if giftmessage else None
        ret['giftmessage'] = giftmessage
    elif goldtype not in ('onetime', 'autorenew', 'creddits', 'code'):
        raise GoldException('bad goldtype')

    return ret

def validate_blob(custom):
    """Validate payment_blob and return a dict with everything looked up."""
    ret = {}

    if not custom:
        raise GoldException("no custom")

    payment_blob = g.hardcache.get("payment_blob-%s" % str(custom))
    if not payment_blob:
        raise GoldException("no payment_blob")

    if not ("account_id" in payment_blob and "account_name" in payment_blob):
        raise GoldException("no account_id")

    try:
        buyer = Account._byID(payment_blob["account_id"], data=True)
        ret["buyer"] = buyer
    except NotFound:
        raise GoldException("bad account_id")

    if not buyer.name.lower() == payment_blob["account_name"].lower():
        raise GoldException("buyer mismatch")

    goldtype = payment_blob["goldtype"]
    ret["goldtype"] = goldtype

    if goldtype == "gift":
        recipient_name = payment_blob.get("recipient", None)
        if not recipient_name:
            raise GoldException("gift missing recpient")
        try:
            recipient = Account._by_name(recipient_name)
            ret["recipient"] = recipient
        except NotFound:
            raise GoldException("bad recipient")
        comment_fullname = payment_blob.get("comment", None)
        if comment_fullname:
            try:
                ret["comment"] = Comment._by_fullname(comment_fullname)
            except NotFound:
                raise GoldException("bad comment")
        ret["signed"] = payment_blob.get("signed", False)
        giftmessage = payment_blob.get("giftmessage")
        giftmessage = _force_unicode(giftmessage) if giftmessage else None
        ret["giftmessage"] = giftmessage
    elif goldtype not in ("onetime", "autorenew", "creddits"):
        raise GoldException("bad goldtype")

    return ret

def store_keys(key, maxes):
    # we're building queries using queries.py, but we could make the
    # queries ourselves if we wanted to avoid the individual lookups
    # for accounts and subreddits.

    # Note that we're only generating the 'sr-' type queries here, but
    # we're also able to process the other listings generated by the
    # old migrate.mr_permacache for convenience

    userrel_fns = dict(liked = queries.get_liked,
                       disliked = queries.get_disliked,
                       saved = queries.get_saved,
                       hidden = queries.get_hidden)

    if key.startswith('user-'):
        acc_str, keytype, account_id = key.split('-')
        account_id = int(account_id)
        fn = queries.get_submitted if keytype == 'submitted' else queries.get_comments
        q = fn(Account._byID(account_id), 'new', 'all')
        q._insert_tuples([(fname, float(timestamp))
                    for (timestamp, fname)
                    in maxes])

    elif key.startswith('sr-'):
        sr_str, sort, time, sr_id = key.split('-')
        sr_id = int(sr_id)

        if sort == 'controversy':
            # I screwed this up in the mapper and it's too late to fix
            # it
            sort = 'controversial'

        q = queries.get_links(Subreddit._byID(sr_id), sort, time)
        q._insert_tuples([tuple([item[-1]] + map(float, item[:-1]))
                    for item in maxes])
    elif key.startswith('domain/'):
        d_str, sort, time, domain = key.split('/')
        q = queries.get_domain_links(domain, sort, time)
        q._insert_tuples([tuple([item[-1]] + map(float, item[:-1]))
                    for item in maxes])


    elif key.split('-')[0] in userrel_fns:
        key_type, account_id = key.split('-')
        account_id = int(account_id)
        fn = userrel_fns[key_type]
        q = fn(Account._byID(account_id))
        q._insert_tuples([tuple([item[-1]] + map(float, item[:-1]))
                    for item in maxes])

    def process_response(self, res):
        from r2.models import Account

        fullname = res.merchantcustomerid.contents[0]
        name = res.description.contents[0]
        customer_id = int(res.customerprofileid.contents[0])
        acct = Account._by_name(name)

        # make sure we are updating the correct account!
        if acct.name == name:
            CustomerID.set(acct, customer_id)
        else:
            raise AuthorizeNetException, "account name doesn't match authorize.net account"

        # parse the ship-to list, and make sure the Account is up todate
        ship_to = []
        for profile in res.findAll("shiptolist"):
            a = Address.fromXML(profile)
            ShippingAddress.add(acct, a.customerAddressId)
            ship_to.append(a)

        # parse the payment profiles, and ditto
        profiles = []
        for profile in res.findAll("paymentprofiles"):
            a = Address.fromXML(profile)
            cc = CreditCard.fromXML(profile.payment)
            payprof = PaymentProfile(a, cc, int(a.customerPaymentProfileId))
            PayID.add(acct, a.customerPaymentProfileId)
            profiles.append(payprof)

        return acct, Profile(acct, profiles, ship_to)

    def process_response(self):
        data = request.POST

        transaction_id = 'RG%s' % data['transaction_id']
        pennies = int(data['pennies'])
        months = int(data['months'])
        status = 'succeeded'

        buyer_name = data['buyer']
        goldtype = data['goldtype']

        buyer = Account._by_name(buyer_name)

        blob = {
            'goldtype': goldtype,
            'account_id': buyer._id,
            'account_name': buyer.name,
            'status': 'initialized',
        }

        if goldtype == 'gift':
            blob['recipient'] = data['recipient']
            giftmessage = data.get('giftmessage', None)
            blob['giftmessage'] = _force_utf8(giftmessage)
            signed = data.get('signed')
            blob['signed'] = True if signed == 'True' else False

        passthrough = generate_blob(blob)

        return status, passthrough, transaction_id, pennies, months

def upgrade_messages(update_comments=True, update_messages=True, update_trees=True):
    from r2.lib.db import queries
    from r2.lib import comment_tree, cache
    from r2.models import Account
    from pylons import app_globals as g

    accounts = set()

    def batch_fn(items):
        g.reset_caches()
        return items

    if update_messages or update_trees:
        q = Message._query(Message.c.new == True, sort=desc("_date"), data=True)
        for m in fetch_things2(q, batch_fn=batch_fn):
            print m, m._date
            if update_messages:
                accounts = accounts | queries.set_unread(m, m.new)
            else:
                accounts.add(m.to_id)
    if update_comments:
        q = Comment._query(Comment.c.new == True, sort=desc("_date"))
        q._filter(Comment.c._id < 26152162676)

        for m in fetch_things2(q, batch_fn=batch_fn):
            print m, m._date
            queries.set_unread(m, True)

    print "Precomputing comment trees for %d accounts" % len(accounts)

    for i, a in enumerate(accounts):
        if not isinstance(a, Account):
            a = Account._byID(a)
        print i, a
        comment_tree.user_messages(a)

    def pre(self):
        set_extension(request.environ, "json")
        MinimalController.pre(self)
        require_https()

        try:
            access_token = OAuth2AccessToken.get_token(self._get_bearer_token())
            require(access_token)
            require(access_token.check_valid())
            c.oauth2_access_token = access_token
            account = Account._byID36(access_token.user_id, data=True)
            require(account)
            require(not account._deleted)
            c.oauth_user = account
        except RequirementException:
            self._auth_error(401, "invalid_token")

        handler = self._get_action_handler()
        if handler:
            oauth2_perms = getattr(handler, "oauth2_perms", None)
            if oauth2_perms:
                grant = OAuth2Scope(access_token.scope)
                if grant.subreddit_only and c.site.name not in grant.subreddits:
                    self._auth_error(403, "insufficient_scope")
                required_scopes = set(oauth2_perms['allowed_scopes'])
                if not (grant.scopes >= required_scopes):
                    self._auth_error(403, "insufficient_scope")
            else:
                self._auth_error(400, "invalid_request")

def _promo_email(thing, kind, body = "", **kw):
    from r2.lib.pages import Promo_Email
    a = Account._byID(thing.author_id, True)
    body = Promo_Email(link = thing, kind = kind,
                       body = body, **kw).render(style = "email")
    return _system_email(a.email, body, kind, thing = thing,
                         reply_to = "[email protected]")

    def _get_sr_restriction(sr):
        """Return a cloudsearch appropriate query string that restricts
        results to only contain results from self.sr
        
        """
        bq = []
        if (not sr) or sr == All or isinstance(sr, DefaultSR):
            return None
        elif isinstance(sr, MultiReddit):
            bq = ["(or"]
            for sr_id in sr.sr_ids:
                bq.append("sr_id:%s" % sr_id)
            bq.append(")")
        elif isinstance(sr, DomainSR):
            bq = ["site:'%s'" % sr.domain]
        elif sr == Friends:
            if not c.user_is_loggedin or not c.user.friends:
                return None
            bq = ["(or"]
            # The query limit is roughly 8k bytes. Limit to 200 friends to
            # avoid getting too close to that limit
            friend_ids = c.user.friends[:200]
            friends = ["author_fullname:'%s'" % Account._fullname_from_id36(r2utils.to36(id_)) for id_ in friend_ids]
            bq.extend(friends)
            bq.append(")")
        elif isinstance(sr, ModContribSR):
            bq = ["(or"]
            for sr_id in sr.sr_ids:
                bq.append("sr_id:%s" % sr_id)
            bq.append(")")
        elif not isinstance(sr, FakeSubreddit):
            bq = ["sr_id:%s" % sr._id]

        return " ".join(bq)

def update_karmas():
    for pair in to_update():
        user = Account._byID(pair[0], True)
        sr = Subreddit._byID(pair[1], True)

        print user.name, sr.name
        user.incr_karma('comment', sr, 20)

def edit_campaign(link, campaign, dates, bid, sr):
    sr_name = sr.name if sr else '' # empty string means target to all
    try:
        # if the bid amount changed, cancel any pending transactions
        if campaign.bid != bid:
            void_campaign(link, campaign)

        # update the schedule
        PromotionWeights.reschedule(link, campaign._id, sr_name,
                                    dates[0], dates[1], bid)

        # update values in the db
        campaign.update(dates[0], dates[1], bid, sr_name, campaign.trans_id, commit=True)

        # record the transaction
        text = 'updated campaign %s. (bid: %0.2f)' % (campaign._id, bid)
        PromotionLog.add(link, text)

        # make it a freebie, if applicable
        author = Account._byID(link.author_id, True)
        if getattr(author, "complimentary_promos", False):
            free_campaign(link, campaign, c.user)

    except Exception, e: # record error and rethrow 
        g.log.error("Failed to update PromoCampaign %s on link %d. Error was: %r" %
                    (campaign._id, link._id, e))
        try: # wrapped in try/except so orig error won't be lost if commit fails
            text = 'update FAILED. (campaign: %s, bid: %.2f)' % (campaign._id,
                                                                 bid)
            PromotionLog.add(link, text)
        except:
            pass
        raise e

def new_comment(comment, inbox_rels):
    author = Account._byID(comment.author_id)
    job = [get_comments(author, "new", "all")]
    if comment._deleted:
        job.append(get_all_comments())
        add_queries(job, delete_items=comment)
    else:
        # if comment._spam:
        #    sr = Subreddit._byID(comment.sr_id)
        #    job.append(get_spam_comments(sr))
        add_queries(job, insert_items=comment)
        amqp.add_item("new_comment", comment._fullname)
        if not g.amqp_host:
            l = Link._byID(comment.link_id, data=True)
            add_comment_tree(comment, l)

    # note that get_all_comments() is updated by the amqp process
    # r2.lib.db.queries.run_new_comments

    if inbox_rels:
        for inbox_rel in tup(inbox_rels):
            inbox_owner = inbox_rel._thing1
            if inbox_rel._name == "inbox":
                add_queries([get_inbox_comments(inbox_owner)], insert_items=inbox_rel)
            else:
                add_queries([get_inbox_selfreply(inbox_owner)], insert_items=inbox_rel)
            set_unread(comment, inbox_owner, True)

 def _get_sr_restriction(sr):
     '''Return a solr-appropriate query string that restricts
     results to only contain results from self.sr
     
     '''
     bq = []
     if (not sr) or sr == All or isinstance(sr, DefaultSR):
         return None
     elif isinstance(sr, MultiReddit):
         for sr_id in sr.sr_ids:
             bq.append("sr_id:%s" % sr_id)
     elif isinstance(sr, DomainSR):
         bq = ["site:'%s'" % sr.domain]
     elif sr == Friends:
         if not c.user_is_loggedin or not c.user.friends:
             return None
         friend_ids = c.user.friends
         friends = ["author_fullname:'%s'" %
                    Account._fullname_from_id36(r2utils.to36(id_))
                    for id_ in friend_ids]
         bq.extend(friends)
     elif isinstance(sr, ModContribSR):
         for sr_id in sr.sr_ids:
             bq.append("sr_id:%s" % sr_id)
     elif not isinstance(sr, FakeSubreddit):
         bq = ["sr_id:%s" % sr._id]
     return ' OR '.join(bq)

    def report(self):
        different = 0
        total = len(self.new_values)
        logged_keys = set()

        for account_id, pairs in self.new_values.iteritems():
            try:
                account = Account._byID(account_id, data=True)
            except NotFound:
                continue

            if self.migrate:
                for k, v in list(pairs.iteritems()):
                    _, dir, kind, sr = k.split('_')
                    old_total = getattr(account, '{0}_{1}_karma'.format(sr, kind), 0)
                    new_total = pairs['karma_ups_{0}_{1}'.format(kind, sr)] - \
                                pairs['karma_downs_{0}_{1}'.format(kind, sr)]
                    if old_total != new_total:
                        different += 1
                        if (account.name, kind, sr) not in logged_keys:
                            logged_keys.add((account.name, kind, sr))
                            print('{0}["{1}_{2}"] differs - old={3}, new={4}'.format(
                                account.name, kind, sr, old_total, new_total))
            else:
                for k, v in pairs.iteritems():
                    old_v = getattr(account, k, 0)
                    if v != old_v:
                        print('{0} differs - old={1}, new={2}'.format(k, old_v, v))

        print('{0} out of {1} values differed'.format(different, total))

    def _gift_using_creddits(self, recipient, months=1, thing_fullname=None, proxying_for=None):
        with creddits_lock(c.user):
            if not c.user.employee and c.user.gold_creddits < months:
                err = RedditError("INSUFFICIENT_CREDDITS")
                self.on_validation_error(err)

            note = None
            buyer = c.user
            if c.user.name.lower() in g.live_config["proxy_gilding_accounts"]:
                note = "proxy-%s" % c.user.name
                if proxying_for:
                    try:
                        buyer = Account._by_name(proxying_for)
                    except NotFound:
                        pass

            send_gift(
                buyer=buyer,
                recipient=recipient,
                months=months,
                days=months * 31,
                signed=False,
                giftmessage=None,
                thing_fullname=thing_fullname,
                note=note,
            )

            if not c.user.employee:
                c.user.gold_creddits -= months
                c.user._commit()

    def authenticate_with_token(self):
        set_extension(request.environ, "json")
        set_content_type()
        require_https()
        require_domain(g.oauth_domain)

        try:
            access_token = OAuth2AccessToken.get_token(self._get_bearer_token())
            require(access_token)
            require(access_token.check_valid())
            c.oauth2_access_token = access_token
            account = Account._byID36(access_token.user_id, data=True)
            require(account)
            require(not account._deleted)
            c.oauth_user = account
        except RequirementException:
            self._auth_error(401, "invalid_token")

        handler = self._get_action_handler()
        if handler:
            oauth2_perms = getattr(handler, "oauth2_perms", None)
            if oauth2_perms or True:
                grant = OAuth2Scope(access_token.scope)
                required = set(oauth2_perms['allowed_scopes'])
                if not grant.has_access(c.site.name, required):
                    self._auth_error(403, "insufficient_scope")
                c.oauth_scope = grant
            else:
                self._auth_error(400, "invalid_request")

def charge_campaign(link, campaign):
    if charged_or_not_needed(campaign):
        return

    user = Account._byID(link.author_id)
    success, reason = authorize.charge_transaction(user, campaign.trans_id,
                                                   campaign._id)

    if not success:
        if reason == authorize.TRANSACTION_NOT_FOUND:
            # authorization hold has expired
            original_trans_id = campaign.trans_id
            campaign.trans_id = NO_TRANSACTION
            campaign._commit()
            text = ('voided expired transaction for %s: (trans_id: %d)'
                    % (campaign, original_trans_id))
            PromotionLog.add(link, text)
        return

    hooks.get_hook('promote.edit_campaign').call(link=link, campaign=campaign)

    if not is_promoted(link):
        update_promote_status(link, PROMOTE_STATUS.pending)

    emailer.queue_promo(link, campaign.bid, campaign.trans_id)
    text = ('auth charge for campaign %s, trans_id: %d' %
            (campaign._id, campaign.trans_id))
    PromotionLog.add(link, text)