def get_deleted_comments(user_id):
    return Comment._query(
        Comment.c.author_id == user_id,
        Comment.c._deleted == True,
        Comment.c._spam == (True, False),
        sort=db_sort("new"),
    )

    def get_all_comments(self):
        from r2.lib.db import queries
        from r2.models import Comment
        from r2.controllers.errors import UserRequiredException

        if not c.user_is_loggedin:
            raise UserRequiredException

        friends = self.get_important_friends(c.user._id)

        if not friends:
            return []

        if g.use_query_cache:
            # with the precomputer enabled, this Subreddit only supports
            # being sorted by 'new'. it would be nice to have a
            # cleaner UI than just blatantly ignoring their sort,
            # though
            sort = "new"
            time = "all"

            friends = Account._byID(friends, return_dict=False)

            crs = [queries.get_comments(friend, sort, time) for friend in friends]
            return queries.MergedCachedResults(crs)

        else:
            q = Comment._query(Comment.c.author_id == friends, sort=desc("_date"), data=True)
            return q

    def process_comment(self, comment_data, comment, post):
        # Prepare data for import
        ip = '127.0.0.1'
        if comment_data:
            naive_date = datetime.datetime.strptime(comment_data['dateCreated'], DATE_FORMAT)
            local_date = INPUT_TIMEZONE.localize(naive_date, is_dst=False) # Pick the non daylight savings time
            utc_date = local_date.astimezone(pytz.utc)

            # Determine account to use for this comment
            account = self._get_or_create_account(comment_data['author'], comment_data['authorEmail'])

        if comment_data and not comment:
            # Create new comment
            comment, inbox_rel = Comment._new(account, post, None, comment_data['body'], ip, date=utc_date)
            comment.is_html = True
            comment.ob_imported = True
            comment._commit()
        elif comment_data and comment:
            # Overwrite existing comment
            comment.author_id = account._id
            comment.body = comment_data['body']
            comment.ip = ip
            comment._date = utc_date
            comment.is_html = True
            comment.ob_imported = True
            comment._commit()
        elif not comment_data and comment:
            # Not enough comment data being imported to overwrite all comments
            print 'WARNING: More comments in lesswrong than we are importing, ignoring additional comment in lesswrong'

def process_comments_on_post(post, comments):
    for comment in comments:
        if comment_exists(post, comment):
            continue

        # Prepare data for import
        ip = '127.0.0.1'
        naive_date = datetime.datetime.strptime(comment['dateCreated'], DATE_FORMAT)
        local_date = INPUT_TIMEZONE.localize(naive_date, is_dst=False) # Pick the non daylight savings time
        utc_date = local_date.astimezone(pytz.utc)

        # Determine account to use for this comment
        account = get_or_create_account(comment['author'])

        if not dryrun:
            # Create new comment
            new_comment, inbox_rel = Comment._new(account, post, None, comment['body'], ip, date=utc_date)
            new_comment.is_html = True
            new_comment.ob_imported = True
            new_comment._commit()

        try:
            print " Imported as '%s' %s" % (account.name.decode('utf-8').encode('utf-8'), comment_excerpt(comment).decode('utf-8').encode('utf-8'))
        except UnicodeError:
            print " Imported comment"

def activate_names_requested_in(link):
    tree = get_comment_tree(link)
    acceptable_names = []
    if tree.tree:
        top_level_cids = tree.tree[None]
        comments = chain.from_iterable(Comment._byID(chunk, return_dict=False,
                                                     data=True)
                                       for chunk in in_chunks(top_level_cids))

        for comment in sorted(comments, key=lambda c: c._ups, reverse=True):
            if comment._spam or comment._deleted:
                continue

            sanitized = comment.body.strip()
            match = valid_name_re.search(sanitized)
            if match:
                acceptable_names.append((comment, match.group(1)))

    # we activate one name for each 100% of rev goal met
    names = acceptable_names[:link.revenue_bucket]
    activate_names(link, names)

    activated_names = [name for comment, name in names]
    link.server_names = activated_names
    link.flair_text = ", ".join(activated_names) if names else "/dev/null"
    link.flair_css_class = "goal-bucket-%d" % link.revenue_bucket
    link._commit()

    def process_comment(self, comment_data, comment, post, comment_dictionary):
        # Prepare data for import
        ip = '127.0.0.1'
        if comment_data:
            naive_date = datetime.datetime.strptime(comment_data['dateCreated'], DATE_FORMAT)
            local_date = INPUT_TIMEZONE.localize(naive_date, is_dst=False) # Pick the non daylight savings time
            utc_date = local_date.astimezone(pytz.utc)

            # Determine account to use for this comment
            account = self._get_or_create_account(comment_data['author'], comment_data['authorEmail'])

        if comment_data and not comment_data['author'].endswith("| The Effective Altruism Blog"):
            if not comment:
                # Create new comment
                comment, inbox_rel = Comment._new(account, post, None, comment_data['body'], ip, date=utc_date)
                if str(comment_data['commentParent']) in comment_dictionary:
                    comment.parent_id = comment_dictionary[str(comment_data['commentParent'])]
                comment.is_html = True
                comment.ob_imported = True
                comment._commit()
                comment_dictionary[str(comment_data['commentId'])] = comment._id
            else:
                # Overwrite existing comment
                if str(comment_data['commentParent']) in comment_dictionary:
                    comment.parent_id = comment_dictionary[str(comment_data['commentParent'])]
                comment.author_id = account._id
                comment.body = comment_data['body']
                comment.ip = ip
                comment._date = utc_date
                comment.is_html = True
                comment.ob_imported = True
                comment._commit()
                comment_dictionary[str(comment_data['commentId'])] = comment._id

    def _mock_comment(id=1, author_id=1, link_id=1, sr_id=1, **kwargs):
        kwargs['id'] = id
        kwargs['author_id'] = author_id
        kwargs['link_id'] = link_id

        comment = Comment(**kwargs)
        VByName.run = MagicMock(return_value=comment)

        link = Link(id=link_id)
        Link._byID = MagicMock(return_value=link)

        sr = Subreddit(id=sr_id)
        comment.subreddit_slow = MagicMock(return_value=sr)
        comment.subreddit_slow.is_moderator = MagicMock(return_value=False)
        link.subreddit = sr

        return comment

    def _run_new_comment(msg):
        fname = msg.body
        comment = Comment._by_fullname(fname,data=True)
        sr = Subreddit._byID(comment.sr_id)

        add_queries([get_all_comments(),
                     get_sr_comments(sr)],
                    insert_items = [comment])

    def _run_new_comments(msgs, chan):
        fnames = [msg.body for msg in msgs]

        comments = Comment._by_fullname(fnames, data=True, return_dict=False)
        add_queries([get_all_comments()], insert_items=comments)

        bysrid = _by_srid(comments, False)
        for srid, sr_comments in bysrid.iteritems():
            add_queries([_get_sr_comments(srid)], insert_items=sr_comments)

 def post_process_post(self, post):
     """Perform post processsing to rewrite URLs and generate mapping
        between old and new permalinks"""
     post.article = self.rewrite_ob_urls(post.article)
     post._commit()
     
     comments = Comment._query(Comment.c.link_id == post._id, data = True)
     for comment in comments:
         comment.body = self.rewrite_ob_urls(comment.body)
         comment._commit()

def comment_reply_effect(comment):
    if comment.parent_id is not None:
        parent = Comment._byID(comment.parent_id, data=True)
    else:
        parent = Link._byID(comment.link_id, data=True)
    all_effects = effects.get_all_effects([parent._fullname])
    parent_effects = all_effects.get(parent._fullname, [])
    for item_name in parent_effects:
        item = items.get_item(item_name)
        item.on_reply(c.user, parent)

def _populate(after_id=None, estimate=54301242):
    from r2.models import Comment, CommentSortsCache, desc
    from r2.lib.db import tdb_cassandra
    from r2.lib import utils

    # larger has a chance to decrease the number of Cassandra writes,
    # but the probability is low
    chunk_size = 5000

    q = Comment._query(Comment.c._spam == (True, False), Comment.c._deleted == (True, False), sort=desc("_date"))

    if after_id is not None:
        q._after(Comment._byID(after_id))

    q = utils.fetch_things2(q, chunk_size=chunk_size)
    q = utils.progress(q, verbosity=chunk_size, estimate=estimate)

    for chunk in utils.in_chunks(q, chunk_size):
        chunk = filter(lambda x: hasattr(x, "link_id"), chunk)
        update_comment_votes(chunk, write_consistency_level=tdb_cassandra.CL.ONE)

    def perform_actions(self, item, data):
        """Execute all the rule's actions against the item."""
        for key, target in self.targets.iteritems():
            target_item = self.get_target_item(item, data, key)
            target.perform_actions(target_item, data)

        if self.comment:
            comment = self.build_message(self.comment, item, data, disclaimer=True)

            # TODO: shouldn't have to do all this manually
            if isinstance(item, Comment):
                link = data["link"]
                parent_comment = item
            else:
                link = item
                parent_comment = None
            new_comment, inbox_rel = Comment._new(
                ACCOUNT, link, parent_comment, comment, None)
            new_comment.distinguished = "yes"
            new_comment._commit()
            queries.queue_vote(ACCOUNT, new_comment, True, None)
            queries.new_comment(new_comment, inbox_rel)

            g.stats.simple_event("automoderator.comment")

        if self.modmail:
            message = self.build_message(self.modmail, item, data, permalink=True)
            subject = replace_placeholders(
                self.modmail_subject, data, self.matches)
            subject = subject[:100]

            new_message, inbox_rel = Message._new(ACCOUNT, data["subreddit"],
                subject, message, None)
            new_message.distinguished = "yes"
            new_message._commit()
            queries.new_message(new_message, inbox_rel)

            g.stats.simple_event("automoderator.modmail")

        if self.message and not data["author"]._deleted:
            message = self.build_message(self.message, item, data,
                disclaimer=True, permalink=True)
            subject = replace_placeholders(
                self.message_subject, data, self.matches)
            subject = subject[:100]

            new_message, inbox_rel = Message._new(ACCOUNT, data["author"],
                subject, message, None)
            queries.new_message(new_message, inbox_rel)

            g.stats.simple_event("automoderator.message")

        PerformedRulesByThing.mark_performed(item, self)

    def _mock_comment(id=1, author_id=1, link_id=1, sr_id=1, can_comment=True,
                      can_view_promo=True, is_moderator=False, **kwargs):
        kwargs['id'] = id
        kwargs['author_id'] = author_id
        kwargs['link_id'] = link_id

        comment = Comment(**kwargs)
        VByName.run = MagicMock(return_value=comment)

        link = Link(id=link_id)
        Link._byID = MagicMock(return_value=link)

        sr = Subreddit(id=sr_id)
        comment.subreddit_slow = sr
        link.subreddit_slow = sr

        Subreddit.can_comment = MagicMock(return_value=can_comment)
        Link.can_view_promo = MagicMock(return_value=can_view_promo)
        Subreddit.is_moderator = MagicMock(return_value=is_moderator)

        return comment

def validate_blob(custom):
    """Validate payment_blob and return a dict with everything looked up."""
    ret = {}

    if not custom:
        raise GoldException('no custom')

    payment_blob = g.hardcache.get('payment_blob-%s' % str(custom))
    if not payment_blob:
        raise GoldException('no payment_blob')

    if not ('account_id' in payment_blob and
            'account_name' in payment_blob):
        raise GoldException('no account_id')

    try:
        buyer = Account._byID(payment_blob['account_id'], data=True)
        ret['buyer'] = buyer
    except NotFound:
        raise GoldException('bad account_id')

    if not buyer.name.lower() == payment_blob['account_name'].lower():
        raise GoldException('buyer mismatch')

    goldtype = payment_blob['goldtype']
    ret['goldtype'] = goldtype

    if goldtype == 'gift':
        recipient_name = payment_blob.get('recipient', None)
        if not recipient_name:
            raise GoldException('gift missing recpient')
        try:
            recipient = Account._by_name(recipient_name)
            ret['recipient'] = recipient
        except NotFound:
            raise GoldException('bad recipient')
        comment_fullname = payment_blob.get('comment', None)
        if comment_fullname:
            try:
                ret['comment'] = Comment._by_fullname(comment_fullname)
            except NotFound:
                raise GoldException('bad comment')
        ret['signed'] = payment_blob.get('signed', False)
        giftmessage = payment_blob.get('giftmessage')
        giftmessage = _force_unicode(giftmessage) if giftmessage else None
        ret['giftmessage'] = giftmessage
    elif goldtype not in ('onetime', 'autorenew', 'creddits'):
        raise GoldException('bad goldtype')

    return ret

def validate_blob(custom):
    """Validate payment_blob and return a dict with everything looked up."""
    ret = {}

    if not custom:
        raise GoldException("no custom")

    payment_blob = g.hardcache.get("payment_blob-%s" % str(custom))
    if not payment_blob:
        raise GoldException("no payment_blob")

    if not ("account_id" in payment_blob and "account_name" in payment_blob):
        raise GoldException("no account_id")

    try:
        buyer = Account._byID(payment_blob["account_id"], data=True)
        ret["buyer"] = buyer
    except NotFound:
        raise GoldException("bad account_id")

    if not buyer.name.lower() == payment_blob["account_name"].lower():
        raise GoldException("buyer mismatch")

    goldtype = payment_blob["goldtype"]
    ret["goldtype"] = goldtype

    if goldtype == "gift":
        recipient_name = payment_blob.get("recipient", None)
        if not recipient_name:
            raise GoldException("gift missing recpient")
        try:
            recipient = Account._by_name(recipient_name)
            ret["recipient"] = recipient
        except NotFound:
            raise GoldException("bad recipient")
        comment_fullname = payment_blob.get("comment", None)
        if comment_fullname:
            try:
                ret["comment"] = Comment._by_fullname(comment_fullname)
            except NotFound:
                raise GoldException("bad comment")
        ret["signed"] = payment_blob.get("signed", False)
        giftmessage = payment_blob.get("giftmessage")
        giftmessage = _force_unicode(giftmessage) if giftmessage else None
        ret["giftmessage"] = giftmessage
    elif goldtype not in ("onetime", "autorenew", "creddits"):
        raise GoldException("bad goldtype")

    return ret

    def _run_commentstree(msg):
        fname = msg.body
        comment = Comment._by_fullname(fname, data=True)

        link = Link._byID(comment.link_id,
                          data=True)

        try:
            add_comment_tree(comment, link)
        except KeyError:
            # Hackity hack. Try to recover from a corrupted comment
            # tree
            print "Trying to fix broken comments-tree."
            link_comments(link._id, _update=True)
            add_comment_tree(comment, link)

def activate_names(link, names):
    for comment, name in names:
        # find a slot to assign a name to. we'll prefer nodes that are
        # currently empty, and failing that find the least-recently-modified
        # node.
        ROOT = "/gold/server-names"
        slot_names = g.zookeeper.get_children(ROOT)
        slots = [(slot_name, g.zookeeper.get(os.path.join(ROOT, slot_name)))
                 for slot_name in slot_names]
        slots.sort(key=lambda (path, (data, stat)): (bool(data), stat.mtime))
        slot_path = os.path.join(ROOT, slots[0][0])

        comment_data = {'name': str(name),
                        'permalink': comment.make_permalink_slow()}
        g.zookeeper.set(slot_path, json.dumps(comment_data))

        lock = g.zookeeper.Lock(slot_path)
        lock_contenders = lock.contenders()
        old_name = lock_contenders[0] if lock_contenders else ""
        old_name = old_name or "one of our servers"

        # reply to the user
        wp = WikiPage.get(SERVERNAME_SR, "templates/success-reply")
        template = random.choice(wp._get("content").split("\r\n---\r\n"))
        comment, inbox_rel = Comment._new(
            author=SYSTEM_ACCOUNT,
            link=link,
            parent=comment,
            body=template % {
                "old-name": old_name,
                "new-name": name,
            },
            ip="127.0.0.1",
        )
        queries.queue_vote(SYSTEM_ACCOUNT, comment, dir=True, ip="127.0.0.1")
        queries.new_comment(comment, inbox_rel)

        # update the link's text
        wp = WikiPage.get(SERVERNAME_SR, "templates/goldisms")
        goldism = random.choice(wp._get("content").split("\r\n---\r\n"))
        wp = WikiPage.get(SERVERNAME_SR, "templates/selftext-success")
        template = wp._get("content")
        link.selftext = template % {
            "old-name": old_name,
            "new-name": name,
            "goldism": goldism,
        }
        link._commit()

    def _run_commentstree(msgs, chan):
        fnames = [msg.body for msg in msgs]
        comments = Comment._by_fullname(fnames, data=True, return_dict=False)

        links = Link._byID(set(cm.link_id for cm in comments), data=True, return_dict=True)

        # add the comment to the comments-tree
        for comment in comments:
            l = links[comment.link_id]
            try:
                add_comment_tree(comment, l)
            except KeyError:
                # Hackity hack. Try to recover from a corrupted
                # comment tree
                print "Trying to fix broken comments-tree."
                link_comments(l._id, _update=True)
                add_comment_tree(comment, l)

def comment_exists(post, comment):
    # Check if this comment already exists using brutal compare on content
    # BeautifulSoup is used to parse as HTML in order to remove markup
    content = ''.join(BeautifulSoup(comment['body']).findAll(text=True))
    key = re_non_alphanum.sub('', content)
    existing_comments = Comment._query(Comment.c.link_id == post._id, Comment.c.ob_imported == True, data=True)
    for existing_comment in existing_comments:
        author = Account._byID(existing_comment.author_id, data=True)
        content = ''.join(BeautifulSoup(existing_comment.body).findAll(text=True))
        existing_key = re_non_alphanum.sub('', content)
        if key == existing_key:
            print " Skipping existing %s" % comment_excerpt(comment)
            return True
        # else:
        #     print "%s *|NOT|* %s" % (key, existing_key)

    return False