def test_recent_helpful_votes(self):
        """Recent helpful votes are indexed properly."""
        # Create a document and verify it doesn't show up in a
        # query for recent_helpful_votes__gt=0.
        r = revision(is_approved=True, save=True)
        self.refresh()
        eq_(Document.search().filter(
            document_recent_helpful_votes__gt=0).count(), 0)

        # Add an unhelpful vote, it still shouldn't show up.
        helpful_vote(revision=r, helpful=False, save=True)
        r.document.save()  # Votes don't trigger a reindex.
        self.refresh()
        eq_(Document.search().filter(
            document_recent_helpful_votes__gt=0).count(), 0)

        # Add an helpful vote created 31 days ago, it still shouldn't show up.
        created = datetime.now() - timedelta(days=31)
        helpful_vote(revision=r, helpful=True, created=created, save=True)
        r.document.save()  # Votes don't trigger a reindex.
        self.refresh()
        eq_(Document.search().filter(
            document_recent_helpful_votes__gt=0).count(), 0)

        # Add an helpful vote created 29 days ago, it should show up now.
        created = datetime.now() - timedelta(days=29)
        helpful_vote(revision=r, helpful=True, created=created, save=True)
        r.document.save()  # Votes don't trigger a reindex.
        self.refresh()
        eq_(Document.search().filter(
            document_recent_helpful_votes__gt=0).count(), 1)

    def test_add_and_delete(self):
        """Adding a doc should add it to the search index; deleting should
        delete it."""
        doc = document(save=True)
        revision(document=doc, is_approved=True, save=True)
        self.refresh()
        eq_(Document.search().count(), 1)

        doc.delete()
        self.refresh()
        eq_(Document.search().count(), 0)

def document(save=False, **kwargs):
    """Return an empty document with enough stuff filled out that it can be
    saved."""
    defaults = {'category': CATEGORIES[0][0], 'title': str(datetime.now())}
    defaults.update(kwargs)
    if 'slug' not in kwargs:
        defaults['slug'] = slugify(defaults['title'])
    d = Document(**defaults)
    if save:
        d.save()
    return d

 def test_wiki_no_revisions(self):
     """Don't index documents without approved revisions"""
     # Create a document with no revisions and make sure the
     # document is not in the index.
     doc = document(save=True)
     self.refresh()
     eq_(Document.search().count(), 0)
     # Create a revision that's not approved and make sure the
     # document is still not in the index.
     revision(document=doc, is_approved=False, save=True)
     self.refresh()
     eq_(Document.search().count(), 0)

    def test_wiki_keywords(self):
        """Make sure updating keywords updates the index."""
        # Create a document with a revision with no keywords. It
        # shouldn't show up with a document_keywords term query for
        # 'wool' since it has no keywords.
        doc = document(title=u'wool hats')
        doc.save()
        revision(document=doc, is_approved=True, save=True)
        self.refresh()
        eq_(Document.search().query(document_keywords='wool').count(), 0)

        revision(document=doc, is_approved=True, keywords='wool', save=True)
        self.refresh()

        eq_(Document.search().query(document_keywords='wool').count(), 1)

    def test_wiki_redirects(self):
        """Make sure we don't index redirects"""
        # First create a revision that doesn't have a redirect and
        # make sure it's in the index.
        doc = document(title=u"wool hats")
        doc.save()
        revision(document=doc, is_approved=True, save=True)
        self.refresh()
        eq_(Document.search().query("wool").count(), 1)

        # Now create a revision that is a redirect and make sure the
        # document is removed from the index.
        revision(document=doc, content=REDIRECT_CONTENT, is_approved=True, save=True)
        self.refresh()
        eq_(Document.search().query("wool").count(), 0)

 def test_document_translate_fallback(self):
     d_en = document(locale='en-US',
                     title=u'How to delete Google Chrome?',
                     save=True)
     invalid_translate = reverse('wiki.document', locale='tr',
                                 args=[d_en.slug])
     self.assertEqual(d_en, Document.from_url(invalid_translate))

def _es_documents_for(locale, topics, products):
    """ES implementation of documents_for."""
    s = Document.search().values_dict(
        'id', 'document_title', 'url').filter(document_locale=locale)
    for topic in topics:
        s = s.filter(document_topic=topic.slug)
    for product in products or []:
        s = s.filter(document_product=product.slug)

    return list(s.order_by('-document_recent_helpful_votes')[:100])

    def test_translations_get_parent_tags(self):
        doc1 = document(title=u'Audio too loud')
        doc1.save()
        revision(document=doc1, is_approved=True, save=True)
        doc1.tags.add(u'desktop')
        doc1.tags.add(u'windows')

        doc2 = document(title=u'Audio too loud bork bork',
                        parent=doc1)
        doc2.save()
        revision(document=doc2, is_approved=True, save=True)
        doc2.tags.add(u'badtag')

        # Verify the parent has the right tags.
        doc_dict = Document.extract_document(doc1.id)
        eq_(doc_dict['tag'], [u'desktop', u'windows'])

        # Verify the translation has the parent's tags.
        doc_dict = Document.extract_document(doc2.id)
        eq_(doc_dict['tag'], [u'desktop', u'windows'])

def _postdoc(request, is_image):
    docpath = u'{0}/{1}'.format(
            settings.MEDIA_ROOT,
            is_image and u'images' or u'documents'
    )

    if not os.path.exists(docpath):
        os.mkdir(docpath)

    f = request.FILES[u'file']

    fd = open(u'{0}/{1}'.format(docpath, f.name), u'wb+')
    for chunk in f.chunks():
        fd.write(chunk)
    fd.close()

    url = u'{0}/{1}/{2}'.format(
            settings.MEDIA_URL,
            is_image and u'images' or u'documents',
            f.name)

    try:
        doc = Document.objects.get(path=url)

    except Document.DoesNotExist:
        doc = Document()

        doc.is_image = is_image
        doc.path = url

        doc.wikipath = request.POST[u'page']
        doc.save()

    return HttpResponse(doc.path)

def suggestions(request):
    """A simple search view that returns OpenSearch suggestions."""
    mimetype = 'application/x-suggestions+json'

    term = request.GET.get('q')
    if not term:
        return HttpResponseBadRequest(mimetype=mimetype)

    site = Site.objects.get_current()
    locale = locale_or_default(request.locale)
    try:
        query = dict(('%s__text' % field, term)
                     for field in Document.get_query_fields())
        wiki_s = (Document.search()
                  .filter(document_is_archived=False)
                  .filter(document_locale=locale)
                  .values_dict('document_title', 'url')
                  .query(or_=query)[:5])

        query = dict(('%s__text' % field, term)
                     for field in Question.get_query_fields())
        question_s = (Question.search()
                      .filter(question_has_helpful=True)
                      .values_dict('question_title', 'url')
                      .query(or_=query)[:5])

        results = list(chain(question_s, wiki_s))
    except (ESTimeoutError, ESMaxRetryError, ESException):
        # If we have ES problems, we just send back an empty result
        # set.
        results = []

    urlize = lambda r: u'https://%s%s' % (site, r['url'])
    titleize = lambda r: (r['document_title'] if 'document_title' in r
                          else r['question_title'])
    data = [term,
            [titleize(r) for r in results],
            [],
            [urlize(r) for r in results]]
    return HttpResponse(json.dumps(data), mimetype=mimetype)

    def test_wiki_tags(self):
        """Make sure that adding tags to a Document causes it to
        refresh the index.

        """
        tag = u'hiphop'
        eq_(Document.search().filter(tag=tag).count(), 0)
        doc = document(save=True)
        revision(document=doc, is_approved=True, save=True)
        self.refresh()
        eq_(Document.search().filter(tag=tag).count(), 0)
        doc.tags.add(tag)
        self.refresh()
        eq_(Document.search().filter(tag=tag).count(), 1)
        doc.tags.remove(tag)
        self.refresh()

        # Make sure the document itself is still there and that we didn't
        # accidentally delete it through screwed up signal handling:
        eq_(Document.search().filter().count(), 1)

        eq_(Document.search().filter(tag=tag).count(), 0)

    def test_translations_get_parent_tags(self):
        doc1 = document(title=u"Audio too loud")
        doc1.save()
        revision(document=doc1, is_approved=True, save=True)
        doc1.topics.add(topic(slug="cookies", save=True))
        doc1.topics.add(topic(slug="general", save=True))
        doc1.products.add(product(slug="desktop", save=True))

        doc2 = document(title=u"Audio too loud bork bork", parent=doc1)
        doc2.save()
        revision(document=doc2, is_approved=True, save=True)
        doc2.tags.add(u"badtag")

        # Verify the parent has the right tags.
        doc_dict = Document.extract_document(doc1.id)
        eq_(doc_dict["document_topic"], [u"cookies", u"general"])
        eq_(doc_dict["document_product"], [u"desktop"])

        # Verify the translation has the parent's tags.
        doc_dict = Document.extract_document(doc2.id)
        eq_(doc_dict["document_topic"], [u"cookies", u"general"])
        eq_(doc_dict["document_product"], [u"desktop"])

    def test_wiki_products(self):
        """Make sure that adding products to a Document causes it to
        refresh the index.

        """
        p = product(slug=u'desktop', save=True)
        eq_(Document.search().filter(product=p.slug).count(), 0)
        doc = document(save=True)
        revision(document=doc, is_approved=True, save=True)
        self.refresh()
        eq_(Document.search().filter(product=p.slug).count(), 0)
        doc.products.add(p)
        self.refresh()
        eq_(Document.search().filter(product=p.slug).count(), 1)
        doc.products.remove(p)
        self.refresh()

        # Make sure the document itself is still there and that we didn't
        # accidentally delete it through screwed up signal handling:
        eq_(Document.search().filter().count(), 1)

        eq_(Document.search().filter(product=p.slug).count(), 0)

    def test_wiki_topics(self):
        """Make sure that adding topics to a Document causes it to
        refresh the index.

        """
        t = topic(slug=u"hiphop", save=True)
        eq_(Document.search().filter(document_topic=t.slug).count(), 0)
        doc = document(save=True)
        revision(document=doc, is_approved=True, save=True)
        self.refresh()
        eq_(Document.search().filter(document_topic=t.slug).count(), 0)
        doc.topics.add(t)
        self.refresh()
        eq_(Document.search().filter(document_topic=t.slug).count(), 1)
        doc.topics.clear()
        self.refresh()

        # Make sure the document itself is still there and that we didn't
        # accidentally delete it through screwed up signal handling:
        eq_(Document.search().filter().count(), 1)

        eq_(Document.search().filter(document_topic=t.slug).count(), 0)

def _es_documents_for(locale, topics, products=None):
    """ES implementation of documents_for."""
    s = (Document.search()
        .values_dict('id', 'document_title', 'url', 'document_parent_id')
        .filter(document_locale=locale, document_is_archived=False,
                document_category__in=settings.IA_DEFAULT_CATEGORIES))

    for topic in topics:
        s = s.filter(document_topic=topic.slug)
    for product in products or []:
        s = s.filter(document_product=product.slug)

    return list(s.order_by('-document_recent_helpful_votes')[:100])

    def test_translations_get_parent_tags(self):
        doc1 = document(title=u'Audio too loud')
        doc1.save()
        revision(document=doc1, is_approved=True, save=True)
        doc1.topics.add(topic(slug='cookies', save=True))
        doc1.topics.add(topic(slug='general', save=True))
        doc1.products.add(product(slug='desktop', save=True))

        doc2 = document(title=u'Audio too loud bork bork',
                        parent=doc1)
        doc2.save()
        revision(document=doc2, is_approved=True, save=True)
        doc2.tags.add(u'badtag')

        # Verify the parent has the right tags.
        doc_dict = Document.extract_document(doc1.id)
        eq_(doc_dict['topic'], [u'cookies', u'general'])
        eq_(doc_dict['product'], [u'desktop'])

        # Verify the translation has the parent's tags.
        doc_dict = Document.extract_document(doc2.id)
        eq_(doc_dict['topic'], [u'cookies', u'general'])
        eq_(doc_dict['product'], [u'desktop'])

def troubleshooting_view(request):
    # Build a list of the most recently indexed 50 wiki documents.
    last_50_indexed = list(_fix_value_dicts(Document.search().values_dict().order_by("-indexed_on")[:50]))

    last_50_reviewed = list(
        Document.uncached.filter(current_revision__is_approved=True).order_by("-current_revision__reviewed")[:50]
    )

    diff_list = diff_it_for_realz(last_50_indexed, last_50_reviewed)

    return render_to_response(
        "search/admin/troubleshooting.html",
        {"title": "Index Troubleshooting", "diffs": diff_list},
        RequestContext(request, {}),
    )

def pageviews_by_document(start_date, end_date):
    """Return the number of pageviews by document in a given date range.

    * Only returns en-US documents for now since that's what we did with
    webtrends.

    Returns a dict with pageviews for each document:
        {<document_id>: <pageviews>,
         1: 42,
         7: 1337,...}
    """
    counts = {}
    request = _build_request()
    start_index = 1
    max_results = 10000

    while True:  # To deal with pagination

        @retry_503
        def _make_request():
            return request.get(
                ids='ga:' + profile_id,
                start_date=str(start_date),
                end_date=str(end_date),
                metrics='ga:pageviews',
                dimensions='ga:pagePath',
                filters='ga:pagePathLevel2==/kb/;ga:pagePathLevel1==/en-US/',
                max_results=max_results,
                start_index=start_index).execute()

        results = _make_request()

        for result in results['rows']:
            path = result[0]
            pageviews = int(result[1])
            doc = Document.from_url(path, id_only=True, check_host=False)
            if not doc:
                continue

            # The same document can appear multiple times due to url params.
            counts[doc.pk] = counts.get(doc.pk, 0) + pageviews

        # Move to next page of results.
        start_index += max_results
        if start_index > results['totalResults']:
            break

    return counts

def _create_document(title='Test Document'):
    d = Document(title=title, html='<div>Lorem Ipsum</div>',
                 category=1, locale='en-US')
    d.save()
    r = Revision(document=d, keywords='key1, key2', summary='lipsum',
                 content='<div>Lorem Ipsum</div>', creator_id=118577,
                 significance=SIGNIFICANCES[0][0])
    r.save()
    d.current_revision = r
    d.save()
    return d