def setUp(self):
        self.feed = self.feed_factory()
        self.data_es = [
            feed_item.get_indexer().extract_document(None, obj=feed_item)
            for feed_item in self.feed]

        # Denormalize feed elements into the serializer context.
        self.app_map = {}
        self.feed_element_map = defaultdict(dict)
        for i, feed_item in enumerate(self.data_es):
            feed_element = getattr(self.feed[i], feed_item['item_type'])
            self.feed_element_map[feed_item['item_type']][feed_element.id] = (
                feed_element.get_indexer().extract_document(None,
                                                            obj=feed_element))

            # Denormalize apps into serializer context.
            if hasattr(feed_element, 'apps'):
                for app in feed_element.apps():
                    self.app_map[app.id] = WebappIndexer.extract_document(
                        None, obj=app)
            else:
                self.app_map[feed_element.app_id] = (
                    WebappIndexer.extract_document(feed_element.app_id))
        self.context = {
            'app_map': self.app_map,
            'feed_element_map': self.feed_element_map,
            'request': mkt.site.tests.req_factory_factory('')
        }

 def tearDown(self):
     # Cleanup to remove these from the index.
     self.app1.delete()
     self.app2.delete()
     unindex_webapps([self.app1.id, self.app2.id])
     # Required to purge the suggestions data structure. In Lucene, a
     # document is not deleted from a segment, just marked as deleted.
     WebappIndexer.get_es().indices.optimize(index=WebappIndexer.get_index(), only_expunge_deletes=True)

def index_webapps(ids, **kw):
    # DEPRECATED: call WebappIndexer.index_ids directly.
    homescreens = set(
        Webapp.tags.through.objects.filter(
            webapp_id__in=ids,
            tag__tag_text='homescreen')
        .values_list('webapp_id', flat=True))
    webapps = set(ids) - homescreens
    if homescreens:
        HomescreenIndexer.index_ids(list(homescreens), no_delay=True)
    if webapps:
        WebappIndexer.index_ids(list(webapps), no_delay=True)

 def mget_apps(self, app_ids):
     """
     Takes a list of app_ids. Does an ES mget.
     Returns an app_map for serializer context.
     """
     app_map = {}
     es = WebappIndexer.get_es()
     apps = es.mget(body={'ids': app_ids}, index=WebappIndexer.get_index(),
                    doc_type=WebappIndexer.get_mapping_type_name())
     for app in apps['docs']:
         # Store the apps to attach to feed elements later.
         app = app['_source']
         app_map[app['id']] = app
     return app_map

    def get_updates_queue(self):
        # Updated apps, i.e. apps that have been published but have new
        # unreviewed versions, go in this queue.
        if self.use_es:
            must = [
                es_filter.Terms(status=mkt.WEBAPPS_APPROVED_STATUSES),
                es_filter.Term(**{'latest_version.status':
                                  mkt.STATUS_PENDING}),
                es_filter.Terms(app_type=[mkt.ADDON_WEBAPP_PACKAGED,
                                          mkt.ADDON_WEBAPP_PRIVILEGED]),
                es_filter.Term(is_disabled=False),
                es_filter.Term(is_escalated=False),
            ]
            return WebappIndexer.search().filter('bool', must=must)

        return (Version.objects.filter(
            # Note: this will work as long as we disable files of existing
            # unreviewed versions when a new version is uploaded.
            files__status=mkt.STATUS_PENDING,
            addon__disabled_by_user=False,
            addon__is_packaged=True,
            addon__status__in=mkt.WEBAPPS_APPROVED_STATUSES)
            .exclude(addon__id__in=self.excluded_ids)
            .exclude(addon__tags__tag_text='homescreen')
            .order_by('nomination', 'created')
            .select_related('addon', 'files').no_transforms())

def index_webapps(ids, **kw):
    """TODO: use search/indexers.py:index."""
    task_log.info('Indexing apps %s-%s. [%s]' % (ids[0], ids[-1], len(ids)))

    index = kw.pop('index', WebappIndexer.get_index())
    # Note: If reindexing is currently occurring, `get_indices` will return
    # more than one index.
    indices = Reindexing.get_indices(index)

    es = WebappIndexer.get_es(urls=settings.ES_URLS)
    qs = Webapp.indexing_transformer(Webapp.with_deleted.no_cache().filter(
        id__in=ids))
    for obj in qs:
        doc = WebappIndexer.extract_document(obj.id, obj)
        for idx in indices:
            WebappIndexer.index(doc, id_=obj.id, es=es, index=idx)

 def test_mapping_properties(self):
     # Spot check a few of the key properties.
     mapping = WebappIndexer.get_mapping()
     keys = mapping['webapp']['properties'].keys()
     for k in ('id', 'app_slug', 'category', 'default_locale',
               'description', 'device', 'features', 'name', 'status'):
         ok_(k in keys, 'Key %s not found in mapping properties' % k)

def app_search(request):
    results = []
    q = request.GET.get("q", u"").lower().strip()
    limit = lkp.MAX_RESULTS if request.GET.get("all_results") else lkp.SEARCH_LIMIT
    fields = ("name", "app_slug")
    non_es_fields = ["id", "name__localized_string"] + list(fields)

    if q.isnumeric():
        qs = Webapp.objects.filter(pk=q).values(*non_es_fields)[:limit]
    else:
        # Try to load by GUID:
        qs = Webapp.objects.filter(guid=q).values(*non_es_fields)[:limit]
        if not qs.count():
            qs = WebappIndexer.search().query(_expand_query(q, fields))[:limit]
            # TODO: Update to `.fields(...)` when the DSL supports it.
            qs = qs.execute()
    for app in qs:
        if isinstance(app, dict):
            # This is a result from the database.
            app["url"] = reverse("lookup.app_summary", args=[app["id"]])
            app["name"] = app["name__localized_string"]
            results.append(app)
        else:
            # This is a result from elasticsearch which returns `Result`
            # objects and name as a list, one for each locale.
            for name in app.name:
                results.append(
                    {
                        "id": app.id,
                        "url": reverse("lookup.app_summary", args=[app.id]),
                        "app_slug": app.get("app_slug"),
                        "name": name,
                    }
                )
    return {"results": results}

    def get_apps(self, request, app_ids):
        """
        Takes a list of app_ids. Gets the apps, including filters.
        Returns an app_map for serializer context.
        """
        sq = WebappIndexer.search()
        if request.QUERY_PARAMS.get('filtering', '1') == '1':
            # With filtering (default).
            for backend in self.filter_backends:
                sq = backend().filter_queryset(request, sq, self)
        sq = WebappIndexer.filter_by_apps(app_ids, sq)

        # Store the apps to attach to feed elements later.
        with statsd.timer('mkt.feed.views.apps_query'):
            apps = sq.execute().hits
        return dict((app.id, app) for app in apps)

    def field_to_native_es(self, obj, request):
        """
        A version of field_to_native that uses ElasticSearch to fetch the apps
        belonging to the collection instead of SQL.

        Relies on a FeaturedSearchView instance in self.context['view']
        to properly rehydrate results returned by ES.
        """
        device = self._get_device(request)

        app_filters = {'profile': get_feature_profile(request)}
        if device and device != amo.DEVICE_DESKTOP:
            app_filters['device'] = device.id

        qs = WebappIndexer.get_app_filter(request, app_filters)
        qs = qs.filter('term', **{'collection.id': obj.pk})

        qs = qs.sort({
            'collection.order': {
                'order': 'asc',
                'nested_filter': {
                    'term': {'collection.id': obj.pk}
                }
            }
        })

        return self.to_native(qs, use_es=True)

def app_search(request):
    results = []
    q = request.GET.get('q', u'').lower().strip()
    limit = (lkp.MAX_RESULTS if request.GET.get('all_results')
             else lkp.SEARCH_LIMIT)
    fields = ('name', 'app_slug')
    non_es_fields = ['id', 'name__localized_string'] + list(fields)

    if q.isnumeric():
        qs = Webapp.objects.filter(pk=q).values(*non_es_fields)[:limit]
    else:
        # Try to load by GUID:
        qs = Webapp.objects.filter(guid=q).values(*non_es_fields)[:limit]
        if not qs.count():
            qs = (WebappIndexer.search()
                  .query(_expand_query(q, fields))[:limit])
                  # TODO: Update to `.fields(...)` when the DSL supports it.
            qs = qs.execute()
    for app in qs:
        if isinstance(app, dict):
            # This is a result from the database.
            app['url'] = reverse('lookup.app_summary', args=[app['id']])
            app['name'] = app['name__localized_string']
            results.append(app)
        else:
            # This is a result from elasticsearch which returns `Result`
            # objects and name as a list, one for each locale.
            for name in app.name:
                results.append({
                    'id': app.id,
                    'url': reverse('lookup.app_summary', args=[app.id]),
                    'app_slug': app.get('app_slug'),
                    'name': name,
                })
    return {'results': results}

    def list(self, request, *args, **kwargs):
        if (not settings.RECOMMENDATIONS_ENABLED or
                not settings.RECOMMENDATIONS_API_URL or
                not self.request.user.is_authenticated()):
            return self._popular()
        else:
            app_ids = []
            url = '{base_url}/api/v2/recommend/{limit}/{user_hash}/'.format(
                base_url=settings.RECOMMENDATIONS_API_URL,
                limit=20, user_hash=self.request.user.recommendation_hash)

            try:
                with statsd.timer('recommendation.get'):
                    resp = requests.get(
                        url, timeout=settings.RECOMMENDATIONS_API_TIMEOUT)
                if resp.status_code == 200:
                    app_ids = resp.json()['recommendations']
            except Timeout as e:
                log.warning(u'Recommendation timeout: {error}'.format(error=e))
            except RequestException as e:
                # On recommendation API exceptions we return popular.
                log.error(u'Recommendation exception: {error}'.format(error=e))

            if not app_ids:
                # Fall back to a popularity search.
                return self._popular()

            sq = WebappIndexer.get_app_filter(self.request, app_ids=app_ids)
            return Response({
                'objects': self.serializer_class(
                    sq.execute().hits, many=True,
                    context={'request': self.request}).data})

    def test_single_hit(self):
        """Test the ESPaginator only queries ES one time."""
        es = WebappIndexer.get_es()
        orig_search = es.search
        es.counter = 0

        def monkey_search(*args, **kwargs):
            es.counter += 1
            return orig_search(*args, **kwargs)

        es.search = monkey_search

        ESPaginator(WebappIndexer.search(), 5).object_list.execute()
        eq_(es.counter, 1)

        es.search = orig_search

 def test_app_ids(self):
     """
     Test all apps are returned if app IDs is passed. Natural ES limit is
     10.
     """
     sq = WebappIndexer.filter_by_apps(app_ids=self.app_ids)
     results = sq.execute().hits
     eq_(len(results), 11)

 def _filter(self, req, filters, **kwargs):
     form = self.form_class(filters)
     if form.is_valid():
         qs = WebappIndexer.from_search(self.req, **kwargs)
         return _filter_search(
             self.req, qs, form.cleaned_data).to_dict()
     else:
         return form.errors.copy()

 def _filter(self, req=None, data=None):
     req = req or RequestFactory().get('/', data=data or {})
     req.user = AnonymousUser()
     queryset = WebappIndexer.search()
     for filter_class in self.filter_classes:
         queryset = filter_class().filter_queryset(req, queryset,
                                                   self.view_class)
     return queryset.to_dict()

 def test_no_filter(self):
     # Set a couple apps as non-public, the count should decrease.
     self.apps[0].update(status=amo.STATUS_REJECTED)
     self.apps[1].update(status=amo.STATUS_PENDING)
     self.refresh('webapp')
     sq = WebappIndexer.get_app_filter(self.request, app_ids=self.app_ids)
     results = sq.execute().hits
     eq_(len(results), 9)

 def test_indexable(self):
     homescreen = app_factory(name=u'Elegant Waffle',
                              description=u'homescreen runner',
                              created=self.days_ago(5),
                              manifest_url='http://h.testmanifest.com')
     Tag(tag_text='homescreen').save_tag(homescreen)
     homescreen.save()
     q = WebappIndexer.get_indexable()
     eq_(list(q), [self.app])

 def _filter(self, req, filters, **kwargs):
     form = self.form_class(filters)
     if form.is_valid():
         form_data = form.cleaned_data
         sq = WebappIndexer.get_app_filter(
             self.req, search_form_to_es_fields(form_data))
         return _sort_search(self.req, sq, form_data).to_dict()
     else:
         return form.errors.copy()

    def setUp(self):
        self.apps = [amo.tests.app_factory() for i in range(3)]
        self.app_ids = [app.id for app in self.apps]

        self.brand = self.feed_brand_factory(app_ids=self.app_ids)
        self.data_es = self.brand.get_indexer().extract_document(
            None, obj=self.brand)

        self.app_map = dict((app.id, WebappIndexer.extract_document(app.id))
                            for app in self.apps)