class Processor(object):

    def __init__(self, solr_server_url):
        self.server = Solr(solr_server_url)

    def process(self, fname):
        base, _ = os.path.splitext(os.path.basename(fname))
        url = DOCUMENT_URL + base + '.html'
        fp = open(fname)
        title = None
        while not title:
            title = fp.next().strip()
        content = ''
        for line in fp:
            s = line.strip()
            if s and not s.startswith(('**', '==', '--')):
                content += s
        fp.close()
        document_id = u"%s-%s" % (DOCUMENT_SITE_ID, title)
        logging.info("new document: %s" % (document_id,))
        t = os.path.getmtime(fname)
        doc = {
            'id': hashlib.sha1(document_id.encode('utf-8')).hexdigest(),
            'site': DOCUMENT_SITE_ID,
            'url': url,
            'title': title,
            'content': content,
            'last_modified': datetime.datetime.fromtimestamp(t)
        }
        self.server.add([doc])

    def test_custom_results_class(self):
        solr = Solr('http://localhost:8983/solr/core0', results_cls=dict)

        results = solr.search(q='*:*')
        assert isinstance(results, dict)
        assert 'responseHeader' in results
        assert 'response' in results

    def test_dismax_loc(self):
        """docstring for test_dismax"""

        conn = Solr(SOLR_URL)

        loc_name = 'downing st'
        loc = TEST_LOCS[loc_name]
        print '\n\n*** %s ' % loc_name, loc
    
        kwords = 'heart'
        kw = {
            'rows': settings.SOLR_ROWS,
            'fl': '*,score',
            'qt': 'resources',
            'sfield': 'pt_location',
            'pt': loc,
            'bf': 'recip(geodist(),2,200,20)^20',
            'sort': 'score desc',
        }
    
        results = conn.search(kwords, **kw)

        print '\n--\nsearch on [%s] : ' % (kwords)
        for result in results:
            print '-', result['score'], result['title'], result.get('pt_location', '')

 def delete(self, *args, **kwargs):
     """docstring for delete"""
     for c in self.curations:
         c.delete()
     conn = Solr(settings.SOLR_URL)
     conn.delete(q='id:%s' % self.id)        
     super(Resource, self).delete(*args, **kwargs)

class SolrSearchBackend(BaseSearchBackend):
    # Word reserved by Solr for special use.
    RESERVED_WORDS = ("AND", "NOT", "OR", "TO")

    # Characters reserved by Solr for special use.
    # The '\\' must come first, so as not to overwrite the other slash replacements.
    RESERVED_CHARACTERS = ("\\", "+", "-", "&&", "||", "!", "(", ")", "{", "}", "[", "]", "^", '"', "~", "*", "?", ":")

    def __init__(self, connection_alias, **connection_options):
        super(SolrSearchBackend, self).__init__(connection_alias, **connection_options)

        if not "URL" in connection_options:
            raise ImproperlyConfigured(
                "You must specify a 'URL' in your settings for connection '%s'." % connection_alias
            )

        self.conn = Solr(connection_options["URL"], timeout=self.timeout)
        self.log = logging.getLogger("haystack")

    def update(self, index, iterable, commit=True):
        docs = []

        try:
            for obj in iterable:
                docs.append(index.full_prepare(obj))
        except UnicodeDecodeError:
            sys.stderr.write("Chunk failed.\n")

        if len(docs) > 0:
            try:
                self.conn.add(docs, commit=commit, boost=index.get_field_weights())
            except (IOError, SolrError), e:
                self.log.error("Failed to add documents to Solr: %s", e)

    def handle(self, port="15672", queues="solr,priority", *args, **options):
        # Determine the number of tasks still in Rabbit
        msg_count = 0
        for queue in queues.split(','):
            # Get the queue data from the rabbit Management API
            uri = "http://%(broker)s:%(port)s/api/queues/dseo-vhost/%(queue)s" % {'broker': settings.BROKER_HOST,
                                                                                  'port': port,
                                                                                  'queue': queue}
            resp = requests.get(uri, auth=(settings.BROKER_USER, settings.BROKER_PASSWORD))
            data = json.loads(resp.content)
            msg_count += data["messages_ready"]
            msg_count += data["messages_unacknowledged"]

        # If we find that having a couple long running messages can lead to false positives,
        # we can change the cutoff here.
        if msg_count <= self.msg_cutoff:
            print "No messages in the queue(s), do not raise an alert."
            return

        # Determine the number of recently updated jobs in solr.
        conn = Solr(settings.HAYSTACK_CONNECTIONS['default']['URL'])
        start_date = datetime.now() - self.look_behind
        solr_count = conn.search("date_added:[%s TO NOW]" % (start_date.iso_format() + "Z")).hits

        if solr_count <= self.solr_cutoff:
            msg = "Found %s messages in rabbit, but saw %s updated job in the last hour.  Perhaps the workers are not responding?" % (msg_count, solr_count)
            send_mail("Rabbit & Solr Monitoring",
                      msg,
                      "[email protected]",
                      "[email protected]")

 def clear_missing(self, verbose=False):
     conn = Solr(settings.SOLR_URL)
     start = 0
     to_delete = []
     pb = None
     if verbose: print "Checking for indexed records no longer in database"
     while True:
         if verbose and pb: pb.update(start)
         result = conn.search('*:*', sort='id asc', start=start, rows=500, fields=['id'])
         if not result:
             break
         if verbose and not pb: pb = ProgressBar(result.hits)
         ids = [int(r['id']) for r in result]
         records = Record.objects.filter(id__in=ids).values_list('id', flat=True)
         for r in records:
             ids.remove(r)
         to_delete.extend(ids)
         start += 500
     if verbose and pb: pb.done()
     pb = None
     if verbose and to_delete:
         print "Removing unneeded records from index"
         pb = ProgressBar(len(to_delete))
     while to_delete:
         if verbose and pb: pb.update(pb.total - len(to_delete))
         conn.delete(q='id:(%s)' % ' '.join(map(str, to_delete[:500])))
         to_delete = to_delete[500:]
     if verbose and pb: pb.done()

def clear_solr(buid):
    """Delete all jobs for a given business unit/job source."""
    conn = Solr(settings.HAYSTACK_CONNECTIONS["default"]["URL"])
    hits = conn.search(q="*:*", rows=1, mlt="false", facet="false").hits
    logging.info("BUID:%s - SOLR - Deleting all %s jobs" % (buid, hits))
    conn.delete(q="buid:%s" % buid)
    logging.info("BUID:%s - SOLR - All jobs deleted." % buid)

def delete_response(request, id):
    if request.method == "POST":
        r = Response.objects.get(id=id)
        conn = Solr(settings.SOLR_BASE)
        conn.delete("result:%d" % r.id)
        r.delete()
    return HttpResponseRedirect("/")

 def reindex(self, remove=False):
     """docstring for reindex"""
     conn = Solr(settings.SOLR_URL)
     # needs wildcard to remove indexing for multiple locations: <id>_<n>
     conn.delete(q="id:%s*" % self.id)
     if not remove:
         self.index(conn)

def search(request):
    query = request.GET.get('q', '')
    if not query:
        return dict(query=query)
    conn = Solr(settings.SOLR_BASE)
    results = [res for res in [extract_response(r) for r in conn.search(query)]
               if res is not None]
    return dict(query=query, responses=results)

def search_results(request):
    conn = Solr(request.registry.settings['solr_base_url'], decoder=decoder)
    params = request.GET.copy()
    q = params.pop('q', None)
    if q is None:
        return HTTPFound('http://2012.haip.cc/')

    params.update({
        'facet': 'true',
        'facet.limit': 20,
        'facet.mincount': 1,
        'facet.sort': 'count',
        'facet.field': ['language', 'author_exact', 'year'],
        'fl': '*',
    })

    # TODO: get cover data, description from https://developers.google.com/books/docs/v1/reference/volumes
    # TODO: refactor logic from template to view
    # TODO: tests

    # first do request without fq so we get all facet values
    params_temp = params.copy()
    if 'fq' in params_temp:
        del params_temp['fq']
    facet_fields = conn.search(q, **params_temp).facets['facet_fields']

    # workaround due to limitation that kwargs can't handle multidict
    if 'fq' in params:
        params['fq'] = ' AND '.join(params.getall('fq'))

    log.debug(params)
    results = conn.search(q, **params)
    log.debug(results)

    allowed_networks = request.registry.settings['allowed_networks'].split(',')
    if request.client_addr in iptools.IpRangeList(*allowed_networks):
        is_trusted_ip = True
    else: 
        is_trusted_ip = False

    out = {
        'results': list(results),
        'q': q,
        'facet_fields': facet_fields,
        'facets': params.get('fq', []),
    }

    if request.matched_route.name.endswith('json'):
        return out
    else:
        out.update({
            'with_facet': with_facet,
            'without_facet': without_facet,
            'format_byte_size': format_byte_size,
            'format_facet': format_facet,
            'is_trusted_ip': is_trusted_ip,
        })
        return out

class SolrSearchBackend(BaseSearchBackend):
    # Word reserved by Solr for special use.
    RESERVED_WORDS = (
        'AND',
        'NOT',
        'OR',
        'TO',
    )

    # Characters reserved by Solr for special use.
    # The '\\' must come first, so as not to overwrite the other slash replacements.
    RESERVED_CHARACTERS = (
        '\\', '+', '-', '&&', '||', '!', '(', ')', '{', '}',
        '[', ']', '^', '"', '~', '*', '?', ':',
    )

    def __init__(self, connection_alias, **connection_options):
        super(SolrSearchBackend, self).__init__(connection_alias, **connection_options)

        if not 'URL' in connection_options:
            raise ImproperlyConfigured("You must specify a 'URL' in your settings for connection '%s'." % connection_alias)

        self.connection_options = connection_options
        self.conn = Solr(connection_options['URL'], timeout=self.timeout)
        self.log = logging.getLogger('haystack')

    def update(self, index, iterable, commit=None):
        docs = []

        if commit == None:
            commit = self.connection_options.get('COMMIT_UPDATES', True)

        for obj in iterable:
            try:
                docs.append(index.full_prepare(obj))
            except UnicodeDecodeError:
                if not self.silently_fail:
                    raise

                # We'll log the object identifier but won't include the actual object
                # to avoid the possibility of that generating encoding errors while
                # processing the log message:
                self.log.error(u"UnicodeDecodeError while preparing object for update", exc_info=True, extra={
                    "data": {
                        "index": index,
                        "object": get_identifier(obj)
                    }
                })

        if len(docs) > 0:
            try:
                self.conn.add(docs, commit=commit, boost=index.get_field_weights())
            except (IOError, SolrError), e:
                if not self.silently_fail:
                    raise

                self.log.error("Failed to add documents to Solr: %s", e)

def admin_readercycle(request):
    solr_url = settings.HAYSTACK_CONNECTIONS['default']['URL']
    try:
        solr = Solr(solr_url)
        solr.readercycle()
        return HttpResponse("Cycled")
    except (IOError, SolrError), e:
        msg = "Failed to cycle Solr %s %s" % (solr_url, e)
        return HttpResponse(msg)

def delete_url(request, id):
    if request.method == "POST":
        u = Url.objects.get(id=id)
        for r in u.response_set.all():
            conn = Solr(settings.SOLR_BASE)
            conn.delete("result:%d" % r.id)
            r.delete()
        u.delete()
    return HttpResponseRedirect("/")

def add_to_solr(response, body):
    conn = Solr(settings.SOLR_BASE)
    conn.add(
        [
            dict(
                id="response:%d" % response.id,
                name=response.url.url,
                text=body.encode('utf-8'),
            )
        ])

def admin_cdrs_import(request):
    if (request.method != 'POST'):
        return render_to_response('portal/admin_cdrs.html', {})

    created = 0
    updated = 0
    solr = Solr(settings.CDRS_SOLR_URL)

    application = request.POST.get('application', '')
    collection_id = request.POST.get('collection_id', '')
    import_classification = request.POST.get('import_classification', '')
    dt = request.POST.get('last_import_date', '')
    tm = urllib.unquote(request.POST.get('last_import_time', '00:00'))

    q = 'import_classifications:"' + import_classification + '"'
    options = {'qt': 'forest-data'}

    last_import_date = LastImportDate.get_last_import_date(dt, tm, application)
    if last_import_date:
        utc = last_import_date.astimezone(FixedOffset(0))
        q += ' AND last_modified:[' + utc.strftime(
            '%Y-%m-%dT%H:%M:%SZ') + ' TO NOW]'

    try:
        collections = urllib.unquote(collection_id).split(",")
        for c in collections:
            # Get list of datasets in each collection id
            record_count = SolrUtilities().get_count_by_lastmodified(
                c, import_classification, last_import_date)
            retrieved = 0
            while (retrieved < record_count):
                to_retrieve = min(1000, record_count - retrieved)
                options['collection_id'] = c
                options['start'] = str(retrieved)
                options['rows'] = str(to_retrieve)

                results = solr.search(q, **options)
                for result in results:
                    if 'dataset_id' in result:
                        if process_metadata(result):
                            created += 1
                        else:
                            updated += 1

                retrieved = retrieved + to_retrieve

        # Update the last import date
        lid = LastImportDate.update_last_import_date(application)
        cache.set('solr_import_date', lid.strftime('%Y-%m-%d'))
        cache.set('solr_import_time', lid.strftime('%H:%M:%S'))
        cache.set('solr_created', created)
        cache.set('solr_updated', updated)
    except Exception, e:
        cache.set('solr_error', str(e))

def check_solr(using='solr'):
    try:
        from pysolr import Solr, SolrError
    except ImportError:
        raise SkipTest("pysolr  not installed.")

    solr = Solr(settings.HAYSTACK_CONNECTIONS[using]['URL'])
    try:
        solr.search('*:*')
    except SolrError as e:
        raise SkipTest("solr not running on %r" % settings.HAYSTACK_CONNECTIONS[using]['URL'], e)

def _solr_results_chunk(tup, buid, step):
    """
    Takes a (start_index, stop_index) tuple and gets the results in that
    range from the Solr index.

    """
    conn = Solr(settings.HAYSTACK_CONNECTIONS["default"]["URL"])
    results = conn.search(
        "*:*", fq="buid:%s" % buid, fl="uid", rows=step, start=tup[0], facet="false", mlt="false"
    ).docs
    return set([i["uid"] for i in results])

def delete_domain(request, id):
    d = Domain.objects.get(id=id)
    if request.method == "POST":
        for u in d.url_set.all():
            for r in u.response_set.all():
                conn = Solr(settings.SOLR_BASE)
                conn.delete("result:%d" % r.id)
                r.delete()
            u.delete()
        d.delete()
    return HttpResponseRedirect("/")