def _get_count(self):
     "Returns the total number of objects, across all pages."
     if self._count is None:
         solr = SolrConnection(settings.SOLR)  # TODO: maybe keep connection around?
         solr_response = solr.query(self._q, fields=['id'])
         self._count = int(solr_response.results.numFound)
     return self._count

def oralookup(pid=None, uuid=None, fields_to_return="f_name, f_subject, f_keyphrase, faculty, f_institution, thesis_type, content_type, collection", endpoint="http://ora.ouls.ox.ac.uk:8080/solr/select"):
  s = SolrConnection(endpoint)
  results = {}
  query = ""
  if pid:
    pid = "\:".join(pid.split(":"))
    query = "id:%s" % pid
  elif uuid:
    query = "id:uuid\:%s" % uuid
  else:
    return results
  # Running actual query (3 tries, failover)
  tries = 0
  while(tries != 3):
    try:
      r = s.query(q = query, fields = fields_to_return)
      logger.debug("Solr response: %s" % r.header)
      tries = 3
    except BadStatusLine:
      sleep(0.5)
      tries = tries + 1
  try:
    assert len(r.results) == 1
    return r.results[0]
  except ValueError:
    logger.warn("Couldn't parse json response from Solr endpoint: %s" % r)
    return {}
  except AssertionError:
    logger.warn("Couldn't assert that only a single result was fetched: %s" % results)
    return {}

def titlelookup(pid):
    s = SolrConnection("ora.ouls.ox.ac.uk:8080")
    results = {}
    query = ""
    if pid:
        pid = "\:".join(pid.split(":"))
        query = "id:%s" % pid
    elif uuid:
        query = "id:uuid\:%s" % uuid
    else:
        return results
    # Running actual query (3 tries, failover)
    tries = 0
    while tries != 3:
        try:
            r = s.search(q=query, wt="json", fl="title")
            logger.debug("Solr response: %s" % r)
            tries = 3
        except BadStatusLine:
            sleep(0.5)
            tries = tries + 1
    try:
        results = simplejson.loads(r)
        assert results["response"]["numFound"] == 1
        doc = results["response"]["docs"][0]
        return doc["title"]
    except ValueError:
        logger.warn("Couldn't parse json response from Solr endpoint: %s" % r)
        return {}
    except AssertionError:
        logger.warn("Couldn't assert that only a single result was fetched: %s" % results)
        return {}

def main():
    opts, args = parse_opts()
    logging.basicConfig(filename=opts.log_file, level=logging.DEBUG,
                        format='[%(asctime)s|%(levelname)s|%(name)s|%(threadName)s|%(message)s]')

    solr = SolrConnection(opts.solr_uri)
    protocol = LineProtocol()
    for request in protocol.input():
        try:
            query = build_query(request)
            if query is None:
                protocol.output(query_failed(), True)
                continue
            log.debug("Running query: " + str(query))
            results = solr.search(**query)
            if results is None:
                protocol.output({'code' : 400})
                continue
            resp = json.loads(results)
            ret = {
                'code' : 200,
                'json' : resp['response']
            }
            protocol.output(ret, True)
        except Exception:
            log.exception("Uncaught exception")
    return 0

def word_matches_for_page(page_id, words):
    """
    Gets a list of pre-analyzed words for a list of words on a particular
    page. So if you pass in 'manufacturer' you can get back a list like
    ['Manufacturer', 'manufacturers', 'MANUFACTURER'] etc ...
    """
    solr = SolrConnection(settings.SOLR)

    # Make sure page_id is of type str, else the following string
    # operation may result in a UnicodeDecodeError. For example, see
    # ticket #493
    if not isinstance(page_id, str):
        page_id = str(page_id)

    ocr_list = ['ocr', ]
    ocr_list.extend(['ocr_%s' % l for l in settings.SOLR_LANGUAGES])
    ocrs = ' OR '.join([query_join(words, o) for o in ocr_list])
    q = 'id:%s AND (%s)' % (page_id, ocrs)
    params = {"hl.snippets": 100, "hl.requireFieldMatch": 'true', "hl.maxAnalyzedChars": '102400'}
    response = solr.query(q, fields=['id'], highlight=ocr_list, **params)

    if page_id not in response.highlighting:
        return []

    words = set()
    for ocr in ocr_list:
        if ocr in response.highlighting[page_id]:
            for context in response.highlighting[page_id][ocr]:
                words.update(find_words(context))
    return list(words)

 def solr_delete(self):
     """
     Remove from solr index
     """
     solr_conn = SolrConnection(settings.SOLR_URL, persistent=False)
     solr_conn.delete_query('id:%s' % self.id)
     solr_conn.commit()

 def solr_index(self):
     """
     Write out to solr
     """
     solr_conn = SolrConnection(settings.SOLR_URL, persistent=False)
     solr_conn.add(**self.solr_doc)
     solr_conn.commit()

def index_evidence(evidence):
    evidence_medicine_list = []

    evidence_medicine = MedicineEvidenceSummary.objects.filter(evidence=evidence.id)
    for evimed in evidence_medicine: 
        if evimed.medicine.name not in evidence_medicine_list:
                evidence_medicine_list.append(evimed.medicine.name)

    # try to create a connection to a solr server and send medicine
    try:
        solr = SolrConnection(settings.SOLR_URL)
        solr.add(
            id = "evidence-%s-%s" % (evidence.language, evidence.id), 
            type = "evidence",
            title = evidence.title,            
            description = evidence.description,
            context = evidence.context,
            question = evidence.question,
            link = evidence.link,
            file = evidence.file,
            language = evidence.language,
            evidence_medicine = evidence_medicine_list,
        )
        response = solr.commit()
    except Exception as ex: 
        return False

    return True

def oralookup(
    pid=None,
    uuid=None,
    fields_to_return="f_name, f_subject, f_keyphrase, faculty, f_institution, thesis_type, content_type, collection",
):
    s = SolrConnection("ora.ouls.ox.ac.uk:8080")
    results = {}
    query = ""
    if pid:
        pid = "\:".join(pid.split(":"))
        query = "id:%s" % pid
    elif uuid:
        query = "id:uuid\:%s" % uuid
    else:
        return results
    # Running actual query (3 tries, failover)
    tries = 0
    while tries != 3:
        try:
            r = s.search(q=query, wt="json", fl=fields_to_return)
            logger.debug("Solr response: %s" % r)
            tries = 3
        except BadStatusLine:
            sleep(0.5)
            tries = tries + 1
    try:
        results = simplejson.loads(r)
        assert results["response"]["numFound"] == 1
        return results["response"]["docs"][0]
    except ValueError:
        logger.warn("Couldn't parse json response from Solr endpoint: %s" % r)
        return {}
    except AssertionError:
        logger.warn("Couldn't assert that only a single result was fetched: %s" % results)
        return {}

def index_title(title, solr=None):
    if solr is None:
        solr = SolrConnection(settings.SOLR)
    LOGGER.info("indexing title: lccn=%s", title.lccn)
    try:
        solr.add(**title.solr_doc)
    except Exception as e:
        LOGGER.exception(e)

def index_title(title, solr=None):
    if solr==None:
        solr = SolrConnection(settings.SOLR)
    _log.info("indexing title: lccn=%s" % title.lccn)
    try:
        solr.add(**title.solr_doc)
    except Exception, e:
        _log.exception(e)

def get_page_text(page):
    no_text = ["Text not available"]
    solr = SolrConnection(settings.SOLR)
    query = 'id:"%s"' % page.url
    solr_results = solr.query(query)
    results_attribute = getattr(solr_results, 'results', None)
    if isinstance(results_attribute, list) and len(results_attribute) > 0:
        return results_attribute[0].get('ocr', no_text)
    else:
        return no_text

def similar_pages(page):
    solr = SolrConnection(settings.SOLR)
    d = page.issue.date_issued
    year, month, day = '{0:02d}'.format(d.year), '{0:02d}'.format(d.month), '{0:02d}'.format(d.day) 
    date = ''.join(map(str, (year, month, day)))

    query = '+type:page AND date:%s AND %s AND NOT(lccn:%s)' % (date, query_join(map(lambda p: p.city, 
                                           page.issue.title.places.all()), 'city'), page.issue.title.lccn)
    response = solr.query(query, rows=25)
    results = response.results
    return map(lambda kwargs: utils.get_page(**kwargs), 
               map(lambda r: urlresolvers.resolve(r['id']).kwargs, results))

def execute_solr_query(query, fields, sort, sort_order, rows, start):
    solr = SolrConnection(settings.SOLR) # TODO: maybe keep connection around?
    solr_response = solr.query(query,
                               fields=['lccn', 'title',
                                       'edition',
                                       'place_of_publication',
                                       'start_year', 'end_year',
                                       'language'],
                               rows=rows,
                               sort=sort,
                               sort_order=sort_order,
                               start=start)
    return solr_response

def _refresh(field=None, data=None, path = None, isCron = None):
	from solr import SolrConnection
	from ID3 import *
	s = SolrConnection(SOLR_URL)
	if path and path != '*':
		#called by user		
		pathsArr = path.split(',')		
	else:
		#called from cron		
		pathsArr = folderpaths
	matches = []
	#handles modify, add
	#deletion will be handled in search when file in solr but not in path
	time.time()
	for path in pathsArr:
		for root, dirnames, filenames in os.walk(path):
			for extension in ['txt', 'log', 'py', 'pl', 'sql', 'mp3']:
				for filename in fnmatch.filter(filenames, '*.' + extension):				
					fullName = os.path.join(root, filename)
					if os.path.getsize(fullName) > 8800000:
						continue
					#print fullName
					if not isCron or (time.time() - os.path.getmtime(fullName) < 24*60*60):				
						try:
							#data = open(fullName, 'r').read().decode('raw_unicode_escape').replace('\n',' ').replace('\t',' ')
							if filename.endswith(('.txt', '.log', '.py', '.pl', '.sql')):								
								data = open(fullName, 'r').read()
								data = filterTxt(data)
							else:								
								audiofile = ID3(fullName)
								audiofilekeys = audiofile.keys()
								if 'TITLE' in audiofilekeys:
									data = audiofile['TITLE'] + " "
								if 'ARTIST' in audiofilekeys:
									data += audiofile['ARTIST'] + " "
								if 'ALBUM' in audiofilekeys:
									data += audiofile['ALBUM'] + " "
								if not data:
									data = ''
								data = data.strip()
							fullName = filterTxt(fullName)
							filename = filterTxt(filename)						
							s.add(id = fullName, name = filename, txt = data)
							s.commit()
						except:																	
							pass
							#print data
							#print traceback.format_exc()
							#print fullName	
							#sys.exit()					
						gc.collect()

class Command(BaseCommand):
    user_option = optparse.make_option('--user',
        action='store', dest='user',
        help='name of user whose entries to purge')
    option_list = BaseCommand.option_list + (user_option,)
    help = "index all or user-specific entries in solr"
    args = 'an optional username'

    def handle(self, *args, **options):
        self.solr = SolrConnection(SOLR_URL)
        self.cursor = connection.cursor()
        if options['user']:
            print "indexing user"
            self.index_entries(user=options['user'])
        else:
            print 'indexing everything'
            self.index_entries()
        print 'committing'
        self.solr.commit()
        print 'optimizing'
        self.solr.optimize()

    def index_entries(self, user=''):
        counter = 0
        entries = m.Entry.objects.all()
        if user:
            entries = entries.filter(user__username=user)
        docs = []
        print 'entry count:', entries.count()
        SLICE_SIZE = MAX_DOCS_PER_ADD * COMMIT_FREQUENCY 
        slices = [x for x in range(entries.count()) \
            if x % SLICE_SIZE == 0]
        for s in slices:
            print 'indexing %s to %s...' % (s, s+SLICE_SIZE)
            entry_slice = entries[s:s+SLICE_SIZE]
            for entry in entry_slice:
                counter += 1
                docs.append(entry.solr_doc)
                if len(docs) == MAX_DOCS_PER_ADD:
                    try:
                        self.solr.add_many(docs)
                    except:
                        print 'BAD RECORD:', [d['id'] for d in docs]
                    del(docs)
                    docs = []
                    reset_queries()
                    if counter % (COMMIT_FREQUENCY * MAX_DOCS_PER_ADD) == 0:
                        print 'committing at count:', counter
                        self.solr.commit()
        # Don't miss the leftovers
        self.solr.add_many(docs)

	def __init__(self, channel_id):

		self.channel_id = channel_id

		# Build up a Solr query
		filters = []
		filters.append('type:request')
		filters.append('channel_id:%s' % channel_id)

		# Make the request to Solr
		solr = SolrConnection(settings.SOLR_URL)
		response = solr.select(q = ' AND '.join(filters), rows = 10, fields = 'datetime, id', sort = 'datetime', sort_order = 'asc')

		self.requests = response.results

def index_missing_pages():
    """
    index all pages that are missing from solr in the database
    """
    solr = SolrConnection(settings.SOLR)
    count = 0
    pages = models.Page.objects.filter(indexed=False).all()
    number_of_pages = len(pages)
    for page in pages:
        LOGGER.info("[%s of %s] indexing page: %s", count, number_of_pages, page.url)
        solr.add(**page.solr_doc)
        count += 1
        page.indexed = True
        page.save()
    solr.commit()

def execute_solr_query(query, fields, sort, sort_order, rows, start):
    # default arg_separator - underscore wont work if fields to facet on
    # themselves have underscore in them
    solr = SolrConnection(settings.SOLR)  # TODO: maybe keep connection around?
    solr_response = solr.query(query,
                               fields=['lccn', 'title',
                                       'edition',
                                       'place_of_publication',
                                       'start_year', 'end_year',
                                       'language'],
                               rows=rows,
                               sort=sort,
                               sort_order=sort_order,
                               start=start)
    return solr_response

	def finished(self, **kwargs):

		source_id = kwargs['source_id']
		track_id = kwargs['track_id']

		# Build up a Solr query
		filters = []
		filters.append('type:request')
		filters.append('channel_id:%s' % self.channel_id)
		filters.append('request_source_id:%s' % source_id)
		filters.append('request_track_id:%s' % track_id)

		# Make the request to Solr
		solr = SolrConnection(settings.SOLR_URL)
		solr.delete_query(' AND '.join(filters))
		solr.commit()