def term_facet(host='localhost:9200',
               terms=['bibleverse'],
               _type='habakkuk',
               date_filter=[],
               size=10):
    ret = []
    conn = ES(host)
    q = MatchAllQuery()
    if date_filter:
        start,end = date_filter
        q = FilteredQuery(q, RangeFilter(qrange=ESRange('created_at_date',start,end,include_upper=False)))

    q = q.search(size=0)
    for term in terms:
        q.facet.add_term_facet(term,order='count',size=size)
        
    print json.dumps(json.loads(q.to_search_json()),indent=2)

    resultset = conn.search(query=q, indices=_type+'-*', doc_types=[_type])
    for facet in resultset.facets:
        print "Total",facet,resultset.facets[facet]['total']
        for row in resultset.facets[facet]['terms']:
            print "\t",row['term'],row['count']
            ret.append((facet,row['term']))
        
    return ret

class StashHandler(logging.Handler):
    def __init__(self, constr, whitelist=None, blacklist=None):
        logging.Handler.__init__(self)
        self.conn = ES(constr)
        if blacklist is None:
            blacklist = set()
        self.whitelist = whitelist
        self.blacklist = blacklist
        self.record_type = 'record'

    @property
    def index_name(self):
        return 'logstash-'+datetime.date.today().strftime('%Y.%m.%d')

    def emit(self, record):
        if self.whitelist is None:
            d = { k: record.__dict__[k] for k in record.__dict__ if k not in self.blacklist }
        else:
            d = { k: record.__dict__[k] for k in record.__dict__ if k in self.whitelist and k not in self.blacklist }
        entry = {
            "@fields": d,
            "@message": record.msg, 
            "@source": "gelf://localhost", 
            "@source_host": "gelf://localhost", 
            "@source_path": "/", 
            "@tags": [], 
            "@timestamp": datetime.datetime.utcnow().isoformat(), 
            "@type": self.record_type}
        self.conn.index(entry, self.index_name, self.record_type)

    def handle(self, *args, **kwargs):
        elastic = ES(settings.SEARCH_HOSTS)

        pp = pprint.PrettyPrinter(indent=4)
        pp.pprint(elastic.get_indices())

        elastic.connection.close()

class ElasticSearchPipeline(object):
    def __init__(self):

        self.settings = get_project_settings()

        basic_auth = {'username': self.settings['ELASTICSEARCH_USERNAME'], 'password': self.settings['ELASTICSEARCH_PASSWORD']}

        if self.settings['ELASTICSEARCH_PORT']:

            uri = "%s:%d" % (self.settings['ELASTICSEARCH_SERVER'], self.settings['ELASTICSEARCH_PORT'])
        else:
            uri = "%s" % (self.settings['ELASTICSEARCH_SERVER'])

        self.es = ES([uri], basic_auth=basic_auth)

    def process_item(self, item, spider):
        if self.__get_uniq_key() is None:
            log.msg("ELASTICSEARCH_UNIQ_KEY is NONE")
            self.es.index(dict(item), self.settings['ELASTICSEARCH_INDEX'], self.settings['ELASTICSEARCH_TYPE'],
                          id=item['id'], op_type='create',)
        else:
            log.msg("Generation SHA1")
            self.es.index(dict(item), self.settings['ELASTICSEARCH_INDEX'], self.settings['ELASTICSEARCH_TYPE'],
                          hashlib.sha1(item[self.__get_uniq_key()]).hexdigest())
        log.msg("Item send to Elastic Search %s" %
                    (self.settings['ELASTICSEARCH_INDEX']),
                    level=log.DEBUG, spider=spider)
        return item

    def __get_uniq_key(self):
        if not self.settings['ELASTICSEARCH_UNIQ_KEY'] or self.settings['ELASTICSEARCH_UNIQ_KEY'] == "":
            return None
        return self.settings['ELASTICSEARCH_UNIQ_KEY']

def index():
    """docstring for fname"""
    import time
    fptr = open(sys.argv[1], 'rb')
    line_count = 0
    conn = ES(["localhost:9200"])
    #conn.create_index('test-index')
    start = time.clock()
    numb_exceptions = 0

    for line in fptr:
        if ((line_count % 10000) == 0):
            end = time.clock()
            minutes = (end - start) / 60.0
            print 'Done with %d took %f min. ' %(line_count, minutes)
            print 'number of exceptions ', numb_exceptions
        line_count += 1
        data = json.loads(line)
        post_id = int(data['post_id'])
        if post_id and data:
            try:
                conn.index(data, "test-index", "test-type", post_id)
            except Exception:
                numb_exceptions += 1
                continue

    print 'number of exceptions ', numb_exceptions

def facets(host='localhost:9200',
          facet_terms=['bibleverse'],
          _type='habakkuk',
          date_filter=[],
          size=10):
    ret = {}
    conn = ES(host)
    q = MatchAllQuery()
    if date_filter:
        start,end = date_filter
        q = FilteredQuery(q, RangeFilter(qrange=ESRange('created_at_date',
                                                        start.isoformat(),
                                                        end.isoformat(),
                                                        include_upper=False)))

    q = q.search(size=0)
    for term in facet_terms:
        q.facet.add_term_facet(term,order='count',size=size)
        
    es_logger.info(q.serialize())

    resultset = conn.search(query=q, indices=_type+'-*', doc_types=[_type])
    for facet in resultset.facets:
        ret[facet] = []
        for row in resultset.facets[facet]['terms']:
            ret[facet].append({"value":row['term'],"count":row['count']})

    logger.debug("facets return|'%s'"%json.dumps(ret))
    return ret

class ElasticSearchPipeline(object):
    def __init__(self):
        self.settings = get_project_settings()

        basic_auth = {'username': self.settings['ELASTICSEARCH_USERNAME'], 'password': self.settings['ELASTICSEARCH_PASSWORD']}

        if self.settings['ELASTICSEARCH_PORT']:
            uri = "%s:%d" % (self.settings['ELASTICSEARCH_SERVER'], self.settings['ELASTICSEARCH_PORT'])
        else:
            uri = "%s" % (self.settings['ELASTICSEARCH_SERVER'])

        self.es = ES([uri], basic_auth=basic_auth)

    def index_item(self, item):
        if self.settings['ELASTICSEARCH_UNIQ_KEY']:
            local_id = hashlib.sha1(item[uniq_key)]).hexdigest()
            log.msg("Generated unique key %s" % local_id, level=self.settings['ELASTICSEARCH_LOG_LEVEL'])
            op_type = 'none'
        else:
            op_type = 'create'
            local_id = item['id']
        self.es.index(dict(item),
                      self.settings['ELASTICSEARCH_INDEX'],
                      self.settings['ELASTICSEARCH_TYPE'],
                      id=local_id,
                      op_type=op_type)

def get_related_videos(video):
    related_videos = []
    conn = ES(['127.0.0.1:9200'])
    conn.default_indices = VIDEO_INDEX
    conn.refresh(VIDEO_INDEX)
    q = {
        "query": {
                 "bool": {
                           "should": [
                                       {"term"  : { "uid" : video.uid } },
                                       {"terms" : { "category" : [video.category]}},
                                       {"terms" : { "topic" : [video.topic]}},
                                       {"terms" : { "language" : [video.language]}}
                                       ],
                           "minimum_should_match" : 1
                           }
                 }
        }
    try:
        query = json.dumps(q)
        url = "http://localhost:9200/%s/_search" % VIDEO_INDEX
        response = urllib2.urlopen(url, query)
        result = json.loads(response.read())
        for res in result['hits']['hits']:
            related_videos.append(res['_source'])
    except Exception:
        pass
    return related_videos

 def es_index(self):
     conn = ES(settings.ES_SERVERS, basic_auth=settings.ES_AUTH)
     conn.index(
         doc=self.get_search_kwargs(),
         index=self.tenant.slug,
         doc_type=self.Meta.document_type,
         id=unicode(self.id)
     )

def get_es(**overrides):
    """Return one pyes.es.ES object

    :arg overrides: Allows you to override defaults to create the ES.

    Things you can override:

    * default_indexes
    * timeout
    * dump_curl

    Values for these correspond with the arguments to pyes.es.ES.

    For example, if you wanted to create an ES for indexing with a timeout
    of 30 seconds, you'd do:

    >>> es = get_es(timeout=30)

    If you wanted to create an ES for debugging that dumps curl
    commands to stdout, you could do:

    >>> class CurlDumper(object):
    ...     def write(self, s):
    ...         print s
    ...
    >>> es = get_es(dump_curl=CurlDumper())
    """
    if overrides or not hasattr(_local, 'es'):
        defaults = {
            'default_indexes': DEFAULT_INDEXES,
            'timeout': DEFAULT_TIMEOUT,
            'dump_curl': DEFAULT_DUMP_CURL,
            }

        defaults.update(overrides)
        if (not thrift_enable and
            not settings.ES_HOSTS[0].split(':')[1].startswith('92')):
            raise ValueError('ES_HOSTS is not set to a valid port starting '
                             'with 9200-9299 range. Other ports are valid '
                             'if using pythrift.')
        es = ES(settings.ES_HOSTS, **defaults)

        # pyes 0.15 does this lame thing where it ignores dump_curl in
        # the ES constructor and always sets it to None. So what we do
        # is set it manually after the ES has been created and
        # defaults['dump_curl'] is truthy. This might not work for all
        # values of dump_curl.
        if VERSION[0:2] == (0, 15):
            es.dump_curl = (defaults['dump_curl']
                            if defaults['dump_curl'] else None)

        # Cache the es if there weren't any overrides.
        if not overrides:
            _local.es = es
    else:
        es = _local.es

    return es

def get_es(hosts=None, default_indexes=None, timeout=None, dump_curl=None,
           **settings):
    """Create an ES object and return it.

    :arg hosts: list of uris; ES hosts to connect to, defaults to
        ``['localhost:9200']``
    :arg default_indexes: list of strings; the default indexes to use,
        defaults to 'default'
    :arg timeout: int; the timeout in seconds, defaults to 5
    :arg dump_curl: function or None; function that dumps curl output,
        see docs, defaults to None
    :arg settings: other settings to pass into `pyes.es.ES`

    Examples:

    >>> es = get_es()


    >>> es = get_es(hosts=['localhost:9200'])


    >>> es = get_es(timeout=30)  # good for indexing


    >>> es = get_es(default_indexes=['sumo_prod_20120627']


    >>> class CurlDumper(object):
    ...     def write(self, text):
    ...         print text
    ...
    >>> es = get_es(dump_curl=CurlDumper())

    """
    # Cheap way of de-None-ifying things
    hosts = hosts or DEFAULT_HOSTS
    default_indexes = default_indexes or DEFAULT_INDEXES
    timeout = timeout if timeout is not None else DEFAULT_TIMEOUT
    dump_curl = dump_curl or DEFAULT_DUMP_CURL

    if not isinstance(default_indexes, list):
        default_indexes = [default_indexes]

    es = ES(hosts,
            default_indexes=default_indexes,
            timeout=timeout,
            dump_curl=dump_curl,
            **settings)

    # pyes 0.15 does this lame thing where it ignores dump_curl in
    # the ES constructor and always sets it to None. So what we do
    # is set it manually after the ES has been created and
    # defaults['dump_curl'] is truthy. This might not work for all
    # values of dump_curl.
    if PYES_VERSION[0:2] == (0, 15) and dump_curl is not None:
        es.dump_curl = dump_curl

    return es

    def tearDown(self):
        self.log.warning("before tearDown es")
	self._unlink_es_cluster()
	self._stop_es_replication()
	if self.es_host != None:
		conn = ES(self.es_host + ":9200")
	        conn.delete_index_if_exists("default")
        super(ElasticSearchSupport, self).tearDown()
        self.log.warning("after tearDown es")

class BaseElasticSearchClient(BaseClient):

    def __init__(self, servers, index):
        """
        @param servers: Make sure to include the port with the server address
        @param index: Document index
        @return:
        """
        super(BaseElasticSearchClient, self).__init__()
        self.connection = None
        self.servers = servers
        self.index = index if type(index) is list else [index]

    def connect(self, connection_pool=1):
        update_connection_pool(connection_pool)

        try:
            self.connection = ES(self.servers)
        except NoServerAvailable:
            self._log.error('Failed to connect to elastic search server')
            return False
        return True

    def close(self):
        self.connection = None

    def _create_term_query(self, must_list):
        # TODO: add remaining conditional list functionality.
        query = BoolQuery()
        for term in must_list:
            query.add_must(term)

    def find_term(self, name, value, size=10):
        if not self.connection:
            return

        query = TermQuery(name, value)
        return self.connection.search(query=Search(query, size=size),
                                      indices=self.index)

    def find(self, filter_terms, size=10, doc_types=None):
        if not self.connection:
            return

        query = self._create_term_query(must_list=filter_terms)
        return self.connection.search(query=Search(query, size=size),
                                      indices=self.index,
                                      doc_types=doc_types)

    def find_one(self, filter_terms, size=10, doc_types=None):
        if not self.connection:
            return

        results = self.find(filter_terms=filter_terms, size=size,
                            doc_types=doc_types)
        return results[0] if len(results) > 0 else None

 def es_deindex(self):
     conn = ES(settings.ES_SERVERS, basic_auth=settings.ES_AUTH)
     try:
         conn.delete(
             index=self.tenant.slug,
             doc_type=self.Meta.document_type,
             id=meta.id
         )
     except:
         pass

class ElasticSearchPipeline(object):

    def __init__(self):    
        self.conn = ES('localhost:9200') 
        # self.file = open('urls.csv', 'wb')
        # self.file.write('spider,url' + '\n')

    def process_item(self, item, spider):        
        #self.file.write(spider.name + ',' + spider.start_urls[0] + '\n')
        self.conn.index(dict(item), "qrator", spider.name)
        return item

def job_redirect(request, slug, source, job_id):
    if request.method == 'GET' and request.GET.has_key('redirect'):
        try:
            elastic = ES(settings.SEARCH_HOSTS)
            data = elastic.get(source, 'job', job_id)
            elastic.connection.close()
            return HttpResponseRedirect(data['_source']['details_url'])
        except NotFoundException:
            raise Http404
        
    return direct_to_template(request, 'pages/redirect.html')

def job_redirect(request, slug, source, job_id):
    if request.method == "GET" and request.GET.has_key("redirect"):
        try:
            elastic = ES(settings.SEARCH_HOSTS)
            data = elastic.get(source, "job", job_id)
            elastic.connection.close()
            return HttpResponseRedirect(data["_source"]["details_url"])
        except NotFoundException:
            raise Http404

    return direct_to_template(request, "pages/redirect.html")

def setup_store():
    connection = ES(settings.THUMBNAIL_ELASTIC_SEARCH_SERVERS)
    try:
        connection.create_index_if_missing(settings.THUMBNAIL_ELASTIC_SEARCH_INDEX)
    except:
        pass
    try:
        connection.put_mapping(settings.THUMBNAIL_ELASTIC_SEARCH_DOCUMENT_TYPE,
                               settings.THUMBNAIL_ELASTIC_SEARCH_MAPPING,
                               indexes=[settings.THUMBNAIL_ELASTIC_SEARCH_INDEX,])
    except:
        pass

def main(argv):
    start = 1
    if len(sys.argv) > 1:
        if sys.argv[1]:
            start = sys.argv[1]

    bulksize = 1000

    es = ES(("http", "localhost", 9200), bulk_size=bulksize)

    c0 = 0
    t0 = time.time()
    c1 = 0
    t1 = time.time()
    for n in range(start, start + 1000000):
        result = es.index({ 
                 'a' : random_string_generator(),
                 'b' : random_string_generator(),
                 'c' : random_string_generator(),
                 'd' : random_string_generator(),
                 'e' : random_string_generator(),
                 'f' : random_string_generator(),
                 'g' : random_string_generator(),
                 'h' : random_string_generator(),
                 'i' : random_string_generator(),
                 'j' : random_string_generator(),
                 'k' : random_string_generator(),
                 'l' : random_string_generator(),
                 'm' : random_string_generator(),
                 'n' : random_string_generator(),
                 'o' : random_string_generator(),
                 'p' : random_string_generator(),
                 'q' : random_string_generator(),
                 'r' : random_string_generator(),
                 's' : random_string_generator(),
                 't' : random_string_generator(),
                 'u' : random_string_generator(),
                 'v' : random_string_generator(),
                 'w' : random_string_generator(),
                 'x' : random_string_generator(),
                 'y' : random_string_generator(),
                 'z' : random_string_generator()
             }, 'pyindex', 'pytype', n, bulk=True)
        c0 = c0 + bulksize
        c1 = c1 + bulksize
        if result:
            d0 = (time.time() - t0) 
            d1 = (time.time() - t1) 
            now = datetime.datetime.utcnow()
            print("{0},{1},{2},{3},{4},{5},{6},{7}"
                .format(now.strftime("%Y-%m-%dT%H:%M:%S.%fZ"), result.took, c0, d0, c0/(d0*bulksize), c1, d1, c1/(d1*bulksize)))
            c1 = 0
            t1 = time.time()

def main(options):
    es = ES([options.es_server])
    try:
        es.create_index_if_missing('bzcache')
    except ElasticSearchException:
        # create_index_if_missing is supposed not to raise if the index
        # already existing, but with the ancient pyes / ES server versions
        # we're using it still does.
        pass

    # re-cache all intermittent-failure bugs
    bzcache = BugzillaCache(es_server=options.es_server)
    bzcache.index_bugs_by_keyword('intermittent-failure')