def main():
    #Train the Naive Bayes Classifier
    f=open('./data_set/naivebayes_trained_model.pickle')
    NBClassifier=pickle.load(f)

    #ElasticSearch- Call the es_indexer file to create 'sentiment_analysis' index and store
    #the contents of the tweet file in that Index
    
    es=ElasticSearch('http://localhost:9200/')
    es_indexer()
    ############Indexing into Elasticsearch############
    i=0
    for each in tweet_data():
        i+=1
        testTweet= each
        processedTestTweet=process_tweet(testTweet)
        sentiment=NBClassifier.classify(extract_features(build_feature_vector(processedTestTweet)))
    
        
        es.index("sentiment_analysis","document",{
                     "text": testTweet,
                     "sentiment": sentiment
                         },id=i)
    print "Indexing completed."

    es.refresh(index="sentiment_analysis")
    print "Index refreshed."

    f.close()

    def _get(self):
        """Build and run the ES query
        """
        opts = self.opts

        es = ElasticSearch(opts.url)
        query = {'sort': {'@timestamp': 'desc'},
                 'size': 1}

        if opts.query:
            query['query'] = {
                'filtered': {
                    'query': {
                        'query_string': {
                            'query': opts.query
                        }
                    }
                }
            }

        # ElasticSearch allows us to pass an array of indices. However,
        # it will throw an exception if any of these don't exist. This
        # isn't the right behavior, because there may not actually be
        # a logstash index from X days ago. Instead, we need to iterate
        # through the daily log indexes in reverse order until we get a
        # non-error response.
        result = None
        for index in self._indexes():
            try:
                result = es.search(query, index=index)
                break
            except ElasticHttpNotFoundError, e:
                pass

class ESPipeline(object):
    def __init__(self, *args, **kwargs):
        self.client = ElasticSearch('http://localhost:9200/')

    def process_item(self, item, spider):
        self.client.index('wiki', 'page', dict(item))
        return item

def analyze_post(token, text):
    response = {
        'post_now': False,
        'hours_to_wait': 1,
        'total_score': 0,
        'time_score': 0,
        'text_score': 0,
        'hint': "Building index",
    }

    try:
        data = Newsfeed.filter_only_posts_by_people(token)

    except Exception, e:
        es = ElasticSearch('http://localhost:9200/')

        try:
            es.create_index(token.lower())
            Newsfeed.newsfeed(token, [], 0, None, 1)

            t = threading.Thread(target=Newsfeed.newsfeed, args=(token, [], 0, None, 1500))
            t.setDaemon(True)
            t.start()

        except Exception, e:
            print e.message

class ESDiffs(object):
    """Implementation of Elastic Search as diff backend"""
    def __init__(self):
        self.es = ElasticSearch(settings.ELASTIC_SEARCH_URLS)

    @staticmethod
    def to_id(label, old, new):
        return "%s/%s/%s" % (label, old, new)

    def put(self, label, old_version, new_version, diff):
        """Store a diff between two versions of a regulation node"""
        struct = {
            'label': label,
            'old_version': old_version,
            'new_version': new_version,
            'diff': diff
        }
        self.es.index(settings.ELASTIC_SEARCH_INDEX, 'diff', struct,
                      id=self.to_id(label, old_version, new_version))

    def get(self, label, old_version, new_version):
        """Find the associated diff"""
        try:
            result = self.es.get(settings.ELASTIC_SEARCH_INDEX, 'diff',
                                 self.to_id(label, old_version, new_version))
            return result['_source']['diff']
        except ElasticHttpNotFoundError:
            return None

def get_documents(urls):
    host =  environ['ELASTICSEARCH_SERVER'] if environ.get('ELASTICSEARCH_SERVER') else 'http://localhost:9200'
    es = ElasticSearch(host)
        
    if len(urls) > 0:
        results = {}

        for url in urls:
            query = {
                "query": {
                    "term": {
                        "url": url
                    }
                },
                "fields": ["text"]
            }
        
            res = es.search(query, 
                            index=environ['ELASTICSEARCH_INDEX'] if environ.get('ELASTICSEARCH_INDEX') else 'memex', 
                            doc_type=environ['ELASTICSEARCH_DOC_TYPE'] if environ.get('ELASTICSEARCH_DOC_TYPE') else 'page')
            hits = res['hits']
            try:
                results[url] = hits['hits'][0]['fields']['text'][0]
            except KeyError, e:
                print url, e, " not found in database"
            except IndexError, e:
                print url, e, " not found in database"

def term_search(field, queryStr):
    es_server = 'http://localhost:9200/'
    if environ.get('ELASTICSEARCH_SERVER'):
        es_server = environ['ELASTICSEARCH_SERVER']
    es = ElasticSearch(es_server)

    if len(queryStr) > 0:
        query = {
            "query" : {
                "match": {
                    field: {
                        "query": ' '.join(queryStr),
                        "minimum_should_match":"100%"
                    }
                }
            },
            "fields": ["url"]
        }
        print query
        res = es.search(query, 
                        index=environ['ELASTICSEARCH_INDEX'] if environ.get('ELASTICSEARCH_INDEX') else 'memex', 
                        doc_type=environ['ELASTICSEARCH_DOC_TYPE'] if environ.get('ELASTICSEARCH_DOC_TYPE') else 'page',
                        size=500)

        hits = res['hits']
        urls = []
        for hit in hits['hits']:
            urls.append(hit['_id'])
        return urls

def search(field, queryStr):
    es_server = 'http://localhost:9200/'
    es_index = 'memex'
    es_doc_type = 'page'
    if environ.get('ELASTICSEARCH_SERVER'):
        es_server = environ['ELASTICSEARCH_SERVER']
    if environ.get('ELASTICSEARCH_INDEX'):
        es_index = environ['ELASTICSEARCH_INDEX']
    if environ.get('ELASTICSEARCH_DOC_TYPE'):
        es_doc_type = environ['ELASTICSEARCH_DOC_TYPE']
        
    es = ElasticSearch(es_server)

    if len(queryStr) > 0:
        query = {
            "query": {
                "query_string": {
                    "fields" : [field],
                    "query": ' and  '.join(queryStr[0:]),
                }
            },
            "fields": [field]
        }
        print query
        res = es.search(query, index=es_index, doc_type=es_doc_type, size=500)
        hits = res['hits']
        urls = []
        for hit in hits['hits']:
            urls.append(hit['_id'])
        return urls

class ESNotices(object):
    """Implementation of Elastic Search as notice backend"""
    def __init__(self):
        self.es = ElasticSearch(settings.ELASTIC_SEARCH_URLS)

    def put(self, doc_number, notice):
        """Store a single notice"""
        self.es.index(settings.ELASTIC_SEARCH_INDEX, 'notice', notice,
                      id=doc_number)

    def get(self, doc_number):
        """Find the associated notice"""
        try:
            result = self.es.get(settings.ELASTIC_SEARCH_INDEX, 'notice',
                                 doc_number)

            return result['_source']
        except ElasticHttpNotFoundError:
            return None

    def listing(self, part=None):
        """All notices or filtered by cfr_part"""
        if part:
            query = {'match': {'cfr_part': part}}
        else:
            query = {'match_all': {}}
        query = {'fields': ['effective_on', 'fr_url', 'publication_date'],
                 'query': query}
        notices = []
        results = self.es.search(query, doc_type='notice', size=100,
                                 index=settings.ELASTIC_SEARCH_INDEX)
        for notice in results['hits']['hits']:
            notice['fields']['document_number'] = notice['_id']
            notices.append(notice['fields'])
        return notices

def get_image(url, output_path=""):
    es_server = 'http://localhost:9200/'
    if environ.get('ELASTICSEARCH_SERVER'):
        es_server = environ['ELASTICSEARCH_SERVER']
    es = ElasticSearch(es_server)

    if output_path:
        output_path = output_path+'/'

    if url:
        query = {
            "query": {
                "term": {
                    "url": url
                }
            },
            "fields": ["thumbnail"]
        }
        res = es.search(query, index='memex', doc_type='page')
        hits = res['hits']
        if (len(hits) > 0):
            img = base64.b64decode(hits['hits'][0]['fields']['thumbnail'][0])
            with open(output_path+urllib2.quote(url).replace("/", "%2F")+'.png','wb') as f:
                f.write(img)
        else:
            print "No thumbnail found"

def get_image(url, es_index='memex', es_doc_type='page', es=None):
    if es is None:
        es = ElasticSearch("http://localhost:9200")

    if url:
        query = {
            "query": {
                "term": {
                    "url": url
                }
            },
            "fields": ["thumbnail", "thumbnail_name"]
        }
        res = es.search(query, index=es_index, doc_type=es_doc_type, size=500)

        hits = res['hits']['hits']
        if (len(hits) > 0):
            try:
                img = base64.b64decode(hits[0]['fields']['thumbnail'][0])
                img_name = hits[0]['fields']['thumbnail_name'][0]
                return [img_name, img]
            except KeyError:
                print "No thumbnail found"
        else:
            print "No thumbnail found"
    return [None, None]

def get_context(terms, es_index='memex', es_doc_type='page', es=None):
    if es is None:
        es = ElasticSearch("http://localhost:9200")

    if len(terms) > 0:
        query = {
            "query": { 
                "match": {
                    "text": {
                        "query": ' and  '.join(terms[0:]),
                        "operator" : "and"
                    }
                }
             },
            "highlight" : {
                "fields" : {
                    "text": {
                        "fragment_size" : 100, "number_of_fragments" : 1
                    }
                }
            }
        }
        print query
        res = es.search(query, index=es_index, doc_type=es_doc_type, size=500)
        hits = res['hits']

        highlights = []
        for hit in hits['hits']:
            highlights.append(hit['highlight']['text'][0])
        return highlights

def range(field, from_val, to_val, ret_fields=[], epoch=None, es_index='memex', es_doc_type='page', es=None):
    if es is None:
        es = ElasticSearch("http://localhost:9200")

    if not (epoch is None):
        if epoch:
            from_val = datetime.utcfromtimestamp(long(from_val)).strftime('%Y-%m-%dT%H:%M:%S')
            to_val = datetime.utcfromtimestamp(long(to_val)).strftime('%Y-%m-%dT%H:%M:%S')
            
    query = { 
        "query" : { 
            "range" : { 
                field : {
                    "from": from_val,
                    "to": to_val
                }
            },
        },
        "fields": ret_fields
    }

    res = es.search(query, index=es_index, doc_type=es_doc_type, size=500)
    hits = res['hits']['hits']

    results=[]
    for hit in hits:
        results.append(hit['fields'])

    return results

def get_documents(terms, term_field, fields=["text"], es_index='memex', es_doc_type='page', es=None):
    if es is None:
        es = ElasticSearch('http://localhost:9200/')

    if len(terms) > 0:
        results = {}

        for term in terms:
            query = {
                "query": {
                    "term": {
                        term_field: term
                    }
                },
                "fields": fields
            }
        
            res = es.search(query, 
                            index=es_index,
                            doc_type=es_doc_type)

            if res['hits']['hits']:
                hits = res['hits']['hits'][0]

                if not hits.get('fields') is None:
                    hits = hits['fields']
                    record = {}
                    for field in fields:
                        if(not hits.get(field) is None):
                            record[field] = hits[field][0]
                    results[term] = record           
            
    return results

    def set_in_index(self, documentList):
        """
        Store the list of documents in the Elasticsearch index via HTTP APIs

        @type  documentList: List
        @param documentList: List of image layer JSON documents
        """
        #Get the Elasticsearch address from the config file
        cfg = config.load()

        #Store the document list in Elasticsearch
        es = ElasticSearch(cfg.search_options.get("address"))
        try:
            es.bulk_index(cfg.search_options.get("index"), cfg.search_options.get("type"), documentList, id_field='id')
        except InvalidJsonResponseError:
            logger.debug("InvalidJsonResponseError!")
        except Timeout:
            logger.debug("Timeout!")
        except ConnectionError:
            logger.debug("ConnectionError!")
        except ElasticHttpError:
            logger.debug("ElasticHttpError!")
        except InvalidJsonResponseError:
            logger.debug("InvalidJsonResponseError!")
        except ElasticHttpNotFoundError:
            logger.debug("ElasticHttpNotFoundError!")            

def search(request, doc_type, search_args):
    """Search elastic search for any matches in the node's text"""
    query = {
        'fields': ['text', 'label', 'version', 'regulation', 'title',
                   'label_string'],
        'from': search_args.page * search_args.page_size,
        'size': search_args.page_size,
    }
    text_match = {'match': {'text': search_args.q, 'doc_type': doc_type}}
    if search_args.version or search_args.regulation:
        term = {}
        if search_args.version:
            term['version'] = search_args.version
        if search_args.regulation:
            term['regulation'] = search_args.regulation
        if search_args.is_root is not None:
            term['is_root'] = search_args.is_root
        if search_args.is_subpart is not None:
            term['is_subpart'] = search_args.is_subpart
        query['query'] = {'filtered': {
            'query': text_match,
            'filter': {'term': term}
        }}
    else:
        query['query'] = text_match
    es = ElasticSearch(settings.ELASTIC_SEARCH_URLS)
    results = es.search(query, index=settings.ELASTIC_SEARCH_INDEX)

    return success({
        'total_hits': results['hits']['total'],
        'results': transform_results([h['fields'] for h in
                                      results['hits']['hits']])
    })

def get_available_domains(es=None):
    if es is None:
        es = ElasticSearch("http://localhost:9200")
        
    query = {
        "query": {
            "match_all": {}
        },
    }
    res = es.search(query, 
                    index='config',
                    doc_type='domains',
                    size=100
                )

    hits = res['hits']['hits']

    res = []
    for hit in hits:
        res.append(hit['_source'])

    for i in range(0,len(res)):
        res[i]['timestamp'] = long(convert_to_epoch(datetime.strptime(res[i]['timestamp'], '%Y-%m-%dT%H:%M:%S.%f')))
        print datetime.utcfromtimestamp(res[i]['timestamp'])
    return res

def term_search(field, queryStr):
    es_server = 'http://localhost:9200/'
    if environ.get('ELASTICSEARCH_SERVER'):
        es_server = environ['ELASTICSEARCH_SERVER']
    es = ElasticSearch(es_server)

    if len(queryStr) > 0:
        query = {
            "query" : {
                "match": {
                    field: {
                        "query": queryStr,
                        "operator" : "and"
                        }
                    }
                },
            "fields": ["url"]
            }
        print query
        res = es.search(query, index='memex', doc_type='page', size=500)
        hits = res['hits']
        urls = []
        for hit in hits['hits']:
            urls.append(hit['_id'])
        return urls

class ElasticSearchBackend(BaseBackend):
    
    def __init__(self, es_url='http://localhost:9200/', batch_size=10, **kwargs):
        """
        Do what is necessary to create/open the index.
        """
        self.batch_size = batch_size
        self.batch_count = 0
        self.es_url = es_url
        self.fast = kwargs.get('fast', False)
        if kwargs.get('noisy', False):
            from logging import getLogger, StreamHandler, DEBUG
            import sys
            logger = getLogger('pyelasticsearch')
            logger.setLevel(DEBUG)
            logger.addHandler(StreamHandler(sys.stdout))
            
        self.es = ElasticSearch(self.es_url)
        try:
            self.es.count('*')
        except ConnectionError:
            print "Error connecting to ElasticSearch server!"
            raise
        self.urls = defaultdict(set) #track urls to be deleted before committing new content
        self.batches = defaultdict(list) #site: [list of docs]
    
    def create_index(self, name):
        name = name.lower()
        try:
            self.es.create_index(name)
            self.update_mapping(name)
        except Exception, e:
            print e
            return

def get_context(terms):
    es_server = 'http://localhost:9200/'
    if environ.get('ELASTICSEARCH_SERVER'):
        es_server = environ['ELASTICSEARCH_SERVER']
    es = ElasticSearch(es_server)

    if len(terms) > 0:

        query = {
            "query": { 
                "match": {
                    "text": {
                        "query": ' and  '.join(terms[0:]),
                        "operator" : "and"
                    }
                }
             },
            "highlight" : {
                "fields" : {
                    "text": {
                        "fragment_size" : 100, "number_of_fragments" : 1
                    }
                }
            }
        }
        print query
        res = es.search(query, index='memex', doc_type='page')
        hits = res['hits']
        print 'Document found: %d' % hits['total']
        highlights = []
        for hit in hits['hits']:
            highlights.append(hit['highlight']['text'])
        return highlights