def _decode_simple_value(field_cls, data):
    """Used to decode values in stored fields.
    """
    # Overload the Integer type, cf _encode_simple_value
    if issubclass(field_cls, Integer):
        return int(sortable_unserialise(data))
    elif issubclass(field_cls, Decimal):
        return decimal(sortable_unserialise(data))
    # A common field or a new field
    return field_cls.decode(data)

    def two_range(self, field, purpose, q):
        """Check the result of a range search which should return 2 items.

        """
        r = [x for x in q.search(0, 10)]
        self.assertEqual(len(r), 2)
        val = xapian.sortable_unserialise(r[0].get_value("foo", "collsort"))
        self.assertTrue(3 <= val)
        self.assertTrue(val <= 4)
        val = xapian.sortable_unserialise(r[1].get_value("foo", "collsort"))
        self.assertTrue(4 <= val)
        self.assertTrue(val <= 5)

def group_poi(request):
    response = {}
    try:
        response['status'] = 'OK'
        group_type = request.GET.get('gt', 'city_code')
        if group_type == 'admin_code':
            admin_code_spy = xapian.ValueCountMatchSpy(1)
        elif group_type == 'prov_code':
            admin_code_spy = xapian.ValueCountMatchSpy(4)
        else:
            admin_code_spy = xapian.ValueCountMatchSpy(5)
        with contextlib.closing(get_xapian_conn()) as xapian_database:
            poi_query_parser = get_poi_query_parser()
            poi_query_parser.set_database(xapian_database)
            make_group_matches(
                request, poi_query_parser, admin_code_spy, xapian_database)
            group_result = {}
            for value in admin_code_spy.values():
                code = int(xapian.sortable_unserialise(value.term))
                group_result[code] = value.termfreq
            response['results'] = group_result
    except BaseException as e:
        logger.exception(e)
        response['results'] = []
        response['size'] = 0
        response['status'] = 'ERROR_PARAMETERS'
    return json.dumps(response, ensure_ascii=False, encoding='utf-8')

    def _remove_cached_items(self, docid=None, xapid=None):
        """Remove from the cache any items for the specified document.

        The document may be specified by xappy docid, or by xapian document id.

        """
        if self.cache_manager is None:
            raise errors.IndexerError("CacheManager has been applied to this "
                                      "index, but is not currently set.")

        doc, xapid = self._get_xapdoc(docid, xapid)
        if doc is None:
            return

        #print "Removing docid=%d" % xapid
        # FIXME: this will only remove the hits from the set cache
        # manager, if we have multiple applied caches, the others won't be
        # updated.  This means that currently, if multiple caches are applied
        # and document removals happen, some of the caches will get out of
        # date; multiple caches are therefore not really suitable for use in
        # production systems - they are however useful for experimenting with
        # different caching algorithms.
        for value in doc.values():
            base_slot = self._cache_manager_slot_start
            upper_slot = self._cache_manager_slot_start + self.cache_manager.num_cached_queries()
            if not (base_slot <= value.num < upper_slot):
                continue
            rank = int(self._cache_manager_max_hits -
                       xapian.sortable_unserialise(value.value))
            self.cache_manager.remove_hits(
                value.num - self._cache_manager_slot_start,
                ((rank, xapid),))

 def get_popcon(self, doc):
     """ Return a popcon value from a xapian document """
     popcon_raw = doc.get_value(XapianValues.POPCON)
     if popcon_raw:
         popcon = xapian.sortable_unserialise(popcon_raw)
     else:
         popcon = 0
     return popcon

    def single_range(self, field, purpose, q):
        """Check the result of a range search which should return 1 item.

        """
        r = [x for x in q.search(0, 10)]
        self.assertEqual(len(r), 1)
        val = xapian.sortable_unserialise(r[0].get_value(field, purpose))
        self.assertTrue(3 <= val)
        self.assertTrue(val <= 4.01)

 def deconvert(self, data):
     if data is None:
         return data
     if self.ftype == PyFieldMeta.TYPE_LONG:
         data = data or long(0)
         return long(data)
     elif self.ftype == PyFieldMeta.TYPE_FLOAT:
         return xapian.sortable_unserialise(data)
     else:
         return data.decode('utf-8')

 def remove_cached_items(self, iconn, doc, xapid):
     #print "Removing docid=%d" % xapid
     for value in doc.values():
         base_slot = cache_manager_slot_start(iconn, self.id)
         upper_slot = base_slot + self.num_cached_queries()
         if not (base_slot <= value.num < upper_slot):
             continue
         rank = int(CACHE_MANAGER_MAX_HITS -
                    xapian.sortable_unserialise(value.value))
         self.remove_hits(
             value.num - base_slot,
             ((rank, xapid),))

    def extract(self, document):
        if self.number:
            value = document.get_value(self.number)

            content_type = self._get_content_type(value)

            if self._is_float_or_interger(content_type):
                value = xapian.sortable_unserialise(value)

            return value

        return None

    def display_differences(self, ids1, ids2, name1, name2):
        ids1_unique = ids1 - ids2
        ids2_unique = ids2 - ids1
        if ids1_unique or ids2_unique:
            print "results for %s and %s differ" % (name1, name2)
        if ids1_unique:
            print "ids only in %s: " % name1, ids1_unique
        if ids2_unique:
            print "ids only in %s: " % name2, ids2_unique

        for i in ids1 ^ ids2:
            d = self.sconn.get_document(i)
            print "value: ", xapian.sortable_unserialise(d.get_value('price', 'collsort'))
            print "termlist: ", map (lambda t: t.term, d._doc.termlist())

def doc2dict(doc):
    od = OrderedDict()

    url = doc.get_value(VALUE_URL)
    od['url'] = url

    title = doc.get_value(VALUE_TITLE)
    if title:
        od['title'] = title.decode('UTF-8')

    tags = doc.get_value(VALUE_TAGS)
    od['tags'] = tags.decode('UTF-8').split(u'\x1f') if tags else []

    created = xapian.sortable_unserialise(doc.get_value(VALUE_CREATED))
    od['created'] = arrow.get(created)

    archived_val = doc.get_value(VALUE_ARCHIVED)
    if archived_val:
        archived = xapian.sortable_unserialise(archived_val)
        od['archived'] = arrow.get(archived)

    od['notes'] = doc.get_data().decode('UTF-8')

    return od

    def _generate_records(self, mset, select=set(["*"])):
        """
        仅返回item_id,item_type,外部再从memcached、db中读取详细数据
        """
        for m in mset:
            result = {"_did" : m.docid, "_score" : m.percent, "_rank" : m.rank, "_collapse_count" : m.collapse_count, "_weight" : m.weight}
            result['item_id'] = int(xapian.sortable_unserialise(m.document.get_value(DOC_ITEM_ID))) #int
            result['item_type'] = m.document.get_value(DOC_ITEM_TYPE)  #string
            
            if select:
                doc = m.document
                data_str = doc.get_data()
                if len(data_str):
                    data_dict = cPickle.loads(data_str)
                    for key, value in data_dict.items():
                        if key in select or "*" in select:
                            result[key] = value

            yield result

    def size(self):
        """Return the size of the application without dependencies

        Note that this will return the download size if the app is
        not installed and the installed size if it is installed.
        """
        if self._pkg:
            if not self._pkg.installed:
                if self._app.archive_suite:
                    ver = self._get_version_for_archive_suite(self._pkg, self._app.archive_suite)
                    if ver:
                        return ver.size
                return self._pkg.candidate.size
            else:
                return self._pkg.installed.size
        elif self._doc:
            size = self._doc.get_value(XapianValues.DOWNLOAD_SIZE)
            if size:
                return xapian.sortable_unserialise(self._doc.get_value(XapianValues.DOWNLOAD_SIZE))

    def _remove_cached_items(self, docid=None, xapid=None):
        """Remove from the cache any items for the specified document.

        The document may be specified by xappy docid, or by xapian document id.

        """
        if self.cache_manager is None:
            raise errors.IndexerError("CacheManager has been applied to this "
                                      "index, but is not currently set.")

        doc, xapid = self._get_xapdoc(docid, xapid)
        if doc is None:
            return

        #print "Removing docid=%d" % xapid
        for value in doc.values():
            if value.num < self._cache_manager_slot_start:
                continue
            rank = int(self._cache_manager_max_hits -
                       xapian.sortable_unserialise(value.value))
            self.cache_manager.remove_hits(
                value.num - self._cache_manager_slot_start,
                ((rank, xapid),))

 def get_weight(self, doc):
     val = doc.get_value(self.field, self.purpose)
     val = xapian.sortable_unserialise(val)
     if val > self.maxval:
         return self.maxval
     return val

#.........这里部分代码省略.........
    expect(term, b"out\xe9r")

    # Check simple stopper
    stop = xapian.SimpleStopper()
    qp.set_stopper(stop)
    expect(stop(b'a'), False)
    expect_query(qp.parse_query(b"foo bar a", qp.FLAG_BOOLEAN),
                 "([email protected] AND [email protected] AND [email protected])")

    stop.add(b'a')
    expect(stop(b'a'), True)
    expect_query(qp.parse_query(b"foo bar a", qp.FLAG_BOOLEAN),
                 "([email protected] AND [email protected])")

    # Feature test for custom Stopper
    class my_b_stopper(xapian.Stopper):
        def __call__(self, term):
            return term == b"b"

        def get_description(self):
            return "my_b_stopper"

    stop = my_b_stopper()
    expect(stop.get_description(), "my_b_stopper")
    qp.set_stopper(stop)
    expect(stop(b'a'), False)
    expect_query(qp.parse_query(b"foo bar a", qp.FLAG_BOOLEAN),
                 "([email protected] AND [email protected] AND [email protected])")

    expect(stop(b'b'), True)
    expect_query(qp.parse_query(b"foo bar b", qp.FLAG_BOOLEAN),
                 "([email protected] AND [email protected])")

    # Test TermGenerator
    termgen = xapian.TermGenerator()
    doc = xapian.Document()
    termgen.set_document(doc)
    termgen.index_text(b'foo bar baz foo')
    expect([(item.term, item.wdf, [pos for pos in item.positer]) for item in doc.termlist()], [(b'bar', 1, [2]), (b'baz', 1, [3]), (b'foo', 2, [1, 4])])


    # Check DateValueRangeProcessor works
    context("checking that DateValueRangeProcessor works")
    qp = xapian.QueryParser()
    vrpdate = xapian.DateValueRangeProcessor(1, 1, 1960)
    qp.add_valuerangeprocessor(vrpdate)
    query = qp.parse_query(b'12/03/99..12/04/01')
    expect(str(query), 'Query(0 * VALUE_RANGE 1 19991203 20011204)')

    # Regression test for bug#193, fixed in 1.0.3.
    context("running regression test for bug#193")
    vrp = xapian.NumberValueRangeProcessor(0, b'$', True)
    a = '$10'
    b = '20'
    slot, a, b = vrp(a, b.encode('utf-8'))
    expect(slot, 0)
    expect(xapian.sortable_unserialise(a), 10)
    expect(xapian.sortable_unserialise(b), 20)

    # Feature test for xapian.FieldProcessor
    context("running feature test for xapian.FieldProcessor")
    class testfieldprocessor(xapian.FieldProcessor):
        def __call__(self, s):
            if s == 'spam':
                raise Exception('already spam')
            return xapian.Query("spam")

    qp.add_prefix('spam', testfieldprocessor())
    qp.add_boolean_prefix('boolspam', testfieldprocessor())
    query = qp.parse_query('spam:ignored')
    expect(str(query), 'Query(spam)')

    # FIXME: This doesn't currently work:
    # expect_exception(Exception, 'already spam', qp.parse_query, 'spam:spam')

    # Regression tests copied from PHP (probably always worked in python, but
    # let's check...)
    context("running regression tests for issues which were found in PHP")

    # PHP overload resolution involving boolean types failed.
    enq.set_sort_by_value(1, True)

    # Regression test - fixed in 0.9.10.1.
    oqparser = xapian.QueryParser()
    oquery = oqparser.parse_query(b"I like tea")

    # Regression test for bug#192 - fixed in 1.0.3.
    enq.set_cutoff(100)

    # Test setting and getting metadata
    expect(db.get_metadata(b'Foo'), b'')
    db.set_metadata(b'Foo', b'Foo')
    expect(db.get_metadata(b'Foo'), b'Foo')
    expect_exception(xapian.InvalidArgumentError, "Empty metadata keys are invalid", db.get_metadata, b'')
    expect_exception(xapian.InvalidArgumentError, "Empty metadata keys are invalid", db.set_metadata, b'', b'Foo')
    expect_exception(xapian.InvalidArgumentError, "Empty metadata keys are invalid", db.get_metadata, b'')

    # Test OP_SCALE_WEIGHT and corresponding constructor
    expect_query(xapian.Query(xapian.Query.OP_SCALE_WEIGHT, xapian.Query(b'foo'), 5),
                 "5 * foo")

    #parser.set_stemming_strategy(xapian.QueryParser.STEM_ALL)
    parser.set_database(db)
    #parser.add_prefix("pkg", "AP")
    query = parser.parse_query(search_term, 
                               xapian.QueryParser.FLAG_PARTIAL|
                               xapian.QueryParser.FLAG_WILDCARD)

    enquire = xapian.Enquire(db)
    enquire.set_sort_by_value_then_relevance(XAPIAN_VALUE_POPCON)
    enquire.set_query(query)
    matches = enquire.get_mset(0, db.get_doccount())
    print "Matches:"
    for m in matches:
        doc = m.document
        popcon = doc.get_value(XAPIAN_VALUE_POPCON)
        print doc.get_data(), "popcon:", xapian.sortable_unserialise(popcon)
        #for t in doc.termlist():
        #    print "'%s': %s (%s); " % (t.term, t.wdf, t.termfreq),
        #print "\n"
        appname = doc.get_data()
    
    # calculate a eset
    print "ESet:"
    rset = xapian.RSet()
    for m in matches:
        rset.add_document(m.docid)
    for m in enquire.get_eset(10, rset):
        print m.term


    # calulate the expansions

#.........这里部分代码省略.........
    expect_query(qp.parse_query("foo o", qp.FLAG_PARTIAL), "([email protected] AND (([email protected] SYNONYM [email protected]) OR [email protected]))")

    expect_query(qp.parse_query("foo outside", qp.FLAG_PARTIAL), "([email protected] AND [email protected])")

    # Test supplying unicode strings
    expect_query(xapian.Query(xapian.Query.OP_OR, (u"foo", u"bar")), "(foo OR bar)")
    expect_query(xapian.Query(xapian.Query.OP_OR, ("foo", u"bar\xa3")), "(foo OR bar\xc2\xa3)")
    expect_query(xapian.Query(xapian.Query.OP_OR, ("foo", "bar\xc2\xa3")), "(foo OR bar\xc2\xa3)")
    expect_query(xapian.Query(xapian.Query.OP_OR, u"foo", u"bar"), "(foo OR bar)")

    expect_query(
        qp.parse_query(u"NOT t\xe9st", qp.FLAG_BOOLEAN + qp.FLAG_PURE_NOT), "(<alldocuments> AND_NOT Zt\xc3\[email protected])"
    )

    doc = xapian.Document()
    doc.set_data(u"Unicode with an acc\xe9nt")
    doc.add_posting(stem(u"out\xe9r"), 1)
    expect(doc.get_data(), u"Unicode with an acc\xe9nt".encode("utf-8"))
    term = doc.termlist().next().term
    expect(term, u"out\xe9r".encode("utf-8"))

    # Check simple stopper
    stop = xapian.SimpleStopper()
    qp.set_stopper(stop)
    expect(stop("a"), False)
    expect_query(qp.parse_query(u"foo bar a", qp.FLAG_BOOLEAN), "([email protected] AND [email protected] AND [email protected])")

    stop.add("a")
    expect(stop("a"), True)
    expect_query(qp.parse_query(u"foo bar a", qp.FLAG_BOOLEAN), "([email protected] AND [email protected])")

    # Feature test for custom Stopper
    class my_b_stopper(xapian.Stopper):
        def __call__(self, term):
            return term == "b"

        def get_description(self):
            return u"my_b_stopper"

    stop = my_b_stopper()
    expect(stop.get_description(), u"my_b_stopper")
    qp.set_stopper(stop)
    expect(stop("a"), False)
    expect_query(qp.parse_query(u"foo bar a", qp.FLAG_BOOLEAN), "([email protected] AND [email protected] AND [email protected])")

    expect(stop("b"), True)
    expect_query(qp.parse_query(u"foo bar b", qp.FLAG_BOOLEAN), "([email protected] AND [email protected])")

    # Test TermGenerator
    termgen = xapian.TermGenerator()
    doc = xapian.Document()
    termgen.set_document(doc)
    termgen.index_text("foo bar baz foo")
    expect(
        [(item.term, item.wdf, [pos for pos in item.positer]) for item in doc.termlist()],
        [("bar", 1, [2]), ("baz", 1, [3]), ("foo", 2, [1, 4])],
    )

    # Check DateValueRangeProcessor works
    context("checking that DateValueRangeProcessor works")
    qp = xapian.QueryParser()
    vrpdate = xapian.DateValueRangeProcessor(1, 1, 1960)
    qp.add_valuerangeprocessor(vrpdate)
    query = qp.parse_query("12/03/99..12/04/01")
    expect(str(query), "Query(0 * VALUE_RANGE 1 19991203 20011204)")

    # Regression test for bug#193, fixed in 1.0.3.
    context("running regression test for bug#193")
    vrp = xapian.NumberValueRangeProcessor(0, "$", True)
    a = "$10"
    b = "20"
    slot, a, b = vrp(a, b)
    expect(slot, 0)
    expect(xapian.sortable_unserialise(a), 10)
    expect(xapian.sortable_unserialise(b), 20)

    # Regression tests copied from PHP (probably always worked in python, but
    # let's check...)
    context("running regression tests for issues which were found in PHP")

    # PHP overload resolution involving boolean types failed.
    enq.set_sort_by_value(1, True)

    # Regression test - fixed in 0.9.10.1.
    oqparser = xapian.QueryParser()
    oquery = oqparser.parse_query("I like tea")

    # Regression test for bug#192 - fixed in 1.0.3.
    enq.set_cutoff(100)

    # Test setting and getting metadata
    expect(db.get_metadata("Foo"), "")
    db.set_metadata("Foo", "Foo")
    expect(db.get_metadata("Foo"), "Foo")
    expect_exception(xapian.InvalidArgumentError, "Empty metadata keys are invalid", db.get_metadata, "")
    expect_exception(xapian.InvalidArgumentError, "Empty metadata keys are invalid", db.set_metadata, "", "Foo")
    expect_exception(xapian.InvalidArgumentError, "Empty metadata keys are invalid", db.get_metadata, "")

    # Test OP_SCALE_WEIGHT and corresponding constructor
    expect_query(xapian.Query(xapian.Query.OP_SCALE_WEIGHT, xapian.Query("foo"), 5), "5 * foo")

import heapq
import os
import sys
import xapian

sys.path.insert(0, "../")
from softwarecenter.enums import *
from softwarecenter.utils import *

if __name__ == "__main__":

    topn = 20
    if len(sys.argv) > 1:
        topn = int(sys.argv[1])

    pathname = os.path.join(XAPIAN_BASE_PATH, "xapian")
    db = xapian.Database(pathname)

    heap = []
    for m in db.postlist(""):
        doc = db.get_document(m.docid)
        pkgname = doc.get_value(XAPIAN_VALUE_PKGNAME)
        appname = doc.get_value(XAPIAN_VALUE_APPNAME)
        summary = doc.get_value(XAPIAN_VALUE_SUMMARY)
        popcon = xapian.sortable_unserialise(doc.get_value(XAPIAN_VALUE_POPCON))
        heapq.heappush(heap, (popcon, appname, pkgname, summary))

    for (popcon, appname, pkgname, summary) in heapq.nlargest(topn, heap):
        print "[%i] %s - %s [%s]" % (popcon, appname, summary, pkgname)

    def query(self, querystring=None, qtype=None, begin=None, end=None, keywords=[], hashtags=[], synonymslist=[], emotiononly=False):
        if qtype == 'hy':
            self.qp.add_valuerangeprocessor(xapian.NumberValueRangeProcessor(self.timestampvi, ''))
            querystring = begin + '..' + end

            if emotiononly:
                self.qp.add_valuerangeprocessor(xapian.NumberValueRangeProcessor(self.emotiononlyvi, 'f', False))
                querystring += ' 1.0..1.0f'

            query = self.qp.parse_query(querystring)
            print "Parsed query is: %s" % [str(query)]

            self.enquire.set_query(query)
            #matches = self.enquire.get_mset(0, self.maxitems)
            matches = self.enquire.get_mset(0, 10000)
            # Display the results.
            print "%i results found." % matches.size()

            if not self.lowkeywords_proc(matches):
                return
            emotions_list, keywords_list = self.keywords_and_emotions_list_proc(matches)

            return emotions_list, keywords_list

        if qtype == 'yq':
            self.qp.add_valuerangeprocessor(xapian.NumberValueRangeProcessor(self.timestampvi, ''))
            querystring = begin + '..' + end
            query = self.qp.parse_query(querystring)
            print "Parsed query is: %s" % [str(query)]

            self.enquire.set_query(query)
            #matches = self.enquire.get_mset(0,10)
            matches = self.enquire.get_mset(0, self.maxitems)

            # Display the results.
            print "%i results found." % matches.size()

            keywords_arr = []
            for m in matches:
                #hashtag
                hashtags = json.loads(m.document.get_value(self.hashtagsvi))

                #keywords
                keywords_hash = json.loads(m.document.get_value(self.keywordsvi))
                keywords_arr.append(keywords_hash)
                #keywords_counter += Counter(json.loads(m.document.get_value(self.keywordsvi)))

            print 'mapreduce begin: ', str(time.strftime("%H:%M:%S", time.gmtime()))
            mapper = SimpleMapReduce(hasharr_to_list, count_words)
            word_counts = mapper(keywords_arr)
            keywords_hash = {}
            for word, count in word_counts:
                keywords_hash[word] = count
            for synonyms in synonymslist:
                if len(synonyms) >= 2 and synonyms[0] in keywords_hash:
                    for word in synonyms[1:]:
                        if word in keywords_hash:
                            keywords_hash[synonyms[0]] += keywords_hash[word]
                            del keywords_hash[word]
            print 'mapreduce end: ', str(time.strftime("%H:%M:%S", time.gmtime()))

            #print keywords_counter
            return hashtags, keywords_hash

        if qtype == 'lh':
            self.qp.add_valuerangeprocessor(xapian.NumberValueRangeProcessor(self.timestampvi, ''))
            timequerystr = begin + '..' + end
            timequery = self.qp.parse_query(timequerystr)

            hashtags = ['H' + hashtag.lower() for hashtag in hashtags]
            keywords = [keyword.lower() for keyword in keywords]
            keywords.extend(hashtags)
            if len(keywords) > 0:
                wordsquery = xapian.Query(xapian.Query.OP_OR, keywords)
            else:
                return None

            query = xapian.Query(xapian.Query.OP_AND, [timequery, wordsquery])
            print "Parsed query is: %s" % [str(query)]

            self.enquire.set_query(query)
            self.enquire.set_sort_by_value(self.timestampvi, False)
            #matches = self.enquire.get_mset(0,10)
            matches = self.enquire.get_mset(0, self.maxitems)

            # Display the results.
            print "%i results found." % matches.size()

            results = []
            for m in matches:
                result = {}
                result['location'] = m.document.get_value(self.loctvi)
                result['repost_location'] = m.document.get_value(self.reploctvi)
                result['timestamp'] = xapian.sortable_unserialise(m.document.get_value(self.timestampvi))
                results.append(result)

            return results