def test_bigtable():
    with TempStorage("bigtable") as st:
        def randstring(min, max):
            return "".join(chr(randint(1, 255))
                           for _ in xrange(randint(min, max)))

        count = 100000
        samp = dict((randstring(1,50), randstring(1,50))
                    for _ in xrange(count))

        fhw = HashWriter(st.create_file("big.hsh"))
        fhw.add_all(iteritems(samp))
        fhw.close()

        fhr = HashReader(st.open_file("big.hsh"))
        keys = list(samp.keys())
        shuffle(keys)
        for key in keys:
            assert_equal(samp[key], fhr[key])

        set1 = set(iteritems(samp))
        set2 = set(fhr.items())
        assert_equal(set1, set2)

        fhr.close()

 def embed(self, other):
     # Copy all transitions from the other NFA into this one
     for s, othertrans in iteritems(other.transitions):
         trans = self.transitions.setdefault(s, {})
         for label, otherdests in iteritems(othertrans):
             dests = trans.setdefault(label, set())
             dests.update(otherdests)

def reverse_nfa(n):
    s = object()
    nfa = NFA(s)
    for src, trans in iteritems(n.transitions):
        for label, destset in iteritems(trans):
            for dest in destset:
                nfa.add_transition(dest, label, src)
    for finalstate in n.final_states:
        nfa.add_transition(s, EPSILON, finalstate)
    nfa.add_final_state(n.initial)
    return nfa

    def expanded_terms(self, number, normalize=True):
        """Returns the N most important terms in the vectors added so far.
        
        :param number: The number of terms to return.
        :param normalize: Whether to normalize the weights.
        :returns: A list of ("term", weight) tuples.
        """

        model = self.model
        fieldname = self.fieldname
        ixreader = self.ixreader
        tlist = []
        maxweight = 0

        # If no terms have been added, return an empty list
        if not self.topN_weight:
            return []

        for word, weight in iteritems(self.topN_weight):
            if (fieldname, word) in ixreader:
                cf = ixreader.frequency(fieldname, word)
                score = model.score(weight, cf, self.top_total)
                if score > maxweight:
                    maxweight = score
                tlist.append((score, word))

        if normalize:
            norm = model.normalizer(maxweight, self.top_total)
        else:
            norm = maxweight
        tlist = [(weight / norm, t) for weight, t in tlist]
        tlist.sort(key=lambda x: (0 - x[0], x[1]))

        return [(t, weight) for weight, t in tlist[:number]]

    def expanded_terms(self, number, normalize=True):
        """Returns the N most important terms in the vectors added so far.
        
        :param number: The number of terms to return.
        :param normalize: Whether to normalize the weights.
        :returns: A list of ("term", weight) tuples.
        """

        model = self.model
        tlist = []
        maxweight = 0
        collection_freq = self.collection_freq

        for word, weight in iteritems(self.topN_weight):
            if word in collection_freq:
                score = model.score(weight, collection_freq[word],
                                    self.top_total)
                if score > maxweight:
                    maxweight = score
                tlist.append((score, word))

        if normalize:
            norm = model.normalizer(maxweight, self.top_total)
        else:
            norm = maxweight
        tlist = [(weight / norm, t) for weight, t in tlist]
        tlist.sort(key=lambda x: (0 - x[0], x[1]))

        return [(t, weight) for weight, t in tlist[:number]]

 def from_file(cls, dbfile, doccount=None):
     obj = cls()
     obj._read_header(dbfile, doccount)
     for fieldname, start in iteritems(obj.starts):
         obj.lengths[fieldname] = dbfile.get_array(start, "B", obj._count)
     dbfile.close()
     return obj

def test_random_hash():
    with TempStorage("randomhash") as st:
        domain = "abcdefghijklmnopqrstuvwxyz"
        domain += domain.upper()
        times = 1000
        minlen = 1
        maxlen = len(domain)
        
        samp = dict((randstring(domain, minlen, maxlen),
                     randstring(domain, minlen, maxlen)) for _ in xrange(times))
        
        hwf = st.create_file("test.hsh")
        hw = HashWriter(hwf)
        for k, v in iteritems(samp):
            hw.add(k, v)
        hw.close()
        
        keys = list(samp.keys())
        random.shuffle(keys)
        hrf = st.open_file("test.hsh")
        hr = HashReader(hrf)
        for k in keys:
            v = hr[k]
            assert_equal(v, b(samp[k]))
        hr.close()

    def word_values(self, value, analyzer, **kwargs):
        fb = self.field_boost
        seen = defaultdict(list)

        kwargs["positions"] = True
        kwargs["chars"] = True
        kwargs["boosts"] = True
        for t in tokens(value, analyzer, kwargs):
            seen[t.text].append((t.pos, t.startchar, t.endchar, t.boost))

        for w, poses in iteritems(seen):
            # posns_chars_boosts = [(pos, startchar, endchar, boost), ...]
            codes = []
            posbase = 0
            charbase = 0
            summedboost = 0
            for pos, startchar, endchar, boost in poses:
                codes.append((pos - posbase, startchar - charbase,
                              endchar - startchar, boost))
                posbase = pos
                charbase = endchar
                summedboost += boost

            value = (pack_uint(len(poses)) + pack_float(summedboost * fb)
                     + dumps(codes, -1)[2:-1])

            yield (w, len(poses), summedboost * fb, value)

def test_random_hash():
    from string import ascii_letters as domain

    times = 1000
    minlen = 1
    maxlen = len(domain)

    def randstring():
        s = "".join(random.sample(domain, random.randint(minlen, maxlen)))
        return b(s)

    with TempStorage("randomhash") as st:
        samp = dict((randstring(), randstring()) for _ in xrange(times))

        hw = HashWriter(st.create_file("test.hsh"))
        for k, v in iteritems(samp):
            hw.add(k, v)
        hw.close()

        keys = list(samp.keys())
        random.shuffle(keys)
        hr = HashReader.open(st, "test.hsh")
        for k in keys:
            assert hr[k] == samp[k]
        hr.close()

 def __repr__(self):
     attrs = ""
     if self.__dict__:
         attrs = ", ".join("%s=%r" % (key, value)
                           for key, value
                           in iteritems(self.__dict__))
     return self.__class__.__name__ + "(%s)" % attrs

 def stored_fields(self, docnum):
     if self.is_closed:
         raise ReaderClosed
     assert docnum >= 0
     schema = self.schema
     sfs = self._perdoc.stored_fields(docnum)
     # Double-check with schema to filter out removed fields
     return dict(item for item in iteritems(sfs) if item[0] in schema)

 def __hash__(self):
     if self._hash is not None:
         return self._hash
     h = int(self.final)
     for key, node in iteritems(self._edges):
         h ^= hash(key) ^ hash(node)
     self._hash = h
     return h

    def __init__(self, dbfile):
        super(TermIndexReader, self).__init__(dbfile)

        dbfile.seek(self.indexbase + self.length * _LONG_SIZE)
        self.fieldmap = dbfile.read_pickle()
        self.names = [None] * len(self.fieldmap)
        for name, num in iteritems(self.fieldmap):
            self.names[num] = name

 def flush(self):
     for fieldname, lst in iteritems(self.postbuf):
         con = self._con(fieldname)
         con.executemany("insert into postings values (?, ?, ?, ?)", lst)
         con.commit()
         con.close()
     self.postbuf = defaultdict(list)
     self.bufsize = 0
     self._flushed = True

 def extract(self, match):
     d = match.groupdict()
     for key, value in iteritems(d):
         try:
             value = int(value)
             d[key] = value
         except (ValueError, TypeError):
             pass
     return Props(**d)

    def __init__(self, dbfile, postfile):
        CodedOrderedReader.__init__(self, dbfile)
        self.postfile = postfile

        dbfile.seek(self.indexbase + self.length * _LONG_SIZE)
        self.fieldmap = dbfile.read_pickle()
        self.names = [None] * len(self.fieldmap)
        for name, num in iteritems(self.fieldmap):
            self.names[num] = name

 def _suggestions(self, text, maxdist, prefix):
     op = self.op
     seen = {}
     for corr in self.correctors:
         for score, sug in corr._suggestions(text, maxdist, prefix):
             if sug in seen:
                 seen[sug] = op(seen[sug], score)
             else:
                 seen[sug] = score
     return iteritems(seen)

    def __init__(self, codec, dbfile, length, postfile):
        self._codec = codec
        self._dbfile = dbfile
        self._tindex = filetables.OrderedHashReader(dbfile, length)
        self._fieldmap = self._tindex.extras["fieldmap"]
        self._postfile = postfile

        self._fieldunmap = [None] * len(self._fieldmap)
        for fieldname, num in iteritems(self._fieldmap):
            self._fieldunmap[num] = fieldname

 def _print_line(self, indent, command, **kwargs):
     self._dbfile.write(b("  ") * indent)
     self._dbfile.write(command.encode("latin1"))
     for k, v in iteritems(kwargs):
         if isinstance(v, memoryview):
             v = bytes(v)
         if v is not None and not isinstance(v, _reprable):
             raise TypeError(type(v))
         self._dbfile.write(("\t%s=%r" % (k, v)).encode("latin1"))
     self._dbfile.write(b("\n"))

def renumber_dfa(dfa, base=0):
    c = itertools.count(base)
    mapping = {}

    def remap(state):
        if state in mapping:
            newnum = mapping[state]
        else:
            newnum = next(c)
            mapping[state] = newnum
        return newnum

    newdfa = DFA(remap(dfa.initial))
    for src, trans in iteritems(dfa.transitions):
        for label, dest in iteritems(trans):
            newdfa.add_transition(remap(src), label, remap(dest))
    for finalstate in dfa.final_states:
        newdfa.add_final_state(remap(finalstate))
    for src, dest in iteritems(dfa.defaults):
        newdfa.set_default_transition(remap(src), remap(dest))
    return newdfa