def word_datas(self, value, **kwargs):
        seen = defaultdict(int)
        if self.boost_as_freq:
            for t in unstopped(self.analyzer(value, boosts=True)):
                seen[t.text] += int(t.boost)
        else:
            for t in unstopped(self.analyzer(value)):
                seen[t.text] += 1

        return ((w, freq, freq) for w, freq in seen.iteritems())

 def word_values(self, value, **kwargs):
     seen = defaultdict(int)
     if self.boost_as_freq:
         for t in unstopped(self.analyzer(value, boosts=True, **kwargs)):
             seen[t.text] += int(t.boost)
     else:
         for t in unstopped(self.analyzer(value, **kwargs)):
             seen[t.text] += 1
     
     encode = self.encode
     return ((w, freq, float(freq), encode(freq))
             for w, freq in seen.iteritems())

 def word_values(self, value, doc_boost = 1.0, **kwargs):
     seen = defaultdict(int)
     for t in unstopped(self.analyzer(value, **kwargs)):
         seen[t.text] += 1
     
     encode = self.encode
     return ((w, freq, encode((freq, doc_boost))) for w, freq in seen.iteritems())

 def word_datas(self, value, start_pos = 0, start_char = 0, **kwargs):
     seen = defaultdict(list)
     
     for t in unstopped(self.analyzer(value, positions = True, chars = True,
                                      start_pos = start_pos, start_char = start_char)):
         seen[t.text].append((t.pos, start_char + t.startchar, start_char + t.endchar))
     
     return ((w, len(ls), ls) for w, ls in seen.iteritems())

 def word_values(self, value, doc_boost=1.0, **kwargs):
     freqs = defaultdict(int)
     weights = defaultdict(float)
     for t in unstopped(self.analyzer(value, boosts=True, **kwargs)):
         weights[t.text] += t.boost
         freqs[t.text] += 1
     
     encode = self.encode
     return ((w, freq, weights[w] * doc_boost, encode((freq, doc_boost)))
             for w, freq in freqs.iteritems())

    def word_values(self, value, **kwargs):
        fb = self.field_boost
        freqs = defaultdict(int)
        weights = defaultdict(float)

        for t in unstopped(self.analyzer(value, boosts=True, **kwargs)):
            freqs[t.text] += 1
            weights[t.text] += t.boost

        encode = self.encode
        return ((w, freq, weights[w] * fb, encode(freq))
                for w, freq in freqs.iteritems())

def tokens(value, analyzer, kwargs):
    if isinstance(value, (tuple, list)):
        gen = entoken(value, **kwargs)
    else:
        gen = analyzer(value, **kwargs)
    return unstopped(gen)