/**
  * @see Zend_Search_Lucene_Analysis_TokenFilter
  */
 public function normalize(Zend_Search_Lucene_Analysis_Token $srcToken)
 {
     $text = $this->stemmer->doStem($srcToken->getTermText());
     $newToken = new Zend_Search_Lucene_Analysis_Token($text, $srcToken->getStartOffset(), $srcToken->getEndOffset());
     $newToken->setPositionIncrement($srcToken->getPositionIncrement());
     return $newToken;
 }

 /**
  * Normalize Token or remove it (if null is returned)
  *
  * @param Zend_Search_Lucene_Analysis_Token $srcToken
  * @return Zend_Search_Lucene_Analysis_Token
  */
 public function normalize(Zend_Search_Lucene_Analysis_Token $srcToken) {
     if (array_key_exists($srcToken->getTermText(), $this->_stopSet)) {
         return null;
     } else {
         return $srcToken;
     }
 }

 /**
  * Normalize Token or remove it (if null is returned)
  *
  * @param Zend_Search_Lucene_Analysis_Token $srcToken
  * @return Zend_Search_Lucene_Analysis_Token
  */
 public function normalize(Zend_Search_Lucene_Analysis_Token $srcToken) {
     if (strlen($srcToken->getTermText()) < $this->length) {
         return null;
     } else {
         return $srcToken;
     }
 }

 /**
  * If not numeric, calls the parent method.
  *
  * @param Zend_Search_Lucene_Analysis_Token $srcToken
  * @return Zend_Search_Lucene_Analysis_Token
  */
 public function normalize(Zend_Search_Lucene_Analysis_Token $srcToken)
 {
     if (!ctype_digit($srcToken->getTermText())) {
         return parent::normalize($srcToken);
     } else {
         return $srcToken;
     }
 }

 /**
  * Normalize Token or remove it (if null is returned).
  *
  * @param Zend_Search_Lucene_Analysis_Token $srcToken
  * @return Zend_Search_Lucene_Analysis_Token
  */
 public function normalize(Zend_Search_Lucene_Analysis_Token $srcToken)
 {
     // gets token text, invokes hook_search_preprocess().
     $processed_text = $srcToken->getTermText();
     search_invoke_preprocess($processed_text);
     // returns the new processed token
     $newToken = new Zend_Search_Lucene_Analysis_Token($processed_text, $srcToken->getStartOffset(), $srcToken->getEndOffset());
     $newToken->setPositionIncrement($srcToken->getPositionIncrement());
     return $newToken;
 }

    /**
     * Normalize Token or remove it (if null is returned)
     *
     * @param Zend_Search_Lucene_Analysis_Token $srcToken
     * @return Zend_Search_Lucene_Analysis_Token
     */
    public function normalize(Zend_Search_Lucene_Analysis_Token $srcToken)
    {
        $newToken = new Zend_Search_Lucene_Analysis_Token(
                                     strtolower( $srcToken->getTermText() ),
                                     $srcToken->getStartOffset(),
                                     $srcToken->getEndOffset());

        $newToken->setPositionIncrement($srcToken->getPositionIncrement());

        return $newToken;
    }

 /**
  * Normalize Token or remove it (if null is returned)
  *
  * @param Zend_Search_Lucene_Analysis_Token $srcToken
  * @return Zend_Search_Lucene_Analysis_Token
  */
 public function normalize(Zend_Search_Lucene_Analysis_Token $srcToken)
 {
     if ($this->mbString) {
         $value = mb_strtolower($srcToken->getTermText(), 'utf8');
     } else {
         $value = strtolower($srcToken->getTermText());
     }
     $newToken = new Zend_Search_Lucene_Analysis_Token($value, $srcToken->getStartOffset(), $srcToken->getEndOffset());
     $newToken->setPositionIncrement($srcToken->getPositionIncrement());
     return $newToken;
 }

 /**
  * Normalize Token or remove it (if null is returned)
  *
  * @param Zend_Search_Lucene_Analysis_Token $srcToken
  * @return Zend_Search_Lucene_Analysis_Token
  */
 public function normalize(Zend_Search_Lucene_Analysis_Token $po_srctoken)
 {
     $vo_lang_analyzer = new LanguageDetection();
     $vs_original_string = $po_srctoken->getTermText();
     $vs_lang_code = $vo_lang_analyzer->analyze($vs_original_string);
     /* stem text with respect to language that has been detected */
     $vo_stemmer = new SnoballStemmer();
     if ($vs_lang_code) {
         $vs_stemmed_string = $vo_stemmer->stem($vs_original_string, $vs_lang_code);
     } else {
         /* if language could not be detected, don't do any stemming at all */
         $vs_stemmed_string = $vs_original_string;
     }
     /* build new token to return */
     $vo_new_token = new Zend_Search_Lucene_Analysis_Token($vs_stemmed_string, $po_srctoken->getStartOffset(), $po_srctoken->getEndOffset());
     $vo_new_token->setPositionIncrement($po_srctoken->getPositionIncrement());
     return $vo_new_token;
 }

 /**
  * Normalize Token or remove it (if null is returned)
  *
  * @param Zend_Search_Lucene_Analysis_Token $srcToken
  * @return Zend_Search_Lucene_Analysis_Token
  */
 public function normalize(Zend_Search_Lucene_Analysis_Token $srcToken)
 {
     $srcToken->setTermText(mb_strtolower($srcToken->getTermText(), 'UTF-8'));
     return $srcToken;
 }

 /**
  * Normalize Token or remove it (if null is returned)
  *
  * @param Zend_Search_Lucene_Analysis_Token $srcToken
  * @return Zend_Search_Lucene_Analysis_Token
  */
 public function normalize(Zend_Search_Lucene_Analysis_Token $srcToken)
 {
     //iconv("utf-8", "us-ascii//TRANSLIT", $url); // TRANSLIT does the whole job
     // We could use also remove_accents() in uri.php
     // Problem: ñ -> n
     //$token = strtolower(iconv("utf-8", "us-ascii//TRANSLIT", $srcToken->getTermText()));
     $token = strtolower($srcToken->getTermText());
     if (strlen($token) < 2 || array_key_exists($token, $this->_stopSet)) {
         return null;
     }
     $newToken = new Zend_Search_Lucene_Analysis_Token($token, $srcToken->getStartOffset(), $srcToken->getEndOffset());
     $newToken->setPositionIncrement($srcToken->getPositionIncrement());
     return $newToken;
 }

<?php

/**
 * This file is part of the sfLucene package.
 * (c) Carl Vondrick <[email protected]>
 *
 * For the full copyright and license information, please view the LICENSE
 * file that was distributed with this source code.
 */
require dirname(__FILE__) . '/../../bootstrap/unit.php';
require 'util/xfLuceneZendManager.class.php';
require 'stemmer/xfLuceneStemmerTokenFilter.class.php';
require 'stemmer/xfLuceneStemmer.interface.php';
require 'stemmer/xfLuceneStemmerPorter.class.php';
require 'vendor/PorterStemmer/PorterStemmer.class.php';
$t = new lime_test(2, new lime_output_color());
$s = new xfLuceneStemmerPorter();
$filter = new xfLuceneStemmerTokenFilter($s);
$token = new Zend_Search_Lucene_Analysis_Token('nationalize', 10, 21);
$token->setPositionIncrement(0);
$response = $filter->normalize($token);
$t->isa_ok($response, 'Zend_Search_Lucene_Analysis_Token', '->normalize() returns a Zend_Search_Lucene_Analysis_Token');
$t->is($response->getTermText(), 'nation', '->normalize() consults the stemmer');

 /**
  * Tokenization stream API
  * Get next token
  * Returns null at the end of stream
  *
  * @return Zend_Search_Lucene_Analysis_Token|null
  */
 public function nextToken()
 {
     // есть ли нам откуда брать данные?
     if (!$this->num_chunks) {
         return null;
     }
     // сначала отдаём уже имеющиеся токены
     if (sizeof($this->token_stack)) {
         return array_pop($this->token_stack);
     }
     while ($this->num_chunks > $this->current_chunk) {
         $word = $this->input_chunks[$this->current_chunk + 1];
         // специальный случай: идентификаторы сайтов и разделов в виде sub123, site5
         if ($this->ignore_numbers && ($word == 'site' || $word == 'sub') && preg_match("/^(\\d+)/", $this->input_chunks[$this->current_chunk + 2], $matches)) {
             $word .= $matches[1];
         }
         $word_length = mb_strlen($word, 'UTF-8');
         $delimiter_length = mb_strlen($this->input_chunks[$this->current_chunk], 'UTF-8');
         $start_position = $this->current_chunk == 1 ? 0 : $this->char_position + $delimiter_length + 1;
         $end_position = $start_position + $word_length;
         // готовимся к следующему циклу
         $this->char_position = $end_position;
         $this->current_chunk += 2;
         if (!$word_length) {
             continue;
         }
         // на входе была строка без значащих символов?
         // применяем фильтры
         $processed = $this->apply_nc_filters($word);
         $count = sizeof($processed);
         if ($count > 0) {
             for ($i = 1; $i < $count; $i++) {
                 // i.e. if $count > 1
                 $token = new Zend_Search_Lucene_Analysis_Token($processed[$i], $start_position, $end_position);
                 // умная книга Lucene in Action советует установить $token->setPositionIncrement(0),
                 // но, по-моему, разницы нет (в исходниках ZSL отмечено "todo: Process
                 // $token->getPositionIncrement()" - может быть, в будущем заработает)
                 $token->setPositionIncrement(0);
                 $this->token_stack[] = $token;
             }
             return new Zend_Search_Lucene_Analysis_Token($processed[0], $start_position, $end_position);
         }
     }
     return null;
 }