public override BytesRef Encode(char[] buffer, int offset, int length)
 {
     float payload = float.Parse(new string(buffer, offset, length)); //TODO: improve this so that we don't have to new Strings
     byte[] bytes = PayloadHelper.EncodeFloat(payload);
     BytesRef result = new BytesRef(bytes);
     return result;
 }

 public override BytesRef Encode(char[] buffer, int offset, int length)
 {
     int payload = ArrayUtil.ParseInt(buffer, offset, length); //TODO: improve this so that we don't have to new Strings
     byte[] bytes = PayloadHelper.EncodeInt(payload);
     BytesRef result = new BytesRef(bytes);
     return result;
 }

 internal virtual byte[] Decompress(byte[] compressed, int originalLength, int offset, int length)
 {
     Decompressor decompressor = Mode.NewDecompressor();
     BytesRef bytes = new BytesRef();
     decompressor.Decompress(new ByteArrayDataInput(compressed), originalLength, offset, length, bytes);
     return Arrays.CopyOfRange(bytes.Bytes, bytes.Offset, bytes.Offset + bytes.Length);
 }

        public void TestBlendingType()
        {

            BytesRef pl = new BytesRef("lake");
            long w = 20;

            Input[] keys = new Input[]{
                new Input("top of the lake", w, pl)
            };

            DirectoryInfo tempDir = CreateTempDir("BlendedInfixSuggesterTest");
            Analyzer a = new StandardAnalyzer(TEST_VERSION_CURRENT, CharArraySet.EMPTY_SET);

            // BlenderType.LINEAR is used by default (remove position*10%)
            BlendedInfixSuggester suggester = new BlendedInfixSuggester(TEST_VERSION_CURRENT, NewFSDirectory(tempDir), a);
            suggester.Build(new InputArrayIterator(keys));

            assertEquals(w, GetInResults(suggester, "top", pl, 1));
            assertEquals((int)(w * (1 - 0.10 * 2)), GetInResults(suggester, "the", pl, 1));
            assertEquals((int)(w * (1 - 0.10 * 3)), GetInResults(suggester, "lake", pl, 1));

            suggester.Dispose();

            // BlenderType.RECIPROCAL is using 1/(1+p) * w where w is weight and p the position of the word
            suggester = new BlendedInfixSuggester(TEST_VERSION_CURRENT, NewFSDirectory(tempDir), a, a,
                                                  AnalyzingInfixSuggester.DEFAULT_MIN_PREFIX_CHARS, BlendedInfixSuggester.BlenderType.POSITION_RECIPROCAL, 1);
            suggester.Build(new InputArrayIterator(keys));

            assertEquals(w, GetInResults(suggester, "top", pl, 1));
            assertEquals((int)(w * 1 / (1 + 2)), GetInResults(suggester, "the", pl, 1));
            assertEquals((int)(w * 1 / (1 + 3)), GetInResults(suggester, "lake", pl, 1));

            suggester.Dispose();
        }

 public MockVariableLengthPayloadFilter(Random random, TokenStream @in)
     : base(@in)
 {
     this.Random = random;
     this.Payload = new BytesRef(Bytes);
     this.PayloadAtt = AddAttribute<IPayloadAttribute>();
 }

 public DocFreqValueSource(string field, string val, string indexedField, BytesRef indexedBytes)
 {
     this.field = field;
     this.val = val;
     this.indexedField = indexedField;
     this.indexedBytes = indexedBytes;
 }

 internal TermStats(string field, BytesRef termtext, int df, long tf)
 {
     this.termtext = BytesRef.DeepCopyOf(termtext);
     this.Field = field;
     this.DocFreq = df;
     this.TotalTermFreq = tf;
 }

        public virtual void TestFarsiRangeFilterCollating(Analyzer analyzer, BytesRef firstBeg, BytesRef firstEnd, BytesRef secondBeg, BytesRef secondEnd)
        {
            Directory dir = NewDirectory();
            IndexWriter writer = new IndexWriter(dir, new IndexWriterConfig(TEST_VERSION_CURRENT, analyzer));
            Document doc = new Document();
            doc.Add(new TextField("content", "\u0633\u0627\u0628", Field.Store.YES));
            doc.Add(new StringField("body", "body", Field.Store.YES));
            writer.AddDocument(doc);
            writer.Dispose();
            IndexReader reader = DirectoryReader.Open(dir);
            IndexSearcher searcher = new IndexSearcher(reader);
            Query query = new TermQuery(new Term("body", "body"));

            // Unicode order would include U+0633 in [ U+062F - U+0698 ], but Farsi
            // orders the U+0698 character before the U+0633 character, so the single
            // index Term below should NOT be returned by a TermRangeFilter with a Farsi
            // Collator (or an Arabic one for the case when Farsi searcher not
            // supported).
            ScoreDoc[] result = searcher.Search(query, new TermRangeFilter("content", firstBeg, firstEnd, true, true), 1).ScoreDocs;
            Assert.AreEqual(0, result.Length, "The index Term should not be included.");

            result = searcher.Search(query, new TermRangeFilter("content", secondBeg, secondEnd, true, true), 1).ScoreDocs;
            Assert.AreEqual(1, result.Length, "The index Term should be included.");

            reader.Dispose();
            dir.Dispose();
        }

        public virtual void AddValue(int docID, BytesRef value)
        {
            if (value == null)
            {
                throw new System.ArgumentException("field \"" + FieldInfo.Name + "\": null value not allowed");
            }
            if (value.Length > (ByteBlockPool.BYTE_BLOCK_SIZE - 2))
            {
                throw new System.ArgumentException("DocValuesField \"" + FieldInfo.Name + "\" is too large, must be <= " + (ByteBlockPool.BYTE_BLOCK_SIZE - 2));
            }

            if (docID != CurrentDoc)
            {
                FinishCurrentDoc();
            }

            // Fill in any holes:
            while (CurrentDoc < docID)
            {
                PendingCounts.Add(0); // no values
                CurrentDoc++;
            }

            AddOneValue(value);
            UpdateBytesUsed();
        }

 public virtual void TestSize()
 {
     BytesRef @ref = new BytesRef();
     int num = AtLeast(2);
     for (int j = 0; j < num; j++)
     {
         int mod = 1 + Random().Next(39);
         for (int i = 0; i < 797; i++)
         {
             string str;
             do
             {
                 str = TestUtil.RandomRealisticUnicodeString(Random(), 1000);
             } while (str.Length == 0);
             @ref.CopyChars(str);
             int count = Hash.Size();
             int key = Hash.Add(@ref);
             if (key < 0)
             {
                 Assert.AreEqual(Hash.Size(), count);
             }
             else
             {
                 Assert.AreEqual(Hash.Size(), count + 1);
             }
             if (i % mod == 0)
             {
                 Hash.Clear();
                 Assert.AreEqual(0, Hash.Size());
                 Hash.Reinit();
             }
         }
     }
 }

        public static void Main(string[] args)
        {
            FileInfo input = new FileInfo("/home/dweiss/tmp/shuffled.dict");

            int buckets = 20;
            int shareMaxTail = 10;

            ExternalRefSorter sorter = new ExternalRefSorter(new OfflineSorter());
            FSTCompletionBuilder builder = new FSTCompletionBuilder(buckets, sorter, shareMaxTail);

            TextReader reader =
                new StreamReader(
                    new FileStream(input.FullName, FileMode.Open), Encoding.UTF8);

            BytesRef scratch = new BytesRef();
            string line;
            int count = 0;
            while ((line = reader.ReadLine()) != null)
            {
                scratch.CopyChars(line);
                builder.Add(scratch, count % buckets);
                if ((count++ % 100000) == 0)
                {
                    Console.WriteLine("Line: " + count);
                }
            }

            Console.WriteLine("Building FSTCompletion.");
            FSTCompletion completion = builder.Build();

            FileInfo fstFile = new FileInfo("completion.fst");
            Console.WriteLine("Done. Writing automaton: " + fstFile.FullName);
            completion.FST.Save(fstFile);
            sorter.Dispose();
        }

        protected virtual void TruncatedToPrefixAndPattern()
        {
            int i = 0;
            while ((i < truncated.Length) && MatchingChar(truncated[i]))
            {
                i++;
            }
            prefix = truncated.Substring(0, i);
            prefixRef = new BytesRef(prefix);

            StringBuilder re = new StringBuilder();
            // LUCENENET NOTE: To mimic Java's matches() method, we alter
            // the Regex to match the entire string. This makes the Regex
            // fail fast when not at the beginning of the string, which is
            // more efficient than testing the length after a successful match.
            // http://stackoverflow.com/a/12547528/181087
            re.Append(@"\A(?:");
            while (i < truncated.Length)
            {
                AppendRegExpForChar(truncated[i], re);
                i++;
            }
            re.Append(@")\z");
            pattern = new Regex(re.ToString(), RegexOptions.Compiled);
        }

        private void SumValues(IList<FacetsCollector.MatchingDocs> matchingDocs)
        {
            //System.out.println("count matchingDocs=" + matchingDocs + " facetsField=" + facetsFieldName);
            foreach (FacetsCollector.MatchingDocs hits in matchingDocs)
            {
                BinaryDocValues dv = hits.context.AtomicReader.GetBinaryDocValues(IndexFieldName);
                if (dv == null) // this reader does not have DocValues for the requested category list
                {
                    continue;
                }

                DocIdSetIterator docs = hits.bits.GetIterator();

                int doc;
                while ((doc = docs.NextDoc()) != DocIdSetIterator.NO_MORE_DOCS)
                {
                    //System.out.println("  doc=" + doc);
                    // TODO: use OrdinalsReader?  we'd need to add a
                    // BytesRef getAssociation()?
                    BytesRef bytesRef = new BytesRef();
                    dv.Get(doc, bytesRef);
                    byte[] bytes = bytesRef.Bytes;
                    int end = bytesRef.Offset + bytesRef.Length;
                    int offset = bytesRef.Offset;
                    while (offset < end)
                    {
                        int ord = ((bytes[offset] & 0xFF) << 24) | ((bytes[offset + 1] & 0xFF) << 16) | ((bytes[offset + 2] & 0xFF) << 8) | (bytes[offset + 3] & 0xFF);
                        offset += 4;
                        int value = ((bytes[offset] & 0xFF) << 24) | ((bytes[offset + 1] & 0xFF) << 16) | ((bytes[offset + 2] & 0xFF) << 8) | (bytes[offset + 3] & 0xFF);
                        offset += 4;
                        values[ord] += value;
                    }
                }
            }
        }

        public void TestBlendedSort()
        {

            BytesRef payload = new BytesRef("star");

            Input[] keys = new Input[]{
                new Input("star wars: episode v - the empire strikes back", 8, payload)
            };

            DirectoryInfo tempDir = CreateTempDir("BlendedInfixSuggesterTest");

            Analyzer a = new StandardAnalyzer(TEST_VERSION_CURRENT, CharArraySet.EMPTY_SET);
            BlendedInfixSuggester suggester = new BlendedInfixSuggester(TEST_VERSION_CURRENT, NewFSDirectory(tempDir), a, a,
                                                                        AnalyzingInfixSuggester.DEFAULT_MIN_PREFIX_CHARS,
                                                                        BlendedInfixSuggester.BlenderType.POSITION_LINEAR,
                                                                        BlendedInfixSuggester.DEFAULT_NUM_FACTOR);
            suggester.Build(new InputArrayIterator(keys));

            // we query for star wars and check that the weight
            // is smaller when we search for tokens that are far from the beginning

            long w0 = GetInResults(suggester, "star ", payload, 1);
            long w1 = GetInResults(suggester, "war", payload, 1);
            long w2 = GetInResults(suggester, "empire ba", payload, 1);
            long w3 = GetInResults(suggester, "back", payload, 1);
            long w4 = GetInResults(suggester, "bacc", payload, 1);

            assertTrue(w0 > w1);
            assertTrue(w1 > w2);
            assertTrue(w2 > w3);

            assertTrue(w4 < 0);

            suggester.Dispose();
        }

 /// <summary>
 /// Creates a new iterator, buffering entries from the specified iterator </summary>
 public BufferedInputIterator(InputIterator source)
 {
     BytesRef spare;
     int freqIndex = 0;
     hasPayloads = source.HasPayloads;
     hasContexts_Renamed = source.HasContexts;
     while ((spare = source.Next()) != null)
     {
         entries.Append(spare);
         if (hasPayloads)
         {
             payloads.Append(source.Payload);
         }
         if (hasContexts_Renamed)
         {
             contextSets.Add(source.Contexts);
         }
         if (freqIndex >= freqs.Length)
         {
             freqs = ArrayUtil.Grow(freqs, freqs.Length + 1);
         }
         freqs[freqIndex++] = source.Weight;
     }
     comp = source.Comparator;
 }

 public virtual void TestEmpty()
 {
     BytesRef b = new BytesRef();
     Assert.AreEqual(BytesRef.EMPTY_BYTES, b.Bytes);
     Assert.AreEqual(0, b.Offset);
     Assert.AreEqual(0, b.Length);
 }

        public virtual void TestSimpleDictionary()
        {
            using (System.IO.Stream affixStream = this.GetType().getResourceAsStream("simple.aff"))
            {
                using (System.IO.Stream dictStream = this.GetType().getResourceAsStream("simple.dic"))
                {

                    Dictionary dictionary = new Dictionary(affixStream, dictStream);
                    assertEquals(3, dictionary.LookupSuffix(new char[] { 'e' }, 0, 1).Length);
                    assertEquals(1, dictionary.LookupPrefix(new char[] { 's' }, 0, 1).Length);
                    IntsRef ordList = dictionary.LookupWord(new char[] { 'o', 'l', 'r' }, 0, 3);
                    assertNotNull(ordList);
                    assertEquals(1, ordList.Length);

                    BytesRef @ref = new BytesRef();
                    dictionary.flagLookup.Get(ordList.Ints[0], @ref);
                    char[] flags = Dictionary.DecodeFlags(@ref);
                    assertEquals(1, flags.Length);

                    ordList = dictionary.LookupWord(new char[] { 'l', 'u', 'c', 'e', 'n' }, 0, 5);
                    assertNotNull(ordList);
                    assertEquals(1, ordList.Length);
                    dictionary.flagLookup.Get(ordList.Ints[0], @ref);
                    flags = Dictionary.DecodeFlags(@ref);
                    assertEquals(1, flags.Length);
                }
            }
        }

 public virtual void TestCopyBytes()
 {
     sbyte[] bytes = new sbyte[] { (sbyte)'a', (sbyte)'b', (sbyte)'c', (sbyte)'d' };
     BytesRef b = new BytesRef(bytes, 1, 3); // bcd
     b.CopyBytes(new BytesRef("bcde"));
     Assert.AreEqual("bcde", b.Utf8ToString());
 }

 public SrndPrefixQuery(string prefix, bool quoted, char truncator)
     : base(quoted)
 {
     this.prefix = prefix;
     prefixRef = new BytesRef(prefix);
     this.truncator = truncator;
 }

 public virtual void TestAppend()
 {
     var bytes = new[] { (byte)'a', (byte)'b', (byte)'c', (byte)'d' };
     BytesRef b = new BytesRef(bytes, 1, 3); // bcd
     b.Append(new BytesRef("e"));
     Assert.AreEqual("bcde", b.Utf8ToString());
 }