public MockVariableLengthPayloadFilter(Random random, TokenStream @in)
     : base(@in)
 {
     this.Random = random;
     this.Payload = new BytesRef(Bytes);
     this.PayloadAtt = AddAttribute<IPayloadAttribute>();
 }

        private void SumValues(IList<FacetsCollector.MatchingDocs> matchingDocs)
        {
            //System.out.println("count matchingDocs=" + matchingDocs + " facetsField=" + facetsFieldName);
            foreach (FacetsCollector.MatchingDocs hits in matchingDocs)
            {
                BinaryDocValues dv = hits.context.AtomicReader.GetBinaryDocValues(IndexFieldName);
                if (dv == null) // this reader does not have DocValues for the requested category list
                {
                    continue;
                }

                DocIdSetIterator docs = hits.bits.GetIterator();

                int doc;
                while ((doc = docs.NextDoc()) != DocIdSetIterator.NO_MORE_DOCS)
                {
                    //System.out.println("  doc=" + doc);
                    // TODO: use OrdinalsReader?  we'd need to add a
                    // BytesRef getAssociation()?
                    BytesRef bytesRef = new BytesRef();
                    dv.Get(doc, bytesRef);
                    byte[] bytes = bytesRef.Bytes;
                    int end = bytesRef.Offset + bytesRef.Length;
                    int offset = bytesRef.Offset;
                    while (offset < end)
                    {
                        int ord = ((bytes[offset] & 0xFF) << 24) | ((bytes[offset + 1] & 0xFF) << 16) | ((bytes[offset + 2] & 0xFF) << 8) | (bytes[offset + 3] & 0xFF);
                        offset += 4;
                        int value = ((bytes[offset] & 0xFF) << 24) | ((bytes[offset + 1] & 0xFF) << 16) | ((bytes[offset + 2] & 0xFF) << 8) | (bytes[offset + 3] & 0xFF);
                        offset += 4;
                        values[ord] += Number.IntBitsToFloat(value);
                    }
                }
            }
        }

        public virtual void AddValue(int docID, BytesRef value)
        {
            if (value == null)
            {
                throw new System.ArgumentException("field \"" + FieldInfo.Name + "\": null value not allowed");
            }
            if (value.Length > (ByteBlockPool.BYTE_BLOCK_SIZE - 2))
            {
                throw new System.ArgumentException("DocValuesField \"" + FieldInfo.Name + "\" is too large, must be <= " + (ByteBlockPool.BYTE_BLOCK_SIZE - 2));
            }

            if (docID != CurrentDoc)
            {
                FinishCurrentDoc();
            }

            // Fill in any holes:
            while (CurrentDoc < docID)
            {
                PendingCounts.Add(0); // no values
                CurrentDoc++;
            }

            AddOneValue(value);
            UpdateBytesUsed();
        }

        /// <summary>
        /// Enumerates all terms greater/equal than <code>lowerTerm</code>
        /// but less/equal than <code>upperTerm</code>.
        ///
        /// If an endpoint is null, it is said to be "open". Either or both
        /// endpoints may be open.  Open endpoints may not be exclusive
        /// (you can't select all but the first or last term without
        /// explicitly specifying the term to exclude.)
        /// </summary>
        /// <param name="tenum">
        ///          TermsEnum to filter </param>
        /// <param name="lowerTerm">
        ///          The term text at the lower end of the range </param>
        /// <param name="upperTerm">
        ///          The term text at the upper end of the range </param>
        /// <param name="includeLower">
        ///          If true, the <code>lowerTerm</code> is included in the range. </param>
        /// <param name="includeUpper">
        ///          If true, the <code>upperTerm</code> is included in the range. </param>
        public TermRangeTermsEnum(TermsEnum tenum, BytesRef lowerTerm, BytesRef upperTerm, bool includeLower, bool includeUpper)
            : base(tenum)
        {
            // do a little bit of normalization...
            // open ended range queries should always be inclusive.
            if (lowerTerm == null)
            {
                this.LowerBytesRef = new BytesRef();
                this.IncludeLower = true;
            }
            else
            {
                this.LowerBytesRef = lowerTerm;
                this.IncludeLower = includeLower;
            }

            if (upperTerm == null)
            {
                this.IncludeUpper = true;
                UpperBytesRef = null;
            }
            else
            {
                this.IncludeUpper = includeUpper;
                UpperBytesRef = upperTerm;
            }

            InitialSeekTerm = LowerBytesRef;
            TermComp = Comparator;
        }

        public virtual void TestFixedBinary()
        {
            BaseDirectoryWrapper dir = NewFSDirectory(CreateTempDir("2BFixedBinary"));
            if (dir is MockDirectoryWrapper)
            {
                ((MockDirectoryWrapper)dir).Throttling = MockDirectoryWrapper.Throttling_e.NEVER;
            }

            IndexWriter w = new IndexWriter(dir, new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random()))
               .SetMaxBufferedDocs(IndexWriterConfig.DISABLE_AUTO_FLUSH).SetRAMBufferSizeMB(256.0).SetMergeScheduler(new ConcurrentMergeScheduler()).SetMergePolicy(NewLogMergePolicy(false, 10)).SetOpenMode(IndexWriterConfig.OpenMode_e.CREATE));

            Document doc = new Document();
            var bytes = new byte[4];
            BytesRef data = new BytesRef(bytes);
            BinaryDocValuesField dvField = new BinaryDocValuesField("dv", data);
            doc.Add(dvField);

            for (int i = 0; i < int.MaxValue; i++)
            {
                bytes[0] = (byte)(i >> 24);
                bytes[1] = (byte)(i >> 16);
                bytes[2] = (byte)(i >> 8);
                bytes[3] = (byte)i;
                w.AddDocument(doc);
                if (i % 100000 == 0)
                {
                    Console.WriteLine("indexed: " + i);
                    Console.Out.Flush();
                }
            }

            w.ForceMerge(1);
            w.Dispose();

            Console.WriteLine("verifying...");
            Console.Out.Flush();

            DirectoryReader r = DirectoryReader.Open(dir);
            int expectedValue = 0;
            foreach (AtomicReaderContext context in r.Leaves)
            {
                AtomicReader reader = context.AtomicReader;
                BytesRef scratch = new BytesRef();
                BinaryDocValues dv = reader.GetBinaryDocValues("dv");
                for (int i = 0; i < reader.MaxDoc; i++)
                {
                    bytes[0] = (byte)(expectedValue >> 24);
                    bytes[1] = (byte)(expectedValue >> 16);
                    bytes[2] = (byte)(expectedValue >> 8);
                    bytes[3] = (byte)expectedValue;
                    dv.Get(i, scratch);
                    Assert.AreEqual(data, scratch);
                    expectedValue++;
                }
            }

            r.Dispose();
            dir.Dispose();
        }

 private DocTermOrdsRangeFilter(string field, BytesRef lowerVal, BytesRef upperVal, bool includeLower, bool includeUpper)
 {
     this.Field_Renamed = field;
     this.LowerVal_Renamed = lowerVal;
     this.UpperVal_Renamed = upperVal;
     this.IncludeLower = includeLower;
     this.IncludeUpper = includeUpper;
 }

 public BinaryDocValuesFieldUpdates(string field, int maxDoc)
     : base(field, Type_e.BINARY)
 {
     DocsWithField = new FixedBitSet(64);
     Docs = new PagedMutable(1, 1024, PackedInts.BitsRequired(maxDoc - 1), PackedInts.COMPACT);
     Offsets = new PagedGrowableWriter(1, 1024, 1, PackedInts.FAST);
     Lengths = new PagedGrowableWriter(1, 1024, 1, PackedInts.FAST);
     Values = new BytesRef(16); // start small
     Size = 0;
 }

 protected internal override AcceptStatus Accept(BytesRef term)
 {
     if (StringHelper.StartsWith(term, PrefixRef))
     {
         return AcceptStatus.YES;
     }
     else
     {
         return AcceptStatus.END;
     }
 }

 public MockFixedLengthPayloadFilter(Random random, TokenStream @in, int length)
     : base(@in)
 {
     if (length < 0)
     {
         throw new System.ArgumentException("length must be >= 0");
     }
     this.Random = random;
     this.Bytes = new byte[length];
     this.Payload = new BytesRef(Bytes);
     this.PayloadAtt = AddAttribute<IPayloadAttribute>();
 }

        public virtual void Test()
        {
            Directory dir = NewDirectory();
            IndexWriter w = new IndexWriter(dir, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random())).SetMergePolicy(NewLogMergePolicy()));

            IList<long?> numbers = new List<long?>();
            IList<BytesRef> binary = new List<BytesRef>();
            IList<BytesRef> sorted = new List<BytesRef>();
            int numDocs = AtLeast(100);
            for (int i = 0; i < numDocs; i++)
            {
                Document d = new Document();
                long number = Random().NextLong();
                d.Add(new NumericDocValuesField("number", number));
                BytesRef bytes = new BytesRef(TestUtil.RandomRealisticUnicodeString(Random()));
                d.Add(new BinaryDocValuesField("bytes", bytes));
                binary.Add(bytes);
                bytes = new BytesRef(TestUtil.RandomRealisticUnicodeString(Random()));
                d.Add(new SortedDocValuesField("sorted", bytes));
                sorted.Add(bytes);
                w.AddDocument(d);
                numbers.Add(number);
            }

            w.ForceMerge(1);
            IndexReader r = w.Reader;
            w.Dispose();

            Assert.AreEqual(1, r.Leaves.Count);
            AtomicReader ar = (AtomicReader)r.Leaves[0].Reader;

            int numThreads = TestUtil.NextInt(Random(), 2, 5);
            IList<ThreadClass> threads = new List<ThreadClass>();
            CountDownLatch startingGun = new CountDownLatch(1);
            for (int t = 0; t < numThreads; t++)
            {
                Random threadRandom = new Random(Random().Next());
                ThreadClass thread = new ThreadAnonymousInnerClassHelper(this, numbers, binary, sorted, numDocs, ar, startingGun, threadRandom);
                thread.Start();
                threads.Add(thread);
            }

            startingGun.countDown();

            foreach (ThreadClass thread in threads)
            {
                thread.Join();
            }

            r.Dispose();
            dir.Dispose();
        }

 public virtual DocsAndPositionsEnum GetDocsAndPositions(AtomicReader reader, BytesRef bytes, Bits liveDocs)
 {
     Terms terms = reader.Terms(FieldName);
     if (terms != null)
     {
         TermsEnum te = terms.Iterator(null);
         if (te.SeekExact(bytes))
         {
             return te.DocsAndPositions(liveDocs, null);
         }
     }
     return null;
 }

 public override void Get(int docID, BytesRef result)
 {
     int ord = GetOrd(docID);
     if (ord == -1)
     {
         result.Bytes = BytesRef.EMPTY_BYTES;
         result.Length = 0;
         result.Offset = 0;
     }
     else
     {
         LookupOrd(ord, result);
     }
 }

 /// <summary>
 /// Creates this from <paramref name="dim"/> and <paramref name="path"/> and an
 /// association 
 /// </summary>
 public AssociationFacetField(BytesRef assoc, string dim, params string[] path)
     : base("dummy", TYPE)
 {
     FacetField.VerifyLabel(dim);
     foreach (string label in path)
     {
         FacetField.VerifyLabel(label);
     }
     this.Dim = dim;
     this.Assoc = assoc;
     if (path.Length == 0)
     {
         throw new System.ArgumentException("path must have at least one element");
     }
     this.Path = path;
 }

        public virtual void TestBinary()
        {
            Directory dir = NewDirectory();
            Document doc = new Document();
            BytesRef @ref = new BytesRef();
            Field field = new BinaryDocValuesField("bytes", @ref);
            doc.Add(field);

            IndexWriterConfig iwc = NewIndexWriterConfig(Random(), TEST_VERSION_CURRENT, null);
            iwc.SetMergePolicy(NewLogMergePolicy());
            RandomIndexWriter iw = new RandomIndexWriter(Random(), dir, iwc);

            int numDocs = AtLeast(500);
            for (int i = 0; i < numDocs; i++)
            {
                @ref.CopyChars(TestUtil.RandomUnicodeString(Random()));
                iw.AddDocument(doc);
                if (Random().Next(17) == 0)
                {
                    iw.Commit();
                }
            }
            DirectoryReader ir = iw.Reader;
            iw.ForceMerge(1);
            DirectoryReader ir2 = iw.Reader;
            AtomicReader merged = GetOnlySegmentReader(ir2);
            iw.Dispose();

            BinaryDocValues multi = MultiDocValues.GetBinaryValues(ir, "bytes");
            BinaryDocValues single = merged.GetBinaryDocValues("bytes");
            BytesRef actual = new BytesRef();
            BytesRef expected = new BytesRef();
            for (int i = 0; i < numDocs; i++)
            {
                single.Get(i, expected);
                multi.Get(i, actual);
                Assert.AreEqual(expected, actual);
            }
            ir.Dispose();
            ir2.Dispose();
            dir.Dispose();
        }

        public virtual void TestBinary()
        {
            Directory dir = NewDirectory();
            RandomIndexWriter iw = new RandomIndexWriter(Random(), dir);
            BytesRef bytes = new BytesRef(2);
            BinaryTokenStream tokenStream = new BinaryTokenStream(bytes);

            for (int i = 0; i < 256; i++)
            {
                bytes.Bytes[0] = (byte)i;
                bytes.Bytes[1] = unchecked((byte)(255 - i));
                bytes.Length = 2;
                Document doc = new Document();
                FieldType customType = new FieldType();
                customType.Stored = true;
                doc.Add(new Field("id", "" + i, customType));
                doc.Add(new TextField("bytes", tokenStream));
                iw.AddDocument(doc);
            }

            IndexReader ir = iw.Reader;
            iw.Dispose();

            IndexSearcher @is = NewSearcher(ir);

            for (int i = 0; i < 256; i++)
            {
                bytes.Bytes[0] = (byte)i;
                bytes.Bytes[1] = unchecked((byte)(255 - i));
                bytes.Length = 2;
                TopDocs docs = @is.Search(new TermQuery(new Term("bytes", bytes)), 5);
                Assert.AreEqual(1, docs.TotalHits);
                Assert.AreEqual("" + i, @is.Doc(docs.ScoreDocs[0].Doc).Get("id"));
            }

            ir.Dispose();
            dir.Dispose();
        }

 public DocTermOrdsRangeFilterAnonymousInnerClassHelper(string field, BytesRef lowerVal, BytesRef upperVal, bool includeLower, bool includeUpper)
     : base(field, lowerVal, upperVal, includeLower, includeUpper)
 {
     this.Field = field;
     this.LowerVal = lowerVal;
     this.UpperVal = upperVal;
     this.IncludeLower = includeLower;
     this.IncludeUpper = includeUpper;
 }

 /// <summary>
 /// Creates a BytesRef range filter using <seealso cref="IFieldCache#getTermsIndex"/>. this works with all
 /// fields containing zero or one term in the field. The range can be half-open by setting one
 /// of the values to <code>null</code>.
 /// </summary>
 public static DocTermOrdsRangeFilter NewBytesRefRange(string field, BytesRef lowerVal, BytesRef upperVal, bool includeLower, bool includeUpper)
 {
     return new DocTermOrdsRangeFilterAnonymousInnerClassHelper(field, lowerVal, upperVal, includeLower, includeUpper);
 }

 protected internal override AcceptStatus Accept(BytesRef term)
 {
     return StringHelper.StartsWith(term, Prefix) ? AcceptStatus.YES : AcceptStatus.NO;
 }

 public PrefixTermsEnum(TermsEnum tenum, BytesRef prefixText)
     : base(tenum)
 {
     InitialSeekTerm = this.PrefixRef = prefixText;
 }

 public override float ScorePayload(int docId, int start, int end, BytesRef payload)
 {
     //we know it is size 4 here, so ignore the offset/length
     return payload.Bytes[payload.Offset];
 }