本文整理汇总了C#中Lucene.Net.Analysis.Token类的典型用法代码示例。如果您正苦于以下问题：C# Token类的具体用法？C# Token怎么用？C# Token使用的例子？那么恭喜您, 这里精选的类代码示例或许可以为您提供帮助。
Token类属于Lucene.Net.Analysis命名空间，在下文中一共展示了Token类的20个代码示例，这些例子默认根据受欢迎程度排序。您可以为喜欢或者感觉有用的代码点赞，您的评价将有助于我们的系统推荐出更棒的C#代码示例。
示例1: TestCtor
public virtual void TestCtor()
{
Token t = new Token();
char[] content = "hello".ToCharArray();
t.SetTermBuffer(content, 0, content.Length);
char[] buf = t.TermBuffer();
Assert.AreNotEqual(t.TermBuffer(), content);
Assert.AreEqual("hello", t.Term);
Assert.AreEqual("word", t.Type);
Assert.AreEqual(0, t.Flags);
t = new Token(6, 22);
t.SetTermBuffer(content, 0, content.Length);
Assert.AreEqual("hello", t.Term);
Assert.AreEqual("(hello,6,22)", t.ToString());
Assert.AreEqual("word", t.Type);
Assert.AreEqual(0, t.Flags);
t = new Token(6, 22, 7);
t.SetTermBuffer(content, 0, content.Length);
Assert.AreEqual("hello", t.Term);
Assert.AreEqual("(hello,6,22)", t.ToString());
Assert.AreEqual(7, t.Flags);
t = new Token(6, 22, "junk");
t.SetTermBuffer(content, 0, content.Length);
Assert.AreEqual("hello", t.Term);
Assert.AreEqual("(hello,6,22,type=junk)", t.ToString());
Assert.AreEqual(0, t.Flags);
}
开发者ID:Nangal，项目名称:lucene.net，代码行数:30，代码来源:TestToken.cs
示例2: GetTokenPositioner
public override TokenPositioner GetTokenPositioner(Token token)
{
return
token.GetPositionIncrement() == 0
? TokenPositioner.NewRow
: TokenPositioner.NewColumn;
}
开发者ID:kstenson，项目名称:NHibernate.Search，代码行数:7，代码来源:TwoDimensionalNonWeightedSynonymTokenSettingsCodec.cs
示例3: Next
public override Token Next(Token reusableToken)
{
System.Diagnostics.Debug.Assert(reusableToken != null);
Token nextToken = input.Next(reusableToken);
sink.Add(nextToken);
return nextToken;
}
开发者ID:Inzaghi2012，项目名称:teamlab.v7.5，代码行数:7，代码来源:TeeTokenFilter.cs
示例4: NextToken
public static Token NextToken(TokenStream input, Token reusableToken)
{
if (input == null)
return null;
if (!input.IncrementToken())
return null;
ITermAttribute termAtt = input.GetAttribute<ITermAttribute>();
IOffsetAttribute offsetAtt = input.GetAttribute<IOffsetAttribute>();
ITypeAttribute typeAtt = input.GetAttribute<ITypeAttribute>();
if (reusableToken == null)
{
reusableToken = new Token();
}
reusableToken.Clear();
if (termAtt != null)
reusableToken.SetTermBuffer(termAtt.TermBuffer(), 0, termAtt.TermLength());
if (offsetAtt != null)
{
reusableToken.StartOffset = offsetAtt.StartOffset;
reusableToken.EndOffset = offsetAtt.EndOffset;
}
if (typeAtt != null)
reusableToken.Type = typeAtt.Type;
return reusableToken;
}
开发者ID:vebin，项目名称:Lucene.Net.Analysis.MMSeg，代码行数:31，代码来源:TokenUtils.cs
示例5: Add
public override void Add(Token t)
{
if (t != null && t.Term().ToUpper().Equals("The".ToUpper()))
{
base.Add(t);
}
}
开发者ID:Rationalle，项目名称:ravendb，代码行数:7，代码来源:TestTeeTokenFilter.cs
示例6: Next
public override Token Next(/* in */ Token reusableToken)
{
System.Diagnostics.Debug.Assert(reusableToken != null);
Token nextToken = input.Next(reusableToken);
if (nextToken != null)
{
char[] buffer = nextToken.TermBuffer();
int length = nextToken.TermLength();
// If no characters actually require rewriting then we
// just return token as-is:
for (int i = 0; i < length; i++)
{
char c = buffer[i];
if (c >= '\u00c0' && c <= '\ufb06')
{
RemoveAccents(buffer, length);
nextToken.SetTermBuffer(output, 0, outputPos);
break;
}
}
return nextToken;
}
else
return null;
}
开发者ID:cqm0609，项目名称:lucene-file-finder，代码行数:25，代码来源:ISOLatin1AccentFilter.cs
示例7: GetWeight
/// <summary>
/// Returns a 32 bit float from the payload, or 1f it null.
/// </summary>
/// <param name="token"></param>
/// <returns></returns>
public override float GetWeight(Token token)
{
if (token.GetPayload() == null || token.GetPayload().GetData() == null)
return 1f;
return PayloadHelper.DecodeFloat(token.GetPayload().GetData());
}
开发者ID:kstenson，项目名称:NHibernate.Search，代码行数:12，代码来源:SimpleThreeDimensionalTokenSettingsCodec.cs
示例8: SetWeight
/// <summary>
/// Stores a 32 bit float in the payload, or set it to null if 1f;
/// </summary>
/// <param name="token"></param>
/// <param name="weight"></param>
public override void SetWeight(Token token, float weight)
{
token.SetPayload(
weight == 1f
? null
: new Payload(PayloadHelper.EncodeFloat(weight))
);
}
开发者ID:kstenson，项目名称:NHibernate.Search，代码行数:13，代码来源:SimpleThreeDimensionalTokenSettingsCodec.cs
示例9: CutLeterDigitFilter
public CutLeterDigitFilter(TokenStream input)
: base(input)
{
reusableToken = new Token();
termAtt = AddAttribute<ITermAttribute>();
offsetAtt = AddAttribute<IOffsetAttribute>();
typeAtt = AddAttribute<ITypeAttribute>();
}
开发者ID:vebin，项目名称:Lucene.Net.Analysis.MMSeg，代码行数:8，代码来源:CutLeterDigitFilter.cs
示例10: Next
public override Token Next()
{
Token t = this.input.Next();
if (t != null)
{
t = new Token(_replaceDiacritics(t.TermText()), t.StartOffset(), t.EndOffset()/*, "DiacriticFiltered"*/);
}
return t;
}
开发者ID:usmanghani，项目名称:Misc，代码行数:10，代码来源:DiacriticAnalyzer.cs
示例11: TestToString
public virtual void TestToString()
{
char[] b = new char[]{'a', 'l', 'o', 'h', 'a'};
Token t = new Token("", 0, 5);
t.SetTermBuffer(b, 0, 5);
Assert.AreEqual("(aloha,0,5)", t.ToString());
t.SetTermText("hi there");
Assert.AreEqual("(hi there,0,5)", t.ToString());
}
开发者ID:vikasraz，项目名称:indexsearchutils，代码行数:10，代码来源:TestToken.cs
示例12: AddToken
void AddToken(Token oriToken, int termBufferOffset, int termBufferLength, byte type)
{
Token token = new Token(oriToken.TermBuffer(), termBufferOffset, termBufferLength,
oriToken.StartOffset + termBufferOffset, oriToken.StartOffset + termBufferOffset + termBufferLength);
if (type == (byte)UnicodeCategory.DecimalDigitNumber)
token.Type = Word.TYPE_DIGIT;
else
token.Type = Word.TYPE_LETTER;
tokenQueue.Enqueue(token);
}
开发者ID:vebin，项目名称:Lucene.Net.Analysis.MMSeg，代码行数:10，代码来源:CutLeterDigitFilter.cs
示例13: Next
public override Token Next(Token result)
{
result = input.Next(result);
if (result != null)
{
if (stemmer.Stem(result.TermBuffer(), 0, result.termLength))
result.SetTermBuffer(stemmer.GetResultBuffer(), 0, stemmer.GetResultLength());
return result;
}
else
return null;
}
开发者ID:vikasraz，项目名称:indexsearchutils，代码行数:12，代码来源:PorterStemFilter.cs
示例14: TestResize
public virtual void TestResize()
{
Token t = new Token();
char[] content = "hello".ToCharArray();
t.SetTermBuffer(content, 0, content.Length);
for (int i = 0; i < 2000; i++)
{
t.ResizeTermBuffer(i);
Assert.IsTrue(i <= t.TermBuffer().Length);
Assert.AreEqual("hello", t.Term);
}
}
开发者ID:Nangal，项目名称:lucene.net，代码行数:12，代码来源:TestToken.cs
示例15: Next
public override Token Next(/* in */ Token reusableToken)
{
System.Diagnostics.Debug.Assert(reusableToken != null);
Token nextToken = input.Next(reusableToken);
if (nextToken == null)
return null;
if (stemmer.Stem(nextToken.TermBuffer(), 0, nextToken.TermLength()))
nextToken.SetTermBuffer(stemmer.GetResultBuffer(), 0, stemmer.GetResultLength());
return nextToken;
}
开发者ID:cqm0609，项目名称:lucene-file-finder，代码行数:12，代码来源:PorterStemFilter.cs
示例16: Next
public override Token Next(/* in */ Token reusableToken)
{
System.Diagnostics.Debug.Assert(reusableToken != null);
reusableToken.Clear();
int length = 0;
int start = bufferIndex;
char[] buffer = reusableToken.TermBuffer();
while (true)
{
if (bufferIndex >= dataLen)
{
offset += dataLen;
dataLen = input is Lucene.Net.Index.ReusableStringReader ? ((Lucene.Net.Index.ReusableStringReader) input).Read(ioBuffer) : input.Read((System.Char[]) ioBuffer, 0, ioBuffer.Length);
if (dataLen <= 0)
{
if (length > 0)
break;
else
return null;
}
bufferIndex = 0;
}
char c = ioBuffer[bufferIndex++];
if (IsTokenChar(c))
{
// if it's a token char
if (length == 0)
// start of token
start = offset + bufferIndex - 1;
else if (length == buffer.Length)
buffer = reusableToken.ResizeTermBuffer(1 + length);
buffer[length++] = Normalize(c); // buffer it, normalized
if (length == MAX_WORD_LEN)
// buffer overflow!
break;
}
else if (length > 0)
// at non-Letter w/ chars
break; // return 'em
}
reusableToken.SetTermLength(length);
reusableToken.SetStartOffset(start);
reusableToken.SetEndOffset(start + length);
return reusableToken;
}
开发者ID:cqm0609，项目名称:lucene-file-finder，代码行数:52，代码来源:CharTokenizer.cs
示例17: SingleTokenTokenStream
public SingleTokenTokenStream(Token token)
{
Debug.Assert(token != null, "Token was null!");
_singleToken = (Token) token.Clone();
// ReSharper disable DoNotCallOverridableMethodsInConstructor
_tokenAtt = (AttributeImpl) AddAttribute(typeof (TermAttribute));
// ReSharper restore DoNotCallOverridableMethodsInConstructor
Debug.Assert(_tokenAtt is Token || _tokenAtt.GetType().Name.Equals(typeof (TokenWrapper).Name),
"Token Attribute is the wrong type! Type was: " + _tokenAtt.GetType().Name + " but expected " +
typeof (TokenWrapper).Name);
}
开发者ID:kstenson，项目名称:NHibernate.Search，代码行数:13，代码来源:SingleTokenTokenStream.cs
示例18: Next
public override Token Next(Token reusableToken)
{
System.Diagnostics.Debug.Assert(reusableToken != null);
if (iter == null)
iter = lst.GetEnumerator();
// Since this TokenStream can be reset we have to maintain the tokens as immutable
if (iter.MoveNext())
{
Token nextToken = iter.Current;
return (Token) nextToken.Clone();
}
return null;
}
开发者ID:jhuntsman，项目名称:FlexNet，代码行数:13，代码来源:SinkTokenizer.cs
示例19: GetTokenPositioner
/// <summary>
///
/// </summary>
/// <param name="token"></param>
/// <returns>the token flags int value as TokenPosition</returns>
public override TokenPositioner GetTokenPositioner(Token token)
{
switch (token.GetFlags())
{
case 0:
return TokenPositioner.NewColumn;
case 1:
return TokenPositioner.NewRow;
case 2:
return TokenPositioner.SameRow;
}
throw new IOException("Unknown matrix positioning of token " + token);
}
开发者ID:kstenson，项目名称:NHibernate.Search，代码行数:18，代码来源:SimpleThreeDimensionalTokenSettingsCodec.cs
示例20: Next
public override Token Next(Token token)
{
token.Clear();
int length = 0;
int start = bufferIndex;
char[] buffer = token.TermBuffer();
while (true)
{
if (bufferIndex >= dataLen)
{
offset += dataLen;
dataLen = input is Lucene.Net.Index.DocumentsWriter.ReusableStringReader ? ((Lucene.Net.Index.DocumentsWriter.ReusableStringReader) input).Read(ioBuffer) : input.Read((System.Char[]) ioBuffer, 0, ioBuffer.Length);
if (dataLen <= 0)
{
if (length > 0)
break;
else
return null;
}
bufferIndex = 0;
}
char c = ioBuffer[bufferIndex++];
if (IsTokenChar(c))
{
// if it's a token char
if (length == 0)
// start of token
start = offset + bufferIndex - 1;
else if (length == buffer.Length)
buffer = token.ResizeTermBuffer(1 + length);
buffer[length++] = Normalize(c); // buffer it, normalized
if (length == MAX_WORD_LEN)
// buffer overflow!
break;
}
else if (length > 0)
// at non-Letter w/ chars
break; // return 'em
}
token.termLength = length;
token.startOffset = start;
token.endOffset = start + length;
return token;
}
开发者ID:vikasraz，项目名称:indexsearchutils，代码行数:51，代码来源:CharTokenizer.cs
注：本文中的Lucene.Net.Analysis.Token类示例由纯净天空整理自Github/MSDocs等源码及文档管理平台，相关代码片段筛选自各路编程大神贡献的开源项目，源码版权归原作者所有，传播和使用请参考对应项目的License；未经允许，请勿转载。 |
客服电话
APP下载
官方微信
请发表评论