import com.intellij.openapi.util.text.LineTokenizer; //导入依赖的package包/类
public Dimension getPreferredSize(JComponent c) {
  FontMetrics metrics = c.getFontMetrics(c.getFont());
  String tipText = ((JToolTip)c).getTipText();
  if (tipText == null) {
    tipText = "";
  }
  int maxWidth = 0;
  myLines.clear();

  final String[] lines = LineTokenizer.tokenize(tipText.toCharArray(), false);
  for (String line : lines) {
    myLines.add(line);
    int width = SwingUtilities.computeStringWidth(metrics, line);
    if (width > maxWidth) {
      maxWidth = width;
    }
  }

  int height = metrics.getHeight() * ((lines.length < 1)? 1 : lines.length);
  return new Dimension(maxWidth + 6, height + 4);
}
 

import com.intellij.openapi.util.text.LineTokenizer; //导入依赖的package包/类
@Override
public Dimension getPreferredSize() {
  if (myPreferredSize == null) {
    int maxLineLength = 0;
    int linesCount = 0;

    for (LineTokenizer lt = new LineTokenizer(myRawText); !lt.atEnd(); lt.advance()) {
      maxLineLength = Math.max(maxLineLength, lt.getLength());
      linesCount++;
    }

    FontMetrics fontMetrics = ((EditorImpl)getEditor()).getFontMetrics(myTextAttributes != null ? myTextAttributes.getFontType() : Font.PLAIN);
    int preferredHeight = getEditor().getLineHeight() * Math.max(1, linesCount);
    int preferredWidth = fontMetrics.charWidth('m') * maxLineLength;

    Insets insets = getInsets();
    if (insets != null) {
      preferredHeight += insets.top + insets.bottom;
      preferredWidth += insets.left + insets.right;
    }

    myPreferredSize = new Dimension(preferredWidth, preferredHeight);
  }
  return myPreferredSize;
}
 

import com.intellij.openapi.util.text.LineTokenizer; //导入依赖的package包/类
private static void appendAbbreviated(StringBuilder to, String text, int start, int end,
                                      FontMetrics metrics, int maxWidth, boolean replaceLineTerminators) {
  int abbreviationLength = abbreviationLength(text, start, end, metrics, maxWidth, replaceLineTerminators);

  if (!replaceLineTerminators) {
    to.append(text, start, start + abbreviationLength);
  }
  else {
    CharSequenceSubSequence subSeq = new CharSequenceSubSequence(text, start, start + abbreviationLength);
    for (LineTokenizer lt = new LineTokenizer(subSeq); !lt.atEnd(); lt.advance()) {
      to.append(subSeq, lt.getOffset(), lt.getOffset() + lt.getLength());
      if (lt.getLineSeparatorLength() > 0) {
        to.append(RETURN_SYMBOL);
      }
    }
  }

  if (abbreviationLength != end - start) {
    to.append(ABBREVIATION_SUFFIX);
  }
}
 

import com.intellij.openapi.util.text.LineTokenizer; //导入依赖的package包/类
private static int abbreviationLength(String text, int start, int end, FontMetrics metrics, int maxWidth, boolean replaceSeparators) {
  if (metrics.charWidth('m') * (end - start) <= maxWidth) return end - start;

  int abbrWidth = metrics.charWidth(ABBREVIATION_SUFFIX);
  int abbrLength = 0;

  CharSequenceSubSequence subSeq = new CharSequenceSubSequence(text, start, end);
  for (LineTokenizer lt = new LineTokenizer(subSeq); !lt.atEnd(); lt.advance()) {
    for (int i = 0; i < lt.getLength(); i++, abbrLength++) {
      abbrWidth += metrics.charWidth(subSeq.charAt(lt.getOffset() + i));
      if (abbrWidth >= maxWidth) return abbrLength;
    }
    if (replaceSeparators && lt.getLineSeparatorLength() != 0) {
      abbrWidth += metrics.charWidth(RETURN_SYMBOL);
      if (abbrWidth >= maxWidth) return abbrLength;
      abbrLength += lt.getLineSeparatorLength();
    }
  }

  return abbrLength;
}
 

import com.intellij.openapi.util.text.LineTokenizer; //导入依赖的package包/类
/**
 * Extracts paths from site.py generated by buildout 1.5+
 *
 * @param vFile path to site.py
 * @return extracted paths
 */
public static List<String> extractFromSitePy(VirtualFile vFile) throws IOException {
  List<String> result = new ArrayList<String>();
  String text = VfsUtil.loadText(vFile);
  String[] lines = LineTokenizer.tokenize(text, false);
  int index = 0;
  while (index < lines.length && !lines[index].startsWith("def addsitepackages(")) {
    index++;
  }
  while (index < lines.length && !lines[index].trim().startsWith("buildout_paths = [")) {
    index++;
  }
  index++;
  while (index < lines.length && !lines[index].trim().equals("]")) {
    String line = lines[index].trim();
    if (line.endsWith(",")) {
      line = line.substring(0, line.length() - 1);
    }
    if (line.startsWith("'") && line.endsWith("'")) {
      result.add(StringUtil.unescapeStringCharacters(line.substring(1, line.length() - 1)));
    }
    index++;
  }
  return result;
}
 

import com.intellij.openapi.util.text.LineTokenizer; //导入依赖的package包/类
@NotNull
@Override
public List<GenerationNode> expand(int numberInIteration,
                                   int totalIterations, String surroundedText,
                                   CustomTemplateCallback callback,
                                   boolean insertSurroundedTextAtTheEnd, GenerationNode parent) {
  if (surroundedText == null) {
    return myOperand.expand(numberInIteration, totalIterations, null, callback, insertSurroundedTextAtTheEnd, parent);
  }
  String[] lines = LineTokenizer.tokenize(surroundedText, false);
  List<GenerationNode> result = new ArrayList<GenerationNode>();
  for (int i = 0; i < lines.length; i++) {
    result.addAll(myOperand.expand(i, lines.length, lines[i].trim(), callback, insertSurroundedTextAtTheEnd, parent));
  }
  return result;
}
 

import com.intellij.openapi.util.text.LineTokenizer; //导入依赖的package包/类
@NotNull
private Map<String, String> readPackedBranches() {
  if (!myPackedRefsFile.exists()) {
    return Collections.emptyMap();
  }
  try {
    String content = DvcsUtil.tryLoadFile(myPackedRefsFile);
    return ContainerUtil.map2MapNotNull(LineTokenizer.tokenize(content, false), new Function<String, Pair<String, String>>() {
      @Override
      public Pair<String, String> fun(String line) {
        return parsePackedRefsLine(line);
      }
    });
  }
  catch (RepoStateException e) {
    return Collections.emptyMap();
  }
}
 

import com.intellij.openapi.util.text.LineTokenizer; //导入依赖的package包/类
@NotNull
@Override
public List<GenerationNode> expand(int numberInIteration,
                                   int totalIterations, String surroundedText,
                                   CustomTemplateCallback callback,
                                   boolean insertSurroundedTextAtTheEnd, GenerationNode parent) {
  if (surroundedText == null) {
    return myOperand.expand(numberInIteration, totalIterations, surroundedText, callback, insertSurroundedTextAtTheEnd, parent);
  }
  String[] lines = LineTokenizer.tokenize(surroundedText, false);
  List<GenerationNode> result = new ArrayList<GenerationNode>();
  for (int i = 0; i < lines.length; i++) {
    result.addAll(myOperand.expand(i, lines.length, lines[i].trim(), callback, insertSurroundedTextAtTheEnd, parent));
  }
  return result;
}
 

import com.intellij.openapi.util.text.LineTokenizer; //导入依赖的package包/类
private static void appendAbbreviated(StringBuilder to, String text, int start, int end, FontMetrics metrics, int maxWidth, boolean replaceLineTerminators) {
  int abbreviationLength = abbreviationLength(text, start, end, metrics, maxWidth, replaceLineTerminators);

  if (!replaceLineTerminators) {
    to.append(text, start, start + abbreviationLength);
  }
  else {
    CharSequenceSubSequence subSeq = new CharSequenceSubSequence(text, start, start + abbreviationLength);
    for (LineTokenizer lt = new LineTokenizer(subSeq); !lt.atEnd(); lt.advance()) {
      to.append(subSeq, lt.getOffset(), lt.getOffset() + lt.getLength());
      if (lt.getLineSeparatorLength() > 0) {
        to.append(RETURN_SYMBOL);
      }
    }
  }

  if (abbreviationLength != end - start) {
    to.append(ABBREVIATION_SUFFIX);
  }
}
 

import com.intellij.openapi.util.text.LineTokenizer; //导入依赖的package包/类
@NotNull
@Override
public String preprocessOnPaste(final Project project, final PsiFile file, final Editor editor, String text, final RawText rawText) {
  final Document document = editor.getDocument();
  PsiDocumentManager.getInstance(project).commitDocument(document);
  final SelectionModel selectionModel = editor.getSelectionModel();

  // pastes in block selection mode (column mode) are not handled by a CopyPasteProcessor
  final int selectionStart = selectionModel.getSelectionStart();
  final int selectionEnd = selectionModel.getSelectionEnd();
  PsiElement token = findLiteralTokenType(file, selectionStart, selectionEnd);
  if (token == null) {
    return text;
  }

  if (isStringLiteral(token)) {
    StringBuilder buffer = new StringBuilder(text.length());
    @NonNls String breaker = getLineBreaker(token);
    final String[] lines = LineTokenizer.tokenize(text.toCharArray(), false, true);
    for (int i = 0; i < lines.length; i++) {
      buffer.append(escapeCharCharacters(lines[i], token));
      if (i != lines.length - 1) {
        buffer.append(breaker);
      }
      else if (text.endsWith("\n")) {
        buffer.append("\\n");
      }
    }
    text = buffer.toString();
  }
  else if (isCharLiteral(token)) {
    return escapeCharCharacters(text, token);
  }
  return text;
}
 

import com.intellij.openapi.util.text.LineTokenizer; //导入依赖的package包/类
private static String stripSpaces(String text) {
  String[] lines = LineTokenizer.tokenize(text.toCharArray(), false);
  StringBuilder buf = new StringBuilder(text.length());
  for (int i = 0; i < lines.length; i++) {
    if (i > 0) buf.append('\n');
    buf.append(rTrim(lines[i]));
  }
  return buf.toString();
}
 

import com.intellij.openapi.util.text.LineTokenizer; //导入依赖的package包/类
private void updateText(Rectangle clip) {
  FontMetrics fontMetrics = ((EditorImpl)getEditor()).getFontMetrics(myTextAttributes != null ? myTextAttributes.getFontType() : Font.PLAIN);
  Insets insets = getInsets();
  int maxLineWidth = getWidth() - (insets != null ? insets.left + insets.right : 0);

  myDocumentTextBuilder.setLength(0);

  boolean singleLineMode = getHeight() / (float)getEditor().getLineHeight() < 1.1f;
  if (singleLineMode) {
    appendAbbreviated(myDocumentTextBuilder, myRawText, 0, myRawText.length(), fontMetrics, maxLineWidth, true);
  }
  else {
    int lineHeight = getEditor().getLineHeight();
    int firstVisibleLine = clip.y / lineHeight;
    float visibleLinesCountFractional = clip.height / (float)lineHeight;
    int linesToAppend = 1 + (int)visibleLinesCountFractional;

    LineTokenizer lt = new LineTokenizer(myRawText);
    for (int line = 0; !lt.atEnd() && line < firstVisibleLine; lt.advance(), line++) {
      myDocumentTextBuilder.append('\n');
    }

    for (int line = 0; !lt.atEnd() && line < linesToAppend; lt.advance(), line++) {
      int start = lt.getOffset();
      int end = start + lt.getLength();
      appendAbbreviated(myDocumentTextBuilder, myRawText, start, end, fontMetrics, maxLineWidth, false);
      if (lt.getLineSeparatorLength() > 0) {
        myDocumentTextBuilder.append('\n');
      }
    }
  }

  setTextToEditor(myDocumentTextBuilder.toString());
}
 

import com.intellij.openapi.util.text.LineTokenizer; //导入依赖的package包/类
private static LineSet createLineSet(CharSequence text, boolean markModified) {
  TIntArrayList starts = new TIntArrayList();
  TByteArrayList flags = new TByteArrayList();

  LineTokenizer lineTokenizer = new LineTokenizer(text);
  while (!lineTokenizer.atEnd()) {
    starts.add(lineTokenizer.getOffset());
    flags.add((byte) (lineTokenizer.getLineSeparatorLength() | (markModified ? MODIFIED_MASK : 0)));
    lineTokenizer.advance();
  }
  return new LineSet(starts.toNativeArray(), flags.toNativeArray(), text.length());
}
 

import com.intellij.openapi.util.text.LineTokenizer; //导入依赖的package包/类
public GenericPatchApplier(final CharSequence text, List<PatchHunk> hunks) {
  debug("GenericPatchApplier created, hunks: " + hunks.size());
  myLines = new ArrayList<String>();
  Collections.addAll(myLines, LineTokenizer.tokenize(text, false));
  myHunks = hunks;
  myTransformations = new TreeMap<TextRange, MyAppliedData>(new Comparator<TextRange>() {
    @Override
    public int compare(TextRange o1, TextRange o2) {
      return new Integer(o1.getStartOffset()).compareTo(new Integer(o2.getStartOffset()));
    }
  });
  myNotExact = new ArrayList<SplitHunk>();
  myNotBound = new ArrayList<SplitHunk>();
}
 

import com.intellij.openapi.util.text.LineTokenizer; //导入依赖的package包/类
public static String[] removeCommonIndentation(@NotNull final String docstring) {
  // detect common indentation
  final String[] lines = LineTokenizer.tokenize(docstring, false);
  boolean isFirst = true;
  int cutWidth = Integer.MAX_VALUE;
  int firstIndentedLine = 0;
  for (String frag : lines) {
    if (frag.length() == 0) continue;
    int padWidth = 0;
    final Matcher matcher = ourSpacesPattern.matcher(frag);
    if (matcher.find()) {
      padWidth = matcher.end();
    }
    if (isFirst) {
      isFirst = false;
      if (padWidth == 0) {    // first line may have zero padding
        firstIndentedLine = 1;
        continue;
      }
    }
    if (padWidth < cutWidth) cutWidth = padWidth;
  }
  // remove common indentation
  if (cutWidth > 0 && cutWidth < Integer.MAX_VALUE) {
    for (int i = firstIndentedLine; i < lines.length; i += 1) {
      if (lines[i].length() >= cutWidth) {
        lines[i] = lines[i].substring(cutWidth);
      }
    }
  }
  final List<String> result = new ArrayList<String>();
  for (String line : lines) {
    if (line.startsWith(PyConsoleUtil.ORDINARY_PROMPT)) break;
    result.add(line);
  }
  return ArrayUtil.toStringArray(result);
}
 

import com.intellij.openapi.util.text.LineTokenizer; //导入依赖的package包/类
@Override
protected String getLanguagePlaceholderText(@NotNull ASTNode node, @NotNull TextRange range) {
  if (PyFileImpl.isImport(node, false)) {
    return "import ...";
  }
  if (node.getElementType() == PyElementTypes.STRING_LITERAL_EXPRESSION) {
    final String stringValue = ((PyStringLiteralExpression)node.getPsi()).getStringValue().trim();
    final String[] lines = LineTokenizer.tokenize(stringValue, true);
    if (lines.length > 2 && lines[1].trim().length() == 0) {
      return "\"\"\"" + lines [0].trim() + "...\"\"\"";
    }
    return "\"\"\"...\"\"\"";
  }
  return "...";
}
 

import com.intellij.openapi.util.text.LineTokenizer; //导入依赖的package包/类
@NotNull
@Override
public String preprocessOnPaste(Project project, PsiFile file, Editor editor, String text, RawText rawText) {
  final Document document = editor.getDocument();
  PsiDocumentManager.getInstance(project).commitDocument(document);
  final SelectionModel selectionModel = editor.getSelectionModel();

  // pastes in block selection mode (column mode) are not handled by a CopyPasteProcessor
  final int selectionStart = selectionModel.getSelectionStart();
  final int selectionEnd = selectionModel.getSelectionEnd();
  PsiElement token = findLiteralTokenType(file, selectionStart, selectionEnd);
  if (token == null) {
    return text;
  }

  if (isStringLiteral(token)) {
    StringBuilder buffer = new StringBuilder(text.length());
    @NonNls String breaker = getLineBreaker(token);
    final String[] lines = LineTokenizer.tokenize(text.toCharArray(), false, true);
    for (int i = 0; i < lines.length; i++) {
      buffer.append(escapeCharCharacters(lines[i], token));
      if (i != lines.length - 1 || "\n".equals(breaker) && text.endsWith("\n")) {
        buffer.append(breaker);
      }
    }
    text = buffer.toString();
  }
  return text;
}
 

import com.intellij.openapi.util.text.LineTokenizer; //导入依赖的package包/类
private void updateSegments(CharSequence newText, int oldStartLine, int oldEndLine, int offset1,
                            DocumentEventImpl e) {
  int count = 0;
  LineTokenizer lineTokenizer = new LineTokenizer(newText);
  for (int index = oldStartLine; index <= oldEndLine; index++) {
    if (!lineTokenizer.atEnd()) {
      setSegmentAt(mySegments, index, lineTokenizer, offset1, true);
      lineTokenizer.advance();
    } else {
      mySegments.remove(index, oldEndLine + 1);
      break;
    }
    count++;
  }
  if (!lineTokenizer.atEnd()) {
    SegmentArrayWithData insertSegments = new SegmentArrayWithData();
    int i = 0;
    while (!lineTokenizer.atEnd()) {
      setSegmentAt(insertSegments, i, lineTokenizer, offset1, true);
      lineTokenizer.advance();
      count++;
      i++;
    }
    mySegments.insert(insertSegments, oldEndLine + 1);
  }
  int shift = e.getNewLength() - e.getOldLength();
  mySegments.shiftSegments(oldStartLine + count, shift);
}
 

import com.intellij.openapi.util.text.LineTokenizer; //导入依赖的package包/类
private static void setSegmentAt(SegmentArrayWithData segmentArrayWithData, int index, LineTokenizer lineTokenizer, int offsetShift, boolean isModified) {
  int offset = lineTokenizer.getOffset() + offsetShift;
  int length = lineTokenizer.getLength();
  int separatorLength = lineTokenizer.getLineSeparatorLength();
  int separatorAndModifiedFlag = separatorLength;
  if(isModified) {
    separatorAndModifiedFlag |= MODIFIED_MASK;
  }
  segmentArrayWithData.setElementAt(index, offset, offset + length + separatorLength, separatorAndModifiedFlag);
}
 

import com.intellij.openapi.util.text.LineTokenizer; //导入依赖的package包/类
private void initSegments(CharSequence text, boolean toSetModified) {
  mySegments.removeAll();
  LineTokenizer lineTokenizer = new LineTokenizer(text);
  int i = 0;
  while(!lineTokenizer.atEnd()) {
    setSegmentAt(mySegments, i, lineTokenizer, 0, toSetModified);
    i++;
    lineTokenizer.advance();
  }
  // We add empty line at the end, if the last line ends by line separator.
  addEmptyLineAtEnd();
}