import de.l3s.boilerpipe.BoilerpipeExtractor; //导入依赖的package包/类
/**
 * returns the article from an document with its basic html structure.
 *
 * @param HTMLDocument
 * @param URI the uri from the document for resolving the relative anchors in the document to absolute anchors
 * @return String
 */
public String process(HTMLDocument htmlDoc, URI docUri, final BoilerpipeExtractor extractor) {

    final HTMLHighlighter hh = HTMLHighlighter.newExtractingInstance();
    hh.setOutputHighlightOnly(true);

    TextDocument doc;

    String text = "";
    try {
        doc = new BoilerpipeSAXInput(htmlDoc.toInputSource()).getTextDocument();
        extractor.process(doc);
        final InputSource is = htmlDoc.toInputSource();
        text = hh.process(doc, is);
    } catch (Exception ex) {
        return null;
    }


    return removeNotAllowedTags(text, docUri);
}
 

import de.l3s.boilerpipe.BoilerpipeExtractor; //导入依赖的package包/类
/**
 * parses the media (picture, video) out of doc
 * 
 * @param doc document to parse the media out
 * @param extractor extractor to use
 * @return list of extracted media, with size = 0 if no media found
 */
public List<Media> process(String doc, final BoilerpipeExtractor extractor) {
	final HTMLDocument htmlDoc = new HTMLDocument(doc);
	List<Media> media = new ArrayList<Media>();
	TextDocument tdoc;

	try {
		tdoc = new BoilerpipeSAXInput(htmlDoc.toInputSource()).getTextDocument();
		extractor.process(tdoc);
		final InputSource is = htmlDoc.toInputSource();
		media = process(tdoc, is);
	} catch (Exception e) {
		return null;
	}
	return media;
}
 

import de.l3s.boilerpipe.BoilerpipeExtractor; //导入依赖的package包/类
/**
 * Fetches the given {@link URL} using {@link HTMLFetcher} and processes the
 * retrieved HTML using the specified {@link BoilerpipeExtractor}.
 *
 *            The processed {@link TextDocument}.
 *            The original HTML document.
 * @return A List of enclosed links
 * @throws BoilerpipeProcessingException
 */
public List<String> process(final URL url, final BoilerpipeExtractor extractor)
throws IOException, BoilerpipeProcessingException, SAXException {
    final HTMLDocument htmlDoc = HTMLFetcher.fetch(url);

    final TextDocument doc = new BoilerpipeSAXInput(htmlDoc.toInputSource())
    .getTextDocument();
    extractor.process(doc);

    final InputSource is = htmlDoc.toInputSource();

    return process(doc, is);
}
 

import de.l3s.boilerpipe.BoilerpipeExtractor; //导入依赖的package包/类
public String process(final URL url, final BoilerpipeExtractor extractor)
        throws IOException, BoilerpipeProcessingException, SAXException {
    final HTMLDocument htmlDoc = HTMLFetcher.fetch(url);

    // Added to fix bug with unicode characters not being recognized by SAX parser on AppEngine (bug while appending chars to StringBuffer by offset)
    htmlDoc.encodeEscapedCharsAsText();

    // Added to support including images in extracted HTML output
    if (includeImages)
        htmlDoc.encodeImageTagsAsText();

    final TextDocument doc = new BoilerpipeSAXInput(htmlDoc.toInputSource())
                                     .getTextDocument();
    extractor.process(doc);

    final InputSource is = htmlDoc.toInputSource();

    String finalHtml = process(doc, is);

    // Added to fix bug with unicode characters not being recognized by SAX parser on AppEngine (bug while appending chars to StringBuffer by offset)
    finalHtml = HTMLDocument.restoreTextEncodedEscapedChars(finalHtml, htmlDoc.getCharset().name());

    // Added to support including images in extracted HTML output
    if (includeImages)
        finalHtml = HTMLDocument.restoreTextEncodedImageTags(finalHtml, htmlDoc.getCharset().name());

    return finalHtml;
}
 

import de.l3s.boilerpipe.BoilerpipeExtractor; //导入依赖的package包/类
/**
 * Fetches the given {@link java.net.URL} using {@link de.l3s.boilerpipe.sax.HTMLFetcher} and processes the
 * retrieved HTML using the specified {@link BoilerpipeExtractor}.
 * 
 *            The processed {@link TextDocument}.
 *            The original HTML document.
 * @return A List of enclosed {@link Image}s
 * @throws BoilerpipeProcessingException
 */
public List<Image> process(final URL url, final BoilerpipeExtractor extractor)
		throws IOException, BoilerpipeProcessingException, SAXException {
	final HTMLDocument htmlDoc = HTMLFetcher.fetch(url);

	final TextDocument doc = new BoilerpipeSAXInput(htmlDoc.toInputSource())
			.getTextDocument();
	extractor.process(doc);

	final InputSource is = htmlDoc.toInputSource();

	return process(doc, is);
}
 

import de.l3s.boilerpipe.BoilerpipeExtractor; //导入依赖的package包/类
/**
 * Fetches the given {@link URL} using {@link HTMLFetcher} and processes the
 * retrieved HTML using the specified {@link BoilerpipeExtractor}.
 * 
 * @param doc
 *            The processed {@link TextDocument}.
 * @param is
 *            The original HTML document.
 * @return A List of enclosed {@link Image}s
 * @throws BoilerpipeProcessingException
 */
public List<Image> process(final URL url, final BoilerpipeExtractor extractor)
		throws IOException, BoilerpipeProcessingException, SAXException {
	final HTMLDocument htmlDoc = HTMLFetcher.fetch(url);

	final TextDocument doc = new BoilerpipeSAXInput(htmlDoc.toInputSource())
			.getTextDocument();
	extractor.process(doc);

	final InputSource is = htmlDoc.toInputSource();

	return process(doc, is);
}
 

import de.l3s.boilerpipe.BoilerpipeExtractor; //导入依赖的package包/类
public static void main(String[] args) throws InterruptedException, IOException {
  List<String> lines = Files.readLines(new File("data/query.tsv"), Charsets.UTF_8);
  Set<String> ids = new HashSet<>();
  for (String line : lines) {
    ids.add(line.split("\t")[0]);
  }
  BoilerpipeExtractor extractor = CommonExtractors.ARTICLE_EXTRACTOR;
  ExecutorService es = Executors.newFixedThreadPool(10);
  System.out.println(ids.size());
  DecimalFormat df = new DecimalFormat("00");
  for (String id : ids) {
    String googleHtml = Files.toString(new File("data/googlerp", id + ".html"), Charsets.UTF_8);
    Matcher matcher = pattern.matcher(googleHtml);
    int count = 0;
    while (matcher.find()) {
      count++;
      // check existence
      File docHtmlFile = new File("data/context", id + "-" + df.format(count) + ".html");
      File docTextFile = new File("data/context", id + "-" + df.format(count) + ".txt");
      if (docHtmlFile.exists() && docTextFile.exists()) {
        continue;
      }
      // get url
      String url = matcher.group(1);
      if (url.contains("wikihow") || url.contains("google")) {
        continue;
      }
      es.execute(() -> {
        System.out.println(id + " " + url);
        // download url
        try {
          String docHtml = Request.Get(url).connectTimeout(2000).socketTimeout(2000).execute()
                  .returnContent().asString();
          Files.write(docHtml, docHtmlFile, Charsets.UTF_8);
          String docText = extractor.getText(docHtml);
          Files.write(docText, docTextFile, Charsets.UTF_8);
        } catch (Exception e) {
          e.printStackTrace();
        }
      });
    }
  }
  es.shutdown();
  if (!es.awaitTermination(5, TimeUnit.MINUTES)) {
    System.out.println("Timeout occurs for one or some concept retrieval service.");
  }
}
 

import de.l3s.boilerpipe.BoilerpipeExtractor; //导入依赖的package包/类
public static void downloadSearchResult() throws IOException, BoilerpipeProcessingException,
        SAXException, URISyntaxException, InterruptedException {
  List<String> lines = Files.readLines(new File("data/e2e-apkbc-suggested-query.tsv"),
          Charsets.UTF_8);
  Set<String> ids = new HashSet<>();
  for (String line : lines) {
    ids.add(line.split("\t")[0]);
  }
  Pattern pattern = Pattern.compile("<a href=\"([^>\"]*)\" onmousedown=\"");
  BoilerpipeExtractor extractor = CommonExtractors.ARTICLE_EXTRACTOR;
  ExecutorService es = Executors.newFixedThreadPool(10);
  System.out.println(ids.size());
  DecimalFormat df = new DecimalFormat("00");
  for (String id : ids) {
    String googleHtml = Files.toString(new File("data/e2e-googlerp", id + ".html"),
            Charsets.UTF_8);
    Matcher matcher = pattern.matcher(googleHtml);
    int count = 0;
    while (matcher.find()) {
      count++;
      // check existence
      File docHtmlFile = new File("data/e2e-context", id + "-" + df.format(count) + ".html");
      File docTextFile = new File("data/e2e-context", id + "-" + df.format(count) + ".txt");
      if (docHtmlFile.exists() && docTextFile.exists()) {
        continue;
      }
      // get url
      String url = matcher.group(1);
      if (url.contains("wikihow") || url.contains("google")) {
        continue;
      }
      es.execute(() -> {
        System.out.println(id + " " + url);
        // download url
        try {
          String docHtml = Request.Get(url).connectTimeout(2000).socketTimeout(2000).execute()
                  .returnContent().asString();
          Files.write(docHtml, docHtmlFile, Charsets.UTF_8);
          String docText = extractor.getText(docHtml);
          Files.write(docText, docTextFile, Charsets.UTF_8);
        } catch (Exception e) {
          e.printStackTrace();
        }
      });
    }
  }
  es.shutdown();
  if (!es.awaitTermination(5, TimeUnit.SECONDS)) {
    System.out.println("Timeout occurs for one or some concept retrieval service.");
  }
}
 

import de.l3s.boilerpipe.BoilerpipeExtractor; //导入依赖的package包/类
/**
 * Creates a new boilerpipe-based content extractor, using the given
 * extraction rules. The extracted main content will be passed to the
 * <delegate> content handler.
 *
 * @param delegate
 *            The {@link ContentHandler} object
 * @param extractor
 *            Extraction rules to use, e.g. {@link ArticleExtractor}
 */
public BoilerpipeContentHandler(ContentHandler delegate, BoilerpipeExtractor extractor) {
    this.td = null;
    this.delegate = delegate;
    this.extractor = extractor;
}