import org.jwat.warc.WarcReader; //导入依赖的package包/类
private RecordWithOffsetsAndURL getNextResponseRecord(WarcReader warcReader)
		throws IOException {
	WarcRecord wr;
	while (true) {
		try {
			wr = warcReader.getNextRecord();
		} catch (IOException e) {
			continue;
		}
		if (wr == null)
			return null;

		long offset = warcReader.getStartOffset();
		String type = wr.getHeader("WARC-Type").value;
		if (type.equals("response")) {
			byte[] rawContent = IOUtils.toByteArray(wr
					.getPayloadContent());
			long endOffset = warcReader.getOffset();
			String url = wr.getHeader(WARC_TARGET_URI).value;
			return new RecordWithOffsetsAndURL(rawContent, offset, endOffset, url);
		}
	}
}
 

import org.jwat.warc.WarcReader; //导入依赖的package包/类
/**
 * Set digest options for WARC reader.
 * @param reader WARC reader instance
 */
protected void setReaderOptions(WarcReader reader) throws JHOVE2Exception {
    reader.setBlockDigestEnabled(bComputeBlockDigest);
    reader.setPayloadDigestEnabled(bComputePayloadDigest);
    if (!reader.setBlockDigestAlgorithm(blockDigestAlgorithm)) {
        throw new JHOVE2Exception("Invalid block digest algorithm: " + blockDigestAlgorithm);
    }
    if (!reader.setPayloadDigestAlgorithm(payloadDigestAlgorithm)) {
        throw new JHOVE2Exception("Invalid payload digest algorithm: " + payloadDigestAlgorithm);
    }
    reader.setBlockDigestEncoding(blockDigestEncoding);
    reader.setPayloadDigestEncoding(payloadDigestEncoding);
    if (bStrictTargetUriValidation) {
    	reader.setWarcTargetUriProfile(UriProfile.RFC3986);
    } else {
    	reader.setWarcTargetUriProfile(UriProfile.RFC3986_ABS_16BIT_LAX);
    }
    if (bStrictUriValidation) {
    	reader.setUriProfile(UriProfile.RFC3986);
    } else {
    	reader.setUriProfile(UriProfile.RFC3986_ABS_16BIT_LAX);
    }
}
 

import org.jwat.warc.WarcReader; //导入依赖的package包/类
/**
 * Parse WARC records that are not encased in GZip entries. Parsing should
 * should be straight forward with all records accessible through the same
 * source.
 * @param jhove2 the JHove2 characterization context
 * @param sourceFactory JHove2 source factory
 * @param parentSource WARC source unit
 * @param reader WARC reader used to parse records
 * @throws EOFException if EOF occurs prematurely
 * @throws IOException if an IO error occurs while processing
 * @throws JHOVE2Exception if a serious problem needs to be reported
 */
protected void parseRecordsUncompressed(JHOVE2 jhove2, SourceFactory sourceFactory,
        Source parentSource, WarcReader reader)
                throws EOFException, IOException, JHOVE2Exception {
    WarcRecord record;
    // Ensure a WARC reader could be instantiated.
    if (reader != null) {
        parentSource.setIsAggregate(true);
        /*
        * Loop through available records.
        */
        while ((record = reader.getNextRecord()) != null) {
            processRecord(jhove2, sourceFactory, parentSource, record);
        }
    } else {
        throw new JHOVE2Exception("WarcReader is null");
    }
}
 

import org.jwat.warc.WarcReader; //导入依赖的package包/类
/**
 * Parse WARC record(s) where the source has been identified as a source of
 * a GZip module instance. Since each record will presumably be parse from
 * a different source alternative methods in the WARC reader will be used.
 * @param jhove2 the JHove2 characterization context
 * @param sourceFactory JHove2 source factory
 * @param parentSource WARC source unit
 * @param reader WARC reader used to parse records
 * @param offset record offset relative to input stream
 * @throws EOFException if EOF occurs prematurely
 * @throws IOException if an IO error occurs while processing
 * @throws JHOVE2Exception if a serious problem needs to be reported
 */
protected void parseRecordsCompressed(JHOVE2 jhove2, SourceFactory sourceFactory,
        Source parentSource, WarcReader reader, Long offset)
                throws EOFException, IOException, JHOVE2Exception {
    WarcRecord record;
    // Ensure a WARC reader could be instantiated.
    if (reader != null) {
        parentSource.setIsAggregate(true);
        InputStream in = parentSource.getInputStream();
        /*
         * Loop through available records.
         */
        while ((record = reader.getNextRecordFrom(in, offset, 8192)) != null) {
            processRecord(jhove2, sourceFactory, parentSource, record);
        }
    } else {
        throw new JHOVE2Exception("WarcReader is null");
    }
}
 

import org.jwat.warc.WarcReader; //导入依赖的package包/类
public String fullText(String s3Link, long startOffset, long endOffset)
		throws IOException, ServiceException {
	S3Object inputObject = s3.getObject("aws-publicdatasets", s3Link, null,
			null, null, null, startOffset, endOffset);
	WarcReader warcReader = WarcReaderFactory
			.getReaderCompressed(inputObject.getDataInputStream());
	WarcRecord wr = warcReader.getNextRecord();
	String s = IOUtils.toString(wr.getPayloadContent());
	inputObject.closeDataInputStream();
	warcReader.close();
	return s;
}
 

import org.jwat.warc.WarcReader; //导入依赖的package包/类
@Test
public void test() throws IOException {
    Path crawlPath = temp.newFolder().toPath();
    Path outdir = temp.newFolder().toPath();
    TestUtils.unzip(HttrackRecordTest.class.getResourceAsStream("testcrawl-3.49-2.zip"), crawlPath);

    Httrack2Warc httrack2Warc = new Httrack2Warc();
    httrack2Warc.setOutputDirectory(outdir);
    httrack2Warc.convert(crawlPath);

    StringBuilder summary = new StringBuilder();
    try (WarcReader warcReader = WarcReaderFactory.getReaderCompressed(Files.newInputStream(outdir.resolve("crawl-0.warc.gz")))) {
        for (WarcRecord warcRecord: warcReader) {
            String type = getHeader(warcRecord, "WARC-Type");
            String url = getHeader(warcRecord, "WARC-Target-URI");
            summary.append(type).append(" ").append(url).append("\n");
            if (type.equals("request") || type.equals("response")) {
                HttpHeader httpHeader = warcRecord.getHttpHeader();
                assertEquals("HTTP/1.1", httpHeader.httpVersion);
            } else if (type.equals("warcinfo")) {
                String payload = slurp(warcRecord.getPayloadContent());
                assertEquals("software: HTTrack/3.49-2 http://www.httrack.com/\r\n" +
                        "software: httrack2warc https://github.com/nla/httrack2warc\r\n" +
                        "httrackOptions: -%H http://test.example.org/\r\n", payload);
            }
        }
    }

    assertEquals("warcinfo null\n" +
            "response http://test.example.org/\n" +
            "request http://test.example.org/\n" +
            "metadata http://test.example.org/\n" +
            "response http://test.example.org/style.css\n" +
            "request http://test.example.org/style.css\n" +
            "metadata http://test.example.org/style.css\n" +
            "response http://test.example.org/query.html?page=1&query=2&FOO=3&&BaR=4&&#anchor\n" +
            "request http://test.example.org/query.html?page=1&query=2&FOO=3&&BaR=4&&#anchor\n" +
            "metadata http://test.example.org/query.html?page=1&query=2&FOO=3&&BaR=4&&#anchor\n" +
            "response http://test.example.org/another\n" +
            "request http://test.example.org/another\n" +
            "metadata http://test.example.org/another\n" +
            "response http://test.example.org/image.gif\n" +
            "request http://test.example.org/image.gif\n" +
            "metadata http://test.example.org/image.gif\n",
            summary.toString());
}