import it.unimi.dsi.logging.ProgressLogger; //导入依赖的package包/类
/** Creates the thread.
 *
 * @param frontier the frontier instantiating the thread.
 * @param distributor the distributor used.
 */
public StatsThread(final Frontier frontier, final Distributor distributor) {
	this.frontier = frontier;
	this.distributor = distributor;

	requestLogger = new ProgressLogger(LOGGER, Long.MAX_VALUE, TimeUnit.MILLISECONDS, "requests");
	requestLogger.displayFreeMemory = requestLogger.displayLocalSpeed = true;
	requestLogger.speedTimeUnit = TimeUnit.SECONDS;
	requestLogger.itemTimeUnit = TimeUnit.MILLISECONDS;

	resourceLogger = new ProgressLogger(LOGGER, Long.MAX_VALUE, TimeUnit.MILLISECONDS, "resources");
	resourceLogger.displayLocalSpeed = true;
	resourceLogger.speedTimeUnit = TimeUnit.SECONDS;
	resourceLogger.itemTimeUnit = TimeUnit.MILLISECONDS;

	transferredBytesLogger = new ProgressLogger(LOGGER, Long.MAX_VALUE, TimeUnit.MILLISECONDS, "bytes");
	transferredBytesLogger.displayLocalSpeed = true;
	transferredBytesLogger.speedTimeUnit = TimeUnit.SECONDS;
	transferredBytesLogger.itemTimeUnit = TimeUnit.NANOSECONDS;

	receivedURLsLogger = new ProgressLogger(LOGGER, Long.MAX_VALUE, TimeUnit.MILLISECONDS, "receivedURLs");
	receivedURLsLogger.displayLocalSpeed = true;
	receivedURLsLogger.speedTimeUnit = TimeUnit.SECONDS;
}
 

import it.unimi.dsi.logging.ProgressLogger; //导入依赖的package包/类
@Override
public Void call() throws Exception {
	final ProgressLogger pl = new ProgressLogger(LOGGER, 1, TimeUnit.MINUTES, "records");

	pl.itemsName = "records";
	pl.displayFreeMemory = true;
	pl.displayLocalSpeed = true;
	pl.start("Scanning...");

	try {
		for(;;) {
			final Result<?>[] result = queue.take();
			if (result == Result.END_OF_RESULTS) {
				pl.done();
				return null;
			}
			for (final Result<?> r : result) if (r != null) r.write();
			pl.lightUpdate();
		}
	}
	catch(final Exception e) {
		LOGGER.error("Exception in flushing thread", e);
		throw e;
	}
}
 

import it.unimi.dsi.logging.ProgressLogger; //导入依赖的package包/类
/**
 * Returns a list of pointers to a GZIP archive entries positions (including the end of file).
 *
 * @param in the stream from which to read the GZIP archive.
 * @param pl a progress logger.
 * @return a list of longs where the <em>i</em>-th long is the offset in the stream of the first byte of the <em>i</em>-th archive entry.
 */
public static LongBigArrayBigList index(final InputStream in, final ProgressLogger pl) throws IOException {
	final LongBigArrayBigList pointers = new LongBigArrayBigList();
	long current = 0;
	final GZIPArchiveReader gzar = new GZIPArchiveReader(in);
	GZIPArchive.ReadEntry re;
	for (;;) {
		re = gzar.skipEntry();
		if (re == null) break;
		pointers.add(current);
		current += re.compressedSkipLength;
		if (pl != null) pl.lightUpdate();
	}
	in.close();
	return pointers;
}
 

import it.unimi.dsi.logging.ProgressLogger; //导入依赖的package包/类
public static void main(String[] arg) throws IOException, JSAPException {
	final SimpleJSAP jsap = new SimpleJSAP(GZIPIndexer.class.getName(), "Computes and stores a quasi-succinct index for a compressed archive.",
			new Parameter[] {
				new UnflaggedOption("archive", JSAP.STRING_PARSER, JSAP.REQUIRED, "The name a GZIP's archive."),
				new UnflaggedOption("index", JSAP.STRING_PARSER, JSAP.REQUIRED, "The output (a serialized LongBigList of pointers to the records in the archive) filename."),
			}
	);

	final JSAPResult jsapResult = jsap.parse(arg);
	if (jsap.messagePrinted()) return;

	final FastBufferedInputStream input = new FastBufferedInputStream(new FileInputStream(jsapResult.getString("archive")));
	ProgressLogger pl = new ProgressLogger(LOGGER, 1, TimeUnit.MINUTES, "records");
	pl.start("Scanning...");
	final EliasFanoMonotoneLongBigList list = new EliasFanoMonotoneLongBigList(index(input, pl));
	pl.done();
	BinIO.storeObject(list, jsapResult.getString("index"));
}
 

import it.unimi.dsi.logging.ProgressLogger; //导入依赖的package包/类
@SuppressWarnings({ "rawtypes", "unchecked" })
public static Map invert(Map inputMap) throws InstantiationException,
		IllegalAccessException, InvocationTargetException,
		NoSuchMethodException, ClassNotFoundException {
	LOGGER.info("Inverting map...");
	Map outputMap = (Map) invertMapType(inputMap.getClass()).getConstructor(new Class[] {}).newInstance(new Object[] {});
	
	ProgressLogger pl = new ProgressLogger(LOGGER, "entries");
	pl.expectedUpdates = inputMap.size();
	pl.start();
	
	for (Object entryObj : inputMap.entrySet()) {
		Map.Entry entry = (Map.Entry) entryObj;
		Object oldValue = outputMap.put(entry.getValue(), entry.getKey());
		if (oldValue != null)
			throw new IllegalArgumentException(
					"The value " + entry.getValue() + " is associated to both '" +
					oldValue + "' and '" + entry.getKey() + "'. The map is not" +
					"bijective"
			);
		pl.lightUpdate();
	}
	pl.done();
	return outputMap;
}
 

import it.unimi.dsi.logging.ProgressLogger; //导入依赖的package包/类
public LatentMatrixEstimator(ImmutableGraph graph, Int2ObjectMap<IntSet> node2cat, String output, Matrix initialMatrix) {
	
	rnd = RandomSingleton.get();
	pl = new ProgressLogger(LOGGER, "node couples");
	
	this.graph = graph;
	this.node2cat = node2cat;
	if (graph.numNodes() != node2cat.size()) {
		LOGGER.warn("node2cat file and graph file have a different number of nodes: " +
				"respectively, " + node2cat.size() + " and " + graph.numNodes());
	}
	
	numNodes = graph.numNodes();
	
	classifier = new PAClassifier(initialMatrix);
	this.output = output;
	nArcsLearned = 0;
	
}
 

import it.unimi.dsi.logging.ProgressLogger; //导入依赖的package包/类
public ClassifierResults[] computeNExperiments(final int nExperiments, final int nSamples) {
	final ClassifierResults[] allResults = new ClassifierResults[nExperiments];
	ProgressLogger pl = new ProgressLogger(LOGGER, "experiments");
	pl.expectedUpdates = nExperiments;
	pl.start("Beginning experiments...");
	
	for (int iExperiment = 0; iExperiment < nExperiments; iExperiment++) {
		if (verbose) LOGGER.info("Starting experiment #" + (iExperiment+1) + "...");
		ClassifierResults experimentResult = computeOneExperiment(createTestingSet(nSamples));
		if (verbose) LOGGER.info("Results for experiment #" + (iExperiment+1) + ":\n\n" + experimentResult + "\n");
		allResults[iExperiment] = experimentResult;
		
		pl.update();
	}
	pl.done();
	
	return allResults;
}
 

import it.unimi.dsi.logging.ProgressLogger; //导入依赖的package包/类
public static SummaryStatistics computeStatistics(UnexpectednessScorer scorer) {
	int numNodes = scorer.graph.numNodes();
	SummaryStatistics stats = new SummaryStatistics();
	
	ProgressLogger pl = new ProgressLogger(LOGGER, "docs");
	pl.expectedUpdates = numNodes;
	pl.start("Finding statistics for values of " + scorer + "...");

	for (int i = 0; i < numNodes; i++) {
		for (double x : scorer.scores(i).values()) {
			stats.addValue(x);
			if (Double.isInfinite(x) || Double.isNaN(x))
				throw new ArithmeticException(
						"Scorer " + scorer + " has returned value "
						+ x + " for a result of document " + i);
		}
		pl.lightUpdate();
	}
	pl.done();
	
	return stats;
}
 

import it.unimi.dsi.logging.ProgressLogger; //导入依赖的package包/类
public static SummaryStatistics computeApproxStatistics(UnexpectednessScorer scorer) {
	int numNodes = scorer.graph.numNodes();
	int sampleSize = 10000;
	if (sampleSize > numNodes) sampleSize = numNodes;

	SummaryStatistics stats = new SummaryStatistics();
	
	ProgressLogger pl = new ProgressLogger(LOGGER, "docs");
	pl.expectedUpdates = sampleSize;
	pl.start("Finding statistics for values of " + scorer + " (sample of "+sampleSize+")...");

	for (int i: RandomSingleton.get().ints(sampleSize, 0, numNodes).toArray()) {
		for (double x : scorer.scores(i).values()) {
			stats.addValue(x);
			if (Double.isInfinite(x) || Double.isNaN(x))
				throw new ArithmeticException(
						"Scorer " + scorer + " has returned value "
						+ x + " for a results of document " + i);
		}
		pl.lightUpdate();
	}
	pl.done();
	
	LOGGER.info(scorer + " -- sample of " + numNodes + " -- " +  stats.toString());
	return stats;
}
 

import it.unimi.dsi.logging.ProgressLogger; //导入依赖的package包/类
public void compute() {
	ProgressLogger pl = new ProgressLogger(LOGGER, "pages");
	pl.expectedUpdates = page2cat.size();
	pl.start("Moving old categories to closest milestones...");
	for (IntSet entry : page2cat.values()) {
		IntSet newCategories = new IntOpenHashSet();
		int milestone;
		for (int cat : entry) {
			milestone = closestMilestones[cat];
			if (milestone != -1)
				newCategories.add(milestone);
		}
		entry.clear();
		entry.addAll(newCategories);
		pl.lightUpdate();
	}
	pl.done();
	
}
 

import it.unimi.dsi.logging.ProgressLogger; //导入依赖的package包/类
/** Creates a chunked hash store with given transformation strategy, hash width and progress logger.
 *
 * @param transform a transformation strategy for the elements.
 * @param tempDir a temporary directory for the store files, or {@code null} for the current directory.
 * @param hashWidthOrCountValues if positive, no associated data is saved in the store: {@link Chunk#data(long)} will return this many lower bits
 * of the first of the three hashes associated with the key; zero, values are stored; if negative, values are stored and a map from values
 * to their frequency is computed.
 * @param pl a progress logger, or {@code null}.
 */

public ChunkedHashStore(final TransformationStrategy<? super T> transform, final File tempDir, final int hashWidthOrCountValues, final ProgressLogger pl) throws IOException {
	this.transform = transform;
	this.pl = pl;
	this.tempDir = tempDir;

	this.hashMask = hashWidthOrCountValues <= 0 ? 0 : -1L >>> Long.SIZE - hashWidthOrCountValues;
	if (hashWidthOrCountValues < 0) value2FrequencyMap = new Long2LongOpenHashMap();

	file = new File[DISK_CHUNKS];
	writableByteChannel = new WritableByteChannel[DISK_CHUNKS];
	byteBuffer = new ByteBuffer[DISK_CHUNKS];
	// Create disk chunks
	for(int i = 0; i < DISK_CHUNKS; i++) {
		byteBuffer[i] = ByteBuffer.allocateDirect(BUFFER_SIZE).order(ByteOrder.nativeOrder());
		writableByteChannel[i] = new FileOutputStream(file[i] = File.createTempFile(ChunkedHashStore.class.getSimpleName(), String.valueOf(i), tempDir)).getChannel();
		file[i].deleteOnExit();
	}

	count = new int[DISK_CHUNKS];
}
 

import it.unimi.dsi.logging.ProgressLogger; //导入依赖的package包/类
public static void run( final FastBufferedReader fbr, final DataOutputStream dos, ProgressLogger pl ) throws IOException {
	final MutableString s = new MutableString();
	Object2IntOpenHashMap<MutableString> map = new Object2IntOpenHashMap<MutableString>();
	map.defaultReturnValue( -1 );
	int slash, hostIndex = -1;

	if ( pl != null ) pl.start( "Reading URLS..." );
	while( fbr.readLine( s ) != null ) {
		slash = s.indexOf( '/', 8 );
		// Fix for non-BURL URLs
		if ( slash != -1 ) s.length( slash );
		if ( ( hostIndex = map.getInt( s ) ) == -1 ) map.put( s.copy(), hostIndex = map.size() );
		dos.writeInt( hostIndex );
		if ( pl != null ) pl.lightUpdate();
	}
	
	if ( pl != null ) pl.done();
}
 

import it.unimi.dsi.logging.ProgressLogger; //导入依赖的package包/类
/** Creates a new approximator for the neighbourhood function.
 * 
 * @param g the graph whosee neighbourhood function you want to compute.
 * @param log2m the logarithm of the number of registers per counter.
 * @param pl a progress logger, or <code>null</code>.
 */
public SequentialHyperBall( final ImmutableGraph g, final int log2m, final ProgressLogger pl, final long seed ) throws IOException {
	super( g.numNodes(), g.numNodes(), ensureEnoughRegisters( log2m ), seed );
	
	if ( pl != null ) pl.logger().info( "Precision: " + Util.format( 100 * HyperLogLogCounterArray.relativeStandardDeviation( log2m ) ) + "% (" + m  + " registers/counter, " + registerSize + " bits/counter)" );

	this.g = g;
	this.pl = pl;
	
	numNodes = g.numNodes();
	squareNumNodes = (double)numNodes * numNodes;

	tempFile = File.createTempFile( SequentialHyperBall.class.getName(), "temp" );
	tempFile.deleteOnExit();
	dos = new DataOutputStream( new FastBufferedOutputStream( fos = new FileOutputStream( tempFile ) ) );
	fbis = new FastBufferedInputStream( new FileInputStream( tempFile ) );
	
	accumulator = new long[ counterLongwords ];
	mask = new long[ counterLongwords ];
}
 

import it.unimi.dsi.logging.ProgressLogger; //导入依赖的package包/类
/** Samples a graph via breadth-first visits.
 * 
 * <p>This method will estimate the cumulative distribution function of distances of 
 * a strongly connected graph. If there is more than one connected component, a warning will be given, specifying the size of the component. An {@link IllegalStateException}
 * will be thrown if the algorithm detects that the graph is not strongly connected, but this is not guaranteed to happen.
 * 
 * @param graph a graph.
 * @param k a number of samples.
 * @param naive sample naively: do not stop sampling even when detecting the lack of strong connection.
 * @param threads the requested number of threads (0 for {@link Runtime#availableProcessors()}).
 * @return an array of samples.
 */
protected static int[][] sample( final ImmutableGraph graph, final int k, final boolean naive, final int threads ) {
	ParallelBreadthFirstVisit visit = new ParallelBreadthFirstVisit( graph, threads, false, new ProgressLogger( LOGGER, "nodes" ) );

	final XorShift1024StarRandom random = new XorShift1024StarRandom();

	int componentSize = -1;
	final int[][] result = new int[ k ][];
	for( int i = k; i-- != 0; ) {
		// After the first iteration, we pick a node from the visit queue, unless we are sampling naively.
		visit.clear();
		visit.visit( visit.queue.isEmpty() || naive ? random.nextInt( visit.graph.numNodes() ) : visit.queue.getInt( random.nextInt( visit.queue.size() ) ), componentSize );
		if ( !naive ) componentSize = visitedNodes( visit, componentSize );
		final int maxDistance = visit.maxDistance();
		result[ i ] = new int[ maxDistance + 1 ];
		for( int d = 0; d <= maxDistance; d++ ) result[ i ][ d ] = visit.cutPoints.getInt( d + 1 );
	}
	
	return result;
}
 

import it.unimi.dsi.logging.ProgressLogger; //导入依赖的package包/类
public static void main( String arg[] ) throws IOException, JSAPException, IllegalArgumentException, ClassNotFoundException, IllegalAccessException, InvocationTargetException, InstantiationException, NoSuchMethodException {
	SimpleJSAP jsap = new SimpleJSAP( SequentialHyperBall.class.getName(), "Prints an approximation of the neighbourhood function.",
		new Parameter[] {
			new FlaggedOption( "log2m", JSAP.INTEGER_PARSER, JSAP.NO_DEFAULT, JSAP.REQUIRED, 'l', "log2m", "The logarithm of the number of registers." ),
			new FlaggedOption( "upperBound", JSAP.LONGSIZE_PARSER, Long.toString( Long.MAX_VALUE ), JSAP.NOT_REQUIRED, 'u', "upper-bound", "An upper bound to the number of iteration (default: the graph size)." ),
			new FlaggedOption( "threshold", JSAP.DOUBLE_PARSER, Double.toString( 1E-3 ), JSAP.NOT_REQUIRED, 't', "threshould", "A threshould that will be used to stop the computation by absolute or relative increment." ),
			new Switch( "spec", 's', "spec", "The source is not a basename but rather a specification of the form <ImmutableGraphImplementation>(arg,arg,...)." ),
			new UnflaggedOption( "basename", JSAP.STRING_PARSER, JSAP.NO_DEFAULT, JSAP.REQUIRED, JSAP.NOT_GREEDY, "The basename of the graph." ),
		}		
	);

	JSAPResult jsapResult = jsap.parse( arg );
	if ( jsap.messagePrinted() ) System.exit( 1 );
	
	final boolean spec = jsapResult.getBoolean( "spec" );
	final String basename = jsapResult.getString( "basename" );
	final ProgressLogger pl = new ProgressLogger( LOGGER );
	final int log2m = jsapResult.getInt( "log2m" );
	
	final ImmutableGraph graph = spec ? ObjectParser.fromSpec( basename, ImmutableGraph.class, GraphClassParser.PACKAGE ) : ImmutableGraph.loadOffline( basename );
	
	SequentialHyperBall shb = new SequentialHyperBall( graph, log2m, pl, Util.randomSeed() );
	TextIO.storeDoubles( shb.approximateNeighbourhoodFunction( jsapResult.getLong( "upperBound" ), jsapResult.getDouble( "threshold" ) ), System.out );
	shb.close();
}
 

import it.unimi.dsi.logging.ProgressLogger; //导入依赖的package包/类
/** Write the offset file to a given bit stream.
 * @param obs the output bit stream to which offsets will be written.
 * @param pl a progress logger, or <code>null</code>.  
 */
public void writeOffsets( final OutputBitStream obs, final ProgressLogger pl ) throws IOException {
	final BVGraphNodeIterator nodeIterator = (BVGraphNodeIterator) nodeIterator( 0 );
	int n = numNodes();
	long lastOffset = 0;
	while( n-- != 0 ) {
			// We fetch the current position of the underlying input bit stream, which is at the start of the next node.
			writeOffset( obs, nodeIterator.ibs.readBits() - lastOffset );
			lastOffset = nodeIterator.ibs.readBits();
			nodeIterator.nextInt();
			nodeIterator.outdegree();
			nodeIterator.successorArray();
			if ( pl != null ) pl.update();
	}
	writeOffset( obs, nodeIterator.ibs.readBits() - lastOffset );
}
 

import it.unimi.dsi.logging.ProgressLogger; //导入依赖的package包/类
@Override
public void writeOffsets( final OutputBitStream obs, final ProgressLogger pl ) throws IOException {
    final HdfsBVGraphNodeIterator nodeIterator = (HdfsBVGraphNodeIterator) nodeIterator( 0 );
    int n = numNodes();
    long lastOffset = 0;
    while( n-- != 0 ) {
        writeOffset( obs, nodeIterator.ibs.readBits() - lastOffset );
        lastOffset = nodeIterator.ibs.readBits();
        nodeIterator.nextInt();
        nodeIterator.outdegree();
        nodeIterator.successorArray();
        if ( pl != null ) pl.update();
    }
    writeOffset( obs, nodeIterator.ibs.readBits() - lastOffset );
}
 

import it.unimi.dsi.logging.ProgressLogger; //导入依赖的package包/类
public static void main(String[] arg) throws IOException {
	if (arg.length == 0) {
		System.err.println("Usage: " + BuildRepetitionSet.class.getSimpleName() + " REPETITIONSET");
		System.exit(1);
	}

	final FastBufferedReader fastBufferedReader = new FastBufferedReader(new InputStreamReader(System.in, Charsets.US_ASCII));
	final MutableString s = new MutableString();
	final LongOpenHashSet repeatedSet = new LongOpenHashSet();
	final String outputFilename = arg[0];
	final ProgressLogger pl = new ProgressLogger();

	MutableString lastUrl = new MutableString();
	pl.itemsName = "lines";
	pl.start("Reading... ");
	while(fastBufferedReader.readLine(s) != null) {
		final int firstTab = s.indexOf('\t');
		final int secondTab = s.indexOf('\t', firstTab + 1);
		MutableString url = s.substring(secondTab + 1);
		if (url.equals(lastUrl)) {
			final int storeIndex = Integer.parseInt(new String(s.array(), 0, firstTab));
			final long storePosition = Long.parseLong(new String(s.array(), firstTab + 1, secondTab - firstTab - 1));
			repeatedSet.add((long)storeIndex << 48 | storePosition);
			System.out.print(storeIndex);
			System.out.print('\t');
			System.out.print(storePosition);
			System.out.print('\t');
			System.out.println(url);
		}

		lastUrl = url;
		pl.lightUpdate();
	}

	pl.done();

	fastBufferedReader.close();
	BinIO.storeObject(repeatedSet, outputFilename);
}
 

import it.unimi.dsi.logging.ProgressLogger; //导入依赖的package包/类
@SuppressWarnings("resource")
public static int[] writeRecords(final String path, final int numRecords, final WarcRecord[] randomRecords, final int parallel) throws IOException, InterruptedException {
	final ProgressLogger pl = new ProgressLogger(LOGGER, "records");
	if (parallel <= 1) pl.expectedUpdates = numRecords;
	final ProgressLogger plb = new ProgressLogger(LOGGER, "KB");
	final CountingOutputStream cos = new CountingOutputStream(new FastBufferedOutputStream(new FileOutputStream (path)));
	final WarcWriter ww;
	if (parallel == 0) {
		ww = new UncompressedWarcWriter(cos);
		pl.start("Writing records…");
	} else if (parallel == 1) {
		ww = new CompressedWarcWriter(cos);
		pl.start("Writing records (compressed)…");
	} else {
		ww = null;
		pl.start("SHOULD NOT HAPPEN");
		throw new IllegalStateException();
	}
	plb.start();
	long written = 0;
	final int[] position = new int[numRecords];
	for (int i = 0; i < numRecords; i++) {
		final int pos = RandomTestMocks.RNG.nextInt(randomRecords.length);
		position[i] = pos;
		ww.write(randomRecords[pos]);
		if (parallel <= 0) {
			pl.lightUpdate();
			plb.update((cos.getCount() - written) / 1024);
		}
		written = cos.getCount();
	}
	ww.close();
	pl.done(numRecords);
	plb.done(cos.getCount());
	return position;
}
 

import it.unimi.dsi.logging.ProgressLogger; //导入依赖的package包/类
public void toMatrixMarketFormat(Writer writer) throws IOException {
	writer.write("%%MatrixMarket matrix coordinate real general\n");
	
	int entriesToWrite = numberOfNonZeroEntries();
	
	if (entriesToWrite == 0)
		throw new RuntimeException("Empty matrix. Aborting Matrix Market export.");
	
	int size = computeSize();
	
	writer.write(size + " " + size + " " + entriesToWrite + "\n");
	
	ObjectIterator<Element> elements = entries();
	
	ProgressLogger pl = new ProgressLogger(LOGGER, "elements");
	pl.expectedUpdates = entriesToWrite;
	pl.start("Exporting matrix to Matrix Market format...");
	
	while (elements.hasNext()) {
			Element e = elements.next();
			if (e.value != 0) {
				writer.write((e.i+1) + " " + (e.j+1) + " " + e.value + "\n");
				entriesToWrite--;
				pl.lightUpdate();
			}
		}
	
	pl.done();
		
	if (entriesToWrite != 0)
		throw new IllegalStateException();
}