import org.elasticsearch.common.blobstore.BlobMetaData; //导入依赖的package包/类
private long listBlobsToGetLatestIndexId() throws IOException {
    Map<String, BlobMetaData> blobs = snapshotsBlobContainer.listBlobsByPrefix(INDEX_FILE_PREFIX);
    long latest = RepositoryData.EMPTY_REPO_GEN;
    if (blobs.isEmpty()) {
        // no snapshot index blobs have been written yet
        return latest;
    }
    for (final BlobMetaData blobMetaData : blobs.values()) {
        final String blobName = blobMetaData.name();
        try {
            final long curr = Long.parseLong(blobName.substring(INDEX_FILE_PREFIX.length()));
            latest = Math.max(latest, curr);
        } catch (NumberFormatException nfe) {
            // the index- blob wasn't of the format index-N where N is a number,
            // no idea what this blob is but it doesn't belong in the repository!
            logger.debug("[{}] Unknown blob in the repository: {}", metadata.name(), blobName);
        }
    }
    return latest;
}
 

import org.elasticsearch.common.blobstore.BlobMetaData; //导入依赖的package包/类
/**
 * Finds the next available blob number
 *
 * @param blobs list of blobs in the repository
 * @return next available blob number
 */
protected long findLatestFileNameGeneration(Map<String, BlobMetaData> blobs) {
    long generation = -1;
    for (String name : blobs.keySet()) {
        if (!name.startsWith(DATA_BLOB_PREFIX)) {
            continue;
        }
        name = BlobStoreIndexShardSnapshot.FileInfo.canonicalName(name);
        try {
            long currentGen = Long.parseLong(name.substring(DATA_BLOB_PREFIX.length()), Character.MAX_RADIX);
            if (currentGen > generation) {
                generation = currentGen;
            }
        } catch (NumberFormatException e) {
            logger.warn("file [{}] does not conform to the '{}' schema", name, DATA_BLOB_PREFIX);
        }
    }
    return generation;
}
 

import org.elasticsearch.common.blobstore.BlobMetaData; //导入依赖的package包/类
/**
 * Checks if snapshot file already exists in the list of blobs
 *
 * @param fileInfo file to check
 * @param blobs    list of blobs
 * @return true if file exists in the list of blobs
 */
private boolean snapshotFileExistsInBlobs(BlobStoreIndexShardSnapshot.FileInfo fileInfo, Map<String, BlobMetaData> blobs) {
    BlobMetaData blobMetaData = blobs.get(fileInfo.name());
    if (blobMetaData != null) {
        return blobMetaData.length() == fileInfo.length();
    } else if (blobs.containsKey(fileInfo.partName(0))) {
        // multi part file sum up the size and check
        int part = 0;
        long totalSize = 0;
        while (true) {
            blobMetaData = blobs.get(fileInfo.partName(part++));
            if (blobMetaData == null) {
                break;
            }
            totalSize += blobMetaData.length();
        }
        return totalSize == fileInfo.length();
    }
    // no file, not exact and not multipart
    return false;
}
 

import org.elasticsearch.common.blobstore.BlobMetaData; //导入依赖的package包/类
@Override
public Map<String, BlobMetaData> listBlobsByPrefix(String blobNamePrefix) throws IOException {
    // If we get duplicate files we should just take the last entry
    Map<String, BlobMetaData> builder = new HashMap<>();

    blobNamePrefix = blobNamePrefix == null ? "" : blobNamePrefix;
    try (DirectoryStream<Path> stream = Files.newDirectoryStream(path, blobNamePrefix + "*")) {
        for (Path file : stream) {
            final BasicFileAttributes attrs = Files.readAttributes(file, BasicFileAttributes.class);
            if (attrs.isRegularFile()) {
                builder.put(file.getFileName().toString(), new PlainBlobMetaData(file.getFileName().toString(), attrs.size()));
            }
        }
    }
    return unmodifiableMap(builder);
}
 

import org.elasticsearch.common.blobstore.BlobMetaData; //导入依赖的package包/类
public void testCompressionIsApplied() throws IOException {
    BlobStore blobStore = createTestBlobStore();
    BlobContainer blobContainer = blobStore.blobContainer(BlobPath.cleanPath());
    StringBuilder veryRedundantText = new StringBuilder();
    for (int i = 0; i < randomIntBetween(100, 300); i++) {
        veryRedundantText.append("Blah ");
    }
    ChecksumBlobStoreFormat<BlobObj> checksumFormat = new ChecksumBlobStoreFormat<>(BLOB_CODEC, "%s", BlobObj::fromXContent,
        xContentRegistry(), false, randomBoolean() ? XContentType.SMILE : XContentType.JSON);
    ChecksumBlobStoreFormat<BlobObj> checksumFormatComp = new ChecksumBlobStoreFormat<>(BLOB_CODEC, "%s", BlobObj::fromXContent,
        xContentRegistry(), true, randomBoolean() ? XContentType.SMILE : XContentType.JSON);
    BlobObj blobObj = new BlobObj(veryRedundantText.toString());
    checksumFormatComp.write(blobObj, blobContainer, "blob-comp");
    checksumFormat.write(blobObj, blobContainer, "blob-not-comp");
    Map<String, BlobMetaData> blobs = blobContainer.listBlobsByPrefix("blob-");
    assertEquals(blobs.size(), 2);
    assertThat(blobs.get("blob-not-comp").length(), greaterThan(blobs.get("blob-comp").length()));
}
 

import org.elasticsearch.common.blobstore.BlobMetaData; //导入依赖的package包/类
@Override
public Map<String, BlobMetaData> listBlobsByPrefix(String account, LocationMode mode, String container, String keyPath, String prefix) throws URISyntaxException, StorageException {
    // NOTE: this should be here: if (prefix == null) prefix = "";
    // however, this is really inefficient since deleteBlobsByPrefix enumerates everything and
    // then does a prefix match on the result; it should just call listBlobsByPrefix with the prefix!

    logger.debug("listing container [{}], keyPath [{}], prefix [{}]", container, keyPath, prefix);
    MapBuilder<String, BlobMetaData> blobsBuilder = MapBuilder.newMapBuilder();
    CloudBlobClient client = this.getSelectedClient(account, mode);
    CloudBlobContainer blobContainer = client.getContainerReference(container);

    SocketAccess.doPrivilegedVoidException(() -> {
        if (blobContainer.exists()) {
            for (ListBlobItem blobItem : blobContainer.listBlobs(keyPath + (prefix == null ? "" : prefix))) {
                URI uri = blobItem.getUri();
                logger.trace("blob url [{}]", uri);

                // uri.getPath is of the form /container/keyPath.* and we want to strip off the /container/
                // this requires 1 + container.length() + 1, with each 1 corresponding to one of the /
                String blobPath = uri.getPath().substring(1 + container.length() + 1);

                CloudBlockBlob blob = blobContainer.getBlockBlobReference(blobPath);

                // fetch the blob attributes from Azure (getBlockBlobReference does not do this)
                // this is needed to retrieve the blob length (among other metadata) from Azure Storage
                blob.downloadAttributes();

                BlobProperties properties = blob.getProperties();
                String name = blobPath.substring(keyPath.length());
                logger.trace("blob url [{}], name [{}], size [{}]", uri, name, properties.getLength());
                blobsBuilder.put(name, new PlainBlobMetaData(name, properties.getLength()));
            }
        }
    });

    return blobsBuilder.immutableMap();
}
 

import org.elasticsearch.common.blobstore.BlobMetaData; //导入依赖的package包/类
@Override
public Map<String, BlobMetaData> listBlobsByPrefix(String account, LocationMode mode, String container, String keyPath, String prefix) {
    MapBuilder<String, BlobMetaData> blobsBuilder = MapBuilder.newMapBuilder();
    blobs.forEach((String blobName, ByteArrayOutputStream bos) -> {
        final String checkBlob;
        if (keyPath != null && !keyPath.isEmpty()) {
            // strip off key path from the beginning of the blob name
            checkBlob = blobName.replace(keyPath, "");
        } else {
            checkBlob = blobName;
        }
        if (prefix == null || startsWithIgnoreCase(checkBlob, prefix)) {
            blobsBuilder.put(blobName, new PlainBlobMetaData(checkBlob, bos.size()));
        }
    });
    return blobsBuilder.immutableMap();
}
 

import org.elasticsearch.common.blobstore.BlobMetaData; //导入依赖的package包/类
/**
 * Finds the next available blob number
 *
 * @param blobs list of blobs in the repository
 * @return next available blob number
 */
protected long findLatestFileNameGeneration(Map<String, BlobMetaData> blobs) {
    long generation = -1;
    for (String name : blobs.keySet()) {
        if (!name.startsWith(DATA_BLOB_PREFIX)) {
            continue;
        }
        name = FileInfo.canonicalName(name);
        try {
            long currentGen = Long.parseLong(name.substring(DATA_BLOB_PREFIX.length()), Character.MAX_RADIX);
            if (currentGen > generation) {
                generation = currentGen;
            }
        } catch (NumberFormatException e) {
            logger.warn("file [{}] does not conform to the '{}' schema", name, DATA_BLOB_PREFIX);
        }
    }
    return generation;
}
 

import org.elasticsearch.common.blobstore.BlobMetaData; //导入依赖的package包/类
@Override
public ImmutableMap<String, BlobMetaData> listBlobsByPrefix(String blobNamePrefix) throws IOException {
    // using MapBuilder and not ImmutableMap.Builder as it seems like File#listFiles might return duplicate files!
    MapBuilder<String, BlobMetaData> builder = MapBuilder.newMapBuilder();

    blobNamePrefix = blobNamePrefix == null ? "" : blobNamePrefix;
    try (DirectoryStream<Path> stream = Files.newDirectoryStream(path, blobNamePrefix + "*")) {
        for (Path file : stream) {
            final BasicFileAttributes attrs = Files.readAttributes(file, BasicFileAttributes.class);
            if (attrs.isRegularFile()) {
                builder.put(file.getFileName().toString(), new PlainBlobMetaData(file.getFileName().toString(), attrs.size()));
            }
        }
    }
    return builder.immutableMap();
}
 

import org.elasticsearch.common.blobstore.BlobMetaData; //导入依赖的package包/类
@Override
public Map<String, BlobMetaData> listBlobs() throws IOException {
    FileStatus[] files = SecurityUtils.execute(blobStore.fileSystemFactory(), new FsCallback<FileStatus[]>() {
        @Override
        public FileStatus[] doInHdfs(FileSystem fs) throws IOException {
            return fs.listStatus(path);
        }
    });
    if (files == null || files.length == 0) {
        return Collections.emptyMap();
    }
    Map<String, BlobMetaData> map = new LinkedHashMap<String, BlobMetaData>();
    for (FileStatus file : files) {
        map.put(file.getPath().getName(), new PlainBlobMetaData(file.getPath().getName(), file.getLen()));
    }
    return Collections.unmodifiableMap(map);
}
 

import org.elasticsearch.common.blobstore.BlobMetaData; //导入依赖的package包/类
@Override
public ImmutableMap<String, BlobMetaData> listBlobsByPrefix(@Nullable String blobNamePrefix) throws IOException {
    ImmutableMap.Builder<String, BlobMetaData> blobsBuilder = ImmutableMap.builder();

    List<GridFSDBFile> files;
    if (blobNamePrefix != null) {
        files = blobStore.gridFS().find(new BasicDBObject("filename", "/^" + buildKey(blobNamePrefix) + "/"));
    } else {
        files = blobStore.gridFS().find(new BasicDBObject("filename", "/^" + keyPath + "/"));
    }
     if (files != null && !files.isEmpty()) {
        for (GridFSDBFile file : files) {
            String name = file.getFilename().substring(keyPath.length());
            blobsBuilder.put(name, new PlainBlobMetaData(name, file.getLength()));
        }
    }
    return blobsBuilder.build();
}
 

import org.elasticsearch.common.blobstore.BlobMetaData; //导入依赖的package包/类
/**
 * Get the blobs matching a given prefix
 * @param blobNamePrefix The prefix to look for blobs with
 * @return blobs metadata
 */
@Override
public ImmutableMap<String, BlobMetaData> listBlobsByPrefix(@Nullable final String blobNamePrefix) {
    return SwiftPerms.exec(new PrivilegedAction<ImmutableMap<String, BlobMetaData>>() {
        @Override
        public ImmutableMap<String, BlobMetaData> run() {
            ImmutableMap.Builder<String, BlobMetaData> blobsBuilder = ImmutableMap.builder();
            Collection<DirectoryOrObject> files;
            if (blobNamePrefix != null) {
                files = blobStore.swift().listDirectory(new Directory(buildKey(blobNamePrefix), '/'));
            } else {
                files = blobStore.swift().listDirectory(new Directory(keyPath, '/'));
            }
            if (files != null && !files.isEmpty()) {
                for (DirectoryOrObject object : files) {
                    if (object.isObject()) {
                        String name = object.getName().substring(keyPath.length());
                        blobsBuilder.put(name, new PlainBlobMetaData(name, object.getAsObject().getContentLength()));
                    }
                }
            }
            return blobsBuilder.build();
        }
    });
}
 

import org.elasticsearch.common.blobstore.BlobMetaData; //导入依赖的package包/类
@Override
public Map<String, BlobMetaData> listBlobsByPrefix(
    String blobNamePrefix) throws IOException {
    try {
        final Vector<LsEntry> entries = blobStore.getClient().ls(path());
        if (entries.isEmpty()) {
            return new HashMap<>();
        }

        final String namePrefix = blobNamePrefix == null ? ""
            : blobNamePrefix;
        final MapBuilder<String, BlobMetaData> builder = MapBuilder
            .newMapBuilder();
        for (final LsEntry entry : entries) {
            if (entry.getAttrs().isReg()
                && entry.getFilename().startsWith(namePrefix)) {
                builder.put(entry.getFilename(), new PlainBlobMetaData(
                    entry.getFilename(), entry.getAttrs().getSize()));
            }
        }
        return builder.immutableMap();
    } catch (Exception e) {
        throw new IOException("Failed to load files in " + path().buildAsString("/"), e);
    }
}
 

import org.elasticsearch.common.blobstore.BlobMetaData; //导入依赖的package包/类
@Override
public ImmutableMap<String, BlobMetaData> listBlobs(URL url) throws IOException {
    HttpURLConnection con = (HttpURLConnection) url.openConnection();
    try {
        if (basicAuth != null) {
            con.setRequestProperty("Authorization", basicAuth);
        }
        setPropFindMethod(con);

        int statusCode = con.getResponseCode();
        String responseMessage = con.getResponseMessage();
        if (statusCode >= SC_OK && statusCode < SC_MULTIPLE_CHOICES) {
            return PropFindResponseParser.parse(con.getInputStream());
        }
        if (statusCode == SC_NOT_FOUND) {
            return emptyMap;
        }
        throw new IOException("status code: " + statusCode + " message: " + responseMessage);
    } finally {
        con.disconnect();
    }
}
 

import org.elasticsearch.common.blobstore.BlobMetaData; //导入依赖的package包/类
@Override
public ImmutableMap<String, BlobMetaData> listBlobs(URL url) throws IOException {
    ImmutableMap.Builder<String, BlobMetaData> builder = ImmutableMap.builder();
    DavEntry davEntry = getEntry(url);
    if (davEntry == null) {
        return builder.build();
    }
    for (Map.Entry<String, DavEntry> resourceEntry : davEntry.children.entrySet()) {
        BlobMetaData blobMetaData = new MockBlobMetaData(
            resourceEntry.getKey(),
            resourceEntry.getValue().outputStream != null
                ? resourceEntry.getValue().outputStream.size()
                : 0);
        builder.put(resourceEntry.getKey(), blobMetaData);
    }
    return builder.build();
}
 

import org.elasticsearch.common.blobstore.BlobMetaData; //导入依赖的package包/类
@Override
public Map<String, BlobMetaData> listBlobsByPrefix(@Nullable String prefix) throws IOException {
    logger.trace("listBlobsByPrefix({})", prefix);

    try {
        return blobStore.listBlobsByPrefix(blobStore.container(), keyPath, prefix);
    } catch (URISyntaxException | StorageException e) {
        logger.warn("can not access [{}] in container {{}}: {}", prefix, blobStore.container(), e.getMessage());
        throw new IOException(e);
    }
}
 

import org.elasticsearch.common.blobstore.BlobMetaData; //导入依赖的package包/类
@Override
public Map<String, BlobMetaData> listBlobsByPrefix(@Nullable String blobNamePrefix) throws IOException {
    return AccessController.doPrivileged((PrivilegedAction<Map<String, BlobMetaData>>) () -> {
        MapBuilder<String, BlobMetaData> blobsBuilder = MapBuilder.newMapBuilder();
        AmazonS3 client = blobStore.client();
        SocketAccess.doPrivilegedVoid(() -> {
            ObjectListing prevListing = null;
            while (true) {
                ObjectListing list;
                if (prevListing != null) {
                    list = client.listNextBatchOfObjects(prevListing);
                } else {
                    if (blobNamePrefix != null) {
                        list = client.listObjects(blobStore.bucket(), buildKey(blobNamePrefix));
                    } else {
                        list = client.listObjects(blobStore.bucket(), keyPath);
                    }
                }
                for (S3ObjectSummary summary : list.getObjectSummaries()) {
                    String name = summary.getKey().substring(keyPath.length());
                    blobsBuilder.put(name, new PlainBlobMetaData(name, summary.getSize()));
                }
                if (list.isTruncated()) {
                    prevListing = list;
                } else {
                    break;
                }
            }
        });
        return blobsBuilder.immutableMap();
    });
}
 

import org.elasticsearch.common.blobstore.BlobMetaData; //导入依赖的package包/类
@Override
public Map<String, BlobMetaData> listBlobsByPrefix(@Nullable final String prefix) throws IOException {
    FileStatus[] files = store.execute(fileContext -> (fileContext.util().listStatus(path,
        path -> prefix == null || path.getName().startsWith(prefix))));
    Map<String, BlobMetaData> map = new LinkedHashMap<String, BlobMetaData>();
    for (FileStatus file : files) {
        map.put(file.getPath().getName(), new PlainBlobMetaData(file.getPath().getName(), file.getLen()));
    }
    return Collections.unmodifiableMap(map);
}
 

import org.elasticsearch.common.blobstore.BlobMetaData; //导入依赖的package包/类
@Override
public void deleteBlobsByPrefix(final String blobNamePrefix) throws IOException {
    Map<String, BlobMetaData> blobs = listBlobsByPrefix(blobNamePrefix);
    for (BlobMetaData blob : blobs.values()) {
        deleteBlob(blob.name());
    }
}
 

import org.elasticsearch.common.blobstore.BlobMetaData; //导入依赖的package包/类
/**
 * This operation is not supported by URLBlobContainer
 */
@Override
public Map<String, BlobMetaData> listBlobs() throws IOException {
    throw new UnsupportedOperationException("URL repository doesn't support this operation");
}