import org.apache.commons.vfs2.AllFileSelector; //导入依赖的package包/类
private static Resources findResources(FileObject rootDir, String inputEncoding) throws IOException {
	Resources result = new Resources();
	FileObject[] allFiles = rootDir.findFiles(new AllFileSelector());
	for(int i = 0; i < allFiles.length; i++) {
		FileObject file = allFiles[i];
		if (file.getType() == FileType.FOLDER) {
			continue;
		}
		MediaType mediaType = MediatypeService.determineMediaType(file.getName().getBaseName()); 
		if(mediaType == null) {
			continue;
		}
		String href = file.getName().toString().substring(rootDir.getName().toString().length() + 1);
		byte[] resourceData = IOUtils.toByteArray(file.getContent().getInputStream());
		if(mediaType == MediatypeService.XHTML && ! nl.siegmann.epublib.Constants.CHARACTER_ENCODING.equalsIgnoreCase(inputEncoding)) {
			resourceData = ResourceUtil.recode(inputEncoding, nl.siegmann.epublib.Constants.CHARACTER_ENCODING, resourceData);
		}
		Resource fileResource = new Resource(null, resourceData, href, mediaType);
		result.add(fileResource);
	}
	return result;
}
 

import org.apache.commons.vfs2.AllFileSelector; //导入依赖的package包/类
@BeforeClass
public static void setup() throws Exception {
  // Create a test hadoop configuration
  FileObject ramRoot = VFS.getManager().resolveFile( HADOOP_CONFIGURATIONS_PATH );
  if ( ramRoot.exists() ) {
    ramRoot.delete( new AllFileSelector() );
  }
  ramRoot.createFolder();

  // Create the implementation jars
  ramRoot.resolveFile( "xercesImpl-2.9.1.jar" ).createFile();
  ramRoot.resolveFile( "xml-apis-1.3.04.jar" ).createFile();
  ramRoot.resolveFile( "xml-apis-ext-1.3.04.jar" ).createFile();
  ramRoot.resolveFile( "xerces-version-1.8.0.jar" ).createFile();
  ramRoot.resolveFile( "xercesImpl2-2.9.1.jar" ).createFile();
  ramRoot.resolveFile( "pentaho-hadoop-shims-api-61.2016.04.01-196.jar" ).createFile();
  ramRoot.resolveFile( "commands-3.3.0-I20070605-0010.jar" ).createFile();
  ramRoot.resolveFile( "postgresql-9.3-1102-jdbc4.jar" ).createFile();
  ramRoot.resolveFile( "trilead-ssh2-build213.jar" ).createFile();
  ramRoot.resolveFile( "trilead-ssh2-build215.jar" ).createFile();
}
 

import org.apache.commons.vfs2.AllFileSelector; //导入依赖的package包/类
@BeforeClass
public static void setup() throws Exception {
  // Create a test hadoop configuration
  FileObject ramRoot = VFS.getManager().resolveFile( CONFIG_PROPERTY_CLASSPATH );
  if ( ramRoot.exists() ) {
    ramRoot.delete( new AllFileSelector() );
  }
  ramRoot.createFolder();

  // Create the implementation jars
  ramRoot.resolveFile( "hadoop-mapreduce-client-app-2.7.0-mapr-1602.jar" ).createFile();
  ramRoot.resolveFile( "hadoop-mapreduce-client-common-2.7.0-mapr-1602.jar" ).createFile();
  ramRoot.resolveFile( "hadoop-mapreduce-client-contrib-2.7.0-mapr-1602.jar" ).createFile();
  ramRoot.resolveFile( "hadoop-mapreduce-client-core-2.7.0-mapr-1602.jar" ).createFile();
  ramRoot.resolveFile( "hadoop-mapreduce-client-hs-2.7.0-mapr-1602.jar" ).createFile();

  pmrFolder = tempFolder.newFolder( "pmr" );
  urlTestResources = Thread.currentThread().getContextClassLoader().getResource( PMR_PROPERTIES );
  Files.copy( Paths.get( urlTestResources.toURI() ), Paths.get( pmrFolder.getAbsolutePath(), PMR_PROPERTIES ) );
}
 

import org.apache.commons.vfs2.AllFileSelector; //导入依赖的package包/类
@Test
public void stageForCache() throws Exception {
  DistributedCacheUtilImpl ch = new DistributedCacheUtilImpl( TEST_CONFIG );

  // Copy the contents of test folder
  FileObject source = DistributedCacheTestUtil.createTestFolderWithContent();

  try {
    Path root = new Path( "bin/test/stageArchiveForCacheTest" );
    Path dest = new Path( root, "org/pentaho/mapreduce/" );

    Configuration conf = new Configuration();
    FileSystem fs = DistributedCacheTestUtil.getLocalFileSystem( conf );

    DistributedCacheTestUtil.stageForCacheTester( ch, source, fs, root, dest, 6, 6 );
  } finally {
    source.delete( new AllFileSelector() );
  }
}
 

import org.apache.commons.vfs2.AllFileSelector; //导入依赖的package包/类
@Test
public void stageForCache_destination_exists() throws Exception {
  DistributedCacheUtilImpl ch = new DistributedCacheUtilImpl( TEST_CONFIG );

  Configuration conf = new Configuration();
  FileSystem fs = DistributedCacheTestUtil.getLocalFileSystem( conf );

  FileObject source = DistributedCacheTestUtil.createTestFolderWithContent();
  try {
    Path root = new Path( "bin/test/stageForCache_destination_exists" );
    Path dest = new Path( root, "dest" );

    fs.mkdirs( dest );
    assertTrue( fs.exists( dest ) );
    assertTrue( fs.getFileStatus( dest ).isDir() );

    DistributedCacheTestUtil.stageForCacheTester( ch, source, fs, root, dest, 6, 6 );
  } finally {
    source.delete( new AllFileSelector() );
  }
}
 

import org.apache.commons.vfs2.AllFileSelector; //导入依赖的package包/类
@Test
public void stagePluginsForCache() throws Exception {
  DistributedCacheUtilImpl ch = new DistributedCacheUtilImpl( TEST_CONFIG );

  Configuration conf = new Configuration();
  FileSystem fs = DistributedCacheTestUtil.getLocalFileSystem( conf );

  Path pluginsDir = new Path( "bin/test/plugins-installation-dir" );

  FileObject pluginDir = DistributedCacheTestUtil.createTestFolderWithContent();

  try {
    ch.stagePluginsForCache( fs, pluginsDir, "bin/test/sample-folder" );
    Path pluginInstallPath = new Path( pluginsDir, "bin/test/sample-folder" );
    assertTrue( fs.exists( pluginInstallPath ) );
    ContentSummary summary = fs.getContentSummary( pluginInstallPath );
    assertEquals( 6, summary.getFileCount() );
    assertEquals( 6, summary.getDirectoryCount() );
  } finally {
    pluginDir.delete( new AllFileSelector() );
    fs.delete( pluginsDir, true );
  }
}
 

import org.apache.commons.vfs2.AllFileSelector; //导入依赖的package包/类
@Test
public void installKettleEnvironment() throws Exception {
  DistributedCacheUtilImpl ch = new DistributedCacheUtilImpl( TEST_CONFIG );

  Configuration conf = new Configuration();
  FileSystem fs = DistributedCacheTestUtil.getLocalFileSystem( conf );

  // This "empty pmr" contains a lib/ folder but with no content
  FileObject pmrArchive = KettleVFS.getFileObject( getClass().getResource( "/empty-pmr.zip" ).toURI().getPath() );

  FileObject bigDataPluginDir = DistributedCacheTestUtil.createTestFolderWithContent( DistributedCacheUtilImpl.PENTAHO_BIG_DATA_PLUGIN_FOLDER_NAME );

  Path root = new Path( "bin/test/installKettleEnvironment" );
  try {
    ch.installKettleEnvironment( pmrArchive, fs, root, bigDataPluginDir, null );
    assertTrue( ch.isKettleEnvironmentInstalledAt( fs, root ) );
  } finally {
    bigDataPluginDir.delete( new AllFileSelector() );
    fs.delete( root, true );
  }
}
 

import org.apache.commons.vfs2.AllFileSelector; //导入依赖的package包/类
@Test
public void installKettleEnvironment_additional_plugins() throws Exception {
  DistributedCacheUtilImpl ch = new DistributedCacheUtilImpl( TEST_CONFIG );

  Configuration conf = new Configuration();
  FileSystem fs = DistributedCacheTestUtil.getLocalFileSystem( conf );

  // This "empty pmr" contains a lib/ folder but with no content
  FileObject pmrArchive = KettleVFS.getFileObject( getClass().getResource( "/empty-pmr.zip" ).toURI().getPath() );
  FileObject bigDataPluginDir = DistributedCacheTestUtil.createTestFolderWithContent( DistributedCacheUtilImpl.PENTAHO_BIG_DATA_PLUGIN_FOLDER_NAME );

  String pluginName = "additional-plugin";
  FileObject additionalPluginDir = DistributedCacheTestUtil.createTestFolderWithContent( pluginName );
  Path root = new Path( "bin/test/installKettleEnvironment" );
  try {
    ch.installKettleEnvironment( pmrArchive, fs, root, bigDataPluginDir, "bin/test/" + pluginName );
    assertTrue( ch.isKettleEnvironmentInstalledAt( fs, root ) );
    assertTrue( fs.exists( new Path( root, "plugins/bin/test/" + pluginName ) ) );
  } finally {
    bigDataPluginDir.delete( new AllFileSelector() );
    additionalPluginDir.delete( new AllFileSelector() );
    fs.delete( root, true );
  }
}
 

import org.apache.commons.vfs2.AllFileSelector; //导入依赖的package包/类
@Test
public void extractToTemp() throws Exception {
  DistributedCacheUtilImpl ch = new DistributedCacheUtilImpl( TEST_CONFIG );

  FileObject archive = KettleVFS.getFileObject( getClass().getResource( "/pentaho-mapreduce-sample.jar" ).toURI().getPath() );
  FileObject extracted = ch.extractToTemp( archive );

  assertNotNull( extracted );
  assertTrue( extracted.exists() );
  try {
    // There should be 3 files and 5 directories inside the root folder (which is the 9th entry)
    assertTrue( extracted.findFiles( new AllFileSelector() ).length == 9 );
  } finally {
    // clean up after ourself
    ch.deleteDirectory( extracted );
  }
}
 

import org.apache.commons.vfs2.AllFileSelector; //导入依赖的package包/类
@Test
public void findFiles_vfs() throws Exception {
  DistributedCacheUtilImpl ch = new DistributedCacheUtilImpl( TEST_CONFIG );

  FileObject testFolder = DistributedCacheTestUtil.createTestFolderWithContent();

  try {
    // Simply test we can find the jar files in our test folder
    List<String> jars = ch.findFiles( testFolder, "jar" );
    assertEquals( 4, jars.size() );

    // Look for all files and folders
    List<String> all = ch.findFiles( testFolder, null );
    assertEquals( 12, all.size() );
  } finally {
    testFolder.delete( new AllFileSelector() );
  }
}
 

import org.apache.commons.vfs2.AllFileSelector; //导入依赖的package包/类
@Test
public void stageForCache_destination_no_overwrite() throws Exception {
  DistributedCacheUtilImpl ch = new DistributedCacheUtilImpl( TEST_CONFIG );

  Configuration conf = new Configuration();
  FileSystem fs = DistributedCacheTestUtil.getLocalFileSystem( conf );

  FileObject source = DistributedCacheTestUtil.createTestFolderWithContent();
  try {
    Path root = new Path( "bin/test/stageForCache_destination_exists" );
    Path dest = new Path( root, "dest" );

    fs.mkdirs( dest );
    assertTrue( fs.exists( dest ) );
    assertTrue( fs.getFileStatus( dest ).isDir() );
    try {
      ch.stageForCache( source, fs, dest, false );
    } catch ( KettleFileException ex ) {
      assertTrue( ex.getMessage(), ex.getMessage().contains( "Destination exists" ) );
    } finally {
      fs.delete( root, true );
    }
  } finally {
    source.delete( new AllFileSelector() );
  }
}
 

import org.apache.commons.vfs2.AllFileSelector; //导入依赖的package包/类
private URL[] prepareJarFiles( FileObject zipFile ) throws Exception {

    // zip:file:///tmp/foo.zip
    FileInputList fileList = FileInputList.createFileList( this, new String[] { "zip:" + zipFile.toString(), },
      new String[] { ".*\\.jar$", }, // Include mask: only jar files
      new String[] { ".*classpath\\.jar$", }, // Exclude mask: only jar files
      new String[] { "Y", }, // File required
      new boolean[] { true, } ); // Search sub-directories

    List<URL> files = new ArrayList<URL>();

    // Copy the jar files in the temp folder...
    //
    for ( FileObject file : fileList.getFiles() ) {
      FileObject jarfilecopy =
        KettleVFS.createTempFile(
          file.getName().getBaseName(), ".jar", environmentSubstitute( "${java.io.tmpdir}" ) );
      jarfilecopy.copyFrom( file, new AllFileSelector() );
      files.add( jarfilecopy.getURL() );
    }

    return files.toArray( new URL[files.size()] );
  }
 

import org.apache.commons.vfs2.AllFileSelector; //导入依赖的package包/类
@Override public File localFile(FileObject resource, FileObject dir) {
    if(resource instanceof LocalFile) {
        return FileUtils.toFile(resource);
    }

    final File localDir = localPath(dir);
    if(localDir == null) {
        throw new MetaborgRuntimeException("Replication directory " + dir
            + " is not on the local filesystem, cannot get local file for " + resource);
    }
    try {
        dir.createFolder();

        final FileObject copyLoc;
        if(resource.getType() == FileType.FOLDER) {
            copyLoc = dir;
        } else {
            copyLoc = dir.resolveFile(resource.getName().getBaseName());
        }
        copyLoc.copyFrom(resource, new AllFileSelector());

        return localDir;
    } catch(FileSystemException e) {
        throw new MetaborgRuntimeException("Could not get local file for " + resource, e);
    }
}
 

import org.apache.commons.vfs2.AllFileSelector; //导入依赖的package包/类
@Override
public void download(String remotePath, Path local) throws IOException {
    LocalFile localFileObject = (LocalFile) fileSystemManager.resolveFile(local.toUri().toString());
    FileObject remoteFileObject = remoteRootDirectory.resolveFile(remotePath);
    try {
        localFileObject.copyFrom(remoteFileObject, new AllFileSelector());
    } finally {
        localFileObject.close();
        remoteFileObject.close();
    }

}
 

import org.apache.commons.vfs2.AllFileSelector; //导入依赖的package包/类
@Override
public void upload(Path local, String remotePath) throws IOException {
    LocalFile localFileObject = (LocalFile) fileSystemManager.resolveFile(local.toUri().toString());
    FileObject remoteFileObject = remoteRootDirectory.resolveFile(remotePath);
    try {
        remoteFileObject.copyFrom(localFileObject, new AllFileSelector());
    } finally {
        localFileObject.close();
        remoteFileObject.close();
    }
}
 

import org.apache.commons.vfs2.AllFileSelector; //导入依赖的package包/类
@Override
public void copy(String oldRemotePath, String newRemotePath) throws IOException {
    FileObject newRemoteFileObject = remoteRootDirectory.resolveFile(newRemotePath);
    FileObject oldRemoteFileObject = remoteRootDirectory.resolveFile(oldRemotePath);
    try {
        newRemoteFileObject.copyFrom(oldRemoteFileObject, new AllFileSelector());
    } finally {
        oldRemoteFileObject.close();
        newRemoteFileObject.close();
    }
}
 

import org.apache.commons.vfs2.AllFileSelector; //导入依赖的package包/类
/**
 * Finds a folder to upgrade from based on the "newFolder" parameter -
 * upgrades are performed only within the same major version.
 *
 * @param newFolder
 *            The folder we want to upgrade to (the new version)
 * @return true if upgrade was successful, false otherwise
 */
public boolean upgrade(final FileObject newFolder) {
    try {
        if (newFolder.getChildren().length != 0) {
            // if the folder is not new then we don't want to touch it
            return false;
        }

        final FileObject upgradeFromFolderCandidate = findUpgradeCandidate(newFolder);

        if (upgradeFromFolderCandidate == null) {
            logger.info("Did not find a suitable upgrade candidate");
            return false;
        }

        logger.info("Upgrading DATACLEANER_HOME from : {}", upgradeFromFolderCandidate);
        newFolder.copyFrom(upgradeFromFolderCandidate, new AllFileSelector());

        // special handling of userpreferences.dat - we only want to keep
        // the good parts ;-)
        final UserPreferencesUpgrader userPreferencesUpgrader = new UserPreferencesUpgrader(newFolder);
        userPreferencesUpgrader.upgrade();

        // Overwrite example jobs
        final List<String> allFilePaths = DataCleanerHome.getAllInitialFiles();
        for (final String filePath : allFilePaths) {
            overwriteFileWithDefaults(newFolder, filePath);
        }
        return true;
    } catch (final FileSystemException e) {
        logger.warn("Exception occured during upgrading: {}", e);
        return false;
    }
}
 

import org.apache.commons.vfs2.AllFileSelector; //导入依赖的package包/类
/** Deletes a file or folder. */
@Override
public void delete(String filePath) {
    assertWritingPermitted("delete()");
    File.verifyFilePath(filePath);
    try {
        getFileObject(filePath).delete(new AllFileSelector());
        logger.debug("Deleted {}", filePath);
    }
    catch (IOException e) {
        throw new TechnicalException("Error deleting file", e);
    }
}
 

import org.apache.commons.vfs2.AllFileSelector; //导入依赖的package包/类
@BeforeClass
public static void setup() throws Exception {
  // Create a test hadoop configuration "a"
  FileObject ramRoot = VFS.getManager().resolveFile( HADOOP_CONFIGURATIONS_PATH );
  FileObject aConfigFolder = ramRoot.resolveFile( "a" );
  if ( aConfigFolder.exists() ) {
    aConfigFolder.delete( new AllFileSelector() );
  }
  aConfigFolder.createFolder();

  assertEquals( FileType.FOLDER, aConfigFolder.getType() );

  // Create the properties file for the configuration as hadoop-configurations/a/config.properties
  configFile = aConfigFolder.resolveFile( "config.properties" );
  Properties p = new Properties();
  p.setProperty( "name", "Test Configuration A" );
  p.setProperty( "classpath", "" );
  p.setProperty( "ignore.classes", "" );
  p.setProperty( "library.path", "" );
  p.setProperty( "required.classes", HadoopConfigurationLocatorTest.class.getName() );
  p.store( configFile.getContent().getOutputStream(), "Test Configuration A" );
  configFile.close();

  // Create the implementation jar
  FileObject implJar = aConfigFolder.resolveFile( "a-config.jar" );
  implJar.createFile();

  // Use ShrinkWrap to create the jar and write it out to VFS
  JavaArchive archive = ShrinkWrap.create( JavaArchive.class, "a-configuration.jar" ).addAsServiceProvider(
    HadoopShim.class, MockHadoopShim.class )
    .addClass( MockHadoopShim.class );
  archive.as( ZipExporter.class ).exportTo( implJar.getContent().getOutputStream() );
}
 

import org.apache.commons.vfs2.AllFileSelector; //导入依赖的package包/类
@Test
public void findFiles_hdfs_native() throws Exception {
  DistributedCacheUtilImpl ch = new DistributedCacheUtilImpl( TEST_CONFIG );

  // Copy the contents of test folder
  FileObject source = DistributedCacheTestUtil.createTestFolderWithContent();
  Path root = new Path( "bin/test/stageArchiveForCacheTest" );
  Configuration conf = new Configuration();
  FileSystem fs = DistributedCacheTestUtil.getLocalFileSystem( conf );
  Path dest = new Path( root, "org/pentaho/mapreduce/" );
  try {
    try {
      ch.stageForCache( source, fs, dest, true );

      List<Path> files = ch.findFiles( fs, dest, null );
      assertEquals( 5, files.size() );

      files = ch.findFiles( fs, dest, Pattern.compile( ".*jar$" ) );
      assertEquals( 2, files.size() );

      files = ch.findFiles( fs, dest, Pattern.compile( ".*folder$" ) );
      assertEquals( 1, files.size() );
    } finally {
      fs.delete( root, true );
    }
  } finally {
    source.delete( new AllFileSelector() );
  }
}