本文整理汇总了Java中org.apache.hadoop.mapred.TestRackAwareTaskPlacement类的典型用法代码示例。如果您正苦于以下问题：Java TestRackAwareTaskPlacement类的具体用法？Java TestRackAwareTaskPlacement怎么用？Java TestRackAwareTaskPlacement使用的例子？那么恭喜您, 这里精选的类代码示例或许可以为您提供帮助。
TestRackAwareTaskPlacement类属于org.apache.hadoop.mapred包，在下文中一共展示了TestRackAwareTaskPlacement类的3个代码示例，这些例子默认根据受欢迎程度排序。您可以为喜欢或者感觉有用的代码点赞，您的评价将有助于我们的系统推荐出更棒的Java代码示例。
示例1: testCachingAtLevel
import org.apache.hadoop.mapred.TestRackAwareTaskPlacement; //导入依赖的package包/类
private void testCachingAtLevel(int level) throws Exception {
String namenode = null;
MiniDFSCluster dfs = null;
MiniMRCluster mr = null;
FileSystem fileSys = null;
String testName = "TestMultiLevelCaching";
try {
final int taskTrackers = 1;
// generate the racks
// use rack1 for data node
String rack1 = getRack(0, level);
// use rack2 for task tracker
String rack2 = getRack(1, level);
Configuration conf = new Configuration();
// Run a datanode on host1 under /a/b/c/..../d1/e1/f1
dfs = new MiniDFSCluster(conf, 1, true, new String[] {rack1},
new String[] {"host1.com"});
dfs.waitActive();
fileSys = dfs.getFileSystem();
if (!fileSys.mkdirs(inDir)) {
throw new IOException("Mkdirs failed to create " + inDir.toString());
}
UtilsForTests.writeFile(dfs.getNameNode(), conf,
new Path(inDir + "/file"), (short)1);
namenode = (dfs.getFileSystem()).getUri().getHost() + ":" +
(dfs.getFileSystem()).getUri().getPort();
// Run a job with the (only)tasktracker on host2 under diff topology
// e.g /a/b/c/..../d2/e2/f2.
JobConf jc = new JobConf();
// cache-level = level (unshared levels) + 1(topmost shared node i.e /a)
// + 1 (for host)
jc.setInt("mapred.task.cache.levels", level + 2);
mr = new MiniMRCluster(taskTrackers, namenode, 1, new String[] {rack2},
new String[] {"host2.com"}, jc);
/* The job is configured with 1 map for one (non-splittable) file.
* Since the datanode is running under different subtree, there is no
* node-level data locality but there should be topological locality.
*/
TestRackAwareTaskPlacement.launchJobAndTestCounters(
testName, mr, fileSys, inDir, outputPath, 1, 1, 0, 0);
mr.shutdown();
} finally {
fileSys.delete(inDir, true);
fileSys.delete(outputPath, true);
if (dfs != null) {
dfs.shutdown();
}
}
}
开发者ID:Nextzero，项目名称:hadoop-2.6.0-cdh5.4.3，代码行数:52，代码来源:TestMultipleLevelCaching.java
示例2: testCachingAtLevel
import org.apache.hadoop.mapred.TestRackAwareTaskPlacement; //导入依赖的package包/类
private void testCachingAtLevel(int level) throws IOException {
String namenode = null;
MiniDFSCluster dfs = null;
MiniMRCluster mr = null;
FileSystem fileSys = null;
String testName = "TestMultiLevelCaching";
try {
final int taskTrackers = 1;
// generate the racks
// use rack1 for data node
String rack1 = getRack(0, level);
// use rack2 for task tracker
String rack2 = getRack(1, level);
Configuration conf = new Configuration();
// Run a datanode on host1 under /a/b/c/..../d1/e1/f1
dfs = new MiniDFSCluster(conf, 1, true, new String[] {rack1},
new String[] {"127.0.0.1"});
dfs.waitActive();
fileSys = dfs.getFileSystem();
if (!fileSys.mkdirs(inDir)) {
throw new IOException("Mkdirs failed to create " + inDir.toString());
}
UtilsForTests.writeFile(dfs.getNameNode(), conf,
new Path(inDir + "/file"), (short)1);
namenode = (dfs.getFileSystem()).getUri().getHost() + ":" +
(dfs.getFileSystem()).getUri().getPort();
// Run a job with the (only)tasktracker on host2 under diff topology
// e.g /a/b/c/..../d2/e2/f2.
JobConf jc = new JobConf();
// cache-level = level (unshared levels) + 1(topmost shared node i.e /a)
// + 1 (for host)
jc.setInt("mapred.task.cache.levels", level + 2);
mr = new MiniMRCluster(taskTrackers, namenode, 1, new String[] {rack2},
new String[] {"127.0.0.1"}, jc);
/* The job is configured with 1 map for one (non-splittable) file.
* Since the datanode is running under different subtree, there is no
* node-level data locality but there should be topological locality.
*/
TestRackAwareTaskPlacement.launchJobAndTestCounters(
testName, mr, fileSys, inDir, outputPath, 1, 0, 1, 0);
mr.shutdown();
} finally {
fileSys.delete(inDir, true);
fileSys.delete(outputPath, true);
if (dfs != null) {
dfs.shutdown();
}
}
}
开发者ID:rhli，项目名称:hadoop-EAR，代码行数:52，代码来源:TestMultipleLevelCaching.java
示例3: testCachingAtLevel
import org.apache.hadoop.mapred.TestRackAwareTaskPlacement; //导入依赖的package包/类
private void testCachingAtLevel(int level) throws IOException {
String namenode = null;
MiniDFSCluster dfs = null;
MiniMRCluster mr = null;
FileSystem fileSys = null;
String testName = "TestMultiLevelCaching";
try {
final int taskTrackers = 1;
// generate the racks
// use rack1 for data node
String rack1 = getRack(0, level);
// use rack2 for task tracker
String rack2 = getRack(1, level);
Configuration conf = new Configuration();
// Run a datanode on host1 under /a/b/c/..../d1/e1/f1
dfs = new MiniDFSCluster(conf, 1, true, new String[] {rack1},
new String[] {"host1.com"});
dfs.waitActive();
fileSys = dfs.getFileSystem();
if (!fileSys.mkdirs(inDir)) {
throw new IOException("Mkdirs failed to create " + inDir.toString());
}
UtilsForTests.writeFile(dfs.getNameNode(), conf,
new Path(inDir + "/file"), (short)1);
namenode = (dfs.getFileSystem()).getUri().getHost() + ":" +
(dfs.getFileSystem()).getUri().getPort();
// Run a job with the (only)tasktracker on host2 under diff topology
// e.g /a/b/c/..../d2/e2/f2.
JobConf jc = new JobConf();
// cache-level = level (unshared levels) + 1(topmost shared node i.e /a)
// + 1 (for host)
jc.setInt("mapred.task.cache.levels", level + 2);
mr = new MiniMRCluster(taskTrackers, namenode, 1, new String[] {rack2},
new String[] {"host2.com"}, jc);
/* The job is configured with 1 map for one (non-splittable) file.
* Since the datanode is running under different subtree, there is no
* node-level data locality but there should be topological locality.
*/
TestRackAwareTaskPlacement.launchJobAndTestCounters(
testName, mr, fileSys, inDir, outputPath, 1, 1, 0, 0);
mr.shutdown();
} finally {
fileSys.delete(inDir, true);
fileSys.delete(outputPath, true);
if (dfs != null) {
dfs.shutdown();
}
}
}
开发者ID:Seagate，项目名称:hadoop-on-lustre，代码行数:52，代码来源:TestMultipleLevelCaching.java
注：本文中的org.apache.hadoop.mapred.TestRackAwareTaskPlacement类示例整理自Github/MSDocs等源码及文档管理平台，相关代码片段筛选自各路编程大神贡献的开源项目，源码版权归原作者所有，传播和使用请参考对应项目的License；未经允许，请勿转载。 |
客服电话
APP下载
官方微信
请发表评论