本文整理汇总了Java中org.apache.accumulo.core.client.mapreduce.InputFormatBase类的典型用法代码示例。如果您正苦于以下问题:Java InputFormatBase类的具体用法?Java InputFormatBase怎么用?Java InputFormatBase使用的例子?那么恭喜您, 这里精选的类代码示例或许可以为您提供帮助。
InputFormatBase类属于org.apache.accumulo.core.client.mapreduce包,在下文中一共展示了InputFormatBase类的12个代码示例,这些例子默认根据受欢迎程度排序。您可以为喜欢或者感觉有用的代码点赞,您的评价将有助于我们的系统推荐出更棒的Java代码示例。
示例1: setupAccumuloInput
import org.apache.accumulo.core.client.mapreduce.InputFormatBase; //导入依赖的package包/类
@Override
protected void setupAccumuloInput(final Job job) throws AccumuloSecurityException {
// set up accumulo input
if (!hdfsInput) {
job.setInputFormatClass(AccumuloInputFormat.class);
} else {
job.setInputFormatClass(AccumuloHDFSFileInputFormat.class);
}
AbstractInputFormat.setConnectorInfo(job, userName, new PasswordToken(pwd));
InputFormatBase.setInputTableName(job, RdfCloudTripleStoreUtils.layoutPrefixToTable(rdfTableLayout, tablePrefix));
AbstractInputFormat.setScanAuthorizations(job, authorizations);
if (!mock) {
AbstractInputFormat.setZooKeeperInstance(job, new ClientConfiguration().withInstance(instance).withZkHosts(zk));
} else {
AbstractInputFormat.setMockInstance(job, instance);
}
if (ttl != null) {
final IteratorSetting setting = new IteratorSetting(1, "fi", AgeOffFilter.class);
AgeOffFilter.setTTL(setting, Long.valueOf(ttl));
InputFormatBase.addIterator(job, setting);
}
for (final IteratorSetting iteratorSetting : AccumuloRyaUtils.COMMON_REG_EX_FILTER_SETTINGS) {
InputFormatBase.addIterator(job, iteratorSetting);
}
}
开发者ID:apache,项目名称:incubator-rya,代码行数:26,代码来源:MergeTool.java
示例2: setOfflineScanner
import org.apache.accumulo.core.client.mapreduce.InputFormatBase; //导入依赖的package包/类
public void setOfflineScanner(String tableName) {
/* int index = 0;
String clonedTableName = tableName + "_clone_" + index;
while(conn.tableOperations().exists(clonedTableName)){
clonedTableName = tableName + "_clone_" + index;
index++;
}
try {
conn.tableOperations().clone(tableName, clonedTableName, true, null, null);
conn.tableOperations().offline(clonedTableName);
setTableName(clonedTableName);*/
InputFormatBase.setOfflineTableScan(job, true);
/* } catch (AccumuloException | AccumuloSecurityException
| TableNotFoundException | TableExistsException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}*/
}
开发者ID:boozallen,项目名称:cognition,代码行数:20,代码来源:AccumuloConfiguration.java
示例3: getEdgeRDD
import org.apache.accumulo.core.client.mapreduce.InputFormatBase; //导入依赖的package包/类
public RDD<Tuple2<Object, Edge>> getEdgeRDD(SparkContext sc, Configuration conf) throws IOException, AccumuloSecurityException{
// Load configuration parameters
zk = MRUtils.getACZK(conf);
instance = MRUtils.getACInstance(conf);
userName = MRUtils.getACUserName(conf);
pwd = MRUtils.getACPwd(conf);
mock = MRUtils.getACMock(conf, false);
tablePrefix = MRUtils.getTablePrefix(conf);
// Set authorizations if specified
String authString = conf.get(MRUtils.AC_AUTH_PROP);
if (authString != null && !authString.isEmpty()) {
authorizations = new Authorizations(authString.split(","));
conf.set(ConfigUtils.CLOUDBASE_AUTHS, authString); // for consistency
}
else {
authorizations = AccumuloRdfConstants.ALL_AUTHORIZATIONS;
}
// Set table prefix to the default if not set
if (tablePrefix == null) {
tablePrefix = RdfCloudTripleStoreConstants.TBL_PRFX_DEF;
MRUtils.setTablePrefix(conf, tablePrefix);
}
// Check for required configuration parameters
Preconditions.checkNotNull(instance, "Accumulo instance name [" + MRUtils.AC_INSTANCE_PROP + "] not set.");
Preconditions.checkNotNull(userName, "Accumulo username [" + MRUtils.AC_USERNAME_PROP + "] not set.");
Preconditions.checkNotNull(pwd, "Accumulo password [" + MRUtils.AC_PWD_PROP + "] not set.");
Preconditions.checkNotNull(tablePrefix, "Table prefix [" + MRUtils.TABLE_PREFIX_PROPERTY + "] not set.");
RdfCloudTripleStoreConstants.prefixTables(tablePrefix);
// If connecting to real accumulo, set additional parameters and require zookeepers
if (!mock) conf.set(ConfigUtils.CLOUDBASE_ZOOKEEPERS, zk); // for consistency
// Ensure consistency between alternative configuration properties
conf.set(ConfigUtils.CLOUDBASE_INSTANCE, instance);
conf.set(ConfigUtils.CLOUDBASE_USER, userName);
conf.set(ConfigUtils.CLOUDBASE_PASSWORD, pwd);
conf.setBoolean(ConfigUtils.USE_MOCK_INSTANCE, mock);
conf.set(RdfCloudTripleStoreConfiguration.CONF_TBL_PREFIX, tablePrefix);
Job job = Job.getInstance(conf, sc.appName());
ClientConfiguration clientConfig = new ClientConfiguration().with(ClientProperty.INSTANCE_NAME, instance).with(ClientProperty.INSTANCE_ZK_HOST, zk);
RyaInputFormat.setTableLayout(job, TABLE_LAYOUT.SPO);
RyaInputFormat.setConnectorInfo(job, userName, new PasswordToken(pwd));
RyaInputFormat.setZooKeeperInstance(job, clientConfig);
RyaInputFormat.setScanAuthorizations(job, authorizations);
String tableName = RdfCloudTripleStoreUtils.layoutPrefixToTable(TABLE_LAYOUT.SPO, tablePrefix);
InputFormatBase.setInputTableName(job, tableName);
return sc.newAPIHadoopRDD(job.getConfiguration(), GraphXEdgeInputFormat.class, Object.class, Edge.class);
}
开发者ID:apache,项目名称:incubator-rya,代码行数:50,代码来源:GraphXGraphGenerator.java
示例4: getInstance
import org.apache.accumulo.core.client.mapreduce.InputFormatBase; //导入依赖的package包/类
public static Instance getInstance(JobContext conf) {
return InputFormatBase.getInstance(conf);
}
开发者ID:apache,项目名称:incubator-rya,代码行数:4,代码来源:MRUtils.java
示例5: getPassword
import org.apache.accumulo.core.client.mapreduce.InputFormatBase; //导入依赖的package包/类
public static AuthenticationToken getPassword(JobContext conf) {
return InputFormatBase.getAuthenticationToken(conf);
}
开发者ID:apache,项目名称:incubator-rya,代码行数:4,代码来源:MRUtils.java
示例6: getUsername
import org.apache.accumulo.core.client.mapreduce.InputFormatBase; //导入依赖的package包/类
public static String getUsername(JobContext conf) {
return InputFormatBase.getPrincipal(conf);
}
开发者ID:apache,项目名称:incubator-rya,代码行数:4,代码来源:MRUtils.java
示例7: getTablename
import org.apache.accumulo.core.client.mapreduce.InputFormatBase; //导入依赖的package包/类
public static String getTablename(JobContext conf) {
return InputFormatBase.getInputTableName(conf);
}
开发者ID:apache,项目名称:incubator-rya,代码行数:4,代码来源:MRUtils.java
示例8: run
import org.apache.accumulo.core.client.mapreduce.InputFormatBase; //导入依赖的package包/类
@Override
public int run(final String[] strings) throws Exception {
useMergeFileInput = conf.getBoolean(USE_MERGE_FILE_INPUT, false);
log.info("Setting up Merge Tool...");
setup();
if (useMergeFileInput) {
// When using file input mode the child instance will use a temporary table in the parent instance to
// store the child table data. The two tables will then be merged together.
copyParentPropertiesToChild(conf);
}
for (final String table : tables) {
final String childTable = table.replaceFirst(tablePrefix, childTablePrefix);
final String jobName = "Merge Tool, merging Child Table: " + childTable + ", into Parent Table: " + table + ", " + System.currentTimeMillis();
log.info("Initializing job: " + jobName);
conf.set(MRUtils.JOB_NAME_PROP, jobName);
conf.set(TABLE_NAME_PROP, table);
final Job job = Job.getInstance(conf);
job.setJarByClass(MergeTool.class);
if (useMergeFileInput) {
importChildFilesToTempParentTable(childTable);
}
setupAccumuloInput(job);
InputFormatBase.setInputTableName(job, table);
// Set input output of the particular job
job.setMapOutputKeyClass(Text.class);
job.setMapOutputValueClass(Mutation.class);
job.setOutputKeyClass(Text.class);
job.setOutputValueClass(Mutation.class);
setupAccumuloOutput(job, table);
// Set mapper and reducer classes
job.setMapperClass(MergeToolMapper.class);
job.setReducerClass(Reducer.class);
// Submit the job
final Date beginTime = new Date();
log.info("Job for table \"" + table + "\" started: " + beginTime);
final int exitCode = job.waitForCompletion(true) ? 0 : 1;
if (useMergeFileInput && StringUtils.isNotBlank(tempChildAuths)) {
// Clear any of the temporary child auths given to the parent
final AccumuloRdfConfiguration parentAccumuloRdfConfiguration = new AccumuloRdfConfiguration(conf);
parentAccumuloRdfConfiguration.setTablePrefix(tablePrefix);
final Connector parentConnector = AccumuloRyaUtils.setupConnector(parentAccumuloRdfConfiguration);
final SecurityOperations secOps = parentConnector.securityOperations();
AccumuloRyaUtils.removeUserAuths(userName, secOps, tempChildAuths);
}
if (exitCode == 0) {
final Date endTime = new Date();
log.info("Job for table \"" + table + "\" finished: " + endTime);
log.info("The job took " + (endTime.getTime() - beginTime.getTime()) / 1000 + " seconds.");
} else {
log.error("Job for table \"" + table + "\" Failed!!!");
return exitCode;
}
}
return 0;
}
开发者ID:apache,项目名称:incubator-rya,代码行数:71,代码来源:MergeTool.java
示例9: setRanges
import org.apache.accumulo.core.client.mapreduce.InputFormatBase; //导入依赖的package包/类
public void setRanges(Collection<Range> ranges) {
InputFormatBase.setRanges(job, ranges);
}
开发者ID:boozallen,项目名称:cognition,代码行数:4,代码来源:AccumuloConfiguration.java
示例10: setTableName
import org.apache.accumulo.core.client.mapreduce.InputFormatBase; //导入依赖的package包/类
public void setTableName(String tableName) {
InputFormatBase.setInputTableName(job, tableName);
}
开发者ID:boozallen,项目名称:cognition,代码行数:4,代码来源:AccumuloConfiguration.java
示例11: addIterator
import org.apache.accumulo.core.client.mapreduce.InputFormatBase; //导入依赖的package包/类
public void addIterator(IteratorSetting is) {
InputFormatBase.addIterator(job, is);
}
开发者ID:boozallen,项目名称:cognition,代码行数:4,代码来源:AccumuloConfiguration.java
示例12: fetchColumns
import org.apache.accumulo.core.client.mapreduce.InputFormatBase; //导入依赖的package包/类
public void fetchColumns(Collection<Pair<Text, Text>> columnFamilyColumnQualifierPairs) {
InputFormatBase.fetchColumns(job, columnFamilyColumnQualifierPairs);
}
开发者ID:boozallen,项目名称:cognition,代码行数:4,代码来源:AccumuloConfiguration.java
注:本文中的org.apache.accumulo.core.client.mapreduce.InputFormatBase类示例整理自Github/MSDocs等源码及文档管理平台,相关代码片段筛选自各路编程大神贡献的开源项目,源码版权归原作者所有,传播和使用请参考对应项目的License;未经允许,请勿转载。 |
请发表评论