本文整理汇总了Java中org.apache.hadoop.hive.common.StatsSetupConst类的典型用法代码示例。如果您正苦于以下问题:Java StatsSetupConst类的具体用法?Java StatsSetupConst怎么用?Java StatsSetupConst使用的例子?那么恭喜您, 这里精选的类代码示例或许可以为您提供帮助。
StatsSetupConst类属于org.apache.hadoop.hive.common包,在下文中一共展示了StatsSetupConst类的14个代码示例,这些例子默认根据受欢迎程度排序。您可以为喜欢或者感觉有用的代码点赞,您的评价将有助于我们的系统推荐出更棒的Java代码示例。
示例1: newTable
import org.apache.hadoop.hive.common.StatsSetupConst; //导入依赖的package包/类
private Table newTable() {
Table table = new Table();
table.setDbName(DB_NAME);
table.setTableName(TABLE_NAME);
table.setTableType(TableType.EXTERNAL_TABLE.name());
StorageDescriptor sd = new StorageDescriptor();
sd.setLocation(tableLocation);
table.setSd(sd);
HashMap<String, String> parameters = new HashMap<>();
parameters.put(StatsSetupConst.ROW_COUNT, "1");
table.setParameters(parameters);
table.setPartitionKeys(PARTITIONS);
return table;
}
开发者ID:HotelsDotCom,项目名称:circus-train,代码行数:18,代码来源:ReplicaTest.java
示例2: newPartition
import org.apache.hadoop.hive.common.StatsSetupConst; //导入依赖的package包/类
@Test
public void newPartition() {
Path replicaPartitionPath = new Path(REPLICA_DATA_DESTINATION, REPLICA_PARTITION_SUBPATH);
Partition replica = factory.newReplicaPartition(EVENT_ID, sourceTable, sourcePartition, DB_NAME, TABLE_NAME,
replicaPartitionPath, FULL);
assertThat(replica.getDbName(), is(sourceTable.getDbName()));
assertThat(replica.getTableName(), is(sourceTable.getTableName()));
assertThat(replica.getSd().getInputFormat(), is(INPUT_FORMAT));
assertThat(replica.getSd().getOutputFormat(), is(OUTPUT_FORMAT));
assertThat(replica.getSd().getLocation(), is(replicaPartitionPath.toUri().toString()));
assertThat(replica.getParameters().get("com.hotels.bdp.circustrain.source.table"), is(DB_NAME + "." + TABLE_NAME));
assertThat(replica.getParameters().get("com.hotels.bdp.circustrain.source.metastore.uris"),
is(SOURCE_META_STORE_URIS));
assertThat(replica.getParameters().get("com.hotels.bdp.circustrain.source.location"), is(PARTITION_LOCATION));
assertThat(replica.getParameters().get("com.hotels.bdp.circustrain.replication.event"), is(EVENT_ID));
assertThat(replica.getParameters().get("com.hotels.bdp.circustrain.last.replicated"), is(not(nullValue())));
assertThat(replica.getParameters().get("com.hotels.bdp.circustrain.replication.mode"), is(FULL.name()));
assertThat(replica.getParameters().get("DO_NOT_UPDATE_STATS"), is("true"));
assertThat(replica.getParameters().get("STATS_GENERATED_VIA_STATS_TASK"), is("true"));
assertThat(replica.getParameters().get("STATS_GENERATED"), is("true"));
assertThat(replica.getParameters().get(StatsSetupConst.ROW_COUNT), is("1"));
}
开发者ID:HotelsDotCom,项目名称:circus-train,代码行数:24,代码来源:ReplicaTableFactoryTest.java
示例3: newPartitionWithTransformation
import org.apache.hadoop.hive.common.StatsSetupConst; //导入依赖的package包/类
@Test
public void newPartitionWithTransformation() {
ReplicaTableFactory factory = new ReplicaTableFactory(SOURCE_META_STORE_URIS, TableTransformation.IDENTITY,
PARTITION_TRANSFORMATION, ColumnStatisticsTransformation.IDENTITY);
Path replicaPartitionPath = new Path(REPLICA_DATA_DESTINATION, REPLICA_PARTITION_SUBPATH);
Partition replica = factory.newReplicaPartition(EVENT_ID, sourceTable, sourcePartition, DB_NAME, TABLE_NAME,
replicaPartitionPath, FULL);
assertThat(replica.getDbName(), is(sourceTable.getDbName()));
assertThat(replica.getTableName(), is(sourceTable.getTableName()));
assertThat(replica.getSd().getInputFormat(), is("newInputFormat"));
assertThat(replica.getSd().getOutputFormat(), is(OUTPUT_FORMAT));
assertThat(replica.getSd().getLocation(), is(replicaPartitionPath.toUri().toString()));
assertThat(replica.getParameters().get("com.hotels.bdp.circustrain.source.table"), is(DB_NAME + "." + TABLE_NAME));
assertThat(replica.getParameters().get("com.hotels.bdp.circustrain.source.metastore.uris"),
is(SOURCE_META_STORE_URIS));
assertThat(replica.getParameters().get("com.hotels.bdp.circustrain.source.location"), is(PARTITION_LOCATION));
assertThat(replica.getParameters().get("com.hotels.bdp.circustrain.replication.event"), is(EVENT_ID));
assertThat(replica.getParameters().get("com.hotels.bdp.circustrain.last.replicated"), is(not(nullValue())));
assertThat(replica.getParameters().get("DO_NOT_UPDATE_STATS"), is("true"));
assertThat(replica.getParameters().get("STATS_GENERATED_VIA_STATS_TASK"), is("true"));
assertThat(replica.getParameters().get("STATS_GENERATED"), is("true"));
assertThat(replica.getParameters().get(StatsSetupConst.ROW_COUNT), is("1"));
}
开发者ID:HotelsDotCom,项目名称:circus-train,代码行数:26,代码来源:ReplicaTableFactoryTest.java
示例4: getStatsFromProps
import org.apache.hadoop.hive.common.StatsSetupConst; //导入依赖的package包/类
/**
* Get the stats from table properties. If not found -1 is returned for each stats field.
* CAUTION: stats may not be up-to-date with the underlying data. It is always good to run the ANALYZE command on
* Hive table to have up-to-date stats.
* @param properties
* @return
*/
private HiveStats getStatsFromProps(final Properties properties) {
long numRows = -1;
long sizeInBytes = -1;
try {
final String numRowsProp = properties.getProperty(StatsSetupConst.ROW_COUNT);
if (numRowsProp != null) {
numRows = Long.valueOf(numRowsProp);
}
final String sizeInBytesProp = properties.getProperty(StatsSetupConst.TOTAL_SIZE);
if (sizeInBytesProp != null) {
sizeInBytes = Long.valueOf(sizeInBytesProp);
}
} catch (final NumberFormatException e) {
logger.error("Failed to parse Hive stats in metastore.", e);
// continue with the defaults.
}
return new HiveStats(numRows, sizeInBytes);
}
开发者ID:dremio,项目名称:dremio-oss,代码行数:28,代码来源:DatasetBuilder.java
示例5: getStatsFromProps
import org.apache.hadoop.hive.common.StatsSetupConst; //导入依赖的package包/类
/**
* Get the stats from table properties. If not found -1 is returned for each stats field.
* CAUTION: stats may not be up-to-date with the underlying data. It is always good to run the ANALYZE command on
* Hive table to have up-to-date stats.
*
* @param properties the source of table stats
* @return {@link HiveStats} instance with rows number and size in bytes from specified properties
*/
private HiveStats getStatsFromProps(final Properties properties) {
long numRows = -1;
long sizeInBytes = -1;
try {
final String numRowsProp = properties.getProperty(StatsSetupConst.ROW_COUNT);
if (numRowsProp != null) {
numRows = Long.valueOf(numRowsProp);
}
final String sizeInBytesProp = properties.getProperty(StatsSetupConst.TOTAL_SIZE);
if (sizeInBytesProp != null) {
sizeInBytes = Long.valueOf(sizeInBytesProp);
}
} catch (final NumberFormatException e) {
logger.error("Failed to parse Hive stats in metastore.", e);
// continue with the defaults.
}
return new HiveStats(numRows, sizeInBytes);
}
开发者ID:axbaretto,项目名称:drill,代码行数:29,代码来源:HiveMetadataProvider.java
示例6: newPartition
import org.apache.hadoop.hive.common.StatsSetupConst; //导入依赖的package包/类
private Partition newPartition(String... values) {
Partition partition = new Partition();
partition.setDbName(DB_NAME);
partition.setTableName(TABLE_NAME);
StorageDescriptor sd = new StorageDescriptor();
sd.setLocation(new Path(tableLocation, partitionName(values)).toUri().toString());
sd.setCols(FIELDS);
partition.setSd(sd);
HashMap<String, String> parameters = new HashMap<>();
parameters.put(StatsSetupConst.ROW_COUNT, "1");
partition.setParameters(parameters);
partition.setValues(Arrays.asList(values));
return partition;
}
开发者ID:HotelsDotCom,项目名称:circus-train,代码行数:15,代码来源:ReplicaTest.java
示例7: newTable
import org.apache.hadoop.hive.common.StatsSetupConst; //导入依赖的package包/类
@Test
public void newTable() {
TableAndStatistics replicaAndStats = factory.newReplicaTable(EVENT_ID, sourceTableAndStats, DB_NAME, TABLE_NAME,
REPLICA_DATA_DESTINATION, FULL);
Table replica = replicaAndStats.getTable();
assertThat(replica.getDbName(), is(sourceTable.getDbName()));
assertThat(replica.getTableName(), is(sourceTable.getTableName()));
assertThat(replica.getSd().getInputFormat(), is(INPUT_FORMAT));
assertThat(replica.getSd().getOutputFormat(), is(OUTPUT_FORMAT));
assertThat(replica.getSd().getLocation(), is(REPLICA_DATA_DESTINATION.toUri().toString()));
assertThat(replica.getParameters().get("com.hotels.bdp.circustrain.source.table"), is(DB_NAME + "." + TABLE_NAME));
assertThat(replica.getParameters().get("com.hotels.bdp.circustrain.source.metastore.uris"),
is(SOURCE_META_STORE_URIS));
assertThat(replica.getParameters().get("com.hotels.bdp.circustrain.source.location"), is(TABLE_LOCATION));
assertThat(replica.getParameters().get("com.hotels.bdp.circustrain.replication.event"), is(EVENT_ID));
assertThat(replica.getParameters().get("com.hotels.bdp.circustrain.last.replicated"), is(not(nullValue())));
assertThat(replica.getParameters().get("com.hotels.bdp.circustrain.replication.mode"), is(FULL.name()));
assertThat(replica.getParameters().get("DO_NOT_UPDATE_STATS"), is("true"));
assertThat(replica.getParameters().get("STATS_GENERATED_VIA_STATS_TASK"), is("true"));
assertThat(replica.getParameters().get("STATS_GENERATED"), is("true"));
assertThat(replica.getParameters().get(StatsSetupConst.ROW_COUNT), is("1"));
assertThat(replica.getTableType(), is(TableType.EXTERNAL_TABLE.name()));
assertThat(replica.getParameters().get("EXTERNAL"), is("TRUE"));
assertTrue(MetaStoreUtils.isExternalTable(replica));
assertThat(replicaAndStats.getStatistics(), is(nullValue()));
}
开发者ID:HotelsDotCom,项目名称:circus-train,代码行数:29,代码来源:ReplicaTableFactoryTest.java
示例8: newTableWithTransformation
import org.apache.hadoop.hive.common.StatsSetupConst; //导入依赖的package包/类
@Test
public void newTableWithTransformation() {
ReplicaTableFactory factory = new ReplicaTableFactory(SOURCE_META_STORE_URIS, TABLE_TRANSFORMATION,
PartitionTransformation.IDENTITY, ColumnStatisticsTransformation.IDENTITY);
TableAndStatistics replicaAndStats = factory.newReplicaTable(EVENT_ID, sourceTableAndStats, DB_NAME, TABLE_NAME,
REPLICA_DATA_DESTINATION, FULL);
Table replica = replicaAndStats.getTable();
assertThat(replica.getDbName(), is(sourceTable.getDbName()));
assertThat(replica.getTableName(), is(sourceTable.getTableName()));
assertThat(replica.getSd().getInputFormat(), is(INPUT_FORMAT));
assertThat(replica.getSd().getOutputFormat(), is("newOutputFormat"));
assertThat(replica.getSd().getLocation(), is(REPLICA_DATA_DESTINATION.toUri().toString()));
assertThat(replica.getParameters().get("com.hotels.bdp.circustrain.source.table"), is(DB_NAME + "." + TABLE_NAME));
assertThat(replica.getParameters().get("com.hotels.bdp.circustrain.source.metastore.uris"),
is(SOURCE_META_STORE_URIS));
assertThat(replica.getParameters().get("com.hotels.bdp.circustrain.source.location"), is(TABLE_LOCATION));
assertThat(replica.getParameters().get("com.hotels.bdp.circustrain.replication.event"), is(EVENT_ID));
assertThat(replica.getParameters().get("com.hotels.bdp.circustrain.last.replicated"), is(not(nullValue())));
assertThat(replica.getParameters().get("DO_NOT_UPDATE_STATS"), is("true"));
assertThat(replica.getParameters().get("STATS_GENERATED_VIA_STATS_TASK"), is("true"));
assertThat(replica.getParameters().get("STATS_GENERATED"), is("true"));
assertThat(replica.getParameters().get(StatsSetupConst.ROW_COUNT), is("1"));
assertThat(replicaAndStats.getStatistics(), is(nullValue()));
}
开发者ID:HotelsDotCom,项目名称:circus-train,代码行数:28,代码来源:ReplicaTableFactoryTest.java
示例9: newView
import org.apache.hadoop.hive.common.StatsSetupConst; //导入依赖的package包/类
@Test
public void newView() {
sourceTableAndStats.getTable().setTableType(TableType.VIRTUAL_VIEW.name());
sourceTableAndStats.getTable().getSd().setInputFormat(null);
sourceTableAndStats.getTable().getSd().setOutputFormat(null);
sourceTableAndStats.getTable().getSd().setLocation(null);
TableAndStatistics replicaAndStats = factory.newReplicaTable(EVENT_ID, sourceTableAndStats, DB_NAME, TABLE_NAME,
null, FULL);
Table replica = replicaAndStats.getTable();
assertThat(replica.getDbName(), is(sourceTable.getDbName()));
assertThat(replica.getTableName(), is(sourceTable.getTableName()));
assertThat(replica.getSd().getInputFormat(), is(nullValue()));
assertThat(replica.getSd().getOutputFormat(), is(nullValue()));
assertThat(replica.getSd().getLocation(), is(nullValue()));
assertThat(replica.getParameters().get("com.hotels.bdp.circustrain.source.table"), is(DB_NAME + "." + TABLE_NAME));
assertThat(replica.getParameters().get("com.hotels.bdp.circustrain.source.metastore.uris"),
is(SOURCE_META_STORE_URIS));
assertThat(replica.getParameters().get("com.hotels.bdp.circustrain.source.location"), is(""));
assertThat(replica.getParameters().get("com.hotels.bdp.circustrain.replication.event"), is(EVENT_ID));
assertThat(replica.getParameters().get("com.hotels.bdp.circustrain.last.replicated"), is(not(nullValue())));
assertThat(replica.getParameters().get("com.hotels.bdp.circustrain.replication.mode"), is(FULL.name()));
assertThat(replica.getParameters().get("DO_NOT_UPDATE_STATS"), is("true"));
assertThat(replica.getParameters().get("STATS_GENERATED_VIA_STATS_TASK"), is("true"));
assertThat(replica.getParameters().get("STATS_GENERATED"), is("true"));
assertThat(replica.getParameters().get(StatsSetupConst.ROW_COUNT), is("1"));
assertThat(replica.getTableType(), is(TableType.VIRTUAL_VIEW.name()));
assertTrue(MetaStoreUtils.isView(replica));
assertThat(replicaAndStats.getStatistics(), is(nullValue()));
}
开发者ID:HotelsDotCom,项目名称:circus-train,代码行数:33,代码来源:ReplicaTableFactoryTest.java
示例10: addPartition
import org.apache.hadoop.hive.common.StatsSetupConst; //导入依赖的package包/类
private void addPartition(String databaseName, String tableName, CatalogProtos.PartitionDescProto
partitionDescProto) {
HiveCatalogStoreClientPool.HiveCatalogStoreClient client = null;
try {
client = clientPool.getClient();
Partition partition = new Partition();
partition.setDbName(databaseName);
partition.setTableName(tableName);
Map<String, String> params = new HashMap<>();
params.put(StatsSetupConst.TOTAL_SIZE, Long.toString(partitionDescProto.getNumBytes()));
partition.setParameters(params);
List<String> values = Lists.newArrayList();
for(CatalogProtos.PartitionKeyProto keyProto : partitionDescProto.getPartitionKeysList()) {
values.add(keyProto.getPartitionValue());
}
partition.setValues(values);
Table table = client.getHiveClient().getTable(databaseName, tableName);
StorageDescriptor sd = table.getSd();
sd.setLocation(partitionDescProto.getPath());
partition.setSd(sd);
client.getHiveClient().add_partition(partition);
} catch (Exception e) {
throw new TajoInternalError(e);
} finally {
if (client != null) {
client.release();
}
}
}
开发者ID:apache,项目名称:tajo,代码行数:36,代码来源:HiveCatalogStore.java
示例11: getHiveTableRows
import org.apache.hadoop.hive.common.StatsSetupConst; //导入依赖的package包/类
@Override
public long getHiveTableRows(String database, String tableName) throws Exception {
Table table = getMetaStoreClient().getTable(database, tableName);
return getBasicStatForTable(new org.apache.hadoop.hive.ql.metadata.Table(table), StatsSetupConst.ROW_COUNT);
}
开发者ID:apache,项目名称:kylin,代码行数:6,代码来源:CLIHiveClient.java
示例12: getPartitionsFromHiveMetaStore
import org.apache.hadoop.hive.common.StatsSetupConst; //导入依赖的package包/类
/**
* Get list of partitions matching specified filter.
*
* For example, consider you have a partitioned table for three columns (i.e., col1, col2, col3).
* Assume that an user want to give a condition WHERE (col1 ='1' or col1 = '100') and col3 > 20 .
*
* Then, the filter string would be written as following:
* (col1 =\"1\" or col1 = \"100\") and col3 > 20
*
*
* @param databaseName
* @param tableName
* @param filter
* @return
*/
private List<PartitionDescProto> getPartitionsFromHiveMetaStore(String databaseName, String tableName,
String filter) {
HiveCatalogStoreClientPool.HiveCatalogStoreClient client = null;
List<PartitionDescProto> partitions = null;
TableDescProto tableDesc = null;
List<ColumnProto> parititonColumns = null;
try {
partitions = new ArrayList<>();
client = clientPool.getClient();
List<Partition> hivePartitions = client.getHiveClient().listPartitionsByFilter(databaseName, tableName
, filter, (short) -1);
tableDesc = getTable(databaseName, tableName);
parititonColumns = tableDesc.getPartition().getExpressionSchema().getFieldsList();
StringBuilder partitionName = new StringBuilder();
for (Partition hivePartition : hivePartitions) {
CatalogProtos.PartitionDescProto.Builder builder = CatalogProtos.PartitionDescProto.newBuilder();
builder.setPath(hivePartition.getSd().getLocation());
partitionName.delete(0, partitionName.length());
for (int i = 0; i < parititonColumns.size(); i++) {
if (i > 0) {
partitionName.append(File.separator);
}
partitionName.append(IdentifierUtil.extractSimpleName(parititonColumns.get(i).getName()));
partitionName.append("=");
partitionName.append(hivePartition.getValues().get(i));
}
builder.setPartitionName(partitionName.toString());
Map<String, String> params = hivePartition.getParameters();
if (params != null) {
if (params.get(StatsSetupConst.TOTAL_SIZE) != null) {
builder.setNumBytes(Long.parseLong(params.get(StatsSetupConst.TOTAL_SIZE)));
}
}
partitions.add(builder.build());
}
} catch (Exception e) {
throw new TajoInternalError(e);
} finally {
if (client != null) {
client.release();
}
}
return partitions;
}
开发者ID:apache,项目名称:tajo,代码行数:69,代码来源:HiveCatalogStore.java
示例13: getFileSizeForTable
import org.apache.hadoop.hive.common.StatsSetupConst; //导入依赖的package包/类
public long getFileSizeForTable(Table table) {
return getBasicStatForTable(new org.apache.hadoop.hive.ql.metadata.Table(table), StatsSetupConst.TOTAL_SIZE);
}
开发者ID:KylinOLAP,项目名称:Kylin,代码行数:4,代码来源:HiveClient.java
示例14: getFileNumberForTable
import org.apache.hadoop.hive.common.StatsSetupConst; //导入依赖的package包/类
public long getFileNumberForTable(Table table) {
return getBasicStatForTable(new org.apache.hadoop.hive.ql.metadata.Table(table), StatsSetupConst.NUM_FILES);
}
开发者ID:KylinOLAP,项目名称:Kylin,代码行数:4,代码来源:HiveClient.java
注:本文中的org.apache.hadoop.hive.common.StatsSetupConst类示例整理自Github/MSDocs等源码及文档管理平台,相关代码片段筛选自各路编程大神贡献的开源项目,源码版权归原作者所有,传播和使用请参考对应项目的License;未经允许,请勿转载。 |
请发表评论