本文整理汇总了Java中parquet.schema.PrimitiveType.PrimitiveTypeName类的典型用法代码示例。如果您正苦于以下问题:Java PrimitiveTypeName类的具体用法?Java PrimitiveTypeName怎么用?Java PrimitiveTypeName使用的例子?那么恭喜您, 这里精选的类代码示例或许可以为您提供帮助。
PrimitiveTypeName类属于parquet.schema.PrimitiveType包,在下文中一共展示了PrimitiveTypeName类的20个代码示例,这些例子默认根据受欢迎程度排序。您可以为喜欢或者感觉有用的代码点赞,您的评价将有助于我们的系统推荐出更棒的Java代码示例。
示例1: ColumnReader
import parquet.schema.PrimitiveType.PrimitiveTypeName; //导入依赖的package包/类
protected ColumnReader(ParquetRecordReader parentReader, int allocateSize, ColumnDescriptor descriptor,
ColumnChunkMetaData columnChunkMetaData, boolean fixedLength, V v, SchemaElement schemaElement) throws ExecutionSetupException {
this.parentReader = parentReader;
this.columnDescriptor = descriptor;
this.columnChunkMetaData = columnChunkMetaData;
this.isFixedLength = fixedLength;
this.schemaElement = schemaElement;
this.valueVec = v;
this.pageReader = new PageReader(this, parentReader.getFileSystem(), parentReader.getHadoopPath(), columnChunkMetaData);
if (columnDescriptor.getType() != PrimitiveType.PrimitiveTypeName.BINARY) {
if (columnDescriptor.getType() == PrimitiveTypeName.FIXED_LEN_BYTE_ARRAY) {
dataTypeLengthInBits = columnDescriptor.getTypeLength() * 8;
} else {
dataTypeLengthInBits = ParquetRecordReader.getTypeLengthInBits(columnDescriptor.getType());
}
}
}
开发者ID:skhalifa,项目名称:QDrill,代码行数:20,代码来源:ColumnReader.java
示例2: IntColumnChunkMetaData
import parquet.schema.PrimitiveType.PrimitiveTypeName; //导入依赖的package包/类
/**
* @param path column identifier
* @param type type of the column
* @param codec
* @param encodings
* @param statistics
* @param firstDataPage
* @param dictionaryPageOffset
* @param valueCount
* @param totalSize
* @param totalUncompressedSize
*/
IntColumnChunkMetaData(
ColumnPath path,
PrimitiveTypeName type,
CompressionCodecName codec,
Set<Encoding> encodings,
Statistics statistics,
long firstDataPage,
long dictionaryPageOffset,
long valueCount,
long totalSize,
long totalUncompressedSize) {
super(ColumnChunkProperties.get(path, type, codec, encodings));
this.firstDataPage = positiveLongToInt(firstDataPage);
this.dictionaryPageOffset = positiveLongToInt(dictionaryPageOffset);
this.valueCount = positiveLongToInt(valueCount);
this.totalSize = positiveLongToInt(totalSize);
this.totalUncompressedSize = positiveLongToInt(totalUncompressedSize);
this.statistics = statistics;
}
开发者ID:grokcoder,项目名称:pbase,代码行数:32,代码来源:ColumnChunkMetaData.java
示例3: LongColumnChunkMetaData
import parquet.schema.PrimitiveType.PrimitiveTypeName; //导入依赖的package包/类
/**
* @param path column identifier
* @param type type of the column
* @param codec
* @param encodings
* @param statistics
* @param firstDataPageOffset
* @param dictionaryPageOffset
* @param valueCount
* @param totalSize
* @param totalUncompressedSize
*/
LongColumnChunkMetaData(
ColumnPath path,
PrimitiveTypeName type,
CompressionCodecName codec,
Set<Encoding> encodings,
Statistics statistics,
long firstDataPageOffset,
long dictionaryPageOffset,
long valueCount,
long totalSize,
long totalUncompressedSize) {
super(ColumnChunkProperties.get(path, type, codec, encodings));
this.firstDataPageOffset = firstDataPageOffset;
this.dictionaryPageOffset = dictionaryPageOffset;
this.valueCount = valueCount;
this.totalSize = totalSize;
this.totalUncompressedSize = totalUncompressedSize;
this.statistics = statistics;
}
开发者ID:grokcoder,项目名称:pbase,代码行数:32,代码来源:ColumnChunkMetaData.java
示例4: getPrimitive
import parquet.schema.PrimitiveType.PrimitiveTypeName; //导入依赖的package包/类
public PrimitiveTypeName getPrimitive(Type type) {
switch (type) {
case BYTE_ARRAY: // TODO: rename BINARY and remove this switch
return PrimitiveTypeName.BINARY;
case INT64:
return PrimitiveTypeName.INT64;
case INT32:
return PrimitiveTypeName.INT32;
case BOOLEAN:
return PrimitiveTypeName.BOOLEAN;
case FLOAT:
return PrimitiveTypeName.FLOAT;
case DOUBLE:
return PrimitiveTypeName.DOUBLE;
case INT96:
return PrimitiveTypeName.INT96;
case FIXED_LEN_BYTE_ARRAY:
return PrimitiveTypeName.FIXED_LEN_BYTE_ARRAY;
default:
throw new RuntimeException("Unknown type " + type);
}
}
开发者ID:grokcoder,项目名称:pbase,代码行数:23,代码来源:ParquetMetadataConverter.java
示例5: getType
import parquet.schema.PrimitiveType.PrimitiveTypeName; //导入依赖的package包/类
Type getType(PrimitiveTypeName type) {
switch (type) {
case INT64:
return Type.INT64;
case INT32:
return Type.INT32;
case BOOLEAN:
return Type.BOOLEAN;
case BINARY:
return Type.BYTE_ARRAY;
case FLOAT:
return Type.FLOAT;
case DOUBLE:
return Type.DOUBLE;
case INT96:
return Type.INT96;
case FIXED_LEN_BYTE_ARRAY:
return Type.FIXED_LEN_BYTE_ARRAY;
default:
throw new RuntimeException("Unknown primitive type " + type);
}
}
开发者ID:grokcoder,项目名称:pbase,代码行数:23,代码来源:ParquetMetadataConverter.java
示例6: getTypeName
import parquet.schema.PrimitiveType.PrimitiveTypeName; //导入依赖的package包/类
private static PrimitiveTypeName getTypeName(Type type)
{
switch (type) {
case BYTE_ARRAY:
return PrimitiveTypeName.BINARY;
case INT64:
return PrimitiveTypeName.INT64;
case INT32:
return PrimitiveTypeName.INT32;
case BOOLEAN:
return PrimitiveTypeName.BOOLEAN;
case FLOAT:
return PrimitiveTypeName.FLOAT;
case DOUBLE:
return PrimitiveTypeName.DOUBLE;
case INT96:
return PrimitiveTypeName.INT96;
case FIXED_LEN_BYTE_ARRAY:
return PrimitiveTypeName.FIXED_LEN_BYTE_ARRAY;
default:
throw new IllegalArgumentException("Unknown type " + type);
}
}
开发者ID:y-lan,项目名称:presto,代码行数:24,代码来源:ParquetMetadataReader.java
示例7: convertField
import parquet.schema.PrimitiveType.PrimitiveTypeName; //导入依赖的package包/类
private PrimitiveType convertField( SchemaDescription.Field f ) {
Repetition rep = f.allowNull ? Repetition.OPTIONAL : Repetition.REQUIRED;
switch ( f.pentahoValueMetaType ) {
case ValueMetaInterface.TYPE_NUMBER:
return new PrimitiveType( rep, PrimitiveTypeName.DOUBLE, f.formatFieldName );
case ValueMetaInterface.TYPE_STRING:
return new PrimitiveType( rep, PrimitiveTypeName.BINARY, f.formatFieldName, OriginalType.UTF8 );
case ValueMetaInterface.TYPE_BOOLEAN:
return new PrimitiveType( rep, PrimitiveTypeName.BOOLEAN, f.formatFieldName );
case ValueMetaInterface.TYPE_INTEGER:
return new PrimitiveType( rep, PrimitiveTypeName.INT64, f.formatFieldName, OriginalType.INT_64 );
case ValueMetaInterface.TYPE_BIGNUMBER:
return new PrimitiveType( rep, PrimitiveTypeName.DOUBLE, f.formatFieldName );
case ValueMetaInterface.TYPE_SERIALIZABLE:
case ValueMetaInterface.TYPE_BINARY:
return new PrimitiveType( rep, PrimitiveTypeName.BINARY, f.formatFieldName );
case ValueMetaInterface.TYPE_DATE:
case ValueMetaInterface.TYPE_TIMESTAMP:
return new PrimitiveType( rep, PrimitiveTypeName.INT64, f.formatFieldName, OriginalType.TIMESTAMP_MILLIS );
case ValueMetaInterface.TYPE_INET:
return new PrimitiveType( rep, PrimitiveTypeName.BINARY, f.formatFieldName );
default:
throw new RuntimeException( "Undefined type: " + f.pentahoValueMetaType );
}
}
开发者ID:pentaho,项目名称:pentaho-hadoop-shims,代码行数:26,代码来源:ParquetConverter.java
示例8: showDetails
import parquet.schema.PrimitiveType.PrimitiveTypeName; //导入依赖的package包/类
private static void showDetails(PrettyPrintWriter out, PrimitiveType type, int depth, MessageType container, List<String> cpath) {
String name = Strings.repeat(".", depth) + type.getName();
OriginalType otype = type.getOriginalType();
Repetition rep = type.getRepetition();
PrimitiveTypeName ptype = type.getPrimitiveTypeName();
out.format("%s: %s %s", name, rep, ptype);
if (otype != null) out.format(" O:%s", otype);
if (container != null) {
cpath.add(type.getName());
String[] paths = cpath.toArray(new String[cpath.size()]);
cpath.remove(cpath.size() - 1);
ColumnDescriptor desc = container.getColumnDescription(paths);
int defl = desc.getMaxDefinitionLevel();
int repl = desc.getMaxRepetitionLevel();
out.format(" R:%d D:%d", repl, defl);
}
out.println();
}
开发者ID:wesleypeck,项目名称:parquet-tools,代码行数:23,代码来源:MetadataUtils.java
示例9: ColumnMetadata
import parquet.schema.PrimitiveType.PrimitiveTypeName; //导入依赖的package包/类
public ColumnMetadata(SchemaPath name, PrimitiveTypeName primitiveType, OriginalType originalType,
Object max, Object min, Long nulls) {
this.name = name;
this.primitiveType = primitiveType;
this.originalType = originalType;
this.max = max;
this.min = min;
this.nulls = nulls;
}
开发者ID:skhalifa,项目名称:QDrill,代码行数:10,代码来源:Metadata.java
示例10: getPrimitiveType
import parquet.schema.PrimitiveType.PrimitiveTypeName; //导入依赖的package包/类
private PrimitiveType getPrimitiveType(MaterializedField field) {
MinorType minorType = field.getType().getMinorType();
String name = field.getLastName();
PrimitiveTypeName primitiveTypeName = ParquetTypeHelper.getPrimitiveTypeNameForMinorType(minorType);
Repetition repetition = ParquetTypeHelper.getRepetitionForDataMode(field.getDataMode());
OriginalType originalType = ParquetTypeHelper.getOriginalTypeForMinorType(minorType);
DecimalMetadata decimalMetadata = ParquetTypeHelper.getDecimalMetadataForField(field);
int length = ParquetTypeHelper.getLengthForMinorType(minorType);
return new PrimitiveType(repetition, primitiveTypeName, length, name, originalType, decimalMetadata, null);
}
开发者ID:skhalifa,项目名称:QDrill,代码行数:11,代码来源:ParquetRecordWriter.java
示例11: ColumnChunkProperties
import parquet.schema.PrimitiveType.PrimitiveTypeName; //导入依赖的package包/类
private ColumnChunkProperties(CompressionCodecName codec,
ColumnPath path,
PrimitiveTypeName type,
Set<Encoding> encodings) {
super();
this.codec = codec;
this.path = path;
this.type = type;
this.encodings = encodings;
}
开发者ID:grokcoder,项目名称:pbase,代码行数:11,代码来源:ColumnChunkProperties.java
示例12: get
import parquet.schema.PrimitiveType.PrimitiveTypeName; //导入依赖的package包/类
@Deprecated
public static ColumnChunkMetaData get(
ColumnPath path,
PrimitiveTypeName type,
CompressionCodecName codec,
Set<Encoding> encodings,
long firstDataPage,
long dictionaryPageOffset,
long valueCount,
long totalSize,
long totalUncompressedSize) {
// to save space we store those always positive longs in ints when they fit.
if (positiveLongFitsInAnInt(firstDataPage)
&& positiveLongFitsInAnInt(dictionaryPageOffset)
&& positiveLongFitsInAnInt(valueCount)
&& positiveLongFitsInAnInt(totalSize)
&& positiveLongFitsInAnInt(totalUncompressedSize)) {
return new IntColumnChunkMetaData(
path, type, codec, encodings,
new BooleanStatistics(),
firstDataPage,
dictionaryPageOffset,
valueCount,
totalSize,
totalUncompressedSize);
} else {
return new LongColumnChunkMetaData(
path, type, codec, encodings,
new BooleanStatistics(),
firstDataPage,
dictionaryPageOffset,
valueCount,
totalSize,
totalUncompressedSize);
}
}
开发者ID:grokcoder,项目名称:pbase,代码行数:37,代码来源:ColumnChunkMetaData.java
示例13: makeBlockFromStats
import parquet.schema.PrimitiveType.PrimitiveTypeName; //导入依赖的package包/类
public static BlockMetaData makeBlockFromStats(IntStatistics stats, long valueCount) {
BlockMetaData blockMetaData = new BlockMetaData();
ColumnChunkMetaData column = ColumnChunkMetaData.get(ColumnPath.get("foo"),
PrimitiveTypeName.INT32,
CompressionCodecName.GZIP,
new HashSet<Encoding>(Arrays.asList(Encoding.PLAIN)),
stats,
100l, 100l, valueCount, 100l, 100l);
blockMetaData.addColumn(column);
blockMetaData.setTotalByteSize(200l);
blockMetaData.setRowCount(valueCount);
return blockMetaData;
}
开发者ID:grokcoder,项目名称:pbase,代码行数:15,代码来源:TestInputFormat.java
示例14: newBlock
import parquet.schema.PrimitiveType.PrimitiveTypeName; //导入依赖的package包/类
private BlockMetaData newBlock(long start, long compressedBlockSize) {
BlockMetaData blockMetaData = new BlockMetaData();
long uncompressedSize = compressedBlockSize * 2;//assuming the compression ratio is 2
ColumnChunkMetaData column = ColumnChunkMetaData.get(ColumnPath.get("foo"),
PrimitiveTypeName.BINARY,
CompressionCodecName.GZIP,
new HashSet<Encoding>(Arrays.asList(Encoding.PLAIN)),
new BinaryStatistics(),
start, 0l, 0l, compressedBlockSize, uncompressedSize);
blockMetaData.addColumn(column);
blockMetaData.setTotalByteSize(uncompressedSize);
return blockMetaData;
}
开发者ID:grokcoder,项目名称:pbase,代码行数:14,代码来源:TestInputFormat.java
示例15: newMD
import parquet.schema.PrimitiveType.PrimitiveTypeName; //导入依赖的package包/类
private ColumnChunkMetaData newMD(long big) {
Set<Encoding> e = new HashSet<Encoding>();
PrimitiveTypeName t = BINARY;
ColumnPath p = ColumnPath.get("foo");
CompressionCodecName c = CompressionCodecName.GZIP;
BinaryStatistics s = new BinaryStatistics();
ColumnChunkMetaData md = ColumnChunkMetaData.get(p, t, c, e, s,
big, 0, 0, 0, 0);
return md;
}
开发者ID:grokcoder,项目名称:pbase,代码行数:11,代码来源:TestColumnChunkMetaData.java
示例16: testConvertToThriftStatistics
import parquet.schema.PrimitiveType.PrimitiveTypeName; //导入依赖的package包/类
@Test
public void testConvertToThriftStatistics() throws Exception {
long[] longArray = new long[]{39L, 99L, 12L, 1000L, 65L, 542L, 2533461316L, -253346131996L, Long.MAX_VALUE, Long.MIN_VALUE};
LongStatistics parquetMRstats = new LongStatistics();
for (long l : longArray) {
parquetMRstats.updateStats(l);
}
Statistics thriftStats = parquet.format.converter.ParquetMetadataConverter.toParquetStatistics(parquetMRstats);
LongStatistics convertedBackStats = (LongStatistics) parquet.format.converter.ParquetMetadataConverter.fromParquetStatistics(thriftStats, PrimitiveTypeName.INT64);
assertEquals(parquetMRstats.getMax(), convertedBackStats.getMax());
assertEquals(parquetMRstats.getMin(), convertedBackStats.getMin());
assertEquals(parquetMRstats.getNumNulls(), convertedBackStats.getNumNulls());
}
开发者ID:grokcoder,项目名称:pbase,代码行数:16,代码来源:TestParquetFileWriter.java
示例17: getIntColumnMeta
import parquet.schema.PrimitiveType.PrimitiveTypeName; //导入依赖的package包/类
private static ColumnChunkMetaData getIntColumnMeta(IntStatistics stats, long valueCount) {
return ColumnChunkMetaData.get(ColumnPath.get("int", "column"),
PrimitiveTypeName.INT32,
CompressionCodecName.GZIP,
new HashSet<Encoding>(Arrays.asList(Encoding.PLAIN)),
stats,
0L, 0L, valueCount, 0L, 0L);
}
开发者ID:grokcoder,项目名称:pbase,代码行数:9,代码来源:TestStatisticsFilter.java
示例18: getDoubleColumnMeta
import parquet.schema.PrimitiveType.PrimitiveTypeName; //导入依赖的package包/类
private static ColumnChunkMetaData getDoubleColumnMeta(DoubleStatistics stats, long valueCount) {
return ColumnChunkMetaData.get(ColumnPath.get("double", "column"),
PrimitiveTypeName.DOUBLE,
CompressionCodecName.GZIP,
new HashSet<Encoding>(Arrays.asList(Encoding.PLAIN)),
stats,
0L, 0L, valueCount, 0L, 0L);
}
开发者ID:grokcoder,项目名称:pbase,代码行数:9,代码来源:TestStatisticsFilter.java
示例19: testSchemaConverterDecimal
import parquet.schema.PrimitiveType.PrimitiveTypeName; //导入依赖的package包/类
@Test
public void testSchemaConverterDecimal() {
ParquetMetadataConverter converter = new ParquetMetadataConverter();
List<SchemaElement> schemaElements = converter.toParquetSchema(
Types.buildMessage()
.required(PrimitiveTypeName.BINARY)
.as(OriginalType.DECIMAL).precision(9).scale(2)
.named("aBinaryDecimal")
.optional(PrimitiveTypeName.FIXED_LEN_BYTE_ARRAY).length(4)
.as(OriginalType.DECIMAL).precision(9).scale(2)
.named("aFixedDecimal")
.named("Message")
);
List<SchemaElement> expected = Lists.newArrayList(
new SchemaElement("Message").setNum_children(2),
new SchemaElement("aBinaryDecimal")
.setRepetition_type(FieldRepetitionType.REQUIRED)
.setType(Type.BYTE_ARRAY)
.setConverted_type(ConvertedType.DECIMAL)
.setPrecision(9).setScale(2),
new SchemaElement("aFixedDecimal")
.setRepetition_type(FieldRepetitionType.OPTIONAL)
.setType(Type.FIXED_LEN_BYTE_ARRAY)
.setType_length(4)
.setConverted_type(ConvertedType.DECIMAL)
.setPrecision(9).setScale(2)
);
Assert.assertEquals(expected, schemaElements);
}
开发者ID:grokcoder,项目名称:pbase,代码行数:30,代码来源:TestParquetMetadataConverter.java
示例20: convertColumn
import parquet.schema.PrimitiveType.PrimitiveTypeName; //导入依赖的package包/类
private Type convertColumn(Column column) {
TajoDataTypes.Type type = column.getDataType().getType();
switch (type) {
case BOOLEAN:
return primitive(column.getSimpleName(),
PrimitiveType.PrimitiveTypeName.BOOLEAN);
case BIT:
case INT2:
case INT4:
return primitive(column.getSimpleName(),
PrimitiveType.PrimitiveTypeName.INT32);
case INT8:
return primitive(column.getSimpleName(),
PrimitiveType.PrimitiveTypeName.INT64);
case FLOAT4:
return primitive(column.getSimpleName(),
PrimitiveType.PrimitiveTypeName.FLOAT);
case FLOAT8:
return primitive(column.getSimpleName(),
PrimitiveType.PrimitiveTypeName.DOUBLE);
case CHAR:
case TEXT:
return primitive(column.getSimpleName(),
PrimitiveType.PrimitiveTypeName.BINARY,
OriginalType.UTF8);
case PROTOBUF:
return primitive(column.getSimpleName(),
PrimitiveType.PrimitiveTypeName.BINARY);
case BLOB:
return primitive(column.getSimpleName(),
PrimitiveType.PrimitiveTypeName.BINARY);
case INET4:
case INET6:
return primitive(column.getSimpleName(),
PrimitiveType.PrimitiveTypeName.BINARY);
default:
throw new RuntimeException("Cannot convert Tajo type: " + type);
}
}
开发者ID:gruter,项目名称:tajo-cdh,代码行数:40,代码来源:TajoSchemaConverter.java
注:本文中的parquet.schema.PrimitiveType.PrimitiveTypeName类示例整理自Github/MSDocs等源码及文档管理平台,相关代码片段筛选自各路编程大神贡献的开源项目,源码版权归原作者所有,传播和使用请参考对应项目的License;未经允许,请勿转载。 |
请发表评论