本文整理汇总了Java中parquet.schema.OriginalType类的典型用法代码示例。如果您正苦于以下问题:Java OriginalType类的具体用法?Java OriginalType怎么用?Java OriginalType使用的例子?那么恭喜您, 这里精选的类代码示例或许可以为您提供帮助。
OriginalType类属于parquet.schema包,在下文中一共展示了OriginalType类的14个代码示例,这些例子默认根据受欢迎程度排序。您可以为喜欢或者感觉有用的代码点赞,您的评价将有助于我们的系统推荐出更棒的Java代码示例。
示例1: convertField
import parquet.schema.OriginalType; //导入依赖的package包/类
private static SchemaDescription.Field convertField( SchemaDescription schema, Type t ) {
boolean allowNull = t.getRepetition() != Repetition.REQUIRED;
switch ( t.asPrimitiveType().getPrimitiveTypeName() ) {
case BINARY:
return schema.new Field( t.getName(), t.getName(), ValueMetaInterface.TYPE_STRING, allowNull );
case BOOLEAN:
return schema.new Field( t.getName(), t.getName(), ValueMetaInterface.TYPE_BOOLEAN, allowNull );
case DOUBLE:
case FLOAT:
return schema.new Field( t.getName(), t.getName(), ValueMetaInterface.TYPE_NUMBER, allowNull );
case INT32:
case INT64:
if ( t.getOriginalType() == OriginalType.DATE || t.getOriginalType() == OriginalType.TIME_MILLIS
|| t.getOriginalType() == OriginalType.TIMESTAMP_MILLIS ) {
return schema.new Field( t.getName(), t.getName(), ValueMetaInterface.TYPE_DATE, allowNull );
} else {
return schema.new Field( t.getName(), t.getName(), ValueMetaInterface.TYPE_INTEGER, allowNull );
}
case INT96:
return schema.new Field( t.getName(), t.getName(), ValueMetaInterface.TYPE_DATE, allowNull );
default:
throw new RuntimeException( "Undefined type: " + t );
}
}
开发者ID:pentaho,项目名称:pentaho-hadoop-shims,代码行数:25,代码来源:ParquetConverter.java
示例2: showDetails
import parquet.schema.OriginalType; //导入依赖的package包/类
private static void showDetails(PrettyPrintWriter out, PrimitiveType type, int depth, MessageType container, List<String> cpath) {
String name = Strings.repeat(".", depth) + type.getName();
OriginalType otype = type.getOriginalType();
Repetition rep = type.getRepetition();
PrimitiveTypeName ptype = type.getPrimitiveTypeName();
out.format("%s: %s %s", name, rep, ptype);
if (otype != null) out.format(" O:%s", otype);
if (container != null) {
cpath.add(type.getName());
String[] paths = cpath.toArray(new String[cpath.size()]);
cpath.remove(cpath.size() - 1);
ColumnDescriptor desc = container.getColumnDescription(paths);
int defl = desc.getMaxDefinitionLevel();
int repl = desc.getMaxRepetitionLevel();
out.format(" R:%d D:%d", repl, defl);
}
out.println();
}
开发者ID:wesleypeck,项目名称:parquet-tools,代码行数:23,代码来源:MetadataUtils.java
示例3: createConverter
import parquet.schema.OriginalType; //导入依赖的package包/类
private Converter createConverter(Type field) {
if (field.isPrimitive()) {
OriginalType otype = field.getOriginalType();
if (otype != null) {
switch (otype) {
case MAP: break;
case LIST: break;
case UTF8: return new StringConverter(field.getName());
case MAP_KEY_VALUE: break;
case ENUM: break;
}
}
return new SimplePrimitiveConverter(field.getName());
}
return new SimpleRecordConverter(field.asGroupType(), field.getName(), this);
}
开发者ID:wesleypeck,项目名称:parquet-tools,代码行数:19,代码来源:SimpleRecordConverter.java
示例4: getOriginalType
import parquet.schema.OriginalType; //导入依赖的package包/类
private OriginalType getOriginalType(Type type, String[] path, int depth) {
if (type.isPrimitive()) {
return type.getOriginalType();
}
Type t = ((GroupType) type).getType(path[depth]);
return getOriginalType(t, path, depth + 1);
}
开发者ID:skhalifa,项目名称:QDrill,代码行数:8,代码来源:Metadata.java
示例5: ColumnMetadata
import parquet.schema.OriginalType; //导入依赖的package包/类
public ColumnMetadata(SchemaPath name, PrimitiveTypeName primitiveType, OriginalType originalType,
Object max, Object min, Long nulls) {
this.name = name;
this.primitiveType = primitiveType;
this.originalType = originalType;
this.max = max;
this.min = min;
this.nulls = nulls;
}
开发者ID:skhalifa,项目名称:QDrill,代码行数:10,代码来源:Metadata.java
示例6: getPrimitiveType
import parquet.schema.OriginalType; //导入依赖的package包/类
private PrimitiveType getPrimitiveType(MaterializedField field) {
MinorType minorType = field.getType().getMinorType();
String name = field.getLastName();
PrimitiveTypeName primitiveTypeName = ParquetTypeHelper.getPrimitiveTypeNameForMinorType(minorType);
Repetition repetition = ParquetTypeHelper.getRepetitionForDataMode(field.getDataMode());
OriginalType originalType = ParquetTypeHelper.getOriginalTypeForMinorType(minorType);
DecimalMetadata decimalMetadata = ParquetTypeHelper.getDecimalMetadataForField(field);
int length = ParquetTypeHelper.getLengthForMinorType(minorType);
return new PrimitiveType(repetition, primitiveTypeName, length, name, originalType, decimalMetadata, null);
}
开发者ID:skhalifa,项目名称:QDrill,代码行数:11,代码来源:ParquetRecordWriter.java
示例7: testSchemaConverterDecimal
import parquet.schema.OriginalType; //导入依赖的package包/类
@Test
public void testSchemaConverterDecimal() {
ParquetMetadataConverter converter = new ParquetMetadataConverter();
List<SchemaElement> schemaElements = converter.toParquetSchema(
Types.buildMessage()
.required(PrimitiveTypeName.BINARY)
.as(OriginalType.DECIMAL).precision(9).scale(2)
.named("aBinaryDecimal")
.optional(PrimitiveTypeName.FIXED_LEN_BYTE_ARRAY).length(4)
.as(OriginalType.DECIMAL).precision(9).scale(2)
.named("aFixedDecimal")
.named("Message")
);
List<SchemaElement> expected = Lists.newArrayList(
new SchemaElement("Message").setNum_children(2),
new SchemaElement("aBinaryDecimal")
.setRepetition_type(FieldRepetitionType.REQUIRED)
.setType(Type.BYTE_ARRAY)
.setConverted_type(ConvertedType.DECIMAL)
.setPrecision(9).setScale(2),
new SchemaElement("aFixedDecimal")
.setRepetition_type(FieldRepetitionType.OPTIONAL)
.setType(Type.FIXED_LEN_BYTE_ARRAY)
.setType_length(4)
.setConverted_type(ConvertedType.DECIMAL)
.setPrecision(9).setScale(2)
);
Assert.assertEquals(expected, schemaElements);
}
开发者ID:grokcoder,项目名称:pbase,代码行数:30,代码来源:TestParquetMetadataConverter.java
示例8: convertColumn
import parquet.schema.OriginalType; //导入依赖的package包/类
private Type convertColumn(Column column) {
TajoDataTypes.Type type = column.getDataType().getType();
switch (type) {
case BOOLEAN:
return primitive(column.getSimpleName(),
PrimitiveType.PrimitiveTypeName.BOOLEAN);
case BIT:
case INT2:
case INT4:
return primitive(column.getSimpleName(),
PrimitiveType.PrimitiveTypeName.INT32);
case INT8:
return primitive(column.getSimpleName(),
PrimitiveType.PrimitiveTypeName.INT64);
case FLOAT4:
return primitive(column.getSimpleName(),
PrimitiveType.PrimitiveTypeName.FLOAT);
case FLOAT8:
return primitive(column.getSimpleName(),
PrimitiveType.PrimitiveTypeName.DOUBLE);
case CHAR:
case TEXT:
return primitive(column.getSimpleName(),
PrimitiveType.PrimitiveTypeName.BINARY,
OriginalType.UTF8);
case PROTOBUF:
return primitive(column.getSimpleName(),
PrimitiveType.PrimitiveTypeName.BINARY);
case BLOB:
return primitive(column.getSimpleName(),
PrimitiveType.PrimitiveTypeName.BINARY);
case INET4:
case INET6:
return primitive(column.getSimpleName(),
PrimitiveType.PrimitiveTypeName.BINARY);
default:
throw new RuntimeException("Cannot convert Tajo type: " + type);
}
}
开发者ID:gruter,项目名称:tajo-cdh,代码行数:40,代码来源:TajoSchemaConverter.java
示例9: getType
import parquet.schema.OriginalType; //导入依赖的package包/类
private MajorType getType(PrimitiveTypeName type, OriginalType originalType) {
if (originalType != null) {
switch (originalType) {
case DECIMAL:
return Types.optional(MinorType.DECIMAL18);
case DATE:
return Types.optional(MinorType.DATE);
case TIME_MILLIS:
return Types.optional(MinorType.TIME);
case TIMESTAMP_MILLIS:
return Types.optional(MinorType.TIMESTAMP);
case UTF8:
return Types.optional(MinorType.VARCHAR);
case UINT_8:
return Types.optional(MinorType.UINT1);
case UINT_16:
return Types.optional(MinorType.UINT2);
case UINT_32:
return Types.optional(MinorType.UINT4);
case UINT_64:
return Types.optional(MinorType.UINT8);
case INT_8:
return Types.optional(MinorType.TINYINT);
case INT_16:
return Types.optional(MinorType.SMALLINT);
}
}
switch (type) {
case BOOLEAN:
return Types.optional(MinorType.BIT);
case INT32:
return Types.optional(MinorType.INT);
case INT64:
return Types.optional(MinorType.BIGINT);
case FLOAT:
return Types.optional(MinorType.FLOAT4);
case DOUBLE:
return Types.optional(MinorType.FLOAT8);
case BINARY:
case FIXED_LEN_BYTE_ARRAY:
case INT96:
return Types.optional(MinorType.VARBINARY);
default:
// Should never hit this
throw new UnsupportedOperationException("Unsupported type:" + type);
}
}
开发者ID:skhalifa,项目名称:QDrill,代码行数:49,代码来源:ParquetGroupScan.java
示例10: getOriginalTypeForMinorType
import parquet.schema.OriginalType; //导入依赖的package包/类
public static OriginalType getOriginalTypeForMinorType(MinorType minorType) {
return originalTypeMap.get(minorType);
}
开发者ID:skhalifa,项目名称:QDrill,代码行数:4,代码来源:ParquetTypeHelper.java
示例11: testExportParquetString
import parquet.schema.OriginalType; //导入依赖的package包/类
@Test
public void testExportParquetString() throws Exception {
List<Integer> dynamicCols = new ArrayList<>();
List<Type> schemaTypes = new ArrayList<>();
schemaTypes.add(new PrimitiveType(Type.Repetition.OPTIONAL, PrimitiveType.PrimitiveTypeName.BINARY, "c1", OriginalType.UTF8));
schemaTypes.add(new PrimitiveType(Type.Repetition.OPTIONAL, PrimitiveType.PrimitiveTypeName.BINARY, "c2", OriginalType.UTF8));
schemaTypes.add(new PrimitiveType(Type.Repetition.OPTIONAL, PrimitiveType.PrimitiveTypeName.BINARY, "v1", OriginalType.UTF8));
schemaTypes.add(new PrimitiveType(Type.Repetition.OPTIONAL, PrimitiveType.PrimitiveTypeName.BINARY, "v2", OriginalType.UTF8));
schemaTypes.add(new PrimitiveType(Type.Repetition.OPTIONAL, PrimitiveType.PrimitiveTypeName.BINARY, "s1", OriginalType.UTF8));
schemaTypes.add(new PrimitiveType(Type.Repetition.OPTIONAL, PrimitiveType.PrimitiveTypeName.BINARY, "charnull", OriginalType.UTF8));
schemaTypes.add(new PrimitiveType(Type.Repetition.OPTIONAL, PrimitiveType.PrimitiveTypeName.BINARY, "varcharnull", OriginalType.UTF8));
schemaTypes.add(new PrimitiveType(Type.Repetition.OPTIONAL, PrimitiveType.PrimitiveTypeName.BINARY, "stringnull", OriginalType.UTF8));
List<List<Object>> dataSet = new ArrayList<>();
List<Object> row = new ArrayList<>();
row.add("a");
row.add("aaaaaaaaaa");
row.add("b");
row.add("bbbbbbbbbb");
row.add("cccccccccccccccccccc");
row.add(null);
row.add(null);
row.add(null);
addRow(dataSet, row);
ExaIterator iter = new ExaIteratorDummy(dataSet);
File tempFile = new File(testFolder.getRoot(),UUID.randomUUID().toString().replaceAll("-", "") + ".parq");
HdfsSerDeExportService.exportToParquetTable(testFolder.getRoot().toString(), "hdfs", false, null, tempFile.getName(), null, "uncompressed", schemaTypes, FIRST_DATA_COLUMN, dynamicCols, iter);
ExaIterator ctx = mock(ExaIterator.class);
List<HCatTableColumn> columns = new ArrayList<>();
columns.add(new HCatTableColumn("c1", "char(1)"));
columns.add(new HCatTableColumn("c2", "char(12)"));
columns.add(new HCatTableColumn("v1", "varchar(1)"));
columns.add(new HCatTableColumn("v2", "varchar(10)"));
columns.add(new HCatTableColumn("s1", "string"));
columns.add(new HCatTableColumn("charnull", "char(1)"));
columns.add(new HCatTableColumn("varcharnulll", "varchar(1)"));
columns.add(new HCatTableColumn("stringnull", "string"));
List<HCatTableColumn> partitionColumns = null;
importFile(ctx, columns, partitionColumns, tempFile.getCanonicalPath(), PARQUET_INPUT_FORMAT_CLASS_NAME, PARQUET_SERDE_CLASS_NAME);
int expectedNumRows = 1;
verify(ctx, times(expectedNumRows)).emit(anyVararg());
verify(ctx, times(1)).emit(
eq("a"),
eq("aaaaaaaaaa "),
eq("b"),
eq("bbbbbbbbbb"),
eq("cccccccccccccccccccc"),
eq(null),
eq(null),
eq(null)
);
}
开发者ID:EXASOL,项目名称:hadoop-etl-udfs,代码行数:59,代码来源:HdfsSerDeExportServiceTest.java
示例12: getOriginalType
import parquet.schema.OriginalType; //导入依赖的package包/类
OriginalType getOriginalType(ConvertedType type) {
switch (type) {
case UTF8:
return OriginalType.UTF8;
case MAP:
return OriginalType.MAP;
case MAP_KEY_VALUE:
return OriginalType.MAP_KEY_VALUE;
case LIST:
return OriginalType.LIST;
case ENUM:
return OriginalType.ENUM;
case DECIMAL:
return OriginalType.DECIMAL;
case DATE:
return OriginalType.DATE;
case TIME_MILLIS:
return OriginalType.TIME_MILLIS;
case TIMESTAMP_MILLIS:
return OriginalType.TIMESTAMP_MILLIS;
case INTERVAL:
return OriginalType.INTERVAL;
case INT_8:
return OriginalType.INT_8;
case INT_16:
return OriginalType.INT_16;
case INT_32:
return OriginalType.INT_32;
case INT_64:
return OriginalType.INT_64;
case UINT_8:
return OriginalType.UINT_8;
case UINT_16:
return OriginalType.UINT_16;
case UINT_32:
return OriginalType.UINT_32;
case UINT_64:
return OriginalType.UINT_64;
case JSON:
return OriginalType.JSON;
case BSON:
return OriginalType.BSON;
default:
throw new RuntimeException("Unknown converted type " + type);
}
}
开发者ID:grokcoder,项目名称:pbase,代码行数:47,代码来源:ParquetMetadataConverter.java
示例13: getConvertedType
import parquet.schema.OriginalType; //导入依赖的package包/类
ConvertedType getConvertedType(OriginalType type) {
switch (type) {
case UTF8:
return ConvertedType.UTF8;
case MAP:
return ConvertedType.MAP;
case MAP_KEY_VALUE:
return ConvertedType.MAP_KEY_VALUE;
case LIST:
return ConvertedType.LIST;
case ENUM:
return ConvertedType.ENUM;
case DECIMAL:
return ConvertedType.DECIMAL;
case DATE:
return ConvertedType.DATE;
case TIME_MILLIS:
return ConvertedType.TIME_MILLIS;
case TIMESTAMP_MILLIS:
return ConvertedType.TIMESTAMP_MILLIS;
case INTERVAL:
return ConvertedType.INTERVAL;
case INT_8:
return ConvertedType.INT_8;
case INT_16:
return ConvertedType.INT_16;
case INT_32:
return ConvertedType.INT_32;
case INT_64:
return ConvertedType.INT_64;
case UINT_8:
return ConvertedType.UINT_8;
case UINT_16:
return ConvertedType.UINT_16;
case UINT_32:
return ConvertedType.UINT_32;
case UINT_64:
return ConvertedType.UINT_64;
case JSON:
return ConvertedType.JSON;
case BSON:
return ConvertedType.BSON;
default:
throw new RuntimeException("Unknown original type " + type);
}
}
开发者ID:grokcoder,项目名称:pbase,代码行数:47,代码来源:ParquetMetadataConverter.java
示例14: getOriginalType
import parquet.schema.OriginalType; //导入依赖的package包/类
private static OriginalType getOriginalType(ConvertedType type)
{
switch (type) {
case UTF8:
return OriginalType.UTF8;
case MAP:
return OriginalType.MAP;
case MAP_KEY_VALUE:
return OriginalType.MAP_KEY_VALUE;
case LIST:
return OriginalType.LIST;
case ENUM:
return OriginalType.ENUM;
case DECIMAL:
return OriginalType.DECIMAL;
case DATE:
return OriginalType.DATE;
case TIME_MILLIS:
return OriginalType.TIME_MILLIS;
case TIMESTAMP_MILLIS:
return OriginalType.TIMESTAMP_MILLIS;
case INTERVAL:
return OriginalType.INTERVAL;
case INT_8:
return OriginalType.INT_8;
case INT_16:
return OriginalType.INT_16;
case INT_32:
return OriginalType.INT_32;
case INT_64:
return OriginalType.INT_64;
case UINT_8:
return OriginalType.UINT_8;
case UINT_16:
return OriginalType.UINT_16;
case UINT_32:
return OriginalType.UINT_32;
case UINT_64:
return OriginalType.UINT_64;
case JSON:
return OriginalType.JSON;
case BSON:
return OriginalType.BSON;
default:
throw new IllegalArgumentException("Unknown converted type " + type);
}
}
开发者ID:y-lan,项目名称:presto,代码行数:48,代码来源:ParquetMetadataReader.java
注:本文中的parquet.schema.OriginalType类示例整理自Github/MSDocs等源码及文档管理平台,相关代码片段筛选自各路编程大神贡献的开源项目,源码版权归原作者所有,传播和使用请参考对应项目的License;未经允许,请勿转载。 |
请发表评论