import org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe; //导入依赖的package包/类
@Test
public void testParquet()
        throws Exception
{
    List<TestColumn> testColumns = getTestColumnsSupportedByParquet();

    HiveOutputFormat<?, ?> outputFormat = new MapredParquetOutputFormat();
    InputFormat<?, ?> inputFormat = new MapredParquetInputFormat();
    @SuppressWarnings("deprecation")
    SerDe serde = new ParquetHiveSerDe();
    File file = File.createTempFile("presto_test", "parquet");
    file.delete();
    try {
        FileSplit split = createTestFile(file.getAbsolutePath(), outputFormat, serde, null, testColumns, NUM_ROWS);
        HiveRecordCursorProvider cursorProvider = new ParquetRecordCursorProvider(false);
        testCursorProvider(cursorProvider, split, inputFormat, serde, testColumns, NUM_ROWS);
    }
    finally {
        //noinspection ResultOfMethodCallIgnored
        file.delete();
    }
}
 

import org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe; //导入依赖的package包/类
@Test(enabled = false)
public void testParquetPageSource()
        throws Exception
{
    List<TestColumn> testColumns = getTestColumnsSupportedByParquet();
    testColumns = testColumns.stream()
            .filter(column -> column.getObjectInspector().getCategory() == Category.PRIMITIVE)
            .collect(toList());

    HiveOutputFormat<?, ?> outputFormat = new MapredParquetOutputFormat();
    InputFormat<?, ?> inputFormat = new MapredParquetInputFormat();
    @SuppressWarnings("deprecation")
    SerDe serde = new ParquetHiveSerDe();
    File file = File.createTempFile("presto_test", "parquet");
    file.delete();
    try {
        FileSplit split = createTestFile(file.getAbsolutePath(), outputFormat, serde, null, testColumns, NUM_ROWS);
        TestingConnectorSession session = new TestingConnectorSession(
                new HiveSessionProperties(new HiveClientConfig().setParquetOptimizedReaderEnabled(true)).getSessionProperties());
        testPageSourceFactory(new ParquetPageSourceFactory(TYPE_MANAGER, false), split, inputFormat, serde, testColumns, session);
    }
    finally {
        //noinspection ResultOfMethodCallIgnored
        file.delete();
    }
}
 

import org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe; //导入依赖的package包/类
@Test
public void testParquetUseColumnNames()
        throws Exception
{
    List<TestColumn> testColumns = getTestColumnsSupportedByParquet();

    HiveOutputFormat<?, ?> outputFormat = new MapredParquetOutputFormat();
    InputFormat<?, ?> inputFormat = new MapredParquetInputFormat();
    @SuppressWarnings("deprecation")
    SerDe serde = new ParquetHiveSerDe();
    File file = File.createTempFile("presto_test", "parquet");
    file.delete();
    try {
        FileSplit split = createTestFile(file.getAbsolutePath(), outputFormat, serde, null, testColumns, NUM_ROWS);
        // Reverse the order of the columns to test access by name, not by index
        Collections.reverse(testColumns);
        HiveRecordCursorProvider cursorProvider = new ParquetRecordCursorProvider(true);
        testCursorProvider(cursorProvider, split, inputFormat, serde, testColumns, NUM_ROWS);
    }
    finally {
        //noinspection ResultOfMethodCallIgnored
        file.delete();
    }
}
 

import org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe; //导入依赖的package包/类
private void deserializeAndSerializeLazySimple(final ParquetHiveSerDe serDe, final ArrayWritable t) throws SerDeException {

    // Get the row structure
    final StructObjectInspector oi = (StructObjectInspector) serDe.getObjectInspector();

    // Deserialize
    final Object row = serDe.deserialize(t);
    assertEquals("deserialization gives the wrong object class", row.getClass(), ArrayWritable.class);
    assertEquals("size correct after deserialization", serDe.getSerDeStats().getRawDataSize(), t.get().length);
    assertEquals("deserialization gives the wrong object", t, row);

    // Serialize
    final ArrayWritable serializedArr = (ArrayWritable) serDe.serialize(row, oi);
    assertEquals("size correct after serialization", serDe.getSerDeStats().getRawDataSize(), serializedArr.get().length);
    assertTrue("serialized object should be equal to starting object", arrayWritableEquals(t, serializedArr));
  }
 

import org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe; //导入依赖的package包/类
public static String getDataFormat(StorageDescriptor descriptor) {
  Preconditions.checkNotNull(descriptor);

  String serde = descriptor.getSerdeInfo().getSerializationLib();
  String inputFormat = descriptor.getInputFormat();

  if (LazySimpleSerDe.class.getName().equals(serde)) {
    if (TextInputFormat.class.getName().equals(inputFormat)) {
      return BuiltinStorages.TEXT;
    } else if (SequenceFileInputFormat.class.getName().equals(inputFormat)) {
      return BuiltinStorages.SEQUENCE_FILE;
    } else {
      throw new TajoRuntimeException(new UnknownDataFormatException(inputFormat));
    }
  } else if (LazyBinarySerDe.class.getName().equals(serde)) {
    if (SequenceFileInputFormat.class.getName().equals(inputFormat)) {
      return BuiltinStorages.SEQUENCE_FILE;
    } else {
      throw new TajoRuntimeException(new UnknownDataFormatException(inputFormat));
    }
  } else if (LazyBinaryColumnarSerDe.class.getName().equals(serde) || ColumnarSerDe.class.getName().equals(serde)) {
    if (RCFileInputFormat.class.getName().equals(inputFormat)) {
      return BuiltinStorages.RCFILE;
    } else {
      throw new TajoRuntimeException(new UnknownDataFormatException(inputFormat));
    }
  } else if (ParquetHiveSerDe.class.getName().equals(serde)) {
    return BuiltinStorages.PARQUET;
  } else if (AvroSerDe.class.getName().equals(serde)) {
    return BuiltinStorages.AVRO;
  } else if (OrcSerde.class.getName().equals(serde)) {
    return BuiltinStorages.ORC;
  } else if (RegexSerDe.class.getName().equals(serde)) {
    return BuiltinStorages.REGEX;
  } else {
    throw new TajoRuntimeException(new UnknownDataFormatException(inputFormat));
  }
}
 

import org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe; //导入依赖的package包/类
private static GroupType convertMapType(final String name, final MapTypeInfo typeInfo) {
  final Type keyType = convertType(ParquetHiveSerDe.MAP_KEY.toString(),
      typeInfo.getMapKeyTypeInfo(), Repetition.REQUIRED);
  final Type valueType = convertType(ParquetHiveSerDe.MAP_VALUE.toString(),
      typeInfo.getMapValueTypeInfo());
  return ConversionPatterns.mapType(Repetition.OPTIONAL, name, keyType, valueType);
}
 

import org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe; //导入依赖的package包/类
@Test
public void testParquetThrift()
        throws Exception
{
    RowType nameType = new RowType(ImmutableList.of(VARCHAR, VARCHAR), Optional.empty());
    RowType phoneType = new RowType(ImmutableList.of(VARCHAR, VARCHAR), Optional.empty());
    RowType personType = new RowType(ImmutableList.of(nameType, BIGINT, VARCHAR, new ArrayType(phoneType)), Optional.empty());

    List<TestColumn> testColumns = ImmutableList.<TestColumn>of(
        new TestColumn(
            "persons",
            getStandardListObjectInspector(
                getStandardStructObjectInspector(
                    ImmutableList.of("name", "id", "email", "phones"),
                    ImmutableList.<ObjectInspector>of(
                        getStandardStructObjectInspector(
                          ImmutableList.of("first_name", "last_name"),
                          ImmutableList.of(javaStringObjectInspector, javaStringObjectInspector)
                        ),
                        javaIntObjectInspector,
                        javaStringObjectInspector,
                        getStandardListObjectInspector(
                          getStandardStructObjectInspector(
                            ImmutableList.of("number", "type"),
                            ImmutableList.of(javaStringObjectInspector, javaStringObjectInspector)
                          )
                        )
                    )
                )
            ),
            null,
            arrayBlockOf(personType,
                    rowBlockOf(ImmutableList.of(nameType, BIGINT, VARCHAR, new ArrayType(phoneType)),
                            rowBlockOf(ImmutableList.of(VARCHAR, VARCHAR), "Bob", "Roberts"),
                            0,
                            "[email protected]",
                            arrayBlockOf(phoneType, rowBlockOf(ImmutableList.of(VARCHAR, VARCHAR), "1234567890", null))
                    )
            )
        )
    );

    InputFormat<?, ?> inputFormat = new MapredParquetInputFormat();
    @SuppressWarnings("deprecation")
    SerDe serde = new ParquetHiveSerDe();
    File file = new File(this.getClass().getClassLoader().getResource("addressbook.parquet").getPath());
    FileSplit split = new FileSplit(new Path(file.getAbsolutePath()), 0, file.length(), new String[0]);
    HiveRecordCursorProvider cursorProvider = new ParquetRecordCursorProvider(false);
    testCursorProvider(cursorProvider, split, inputFormat, serde, testColumns, 1);
}
 

import org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe; //导入依赖的package包/类
private static GroupType convertArrayType(final String name, final ListTypeInfo typeInfo) {
  final TypeInfo subType = typeInfo.getListElementTypeInfo();
  return listWrapper(name, OriginalType.LIST, new GroupType(Repetition.REPEATED,
      ParquetHiveSerDe.ARRAY.toString(), convertType("array_element", subType)));
}
 

import org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe; //导入依赖的package包/类
public void testParquetHiveSerDe() throws Throwable {
  try {
    // Create the SerDe
    System.out.println("test: testParquetHiveSerDe");

    final ParquetHiveSerDe serDe = new ParquetHiveSerDe();
    final Configuration conf = new Configuration();
    final Properties tbl = createProperties();
    serDe.initialize(conf, tbl);

    // Data
    final Writable[] arr = new Writable[8];

    arr[0] = new ByteWritable((byte) 123);
    arr[1] = new ShortWritable((short) 456);
    arr[2] = new IntWritable(789);
    arr[3] = new LongWritable(1000l);
    arr[4] = new DoubleWritable((double) 5.3);
    arr[5] = new BinaryWritable(Binary.fromString("hive and hadoop and parquet. Big family."));

    final Writable[] mapContainer = new Writable[1];
    final Writable[] map = new Writable[3];
    for (int i = 0; i < 3; ++i) {
      final Writable[] pair = new Writable[2];
      pair[0] = new BinaryWritable(Binary.fromString("key_" + i));
      pair[1] = new IntWritable(i);
      map[i] = new ArrayWritable(Writable.class, pair);
    }
    mapContainer[0] = new ArrayWritable(Writable.class, map);
    arr[6] = new ArrayWritable(Writable.class, mapContainer);

    final Writable[] arrayContainer = new Writable[1];
    final Writable[] array = new Writable[5];
    for (int i = 0; i < 5; ++i) {
      array[i] = new BinaryWritable(Binary.fromString("elem_" + i));
    }
    arrayContainer[0] = new ArrayWritable(Writable.class, array);
    arr[7] = new ArrayWritable(Writable.class, arrayContainer);

    final ArrayWritable arrWritable = new ArrayWritable(Writable.class, arr);
    // Test
    deserializeAndSerializeLazySimple(serDe, arrWritable);
    System.out.println("test: testParquetHiveSerDe - OK");

  } catch (final Throwable e) {
    e.printStackTrace();
    throw e;
  }
}