import org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName; //导入依赖的package包/类
protected ColumnReader(DeprecatedParquetVectorizedReader parentReader, int allocateSize, ColumnDescriptor descriptor,
                       ColumnChunkMetaData columnChunkMetaData, boolean fixedLength, V v, SchemaElement schemaElement) throws ExecutionSetupException {
  this.parentReader = parentReader;
  this.columnDescriptor = descriptor;
  this.columnChunkMetaData = columnChunkMetaData;
  this.isFixedLength = fixedLength;
  this.schemaElement = schemaElement;
  this.valueVec =  v;
  this.pageReader = (parentReader.getSingleStream() != null)?
    new DeprecatedSingleStreamPageReader(this, parentReader.getSingleStream(), parentReader.getHadoopPath(), columnChunkMetaData) :
    new PageReader(this, parentReader.getFileSystem(), parentReader.getHadoopPath(), columnChunkMetaData);

  if (columnDescriptor.getType() != PrimitiveType.PrimitiveTypeName.BINARY) {
    if (columnDescriptor.getType() == PrimitiveTypeName.FIXED_LEN_BYTE_ARRAY) {
      dataTypeLengthInBits = columnDescriptor.getTypeLength() * 8;
    } else if (columnDescriptor.getType() == PrimitiveTypeName.INT96
      && (valueVec instanceof TimeStampMilliVector || valueVec instanceof NullableTimeStampMilliVector)) {
      // if int 96 column is being read as a Timestamp, this truncates the time format used by Impala
      // dataTypeLengthInBits is only ever used when computing offsets into the destination vector, so it
      // needs to be set to the bit width of the resulting Arrow type, usually this matches the input length
      dataTypeLengthInBits = 64;
    } else {
      dataTypeLengthInBits = DeprecatedParquetVectorizedReader.getTypeLengthInBits(columnDescriptor.getType());
    }
  }
}
 

import org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName; //导入依赖的package包/类
@Override
public void serialize(ColumnMetadata_v2 value, JsonGenerator jgen, SerializerProvider provider)
    throws IOException, JsonProcessingException {
  jgen.writeStartObject();
  jgen.writeArrayFieldStart("name");
  for (String n : value.name) {
    jgen.writeString(n);
  }
  jgen.writeEndArray();
  if (value.mxValue != null) {
    Object val;
    if (value.primitiveType == PrimitiveTypeName.BINARY && value.mxValue != null) {
      val = new String(((Binary) value.mxValue).getBytes());
    } else {
      val = value.mxValue;
    }
    jgen.writeObjectField("mxValue", val);
  }
  if (value.nulls != null) {
    jgen.writeObjectField("nulls", value.nulls);
  }
  jgen.writeEndObject();
}
 

import org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName; //导入依赖的package包/类
private void testStillUseStatsWithSignedSortOrderIfSingleValue(StatsHelper helper) {
  ParquetMetadataConverter converter = new ParquetMetadataConverter();
  BinaryStatistics stats = new BinaryStatistics();
  stats.incrementNumNulls();
  stats.updateStats(Binary.fromString("A"));
  stats.incrementNumNulls();
  stats.updateStats(Binary.fromString("A"));
  stats.incrementNumNulls();

  PrimitiveType binaryType = Types.required(PrimitiveTypeName.BINARY).as(OriginalType.UTF8).named("b");
  Statistics convertedStats = converter.fromParquetStatistics(
      Version.FULL_VERSION,
      ParquetMetadataConverter.toParquetStatistics(stats),
      binaryType);

  Assert.assertFalse("Stats should not be empty: " + convertedStats, convertedStats.isEmpty());
  Assert.assertArrayEquals("min == max: " + convertedStats, convertedStats.getMaxBytes(), convertedStats.getMinBytes());
}
 

import org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName; //导入依赖的package包/类
/**
 * Returns the typed statistics object based on the passed type parameter
 * @param type PrimitiveTypeName type of the column
 * @return instance of a typed statistics class
 * @deprecated Use {@link #createStats(Type)} instead
 */
@Deprecated
public static Statistics getStatsBasedOnType(PrimitiveTypeName type) {
  switch (type) {
    case INT32:
      return new IntStatistics();
    case INT64:
      return new LongStatistics();
    case FLOAT:
      return new FloatStatistics();
    case DOUBLE:
      return new DoubleStatistics();
    case BOOLEAN:
      return new BooleanStatistics();
    case BINARY:
      return new BinaryStatistics();
    case INT96:
      return new BinaryStatistics();
    case FIXED_LEN_BYTE_ARRAY:
      return new BinaryStatistics();
    default:
      throw new UnknownColumnTypeException(type);
  }
}
 

import org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName; //导入依赖的package包/类
@Test
public void testConvertToThriftStatistics() throws Exception {
  long[] longArray = new long[] {39L, 99L, 12L, 1000L, 65L, 542L, 2533461316L, -253346131996L, Long.MAX_VALUE, Long.MIN_VALUE};
  LongStatistics parquetMRstats = new LongStatistics();

  for (long l: longArray) {
    parquetMRstats.updateStats(l);
  }
  final String createdBy =
      "parquet-mr version 1.8.0 (build d4d5a07ec9bd262ca1e93c309f1d7d4a74ebda4c)";
  Statistics thriftStats =
      org.apache.parquet.format.converter.ParquetMetadataConverter.toParquetStatistics(parquetMRstats);
  LongStatistics convertedBackStats =
      (LongStatistics) org.apache.parquet.format.converter.ParquetMetadataConverter.fromParquetStatistics(
          createdBy, thriftStats, PrimitiveTypeName.INT64);

  assertEquals(parquetMRstats.getMax(), convertedBackStats.getMax());
  assertEquals(parquetMRstats.getMin(), convertedBackStats.getMin());
  assertEquals(parquetMRstats.getNumNulls(), convertedBackStats.getNumNulls());
}
 

import org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName; //导入依赖的package包/类
@Test
public void testV2OnlyStats() {
  testV2OnlyStats(Types.optional(PrimitiveTypeName.INT32).as(OriginalType.UINT_8).named(""),
      0x7F,
      0x80);
  testV2OnlyStats(Types.optional(PrimitiveTypeName.INT32).as(OriginalType.UINT_16).named(""),
      0x7FFF,
      0x8000);
  testV2OnlyStats(Types.optional(PrimitiveTypeName.INT32).as(OriginalType.UINT_32).named(""),
      0x7FFFFFFF,
      0x80000000);
  testV2OnlyStats(Types.optional(PrimitiveTypeName.INT64).as(OriginalType.UINT_64).named(""),
      0x7FFFFFFFFFFFFFFFL,
      0x8000000000000000L);
  testV2OnlyStats(Types.optional(PrimitiveTypeName.BINARY).as(OriginalType.DECIMAL).precision(6).named(""),
      new BigInteger("-765875"),
      new BigInteger("876856"));
  testV2OnlyStats(
      Types.optional(PrimitiveTypeName.FIXED_LEN_BYTE_ARRAY).length(14).as(OriginalType.DECIMAL).precision(7)
          .named(""),
      new BigInteger("-6769643"),
      new BigInteger("9864675"));
}
 

import org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName; //导入依赖的package包/类
private void validate(PrimitiveTypeName p) {
  Type currentType = types.peek().asGroupType().getType(fields.peek());
  int c = fieldValueCount.pop() + 1;
  fieldValueCount.push(c);
  LOG.debug("validate {} for {}",p ,currentType.getName());
  switch (currentType.getRepetition()) {
    case OPTIONAL:
    case REQUIRED:
      if (c > 1) {
        throw new InvalidRecordException("repeated value when the type is not repeated in " + currentType);
      }
      break;
    case REPEATED:
      break;
    default:
      throw new InvalidRecordException("unknown repetition " + currentType.getRepetition() + " in " + currentType);
  }
  if (!currentType.isPrimitive() || currentType.asPrimitiveType().getPrimitiveTypeName() != p) {
    throw new InvalidRecordException("expected type " + p + " but got "+ currentType);
  }
}
 

import org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName; //导入依赖的package包/类
@Test
public void testEachPrimitiveType() {
  MessageTypeBuilder builder = buildMessage();
  StringBuilder schema = new StringBuilder();
  schema.append("message EachType {\n");
  for (PrimitiveTypeName type : PrimitiveTypeName.values()) {
    // add a schema entry, e.g., "  required int32 int32_;\n"
    if (type == FIXED_LEN_BYTE_ARRAY) {
      schema.append("  required fixed_len_byte_array(3) fixed_;");
      builder.required(FIXED_LEN_BYTE_ARRAY).length(3).named("fixed_");
    } else {
      schema.append("  required ").append(type)
      .append(" ").append(type).append("_;\n");
      builder.required(type).named(type.toString() + "_");
    }
  }
  schema.append("}\n");
  MessageType expected = builder.named("EachType");

  MessageType parsed = parseMessageType(schema.toString());

  assertEquals(expected, parsed);
  MessageType reparsed = parseMessageType(parsed.toString());
  assertEquals(expected, reparsed);
}
 

import org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName; //导入依赖的package包/类
@Test
public void test() throws IOException {
  MemPageStore memPageStore = new MemPageStore(10);
  ColumnDescriptor col = new ColumnDescriptor(path , PrimitiveTypeName.INT64, 2, 2);
  LongStatistics stats = new LongStatistics();
  PageWriter pageWriter = memPageStore.getPageWriter(col);
  pageWriter.writePage(BytesInput.from(new byte[735]), 209, stats, BIT_PACKED, BIT_PACKED, PLAIN);
  pageWriter.writePage(BytesInput.from(new byte[743]), 209, stats, BIT_PACKED, BIT_PACKED, PLAIN);
  pageWriter.writePage(BytesInput.from(new byte[743]), 209, stats, BIT_PACKED, BIT_PACKED, PLAIN);
  pageWriter.writePage(BytesInput.from(new byte[735]), 209, stats, BIT_PACKED, BIT_PACKED, PLAIN);
  PageReader pageReader = memPageStore.getPageReader(col);
  long totalValueCount = pageReader.getTotalValueCount();
  System.out.println(totalValueCount);
  int total = 0;
  do {
    DataPage readPage = pageReader.readPage();
    total += readPage.getValueCount();
    System.out.println(readPage);
    // TODO: assert
  } while (total < totalValueCount);
}
 

import org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName; //导入依赖的package包/类
public PrimitiveTypeName getPrimitive(Type type) {
  switch (type) {
    case BYTE_ARRAY: // TODO: rename BINARY and remove this switch
      return PrimitiveTypeName.BINARY;
    case INT64:
      return PrimitiveTypeName.INT64;
    case INT32:
      return PrimitiveTypeName.INT32;
    case BOOLEAN:
      return PrimitiveTypeName.BOOLEAN;
    case FLOAT:
      return PrimitiveTypeName.FLOAT;
    case DOUBLE:
      return PrimitiveTypeName.DOUBLE;
    case INT96:
      return PrimitiveTypeName.INT96;
    case FIXED_LEN_BYTE_ARRAY:
      return PrimitiveTypeName.FIXED_LEN_BYTE_ARRAY;
    default:
      throw new RuntimeException("Unknown type " + type);
  }
}
 

import org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName; //导入依赖的package包/类
@Test
public void testIgnoreStatsWithSignedSortOrder() {
  ParquetMetadataConverter converter = new ParquetMetadataConverter();
  BinaryStatistics stats = new BinaryStatistics();
  stats.incrementNumNulls();
  stats.updateStats(Binary.fromString("A"));
  stats.incrementNumNulls();
  stats.updateStats(Binary.fromString("z"));
  stats.incrementNumNulls();

  PrimitiveType binaryType = Types.required(PrimitiveTypeName.BINARY)
      .as(OriginalType.UTF8).named("b");
  Statistics convertedStats = converter.fromParquetStatistics(
      Version.FULL_VERSION,
      StatsHelper.V1.toParquetStatistics(stats),
      binaryType);

  Assert.assertTrue("Stats should be empty: " + convertedStats, convertedStats.isEmpty());
}
 

import org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName; //导入依赖的package包/类
@Test
public void testFloatingPointStringIndependentFromLocale() {
  Statistics<?> floatStats = Statistics.createStats(Types.optional(PrimitiveTypeName.FLOAT).named("test-float"));
  floatStats.updateStats(123.456f);
  Statistics<?> doubleStats = Statistics.createStats(Types.optional(PrimitiveTypeName.DOUBLE).named("test-double"));
  doubleStats.updateStats(12345.6789);

  Locale defaultLocale = Locale.getDefault();
  try {
    // Set the locale to French where the decimal separator would be ',' instead of '.'
    Locale.setDefault(Locale.FRENCH);
    assertEquals("min: 123.456, max: 123.456, num_nulls: 0", floatStats.toString());
    assertEquals("min: 12345.6789, max: 12345.6789, num_nulls: 0", doubleStats.toString());
  } finally {
    Locale.setDefault(defaultLocale);
  }
}
 

import org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName; //导入依赖的package包/类
@Test
public void testPrimitiveTypeConstruction() {
  PrimitiveTypeName[] types = new PrimitiveTypeName[] {
      BOOLEAN, INT32, INT64, INT96, FLOAT, DOUBLE, BINARY
  };
  for (PrimitiveTypeName type : types) {
    String name = type.toString() + "_";
    for (Type.Repetition repetition : Type.Repetition.values()) {
      PrimitiveType expected = new PrimitiveType(repetition, type, name);
      PrimitiveType built = Types.primitive(type, repetition).named(name);
      Assert.assertEquals(expected, built);
      switch (repetition) {
        case REQUIRED:
          built = Types.required(type).named(name);
          break;
        case OPTIONAL:
          built = Types.optional(type).named(name);
          break;
        case REPEATED:
          built = Types.repeated(type).named(name);
          break;
      }
      Assert.assertEquals(expected, built);
    }
  }
}
 

import org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName; //导入依赖的package包/类
@Test
public void testDECIMALAnnotationRejectsUnsupportedTypes() {
  PrimitiveTypeName[] unsupported = new PrimitiveTypeName[]{
      BOOLEAN, INT96, DOUBLE, FLOAT
  };
  for (final PrimitiveTypeName type : unsupported) {
    assertThrows("Should reject non-binary type: " + type,
        IllegalStateException.class, new Callable<Type>() {
          @Override
          public Type call() throws Exception {
            return Types.required(type)
                .as(DECIMAL).precision(9).scale(2)
                .named("d");
          }
        });
  }
}
 

import org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName; //导入依赖的package包/类
@Deprecated
public static ColumnChunkMetaData get(
    ColumnPath path,
    PrimitiveTypeName type,
    CompressionCodecName codec,
    Set<Encoding> encodings,
    Statistics statistics,
    long firstDataPage,
    long dictionaryPageOffset,
    long valueCount,
    long totalSize,
    long totalUncompressedSize) {
  return get(
      path, type, codec, null, encodings, statistics, firstDataPage, dictionaryPageOffset,
      valueCount, totalSize, totalUncompressedSize);
}
 

import org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName; //导入依赖的package包/类
/**
 * When reading the very first footer, any column is a potential partition column. So for the first footer, we check
 * every column to see if it is single valued, and if so, add it to the list of potential partition columns. For the
 * remaining footers, we will not find any new partition columns, but we may discover that what was previously a
 * potential partition column now no longer qualifies, so it needs to be removed from the list.
 * @return whether column is a potential partition column
 */
private boolean checkForPartitionColumn(ParquetFileMetadata fileMetadata, ColumnMetadata columnMetadata, boolean first, long rowCount) {
  SchemaPath schemaPath = SchemaPath.getCompoundPath(columnMetadata.getName());
  if (schemaPath.getAsUnescapedPath().equals(UPDATE_COLUMN)) {
    return true;
  }
  final PrimitiveTypeName primitiveType = fileMetadata.getPrimitiveType(columnMetadata.getName());
  final OriginalType originalType = fileMetadata.getOriginalType(columnMetadata.getName());

  if (first) {
    if (hasSingleValue(columnMetadata, rowCount)) {
      columnTypeMap.put(schemaPath, getType(primitiveType, originalType));
      return true;
    } else {
      return false;
    }
  } else {
    if (!columnTypeMap.keySet().contains(schemaPath)) {
      return false;
    } else {
      if (!hasSingleValue(columnMetadata, rowCount)) {
        columnTypeMap.remove(schemaPath);
        return false;
      }
      if (!getType(primitiveType, originalType).equals(columnTypeMap.get(schemaPath))) {
        columnTypeMap.remove(schemaPath);
        return false;
      }
    }
  }
  return true;
}
 

import org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName; //导入依赖的package包/类
private PrimitiveType getPrimitiveType(Field field) {
  MajorType majorType = getMajorTypeForField(field);
  MinorType minorType = majorType.getMinorType();
  String name = field.getName();
  PrimitiveTypeName primitiveTypeName = ParquetTypeHelper.getPrimitiveTypeNameForMinorType(minorType);
  if (primitiveTypeName == null) {
    return null;
  }
  OriginalType originalType = ParquetTypeHelper.getOriginalTypeForMinorType(minorType);
  int length = ParquetTypeHelper.getLengthForMinorType(minorType);
  DecimalMetadata decimalMetadata  = ParquetTypeHelper.getDecimalMetadataForField(majorType);
  return new PrimitiveType(OPTIONAL, primitiveTypeName, length, name, originalType, decimalMetadata, null);
}
 

import org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName; //导入依赖的package包/类
public ColumnMetadata_v1(SchemaPath name, PrimitiveTypeName primitiveType, OriginalType originalType,
    Object max, Object min, Long nulls) {
  this.name = name;
  this.primitiveType = primitiveType;
  this.originalType = originalType;
  this.max = max;
  this.min = min;
  this.nulls = nulls;
}
 

import org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName; //导入依赖的package包/类
@JsonProperty(value = "min")
public Object getMin() {
  if (primitiveType == PrimitiveTypeName.BINARY && min != null) {
    return new String(((Binary) min).getBytes());
  }
  return min;
}
 

import org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName; //导入依赖的package包/类
@JsonProperty(value = "max")
public Object getMax() {
  if (primitiveType == PrimitiveTypeName.BINARY && max != null) {
    return new String(((Binary) max).getBytes());
  }
  return max;
}