import org.apache.parquet.io.api.Converter; //导入依赖的package包/类
private Converter groupConverter(OutputMutator mutator,
    List<Field> arrowSchema, GroupType groupType, PathSegment colNextChild, final String nameForChild) {
  Collection<SchemaPath> c = new ArrayList<>();

  while (colNextChild != null) {
    if (colNextChild.isNamed()) {
      break;
    }
    colNextChild = colNextChild.getChild();
  }

  if (colNextChild != null) {
    SchemaPath s = new SchemaPath(colNextChild.getNameSegment());
    c.add(s);
  }

  if (arrowSchema != null) {
    return groupConverterFromArrowSchema(nameForChild, groupType.getName(), groupType, c);
  }

  return defaultGroupConverter(mutator, groupType, nameForChild, c, null);
}
 

import org.apache.parquet.io.api.Converter; //导入依赖的package包/类
public ParquetValueConverter(GroupType schema, ParentContainerUpdater updater)
{
    super(updater);

    ArrayList<String> fieldNames = new ArrayList<>();
    for (Type type : schema.getFields()) {
        fieldNames.add(type.getName());
    }
    this.currentMap = new InternalMap(fieldNames);

    this.fieldConverters = new Converter[schema.getFieldCount()];
    int i = 0;
    for (Type field : schema.getFields()) {
        InternalMapUpdater update = new InternalMapUpdater(currentMap, i);
        fieldConverters[i++] = newFieldConverter(field, update);
    }
}
 

import org.apache.parquet.io.api.Converter; //导入依赖的package包/类
private Converter newFieldConverter(Type parquetType, ParentContainerUpdater updater)
{
    if (parquetType.isRepetition(Type.Repetition.REPEATED) && parquetType.getOriginalType() != OriginalType.LIST) {
        // A repeated field that is neither contained by a `LIST`- or `MAP`-annotated group nor
        // annotated by `LIST` or `MAP` should be interpreted as a required list of required
        // elements where the element type is the type of the field.
        if (parquetType.isPrimitive()) {
            return new RepeatedPrimitiveConverter(parquetType, updater);
        }
        else {
            return new RepeatedGroupConverter(parquetType, updater);
        }
    }
    else {
        return newConverter(parquetType, updater);
    }
}
 

import org.apache.parquet.io.api.Converter; //导入依赖的package包/类
private Map<Integer, Converter> buildFieldToConverter(final GroupType schema) {
    final Map<Integer, Converter> fieldToConverter = new HashMap<>(fieldCount);
    int i = 0;
    for (final Type field : schema.getFields()) {
        final String[] newColumnPath = new String[columnPath.length + 1];
        int j = 0;
        for (final String part : columnPath) {
            newColumnPath[j] = part;
            j++;
        }
        newColumnPath[j] = field.getName();
        if (field.isPrimitive()) {
            fieldToConverter.put(i, new PrimitiveConverter(parquetColumnToObject, field.asPrimitiveType().getPrimitiveTypeName().javaType.getSimpleName(), newColumnPath, field.getOriginalType()));
        } else {
            fieldToConverter.put(i, new BypassGroupConverter(parquetColumnToObject, field.asGroupType(), newColumnPath));
        }
        i++;
    }
    return fieldToConverter;
}
 

import org.apache.parquet.io.api.Converter; //导入依赖的package包/类
SimpleGroupConverter(SimpleGroupConverter parent, int index, GroupType schema) {
  this.parent = parent;
  this.index = index;

  converters = new Converter[schema.getFieldCount()];

  for (int i = 0; i < converters.length; i++) {
    final Type type = schema.getType(i);
    if (type.isPrimitive()) {
      converters[i] = new SimplePrimitiveConverter(this, i);
    } else {
      converters[i] = new SimpleGroupConverter(this, i, type.asGroupType());
    }

  }
}
 

import org.apache.parquet.io.api.Converter; //导入依赖的package包/类
@Override
public Converter getConverter(int fieldIndex) {

  // get the real converter from the delegate
  Converter delegateConverter = checkNotNull(delegate.getConverter(fieldIndex), "delegate converter");

  // determine the indexFieldPath for the converter proxy we're about to make, which is
  // this converter's path + the requested fieldIndex
  List<Integer> newIndexFieldPath = new ArrayList<Integer>(indexFieldPath.size() + 1);
  newIndexFieldPath.addAll(indexFieldPath);
  newIndexFieldPath.add(fieldIndex);

  if (delegateConverter.isPrimitive()) {
    PrimitiveColumnIO columnIO = getColumnIO(newIndexFieldPath);
    ColumnPath columnPath = ColumnPath.get(columnIO.getColumnDescriptor().getPath());
    ValueInspector[] valueInspectors = getValueInspectors(columnPath);
    return new FilteringPrimitiveConverter(delegateConverter.asPrimitiveConverter(), valueInspectors);
  } else {
    return new FilteringGroupConverter(delegateConverter.asGroupConverter(), newIndexFieldPath, valueInspectorsByColumn, columnIOsByIndexFieldPath);
  }

}
 

import org.apache.parquet.io.api.Converter; //导入依赖的package包/类
private Converter newConverter(List<TProtocol> events, Type type, ThriftField field) {
  switch (field.getType().getType()) {
  case LIST:
    return new ListConverter(events, type.asGroupType(), field);
  case SET:
    return new SetConverter(events, type.asGroupType(), field);
  case MAP:
    return new MapConverter(events, type.asGroupType(), field);
  case STRUCT:
    return new StructConverter(events, type.asGroupType(), field);
  case STRING:
    return new FieldStringConverter(events, field);
  case ENUM:
    return new FieldEnumConverter(events, field);
  default:
    return new FieldPrimitiveConverter(events, field);
  }
}
 

import org.apache.parquet.io.api.Converter; //导入依赖的package包/类
protected static Converter getConverterFromDescription(final Type type, final int index,
    final HiveGroupConverter parent) {
  if (type == null) {
    return null;
  }
  if (type.isPrimitive()) {
    return ETypeConverter.getNewConverter(type.asPrimitiveType().getPrimitiveTypeName().javaType,
        index, parent);
  } else {
    if (type.asGroupType().getRepetition() == Repetition.REPEATED) {
      return new ArrayWritableGroupConverter(type.asGroupType(), parent, index);
    } else {
      return new DataWritableGroupConverter(type.asGroupType(), parent, index);
    }
  }
}
 

import org.apache.parquet.io.api.Converter; //导入依赖的package包/类
public DataWritableGroupConverter(final GroupType selectedGroupType,
    final HiveGroupConverter parent, final int index, final GroupType containingGroupType) {
  this.parent = parent;
  this.index = index;
  final int totalFieldCount = containingGroupType.getFieldCount();
  final int selectedFieldCount = selectedGroupType.getFieldCount();

  currentArr = new Object[totalFieldCount];
  converters = new Converter[selectedFieldCount];

  List<Type> selectedFields = selectedGroupType.getFields();
  for (int i = 0; i < selectedFieldCount; i++) {
    Type subtype = selectedFields.get(i);
    if (containingGroupType.getFields().contains(subtype)) {
      converters[i] = getConverterFromDescription(subtype,
          containingGroupType.getFieldIndex(subtype.getName()), this);
    } else {
      throw new IllegalStateException("Group type [" + containingGroupType +
          "] does not contain requested field: " + subtype);
    }
  }
}
 

import org.apache.parquet.io.api.Converter; //导入依赖的package包/类
private Converter newMessageConverter(final Message.Builder parentBuilder, final Descriptors.FieldDescriptor fieldDescriptor, Type parquetType) {

    boolean isRepeated = fieldDescriptor.isRepeated();

    ParentValueContainer parent;

    if (isRepeated) {
      parent = new ParentValueContainer() {
        @Override
        public void add(Object value) {
          parentBuilder.addRepeatedField(fieldDescriptor, value);
        }
      };
    } else {
      parent = new ParentValueContainer() {
        @Override
        public void add(Object value) {
          parentBuilder.setField(fieldDescriptor, value);
        }
      };
    }

    return newScalarConverter(parent, parentBuilder, fieldDescriptor, parquetType);
  }
 

import org.apache.parquet.io.api.Converter; //导入依赖的package包/类
private Converter newScalarConverter(ParentValueContainer pvc, Message.Builder parentBuilder, Descriptors.FieldDescriptor fieldDescriptor, Type parquetType) {

    JavaType javaType = fieldDescriptor.getJavaType();

    switch (javaType) {
      case STRING: return new ProtoStringConverter(pvc);
      case FLOAT: return new ProtoFloatConverter(pvc);
      case DOUBLE: return new ProtoDoubleConverter(pvc);
      case BOOLEAN: return new ProtoBooleanConverter(pvc);
      case BYTE_STRING: return new ProtoBinaryConverter(pvc);
      case ENUM: return new ProtoEnumConverter(pvc, fieldDescriptor);
      case INT: return new ProtoIntConverter(pvc);
      case LONG: return new ProtoLongConverter(pvc);
      case MESSAGE: {
        Message.Builder subBuilder = parentBuilder.newBuilderForField(fieldDescriptor);
        return new ProtoMessageConverter(pvc, subBuilder, parquetType.asGroupType());
      }
    }

    throw new UnsupportedOperationException(String.format("Cannot convert type: %s" +
            " (Parquet type: %s) ", javaType, parquetType));
  }
 

import org.apache.parquet.io.api.Converter; //导入依赖的package包/类
public AvroUnionConverter(ParentValueContainer parent, Type parquetSchema,
                          Schema avroSchema, GenericData model) {
  this.parent = parent;
  GroupType parquetGroup = parquetSchema.asGroupType();
  this.memberConverters = new Converter[ parquetGroup.getFieldCount()];

  int parquetIndex = 0;
  for (int index = 0; index < avroSchema.getTypes().size(); index++) {
    Schema memberSchema = avroSchema.getTypes().get(index);
    if (!memberSchema.getType().equals(Schema.Type.NULL)) {
      Type memberType = parquetGroup.getType(parquetIndex);
      memberConverters[parquetIndex] = newConverter(memberSchema, memberType, model, new ParentValueContainer() {
        @Override
        public void add(Object value) {
          Preconditions.checkArgument(memberValue==null, "Union is resolving to more than one type");
          memberValue = value;
        }
      });
      parquetIndex++; // Note for nulls the parquetIndex id not increased
    }
  }
}
 

import org.apache.parquet.io.api.Converter; //导入依赖的package包/类
public AvroUnionConverter(ParentValueContainer parent, Type parquetSchema,
                          Schema avroSchema, GenericData model) {
  super(parent);
  GroupType parquetGroup = parquetSchema.asGroupType();
  this.memberConverters = new Converter[ parquetGroup.getFieldCount()];

  int parquetIndex = 0;
  for (int index = 0; index < avroSchema.getTypes().size(); index++) {
    Schema memberSchema = avroSchema.getTypes().get(index);
    if (!memberSchema.getType().equals(Schema.Type.NULL)) {
      Type memberType = parquetGroup.getType(parquetIndex);
      memberConverters[parquetIndex] = newConverter(memberSchema, memberType, model, new ParentValueContainer() {
        @Override
        public void add(Object value) {
          Preconditions.checkArgument(
              AvroUnionConverter.this.memberValue == null,
              "Union is resolving to more than one type");
          memberValue = value;
        }
      });
      parquetIndex++; // Note for nulls the parquetIndex id not increased
    }
  }
}
 

import org.apache.parquet.io.api.Converter; //导入依赖的package包/类
protected void addChildConverter(OutputMutator mutator,
    List<Field> arrowSchema, Iterator<SchemaPath> colIterator, Type type, Function<String, String> childNameResolver) {
  // Match the name of the field in the schema definition to the name of the field in the query.
  String name = null;
  SchemaPath col;
  PathSegment colPath;
  PathSegment colNextChild = null;
  while (colIterator.hasNext()) {
    col = colIterator.next();
    colPath = col.getRootSegment();
    colNextChild = colPath.getChild();

    if (colPath.isNamed() && (!colPath.getNameSegment().getPath().equals("*"))) {
      name = colPath.getNameSegment().getPath();
      // We may have a field that does not exist in the schema
      if (!name.equalsIgnoreCase(type.getName())) {
        continue;
      }
    }
    break;
  }
  if (name == null) {
    name = type.getName();
  }

  final String nameForChild = childNameResolver.apply(name);
  final Converter converter = type.isPrimitive() ?
    getConverterForType(nameForChild, type.asPrimitiveType())
    : groupConverter(mutator, arrowSchema, type.asGroupType(), colNextChild, nameForChild);
  converters.add(converter);
}
 

import org.apache.parquet.io.api.Converter; //导入依赖的package包/类
Converter groupConverterFromArrowSchema(String nameForChild, String fieldName, GroupType groupType, Collection<SchemaPath> c) {
  final Field arrowField = Schema.findField(arrowSchema, fieldName);
  final ArrowTypeID arrowTypeType = arrowField.getType().getTypeID();
  final List<Field> arrowChildren = arrowField.getChildren();
  if (arrowTypeType == ArrowTypeID.Union) {
    // if it's a union we will add the children directly to the parent
    return new UnionGroupConverter(mutator, getWriterProvider(), groupType, c, options, arrowChildren, nameForChild, containsCorruptedDates, readInt96AsTimeStamp);
  } else if (arrowTypeType == ArrowTypeID.List) {
    // make sure the parquet schema matches the arrow schema and delegate handling the logical list to defaultGroupConverter()
    Preconditions.checkState(groupType.getOriginalType() == OriginalType.LIST, "parquet schema doesn't match the arrow schema for LIST " + nameForChild);
  }

  return defaultGroupConverter(mutator, groupType, nameForChild, c, arrowChildren);
}
 

import org.apache.parquet.io.api.Converter; //导入依赖的package包/类
Converter defaultGroupConverter(OutputMutator mutator, GroupType groupType, final String nameForChild,
                                Collection<SchemaPath> c, List<Field> arrowSchema) {

  if (groupType.getOriginalType() == OriginalType.LIST && LogicalListL1Converter.isSupportedSchema(groupType)) {
    return new LogicalListL1Converter(
      nameForChild,
      mutator,
      getWriterProvider(),
      groupType,
      c,
      options,
      arrowSchema,
      containsCorruptedDates,
      readInt96AsTimeStamp);
  }

  final MapWriter map;
  if (groupType.isRepetition(REPEATED)) {
    if (arrowSchema != null) {
      //TODO assert this should never occur at this level
      // only parquet writer that writes arrowSchema doesn't write repeated fields except
      // as part of a LOGICAL LIST, thus this scenario (repeated + arrow schema present) can
      // only happen in LogicalList converter
      arrowSchema = handleRepeatedField(arrowSchema, groupType);
    }
    map = list(nameForChild).map();
  } else {
    map = getWriterProvider().map(nameForChild);
  }
  return new StructGroupConverter(mutator, map, groupType, c, options, arrowSchema, containsCorruptedDates, readInt96AsTimeStamp);
}
 

import org.apache.parquet.io.api.Converter; //导入依赖的package包/类
@Override
public void start()
{
    int i = 0;
    for (Converter converter : fieldConverters) {
        ((HasParentContainerUpdater) converter).getUpdater().start();
        currentMap.set(i, ValueFactory.newNil());
        i += 1;
    }
}
 

import org.apache.parquet.io.api.Converter; //导入依赖的package包/类
@Override
public void end()
{
    for (Converter converter : fieldConverters) {
        ((HasParentContainerUpdater) converter).getUpdater().end();
    }
    getUpdater().set(currentMap.build());
}
 

import org.apache.parquet.io.api.Converter; //导入依赖的package包/类
private Converter newConverter(Type parquetType, ParentContainerUpdater updater)
{
    if (parquetType.isPrimitive()) {
        return newConverterForPrimitiveField(parquetType.asPrimitiveType(), updater);
    }
    else {
        return newConverterForGroupField(parquetType.asGroupType(), updater);
    }
}
 

import org.apache.parquet.io.api.Converter; //导入依赖的package包/类
private Map<Integer, Converter> buildFieldToConverter(final MessageType schema) {
    final Map<Integer, Converter> fieldToConverter = new HashMap<>(fieldCount);
    int i = 0;
    for (final Type field : schema.getFields()) {
        if (field.isPrimitive()) {
            fieldToConverter.put(i, new PrimitiveConverter(parquetColumnToObject, field.asPrimitiveType().getPrimitiveTypeName().javaType.getSimpleName(), new String[]{field.getName()}, field.getOriginalType()));
        } else {
            fieldToConverter.put(i, new BypassGroupConverter(parquetColumnToObject, field.asGroupType(), new String[]{field.getName()}));
        }
        i++;
    }
    return fieldToConverter;
}