in paimon-arrow/src/main/java/org/apache/paimon/arrow/ArrowUtils.java [95:190]
public static Field toArrowField(String fieldName, int fieldId, DataType dataType, int depth) {
FieldType fieldType = dataType.accept(ArrowFieldTypeConversion.ARROW_FIELD_TYPE_VISITOR);
fieldType =
new FieldType(
fieldType.isNullable(),
fieldType.getType(),
fieldType.getDictionary(),
Collections.singletonMap(PARQUET_FIELD_ID, String.valueOf(fieldId)));
List<Field> children = null;
if (dataType instanceof ArrayType) {
Field field =
toArrowField(
ListVector.DATA_VECTOR_NAME,
fieldId,
((ArrayType) dataType).getElementType(),
depth + 1);
FieldType typeInner = field.getFieldType();
field =
new Field(
field.getName(),
new FieldType(
typeInner.isNullable(),
typeInner.getType(),
typeInner.getDictionary(),
Collections.singletonMap(
PARQUET_FIELD_ID,
String.valueOf(
SpecialFields.getArrayElementFieldId(
fieldId, depth + 1)))),
field.getChildren());
children = Collections.singletonList(field);
} else if (dataType instanceof MapType) {
MapType mapType = (MapType) dataType;
Field keyField =
toArrowField(
MapVector.KEY_NAME, fieldId, mapType.getKeyType().notNull(), depth + 1);
FieldType keyType = keyField.getFieldType();
keyField =
new Field(
keyField.getName(),
new FieldType(
keyType.isNullable(),
keyType.getType(),
keyType.getDictionary(),
Collections.singletonMap(
PARQUET_FIELD_ID,
String.valueOf(
SpecialFields.getMapKeyFieldId(
fieldId, depth + 1)))),
keyField.getChildren());
Field valueField =
toArrowField(
MapVector.VALUE_NAME,
fieldId,
mapType.getValueType().notNull(),
depth + 1);
FieldType valueType = valueField.getFieldType();
valueField =
new Field(
valueField.getName(),
new FieldType(
valueType.isNullable(),
valueType.getType(),
valueType.getDictionary(),
Collections.singletonMap(
PARQUET_FIELD_ID,
String.valueOf(
SpecialFields.getMapValueFieldId(
fieldId, depth + 1)))),
valueField.getChildren());
FieldType structType =
new FieldType(
false,
Types.MinorType.STRUCT.getType(),
null,
Collections.singletonMap(PARQUET_FIELD_ID, String.valueOf(fieldId)));
Field mapField =
new Field(
MapVector.DATA_VECTOR_NAME,
// data vector, key vector and value vector CANNOT be null
structType,
Arrays.asList(keyField, valueField));
children = Collections.singletonList(mapField);
} else if (dataType instanceof RowType) {
RowType rowType = (RowType) dataType;
children = new ArrayList<>();
for (DataField field : rowType.getFields()) {
children.add(toArrowField(field.name(), field.id(), field.type(), 0));
}
}
return new Field(fieldName, fieldType, children);
}