in parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ParquetFileWriter.java [445:522]
private ParquetFileWriter(
OutputFile file,
MessageType schema,
Mode mode,
long rowGroupSize,
int maxPaddingSize,
int columnIndexTruncateLength,
int statisticsTruncateLength,
boolean pageWriteChecksumEnabled,
FileEncryptionProperties encryptionProperties,
InternalFileEncryptor encryptor,
ByteBufferAllocator allocator)
throws IOException {
TypeUtil.checkValidWriteSchema(schema);
this.schema = schema;
long blockSize = rowGroupSize;
if (file.supportsBlockSize()) {
blockSize = Math.max(file.defaultBlockSize(), rowGroupSize);
this.alignment = PaddingAlignment.get(blockSize, rowGroupSize, maxPaddingSize);
} else {
this.alignment = NoAlignment.get(rowGroupSize);
}
if (mode == Mode.OVERWRITE) {
this.out = file.createOrOverwrite(blockSize);
} else {
this.out = file.create(blockSize);
}
this.encodingStatsBuilder = new EncodingStats.Builder();
this.columnIndexTruncateLength = columnIndexTruncateLength;
this.pageWriteChecksumEnabled = pageWriteChecksumEnabled;
this.crc = pageWriteChecksumEnabled ? new CRC32() : null;
this.crcAllocator = pageWriteChecksumEnabled
? ReusingByteBufferAllocator.strict(allocator == null ? new HeapByteBufferAllocator() : allocator)
: null;
this.metadataConverter = new ParquetMetadataConverter(statisticsTruncateLength);
if (null == encryptionProperties && null == encryptor) {
this.fileEncryptor = null;
return;
}
if (null == encryptionProperties) {
encryptionProperties = encryptor.getEncryptionProperties();
}
// Verify that every encrypted column is in file schema
Map<ColumnPath, ColumnEncryptionProperties> columnEncryptionProperties =
encryptionProperties.getEncryptedColumns();
if (null != columnEncryptionProperties) { // if null, every column in file schema will be encrypted with footer
// key
for (Map.Entry<ColumnPath, ColumnEncryptionProperties> entry : columnEncryptionProperties.entrySet()) {
String[] path = entry.getKey().toArray();
if (!schema.containsPath(path)) {
StringBuilder columnList = new StringBuilder();
columnList.append("[");
for (String[] columnPath : schema.getPaths()) {
columnList
.append(ColumnPath.get(columnPath).toDotString())
.append("], [");
}
throw new ParquetCryptoRuntimeException(
"Encrypted column [" + entry.getKey().toDotString() + "] not in file schema column list: "
+ columnList.substring(0, columnList.length() - 3));
}
}
}
if (null == encryptor) {
this.fileEncryptor = new InternalFileEncryptor(encryptionProperties);
} else {
this.fileEncryptor = encryptor;
}
}