in aws/src/main/java/org/apache/iceberg/aws/s3/S3FileIOProperties.java [585:704]
public S3FileIOProperties(Map<String, String> properties) {
this.sseType = properties.getOrDefault(SSE_TYPE, SSE_TYPE_NONE);
this.sseKey = properties.get(SSE_KEY);
this.sseMd5 = properties.get(SSE_MD5);
this.accessKeyId = properties.get(ACCESS_KEY_ID);
this.secretAccessKey = properties.get(SECRET_ACCESS_KEY);
this.sessionToken = properties.get(SESSION_TOKEN);
if (SSE_TYPE_CUSTOM.equals(sseType)) {
Preconditions.checkArgument(
null != sseKey, "Cannot initialize SSE-C S3FileIO with null encryption key");
Preconditions.checkArgument(
null != sseMd5, "Cannot initialize SSE-C S3FileIO with null encryption key MD5");
}
this.endpoint = properties.get(ENDPOINT);
this.multipartUploadThreads =
PropertyUtil.propertyAsInt(
properties, MULTIPART_UPLOAD_THREADS, Runtime.getRuntime().availableProcessors());
this.isPathStyleAccess =
PropertyUtil.propertyAsBoolean(properties, PATH_STYLE_ACCESS, PATH_STYLE_ACCESS_DEFAULT);
this.isUseArnRegionEnabled =
PropertyUtil.propertyAsBoolean(
properties, USE_ARN_REGION_ENABLED, USE_ARN_REGION_ENABLED_DEFAULT);
this.isAccelerationEnabled =
PropertyUtil.propertyAsBoolean(
properties, ACCELERATION_ENABLED, ACCELERATION_ENABLED_DEFAULT);
this.isDualStackEnabled =
PropertyUtil.propertyAsBoolean(properties, DUALSTACK_ENABLED, DUALSTACK_ENABLED_DEFAULT);
this.isCrossRegionAccessEnabled =
PropertyUtil.propertyAsBoolean(
properties, CROSS_REGION_ACCESS_ENABLED, CROSS_REGION_ACCESS_ENABLED_DEFAULT);
try {
this.multiPartSize =
PropertyUtil.propertyAsInt(properties, MULTIPART_SIZE, MULTIPART_SIZE_DEFAULT);
} catch (NumberFormatException e) {
throw new IllegalArgumentException(
String.format(
"Input malformed or exceeded maximum multipart upload size 5GB: %s",
properties.get(MULTIPART_SIZE)));
}
this.multipartThresholdFactor =
PropertyUtil.propertyAsDouble(
properties, MULTIPART_THRESHOLD_FACTOR, MULTIPART_THRESHOLD_FACTOR_DEFAULT);
Preconditions.checkArgument(
multipartThresholdFactor >= 1.0, "Multipart threshold factor must be >= to 1.0");
Preconditions.checkArgument(
multiPartSize >= MULTIPART_SIZE_MIN,
"Minimum multipart upload object size must be larger than 5 MB.");
this.stagingDirectory =
PropertyUtil.propertyAsString(
properties, STAGING_DIRECTORY, System.getProperty("java.io.tmpdir"));
String aclType = properties.get(ACL);
this.acl = ObjectCannedACL.fromValue(aclType);
Preconditions.checkArgument(
acl == null || !acl.equals(ObjectCannedACL.UNKNOWN_TO_SDK_VERSION),
"Cannot support S3 CannedACL " + aclType);
this.isChecksumEnabled =
PropertyUtil.propertyAsBoolean(properties, CHECKSUM_ENABLED, CHECKSUM_ENABLED_DEFAULT);
this.deleteBatchSize =
PropertyUtil.propertyAsInt(properties, DELETE_BATCH_SIZE, DELETE_BATCH_SIZE_DEFAULT);
Preconditions.checkArgument(
deleteBatchSize > 0 && deleteBatchSize <= DELETE_BATCH_SIZE_MAX,
String.format("Deletion batch size must be between 1 and %s", DELETE_BATCH_SIZE_MAX));
this.writeTags = toS3Tags(properties, WRITE_TAGS_PREFIX);
this.isWriteTableTagEnabled =
PropertyUtil.propertyAsBoolean(
properties, WRITE_TABLE_TAG_ENABLED, WRITE_TABLE_TAG_ENABLED_DEFAULT);
this.isWriteNamespaceTagEnabled =
PropertyUtil.propertyAsBoolean(
properties, WRITE_NAMESPACE_TAG_ENABLED, WRITE_NAMESPACE_TAG_ENABLED_DEFAULT);
this.deleteTags = toS3Tags(properties, DELETE_TAGS_PREFIX);
this.deleteThreads =
PropertyUtil.propertyAsInt(
properties, DELETE_THREADS, Runtime.getRuntime().availableProcessors());
this.isDeleteEnabled =
PropertyUtil.propertyAsBoolean(properties, DELETE_ENABLED, DELETE_ENABLED_DEFAULT);
this.bucketToAccessPointMapping =
PropertyUtil.propertiesWithPrefix(properties, ACCESS_POINTS_PREFIX);
this.isPreloadClientEnabled =
PropertyUtil.propertyAsBoolean(
properties, PRELOAD_CLIENT_ENABLED, PRELOAD_CLIENT_ENABLED_DEFAULT);
this.isRemoteSigningEnabled =
PropertyUtil.propertyAsBoolean(
properties, REMOTE_SIGNING_ENABLED, REMOTE_SIGNING_ENABLED_DEFAULT);
this.writeStorageClass = properties.get(WRITE_STORAGE_CLASS);
this.allProperties = SerializableMap.copyOf(properties);
this.isS3AccessGrantsEnabled =
PropertyUtil.propertyAsBoolean(
properties, S3_ACCESS_GRANTS_ENABLED, S3_ACCESS_GRANTS_ENABLED_DEFAULT);
this.isS3AccessGrantsFallbackToIamEnabled =
PropertyUtil.propertyAsBoolean(
properties,
S3_ACCESS_GRANTS_FALLBACK_TO_IAM_ENABLED,
S3_ACCESS_GRANTS_FALLBACK_TO_IAM_ENABLED_DEFAULT);
this.s3RetryNumRetries =
PropertyUtil.propertyAsInt(properties, S3_RETRY_NUM_RETRIES, S3_RETRY_NUM_RETRIES_DEFAULT);
this.s3RetryMinWaitMs =
PropertyUtil.propertyAsLong(properties, S3_RETRY_MIN_WAIT_MS, S3_RETRY_MIN_WAIT_MS_DEFAULT);
this.s3RetryMaxWaitMs =
PropertyUtil.propertyAsLong(properties, S3_RETRY_MAX_WAIT_MS, S3_RETRY_MAX_WAIT_MS_DEFAULT);
this.s3DirectoryBucketListPrefixAsDirectory =
PropertyUtil.propertyAsBoolean(
properties,
S3_DIRECTORY_BUCKET_LIST_PREFIX_AS_DIRECTORY,
S3_DIRECTORY_BUCKET_LIST_PREFIX_AS_DIRECTORY_DEFAULT);
this.isS3AnalyticsAcceleratorEnabled =
PropertyUtil.propertyAsBoolean(
properties, S3_ANALYTICS_ACCELERATOR_ENABLED, S3_ANALYTICS_ACCELERATOR_ENABLED_DEFAULT);
this.s3AnalyticsacceleratorProperties =
PropertyUtil.propertiesWithPrefix(properties, S3_ANALYTICS_ACCELERATOR_PROPERTIES_PREFIX);
this.isS3CRTEnabled =
PropertyUtil.propertyAsBoolean(properties, S3_CRT_ENABLED, S3_CRT_ENABLED_DEFAULT);
this.s3CrtMaxConcurrency =
PropertyUtil.propertyAsInt(
properties, S3_CRT_MAX_CONCURRENCY, S3_CRT_MAX_CONCURRENCY_DEFAULT);
ValidationException.check(
keyIdAccessKeyBothConfigured(),
"S3 client access key ID and secret access key must be set at the same time");
}