in seatunnel-datasource/seatunnel-datasource-plugins/datasource-s3redshift/src/main/java/org/apache/seatunnel/datasource/plugin/redshift/s3/HadoopS3AConfiguration.java [42:96]
public static Configuration getConfiguration(Map<String, String> s3Options) {
if (!s3Options.containsKey(S3RedshiftOptionRule.BUCKET.key())) {
throw new IllegalArgumentException(
"S3Redshift datasource bucket is null, please check your config");
}
if (!s3Options.containsKey(S3RedshiftOptionRule.FS_S3A_ENDPOINT.key())) {
throw new IllegalArgumentException(
"S3Redshift datasource endpoint is null, please check your config");
}
String bucket = s3Options.get(S3RedshiftOptionRule.BUCKET.key());
String protocol = DEFAULT_PROTOCOL;
if (bucket.startsWith(S3A_PROTOCOL)) {
protocol = S3A_PROTOCOL;
}
String fsImpl = protocol.equals(S3A_PROTOCOL) ? HDFS_S3A_IMPL : HDFS_S3N_IMPL;
Configuration hadoopConf = new Configuration();
hadoopConf.set(FS_DEFAULT_NAME_KEY, bucket);
hadoopConf.set(
S3RedshiftOptionRule.FS_S3A_ENDPOINT.key(),
s3Options.get(S3RedshiftOptionRule.FS_S3A_ENDPOINT.key()));
hadoopConf.set(formatKey(protocol, HDFS_IMPL_KEY), fsImpl);
if (s3Options.containsKey(S3RedshiftOptionRule.HADOOP_S3_PROPERTIES.key())) {
Arrays.stream(
s3Options
.get(S3RedshiftOptionRule.HADOOP_S3_PROPERTIES.key())
.split("\n"))
.map(String::trim)
.filter(StringUtils::isNotBlank)
.forEach(
line -> {
String[] kv = line.split("=");
if (kv.length == 2) {
hadoopConf.set(kv[0].trim(), kv[1].trim());
}
});
}
if (S3RedshiftOptionRule.S3aAwsCredentialsProvider.SimpleAWSCredentialsProvider
.getProvider()
.equals(s3Options.get(S3RedshiftOptionRule.S3A_AWS_CREDENTIALS_PROVIDER.key()))) {
hadoopConf.set(
S3RedshiftOptionRule.S3A_AWS_CREDENTIALS_PROVIDER.key(),
s3Options.get(S3RedshiftOptionRule.S3A_AWS_CREDENTIALS_PROVIDER.key()));
hadoopConf.set(
"fs.s3a.access.key", s3Options.get(S3RedshiftOptionRule.ACCESS_KEY.key()));
hadoopConf.set(
"fs.s3a.secret.key", s3Options.get(S3RedshiftOptionRule.SECRET_KEY.key()));
} else {
hadoopConf.set(
S3RedshiftOptionRule.S3A_AWS_CREDENTIALS_PROVIDER.key(),
s3Options.get(S3RedshiftOptionRule.S3A_AWS_CREDENTIALS_PROVIDER.key()));
}
return hadoopConf;
}