def initializeSparkSessionForIcebergTable()

in scala-spark-sdk/src/main/scala/software/amazon/sagemaker/featurestore/sparksdk/helpers/SparkSessionInitializer.scala [98:127]

27 lines of code
3 McCabe index (conditional complexity)


  def initializeSparkSessionForIcebergTable(
      sparkSession: SparkSession,
      offlineStoreEncryptionKmsKeyId: String,
      resolvedOutputS3Uri: String,
      dataCatalogName: String,
      assumeRoleArn: String,
      region: String
  ): Unit = {

    if (offlineStoreEncryptionKmsKeyId != null) {
      sparkSession.conf.set(f"spark.sql.catalog.$dataCatalogName.s3.sse.key", offlineStoreEncryptionKmsKeyId)
    }

    if (assumeRoleArn != null) {
      sparkSession.conf.set(f"spark.sql.catalog.$dataCatalogName.client.assume-role.arn", assumeRoleArn)
      sparkSession.conf.set(f"spark.sql.catalog.$dataCatalogName.client.assume-role.region", region)
      sparkSession.conf.set(
        f"spark.sql.catalog.$dataCatalogName.client.factory",
        "org.apache.iceberg.aws.AssumeRoleAwsClientFactory"
      )
    }

    sparkSession.conf.set(f"spark.sql.catalog.$dataCatalogName.s3.sse.type", "kms")
    sparkSession.conf.set(f"spark.sql.catalog.$dataCatalogName.warehouse", resolvedOutputS3Uri)
    sparkSession.conf.set(f"spark.sql.catalog.$dataCatalogName", "org.apache.iceberg.spark.SparkCatalog")
    sparkSession.conf.set(f"spark.sql.catalog.$dataCatalogName.catalog-impl", "org.apache.iceberg.aws.glue.GlueCatalog")
    sparkSession.conf.set(f"spark.sql.catalog.$dataCatalogName.io-impl", "org.apache.iceberg.aws.s3.S3FileIO")
    sparkSession.conf.set(f"spark.sql.catalog.$dataCatalogName.glue.skip-name-validation", "true")
    sparkSession.conf.set(f"spark.sql.catalog.$dataCatalogName.glue.skip-archive", "true")
  }