def listAll()

in magenta-lib/src/main/scala/magenta/artifact/S3Location.scala [28:71]
39 lines of code
3 McCabe index (conditional complexity)

  def listAll(bucket: String)(implicit s3Client: S3Client): Seq[S3Object] =
    listObjects(bucket, None)

  def listObjects(location: S3Location)(implicit
      s3Client: S3Client
  ): Seq[S3Object] =
    listObjects(location.bucket, Some(location.key))

  val maxKeysInBucketListing =
    1000 // AWS won't return more than this, even if you set the parameter to a larger value

  private def listObjects(bucket: String, prefix: Option[String])(implicit
      s3Client: S3Client
  ): Seq[S3Object] = {
    def request(continuationToken: Option[String]): ListObjectsV2Request =
      ListObjectsV2Request
        .builder()
        .bucket(bucket)
        .prefix(prefix.orNull)
        .maxKeys(maxKeysInBucketListing)
        .continuationToken(continuationToken.orNull)
        .build()

    @tailrec
    def pageListings(
        acc: Seq[ListObjectsV2Response],
        previousListing: ListObjectsV2Response
    ): Seq[ListObjectsV2Response] = {
      if (!previousListing.isTruncated) {
        acc
      } else {
        val listing = s3Client.listObjectsV2(
          request(Some(previousListing.nextContinuationToken))
        )
        pageListings(acc :+ listing, listing)
      }
    }

    val initialListing = s3Client.listObjectsV2(request(None))
    for {
      summaries <- pageListings(Seq(initialListing), initialListing)
      summary <- summaries.contents.asScala
    } yield S3Object(bucket, summary.key, summary.size)
  }