void S3Init::list_files()

in awsio/csrc/io/s3/s3_io.cpp [387:427]

• 38 lines of code
• 7 McCabe index (conditional complexity)

void S3Init::list_files(const std::string &file_url,
                        std::vector<std::string> *filenames) {
    std::string bucket, prefix;
    parseS3Path(file_url, &bucket, &prefix);
    Aws::String default_key = "";
    if (prefix.empty()) {
        default_key = "/";
    }

    Aws::S3::Model::ListObjectsRequest listObjectsRequest;
    listObjectsRequest.WithBucket(bucket.c_str())
        .WithPrefix(prefix.c_str())
        .WithMaxKeys(S3GetFilesMaxKeys);

    Aws::S3::Model::ListObjectsResult listObjectsResult;
    do {
        auto listObjectsOutcome =
            this->initializeS3Client()->ListObjects(listObjectsRequest);
        if (!listObjectsOutcome.IsSuccess()) {
            Aws::String const &error_aws =
                listObjectsOutcome.GetError().GetMessage();
            std::string error_str(error_aws.c_str(), error_aws.size());
            throw std::invalid_argument(error_str);
        }

        listObjectsResult = listObjectsOutcome.GetResult();
        Aws::Vector<Aws::S3::Model::Object> objects = listObjectsResult.GetContents();
        if (!objects.empty()) {
            for (const auto &object : objects) {
                Aws::String key = default_key + object.GetKey();
                if (key.back() == '/') {
                    continue;
                }
                Aws::String bucket_aws(bucket.c_str(), bucket.size());
                Aws::String entry = "s3://" + bucket_aws + "/" + object.GetKey();
                filenames->push_back(entry.c_str());
            }
            listObjectsRequest.SetMarker(listObjectsResult.GetContents().back().GetKey());
        }
    } while (listObjectsResult.GetIsTruncated());
}