'Unable to retrieve all the subfolders in S3 (Scala)
I am unable to retrieve all the S3 paths for each sub-folders with this code.
package com.amazon.music.mdxsparkworkflows.utils
import com.amazonaws.services.s3.{AmazonS3, AmazonS3ClientBuilder}
import com.amazonaws.services.s3.model.{ListObjectsRequest, ObjectListing}
import scala.collection.JavaConversions._
object S3Utils {
def getS3Client(region:String = "us-east-1"): AmazonS3 ={
AmazonS3ClientBuilder.standard()
.withRegion(region)
.build()
}
def getS3CommonPrefixes(s3BucketName: String, s3PrefixPath: String): List[String] = {
val listObjectsRequest: ListObjectsRequest = new ListObjectsRequest()
.withBucketName(s3BucketName)
.withPrefix(s3PrefixPath)
.withDelimiter("/")
val s3Client = getS3Client()
val s3Objects: ObjectListing = s3Client.listObjects(listObjectsRequest)
s3Objects.getCommonPrefixes.toList
}
}
It only retrieves for a single partition then stops.
Sources
This article follows the attribution requirements of Stack Overflow and is licensed under CC BY-SA 3.0.
Source: Stack Overflow
| Solution | Source |
|---|
