'Unable to retrieve all the subfolders in S3 (Scala)

I am unable to retrieve all the S3 paths for each sub-folders with this code.

package com.amazon.music.mdxsparkworkflows.utils

import com.amazonaws.services.s3.{AmazonS3, AmazonS3ClientBuilder}
import com.amazonaws.services.s3.model.{ListObjectsRequest, ObjectListing}
import scala.collection.JavaConversions._

object S3Utils {
  def getS3Client(region:String = "us-east-1"): AmazonS3 ={
    AmazonS3ClientBuilder.standard()
      .withRegion(region)
      .build()
  }

  def getS3CommonPrefixes(s3BucketName: String, s3PrefixPath: String): List[String] = {
    val listObjectsRequest: ListObjectsRequest = new ListObjectsRequest()
      .withBucketName(s3BucketName)
      .withPrefix(s3PrefixPath)
      .withDelimiter("/")
    val s3Client = getS3Client()
    val s3Objects: ObjectListing = s3Client.listObjects(listObjectsRequest)

    s3Objects.getCommonPrefixes.toList
  }
}

It only retrieves for a single partition then stops.



Sources

This article follows the attribution requirements of Stack Overflow and is licensed under CC BY-SA 3.0.

Source: Stack Overflow

Solution Source