Python process 1 file at a time using boto3

Question

I have a Python script that needs to read and process 100 files. How can my script run while only processing 1 file at a time? My print(len(session_results)) should have 2 separate lengths.

def fetch_endpoint(endpoint: str, date: str) -> str:
    """Read files from s3 bucket and aggregate them via generator function"""
    client = aws_s3_client_auth()
    resource = aws_s3_resource_auth()
    print(f"{endpoint.upper()}/{date}/")
    paginator = client.get_paginator('list_objects_v2')
    operation_parameters: dict[str] = {
        "Bucket": RAW_BUCKET,
        "Prefix": f"{endpoint.upper()}/{date}/"
    }
    keys_to_process: list[str] = []
    page_iterator = paginator.paginate(**operation_parameters, PaginationConfig={'MaxItems': 100})
    for page in page_iterator:
        if page.get("KeyCount") == 0:
            return print(f'No data was found for {endpoint} on {date}')
        else:
            for content in page.get("Contents"):
                # print(content)
                keys_to_process.append(content.get("Key"))
            print(keys_to_process)
            index = 0
            while index < len(keys_to_process):
                print(keys_to_process[index])
                client_events = resource.Object(RAW_BUCKET, key=keys_to_process[index])
                file_content = client_events.get()['Body'].read().decode('utf-8')
                for line in file_content.splitlines():
                    data = json.loads(line)
                    index += 1
                    yield data


def session(endpoint: str, date: str) -> None:
    """Make a request to s3 to read session in the RAW_BUCKET"""
    session_results = [line for line in fetch_endpoint(endpoint, date)]
    print(len(session_results))
    return None

Answer 1

session_results is a list where each element in the list is something that was yielded by fetch_endpoint().

When you print(len(session_results)) you are telling Python to tell you how many elements are in session_results.

I'm not really sure what you mean by "should have 2 separate lengths." It is a list, it can only have 1 length at a time.