I am processing significant amount of files (about 9000 files) via S3
I am using next_token = response.get("NextContinuationToken")
to process parts of keys
I found a problem -- next_token
value is always the same (name of object in my s3). Thus I have an infinite loop.
What is wrong with my code? Why do I have the same value of next_token
variable
My code
import boto3import botocoreclass S3Manager: def __init__(self, config) -> None: # Set certs and secrets here aws_config = boto3.session.Config ( signature_version='s3v4', connect_timeout=100, client_cert=ssl_certs ) self.s3_client = boto3.client(storage_type, endpoint_url=self.endpoint, aws_access_key_id=access_key, aws_secret_access_key=secret_key, aws_session_token=None, config=aws_config, ) def get_folders(self, bucket_name, prefix=""): folders = [] default_kwargs = {"Bucket": bucket_name,"Prefix": prefix } next_token = "" while next_token is not None: updated_kwargs = default_kwargs.copy() if next_token != "": updated_kwargs["ContinuationToken"] = next_token response = self.s3_client.list_objects_v2(**default_kwargs) # Appending data to folders list next_token = response.get("NextContinuationToken") self.logger.debug(f"DEBUG_ S3 next_token {next_token}") # see the same value every iteration return folders