Skip to content

armgit / ente

Public Imported from GitHub
0

πŸ’š End-to-end encrypted cloud for everything.

[server] Handle S3 prefix cleanup pagination (#13218)

neeraj-pilot3db6f4a49e2c
diff --git a/server/pkg/controller/object_cleanup.go b/server/pkg/controller/object_cleanup.go
index e496d84a4e..ac950d1a03 100644
--- a/server/pkg/controller/object_cleanup.go
+++ b/server/pkg/controller/object_cleanup.go
@@ -175,23 +175,36 @@ func (c *ObjectCleanupController) AddTempObject(object ente.TempObject) error {
func (c *ObjectCleanupController) DeleteAllObjectsWithPrefix(prefix string, dc string) error {
s3Client := c.S3Config.GetS3Client(dc)
bucket := c.S3Config.GetBucket(dc)
- output, err := s3Client.ListObjectsV2(&s3.ListObjectsV2Input{
- Bucket: bucket,
- Prefix: &prefix,
- })
- if err != nil {
- log.WithFields(log.Fields{
- "prefix": prefix,
- "dc": dc,
- }).WithError(err).Error("Failed to list objects")
- return stacktrace.Propagate(err, "")
- }
var keys []string
- for _, obj := range output.Contents {
- keys = append(keys, *obj.Key)
+ var continuationToken *string
+ for {
+ output, err := s3Client.ListObjectsV2(&s3.ListObjectsV2Input{
+ Bucket: bucket,
+ Prefix: &prefix,
+ ContinuationToken: continuationToken,
+ })
+ if err != nil {
+ log.WithFields(log.Fields{
+ "prefix": prefix,
+ "dc": dc,
+ }).WithError(err).Error("Failed to list objects")
+ return stacktrace.Propagate(err, "")
+ }
+ for _, obj := range output.Contents {
+ keys = append(keys, *obj.Key)
+ }
+ if !aws.BoolValue(output.IsTruncated) {
+ break
+ }
+ nextContinuationToken := output.NextContinuationToken
+ if aws.StringValue(nextContinuationToken) == "" ||
+ aws.StringValue(nextContinuationToken) == aws.StringValue(continuationToken) {
+ return errors.New("truncated object listing did not provide a new continuation token")
+ }
+ continuationToken = nextContinuationToken
}
for _, key := range keys {
- err = c.DeleteObjectFromDataCenter(key, dc)
+ err := c.DeleteObjectFromDataCenter(key, dc)
if err != nil {
log.WithFields(log.Fields{
"object_key": key,