Skip to content

SeaweedFS is a distributed storage system for object storage (S3), file systems, Iceberg table buckets, and Lance vector buckets, designed to handle billions of files with O(1) disk access and effortless horizontal scaling.

23.9 KB
Raw Blame
package s3api

import (
	"context"
	"encoding/xml"
	"errors"
	"io"
	"net/http"
	"strings"

	"github.com/seaweedfs/seaweedfs/weed/glog"
	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
	"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
	"github.com/seaweedfs/seaweedfs/weed/s3api/s3err"
	stats_collect "github.com/seaweedfs/seaweedfs/weed/stats"
	"github.com/seaweedfs/seaweedfs/weed/util"
)

const (
	deleteMultipleObjectsLimit = 1000
)

type deleteMutationResult struct {
	versionId    string
	deleteMarker bool
}

func deleteErrorFromCode(code s3err.ErrorCode, key, versionId string) DeleteError {
	apiErr := s3err.GetAPIError(code)
	return DeleteError{
		Code:      apiErr.Code,
		Message:   apiErr.Description,
		Key:       key,
		VersionId: versionId,
	}
}

func validateDeleteObjectIdentifier(object ObjectIdentifier) s3err.ErrorCode {
	if !s3_constants.IsValidObjectKey(object.Key) || !isValidVersionID(object.VersionId) {
		return s3err.ErrInvalidRequest
	}
	return s3err.ErrNone
}

// isMissingDeleteConditionTarget normalizes missing-target detection for conditional deletes.
// Prefer errors.Is(err, filer_pb.ErrNotFound) and errors.Is(err, ErrDeleteMarker); keep the
// string-based fallback only as a defensive bridge for filer paths that still return plain text.
func isMissingDeleteConditionTarget(err error) bool {
	if err == nil {
		return false
	}
	if errors.Is(err, filer_pb.ErrNotFound) || errors.Is(err, ErrDeleteMarker) {
		return true
	}

	lowerErr := strings.ToLower(err.Error())
	return strings.Contains(lowerErr, "not found")
}

func (s3a *S3ApiServer) resolveDeleteConditionalEntry(bucket, object, versionId, versioningState string) (*filer_pb.Entry, error) {
	if !isValidVersionID(versionId) {
		return nil, errInvalidVersionID
	}
	normalizedObject := s3_constants.NormalizeObjectKey(object)

	if versionId != "" {
		if versionId == "null" {
			return s3a.getObjectEntryRoutedByKey(bucket, normalizedObject)
		}
		return s3a.getObjectScopedEntryRoutedByKey(bucket, normalizedObject, s3a.getVersionedObjectDir(bucket, normalizedObject), s3a.getVersionFileName(versionId))
	}

	switch versioningState {
	case s3_constants.VersioningEnabled:
		entry, err := s3a.getLatestObjectVersion(bucket, normalizedObject)
		if err != nil {
			return nil, err
		}
		return normalizeConditionalTargetEntry(entry), nil
	default:
		entry, err := s3a.resolveObjectEntry(bucket, normalizedObject, "")
		if err != nil {
			return nil, err
		}
		return normalizeConditionalTargetEntry(entry), nil
	}
}

func (s3a *S3ApiServer) validateDeleteIfMatch(entry *filer_pb.Entry, ifMatch string, missingCode s3err.ErrorCode) s3err.ErrorCode {
	if ifMatch == "" {
		return s3err.ErrNone
	}
	if entry == nil {
		return missingCode
	}
	if ifMatch == "*" {
		return s3err.ErrNone
	}
	if !s3a.etagMatches(ifMatch, s3a.getObjectETag(entry)) {
		return s3err.ErrPreconditionFailed
	}
	return s3err.ErrNone
}

func (s3a *S3ApiServer) checkDeleteIfMatch(bucket, object, versionId, versioningState, ifMatch string, missingCode s3err.ErrorCode) s3err.ErrorCode {
	if ifMatch == "" {
		return s3err.ErrNone
	}

	entry, err := s3a.resolveDeleteConditionalEntry(bucket, object, versionId, versioningState)
	if err != nil {
		if isMissingDeleteConditionTarget(err) {
			return missingCode
		}
		glog.Errorf("checkDeleteIfMatch: failed to resolve %s/%s (versionId=%s): %v", bucket, object, versionId, err)
		return s3err.ErrInternalError
	}

	return s3a.validateDeleteIfMatch(entry, ifMatch, missingCode)
}

func (s3a *S3ApiServer) deleteVersionedObject(r *http.Request, bucket, object, versionId, versioningState string) (deleteMutationResult, s3err.ErrorCode) {
	var result deleteMutationResult

	// The key "dir/" is the filer directory itself, which a delete marker cannot stand
	// in for without hiding the children underneath it. It is not a versioned object,
	// so it is deleted the way an unversioned bucket deletes it.
	if versionId == "" && strings.HasSuffix(object, "/") {
		return result, s3a.deleteDirectoryMarker(r, bucket, object)
	}

	switch {
	case versionId != "":
		versionEntry, versionLookupErr := s3a.getSpecificObjectVersion(bucket, object, versionId)
		if versionLookupErr == nil && versionEntry != nil && versionEntry.Extended != nil {
			if deleteMarker, ok := versionEntry.Extended[s3_constants.ExtDeleteMarkerKey]; ok && string(deleteMarker) == "true" {
				result.deleteMarker = true
			}
		}
		governanceBypassAllowed := s3a.evaluateGovernanceBypassRequest(r, bucket, object)
		if err := s3a.enforceObjectLockProtections(r, bucket, object, versionId, governanceBypassAllowed); err != nil {
			glog.V(2).Infof("deleteVersionedObject: object lock check failed for %s/%s version %s: %v", bucket, object, versionId, err)
			return result, s3err.ErrAccessDenied
		}
		if err := s3a.deleteSpecificObjectVersion(r.Context(), bucket, object, versionId, false); err != nil {
			glog.Errorf("deleteVersionedObject: failed to delete specific version %s for %s/%s: %v", versionId, bucket, object, err)
			return result, s3err.ErrInternalError
		}
		result.versionId = versionId
		return result, s3err.ErrNone

	case versioningState == s3_constants.VersioningEnabled:
		deleteMarkerVersionId, err := s3a.createDeleteMarker(bucket, object)
		if err != nil {
			glog.Errorf("deleteVersionedObject: failed to create delete marker for %s/%s: %v", bucket, object, err)
			return result, s3err.ErrInternalError
		}
		result.versionId = deleteMarkerVersionId
		result.deleteMarker = true
		return result, s3err.ErrNone

	case versioningState == s3_constants.VersioningSuspended:
		governanceBypassAllowed := s3a.evaluateGovernanceBypassRequest(r, bucket, object)
		if err := s3a.enforceObjectLockProtections(r, bucket, object, "null", governanceBypassAllowed); err != nil {
			glog.V(2).Infof("deleteVersionedObject: object lock check failed for %s/%s null version: %v", bucket, object, err)
			return result, s3err.ErrAccessDenied
		}
		// Delete the null version on the owner the routed reads consult: on
		// the default filer an absent entry is a successful delete, so a null
		// version that only reached the owner would survive a reported
		// success. An unsettled routed attempt falls back to the lock path.
		nullDeleted := false
		if owner := s3a.routableWriteOwner(bucket, object); owner != "" {
			worm, lockErr := s3a.isObjectLockEnabled(bucket)
			if lockErr == nil {
				code, settled := s3a.routedDeleteNullVersion(owner, bucket, object, worm, worm && governanceBypassAllowed, false, nil)
				if settled {
					if code != s3err.ErrNone {
						return result, code
					}
					nullDeleted = true
				}
			}
		}
		if !nullDeleted {
			if err := s3a.deleteSpecificObjectVersion(r.Context(), bucket, object, "null", false); err != nil {
				glog.Errorf("deleteVersionedObject: failed to delete null version for %s/%s: %v", bucket, object, err)
				return result, s3err.ErrInternalError
			}
		}
		// Suspended versioning overwrites the null version with a single null delete
		// marker (S3 spec), so the marker replaces any prior one instead of piling up.
		if err := s3a.createNullDeleteMarker(bucket, object); err != nil {
			glog.Errorf("deleteVersionedObject: failed to create null delete marker for suspended versioning %s/%s: %v", bucket, object, err)
			return result, s3err.ErrInternalError
		}
		result.versionId = "null"
		result.deleteMarker = true
		return result, s3err.ErrNone
	}

	glog.Errorf("deleteVersionedObject: unsupported versioning state %q for %s/%s", versioningState, bucket, object)
	return result, s3err.ErrInternalError
}

// deleteUnversionedObjectWithClient removes the bare object entry. When
// metadataOnly is true the filer skips per-chunk DeleteFile RPCs and
// relies on the volume's natural TTL to reclaim chunks; pass true only
// when the entry's Attributes.TtlSec > 0 so the volume is guaranteed to
// drop the chunks on its own.
func (s3a *S3ApiServer) deleteUnversionedObjectWithClient(ctx context.Context, client filer_pb.SeaweedFilerClient, bucket, object string, metadataOnly bool) error {
	if !s3_constants.IsValidBucketName(bucket) || !s3_constants.IsValidObjectKey(object) {
		return errors.New("invalid bucket or object path")
	}
	target := util.NewFullPath(s3a.bucketDir(bucket), object)
	dir, name := target.DirAndName()
	// The caller holds one client for a whole batch, so a dropped reply is
	// replayed on that client rather than by re-entering WithFilerClient.
	return retryFilerOp(ctx, "delete "+string(target), func() error {
		return deleteObjectEntry(ctx, client, dir, name, !metadataOnly, false)
	})
}

func (s3a *S3ApiServer) DeleteObjectHandler(w http.ResponseWriter, r *http.Request) {

	bucket, object := s3_constants.GetBucketAndObject(r)
	glog.Infof("DeleteObjectHandler %s %s", bucket, object)
	// The filer ops below each retry, and a failover walk runs the whole set
	// once per filer, so the backoff comes out of one allowance held here.
	r = r.WithContext(withFilerRetryBudget(r.Context(), filerRetryRequestBudget))
	if _, err := s3a.validateTableBucketObjectPathOrVerify(bucket, object); err != nil {
		s3err.WriteErrorResponse(w, r, s3err.ErrAccessDenied)
		return
	}

	// Check for specific version ID in query parameters
	versionId := r.URL.Query().Get("versionId")

	// Get detailed versioning state for proper handling of suspended vs enabled versioning
	versioningState, err := s3a.getVersioningState(bucket)
	if err != nil {
		if err == filer_pb.ErrNotFound {
			s3err.WriteErrorResponse(w, r, s3err.ErrNoSuchBucket)
			return
		}
		glog.Errorf("Error checking versioning status for bucket %s: %v", bucket, err)
		s3err.WriteErrorResponse(w, r, s3err.ErrInternalError)
		return
	}

	versioningConfigured := (versioningState != "")

	if errCode := s3a.checkVersionIdConfigured(bucket, versionId); errCode != s3err.ErrNone {
		s3err.WriteErrorResponse(w, r, errCode)
		return
	}

	if errCode := s3a.recheckTagConditionedPolicyWithLookup(r, bucket, object, versionId, string(s3_constants.ACTION_WRITE), "DeleteObjectHandler"); errCode != s3err.ErrNone {
		s3err.WriteErrorResponse(w, r, errCode)
		return
	}

	var auditLog *s3err.AccessLog
	if s3err.Logger != nil {
		auditLog = s3err.GetAccessLog(r, http.StatusNoContent, s3err.ErrNone)
	}

	if ifMatchResult := s3a.checkDeleteIfMatch(bucket, object, versionId, versioningState, r.Header.Get(s3_constants.IfMatch), s3err.ErrPreconditionFailed); ifMatchResult != s3err.ErrNone {
		s3err.WriteErrorResponse(w, r, ifMatchResult)
		return
	}

	var deleteResult deleteMutationResult
	var deleteCode s3err.ErrorCode

	// A trailing-slash key is a directory marker in every bucket, versioned or not, and
	// is deleted the same way: the raw delete below cannot handle a directory that still
	// has children, and versioning has nothing to add to a key that is not an object.
	deleteHandled := false
	if versionId == "" && strings.HasSuffix(object, "/") {
		deleteCode, deleteHandled = s3a.withObjectWriteLock(bucket, object, func() s3err.ErrorCode {
			return s3a.checkDeleteIfMatch(bucket, object, versionId, versioningState, r.Header.Get(s3_constants.IfMatch), s3err.ErrPreconditionFailed)
		}, func() s3err.ErrorCode {
			return s3a.deleteDirectoryMarker(r, bucket, object)
		}), true
	}

	// Fast path: route the delete to the owner filer under its per-path lock;
	// routedObjectOwner excludes versioned/object-lock buckets.
	if !deleteHandled && !versioningConfigured {
		if owner, ownerOk := s3a.routedObjectOwner(bucket, object); ownerOk {
			resp, err := s3a.routedDelete(owner, bucket, object, buildDeleteCondition(r))
			switch {
			case err != nil:
				glog.Warningf("DeleteObjectHandler: routed delete to %s failed for %s/%s, falling back to lock: %v", owner, bucket, object, err)
			case resp.ErrorCode == filer_pb.FilerError_PRECONDITION_FAILED:
				deleteCode, deleteHandled = s3err.ErrPreconditionFailed, true
			case resp.Error != "":
				// Non-precondition error: fall back (the lock path handles cases
				// the raw delete cannot, e.g. a non-empty directory marker).
				glog.Warningf("DeleteObjectHandler: routed delete to %s returned %q for %s/%s, falling back to lock", owner, resp.Error, bucket, object)
			default:
				deleteCode, deleteHandled = s3err.ErrNone, true
			}
		}
	}
	// Versioned/suspended delete with no specific version: route off the lock when
	// the bucket has an owner. createDeleteMarker routes its own pointer flip; a
	// delete marker never removes a locked version, so object-lock buckets route
	// here too. A conditional delete stays on the lock path, which re-checks
	// If-Match under the lock, so concurrent ones cannot all pass the check above.
	if !deleteHandled && versioningConfigured && versionId == "" && r.Header.Get(s3_constants.IfMatch) == "" {
		if owner := s3a.routableWriteOwner(bucket, object); owner != "" {
			deleteResult, deleteCode = s3a.deleteVersionedObject(r, bucket, object, versionId, versioningState)
			deleteHandled = true
		}
	}
	// Specific-version delete: route off the lock. A real version recomputes the
	// .versions pointer excluding it and deletes the version file; the null
	// version is the regular object entry, deleted directly. Object-lock buckets
	// gate the delete on the version's WORM guards, evaluated on the owner — for
	// governance bypass the retention guard is scoped to COMPLIANCE so the filer
	// allows a governance-mode delete while still denying compliance and legal
	// hold, without the gateway reading the version. If-Match rides the same
	// transaction, so concurrent conditional deletes of one version cannot both
	// pass.
	if !deleteHandled && versionId != "" {
		worm, lockErr := s3a.isObjectLockEnabled(bucket)
		bypass := worm && s3a.evaluateGovernanceBypassRequest(r, bucket, object)
		ifMatch := buildDeleteCondition(r)
		if lockErr == nil {
			if owner := s3a.objectWriteOwner(bucket, object); owner != "" {
				deleteResult.versionId = versionId
				if ve, vErr := s3a.getSpecificObjectVersion(bucket, object, versionId); vErr == nil && ve != nil && ve.Extended != nil {
					if dm, ok := ve.Extended[s3_constants.ExtDeleteMarkerKey]; ok && string(dm) == "true" {
						deleteResult.deleteMarker = true
					}
				}
				if versionId == "null" {
					deleteCode, deleteHandled = s3a.routedDeleteNullVersion(owner, bucket, object, worm, bypass, false, ifMatch)
				} else {
					// A version id never changes kind, so a marker read here is
					// the one deleted; markers carry no WORM guard.
					deleteCode, deleteHandled = s3a.routedDeleteSpecificVersion(owner, bucket, object, versionId, worm && !deleteResult.deleteMarker, bypass, false, ifMatch), true
				}
			}
		}
	}
	if !deleteHandled {
		deleteCode = s3a.withObjectWriteLock(bucket, object, func() s3err.ErrorCode {
			return s3a.checkDeleteIfMatch(bucket, object, versionId, versioningState, r.Header.Get(s3_constants.IfMatch), s3err.ErrPreconditionFailed)
		}, func() s3err.ErrorCode {
			if versioningConfigured {
				result, errCode := s3a.deleteVersionedObject(r, bucket, object, versionId, versioningState)
				if errCode != s3err.ErrNone {
					return errCode
				}
				deleteResult = result
				return s3err.ErrNone
			}

			governanceBypassAllowed := s3a.evaluateGovernanceBypassRequest(r, bucket, object)
			if err := s3a.enforceObjectLockProtections(r, bucket, object, "", governanceBypassAllowed); err != nil {
				glog.V(2).Infof("DeleteObjectHandler: object lock check failed for %s/%s: %v", bucket, object, err)
				return s3err.ErrAccessDenied
			}

			if err := s3a.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
				return s3a.deleteUnversionedObjectWithClient(r.Context(), client, bucket, object, false)
			}); err != nil {
				glog.Errorf("DeleteObjectHandler: failed to delete %s/%s: %v", bucket, object, err)
				return s3err.ErrInternalError
			}

			return s3err.ErrNone
		})
	}
	if deleteCode != s3err.ErrNone {
		s3err.WriteErrorResponse(w, r, deleteCode)
		return
	}

	if deleteResult.versionId != "" {
		w.Header().Set("x-amz-version-id", deleteResult.versionId)
	}
	if deleteResult.deleteMarker {
		w.Header().Set("x-amz-delete-marker", "true")
	}

	if auditLog != nil {
		auditLog.Key = strings.TrimPrefix(object, "/")
		s3err.PostAccessLog(*auditLog)
		s3err.MarkAuditLogged(r)
	}

	stats_collect.RecordBucketActiveTime(bucket)
	stats_collect.S3DeletedObjectsCounter.WithLabelValues(bucket).Inc()
	w.WriteHeader(http.StatusNoContent)
}

// ObjectIdentifier represents an object to be deleted with its key name and optional version ID.
type ObjectIdentifier struct {
	Key                   string `xml:"Key"`
	VersionId             string `xml:"VersionId,omitempty"`
	ETag                  string `xml:"ETag,omitempty"`
	DeleteMarker          bool   `xml:"DeleteMarker,omitempty"`
	DeleteMarkerVersionId string `xml:"DeleteMarkerVersionId,omitempty"`
}

// DeleteObjectsRequest - xml carrying the object key names which needs to be deleted.
type DeleteObjectsRequest struct {
	// Element to enable quiet mode for the request
	Quiet bool
	// List of objects to be deleted
	Objects []ObjectIdentifier `xml:"Object"`
}

// DeleteError structure.
type DeleteError struct {
	Code      string `xml:"Code"`
	Message   string `xml:"Message"`
	Key       string `xml:"Key"`
	VersionId string `xml:"VersionId,omitempty"`
}

// DeleteObjectsResponse container for multiple object deletes.
type DeleteObjectsResponse struct {
	XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ DeleteResult" json:"-"`

	// Collection of all deleted objects
	DeletedObjects []ObjectIdentifier `xml:"Deleted,omitempty"`

	// Collection of errors deleting certain objects.
	Errors []DeleteError `xml:"Error,omitempty"`
}

// DeleteMultipleObjectsHandler - Delete multiple objects
func (s3a *S3ApiServer) DeleteMultipleObjectsHandler(w http.ResponseWriter, r *http.Request) {

	bucket, _ := s3_constants.GetBucketAndObject(r)
	glog.V(3).Infof("DeleteMultipleObjectsHandler %s", bucket)

	deleteXMLBytes, err := io.ReadAll(r.Body)
	if err != nil {
		s3err.WriteErrorResponse(w, r, requestBodyErrorCode(err, s3err.ErrInternalError))
		return
	}

	deleteObjects := &DeleteObjectsRequest{}
	if err := xml.Unmarshal(deleteXMLBytes, deleteObjects); err != nil {
		s3err.WriteErrorResponse(w, r, s3err.ErrMalformedXML)
		return
	}

	if len(deleteObjects.Objects) > deleteMultipleObjectsLimit {
		s3err.WriteErrorResponse(w, r, s3err.ErrMalformedXML)
		return
	}

	var deletedObjects []ObjectIdentifier
	var deleteErrors []DeleteError
	var auditLog *s3err.AccessLog

	if s3err.Logger != nil {
		auditLog = s3err.GetAccessLog(r, http.StatusNoContent, s3err.ErrNone)
	}

	// Get detailed versioning state for proper handling of suspended vs enabled versioning
	versioningState, err := s3a.getVersioningState(bucket)
	if err != nil {
		if err == filer_pb.ErrNotFound {
			s3err.WriteErrorResponse(w, r, s3err.ErrNoSuchBucket)
			return
		}
		glog.Errorf("Error checking versioning status for bucket %s: %v", bucket, err)
		s3err.WriteErrorResponse(w, r, s3err.ErrInternalError)
		return
	}

	versioningConfigured := (versioningState != "")
	deletedCount := 0

	// Per-key authorization: keys arrive in the body, so the route Auth middleware
	// only authenticated. Authorize each key via AuthorizeObjectDelete below.
	var identity *Identity
	if id := s3_constants.GetIdentityFromContext(r); id != nil {
		identity, _ = id.(*Identity)
	}

	// The keys below each drive their own bounded filer retries, and the client
	// picks how many keys there are, so the whole batch shares one allowance.
	r = r.WithContext(withFilerRetryBudget(r.Context(), filerRetryRequestBudget))

	err = s3a.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
		// delete file entries
		for _, object := range deleteObjects.Objects {
			if object.Key == "" {
				continue
			}
			if validationCode := validateDeleteObjectIdentifier(object); validationCode != s3err.ErrNone {
				deleteErrors = append(deleteErrors, deleteErrorFromCode(validationCode, object.Key, object.VersionId))
				continue
			}
			key := s3_constants.NormalizeObjectKey(object.Key)
			if key == "" {
				// A key of only slashes can never name a stored entry; like a
				// missing key the delete is an idempotent no-op, and a
				// conditional delete has no target to match.
				if object.ETag != "" {
					deleteErrors = append(deleteErrors, deleteErrorFromCode(s3err.ErrNoSuchKey, object.Key, object.VersionId))
					continue
				}
				deletedCount++
				if !deleteObjects.Quiet {
					deletedObjects = append(deletedObjects, ObjectIdentifier{Key: object.Key, VersionId: object.VersionId})
				}
				continue
			}
			if _, err := s3a.validateTableBucketObjectPathOrVerify(bucket, key); err != nil {
				deleteErrors = append(deleteErrors, deleteErrorFromCode(s3err.ErrAccessDenied, object.Key, object.VersionId))
				continue
			}
			if authErr := s3a.iam.AuthorizeObjectDelete(r, identity, bucket, key, object.VersionId); authErr != s3err.ErrNone {
				// Defer to the recheck, mirroring the copy-source fallback: a
				// tag-conditioned Allow may still grant what phase one denied.
				if authErr != s3err.ErrAccessDenied || s3a.recheckPolicyForKeyWithLookup(r, http.MethodDelete, bucket, key, object.VersionId, string(s3_constants.ACTION_WRITE), "DeleteMultipleObjectsHandler") != s3err.ErrNone {
					deleteErrors = append(deleteErrors, deleteErrorFromCode(authErr, object.Key, object.VersionId))
					continue
				}
			}
			if errCode := s3a.recheckTagConditionedPolicyWithLookup(objectKeyPolicyRequest(r, http.MethodDelete, bucket, key, object.VersionId), bucket, key, object.VersionId, string(s3_constants.ACTION_WRITE), "DeleteMultipleObjectsHandler"); errCode != s3err.ErrNone {
				deleteErrors = append(deleteErrors, deleteErrorFromCode(errCode, object.Key, object.VersionId))
				continue
			}

			var deleteResult deleteMutationResult
			deleteCode := s3a.withObjectWriteLock(bucket, key, func() s3err.ErrorCode {
				return s3a.checkDeleteIfMatch(bucket, key, object.VersionId, versioningState, object.ETag, s3err.ErrNoSuchKey)
			}, func() s3err.ErrorCode {
				if versioningConfigured {
					result, errCode := s3a.deleteVersionedObject(r, bucket, key, object.VersionId, versioningState)
					if errCode != s3err.ErrNone {
						return errCode
					}
					deleteResult = result
					return s3err.ErrNone
				}

				governanceBypassAllowed := s3a.evaluateGovernanceBypassRequest(r, bucket, key)
				if err := s3a.enforceObjectLockProtections(r, bucket, key, "", governanceBypassAllowed); err != nil {
					glog.V(2).Infof("DeleteMultipleObjectsHandler: object lock check failed for %s/%s: %v", bucket, key, err)
					return s3err.ErrAccessDenied
				}

				if strings.HasSuffix(key, "/") {
					return s3a.deleteDirectoryMarker(r, bucket, key)
				}

				if err := s3a.deleteUnversionedObjectWithClient(r.Context(), client, bucket, key, false); err != nil {
					glog.Errorf("DeleteMultipleObjectsHandler: failed to delete %s/%s: %v", bucket, key, err)
					return s3err.ErrInternalError
				}

				return s3err.ErrNone
			})
			if deleteCode != s3err.ErrNone {
				deleteErrors = append(deleteErrors, deleteErrorFromCode(deleteCode, object.Key, object.VersionId))
				continue
			}

			deletedCount++
			if !deleteObjects.Quiet {
				deletedObject := ObjectIdentifier{
					Key:       object.Key,
					VersionId: deleteResult.versionId,
				}
				if deleteResult.deleteMarker {
					deletedObject.DeleteMarker = true
					deletedObject.DeleteMarkerVersionId = deleteResult.versionId
					deletedObject.VersionId = ""
				}
				deletedObjects = append(deletedObjects, deletedObject)
			}

			if auditLog != nil {
				auditLog.Key = object.Key
				s3err.PostAccessLog(*auditLog)
				s3err.MarkAuditLogged(r)
			}
		}

		// Note: Empty folder cleanup is now handled asynchronously by EmptyFolderCleaner
		// which listens to metadata events and uses consistent hashing for coordination

		return nil
	})
	if err != nil {
		glog.Errorf("DeleteMultipleObjectsHandler: failed to initialize filer client for bucket %s: %v", bucket, err)
		s3err.WriteErrorResponse(w, r, s3err.ErrInternalError)
		return
	}

	deleteResp := DeleteObjectsResponse{}
	if !deleteObjects.Quiet {
		deleteResp.DeletedObjects = deletedObjects
	}
	deleteResp.Errors = deleteErrors
	stats_collect.RecordBucketActiveTime(bucket)
	stats_collect.S3DeletedObjectsCounter.WithLabelValues(bucket).Add(float64(deletedCount))

	writeSuccessResponseXML(w, r, deleteResp)

}