"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files
Go azure-blobazure-blob-storageazure-filesbackblaze-b2cloud-storagedropboxencryptionftpfuse-filesystemgogolanggoogle-cloud-storagegoogle-driveonedriveopenstack-swiftrclones3sftpsyncwebdav
s3: add Object Lock support
Chris7d0a8bf850b9
diff --git a/backend/s3/s3.go b/backend/s3/s3.goindex b5bba8327..02398a34f 100644--- a/backend/s3/s3.go+++ b/backend/s3/s3.go@@ -828,6 +828,107 @@ use |-vv| to see the debug level logs.}, {Name: "ibm_resource_instance_id",Help: "IBM service instance id",+ }, {+ Name: "object_lock_mode",+ Help: `Object Lock mode to apply when uploading or copying objects.++Set this to apply Object Lock retention mode to objects.+If not set, no Object Lock mode is applied (even with --metadata).++Note: To enable Object Lock retention, you must set BOTH object_lock_mode+AND object_lock_retain_until_date. Setting only one has no effect.++- GOVERNANCE: Set Object Lock mode to GOVERNANCE+- COMPLIANCE: Set Object Lock mode to COMPLIANCE+- copy: Copy the mode from the source object (requires --metadata)++See: https://docs.aws.amazon.com/AmazonS3/latest/userguide/object-lock.html`,+ Advanced: true,+ Examples: []fs.OptionExample{{+ Value: "GOVERNANCE",+ Help: "Set Object Lock mode to GOVERNANCE",+ }, {+ Value: "COMPLIANCE",+ Help: "Set Object Lock mode to COMPLIANCE",+ }, {+ Value: "copy",+ Help: "Copy from source object (requires --metadata)",+ }},+ }, {+ Name: "object_lock_retain_until_date",+ Help: `Object Lock retention until date to apply when uploading or copying objects.++Set this to apply Object Lock retention date to objects.+If not set, no retention date is applied (even with --metadata).++Note: To enable Object Lock retention, you must set BOTH object_lock_mode+AND object_lock_retain_until_date. Setting only one has no effect.++Accepts:+- RFC 3339 format: 2030-01-02T15:04:05Z+- Duration from now: 365d, 1y, 6M (days, years, months)+- copy: Copy the date from the source object (requires --metadata)++See: https://docs.aws.amazon.com/AmazonS3/latest/userguide/object-lock.html`,+ Advanced: true,+ Examples: []fs.OptionExample{{+ Value: "copy",+ Help: "Copy from source object (requires --metadata)",+ }, {+ Value: "2030-01-01T00:00:00Z",+ Help: "Set specific date (RFC 3339 format)",+ }, {+ Value: "365d",+ Help: "Set retention for 365 days from now",+ }, {+ Value: "1y",+ Help: "Set retention for 1 year from now",+ }},+ }, {+ Name: "object_lock_legal_hold_status",+ Help: `Object Lock legal hold status to apply when uploading or copying objects.++Set this to apply Object Lock legal hold to objects.+If not set, no legal hold is applied (even with --metadata).++Note: Legal hold is independent of retention and can be set separately.++- ON: Enable legal hold+- OFF: Disable legal hold+- copy: Copy the legal hold status from the source object (requires --metadata)++See: https://docs.aws.amazon.com/AmazonS3/latest/userguide/object-lock.html`,+ Advanced: true,+ Examples: []fs.OptionExample{{+ Value: "ON",+ Help: "Enable legal hold",+ }, {+ Value: "OFF",+ Help: "Disable legal hold",+ }, {+ Value: "copy",+ Help: "Copy from source object (requires --metadata)",+ }},+ }, {+ Name: "bypass_governance_retention",+ Help: `Allow deleting or modifying objects locked with GOVERNANCE mode.`,+ Default: false,+ Advanced: true,+ }, {+ Name: "bucket_object_lock_enabled",+ Help: `Enable Object Lock when creating new buckets.`,+ Default: false,+ }, {+ Name: "object_lock_set_after_upload",+ Help: `Set Object Lock via separate API calls after upload.++Use this for S3-compatible providers that don't support setting Object Lock+headers during PUT operations. When enabled, Object Lock is set via separate+PutObjectRetention and PutObjectLegalHold API calls after the upload completes.++This adds extra API calls per object, so only enable if your provider requires it.`,+ Default: false,+ Advanced: true,},}}))}@@ -922,6 +1023,21 @@ var systemMetadataInfo = map[string]fs.MetadataHelp{Example: "2006-01-02T15:04:05.999999999Z07:00",ReadOnly: true,},+ "object-lock-mode": {+ Help: "Object Lock mode: GOVERNANCE or COMPLIANCE",+ Type: "string",+ Example: "GOVERNANCE",+ },+ "object-lock-retain-until-date": {+ Help: "Object Lock retention until date",+ Type: "RFC 3339",+ Example: "2030-01-02T15:04:05Z",+ },+ "object-lock-legal-hold-status": {+ Help: "Object Lock legal hold status: ON or OFF",+ Type: "string",+ Example: "OFF",+ },}// Options defines the configuration for this backend@@ -992,6 +1108,12 @@ type Options struct {IBMInstanceID string `config:"ibm_resource_instance_id"`UseXID fs.Tristate `config:"use_x_id"`SignAcceptEncoding fs.Tristate `config:"sign_accept_encoding"`+ ObjectLockMode string `config:"object_lock_mode"`+ ObjectLockRetainUntilDate string `config:"object_lock_retain_until_date"`+ ObjectLockLegalHoldStatus string `config:"object_lock_legal_hold_status"`+ BypassGovernanceRetention bool `config:"bypass_governance_retention"`+ BucketObjectLockEnabled bool `config:"bucket_object_lock_enabled"`+ ObjectLockSetAfterUpload bool `config:"object_lock_set_after_upload"`}// Fs represents a remote s3 server@@ -1036,6 +1158,11 @@ type Object struct {contentDisposition *string // Content-Disposition: headercontentEncoding *string // Content-Encoding: headercontentLanguage *string // Content-Language: header++ // Object Lock metadata+ objectLockMode *string // Object Lock mode: GOVERNANCE or COMPLIANCE+ objectLockRetainUntilDate *time.Time // Object Lock retention until date+ objectLockLegalHoldStatus *string // Object Lock legal hold: ON or OFF}// safely dereference the pointer, returning a zero T if nil@@ -1056,6 +1183,21 @@ func getHTTPStatusCode(err error) int {return -1}+// parseRetainUntilDate parses a retain until date from a string.+// It accepts RFC 3339 format or duration strings like "365d", "1y", "6m".+func parseRetainUntilDate(s string) (time.Time, error) {+ // First try RFC 3339 format+ if t, err := time.Parse(time.RFC3339, s); err == nil {+ return t, nil+ }+ // Try as a duration from now+ d, err := fs.ParseDuration(s)+ if err != nil {+ return time.Time{}, fmt.Errorf("can't parse %q as RFC 3339 date or duration: %w", s, err)+ }+ return time.Now().Add(d), nil+}+// ------------------------------------------------------------// Name of the remote (as passed into NewFs)@@ -2649,8 +2791,9 @@ func (f *Fs) makeBucket(ctx context.Context, bucket string) error {}return f.cache.Create(bucket, func() error {req := s3.CreateBucketInput{- Bucket: &bucket,- ACL: types.BucketCannedACL(f.opt.BucketACL),+ Bucket: &bucket,+ ACL: types.BucketCannedACL(f.opt.BucketACL),+ ObjectLockEnabledForBucket: &f.opt.BucketObjectLockEnabled,}if f.opt.LocationConstraint != "" {req.CreateBucketConfiguration = &types.CreateBucketConfiguration{@@ -2769,6 +2912,24 @@ func (f *Fs) copy(ctx context.Context, req *s3.CopyObjectInput, dstBucket, dstPareq.StorageClass = types.StorageClass(f.opt.StorageClass)}+ // Apply Object Lock options via headers (unless ObjectLockSetAfterUpload is set)+ // "copy" means: keep the value from source (passed via req from prepareUpload/setFrom functions)+ if !f.opt.ObjectLockSetAfterUpload {+ if f.opt.ObjectLockMode != "" && !strings.EqualFold(f.opt.ObjectLockMode, "copy") {+ req.ObjectLockMode = types.ObjectLockMode(strings.ToUpper(f.opt.ObjectLockMode))+ }+ if f.opt.ObjectLockRetainUntilDate != "" && !strings.EqualFold(f.opt.ObjectLockRetainUntilDate, "copy") {+ retainDate, err := parseRetainUntilDate(f.opt.ObjectLockRetainUntilDate)+ if err != nil {+ return fmt.Errorf("invalid object_lock_retain_until_date: %w", err)+ }+ req.ObjectLockRetainUntilDate = &retainDate+ }+ if f.opt.ObjectLockLegalHoldStatus != "" && !strings.EqualFold(f.opt.ObjectLockLegalHoldStatus, "copy") {+ req.ObjectLockLegalHoldStatus = types.ObjectLockLegalHoldStatus(strings.ToUpper(f.opt.ObjectLockLegalHoldStatus))+ }+ }+if src.bytes >= int64(f.opt.CopyCutoff) {return f.copyMultipart(ctx, req, dstBucket, dstPath, srcBucket, srcPath, src)}@@ -2951,7 +3112,15 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object,}setFrom_s3CopyObjectInput_s3PutObjectInput(&req, ui.req)- if ci.Metadata {+ // Use REPLACE directive if metadata is being modified, otherwise S3 ignores our values+ // This is needed when:+ // 1. --metadata flag is set+ // 2. Any Object Lock option is set (to override or explicitly copy)+ needsReplace := ci.Metadata ||+ f.opt.ObjectLockMode != "" ||+ f.opt.ObjectLockRetainUntilDate != "" ||+ f.opt.ObjectLockLegalHoldStatus != ""+ if needsReplace {req.MetadataDirective = types.MetadataDirectiveReplace}@@ -2959,7 +3128,21 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object,if err != nil {return nil, err}- return f.NewObject(ctx, remote)+ dstObj, err := f.NewObject(ctx, remote)+ if err != nil {+ return nil, err+ }++ // Set Object Lock via separate API calls if requested+ if f.opt.ObjectLockSetAfterUpload {+ if dstObject, ok := dstObj.(*Object); ok {+ if err := dstObject.setObjectLockAfterUpload(ctx, srcObj); err != nil {+ return nil, err+ }+ }+ }++ return dstObj, nil}// Hashes returns the supported hash sets.@@ -3838,6 +4021,19 @@ func (o *Object) setMetaData(resp *s3.HeadObjectOutput) {o.contentEncoding = stringClonePointer(removeAWSChunked(resp.ContentEncoding))o.contentLanguage = stringClonePointer(resp.ContentLanguage)+ // Set Object Lock metadata+ if resp.ObjectLockMode != "" {+ mode := string(resp.ObjectLockMode)+ o.objectLockMode = &mode+ }+ if resp.ObjectLockRetainUntilDate != nil {+ o.objectLockRetainUntilDate = resp.ObjectLockRetainUntilDate+ }+ if resp.ObjectLockLegalHoldStatus != "" {+ status := string(resp.ObjectLockLegalHoldStatus)+ o.objectLockLegalHoldStatus = &status+ }+// If decompressing then size and md5sum are unknownif o.fs.opt.Decompress && deref(o.contentEncoding) == "gzip" {o.bytes = -1@@ -4556,6 +4752,26 @@ func (o *Object) prepareUpload(ctx context.Context, src fs.ObjectInfo, options [case "btime":// write as metadata since we can't set itui.req.Metadata[k] = v+ case "object-lock-mode":+ // Only apply if option is set to "copy" and not using after-upload API+ if strings.EqualFold(o.fs.opt.ObjectLockMode, "copy") && !o.fs.opt.ObjectLockSetAfterUpload {+ ui.req.ObjectLockMode = types.ObjectLockMode(v)+ }+ case "object-lock-retain-until-date":+ // Only apply if option is set to "copy" and not using after-upload API+ if strings.EqualFold(o.fs.opt.ObjectLockRetainUntilDate, "copy") && !o.fs.opt.ObjectLockSetAfterUpload {+ retainDate, err := time.Parse(time.RFC3339, v)+ if err != nil {+ fs.Debugf(o, "failed to parse object-lock-retain-until-date %q: %v", v, err)+ } else {+ ui.req.ObjectLockRetainUntilDate = &retainDate+ }+ }+ case "object-lock-legal-hold-status":+ // Only apply if option is set to "copy" and not using after-upload API+ if strings.EqualFold(o.fs.opt.ObjectLockLegalHoldStatus, "copy") && !o.fs.opt.ObjectLockSetAfterUpload {+ ui.req.ObjectLockLegalHoldStatus = types.ObjectLockLegalHoldStatus(v)+ }default:ui.req.Metadata[k] = v}@@ -4621,6 +4837,25 @@ func (o *Object) prepareUpload(ctx context.Context, src fs.ObjectInfo, options [if o.fs.opt.StorageClass != "" {ui.req.StorageClass = types.StorageClass(o.fs.opt.StorageClass)}++ // Apply Object Lock options via headers (unless ObjectLockSetAfterUpload is set)+ // "copy" means: keep the value from metadata (already applied above in the switch)+ if !o.fs.opt.ObjectLockSetAfterUpload {+ if o.fs.opt.ObjectLockMode != "" && !strings.EqualFold(o.fs.opt.ObjectLockMode, "copy") {+ ui.req.ObjectLockMode = types.ObjectLockMode(strings.ToUpper(o.fs.opt.ObjectLockMode))+ }+ if o.fs.opt.ObjectLockRetainUntilDate != "" && !strings.EqualFold(o.fs.opt.ObjectLockRetainUntilDate, "copy") {+ retainDate, err := parseRetainUntilDate(o.fs.opt.ObjectLockRetainUntilDate)+ if err != nil {+ return ui, fmt.Errorf("invalid object_lock_retain_until_date %q: %w", o.fs.opt.ObjectLockRetainUntilDate, err)+ }+ ui.req.ObjectLockRetainUntilDate = &retainDate+ }+ if o.fs.opt.ObjectLockLegalHoldStatus != "" && !strings.EqualFold(o.fs.opt.ObjectLockLegalHoldStatus, "copy") {+ ui.req.ObjectLockLegalHoldStatus = types.ObjectLockLegalHoldStatus(strings.ToUpper(o.fs.opt.ObjectLockLegalHoldStatus))+ }+ }+// Apply upload optionsfor _, option := range options {key, value := option.Header()@@ -4744,6 +4979,14 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op}fs.Debugf(o, "Multipart upload Etag: %s OK", wantETag)}++ // Set Object Lock via separate API calls if requested+ if o.fs.opt.ObjectLockSetAfterUpload {+ if err := o.setObjectLockAfterUpload(ctx, src); err != nil {+ return err+ }+ }+return err}@@ -4761,6 +5004,9 @@ func (o *Object) Remove(ctx context.Context) error {if o.fs.opt.RequesterPays {req.RequestPayer = types.RequestPayerRequester}+ if o.fs.opt.BypassGovernanceRetention {+ req.BypassGovernanceRetention = &o.fs.opt.BypassGovernanceRetention+ }err := o.fs.pacer.Call(func() (bool, error) {_, err := o.fs.c.DeleteObject(ctx, &req)return o.fs.shouldRetry(ctx, err)@@ -4768,6 +5014,120 @@ func (o *Object) Remove(ctx context.Context) error {return err}+// setObjectRetention sets Object Lock retention on an object via PutObjectRetention API+//+// Note: We use smithyhttp.AddContentChecksumMiddleware to ensure Content-MD5 is+// calculated for the request body. The AWS SDK v2 switched from MD5 to CRC32 as+// the default checksum algorithm, but some S3-compatible providers (e.g. MinIO)+// still require Content-MD5 for PutObjectRetention requests.+// See: https://github.com/aws/aws-sdk-go-v2/discussions/2960+func (o *Object) setObjectRetention(ctx context.Context, mode types.ObjectLockRetentionMode, retainUntilDate time.Time) error {+ bucket, bucketPath := o.split()+ req := s3.PutObjectRetentionInput{+ Bucket: &bucket,+ Key: &bucketPath,+ VersionId: o.versionID,+ Retention: &types.ObjectLockRetention{+ Mode: mode,+ RetainUntilDate: &retainUntilDate,+ },+ }+ if o.fs.opt.RequesterPays {+ req.RequestPayer = types.RequestPayerRequester+ }+ if o.fs.opt.BypassGovernanceRetention {+ req.BypassGovernanceRetention = &o.fs.opt.BypassGovernanceRetention+ }+ return o.fs.pacer.Call(func() (bool, error) {+ _, err := o.fs.c.PutObjectRetention(ctx, &req,+ s3.WithAPIOptions(smithyhttp.AddContentChecksumMiddleware))+ return o.fs.shouldRetry(ctx, err)+ })+}++// setObjectLegalHold sets Object Lock legal hold on an object via PutObjectLegalHold API+//+// Note: We use smithyhttp.AddContentChecksumMiddleware to ensure Content-MD5 is+// calculated for the request body. The AWS SDK v2 switched from MD5 to CRC32 as+// the default checksum algorithm, but some S3-compatible providers (e.g. MinIO)+// still require Content-MD5 for PutObjectLegalHold requests.+// See: https://github.com/aws/aws-sdk-go-v2/discussions/2960+func (o *Object) setObjectLegalHold(ctx context.Context, status types.ObjectLockLegalHoldStatus) error {+ bucket, bucketPath := o.split()+ req := s3.PutObjectLegalHoldInput{+ Bucket: &bucket,+ Key: &bucketPath,+ VersionId: o.versionID,+ LegalHold: &types.ObjectLockLegalHold{+ Status: status,+ },+ }+ if o.fs.opt.RequesterPays {+ req.RequestPayer = types.RequestPayerRequester+ }+ return o.fs.pacer.Call(func() (bool, error) {+ _, err := o.fs.c.PutObjectLegalHold(ctx, &req,+ s3.WithAPIOptions(smithyhttp.AddContentChecksumMiddleware))+ return o.fs.shouldRetry(ctx, err)+ })+}++// setObjectLockAfterUpload sets Object Lock via separate API calls after upload+// This is for S3 providers that don't support Object Lock headers during PUT+func (o *Object) setObjectLockAfterUpload(ctx context.Context, src fs.ObjectInfo) error {+ // Determine the mode+ var mode types.ObjectLockRetentionMode+ modeOpt := o.fs.opt.ObjectLockMode+ if strings.EqualFold(modeOpt, "copy") {+ if srcObj, ok := src.(*Object); ok && srcObj.objectLockMode != nil {+ mode = types.ObjectLockRetentionMode(*srcObj.objectLockMode)+ }+ } else if modeOpt != "" {+ mode = types.ObjectLockRetentionMode(strings.ToUpper(modeOpt))+ }++ // Determine the retain until date+ var retainUntilDate time.Time+ dateOpt := o.fs.opt.ObjectLockRetainUntilDate+ if strings.EqualFold(dateOpt, "copy") {+ if srcObj, ok := src.(*Object); ok && srcObj.objectLockRetainUntilDate != nil {+ retainUntilDate = *srcObj.objectLockRetainUntilDate+ }+ } else if dateOpt != "" {+ var err error+ retainUntilDate, err = parseRetainUntilDate(dateOpt)+ if err != nil {+ return fmt.Errorf("invalid object_lock_retain_until_date %q: %w", dateOpt, err)+ }+ }++ // Set retention if both mode and date are set+ if mode != "" && !retainUntilDate.IsZero() {+ if err := o.setObjectRetention(ctx, mode, retainUntilDate); err != nil {+ return fmt.Errorf("failed to set object retention: %w", err)+ }+ }++ // Determine and set legal hold+ var legalHold types.ObjectLockLegalHoldStatus+ legalHoldOpt := o.fs.opt.ObjectLockLegalHoldStatus+ if strings.EqualFold(legalHoldOpt, "copy") {+ if srcObj, ok := src.(*Object); ok && srcObj.objectLockLegalHoldStatus != nil {+ legalHold = types.ObjectLockLegalHoldStatus(*srcObj.objectLockLegalHoldStatus)+ }+ } else if legalHoldOpt != "" {+ legalHold = types.ObjectLockLegalHoldStatus(strings.ToUpper(legalHoldOpt))+ }++ if legalHold != "" {+ if err := o.setObjectLegalHold(ctx, legalHold); err != nil {+ return fmt.Errorf("failed to set legal hold: %w", err)+ }+ }++ return nil+}+// MimeType of an Object if known, "" otherwisefunc (o *Object) MimeType(ctx context.Context) string {err := o.readMetaData(ctx)@@ -4811,7 +5171,7 @@ func (o *Object) Metadata(ctx context.Context) (metadata fs.Metadata, err error)if err != nil {return nil, err}- metadata = make(fs.Metadata, len(o.meta)+7)+ metadata = make(fs.Metadata, len(o.meta)+10)for k, v := range o.meta {switch k {case metaMtime:@@ -4846,6 +5206,15 @@ func (o *Object) Metadata(ctx context.Context) (metadata fs.Metadata, err error)setMetadata("content-disposition", o.contentDisposition)setMetadata("content-encoding", o.contentEncoding)setMetadata("content-language", o.contentLanguage)++ // Set Object Lock metadata+ setMetadata("object-lock-mode", o.objectLockMode)+ if o.objectLockRetainUntilDate != nil {+ formatted := o.objectLockRetainUntilDate.Format(time.RFC3339)+ setMetadata("object-lock-retain-until-date", &formatted)+ }+ setMetadata("object-lock-legal-hold-status", o.objectLockLegalHoldStatus)+metadata["tier"] = o.GetTier()return metadata, nildiff --git a/backend/s3/s3_internal_test.go b/backend/s3/s3_internal_test.goindex cdca649ae..c4e6637f5 100644--- a/backend/s3/s3_internal_test.go+++ b/backend/s3/s3_internal_test.go@@ -498,10 +498,236 @@ func (f *Fs) InternalTestVersions(t *testing.T) {// Purge gets tested later}+func (f *Fs) InternalTestObjectLock(t *testing.T) {+ ctx := context.Background()++ // Create a temporary bucket with Object Lock enabled to test on.+ // This exercises our BucketObjectLockEnabled option and isolates+ // the test from the main test bucket.+ lockBucket := f.rootBucket + "-object-lock-" + random.String(8)+ lockBucket = strings.ToLower(lockBucket)++ // Try to create bucket with Object Lock enabled+ objectLockEnabled := true+ req := s3.CreateBucketInput{+ Bucket: &lockBucket,+ ACL: types.BucketCannedACL(f.opt.BucketACL),+ ObjectLockEnabledForBucket: &objectLockEnabled,+ }+ if f.opt.LocationConstraint != "" {+ req.CreateBucketConfiguration = &types.CreateBucketConfiguration{+ LocationConstraint: types.BucketLocationConstraint(f.opt.LocationConstraint),+ }+ }+ err := f.pacer.Call(func() (bool, error) {+ _, err := f.c.CreateBucket(ctx, &req)+ return f.shouldRetry(ctx, err)+ })+ if err != nil {+ t.Skipf("Object Lock not supported by this provider: CreateBucket with Object Lock failed: %v", err)+ }++ // Verify Object Lock is actually enabled on the new bucket.+ // Some S3-compatible servers (e.g. rclone serve s3) accept the+ // ObjectLockEnabledForBucket flag but don't actually implement Object Lock.+ var lockCfg *s3.GetObjectLockConfigurationOutput+ err = f.pacer.Call(func() (bool, error) {+ var err error+ lockCfg, err = f.c.GetObjectLockConfiguration(ctx, &s3.GetObjectLockConfigurationInput{+ Bucket: &lockBucket,+ })+ return f.shouldRetry(ctx, err)+ })+ if err != nil || lockCfg.ObjectLockConfiguration == nil ||+ lockCfg.ObjectLockConfiguration.ObjectLockEnabled != types.ObjectLockEnabledEnabled {+ _ = f.pacer.Call(func() (bool, error) {+ _, err := f.c.DeleteBucket(ctx, &s3.DeleteBucketInput{Bucket: &lockBucket})+ return f.shouldRetry(ctx, err)+ })+ t.Skipf("Object Lock not functional on this provider (GetObjectLockConfiguration: %v)", err)+ }++ // Switch f to use the Object Lock bucket for this test+ oldBucket := f.rootBucket+ oldRoot := f.root+ oldRootDir := f.rootDirectory+ f.rootBucket = lockBucket+ f.root = lockBucket+ f.rootDirectory = ""+ defer func() {+ f.rootBucket = oldBucket+ f.root = oldRoot+ f.rootDirectory = oldRootDir+ }()++ // Helper to remove an object with Object Lock protection+ removeLocked := func(t *testing.T, obj fs.Object) {+ t.Helper()+ o := obj.(*Object)+ // Remove legal hold if present+ _ = o.setObjectLegalHold(ctx, types.ObjectLockLegalHoldStatusOff)+ // Enable bypass governance retention for deletion+ o.fs.opt.BypassGovernanceRetention = true+ err := obj.Remove(ctx)+ o.fs.opt.BypassGovernanceRetention = false+ assert.NoError(t, err)+ }++ // Clean up the temporary bucket after all sub-tests+ defer func() {+ // List and remove all object versions+ var objectVersions []types.ObjectIdentifier+ listReq := &s3.ListObjectVersionsInput{Bucket: &lockBucket}+ for {+ var resp *s3.ListObjectVersionsOutput+ err := f.pacer.Call(func() (bool, error) {+ var err error+ resp, err = f.c.ListObjectVersions(ctx, listReq)+ return f.shouldRetry(ctx, err)+ })+ if err != nil {+ t.Logf("Failed to list object versions for cleanup: %v", err)+ break+ }+ for _, v := range resp.Versions {+ objectVersions = append(objectVersions, types.ObjectIdentifier{+ Key: v.Key,+ VersionId: v.VersionId,+ })+ }+ for _, m := range resp.DeleteMarkers {+ objectVersions = append(objectVersions, types.ObjectIdentifier{+ Key: m.Key,+ VersionId: m.VersionId,+ })+ }+ if !aws.ToBool(resp.IsTruncated) {+ break+ }+ listReq.KeyMarker = resp.NextKeyMarker+ listReq.VersionIdMarker = resp.NextVersionIdMarker+ }+ if len(objectVersions) > 0 {+ bypass := true+ _ = f.pacer.Call(func() (bool, error) {+ _, err := f.c.DeleteObjects(ctx, &s3.DeleteObjectsInput{+ Bucket: &lockBucket,+ BypassGovernanceRetention: &bypass,+ Delete: &types.Delete{+ Objects: objectVersions,+ Quiet: aws.Bool(true),+ },+ })+ return f.shouldRetry(ctx, err)+ })+ }+ _ = f.pacer.Call(func() (bool, error) {+ _, err := f.c.DeleteBucket(ctx, &s3.DeleteBucketInput{Bucket: &lockBucket})+ return f.shouldRetry(ctx, err)+ })+ }()++ retainUntilDate := time.Now().UTC().Add(24 * time.Hour).Truncate(time.Second)++ t.Run("Retention", func(t *testing.T) {+ // Set Object Lock options for this test+ f.opt.ObjectLockMode = "GOVERNANCE"+ f.opt.ObjectLockRetainUntilDate = retainUntilDate.Format(time.RFC3339)+ defer func() {+ f.opt.ObjectLockMode = ""+ f.opt.ObjectLockRetainUntilDate = ""+ }()++ // Upload an object with Object Lock retention+ contents := random.String(100)+ item := fstest.NewItem("test-object-lock-retention", contents, fstest.Time("2001-05-06T04:05:06.499999999Z"))+ obj := fstests.PutTestContents(ctx, t, f, &item, contents, true)+ defer func() {+ removeLocked(t, obj)+ }()++ // Read back metadata and verify Object Lock settings+ o := obj.(*Object)+ gotMetadata, err := o.Metadata(ctx)+ require.NoError(t, err)++ assert.Equal(t, "GOVERNANCE", gotMetadata["object-lock-mode"])+ gotRetainDate, err := time.Parse(time.RFC3339, gotMetadata["object-lock-retain-until-date"])+ require.NoError(t, err)+ assert.WithinDuration(t, retainUntilDate, gotRetainDate, time.Second)+ })++ t.Run("LegalHold", func(t *testing.T) {+ // Set Object Lock legal hold option+ f.opt.ObjectLockLegalHoldStatus = "ON"+ defer func() {+ f.opt.ObjectLockLegalHoldStatus = ""+ }()++ // Upload an object with legal hold+ contents := random.String(100)+ item := fstest.NewItem("test-object-lock-legal-hold", contents, fstest.Time("2001-05-06T04:05:06.499999999Z"))+ obj := fstests.PutTestContents(ctx, t, f, &item, contents, true)+ defer func() {+ removeLocked(t, obj)+ }()++ // Verify legal hold is ON+ o := obj.(*Object)+ gotMetadata, err := o.Metadata(ctx)+ require.NoError(t, err)+ assert.Equal(t, "ON", gotMetadata["object-lock-legal-hold-status"])++ // Set legal hold to OFF+ err = o.setObjectLegalHold(ctx, types.ObjectLockLegalHoldStatusOff)+ require.NoError(t, err)++ // Clear cached metadata and re-read+ o.meta = nil+ gotMetadata, err = o.Metadata(ctx)+ require.NoError(t, err)+ assert.Equal(t, "OFF", gotMetadata["object-lock-legal-hold-status"])+ })++ t.Run("SetAfterUpload", func(t *testing.T) {+ // Test the post-upload API path (PutObjectRetention + PutObjectLegalHold)+ f.opt.ObjectLockSetAfterUpload = true+ f.opt.ObjectLockMode = "GOVERNANCE"+ f.opt.ObjectLockRetainUntilDate = retainUntilDate.Format(time.RFC3339)+ f.opt.ObjectLockLegalHoldStatus = "ON"+ defer func() {+ f.opt.ObjectLockSetAfterUpload = false+ f.opt.ObjectLockMode = ""+ f.opt.ObjectLockRetainUntilDate = ""+ f.opt.ObjectLockLegalHoldStatus = ""+ }()++ // Upload an object - lock applied AFTER upload via separate API calls+ contents := random.String(100)+ item := fstest.NewItem("test-object-lock-after-upload", contents, fstest.Time("2001-05-06T04:05:06.499999999Z"))+ obj := fstests.PutTestContents(ctx, t, f, &item, contents, true)+ defer func() {+ removeLocked(t, obj)+ }()++ // Verify all Object Lock settings were applied+ o := obj.(*Object)+ gotMetadata, err := o.Metadata(ctx)+ require.NoError(t, err)++ assert.Equal(t, "GOVERNANCE", gotMetadata["object-lock-mode"])+ gotRetainDate, err := time.Parse(time.RFC3339, gotMetadata["object-lock-retain-until-date"])+ require.NoError(t, err)+ assert.WithinDuration(t, retainUntilDate, gotRetainDate, time.Second)+ assert.Equal(t, "ON", gotMetadata["object-lock-legal-hold-status"])+ })+}+func (f *Fs) InternalTest(t *testing.T) {t.Run("Metadata", f.InternalTestMetadata)t.Run("NoHead", f.InternalTestNoHead)t.Run("Versions", f.InternalTestVersions)+ t.Run("ObjectLock", f.InternalTestObjectLock)}var _ fstests.InternalTester = (*Fs)(nil)diff --git a/backend/s3/s3_test.go b/backend/s3/s3_test.goindex 973314c5c..a262d1703 100644--- a/backend/s3/s3_test.go+++ b/backend/s3/s3_test.go@@ -5,6 +5,7 @@ import ("context""net/http""testing"+ "time""github.com/aws/aws-sdk-go-v2/aws""github.com/rclone/rclone/fs"@@ -92,3 +93,87 @@ var (_ fstests.SetUploadCutoffer = (*Fs)(nil)_ fstests.SetCopyCutoffer = (*Fs)(nil))++func TestParseRetainUntilDate(t *testing.T) {+ now := time.Now()++ tests := []struct {+ name string+ input string+ wantErr bool+ checkFunc func(t *testing.T, result time.Time)+ }{+ {+ name: "RFC3339 date",+ input: "2030-01-15T10:30:00Z",+ wantErr: false,+ checkFunc: func(t *testing.T, result time.Time) {+ expected, _ := time.Parse(time.RFC3339, "2030-01-15T10:30:00Z")+ assert.Equal(t, expected, result)+ },+ },+ {+ name: "RFC3339 date with timezone",+ input: "2030-06-15T10:30:00+02:00",+ wantErr: false,+ checkFunc: func(t *testing.T, result time.Time) {+ expected, _ := time.Parse(time.RFC3339, "2030-06-15T10:30:00+02:00")+ assert.Equal(t, expected, result)+ },+ },+ {+ name: "duration days",+ input: "365d",+ wantErr: false,+ checkFunc: func(t *testing.T, result time.Time) {+ expected := now.Add(365 * 24 * time.Hour)+ diff := result.Sub(expected)+ assert.Less(t, diff.Abs(), 2*time.Second, "result should be ~365 days from now")+ },+ },+ {+ name: "duration hours",+ input: "24h",+ wantErr: false,+ checkFunc: func(t *testing.T, result time.Time) {+ expected := now.Add(24 * time.Hour)+ diff := result.Sub(expected)+ assert.Less(t, diff.Abs(), 2*time.Second, "result should be ~24 hours from now")+ },+ },+ {+ name: "duration minutes",+ input: "30m",+ wantErr: false,+ checkFunc: func(t *testing.T, result time.Time) {+ expected := now.Add(30 * time.Minute)+ diff := result.Sub(expected)+ assert.Less(t, diff.Abs(), 2*time.Second, "result should be ~30 minutes from now")+ },+ },+ {+ name: "invalid input",+ input: "not-a-date",+ wantErr: true,+ },+ {+ name: "empty input",+ input: "",+ wantErr: true,+ },+ }++ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ result, err := parseRetainUntilDate(tt.input)+ if tt.wantErr {+ require.Error(t, err)+ return+ }+ require.NoError(t, err)+ if tt.checkFunc != nil {+ tt.checkFunc(t, result)+ }+ })+ }+}diff --git a/docs/content/s3.md b/docs/content/s3.mdindex b04a735eb..a368a2a8d 100644--- a/docs/content/s3.md+++ b/docs/content/s3.md@@ -904,6 +904,48 @@ section, small files that are not uploaded as multipart, use a different tag, cathe upload to fail. A simple solution is to set the `--s3-upload-cutoff 0` and forceall the files to be uploaded as multipart.+#### Setting Object Lock retention++Rclone supports setting Object Lock retention on uploaded objects with+the following options:++- `--s3-object-lock-mode` - Set the Object Lock mode (GOVERNANCE or COMPLIANCE)+- `--s3-object-lock-retain-until-date` - Set the retention date (RFC3339 format or duration like `365d`, `24h`)+- `--s3-object-lock-legal-hold-status` - Set legal hold (ON or OFF)++Example - upload files with 1 year GOVERNANCE retention:++ rclone copy local:/data remote:bucket \+ --s3-object-lock-mode GOVERNANCE \+ --s3-object-lock-retain-until-date 365d++#### Copying Object Lock settings from source++When using `--metadata`, you can use the special value `copy` to preserve+the source object's Object Lock settings:++ rclone copy source:bucket dest:bucket \+ --metadata \+ --s3-object-lock-mode copy \+ --s3-object-lock-retain-until-date copy++You can also mix copied and explicit values. For example, to change the+mode from COMPLIANCE to GOVERNANCE while preserving the original retention date:++ rclone copy source:bucket dest:bucket \+ --metadata \+ --s3-object-lock-mode GOVERNANCE \+ --s3-object-lock-retain-until-date copy++#### Additional Object Lock options++- `--s3-bypass-governance-retention` - Required to delete or overwrite objects+ with GOVERNANCE mode retention before the retention date expires+- `--s3-bucket-object-lock-enabled` - Enable Object Lock when creating a new+ bucket with `rclone mkdir`+- `--s3-object-lock-set-after-upload` - Set Object Lock via separate API calls+ after upload (for providers that don't support Object Lock headers during PUT)+<!-- autogenerated options start - DO NOT EDIT - instead edit fs.RegInfo in backend/s3/s3.go and run make backenddocs to verify --> <!-- markdownlint-disable-line line-length -->### Standard options