Skip to content

"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files

s3: add Object Lock support

Chris7d0a8bf850b9
diff --git a/backend/s3/s3.go b/backend/s3/s3.go
index b5bba8327..02398a34f 100644
--- a/backend/s3/s3.go
+++ b/backend/s3/s3.go
@@ -828,6 +828,107 @@ use |-vv| to see the debug level logs.
}, {
Name: "ibm_resource_instance_id",
Help: "IBM service instance id",
+ }, {
+ Name: "object_lock_mode",
+ Help: `Object Lock mode to apply when uploading or copying objects.
+
+Set this to apply Object Lock retention mode to objects.
+If not set, no Object Lock mode is applied (even with --metadata).
+
+Note: To enable Object Lock retention, you must set BOTH object_lock_mode
+AND object_lock_retain_until_date. Setting only one has no effect.
+
+- GOVERNANCE: Set Object Lock mode to GOVERNANCE
+- COMPLIANCE: Set Object Lock mode to COMPLIANCE
+- copy: Copy the mode from the source object (requires --metadata)
+
+See: https://docs.aws.amazon.com/AmazonS3/latest/userguide/object-lock.html`,
+ Advanced: true,
+ Examples: []fs.OptionExample{{
+ Value: "GOVERNANCE",
+ Help: "Set Object Lock mode to GOVERNANCE",
+ }, {
+ Value: "COMPLIANCE",
+ Help: "Set Object Lock mode to COMPLIANCE",
+ }, {
+ Value: "copy",
+ Help: "Copy from source object (requires --metadata)",
+ }},
+ }, {
+ Name: "object_lock_retain_until_date",
+ Help: `Object Lock retention until date to apply when uploading or copying objects.
+
+Set this to apply Object Lock retention date to objects.
+If not set, no retention date is applied (even with --metadata).
+
+Note: To enable Object Lock retention, you must set BOTH object_lock_mode
+AND object_lock_retain_until_date. Setting only one has no effect.
+
+Accepts:
+- RFC 3339 format: 2030-01-02T15:04:05Z
+- Duration from now: 365d, 1y, 6M (days, years, months)
+- copy: Copy the date from the source object (requires --metadata)
+
+See: https://docs.aws.amazon.com/AmazonS3/latest/userguide/object-lock.html`,
+ Advanced: true,
+ Examples: []fs.OptionExample{{
+ Value: "copy",
+ Help: "Copy from source object (requires --metadata)",
+ }, {
+ Value: "2030-01-01T00:00:00Z",
+ Help: "Set specific date (RFC 3339 format)",
+ }, {
+ Value: "365d",
+ Help: "Set retention for 365 days from now",
+ }, {
+ Value: "1y",
+ Help: "Set retention for 1 year from now",
+ }},
+ }, {
+ Name: "object_lock_legal_hold_status",
+ Help: `Object Lock legal hold status to apply when uploading or copying objects.
+
+Set this to apply Object Lock legal hold to objects.
+If not set, no legal hold is applied (even with --metadata).
+
+Note: Legal hold is independent of retention and can be set separately.
+
+- ON: Enable legal hold
+- OFF: Disable legal hold
+- copy: Copy the legal hold status from the source object (requires --metadata)
+
+See: https://docs.aws.amazon.com/AmazonS3/latest/userguide/object-lock.html`,
+ Advanced: true,
+ Examples: []fs.OptionExample{{
+ Value: "ON",
+ Help: "Enable legal hold",
+ }, {
+ Value: "OFF",
+ Help: "Disable legal hold",
+ }, {
+ Value: "copy",
+ Help: "Copy from source object (requires --metadata)",
+ }},
+ }, {
+ Name: "bypass_governance_retention",
+ Help: `Allow deleting or modifying objects locked with GOVERNANCE mode.`,
+ Default: false,
+ Advanced: true,
+ }, {
+ Name: "bucket_object_lock_enabled",
+ Help: `Enable Object Lock when creating new buckets.`,
+ Default: false,
+ }, {
+ Name: "object_lock_set_after_upload",
+ Help: `Set Object Lock via separate API calls after upload.
+
+Use this for S3-compatible providers that don't support setting Object Lock
+headers during PUT operations. When enabled, Object Lock is set via separate
+PutObjectRetention and PutObjectLegalHold API calls after the upload completes.
+
+This adds extra API calls per object, so only enable if your provider requires it.`,
+ Default: false,
+ Advanced: true,
},
}}))
}
@@ -922,6 +1023,21 @@ var systemMetadataInfo = map[string]fs.MetadataHelp{
Example: "2006-01-02T15:04:05.999999999Z07:00",
ReadOnly: true,
},
+ "object-lock-mode": {
+ Help: "Object Lock mode: GOVERNANCE or COMPLIANCE",
+ Type: "string",
+ Example: "GOVERNANCE",
+ },
+ "object-lock-retain-until-date": {
+ Help: "Object Lock retention until date",
+ Type: "RFC 3339",
+ Example: "2030-01-02T15:04:05Z",
+ },
+ "object-lock-legal-hold-status": {
+ Help: "Object Lock legal hold status: ON or OFF",
+ Type: "string",
+ Example: "OFF",
+ },
}
// Options defines the configuration for this backend
@@ -992,6 +1108,12 @@ type Options struct {
IBMInstanceID string `config:"ibm_resource_instance_id"`
UseXID fs.Tristate `config:"use_x_id"`
SignAcceptEncoding fs.Tristate `config:"sign_accept_encoding"`
+ ObjectLockMode string `config:"object_lock_mode"`
+ ObjectLockRetainUntilDate string `config:"object_lock_retain_until_date"`
+ ObjectLockLegalHoldStatus string `config:"object_lock_legal_hold_status"`
+ BypassGovernanceRetention bool `config:"bypass_governance_retention"`
+ BucketObjectLockEnabled bool `config:"bucket_object_lock_enabled"`
+ ObjectLockSetAfterUpload bool `config:"object_lock_set_after_upload"`
}
// Fs represents a remote s3 server
@@ -1036,6 +1158,11 @@ type Object struct {
contentDisposition *string // Content-Disposition: header
contentEncoding *string // Content-Encoding: header
contentLanguage *string // Content-Language: header
+
+ // Object Lock metadata
+ objectLockMode *string // Object Lock mode: GOVERNANCE or COMPLIANCE
+ objectLockRetainUntilDate *time.Time // Object Lock retention until date
+ objectLockLegalHoldStatus *string // Object Lock legal hold: ON or OFF
}
// safely dereference the pointer, returning a zero T if nil
@@ -1056,6 +1183,21 @@ func getHTTPStatusCode(err error) int {
return -1
}
+// parseRetainUntilDate parses a retain until date from a string.
+// It accepts RFC 3339 format or duration strings like "365d", "1y", "6m".
+func parseRetainUntilDate(s string) (time.Time, error) {
+ // First try RFC 3339 format
+ if t, err := time.Parse(time.RFC3339, s); err == nil {
+ return t, nil
+ }
+ // Try as a duration from now
+ d, err := fs.ParseDuration(s)
+ if err != nil {
+ return time.Time{}, fmt.Errorf("can't parse %q as RFC 3339 date or duration: %w", s, err)
+ }
+ return time.Now().Add(d), nil
+}
+
// ------------------------------------------------------------
// Name of the remote (as passed into NewFs)
@@ -2649,8 +2791,9 @@ func (f *Fs) makeBucket(ctx context.Context, bucket string) error {
}
return f.cache.Create(bucket, func() error {
req := s3.CreateBucketInput{
- Bucket: &bucket,
- ACL: types.BucketCannedACL(f.opt.BucketACL),
+ Bucket: &bucket,
+ ACL: types.BucketCannedACL(f.opt.BucketACL),
+ ObjectLockEnabledForBucket: &f.opt.BucketObjectLockEnabled,
}
if f.opt.LocationConstraint != "" {
req.CreateBucketConfiguration = &types.CreateBucketConfiguration{
@@ -2769,6 +2912,24 @@ func (f *Fs) copy(ctx context.Context, req *s3.CopyObjectInput, dstBucket, dstPa
req.StorageClass = types.StorageClass(f.opt.StorageClass)
}
+ // Apply Object Lock options via headers (unless ObjectLockSetAfterUpload is set)
+ // "copy" means: keep the value from source (passed via req from prepareUpload/setFrom functions)
+ if !f.opt.ObjectLockSetAfterUpload {
+ if f.opt.ObjectLockMode != "" && !strings.EqualFold(f.opt.ObjectLockMode, "copy") {
+ req.ObjectLockMode = types.ObjectLockMode(strings.ToUpper(f.opt.ObjectLockMode))
+ }
+ if f.opt.ObjectLockRetainUntilDate != "" && !strings.EqualFold(f.opt.ObjectLockRetainUntilDate, "copy") {
+ retainDate, err := parseRetainUntilDate(f.opt.ObjectLockRetainUntilDate)
+ if err != nil {
+ return fmt.Errorf("invalid object_lock_retain_until_date: %w", err)
+ }
+ req.ObjectLockRetainUntilDate = &retainDate
+ }
+ if f.opt.ObjectLockLegalHoldStatus != "" && !strings.EqualFold(f.opt.ObjectLockLegalHoldStatus, "copy") {
+ req.ObjectLockLegalHoldStatus = types.ObjectLockLegalHoldStatus(strings.ToUpper(f.opt.ObjectLockLegalHoldStatus))
+ }
+ }
+
if src.bytes >= int64(f.opt.CopyCutoff) {
return f.copyMultipart(ctx, req, dstBucket, dstPath, srcBucket, srcPath, src)
}
@@ -2951,7 +3112,15 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object,
}
setFrom_s3CopyObjectInput_s3PutObjectInput(&req, ui.req)
- if ci.Metadata {
+ // Use REPLACE directive if metadata is being modified, otherwise S3 ignores our values
+ // This is needed when:
+ // 1. --metadata flag is set
+ // 2. Any Object Lock option is set (to override or explicitly copy)
+ needsReplace := ci.Metadata ||
+ f.opt.ObjectLockMode != "" ||
+ f.opt.ObjectLockRetainUntilDate != "" ||
+ f.opt.ObjectLockLegalHoldStatus != ""
+ if needsReplace {
req.MetadataDirective = types.MetadataDirectiveReplace
}
@@ -2959,7 +3128,21 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object,
if err != nil {
return nil, err
}
- return f.NewObject(ctx, remote)
+ dstObj, err := f.NewObject(ctx, remote)
+ if err != nil {
+ return nil, err
+ }
+
+ // Set Object Lock via separate API calls if requested
+ if f.opt.ObjectLockSetAfterUpload {
+ if dstObject, ok := dstObj.(*Object); ok {
+ if err := dstObject.setObjectLockAfterUpload(ctx, srcObj); err != nil {
+ return nil, err
+ }
+ }
+ }
+
+ return dstObj, nil
}
// Hashes returns the supported hash sets.
@@ -3838,6 +4021,19 @@ func (o *Object) setMetaData(resp *s3.HeadObjectOutput) {
o.contentEncoding = stringClonePointer(removeAWSChunked(resp.ContentEncoding))
o.contentLanguage = stringClonePointer(resp.ContentLanguage)
+ // Set Object Lock metadata
+ if resp.ObjectLockMode != "" {
+ mode := string(resp.ObjectLockMode)
+ o.objectLockMode = &mode
+ }
+ if resp.ObjectLockRetainUntilDate != nil {
+ o.objectLockRetainUntilDate = resp.ObjectLockRetainUntilDate
+ }
+ if resp.ObjectLockLegalHoldStatus != "" {
+ status := string(resp.ObjectLockLegalHoldStatus)
+ o.objectLockLegalHoldStatus = &status
+ }
+
// If decompressing then size and md5sum are unknown
if o.fs.opt.Decompress && deref(o.contentEncoding) == "gzip" {
o.bytes = -1
@@ -4556,6 +4752,26 @@ func (o *Object) prepareUpload(ctx context.Context, src fs.ObjectInfo, options [
case "btime":
// write as metadata since we can't set it
ui.req.Metadata[k] = v
+ case "object-lock-mode":
+ // Only apply if option is set to "copy" and not using after-upload API
+ if strings.EqualFold(o.fs.opt.ObjectLockMode, "copy") && !o.fs.opt.ObjectLockSetAfterUpload {
+ ui.req.ObjectLockMode = types.ObjectLockMode(v)
+ }
+ case "object-lock-retain-until-date":
+ // Only apply if option is set to "copy" and not using after-upload API
+ if strings.EqualFold(o.fs.opt.ObjectLockRetainUntilDate, "copy") && !o.fs.opt.ObjectLockSetAfterUpload {
+ retainDate, err := time.Parse(time.RFC3339, v)
+ if err != nil {
+ fs.Debugf(o, "failed to parse object-lock-retain-until-date %q: %v", v, err)
+ } else {
+ ui.req.ObjectLockRetainUntilDate = &retainDate
+ }
+ }
+ case "object-lock-legal-hold-status":
+ // Only apply if option is set to "copy" and not using after-upload API
+ if strings.EqualFold(o.fs.opt.ObjectLockLegalHoldStatus, "copy") && !o.fs.opt.ObjectLockSetAfterUpload {
+ ui.req.ObjectLockLegalHoldStatus = types.ObjectLockLegalHoldStatus(v)
+ }
default:
ui.req.Metadata[k] = v
}
@@ -4621,6 +4837,25 @@ func (o *Object) prepareUpload(ctx context.Context, src fs.ObjectInfo, options [
if o.fs.opt.StorageClass != "" {
ui.req.StorageClass = types.StorageClass(o.fs.opt.StorageClass)
}
+
+ // Apply Object Lock options via headers (unless ObjectLockSetAfterUpload is set)
+ // "copy" means: keep the value from metadata (already applied above in the switch)
+ if !o.fs.opt.ObjectLockSetAfterUpload {
+ if o.fs.opt.ObjectLockMode != "" && !strings.EqualFold(o.fs.opt.ObjectLockMode, "copy") {
+ ui.req.ObjectLockMode = types.ObjectLockMode(strings.ToUpper(o.fs.opt.ObjectLockMode))
+ }
+ if o.fs.opt.ObjectLockRetainUntilDate != "" && !strings.EqualFold(o.fs.opt.ObjectLockRetainUntilDate, "copy") {
+ retainDate, err := parseRetainUntilDate(o.fs.opt.ObjectLockRetainUntilDate)
+ if err != nil {
+ return ui, fmt.Errorf("invalid object_lock_retain_until_date %q: %w", o.fs.opt.ObjectLockRetainUntilDate, err)
+ }
+ ui.req.ObjectLockRetainUntilDate = &retainDate
+ }
+ if o.fs.opt.ObjectLockLegalHoldStatus != "" && !strings.EqualFold(o.fs.opt.ObjectLockLegalHoldStatus, "copy") {
+ ui.req.ObjectLockLegalHoldStatus = types.ObjectLockLegalHoldStatus(strings.ToUpper(o.fs.opt.ObjectLockLegalHoldStatus))
+ }
+ }
+
// Apply upload options
for _, option := range options {
key, value := option.Header()
@@ -4744,6 +4979,14 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
}
fs.Debugf(o, "Multipart upload Etag: %s OK", wantETag)
}
+
+ // Set Object Lock via separate API calls if requested
+ if o.fs.opt.ObjectLockSetAfterUpload {
+ if err := o.setObjectLockAfterUpload(ctx, src); err != nil {
+ return err
+ }
+ }
+
return err
}
@@ -4761,6 +5004,9 @@ func (o *Object) Remove(ctx context.Context) error {
if o.fs.opt.RequesterPays {
req.RequestPayer = types.RequestPayerRequester
}
+ if o.fs.opt.BypassGovernanceRetention {
+ req.BypassGovernanceRetention = &o.fs.opt.BypassGovernanceRetention
+ }
err := o.fs.pacer.Call(func() (bool, error) {
_, err := o.fs.c.DeleteObject(ctx, &req)
return o.fs.shouldRetry(ctx, err)
@@ -4768,6 +5014,120 @@ func (o *Object) Remove(ctx context.Context) error {
return err
}
+// setObjectRetention sets Object Lock retention on an object via PutObjectRetention API
+//
+// Note: We use smithyhttp.AddContentChecksumMiddleware to ensure Content-MD5 is
+// calculated for the request body. The AWS SDK v2 switched from MD5 to CRC32 as
+// the default checksum algorithm, but some S3-compatible providers (e.g. MinIO)
+// still require Content-MD5 for PutObjectRetention requests.
+// See: https://github.com/aws/aws-sdk-go-v2/discussions/2960
+func (o *Object) setObjectRetention(ctx context.Context, mode types.ObjectLockRetentionMode, retainUntilDate time.Time) error {
+ bucket, bucketPath := o.split()
+ req := s3.PutObjectRetentionInput{
+ Bucket: &bucket,
+ Key: &bucketPath,
+ VersionId: o.versionID,
+ Retention: &types.ObjectLockRetention{
+ Mode: mode,
+ RetainUntilDate: &retainUntilDate,
+ },
+ }
+ if o.fs.opt.RequesterPays {
+ req.RequestPayer = types.RequestPayerRequester
+ }
+ if o.fs.opt.BypassGovernanceRetention {
+ req.BypassGovernanceRetention = &o.fs.opt.BypassGovernanceRetention
+ }
+ return o.fs.pacer.Call(func() (bool, error) {
+ _, err := o.fs.c.PutObjectRetention(ctx, &req,
+ s3.WithAPIOptions(smithyhttp.AddContentChecksumMiddleware))
+ return o.fs.shouldRetry(ctx, err)
+ })
+}
+
+// setObjectLegalHold sets Object Lock legal hold on an object via PutObjectLegalHold API
+//
+// Note: We use smithyhttp.AddContentChecksumMiddleware to ensure Content-MD5 is
+// calculated for the request body. The AWS SDK v2 switched from MD5 to CRC32 as
+// the default checksum algorithm, but some S3-compatible providers (e.g. MinIO)
+// still require Content-MD5 for PutObjectLegalHold requests.
+// See: https://github.com/aws/aws-sdk-go-v2/discussions/2960
+func (o *Object) setObjectLegalHold(ctx context.Context, status types.ObjectLockLegalHoldStatus) error {
+ bucket, bucketPath := o.split()
+ req := s3.PutObjectLegalHoldInput{
+ Bucket: &bucket,
+ Key: &bucketPath,
+ VersionId: o.versionID,
+ LegalHold: &types.ObjectLockLegalHold{
+ Status: status,
+ },
+ }
+ if o.fs.opt.RequesterPays {
+ req.RequestPayer = types.RequestPayerRequester
+ }
+ return o.fs.pacer.Call(func() (bool, error) {
+ _, err := o.fs.c.PutObjectLegalHold(ctx, &req,
+ s3.WithAPIOptions(smithyhttp.AddContentChecksumMiddleware))
+ return o.fs.shouldRetry(ctx, err)
+ })
+}
+
+// setObjectLockAfterUpload sets Object Lock via separate API calls after upload
+// This is for S3 providers that don't support Object Lock headers during PUT
+func (o *Object) setObjectLockAfterUpload(ctx context.Context, src fs.ObjectInfo) error {
+ // Determine the mode
+ var mode types.ObjectLockRetentionMode
+ modeOpt := o.fs.opt.ObjectLockMode
+ if strings.EqualFold(modeOpt, "copy") {
+ if srcObj, ok := src.(*Object); ok && srcObj.objectLockMode != nil {
+ mode = types.ObjectLockRetentionMode(*srcObj.objectLockMode)
+ }
+ } else if modeOpt != "" {
+ mode = types.ObjectLockRetentionMode(strings.ToUpper(modeOpt))
+ }
+
+ // Determine the retain until date
+ var retainUntilDate time.Time
+ dateOpt := o.fs.opt.ObjectLockRetainUntilDate
+ if strings.EqualFold(dateOpt, "copy") {
+ if srcObj, ok := src.(*Object); ok && srcObj.objectLockRetainUntilDate != nil {
+ retainUntilDate = *srcObj.objectLockRetainUntilDate
+ }
+ } else if dateOpt != "" {
+ var err error
+ retainUntilDate, err = parseRetainUntilDate(dateOpt)
+ if err != nil {
+ return fmt.Errorf("invalid object_lock_retain_until_date %q: %w", dateOpt, err)
+ }
+ }
+
+ // Set retention if both mode and date are set
+ if mode != "" && !retainUntilDate.IsZero() {
+ if err := o.setObjectRetention(ctx, mode, retainUntilDate); err != nil {
+ return fmt.Errorf("failed to set object retention: %w", err)
+ }
+ }
+
+ // Determine and set legal hold
+ var legalHold types.ObjectLockLegalHoldStatus
+ legalHoldOpt := o.fs.opt.ObjectLockLegalHoldStatus
+ if strings.EqualFold(legalHoldOpt, "copy") {
+ if srcObj, ok := src.(*Object); ok && srcObj.objectLockLegalHoldStatus != nil {
+ legalHold = types.ObjectLockLegalHoldStatus(*srcObj.objectLockLegalHoldStatus)
+ }
+ } else if legalHoldOpt != "" {
+ legalHold = types.ObjectLockLegalHoldStatus(strings.ToUpper(legalHoldOpt))
+ }
+
+ if legalHold != "" {
+ if err := o.setObjectLegalHold(ctx, legalHold); err != nil {
+ return fmt.Errorf("failed to set legal hold: %w", err)
+ }
+ }
+
+ return nil
+}
+
// MimeType of an Object if known, "" otherwise
func (o *Object) MimeType(ctx context.Context) string {
err := o.readMetaData(ctx)
@@ -4811,7 +5171,7 @@ func (o *Object) Metadata(ctx context.Context) (metadata fs.Metadata, err error)
if err != nil {
return nil, err
}
- metadata = make(fs.Metadata, len(o.meta)+7)
+ metadata = make(fs.Metadata, len(o.meta)+10)
for k, v := range o.meta {
switch k {
case metaMtime:
@@ -4846,6 +5206,15 @@ func (o *Object) Metadata(ctx context.Context) (metadata fs.Metadata, err error)
setMetadata("content-disposition", o.contentDisposition)
setMetadata("content-encoding", o.contentEncoding)
setMetadata("content-language", o.contentLanguage)
+
+ // Set Object Lock metadata
+ setMetadata("object-lock-mode", o.objectLockMode)
+ if o.objectLockRetainUntilDate != nil {
+ formatted := o.objectLockRetainUntilDate.Format(time.RFC3339)
+ setMetadata("object-lock-retain-until-date", &formatted)
+ }
+ setMetadata("object-lock-legal-hold-status", o.objectLockLegalHoldStatus)
+
metadata["tier"] = o.GetTier()
return metadata, nil
diff --git a/backend/s3/s3_internal_test.go b/backend/s3/s3_internal_test.go
index cdca649ae..c4e6637f5 100644
--- a/backend/s3/s3_internal_test.go
+++ b/backend/s3/s3_internal_test.go
@@ -498,10 +498,236 @@ func (f *Fs) InternalTestVersions(t *testing.T) {
// Purge gets tested later
}
+func (f *Fs) InternalTestObjectLock(t *testing.T) {
+ ctx := context.Background()
+
+ // Create a temporary bucket with Object Lock enabled to test on.
+ // This exercises our BucketObjectLockEnabled option and isolates
+ // the test from the main test bucket.
+ lockBucket := f.rootBucket + "-object-lock-" + random.String(8)
+ lockBucket = strings.ToLower(lockBucket)
+
+ // Try to create bucket with Object Lock enabled
+ objectLockEnabled := true
+ req := s3.CreateBucketInput{
+ Bucket: &lockBucket,
+ ACL: types.BucketCannedACL(f.opt.BucketACL),
+ ObjectLockEnabledForBucket: &objectLockEnabled,
+ }
+ if f.opt.LocationConstraint != "" {
+ req.CreateBucketConfiguration = &types.CreateBucketConfiguration{
+ LocationConstraint: types.BucketLocationConstraint(f.opt.LocationConstraint),
+ }
+ }
+ err := f.pacer.Call(func() (bool, error) {
+ _, err := f.c.CreateBucket(ctx, &req)
+ return f.shouldRetry(ctx, err)
+ })
+ if err != nil {
+ t.Skipf("Object Lock not supported by this provider: CreateBucket with Object Lock failed: %v", err)
+ }
+
+ // Verify Object Lock is actually enabled on the new bucket.
+ // Some S3-compatible servers (e.g. rclone serve s3) accept the
+ // ObjectLockEnabledForBucket flag but don't actually implement Object Lock.
+ var lockCfg *s3.GetObjectLockConfigurationOutput
+ err = f.pacer.Call(func() (bool, error) {
+ var err error
+ lockCfg, err = f.c.GetObjectLockConfiguration(ctx, &s3.GetObjectLockConfigurationInput{
+ Bucket: &lockBucket,
+ })
+ return f.shouldRetry(ctx, err)
+ })
+ if err != nil || lockCfg.ObjectLockConfiguration == nil ||
+ lockCfg.ObjectLockConfiguration.ObjectLockEnabled != types.ObjectLockEnabledEnabled {
+ _ = f.pacer.Call(func() (bool, error) {
+ _, err := f.c.DeleteBucket(ctx, &s3.DeleteBucketInput{Bucket: &lockBucket})
+ return f.shouldRetry(ctx, err)
+ })
+ t.Skipf("Object Lock not functional on this provider (GetObjectLockConfiguration: %v)", err)
+ }
+
+ // Switch f to use the Object Lock bucket for this test
+ oldBucket := f.rootBucket
+ oldRoot := f.root
+ oldRootDir := f.rootDirectory
+ f.rootBucket = lockBucket
+ f.root = lockBucket
+ f.rootDirectory = ""
+ defer func() {
+ f.rootBucket = oldBucket
+ f.root = oldRoot
+ f.rootDirectory = oldRootDir
+ }()
+
+ // Helper to remove an object with Object Lock protection
+ removeLocked := func(t *testing.T, obj fs.Object) {
+ t.Helper()
+ o := obj.(*Object)
+ // Remove legal hold if present
+ _ = o.setObjectLegalHold(ctx, types.ObjectLockLegalHoldStatusOff)
+ // Enable bypass governance retention for deletion
+ o.fs.opt.BypassGovernanceRetention = true
+ err := obj.Remove(ctx)
+ o.fs.opt.BypassGovernanceRetention = false
+ assert.NoError(t, err)
+ }
+
+ // Clean up the temporary bucket after all sub-tests
+ defer func() {
+ // List and remove all object versions
+ var objectVersions []types.ObjectIdentifier
+ listReq := &s3.ListObjectVersionsInput{Bucket: &lockBucket}
+ for {
+ var resp *s3.ListObjectVersionsOutput
+ err := f.pacer.Call(func() (bool, error) {
+ var err error
+ resp, err = f.c.ListObjectVersions(ctx, listReq)
+ return f.shouldRetry(ctx, err)
+ })
+ if err != nil {
+ t.Logf("Failed to list object versions for cleanup: %v", err)
+ break
+ }
+ for _, v := range resp.Versions {
+ objectVersions = append(objectVersions, types.ObjectIdentifier{
+ Key: v.Key,
+ VersionId: v.VersionId,
+ })
+ }
+ for _, m := range resp.DeleteMarkers {
+ objectVersions = append(objectVersions, types.ObjectIdentifier{
+ Key: m.Key,
+ VersionId: m.VersionId,
+ })
+ }
+ if !aws.ToBool(resp.IsTruncated) {
+ break
+ }
+ listReq.KeyMarker = resp.NextKeyMarker
+ listReq.VersionIdMarker = resp.NextVersionIdMarker
+ }
+ if len(objectVersions) > 0 {
+ bypass := true
+ _ = f.pacer.Call(func() (bool, error) {
+ _, err := f.c.DeleteObjects(ctx, &s3.DeleteObjectsInput{
+ Bucket: &lockBucket,
+ BypassGovernanceRetention: &bypass,
+ Delete: &types.Delete{
+ Objects: objectVersions,
+ Quiet: aws.Bool(true),
+ },
+ })
+ return f.shouldRetry(ctx, err)
+ })
+ }
+ _ = f.pacer.Call(func() (bool, error) {
+ _, err := f.c.DeleteBucket(ctx, &s3.DeleteBucketInput{Bucket: &lockBucket})
+ return f.shouldRetry(ctx, err)
+ })
+ }()
+
+ retainUntilDate := time.Now().UTC().Add(24 * time.Hour).Truncate(time.Second)
+
+ t.Run("Retention", func(t *testing.T) {
+ // Set Object Lock options for this test
+ f.opt.ObjectLockMode = "GOVERNANCE"
+ f.opt.ObjectLockRetainUntilDate = retainUntilDate.Format(time.RFC3339)
+ defer func() {
+ f.opt.ObjectLockMode = ""
+ f.opt.ObjectLockRetainUntilDate = ""
+ }()
+
+ // Upload an object with Object Lock retention
+ contents := random.String(100)
+ item := fstest.NewItem("test-object-lock-retention", contents, fstest.Time("2001-05-06T04:05:06.499999999Z"))
+ obj := fstests.PutTestContents(ctx, t, f, &item, contents, true)
+ defer func() {
+ removeLocked(t, obj)
+ }()
+
+ // Read back metadata and verify Object Lock settings
+ o := obj.(*Object)
+ gotMetadata, err := o.Metadata(ctx)
+ require.NoError(t, err)
+
+ assert.Equal(t, "GOVERNANCE", gotMetadata["object-lock-mode"])
+ gotRetainDate, err := time.Parse(time.RFC3339, gotMetadata["object-lock-retain-until-date"])
+ require.NoError(t, err)
+ assert.WithinDuration(t, retainUntilDate, gotRetainDate, time.Second)
+ })
+
+ t.Run("LegalHold", func(t *testing.T) {
+ // Set Object Lock legal hold option
+ f.opt.ObjectLockLegalHoldStatus = "ON"
+ defer func() {
+ f.opt.ObjectLockLegalHoldStatus = ""
+ }()
+
+ // Upload an object with legal hold
+ contents := random.String(100)
+ item := fstest.NewItem("test-object-lock-legal-hold", contents, fstest.Time("2001-05-06T04:05:06.499999999Z"))
+ obj := fstests.PutTestContents(ctx, t, f, &item, contents, true)
+ defer func() {
+ removeLocked(t, obj)
+ }()
+
+ // Verify legal hold is ON
+ o := obj.(*Object)
+ gotMetadata, err := o.Metadata(ctx)
+ require.NoError(t, err)
+ assert.Equal(t, "ON", gotMetadata["object-lock-legal-hold-status"])
+
+ // Set legal hold to OFF
+ err = o.setObjectLegalHold(ctx, types.ObjectLockLegalHoldStatusOff)
+ require.NoError(t, err)
+
+ // Clear cached metadata and re-read
+ o.meta = nil
+ gotMetadata, err = o.Metadata(ctx)
+ require.NoError(t, err)
+ assert.Equal(t, "OFF", gotMetadata["object-lock-legal-hold-status"])
+ })
+
+ t.Run("SetAfterUpload", func(t *testing.T) {
+ // Test the post-upload API path (PutObjectRetention + PutObjectLegalHold)
+ f.opt.ObjectLockSetAfterUpload = true
+ f.opt.ObjectLockMode = "GOVERNANCE"
+ f.opt.ObjectLockRetainUntilDate = retainUntilDate.Format(time.RFC3339)
+ f.opt.ObjectLockLegalHoldStatus = "ON"
+ defer func() {
+ f.opt.ObjectLockSetAfterUpload = false
+ f.opt.ObjectLockMode = ""
+ f.opt.ObjectLockRetainUntilDate = ""
+ f.opt.ObjectLockLegalHoldStatus = ""
+ }()
+
+ // Upload an object - lock applied AFTER upload via separate API calls
+ contents := random.String(100)
+ item := fstest.NewItem("test-object-lock-after-upload", contents, fstest.Time("2001-05-06T04:05:06.499999999Z"))
+ obj := fstests.PutTestContents(ctx, t, f, &item, contents, true)
+ defer func() {
+ removeLocked(t, obj)
+ }()
+
+ // Verify all Object Lock settings were applied
+ o := obj.(*Object)
+ gotMetadata, err := o.Metadata(ctx)
+ require.NoError(t, err)
+
+ assert.Equal(t, "GOVERNANCE", gotMetadata["object-lock-mode"])
+ gotRetainDate, err := time.Parse(time.RFC3339, gotMetadata["object-lock-retain-until-date"])
+ require.NoError(t, err)
+ assert.WithinDuration(t, retainUntilDate, gotRetainDate, time.Second)
+ assert.Equal(t, "ON", gotMetadata["object-lock-legal-hold-status"])
+ })
+}
+
func (f *Fs) InternalTest(t *testing.T) {
t.Run("Metadata", f.InternalTestMetadata)
t.Run("NoHead", f.InternalTestNoHead)
t.Run("Versions", f.InternalTestVersions)
+ t.Run("ObjectLock", f.InternalTestObjectLock)
}
var _ fstests.InternalTester = (*Fs)(nil)
diff --git a/backend/s3/s3_test.go b/backend/s3/s3_test.go
index 973314c5c..a262d1703 100644
--- a/backend/s3/s3_test.go
+++ b/backend/s3/s3_test.go
@@ -5,6 +5,7 @@ import (
"context"
"net/http"
"testing"
+ "time"
"github.com/aws/aws-sdk-go-v2/aws"
"github.com/rclone/rclone/fs"
@@ -92,3 +93,87 @@ var (
_ fstests.SetUploadCutoffer = (*Fs)(nil)
_ fstests.SetCopyCutoffer = (*Fs)(nil)
)
+
+func TestParseRetainUntilDate(t *testing.T) {
+ now := time.Now()
+
+ tests := []struct {
+ name string
+ input string
+ wantErr bool
+ checkFunc func(t *testing.T, result time.Time)
+ }{
+ {
+ name: "RFC3339 date",
+ input: "2030-01-15T10:30:00Z",
+ wantErr: false,
+ checkFunc: func(t *testing.T, result time.Time) {
+ expected, _ := time.Parse(time.RFC3339, "2030-01-15T10:30:00Z")
+ assert.Equal(t, expected, result)
+ },
+ },
+ {
+ name: "RFC3339 date with timezone",
+ input: "2030-06-15T10:30:00+02:00",
+ wantErr: false,
+ checkFunc: func(t *testing.T, result time.Time) {
+ expected, _ := time.Parse(time.RFC3339, "2030-06-15T10:30:00+02:00")
+ assert.Equal(t, expected, result)
+ },
+ },
+ {
+ name: "duration days",
+ input: "365d",
+ wantErr: false,
+ checkFunc: func(t *testing.T, result time.Time) {
+ expected := now.Add(365 * 24 * time.Hour)
+ diff := result.Sub(expected)
+ assert.Less(t, diff.Abs(), 2*time.Second, "result should be ~365 days from now")
+ },
+ },
+ {
+ name: "duration hours",
+ input: "24h",
+ wantErr: false,
+ checkFunc: func(t *testing.T, result time.Time) {
+ expected := now.Add(24 * time.Hour)
+ diff := result.Sub(expected)
+ assert.Less(t, diff.Abs(), 2*time.Second, "result should be ~24 hours from now")
+ },
+ },
+ {
+ name: "duration minutes",
+ input: "30m",
+ wantErr: false,
+ checkFunc: func(t *testing.T, result time.Time) {
+ expected := now.Add(30 * time.Minute)
+ diff := result.Sub(expected)
+ assert.Less(t, diff.Abs(), 2*time.Second, "result should be ~30 minutes from now")
+ },
+ },
+ {
+ name: "invalid input",
+ input: "not-a-date",
+ wantErr: true,
+ },
+ {
+ name: "empty input",
+ input: "",
+ wantErr: true,
+ },
+ }
+
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ result, err := parseRetainUntilDate(tt.input)
+ if tt.wantErr {
+ require.Error(t, err)
+ return
+ }
+ require.NoError(t, err)
+ if tt.checkFunc != nil {
+ tt.checkFunc(t, result)
+ }
+ })
+ }
+}
diff --git a/docs/content/s3.md b/docs/content/s3.md
index b04a735eb..a368a2a8d 100644
--- a/docs/content/s3.md
+++ b/docs/content/s3.md
@@ -904,6 +904,48 @@ section, small files that are not uploaded as multipart, use a different tag, ca
the upload to fail. A simple solution is to set the `--s3-upload-cutoff 0` and force
all the files to be uploaded as multipart.
+#### Setting Object Lock retention
+
+Rclone supports setting Object Lock retention on uploaded objects with
+the following options:
+
+- `--s3-object-lock-mode` - Set the Object Lock mode (GOVERNANCE or COMPLIANCE)
+- `--s3-object-lock-retain-until-date` - Set the retention date (RFC3339 format or duration like `365d`, `24h`)
+- `--s3-object-lock-legal-hold-status` - Set legal hold (ON or OFF)
+
+Example - upload files with 1 year GOVERNANCE retention:
+
+ rclone copy local:/data remote:bucket \
+ --s3-object-lock-mode GOVERNANCE \
+ --s3-object-lock-retain-until-date 365d
+
+#### Copying Object Lock settings from source
+
+When using `--metadata`, you can use the special value `copy` to preserve
+the source object's Object Lock settings:
+
+ rclone copy source:bucket dest:bucket \
+ --metadata \
+ --s3-object-lock-mode copy \
+ --s3-object-lock-retain-until-date copy
+
+You can also mix copied and explicit values. For example, to change the
+mode from COMPLIANCE to GOVERNANCE while preserving the original retention date:
+
+ rclone copy source:bucket dest:bucket \
+ --metadata \
+ --s3-object-lock-mode GOVERNANCE \
+ --s3-object-lock-retain-until-date copy
+
+#### Additional Object Lock options
+
+- `--s3-bypass-governance-retention` - Required to delete or overwrite objects
+ with GOVERNANCE mode retention before the retention date expires
+- `--s3-bucket-object-lock-enabled` - Enable Object Lock when creating a new
+ bucket with `rclone mkdir`
+- `--s3-object-lock-set-after-upload` - Set Object Lock via separate API calls
+ after upload (for providers that don't support Object Lock headers during PUT)
+
<!-- autogenerated options start - DO NOT EDIT - instead edit fs.RegInfo in backend/s3/s3.go and run make backenddocs to verify --> <!-- markdownlint-disable-line line-length -->
### Standard options