"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files
Go azure-blobazure-blob-storageazure-filesbackblaze-b2cloud-storagedropboxencryptionftpfuse-filesystemgogolanggoogle-cloud-storagegoogle-driveonedriveopenstack-swiftrclones3sftpsyncwebdav
chunker: clean stale chunks after server-side overwrite - fixes #10123
hahahahahayesyeseysfe813c53f278
diff --git a/backend/chunker/chunker.go b/backend/chunker/chunker.goindex 45063c822..1767b9fcc 100644--- a/backend/chunker/chunker.go+++ b/backend/chunker/chunker.go@@ -1502,11 +1502,21 @@ func (c *chunkingReader) rollback(ctx context.Context, metaObject fs.Object) {func (f *Fs) removeOldChunks(ctx context.Context, remote string) {oldFsObject, err := f.NewObject(ctx, remote)if err == nil {- oldObject := oldFsObject.(*Object)- for _, chunk := range oldObject.chunks {- if err := chunk.Remove(ctx); err != nil {- fs.Errorf(chunk, "Failed to remove old chunk: %v", err)- }+ removeStaleChunks(ctx, oldFsObject.(*Object), nil)+ }+}++func removeStaleChunks(ctx context.Context, oldObject *Object, currentChunks []fs.Object) {+ currentRemotes := make(map[string]struct{}, len(currentChunks))+ for _, chunk := range currentChunks {+ currentRemotes[chunk.Remote()] = struct{}{}+ }+ for _, chunk := range oldObject.chunks {+ if _, found := currentRemotes[chunk.Remote()]; found {+ continue+ }+ if err := chunk.Remove(ctx); err != nil {+ fs.Errorf(chunk, "Failed to remove old chunk: %v", err)}}}@@ -1687,12 +1697,20 @@ func (f *Fs) copyOrMove(ctx context.Context, o *Object, remote string, do copyMo// metadata format which might involve unsupported chunk types.return nil, fmt.Errorf("can't %s this file: %w", opName, err)}+ // Snapshot destination chunks before its metadata is replaced.+ var oldObject *Object+ if oldFsObject, err := f.NewObject(ctx, remote); err == nil {+ oldObject = oldFsObject.(*Object)+ }if !o.isComposite() {fs.Debugf(o, "%s non-chunked object...", opName)oResult, err := do(ctx, o.mainChunk(), remote) // chain operation to a single wrapped chunkif err != nil {return nil, err}+ if oldObject != nil {+ removeStaleChunks(ctx, oldObject, nil)+ }return f.newObject("", oResult, nil), nil}@@ -1758,6 +1776,9 @@ func (f *Fs) copyOrMove(ctx context.Context, o *Object, remote string, do copyMosilentlyRemove(ctx, newObj)return nil, err}+ if oldObject != nil {+ removeStaleChunks(ctx, oldObject, newChunks)+ }return newObj, nil}diff --git a/backend/chunker/chunker_internal_test.go b/backend/chunker/chunker_internal_test.goindex 6285f3b70..801e09664 100644--- a/backend/chunker/chunker_internal_test.go+++ b/backend/chunker/chunker_internal_test.go@@ -11,6 +11,7 @@ import ("strings""testing"+ _ "github.com/rclone/rclone/backend/memory""github.com/rclone/rclone/fs""github.com/rclone/rclone/fs/config/configmap""github.com/rclone/rclone/fs/fspath"@@ -877,6 +878,101 @@ func testChunkerServerSideMove(t *testing.T, f *Fs) {_ = operations.Purge(ctx, f.base, dir)}+func testChunkerServerSideOverwrite(t *testing.T, moveFs *Fs) {+ ctx := context.Background()+ const dir = "serveroverwrite"+ remote := fmt.Sprintf(":chunker,remote=':memory:%s',chunk_size=3B:", dir)+ fsResult, err := fs.NewFs(ctx, remote)+ require.NoError(t, err)+ copyFs, ok := fsResult.(*Fs)+ require.True(t, ok)+ oldMoveChunkSize := moveFs.opt.ChunkSize+ moveFs.opt.ChunkSize = fs.SizeSuffix(3)+ defer func() {+ moveFs.opt.ChunkSize = oldMoveChunkSize+ _ = operations.Purge(ctx, copyFs.base, "")+ _ = operations.Purge(ctx, moveFs.base, dir)+ }()++ type operation struct {+ name string+ fs *Fs+ enabled bool+ do copyMoveFn+ }+ operations := []operation{+ {name: "Copy", fs: copyFs, enabled: copyFs.Features().Copy != nil, do: copyFs.Copy},+ {name: "Move", fs: moveFs, enabled: moveFs.Features().Move != nil, do: moveFs.Move},+ }+ testCases := []struct {+ name string+ sourceContents string+ targetContents string+ }{+ {name: "SmallOverComposite", sourceContents: "hi", targetContents: "abcdef"},+ {name: "FewerChunks", sourceContents: "abcd", targetContents: "abcdefghi"},+ }+ transactionModes := []struct {+ name string+ useNoRename bool+ }{+ {name: "Rename"},+ {name: "NoRename", useNoRename: true},+ }++ for _, operation := range operations {+ t.Run(operation.name, func(t *testing.T) {+ if !operation.enabled {+ t.Skipf("wrapped remote does not support server-side %s", strings.ToLower(operation.name))+ }+ for _, transactionMode := range transactionModes {+ t.Run(transactionMode.name, func(t *testing.T) {+ f := operation.fs+ oldUseNoRename := f.useNoRename+ f.useNoRename = transactionMode.useNoRename+ defer func() { f.useNoRename = oldUseNoRename }()+ for _, testCase := range testCases {+ t.Run(testCase.name, func(t *testing.T) {+ prefix := path.Join(dir, strings.ToLower(operation.name), strings.ToLower(transactionMode.name), strings.ToLower(testCase.name))+ source := testPutFile(ctx, t, f, prefix+"-source", testCase.sourceContents, "source", true)+ target := testPutFile(ctx, t, f, prefix+"-target", testCase.targetContents, "target", true)+ targetObject := target.(*Object)+ require.NotEmpty(t, targetObject.chunks)++ oldChunks := append([]fs.Object(nil), targetObject.chunks...)+ result, err := operation.do(ctx, source, target.Remote())+ require.NoError(t, err)++ reader, err := result.Open(ctx)+ require.NoError(t, err)+ contents, err := io.ReadAll(reader)+ require.NoError(t, err)+ require.NoError(t, reader.Close())+ assert.Equal(t, testCase.sourceContents, string(contents))++ resultObject := result.(*Object)+ currentChunks := make(map[string]struct{}, len(resultObject.chunks))+ for _, chunk := range resultObject.chunks {+ currentChunks[chunk.Remote()] = struct{}{}+ }+ removed := 0+ for _, chunk := range oldChunks {+ if _, found := currentChunks[chunk.Remote()]; found {+ continue+ }+ _, err := f.base.NewObject(ctx, chunk.Remote())+ assert.Equal(t, fs.ErrorObjectNotFound, err)+ removed+++ }+ assert.Positive(t, removed)+ })+ }+ })+ }+ })+ }+}+// Test that md5all creates metadata even for small filesfunc testMD5AllSlow(t *testing.T, f *Fs) {ctx := context.Background()@@ -950,6 +1046,9 @@ func (f *Fs) InternalTest(t *testing.T) {t.Run("ChunkerServerSideMove", func(t *testing.T) {testChunkerServerSideMove(t, f)})+ t.Run("ChunkerServerSideOverwrite", func(t *testing.T) {+ testChunkerServerSideOverwrite(t, f)+ })t.Run("MD5AllSlow", func(t *testing.T) {testMD5AllSlow(t, f)})