Skip to content

"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files

chunker: clean stale chunks after server-side overwrite - fixes #10123

hahahahahayesyeseysfe813c53f278
diff --git a/backend/chunker/chunker.go b/backend/chunker/chunker.go
index 45063c822..1767b9fcc 100644
--- a/backend/chunker/chunker.go
+++ b/backend/chunker/chunker.go
@@ -1502,11 +1502,21 @@ func (c *chunkingReader) rollback(ctx context.Context, metaObject fs.Object) {
func (f *Fs) removeOldChunks(ctx context.Context, remote string) {
oldFsObject, err := f.NewObject(ctx, remote)
if err == nil {
- oldObject := oldFsObject.(*Object)
- for _, chunk := range oldObject.chunks {
- if err := chunk.Remove(ctx); err != nil {
- fs.Errorf(chunk, "Failed to remove old chunk: %v", err)
- }
+ removeStaleChunks(ctx, oldFsObject.(*Object), nil)
+ }
+}
+
+func removeStaleChunks(ctx context.Context, oldObject *Object, currentChunks []fs.Object) {
+ currentRemotes := make(map[string]struct{}, len(currentChunks))
+ for _, chunk := range currentChunks {
+ currentRemotes[chunk.Remote()] = struct{}{}
+ }
+ for _, chunk := range oldObject.chunks {
+ if _, found := currentRemotes[chunk.Remote()]; found {
+ continue
+ }
+ if err := chunk.Remove(ctx); err != nil {
+ fs.Errorf(chunk, "Failed to remove old chunk: %v", err)
}
}
}
@@ -1687,12 +1697,20 @@ func (f *Fs) copyOrMove(ctx context.Context, o *Object, remote string, do copyMo
// metadata format which might involve unsupported chunk types.
return nil, fmt.Errorf("can't %s this file: %w", opName, err)
}
+ // Snapshot destination chunks before its metadata is replaced.
+ var oldObject *Object
+ if oldFsObject, err := f.NewObject(ctx, remote); err == nil {
+ oldObject = oldFsObject.(*Object)
+ }
if !o.isComposite() {
fs.Debugf(o, "%s non-chunked object...", opName)
oResult, err := do(ctx, o.mainChunk(), remote) // chain operation to a single wrapped chunk
if err != nil {
return nil, err
}
+ if oldObject != nil {
+ removeStaleChunks(ctx, oldObject, nil)
+ }
return f.newObject("", oResult, nil), nil
}
@@ -1758,6 +1776,9 @@ func (f *Fs) copyOrMove(ctx context.Context, o *Object, remote string, do copyMo
silentlyRemove(ctx, newObj)
return nil, err
}
+ if oldObject != nil {
+ removeStaleChunks(ctx, oldObject, newChunks)
+ }
return newObj, nil
}
diff --git a/backend/chunker/chunker_internal_test.go b/backend/chunker/chunker_internal_test.go
index 6285f3b70..801e09664 100644
--- a/backend/chunker/chunker_internal_test.go
+++ b/backend/chunker/chunker_internal_test.go
@@ -11,6 +11,7 @@ import (
"strings"
"testing"
+ _ "github.com/rclone/rclone/backend/memory"
"github.com/rclone/rclone/fs"
"github.com/rclone/rclone/fs/config/configmap"
"github.com/rclone/rclone/fs/fspath"
@@ -877,6 +878,101 @@ func testChunkerServerSideMove(t *testing.T, f *Fs) {
_ = operations.Purge(ctx, f.base, dir)
}
+func testChunkerServerSideOverwrite(t *testing.T, moveFs *Fs) {
+ ctx := context.Background()
+ const dir = "serveroverwrite"
+ remote := fmt.Sprintf(":chunker,remote=':memory:%s',chunk_size=3B:", dir)
+ fsResult, err := fs.NewFs(ctx, remote)
+ require.NoError(t, err)
+ copyFs, ok := fsResult.(*Fs)
+ require.True(t, ok)
+ oldMoveChunkSize := moveFs.opt.ChunkSize
+ moveFs.opt.ChunkSize = fs.SizeSuffix(3)
+ defer func() {
+ moveFs.opt.ChunkSize = oldMoveChunkSize
+ _ = operations.Purge(ctx, copyFs.base, "")
+ _ = operations.Purge(ctx, moveFs.base, dir)
+ }()
+
+ type operation struct {
+ name string
+ fs *Fs
+ enabled bool
+ do copyMoveFn
+ }
+ operations := []operation{
+ {name: "Copy", fs: copyFs, enabled: copyFs.Features().Copy != nil, do: copyFs.Copy},
+ {name: "Move", fs: moveFs, enabled: moveFs.Features().Move != nil, do: moveFs.Move},
+ }
+ testCases := []struct {
+ name string
+ sourceContents string
+ targetContents string
+ }{
+ {name: "SmallOverComposite", sourceContents: "hi", targetContents: "abcdef"},
+ {name: "FewerChunks", sourceContents: "abcd", targetContents: "abcdefghi"},
+ }
+ transactionModes := []struct {
+ name string
+ useNoRename bool
+ }{
+ {name: "Rename"},
+ {name: "NoRename", useNoRename: true},
+ }
+
+ for _, operation := range operations {
+ t.Run(operation.name, func(t *testing.T) {
+ if !operation.enabled {
+ t.Skipf("wrapped remote does not support server-side %s", strings.ToLower(operation.name))
+ }
+ for _, transactionMode := range transactionModes {
+ t.Run(transactionMode.name, func(t *testing.T) {
+ f := operation.fs
+ oldUseNoRename := f.useNoRename
+ f.useNoRename = transactionMode.useNoRename
+ defer func() { f.useNoRename = oldUseNoRename }()
+ for _, testCase := range testCases {
+ t.Run(testCase.name, func(t *testing.T) {
+ prefix := path.Join(dir, strings.ToLower(operation.name), strings.ToLower(transactionMode.name), strings.ToLower(testCase.name))
+ source := testPutFile(ctx, t, f, prefix+"-source", testCase.sourceContents, "source", true)
+ target := testPutFile(ctx, t, f, prefix+"-target", testCase.targetContents, "target", true)
+ targetObject := target.(*Object)
+ require.NotEmpty(t, targetObject.chunks)
+
+ oldChunks := append([]fs.Object(nil), targetObject.chunks...)
+ result, err := operation.do(ctx, source, target.Remote())
+ require.NoError(t, err)
+
+ reader, err := result.Open(ctx)
+ require.NoError(t, err)
+ contents, err := io.ReadAll(reader)
+ require.NoError(t, err)
+ require.NoError(t, reader.Close())
+ assert.Equal(t, testCase.sourceContents, string(contents))
+
+ resultObject := result.(*Object)
+ currentChunks := make(map[string]struct{}, len(resultObject.chunks))
+ for _, chunk := range resultObject.chunks {
+ currentChunks[chunk.Remote()] = struct{}{}
+ }
+ removed := 0
+ for _, chunk := range oldChunks {
+ if _, found := currentChunks[chunk.Remote()]; found {
+ continue
+ }
+ _, err := f.base.NewObject(ctx, chunk.Remote())
+ assert.Equal(t, fs.ErrorObjectNotFound, err)
+ removed++
+ }
+ assert.Positive(t, removed)
+ })
+ }
+ })
+ }
+ })
+ }
+}
+
// Test that md5all creates metadata even for small files
func testMD5AllSlow(t *testing.T, f *Fs) {
ctx := context.Background()
@@ -950,6 +1046,9 @@ func (f *Fs) InternalTest(t *testing.T) {
t.Run("ChunkerServerSideMove", func(t *testing.T) {
testChunkerServerSideMove(t, f)
})
+ t.Run("ChunkerServerSideOverwrite", func(t *testing.T) {
+ testChunkerServerSideOverwrite(t, f)
+ })
t.Run("MD5AllSlow", func(t *testing.T) {
testMD5AllSlow(t, f)
})