Skip to content

"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files

operations: fix partial name collisions for non --inplace copies

Nick Craig-Woodc5a3e86df814
diff --git a/fs/operations/copy.go b/fs/operations/copy.go
index 2d42fac07..f7871d292 100644
--- a/fs/operations/copy.go
+++ b/fs/operations/copy.go
@@ -99,10 +99,13 @@ func (c *copy) checkPartial(ctx context.Context) (remoteForCopy string, inplace
// Avoid making the leaf name longer if it's already lengthy to avoid
// trouble with file name length limits.
- // generate a stable random suffix by hashing the fingerprint
- hash := crc32.ChecksumIEEE([]byte(fs.Fingerprint(ctx, c.src, true)))
+ // generate a stable random suffix by hashing the filename and fingerprint
+ hasher := crc32.New(crc32.IEEETable)
+ _, _ = hasher.Write([]byte(c.remote))
+ _, _ = hasher.Write([]byte(fs.Fingerprint(ctx, c.src, true)))
+ hash := hasher.Sum32()
- suffix := fmt.Sprintf(".%x%s", hash, c.ci.PartialSuffix)
+ suffix := fmt.Sprintf(".%08x%s", hash, c.ci.PartialSuffix)
base := path.Base(remoteForCopy)
if len(base) > 100 {
remoteForCopy = TruncateString(remoteForCopy, len(remoteForCopy)-len(suffix)) + suffix
diff --git a/fs/operations/copy_test.go b/fs/operations/copy_test.go
index 1e8f3ae5e..e717f3bef 100644
--- a/fs/operations/copy_test.go
+++ b/fs/operations/copy_test.go
@@ -15,6 +15,7 @@ import (
"github.com/rclone/rclone/fs"
"github.com/rclone/rclone/fs/accounting"
"github.com/rclone/rclone/fs/operations"
+ "github.com/rclone/rclone/fs/sync"
"github.com/rclone/rclone/fstest"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
@@ -427,6 +428,32 @@ func TestCopyLongFileName(t *testing.T) {
r.CheckRemoteItems(t, file2)
}
+func TestCopyLongFileNameCollision(t *testing.T) {
+ ctx := context.Background()
+ ctx, ci := fs.AddConfig(ctx)
+ r := fstest.NewRun(t)
+
+ if !r.Fremote.Features().PartialUploads {
+ t.Skip("Partial uploads not supported")
+ }
+
+ ci.Inplace = false
+ ci.Transfers = 4
+
+ // Write a lot of identical files with long names
+ files := make([]fstest.Item, 10)
+ namePrefix := strings.Repeat("file1", 30)
+ for i := range files {
+ files[i] = r.WriteFile(fmt.Sprintf("%s%02d", namePrefix, i), "file1 contents", t1)
+ }
+ r.CheckLocalItems(t, files...)
+
+ err := sync.CopyDir(ctx, r.Fremote, r.Flocal, false)
+ require.NoError(t, err)
+ r.CheckLocalItems(t, files...)
+ r.CheckRemoteItems(t, files...)
+}
+
func TestCopyFileMaxTransfer(t *testing.T) {
ctx := context.Background()
ctx, ci := fs.AddConfig(ctx)