"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files
Go azure-blobazure-blob-storageazure-filesbackblaze-b2cloud-storagedropboxencryptionftpfuse-filesystemgogolanggoogle-cloud-storagegoogle-driveonedriveopenstack-swiftrclones3sftpsyncwebdav
operations: fix partial name collisions for non --inplace copies
Nick Craig-Woodc5a3e86df814
diff --git a/fs/operations/copy.go b/fs/operations/copy.goindex 2d42fac07..f7871d292 100644--- a/fs/operations/copy.go+++ b/fs/operations/copy.go@@ -99,10 +99,13 @@ func (c *copy) checkPartial(ctx context.Context) (remoteForCopy string, inplace// Avoid making the leaf name longer if it's already lengthy to avoid// trouble with file name length limits.- // generate a stable random suffix by hashing the fingerprint- hash := crc32.ChecksumIEEE([]byte(fs.Fingerprint(ctx, c.src, true)))+ // generate a stable random suffix by hashing the filename and fingerprint+ hasher := crc32.New(crc32.IEEETable)+ _, _ = hasher.Write([]byte(c.remote))+ _, _ = hasher.Write([]byte(fs.Fingerprint(ctx, c.src, true)))+ hash := hasher.Sum32()- suffix := fmt.Sprintf(".%x%s", hash, c.ci.PartialSuffix)+ suffix := fmt.Sprintf(".%08x%s", hash, c.ci.PartialSuffix)base := path.Base(remoteForCopy)if len(base) > 100 {remoteForCopy = TruncateString(remoteForCopy, len(remoteForCopy)-len(suffix)) + suffixdiff --git a/fs/operations/copy_test.go b/fs/operations/copy_test.goindex 1e8f3ae5e..e717f3bef 100644--- a/fs/operations/copy_test.go+++ b/fs/operations/copy_test.go@@ -15,6 +15,7 @@ import ("github.com/rclone/rclone/fs""github.com/rclone/rclone/fs/accounting""github.com/rclone/rclone/fs/operations"+ "github.com/rclone/rclone/fs/sync""github.com/rclone/rclone/fstest""github.com/stretchr/testify/assert""github.com/stretchr/testify/require"@@ -427,6 +428,32 @@ func TestCopyLongFileName(t *testing.T) {r.CheckRemoteItems(t, file2)}+func TestCopyLongFileNameCollision(t *testing.T) {+ ctx := context.Background()+ ctx, ci := fs.AddConfig(ctx)+ r := fstest.NewRun(t)++ if !r.Fremote.Features().PartialUploads {+ t.Skip("Partial uploads not supported")+ }++ ci.Inplace = false+ ci.Transfers = 4++ // Write a lot of identical files with long names+ files := make([]fstest.Item, 10)+ namePrefix := strings.Repeat("file1", 30)+ for i := range files {+ files[i] = r.WriteFile(fmt.Sprintf("%s%02d", namePrefix, i), "file1 contents", t1)+ }+ r.CheckLocalItems(t, files...)++ err := sync.CopyDir(ctx, r.Fremote, r.Flocal, false)+ require.NoError(t, err)+ r.CheckLocalItems(t, files...)+ r.CheckRemoteItems(t, files...)+}+func TestCopyFileMaxTransfer(t *testing.T) {ctx := context.Background()ctx, ci := fs.AddConfig(ctx)