"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files
Go azure-blobazure-blob-storageazure-filesbackblaze-b2cloud-storagedropboxencryptionftpfuse-filesystemgogolanggoogle-cloud-storagegoogle-driveonedriveopenstack-swiftrclones3sftpsyncwebdav
vfs: preserve shrinking truncate on uncached files
47th78be7079b97d
diff --git a/vfs/read_write_test.go b/vfs/read_write_test.goindex b0e05a84e..7a1f14112 100644--- a/vfs/read_write_test.go+++ b/vfs/read_write_test.go@@ -631,6 +631,39 @@ func testRWFileHandleOpenTest(t *testing.T, vfs *VFS, test *openTest) {assert.Equal(t, test.contents, contents)}+// Shrinking an uncached file with Truncate (what a mount does for+// truncate(2) with no open handle) must stick after the upload.+func TestRWFileTruncateShrinkUncached(t *testing.T) {+ for _, cacheMode := range []vfscommon.CacheMode{vfscommon.CacheModeWrites, vfscommon.CacheModeFull} {+ t.Run(cacheMode.String(), func(t *testing.T) {+ opt := vfscommon.Opt+ opt.CacheMode = cacheMode+ opt.WriteBack = writeBackDelay+ r, vfs := newTestVFSOpt(t, &opt)++ file1 := r.WriteObject(context.Background(), "file1", "0123456789abcdef", t1)+ r.CheckRemoteItems(t, file1)++ node, err := vfs.Stat("file1")+ require.NoError(t, err)+ require.NoError(t, node.(*File).Truncate(5))++ vfs.WaitForWriters(waitForWritersDelay)++ obj, err := r.Fremote.NewObject(context.Background(), "file1")+ require.NoError(t, err)+ assert.Equal(t, int64(5), obj.Size())++ rc, err := obj.Open(context.Background())+ require.NoError(t, err)+ contents, err := io.ReadAll(rc)+ require.NoError(t, err)+ require.NoError(t, rc.Close())+ assert.Equal(t, "01234", string(contents))+ })+ }+}+func TestRWFileHandleOpenTests(t *testing.T) {for _, cacheMode := range []vfscommon.CacheMode{vfscommon.CacheModeWrites, vfscommon.CacheModeFull} {t.Run(cacheMode.String(), func(t *testing.T) {diff --git a/vfs/vfscache/item.go b/vfs/vfscache/item.goindex f25f90734..66b075606 100644--- a/vfs/vfscache/item.go+++ b/vfs/vfscache/item.go@@ -1468,11 +1468,25 @@ func (item *Item) WriteAt(b []byte, off int64) (n int, err error) {//// This is used by the downloader to write bytes to the file.//+// Bytes past the end of the item are not written; they are counted+// as processed and skipped.+//// It returns n the total bytes processed and skipped the number of// bytes which were processed but not actually written to the file.func (item *Item) WriteAtNoOverwrite(b []byte, off int64) (n int, skipped int, err error) {item.mu.Lock()+ // Don't write past the end of the item. If the file was shrunk+ // with Truncate while the remote object is still longer, the+ // downloader would otherwise grow the cache file back to the+ // size of the remote object.+ beyond := 0+ if end := off + int64(len(b)); end > item.info.Size {+ keep := max(item.info.Size-off, 0)+ beyond = len(b) - int(keep)+ b = b[:keep]+ }+var (// Range we wish to writer = ranges.Range{Pos: off, Size: int64(len(b))}@@ -1514,6 +1528,10 @@ func (item *Item) WriteAtNoOverwrite(b []byte, off int64) (n int, skipped int, e}}item.mu.Unlock()+ if err == nil {+ n += beyond+ skipped += beyond+ }return n, skipped, err}diff --git a/vfs/vfscache/item_test.go b/vfs/vfscache/item_test.goindex 852dba46e..5403fc5fa 100644--- a/vfs/vfscache/item_test.go+++ b/vfs/vfscache/item_test.go@@ -181,6 +181,20 @@ func TestItemTruncateExisting(t *testing.T) {checkObject(t, r, "existing", contents[:40]+zeroes[:20])}+func TestItemTruncateShrinkUncached(t *testing.T) {+ r, c := newItemTestCache(t)++ contents, obj, item := newFile(t, r, c, "existing")++ // Shrink a file which hasn't been read so the downloader has to+ // fetch the kept part of the file on close.+ require.NoError(t, item.Open(obj))+ require.NoError(t, item.Truncate(40))+ require.NoError(t, item.Close(nil))++ checkObject(t, r, "existing", contents[:40])+}+func TestItemReadAt(t *testing.T) {r, c := newItemTestCache(t)