Skip to content

"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files

vfs: preserve shrinking truncate on uncached files

47th78be7079b97d
diff --git a/vfs/read_write_test.go b/vfs/read_write_test.go
index b0e05a84e..7a1f14112 100644
--- a/vfs/read_write_test.go
+++ b/vfs/read_write_test.go
@@ -631,6 +631,39 @@ func testRWFileHandleOpenTest(t *testing.T, vfs *VFS, test *openTest) {
assert.Equal(t, test.contents, contents)
}
+// Shrinking an uncached file with Truncate (what a mount does for
+// truncate(2) with no open handle) must stick after the upload.
+func TestRWFileTruncateShrinkUncached(t *testing.T) {
+ for _, cacheMode := range []vfscommon.CacheMode{vfscommon.CacheModeWrites, vfscommon.CacheModeFull} {
+ t.Run(cacheMode.String(), func(t *testing.T) {
+ opt := vfscommon.Opt
+ opt.CacheMode = cacheMode
+ opt.WriteBack = writeBackDelay
+ r, vfs := newTestVFSOpt(t, &opt)
+
+ file1 := r.WriteObject(context.Background(), "file1", "0123456789abcdef", t1)
+ r.CheckRemoteItems(t, file1)
+
+ node, err := vfs.Stat("file1")
+ require.NoError(t, err)
+ require.NoError(t, node.(*File).Truncate(5))
+
+ vfs.WaitForWriters(waitForWritersDelay)
+
+ obj, err := r.Fremote.NewObject(context.Background(), "file1")
+ require.NoError(t, err)
+ assert.Equal(t, int64(5), obj.Size())
+
+ rc, err := obj.Open(context.Background())
+ require.NoError(t, err)
+ contents, err := io.ReadAll(rc)
+ require.NoError(t, err)
+ require.NoError(t, rc.Close())
+ assert.Equal(t, "01234", string(contents))
+ })
+ }
+}
+
func TestRWFileHandleOpenTests(t *testing.T) {
for _, cacheMode := range []vfscommon.CacheMode{vfscommon.CacheModeWrites, vfscommon.CacheModeFull} {
t.Run(cacheMode.String(), func(t *testing.T) {
diff --git a/vfs/vfscache/item.go b/vfs/vfscache/item.go
index f25f90734..66b075606 100644
--- a/vfs/vfscache/item.go
+++ b/vfs/vfscache/item.go
@@ -1468,11 +1468,25 @@ func (item *Item) WriteAt(b []byte, off int64) (n int, err error) {
//
// This is used by the downloader to write bytes to the file.
//
+// Bytes past the end of the item are not written; they are counted
+// as processed and skipped.
+//
// It returns n the total bytes processed and skipped the number of
// bytes which were processed but not actually written to the file.
func (item *Item) WriteAtNoOverwrite(b []byte, off int64) (n int, skipped int, err error) {
item.mu.Lock()
+ // Don't write past the end of the item. If the file was shrunk
+ // with Truncate while the remote object is still longer, the
+ // downloader would otherwise grow the cache file back to the
+ // size of the remote object.
+ beyond := 0
+ if end := off + int64(len(b)); end > item.info.Size {
+ keep := max(item.info.Size-off, 0)
+ beyond = len(b) - int(keep)
+ b = b[:keep]
+ }
+
var (
// Range we wish to write
r = ranges.Range{Pos: off, Size: int64(len(b))}
@@ -1514,6 +1528,10 @@ func (item *Item) WriteAtNoOverwrite(b []byte, off int64) (n int, skipped int, e
}
}
item.mu.Unlock()
+ if err == nil {
+ n += beyond
+ skipped += beyond
+ }
return n, skipped, err
}
diff --git a/vfs/vfscache/item_test.go b/vfs/vfscache/item_test.go
index 852dba46e..5403fc5fa 100644
--- a/vfs/vfscache/item_test.go
+++ b/vfs/vfscache/item_test.go
@@ -181,6 +181,20 @@ func TestItemTruncateExisting(t *testing.T) {
checkObject(t, r, "existing", contents[:40]+zeroes[:20])
}
+func TestItemTruncateShrinkUncached(t *testing.T) {
+ r, c := newItemTestCache(t)
+
+ contents, obj, item := newFile(t, r, c, "existing")
+
+ // Shrink a file which hasn't been read so the downloader has to
+ // fetch the kept part of the file on close.
+ require.NoError(t, item.Open(obj))
+ require.NoError(t, item.Truncate(40))
+ require.NoError(t, item.Close(nil))
+
+ checkObject(t, r, "existing", contents[:40])
+}
+
func TestItemReadAt(t *testing.T) {
r, c := newItemTestCache(t)