"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files
Go azure-blobazure-blob-storageazure-filesbackblaze-b2cloud-storagedropboxencryptionftpfuse-filesystemgogolanggoogle-cloud-storagegoogle-driveonedriveopenstack-swiftrclones3sftpsyncwebdav
vfs: keep a record of the file size in the writeback queue
Nick Craig-Woodbfec15950450
diff --git a/vfs/vfscache/item.go b/vfs/vfscache/item.goindex 702847719..b76688db1 100644--- a/vfs/vfscache/item.go+++ b/vfs/vfscache/item.go@@ -737,7 +737,7 @@ func (item *Item) Close(storeFn StoreFn) (err error) {item.c.writeback.SetID(&item.writeBackID)id := item.writeBackIDitem.mu.Unlock()- item.c.writeback.Add(id, item.name, item.modified, func(ctx context.Context) error {+ item.c.writeback.Add(id, item.name, item.info.Size, item.modified, func(ctx context.Context) error {return item.store(ctx, storeFn)})item.mu.Lock()diff --git a/vfs/vfscache/writeback/writeback.go b/vfs/vfscache/writeback/writeback.goindex 9801a2d3c..33c5616c2 100644--- a/vfs/vfscache/writeback/writeback.go+++ b/vfs/vfscache/writeback/writeback.go@@ -62,6 +62,7 @@ func New(ctx context.Context, opt *vfscommon.Options) *WriteBack {// writeBack.mu must be held to manipulate thistype writeBackItem struct {name string // name of the item so we don't have to read it from item+ size int64 // size of the item so we don't have to read it from itemid Handle // id of the itemindex int // index into the priority queue for updateexpiry time.Time // When this expires we will write it back@@ -135,10 +136,11 @@ func (wb *WriteBack) _newExpiry() time.Time {// make a new writeBackItem//// call with the lock held-func (wb *WriteBack) _newItem(id Handle, name string) *writeBackItem {+func (wb *WriteBack) _newItem(id Handle, name string, size int64) *writeBackItem {wb.SetID(&id)wbItem := &writeBackItem{name: name,+ size: size,expiry: wb._newExpiry(),delay: time.Duration(wb.opt.WriteBack),id: id,@@ -256,13 +258,13 @@ func (wb *WriteBack) SetID(pid *Handle) {//// If modified is false then it it doesn't cancel a pending upload if// there is one as there is no need.-func (wb *WriteBack) Add(id Handle, name string, modified bool, putFn PutFn) Handle {+func (wb *WriteBack) Add(id Handle, name string, size int64, modified bool, putFn PutFn) Handle {wb.mu.Lock()defer wb.mu.Unlock()wbItem, ok := wb.lookup[id]if !ok {- wbItem = wb._newItem(id, name)+ wbItem = wb._newItem(id, name, size)} else {if wbItem.uploading && modified {// We are uploading already so cancel the upload@@ -272,6 +274,7 @@ func (wb *WriteBack) Add(id Handle, name string, modified bool, putFn PutFn) Hanwb.items._update(wbItem, wb._newExpiry())}wbItem.putFn = putFn+ wbItem.size = sizewb._resetTimer()return wbItem.id}diff --git a/vfs/vfscache/writeback/writeback_test.go b/vfs/vfscache/writeback/writeback_test.goindex 64645eccf..21d8bae7e 100644--- a/vfs/vfscache/writeback/writeback_test.go+++ b/vfs/vfscache/writeback/writeback_test.go@@ -122,15 +122,15 @@ func TestWriteBackItemCRUD(t *testing.T) {// _peekItem emptyassert.Nil(t, wb._peekItem())- wbItem1 := wb._newItem(0, "one")+ wbItem1 := wb._newItem(0, "one", 10)checkOnHeap(t, wb, wbItem1)checkInLookup(t, wb, wbItem1)- wbItem2 := wb._newItem(0, "two")+ wbItem2 := wb._newItem(0, "two", 10)checkOnHeap(t, wb, wbItem2)checkInLookup(t, wb, wbItem2)- wbItem3 := wb._newItem(0, "three")+ wbItem3 := wb._newItem(0, "three", 10)checkOnHeap(t, wb, wbItem3)checkInLookup(t, wb, wbItem3)@@ -201,7 +201,7 @@ func TestWriteBackResetTimer(t *testing.T) {// Check timer is stoppedassertTimerRunning(t, wb, false)- _ = wb._newItem(0, "three")+ _ = wb._newItem(0, "three", 10)// Reset the timer on an queue with stuffwb._resetTimer()@@ -297,7 +297,7 @@ func TestWriteBackAddOK(t *testing.T) {wb.SetID(&inID)assert.Equal(t, Handle(1), inID)- id := wb.Add(inID, "one", true, pi.put)+ id := wb.Add(inID, "one", 10, true, pi.put)assert.Equal(t, inID, id)wbItem := wb.lookup[id]checkOnHeap(t, wb, wbItem)@@ -321,7 +321,7 @@ func TestWriteBackAddFailRetry(t *testing.T) {pi := newPutItem(t)- id := wb.Add(0, "one", true, pi.put)+ id := wb.Add(0, "one", 10, true, pi.put)wbItem := wb.lookup[id]checkOnHeap(t, wb, wbItem)checkInLookup(t, wb, wbItem)@@ -354,8 +354,9 @@ func TestWriteBackAddUpdate(t *testing.T) {pi := newPutItem(t)- id := wb.Add(0, "one", true, pi.put)+ id := wb.Add(0, "one", 10, true, pi.put)wbItem := wb.lookup[id]+ assert.Equal(t, int64(10), wbItem.size) // check sizecheckOnHeap(t, wb, wbItem)checkInLookup(t, wb, wbItem)assert.Equal(t, "one", wb.string(t))@@ -367,9 +368,10 @@ func TestWriteBackAddUpdate(t *testing.T) {// Now the upload has started add another onepi2 := newPutItem(t)- id2 := wb.Add(id, "one", true, pi2.put)+ id2 := wb.Add(id, "one", 20, true, pi2.put)assert.Equal(t, id, id2)- checkOnHeap(t, wb, wbItem) // object awaiting writeback time+ assert.Equal(t, int64(20), wbItem.size) // check size has changed+ checkOnHeap(t, wb, wbItem) // object awaiting writeback timecheckInLookup(t, wb, wbItem)// check the previous transfer was cancelled@@ -393,7 +395,7 @@ func TestWriteBackAddUpdateNotModified(t *testing.T) {pi := newPutItem(t)- id := wb.Add(0, "one", false, pi.put)+ id := wb.Add(0, "one", 10, false, pi.put)wbItem := wb.lookup[id]checkOnHeap(t, wb, wbItem)checkInLookup(t, wb, wbItem)@@ -406,7 +408,7 @@ func TestWriteBackAddUpdateNotModified(t *testing.T) {// Now the upload has started add another onepi2 := newPutItem(t)- id2 := wb.Add(id, "one", false, pi2.put)+ id2 := wb.Add(id, "one", 10, false, pi2.put)assert.Equal(t, id, id2)checkNotOnHeap(t, wb, wbItem) // object still being transferredcheckInLookup(t, wb, wbItem)@@ -432,7 +434,7 @@ func TestWriteBackAddUpdateNotStarted(t *testing.T) {pi := newPutItem(t)- id := wb.Add(0, "one", true, pi.put)+ id := wb.Add(0, "one", 10, true, pi.put)wbItem := wb.lookup[id]checkOnHeap(t, wb, wbItem)checkInLookup(t, wb, wbItem)@@ -441,7 +443,7 @@ func TestWriteBackAddUpdateNotStarted(t *testing.T) {// Immediately add another upload before the first has startedpi2 := newPutItem(t)- id2 := wb.Add(id, "one", true, pi2.put)+ id2 := wb.Add(id, "one", 10, true, pi2.put)assert.Equal(t, id, id2)checkOnHeap(t, wb, wbItem) // object still awaiting transfercheckInLookup(t, wb, wbItem)@@ -470,7 +472,7 @@ func TestWriteBackGetStats(t *testing.T) {pi := newPutItem(t)- wb.Add(0, "one", true, pi.put)+ wb.Add(0, "one", 10, true, pi.put)inProgress, queued := wb.Stats()assert.Equal(t, queued, 1)@@ -506,7 +508,7 @@ func TestWriteBackMaxQueue(t *testing.T) {for i := 0; i < toTransfer; i++ {pi := newPutItem(t)pis = append(pis, pi)- wb.Add(0, fmt.Sprintf("number%d", 1), true, pi.put)+ wb.Add(0, fmt.Sprintf("number%d", 1), 10, true, pi.put)}inProgress, queued := wb.Stats()@@ -551,7 +553,7 @@ func TestWriteBackRename(t *testing.T) {// add itempi1 := newPutItem(t)- id := wb.Add(0, "one", true, pi1.put)+ id := wb.Add(0, "one", 10, true, pi1.put)wbItem := wb.lookup[id]checkOnHeap(t, wb, wbItem)checkInLookup(t, wb, wbItem)@@ -566,7 +568,7 @@ func TestWriteBackRename(t *testing.T) {// add itempi2 := newPutItem(t)- id = wb.Add(id, "two", true, pi2.put)+ id = wb.Add(id, "two", 10, true, pi2.put)wbItem = wb.lookup[id]checkOnHeap(t, wb, wbItem)checkInLookup(t, wb, wbItem)@@ -591,9 +593,9 @@ func TestWriteBackRenameDuplicates(t *testing.T) {wb, cancel := newTestWriteBack(t)defer cancel()- // add item "one"+ // add item "one", 10pi1 := newPutItem(t)- id1 := wb.Add(0, "one", true, pi1.put)+ id1 := wb.Add(0, "one", 10, true, pi1.put)wbItem1 := wb.lookup[id1]checkOnHeap(t, wb, wbItem1)checkInLookup(t, wb, wbItem1)@@ -605,7 +607,7 @@ func TestWriteBackRenameDuplicates(t *testing.T) {// add item "two"pi2 := newPutItem(t)- id2 := wb.Add(0, "two", true, pi2.put)+ id2 := wb.Add(0, "two", 10, true, pi2.put)wbItem2 := wb.lookup[id2]checkOnHeap(t, wb, wbItem2)checkInLookup(t, wb, wbItem2)@@ -641,7 +643,7 @@ func TestWriteBackCancelUpload(t *testing.T) {// add itempi := newPutItem(t)- id := wb.Add(0, "one", true, pi.put)+ id := wb.Add(0, "one", 10, true, pi.put)wbItem := wb.lookup[id]checkOnHeap(t, wb, wbItem)checkInLookup(t, wb, wbItem)